Add generator plugin

README.md

@@ -1,6 +1,6 @@
 # Omorphia
 
-*The Modrinth component library, in Svelte*
+_The Modrinth component library, in Svelte_
 
 ---
 
@@ -8,6 +8,8 @@
 
 Learn more at [omorphia.modrinth.com.](https://omorphia.modrinth.com)
 
+Requires Node v16.5+.
+
 ## Developing
 
 The library lives in the `src/package` folder, and the documentation lives in the `src/routes` folder.

package.json

@@ -26,6 +26,7 @@
         "mdsvex": "^0.10.5",
         "mdsvexamples": "^0.3.0",
         "nodemon": "^2.0.15",
+        "prettier": "^2.6.2",
         "sveld": "^0.13.4",
         "svelte": "^3.48.0",
         "svelte-check": "^2.2.6",
@@ -59,9 +60,12 @@
         "@iconify-json/lucide": "^1.1.7",
         "@poppanator/sveltekit-svg": "^0.3.1",
         "autoprefixer": "^10.4.2",
+        "cli-progress": "^3.11.1",
         "cssnano": "^5.1.1",
+        "fast-average-color-node": "^2.2.0",
         "highlight.js": "^11.5.0",
         "insane": "^2.6.2",
+        "jimp": "^0.16.1",
         "marked": "^4.0.12",
         "postcss": "^8.4.8",
         "postcss-easy-import": "^4.0.0",
@@ -77,6 +81,7 @@
         "svelte-tiny-virtual-list": "^2.0.1",
         "svelte-use-click-outside": "^1.0.0",
         "throttle-debounce": "^3.0.1",
+        "undici": "^5.2.0",
         "unplugin-icons": "^0.13.3"
     }
 }

src/package/config/svelte.config.js

@@ -4,7 +4,7 @@ import svelteSvg from '@poppanator/sveltekit-svg';
 
 export const preprocess = sveltePreprocess({
     postcss: true,
-})
+});
 
 export const plugins = [
     svelteSvg(),
@@ -13,4 +13,4 @@ export const plugins = [
         defaultClass: 'icon',
         scale: 1,
     }),
-]
+];

src/package/plugins/generator/index.ts

@@ -0,0 +1,71 @@
+import { promises as fs } from 'fs';
+import { landingPage } from './outputs/landingPage.js';
+import { projectColors } from './outputs/projectColors.js';
+import { gameVersions } from './outputs/gameVersions.js';
+import { tags } from './outputs/tags.js';
+
+const API_URL = 'https://api.modrinth.com/v2/'; //TODO Remove
+process.env.VITE_API_URL || process.env?.NODE_ENV === 'development'
+    ? 'https://staging-api.modrinth.com/v2/'
+    : 'https://api.modrinth.com/v2/';
+
+// Time to live: 7 days
+const TTL = 7 * 24 * 60 * 60 * 1000;
+
+export default function Generator(options: PluginOptions) {
+    return {
+        name: 'rollup-plugin-omorphia-generator',
+        async buildStart() {
+            let state: State = {};
+
+            try {
+                state = JSON.parse(await fs.readFile('./generated/state.json', 'utf8'));
+            } catch {
+                // File doesn't exist, create folder
+                await fs.mkdir('./generated', { recursive: true });
+
+                await fs.writeFile(
+                    './generated/state.json',
+                    JSON.stringify(
+                        {
+                            options,
+                        },
+                        null,
+                        2
+                    )
+                );
+            }
+
+            // Don't generate if the last generation was less than TTL and the options are the same
+            if (
+                state?.lastGenerated &&
+                new Date(state.lastGenerated).getTime() + TTL > new Date().getTime() &&
+                JSON.stringify(state.options) === JSON.stringify(options)
+            ) {
+                return;
+            }
+
+            if (options.landingPage) await landingPage(API_URL);
+            if (options.projectColors) await projectColors(API_URL);
+            if (options.gameVersions) await gameVersions(API_URL);
+            if (options.tags) await tags(API_URL);
+
+            // Write new state
+            state.lastGenerated = new Date().toISOString();
+            state.options = options;
+            await fs.writeFile('./generated/state.json', JSON.stringify(state, null, 2));
+        },
+    };
+}
+
+export interface PluginOptions {
+    projectColors: boolean;
+    landingPage: boolean;
+    gameVersions: boolean;
+    tags: boolean;
+}
+
+interface State {
+    lastGenerated?: string;
+    options?: PluginOptions;
+}

src/package/plugins/generator/outputs/gameVersions.ts

@@ -0,0 +1,24 @@
+import { fetch } from 'undici';
+import { promises as fs } from 'fs';
+import cliProgress from 'cli-progress';
+
+export async function gameVersions(API_URL: string) {
+    const progressBar = new cliProgress.SingleBar({
+        format: 'Generating game versions  | {bar} | {percentage}%',
+        barCompleteChar: '\u2588',
+        barIncompleteChar: '\u2591',
+        hideCursor: true,
+    });
+
+    progressBar.start(2, 0);
+
+    const gameVersions = await (await fetch(API_URL + 'tag/game_version')).json();
+
+    progressBar.increment();
+
+    // Write JSON file
+    await fs.writeFile('./generated/gameVersions.json', JSON.stringify(gameVersions));
+    progressBar.increment();
+
+    progressBar.stop();
+}

src/package/plugins/generator/outputs/landingPage.ts

@@ -0,0 +1,44 @@
+import { fetch } from 'undici';
+import { promises as fs } from 'fs';
+import sharp from 'sharp';
+import FastAverageColor from 'fast-average-color';
+import cliProgress from 'cli-progress';
+
+export async function landingPage(API_URL: string) {
+    const progressBar = new cliProgress.SingleBar({
+        format: 'Generating landing page   | {bar} | {percentage}% || {value}/{total} mods',
+        barCompleteChar: '\u2588',
+        barIncompleteChar: '\u2591',
+        hideCursor: true,
+    });
+
+    progressBar.start(100, 0);
+
+    // Fetch top 100 mods
+    const response = (await (
+        await fetch(API_URL + 'search?limit=100&facets=[["project_type:mod"]]')
+    ).json()) as Record<string, any>;
+
+    // Simplified array with the format: ['id', 'slug', 'icon_extension']
+    const compressed = response.hits
+        .filter((project) => project.icon_url)
+        .map((project) => {
+            progressBar.increment();
+            return [
+                project.project_id,
+                project.slug || '',
+                project.icon_url.match(/\.[0-9a-z]+$/i)[0].substring(1),
+            ];
+        });
+
+    // Write JSON file
+    await fs.writeFile(
+        './generated/landingPage.json',
+        JSON.stringify({
+            mods: compressed,
+            random: Math.random(),
+        })
+    );
+
+    progressBar.stop();
+}

src/package/plugins/generator/outputs/projectColors.ts

@@ -0,0 +1,106 @@
+import { fetch } from 'undici';
+import { promises as fs, createWriteStream } from 'fs';
+import cliProgress from 'cli-progress';
+import Jimp from 'jimp';
+import { getAverageColor } from 'fast-average-color-node';
+
+// Note: This function has issues and will occasionally fail with some project icons. It averages at a 99.4% success rate. Most issues are from ECONNRESET errors & Jimp not being able to handle webp & svg images.
+export async function projectColors(API_URL: string) {
+    const progressBar = new cliProgress.SingleBar({
+        format: 'Generating project colors | {bar} | {percentage}% || {value}/{total} projects',
+        barCompleteChar: '\u2588',
+        barIncompleteChar: '\u2591',
+        hideCursor: true,
+    });
+
+    // Get total number of projects
+    const projectCount = (
+        (await (await fetch(API_URL + 'search?limit=0')).json()) as Record<string, any>
+    ).total_hits;
+
+    progressBar.start(projectCount, 0);
+
+    const writeStream = createWriteStream('./generated/projects.json');
+
+    writeStream.write('{');
+
+    // Used to form the JSON string (so that the first doesn't have a comma prefix)
+    let first = true;
+
+    let completed = 0;
+
+    // Number of pages through search to fetch
+    const requestCount = Math.ceil(projectCount / 100);
+    await Promise.allSettled(
+        Array.from({ length: requestCount }, async (_, index) => {
+            const response = await fetch(API_URL + `search?limit=100&offset=${index * 100}`);
+            if (!response.ok) {
+                throw new Error(`Failed to fetch projects: ${response.statusText}`);
+            }
+
+            // Get project hits & use map to get rid of extra data
+            const hits = ((await response.json()) as Record<string, any>).hits.map((project) => ({
+                project_id: project.project_id,
+                slug: project.slug,
+                title: project.title,
+                icon_url: project.icon_url,
+            }));
+
+            // Try parsing the icon of each project
+            await Promise.allSettled(
+                hits.map(async (project) => {
+                    if (
+                        project.icon_url &&
+                        // Jimp doesn't support webp or svg
+                        !project.icon_url.endsWith('.webp') &&
+                        !project.icon_url.endsWith('.svg')
+                    ) {
+                        try {
+                            const image = await Jimp.read(
+                                project.icon_url.replace('cdn', 'cdn-raw') // Skip redirect to raw CDN
+                            );
+
+                            // Resize image before getting average color (faster)
+                            image.resize(256, 256);
+
+                            // Get bottom edge of image
+                            const edge = image.clone().crop(0, 255, 256, 1);
+
+                            const buffer = await edge.getBufferAsync(Jimp.AUTO);
+                            let color = (await getAverageColor(buffer)).hexa;
+
+                            // If the edge is transparent, use the average color of the entire image
+                            if (color === '#00000000') {
+                                const buffer = await image.getBufferAsync(Jimp.AUTO);
+                                color = (await getAverageColor(buffer)).hexa;
+                            }
+
+                            // Remove color transparency
+                            color = color.replace(/.{2}$/, '');
+
+                            // Only use comma prefix if not first
+                            let prefix = ',';
+                            if (first) {
+                                prefix = '';
+                                first = false;
+                            }
+
+                            writeStream.write(`${prefix}"${project.project_id}":"${color}"`);
+                            completed++;
+                        } catch (error) {
+                            // Ignore errors
+                            // console.log(error);
+                        }
+                    }
+                    progressBar.increment();
+                })
+            );
+        })
+    );
+
+    writeStream.write('}');
+    writeStream.end();
+
+    progressBar.stop();
+    console.log(`Failed to parse ${projectCount - completed} project icons.`);
+}

src/package/plugins/generator/outputs/tags.ts

@@ -0,0 +1,50 @@
+import { fetch } from 'undici';
+import { promises as fs } from 'fs';
+import cliProgress from 'cli-progress';
+
+export async function tags(API_URL: string) {
+    const progressBar = new cliProgress.SingleBar({
+        format: 'Generating tags           | {bar} | {percentage}%',
+        barCompleteChar: '\u2588',
+        barIncompleteChar: '\u2591',
+        hideCursor: true,
+    });
+
+    progressBar.start(7, 0);
+
+    // eslint-disable-next-line prefer-const
+    let [categories, loaders, licenses, donationPlatforms]: any = await Promise.all([
+        await (await fetch(API_URL + 'tag/category')).json(),
+        await (await fetch(API_URL + 'tag/loader')).json(),
+        await (await fetch(API_URL + 'tag/license')).json(),
+        await (await fetch(API_URL + 'tag/donation_platform')).json(),
+    ]);
+
+    progressBar.update(4);
+
+    // Delete icons from original arrays
+    categories = categories.map(({ icon, ...rest }) => rest);
+    loaders = loaders.map(({ icon, ...rest }) => rest);
+
+    progressBar.increment();
+
+    // Create single object with icons
+    const tagIcons = {
+        ...categories.reduce((a, v) => ({ ...a, [v.name]: v.icon }), {}),
+        ...loaders.reduce((a, v) => ({ ...a, [v.name]: v.icon }), {}),
+    };
+
+    progressBar.increment();
+
+    // Set project types
+    const projectTypes = ['mod', 'modpack'];
+
+    // Write JSON file
+    await fs.writeFile(
+        './generated/tags.json',
+        JSON.stringify({ categories, loaders, projectTypes, licenses, donationPlatforms, tagIcons })
+    );
+    progressBar.increment();
+
+    progressBar.stop();
+}

src/routes/getting-started/generator.md

@@ -0,0 +1,38 @@
+---
+title: Generator plugin
+---
+
+The generator plugin creates static files from API responses to increase performance and perform tasks that would not be possible on the client. It regenerates files every 7 days, or when the plugin settings change.
+
+### Current options
+
+-   `projectColors` (false) generates colors for every project
+-   `tags` (false) copies & parses tags from API
+-   `gameVersions` copes game versions from API
+-   `landingPage` gets icon urls for top 100 mods
+
+> All options are disabled by default
+
+## Configuration
+
+```js
+import Generator from 'omorphia/plugins/generator';
+
+/** @type {import('@sveltejs/kit').Config} */
+const config = {
+    kit: {
+        vite: {
+            plugins: [
+                Generator({
+                    projectColors: true,
+                    tags: true,
+                    gameVersions: true,
+                    landingPage: true,
+                }),
+            ],
+        },
+    },
+};
+
+export default config;
+```

tsconfig.json

@@ -1,21 +1,15 @@
 {
     "compilerOptions": {
         "paths": {
-      "omorphia/*": [
+            "omorphia/*": ["src/package/*"],
-        "src/package/*"
+            "omorphia": ["src/package"],
-      ],
+            "$package/*": ["src/package/*"],
-      "omorphia": [
+            "$routes/*": ["src/routes/*"],
-        "src/package"
+            "$lib": ["src/package"],
-      ],
+            "$lib/*": ["src/package/*"]
-      "$package/*": [
+        },
-        "src/package/*"
+        "resolveJsonModule": true,
-      ],
+        "esModuleInterop": true
-      "$routes/*": [
-        "src/routes/*"
-      ],
-      "$lib":["src/package"],
-      "$lib/*":["src/package/*"]
-    }
     },
     "extends": "./.svelte-kit/tsconfig.json"
 }