Add generator plugin

This commit is contained in:
venashial 2022-05-20 00:10:15 -07:00
parent 06c210f5c8
commit 0f6263d82e
11 changed files with 1446 additions and 147 deletions

View File

@ -1,6 +1,6 @@
# Omorphia
*The Modrinth component library, in Svelte*
_The Modrinth component library, in Svelte_
---
@ -8,6 +8,8 @@
Learn more at [omorphia.modrinth.com.](https://omorphia.modrinth.com)
Requires Node v16.5+.
## Developing
The library lives in the `src/package` folder, and the documentation lives in the `src/routes` folder.

View File

@ -1,82 +1,87 @@
{
"name": "omorphia",
"version": "0.0.19",
"description": "A beautiful Svelte component & style library",
"scripts": {
"dev": "svelte-kit dev",
"build": "svelte-kit build",
"package": "svelte-kit package",
"preview": "svelte-kit preview",
"prepare": "svelte-kit sync",
"generate": "node scripts/generate.cjs",
"generate:watch": "nodemon --watch app --watch libs app/server.js",
"check": "svelte-check --tsconfig ./tsconfig.json",
"check:watch": "svelte-check --tsconfig ./tsconfig.json --watch",
"lint": "prettier --ignore-path .gitignore --check --plugin-search-dir=. . && eslint --ignore-path .gitignore .",
"format": "prettier --ignore-path .gitignore --write --plugin-search-dir=. ."
},
"devDependencies": {
"@sveltejs/adapter-static": "^1.0.0-next.29",
"@sveltejs/kit": "next",
"@typescript-eslint/eslint-plugin": "^5.10.1",
"@typescript-eslint/parser": "^5.10.1",
"eslint": "^7.32.0",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-svelte3": "^3.2.1",
"mdsvex": "^0.10.5",
"mdsvexamples": "^0.3.0",
"nodemon": "^2.0.15",
"sveld": "^0.13.4",
"svelte": "^3.48.0",
"svelte-check": "^2.2.6",
"svelte-preprocess": "^4.10.1",
"svelte2tsx": "^0.5.5",
"tslib": "^2.3.1",
"typescript": "~4.6.2"
},
"type": "module",
"svelte": "index.js",
"repository": {
"type": "git",
"url": "https://github.com/modrinth/omorphia.git"
},
"keywords": [
"UI",
"framework",
"components",
"library"
],
"license": "MIT",
"bugs": {
"url": "https://github.com/modrinth/omorphia/issues"
},
"homepage": "https://omorphia.modrinth.com",
"dependencies": {
"@iconify-json/carbon": "^1.1.1",
"@iconify-json/fa-regular": "^1.1.1",
"@iconify-json/heroicons-outline": "^1.1.1",
"@iconify-json/heroicons-solid": "^1.1.1",
"@iconify-json/lucide": "^1.1.7",
"@poppanator/sveltekit-svg": "^0.3.1",
"autoprefixer": "^10.4.2",
"cssnano": "^5.1.1",
"highlight.js": "^11.5.0",
"insane": "^2.6.2",
"marked": "^4.0.12",
"postcss": "^8.4.8",
"postcss-easy-import": "^4.0.0",
"postcss-extend-rule": "^4.0.0",
"postcss-import": "^14.0.2",
"postcss-import-ext-glob": "^2.0.1",
"postcss-load-config": "^3.1.4",
"postcss-nested": "^5.0.6",
"postcss-preset-env": "^7.4.2",
"postcss-pxtorem": "^6.0.0",
"postcss-strip-inline-comments": "^0.1.5",
"sanitize.css": "^13.0.0",
"svelte-tiny-virtual-list": "^2.0.1",
"svelte-use-click-outside": "^1.0.0",
"throttle-debounce": "^3.0.1",
"unplugin-icons": "^0.13.3"
}
"name": "omorphia",
"version": "0.0.19",
"description": "A beautiful Svelte component & style library",
"scripts": {
"dev": "svelte-kit dev",
"build": "svelte-kit build",
"package": "svelte-kit package",
"preview": "svelte-kit preview",
"prepare": "svelte-kit sync",
"generate": "node scripts/generate.cjs",
"generate:watch": "nodemon --watch app --watch libs app/server.js",
"check": "svelte-check --tsconfig ./tsconfig.json",
"check:watch": "svelte-check --tsconfig ./tsconfig.json --watch",
"lint": "prettier --ignore-path .gitignore --check --plugin-search-dir=. . && eslint --ignore-path .gitignore .",
"format": "prettier --ignore-path .gitignore --write --plugin-search-dir=. ."
},
"devDependencies": {
"@sveltejs/adapter-static": "^1.0.0-next.29",
"@sveltejs/kit": "next",
"@typescript-eslint/eslint-plugin": "^5.10.1",
"@typescript-eslint/parser": "^5.10.1",
"eslint": "^7.32.0",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-svelte3": "^3.2.1",
"mdsvex": "^0.10.5",
"mdsvexamples": "^0.3.0",
"nodemon": "^2.0.15",
"prettier": "^2.6.2",
"sveld": "^0.13.4",
"svelte": "^3.48.0",
"svelte-check": "^2.2.6",
"svelte-preprocess": "^4.10.1",
"svelte2tsx": "^0.5.5",
"tslib": "^2.3.1",
"typescript": "~4.6.2"
},
"type": "module",
"svelte": "index.js",
"repository": {
"type": "git",
"url": "https://github.com/modrinth/omorphia.git"
},
"keywords": [
"UI",
"framework",
"components",
"library"
],
"license": "MIT",
"bugs": {
"url": "https://github.com/modrinth/omorphia/issues"
},
"homepage": "https://omorphia.modrinth.com",
"dependencies": {
"@iconify-json/carbon": "^1.1.1",
"@iconify-json/fa-regular": "^1.1.1",
"@iconify-json/heroicons-outline": "^1.1.1",
"@iconify-json/heroicons-solid": "^1.1.1",
"@iconify-json/lucide": "^1.1.7",
"@poppanator/sveltekit-svg": "^0.3.1",
"autoprefixer": "^10.4.2",
"cli-progress": "^3.11.1",
"cssnano": "^5.1.1",
"fast-average-color-node": "^2.2.0",
"highlight.js": "^11.5.0",
"insane": "^2.6.2",
"jimp": "^0.16.1",
"marked": "^4.0.12",
"postcss": "^8.4.8",
"postcss-easy-import": "^4.0.0",
"postcss-extend-rule": "^4.0.0",
"postcss-import": "^14.0.2",
"postcss-import-ext-glob": "^2.0.1",
"postcss-load-config": "^3.1.4",
"postcss-nested": "^5.0.6",
"postcss-preset-env": "^7.4.2",
"postcss-pxtorem": "^6.0.0",
"postcss-strip-inline-comments": "^0.1.5",
"sanitize.css": "^13.0.0",
"svelte-tiny-virtual-list": "^2.0.1",
"svelte-use-click-outside": "^1.0.0",
"throttle-debounce": "^3.0.1",
"undici": "^5.2.0",
"unplugin-icons": "^0.13.3"
}
}

1039
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@ -3,14 +3,14 @@ import Icons from 'unplugin-icons/vite';
import svelteSvg from '@poppanator/sveltekit-svg';
export const preprocess = sveltePreprocess({
postcss: true,
})
postcss: true,
});
export const plugins = [
svelteSvg(),
Icons({
compiler: 'svelte',
defaultClass: 'icon',
scale: 1,
}),
]
svelteSvg(),
Icons({
compiler: 'svelte',
defaultClass: 'icon',
scale: 1,
}),
];

View File

@ -0,0 +1,71 @@
import { promises as fs } from 'fs';
import { landingPage } from './outputs/landingPage.js';
import { projectColors } from './outputs/projectColors.js';
import { gameVersions } from './outputs/gameVersions.js';
import { tags } from './outputs/tags.js';
const API_URL = 'https://api.modrinth.com/v2/'; //TODO Remove
process.env.VITE_API_URL || process.env?.NODE_ENV === 'development'
? 'https://staging-api.modrinth.com/v2/'
: 'https://api.modrinth.com/v2/';
// Time to live: 7 days
const TTL = 7 * 24 * 60 * 60 * 1000;
export default function Generator(options: PluginOptions) {
return {
name: 'rollup-plugin-omorphia-generator',
async buildStart() {
let state: State = {};
try {
state = JSON.parse(await fs.readFile('./generated/state.json', 'utf8'));
} catch {
// File doesn't exist, create folder
await fs.mkdir('./generated', { recursive: true });
await fs.writeFile(
'./generated/state.json',
JSON.stringify(
{
options,
},
null,
2
)
);
}
// Don't generate if the last generation was less than TTL and the options are the same
if (
state?.lastGenerated &&
new Date(state.lastGenerated).getTime() + TTL > new Date().getTime() &&
JSON.stringify(state.options) === JSON.stringify(options)
) {
return;
}
if (options.landingPage) await landingPage(API_URL);
if (options.projectColors) await projectColors(API_URL);
if (options.gameVersions) await gameVersions(API_URL);
if (options.tags) await tags(API_URL);
// Write new state
state.lastGenerated = new Date().toISOString();
state.options = options;
await fs.writeFile('./generated/state.json', JSON.stringify(state, null, 2));
},
};
}
export interface PluginOptions {
projectColors: boolean;
landingPage: boolean;
gameVersions: boolean;
tags: boolean;
}
interface State {
lastGenerated?: string;
options?: PluginOptions;
}

View File

@ -0,0 +1,24 @@
import { fetch } from 'undici';
import { promises as fs } from 'fs';
import cliProgress from 'cli-progress';
export async function gameVersions(API_URL: string) {
const progressBar = new cliProgress.SingleBar({
format: 'Generating game versions | {bar} | {percentage}%',
barCompleteChar: '\u2588',
barIncompleteChar: '\u2591',
hideCursor: true,
});
progressBar.start(2, 0);
const gameVersions = await (await fetch(API_URL + 'tag/game_version')).json();
progressBar.increment();
// Write JSON file
await fs.writeFile('./generated/gameVersions.json', JSON.stringify(gameVersions));
progressBar.increment();
progressBar.stop();
}

View File

@ -0,0 +1,44 @@
import { fetch } from 'undici';
import { promises as fs } from 'fs';
import sharp from 'sharp';
import FastAverageColor from 'fast-average-color';
import cliProgress from 'cli-progress';
export async function landingPage(API_URL: string) {
const progressBar = new cliProgress.SingleBar({
format: 'Generating landing page | {bar} | {percentage}% || {value}/{total} mods',
barCompleteChar: '\u2588',
barIncompleteChar: '\u2591',
hideCursor: true,
});
progressBar.start(100, 0);
// Fetch top 100 mods
const response = (await (
await fetch(API_URL + 'search?limit=100&facets=[["project_type:mod"]]')
).json()) as Record<string, any>;
// Simplified array with the format: ['id', 'slug', 'icon_extension']
const compressed = response.hits
.filter((project) => project.icon_url)
.map((project) => {
progressBar.increment();
return [
project.project_id,
project.slug || '',
project.icon_url.match(/\.[0-9a-z]+$/i)[0].substring(1),
];
});
// Write JSON file
await fs.writeFile(
'./generated/landingPage.json',
JSON.stringify({
mods: compressed,
random: Math.random(),
})
);
progressBar.stop();
}

View File

@ -0,0 +1,106 @@
import { fetch } from 'undici';
import { promises as fs, createWriteStream } from 'fs';
import cliProgress from 'cli-progress';
import Jimp from 'jimp';
import { getAverageColor } from 'fast-average-color-node';
// Note: This function has issues and will occasionally fail with some project icons. It averages at a 99.4% success rate. Most issues are from ECONNRESET errors & Jimp not being able to handle webp & svg images.
export async function projectColors(API_URL: string) {
const progressBar = new cliProgress.SingleBar({
format: 'Generating project colors | {bar} | {percentage}% || {value}/{total} projects',
barCompleteChar: '\u2588',
barIncompleteChar: '\u2591',
hideCursor: true,
});
// Get total number of projects
const projectCount = (
(await (await fetch(API_URL + 'search?limit=0')).json()) as Record<string, any>
).total_hits;
progressBar.start(projectCount, 0);
const writeStream = createWriteStream('./generated/projects.json');
writeStream.write('{');
// Used to form the JSON string (so that the first doesn't have a comma prefix)
let first = true;
let completed = 0;
// Number of pages through search to fetch
const requestCount = Math.ceil(projectCount / 100);
await Promise.allSettled(
Array.from({ length: requestCount }, async (_, index) => {
const response = await fetch(API_URL + `search?limit=100&offset=${index * 100}`);
if (!response.ok) {
throw new Error(`Failed to fetch projects: ${response.statusText}`);
}
// Get project hits & use map to get rid of extra data
const hits = ((await response.json()) as Record<string, any>).hits.map((project) => ({
project_id: project.project_id,
slug: project.slug,
title: project.title,
icon_url: project.icon_url,
}));
// Try parsing the icon of each project
await Promise.allSettled(
hits.map(async (project) => {
if (
project.icon_url &&
// Jimp doesn't support webp or svg
!project.icon_url.endsWith('.webp') &&
!project.icon_url.endsWith('.svg')
) {
try {
const image = await Jimp.read(
project.icon_url.replace('cdn', 'cdn-raw') // Skip redirect to raw CDN
);
// Resize image before getting average color (faster)
image.resize(256, 256);
// Get bottom edge of image
const edge = image.clone().crop(0, 255, 256, 1);
const buffer = await edge.getBufferAsync(Jimp.AUTO);
let color = (await getAverageColor(buffer)).hexa;
// If the edge is transparent, use the average color of the entire image
if (color === '#00000000') {
const buffer = await image.getBufferAsync(Jimp.AUTO);
color = (await getAverageColor(buffer)).hexa;
}
// Remove color transparency
color = color.replace(/.{2}$/, '');
// Only use comma prefix if not first
let prefix = ',';
if (first) {
prefix = '';
first = false;
}
writeStream.write(`${prefix}"${project.project_id}":"${color}"`);
completed++;
} catch (error) {
// Ignore errors
// console.log(error);
}
}
progressBar.increment();
})
);
})
);
writeStream.write('}');
writeStream.end();
progressBar.stop();
console.log(`Failed to parse ${projectCount - completed} project icons.`);
}

View File

@ -0,0 +1,50 @@
import { fetch } from 'undici';
import { promises as fs } from 'fs';
import cliProgress from 'cli-progress';
export async function tags(API_URL: string) {
const progressBar = new cliProgress.SingleBar({
format: 'Generating tags | {bar} | {percentage}%',
barCompleteChar: '\u2588',
barIncompleteChar: '\u2591',
hideCursor: true,
});
progressBar.start(7, 0);
// eslint-disable-next-line prefer-const
let [categories, loaders, licenses, donationPlatforms]: any = await Promise.all([
await (await fetch(API_URL + 'tag/category')).json(),
await (await fetch(API_URL + 'tag/loader')).json(),
await (await fetch(API_URL + 'tag/license')).json(),
await (await fetch(API_URL + 'tag/donation_platform')).json(),
]);
progressBar.update(4);
// Delete icons from original arrays
categories = categories.map(({ icon, ...rest }) => rest);
loaders = loaders.map(({ icon, ...rest }) => rest);
progressBar.increment();
// Create single object with icons
const tagIcons = {
...categories.reduce((a, v) => ({ ...a, [v.name]: v.icon }), {}),
...loaders.reduce((a, v) => ({ ...a, [v.name]: v.icon }), {}),
};
progressBar.increment();
// Set project types
const projectTypes = ['mod', 'modpack'];
// Write JSON file
await fs.writeFile(
'./generated/tags.json',
JSON.stringify({ categories, loaders, projectTypes, licenses, donationPlatforms, tagIcons })
);
progressBar.increment();
progressBar.stop();
}

View File

@ -0,0 +1,38 @@
---
title: Generator plugin
---
The generator plugin creates static files from API responses to increase performance and perform tasks that would not be possible on the client. It regenerates files every 7 days, or when the plugin settings change.
### Current options
- `projectColors` (false) generates colors for every project
- `tags` (false) copies & parses tags from API
- `gameVersions` copes game versions from API
- `landingPage` gets icon urls for top 100 mods
> All options are disabled by default
## Configuration
```js
import Generator from 'omorphia/plugins/generator';
/** @type {import('@sveltejs/kit').Config} */
const config = {
kit: {
vite: {
plugins: [
Generator({
projectColors: true,
tags: true,
gameVersions: true,
landingPage: true,
}),
],
},
},
};
export default config;
```

View File

@ -1,21 +1,15 @@
{
"compilerOptions": {
"paths": {
"omorphia/*": [
"src/package/*"
],
"omorphia": [
"src/package"
],
"$package/*": [
"src/package/*"
],
"$routes/*": [
"src/routes/*"
],
"$lib":["src/package"],
"$lib/*":["src/package/*"]
}
},
"extends": "./.svelte-kit/tsconfig.json"
"compilerOptions": {
"paths": {
"omorphia/*": ["src/package/*"],
"omorphia": ["src/package"],
"$package/*": ["src/package/*"],
"$routes/*": ["src/routes/*"],
"$lib": ["src/package"],
"$lib/*": ["src/package/*"]
},
"resolveJsonModule": true,
"esModuleInterop": true
},
"extends": "./.svelte-kit/tsconfig.json"
}