Bommels05/Modrinth
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Batch inserts [MOD-555] (#726)

Browse Source 
* Batch a bunch of inserts, but still more to do

* Insert many for clickhouse (+ tests)

* Batch the remaining ones except those requiring deduplication

* Risky dedups

* Bit o cleanup and formatting

* cargo sqlx prepare

* Add test around batch editing project categories

* Add struct to satisfy clippy

* Fix silly mistake that was caught by the tests!

* Leave room for growth in dummy_data
This commit is contained in:
Jackson Kruger 2023-10-11 13:32:58 -05:00 committed by GitHub
parent dfa43f3c5a
commit d92272ffa0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 1208 additions and 929 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Cargo.lock generated
View File

@ -1284,6 +1284,17 @@ dependencies = [
"uuid 1.4.0",
]
[[package]]
name = "derive-new"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "derive_builder"
version = "0.12.0"
@ -2260,6 +2271,7 @@ dependencies = [
"color-thief",
"dashmap",
"deadpool-redis",
"derive-new",
"dotenvy",
"env_logger",
"flate2",

2
Cargo.toml
View File

@ -106,5 +106,7 @@ woothee = "0.13.0"
lettre = "0.10.4"
derive-new = "0.5.9"
[dev-dependencies]
actix-http = "3.4.0"

480
sqlx-data.json
View File

@ -265,19 +265,6 @@
},
"query": "\n SELECT o.id, o.title, o.team_id, o.description, o.icon_url, o.color\n FROM organizations o\n WHERE o.id = ANY($1) OR o.title = ANY($2)\n GROUP BY o.id;\n "
},
"05baeb26d9856218e5c6f8856a96788b2a7ac3536ff9412a50552cef1d561a1e": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Int4"
]
}
},
"query": "\n INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)\n VALUES ($1, $2, FALSE)\n "
},
"061a3e43df9464263aaf1555a27c1f4b6a0f381282f4fa75cc13b1d354857578": {
"describe": {
"columns": [
@ -595,24 +582,6 @@
},
"query": "\n SELECT SUM(pv.amount) amount\n FROM payouts_values pv\n WHERE pv.user_id = $1\n "
},
"0c2addb0d7a87fa558821ff8e943bbb751fb2bdc22d1a5368f61cc7827586840": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Int8",
"Varchar",
"Varchar",
"Bool",
"Int4",
"Varchar"
]
}
},
"query": "\n INSERT INTO files (id, version_id, url, filename, is_primary, size, file_type)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n "
},
"0c6b7e51b0b9115d95b5dbb9bb88a3e266b78ae9375a90261503c2cccd5bdf1b": {
"describe": {
"columns": [],
@ -870,21 +839,6 @@
},
"query": "SELECT EXISTS(SELECT 1 FROM threads WHERE id=$1)"
},
"196c8ac2228e199f23eaf980f7ea15b37f76e66bb81da1115a754aad0be756e4": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Int8",
"Numeric",
"Timestamptz"
]
}
},
"query": "\n INSERT INTO payouts_values (user_id, mod_id, amount, created)\n VALUES ($1, $2, $3, $4)\n "
},
"19c7498a01f51b8220245a53490916191a153fa1fe14404d39ab2980e3386058": {
"describe": {
"columns": [],
@ -935,6 +889,34 @@
},
"query": "\n UPDATE collections\n SET icon_url = NULL, color = NULL\n WHERE (id = $1)\n "
},
"1b66b5d566aa6a969bacbb7897af829a569e13a619db295d2e6abcdb89fcac17": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8Array",
"Int8Array"
]
}
},
"query": "\n INSERT INTO collections_mods (collection_id, mod_id)\n SELECT * FROM UNNEST ($1::int8[], $2::int8[])\n ON CONFLICT DO NOTHING\n "
},
"1c30a8a31b031f194f70dc2a3bac8e131513dd7e9d2c46432ca797f6422c6ecf": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8Array",
"Int8Array",
"NumericArray",
"TimestamptzArray"
]
}
},
"query": "\n INSERT INTO payouts_values (user_id, mod_id, amount, created)\n SELECT * FROM UNNEST ($1::bigint[], $2::bigint[], $3::numeric[], $4::timestamptz[])\n "
},
"1cefe4924d3c1f491739858ce844a22903d2dbe26f255219299f1833a10ce3d7": {
"describe": {
"columns": [
@ -1201,6 +1183,24 @@
},
"query": "\n UPDATE team_members\n SET permissions = $1\n WHERE (team_id = $2 AND user_id = $3)\n "
},
"24ae57ca296554a29b414caca866cfe7ab956ea28450d40a564498c3d27b937f": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8Array",
"Int8Array",
"VarcharArray",
"VarcharArray",
"BoolArray",
"Int4Array",
"VarcharArray"
]
}
},
"query": "\n INSERT INTO files (id, version_id, url, filename, is_primary, size, file_type)\n SELECT * FROM UNNEST($1::bigint[], $2::bigint[], $3::varchar[], $4::varchar[], $5::bool[], $6::integer[], $7::varchar[])\n "
},
"25131559cb73a088000ab6379a769233440ade6c7511542da410065190d203fc": {
"describe": {
"columns": [
@ -1233,18 +1233,6 @@
},
"query": "\n DELETE FROM threads_members\n WHERE thread_id = $1\n "
},
"299b8ea6e7a0048fa389cc4432715dc2a09e227d2f08e91167a43372a7ac6e35": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8"
]
}
},
"query": "\n DELETE FROM mods_categories\n WHERE joining_mod_id = $1 AND is_additional = FALSE\n "
},
"29e171bd746ac5dc1fabae4c9f81c3d1df4e69c860b7d0f6a907377664199217": {
"describe": {
"columns": [
@ -1265,19 +1253,6 @@
},
"query": "\n SELECT id FROM reports\n WHERE closed = FALSE\n ORDER BY created ASC\n LIMIT $1;\n "
},
"29e657d26f0fb24a766f5b5eb6a94d01d1616884d8ca10e91536e974d5b585a6": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int4",
"Int8"
]
}
},
"query": "\n INSERT INTO loaders_versions (loader_id, version_id)\n VALUES ($1, $2)\n "
},
"29fcff0f1d36bd1a9e0c8c4005209308f0c5f383e4e52ed8c6b989994ead32df": {
"describe": {
"columns": [],
@ -1462,6 +1437,19 @@
},
"query": "\n SELECT id FROM users\n WHERE email = $1\n "
},
"3151420021b0c5a85f7c338e67be971915ff89073815e27fa6af5254db22dce8": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int4Array",
"Int8Array"
]
}
},
"query": "\n INSERT INTO loaders_versions (loader_id, version_id)\n SELECT * FROM UNNEST($1::integer[], $2::bigint[])\n "
},
"320d73cd900a6e00f0e74b7a8c34a7658d16034b01a35558cb42fa9c16185eb5": {
"describe": {
"columns": [
@ -1768,31 +1756,6 @@
},
"query": "\n UPDATE mods\n SET title = $1\n WHERE (id = $2)\n "
},
"3f2f05653552ce8c1be95ce0a922ab41f52f40f8ff6c91c6621481102c8f35e3": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int4",
"Int8"
]
}
},
"query": "\n INSERT INTO game_versions_versions (game_version_id, joining_version_id)\n VALUES ($1, $2)\n "
},
"3fcfed18cbfb37866e0fa57a4e95efb326864f8219941d1b696add39ed333ad1": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8"
]
}
},
"query": "\n DELETE FROM mods_categories\n WHERE joining_mod_id = $1 AND is_additional = TRUE\n "
},
"40f7c5bec98fe3503d6bd6db2eae5a4edb8d5d6efda9b9dc124f344ae5c60e08": {
"describe": {
"columns": [],
@ -2009,6 +1972,20 @@
},
"query": "\n UPDATE uploaded_images\n SET thread_message_id = $1\n WHERE id = $2\n "
},
"473db826b691ae1131990ef0927cfe5b63d48829dd41edb7def22248d5668ac7": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8Array",
"Int4Array",
"VarcharArray"
]
}
},
"query": "\n INSERT INTO mods_donations (\n joining_mod_id, joining_platform_id, url\n )\n SELECT * FROM UNNEST($1::bigint[], $2::int[], $3::varchar[])\n "
},
"4778d2f5994fda2f978fa53e0840c1a9a2582ef0434a5ff7f21706f1dc4edcf4": {
"describe": {
"columns": [],
@ -2055,19 +2032,6 @@
},
"query": "\n SELECT d.dependency_id, COALESCE(vd.mod_id, 0) mod_id, d.mod_dependency_id\n FROM versions v\n INNER JOIN dependencies d ON d.dependent_id = v.id\n LEFT JOIN versions vd ON d.dependency_id = vd.id\n WHERE v.mod_id = $1\n "
},
"489913b3c32631fb329a3259cfe620d65053e2abf425a0d3f1bc01f1cdbdd73d": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Int8"
]
}
},
"query": "\n INSERT INTO collections_mods (collection_id, mod_id)\n VALUES ($1, $2)\n ON CONFLICT DO NOTHING\n "
},
"49813a96f007216072d69468aae705d73d5b85dcdd64a22060009b12d947ed5a": {
"describe": {
"columns": [],
@ -2649,6 +2613,23 @@
},
"query": "SELECT id FROM users WHERE gitlab_id = $1"
},
"5d65f89c020ae032f26d742c37afe47876911eb3a16a6852299b98f2a8251fb4": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8Array",
"VarcharArray",
"BoolArray",
"VarcharArray",
"VarcharArray",
"Int8Array"
]
}
},
"query": "\n INSERT INTO mods_gallery (\n mod_id, image_url, featured, title, description, ordering\n )\n SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::bool[], $4::varchar[], $5::varchar[], $6::bigint[])\n "
},
"5d7425cfa91e332bf7cc14aa5c300b997e941c49757606f6b906cb5e060d3179": {
"describe": {
"columns": [],
@ -2725,22 +2706,6 @@
},
"query": "\n UPDATE mods_gallery\n SET ordering = $2\n WHERE id = $1\n "
},
"5f94e9e767ec4be7f9136b991b4a29373dbe48feb2f61281e3212721095ed675": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Varchar",
"Int8",
"Int8",
"Varchar"
]
}
},
"query": "\n INSERT INTO dependencies (dependent_id, dependency_type, dependency_id, mod_dependency_id, dependency_file_name)\n VALUES ($1, $2, $3, $4, $5)\n "
},
"60a251aea1efbc7d9357255e520f0ac13f3697fecb84b1e9edd5d9ea61fe0cb0": {
"describe": {
"columns": [
@ -3187,19 +3152,6 @@
},
"query": "\n UPDATE mods\n SET loaders = (\n SELECT COALESCE(ARRAY_AGG(DISTINCT l.loader) filter (where l.loader is not null), array[]::varchar[])\n FROM versions v\n INNER JOIN loaders_versions lv ON lv.version_id = v.id\n INNER JOIN loaders l on lv.loader_id = l.id\n WHERE v.mod_id = mods.id AND v.status != ALL($2)\n )\n WHERE id = $1\n "
},
"6c7aeb0db4a4fb3387c37b8d7aca6fdafaa637fd883a44416b56270aeebb7a01": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int4",
"Int8"
]
}
},
"query": "\n INSERT INTO loaders_versions (loader_id, version_id)\n VALUES ($1, $2)\n "
},
"6c8b8a2f11c0b4e7a5973547fe1611a0fa4ef366d5c8a91d9fb9a1360ea04d46": {
"describe": {
"columns": [
@ -3358,18 +3310,38 @@
},
"query": "SELECT EXISTS(SELECT 1 FROM uploaded_images WHERE id=$1)"
},
"70b510956a40583eef8c57dcced71c67f525eee455ae8b09e9b2403668068751": {
"7075dc0343dab7c4dd4469b4af095232dcdd056a15d928a6d93556daf6fd327c": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8Array",
"Int8Array",
"Int8Array",
"VarcharArray",
"Int8Array",
"Int8Array",
"BoolArray",
"NumericArray",
"Int8Array"
]
}
},
"query": "\n INSERT INTO team_members (id, team_id, user_id, role, permissions, organization_permissions, accepted, payouts_split, ordering)\n SELECT * FROM UNNEST ($1::int8[], $2::int8[], $3::int8[], $4::varchar[], $5::int8[], $6::int8[], $7::bool[], $8::numeric[], $9::int8[])\n "
},
"70c812c6a0d29465569169afde42c74a353a534aeedd5cdd81bceb2a7de6bc78": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Int8"
"Bool"
]
}
},
"query": "\n INSERT INTO threads_members (\n thread_id, user_id\n )\n VALUES (\n $1, $2\n )\n "
"query": "\n DELETE FROM mods_categories\n WHERE joining_mod_id = $1 AND is_additional = $2\n "
},
"71abd207410d123f9a50345ddcddee335fea0d0cc6f28762713ee01a36aee8a0": {
"describe": {
@ -3743,19 +3715,6 @@
},
"query": "\n SELECT id FROM mods_gallery\n WHERE image_url = $1\n "
},
"7cb691738c28e0d1f28c84ba2dbcfa21a6dbd859bcf0f565f90cd7ce2ea5aa1c": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Int4"
]
}
},
"query": "\n INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)\n VALUES ($1, $2, FALSE)\n "
},
"7e030d43f3412e7df63c970f873d0a73dd2deb9857aa6f201ec5eec628eb336c": {
"describe": {
"columns": [],
@ -3807,6 +3766,22 @@
},
"query": "\n DELETE FROM historical_payouts\n WHERE user_id = $1\n "
},
"8475c7cb94786576012b16d53a017cb250f0de99b76746d8725798daa3345c5e": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8Array",
"VarcharArray",
"Int8Array",
"Int8Array",
"VarcharArray"
]
}
},
"query": "\n INSERT INTO dependencies (dependent_id, dependency_type, dependency_id, mod_dependency_id, dependency_file_name)\n SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::bigint[], $4::bigint[], $5::varchar[])\n "
},
"85463fa221147ee8d409fc92ed681fa27df683e7c80b8dd8616ae94dc1205c24": {
"describe": {
"columns": [],
@ -3820,23 +3795,6 @@
},
"query": "\n UPDATE versions\n SET author_id = $1\n WHERE (author_id = $2)\n "
},
"85b40877c48fc4f23039c1b556007f92056a015f160fe1059b0d3b13615af0fb": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Int8"
]
}
},
"query": "\n INSERT INTO mods_gallery (\n mod_id, image_url, featured, title, description, ordering\n )\n VALUES (\n $1, $2, $3, $4, $5, $6\n )\n "
},
"85c6de008681d9fc9dc51b17330bed09204010813111e66a7ca84bc0e603f537": {
"describe": {
"columns": [
@ -4820,6 +4778,20 @@
},
"query": "\n SELECT m.id id, tm.user_id user_id, tm.payouts_split payouts_split\n FROM mods m\n INNER JOIN team_members tm on m.team_id = tm.team_id AND tm.accepted = TRUE\n WHERE m.id = ANY($1) AND m.monetization_status = $2\n "
},
"b86145932b1f919fc82414c303ade80f62d4c1bc155f948359b5f6578c680244": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8Array",
"Int4Array",
"BoolArray"
]
}
},
"query": "\n INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)\n SELECT * FROM UNNEST ($1::bigint[], $2::int[], $3::bool[])\n "
},
"b903ac4e686ef85ba28d698c668da07860e7f276b261d8f2cebb74e73b094970": {
"describe": {
"columns": [],
@ -4844,19 +4816,6 @@
},
"query": "\n INSERT INTO teams (id)\n VALUES ($1)\n "
},
"b96ab39ab9624bfcdc8675107544307af9892504c4cbc40e4e7c40a1e4e83e14": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int4",
"Int8"
]
}
},
"query": "\n INSERT INTO game_versions_versions (game_version_id, joining_version_id)\n VALUES ($1, $2)\n "
},
"b971cecafab7046c5952447fd78a6e45856841256d812ce9ae3c07f903c5cc62": {
"describe": {
"columns": [],
@ -5157,20 +5116,6 @@
},
"query": "\n UPDATE users\n SET payout_wallet = NULL, payout_wallet_type = NULL, payout_address = NULL\n WHERE (id = $1)\n "
},
"c545a74e902c5c63bca1057b76e94b9547ee21fadbc61964f45837915d5f4608": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Int4",
"Varchar"
]
}
},
"query": "\n INSERT INTO mods_donations (\n joining_mod_id, joining_platform_id, url\n )\n VALUES (\n $1, $2, $3\n )\n "
},
"c55d2132e3e6e92dd50457affab758623dca175dc27a2d3cd4aace9cfdecf789": {
"describe": {
"columns": [],
@ -5217,26 +5162,6 @@
},
"query": "\n UPDATE mods\n SET client_side = $1\n WHERE (id = $2)\n "
},
"c6060a389343c9f35aea5d931518b85ab7c71b6ba74eae7b4b51d881f1798c6e": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Int8",
"Int8",
"Varchar",
"Int8",
"Int8",
"Bool",
"Numeric",
"Int8"
]
}
},
"query": "\n INSERT INTO team_members (id, team_id, user_id, role, permissions, organization_permissions, accepted, payouts_split, ordering)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n "
},
"c8a27a122160a0896914c786deef9e8193eb240501d30d5ffb4129e2103efd3d": {
"describe": {
"columns": [],
@ -5249,6 +5174,19 @@
},
"query": "\n UPDATE versions\n SET status = requested_status\n WHERE status = $1 AND date_published < CURRENT_DATE AND requested_status IS NOT NULL\n "
},
"c8c0bf5d298810a7a30caf03d7437af757303fa9aa0f500b83476e65cec7f1e9": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8Array",
"Int8Array"
]
}
},
"query": "\n INSERT INTO threads_members (\n thread_id, user_id\n )\n SELECT * FROM UNNEST ($1::int8[], $2::int8[])\n "
},
"c8fde56e5d03eda085519b4407768de7ddf48cae18ce7138a97e8e8fba967e15": {
"describe": {
"columns": [
@ -5376,20 +5314,6 @@
},
"query": "\n SELECT id, user_id, session, created, last_login, expires, refresh_expires, os, platform,\n city, country, ip, user_agent\n FROM sessions\n WHERE id = ANY($1) OR session = ANY($2)\n ORDER BY created DESC\n "
},
"cb57ae673f1a7e50cc319efddb9bdc82e2251596bcf85aea52e8def343e423b8": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Varchar",
"Bytea"
]
}
},
"query": "\n INSERT INTO hashes (file_id, algorithm, hash)\n VALUES ($1, $2, $3)\n "
},
"cb82bb6e22690fd5fee18bbc2975503371814ef1cbf95f32c195bfe7542b2b20": {
"describe": {
"columns": [],
@ -5408,19 +5332,6 @@
},
"query": "\n INSERT INTO team_members (\n id, team_id, user_id, role, permissions, organization_permissions, accepted\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n "
},
"ccce60dc60ca6c4ea1142ab6d0d81bdb1ee9ed97c992695324aec015e0e190bf": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Int8"
]
}
},
"query": "\n INSERT INTO collections_mods (collection_id, mod_id)\n VALUES ($1, $2)\n ON CONFLICT DO NOTHING\n "
},
"ccd913bb2f3006ffe881ce2fc4ef1e721d18fe2eed6ac62627046c955129610c": {
"describe": {
"columns": [
@ -5453,19 +5364,6 @@
},
"query": "\n DELETE FROM hashes\n WHERE file_id = $1\n "
},
"cdf20036b29b61da40bf990c9ab04c509297a4d65bc9b136c9fb20f1e97e1149": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Int4"
]
}
},
"query": "\n INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)\n VALUES ($1, $2, FALSE)\n "
},
"ce20a9c53249e255be7312819f505d935d3ab2ee3c21a6422e5b12155c159bd7": {
"describe": {
"columns": [
@ -5660,6 +5558,20 @@
},
"query": "\n INSERT INTO notifications (\n id, user_id, body\n )\n VALUES (\n $1, $2, $3\n )\n "
},
"d2e826d4fa4e3e730cc84c97964c0c5fdd25cd49ddff8c593bd9b8a3b4d5ff1e": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8Array",
"VarcharArray",
"ByteaArray"
]
}
},
"query": "\n INSERT INTO hashes (file_id, algorithm, hash)\n SELECT * FROM UNNEST($1::bigint[], $2::varchar[], $3::bytea[])\n "
},
"d331ca8f22da418cf654985c822ce4466824beaa00dea64cde90dc651a03024b": {
"describe": {
"columns": [],
@ -5791,19 +5703,6 @@
},
"query": "\n SELECT id, team_id, role AS member_role, permissions, organization_permissions,\n accepted, payouts_split, role,\n ordering, user_id\n FROM team_members\n WHERE (team_id = ANY($1) AND user_id = $2 AND accepted = TRUE)\n ORDER BY ordering\n "
},
"d59a0ca4725d40232eae8bf5735787e1b76282c390d2a8d07fb34e237a0b2132": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Int4"
]
}
},
"query": "\n INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)\n VALUES ($1, $2, TRUE)\n "
},
"d6453e50041b5521fa9e919a9162e533bb9426f8c584d98474c6ad414db715c8": {
"describe": {
"columns": [
@ -6505,6 +6404,19 @@
},
"query": "\n SELECT name FROM project_types pt\n INNER JOIN mods ON mods.project_type = pt.id\n WHERE mods.id = $1\n "
},
"efdaae627a24efdf522c913cfd3600d6331e30dffbba8c2d318e44e260ac5f59": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8Array",
"Int8Array"
]
}
},
"query": "\n INSERT INTO collections_mods (collection_id, mod_id)\n SELECT * FROM UNNEST($1::bigint[], $2::bigint[])\n ON CONFLICT DO NOTHING\n "
},
"f1525930830e17b5ee8feb796d9950dd3741131965f050840fa75423b5a54f01": {
"describe": {
"columns": [],
@ -6723,6 +6635,19 @@
},
"query": "SELECT EXISTS(SELECT 1 FROM mods m INNER JOIN team_members tm ON tm.team_id = m.team_id AND user_id = $2 WHERE m.id = $1)"
},
"fa54ed32004b883daa44eeb413fc2e07b45883608afc6ac91ac6f74736a12256": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int4Array",
"Int8Array"
]
}
},
"query": "\n INSERT INTO game_versions_versions (game_version_id, joining_version_id)\n SELECT * FROM UNNEST($1::integer[], $2::bigint[])\n "
},
"fb955ca41b95120f66c98c0b528b1db10c4be4a55e9641bb104d772e390c9bb7": {
"describe": {
"columns": [
@ -6743,19 +6668,6 @@
},
"query": "SELECT EXISTS(SELECT 1 FROM notifications WHERE id=$1)"
},
"fcd15905507769ab7f9839d64d1be3ee3f61cd555aee57dace76f8e53e91d344": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Int8",
"Int4"
]
}
},
"query": "\n INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)\n VALUES ($1, $2, TRUE)\n "
},
"fce67ce3d0c27c64af85fb7d36661513bc5ea2e96fcf12f3a51c97999b01b83c": {
"describe": {
"columns": [

6
src/clickhouse/mod.rs
View File

@ -6,8 +6,12 @@ mod fetch;
pub use fetch::*;
pub async fn init_client() -> clickhouse::error::Result<clickhouse::Client> {
let database = dotenvy::var("CLICKHOUSE_DATABASE").unwrap();
init_client_with_database(&dotenvy::var("CLICKHOUSE_DATABASE").unwrap()).await
}
pub async fn init_client_with_database(
database: &str,
) -> clickhouse::error::Result<clickhouse::Client> {
let client = {
let mut http_connector = HttpConnector::new();
http_connector.enforce_http(false); // allow https URLs

26
src/database/models/collection_item.rs
View File

@ -81,19 +81,19 @@ impl Collection {
.execute(&mut *transaction)
.await?;
for project_id in self.projects.iter() {
sqlx::query!(
"
INSERT INTO collections_mods (collection_id, mod_id)
VALUES ($1, $2)
ON CONFLICT DO NOTHING
",
self.id as CollectionId,
*project_id as ProjectId,
)
.execute(&mut *transaction)
.await?;
}
let (collection_ids, project_ids): (Vec<_>, Vec<_>) =
self.projects.iter().map(|p| (self.id.0, p.0)).unzip();
sqlx::query!(
"
INSERT INTO collections_mods (collection_id, mod_id)
SELECT * FROM UNNEST($1::bigint[], $2::bigint[])
ON CONFLICT DO NOTHING
",
&collection_ids[..],
&project_ids[..],
)
.execute(&mut *transaction)
.await?;
Ok(())
}

144
src/database/models/project_item.rs
View File

@ -5,6 +5,7 @@ use crate::database::redis::RedisPool;
use crate::models::ids::base62_impl::{parse_base62, to_base62};
use crate::models::projects::{MonetizationStatus, ProjectStatus};
use chrono::{DateTime, Utc};
use itertools::Itertools;
use serde::{Deserialize, Serialize};
pub const PROJECTS_NAMESPACE: &str = "projects";
@ -20,23 +21,25 @@ pub struct DonationUrl {
}
impl DonationUrl {
pub async fn insert_project(
&self,
pub async fn insert_many_projects(
donation_urls: Vec<Self>,
project_id: ProjectId,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<(), sqlx::error::Error> {
let (project_ids, platform_ids, urls): (Vec<_>, Vec<_>, Vec<_>) = donation_urls
.into_iter()
.map(|url| (project_id.0, url.platform_id.0, url.url))
.multiunzip();
sqlx::query!(
"
INSERT INTO mods_donations (
joining_mod_id, joining_platform_id, url
)
VALUES (
$1, $2, $3
)
SELECT * FROM UNNEST($1::bigint[], $2::int[], $3::varchar[])
",
project_id as ProjectId,
self.platform_id as DonationPlatformId,
self.url,
&project_ids[..],
&platform_ids[..],
&urls[..],
)
.execute(&mut *transaction)
.await?;
@ -56,26 +59,76 @@ pub struct GalleryItem {
}
impl GalleryItem {
pub async fn insert(
&self,
pub async fn insert_many(
items: Vec<Self>,
project_id: ProjectId,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<(), sqlx::error::Error> {
let (project_ids, image_urls, featureds, titles, descriptions, orderings): (
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
) = items
.into_iter()
.map(|gi| {
(
project_id.0,
gi.image_url,
gi.featured,
gi.title,
gi.description,
gi.ordering,
)
})
.multiunzip();
sqlx::query!(
"
INSERT INTO mods_gallery (
mod_id, image_url, featured, title, description, ordering
)
VALUES (
$1, $2, $3, $4, $5, $6
)
SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::bool[], $4::varchar[], $5::varchar[], $6::bigint[])
",
project_id as ProjectId,
self.image_url,
self.featured,
self.title,
self.description,
self.ordering
&project_ids[..],
&image_urls[..],
&featureds[..],
&titles[..] as &[Option<String>],
&descriptions[..] as &[Option<String>],
&orderings[..]
)
.execute(&mut *transaction)
.await?;
Ok(())
}
}
#[derive(derive_new::new)]
pub struct ModCategory {
project_id: ProjectId,
category_id: CategoryId,
is_additional: bool,
}
impl ModCategory {
pub async fn insert_many(
items: Vec<Self>,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<(), DatabaseError> {
let (project_ids, category_ids, is_additionals): (Vec<_>, Vec<_>, Vec<_>) = items
.into_iter()
.map(|mc| (mc.project_id.0, mc.category_id.0, mc.is_additional))
.multiunzip();
sqlx::query!(
"
INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)
SELECT * FROM UNNEST ($1::bigint[], $2::int[], $3::bool[])
",
&project_ids[..],
&category_ids[..],
&is_additionals[..]
)
.execute(&mut *transaction)
.await?;
@ -160,46 +213,35 @@ impl ProjectBuilder {
};
project_struct.insert(&mut *transaction).await?;
let ProjectBuilder {
donation_urls,
gallery_items,
categories,
additional_categories,
..
} = self;
for mut version in self.initial_versions {
version.project_id = self.project_id;
version.insert(&mut *transaction).await?;
}
for donation in self.donation_urls {
donation
.insert_project(self.project_id, &mut *transaction)
.await?;
}
for gallery in self.gallery_items {
gallery.insert(self.project_id, &mut *transaction).await?;
}
for category in self.categories {
sqlx::query!(
"
INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)
VALUES ($1, $2, FALSE)
",
self.project_id as ProjectId,
category as CategoryId,
)
.execute(&mut *transaction)
DonationUrl::insert_many_projects(donation_urls, self.project_id, &mut *transaction)
.await?;
}
for category in self.additional_categories {
sqlx::query!(
"
INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)
VALUES ($1, $2, TRUE)
",
self.project_id as ProjectId,
category as CategoryId,
GalleryItem::insert_many(gallery_items, self.project_id, &mut *transaction).await?;
let project_id = self.project_id;
let mod_categories = categories
.into_iter()
.map(|c| ModCategory::new(project_id, c, false))
.chain(
additional_categories
.into_iter()
.map(|c| ModCategory::new(project_id, c, true)),
)
.execute(&mut *transaction)
.await?;
}
.collect_vec();
ModCategory::insert_many(mod_categories, &mut *transaction).await?;
Project::update_game_versions(self.project_id, &mut *transaction).await?;
Project::update_loaders(self.project_id, &mut *transaction).await?;

73
src/database/models/team_item.rs
View File

@ -41,26 +41,61 @@ impl TeamBuilder {
.execute(&mut *transaction)
.await?;
for member in self.members {
let team_member_id = generate_team_member_id(&mut *transaction).await?;
sqlx::query!(
"
INSERT INTO team_members (id, team_id, user_id, role, permissions, organization_permissions, accepted, payouts_split, ordering)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
",
team_member_id as TeamMemberId,
team.id as TeamId,
member.user_id as UserId,
member.role,
member.permissions.bits() as i64,
member.organization_permissions.map(|p| p.bits() as i64),
member.accepted,
member.payouts_split,
member.ordering,
)
.execute(&mut *transaction)
.await?;
let mut team_member_ids = Vec::new();
for _ in self.members.iter() {
team_member_ids.push(generate_team_member_id(&mut *transaction).await?.0);
}
let TeamBuilder { members } = self;
let (
team_ids,
user_ids,
roles,
permissions,
organization_permissions,
accepteds,
payouts_splits,
orderings,
): (
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
) = members
.into_iter()
.map(|m| {
(
team.id.0,
m.user_id.0,
m.role,
m.permissions.bits() as i64,
m.organization_permissions.map(|p| p.bits() as i64),
m.accepted,
m.payouts_split,
m.ordering,
)
})
.multiunzip();
sqlx::query!(
"
INSERT INTO team_members (id, team_id, user_id, role, permissions, organization_permissions, accepted, payouts_split, ordering)
SELECT * FROM UNNEST ($1::int8[], $2::int8[], $3::int8[], $4::varchar[], $5::int8[], $6::int8[], $7::bool[], $8::numeric[], $9::int8[])
",
&team_member_ids[..],
&team_ids[..],
&user_ids[..],
&roles[..],
&permissions[..],
&organization_permissions[..] as &[Option<i64>],
&accepteds[..],
&payouts_splits[..],
&orderings[..],
)
.execute(&mut *transaction)
.await?;
Ok(team_id)
}

28
src/database/models/thread_item.rs
View File

@ -90,22 +90,20 @@ impl ThreadBuilder {
.execute(&mut *transaction)
.await?;
for member in &self.members {
sqlx::query!(
"
INSERT INTO threads_members (
thread_id, user_id
)
VALUES (
$1, $2
)
",
thread_id as ThreadId,
*member as UserId,
let (thread_ids, members): (Vec<_>, Vec<_>) =
self.members.iter().map(|m| (thread_id.0, m.0)).unzip();
sqlx::query!(
"
INSERT INTO threads_members (
thread_id, user_id
)
.execute(&mut *transaction)
.await?;
}
SELECT * FROM UNNEST ($1::int8[], $2::int8[])
",
&thread_ids[..],
&members[..],
)
.execute(&mut *transaction)
.await?;
Ok(thread_id)
}

255
src/database/models/version_item.rs
View File

@ -39,12 +39,59 @@ pub struct DependencyBuilder {
}
impl DependencyBuilder {
pub async fn insert(
self,
pub async fn insert_many(
builders: Vec<Self>,
version_id: VersionId,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<(), DatabaseError> {
let project_id = if let Some(project_id) = self.project_id {
let mut project_ids = Vec::new();
for dependency in builders.iter() {
project_ids.push(
dependency
.try_get_project_id(transaction)
.await?
.map(|id| id.0),
);
}
let (version_ids, dependency_types, dependency_ids, filenames): (
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
) = builders
.into_iter()
.map(|d| {
(
version_id.0,
d.dependency_type,
d.version_id.map(|v| v.0),
d.file_name,
)
})
.multiunzip();
sqlx::query!(
"
INSERT INTO dependencies (dependent_id, dependency_type, dependency_id, mod_dependency_id, dependency_file_name)
SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::bigint[], $4::bigint[], $5::varchar[])
",
&version_ids[..],
&dependency_types[..],
&dependency_ids[..] as &[Option<i64>],
&project_ids[..] as &[Option<i64>],
&filenames[..] as &[Option<String>],
)
.execute(&mut *transaction)
.await?;
Ok(())
}
async fn try_get_project_id(
&self,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<Option<ProjectId>, DatabaseError> {
Ok(if let Some(project_id) = self.project_id {
Some(project_id)
} else if let Some(version_id) = self.version_id {
sqlx::query!(
@ -58,23 +105,7 @@ impl DependencyBuilder {
.map(|x| ProjectId(x.mod_id))
} else {
None
};
sqlx::query!(
"
INSERT INTO dependencies (dependent_id, dependency_type, dependency_id, mod_dependency_id, dependency_file_name)
VALUES ($1, $2, $3, $4, $5)
",
version_id as VersionId,
self.dependency_type,
self.version_id.map(|x| x.0),
project_id.map(|x| x.0),
self.file_name,
)
.execute(&mut *transaction)
.await?;
Ok(())
})
}
}
@ -89,42 +120,70 @@ pub struct VersionFileBuilder {
}
impl VersionFileBuilder {
pub async fn insert(
self,
pub async fn insert_many(
version_files: Vec<Self>,
version_id: VersionId,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<FileId, DatabaseError> {
let file_id = generate_file_id(&mut *transaction).await?;
let (file_ids, version_ids, urls, filenames, primary, sizes, file_types): (
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
) = version_files
.iter()
.map(|f| {
(
file_id.0,
version_id.0,
f.url.clone(),
f.filename.clone(),
f.primary,
f.size as i32,
f.file_type.map(|x| x.to_string()),
)
})
.multiunzip();
sqlx::query!(
"
INSERT INTO files (id, version_id, url, filename, is_primary, size, file_type)
VALUES ($1, $2, $3, $4, $5, $6, $7)
SELECT * FROM UNNEST($1::bigint[], $2::bigint[], $3::varchar[], $4::varchar[], $5::bool[], $6::integer[], $7::varchar[])
",
file_id as FileId,
version_id as VersionId,
self.url,
self.filename,
self.primary,
self.size as i32,
self.file_type.map(|x| x.as_str()),
&file_ids[..],
&version_ids[..],
&urls[..],
&filenames[..],
&primary[..],
&sizes[..],
&file_types[..] as &[Option<String>],
)
.execute(&mut *transaction)
.await?;
for hash in self.hashes {
sqlx::query!(
"
INSERT INTO hashes (file_id, algorithm, hash)
VALUES ($1, $2, $3)
",
file_id as FileId,
hash.algorithm,
hash.hash,
)
.execute(&mut *transaction)
.await?;
}
let (file_ids, algorithms, hashes): (Vec<_>, Vec<_>, Vec<_>) = version_files
.into_iter()
.flat_map(|f| {
f.hashes
.into_iter()
.map(|h| (file_id.0, h.algorithm, h.hash))
})
.multiunzip();
sqlx::query!(
"
INSERT INTO hashes (file_id, algorithm, hash)
SELECT * FROM UNNEST($1::bigint[], $2::varchar[], $3::bytea[])
",
&file_ids[..],
&algorithms[..],
&hashes[..],
)
.execute(&mut *transaction)
.await?;
Ok(file_id)
}
@ -170,44 +229,94 @@ impl VersionBuilder {
.execute(&mut *transaction)
.await?;
for file in self.files {
file.insert(self.version_id, transaction).await?;
}
let VersionBuilder {
dependencies,
loaders,
game_versions,
files,
version_id,
..
} = self;
VersionFileBuilder::insert_many(files, self.version_id, transaction).await?;
for dependency in self.dependencies {
dependency.insert(self.version_id, transaction).await?;
}
DependencyBuilder::insert_many(dependencies, self.version_id, transaction).await?;
for loader in self.loaders.clone() {
sqlx::query!(
"
INSERT INTO loaders_versions (loader_id, version_id)
VALUES ($1, $2)
",
loader as LoaderId,
self.version_id as VersionId,
)
.execute(&mut *transaction)
.await?;
}
let loader_versions = loaders
.iter()
.map(|l| LoaderVersion::new(*l, version_id))
.collect_vec();
LoaderVersion::insert_many(loader_versions, &mut *transaction).await?;
for game_version in self.game_versions.clone() {
sqlx::query!(
"
INSERT INTO game_versions_versions (game_version_id, joining_version_id)
VALUES ($1, $2)
",
game_version as GameVersionId,
self.version_id as VersionId,
)
.execute(&mut *transaction)
.await?;
}
let game_version_versions = game_versions
.iter()
.map(|v| VersionVersion::new(*v, version_id))
.collect_vec();
VersionVersion::insert_many(game_version_versions, &mut *transaction).await?;
Ok(self.version_id)
}
}
#[derive(derive_new::new)]
pub struct LoaderVersion {
pub loader_id: LoaderId,
pub version_id: VersionId,
}
impl LoaderVersion {
pub async fn insert_many(
items: Vec<Self>,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<(), DatabaseError> {
let (loader_ids, version_ids): (Vec<_>, Vec<_>) = items
.iter()
.map(|l| (l.loader_id.0, l.version_id.0))
.unzip();
sqlx::query!(
"
INSERT INTO loaders_versions (loader_id, version_id)
SELECT * FROM UNNEST($1::integer[], $2::bigint[])
",
&loader_ids[..],
&version_ids[..],
)
.execute(&mut *transaction)
.await?;
Ok(())
}
}
#[derive(derive_new::new)]
pub struct VersionVersion {
pub game_version_id: GameVersionId,
pub joining_version_id: VersionId,
}
impl VersionVersion {
pub async fn insert_many(
items: Vec<Self>,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<(), DatabaseError> {
let (game_version_ids, version_ids): (Vec<_>, Vec<_>) = items
.into_iter()
.map(|i| (i.game_version_id.0, i.joining_version_id.0))
.unzip();
sqlx::query!(
"
INSERT INTO game_versions_versions (game_version_id, joining_version_id)
SELECT * FROM UNNEST($1::integer[], $2::bigint[])
",
&game_version_ids[..],
&version_ids[..],
)
.execute(&mut *transaction)
.await?;
Ok(())
}
}
#[derive(Clone, Deserialize, Serialize)]
pub struct Version {
pub id: VersionId,

71
src/queue/analytics.rs
View File

@ -1,6 +1,13 @@
use crate::models::analytics::{Download, PageView, Playtime};
use dashmap::DashSet;
#[cfg(test)]
mod tests;
const VIEWS_TABLENAME: &str = "views";
const DOWNLOADS_TABLENAME: &str = "downloads";
const PLAYTIME_TABLENAME: &str = "playtime";
pub struct AnalyticsQueue {
views_queue: DashSet<PageView>,
downloads_queue: DashSet<Download>,
@ -17,54 +24,50 @@ impl AnalyticsQueue {
}
}
pub async fn add_view(&self, page_view: PageView) {
pub fn add_view(&self, page_view: PageView) {
self.views_queue.insert(page_view);
}
pub async fn add_download(&self, download: Download) {
pub fn add_download(&self, download: Download) {
self.downloads_queue.insert(download);
}
pub async fn add_playtime(&self, playtime: Playtime) {
pub fn add_playtime(&self, playtime: Playtime) {
self.playtime_queue.insert(playtime);
}
pub async fn index(&self, client: clickhouse::Client) -> Result<(), clickhouse::error::Error> {
let views_queue = self.views_queue.clone();
self.views_queue.clear();
Self::index_queue(&client, &self.views_queue, VIEWS_TABLENAME).await?;
Self::index_queue(&client, &self.downloads_queue, DOWNLOADS_TABLENAME).await?;
Self::index_queue(&client, &self.playtime_queue, PLAYTIME_TABLENAME).await?;
let downloads_queue = self.downloads_queue.clone();
self.downloads_queue.clear();
Ok(())
}
let playtime_queue = self.playtime_queue.clone();
self.playtime_queue.clear();
if !views_queue.is_empty() || !downloads_queue.is_empty() || !playtime_queue.is_empty() {
let mut views = client.insert("views")?;
for view in views_queue {
views.write(&view).await?;
}
views.end().await?;
let mut downloads = client.insert("downloads")?;
for download in downloads_queue {
downloads.write(&download).await?;
}
downloads.end().await?;
let mut playtimes = client.insert("playtime")?;
for playtime in playtime_queue {
playtimes.write(&playtime).await?;
}
playtimes.end().await?;
async fn index_queue<T>(
client: &clickhouse::Client,
queue: &DashSet<T>,
table_name: &str,
) -> Result<(), clickhouse::error::Error>
where
T: serde::Serialize + Eq + std::hash::Hash + Clone + clickhouse::Row,
{
if queue.is_empty() {
return Ok(());
}
let current_queue = queue.clone();
queue.clear();
let mut inserter = client.inserter(table_name)?;
for row in current_queue {
inserter.write(&row).await?;
inserter.commit().await?;
}
inserter.end().await?;
Ok(())
}
}

128
src/queue/analytics/tests.rs Normal file
View File

@ -0,0 +1,128 @@
use futures::Future;
use uuid::Uuid;
use super::*;
use crate::clickhouse::init_client_with_database;
use std::net::Ipv6Addr;
#[tokio::test]
async fn test_indexing() {
with_test_clickhouse_db(|clickhouse_client| async move {
let analytics = AnalyticsQueue::new();
analytics.add_download(get_default_download());
analytics.add_playtime(get_default_playtime());
analytics.add_view(get_default_views());
analytics.index(clickhouse_client.clone()).await.unwrap();
assert_table_counts(&clickhouse_client, 1, 1, 1).await;
analytics.index(clickhouse_client.clone()).await.unwrap();
assert_table_counts(&clickhouse_client, 1, 1, 1).await;
})
.await;
}
#[tokio::test]
async fn can_insert_many_downloads() {
with_test_clickhouse_db(|clickhouse_client| async move {
let analytics = AnalyticsQueue::new();
let n_downloads = 100_000;
for _ in 0..n_downloads {
analytics.add_download(get_default_download());
}
analytics.index(clickhouse_client.clone()).await.unwrap();
assert_table_count(DOWNLOADS_TABLENAME, &clickhouse_client, n_downloads).await;
})
.await;
}
async fn assert_table_counts(
client: &clickhouse::Client,
downloads: u64,
playtimes: u64,
views: u64,
) {
assert_table_count(DOWNLOADS_TABLENAME, client, downloads).await;
assert_table_count(PLAYTIME_TABLENAME, client, playtimes).await;
assert_table_count(VIEWS_TABLENAME, client, views).await;
}
async fn assert_table_count(table_name: &str, client: &clickhouse::Client, expected_count: u64) {
let count = client
.query(&format!("SELECT COUNT(*) from {table_name}"))
.fetch_one::<u64>()
.await
.unwrap();
assert_eq!(expected_count, count);
}
async fn with_test_clickhouse_db<Fut>(f: impl FnOnce(clickhouse::Client) -> Fut)
where
Fut: Future<Output = ()>,
{
let db_name = format!("test_{}", uuid::Uuid::new_v4().as_simple());
println!("Clickhouse test db: {}", db_name);
let clickhouse_client = init_client_with_database(&db_name)
.await
.expect("A real clickhouse instance should be running locally");
f(clickhouse_client.clone()).await;
clickhouse_client
.query(&format!("DROP DATABASE IF EXISTS {db_name}"))
.execute()
.await
.unwrap();
}
fn get_default_download() -> Download {
Download {
id: Uuid::new_v4(),
recorded: Default::default(),
domain: Default::default(),
site_path: Default::default(),
user_id: Default::default(),
project_id: Default::default(),
version_id: Default::default(),
ip: get_default_ipv6(),
country: Default::default(),
user_agent: Default::default(),
headers: Default::default(),
}
}
fn get_default_playtime() -> Playtime {
Playtime {
id: Uuid::new_v4(),
recorded: Default::default(),
seconds: Default::default(),
user_id: Default::default(),
project_id: Default::default(),
version_id: Default::default(),
loader: Default::default(),
game_version: Default::default(),
parent: Default::default(),
}
}
fn get_default_views() -> PageView {
PageView {
id: Uuid::new_v4(),
recorded: Default::default(),
domain: Default::default(),
site_path: Default::default(),
user_id: Default::default(),
project_id: Default::default(),
ip: get_default_ipv6(),
country: Default::default(),
user_agent: Default::default(),
headers: Default::default(),
}
}
fn get_default_ipv6() -> Ipv6Addr {
Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)
}

31
src/queue/payouts.rs
View File

@ -355,6 +355,8 @@ pub async fn process_payout(
};
let mut clear_cache_users = Vec::new();
let (mut insert_user_ids, mut insert_project_ids, mut insert_payouts, mut insert_starts) =
(Vec::new(), Vec::new(), Vec::new(), Vec::new());
for (id, project) in projects_map {
if let Some(value) = &multipliers.values.get(&(id as u64)) {
let project_multiplier: Decimal =
@ -367,18 +369,10 @@ pub async fn process_payout(
let payout: Decimal = payout * project_multiplier * (split / sum_splits);
if payout > Decimal::ZERO {
sqlx::query!(
"
INSERT INTO payouts_values (user_id, mod_id, amount, created)
VALUES ($1, $2, $3, $4)
",
user_id,
id,
payout,
start
)
.execute(&mut *transaction)
.await?;
insert_user_ids.push(user_id);
insert_project_ids.push(id);
insert_payouts.push(payout);
insert_starts.push(start);
sqlx::query!(
"
@ -399,6 +393,19 @@ pub async fn process_payout(
}
}
sqlx::query!(
"
INSERT INTO payouts_values (user_id, mod_id, amount, created)
SELECT * FROM UNNEST ($1::bigint[], $2::bigint[], $3::numeric[], $4::timestamptz[])
",
&insert_user_ids[..],
&insert_project_ids[..],
&insert_payouts[..],
&insert_starts[..]
)
.execute(&mut *transaction)
.await?;
if !clear_cache_users.is_empty() {
crate::database::models::User::clear_caches(
&clear_cache_users

26
src/routes/analytics.rs
View File

@ -150,7 +150,7 @@ pub async fn page_view_ingest(
view.user_id = user.id.0;
}
analytics_queue.add_view(view).await;
analytics_queue.add_view(view);
Ok(HttpResponse::NoContent().body(""))
}
@ -202,19 +202,17 @@ pub async fn playtime_ingest(
}
if let Some(version) = versions.iter().find(|x| id == x.inner.id.into()) {
analytics_queue
.add_playtime(Playtime {
id: Default::default(),
recorded: Utc::now().timestamp_nanos() / 100_000,
seconds: playtime.seconds as u64,
user_id: user.id.0,
project_id: version.inner.project_id.0 as u64,
version_id: version.inner.id.0 as u64,
loader: playtime.loader,
game_version: playtime.game_version,
parent: playtime.parent.map(|x| x.0).unwrap_or(0),
})
.await;
analytics_queue.add_playtime(Playtime {
id: Default::default(),
recorded: Utc::now().timestamp_nanos() / 100_000,
seconds: playtime.seconds as u64,
user_id: user.id.0,
project_id: version.inner.project_id.0 as u64,
version_id: version.inner.id.0 as u64,
loader: playtime.loader,
game_version: playtime.game_version,
parent: playtime.parent.map(|x| x.0).unwrap_or(0),
});
}
}

64
src/routes/v2/admin.rs
View File

@ -108,40 +108,36 @@ pub async fn count_download(
let ip = crate::routes::analytics::convert_to_ip_v6(&download_body.ip)
.unwrap_or_else(|_| Ipv4Addr::new(127, 0, 0, 1).to_ipv6_mapped());
analytics_queue
.add_download(Download {
id: Uuid::new_v4(),
recorded: Utc::now().timestamp_nanos() / 100_000,
domain: url.host_str().unwrap_or_default().to_string(),
site_path: url.path().to_string(),
user_id: user
.and_then(|(scopes, x)| {
if scopes.contains(Scopes::PERFORM_ANALYTICS) {
Some(x.id.0 as u64)
} else {
None
}
})
.unwrap_or(0),
project_id: project_id as u64,
version_id: version_id as u64,
ip,
country: maxmind.query(ip).await.unwrap_or_default(),
user_agent: download_body
.headers
.get("user-agent")
.cloned()
.unwrap_or_default(),
headers: download_body
.headers
.clone()
.into_iter()
.filter(|x| {
!crate::routes::analytics::FILTERED_HEADERS.contains(&&*x.0.to_lowercase())
})
.collect(),
})
.await;
analytics_queue.add_download(Download {
id: Uuid::new_v4(),
recorded: Utc::now().timestamp_nanos() / 100_000,
domain: url.host_str().unwrap_or_default().to_string(),
site_path: url.path().to_string(),
user_id: user
.and_then(|(scopes, x)| {
if scopes.contains(Scopes::PERFORM_ANALYTICS) {
Some(x.id.0 as u64)
} else {
None
}
})
.unwrap_or(0),
project_id: project_id as u64,
version_id: version_id as u64,
ip,
country: maxmind.query(ip).await.unwrap_or_default(),
user_agent: download_body
.headers
.get("user-agent")
.cloned()
.unwrap_or_default(),
headers: download_body
.headers
.clone()
.into_iter()
.filter(|x| !crate::routes::analytics::FILTERED_HEADERS.contains(&&*x.0.to_lowercase()))
.collect(),
});
Ok(HttpResponse::NoContent().body(""))
}

32
src/routes/v2/collections.rs
View File

@ -15,6 +15,7 @@ use crate::{database, models};
use actix_web::web::Data;
use actix_web::{delete, get, patch, post, web, HttpRequest, HttpResponse};
use chrono::Utc;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use sqlx::PgPool;
use std::sync::Arc;
@ -301,6 +302,11 @@ pub async fn collection_edit(
.execute(&mut *transaction)
.await?;
let collection_item_ids = new_project_ids
.iter()
.map(|_| collection_item.id.0)
.collect_vec();
let mut validated_project_ids = Vec::new();
for project_id in new_project_ids {
let project = database::models::Project::get(project_id, &**pool, &redis)
.await?
@ -309,20 +315,20 @@ pub async fn collection_edit(
"The specified project {project_id} does not exist!"
))
})?;
// Insert- don't throw an error if it already exists
sqlx::query!(
"
INSERT INTO collections_mods (collection_id, mod_id)
VALUES ($1, $2)
ON CONFLICT DO NOTHING
",
collection_item.id as database::models::ids::CollectionId,
project.inner.id as database::models::ids::ProjectId,
)
.execute(&mut *transaction)
.await?;
validated_project_ids.push(project.inner.id.0);
}
// Insert- don't throw an error if it already exists
sqlx::query!(
"
INSERT INTO collections_mods (collection_id, mod_id)
SELECT * FROM UNNEST ($1::int8[], $2::int8[])
ON CONFLICT DO NOTHING
",
&collection_item_ids[..],
&validated_project_ids[..],
)
.execute(&mut *transaction)
.await?;
}
database::models::Collection::clear_cache(collection_item.id, &redis).await?;

595
src/routes/v2/projects.rs
View File

File diff suppressed because it is too large Load Diff

4
src/routes/v2/version_creation.rs
View File

@ -725,9 +725,7 @@ async fn upload_file_to_version_inner(
"At least one file must be specified".to_string(),
));
} else {
for file_builder in file_builders {
file_builder.insert(version_id, &mut *transaction).await?;
}
VersionFileBuilder::insert_many(file_builders, version_id, &mut *transaction).await?;
}
// Clear version cache

39
src/routes/v2/versions.rs
View File

@ -3,6 +3,7 @@ use crate::auth::{
filter_authorized_versions, get_user_from_headers, is_authorized, is_authorized_version,
};
use crate::database;
use crate::database::models::version_item::{DependencyBuilder, LoaderVersion, VersionVersion};
use crate::database::models::{image_item, Organization};
use crate::database::redis::RedisPool;
use crate::models;
@ -450,11 +451,12 @@ pub async fn version_edit(
})
.collect::<Vec<database::models::version_item::DependencyBuilder>>();
for dependency in builders {
dependency
.insert(version_item.inner.id, &mut transaction)
.await?;
}
DependencyBuilder::insert_many(
builders,
version_item.inner.id,
&mut transaction,
)
.await?;
}
}
}
@ -469,6 +471,7 @@ pub async fn version_edit(
.execute(&mut *transaction)
.await?;
let mut version_versions = Vec::new();
for game_version in game_versions {
let game_version_id = database::models::categories::GameVersion::get_id(
&game_version.0,
@ -481,17 +484,9 @@ pub async fn version_edit(
)
})?;
sqlx::query!(
"
INSERT INTO game_versions_versions (game_version_id, joining_version_id)
VALUES ($1, $2)
",
game_version_id as database::models::ids::GameVersionId,
id as database::models::ids::VersionId,
)
.execute(&mut *transaction)
.await?;
version_versions.push(VersionVersion::new(game_version_id, id));
}
VersionVersion::insert_many(version_versions, &mut transaction).await?;
database::models::Project::update_game_versions(
version_item.inner.project_id,
@ -510,6 +505,7 @@ pub async fn version_edit(
.execute(&mut *transaction)
.await?;
let mut loader_versions = Vec::new();
for loader in loaders {
let loader_id =
database::models::categories::Loader::get_id(&loader.0, &mut *transaction)
@ -519,18 +515,9 @@ pub async fn version_edit(
"No database entry for loader provided.".to_string(),
)
})?;
sqlx::query!(
"
INSERT INTO loaders_versions (loader_id, version_id)
VALUES ($1, $2)
",
loader_id as database::models::ids::LoaderId,
id as database::models::ids::VersionId,
)
.execute(&mut *transaction)
.await?;
loader_versions.push(LoaderVersion::new(loader_id, id));
}
LoaderVersion::insert_many(loader_versions, &mut transaction).await?;
database::models::Project::update_loaders(
version_item.inner.project_id,

5
tests/common/database.rs
View File

@ -29,6 +29,7 @@ pub const USER_USER_PAT: &str = "mrp_patuser";
pub const FRIEND_USER_PAT: &str = "mrp_patfriend";
pub const ENEMY_USER_PAT: &str = "mrp_patenemy";
#[derive(Clone)]
pub struct TemporaryDatabase {
pub pool: PgPool,
pub redis_pool: RedisPool,
@ -75,10 +76,14 @@ impl TemporaryDatabase {
.await
.expect("Connection to temporary database failed");
println!("Running migrations on temporary database");
// Performs migrations
let migrations = sqlx::migrate!("./migrations");
migrations.run(&pool).await.expect("Migrations failed");
println!("Migrations complete");
// Gets new Redis pool
let redis_pool = RedisPool::new(Some(temp_database_name.clone()));

10
tests/common/dummy_data.rs
View File

@ -13,6 +13,16 @@ use super::{
environment::TestEnvironment,
};
pub const DUMMY_CATEGORIES: &'static [&str] = &[
"combat",
"decoration",
"economy",
"food",
"magic",
"mobs",
"optimization",
];
pub struct DummyData {
pub alpha_team_id: String,
pub beta_team_id: String,

13
tests/common/environment.rs
View File

@ -3,6 +3,19 @@
use super::{database::TemporaryDatabase, dummy_data};
use crate::common::setup;
use actix_web::{dev::ServiceResponse, test, App};
use futures::Future;
pub async fn with_test_environment<Fut>(f: impl FnOnce(TestEnvironment) -> Fut)
where
Fut: Future<Output = ()>,
{
let test_env = TestEnvironment::build_with_dummy().await;
let db = test_env.db.clone();
f(test_env).await;
db.cleanup().await;
}
// A complete test environment, with a test actix app and a database.
// Must be called in an #[actix_rt::test] context. It also simulates a

22
tests/files/dummy_data.sql
View File

@ -27,10 +27,20 @@ INSERT INTO loaders (id, loader) VALUES (1, 'fabric');
INSERT INTO loaders_project_types (joining_loader_id, joining_project_type_id) VALUES (1,1);
INSERT INTO loaders_project_types (joining_loader_id, joining_project_type_id) VALUES (1,2);
INSERT INTO categories (id, category, project_type) VALUES (1, 'combat', 1);
INSERT INTO categories (id, category, project_type) VALUES (2, 'decoration', 1);
INSERT INTO categories (id, category, project_type) VALUES (3, 'economy', 1);
INSERT INTO categories (id, category, project_type) VALUES
(1, 'combat', 1),
(2, 'decoration', 1),
(3, 'economy', 1),
(4, 'food', 1),
(5, 'magic', 1),
(6, 'mobs', 1),
(7, 'optimization', 1);
INSERT INTO categories (id, category, project_type) VALUES (4, 'combat', 2);
INSERT INTO categories (id, category, project_type) VALUES (5, 'decoration', 2);
INSERT INTO categories (id, category, project_type) VALUES (6, 'economy', 2);
INSERT INTO categories (id, category, project_type) VALUES
(101, 'combat', 2),
(102, 'decoration', 2),
(103, 'economy', 2),
(104, 'food', 2),
(105, 'magic', 2),
(106, 'mobs', 2),
(107, 'optimization', 2);

71
tests/project.rs
View File

@ -1,10 +1,14 @@
use actix_http::StatusCode;
use actix_web::dev::ServiceResponse;
use actix_web::test;
use common::environment::with_test_environment;
use labrinth::database::models::project_item::{PROJECTS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE};
use labrinth::models::ids::base62_impl::parse_base62;
use serde_json::json;
use crate::common::database::*;
use crate::common::dummy_data::DUMMY_CATEGORIES;
use crate::common::{actix::AppendsMultipart, environment::TestEnvironment};
// importing common module.
@ -403,7 +407,7 @@ pub async fn test_patch_project() {
"title": "New successful title",
"description": "New successful description",
"body": "New successful body",
"categories": ["combat"],
"categories": [DUMMY_CATEGORIES[0]],
"license_id": "MIT",
"issues_url": "https://github.com",
"discord_url": "https://discord.gg",
@ -441,7 +445,7 @@ pub async fn test_patch_project() {
assert_eq!(body["title"], json!("New successful title"));
assert_eq!(body["description"], json!("New successful description"));
assert_eq!(body["body"], json!("New successful body"));
assert_eq!(body["categories"], json!(["combat"]));
assert_eq!(body["categories"], json!([DUMMY_CATEGORIES[0]]));
assert_eq!(body["license"]["id"], json!("MIT"));
assert_eq!(body["issues_url"], json!("https://github.com"));
assert_eq!(body["discord_url"], json!("https://discord.gg"));
@ -457,5 +461,68 @@ pub async fn test_patch_project() {
test_env.cleanup().await;
}
#[actix_rt::test]
pub async fn test_bulk_edit_categories() {
with_test_environment(|test_env| async move {
let alpha_project_id = &test_env.dummy.as_ref().unwrap().alpha_project_id;
let beta_project_id = &test_env.dummy.as_ref().unwrap().beta_project_id;
let req = test::TestRequest::patch()
.uri(&format!(
"/v2/projects?ids={}",
urlencoding::encode(&format!("[\"{alpha_project_id}\",\"{beta_project_id}\"]"))
))
.append_header(("Authorization", ADMIN_USER_PAT))
.set_json(json!({
"categories": [DUMMY_CATEGORIES[0], DUMMY_CATEGORIES[3]],
"add_categories": [DUMMY_CATEGORIES[1], DUMMY_CATEGORIES[2]],
"remove_categories": [DUMMY_CATEGORIES[3]],
"additional_categories": [DUMMY_CATEGORIES[4], DUMMY_CATEGORIES[6]],
"add_additional_categories": [DUMMY_CATEGORIES[5]],
"remove_additional_categories": [DUMMY_CATEGORIES[6]],
}))
.to_request();
let resp = test_env.call(req).await;
assert_eq!(resp.status(), StatusCode::NO_CONTENT);
let alpha_body = get_project_body(&test_env, &alpha_project_id, ADMIN_USER_PAT).await;
assert_eq!(alpha_body["categories"], json!(DUMMY_CATEGORIES[0..=2]));
assert_eq!(
alpha_body["additional_categories"],
json!(DUMMY_CATEGORIES[4..=5])
);
let beta_body = get_project_body(&test_env, &beta_project_id, ADMIN_USER_PAT).await;
assert_eq!(beta_body["categories"], alpha_body["categories"]);
assert_eq!(
beta_body["additional_categories"],
alpha_body["additional_categories"],
);
})
.await;
}
async fn get_project(
test_env: &TestEnvironment,
project_slug: &str,
user_pat: &str,
) -> ServiceResponse {
let req = test::TestRequest::get()
.uri(&format!("/v2/project/{project_slug}"))
.append_header(("Authorization", user_pat))
.to_request();
test_env.call(req).await
}
async fn get_project_body(
test_env: &TestEnvironment,
project_slug: &str,
user_pat: &str,
) -> serde_json::Value {
let resp = get_project(test_env, project_slug, user_pat).await;
assert_eq!(resp.status(), StatusCode::OK);
test::read_body_json(resp).await
}
// TODO: Missing routes on projects
// TODO: using permissions/scopes, can we SEE projects existence that we are not allowed to? (ie 401 instead of 404)