From e9f5bd4ac1ab79c56ec2b45271e8d6238fd84b1b Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Sun, 6 Aug 2023 15:34:03 -0700 Subject: [PATCH] Fix redis pool timeout (#669) * Fix redis pool timeout * remove search dep project issues * run fmt + prep --- sqlx-data.json | 338 +++++++++++----------- src/auth/minecraft/stages/bearer_token.rs | 6 +- src/queue/payouts.rs | 24 +- src/routes/mod.rs | 10 +- src/routes/v2/admin.rs | 4 +- src/search/indexing/local_import.rs | 22 +- src/search/indexing/mod.rs | 2 - src/search/mod.rs | 4 - 8 files changed, 191 insertions(+), 219 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 351d888b8..33765861a 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -507,178 +507,6 @@ }, "query": "\n UPDATE notifications\n SET read = TRUE\n WHERE id = ANY($1)\n " }, - "1510e820cdc31b62222d2be6a838e69876fe3c3c742dea188ca1a6b7f894d610": { - "describe": { - "columns": [ - { - "name": "id", - "ordinal": 0, - "type_info": "Int8" - }, - { - "name": "project_type", - "ordinal": 1, - "type_info": "Int4" - }, - { - "name": "title", - "ordinal": 2, - "type_info": "Varchar" - }, - { - "name": "description", - "ordinal": 3, - "type_info": "Varchar" - }, - { - "name": "downloads", - "ordinal": 4, - "type_info": "Int4" - }, - { - "name": "follows", - "ordinal": 5, - "type_info": "Int4" - }, - { - "name": "icon_url", - "ordinal": 6, - "type_info": "Varchar" - }, - { - "name": "published", - "ordinal": 7, - "type_info": "Timestamptz" - }, - { - "name": "approved", - "ordinal": 8, - "type_info": "Timestamptz" - }, - { - "name": "updated", - "ordinal": 9, - "type_info": "Timestamptz" - }, - { - "name": "team_id", - "ordinal": 10, - "type_info": "Int8" - }, - { - "name": "license", - "ordinal": 11, - "type_info": "Varchar" - }, - { - "name": "slug", - "ordinal": 12, - "type_info": "Varchar" - }, - { - "name": "status_name", - "ordinal": 13, - "type_info": "Varchar" - }, - { - "name": "color", - "ordinal": 14, - "type_info": "Int4" - }, - { - "name": "client_side_type", - "ordinal": 15, - "type_info": "Varchar" - }, - { - "name": "server_side_type", - "ordinal": 16, - "type_info": "Varchar" - }, - { - "name": "project_type_name", - "ordinal": 17, - "type_info": "Varchar" - }, - { - "name": "username", - "ordinal": 18, - "type_info": "Varchar" - }, - { - "name": "categories", - "ordinal": 19, - "type_info": "VarcharArray" - }, - { - "name": "additional_categories", - "ordinal": 20, - "type_info": "VarcharArray" - }, - { - "name": "loaders", - "ordinal": 21, - "type_info": "VarcharArray" - }, - { - "name": "versions", - "ordinal": 22, - "type_info": "VarcharArray" - }, - { - "name": "gallery", - "ordinal": 23, - "type_info": "VarcharArray" - }, - { - "name": "featured_gallery", - "ordinal": 24, - "type_info": "VarcharArray" - }, - { - "name": "dependencies", - "ordinal": 25, - "type_info": "Jsonb" - } - ], - "nullable": [ - false, - false, - false, - false, - false, - false, - true, - false, - true, - false, - false, - false, - true, - false, - true, - false, - false, - false, - false, - null, - null, - null, - null, - null, - null, - null - ], - "parameters": { - "Left": [ - "TextArray", - "TextArray", - "Text" - ] - } - }, - "query": "\n SELECT m.id id, m.project_type project_type, m.title title, m.description description, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.published published, m.approved approved, m.updated updated,\n m.team_id team_id, m.license license, m.slug slug, m.status status_name, m.color color,\n cs.name client_side_type, ss.name server_side_type, pt.name project_type_name, u.username username,\n ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is false) categories,\n ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is true) additional_categories,\n ARRAY_AGG(DISTINCT lo.loader) filter (where lo.loader is not null) loaders,\n ARRAY_AGG(DISTINCT gv.version) filter (where gv.version is not null) versions,\n ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is false) gallery,\n ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is true) featured_gallery,\n JSONB_AGG(DISTINCT jsonb_build_object('id', mdep.id, 'dep_type', d.dependency_type)) filter (where mdep.id is not null) dependencies\n FROM mods m\n LEFT OUTER JOIN mods_categories mc ON joining_mod_id = m.id\n LEFT OUTER JOIN categories c ON mc.joining_category_id = c.id\n LEFT OUTER JOIN versions v ON v.mod_id = m.id AND v.status != ALL($1)\n LEFT OUTER JOIN game_versions_versions gvv ON gvv.joining_version_id = v.id\n LEFT OUTER JOIN game_versions gv ON gvv.game_version_id = gv.id\n LEFT OUTER JOIN loaders_versions lv ON lv.version_id = v.id\n LEFT OUTER JOIN loaders lo ON lo.id = lv.loader_id\n LEFT OUTER JOIN mods_gallery mg ON mg.mod_id = m.id\n LEFT OUTER JOIN dependencies d ON d.dependent_id = v.id\n LEFT OUTER JOIN mods mdep ON mdep.id = d.mod_dependency_id\n INNER JOIN project_types pt ON pt.id = m.project_type\n INNER JOIN side_types cs ON m.client_side = cs.id\n INNER JOIN side_types ss ON m.server_side = ss.id\n INNER JOIN team_members tm ON tm.team_id = m.team_id AND tm.role = $3 AND tm.accepted = TRUE\n INNER JOIN users u ON tm.user_id = u.id\n WHERE m.status = ANY($2)\n GROUP BY m.id, cs.id, ss.id, pt.id, u.id;\n " - }, "16049957962ded08751d5a4ddce2ffac17ecd486f61210c51a952508425d83e6": { "describe": { "columns": [], @@ -1598,6 +1426,172 @@ }, "query": "\n SELECT gv.id id, gv.version version_, gv.type type_, gv.created created, gv.major major FROM game_versions gv\n WHERE major = $1 AND type = $2\n ORDER BY created DESC\n " }, + "4514723bdc1eb8a781215075bec51af1cc6fabe88a469338d5a59533eabf80c5": { + "describe": { + "columns": [ + { + "name": "id", + "ordinal": 0, + "type_info": "Int8" + }, + { + "name": "project_type", + "ordinal": 1, + "type_info": "Int4" + }, + { + "name": "title", + "ordinal": 2, + "type_info": "Varchar" + }, + { + "name": "description", + "ordinal": 3, + "type_info": "Varchar" + }, + { + "name": "downloads", + "ordinal": 4, + "type_info": "Int4" + }, + { + "name": "follows", + "ordinal": 5, + "type_info": "Int4" + }, + { + "name": "icon_url", + "ordinal": 6, + "type_info": "Varchar" + }, + { + "name": "published", + "ordinal": 7, + "type_info": "Timestamptz" + }, + { + "name": "approved", + "ordinal": 8, + "type_info": "Timestamptz" + }, + { + "name": "updated", + "ordinal": 9, + "type_info": "Timestamptz" + }, + { + "name": "team_id", + "ordinal": 10, + "type_info": "Int8" + }, + { + "name": "license", + "ordinal": 11, + "type_info": "Varchar" + }, + { + "name": "slug", + "ordinal": 12, + "type_info": "Varchar" + }, + { + "name": "status_name", + "ordinal": 13, + "type_info": "Varchar" + }, + { + "name": "color", + "ordinal": 14, + "type_info": "Int4" + }, + { + "name": "client_side_type", + "ordinal": 15, + "type_info": "Varchar" + }, + { + "name": "server_side_type", + "ordinal": 16, + "type_info": "Varchar" + }, + { + "name": "project_type_name", + "ordinal": 17, + "type_info": "Varchar" + }, + { + "name": "username", + "ordinal": 18, + "type_info": "Varchar" + }, + { + "name": "categories", + "ordinal": 19, + "type_info": "VarcharArray" + }, + { + "name": "additional_categories", + "ordinal": 20, + "type_info": "VarcharArray" + }, + { + "name": "loaders", + "ordinal": 21, + "type_info": "VarcharArray" + }, + { + "name": "versions", + "ordinal": 22, + "type_info": "VarcharArray" + }, + { + "name": "gallery", + "ordinal": 23, + "type_info": "VarcharArray" + }, + { + "name": "featured_gallery", + "ordinal": 24, + "type_info": "VarcharArray" + } + ], + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + false, + true, + false, + false, + false, + true, + false, + true, + false, + false, + false, + false, + null, + null, + null, + null, + null, + null + ], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "Text" + ] + } + }, + "query": "\n SELECT m.id id, m.project_type project_type, m.title title, m.description description, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.published published, m.approved approved, m.updated updated,\n m.team_id team_id, m.license license, m.slug slug, m.status status_name, m.color color,\n cs.name client_side_type, ss.name server_side_type, pt.name project_type_name, u.username username,\n ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is false) categories,\n ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is true) additional_categories,\n ARRAY_AGG(DISTINCT lo.loader) filter (where lo.loader is not null) loaders,\n ARRAY_AGG(DISTINCT gv.version) filter (where gv.version is not null) versions,\n ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is false) gallery,\n ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is true) featured_gallery\n FROM mods m\n LEFT OUTER JOIN mods_categories mc ON joining_mod_id = m.id\n LEFT OUTER JOIN categories c ON mc.joining_category_id = c.id\n LEFT OUTER JOIN versions v ON v.mod_id = m.id AND v.status != ALL($1)\n LEFT OUTER JOIN game_versions_versions gvv ON gvv.joining_version_id = v.id\n LEFT OUTER JOIN game_versions gv ON gvv.game_version_id = gv.id\n LEFT OUTER JOIN loaders_versions lv ON lv.version_id = v.id\n LEFT OUTER JOIN loaders lo ON lo.id = lv.loader_id\n LEFT OUTER JOIN mods_gallery mg ON mg.mod_id = m.id\n INNER JOIN project_types pt ON pt.id = m.project_type\n INNER JOIN side_types cs ON m.client_side = cs.id\n INNER JOIN side_types ss ON m.server_side = ss.id\n INNER JOIN team_members tm ON tm.team_id = m.team_id AND tm.role = $3 AND tm.accepted = TRUE\n INNER JOIN users u ON tm.user_id = u.id\n WHERE m.status = ANY($2)\n GROUP BY m.id, cs.id, ss.id, pt.id, u.id;\n " + }, "4567790f0dc98ff20b596a33161d1f6ac8af73da67fe8c54192724626c6bf670": { "describe": { "columns": [], diff --git a/src/auth/minecraft/stages/bearer_token.rs b/src/auth/minecraft/stages/bearer_token.rs index 5b1e2f056..d6e04c3da 100644 --- a/src/auth/minecraft/stages/bearer_token.rs +++ b/src/auth/minecraft/stages/bearer_token.rs @@ -21,7 +21,7 @@ pub async fn fetch_bearer(token: &str, uhs: &str) -> Result weekday_amount, }; + let mut clear_cache_users = Vec::new(); for (id, project) in projects_map { if let Some(value) = &multipliers.values.get(&(id as u64)) { let project_multiplier: Decimal = @@ -356,8 +357,6 @@ pub async fn process_payout( let sum_splits: Decimal = project.team_members.iter().map(|x| x.1).sum(); - let mut clear_cache_users = Vec::new(); - if sum_splits > Decimal::ZERO { for (user_id, split) in project.team_members { let payout: Decimal = payout * project_multiplier * (split / sum_splits); @@ -387,22 +386,25 @@ pub async fn process_payout( ) .execute(&mut *transaction) .await?; + clear_cache_users.push(user_id); } } } - - crate::database::models::User::clear_caches( - &clear_cache_users - .into_iter() - .map(|x| (crate::database::models::UserId(x), None)) - .collect::>(), - redis, - ) - .await?; } } + if !clear_cache_users.is_empty() { + crate::database::models::User::clear_caches( + &clear_cache_users + .into_iter() + .map(|x| (crate::database::models::UserId(x), None)) + .collect::>(), + redis, + ) + .await?; + } + transaction.commit().await?; Ok(()) diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 4faeac0bc..4a80c6d27 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -1,21 +1,21 @@ -use actix_cors::Cors; use crate::file_hosting::FileHostingError; +use crate::routes::analytics::{page_view_ingest, playtime_ingest}; use crate::util::cors::default_cors; +use crate::util::env::parse_strings_from_var; +use actix_cors::Cors; use actix_files::Files; use actix_web::http::StatusCode; use actix_web::{web, HttpResponse}; use futures::FutureExt; -use crate::routes::analytics::{page_view_ingest, playtime_ingest}; -use crate::util::env::parse_strings_from_var; pub mod v2; pub mod v3; +mod analytics; mod index; mod maven; mod not_found; mod updates; -mod analytics; pub use self::not_found::not_found; @@ -40,7 +40,7 @@ pub fn root_config(cfg: &mut web::ServiceConfig) { allowed_origins.contains(&"*".to_string()) || allowed_origins - .contains(&origin.to_str().unwrap_or_default().to_string()) + .contains(&origin.to_str().unwrap_or_default().to_string()) }) .allowed_methods(vec!["GET", "POST"]) .allowed_headers(vec![ diff --git a/src/routes/v2/admin.rs b/src/routes/v2/admin.rs index 4bb763488..4b0f193fd 100644 --- a/src/routes/v2/admin.rs +++ b/src/routes/v2/admin.rs @@ -135,7 +135,9 @@ pub async fn count_download( .headers .clone() .into_iter() - .filter(|x| !crate::routes::analytics::FILTERED_HEADERS.contains(&&*x.0.to_lowercase())) + .filter(|x| { + !crate::routes::analytics::FILTERED_HEADERS.contains(&&*x.0.to_lowercase()) + }) .collect(), }) .await; diff --git a/src/search/indexing/local_import.rs b/src/search/indexing/local_import.rs index 8af575d85..7c612bee3 100644 --- a/src/search/indexing/local_import.rs +++ b/src/search/indexing/local_import.rs @@ -4,7 +4,6 @@ use log::info; use super::IndexingError; use crate::database::models::ProjectId; use crate::search::UploadSearchProject; -use serde::Deserialize; use sqlx::postgres::PgPool; pub async fn index_local(pool: PgPool) -> Result, IndexingError> { @@ -22,8 +21,7 @@ pub async fn index_local(pool: PgPool) -> Result, Index ARRAY_AGG(DISTINCT lo.loader) filter (where lo.loader is not null) loaders, ARRAY_AGG(DISTINCT gv.version) filter (where gv.version is not null) versions, ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is false) gallery, - ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is true) featured_gallery, - JSONB_AGG(DISTINCT jsonb_build_object('id', mdep.id, 'dep_type', d.dependency_type)) filter (where mdep.id is not null) dependencies + ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is true) featured_gallery FROM mods m LEFT OUTER JOIN mods_categories mc ON joining_mod_id = m.id LEFT OUTER JOIN categories c ON mc.joining_category_id = c.id @@ -33,8 +31,6 @@ pub async fn index_local(pool: PgPool) -> Result, Index LEFT OUTER JOIN loaders_versions lv ON lv.version_id = v.id LEFT OUTER JOIN loaders lo ON lo.id = lv.loader_id LEFT OUTER JOIN mods_gallery mg ON mg.mod_id = m.id - LEFT OUTER JOIN dependencies d ON d.dependent_id = v.id - LEFT OUTER JOIN mods mdep ON mdep.id = d.mod_dependency_id INNER JOIN project_types pt ON pt.id = m.project_type INNER JOIN side_types cs ON m.client_side = cs.id INNER JOIN side_types ss ON m.server_side = ss.id @@ -72,21 +68,6 @@ pub async fn index_local(pool: PgPool) -> Result, Index _ => false, }; - #[derive(Deserialize)] - struct TempDependency { - id: ProjectId, - dep_type: String - } - - let dependencies = serde_json::from_value::>( - m.dependencies.unwrap_or_default(), - ) - .ok() - .unwrap_or_default() - .into_iter() - .map(|x| format!("{}-{}", crate::models::ids::ProjectId::from(x.id), x.dep_type)) - .collect(); - UploadSearchProject { project_id: project_id.to_string(), title: m.title, @@ -112,7 +93,6 @@ pub async fn index_local(pool: PgPool) -> Result, Index open_source, color: m.color.map(|x| x as u32), featured_gallery: m.featured_gallery.unwrap_or_default().first().cloned(), - dependencies, } })) }) diff --git a/src/search/indexing/mod.rs b/src/search/indexing/mod.rs index 9e29f9d45..cea9dd02e 100644 --- a/src/search/indexing/mod.rs +++ b/src/search/indexing/mod.rs @@ -181,7 +181,6 @@ const DEFAULT_DISPLAYED_ATTRIBUTES: &[&str] = &[ "gallery", "featured_gallery", "color", - "dependencies", ]; const DEFAULT_SEARCHABLE_ATTRIBUTES: &[&str] = &["title", "description", "author", "slug"]; @@ -202,7 +201,6 @@ const DEFAULT_ATTRIBUTES_FOR_FACETING: &[&str] = &[ "project_id", "open_source", "color", - "dependencies", ]; const DEFAULT_SORTABLE_ATTRIBUTES: &[&str] = diff --git a/src/search/mod.rs b/src/search/mod.rs index 67c702eda..af8155a11 100644 --- a/src/search/mod.rs +++ b/src/search/mod.rs @@ -99,8 +99,6 @@ pub struct UploadSearchProject { pub modified_timestamp: i64, pub open_source: bool, pub color: Option, - /// format: {project_id}-{dep_type} - pub dependencies: Vec, } #[derive(Serialize, Deserialize, Debug)] @@ -136,8 +134,6 @@ pub struct ResultSearchProject { pub gallery: Vec, pub featured_gallery: Option, pub color: Option, - /// format: {project_id}-{dep_type} - pub dependencies: Vec, } pub async fn search_for_project(