Bommels05/Modrinth
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix redis pool timeout (#669)

Browse Source 
* Fix redis pool timeout

* remove search dep project issues

* run fmt + prep
This commit is contained in:
Geometrically 2023-08-06 15:34:03 -07:00 committed by GitHub
parent 1f4ad732fd
commit e9f5bd4ac1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 191 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

338
sqlx-data.json
View File

@ -507,178 +507,6 @@
}, },
"query": "\n UPDATE notifications\n SET read = TRUE\n WHERE id = ANY($1)\n " "query": "\n UPDATE notifications\n SET read = TRUE\n WHERE id = ANY($1)\n "
}, },
"1510e820cdc31b62222d2be6a838e69876fe3c3c742dea188ca1a6b7f894d610": {
"describe": {
"columns": [
{
"name": "id",
"ordinal": 0,
"type_info": "Int8"
},
{
"name": "project_type",
"ordinal": 1,
"type_info": "Int4"
},
{
"name": "title",
"ordinal": 2,
"type_info": "Varchar"
},
{
"name": "description",
"ordinal": 3,
"type_info": "Varchar"
},
{
"name": "downloads",
"ordinal": 4,
"type_info": "Int4"
},
{
"name": "follows",
"ordinal": 5,
"type_info": "Int4"
},
{
"name": "icon_url",
"ordinal": 6,
"type_info": "Varchar"
},
{
"name": "published",
"ordinal": 7,
"type_info": "Timestamptz"
},
{
"name": "approved",
"ordinal": 8,
"type_info": "Timestamptz"
},
{
"name": "updated",
"ordinal": 9,
"type_info": "Timestamptz"
},
{
"name": "team_id",
"ordinal": 10,
"type_info": "Int8"
},
{
"name": "license",
"ordinal": 11,
"type_info": "Varchar"
},
{
"name": "slug",
"ordinal": 12,
"type_info": "Varchar"
},
{
"name": "status_name",
"ordinal": 13,
"type_info": "Varchar"
},
{
"name": "color",
"ordinal": 14,
"type_info": "Int4"
},
{
"name": "client_side_type",
"ordinal": 15,
"type_info": "Varchar"
},
{
"name": "server_side_type",
"ordinal": 16,
"type_info": "Varchar"
},
{
"name": "project_type_name",
"ordinal": 17,
"type_info": "Varchar"
},
{
"name": "username",
"ordinal": 18,
"type_info": "Varchar"
},
{
"name": "categories",
"ordinal": 19,
"type_info": "VarcharArray"
},
{
"name": "additional_categories",
"ordinal": 20,
"type_info": "VarcharArray"
},
{
"name": "loaders",
"ordinal": 21,
"type_info": "VarcharArray"
},
{
"name": "versions",
"ordinal": 22,
"type_info": "VarcharArray"
},
{
"name": "gallery",
"ordinal": 23,
"type_info": "VarcharArray"
},
{
"name": "featured_gallery",
"ordinal": 24,
"type_info": "VarcharArray"
},
{
"name": "dependencies",
"ordinal": 25,
"type_info": "Jsonb"
}
],
"nullable": [
false,
false,
false,
false,
false,
false,
true,
false,
true,
false,
false,
false,
true,
false,
true,
false,
false,
false,
false,
null,
null,
null,
null,
null,
null,
null
],
"parameters": {
"Left": [
"TextArray",
"TextArray",
"Text"
]
}
},
"query": "\n SELECT m.id id, m.project_type project_type, m.title title, m.description description, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.published published, m.approved approved, m.updated updated,\n m.team_id team_id, m.license license, m.slug slug, m.status status_name, m.color color,\n cs.name client_side_type, ss.name server_side_type, pt.name project_type_name, u.username username,\n ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is false) categories,\n ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is true) additional_categories,\n ARRAY_AGG(DISTINCT lo.loader) filter (where lo.loader is not null) loaders,\n ARRAY_AGG(DISTINCT gv.version) filter (where gv.version is not null) versions,\n ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is false) gallery,\n ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is true) featured_gallery,\n JSONB_AGG(DISTINCT jsonb_build_object('id', mdep.id, 'dep_type', d.dependency_type)) filter (where mdep.id is not null) dependencies\n FROM mods m\n LEFT OUTER JOIN mods_categories mc ON joining_mod_id = m.id\n LEFT OUTER JOIN categories c ON mc.joining_category_id = c.id\n LEFT OUTER JOIN versions v ON v.mod_id = m.id AND v.status != ALL($1)\n LEFT OUTER JOIN game_versions_versions gvv ON gvv.joining_version_id = v.id\n LEFT OUTER JOIN game_versions gv ON gvv.game_version_id = gv.id\n LEFT OUTER JOIN loaders_versions lv ON lv.version_id = v.id\n LEFT OUTER JOIN loaders lo ON lo.id = lv.loader_id\n LEFT OUTER JOIN mods_gallery mg ON mg.mod_id = m.id\n LEFT OUTER JOIN dependencies d ON d.dependent_id = v.id\n LEFT OUTER JOIN mods mdep ON mdep.id = d.mod_dependency_id\n INNER JOIN project_types pt ON pt.id = m.project_type\n INNER JOIN side_types cs ON m.client_side = cs.id\n INNER JOIN side_types ss ON m.server_side = ss.id\n INNER JOIN team_members tm ON tm.team_id = m.team_id AND tm.role = $3 AND tm.accepted = TRUE\n INNER JOIN users u ON tm.user_id = u.id\n WHERE m.status = ANY($2)\n GROUP BY m.id, cs.id, ss.id, pt.id, u.id;\n "
},
"16049957962ded08751d5a4ddce2ffac17ecd486f61210c51a952508425d83e6": { "16049957962ded08751d5a4ddce2ffac17ecd486f61210c51a952508425d83e6": {
"describe": { "describe": {
"columns": [], "columns": [],
@ -1598,6 +1426,172 @@
}, },
"query": "\n SELECT gv.id id, gv.version version_, gv.type type_, gv.created created, gv.major major FROM game_versions gv\n WHERE major = $1 AND type = $2\n ORDER BY created DESC\n " "query": "\n SELECT gv.id id, gv.version version_, gv.type type_, gv.created created, gv.major major FROM game_versions gv\n WHERE major = $1 AND type = $2\n ORDER BY created DESC\n "
}, },
"4514723bdc1eb8a781215075bec51af1cc6fabe88a469338d5a59533eabf80c5": {
"describe": {
"columns": [
{
"name": "id",
"ordinal": 0,
"type_info": "Int8"
},
{
"name": "project_type",
"ordinal": 1,
"type_info": "Int4"
},
{
"name": "title",
"ordinal": 2,
"type_info": "Varchar"
},
{
"name": "description",
"ordinal": 3,
"type_info": "Varchar"
},
{
"name": "downloads",
"ordinal": 4,
"type_info": "Int4"
},
{
"name": "follows",
"ordinal": 5,
"type_info": "Int4"
},
{
"name": "icon_url",
"ordinal": 6,
"type_info": "Varchar"
},
{
"name": "published",
"ordinal": 7,
"type_info": "Timestamptz"
},
{
"name": "approved",
"ordinal": 8,
"type_info": "Timestamptz"
},
{
"name": "updated",
"ordinal": 9,
"type_info": "Timestamptz"
},
{
"name": "team_id",
"ordinal": 10,
"type_info": "Int8"
},
{
"name": "license",
"ordinal": 11,
"type_info": "Varchar"
},
{
"name": "slug",
"ordinal": 12,
"type_info": "Varchar"
},
{
"name": "status_name",
"ordinal": 13,
"type_info": "Varchar"
},
{
"name": "color",
"ordinal": 14,
"type_info": "Int4"
},
{
"name": "client_side_type",
"ordinal": 15,
"type_info": "Varchar"
},
{
"name": "server_side_type",
"ordinal": 16,
"type_info": "Varchar"
},
{
"name": "project_type_name",
"ordinal": 17,
"type_info": "Varchar"
},
{
"name": "username",
"ordinal": 18,
"type_info": "Varchar"
},
{
"name": "categories",
"ordinal": 19,
"type_info": "VarcharArray"
},
{
"name": "additional_categories",
"ordinal": 20,
"type_info": "VarcharArray"
},
{
"name": "loaders",
"ordinal": 21,
"type_info": "VarcharArray"
},
{
"name": "versions",
"ordinal": 22,
"type_info": "VarcharArray"
},
{
"name": "gallery",
"ordinal": 23,
"type_info": "VarcharArray"
},
{
"name": "featured_gallery",
"ordinal": 24,
"type_info": "VarcharArray"
}
],
"nullable": [
false,
false,
false,
false,
false,
false,
true,
false,
true,
false,
false,
false,
true,
false,
true,
false,
false,
false,
false,
null,
null,
null,
null,
null,
null
],
"parameters": {
"Left": [
"TextArray",
"TextArray",
"Text"
]
}
},
"query": "\n SELECT m.id id, m.project_type project_type, m.title title, m.description description, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.published published, m.approved approved, m.updated updated,\n m.team_id team_id, m.license license, m.slug slug, m.status status_name, m.color color,\n cs.name client_side_type, ss.name server_side_type, pt.name project_type_name, u.username username,\n ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is false) categories,\n ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is true) additional_categories,\n ARRAY_AGG(DISTINCT lo.loader) filter (where lo.loader is not null) loaders,\n ARRAY_AGG(DISTINCT gv.version) filter (where gv.version is not null) versions,\n ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is false) gallery,\n ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is true) featured_gallery\n FROM mods m\n LEFT OUTER JOIN mods_categories mc ON joining_mod_id = m.id\n LEFT OUTER JOIN categories c ON mc.joining_category_id = c.id\n LEFT OUTER JOIN versions v ON v.mod_id = m.id AND v.status != ALL($1)\n LEFT OUTER JOIN game_versions_versions gvv ON gvv.joining_version_id = v.id\n LEFT OUTER JOIN game_versions gv ON gvv.game_version_id = gv.id\n LEFT OUTER JOIN loaders_versions lv ON lv.version_id = v.id\n LEFT OUTER JOIN loaders lo ON lo.id = lv.loader_id\n LEFT OUTER JOIN mods_gallery mg ON mg.mod_id = m.id\n INNER JOIN project_types pt ON pt.id = m.project_type\n INNER JOIN side_types cs ON m.client_side = cs.id\n INNER JOIN side_types ss ON m.server_side = ss.id\n INNER JOIN team_members tm ON tm.team_id = m.team_id AND tm.role = $3 AND tm.accepted = TRUE\n INNER JOIN users u ON tm.user_id = u.id\n WHERE m.status = ANY($2)\n GROUP BY m.id, cs.id, ss.id, pt.id, u.id;\n "
},
"4567790f0dc98ff20b596a33161d1f6ac8af73da67fe8c54192724626c6bf670": { "4567790f0dc98ff20b596a33161d1f6ac8af73da67fe8c54192724626c6bf670": {
"describe": { "describe": {
"columns": [], "columns": [],

6
src/auth/minecraft/stages/bearer_token.rs
View File

@ -21,7 +21,7 @@ pub async fn fetch_bearer(token: &str, uhs: &str) -> Result<String, Authenticati
.get("access_token") .get("access_token")
.and_then(serde_json::Value::as_str) .and_then(serde_json::Value::as_str)
.map(String::from) .map(String::from)
.ok_or(AuthenticationError::Custom( .ok_or(AuthenticationError::Custom(format!(
format!("Response didn't contain valid bearer token. body: {body}"), "Response didn't contain valid bearer token. body: {body}"
)) )))
} }

8
src/queue/payouts.rs
View File

@ -349,6 +349,7 @@ pub async fn process_payout(
_ => weekday_amount, _ => weekday_amount,
}; };
let mut clear_cache_users = Vec::new();
for (id, project) in projects_map { for (id, project) in projects_map {
if let Some(value) = &multipliers.values.get(&(id as u64)) { if let Some(value) = &multipliers.values.get(&(id as u64)) {
let project_multiplier: Decimal = let project_multiplier: Decimal =
@ -356,8 +357,6 @@ pub async fn process_payout(
let sum_splits: Decimal = project.team_members.iter().map(|x| x.1).sum(); let sum_splits: Decimal = project.team_members.iter().map(|x| x.1).sum();
let mut clear_cache_users = Vec::new();
if sum_splits > Decimal::ZERO { if sum_splits > Decimal::ZERO {
for (user_id, split) in project.team_members { for (user_id, split) in project.team_members {
let payout: Decimal = payout * project_multiplier * (split / sum_splits); let payout: Decimal = payout * project_multiplier * (split / sum_splits);
@ -387,11 +386,15 @@ pub async fn process_payout(
) )
.execute(&mut *transaction) .execute(&mut *transaction)
.await?; .await?;
clear_cache_users.push(user_id); clear_cache_users.push(user_id);
} }
} }
} }
}
}
if !clear_cache_users.is_empty() {
crate::database::models::User::clear_caches( crate::database::models::User::clear_caches(
&clear_cache_users &clear_cache_users
.into_iter() .into_iter()
@ -401,7 +404,6 @@ pub async fn process_payout(
) )
.await?; .await?;
} }
}
transaction.commit().await?; transaction.commit().await?;

8
src/routes/mod.rs
View File

@ -1,21 +1,21 @@
use actix_cors::Cors;
use crate::file_hosting::FileHostingError; use crate::file_hosting::FileHostingError;
use crate::routes::analytics::{page_view_ingest, playtime_ingest};
use crate::util::cors::default_cors; use crate::util::cors::default_cors;
use crate::util::env::parse_strings_from_var;
use actix_cors::Cors;
use actix_files::Files; use actix_files::Files;
use actix_web::http::StatusCode; use actix_web::http::StatusCode;
use actix_web::{web, HttpResponse}; use actix_web::{web, HttpResponse};
use futures::FutureExt; use futures::FutureExt;
use crate::routes::analytics::{page_view_ingest, playtime_ingest};
use crate::util::env::parse_strings_from_var;
pub mod v2; pub mod v2;
pub mod v3; pub mod v3;
mod analytics;
mod index; mod index;
mod maven; mod maven;
mod not_found; mod not_found;
mod updates; mod updates;
mod analytics;
pub use self::not_found::not_found; pub use self::not_found::not_found;

4
src/routes/v2/admin.rs
View File

@ -135,7 +135,9 @@ pub async fn count_download(
.headers .headers
.clone() .clone()
.into_iter() .into_iter()
.filter(|x| !crate::routes::analytics::FILTERED_HEADERS.contains(&&*x.0.to_lowercase())) .filter(|x| {
!crate::routes::analytics::FILTERED_HEADERS.contains(&&*x.0.to_lowercase())
})
.collect(), .collect(),
}) })
.await; .await;

22
src/search/indexing/local_import.rs
View File

@ -4,7 +4,6 @@ use log::info;
use super::IndexingError; use super::IndexingError;
use crate::database::models::ProjectId; use crate::database::models::ProjectId;
use crate::search::UploadSearchProject; use crate::search::UploadSearchProject;
use serde::Deserialize;
use sqlx::postgres::PgPool; use sqlx::postgres::PgPool;
pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchProject>, IndexingError> { pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchProject>, IndexingError> {
@ -22,8 +21,7 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchProject>, Index
ARRAY_AGG(DISTINCT lo.loader) filter (where lo.loader is not null) loaders, ARRAY_AGG(DISTINCT lo.loader) filter (where lo.loader is not null) loaders,
ARRAY_AGG(DISTINCT gv.version) filter (where gv.version is not null) versions, ARRAY_AGG(DISTINCT gv.version) filter (where gv.version is not null) versions,
ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is false) gallery, ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is false) gallery,
ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is true) featured_gallery, ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is true) featured_gallery
JSONB_AGG(DISTINCT jsonb_build_object('id', mdep.id, 'dep_type', d.dependency_type)) filter (where mdep.id is not null) dependencies
FROM mods m FROM mods m
LEFT OUTER JOIN mods_categories mc ON joining_mod_id = m.id LEFT OUTER JOIN mods_categories mc ON joining_mod_id = m.id
LEFT OUTER JOIN categories c ON mc.joining_category_id = c.id LEFT OUTER JOIN categories c ON mc.joining_category_id = c.id
@ -33,8 +31,6 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchProject>, Index
LEFT OUTER JOIN loaders_versions lv ON lv.version_id = v.id LEFT OUTER JOIN loaders_versions lv ON lv.version_id = v.id
LEFT OUTER JOIN loaders lo ON lo.id = lv.loader_id LEFT OUTER JOIN loaders lo ON lo.id = lv.loader_id
LEFT OUTER JOIN mods_gallery mg ON mg.mod_id = m.id LEFT OUTER JOIN mods_gallery mg ON mg.mod_id = m.id
LEFT OUTER JOIN dependencies d ON d.dependent_id = v.id
LEFT OUTER JOIN mods mdep ON mdep.id = d.mod_dependency_id
INNER JOIN project_types pt ON pt.id = m.project_type INNER JOIN project_types pt ON pt.id = m.project_type
INNER JOIN side_types cs ON m.client_side = cs.id INNER JOIN side_types cs ON m.client_side = cs.id
INNER JOIN side_types ss ON m.server_side = ss.id INNER JOIN side_types ss ON m.server_side = ss.id
@ -72,21 +68,6 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchProject>, Index
_ => false, _ => false,
}; };
#[derive(Deserialize)]
struct TempDependency {
id: ProjectId,
dep_type: String
}
let dependencies = serde_json::from_value::<Vec<TempDependency>>(
m.dependencies.unwrap_or_default(),
)
.ok()
.unwrap_or_default()
.into_iter()
.map(|x| format!("{}-{}", crate::models::ids::ProjectId::from(x.id), x.dep_type))
.collect();
UploadSearchProject { UploadSearchProject {
project_id: project_id.to_string(), project_id: project_id.to_string(),
title: m.title, title: m.title,
@ -112,7 +93,6 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchProject>, Index
open_source, open_source,
color: m.color.map(|x| x as u32), color: m.color.map(|x| x as u32),
featured_gallery: m.featured_gallery.unwrap_or_default().first().cloned(), featured_gallery: m.featured_gallery.unwrap_or_default().first().cloned(),
dependencies,
} }
})) }))
}) })

2
src/search/indexing/mod.rs
View File

@ -181,7 +181,6 @@ const DEFAULT_DISPLAYED_ATTRIBUTES: &[&str] = &[
"gallery", "gallery",
"featured_gallery", "featured_gallery",
"color", "color",
"dependencies",
]; ];
const DEFAULT_SEARCHABLE_ATTRIBUTES: &[&str] = &["title", "description", "author", "slug"]; const DEFAULT_SEARCHABLE_ATTRIBUTES: &[&str] = &["title", "description", "author", "slug"];
@ -202,7 +201,6 @@ const DEFAULT_ATTRIBUTES_FOR_FACETING: &[&str] = &[
"project_id", "project_id",
"open_source", "open_source",
"color", "color",
"dependencies",
]; ];
const DEFAULT_SORTABLE_ATTRIBUTES: &[&str] = const DEFAULT_SORTABLE_ATTRIBUTES: &[&str] =

4
src/search/mod.rs
View File

@ -99,8 +99,6 @@ pub struct UploadSearchProject {
pub modified_timestamp: i64, pub modified_timestamp: i64,
pub open_source: bool, pub open_source: bool,
pub color: Option<u32>, pub color: Option<u32>,
/// format: {project_id}-{dep_type}
pub dependencies: Vec<String>,
} }
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]
@ -136,8 +134,6 @@ pub struct ResultSearchProject {
pub gallery: Vec<String>, pub gallery: Vec<String>,
pub featured_gallery: Option<String>, pub featured_gallery: Option<String>,
pub color: Option<u32>, pub color: Option<u32>,
/// format: {project_id}-{dep_type}
pub dependencies: Vec<String>,
} }
pub async fn search_for_project( pub async fn search_for_project(