Bommels05/Modrinth
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add clickhouse replication, exclude bad prom metrics (#3344)

Browse Source
This commit is contained in:
Jai Agrawal 2025-03-05 15:40:46 -08:00 committed by GitHub
parent 0d223e3ab5
commit c1bb934fc6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 31 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.lock generated
View File

@ -265,9 +265,9 @@ dependencies = [
[[package]]
name = "actix-web-prom"
version = "0.8.0"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76743e67d4e7efa9fc2ac7123de0dd7b2ca592668e19334f1d81a3b077afc6ac"
checksum = "56a34f1825c3ae06567a9d632466809bbf34963c86002e8921b64f32d48d289d"
dependencies = [
"actix-web",
"futures-core",

1
apps/labrinth/.env
View File

@ -95,6 +95,7 @@ SENDY_API_KEY=none
ANALYTICS_ALLOWED_ORIGINS='["http://127.0.0.1:3000", "http://localhost:3000", "https://modrinth.com", "https://www.modrinth.com", "*"]'
CLICKHOUSE_REPLICATED=false
CLICKHOUSE_URL=http://localhost:8123
CLICKHOUSE_USER=default
CLICKHOUSE_PASSWORD=

2
apps/labrinth/Cargo.toml
View File

@ -17,7 +17,7 @@ actix-multipart = "0.6.1"
actix-cors = "0.7.0"
actix-ws = "0.3.0"
actix-files = "0.6.5"
actix-web-prom = { version = "0.8.0", features = ["process"] }
actix-web-prom = { version = "0.9.0", features = ["process"] }
governor = "0.6.3"
tokio = { version = "1.35.1", features = ["sync"] }

29
apps/labrinth/src/clickhouse/mod.rs
View File

@ -35,10 +35,24 @@ pub async fn init_client_with_database(
.execute()
.await?;
let clickhouse_replicated =
dotenvy::var("CLICKHOUSE_REPLICATED").unwrap() == "true";
let cluster_line = if clickhouse_replicated {
"ON cluster '{cluster}'"
} else {
""
};
let engine = if clickhouse_replicated {
"ReplicatedMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}', '{replica}')"
} else {
"MergeTree()"
};
client
.query(&format!(
"
CREATE TABLE IF NOT EXISTS {database}.views
CREATE TABLE IF NOT EXISTS {database}.views {cluster_line}
(
recorded DateTime64(4),
domain String,
@ -53,8 +67,9 @@ pub async fn init_client_with_database(
user_agent String,
headers Array(Tuple(String, String))
)
ENGINE = MergeTree()
ENGINE = {engine}
PRIMARY KEY (project_id, recorded, ip)
SETTINGS index_granularity = 8192
"
))
.execute()
@ -63,7 +78,7 @@ pub async fn init_client_with_database(
client
.query(&format!(
"
CREATE TABLE IF NOT EXISTS {database}.downloads
CREATE TABLE IF NOT EXISTS {database}.downloads {cluster_line}
(
recorded DateTime64(4),
domain String,
@ -78,8 +93,9 @@ pub async fn init_client_with_database(
user_agent String,
headers Array(Tuple(String, String))
)
ENGINE = MergeTree()
ENGINE = {engine}
PRIMARY KEY (project_id, recorded, ip)
SETTINGS index_granularity = 8192
"
))
.execute()
@ -88,7 +104,7 @@ pub async fn init_client_with_database(
client
.query(&format!(
"
CREATE TABLE IF NOT EXISTS {database}.playtime
CREATE TABLE IF NOT EXISTS {database}.playtime {cluster_line}
(
recorded DateTime64(4),
seconds UInt64,
@ -101,8 +117,9 @@ pub async fn init_client_with_database(
game_version String,
parent UInt64
)
ENGINE = MergeTree()
ENGINE = {engine}
PRIMARY KEY (project_id, recorded, user_id)
SETTINGS index_granularity = 8192
"
))
.execute()

1
apps/labrinth/src/lib.rs
View File

@ -473,6 +473,7 @@ pub fn check_env_vars() -> bool {
failed |= true;
}
failed |= check_var::<bool>("CLICKHOUSE_REPLICATED");
failed |= check_var::<String>("CLICKHOUSE_URL");
failed |= check_var::<String>("CLICKHOUSE_USER");
failed |= check_var::<String>("CLICKHOUSE_PASSWORD");

3
apps/labrinth/src/main.rs
View File

@ -92,7 +92,10 @@ async fn main() -> std::io::Result<()> {
let prometheus = PrometheusMetricsBuilder::new("labrinth")
.endpoint("/metrics")
.exclude_regex(r"^/api/v1/.*$")
.exclude_regex(r"^/maven/.*$")
.exclude("/_internal/launcher_socket")
.mask_unmatched_patterns("UNKNOWN")
.build()
.expect("Failed to create prometheus metrics middleware");