Add clickhouse replication, exclude bad prom metrics (#3344)

Cargo.lock

@@ -265,9 +265,9 @@ dependencies = [
 
 [[package]]
 name = "actix-web-prom"
-version = "0.8.0"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76743e67d4e7efa9fc2ac7123de0dd7b2ca592668e19334f1d81a3b077afc6ac"
+checksum = "56a34f1825c3ae06567a9d632466809bbf34963c86002e8921b64f32d48d289d"
 dependencies = [
  "actix-web",
  "futures-core",

apps/labrinth/.env

@@ -95,6 +95,7 @@ SENDY_API_KEY=none
 
 ANALYTICS_ALLOWED_ORIGINS='["http://127.0.0.1:3000", "http://localhost:3000", "https://modrinth.com", "https://www.modrinth.com", "*"]'
 
+CLICKHOUSE_REPLICATED=false
 CLICKHOUSE_URL=http://localhost:8123
 CLICKHOUSE_USER=default
 CLICKHOUSE_PASSWORD=

apps/labrinth/Cargo.toml

@@ -17,7 +17,7 @@ actix-multipart = "0.6.1"
 actix-cors = "0.7.0"
 actix-ws = "0.3.0"
 actix-files = "0.6.5"
-actix-web-prom = { version = "0.8.0", features = ["process"] }
+actix-web-prom = { version = "0.9.0", features = ["process"] }
 governor = "0.6.3"
 
 tokio = { version = "1.35.1", features = ["sync"] }

apps/labrinth/src/clickhouse/mod.rs

@@ -35,10 +35,24 @@ pub async fn init_client_with_database(
         .execute()
         .await?;
 
+    let clickhouse_replicated =
+        dotenvy::var("CLICKHOUSE_REPLICATED").unwrap() == "true";
+    let cluster_line = if clickhouse_replicated {
+        "ON cluster '{cluster}'"
+    } else {
+        ""
+    };
+
+    let engine = if clickhouse_replicated {
+        "ReplicatedMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}', '{replica}')"
+    } else {
+        "MergeTree()"
+    };
+
     client
         .query(&format!(
             "
-            CREATE TABLE IF NOT EXISTS {database}.views
+            CREATE TABLE IF NOT EXISTS {database}.views {cluster_line}
             (
                 recorded DateTime64(4),
                 domain String,
@@ -53,8 +67,9 @@ pub async fn init_client_with_database(
                 user_agent String,
                 headers Array(Tuple(String, String))
             )
-            ENGINE = MergeTree()
+            ENGINE = {engine}
             PRIMARY KEY (project_id, recorded, ip)
+            SETTINGS index_granularity = 8192
             "
         ))
         .execute()
@@ -63,7 +78,7 @@ pub async fn init_client_with_database(
     client
         .query(&format!(
             "
-            CREATE TABLE IF NOT EXISTS {database}.downloads
+            CREATE TABLE IF NOT EXISTS {database}.downloads {cluster_line}
             (
                 recorded DateTime64(4),
                 domain String,
@@ -78,8 +93,9 @@ pub async fn init_client_with_database(
                 user_agent String,
                 headers Array(Tuple(String, String))
             )
-            ENGINE = MergeTree()
+            ENGINE = {engine}
             PRIMARY KEY (project_id, recorded, ip)
+            SETTINGS index_granularity = 8192
             "
         ))
         .execute()
@@ -88,7 +104,7 @@ pub async fn init_client_with_database(
     client
         .query(&format!(
             "
-            CREATE TABLE IF NOT EXISTS {database}.playtime
+            CREATE TABLE IF NOT EXISTS {database}.playtime {cluster_line}
             (
                 recorded DateTime64(4),
                 seconds UInt64,
@@ -101,8 +117,9 @@ pub async fn init_client_with_database(
                 game_version String,
                 parent UInt64
             )
-            ENGINE = MergeTree()
+            ENGINE = {engine}
             PRIMARY KEY (project_id, recorded, user_id)
+            SETTINGS index_granularity = 8192
             "
         ))
         .execute()

apps/labrinth/src/lib.rs

@@ -473,6 +473,7 @@ pub fn check_env_vars() -> bool {
         failed |= true;
     }
 
+    failed |= check_var::<bool>("CLICKHOUSE_REPLICATED");
     failed |= check_var::<String>("CLICKHOUSE_URL");
     failed |= check_var::<String>("CLICKHOUSE_USER");
     failed |= check_var::<String>("CLICKHOUSE_PASSWORD");

apps/labrinth/src/main.rs

@@ -92,7 +92,10 @@ async fn main() -> std::io::Result<()> {
 
     let prometheus = PrometheusMetricsBuilder::new("labrinth")
         .endpoint("/metrics")
+        .exclude_regex(r"^/api/v1/.*$")
+        .exclude_regex(r"^/maven/.*$")
         .exclude("/_internal/launcher_socket")
+        .mask_unmatched_patterns("UNKNOWN")
         .build()
         .expect("Failed to create prometheus metrics middleware");