diff --git a/.env b/.env index dbcdad772..1b2cd8110 100644 --- a/.env +++ b/.env @@ -4,4 +4,8 @@ DEBUG=true MONGODB_ADDR=mongodb://localhost:27017 MEILISEARCH_ADDR=http://localhost:7700 -PORT=8000 \ No newline at end of file +PORT=8000 + +BACKBLAZE_KEY_ID=none +BACKBLAZE_KEY=none +BACKBLAZE_BUCKET_ID=none \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5e97ca4d2..007b10587 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,3 +37,7 @@ jobs: - uses: actions-rs/cargo@v1 with: command: test + env: + BACKBLAZE_BUCKET_ID: ${{ secrets.BACKBLAZE_BUCKET_ID }} + BACKBLAZE_KEY: ${{ secrets.BACKBLAZE_KEY }} + BACKBLAZE_KEY_ID: ${{ secrets.BACKBLAZE_KEY_ID }} \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 73f69e095..000000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml deleted file mode 100644 index 7fb4bfd3a..000000000 --- a/.idea/dataSources.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - postgresql - true - org.postgresql.Driver - jdbc:postgresql://localhost:5432/postgres - - - \ No newline at end of file diff --git a/.idea/deployment.xml b/.idea/deployment.xml deleted file mode 100644 index 52206eb2c..000000000 --- a/.idea/deployment.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 12a0a782e..000000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/jsLibraryMappings.xml b/.idea/jsLibraryMappings.xml deleted file mode 100644 index 5cff58334..000000000 --- a/.idea/jsLibraryMappings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/Fabricate.iml b/.idea/labrinth.iml similarity index 66% rename from .idea/Fabricate.iml rename to .idea/labrinth.iml index 9392e8ef5..c254557e1 100644 --- a/.idea/Fabricate.iml +++ b/.idea/labrinth.iml @@ -7,8 +7,5 @@ - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml index 26dc67132..0d8b557a8 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -2,7 +2,7 @@ - + \ No newline at end of file diff --git a/.idea/sqldialects.xml b/.idea/sqldialects.xml deleted file mode 100644 index 6df4889b0..000000000 --- a/.idea/sqldialects.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 328eba1a5..95da4728b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "fabricate" +name = "modrinth" version = "0.1.0" #Team members, please add your emails and usernames authors = ["geometrically ", "Redblueflame ", "Aeledfyr ", "cfanoulis"] @@ -12,7 +12,7 @@ actix-web = "2.0" actix-rt = "1.1.1" actix-files = "0.2.2" -reqwest = "0.10.4" +reqwest = {version="0.10.6", features=["json"]} meilisearch-sdk = "0.1.4" @@ -32,4 +32,7 @@ thiserror = "1.0.20" async-trait = "0.1.36" futures = "0.3.5" -futures-timer = "3.0.2" \ No newline at end of file +futures-timer = "3.0.2" + +base64 = "0.12.3" +sha1 = {version="0.6.0", features=["std"]} \ No newline at end of file diff --git a/src/file_hosting/authorization.rs b/src/file_hosting/authorization.rs new file mode 100644 index 000000000..882d642e6 --- /dev/null +++ b/src/file_hosting/authorization.rs @@ -0,0 +1,73 @@ +use crate::file_hosting::FileHostingError; +use base64::encode; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct AuthorizationPermissions { + bucket_id: Option, + bucket_name: Option, + capabilities: Vec, + name_prefix: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct AuthorizationData { + pub absolute_minimum_part_size: i32, + pub account_id: String, + pub allowed: AuthorizationPermissions, + pub api_url: String, + pub authorization_token: String, + pub download_url: String, + pub recommended_part_size: i32, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct UploadUrlData { + pub bucket_id: String, + pub upload_url: String, + pub authorization_token: String, +} + +pub async fn authorize_account( + key_id: String, + application_key: String, +) -> Result { + let combined_key = format!("{}:{}", key_id, application_key); + let formatted_key = format!("Basic {}", encode(combined_key)); + + Ok(reqwest::Client::new() + .get("https://api.backblazeb2.com/b2api/v2/b2_authorize_account") + .header(reqwest::header::CONTENT_TYPE, "application/json") + .header(reqwest::header::AUTHORIZATION, formatted_key) + .send() + .await? + .json() + .await?) +} + +pub async fn get_upload_url( + authorization_data: AuthorizationData, + bucket_id: String, +) -> Result { + Ok(reqwest::Client::new() + .post(&format!("{}/b2api/v2/b2_get_upload_url", authorization_data.api_url).to_string()) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .header( + reqwest::header::AUTHORIZATION, + authorization_data.authorization_token, + ) + .body( + json!({ + "bucketId": bucket_id, + }) + .to_string(), + ) + .send() + .await? + .json() + .await?) +} diff --git a/src/file_hosting/delete.rs b/src/file_hosting/delete.rs new file mode 100644 index 000000000..52ff6e89c --- /dev/null +++ b/src/file_hosting/delete.rs @@ -0,0 +1,41 @@ +use crate::file_hosting::{AuthorizationData, FileHostingError}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct DeleteFileData { + pub file_id: String, + pub file_name: String, +} + +pub async fn delete_file_version( + authorization_data: AuthorizationData, + file_id: String, + file_name: String, +) -> Result { + Ok(reqwest::Client::new() + .post( + &format!( + "{}/b2api/v2/b2_delete_file_version", + authorization_data.api_url + ) + .to_string(), + ) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .header( + reqwest::header::AUTHORIZATION, + authorization_data.authorization_token, + ) + .body( + json!({ + "fileName": file_name, + "fileId": file_id + }) + .to_string(), + ) + .send() + .await? + .json() + .await?) +} diff --git a/src/file_hosting/mod.rs b/src/file_hosting/mod.rs new file mode 100644 index 000000000..83961f873 --- /dev/null +++ b/src/file_hosting/mod.rs @@ -0,0 +1,79 @@ +use thiserror::Error; + +mod authorization; +mod delete; +mod upload; + +pub use authorization::authorize_account; +pub use authorization::get_upload_url; +pub use authorization::AuthorizationData; +pub use authorization::AuthorizationPermissions; +pub use authorization::UploadUrlData; + +pub use upload::upload_file; +pub use upload::UploadFileData; + +pub use delete::delete_file_version; +pub use delete::DeleteFileData; + +#[derive(Error, Debug)] +pub enum FileHostingError { + #[error("Error while accessing the data from remote")] + RemoteWebsiteError(#[from] reqwest::Error), + #[error("Error while serializing or deserializing JSON")] + SerDeError(#[from] serde_json::Error), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[actix_rt::test] + async fn test_authorization() { + let authorization_data = authorize_account( + dotenv::var("BACKBLAZE_KEY_ID").unwrap(), + dotenv::var("BACKBLAZE_KEY").unwrap(), + ) + .await + .unwrap(); + + get_upload_url( + authorization_data, + dotenv::var("BACKBLAZE_BUCKET_ID").unwrap(), + ) + .await + .unwrap(); + } + + #[actix_rt::test] + async fn test_file_management() { + let authorization_data = authorize_account( + dotenv::var("BACKBLAZE_KEY_ID").unwrap(), + dotenv::var("BACKBLAZE_KEY").unwrap(), + ) + .await + .unwrap(); + let upload_url_data = get_upload_url( + authorization_data.clone(), + dotenv::var("BACKBLAZE_BUCKET_ID").unwrap(), + ) + .await + .unwrap(); + let upload_data = upload_file( + upload_url_data, + "text/plain".to_string(), + "test.txt".to_string(), + "test file".to_string().into_bytes(), + ) + .await + .unwrap(); + + delete_file_version( + authorization_data, + upload_data.file_id, + upload_data.file_name, + ) + .await + .unwrap(); + } +} diff --git a/src/file_hosting/upload.rs b/src/file_hosting/upload.rs new file mode 100644 index 000000000..8d3fb232d --- /dev/null +++ b/src/file_hosting/upload.rs @@ -0,0 +1,41 @@ +use crate::file_hosting::authorization::UploadUrlData; +use crate::file_hosting::FileHostingError; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct UploadFileData { + pub file_id: String, + pub file_name: String, + pub account_id: String, + pub bucket_id: String, + pub content_length: u32, + pub content_sha1: String, + pub content_md5: Option, + pub content_type: String, + pub upload_timestamp: u64, +} + +//Content Types found here: https://www.backblaze.com/b2/docs/content-types.html +pub async fn upload_file( + url_data: UploadUrlData, + content_type: String, + file_name: String, + file_bytes: Vec, +) -> Result { + Ok(reqwest::Client::new() + .post(&url_data.upload_url) + .header(reqwest::header::AUTHORIZATION, url_data.authorization_token) + .header("X-Bz-File-Name", file_name) + .header(reqwest::header::CONTENT_TYPE, content_type) + .header(reqwest::header::CONTENT_LENGTH, file_bytes.len()) + .header( + "X-Bz-Content-Sha1", + sha1::Sha1::from(&file_bytes).hexdigest(), + ) + .body(file_bytes) + .send() + .await? + .json() + .await?) +} diff --git a/src/main.rs b/src/main.rs index 029c3b29a..4eea7904b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,6 +7,7 @@ use std::env; use std::fs::File; mod database; +mod file_hosting; mod models; mod routes; mod search; diff --git a/src/routes/index.rs b/src/routes/index.rs index 8db5bf793..1a8d3f052 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -1,4 +1,4 @@ -use actix_web::{get, web, HttpResponse}; +use actix_web::{get, HttpResponse}; use serde_json::json; #[get("/")] diff --git a/src/search/indexing/curseforge_import.rs b/src/search/indexing/curseforge_import.rs index 1ffc04d4e..e1fa9411c 100644 --- a/src/search/indexing/curseforge_import.rs +++ b/src/search/indexing/curseforge_import.rs @@ -65,10 +65,7 @@ pub async fn index_curseforge( let text = &res.text().await?; let curseforge_mods: Vec = serde_json::from_str(text)?; - let mut max_index = 0; - for curseforge_mod in curseforge_mods { - max_index = curseforge_mod.id; if curseforge_mod.game_slug != "minecraft" || !curseforge_mod.website_url.contains("/mc-mods/") { diff --git a/src/search/indexing/local_import.rs b/src/search/indexing/local_import.rs index ed3fd7035..b7062978a 100644 --- a/src/search/indexing/local_import.rs +++ b/src/search/indexing/local_import.rs @@ -1,13 +1,11 @@ use bson::doc; -use bson::Bson; use futures::StreamExt; use log::info; -use meilisearch_sdk::client::Client; use crate::database::models::Item; use crate::database::{Mod, Version}; -use crate::search::{SearchError, SearchMod, SearchRequest}; +use crate::search::{SearchError, SearchMod}; pub async fn index_local(client: mongodb::Client) -> Result, SearchError> { info!("Indexing local mods!"); diff --git a/src/search/indexing/mod.rs b/src/search/indexing/mod.rs index e848fb66a..feaa19b72 100644 --- a/src/search/indexing/mod.rs +++ b/src/search/indexing/mod.rs @@ -2,7 +2,6 @@ pub mod curseforge_import; pub mod local_import; -use crate::database::DatabaseError; use crate::search::indexing::curseforge_import::index_curseforge; use crate::search::indexing::local_import::index_local; use crate::search::{SearchError, SearchMod};