Add Backblaze Driver (#32)

* Backblaze Driver

* Update action to work with new tests

* Fix minor issues

* Run Formatter + Switch to reqwest json parser

.env

@@ -4,4 +4,8 @@ DEBUG=true
 MONGODB_ADDR=mongodb://localhost:27017
 MEILISEARCH_ADDR=http://localhost:7700
 
-PORT=8000
+PORT=8000
+
+BACKBLAZE_KEY_ID=none
+BACKBLAZE_KEY=none
+BACKBLAZE_BUCKET_ID=none

.github/workflows/tests.yml

@@ -37,3 +37,7 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: test
+        env:
+          BACKBLAZE_BUCKET_ID: ${{ secrets.BACKBLAZE_BUCKET_ID }}
+          BACKBLAZE_KEY: ${{ secrets.BACKBLAZE_KEY }}
+          BACKBLAZE_KEY_ID: ${{ secrets.BACKBLAZE_KEY_ID }}

.idea/.gitignore

@@ -1,8 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
-# Datasource local storage ignored files
-/dataSources/
-/dataSources.local.xml
-# Editor-based HTTP Client requests
-/httpRequests/

.idea/dataSources.xml

@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="DataSourceManagerImpl" format="xml" multifile-model="true">
-    <data-source source="LOCAL" name="postgres@localhost/fabricate" uuid="78c95043-78e4-4469-8d73-d6d45e07a4f1">
-      <driver-ref>postgresql</driver-ref>
-      <synchronize>true</synchronize>
-      <jdbc-driver>org.postgresql.Driver</jdbc-driver>
-      <jdbc-url>jdbc:postgresql://localhost:5432/postgres</jdbc-url>
-    </data-source>
-  </component>
-</project>

.idea/deployment.xml

@@ -1,14 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="PublishConfigData">
-    <serverData>
-      <paths name="Remote Host (b864ed82-8c7b-4ccd-920b-a7b8ba7cc008)">
-        <serverdata>
-          <mappings>
-            <mapping local="$PROJECT_DIR$" web="/" />
-          </mappings>
-        </serverdata>
-      </paths>
-    </serverData>
-  </component>
-</project>

.idea/inspectionProfiles/Project_Default.xml

@@ -1,36 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <profile version="1.0">
-    <option name="myName" value="Project Default" />
-    <inspection_tool class="DuplicatedCode" enabled="true" level="WEAK WARNING" enabled_by_default="true">
-      <Languages>
-        <language minSize="56" name="Rust" />
-      </Languages>
-    </inspection_tool>
-    <inspection_tool class="HtmlUnknownAttribute" enabled="true" level="WARNING" enabled_by_default="true">
-      <option name="myValues">
-        <value>
-          <list size="1">
-            <item index="0" class="java.lang.String" itemvalue="white" />
-          </list>
-        </value>
-      </option>
-      <option name="myCustomValuesEnabled" value="true" />
-    </inspection_tool>
-    <inspection_tool class="HtmlUnknownTag" enabled="true" level="WARNING" enabled_by_default="true">
-      <option name="myValues">
-        <value>
-          <list size="7">
-            <item index="0" class="java.lang.String" itemvalue="nobr" />
-            <item index="1" class="java.lang.String" itemvalue="noembed" />
-            <item index="2" class="java.lang.String" itemvalue="comment" />
-            <item index="3" class="java.lang.String" itemvalue="noscript" />
-            <item index="4" class="java.lang.String" itemvalue="embed" />
-            <item index="5" class="java.lang.String" itemvalue="script" />
-            <item index="6" class="java.lang.String" itemvalue="style" />
-          </list>
-        </value>
-      </option>
-      <option name="myCustomValuesEnabled" value="true" />
-    </inspection_tool>
-  </profile>
-</component>

.idea/jsLibraryMappings.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="JavaScriptLibraryMappings">
-    <file url="file://$PROJECT_DIR$" libraries="{default, highlight, tomorrow}" />
-  </component>
-</project>

.idea/labrinth.iml

@@ -7,8 +7,5 @@
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
-    <orderEntry type="library" name="default" level="application" />
-    <orderEntry type="library" name="highlight" level="application" />
-    <orderEntry type="library" name="tomorrow" level="application" />
   </component>
 </module>

.idea/modules.xml

@@ -2,7 +2,7 @@
 <project version="4">
   <component name="ProjectModuleManager">
     <modules>
-      <module fileurl="file://$PROJECT_DIR$/../fabricate/.idea/Fabricate.iml" filepath="$PROJECT_DIR$/../fabricate/.idea/Fabricate.iml" />
+      <module fileurl="file://$PROJECT_DIR$/.idea/labrinth.iml" filepath="$PROJECT_DIR$/.idea/labrinth.iml" />
     </modules>
   </component>
 </project>

.idea/sqldialects.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="SqlDialectMappings">
-    <file url="PROJECT" dialect="PostgreSQL" />
-  </component>
-</project>

Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "fabricate"
+name = "modrinth"
 version = "0.1.0"
 #Team members, please add your emails and usernames
 authors = ["geometrically <jai.a@tuta.io>", "Redblueflame <contact@redblueflame.com>", "Aeledfyr <aeledfyr@gmail.com>", "cfanoulis"]
@@ -12,7 +12,7 @@ actix-web = "2.0"
 actix-rt = "1.1.1"
 actix-files = "0.2.2"
 
-reqwest = "0.10.4"
+reqwest = {version="0.10.6", features=["json"]}
 
 meilisearch-sdk = "0.1.4"
 
@@ -32,4 +32,7 @@ thiserror = "1.0.20"
 async-trait = "0.1.36"
 
 futures = "0.3.5"
-futures-timer = "3.0.2"
+futures-timer = "3.0.2"
+
+base64 = "0.12.3"
+sha1 = {version="0.6.0", features=["std"]}

src/file_hosting/authorization.rs

@@ -0,0 +1,73 @@
+use crate::file_hosting::FileHostingError;
+use base64::encode;
+use serde::{Deserialize, Serialize};
+use serde_json::json;
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct AuthorizationPermissions {
+    bucket_id: Option<String>,
+    bucket_name: Option<String>,
+    capabilities: Vec<String>,
+    name_prefix: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct AuthorizationData {
+    pub absolute_minimum_part_size: i32,
+    pub account_id: String,
+    pub allowed: AuthorizationPermissions,
+    pub api_url: String,
+    pub authorization_token: String,
+    pub download_url: String,
+    pub recommended_part_size: i32,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct UploadUrlData {
+    pub bucket_id: String,
+    pub upload_url: String,
+    pub authorization_token: String,
+}
+
+pub async fn authorize_account(
+    key_id: String,
+    application_key: String,
+) -> Result<AuthorizationData, FileHostingError> {
+    let combined_key = format!("{}:{}", key_id, application_key);
+    let formatted_key = format!("Basic {}", encode(combined_key));
+
+    Ok(reqwest::Client::new()
+        .get("https://api.backblazeb2.com/b2api/v2/b2_authorize_account")
+        .header(reqwest::header::CONTENT_TYPE, "application/json")
+        .header(reqwest::header::AUTHORIZATION, formatted_key)
+        .send()
+        .await?
+        .json()
+        .await?)
+}
+
+pub async fn get_upload_url(
+    authorization_data: AuthorizationData,
+    bucket_id: String,
+) -> Result<UploadUrlData, FileHostingError> {
+    Ok(reqwest::Client::new()
+        .post(&format!("{}/b2api/v2/b2_get_upload_url", authorization_data.api_url).to_string())
+        .header(reqwest::header::CONTENT_TYPE, "application/json")
+        .header(
+            reqwest::header::AUTHORIZATION,
+            authorization_data.authorization_token,
+        )
+        .body(
+            json!({
+                "bucketId": bucket_id,
+            })
+            .to_string(),
+        )
+        .send()
+        .await?
+        .json()
+        .await?)
+}

src/file_hosting/delete.rs

@@ -0,0 +1,41 @@
+use crate::file_hosting::{AuthorizationData, FileHostingError};
+use serde::{Deserialize, Serialize};
+use serde_json::json;
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct DeleteFileData {
+    pub file_id: String,
+    pub file_name: String,
+}
+
+pub async fn delete_file_version(
+    authorization_data: AuthorizationData,
+    file_id: String,
+    file_name: String,
+) -> Result<DeleteFileData, FileHostingError> {
+    Ok(reqwest::Client::new()
+        .post(
+            &format!(
+                "{}/b2api/v2/b2_delete_file_version",
+                authorization_data.api_url
+            )
+            .to_string(),
+        )
+        .header(reqwest::header::CONTENT_TYPE, "application/json")
+        .header(
+            reqwest::header::AUTHORIZATION,
+            authorization_data.authorization_token,
+        )
+        .body(
+            json!({
+                "fileName": file_name,
+                "fileId": file_id
+            })
+            .to_string(),
+        )
+        .send()
+        .await?
+        .json()
+        .await?)
+}

src/file_hosting/mod.rs

@@ -0,0 +1,79 @@
+use thiserror::Error;
+
+mod authorization;
+mod delete;
+mod upload;
+
+pub use authorization::authorize_account;
+pub use authorization::get_upload_url;
+pub use authorization::AuthorizationData;
+pub use authorization::AuthorizationPermissions;
+pub use authorization::UploadUrlData;
+
+pub use upload::upload_file;
+pub use upload::UploadFileData;
+
+pub use delete::delete_file_version;
+pub use delete::DeleteFileData;
+
+#[derive(Error, Debug)]
+pub enum FileHostingError {
+    #[error("Error while accessing the data from remote")]
+    RemoteWebsiteError(#[from] reqwest::Error),
+    #[error("Error while serializing or deserializing JSON")]
+    SerDeError(#[from] serde_json::Error),
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[actix_rt::test]
+    async fn test_authorization() {
+        let authorization_data = authorize_account(
+            dotenv::var("BACKBLAZE_KEY_ID").unwrap(),
+            dotenv::var("BACKBLAZE_KEY").unwrap(),
+        )
+        .await
+        .unwrap();
+
+        get_upload_url(
+            authorization_data,
+            dotenv::var("BACKBLAZE_BUCKET_ID").unwrap(),
+        )
+        .await
+        .unwrap();
+    }
+
+    #[actix_rt::test]
+    async fn test_file_management() {
+        let authorization_data = authorize_account(
+            dotenv::var("BACKBLAZE_KEY_ID").unwrap(),
+            dotenv::var("BACKBLAZE_KEY").unwrap(),
+        )
+        .await
+        .unwrap();
+        let upload_url_data = get_upload_url(
+            authorization_data.clone(),
+            dotenv::var("BACKBLAZE_BUCKET_ID").unwrap(),
+        )
+        .await
+        .unwrap();
+        let upload_data = upload_file(
+            upload_url_data,
+            "text/plain".to_string(),
+            "test.txt".to_string(),
+            "test file".to_string().into_bytes(),
+        )
+        .await
+        .unwrap();
+
+        delete_file_version(
+            authorization_data,
+            upload_data.file_id,
+            upload_data.file_name,
+        )
+        .await
+        .unwrap();
+    }
+}

src/file_hosting/upload.rs

@@ -0,0 +1,41 @@
+use crate::file_hosting::authorization::UploadUrlData;
+use crate::file_hosting::FileHostingError;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct UploadFileData {
+    pub file_id: String,
+    pub file_name: String,
+    pub account_id: String,
+    pub bucket_id: String,
+    pub content_length: u32,
+    pub content_sha1: String,
+    pub content_md5: Option<String>,
+    pub content_type: String,
+    pub upload_timestamp: u64,
+}
+
+//Content Types found here: https://www.backblaze.com/b2/docs/content-types.html
+pub async fn upload_file(
+    url_data: UploadUrlData,
+    content_type: String,
+    file_name: String,
+    file_bytes: Vec<u8>,
+) -> Result<UploadFileData, FileHostingError> {
+    Ok(reqwest::Client::new()
+        .post(&url_data.upload_url)
+        .header(reqwest::header::AUTHORIZATION, url_data.authorization_token)
+        .header("X-Bz-File-Name", file_name)
+        .header(reqwest::header::CONTENT_TYPE, content_type)
+        .header(reqwest::header::CONTENT_LENGTH, file_bytes.len())
+        .header(
+            "X-Bz-Content-Sha1",
+            sha1::Sha1::from(&file_bytes).hexdigest(),
+        )
+        .body(file_bytes)
+        .send()
+        .await?
+        .json()
+        .await?)
+}

src/main.rs

@@ -7,6 +7,7 @@ use std::env;
 use std::fs::File;
 
 mod database;
+mod file_hosting;
 mod models;
 mod routes;
 mod search;

src/routes/index.rs

@@ -1,4 +1,4 @@
-use actix_web::{get, web, HttpResponse};
+use actix_web::{get, HttpResponse};
 use serde_json::json;
 
 #[get("/")]

src/search/indexing/curseforge_import.rs

@@ -65,10 +65,7 @@ pub async fn index_curseforge(
     let text = &res.text().await?;
     let curseforge_mods: Vec<CurseForgeMod> = serde_json::from_str(text)?;
 
-    let mut max_index = 0;
-
     for curseforge_mod in curseforge_mods {
-        max_index = curseforge_mod.id;
         if curseforge_mod.game_slug != "minecraft"
             || !curseforge_mod.website_url.contains("/mc-mods/")
         {

src/search/indexing/local_import.rs

@@ -1,13 +1,11 @@
 use bson::doc;
-use bson::Bson;
 use futures::StreamExt;
 use log::info;
-use meilisearch_sdk::client::Client;
 
 use crate::database::models::Item;
 use crate::database::{Mod, Version};
 
-use crate::search::{SearchError, SearchMod, SearchRequest};
+use crate::search::{SearchError, SearchMod};
 
 pub async fn index_local(client: mongodb::Client) -> Result<Vec<SearchMod>, SearchError> {
     info!("Indexing local mods!");

src/search/indexing/mod.rs

@@ -2,7 +2,6 @@
 pub mod curseforge_import;
 pub mod local_import;
 
-use crate::database::DatabaseError;
 use crate::search::indexing::curseforge_import::index_curseforge;
 use crate::search::indexing::local_import::index_local;
 use crate::search::{SearchError, SearchMod};