From 45dbf5393f62fb019ff6ff82b3495c18e856d0d7 Mon Sep 17 00:00:00 2001 From: Jai A Date: Thu, 22 Dec 2022 19:01:41 -0700 Subject: [PATCH] Bump versions + switch AWS/S3 library --- .env | 4 - daedalus/Cargo.toml | 6 +- daedalus/src/lib.rs | 12 +-- daedalus_client/Cargo.toml | 20 +++-- daedalus_client/src/fabric.rs | 18 ++--- daedalus_client/src/forge.rs | 36 ++++----- daedalus_client/src/main.rs | 135 ++++++++++--------------------- daedalus_client/src/minecraft.rs | 6 +- 8 files changed, 90 insertions(+), 147 deletions(-) diff --git a/.env b/.env index dc87710fe..34bf98cd1 100644 --- a/.env +++ b/.env @@ -8,7 +8,3 @@ S3_SECRET=none S3_URL=none S3_REGION=none S3_BUCKET_NAME=none - -DO_INTEGRATION=false -DO_ACCESS_KEY=none -DO_ENDPOINT_ID=none \ No newline at end of file diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 9b5929c52..3909d8403 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.16" +version = "0.1.17" authors = ["Jai A "] edition = "2018" license = "MIT" @@ -21,5 +21,5 @@ chrono = { version = "0.4", features = ["serde"] } bytes = "1" thiserror = "1.0" tokio = { version = "1", features = ["full"] } -sha1 = { version = "0.6.0", features = ["std"]} -bincode = {version = "2.0.0-rc", features = ["serde"], optional = true} +sha1 = { version = "0.6.1", features = ["std"]} +bincode = {version = "2.0.0-rc.2", features = ["serde"], optional = true} diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 77657323d..678bf1c4a 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -45,7 +45,7 @@ pub enum Error { pub fn get_path_from_artifact(artifact: &str) -> Result { let name_items = artifact.split(':').collect::>(); - let package = name_items.get(0).ok_or_else(|| { + let package = name_items.first().ok_or_else(|| { Error::ParseError(format!("Unable to find package for library {}", &artifact)) })?; let name = name_items.get(1).ok_or_else(|| { @@ -60,14 +60,14 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { })? .split('@') .collect::>(); - let version = version_ext.get(0).ok_or_else(|| { + let version = version_ext.first().ok_or_else(|| { Error::ParseError(format!("Unable to find version for library {}", &artifact)) })?; let ext = version_ext.get(1); Ok(format!( "{}/{}/{}/{}-{}.{}", - package.replace(".", "/"), + package.replace('.', "/"), name, version, name, @@ -86,14 +86,14 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { })? .split('@') .collect::>(); - let data = data_ext.get(0).ok_or_else(|| { + let data = data_ext.first().ok_or_else(|| { Error::ParseError(format!("Unable to find data for library {}", &artifact)) })?; let ext = data_ext.get(1); Ok(format!( "{}/{}/{}/{}-{}-{}.{}", - package.replace(".", "/"), + package.replace('.', "/"), name, version, name, @@ -115,7 +115,7 @@ pub async fn download_file_mirrors( } for (index, mirror) in mirrors.iter().enumerate() { - let result = download_file(&*format!("{}{}", mirror, base), sha1).await; + let result = download_file(&format!("{}{}", mirror, base), sha1).await; if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) { return result; diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index bf4473328..035588722 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.14" +version = "0.1.17" authors = ["Jai A "] edition = "2018" @@ -9,19 +9,17 @@ edition = "2018" [dependencies] daedalus = { path = "../daedalus" } tokio = { version = "1", features = ["full"] } -futures = "0.3.17" -dotenv = "0.15.0" -log = "0.4.8" -env_logger="0.9.0" +futures = "0.3.25" +dotenvy = "0.15.6" +log = "0.4.17" +env_logger= "0.10.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" lazy_static = "1.4.0" thiserror = "1.0" -reqwest = "0.11.4" -zip = "0.5.13" +reqwest = "0.11.13" +zip = "0.6.3" semver = "1.0" chrono = { version = "0.4", features = ["serde"] } -bytes = "1.1.0" - -rusoto_core = "0.47.0" -rusoto_s3 = "0.47.0" +bytes = "1.3.0" +rust-s3 = "0.32.3" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index fdc3cf46c..3aef4fdc4 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -13,7 +13,7 @@ pub async fn retrieve_data( uploaded_files: &mut Vec, ) -> Result<(), Error> { let mut list = fetch_fabric_versions(None).await?; - let old_manifest = daedalus::modded::fetch_manifest(&*format!( + let old_manifest = daedalus::modded::fetch_manifest(&format!( "fabric/v{}/manifest.json", daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, )) @@ -47,11 +47,7 @@ pub async fn retrieve_data( } } - list.loader = list - .loader - .into_iter() - .filter(|x| loaders.iter().any(|val| val.1 == x.version)) - .collect(); + list.loader.retain(|x| loaders.iter().any(|val| val.1 == x.version)) } let mut version_futures = Vec::new(); @@ -79,7 +75,7 @@ pub async fn retrieve_data( } let version = - fetch_fabric_version(&*game_version.version, &*loader).await?; + fetch_fabric_version(&game_version.version, &loader).await?; Ok::, String, PartialVersionInfo)>, Error>(Some( (stable, loader, version), @@ -106,10 +102,10 @@ pub async fn retrieve_data( } } - let artifact_path = daedalus::get_path_from_artifact(&*lib.name)?; + let artifact_path = daedalus::get_path_from_artifact(&lib.name)?; let artifact = daedalus::download_file( - &*format!( + &format!( "{}{}", lib.url.unwrap_or_else(|| { "https://maven.fabricmc.net/".to_string() @@ -169,7 +165,7 @@ pub async fn retrieve_data( async move { loader_version_map.push(LoaderVersion { id: format!("{}-{}", inherits_from, loader), - url: format_url(&*version_path), + url: format_url(&version_path), stable: *stable, }); } @@ -280,7 +276,7 @@ async fn fetch_fabric_version( ) -> Result { Ok(serde_json::from_slice( &download_file( - &*format!( + &format!( "{}/versions/loader/{}/{}/profile/json", FABRIC_META_URL, version_number, loader_version ), diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index c0e841c5e..25da023af 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -28,7 +28,7 @@ pub async fn retrieve_data( uploaded_files: &mut Vec, ) -> Result<(), Error> { let maven_metadata = fetch_maven_metadata(None).await?; - let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( + let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "forge/v{}/manifest.json", daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, ))) @@ -69,7 +69,7 @@ pub async fn retrieve_data( loader_version_raw.to_string() }; - let version = Version::parse(&*loader_version)?; + let version = Version::parse(&loader_version)?; if FORGE_MANIFEST_V1_QUERY.matches(&version) || FORGE_MANIFEST_V2_QUERY_P1.matches(&version) @@ -104,7 +104,7 @@ pub async fn retrieve_data( { let versions = versions_mutex.lock().await; let version = versions.iter().find(|x| - x.id == minecraft_version).map(|x| x.loaders.iter().find(|x| x.id == loader_version_full)).flatten(); + x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); if let Some(version) = version { return Ok::, Error>(Some(version.clone())); @@ -112,7 +112,7 @@ pub async fn retrieve_data( } info!("Forge - Installer Start {}", loader_version_full.clone()); - let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; + let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; let reader = std::io::Cursor::new(bytes); @@ -125,13 +125,13 @@ pub async fn retrieve_data( let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - Ok::(serde_json::from_str::(&*contents)?) + Ok::(serde_json::from_str::(&contents)?) }).await??; let mut archive_clone = archive.clone(); let file_path = profile.install.file_path.clone(); let forge_universal_bytes = tokio::task::spawn_blocking(move || { - let mut forge_universal_file = archive_clone.by_name(&*file_path)?; + let mut forge_universal_file = archive_clone.by_name(&file_path)?; let mut forge_universal = Vec::new(); forge_universal_file.read_to_end(&mut forge_universal)?; @@ -156,7 +156,7 @@ pub async fn retrieve_data( } let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; + daedalus::get_path_from_artifact(&lib.name)?; let artifact = if lib.name == forge_universal_path { forge_universal_bytes.clone() @@ -164,7 +164,7 @@ pub async fn retrieve_data( let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; daedalus::download_file_mirrors( - &*artifact_path, + &artifact_path, &mirrors, None, ) @@ -216,7 +216,7 @@ pub async fn retrieve_data( return Ok(Some(LoaderVersion { id: loader_version_full, - url: format_url(&*version_path), + url: format_url(&version_path), stable: false })); } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { @@ -227,7 +227,7 @@ pub async fn retrieve_data( let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - Ok::(serde_json::from_str::(&*contents)?) + Ok::(serde_json::from_str::(&contents)?) }).await??; let mut archive_clone = archive.clone(); @@ -237,7 +237,7 @@ pub async fn retrieve_data( let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - Ok::(serde_json::from_str::(&*contents)?) + Ok::(serde_json::from_str::(&contents)?) }).await??; @@ -255,12 +255,12 @@ pub async fn retrieve_data( let mut local_libs : HashMap = HashMap::new(); for lib in &libs { - if lib.downloads.as_ref().map(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).flatten().unwrap_or(false) { + if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { let mut archive_clone = archive.clone(); let lib_name_clone = lib.name.clone(); let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*lib_name_clone)?))?; + let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; let mut lib_bytes = Vec::new(); lib_file.read_to_end(&mut lib_bytes)?; @@ -331,7 +331,7 @@ pub async fn retrieve_data( let now = Instant::now(); let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; + daedalus::get_path_from_artifact(&lib.name)?; { let mut visited_assets = visited_assets.lock().await; @@ -339,7 +339,7 @@ pub async fn retrieve_data( if visited_assets.contains(&lib.name) { if let Some(ref mut downloads) = lib.downloads { if let Some(ref mut artifact) = downloads.artifact { - artifact.url = format_url(&*format!("maven/{}", artifact_path)); + artifact.url = format_url(&format!("maven/{}", artifact_path)); } } else if lib.url.is_some() { lib.url = Some(format_url("maven/")); @@ -357,14 +357,14 @@ pub async fn retrieve_data( local_libs.get(&lib.name).cloned() } else { Some(daedalus::download_file( - &*artifact.url, + &artifact.url, Some(&*artifact.sha1), ) .await?) }; if res.is_some() { - artifact.url = format_url(&*format!("maven/{}", artifact_path)); + artifact.url = format_url(&format!("maven/{}", artifact_path)); } res @@ -431,7 +431,7 @@ pub async fn retrieve_data( return Ok(Some(LoaderVersion { id: loader_version_full, - url: format_url(&*version_path), + url: format_url(&version_path), stable: false })); } diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 4a4f67667..70a4fa6df 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,9 +1,8 @@ use log::{error, warn}; -use rusoto_core::credential::StaticProvider; -use rusoto_core::{HttpClient, Region, RusotoError}; -use rusoto_s3::{PutObjectError, S3Client}; -use rusoto_s3::{PutObjectRequest, S3}; use std::time::Duration; +use s3::{Bucket, Region}; +use s3::creds::Credentials; +use s3::error::S3Error; mod fabric; mod forge; @@ -21,7 +20,7 @@ pub enum Error { TaskError(#[from] tokio::task::JoinError), #[error("Error while uploading file to S3")] S3Error { - inner: RusotoError, + inner: S3Error, file: String, }, #[error("Error while parsing version as semver: {0}")] @@ -70,11 +69,6 @@ async fn main() { Err(err) => error!("{:?}", err), }; } - - match purge_digitalocean_cache(uploaded_files).await { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; } } @@ -82,13 +76,13 @@ fn check_env_vars() -> bool { let mut failed = false; fn check_var(var: &str) -> bool { - if dotenv::var(var) + if dotenvy::var(var) .ok() .and_then(|s| s.parse::().ok()) .is_none() { warn!( - "Variable `{}` missing in dotenv or not of type `{}`", + "Variable `{}` missing in dotenvy or not of type `{}`", var, std::any::type_name::() ); @@ -107,36 +101,31 @@ fn check_env_vars() -> bool { failed |= check_var::("S3_REGION"); failed |= check_var::("S3_BUCKET_NAME"); - failed |= check_var::("DO_INTEGRATION"); - - let do_integration = dotenv::var("DO_INTEGRATION") - .ok() - .map(|x| x.parse::().ok()) - .flatten() - .unwrap_or(false); - - if do_integration { - failed |= check_var::("DO_ACCESS_KEY"); - failed |= check_var::("DO_ENDPOINT_ID"); - } - failed } + lazy_static::lazy_static! { - static ref CLIENT : S3Client = S3Client::new_with( - HttpClient::new().unwrap(), - StaticProvider::new( - dotenv::var("S3_ACCESS_TOKEN").unwrap(), - dotenv::var("S3_SECRET").unwrap(), - None, - None, - ), + static ref CLIENT : Bucket = Bucket::new( + &dotenvy::var("S3_BUCKET_NAME").unwrap(), + if &*dotenvy::var("S3_REGION").unwrap() == "r2" { + Region::R2 { + account_id: dotenvy::var("S3_URL").unwrap(), + } + } else { Region::Custom { - name: dotenv::var("S3_REGION").unwrap(), - endpoint: dotenv::var("S3_URL").unwrap(), - }, - ); + region: dotenvy::var("S3_REGION").unwrap(), + endpoint: dotenvy::var("S3_URL").unwrap(), + } + }, + Credentials::new( + Some(&*dotenvy::var("S3_ACCESS_TOKEN").unwrap()), + Some(&*dotenvy::var("S3_SECRET").unwrap()), + None, + None, + None, + ).unwrap(), + ).unwrap(); } pub async fn upload_file_to_bucket( @@ -145,23 +134,24 @@ pub async fn upload_file_to_bucket( content_type: Option, uploaded_files: &tokio::sync::Mutex>, ) -> Result<(), Error> { - let key = format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path); + let key = format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path); for attempt in 1..=4 { - let result = CLIENT - .put_object(PutObjectRequest { - bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), - key: key.clone(), - body: Some(bytes.clone().into()), - acl: Some("public-read".to_string()), - content_type: content_type.clone(), - ..Default::default() - }) - .await - .map_err(|err| Error::S3Error { - inner: err, - file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), - }); + let result = if let Some(ref content_type) = content_type { + CLIENT.put_object_with_content_type( + key.clone(), + &bytes, + content_type, + ).await + } else { + CLIENT.put_object( + key.clone(), + &bytes, + ).await + }.map_err(|err| Error::S3Error { + inner: err, + file: format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path), + }); match result { Ok(_) => { @@ -185,45 +175,8 @@ pub async fn upload_file_to_bucket( pub fn format_url(path: &str) -> String { format!( "{}/{}/{}", - &*dotenv::var("BASE_URL").unwrap(), - &*dotenv::var("BASE_FOLDER").unwrap(), + &*dotenvy::var("BASE_URL").unwrap(), + &*dotenvy::var("BASE_FOLDER").unwrap(), path ) } - -#[derive(serde::Serialize)] -struct PurgeCacheRequest { - pub files: Vec, -} - -pub async fn purge_digitalocean_cache(files: Vec) -> Result<(), Error> { - if !dotenv::var("DO_INTEGRATION") - .ok() - .map(|x| x.parse::().ok()) - .flatten() - .unwrap_or(false) - { - return Ok(()); - } - - let client = reqwest::Client::new(); - - client - .delete(&format!( - "https://api.digitalocean.com/v2/cdn/endpoints/{}/cache", - &*dotenv::var("DO_ENDPOINT_ID").unwrap() - )) - .header( - "Authorization", - &*format!("Bearer {}", &*dotenv::var("DO_ACCESS_KEY").unwrap()), - ) - .json(&PurgeCacheRequest { files }) - .send() - .await - .map_err(|err| Error::FetchError { - inner: err, - item: "purging digital ocean cache".to_string(), - })?; - - Ok(()) -} diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 58fae665b..56007b9f9 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -8,7 +8,7 @@ use tokio::sync::Mutex; pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result { let old_manifest = - daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!( + daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&format!( "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION )))) @@ -43,12 +43,12 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result