Bommels05/Modrinth
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bump versions + switch AWS/S3 library

Browse Source
This commit is contained in:
Jai A 2022-12-22 19:01:41 -07:00
parent 827c4e31ee
commit 45dbf5393f
No known key found for this signature in database
GPG Key ID: CC88DE86F48BE019
8 changed files with 90 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.env
View File

@ -8,7 +8,3 @@ S3_SECRET=none
S3_URL=none
S3_REGION=none
S3_BUCKET_NAME=none
DO_INTEGRATION=false
DO_ACCESS_KEY=none
DO_ENDPOINT_ID=none

6
daedalus/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "daedalus"
version = "0.1.16"
version = "0.1.17"
authors = ["Jai A <jaiagr+gpg@pm.me>"]
edition = "2018"
license = "MIT"
@ -21,5 +21,5 @@ chrono = { version = "0.4", features = ["serde"] }
bytes = "1"
thiserror = "1.0"
tokio = { version = "1", features = ["full"] }
sha1 = { version = "0.6.0", features = ["std"]}
bincode = {version = "2.0.0-rc", features = ["serde"], optional = true}
sha1 = { version = "0.6.1", features = ["std"]}
bincode = {version = "2.0.0-rc.2", features = ["serde"], optional = true}

12
daedalus/src/lib.rs
View File

@ -45,7 +45,7 @@ pub enum Error {
pub fn get_path_from_artifact(artifact: &str) -> Result<String, Error> {
let name_items = artifact.split(':').collect::<Vec<&str>>();
let package = name_items.get(0).ok_or_else(|| {
let package = name_items.first().ok_or_else(|| {
Error::ParseError(format!("Unable to find package for library {}", &artifact))
})?;
let name = name_items.get(1).ok_or_else(|| {
@ -60,14 +60,14 @@ pub fn get_path_from_artifact(artifact: &str) -> Result<String, Error> {
})?
.split('@')
.collect::<Vec<&str>>();
let version = version_ext.get(0).ok_or_else(|| {
let version = version_ext.first().ok_or_else(|| {
Error::ParseError(format!("Unable to find version for library {}", &artifact))
})?;
let ext = version_ext.get(1);
Ok(format!(
"{}/{}/{}/{}-{}.{}",
package.replace(".", "/"),
package.replace('.', "/"),
name,
version,
name,
@ -86,14 +86,14 @@ pub fn get_path_from_artifact(artifact: &str) -> Result<String, Error> {
})?
.split('@')
.collect::<Vec<&str>>();
let data = data_ext.get(0).ok_or_else(|| {
let data = data_ext.first().ok_or_else(|| {
Error::ParseError(format!("Unable to find data for library {}", &artifact))
})?;
let ext = data_ext.get(1);
Ok(format!(
"{}/{}/{}/{}-{}-{}.{}",
package.replace(".", "/"),
package.replace('.', "/"),
name,
version,
name,
@ -115,7 +115,7 @@ pub async fn download_file_mirrors(
}
for (index, mirror) in mirrors.iter().enumerate() {
let result = download_file(&*format!("{}{}", mirror, base), sha1).await;
let result = download_file(&format!("{}{}", mirror, base), sha1).await;
if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) {
return result;

20
daedalus_client/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "daedalus_client"
version = "0.1.14"
version = "0.1.17"
authors = ["Jai A <jaiagr+gpg@pm.me>"]
edition = "2018"
@ -9,19 +9,17 @@ edition = "2018"
[dependencies]
daedalus = { path = "../daedalus" }
tokio = { version = "1", features = ["full"] }
futures = "0.3.17"
dotenv = "0.15.0"
log = "0.4.8"
env_logger="0.9.0"
futures = "0.3.25"
dotenvy = "0.15.6"
log = "0.4.17"
env_logger= "0.10.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
lazy_static = "1.4.0"
thiserror = "1.0"
reqwest = "0.11.4"
zip = "0.5.13"
reqwest = "0.11.13"
zip = "0.6.3"
semver = "1.0"
chrono = { version = "0.4", features = ["serde"] }
bytes = "1.1.0"
rusoto_core = "0.47.0"
rusoto_s3 = "0.47.0"
bytes = "1.3.0"
rust-s3 = "0.32.3"

18
daedalus_client/src/fabric.rs
View File

@ -13,7 +13,7 @@ pub async fn retrieve_data(
uploaded_files: &mut Vec<String>,
) -> Result<(), Error> {
let mut list = fetch_fabric_versions(None).await?;
let old_manifest = daedalus::modded::fetch_manifest(&*format!(
let old_manifest = daedalus::modded::fetch_manifest(&format!(
"fabric/v{}/manifest.json",
daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
))
@ -47,11 +47,7 @@ pub async fn retrieve_data(
}
}
list.loader = list
.loader
.into_iter()
.filter(|x| loaders.iter().any(|val| val.1 == x.version))
.collect();
list.loader.retain(|x| loaders.iter().any(|val| val.1 == x.version))
}
let mut version_futures = Vec::new();
@ -79,7 +75,7 @@ pub async fn retrieve_data(
}
let version =
fetch_fabric_version(&*game_version.version, &*loader).await?;
fetch_fabric_version(&game_version.version, &loader).await?;
Ok::<Option<(Box<bool>, String, PartialVersionInfo)>, Error>(Some(
(stable, loader, version),
@ -106,10 +102,10 @@ pub async fn retrieve_data(
}
}
let artifact_path = daedalus::get_path_from_artifact(&*lib.name)?;
let artifact_path = daedalus::get_path_from_artifact(&lib.name)?;
let artifact = daedalus::download_file(
&*format!(
&format!(
"{}{}",
lib.url.unwrap_or_else(|| {
"https://maven.fabricmc.net/".to_string()
@ -169,7 +165,7 @@ pub async fn retrieve_data(
async move {
loader_version_map.push(LoaderVersion {
id: format!("{}-{}", inherits_from, loader),
url: format_url(&*version_path),
url: format_url(&version_path),
stable: *stable,
});
}
@ -280,7 +276,7 @@ async fn fetch_fabric_version(
) -> Result<PartialVersionInfo, Error> {
Ok(serde_json::from_slice(
&download_file(
&*format!(
&format!(
"{}/versions/loader/{}/{}/profile/json",
FABRIC_META_URL, version_number, loader_version
),

36
daedalus_client/src/forge.rs
View File

@ -28,7 +28,7 @@ pub async fn retrieve_data(
uploaded_files: &mut Vec<String>,
) -> Result<(), Error> {
let maven_metadata = fetch_maven_metadata(None).await?;
let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!(
let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!(
"forge/v{}/manifest.json",
daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
)))
@ -69,7 +69,7 @@ pub async fn retrieve_data(
loader_version_raw.to_string()
};
let version = Version::parse(&*loader_version)?;
let version = Version::parse(&loader_version)?;
if FORGE_MANIFEST_V1_QUERY.matches(&version)
|| FORGE_MANIFEST_V2_QUERY_P1.matches(&version)
@ -104,7 +104,7 @@ pub async fn retrieve_data(
{
let versions = versions_mutex.lock().await;
let version = versions.iter().find(|x|
x.id == minecraft_version).map(|x| x.loaders.iter().find(|x| x.id == loader_version_full)).flatten();
x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full));
if let Some(version) = version {
return Ok::<Option<LoaderVersion>, Error>(Some(version.clone()));
@ -112,7 +112,7 @@ pub async fn retrieve_data(
}
info!("Forge - Installer Start {}", loader_version_full.clone());
let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?;
let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?;
let reader = std::io::Cursor::new(bytes);
@ -125,13 +125,13 @@ pub async fn retrieve_data(
let mut contents = String::new();
install_profile.read_to_string(&mut contents)?;
Ok::<ForgeInstallerProfileV1, Error>(serde_json::from_str::<ForgeInstallerProfileV1>(&*contents)?)
Ok::<ForgeInstallerProfileV1, Error>(serde_json::from_str::<ForgeInstallerProfileV1>(&contents)?)
}).await??;
let mut archive_clone = archive.clone();
let file_path = profile.install.file_path.clone();
let forge_universal_bytes = tokio::task::spawn_blocking(move || {
let mut forge_universal_file = archive_clone.by_name(&*file_path)?;
let mut forge_universal_file = archive_clone.by_name(&file_path)?;
let mut forge_universal = Vec::new();
forge_universal_file.read_to_end(&mut forge_universal)?;
@ -156,7 +156,7 @@ pub async fn retrieve_data(
}
let artifact_path =
daedalus::get_path_from_artifact(&*lib.name)?;
daedalus::get_path_from_artifact(&lib.name)?;
let artifact = if lib.name == forge_universal_path {
forge_universal_bytes.clone()
@ -164,7 +164,7 @@ pub async fn retrieve_data(
let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"];
daedalus::download_file_mirrors(
&*artifact_path,
&artifact_path,
&mirrors,
None,
)
@ -216,7 +216,7 @@ pub async fn retrieve_data(
return Ok(Some(LoaderVersion {
id: loader_version_full,
url: format_url(&*version_path),
url: format_url(&version_path),
stable: false
}));
} else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) {
@ -227,7 +227,7 @@ pub async fn retrieve_data(
let mut contents = String::new();
install_profile.read_to_string(&mut contents)?;
Ok::<ForgeInstallerProfileV2, Error>(serde_json::from_str::<ForgeInstallerProfileV2>(&*contents)?)
Ok::<ForgeInstallerProfileV2, Error>(serde_json::from_str::<ForgeInstallerProfileV2>(&contents)?)
}).await??;
let mut archive_clone = archive.clone();
@ -237,7 +237,7 @@ pub async fn retrieve_data(
let mut contents = String::new();
install_profile.read_to_string(&mut contents)?;
Ok::<PartialVersionInfo, Error>(serde_json::from_str::<PartialVersionInfo>(&*contents)?)
Ok::<PartialVersionInfo, Error>(serde_json::from_str::<PartialVersionInfo>(&contents)?)
}).await??;
@ -255,12 +255,12 @@ pub async fn retrieve_data(
let mut local_libs : HashMap<String, bytes::Bytes> = HashMap::new();
for lib in &libs {
if lib.downloads.as_ref().map(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).flatten().unwrap_or(false) {
if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) {
let mut archive_clone = archive.clone();
let lib_name_clone = lib.name.clone();
let lib_bytes = tokio::task::spawn_blocking(move || {
let mut lib_file = archive_clone.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*lib_name_clone)?))?;
let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?;
let mut lib_bytes = Vec::new();
lib_file.read_to_end(&mut lib_bytes)?;
@ -331,7 +331,7 @@ pub async fn retrieve_data(
let now = Instant::now();
let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async {
let artifact_path =
daedalus::get_path_from_artifact(&*lib.name)?;
daedalus::get_path_from_artifact(&lib.name)?;
{
let mut visited_assets = visited_assets.lock().await;
@ -339,7 +339,7 @@ pub async fn retrieve_data(
if visited_assets.contains(&lib.name) {
if let Some(ref mut downloads) = lib.downloads {
if let Some(ref mut artifact) = downloads.artifact {
artifact.url = format_url(&*format!("maven/{}", artifact_path));
artifact.url = format_url(&format!("maven/{}", artifact_path));
}
} else if lib.url.is_some() {
lib.url = Some(format_url("maven/"));
@ -357,14 +357,14 @@ pub async fn retrieve_data(
local_libs.get(&lib.name).cloned()
} else {
Some(daedalus::download_file(
&*artifact.url,
&artifact.url,
Some(&*artifact.sha1),
)
.await?)
};
if res.is_some() {
artifact.url = format_url(&*format!("maven/{}", artifact_path));
artifact.url = format_url(&format!("maven/{}", artifact_path));
}
res
@ -431,7 +431,7 @@ pub async fn retrieve_data(
return Ok(Some(LoaderVersion {
id: loader_version_full,
url: format_url(&*version_path),
url: format_url(&version_path),
stable: false
}));
}

135
daedalus_client/src/main.rs
View File

@ -1,9 +1,8 @@
use log::{error, warn};
use rusoto_core::credential::StaticProvider;
use rusoto_core::{HttpClient, Region, RusotoError};
use rusoto_s3::{PutObjectError, S3Client};
use rusoto_s3::{PutObjectRequest, S3};
use std::time::Duration;
use s3::{Bucket, Region};
use s3::creds::Credentials;
use s3::error::S3Error;
mod fabric;
mod forge;
@ -21,7 +20,7 @@ pub enum Error {
TaskError(#[from] tokio::task::JoinError),
#[error("Error while uploading file to S3")]
S3Error {
inner: RusotoError<PutObjectError>,
inner: S3Error,
file: String,
},
#[error("Error while parsing version as semver: {0}")]
@ -70,11 +69,6 @@ async fn main() {
Err(err) => error!("{:?}", err),
};
}
match purge_digitalocean_cache(uploaded_files).await {
Ok(..) => {}
Err(err) => error!("{:?}", err),
};
}
}
@ -82,13 +76,13 @@ fn check_env_vars() -> bool {
let mut failed = false;
fn check_var<T: std::str::FromStr>(var: &str) -> bool {
if dotenv::var(var)
if dotenvy::var(var)
.ok()
.and_then(|s| s.parse::<T>().ok())
.is_none()
{
warn!(
"Variable `{}` missing in dotenv or not of type `{}`",
"Variable `{}` missing in dotenvy or not of type `{}`",
var,
std::any::type_name::<T>()
);
@ -107,36 +101,31 @@ fn check_env_vars() -> bool {
failed |= check_var::<String>("S3_REGION");
failed |= check_var::<String>("S3_BUCKET_NAME");
failed |= check_var::<bool>("DO_INTEGRATION");
let do_integration = dotenv::var("DO_INTEGRATION")
.ok()
.map(|x| x.parse::<bool>().ok())
.flatten()
.unwrap_or(false);
if do_integration {
failed |= check_var::<String>("DO_ACCESS_KEY");
failed |= check_var::<String>("DO_ENDPOINT_ID");
}
failed
}
lazy_static::lazy_static! {
static ref CLIENT : S3Client = S3Client::new_with(
HttpClient::new().unwrap(),
StaticProvider::new(
dotenv::var("S3_ACCESS_TOKEN").unwrap(),
dotenv::var("S3_SECRET").unwrap(),
None,
None,
),
static ref CLIENT : Bucket = Bucket::new(
&dotenvy::var("S3_BUCKET_NAME").unwrap(),
if &*dotenvy::var("S3_REGION").unwrap() == "r2" {
Region::R2 {
account_id: dotenvy::var("S3_URL").unwrap(),
}
} else {
Region::Custom {
name: dotenv::var("S3_REGION").unwrap(),
endpoint: dotenv::var("S3_URL").unwrap(),
},
);
region: dotenvy::var("S3_REGION").unwrap(),
endpoint: dotenvy::var("S3_URL").unwrap(),
}
},
Credentials::new(
Some(&*dotenvy::var("S3_ACCESS_TOKEN").unwrap()),
Some(&*dotenvy::var("S3_SECRET").unwrap()),
None,
None,
None,
).unwrap(),
).unwrap();
}
pub async fn upload_file_to_bucket(
@ -145,23 +134,24 @@ pub async fn upload_file_to_bucket(
content_type: Option<String>,
uploaded_files: &tokio::sync::Mutex<Vec<String>>,
) -> Result<(), Error> {
let key = format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path);
let key = format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path);
for attempt in 1..=4 {
let result = CLIENT
.put_object(PutObjectRequest {
bucket: dotenv::var("S3_BUCKET_NAME").unwrap(),
key: key.clone(),
body: Some(bytes.clone().into()),
acl: Some("public-read".to_string()),
content_type: content_type.clone(),
..Default::default()
})
.await
.map_err(|err| Error::S3Error {
inner: err,
file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path),
});
let result = if let Some(ref content_type) = content_type {
CLIENT.put_object_with_content_type(
key.clone(),
&bytes,
content_type,
).await
} else {
CLIENT.put_object(
key.clone(),
&bytes,
).await
}.map_err(|err| Error::S3Error {
inner: err,
file: format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path),
});
match result {
Ok(_) => {
@ -185,45 +175,8 @@ pub async fn upload_file_to_bucket(
pub fn format_url(path: &str) -> String {
format!(
"{}/{}/{}",
&*dotenv::var("BASE_URL").unwrap(),
&*dotenv::var("BASE_FOLDER").unwrap(),
&*dotenvy::var("BASE_URL").unwrap(),
&*dotenvy::var("BASE_FOLDER").unwrap(),
path
)
}
#[derive(serde::Serialize)]
struct PurgeCacheRequest {
pub files: Vec<String>,
}
pub async fn purge_digitalocean_cache(files: Vec<String>) -> Result<(), Error> {
if !dotenv::var("DO_INTEGRATION")
.ok()
.map(|x| x.parse::<bool>().ok())
.flatten()
.unwrap_or(false)
{
return Ok(());
}
let client = reqwest::Client::new();
client
.delete(&format!(
"https://api.digitalocean.com/v2/cdn/endpoints/{}/cache",
&*dotenv::var("DO_ENDPOINT_ID").unwrap()
))
.header(
"Authorization",
&*format!("Bearer {}", &*dotenv::var("DO_ACCESS_KEY").unwrap()),
)
.json(&PurgeCacheRequest { files })
.send()
.await
.map_err(|err| Error::FetchError {
inner: err,
item: "purging digital ocean cache".to_string(),
})?;
Ok(())
}

6
daedalus_client/src/minecraft.rs
View File

@ -8,7 +8,7 @@ use tokio::sync::Mutex;
pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionManifest, Error> {
let old_manifest =
daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!(
daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&format!(
"minecraft/v{}/manifest.json",
daedalus::minecraft::CURRENT_FORMAT_VERSION
))))
@ -43,12 +43,12 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
let cloned_manifest_mutex = Arc::clone(&cloned_manifest);
let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
let assets_hash = old_version.map(|x| x.assets_index_sha1.clone()).flatten();
let assets_hash = old_version.and_then(|x| x.assets_index_sha1.clone());
async move {
let mut upload_futures = Vec::new();
let mut version_info = daedalus::minecraft::fetch_version_info(version).await?;
let version_info = daedalus::minecraft::fetch_version_info(version).await?;
let version_path = format!(
"minecraft/v{}/versions/{}.json",