diff --git a/daedalus/src/fabric.rs b/daedalus/src/fabric.rs new file mode 100644 index 000000000..052486ffa --- /dev/null +++ b/daedalus/src/fabric.rs @@ -0,0 +1,132 @@ +use crate::minecraft::{Argument, ArgumentType, Library, VersionInfo, VersionType}; +use crate::{download_file, Error}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// The latest version of the format the model structs deserialize to +pub const CURRENT_FORMAT_VERSION: usize = 0; + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// A partial version returned by fabric meta +pub struct PartialVersionInfo { + /// The version ID of the version + pub id: String, + /// The version ID this partial version inherits from + pub inherits_from: String, + /// The time that the version was released + pub release_time: DateTime, + /// The latest time a file in this version was updated + pub time: DateTime, + /// The classpath to the main class to launch the game + pub main_class: String, + /// Arguments passed to the game or JVM + pub arguments: Option>>, + /// Libraries that the version depends on + pub libraries: Vec, + #[serde(rename = "type")] + /// The type of version + pub type_: VersionType, +} + +/// Merges a partial version into a complete one +pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> VersionInfo { + VersionInfo { + arguments: if let Some(partial_args) = partial.arguments { + if let Some(merge_args) = merge.arguments { + Some(partial_args.into_iter().chain(merge_args).collect()) + } else { + Some(partial_args) + } + } else { + merge.arguments + }, + asset_index: merge.asset_index, + assets: merge.assets, + downloads: merge.downloads, + id: merge.id, + libraries: partial + .libraries + .into_iter() + .chain(merge.libraries) + .collect::>(), + main_class: partial.main_class, + minecraft_arguments: merge.minecraft_arguments, + minimum_launcher_version: merge.minimum_launcher_version, + release_time: partial.release_time, + time: partial.time, + type_: partial.type_, + } +} + +/// The default servers for fabric meta +pub const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2"; + +/// Fetches the manifest of a fabric loader version and game version +pub async fn fetch_fabric_version( + version_number: &str, + loader_version: &str, +) -> Result { + Ok(serde_json::from_slice( + &download_file( + &*format!( + "{}/versions/loader/{}/{}/profile/json", + FABRIC_META_URL, version_number, loader_version + ), + None, + ) + .await?, + )?) +} + +/// Fetches the manifest of a game version's URL +pub async fn fetch_fabric_game_version(url: &str) -> Result { + Ok(serde_json::from_slice(&download_file(url, None).await?)?) +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// Versions of fabric components +pub struct FabricVersions { + /// Versions of Minecraft that fabric supports + pub game: Vec, + /// Available versions of the fabric loader + pub loader: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of Minecraft that fabric supports +pub struct FabricGameVersion { + /// The version number of the game + pub version: String, + /// Whether the Minecraft version is stable or not + pub stable: bool, + /// (Modrinth Provided) The URLs to download this version's profile with a loader + /// The key of the map is the loader version, and the value is the URL + pub urls: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of the fabric loader +pub struct FabricLoaderVersion { + /// The separator to get the build number + pub separator: String, + /// The build number + pub build: u32, + /// The maven artifact + pub maven: String, + /// The version number of the fabric loader + pub version: String, + /// Whether the loader is stable or not + pub stable: bool, +} +/// Fetches the list of fabric versions +pub async fn fetch_fabric_versions(url: Option<&str>) -> Result { + Ok(serde_json::from_slice( + &download_file( + url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)), + None, + ) + .await?, + )?) +} diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 8aea3e1ec..92ff542f6 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -4,6 +4,8 @@ #![warn(missing_docs, unused_import_braces, missing_debug_implementations)] +/// Models and methods for fetching metadata for the Fabric mod loader +pub mod fabric; /// Models and methods for fetching metadata for Minecraft pub mod minecraft; @@ -34,6 +36,32 @@ pub enum Error { /// There was an error when managing async tasks #[error("Error while managing asynchronous tasks")] TaskError(#[from] tokio::task::JoinError), + /// Error while parsing input + #[error("{0}")] + ParseError(String), +} + +/// Converts a maven artifact to a path +pub fn get_path_from_artifact(artifact: &str) -> Result { + let name_items = artifact.split(':').collect::>(); + + let package = name_items.get(0).ok_or_else(|| { + Error::ParseError(format!("Unable to find package for library {}", &artifact)) + })?; + let name = name_items.get(1).ok_or_else(|| { + Error::ParseError(format!("Unable to find name for library {}", &artifact)) + })?; + let version = name_items.get(2).ok_or_else(|| { + Error::ParseError(format!("Unable to find version for library {}", &artifact)) + })?; + + Ok(format!( + "{}/{}/{}-{}.jar", + package.replace(".", "/"), + version, + name, + version + )) } /// Downloads a file with retry and checksum functionality diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 89008da16..e90e8f4d4 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -69,7 +69,7 @@ pub struct LatestVersion { } #[derive(Serialize, Deserialize, Debug, Clone)] -/// Data of all game versions of Minecraft +/// Data of all game versions of Minecrafat pub struct VersionManifest { /// A struct containing the latest snapshot and release of the game pub latest: LatestVersion, @@ -181,10 +181,13 @@ pub enum Os { #[derive(Serialize, Deserialize, Debug)] /// A rule which depends on what OS the user is on pub struct OsRule { + #[serde(skip_serializing_if = "Option::is_none")] /// The name of the OS pub name: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// The version of the OS. This is normally a RegEx pub version: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// The architecture of the OS pub arch: Option, } @@ -192,8 +195,10 @@ pub struct OsRule { #[derive(Serialize, Deserialize, Debug)] /// A rule which depends on the toggled features of the launcher pub struct FeatureRule { + #[serde(skip_serializing_if = "Option::is_none")] /// Whether the user is in demo mode pub is_demo_user: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// Whether the user is using the demo resolution pub has_demo_resolution: Option, } @@ -203,8 +208,10 @@ pub struct FeatureRule { pub struct Rule { /// The action the rule takes pub action: RuleAction, + #[serde(skip_serializing_if = "Option::is_none")] /// The OS rule pub os: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// The feature rule pub features: Option, } @@ -212,6 +219,7 @@ pub struct Rule { #[derive(Serialize, Deserialize, Debug)] /// Information delegating the extraction of the library pub struct LibraryExtract { + #[serde(skip_serializing_if = "Option::is_none")] /// Files/Folders to be excluded from the extraction of the library pub exclude: Option>, } @@ -219,14 +227,21 @@ pub struct LibraryExtract { #[derive(Serialize, Deserialize, Debug)] /// A library which the game relies on to run pub struct Library { + #[serde(skip_serializing_if = "Option::is_none")] /// The files the library has - pub downloads: LibraryDownloads, + pub downloads: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// Rules of the extraction of the file pub extract: Option, /// The maven name of the library. The format is `groupId:artifactId:version` pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + /// The URL to the repository where the library can be downloaded + pub url: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// Native files that the library relies on pub natives: Option>, + #[serde(skip_serializing_if = "Option::is_none")] /// Rules deciding whether the library should be downloaded or not pub rules: Option>, } @@ -270,6 +285,7 @@ pub enum ArgumentType { #[serde(rename_all = "camelCase")] /// Information about a version pub struct VersionInfo { + #[serde(skip_serializing_if = "Option::is_none")] /// Arguments passed to the game or JVM pub arguments: Option>>, /// Assets for the game @@ -284,6 +300,7 @@ pub struct VersionInfo { pub libraries: Vec, /// The classpath to the main class to launch the game pub main_class: String, + #[serde(skip_serializing_if = "Option::is_none")] /// (Legacy) Arguments passed to the game pub minecraft_arguments: Option, /// The minimum version of the Minecraft Launcher that can run this version of the game diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs new file mode 100644 index 000000000..890bd5be1 --- /dev/null +++ b/daedalus_client/src/fabric.rs @@ -0,0 +1,188 @@ +use crate::{format_url, upload_file_to_bucket, Error}; +use daedalus::fabric::PartialVersionInfo; +use daedalus::minecraft::Library; +use std::collections::HashMap; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::{Duration, Instant}; + +pub async fn retrieve_data() -> Result<(), Error> { + let mut list = daedalus::fabric::fetch_fabric_versions(None).await?; + + let loaders = RwLock::new(Vec::new()); + let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); + + if let Some(latest) = list.loader.get(0) { + { + let mut loaders = match loaders.write() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + loaders.push(latest.version.clone()); + + if !latest.stable { + if let Some(stable) = list.loader.iter().find(|x| x.stable) { + loaders.push(stable.version.clone()); + } + } + + list.loader = list + .loader + .into_iter() + .filter(|x| loaders.contains(&x.version)) + .collect(); + } + + let mut versions = list + .game + .iter_mut() + .map(|game_version| { + let loaders = match loaders.read() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex); + let game_version_mutex = Mutex::new(HashMap::new()); + + async move { + let versions = futures::future::try_join_all(loaders.clone().into_iter().map( + |loader| async { + let version = daedalus::fabric::fetch_fabric_version( + &*game_version.version, + &*loader, + ) + .await + .expect(&*format!("{}, {}", game_version.version, loader)); + + Ok::<(String, PartialVersionInfo), Error>((loader, version)) + }, + )) + .await?; + + futures::future::try_join_all(versions.into_iter().map( + |(loader, version)| async { + let libs = futures::future::try_join_all( + version.libraries.into_iter().map(|mut lib| async { + { + let mut visited_assets = + match visited_artifacts_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); + + return Ok(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; + + let artifact = daedalus::download_file( + &*format!( + "{}{}", + lib.url.unwrap_or_else(|| { + "https://maven.fabricmc.net/".to_string() + }), + artifact_path + ), + None, + ) + .await?; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + ) + .await?; + + Ok::(lib) + }), + ) + .await?; + + let version_path = format!( + "fabric/v{}/versions/{}-{}.json", + daedalus::fabric::CURRENT_FORMAT_VERSION, + version.inherits_from, + loader + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&PartialVersionInfo { + arguments: version.arguments, + id: version.id, + main_class: version.main_class, + release_time: version.release_time, + time: version.time, + type_: version.type_, + inherits_from: version.inherits_from, + libraries: libs, + })?, + Some("application/json".to_string()), + ) + .await?; + + { + let mut game_version_map = match game_version_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + game_version_map.insert(loader, format_url(&*version_path)); + } + + Ok::<(), Error>(()) + }, + )) + .await?; + + game_version.urls = Some( + match game_version_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } + .clone(), + ); + + Ok::<(), Error>(()) + } + }) + .peekable(); + + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(10).collect(); + futures::future::try_join_all(chunk).await?; + + std::thread::sleep(Duration::from_secs(1)); + + chunk_index += 1; + + let elapsed = now.elapsed(); + println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + + upload_file_to_bucket( + format!( + "fabric/v{}/manifest.json", + daedalus::fabric::CURRENT_FORMAT_VERSION, + ), + serde_json::to_vec(&list)?, + Some("application/json".to_string()), + ) + .await?; + + Ok(()) +} diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index bb10303ac..7ac9a1861 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,9 +1,10 @@ use log::{error, info, warn}; use rusoto_core::credential::StaticProvider; use rusoto_core::{HttpClient, Region, RusotoError}; -use rusoto_s3::{S3Client, PutObjectError}; +use rusoto_s3::{PutObjectError, S3Client}; use rusoto_s3::{PutObjectRequest, S3}; +mod fabric; mod minecraft; #[derive(thiserror::Error, Debug)] @@ -13,10 +14,7 @@ pub enum Error { #[error("Error while deserializing JSON")] SerdeError(#[from] serde_json::Error), #[error("Unable to fetch {item}")] - FetchError { - inner: reqwest::Error, - item: String, - }, + FetchError { inner: reqwest::Error, item: String }, #[error("Error while managing asynchronous tasks")] TaskError(#[from] tokio::task::JoinError), #[error("Error while uploading file to S3")] @@ -34,7 +32,7 @@ async fn main() { return; } - minecraft::retrieve_data().await.unwrap(); + fabric::retrieve_data().await.unwrap(); } fn check_env_vars() -> bool { @@ -96,7 +94,11 @@ lazy_static::lazy_static! { ); } -pub async fn upload_file_to_bucket(path: String, bytes: Vec, content_type: Option) -> Result<(), Error> { +pub async fn upload_file_to_bucket( + path: String, + bytes: Vec, + content_type: Option, +) -> Result<(), Error> { CLIENT .put_object(PutObjectRequest { bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), @@ -109,7 +111,7 @@ pub async fn upload_file_to_bucket(path: String, bytes: Vec, content_type: O .await .map_err(|err| Error::S3Error { inner: err, - file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path) + file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), })?; Ok(()) diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 1336e9ffb..ddcf286e7 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -6,14 +6,13 @@ use std::time::{Duration, Instant}; pub async fn retrieve_data() -> Result<(), Error> { let old_manifest = daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!( - "minecraft/v{}/version_manifest.json", + "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION )))) .await .ok(); - let mut manifest = daedalus::minecraft::fetch_version_manifest(None) - .await?; + let mut manifest = daedalus::minecraft::fetch_version_manifest(None).await?; let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); @@ -44,12 +43,7 @@ pub async fn retrieve_data() -> Result<(), Error> { async move { let mut upload_futures = Vec::new(); - let now = Instant::now(); - let mut version_println = daedalus::minecraft::fetch_version_info(version) - .await - ?; - let elapsed = now.elapsed(); - println!("Version {} Elapsed: {:.2?}", version.id, elapsed); + let mut version_info = daedalus::minecraft::fetch_version_info(version).await?; let version_path = format!( "minecraft/v{}/versions/{}.json", @@ -59,9 +53,9 @@ pub async fn retrieve_data() -> Result<(), Error> { let assets_path = format!( "minecraft/v{}/assets/{}.json", daedalus::minecraft::CURRENT_FORMAT_VERSION, - version_println.asset_index.id + version_info.asset_index.id ); - let assets_index_url = version_println.asset_index.url.clone(); + let assets_index_url = version_info.asset_index.url.clone(); { let mut cloned_manifest = match cloned_manifest_mutex.lock() { @@ -76,10 +70,10 @@ pub async fn retrieve_data() -> Result<(), Error> { .unwrap(); cloned_manifest.versions[position].url = format_url(&version_path); cloned_manifest.versions[position].assets_index_sha1 = - Some(version_println.asset_index.sha1.clone()); + Some(version_info.asset_index.sha1.clone()); cloned_manifest.versions[position].assets_index_url = Some(format_url(&assets_path)); - version_println.asset_index.url = format_url(&assets_path); + version_info.asset_index.url = format_url(&assets_path); } let mut download_assets = false; @@ -90,9 +84,9 @@ pub async fn retrieve_data() -> Result<(), Error> { Err(poisoned) => poisoned.into_inner(), }; - if !visited_assets.contains(&version_println.asset_index.id) { + if !visited_assets.contains(&version_info.asset_index.id) { if let Some(assets_hash) = assets_hash { - if version_println.asset_index.sha1 != assets_hash { + if version_info.asset_index.sha1 != assets_hash { download_assets = true; } } else { @@ -101,26 +95,29 @@ pub async fn retrieve_data() -> Result<(), Error> { } if download_assets { - visited_assets.push(version_println.asset_index.id.clone()); + visited_assets.push(version_info.asset_index.id.clone()); } } if download_assets { let assets_index = - download_file(&assets_index_url, Some(&version_println.asset_index.sha1)) + download_file(&assets_index_url, Some(&version_info.asset_index.sha1)) .await?; { - upload_futures - .push(upload_file_to_bucket(assets_path, assets_index.to_vec(), Some("application/json".to_string()))); + upload_futures.push(upload_file_to_bucket( + assets_path, + assets_index.to_vec(), + Some("application/json".to_string()), + )); } } { upload_futures.push(upload_file_to_bucket( version_path, - serde_json::to_vec(&version_println)?, - Some("application/json".to_string()) + serde_json::to_vec(&version_info)?, + Some("application/json".to_string()), )); } @@ -154,14 +151,14 @@ pub async fn retrieve_data() -> Result<(), Error> { upload_file_to_bucket( format!( - "minecraft/v{}/version_manifest.json", + "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION ), serde_json::to_vec(&*match cloned_manifest.lock() { Ok(guard) => guard, Err(poisoned) => poisoned.into_inner(), })?, - Some("application/json".to_string()) + Some("application/json".to_string()), ) .await?;