diff --git a/.env b/.env index 4e9b803ba..56db8578a 100644 --- a/.env +++ b/.env @@ -1,3 +1,5 @@ +RUST_LOG=info,error + BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com BASE_FOLDER=gamedata diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index a4ac00599..5e0f5e422 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -90,7 +90,7 @@ pub async fn download_file_mirrors( pub async fn download_file(url: &str, sha1: Option<&str>) -> Result { let client = reqwest::Client::builder() .tcp_keepalive(Some(std::time::Duration::from_secs(10))) - .timeout(std::time::Duration::from_secs(30)) + .timeout(std::time::Duration::from_secs(15)) .build() .map_err(|err| Error::FetchError { inner: err, diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index cff051fa6..c631b6193 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -10,6 +10,15 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; +/// A data variable entry that depends on the side of the installation +#[derive(Serialize, Deserialize, Debug)] +pub struct SidedDataEntry { + /// The value on the client + pub client: String, + /// The value on the server + pub server: String, +} + #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// A partial version returned by fabric meta @@ -31,6 +40,26 @@ pub struct PartialVersionInfo { #[serde(rename = "type")] /// The type of version pub type_: VersionType, + /// (Forge-only) + pub data: Option>, + /// (Forge-only) The list of processors to run after downloading the files + pub processors: Option>, +} + +/// A processor to be ran after downloading the files +#[derive(Serialize, Deserialize, Debug)] +pub struct Processor { + /// Maven coordinates for the JAR library of this processor. + pub jar: String, + /// Maven coordinates for all the libraries that must be included in classpath when running this processor. + pub classpath: Vec, + /// Arguments for this processor. + pub args: Vec, + /// Represents a map of outputs. Keys and values can be data values + pub outputs: Option>, + /// Which sides this processor shall be ran on. + /// Valid values: client, server, extract + pub sides: Option>, } /// Fetches the version manifest of a game version's URL @@ -72,7 +101,7 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> } } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] /// A manifest containing information about a mod loader's versions pub struct Manifest { @@ -91,7 +120,7 @@ pub enum LoaderType { Stable, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A game version of Minecraft pub struct Version { /// The minecraft version ID @@ -100,7 +129,7 @@ pub struct Version { pub loaders: HashMap, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A version of a Minecraft mod loader pub struct LoaderVersion { /// The version ID of the loader diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 33f1c0638..f36f6f3b1 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -12,6 +12,7 @@ tokio = { version = "1", features = ["full"] } futures = "0.3.17" dotenv = "0.15.0" log = "0.4.8" +env_logger="0.9.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" lazy_static = "1.4.0" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 63f637f40..5ac71f16f 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -145,6 +145,8 @@ pub async fn retrieve_data() -> Result<(), Error> { type_: version.type_, inherits_from: version.inherits_from, libraries: libs, + processors: None, + data: None })?, Some("application/json".to_string()), ) @@ -183,7 +185,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let chunk: Vec<_> = versions.by_ref().take(10).collect(); futures::future::try_join_all(chunk).await?; - std::thread::sleep(Duration::from_secs(1)); + tokio::time::sleep(Duration::from_secs(1)).await; chunk_index += 1; diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index f06d23b00..af31f015f 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -2,7 +2,7 @@ use crate::{format_url, upload_file_to_bucket, Error}; use chrono::{DateTime, Utc}; use daedalus::download_file; use daedalus::minecraft::{Argument, ArgumentType, Library, VersionType}; -use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo}; +use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry}; use lazy_static::lazy_static; use semver::{Version, VersionReq}; use serde::{Deserialize, Serialize}; @@ -15,17 +15,21 @@ use tokio::sync::Mutex; lazy_static! { static ref FORGE_MANIFEST_V1_QUERY: VersionReq = VersionReq::parse(">=8.0.684, <23.5.2851").unwrap(); - static ref FORGE_MANIFEST_V2_QUERY: VersionReq = - VersionReq::parse(">=23.5.2851, <37.0.0").unwrap(); + + static ref FORGE_MANIFEST_V2_QUERY_P1: VersionReq = + VersionReq::parse(">=23.5.2851, <31.2.52").unwrap(); + static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq = + VersionReq::parse(">=32.0.1, <37.0.0").unwrap(); + static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap(); } pub async fn retrieve_data() -> Result<(), Error> { let maven_metadata = fetch_maven_metadata(None).await?; - let old_manifest = daedalus::modded::fetch_manifest(&*format!( + let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( "forge/v{}/manifest.json", daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - )) + ))) .await .ok(); @@ -40,7 +44,9 @@ pub async fn retrieve_data() -> Result<(), Error> { let mut version_futures = Vec::new(); for (minecraft_version, loader_versions) in maven_metadata { - if let Some(loader_version_full) = loader_versions.into_iter().last() { + let mut loaders = Vec::new(); + + for loader_version_full in loader_versions { let loader_version = loader_version_full.split('-').into_iter().nth(1); if let Some(loader_version_raw) = loader_version { @@ -60,57 +66,173 @@ pub async fn retrieve_data() -> Result<(), Error> { let version = Version::parse(&*loader_version)?; - version_futures.push(async { - let versions_mutex = Arc::clone(&versions); - let visited_assets = Arc::clone(&visited_assets_mutex); - async move { - { - if versions_mutex.lock().await.iter().any(|x| { - x.id == minecraft_version - && x.loaders - .get(&LoaderType::Latest) - .map(|x| x.id == loader_version_full) - .unwrap_or(false) - }) { - return Ok(()); - } + if FORGE_MANIFEST_V1_QUERY.matches(&version) || FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { + loaders.push((loader_version_full, version)) + } + } + } + + if let Some((loader_version_full, version)) = loaders.into_iter().last() { + version_futures.push(async { + let versions_mutex = Arc::clone(&versions); + let visited_assets = Arc::clone(&visited_assets_mutex); + async move { + { + if versions_mutex.lock().await.iter().any(|x| { + x.id == minecraft_version + && x.loaders + .get(&LoaderType::Latest) + .map(|x| x.id == loader_version_full) + .unwrap_or(false) + }) { + return Ok(()); } + } - println!("installer start {}", loader_version_full.clone()); - let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; + println!("installer start {}", loader_version_full.clone()); + let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; - let reader = std::io::Cursor::new(&*bytes); + let reader = std::io::Cursor::new(&*bytes); - if let Ok(mut archive) = zip::ZipArchive::new(reader) { - if FORGE_MANIFEST_V1_QUERY.matches(&version) { - let install_profile = { - let mut install_profile = archive.by_name("install_profile.json")?; + if let Ok(mut archive) = zip::ZipArchive::new(reader) { + if FORGE_MANIFEST_V1_QUERY.matches(&version) { + let profile = { + let mut install_profile = archive.by_name("install_profile.json")?; - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; - contents - }; + serde_json::from_str::(&*contents)? + }; - let profile = serde_json::from_str::(&*install_profile)?; + let forge_universal_bytes = { + let mut forge_universal_file = archive.by_name(&*profile.install.file_path)?; + let mut forge_universal = Vec::new(); + forge_universal_file.read_to_end(&mut forge_universal)?; - let forge_universal_bytes = { - let mut forge_universal_file = archive.by_name(&*profile.install.file_path)?; + bytes::Bytes::from(forge_universal) + }; + let forge_universal_path = profile.install.file_path.clone(); + + let now = Instant::now(); + let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { + if let Some(url) = lib.url { + { + let mut visited_assets = visited_assets.lock().await; + + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); + + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; + + let artifact = if lib.name == forge_universal_path { + forge_universal_bytes.clone() + } else { + let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; + + daedalus::download_file_mirrors( + &*artifact_path, + &mirrors, + None, + ) + .await? + }; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + ).await?; + } + + Ok::(lib) + })).await?; + + let elapsed = now.elapsed(); + println!("Elapsed lib DL: {:.2?}", elapsed); + + let new_profile = PartialVersionInfo { + id: profile.version_info.id, + inherits_from: profile.install.minecraft, + release_time: profile.version_info.release_time, + time: profile.version_info.time, + main_class: profile.version_info.main_class, + arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), + libraries: libs, + type_: profile.version_info.type_, + data: None, + processors: None + }; + + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + ).await?; + + let mut map = HashMap::new(); + map.insert(LoaderType::Latest, LoaderVersion { + id: loader_version_full, + url: format_url(&*version_path) + }); + versions_mutex.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + loaders: map + }) + } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { + let profile = { + let mut install_profile = archive.by_name("install_profile.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + serde_json::from_str::(&*contents)? + }; + + let version_info = { + let mut install_profile = archive.by_name("version.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + serde_json::from_str::(&*contents)? + }; + + let forge_universal_bytes = { + if let Some(path) = &profile.path { + let mut forge_universal_file = archive.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*path)?))?; let mut forge_universal = Vec::new(); forge_universal_file.read_to_end(&mut forge_universal)?; - bytes::Bytes::from(forge_universal) - }; - let forge_universal_path = profile.install.file_path.clone(); + Some(bytes::Bytes::from(forge_universal)) + } else { + None + } + }; - let now = Instant::now(); - let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { - if let Some(url) = lib.url { + let now = Instant::now(); + let libs = futures::future::try_join_all(profile.libraries.into_iter().chain(version_info.libraries).map(|mut lib| async { + if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { { let mut visited_assets = visited_assets.lock().await; if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); + artifact.url = format_url(&*format!("maven/{}", artifact.path)); return Ok::(lib); } else { @@ -121,86 +243,74 @@ pub async fn retrieve_data() -> Result<(), Error> { let artifact_path = daedalus::get_path_from_artifact(&*lib.name)?; - let artifact = if lib.name == forge_universal_path { - forge_universal_bytes.clone() + let artifact_bytes = if &*artifact.url == "" { + forge_universal_bytes.clone().unwrap_or_default() } else { - let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; - - daedalus::download_file_mirrors( - &*artifact_path, - &mirrors, - None, + daedalus::download_file( + &*artifact.url, + Some(&*artifact.sha1), ) .await? }; - lib.url = Some(format_url("maven/")); + artifact.url = format_url(&*format!("maven/{}", artifact.path)); upload_file_to_bucket( format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), + artifact_bytes.to_vec(), Some("application/java-archive".to_string()), ).await?; } + } - Ok::(lib) - })).await?; + Ok::(lib) + })).await?; - let elapsed = now.elapsed(); - println!("Elapsed lib DL: {:.2?}", elapsed); + let elapsed = now.elapsed(); + println!("Elapsed lib DL: {:.2?}", elapsed); - let new_profile = PartialVersionInfo { - id: profile.version_info.id, - inherits_from: profile.install.minecraft, - release_time: profile.version_info.release_time, - time: profile.version_info.time, - main_class: profile.version_info.main_class, - arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), - libraries: libs, - type_: profile.version_info.type_, - }; + let new_profile = PartialVersionInfo { + id: version_info.id, + inherits_from: version_info.inherits_from, + release_time: version_info.release_time, + time: version_info.time, + main_class: version_info.main_class, + arguments: version_info.arguments, + libraries: libs, + type_: version_info.type_, + data: Some(profile.data), + processors: Some(profile.processors), + }; - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - ).await?; + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + ).await?; - let mut map = HashMap::new(); - map.insert(LoaderType::Latest, LoaderVersion { - id: loader_version_full, - url: format_url(&*version_path) - }); - versions_mutex.lock().await.push(daedalus::modded::Version { - id: minecraft_version, - loaders: map - }) - } else if FORGE_MANIFEST_V2_QUERY.matches(&version) { - let install_profile = { - let mut install_profile = archive.by_name("install_profile.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - contents - }; - } else if FORGE_MANIFEST_V3_QUERY.matches(&version) { - - } + let mut map = HashMap::new(); + map.insert(LoaderType::Latest, LoaderVersion { + id: loader_version_full, + url: format_url(&*version_path) + }); + versions_mutex.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + loaders: map + }) } - - Ok::<(), Error>(()) - }.await?; + } Ok::<(), Error>(()) - }); - } + }.await?; + + Ok::<(), Error>(()) + }); } } @@ -208,12 +318,13 @@ pub async fn retrieve_data() -> Result<(), Error> { let mut versions_peek = version_futures.into_iter().peekable(); let mut chunk_index = 0; while versions_peek.peek().is_some() { + println!("Chunk {} Start", chunk_index); let now = Instant::now(); let chunk: Vec<_> = versions_peek.by_ref().take(100).collect(); futures::future::try_join_all(chunk).await?; - std::thread::sleep(Duration::from_secs(1)); + tokio::time::sleep(Duration::from_secs(1)).await; chunk_index += 1; @@ -294,4 +405,14 @@ struct ForgeInstallerProfileV1 { #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileV2 {} +struct ForgeInstallerProfileV2 { + pub spec: i32, + pub profile: String, + pub version: String, + pub json: String, + pub path: Option, + pub minecraft: String, + pub data: HashMap, + pub libraries: Vec, + pub processors: Vec, +} \ No newline at end of file diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 14f4fb133..7d0511422 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -34,6 +34,8 @@ pub enum Error { #[tokio::main] async fn main() { + env_logger::init(); + if check_env_vars() { error!("Some environment variables are missing!"); diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 07907c4f3..c6062dbb2 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -137,7 +137,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let chunk: Vec<_> = versions.by_ref().take(100).collect(); futures::future::try_join_all(chunk).await?; - std::thread::sleep(Duration::from_secs(1)); + tokio::time::sleep(Duration::from_secs(1)).await; chunk_index += 1;