Finish newer forge versions

.env

@@ -1,3 +1,5 @@
+RUST_LOG=info,error
+
 BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com
 BASE_FOLDER=gamedata
 

daedalus/src/lib.rs

@@ -90,7 +90,7 @@ pub async fn download_file_mirrors(
 pub async fn download_file(url: &str, sha1: Option<&str>) -> Result<bytes::Bytes, Error> {
     let client = reqwest::Client::builder()
         .tcp_keepalive(Some(std::time::Duration::from_secs(10)))
-        .timeout(std::time::Duration::from_secs(30))
+        .timeout(std::time::Duration::from_secs(15))
         .build()
         .map_err(|err| Error::FetchError {
             inner: err,

daedalus/src/modded.rs

@@ -10,6 +10,15 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0;
 /// The latest version of the format the fabric model structs deserialize to
 pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0;
 
+/// A data variable entry that depends on the side of the installation
+#[derive(Serialize, Deserialize, Debug)]
+pub struct SidedDataEntry {
+    /// The value on the client
+    pub client: String,
+    /// The value on the server
+    pub server: String,
+}
+
 #[derive(Serialize, Deserialize, Debug)]
 #[serde(rename_all = "camelCase")]
 /// A partial version returned by fabric meta
@@ -31,6 +40,26 @@ pub struct PartialVersionInfo {
     #[serde(rename = "type")]
     /// The type of version
     pub type_: VersionType,
+    /// (Forge-only)
+    pub data: Option<HashMap<String, SidedDataEntry>>,
+    /// (Forge-only) The list of processors to run after downloading the files
+    pub processors: Option<Vec<Processor>>,
+}
+
+/// A processor to be ran after downloading the files
+#[derive(Serialize, Deserialize, Debug)]
+pub struct Processor {
+    /// Maven coordinates for the JAR library of this processor.
+    pub jar: String,
+    /// Maven coordinates for all the libraries that must be included in classpath when running this processor.
+    pub classpath: Vec<String>,
+    /// Arguments for this processor.
+    pub args: Vec<String>,
+    /// Represents a map of outputs. Keys and values can be data values
+    pub outputs: Option<HashMap<String, String>>,
+    /// Which sides this processor shall be ran on.
+    /// Valid values: client, server, extract
+    pub sides: Option<Vec<String>>,
 }
 
 /// Fetches the version manifest of a game version's URL
@@ -72,7 +101,7 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) ->
     }
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(rename_all = "camelCase")]
 /// A manifest containing information about a mod loader's versions
 pub struct Manifest {
@@ -91,7 +120,7 @@ pub enum LoaderType {
     Stable,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 ///  A game version of Minecraft
 pub struct Version {
     /// The minecraft version ID
@@ -100,7 +129,7 @@ pub struct Version {
     pub loaders: HashMap<LoaderType, LoaderVersion>,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 /// A version of a Minecraft mod loader
 pub struct LoaderVersion {
     /// The version ID of the loader

daedalus_client/Cargo.toml

@@ -12,6 +12,7 @@ tokio = { version = "1", features = ["full"] }
 futures = "0.3.17"
 dotenv = "0.15.0"
 log = "0.4.8"
+env_logger="0.9.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 lazy_static = "1.4.0"

daedalus_client/src/fabric.rs

@@ -145,6 +145,8 @@ pub async fn retrieve_data() -> Result<(), Error> {
                             type_: version.type_,
                             inherits_from: version.inherits_from,
                             libraries: libs,
+                            processors: None,
+                            data: None
                         })?,
                         Some("application/json".to_string()),
                     )
@@ -183,7 +185,7 @@ pub async fn retrieve_data() -> Result<(), Error> {
             let chunk: Vec<_> = versions.by_ref().take(10).collect();
             futures::future::try_join_all(chunk).await?;
 
-            std::thread::sleep(Duration::from_secs(1));
+            tokio::time::sleep(Duration::from_secs(1)).await;
 
             chunk_index += 1;
 

daedalus_client/src/forge.rs

@@ -2,7 +2,7 @@ use crate::{format_url, upload_file_to_bucket, Error};
 use chrono::{DateTime, Utc};
 use daedalus::download_file;
 use daedalus::minecraft::{Argument, ArgumentType, Library, VersionType};
-use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo};
+use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry};
 use lazy_static::lazy_static;
 use semver::{Version, VersionReq};
 use serde::{Deserialize, Serialize};
@@ -15,17 +15,21 @@ use tokio::sync::Mutex;
 lazy_static! {
     static ref FORGE_MANIFEST_V1_QUERY: VersionReq =
         VersionReq::parse(">=8.0.684, <23.5.2851").unwrap();
-    static ref FORGE_MANIFEST_V2_QUERY: VersionReq =
+
-        VersionReq::parse(">=23.5.2851, <37.0.0").unwrap();
+    static ref FORGE_MANIFEST_V2_QUERY_P1: VersionReq =
+        VersionReq::parse(">=23.5.2851, <31.2.52").unwrap();
+    static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq =
+        VersionReq::parse(">=32.0.1, <37.0.0").unwrap();
+
     static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap();
 }
 
 pub async fn retrieve_data() -> Result<(), Error> {
     let maven_metadata = fetch_maven_metadata(None).await?;
-    let old_manifest = daedalus::modded::fetch_manifest(&*format!(
+    let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!(
         "forge/v{}/manifest.json",
         daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
-    ))
+    )))
     .await
     .ok();
 
@@ -40,7 +44,9 @@ pub async fn retrieve_data() -> Result<(), Error> {
     let mut version_futures = Vec::new();
 
     for (minecraft_version, loader_versions) in maven_metadata {
-        if let Some(loader_version_full) = loader_versions.into_iter().last() {
+        let mut loaders = Vec::new();
+
+        for loader_version_full in loader_versions {
             let loader_version = loader_version_full.split('-').into_iter().nth(1);
 
             if let Some(loader_version_raw) = loader_version {
@@ -60,6 +66,13 @@ pub async fn retrieve_data() -> Result<(), Error> {
 
                 let version = Version::parse(&*loader_version)?;
 
+                if FORGE_MANIFEST_V1_QUERY.matches(&version) || FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) {
+                    loaders.push((loader_version_full, version))
+                }
+            }
+        }
+
+        if let Some((loader_version_full, version)) = loaders.into_iter().last() {
             version_futures.push(async {
                 let versions_mutex = Arc::clone(&versions);
                 let visited_assets = Arc::clone(&visited_assets_mutex);
@@ -83,17 +96,15 @@ pub async fn retrieve_data() -> Result<(), Error> {
 
                     if let Ok(mut archive) = zip::ZipArchive::new(reader) {
                         if FORGE_MANIFEST_V1_QUERY.matches(&version) {
-                                let install_profile = {
+                            let profile = {
                                 let mut install_profile = archive.by_name("install_profile.json")?;
 
                                 let mut contents = String::new();
                                 install_profile.read_to_string(&mut contents)?;
 
-                                    contents
+                                serde_json::from_str::<ForgeInstallerProfileV1>(&*contents)?
                             };
 
-                                let profile = serde_json::from_str::<ForgeInstallerProfileV1>(&*install_profile)?;
-
                             let forge_universal_bytes = {
                                 let mut forge_universal_file = archive.by_name(&*profile.install.file_path)?;
                                 let mut forge_universal =  Vec::new();
@@ -158,6 +169,8 @@ pub async fn retrieve_data() -> Result<(), Error> {
                                 arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()),
                                 libraries: libs,
                                 type_: profile.version_info.type_,
+                                data: None,
+                                processors: None
                             };
 
                             let version_path = format!(
@@ -181,17 +194,115 @@ pub async fn retrieve_data() -> Result<(), Error> {
                                 id: minecraft_version,
                                 loaders: map
                             })
-                            } else if FORGE_MANIFEST_V2_QUERY.matches(&version) {
+                        } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) {
-                                let install_profile = {
+                            let profile = {
                                 let mut install_profile = archive.by_name("install_profile.json")?;
 
                                 let mut contents = String::new();
                                 install_profile.read_to_string(&mut contents)?;
 
-                                    contents
+                                serde_json::from_str::<ForgeInstallerProfileV2>(&*contents)?
                             };
-                            } else if FORGE_MANIFEST_V3_QUERY.matches(&version) {
 
+                            let version_info = {
+                                let mut install_profile = archive.by_name("version.json")?;
+
+                                let mut contents = String::new();
+                                install_profile.read_to_string(&mut contents)?;
+                                serde_json::from_str::<PartialVersionInfo>(&*contents)?
+                            };
+
+                            let forge_universal_bytes = {
+                                if let Some(path) = &profile.path {
+                                    let mut forge_universal_file = archive.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*path)?))?;
+                                    let mut forge_universal =  Vec::new();
+                                    forge_universal_file.read_to_end(&mut forge_universal)?;
+
+                                    Some(bytes::Bytes::from(forge_universal))
+                                } else {
+                                    None
+                                }
+                            };
+
+                            let now = Instant::now();
+                            let libs = futures::future::try_join_all(profile.libraries.into_iter().chain(version_info.libraries).map(|mut lib| async {
+                                if let Some(ref mut downloads) = lib.downloads {
+                                    if let Some(ref mut artifact) = downloads.artifact {
+                                        {
+                                            let mut visited_assets = visited_assets.lock().await;
+
+                                            if visited_assets.contains(&lib.name) {
+                                                artifact.url = format_url(&*format!("maven/{}", artifact.path));
+
+                                                return Ok::<Library, Error>(lib);
+                                            } else {
+                                                visited_assets.push(lib.name.clone())
+                                            }
+                                        }
+
+                                        let artifact_path =
+                                            daedalus::get_path_from_artifact(&*lib.name)?;
+
+                                        let artifact_bytes = if &*artifact.url == "" {
+                                            forge_universal_bytes.clone().unwrap_or_default()
+                                        } else {
+                                            daedalus::download_file(
+                                                &*artifact.url,
+                                                Some(&*artifact.sha1),
+                                            )
+                                                .await?
+                                        };
+
+                                        artifact.url = format_url(&*format!("maven/{}", artifact.path));
+
+                                        upload_file_to_bucket(
+                                            format!("{}/{}", "maven", artifact_path),
+                                            artifact_bytes.to_vec(),
+                                            Some("application/java-archive".to_string()),
+                                        ).await?;
+                                    }
+                                }
+
+                                Ok::<Library, Error>(lib)
+                            })).await?;
+
+                            let elapsed = now.elapsed();
+                            println!("Elapsed lib DL: {:.2?}", elapsed);
+
+                            let new_profile = PartialVersionInfo {
+                                id: version_info.id,
+                                inherits_from: version_info.inherits_from,
+                                release_time: version_info.release_time,
+                                time: version_info.time,
+                                main_class: version_info.main_class,
+                                arguments: version_info.arguments,
+                                libraries: libs,
+                                type_: version_info.type_,
+                                data: Some(profile.data),
+                                processors: Some(profile.processors),
+                            };
+
+                            let version_path = format!(
+                                "forge/v{}/versions/{}.json",
+                                daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
+                                new_profile.id
+                            );
+
+                            upload_file_to_bucket(
+                                version_path.clone(),
+                                serde_json::to_vec(&new_profile)?,
+                                Some("application/json".to_string()),
+                            ).await?;
+
+                            let mut map = HashMap::new();
+                            map.insert(LoaderType::Latest, LoaderVersion {
+                                id: loader_version_full,
+                                url: format_url(&*version_path)
+                            });
+                            versions_mutex.lock().await.push(daedalus::modded::Version {
+                                id: minecraft_version,
+                                loaders: map
+                            })
                         }
                     }
 
@@ -202,18 +313,18 @@ pub async fn retrieve_data() -> Result<(), Error> {
             });
         }
     }
-    }
 
     {
         let mut versions_peek = version_futures.into_iter().peekable();
         let mut chunk_index = 0;
         while versions_peek.peek().is_some() {
+            println!("Chunk {} Start", chunk_index);
             let now = Instant::now();
 
             let chunk: Vec<_> = versions_peek.by_ref().take(100).collect();
             futures::future::try_join_all(chunk).await?;
 
-            std::thread::sleep(Duration::from_secs(1));
+            tokio::time::sleep(Duration::from_secs(1)).await;
 
             chunk_index += 1;
 
@@ -294,4 +405,14 @@ struct ForgeInstallerProfileV1 {
 
 #[derive(Serialize, Deserialize, Debug)]
 #[serde(rename_all = "camelCase")]
-struct ForgeInstallerProfileV2 {}
+struct ForgeInstallerProfileV2 {
+    pub spec: i32,
+    pub profile: String,
+    pub version: String,
+    pub json: String,
+    pub path: Option<String>,
+    pub minecraft: String,
+    pub data: HashMap<String, SidedDataEntry>,
+    pub libraries: Vec<Library>,
+    pub processors: Vec<Processor>,
+}

daedalus_client/src/main.rs

@@ -34,6 +34,8 @@ pub enum Error {
 
 #[tokio::main]
 async fn main() {
+    env_logger::init();
+
     if check_env_vars() {
         error!("Some environment variables are missing!");
 

daedalus_client/src/minecraft.rs

@@ -137,7 +137,7 @@ pub async fn retrieve_data() -> Result<(), Error> {
         let chunk: Vec<_> = versions.by_ref().take(100).collect();
         futures::future::try_join_all(chunk).await?;
 
-        std::thread::sleep(Duration::from_secs(1));
+        tokio::time::sleep(Duration::from_secs(1)).await;
 
         chunk_index += 1;