Fix forge install issues (#18)

* Fix forge install issues

* remove mac garb

daedalus/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "daedalus"
-version = "0.2.0"
+version = "0.2.1"
 authors = ["Jai A <jai@modrinth.com>"]
 edition = "2021"
 license = "MIT"

daedalus_client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "daedalus_client"
-version = "0.2.0"
+version = "0.2.1"
 authors = ["Jai A <jai@modrinth.com>"]
 edition = "2021"
 

daedalus_client/src/fabric.rs

@@ -17,6 +17,7 @@ pub async fn fetch_fabric(
         "fabric",
         "https://meta.fabricmc.net/v2",
         "https://maven.fabricmc.net/",
+        &[],
         semaphore,
         upload_files,
         mirror_artifacts,
@@ -34,7 +35,11 @@ pub async fn fetch_quilt(
         daedalus::modded::CURRENT_QUILT_FORMAT_VERSION,
         "quilt",
         "https://meta.quiltmc.org/v3",
-        "https://meta.quiltmc.org/",
+        "https://maven.quiltmc.org/repository/release/",
+        &[
+            // This version is broken as it contains invalid library coordinates
+            "0.17.5-beta.4",
+        ],
         semaphore,
         upload_files,
         mirror_artifacts,
@@ -48,6 +53,7 @@ async fn fetch(
     mod_loader: &str,
     meta_url: &str,
     maven_url: &str,
+    skip_versions: &[&str],
     semaphore: Arc<Semaphore>,
     upload_files: &DashMap<String, UploadFile>,
     mirror_artifacts: &DashMap<String, MirrorArtifact>,
@@ -76,6 +82,7 @@ async fn fetch(
                     .game_versions
                     .iter()
                     .any(|x| x.loaders.iter().any(|x| x.id == version.version))
+                    && !skip_versions.contains(&&*version.version)
                 {
                     fetch_versions.push(version);
                 }
@@ -98,7 +105,11 @@ async fn fetch(
             (fetch_versions, fetch_intermediary_versions)
         } else {
             (
-                fabric_manifest.loader.iter().collect(),
+                fabric_manifest
+                    .loader
+                    .iter()
+                    .filter(|x| !skip_versions.contains(&&*x.version))
+                    .collect(),
                 fabric_manifest.intermediary.iter().collect(),
             )
         };
@@ -109,7 +120,9 @@ async fn fetch(
         for x in &fetch_intermediary_versions {
             insert_mirrored_artifact(
                 &x.maven,
-                maven_url.to_string(),
+                None,
+                vec![maven_url.to_string()],
+                false,
                 mirror_artifacts,
             )?;
         }
@@ -142,13 +155,24 @@ async fn fetch(
                     let new_name = lib
                         .name
                         .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
+
+                    // Hard-code: This library is not present on fabric's maven, so we fetch it from MC libraries
+                    if &*lib.name == "net.minecraft:launchwrapper:1.12" {
+                        lib.url = Some(
+                            "https://libraries.minecraft.net/".to_string(),
+                        );
+                    }
+
                     // If a library is not intermediary, we add it to mirror artifacts to be mirrored
                     if lib.name == new_name {
                         insert_mirrored_artifact(
                             &new_name,
-                            lib.url
+                            None,
+                            vec![lib
+                                .url
                                 .clone()
-                                .unwrap_or_else(|| maven_url.to_string()),
+                                .unwrap_or_else(|| maven_url.to_string())],
+                            false,
                             mirror_artifacts,
                         )?;
                     } else {

daedalus_client/src/forge.rs

@@ -1,4 +1,6 @@
-use crate::util::{download_file, fetch_json, fetch_xml, format_url};
+use crate::util::{
+    download_file, fetch_json, fetch_xml, format_url, sha1_async,
+};
 use crate::{insert_mirrored_artifact, Error, MirrorArtifact, UploadFile};
 use chrono::{DateTime, Utc};
 use daedalus::get_path_from_artifact;
@@ -246,6 +248,7 @@ async fn fetch(
             raw: bytes::Bytes,
             loader: &ForgeVersion,
             maven_url: &str,
+            mod_loader: &str,
             upload_files: &DashMap<String, UploadFile>,
             mirror_artifacts: &DashMap<String, MirrorArtifact>,
         ) -> Result<PartialVersionInfo, Error> {
@@ -399,16 +402,28 @@ async fn fetch(
                         .into_iter()
                         .map(|mut lib| {
                             // For all libraries besides the forge lib extracted, we mirror them from maven servers
-                            if lib.name != install_profile.install.path {
+                            // unless the URL is empty/null or available on Minecraft's servers
-                                // TODO: add mirrors "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"
+                            if let Some(url) = lib.url {
+                                if lib.name != install_profile.install.path
+                                    && !url.is_empty()
+                                    && !url.contains(
+                                        "https://libraries.minecraft.net/",
+                                    )
+                                {
                                     insert_mirrored_artifact(
                                         &lib.name,
-                                    lib.url.clone().unwrap_or_else(|| {
+                                        None,
-                                        maven_url.to_string()
+                                        vec![
-                                    }),
+                                            url,
+                                            "https://maven.creeperhost.net/"
+                                                .to_string(),
+                                            maven_url.to_string(),
+                                        ],
+                                        false,
                                         mirror_artifacts,
                                     )?;
                                 }
+                            }
 
                             lib.url = Some(format_url("maven/"));
 
@@ -468,6 +483,7 @@ async fn fetch(
                 async fn mirror_forge_library(
                     mut zip: ZipFileReader,
                     mut lib: daedalus::minecraft::Library,
+                    maven_url: &str,
                     upload_files: &DashMap<String, UploadFile>,
                     mirror_artifacts: &DashMap<String, MirrorArtifact>,
                 ) -> Result<daedalus::minecraft::Library, Error>
@@ -480,7 +496,9 @@ async fn fetch(
                         if !artifact.url.is_empty() {
                             insert_mirrored_artifact(
                                 &lib.name,
-                                artifact.url.clone(),
+                                Some(artifact.sha1.clone()),
+                                vec![artifact.url.clone()],
+                                true,
                                 mirror_artifacts,
                             )?;
 
@@ -491,10 +509,18 @@ async fn fetch(
                         }
                     } else if let Some(url) = &lib.url {
                         if !url.is_empty() {
-                            // TODO: add mirrors "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"
                             insert_mirrored_artifact(
                                 &lib.name,
+                                None,
+                                vec![
                                     url.clone(),
+                                    "https://libraries.minecraft.net/"
+                                        .to_string(),
+                                    "https://maven.creeperhost.net/"
+                                        .to_string(),
+                                    maven_url.to_string(),
+                                ],
+                                false,
                                 mirror_artifacts,
                             )?;
 
@@ -531,6 +557,7 @@ async fn fetch(
                         mirror_forge_library(
                             zip.clone(),
                             lib,
+                            maven_url,
                             upload_files,
                             mirror_artifacts,
                         )
@@ -560,7 +587,7 @@ async fn fetch(
                         value: &str,
                         upload_files: &DashMap<String, UploadFile>,
                         libs: &mut Vec<daedalus::minecraft::Library>,
-                        install_profile_path: Option<&str>,
+                        mod_loader: &str,
                         version: &ForgeVersion,
                     ) -> Result<String, Error> {
                         let extract_file =
@@ -595,11 +622,9 @@ async fn fetch(
                             })?;
 
                         let path = format!(
-                            "{}:{}@{}",
+                            "com.modrinth.daedalus:{}-installer-extracts:{}:{}@{}",
-                            install_profile_path.unwrap_or(&*format!(
+                            mod_loader,
-                                "net.minecraftforge:forge:{}",
+                            version.raw,
-                                version.raw
-                            )),
                             file_name,
                             ext
                         );
@@ -634,7 +659,7 @@ async fn fetch(
                             &entry.client,
                             upload_files,
                             &mut version_info.libraries,
-                            install_profile.path.as_deref(),
+                            mod_loader,
                             loader,
                         )
                         .await?
@@ -649,7 +674,7 @@ async fn fetch(
                             &entry.server,
                             upload_files,
                             &mut version_info.libraries,
-                            install_profile.path.as_deref(),
+                            mod_loader,
                             loader,
                         )
                         .await?
@@ -686,6 +711,7 @@ async fn fetch(
                         raw,
                         loader,
                         maven_url,
+                        mod_loader,
                         upload_files,
                         mirror_artifacts,
                     )

daedalus_client/src/main.rs

@@ -72,12 +72,28 @@ async fn main() -> Result<()> {
     futures::future::try_join_all(mirror_artifacts.iter().map(|x| {
         upload_url_to_bucket_mirrors(
             format!("maven/{}", x.key()),
-            x.value().mirrors.iter().map(|x| x.key().clone()).collect(),
+            x.value()
+                .mirrors
+                .iter()
+                .map(|mirror| {
+                    if mirror.entire_url {
+                        mirror.path.clone()
+                    } else {
+                        format!("{}{}", mirror.path, x.key())
+                    }
+                })
+                .collect(),
+            x.sha1.clone(),
             &semaphore,
         )
     }))
     .await?;
 
+    if dotenvy::var("CLOUDFLARE_INTEGRATION")
+        .ok()
+        .and_then(|x| x.parse::<bool>().ok())
+        .unwrap_or(false)
+    {
         if let Ok(token) = dotenvy::var("CLOUDFLARE_TOKEN") {
             if let Ok(zone_id) = dotenvy::var("CLOUDFLARE_ZONE_ID") {
                 let cache_clears = upload_files
@@ -115,6 +131,7 @@ async fn main() -> Result<()> {
                 }
             }
         }
+    }
 
     Ok(())
 }
@@ -125,21 +142,37 @@ pub struct UploadFile {
 }
 
 pub struct MirrorArtifact {
-    pub mirrors: DashSet<String>,
+    pub sha1: Option<String>,
+    pub mirrors: DashSet<Mirror>,
 }
 
+#[derive(Eq, PartialEq, Hash)]
+pub struct Mirror {
+    path: String,
+    entire_url: bool,
+}
+
+#[tracing::instrument(skip(mirror_artifacts))]
 pub fn insert_mirrored_artifact(
     artifact: &str,
-    mirror: String,
+    sha1: Option<String>,
+    mirrors: Vec<String>,
+    entire_url: bool,
     mirror_artifacts: &DashMap<String, MirrorArtifact>,
 ) -> Result<()> {
-    mirror_artifacts
+    let mut val = mirror_artifacts
         .entry(get_path_from_artifact(artifact)?)
         .or_insert(MirrorArtifact {
+            sha1,
             mirrors: DashSet::new(),
-        })
+        });
-        .mirrors
+
-        .insert(mirror);
+    for mirror in mirrors {
+        val.mirrors.insert(Mirror {
+            path: mirror,
+            entire_url,
+        });
+    }
 
     Ok(())
 }

daedalus_client/src/util.rs

@@ -1,6 +1,5 @@
 use crate::{Error, ErrorKind};
-use bytes::{Bytes, BytesMut};
+use bytes::Bytes;
-use futures::StreamExt;
 use s3::creds::Credentials;
 use s3::{Bucket, Region};
 use serde::de::DeserializeOwned;
@@ -95,8 +94,9 @@ pub async fn upload_file_to_bucket(
 }
 
 pub async fn upload_url_to_bucket_mirrors(
-    base: String,
+    upload_path: String,
     mirrors: Vec<String>,
+    sha1: Option<String>,
     semaphore: &Arc<Semaphore>,
 ) -> Result<(), Error> {
     if mirrors.is_empty() {
@@ -108,8 +108,9 @@ pub async fn upload_url_to_bucket_mirrors(
 
     for (index, mirror) in mirrors.iter().enumerate() {
         let result = upload_url_to_bucket(
-            &base,
+            upload_path.clone(),
-            &format!("{}{}", mirror, base),
+            mirror.clone(),
+            sha1.clone(),
             semaphore,
         )
         .await;
@@ -124,152 +125,16 @@ pub async fn upload_url_to_bucket_mirrors(
 
 #[tracing::instrument(skip(semaphore))]
 pub async fn upload_url_to_bucket(
-    path: &str,
+    path: String,
-    url: &str,
+    url: String,
+    sha1: Option<String>,
     semaphore: &Arc<Semaphore>,
 ) -> Result<(), Error> {
-    let _permit = semaphore.acquire().await?;
+    let data = download_file(&url, sha1.as_deref(), semaphore).await?;
 
-    const RETRIES: i32 = 3;
+    upload_file_to_bucket(path, data, None, semaphore).await?;
-    for attempt in 1..=(RETRIES + 1) {
-        tracing::trace!("Attempting streaming file upload, attempt {attempt}");
-
-        let result: Result<(), Error> = {
-            let response =
-                REQWEST_CLIENT.get(url).send().await.map_err(|err| {
-                    ErrorKind::Fetch {
-                        inner: err,
-                        item: url.to_string(),
-                    }
-                })?;
-
-            let content_type = response
-                .headers()
-                .get(reqwest::header::CONTENT_TYPE)
-                .and_then(|ct| ct.to_str().ok())
-                .unwrap_or("application/octet-stream")
-                .to_string();
-
-            let total_size = response.content_length().unwrap_or(0);
-
-            const MIN_PART_SIZE: usize = 5 * 1024 * 1024;
-
-            if total_size < MIN_PART_SIZE as u64 {
-                let data =
-                    response.bytes().await.map_err(|err| ErrorKind::Fetch {
-                        inner: err,
-                        item: url.to_string(),
-                    })?;
-                BUCKET.put_object(&path, &data).await.map_err(|err| {
-                    ErrorKind::S3 {
-                        inner: err,
-                        file: path.to_string(),
-                    }
-                })?;
-            } else {
-                let mut stream = response.bytes_stream();
-
-                let multipart = BUCKET
-                    .initiate_multipart_upload(path, &content_type)
-                    .await
-                    .map_err(|err| ErrorKind::S3 {
-                        inner: err,
-                        file: path.to_string(),
-                    })?;
-
-                let mut parts = Vec::new();
-                let mut buffer = BytesMut::new();
-
-                async fn upload_part(
-                    parts: &mut Vec<s3::serde_types::Part>,
-                    buffer: Vec<u8>,
-                    path: &str,
-                    upload_id: &str,
-                    content_type: &str,
-                ) -> Result<(), Error> {
-                    let part = BUCKET
-                        .put_multipart_chunk(
-                            buffer,
-                            path,
-                            (parts.len() + 1) as u32,
-                            upload_id,
-                            content_type,
-                        )
-                        .await
-                        .map_err(|err| ErrorKind::S3 {
-                            inner: err,
-                            file: path.to_string(),
-                        })?;
-
-                    parts.push(part);
 
     Ok(())
-                }
-
-                while let Some(chunk) = stream.next().await {
-                    let chunk = chunk.map_err(|err| ErrorKind::Fetch {
-                        inner: err,
-                        item: url.to_string(),
-                    })?;
-
-                    buffer.extend_from_slice(&chunk);
-
-                    if buffer.len() >= MIN_PART_SIZE {
-                        upload_part(
-                            &mut parts,
-                            buffer.to_vec(),
-                            path,
-                            &multipart.upload_id,
-                            &content_type,
-                        )
-                        .await?;
-                        buffer.clear();
-                    }
-                }
-
-                if !buffer.is_empty() {
-                    let part = BUCKET
-                        .put_multipart_chunk(
-                            buffer.to_vec(),
-                            path,
-                            (parts.len() + 1) as u32,
-                            &multipart.upload_id,
-                            &content_type,
-                        )
-                        .await
-                        .map_err(|err| ErrorKind::S3 {
-                            inner: err,
-                            file: path.to_string(),
-                        })?;
-
-                    parts.push(part);
-                }
-
-                BUCKET
-                    .complete_multipart_upload(
-                        path,
-                        &multipart.upload_id,
-                        parts,
-                    )
-                    .await
-                    .map_err(|err| ErrorKind::S3 {
-                        inner: err,
-                        file: path.to_string(),
-                    })?;
-            }
-
-            Ok(())
-        };
-
-        match result {
-            Ok(_) => return Ok(()),
-            Err(_) if attempt <= RETRIES => continue,
-            Err(_) => {
-                result?;
-            }
-        }
-    }
-    unreachable!()
 }
 
 #[tracing::instrument(skip(bytes))]
@@ -294,7 +159,7 @@ pub async fn download_file(
     const RETRIES: u32 = 10;
     for attempt in 1..=(RETRIES + 1) {
         let result = REQWEST_CLIENT
-            .get(url)
+            .get(&url.replace("http://", "https://"))
             .send()
             .await
             .and_then(|x| x.error_for_status());