Working mirroring of minecraft metadata

.env

@@ -0,0 +1,11 @@
+BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com
+BASE_FOLDER=gamedata
+
+S3_ACCESS_TOKEN=none
+S3_SECRET=none
+S3_URL=none
+S3_REGION=none
+S3_BUCKET_NAME=none
+
+DO_INTEGRATION=false
+DO_ACCESS_KEY=none

.gitignore

@@ -0,0 +1,121 @@
+### Intellij ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### Intellij Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+# https://plugins.jetbrains.com/plugin/7973-sonarlint
+.idea/**/sonarlint/
+
+# SonarQube Plugin
+# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
+.idea/**/sonarIssues.xml
+
+# Markdown Navigator plugin
+# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
+.idea/**/markdown-navigator.xml
+.idea/**/markdown-navigator-enh.xml
+.idea/**/markdown-navigator/
+
+# Cache file creation bug
+# See https://youtrack.jetbrains.com/issue/JBR-2257
+.idea/$CACHE_FILE$
+
+# CodeStream plugin
+# https://plugins.jetbrains.com/plugin/12206-codestream
+.idea/codestream.xml
+
+### Rust ###
+# Generated by Cargo
+# will have compiled files and executables
+debug/
+target/
+
+# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
+# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
+Cargo.lock
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
+# Editor-based HTTP Client requests
+/httpRequests/

.idea/daedalus.iml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/daedalus/src" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/daedalus_client/src" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/daedalus.iml" filepath="$PROJECT_DIR$/.idea/daedalus.iml" />
+    </modules>
+  </component>
+</project>

.idea/runConfigurations.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="RunConfigurationProducerService">
+    <option name="ignoredProducers">
+      <set>
+        <option value="com.android.tools.idea.compose.preview.runconfiguration.ComposePreviewRunConfigurationProducer" />
+      </set>
+    </option>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

Cargo.toml

@@ -0,0 +1,6 @@
+[workspace]
+
+members = [
+    "daedalus",
+    "daedalus_client"
+]

daedalus/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "daedalus"
+version = "0.1.0"
+authors = ["Jai A <jaiagr+gpg@pm.me>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+reqwest = { version = "0.11", features = ["json"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+chrono = { version = "0.4", features = ["serde"] }
+bytes = "1"
+thiserror = "1.0"
+tokio = { version = "1", features = ["full"] }
+sha1 = { version = "0.6.0", features = ["std"]}

daedalus/src/lib.rs

@@ -0,0 +1,99 @@
+//! # Daedalus
+//!
+//! Daedalus is a library which provides models and methods to fetch metadata about games
+
+#![warn(missing_docs, unused_import_braces, missing_debug_implementations)]
+
+/// Models and methods for fetching metadata for Minecraft
+pub mod minecraft;
+
+#[derive(thiserror::Error, Debug)]
+/// An error type representing possible errors when fetching metadata
+pub enum Error {
+    #[error("Failed to validate file checksum at url {url} with hash {hash} after {tries} tries")]
+    /// A checksum was failed to validate for a file
+    ChecksumFailure {
+        /// The checksum's hash
+        hash: String,
+        /// The URL of the file attempted to be downloaded
+        url: String,
+        /// The amount of tries that the file was downloaded until failure
+        tries: u32,
+    },
+    /// There was an error while deserializing metadata
+    #[error("Error while deserializing JSON")]
+    SerdeError(#[from] serde_json::Error),
+    /// There was a network error when fetching an object
+    #[error("Unable to fetch {item}")]
+    FetchError {
+        /// The internal reqwest error
+        inner: reqwest::Error,
+        /// The item that was failed to be fetched
+        item: String,
+    },
+    /// There was an error when managing async tasks
+    #[error("Error while managing asynchronous tasks")]
+    TaskError(#[from] tokio::task::JoinError),
+}
+
+/// Downloads a file with retry and checksum functionality
+pub async fn download_file(url: &str, sha1: Option<&str>) -> Result<bytes::Bytes, Error> {
+    let client = reqwest::Client::builder()
+        .tcp_keepalive(Some(std::time::Duration::from_secs(10)))
+        .build()
+        .map_err(|err| Error::FetchError {
+            inner: err,
+            item: url.to_string(),
+        })?;
+
+    for attempt in 1..=4 {
+        let result = client.get(url).send().await;
+
+        match result {
+            Ok(x) => {
+                let bytes = x.bytes().await;
+
+                if let Ok(bytes) = bytes {
+                    if let Some(sha1) = sha1 {
+                        if &*get_hash(bytes.clone()).await? != sha1 {
+                            if attempt <= 3 {
+                                continue;
+                            } else {
+                                return Err(Error::ChecksumFailure {
+                                    hash: sha1.to_string(),
+                                    url: url.to_string(),
+                                    tries: attempt,
+                                });
+                            }
+                        }
+                    }
+
+                    return Ok(bytes);
+                } else if attempt <= 3 {
+                    continue;
+                } else if let Err(err) = bytes {
+                    return Err(Error::FetchError {
+                        inner: err,
+                        item: url.to_string(),
+                    });
+                }
+            }
+            Err(_) if attempt <= 3 => continue,
+            Err(err) => {
+                return Err(Error::FetchError {
+                    inner: err,
+                    item: url.to_string(),
+                })
+            }
+        }
+    }
+
+    unreachable!()
+}
+
+/// Computes a checksum of the input bytes
+pub async fn get_hash(bytes: bytes::Bytes) -> Result<String, Error> {
+    let hash = tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()).await?;
+
+    Ok(hash)
+}

daedalus/src/minecraft.rs

@@ -0,0 +1,328 @@
+use crate::{download_file, Error};
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+/// The latest version of the format the model structs deserialize to
+pub const CURRENT_FORMAT_VERSION: usize = 0;
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "snake_case")]
+/// The version type
+pub enum VersionType {
+    /// A major version, which is stable for all players to use
+    Release,
+    /// An experimental version, which is unstable and used for feature previews and beta testing
+    Snapshot,
+    /// The oldest versions before the game was released
+    OldAlpha,
+    /// Early versions of the game
+    OldBeta,
+}
+
+impl VersionType {
+    /// Converts the version type to a string
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            VersionType::Release => "release",
+            VersionType::Snapshot => "snapshot",
+            VersionType::OldAlpha => "old_alpha",
+            VersionType::OldBeta => "old_beta",
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+/// A game version of Minecraft
+pub struct Version {
+    /// A unique identifier of the version
+    pub id: String,
+    #[serde(rename = "type")]
+    /// The release type of the version
+    pub type_: VersionType,
+    /// A link to additional information about the version
+    pub url: String,
+    /// The latest time a file in this version was updated
+    pub time: DateTime<Utc>,
+    /// The time this version was released
+    pub release_time: DateTime<Utc>,
+    /// The SHA1 hash of the additional information about the version
+    pub sha1: String,
+    /// Whether the version supports the latest player safety features
+    pub compliance_level: u32,
+    /// (Modrinth Provided) The link to the assets index for this version
+    /// This is only available when using the Modrinth mirror
+    pub assets_index_url: Option<String>,
+    /// (Modrinth Provided) The SHA1 hash of the assets index for this version
+    /// This is only available when using the Modrinth mirror
+    pub assets_index_sha1: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+/// The latest snapshot and release of the game
+pub struct LatestVersion {
+    /// The version id of the latest release
+    pub release: String,
+    /// The version id of the latest snapshot
+    pub snapshot: String,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+/// Data of all game versions of Minecraft
+pub struct VersionManifest {
+    /// A struct containing the latest snapshot and release of the game
+    pub latest: LatestVersion,
+    /// A list of game versions of Minecraft
+    pub versions: Vec<Version>,
+}
+
+/// The URL to the version manifest
+pub const VERSION_MANIFEST_URL: &str =
+    "https://launchermeta.mojang.com/mc/game/version_manifest_v2.json";
+
+/// Fetches a version manifest from the specified URL. If no URL is specified, the default is used.
+pub async fn fetch_version_manifest(url: Option<&str>) -> Result<VersionManifest, Error> {
+    Ok(serde_json::from_slice(
+        &download_file(url.unwrap_or(VERSION_MANIFEST_URL), None).await?,
+    )?)
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+/// Information about the assets of the game
+pub struct AssetIndex {
+    /// The game version ID the assets are for
+    pub id: String,
+    /// The SHA1 hash of the assets index
+    pub sha1: String,
+    /// The size of the assets index
+    pub size: u32,
+    /// The size of the game version's assets
+    pub total_size: u32,
+    /// A URL to a file which contains information about the version's assets
+    pub url: String,
+}
+
+#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)]
+#[serde(rename_all = "snake_case")]
+/// The type of download
+pub enum DownloadType {
+    /// The download is for the game client
+    Client,
+    /// The download is mappings for the game
+    ClientMappings,
+    /// The download is for the game server
+    Server,
+    /// The download is mappings for the game server
+    ServerMappings,
+    /// The download is for the windows server
+    WindowsServer,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+/// Download information of a file
+pub struct Download {
+    /// The SHA1 hash of the file
+    pub sha1: String,
+    /// The size of the file
+    pub size: u32,
+    /// The URL where the file can be downloaded
+    pub url: String,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+/// Download information of a library
+pub struct LibraryDownload {
+    /// The path that the library should be saved to
+    pub path: String,
+    /// The SHA1 hash of the library
+    pub sha1: String,
+    /// The size of the library
+    pub size: u32,
+    /// The URL where the library can be downloaded
+    pub url: String,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+/// A list of files that should be downloaded for libraries
+pub struct LibraryDownloads {
+    /// The primary library artifact
+    pub artifact: Option<LibraryDownload>,
+    /// Conditional files that may be needed to be downloaded alongside the library
+    /// The HashMap key specifies a classifier as additional information for downloading files
+    pub classifiers: Option<HashMap<String, LibraryDownload>>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "snake_case")]
+/// The action a rule can follow
+pub enum RuleAction {
+    /// The rule's status allows something to be done
+    Allow,
+    /// The rule's status disallows something to be done
+    Disallow,
+}
+
+#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)]
+#[serde(rename_all = "snake_case")]
+/// An enum representing the different types of operating systems
+pub enum Os {
+    /// MacOS
+    Osx,
+    /// Windows
+    Windows,
+    /// Linux and its derivatives
+    Linux,
+    /// The OS is unknown
+    Unknown,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+/// A rule which depends on what OS the user is on
+pub struct OsRule {
+    /// The name of the OS
+    pub name: Option<Os>,
+    /// The version of the OS. This is normally a RegEx
+    pub version: Option<String>,
+    /// The architecture of the OS
+    pub arch: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+/// A rule which depends on the toggled features of the launcher
+pub struct FeatureRule {
+    /// Whether the user is in demo mode
+    pub is_demo_user: Option<bool>,
+    /// Whether the user is using the demo resolution
+    pub has_demo_resolution: Option<bool>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+/// A rule deciding whether a file is downloaded, an argument is used, etc.
+pub struct Rule {
+    /// The action the rule takes
+    pub action: RuleAction,
+    /// The OS rule
+    pub os: Option<OsRule>,
+    /// The feature rule
+    pub features: Option<FeatureRule>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+/// Information delegating the extraction of the library
+pub struct LibraryExtract {
+    /// Files/Folders to be excluded from the extraction of the library
+    pub exclude: Option<Vec<String>>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+/// A library which the game relies on to run
+pub struct Library {
+    /// The files the library has
+    pub downloads: LibraryDownloads,
+    /// Rules of the extraction of the file
+    pub extract: Option<LibraryExtract>,
+    /// The maven name of the library. The format is `groupId:artifactId:version`
+    pub name: String,
+    /// Native files that the library relies on
+    pub natives: Option<HashMap<Os, String>>,
+    /// Rules deciding whether the library should be downloaded or not
+    pub rules: Option<Vec<Rule>>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(untagged)]
+/// A container for an argument or multiple arguments
+pub enum ArgumentValue {
+    /// The container has one argument
+    Single(String),
+    /// The container has multiple arguments
+    Many(Vec<String>),
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(untagged)]
+/// A command line argument passed to a program
+pub enum Argument {
+    /// An argument which is applied no matter what
+    Normal(String),
+    /// An argument which is only applied if certain conditions are met
+    Ruled {
+        /// The rules deciding whether the argument(s) is used or not
+        rules: Vec<Rule>,
+        /// The container of the argument(s) that should be applied accordingly
+        value: ArgumentValue,
+    },
+}
+
+#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)]
+#[serde(rename_all = "snake_case")]
+/// The type of argument
+pub enum ArgumentType {
+    /// The argument is passed to the game
+    Game,
+    /// The argument is passed to the JVM
+    Jvm,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+/// Information about a version
+pub struct VersionInfo {
+    /// Arguments passed to the game or JVM
+    pub arguments: Option<HashMap<ArgumentType, Vec<Argument>>>,
+    /// Assets for the game
+    pub asset_index: AssetIndex,
+    /// The version ID of the assets
+    pub assets: String,
+    /// Game downloads of the version
+    pub downloads: HashMap<DownloadType, Download>,
+    /// The version ID of the version
+    pub id: String,
+    /// Libraries that the version depends on
+    pub libraries: Vec<Library>,
+    /// The classpath to the main class to launch the game
+    pub main_class: String,
+    /// (Legacy) Arguments passed to the game
+    pub minecraft_arguments: Option<String>,
+    /// The minimum version of the Minecraft Launcher that can run this version of the game
+    pub minimum_launcher_version: u32,
+    /// The time that the version was released
+    pub release_time: DateTime<Utc>,
+    /// The latest time a file in this version was updated
+    pub time: DateTime<Utc>,
+    #[serde(rename = "type")]
+    /// The type of version
+    pub type_: VersionType,
+}
+
+/// Fetches detailed information about a version from the manifest
+pub async fn fetch_version_info(version: &Version) -> Result<VersionInfo, Error> {
+    Ok(serde_json::from_slice(
+        &download_file(&version.url, Some(&version.sha1)).await?,
+    )?)
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+/// An asset of the game
+pub struct Asset {
+    /// The SHA1 hash of the asset file
+    pub hash: String,
+    /// The size of the asset file
+    pub size: u32,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+/// An index containing all assets the game needs
+pub struct AssetsIndex {
+    /// A hashmap containing the filename (key) and asset (value)
+    pub objects: HashMap<String, Asset>,
+}
+
+/// Fetches the assets index from the version info
+pub async fn fetch_assets_index(version: &VersionInfo) -> Result<AssetsIndex, Error> {
+    Ok(serde_json::from_slice(
+        &download_file(&version.asset_index.url, Some(&version.asset_index.sha1)).await?,
+    )?)
+}

daedalus_client/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "daedalus_client"
+version = "0.1.0"
+authors = ["Jai A <jaiagr+gpg@pm.me>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+daedalus = { path = "../daedalus" }
+tokio = { version = "1", features = ["full"] }
+futures = "0.3.17"
+dotenv = "0.15.0"
+log = "0.4.8"
+serde_json = "1.0"
+lazy_static = "1.4.0"
+thiserror = "1.0"
+reqwest = "0.11.4"
+
+rusoto_core = "0.47.0"
+rusoto_s3 = "0.47.0"

daedalus_client/src/main.rs

@@ -0,0 +1,125 @@
+use log::{error, info, warn};
+use rusoto_core::credential::StaticProvider;
+use rusoto_core::{HttpClient, Region, RusotoError};
+use rusoto_s3::{S3Client, PutObjectError};
+use rusoto_s3::{PutObjectRequest, S3};
+
+mod minecraft;
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("{0}")]
+    DaedalusError(#[from] daedalus::Error),
+    #[error("Error while deserializing JSON")]
+    SerdeError(#[from] serde_json::Error),
+    #[error("Unable to fetch {item}")]
+    FetchError {
+        inner: reqwest::Error,
+        item: String,
+    },
+    #[error("Error while managing asynchronous tasks")]
+    TaskError(#[from] tokio::task::JoinError),
+    #[error("Error while uploading file to S3")]
+    S3Error {
+        inner: RusotoError<PutObjectError>,
+        file: String,
+    },
+}
+
+#[tokio::main]
+async fn main() {
+    if check_env_vars() {
+        error!("Some environment variables are missing!");
+
+        return;
+    }
+
+    minecraft::retrieve_data().await.unwrap();
+}
+
+fn check_env_vars() -> bool {
+    let mut failed = false;
+
+    fn check_var<T: std::str::FromStr>(var: &str) -> bool {
+        if dotenv::var(var)
+            .ok()
+            .and_then(|s| s.parse::<T>().ok())
+            .is_none()
+        {
+            warn!(
+                "Variable `{}` missing in dotenv or not of type `{}`",
+                var,
+                std::any::type_name::<T>()
+            );
+            true
+        } else {
+            false
+        }
+    }
+
+    failed |= check_var::<String>("BASE_URL");
+    failed |= check_var::<String>("BASE_FOLDER");
+
+    failed |= check_var::<String>("S3_ACCESS_TOKEN");
+    failed |= check_var::<String>("S3_SECRET");
+    failed |= check_var::<String>("S3_URL");
+    failed |= check_var::<String>("S3_REGION");
+    failed |= check_var::<String>("S3_BUCKET_NAME");
+
+    failed |= check_var::<bool>("DO_INTEGRATION");
+
+    let do_integration = dotenv::var("DO_INTEGRATION")
+        .ok()
+        .map(|x| x.parse::<bool>().ok())
+        .flatten();
+
+    if do_integration.unwrap_or(false) {
+        failed |= check_var::<bool>("DO_ACCESS_KEY");
+    }
+
+    failed
+}
+
+lazy_static::lazy_static! {
+    static ref CLIENT : S3Client = S3Client::new_with(
+            HttpClient::new().unwrap(),
+            StaticProvider::new(
+                dotenv::var("S3_ACCESS_TOKEN").unwrap(),
+                dotenv::var("S3_SECRET").unwrap(),
+                None,
+                None,
+            ),
+            Region::Custom {
+                name: dotenv::var("S3_REGION").unwrap(),
+                endpoint: dotenv::var("S3_URL").unwrap(),
+            },
+        );
+}
+
+pub async fn upload_file_to_bucket(path: String, bytes: Vec<u8>, content_type: Option<String>) -> Result<(), Error> {
+    CLIENT
+        .put_object(PutObjectRequest {
+            bucket: dotenv::var("S3_BUCKET_NAME").unwrap(),
+            key: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path),
+            body: Some(bytes.into()),
+            acl: Some("public-read".to_string()),
+            content_type,
+            ..Default::default()
+        })
+        .await
+        .map_err(|err| Error::S3Error {
+            inner: err,
+            file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path)
+        })?;
+
+    Ok(())
+}
+
+pub fn format_url(path: &str) -> String {
+    format!(
+        "{}/{}/{}",
+        &*dotenv::var("BASE_URL").unwrap(),
+        &*dotenv::var("BASE_FOLDER").unwrap(),
+        path
+    )
+}

daedalus_client/src/minecraft.rs

@@ -0,0 +1,172 @@
+use crate::{format_url, upload_file_to_bucket, Error};
+use daedalus::download_file;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+pub async fn retrieve_data() -> Result<(), Error> {
+    let old_manifest =
+        daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!(
+            "minecraft/v{}/version_manifest.json",
+            daedalus::minecraft::CURRENT_FORMAT_VERSION
+        ))))
+        .await
+        .ok();
+
+    let mut manifest = daedalus::minecraft::fetch_version_manifest(None)
+        .await?;
+    let cloned_manifest = Arc::new(Mutex::new(manifest.clone()));
+
+    let visited_assets_mutex = Arc::new(Mutex::new(Vec::new()));
+
+    let now = Instant::now();
+
+    let mut versions = manifest
+        .versions
+        .iter_mut()
+        .map(|version| async {
+            let old_version = if let Some(old_manifest) = &old_manifest {
+                old_manifest.versions.iter().find(|x| x.id == version.id)
+            } else {
+                None
+            };
+
+            if let Some(old_version) = old_version {
+                if old_version.sha1 == version.sha1 {
+                    return Ok(());
+                }
+            }
+
+            let visited_assets_mutex = Arc::clone(&visited_assets_mutex);
+            let cloned_manifest_mutex = Arc::clone(&cloned_manifest);
+
+            let assets_hash = old_version.map(|x| x.assets_index_sha1.clone()).flatten();
+
+            async move {
+                let mut upload_futures = Vec::new();
+
+                let now = Instant::now();
+                let mut version_println = daedalus::minecraft::fetch_version_info(version)
+                    .await
+                    ?;
+                let elapsed = now.elapsed();
+                println!("Version {} Elapsed: {:.2?}", version.id, elapsed);
+
+                let version_path = format!(
+                    "minecraft/v{}/versions/{}.json",
+                    daedalus::minecraft::CURRENT_FORMAT_VERSION,
+                    version.id
+                );
+                let assets_path = format!(
+                    "minecraft/v{}/assets/{}.json",
+                    daedalus::minecraft::CURRENT_FORMAT_VERSION,
+                    version_println.asset_index.id
+                );
+                let assets_index_url = version_println.asset_index.url.clone();
+
+                {
+                    let mut cloned_manifest = match cloned_manifest_mutex.lock() {
+                        Ok(guard) => guard,
+                        Err(poisoned) => poisoned.into_inner(),
+                    };
+
+                    let position = cloned_manifest
+                        .versions
+                        .iter()
+                        .position(|x| version.id == x.id)
+                        .unwrap();
+                    cloned_manifest.versions[position].url = format_url(&version_path);
+                    cloned_manifest.versions[position].assets_index_sha1 =
+                        Some(version_println.asset_index.sha1.clone());
+                    cloned_manifest.versions[position].assets_index_url =
+                        Some(format_url(&assets_path));
+                    version_println.asset_index.url = format_url(&assets_path);
+                }
+
+                let mut download_assets = false;
+
+                {
+                    let mut visited_assets = match visited_assets_mutex.lock() {
+                        Ok(guard) => guard,
+                        Err(poisoned) => poisoned.into_inner(),
+                    };
+
+                    if !visited_assets.contains(&version_println.asset_index.id) {
+                        if let Some(assets_hash) = assets_hash {
+                            if version_println.asset_index.sha1 != assets_hash {
+                                download_assets = true;
+                            }
+                        } else {
+                            download_assets = true;
+                        }
+                    }
+
+                    if download_assets {
+                        visited_assets.push(version_println.asset_index.id.clone());
+                    }
+                }
+
+                if download_assets {
+                    let assets_index =
+                        download_file(&assets_index_url, Some(&version_println.asset_index.sha1))
+                            .await?;
+
+                    {
+                        upload_futures
+                            .push(upload_file_to_bucket(assets_path, assets_index.to_vec(), Some("application/json".to_string())));
+                    }
+                }
+
+                {
+                    upload_futures.push(upload_file_to_bucket(
+                        version_path,
+                        serde_json::to_vec(&version_println)?,
+                        Some("application/json".to_string())
+                    ));
+                }
+
+                let now = Instant::now();
+                futures::future::try_join_all(upload_futures).await?;
+                let elapsed = now.elapsed();
+                println!("Spaces Upload {} Elapsed: {:.2?}", version.id, elapsed);
+
+                Ok::<(), Error>(())
+            }
+            .await?;
+
+            Ok::<(), Error>(())
+        })
+        .peekable();
+
+    let mut chunk_index = 0;
+    while versions.peek().is_some() {
+        let now = Instant::now();
+
+        let chunk: Vec<_> = versions.by_ref().take(100).collect();
+        futures::future::try_join_all(chunk).await?;
+
+        std::thread::sleep(Duration::from_secs(1));
+
+        chunk_index += 1;
+
+        let elapsed = now.elapsed();
+        println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
+    }
+
+    upload_file_to_bucket(
+        format!(
+            "minecraft/v{}/version_manifest.json",
+            daedalus::minecraft::CURRENT_FORMAT_VERSION
+        ),
+        serde_json::to_vec(&*match cloned_manifest.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        })?,
+        Some("application/json".to_string())
+    )
+    .await?;
+
+    let elapsed = now.elapsed();
+    println!("Elapsed: {:.2?}", elapsed);
+
+    Ok(())
+}