commit 4a7f4bde4a4384d92b79768cecaf147a7033e6f8 Author: Jai A Date: Tue Oct 5 22:52:17 2021 -0700 Working mirroring of minecraft metadata diff --git a/.env b/.env new file mode 100644 index 000000000..4e9b803ba --- /dev/null +++ b/.env @@ -0,0 +1,11 @@ +BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com +BASE_FOLDER=gamedata + +S3_ACCESS_TOKEN=none +S3_SECRET=none +S3_URL=none +S3_REGION=none +S3_BUCKET_NAME=none + +DO_INTEGRATION=false +DO_ACCESS_KEY=none \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..dae6eff4c --- /dev/null +++ b/.gitignore @@ -0,0 +1,121 @@ +### Intellij ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Intellij Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + +### Rust ### +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 000000000..73f69e095 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/daedalus.iml b/.idea/daedalus.iml new file mode 100644 index 000000000..ec7bb0139 --- /dev/null +++ b/.idea/daedalus.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 000000000..c73594817 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations.xml b/.idea/runConfigurations.xml new file mode 100644 index 000000000..797acea53 --- /dev/null +++ b/.idea/runConfigurations.xml @@ -0,0 +1,10 @@ + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 000000000..35eb1ddfb --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..dc360c2e5 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,6 @@ +[workspace] + +members = [ + "daedalus", + "daedalus_client" +] \ No newline at end of file diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml new file mode 100644 index 000000000..a52c207f5 --- /dev/null +++ b/daedalus/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "daedalus" +version = "0.1.0" +authors = ["Jai A "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +reqwest = { version = "0.11", features = ["json"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +chrono = { version = "0.4", features = ["serde"] } +bytes = "1" +thiserror = "1.0" +tokio = { version = "1", features = ["full"] } +sha1 = { version = "0.6.0", features = ["std"]} \ No newline at end of file diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs new file mode 100644 index 000000000..8aea3e1ec --- /dev/null +++ b/daedalus/src/lib.rs @@ -0,0 +1,99 @@ +//! # Daedalus +//! +//! Daedalus is a library which provides models and methods to fetch metadata about games + +#![warn(missing_docs, unused_import_braces, missing_debug_implementations)] + +/// Models and methods for fetching metadata for Minecraft +pub mod minecraft; + +#[derive(thiserror::Error, Debug)] +/// An error type representing possible errors when fetching metadata +pub enum Error { + #[error("Failed to validate file checksum at url {url} with hash {hash} after {tries} tries")] + /// A checksum was failed to validate for a file + ChecksumFailure { + /// The checksum's hash + hash: String, + /// The URL of the file attempted to be downloaded + url: String, + /// The amount of tries that the file was downloaded until failure + tries: u32, + }, + /// There was an error while deserializing metadata + #[error("Error while deserializing JSON")] + SerdeError(#[from] serde_json::Error), + /// There was a network error when fetching an object + #[error("Unable to fetch {item}")] + FetchError { + /// The internal reqwest error + inner: reqwest::Error, + /// The item that was failed to be fetched + item: String, + }, + /// There was an error when managing async tasks + #[error("Error while managing asynchronous tasks")] + TaskError(#[from] tokio::task::JoinError), +} + +/// Downloads a file with retry and checksum functionality +pub async fn download_file(url: &str, sha1: Option<&str>) -> Result { + let client = reqwest::Client::builder() + .tcp_keepalive(Some(std::time::Duration::from_secs(10))) + .build() + .map_err(|err| Error::FetchError { + inner: err, + item: url.to_string(), + })?; + + for attempt in 1..=4 { + let result = client.get(url).send().await; + + match result { + Ok(x) => { + let bytes = x.bytes().await; + + if let Ok(bytes) = bytes { + if let Some(sha1) = sha1 { + if &*get_hash(bytes.clone()).await? != sha1 { + if attempt <= 3 { + continue; + } else { + return Err(Error::ChecksumFailure { + hash: sha1.to_string(), + url: url.to_string(), + tries: attempt, + }); + } + } + } + + return Ok(bytes); + } else if attempt <= 3 { + continue; + } else if let Err(err) = bytes { + return Err(Error::FetchError { + inner: err, + item: url.to_string(), + }); + } + } + Err(_) if attempt <= 3 => continue, + Err(err) => { + return Err(Error::FetchError { + inner: err, + item: url.to_string(), + }) + } + } + } + + unreachable!() +} + +/// Computes a checksum of the input bytes +pub async fn get_hash(bytes: bytes::Bytes) -> Result { + let hash = tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()).await?; + + Ok(hash) +} diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs new file mode 100644 index 000000000..89008da16 --- /dev/null +++ b/daedalus/src/minecraft.rs @@ -0,0 +1,328 @@ +use crate::{download_file, Error}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// The latest version of the format the model structs deserialize to +pub const CURRENT_FORMAT_VERSION: usize = 0; + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "snake_case")] +/// The version type +pub enum VersionType { + /// A major version, which is stable for all players to use + Release, + /// An experimental version, which is unstable and used for feature previews and beta testing + Snapshot, + /// The oldest versions before the game was released + OldAlpha, + /// Early versions of the game + OldBeta, +} + +impl VersionType { + /// Converts the version type to a string + pub fn as_str(&self) -> &'static str { + match self { + VersionType::Release => "release", + VersionType::Snapshot => "snapshot", + VersionType::OldAlpha => "old_alpha", + VersionType::OldBeta => "old_beta", + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +/// A game version of Minecraft +pub struct Version { + /// A unique identifier of the version + pub id: String, + #[serde(rename = "type")] + /// The release type of the version + pub type_: VersionType, + /// A link to additional information about the version + pub url: String, + /// The latest time a file in this version was updated + pub time: DateTime, + /// The time this version was released + pub release_time: DateTime, + /// The SHA1 hash of the additional information about the version + pub sha1: String, + /// Whether the version supports the latest player safety features + pub compliance_level: u32, + /// (Modrinth Provided) The link to the assets index for this version + /// This is only available when using the Modrinth mirror + pub assets_index_url: Option, + /// (Modrinth Provided) The SHA1 hash of the assets index for this version + /// This is only available when using the Modrinth mirror + pub assets_index_sha1: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// The latest snapshot and release of the game +pub struct LatestVersion { + /// The version id of the latest release + pub release: String, + /// The version id of the latest snapshot + pub snapshot: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// Data of all game versions of Minecraft +pub struct VersionManifest { + /// A struct containing the latest snapshot and release of the game + pub latest: LatestVersion, + /// A list of game versions of Minecraft + pub versions: Vec, +} + +/// The URL to the version manifest +pub const VERSION_MANIFEST_URL: &str = + "https://launchermeta.mojang.com/mc/game/version_manifest_v2.json"; + +/// Fetches a version manifest from the specified URL. If no URL is specified, the default is used. +pub async fn fetch_version_manifest(url: Option<&str>) -> Result { + Ok(serde_json::from_slice( + &download_file(url.unwrap_or(VERSION_MANIFEST_URL), None).await?, + )?) +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// Information about the assets of the game +pub struct AssetIndex { + /// The game version ID the assets are for + pub id: String, + /// The SHA1 hash of the assets index + pub sha1: String, + /// The size of the assets index + pub size: u32, + /// The size of the game version's assets + pub total_size: u32, + /// A URL to a file which contains information about the version's assets + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] +#[serde(rename_all = "snake_case")] +/// The type of download +pub enum DownloadType { + /// The download is for the game client + Client, + /// The download is mappings for the game + ClientMappings, + /// The download is for the game server + Server, + /// The download is mappings for the game server + ServerMappings, + /// The download is for the windows server + WindowsServer, +} + +#[derive(Serialize, Deserialize, Debug)] +/// Download information of a file +pub struct Download { + /// The SHA1 hash of the file + pub sha1: String, + /// The size of the file + pub size: u32, + /// The URL where the file can be downloaded + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug)] +/// Download information of a library +pub struct LibraryDownload { + /// The path that the library should be saved to + pub path: String, + /// The SHA1 hash of the library + pub sha1: String, + /// The size of the library + pub size: u32, + /// The URL where the library can be downloaded + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A list of files that should be downloaded for libraries +pub struct LibraryDownloads { + /// The primary library artifact + pub artifact: Option, + /// Conditional files that may be needed to be downloaded alongside the library + /// The HashMap key specifies a classifier as additional information for downloading files + pub classifiers: Option>, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "snake_case")] +/// The action a rule can follow +pub enum RuleAction { + /// The rule's status allows something to be done + Allow, + /// The rule's status disallows something to be done + Disallow, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] +#[serde(rename_all = "snake_case")] +/// An enum representing the different types of operating systems +pub enum Os { + /// MacOS + Osx, + /// Windows + Windows, + /// Linux and its derivatives + Linux, + /// The OS is unknown + Unknown, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A rule which depends on what OS the user is on +pub struct OsRule { + /// The name of the OS + pub name: Option, + /// The version of the OS. This is normally a RegEx + pub version: Option, + /// The architecture of the OS + pub arch: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A rule which depends on the toggled features of the launcher +pub struct FeatureRule { + /// Whether the user is in demo mode + pub is_demo_user: Option, + /// Whether the user is using the demo resolution + pub has_demo_resolution: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A rule deciding whether a file is downloaded, an argument is used, etc. +pub struct Rule { + /// The action the rule takes + pub action: RuleAction, + /// The OS rule + pub os: Option, + /// The feature rule + pub features: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +/// Information delegating the extraction of the library +pub struct LibraryExtract { + /// Files/Folders to be excluded from the extraction of the library + pub exclude: Option>, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A library which the game relies on to run +pub struct Library { + /// The files the library has + pub downloads: LibraryDownloads, + /// Rules of the extraction of the file + pub extract: Option, + /// The maven name of the library. The format is `groupId:artifactId:version` + pub name: String, + /// Native files that the library relies on + pub natives: Option>, + /// Rules deciding whether the library should be downloaded or not + pub rules: Option>, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(untagged)] +/// A container for an argument or multiple arguments +pub enum ArgumentValue { + /// The container has one argument + Single(String), + /// The container has multiple arguments + Many(Vec), +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(untagged)] +/// A command line argument passed to a program +pub enum Argument { + /// An argument which is applied no matter what + Normal(String), + /// An argument which is only applied if certain conditions are met + Ruled { + /// The rules deciding whether the argument(s) is used or not + rules: Vec, + /// The container of the argument(s) that should be applied accordingly + value: ArgumentValue, + }, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] +#[serde(rename_all = "snake_case")] +/// The type of argument +pub enum ArgumentType { + /// The argument is passed to the game + Game, + /// The argument is passed to the JVM + Jvm, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// Information about a version +pub struct VersionInfo { + /// Arguments passed to the game or JVM + pub arguments: Option>>, + /// Assets for the game + pub asset_index: AssetIndex, + /// The version ID of the assets + pub assets: String, + /// Game downloads of the version + pub downloads: HashMap, + /// The version ID of the version + pub id: String, + /// Libraries that the version depends on + pub libraries: Vec, + /// The classpath to the main class to launch the game + pub main_class: String, + /// (Legacy) Arguments passed to the game + pub minecraft_arguments: Option, + /// The minimum version of the Minecraft Launcher that can run this version of the game + pub minimum_launcher_version: u32, + /// The time that the version was released + pub release_time: DateTime, + /// The latest time a file in this version was updated + pub time: DateTime, + #[serde(rename = "type")] + /// The type of version + pub type_: VersionType, +} + +/// Fetches detailed information about a version from the manifest +pub async fn fetch_version_info(version: &Version) -> Result { + Ok(serde_json::from_slice( + &download_file(&version.url, Some(&version.sha1)).await?, + )?) +} + +#[derive(Serialize, Deserialize, Debug)] +/// An asset of the game +pub struct Asset { + /// The SHA1 hash of the asset file + pub hash: String, + /// The size of the asset file + pub size: u32, +} + +#[derive(Serialize, Deserialize, Debug)] +/// An index containing all assets the game needs +pub struct AssetsIndex { + /// A hashmap containing the filename (key) and asset (value) + pub objects: HashMap, +} + +/// Fetches the assets index from the version info +pub async fn fetch_assets_index(version: &VersionInfo) -> Result { + Ok(serde_json::from_slice( + &download_file(&version.asset_index.url, Some(&version.asset_index.sha1)).await?, + )?) +} diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml new file mode 100644 index 000000000..4f2fa019e --- /dev/null +++ b/daedalus_client/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "daedalus_client" +version = "0.1.0" +authors = ["Jai A "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +daedalus = { path = "../daedalus" } +tokio = { version = "1", features = ["full"] } +futures = "0.3.17" +dotenv = "0.15.0" +log = "0.4.8" +serde_json = "1.0" +lazy_static = "1.4.0" +thiserror = "1.0" +reqwest = "0.11.4" + +rusoto_core = "0.47.0" +rusoto_s3 = "0.47.0" diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs new file mode 100644 index 000000000..bb10303ac --- /dev/null +++ b/daedalus_client/src/main.rs @@ -0,0 +1,125 @@ +use log::{error, info, warn}; +use rusoto_core::credential::StaticProvider; +use rusoto_core::{HttpClient, Region, RusotoError}; +use rusoto_s3::{S3Client, PutObjectError}; +use rusoto_s3::{PutObjectRequest, S3}; + +mod minecraft; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("{0}")] + DaedalusError(#[from] daedalus::Error), + #[error("Error while deserializing JSON")] + SerdeError(#[from] serde_json::Error), + #[error("Unable to fetch {item}")] + FetchError { + inner: reqwest::Error, + item: String, + }, + #[error("Error while managing asynchronous tasks")] + TaskError(#[from] tokio::task::JoinError), + #[error("Error while uploading file to S3")] + S3Error { + inner: RusotoError, + file: String, + }, +} + +#[tokio::main] +async fn main() { + if check_env_vars() { + error!("Some environment variables are missing!"); + + return; + } + + minecraft::retrieve_data().await.unwrap(); +} + +fn check_env_vars() -> bool { + let mut failed = false; + + fn check_var(var: &str) -> bool { + if dotenv::var(var) + .ok() + .and_then(|s| s.parse::().ok()) + .is_none() + { + warn!( + "Variable `{}` missing in dotenv or not of type `{}`", + var, + std::any::type_name::() + ); + true + } else { + false + } + } + + failed |= check_var::("BASE_URL"); + failed |= check_var::("BASE_FOLDER"); + + failed |= check_var::("S3_ACCESS_TOKEN"); + failed |= check_var::("S3_SECRET"); + failed |= check_var::("S3_URL"); + failed |= check_var::("S3_REGION"); + failed |= check_var::("S3_BUCKET_NAME"); + + failed |= check_var::("DO_INTEGRATION"); + + let do_integration = dotenv::var("DO_INTEGRATION") + .ok() + .map(|x| x.parse::().ok()) + .flatten(); + + if do_integration.unwrap_or(false) { + failed |= check_var::("DO_ACCESS_KEY"); + } + + failed +} + +lazy_static::lazy_static! { + static ref CLIENT : S3Client = S3Client::new_with( + HttpClient::new().unwrap(), + StaticProvider::new( + dotenv::var("S3_ACCESS_TOKEN").unwrap(), + dotenv::var("S3_SECRET").unwrap(), + None, + None, + ), + Region::Custom { + name: dotenv::var("S3_REGION").unwrap(), + endpoint: dotenv::var("S3_URL").unwrap(), + }, + ); +} + +pub async fn upload_file_to_bucket(path: String, bytes: Vec, content_type: Option) -> Result<(), Error> { + CLIENT + .put_object(PutObjectRequest { + bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), + key: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), + body: Some(bytes.into()), + acl: Some("public-read".to_string()), + content_type, + ..Default::default() + }) + .await + .map_err(|err| Error::S3Error { + inner: err, + file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path) + })?; + + Ok(()) +} + +pub fn format_url(path: &str) -> String { + format!( + "{}/{}/{}", + &*dotenv::var("BASE_URL").unwrap(), + &*dotenv::var("BASE_FOLDER").unwrap(), + path + ) +} diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs new file mode 100644 index 000000000..1336e9ffb --- /dev/null +++ b/daedalus_client/src/minecraft.rs @@ -0,0 +1,172 @@ +use crate::{format_url, upload_file_to_bucket, Error}; +use daedalus::download_file; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +pub async fn retrieve_data() -> Result<(), Error> { + let old_manifest = + daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!( + "minecraft/v{}/version_manifest.json", + daedalus::minecraft::CURRENT_FORMAT_VERSION + )))) + .await + .ok(); + + let mut manifest = daedalus::minecraft::fetch_version_manifest(None) + .await?; + let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); + + let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); + + let now = Instant::now(); + + let mut versions = manifest + .versions + .iter_mut() + .map(|version| async { + let old_version = if let Some(old_manifest) = &old_manifest { + old_manifest.versions.iter().find(|x| x.id == version.id) + } else { + None + }; + + if let Some(old_version) = old_version { + if old_version.sha1 == version.sha1 { + return Ok(()); + } + } + + let visited_assets_mutex = Arc::clone(&visited_assets_mutex); + let cloned_manifest_mutex = Arc::clone(&cloned_manifest); + + let assets_hash = old_version.map(|x| x.assets_index_sha1.clone()).flatten(); + + async move { + let mut upload_futures = Vec::new(); + + let now = Instant::now(); + let mut version_println = daedalus::minecraft::fetch_version_info(version) + .await + ?; + let elapsed = now.elapsed(); + println!("Version {} Elapsed: {:.2?}", version.id, elapsed); + + let version_path = format!( + "minecraft/v{}/versions/{}.json", + daedalus::minecraft::CURRENT_FORMAT_VERSION, + version.id + ); + let assets_path = format!( + "minecraft/v{}/assets/{}.json", + daedalus::minecraft::CURRENT_FORMAT_VERSION, + version_println.asset_index.id + ); + let assets_index_url = version_println.asset_index.url.clone(); + + { + let mut cloned_manifest = match cloned_manifest_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + let position = cloned_manifest + .versions + .iter() + .position(|x| version.id == x.id) + .unwrap(); + cloned_manifest.versions[position].url = format_url(&version_path); + cloned_manifest.versions[position].assets_index_sha1 = + Some(version_println.asset_index.sha1.clone()); + cloned_manifest.versions[position].assets_index_url = + Some(format_url(&assets_path)); + version_println.asset_index.url = format_url(&assets_path); + } + + let mut download_assets = false; + + { + let mut visited_assets = match visited_assets_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + if !visited_assets.contains(&version_println.asset_index.id) { + if let Some(assets_hash) = assets_hash { + if version_println.asset_index.sha1 != assets_hash { + download_assets = true; + } + } else { + download_assets = true; + } + } + + if download_assets { + visited_assets.push(version_println.asset_index.id.clone()); + } + } + + if download_assets { + let assets_index = + download_file(&assets_index_url, Some(&version_println.asset_index.sha1)) + .await?; + + { + upload_futures + .push(upload_file_to_bucket(assets_path, assets_index.to_vec(), Some("application/json".to_string()))); + } + } + + { + upload_futures.push(upload_file_to_bucket( + version_path, + serde_json::to_vec(&version_println)?, + Some("application/json".to_string()) + )); + } + + let now = Instant::now(); + futures::future::try_join_all(upload_futures).await?; + let elapsed = now.elapsed(); + println!("Spaces Upload {} Elapsed: {:.2?}", version.id, elapsed); + + Ok::<(), Error>(()) + } + .await?; + + Ok::<(), Error>(()) + }) + .peekable(); + + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(100).collect(); + futures::future::try_join_all(chunk).await?; + + std::thread::sleep(Duration::from_secs(1)); + + chunk_index += 1; + + let elapsed = now.elapsed(); + println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + + upload_file_to_bucket( + format!( + "minecraft/v{}/version_manifest.json", + daedalus::minecraft::CURRENT_FORMAT_VERSION + ), + serde_json::to_vec(&*match cloned_manifest.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + })?, + Some("application/json".to_string()) + ) + .await?; + + let elapsed = now.elapsed(); + println!("Elapsed: {:.2?}", elapsed); + + Ok(()) +}