Bommels05/Modrinth
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3 from modrinth/many-fixes

Browse Source 
Many fixes
This commit is contained in:
Geometrically 2023-04-04 21:17:41 -07:00 committed by GitHub
commit b9de2b4b58
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 761 additions and 604 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.env
View File

@ -1,7 +1,6 @@
RUST_LOG=info,error
BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com
BASE_FOLDER=gamedata
S3_ACCESS_TOKEN=none
S3_SECRET=none

2
Dockerfile
View File

@ -1,4 +1,4 @@
FROM rust:1.65
FROM rust:1.68.2
COPY ./ ./

2
daedalus/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "daedalus"
version = "0.1.17"
version = "0.1.18"
authors = ["Jai A <jaiagr+gpg@pm.me>"]
edition = "2018"
license = "MIT"

52
daedalus/src/lib.rs
View File

@ -46,22 +46,34 @@ pub fn get_path_from_artifact(artifact: &str) -> Result<String, Error> {
let name_items = artifact.split(':').collect::<Vec<&str>>();
let package = name_items.first().ok_or_else(|| {
Error::ParseError(format!("Unable to find package for library {}", &artifact))
Error::ParseError(format!(
"Unable to find package for library {}",
&artifact
))
})?;
let name = name_items.get(1).ok_or_else(|| {
Error::ParseError(format!("Unable to find name for library {}", &artifact))
Error::ParseError(format!(
"Unable to find name for library {}",
&artifact
))
})?;
if name_items.len() == 3 {
let version_ext = name_items
.get(2)
.ok_or_else(|| {
Error::ParseError(format!("Unable to find version for library {}", &artifact))
Error::ParseError(format!(
"Unable to find version for library {}",
&artifact
))
})?
.split('@')
.collect::<Vec<&str>>();
let version = version_ext.first().ok_or_else(|| {
Error::ParseError(format!("Unable to find version for library {}", &artifact))
Error::ParseError(format!(
"Unable to find version for library {}",
&artifact
))
})?;
let ext = version_ext.get(1);
@ -76,18 +88,27 @@ pub fn get_path_from_artifact(artifact: &str) -> Result<String, Error> {
))
} else {
let version = name_items.get(2).ok_or_else(|| {
Error::ParseError(format!("Unable to find version for library {}", &artifact))
Error::ParseError(format!(
"Unable to find version for library {}",
&artifact
))
})?;
let data_ext = name_items
.get(3)
.ok_or_else(|| {
Error::ParseError(format!("Unable to find data for library {}", &artifact))
Error::ParseError(format!(
"Unable to find data for library {}",
&artifact
))
})?
.split('@')
.collect::<Vec<&str>>();
let data = data_ext.first().ok_or_else(|| {
Error::ParseError(format!("Unable to find data for library {}", &artifact))
Error::ParseError(format!(
"Unable to find data for library {}",
&artifact
))
})?;
let ext = data_ext.get(1);
@ -126,10 +147,21 @@ pub async fn download_file_mirrors(
}
/// Downloads a file with retry and checksum functionality
pub async fn download_file(url: &str, sha1: Option<&str>) -> Result<bytes::Bytes, Error> {
pub async fn download_file(
url: &str,
sha1: Option<&str>,
) -> Result<bytes::Bytes, Error> {
let mut headers = reqwest::header::HeaderMap::new();
if let Ok(header) = reqwest::header::HeaderValue::from_str(&format!(
"modrinth/daedalus/{} (support@modrinth.com)",
env!("CARGO_PKG_VERSION")
)) {
headers.insert(reqwest::header::USER_AGENT, header);
}
let client = reqwest::Client::builder()
.tcp_keepalive(Some(std::time::Duration::from_secs(10)))
.timeout(std::time::Duration::from_secs(15))
.default_headers(headers)
.build()
.map_err(|err| Error::FetchError {
inner: err,
@ -183,7 +215,9 @@ pub async fn download_file(url: &str, sha1: Option<&str>) -> Result<bytes::Bytes
/// Computes a checksum of the input bytes
pub async fn get_hash(bytes: bytes::Bytes) -> Result<String, Error> {
let hash = tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()).await?;
let hash =
tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest())
.await?;
Ok(hash)
}

19
daedalus/src/modded.rs
View File

@ -15,6 +15,9 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0;
/// The latest version of the format the fabric model structs deserialize to
pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0;
/// The dummy replace string library names, inheritsFrom, and version names should be replaced with
pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}";
/// A data variable entry that depends on the side of the installation
#[cfg_attr(feature = "bincode", derive(Encode, Decode))]
#[derive(Serialize, Deserialize, Debug)]
@ -93,6 +96,8 @@ pub fn merge_partial_version(
partial: PartialVersionInfo,
merge: VersionInfo,
) -> VersionInfo {
let merge_id = merge.id.clone();
VersionInfo {
arguments: if let Some(partial_args) = partial.arguments {
if let Some(merge_args) = merge.arguments {
@ -126,12 +131,22 @@ pub fn merge_partial_version(
asset_index: merge.asset_index,
assets: merge.assets,
downloads: merge.downloads,
id: partial.id,
id: merge.id,
java_version: merge.java_version,
libraries: partial
.libraries
.into_iter()
.chain(merge.libraries)
.map(|x| Library {
downloads: x.downloads,
extract: x.extract,
name: x.name.replace(DUMMY_REPLACE_STRING, &merge_id),
url: x.url,
natives: x.natives,
rules: x.rules,
checksums: x.checksums,
include_in_classpath: x.include_in_classpath,
})
.collect::<Vec<_>>(),
main_class: if let Some(main_class) = partial.main_class {
main_class
@ -163,6 +178,8 @@ pub struct Manifest {
pub struct Version {
/// The minecraft version ID
pub id: String,
/// Whether the release is stable or not
pub stable: bool,
/// A map that contains loader versions for the game version
pub loaders: Vec<LoaderVersion>,
}

2
daedalus_client/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "daedalus_client"
version = "0.1.17"
version = "0.1.18"
authors = ["Jai A <jaiagr+gpg@pm.me>"]
edition = "2018"

471
daedalus_client/src/fabric.rs
View File

@ -1,266 +1,283 @@
use crate::{format_url, upload_file_to_bucket, Error};
use daedalus::download_file;
use crate::{download_file, format_url, upload_file_to_bucket, Error};
use daedalus::minecraft::{Library, VersionManifest};
use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Version};
use log::info;
use daedalus::modded::{
LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING,
};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::{Mutex, RwLock};
use tokio::sync::{Mutex, RwLock, Semaphore};
pub async fn retrieve_data(
minecraft_versions: &VersionManifest,
uploaded_files: &mut Vec<String>,
semaphore: Arc<Semaphore>,
) -> Result<(), Error> {
let mut list = fetch_fabric_versions(None).await?;
let old_manifest = daedalus::modded::fetch_manifest(&format!(
let mut list = fetch_fabric_versions(None, semaphore.clone()).await?;
let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!(
"fabric/v{}/manifest.json",
daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
))
)))
.await
.ok();
let versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest {
let mut versions = if let Some(old_manifest) = old_manifest {
old_manifest.game_versions
} else {
Vec::new()
}));
};
let loaders_mutex = RwLock::new(Vec::new());
{
let mut loaders = loaders_mutex.write().await;
for loader in &list.loader {
loaders.push((Box::new(loader.stable), loader.version.clone()))
}
list.loader
.retain(|x| loaders.iter().any(|val| val.1 == x.version))
}
const DUMMY_GAME_VERSION: &str = "1.19.4-rc2";
let loader_version_mutex = Mutex::new(Vec::new());
let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new()));
if let Some(latest) = list.loader.get(0) {
let loaders_mutex = Arc::new(RwLock::new(Vec::new()));
let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new()));
{
let mut loaders = loaders_mutex.write().await;
// for loader in &list.loader {
// loaders.push((Box::new(loader.stable), loader.version.clone()))
// }
loaders.push((Box::new(latest.stable), latest.version.clone()));
if !latest.stable {
if let Some(stable) = list.loader.iter().find(|x| x.stable) {
loaders.push((Box::new(stable.stable), stable.version.clone()));
}
}
list.loader.retain(|x| loaders.iter().any(|val| val.1 == x.version))
}
let mut version_futures = Vec::new();
for game_version in list.game.iter_mut() {
let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex);
let loaders_mutex = Arc::clone(&loaders_mutex);
let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
let versions_mutex = Arc::clone(&versions);
version_futures.push(async move {
let loader_version_mutex = Mutex::new(Vec::new());
let versions =
futures::future::try_join_all(
loaders_mutex.read().await.clone().into_iter().map(
|(stable, loader)| async {
{
if versions_mutex.lock().await.iter().any(|x| {
x.id == game_version.version
&& x.loaders.iter().any(|x| x.id == loader)
}) {
return Ok(None);
}
}
let version =
fetch_fabric_version(&game_version.version, &loader).await?;
Ok::<Option<(Box<bool>, String, PartialVersionInfo)>, Error>(Some(
(stable, loader, version),
))
},
),
)
.await?
.into_iter()
.flatten();
futures::future::try_join_all(versions.map(|(stable, loader, version)| async {
let libs = futures::future::try_join_all(version.libraries.into_iter().map(
|mut lib| async {
{
let mut visited_assets = visited_artifacts_mutex.lock().await;
if visited_assets.contains(&lib.name) {
lib.url = Some(format_url("maven/"));
return Ok(lib);
} else {
visited_assets.push(lib.name.clone())
}
}
let artifact_path = daedalus::get_path_from_artifact(&lib.name)?;
let artifact = daedalus::download_file(
&format!(
"{}{}",
lib.url.unwrap_or_else(|| {
"https://maven.fabricmc.net/".to_string()
}),
artifact_path
),
None,
)
.await?;
lib.url = Some(format_url("maven/"));
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
artifact.to_vec(),
Some("application/java-archive".to_string()),
uploaded_files_mutex.as_ref(),
)
.await?;
Ok::<Library, Error>(lib)
},
))
.await?;
let version_path = format!(
"fabric/v{}/versions/{}-{}.json",
daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
version.inherits_from,
&loader
);
let inherits_from = version.inherits_from.clone();
upload_file_to_bucket(
version_path.clone(),
serde_json::to_vec(&PartialVersionInfo {
arguments: version.arguments,
id: version.id,
main_class: version.main_class,
release_time: version.release_time,
time: version.time,
type_: version.type_,
inherits_from: version.inherits_from,
libraries: libs,
minecraft_arguments: version.minecraft_arguments,
processors: None,
data: None,
})?,
Some("application/json".to_string()),
uploaded_files_mutex.as_ref(),
)
.await?;
{
let mut loader_version_map = loader_version_mutex.lock().await;
async move {
loader_version_map.push(LoaderVersion {
id: format!("{}-{}", inherits_from, loader),
url: format_url(&version_path),
stable: *stable,
});
}
.await;
let loader_versions = futures::future::try_join_all(
loaders_mutex.read().await.clone().into_iter().map(
|(stable, loader)| async {
{
if versions.iter().any(|x| {
x.id == DUMMY_REPLACE_STRING
&& x.loaders.iter().any(|x| x.id == loader)
}) {
return Ok(None);
}
}
Ok::<(), Error>(())
}))
let version = fetch_fabric_version(
DUMMY_GAME_VERSION,
&loader,
semaphore.clone(),
)
.await?;
let mut versions = versions_mutex.lock().await;
versions.push(Version {
id: game_version.version.clone(),
loaders: loader_version_mutex.into_inner(),
});
Ok::<Option<(Box<bool>, String, PartialVersionInfo)>, Error>(
Some((stable, loader, version)),
)
},
),
)
.await?;
Ok::<(), Error>(())
});
}
let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new()));
futures::future::try_join_all(loader_versions.into_iter()
.flatten().map(
|(stable, loader, version)| async {
let libs = futures::future::try_join_all(
version.libraries.into_iter().map(|mut lib| async {
{
let mut visited_assets =
visited_artifacts_mutex.lock().await;
let mut versions = version_futures.into_iter().peekable();
let mut chunk_index = 0;
while versions.peek().is_some() {
let now = Instant::now();
if visited_assets.contains(&lib.name) {
lib.url = Some(format_url("maven/"));
let chunk: Vec<_> = versions.by_ref().take(10).collect();
futures::future::try_join_all(chunk).await?;
return Ok(lib);
} else {
visited_assets.push(lib.name.clone())
}
}
tokio::time::sleep(Duration::from_secs(1)).await;
if lib.name.contains(DUMMY_GAME_VERSION) {
lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
lib.url = Some(format_url("maven/"));
futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async {
let semaphore = semaphore.clone();
let uploaded_files_mutex = uploaded_files_mutex.clone();
let lib_name = lib.name.clone();
let lib_url = lib.url.clone();
chunk_index += 1;
async move {
let artifact_path =
daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?;
let elapsed = now.elapsed();
info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
}
let artifact = download_file(
&format!(
"{}{}",
lib_url.unwrap_or_else(|| {
"https://maven.fabricmc.net/".to_string()
}),
artifact_path
),
None,
semaphore.clone(),
)
.await?;
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
artifact.to_vec(),
Some("application/java-archive".to_string()),
&uploaded_files_mutex,
semaphore.clone(),
)
.await?;
Ok::<(), Error>(())
}.await?;
Ok::<(), Error>(())
})).await?;
return Ok(lib);
}
let artifact_path =
daedalus::get_path_from_artifact(&lib.name)?;
let artifact = download_file(
&format!(
"{}{}",
lib.url.unwrap_or_else(|| {
"https://maven.fabricmc.net/".to_string()
}),
artifact_path
),
None,
semaphore.clone(),
)
.await?;
lib.url = Some(format_url("maven/"));
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
artifact.to_vec(),
Some("application/java-archive".to_string()),
&uploaded_files_mutex,
semaphore.clone(),
)
.await?;
Ok::<Library, Error>(lib)
}),
)
.await?;
let version_path = format!(
"fabric/v{}/versions/{}.json",
daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
&loader
);
upload_file_to_bucket(
version_path.clone(),
serde_json::to_vec(&PartialVersionInfo {
arguments: version.arguments,
id: version
.id
.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING),
main_class: version.main_class,
release_time: version.release_time,
time: version.time,
type_: version.type_,
inherits_from: version
.inherits_from
.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING),
libraries: libs,
minecraft_arguments: version.minecraft_arguments,
processors: None,
data: None,
})?,
Some("application/json".to_string()),
&uploaded_files_mutex,
semaphore.clone(),
)
.await?;
{
let mut loader_version_map = loader_version_mutex.lock().await;
async move {
loader_version_map.push(LoaderVersion {
id: loader.to_string(),
url: format_url(&version_path),
stable: *stable,
});
}
.await;
}
Ok::<(), Error>(())
},
))
.await?;
versions.push(Version {
id: DUMMY_REPLACE_STRING.to_string(),
stable: true,
loaders: loader_version_mutex.into_inner(),
});
for version in &list.game {
versions.push(Version {
id: version.version.clone(),
stable: version.stable,
loaders: vec![],
});
}
if let Ok(versions) = Arc::try_unwrap(versions) {
let mut versions = versions.into_inner();
versions.sort_by(|x, y| {
minecraft_versions
.versions
.iter()
.position(|z| x.id == z.id)
.unwrap_or_default()
.cmp(
&minecraft_versions
.versions
.iter()
.position(|z| y.id == z.id)
.unwrap_or_default(),
)
});
versions.sort_by(|x, y| {
minecraft_versions
.versions
for version in &mut versions {
version.loaders.sort_by(|x, y| {
list.loader
.iter()
.position(|z| x.id == z.id)
.position(|z| {
x.id.split('-').next().unwrap_or_default() == &*z.version
})
.unwrap_or_default()
.cmp(
&minecraft_versions
.versions
&list
.loader
.iter()
.position(|z| y.id == z.id)
.position(|z| {
y.id.split('-').next().unwrap_or_default()
== z.version
})
.unwrap_or_default(),
)
});
for version in &mut versions {
version.loaders.sort_by(|x, y| {
list.loader
.iter()
.position(|z| {
x.id.split('-')
.next()
.unwrap_or_default()
== &*z.version
})
.unwrap_or_default()
.cmp(
&list
.loader
.iter()
.position(|z| {
y.id.split('-')
.next()
.unwrap_or_default()
== z.version
})
.unwrap_or_default(),
)
})
}
upload_file_to_bucket(
format!(
"fabric/v{}/manifest.json",
daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
),
serde_json::to_vec(&Manifest {
game_versions: versions,
})?,
Some("application/json".to_string()),
uploaded_files_mutex.as_ref(),
)
.await?;
})
}
upload_file_to_bucket(
format!(
"fabric/v{}/manifest.json",
daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
),
serde_json::to_vec(&Manifest {
game_versions: versions,
})?,
Some("application/json".to_string()),
&uploaded_files_mutex,
semaphore,
)
.await?;
if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) {
uploaded_files.extend(uploaded_files_mutex.into_inner());
}
@ -273,6 +290,7 @@ const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2";
async fn fetch_fabric_version(
version_number: &str,
loader_version: &str,
semaphore: Arc<Semaphore>,
) -> Result<PartialVersionInfo, Error> {
Ok(serde_json::from_slice(
&download_file(
@ -281,6 +299,7 @@ async fn fetch_fabric_version(
FABRIC_META_URL, version_number, loader_version
),
None,
semaphore,
)
.await?,
)?)
@ -319,11 +338,15 @@ struct FabricLoaderVersion {
pub stable: bool,
}
/// Fetches the list of fabric versions
async fn fetch_fabric_versions(url: Option<&str>) -> Result<FabricVersions, Error> {
async fn fetch_fabric_versions(
url: Option<&str>,
semaphore: Arc<Semaphore>,
) -> Result<FabricVersions, Error> {
Ok(serde_json::from_slice(
&download_file(
url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)),
None,
semaphore,
)
.await?,
)?)

663
daedalus_client/src/forge.rs
View File

@ -1,8 +1,14 @@
use crate::{format_url, upload_file_to_bucket, Error};
use crate::{
download_file, download_file_mirrors, format_url, upload_file_to_bucket,
Error,
};
use chrono::{DateTime, Utc};
use daedalus::download_file;
use daedalus::minecraft::{Argument, ArgumentType, Library, VersionManifest, VersionType};
use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry};
use daedalus::minecraft::{
Argument, ArgumentType, Library, VersionManifest, VersionType,
};
use daedalus::modded::{
LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry,
};
use lazy_static::lazy_static;
use log::info;
use semver::{Version, VersionReq};
@ -11,7 +17,7 @@ use std::collections::HashMap;
use std::io::Read;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::Mutex;
use tokio::sync::{Mutex, Semaphore};
lazy_static! {
static ref FORGE_MANIFEST_V1_QUERY: VersionReq =
@ -20,14 +26,16 @@ lazy_static! {
VersionReq::parse(">=23.5.2851, <31.2.52").unwrap();
static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq =
VersionReq::parse(">=32.0.1, <37.0.0").unwrap();
static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap();
static ref FORGE_MANIFEST_V3_QUERY: VersionReq =
VersionReq::parse(">=37.0.0").unwrap();
}
pub async fn retrieve_data(
minecraft_versions: &VersionManifest,
uploaded_files: &mut Vec<String>,
semaphore: Arc<Semaphore>,
) -> Result<(), Error> {
let maven_metadata = fetch_maven_metadata(None).await?;
let maven_metadata = fetch_maven_metadata(None, semaphore.clone()).await?;
let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!(
"forge/v{}/manifest.json",
daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
@ -35,11 +43,12 @@ pub async fn retrieve_data(
.await
.ok();
let old_versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest {
old_manifest.game_versions
} else {
Vec::new()
}));
let old_versions =
Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest {
old_manifest.game_versions
} else {
Vec::new()
}));
let versions = Arc::new(Mutex::new(Vec::new()));
@ -52,13 +61,14 @@ pub async fn retrieve_data(
let mut loaders = Vec::new();
for loader_version_full in loader_versions {
let loader_version = loader_version_full.split('-').into_iter().nth(1);
let loader_version = loader_version_full.split('-').nth(1);
if let Some(loader_version_raw) = loader_version {
// This is a dirty hack to get around Forge not complying with SemVer, but whatever
// Most of this is a hack anyways :(
// Works for all forge versions!
let split = loader_version_raw.split('.').collect::<Vec<&str>>();
let split =
loader_version_raw.split('.').collect::<Vec<&str>>();
let loader_version = if split.len() >= 4 {
if split[0].parse::<i32>().unwrap_or(0) < 6 {
format!("{}.{}.{}", split[0], split[1], split[3])
@ -80,203 +90,209 @@ pub async fn retrieve_data(
}
}
}
version_futures.push(async {
let mut loaders_versions = Vec::new();
{
let mut loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async {
let versions_mutex = Arc::clone(&old_versions);
let visited_assets = Arc::clone(&visited_assets_mutex);
let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
let minecraft_version = minecraft_version.clone();
if !loaders.is_empty() {
version_futures.push(async {
let loaders_versions = Vec::new();
async move {
/// These forge versions are not worth supporting!
const WHITELIST : [&str; 1] = [
// Not supported due to `data` field being `[]` even though the type is a map
"1.12.2-14.23.5.2851"
];
{
let loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async {
let versions_mutex = Arc::clone(&old_versions);
let visited_assets = Arc::clone(&visited_assets_mutex);
let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
let semaphore = Arc::clone(&semaphore);
let minecraft_version = minecraft_version.clone();
if WHITELIST.contains(&&*loader_version_full) {
return Ok(None);
}
async move {
/// These forge versions are not worth supporting!
const WHITELIST : [&str; 1] = [
// Not supported due to `data` field being `[]` even though the type is a map
"1.12.2-14.23.5.2851",
];
{
let versions = versions_mutex.lock().await;
let version = versions.iter().find(|x|
x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full));
if let Some(version) = version {
return Ok::<Option<LoaderVersion>, Error>(Some(version.clone()));
if WHITELIST.contains(&&*loader_version_full) {
return Ok(None);
}
}
info!("Forge - Installer Start {}", loader_version_full.clone());
let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?;
{
let versions = versions_mutex.lock().await;
let version = versions.iter().find(|x|
x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full));
let reader = std::io::Cursor::new(bytes);
if let Some(version) = version {
return Ok::<Option<LoaderVersion>, Error>(Some(version.clone()));
}
}
if let Ok(archive) = zip::ZipArchive::new(reader) {
if FORGE_MANIFEST_V1_QUERY.matches(&version) {
let mut archive_clone = archive.clone();
let profile = tokio::task::spawn_blocking(move || {
let mut install_profile = archive_clone.by_name("install_profile.json")?;
info!("Forge - Installer Start {}", loader_version_full.clone());
let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None, semaphore.clone()).await?;
let mut contents = String::new();
install_profile.read_to_string(&mut contents)?;
let reader = std::io::Cursor::new(bytes);
Ok::<ForgeInstallerProfileV1, Error>(serde_json::from_str::<ForgeInstallerProfileV1>(&contents)?)
}).await??;
if let Ok(archive) = zip::ZipArchive::new(reader) {
if FORGE_MANIFEST_V1_QUERY.matches(&version) {
let mut archive_clone = archive.clone();
let profile = tokio::task::spawn_blocking(move || {
let mut install_profile = archive_clone.by_name("install_profile.json")?;
let mut archive_clone = archive.clone();
let file_path = profile.install.file_path.clone();
let forge_universal_bytes = tokio::task::spawn_blocking(move || {
let mut forge_universal_file = archive_clone.by_name(&file_path)?;
let mut forge_universal = Vec::new();
forge_universal_file.read_to_end(&mut forge_universal)?;
let mut contents = String::new();
install_profile.read_to_string(&mut contents)?;
Ok::<ForgeInstallerProfileV1, Error>(serde_json::from_str::<ForgeInstallerProfileV1>(&contents)?)
}).await??;
let mut archive_clone = archive.clone();
let file_path = profile.install.file_path.clone();
let forge_universal_bytes = tokio::task::spawn_blocking(move || {
let mut forge_universal_file = archive_clone.by_name(&file_path)?;
let mut forge_universal = Vec::new();
forge_universal_file.read_to_end(&mut forge_universal)?;
Ok::<bytes::Bytes, Error>(bytes::Bytes::from(forge_universal))
}).await??;
let forge_universal_path = profile.install.path.clone();
Ok::<bytes::Bytes, Error>(bytes::Bytes::from(forge_universal))
}).await??;
let forge_universal_path = profile.install.path.clone();
let now = Instant::now();
let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async {
if let Some(url) = lib.url {
{
let mut visited_assets = visited_assets.lock().await;
let now = Instant::now();
let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async {
if let Some(url) = lib.url {
{
let mut visited_assets = visited_assets.lock().await;
if visited_assets.contains(&lib.name) {
lib.url = Some(format_url("maven/"));
if visited_assets.contains(&lib.name) {
lib.url = Some(format_url("maven/"));
return Ok::<Library, Error>(lib);
} else {
visited_assets.push(lib.name.clone())
return Ok::<Library, Error>(lib);
} else {
visited_assets.push(lib.name.clone())
}
}
let artifact_path =
daedalus::get_path_from_artifact(&lib.name)?;
let artifact = if lib.name == forge_universal_path {
forge_universal_bytes.clone()
} else {
let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"];
download_file_mirrors(
&artifact_path,
&mirrors,
None,
semaphore.clone(),
)
.await?
};
lib.url = Some(format_url("maven/"));
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
artifact.to_vec(),
Some("application/java-archive".to_string()),
uploaded_files_mutex.as_ref(),
semaphore.clone(),
).await?;
}
let artifact_path =
daedalus::get_path_from_artifact(&lib.name)?;
Ok::<Library, Error>(lib)
})).await?;
let artifact = if lib.name == forge_universal_path {
forge_universal_bytes.clone()
} else {
let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"];
let elapsed = now.elapsed();
info!("Elapsed lib DL: {:.2?}", elapsed);
daedalus::download_file_mirrors(
&artifact_path,
&mirrors,
None,
)
.await?
};
let new_profile = PartialVersionInfo {
id: profile.version_info.id,
inherits_from: profile.install.minecraft,
release_time: profile.version_info.release_time,
time: profile.version_info.time,
main_class: profile.version_info.main_class,
minecraft_arguments: profile.version_info.minecraft_arguments.clone(),
arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()),
libraries: libs,
type_: profile.version_info.type_,
data: None,
processors: None
};
lib.url = Some(format_url("maven/"));
let version_path = format!(
"forge/v{}/versions/{}.json",
daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
new_profile.id
);
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
artifact.to_vec(),
Some("application/java-archive".to_string()),
uploaded_files_mutex.as_ref(),
).await?;
upload_file_to_bucket(
version_path.clone(),
serde_json::to_vec(&new_profile)?,
Some("application/json".to_string()),
uploaded_files_mutex.as_ref(),
semaphore.clone(),
).await?;
return Ok(Some(LoaderVersion {
id: loader_version_full,
url: format_url(&version_path),
stable: false
}));
} else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) {
let mut archive_clone = archive.clone();
let mut profile = tokio::task::spawn_blocking(move || {
let mut install_profile = archive_clone.by_name("install_profile.json")?;
let mut contents = String::new();
install_profile.read_to_string(&mut contents)?;
Ok::<ForgeInstallerProfileV2, Error>(serde_json::from_str::<ForgeInstallerProfileV2>(&contents)?)
}).await??;
let mut archive_clone = archive.clone();
let version_info = tokio::task::spawn_blocking(move || {
let mut install_profile = archive_clone.by_name("version.json")?;
let mut contents = String::new();
install_profile.read_to_string(&mut contents)?;
Ok::<PartialVersionInfo, Error>(serde_json::from_str::<PartialVersionInfo>(&contents)?)
}).await??;
let mut libs : Vec<Library> = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library {
downloads: x.downloads,
extract: x.extract,
name: x.name,
url: x.url,
natives: x.natives,
rules: x.rules,
checksums: x.checksums,
include_in_classpath: false
})).collect();
let mut local_libs : HashMap<String, bytes::Bytes> = HashMap::new();
for lib in &libs {
if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) {
let mut archive_clone = archive.clone();
let lib_name_clone = lib.name.clone();
let lib_bytes = tokio::task::spawn_blocking(move || {
let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?;
let mut lib_bytes = Vec::new();
lib_file.read_to_end(&mut lib_bytes)?;
Ok::<bytes::Bytes, Error>(bytes::Bytes::from(lib_bytes))
}).await??;
local_libs.insert(lib.name.clone(), lib_bytes);
}
}
Ok::<Library, Error>(lib)
})).await?;
let path = profile.path.clone();
let version = profile.version.clone();
let elapsed = now.elapsed();
info!("Elapsed lib DL: {:.2?}", elapsed);
let new_profile = PartialVersionInfo {
id: profile.version_info.id,
inherits_from: profile.install.minecraft,
release_time: profile.version_info.release_time,
time: profile.version_info.time,
main_class: profile.version_info.main_class,
minecraft_arguments: profile.version_info.minecraft_arguments.clone(),
arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()),
libraries: libs,
type_: profile.version_info.type_,
data: None,
processors: None
};
let version_path = format!(
"forge/v{}/versions/{}.json",
daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
new_profile.id
);
upload_file_to_bucket(
version_path.clone(),
serde_json::to_vec(&new_profile)?,
Some("application/json".to_string()),
uploaded_files_mutex.as_ref()
).await?;
return Ok(Some(LoaderVersion {
id: loader_version_full,
url: format_url(&version_path),
stable: false
}));
} else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) {
let mut archive_clone = archive.clone();
let mut profile = tokio::task::spawn_blocking(move || {
let mut install_profile = archive_clone.by_name("install_profile.json")?;
let mut contents = String::new();
install_profile.read_to_string(&mut contents)?;
Ok::<ForgeInstallerProfileV2, Error>(serde_json::from_str::<ForgeInstallerProfileV2>(&contents)?)
}).await??;
let mut archive_clone = archive.clone();
let version_info = tokio::task::spawn_blocking(move || {
let mut install_profile = archive_clone.by_name("version.json")?;
let mut contents = String::new();
install_profile.read_to_string(&mut contents)?;
Ok::<PartialVersionInfo, Error>(serde_json::from_str::<PartialVersionInfo>(&contents)?)
}).await??;
let mut libs : Vec<daedalus::minecraft::Library> = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library {
downloads: x.downloads,
extract: x.extract,
name: x.name,
url: x.url,
natives: x.natives,
rules: x.rules,
checksums: x.checksums,
include_in_classpath: false
})).collect();
let mut local_libs : HashMap<String, bytes::Bytes> = HashMap::new();
for lib in &libs {
if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) {
let mut archive_clone = archive.clone();
let lib_name_clone = lib.name.clone();
let lib_bytes = tokio::task::spawn_blocking(move || {
let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?;
let mut lib_bytes = Vec::new();
lib_file.read_to_end(&mut lib_bytes)?;
Ok::<bytes::Bytes, Error>(bytes::Bytes::from(lib_bytes))
}).await??;
local_libs.insert(lib.name.clone(), lib_bytes);
}
}
let path = profile.path.clone();
let version = profile.version.clone();
for entry in profile.data.values_mut() {
if entry.client.starts_with('/') || entry.server.starts_with('/') {
macro_rules! read_data {
for entry in profile.data.values_mut() {
if entry.client.starts_with('/') || entry.server.starts_with('/') {
macro_rules! read_data {
($value:expr) => {
let mut archive_clone = archive.clone();
let value_clone = $value.clone();
@ -315,178 +331,178 @@ pub async fn retrieve_data(
}
}
if entry.client.starts_with('/') {
read_data!(entry.client);
}
// Do we really need to support server installs? Keeping this here
// just in case
//
// if entry.server.starts_with('/') {
// read_data!(entry.server);
// }
}
}
let now = Instant::now();
let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async {
let artifact_path =
daedalus::get_path_from_artifact(&lib.name)?;
{
let mut visited_assets = visited_assets.lock().await;
if visited_assets.contains(&lib.name) {
if let Some(ref mut downloads) = lib.downloads {
if let Some(ref mut artifact) = downloads.artifact {
artifact.url = format_url(&format!("maven/{}", artifact_path));
}
} else if lib.url.is_some() {
lib.url = Some(format_url("maven/"));
if entry.client.starts_with('/') {
read_data!(entry.client);
}
return Ok::<Library, Error>(lib);
} else {
visited_assets.push(lib.name.clone())
if entry.server.starts_with('/') {
read_data!(entry.server);
}
}
}
let artifact_bytes = if let Some(ref mut downloads) = lib.downloads {
if let Some(ref mut artifact) = downloads.artifact {
let res = if artifact.url.is_empty() {
let now = Instant::now();
let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async {
let artifact_path =
daedalus::get_path_from_artifact(&lib.name)?;
{
let mut visited_assets = visited_assets.lock().await;
if visited_assets.contains(&lib.name) {
if let Some(ref mut downloads) = lib.downloads {
if let Some(ref mut artifact) = downloads.artifact {
artifact.url = format_url(&format!("maven/{}", artifact_path));
}
} else if lib.url.is_some() {
lib.url = Some(format_url("maven/"));
}
return Ok::<Library, Error>(lib);
} else {
visited_assets.push(lib.name.clone())
}
}
let artifact_bytes = if let Some(ref mut downloads) = lib.downloads {
if let Some(ref mut artifact) = downloads.artifact {
let res = if artifact.url.is_empty() {
local_libs.get(&lib.name).cloned()
} else {
Some(download_file(
&artifact.url,
Some(&*artifact.sha1),
semaphore.clone(),
)
.await?)
};
if res.is_some() {
artifact.url = format_url(&format!("maven/{}", artifact_path));
}
res
} else { None }
} else if let Some(ref mut url) = lib.url {
let res = if url.is_empty() {
local_libs.get(&lib.name).cloned()
} else {
Some(daedalus::download_file(
&artifact.url,
Some(&*artifact.sha1),
Some(download_file(
url,
None,
semaphore.clone(),
)
.await?)
};
if res.is_some() {
artifact.url = format_url(&format!("maven/{}", artifact_path));
lib.url = Some(format_url("maven/"));
}
res
} else { None }
} else if let Some(ref mut url) = lib.url {
let res = if url.is_empty() {
local_libs.get(&lib.name).cloned()
} else {
Some(daedalus::download_file(
url,
None,
)
.await?)
};
} else { None };
if res.is_some() {
lib.url = Some(format_url("maven/"));
if let Some(bytes) = artifact_bytes {
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
bytes.to_vec(),
Some("application/java-archive".to_string()),
uploaded_files_mutex.as_ref(),
semaphore.clone(),
).await?;
}
res
} else { None };
Ok::<Library, Error>(lib)
})).await?;
if let Some(bytes) = artifact_bytes {
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
bytes.to_vec(),
Some("application/java-archive".to_string()),
uploaded_files_mutex.as_ref()
).await?;
}
let elapsed = now.elapsed();
info!("Elapsed lib DL: {:.2?}", elapsed);
Ok::<Library, Error>(lib)
})).await?;
let new_profile = PartialVersionInfo {
id: version_info.id,
inherits_from: version_info.inherits_from,
release_time: version_info.release_time,
time: version_info.time,
main_class: version_info.main_class,
minecraft_arguments: version_info.minecraft_arguments,
arguments: version_info.arguments,
libraries: libs,
type_: version_info.type_,
data: Some(profile.data),
processors: Some(profile.processors),
};
let elapsed = now.elapsed();
info!("Elapsed lib DL: {:.2?}", elapsed);
let version_path = format!(
"forge/v{}/versions/{}.json",
daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
new_profile.id
);
let new_profile = PartialVersionInfo {
id: version_info.id,
inherits_from: version_info.inherits_from,
release_time: version_info.release_time,
time: version_info.time,
main_class: version_info.main_class,
minecraft_arguments: version_info.minecraft_arguments,
arguments: version_info.arguments,
libraries: libs,
type_: version_info.type_,
data: Some(profile.data),
processors: Some(profile.processors),
};
upload_file_to_bucket(
version_path.clone(),
serde_json::to_vec(&new_profile)?,
Some("application/json".to_string()),
uploaded_files_mutex.as_ref(),
semaphore.clone(),
).await?;
let version_path = format!(
"forge/v{}/versions/{}.json",
daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
new_profile.id
);
upload_file_to_bucket(
version_path.clone(),
serde_json::to_vec(&new_profile)?,
Some("application/json".to_string()),
uploaded_files_mutex.as_ref()
).await?;
return Ok(Some(LoaderVersion {
id: loader_version_full,
url: format_url(&version_path),
stable: false
}));
return Ok(Some(LoaderVersion {
id: loader_version_full,
url: format_url(&version_path),
stable: false
}));
}
}
Ok(None)
}.await
});
{
let mut versions = loaders_futures.into_iter().peekable();
let mut chunk_index = 0;
while versions.peek().is_some() {
let now = Instant::now();
let chunk: Vec<_> = versions.by_ref().take(1).collect();
futures::future::try_join_all(chunk).await?;
chunk_index += 1;
let elapsed = now.elapsed();
info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
}
Ok(None)
}.await
}).into_iter().peekable()/*.into_iter().flatten().collect()*/;
let mut chunk_index = 0;
while loaders_futures.peek().is_some() {
info!("Loader Chunk {} Start", chunk_index);
let now = Instant::now();
let chunk: Vec<_> = loaders_futures.by_ref().take(10).collect();
let res = futures::future::try_join_all(chunk).await?;
loaders_versions.extend(res.into_iter().flatten());
tokio::time::sleep(Duration::from_secs(1)).await;
chunk_index += 1;
let elapsed = now.elapsed();
info!("Loader Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
}
//futures::future::try_join_all(loaders_futures).await?;
}
}
versions.lock().await.push(daedalus::modded::Version {
id: minecraft_version,
loaders: loaders_versions
versions.lock().await.push(daedalus::modded::Version {
id: minecraft_version,
stable: true,
loaders: loaders_versions
});
Ok::<(), Error>(())
});
Ok::<(), Error>(())
});
}
}
{
let mut versions_peek = version_futures.into_iter().peekable();
let mut versions = version_futures.into_iter().peekable();
let mut chunk_index = 0;
while versions_peek.peek().is_some() {
info!("Chunk {} Start", chunk_index);
while versions.peek().is_some() {
let now = Instant::now();
let chunk: Vec<_> = versions_peek.by_ref().take(1).collect();
let chunk: Vec<_> = versions.by_ref().take(10).collect();
futures::future::try_join_all(chunk).await?;
tokio::time::sleep(Duration::from_secs(1)).await;
chunk_index += 1;
let elapsed = now.elapsed();
info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
}
}
//futures::future::try_join_all(version_futures).await?;
if let Ok(versions) = Arc::try_unwrap(versions) {
let mut versions = versions.into_inner();
@ -495,13 +511,17 @@ pub async fn retrieve_data(
minecraft_versions
.versions
.iter()
.position(|z| x.id == z.id)
.position(|z| {
x.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id
})
.unwrap_or_default()
.cmp(
&minecraft_versions
.versions
.iter()
.position(|z| y.id == z.id)
.position(|z| {
y.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id
})
.unwrap_or_default(),
)
});
@ -534,6 +554,7 @@ pub async fn retrieve_data(
})?,
Some("application/json".to_string()),
uploaded_files_mutex.as_ref(),
semaphore,
)
.await?;
}
@ -554,9 +575,15 @@ const DEFAULT_MAVEN_METADATA_URL: &str =
/// the specified Minecraft version
pub async fn fetch_maven_metadata(
url: Option<&str>,
semaphore: Arc<Semaphore>,
) -> Result<HashMap<String, Vec<String>>, Error> {
Ok(serde_json::from_slice(
&download_file(url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), None).await?,
&download_file(
url.unwrap_or(DEFAULT_MAVEN_METADATA_URL),
None,
semaphore,
)
.await?,
)?)
}

97
daedalus_client/src/main.rs
View File

@ -1,8 +1,10 @@
use log::{error, warn};
use std::time::Duration;
use s3::{Bucket, Region};
use log::{error, info, warn};
use s3::creds::Credentials;
use s3::error::S3Error;
use s3::{Bucket, Region};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Semaphore;
mod fabric;
mod forge;
@ -19,10 +21,7 @@ pub enum Error {
#[error("Error while managing asynchronous tasks")]
TaskError(#[from] tokio::task::JoinError),
#[error("Error while uploading file to S3")]
S3Error {
inner: S3Error,
file: String,
},
S3Error { inner: S3Error, file: String },
#[error("Error while parsing version as semver: {0}")]
SemVerError(#[from] semver::Error),
#[error("Error while reading zip file: {0}")]
@ -31,6 +30,8 @@ pub enum Error {
IoError(#[from] std::io::Error),
#[error("Error while obtaining strong reference to Arc")]
ArcError,
#[error("Error acquiring semaphore: {0}")]
AcquireError(#[from] tokio::sync::AcquireError),
}
#[tokio::main]
@ -43,14 +44,20 @@ async fn main() {
return;
}
let mut timer = tokio::time::interval(Duration::from_secs(10 * 60));
let mut timer = tokio::time::interval(Duration::from_secs(30 * 60));
let semaphore = Arc::new(Semaphore::new(50));
loop {
timer.tick().await;
let mut uploaded_files = Vec::new();
let versions = match minecraft::retrieve_data(&mut uploaded_files).await {
let versions = match minecraft::retrieve_data(
&mut uploaded_files,
semaphore.clone(),
)
.await
{
Ok(res) => Some(res),
Err(err) => {
error!("{:?}", err);
@ -60,11 +67,23 @@ async fn main() {
};
if let Some(manifest) = versions {
match fabric::retrieve_data(&manifest, &mut uploaded_files).await {
match fabric::retrieve_data(
&manifest,
&mut uploaded_files,
semaphore.clone(),
)
.await
{
Ok(..) => {}
Err(err) => error!("{:?}", err),
};
match forge::retrieve_data(&manifest, &mut uploaded_files).await {
match forge::retrieve_data(
&manifest,
&mut uploaded_files,
semaphore.clone(),
)
.await
{
Ok(..) => {}
Err(err) => error!("{:?}", err),
};
@ -93,7 +112,6 @@ fn check_env_vars() -> bool {
}
failed |= check_var::<String>("BASE_URL");
failed |= check_var::<String>("BASE_FOLDER");
failed |= check_var::<String>("S3_ACCESS_TOKEN");
failed |= check_var::<String>("S3_SECRET");
@ -104,7 +122,6 @@ fn check_env_vars() -> bool {
failed
}
lazy_static::lazy_static! {
static ref CLIENT : Bucket = Bucket::new(
&dotenvy::var("S3_BUCKET_NAME").unwrap(),
@ -133,29 +150,29 @@ pub async fn upload_file_to_bucket(
bytes: Vec<u8>,
content_type: Option<String>,
uploaded_files: &tokio::sync::Mutex<Vec<String>>,
semaphore: Arc<Semaphore>,
) -> Result<(), Error> {
let key = format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path);
let _permit = semaphore.acquire().await?;
info!("{} started uploading", path);
let key = path.clone();
for attempt in 1..=4 {
let result = if let Some(ref content_type) = content_type {
CLIENT.put_object_with_content_type(
key.clone(),
&bytes,
content_type,
).await
CLIENT
.put_object_with_content_type(key.clone(), &bytes, content_type)
.await
} else {
CLIENT.put_object(
key.clone(),
&bytes,
).await
}.map_err(|err| Error::S3Error {
CLIENT.put_object(key.clone(), &bytes).await
}
.map_err(|err| Error::S3Error {
inner: err,
file: format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path),
file: path.clone(),
});
match result {
Ok(_) => {
{
info!("{} done uploading", path);
let mut uploaded_files = uploaded_files.lock().await;
uploaded_files.push(key);
}
@ -168,15 +185,39 @@ pub async fn upload_file_to_bucket(
}
}
}
unreachable!()
}
pub fn format_url(path: &str) -> String {
format!(
"{}/{}/{}",
"{}/{}",
&*dotenvy::var("BASE_URL").unwrap(),
&*dotenvy::var("BASE_FOLDER").unwrap(),
path
)
}
pub async fn download_file(
url: &str,
sha1: Option<&str>,
semaphore: Arc<Semaphore>,
) -> Result<bytes::Bytes, Error> {
let _permit = semaphore.acquire().await?;
info!("{} started downloading", url);
let val = daedalus::download_file(url, sha1).await?;
info!("{} finished downloading", url);
Ok(val)
}
pub async fn download_file_mirrors(
base: &str,
mirrors: &[&str],
sha1: Option<&str>,
semaphore: Arc<Semaphore>,
) -> Result<bytes::Bytes, Error> {
let _permit = semaphore.acquire().await?;
info!("{} started downloading", base);
let val = daedalus::download_file_mirrors(base, mirrors, sha1).await?;
info!("{} finished downloading", base);
Ok(val)
}

56
daedalus_client/src/minecraft.rs
View File

@ -1,21 +1,26 @@
use crate::download_file;
use crate::{format_url, upload_file_to_bucket, Error};
use daedalus::download_file;
use daedalus::minecraft::VersionManifest;
use log::info;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::Mutex;
use std::time::Instant;
use tokio::sync::{Mutex, Semaphore};
pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionManifest, Error> {
let old_manifest =
daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&format!(
pub async fn retrieve_data(
uploaded_files: &mut Vec<String>,
semaphore: Arc<Semaphore>,
) -> Result<VersionManifest, Error> {
let old_manifest = daedalus::minecraft::fetch_version_manifest(Some(
&*format_url(&format!(
"minecraft/v{}/manifest.json",
daedalus::minecraft::CURRENT_FORMAT_VERSION
))))
.await
.ok();
)),
))
.await
.ok();
let mut manifest = daedalus::minecraft::fetch_version_manifest(None).await?;
let mut manifest =
daedalus::minecraft::fetch_version_manifest(None).await?;
let cloned_manifest = Arc::new(Mutex::new(manifest.clone()));
let visited_assets_mutex = Arc::new(Mutex::new(Vec::new()));
@ -42,13 +47,16 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
let visited_assets_mutex = Arc::clone(&visited_assets_mutex);
let cloned_manifest_mutex = Arc::clone(&cloned_manifest);
let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
let semaphore = Arc::clone(&semaphore);
let assets_hash = old_version.and_then(|x| x.assets_index_sha1.clone());
let assets_hash =
old_version.and_then(|x| x.assets_index_sha1.clone());
async move {
let mut upload_futures = Vec::new();
let version_info = daedalus::minecraft::fetch_version_info(version).await?;
let version_info =
daedalus::minecraft::fetch_version_info(version).await?;
let version_path = format!(
"minecraft/v{}/versions/{}.json",
@ -63,14 +71,16 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
let assets_index_url = version_info.asset_index.url.clone();
{
let mut cloned_manifest = cloned_manifest_mutex.lock().await;
let mut cloned_manifest =
cloned_manifest_mutex.lock().await;
let position = cloned_manifest
.versions
.iter()
.position(|x| version.id == x.id)
.unwrap();
cloned_manifest.versions[position].url = format_url(&version_path);
cloned_manifest.versions[position].url =
format_url(&version_path);
cloned_manifest.versions[position].assets_index_sha1 =
Some(version_info.asset_index.sha1.clone());
cloned_manifest.versions[position].assets_index_url =
@ -93,14 +103,18 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
}
if download_assets {
visited_assets.push(version_info.asset_index.id.clone());
visited_assets
.push(version_info.asset_index.id.clone());
}
}
if download_assets {
let assets_index =
download_file(&assets_index_url, Some(&version_info.asset_index.sha1))
.await?;
let assets_index = download_file(
&assets_index_url,
Some(&version_info.asset_index.sha1),
semaphore.clone(),
)
.await?;
{
upload_futures.push(upload_file_to_bucket(
@ -108,6 +122,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
assets_index.to_vec(),
Some("application/json".to_string()),
uploaded_files_mutex.as_ref(),
semaphore.clone(),
));
}
}
@ -118,6 +133,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
serde_json::to_vec(&version_info)?,
Some("application/json".to_string()),
uploaded_files_mutex.as_ref(),
semaphore.clone(),
));
}
@ -140,14 +156,13 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
let chunk: Vec<_> = versions.by_ref().take(100).collect();
futures::future::try_join_all(chunk).await?;
tokio::time::sleep(Duration::from_secs(1)).await;
chunk_index += 1;
let elapsed = now.elapsed();
info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
}
}
//futures::future::try_join_all(version_futures).await?;
upload_file_to_bucket(
format!(
@ -157,6 +172,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
serde_json::to_vec(&*cloned_manifest.lock().await)?,
Some("application/json".to_string()),
uploaded_files_mutex.as_ref(),
semaphore,
)
.await?;