Bommels05/Modrinth
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make java stuff public, fix forge erroring out due to ratelimiting

Browse Source
This commit is contained in:
Jai A 2021-12-19 15:09:36 -07:00
parent 5a6c06c8a3
commit 2a588d1e9a
No known key found for this signature in database
GPG Key ID: CC88DE86F48BE019
5 changed files with 305 additions and 283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
daedalus/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "daedalus" name = "daedalus"
version = "0.1.8" version = "0.1.9"
authors = ["Jai A <jaiagr+gpg@pm.me>"] authors = ["Jai A <jaiagr+gpg@pm.me>"]
edition = "2018" edition = "2018"
license = "MIT" license = "MIT"

4
daedalus/src/minecraft.rs
View File

@ -233,9 +233,9 @@ pub struct LibraryExtract {
/// Information about the java version the game needs /// Information about the java version the game needs
pub struct JavaVersion { pub struct JavaVersion {
/// The component needed for the Java installation /// The component needed for the Java installation
component: String, pub component: String,
/// The major Java version number /// The major Java version number
major_version: u32, pub major_version: u32,
} }
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]

2
daedalus_client/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "daedalus_client" name = "daedalus_client"
version = "0.1.8" version = "0.1.9"
authors = ["Jai A <jaiagr+gpg@pm.me>"] authors = ["Jai A <jaiagr+gpg@pm.me>"]
edition = "2018" edition = "2018"

541
daedalus_client/src/forge.rs
View File

@ -27,6 +27,7 @@ pub async fn retrieve_data(
minecraft_versions: &VersionManifest, minecraft_versions: &VersionManifest,
uploaded_files: &mut Vec<String>, uploaded_files: &mut Vec<String>,
) -> Result<(), Error> { ) -> Result<(), Error> {
println!("forg");
let maven_metadata = fetch_maven_metadata(None).await?; let maven_metadata = fetch_maven_metadata(None).await?;
let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!(
"forge/v{}/manifest.json", "forge/v{}/manifest.json",
@ -81,189 +82,192 @@ pub async fn retrieve_data(
} }
} }
version_futures.push(async { version_futures.push(async {
let loaders_versions = futures::future::try_join_all(loaders.into_iter().map(|(loader_version_full, version)| async { let mut loaders_versions = Vec::new();
let versions_mutex = Arc::clone(&old_versions);
let visited_assets = Arc::clone(&visited_assets_mutex);
let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
let minecraft_version = minecraft_version.clone();
async move { {
{ let mut loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async {
let versions = versions_mutex.lock().await; let versions_mutex = Arc::clone(&old_versions);
let version = versions.iter().find(|x| let visited_assets = Arc::clone(&visited_assets_mutex);
x.id == minecraft_version).map(|x| x.loaders.iter().find(|x| x.id == loader_version_full)).flatten(); let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
let minecraft_version = minecraft_version.clone();
if let Some(version) = version { async move {
return Ok::<Option<LoaderVersion>, Error>(Some(version.clone())); {
let versions = versions_mutex.lock().await;
let version = versions.iter().find(|x|
x.id == minecraft_version).map(|x| x.loaders.iter().find(|x| x.id == loader_version_full)).flatten();
if let Some(version) = version {
return Ok::<Option<LoaderVersion>, Error>(Some(version.clone()));
}
} }
}
info!("Forge - Installer Start {}", loader_version_full.clone()); info!("Forge - Installer Start {}", loader_version_full.clone());
let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?;
let reader = std::io::Cursor::new(bytes); let reader = std::io::Cursor::new(bytes);
if let Ok(archive) = zip::ZipArchive::new(reader) { if let Ok(archive) = zip::ZipArchive::new(reader) {
if FORGE_MANIFEST_V1_QUERY.matches(&version) { if FORGE_MANIFEST_V1_QUERY.matches(&version) {
let mut archive_clone = archive.clone(); let mut archive_clone = archive.clone();
let profile = tokio::task::spawn_blocking(move || { let profile = tokio::task::spawn_blocking(move || {
let mut install_profile = archive_clone.by_name("install_profile.json")?; let mut install_profile = archive_clone.by_name("install_profile.json")?;
let mut contents = String::new(); let mut contents = String::new();
install_profile.read_to_string(&mut contents)?; install_profile.read_to_string(&mut contents)?;
Ok::<ForgeInstallerProfileV1, Error>(serde_json::from_str::<ForgeInstallerProfileV1>(&*contents)?) Ok::<ForgeInstallerProfileV1, Error>(serde_json::from_str::<ForgeInstallerProfileV1>(&*contents)?)
}).await??; }).await??;
let mut archive_clone = archive.clone(); let mut archive_clone = archive.clone();
let file_path = profile.install.file_path.clone(); let file_path = profile.install.file_path.clone();
let forge_universal_bytes = tokio::task::spawn_blocking(move || { let forge_universal_bytes = tokio::task::spawn_blocking(move || {
let mut forge_universal_file = archive_clone.by_name(&*file_path)?; let mut forge_universal_file = archive_clone.by_name(&*file_path)?;
let mut forge_universal = Vec::new(); let mut forge_universal = Vec::new();
forge_universal_file.read_to_end(&mut forge_universal)?; forge_universal_file.read_to_end(&mut forge_universal)?;
Ok::<bytes::Bytes, Error>(bytes::Bytes::from(forge_universal)) Ok::<bytes::Bytes, Error>(bytes::Bytes::from(forge_universal))
}).await??; }).await??;
let forge_universal_path = profile.install.path.clone(); let forge_universal_path = profile.install.path.clone();
let now = Instant::now(); let now = Instant::now();
let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async {
if let Some(url) = lib.url { if let Some(url) = lib.url {
{ {
let mut visited_assets = visited_assets.lock().await; let mut visited_assets = visited_assets.lock().await;
if visited_assets.contains(&lib.name) { if visited_assets.contains(&lib.name) {
lib.url = Some(format_url("maven/")); lib.url = Some(format_url("maven/"));
return Ok::<Library, Error>(lib); return Ok::<Library, Error>(lib);
} else { } else {
visited_assets.push(lib.name.clone()) visited_assets.push(lib.name.clone())
}
} }
let artifact_path =
daedalus::get_path_from_artifact(&*lib.name)?;
let artifact = if lib.name == forge_universal_path {
forge_universal_bytes.clone()
} else {
let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"];
daedalus::download_file_mirrors(
&*artifact_path,
&mirrors,
None,
)
.await?
};
lib.url = Some(format_url("maven/"));
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
artifact.to_vec(),
Some("application/java-archive".to_string()),
uploaded_files_mutex.as_ref(),
).await?;
} }
let artifact_path = Ok::<Library, Error>(lib)
daedalus::get_path_from_artifact(&*lib.name)?; })).await?;
let artifact = if lib.name == forge_universal_path { let elapsed = now.elapsed();
forge_universal_bytes.clone() info!("Elapsed lib DL: {:.2?}", elapsed);
} else {
let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"];
daedalus::download_file_mirrors( let new_profile = PartialVersionInfo {
&*artifact_path, id: profile.version_info.id,
&mirrors, inherits_from: profile.install.minecraft,
None, release_time: profile.version_info.release_time,
) time: profile.version_info.time,
.await? main_class: profile.version_info.main_class,
}; minecraft_arguments: profile.version_info.minecraft_arguments.clone(),
arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()),
libraries: libs,
type_: profile.version_info.type_,
data: None,
processors: None
};
lib.url = Some(format_url("maven/")); let version_path = format!(
"forge/v{}/versions/{}.json",
daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
new_profile.id
);
upload_file_to_bucket( upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path), version_path.clone(),
artifact.to_vec(), serde_json::to_vec(&new_profile)?,
Some("application/java-archive".to_string()), Some("application/json".to_string()),
uploaded_files_mutex.as_ref(), uploaded_files_mutex.as_ref()
).await?; ).await?;
return Ok(Some(LoaderVersion {
id: loader_version_full,
url: format_url(&*version_path),
stable: false
}));
} else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) {
let mut archive_clone = archive.clone();
let mut profile = tokio::task::spawn_blocking(move || {
let mut install_profile = archive_clone.by_name("install_profile.json")?;
let mut contents = String::new();
install_profile.read_to_string(&mut contents)?;
Ok::<ForgeInstallerProfileV2, Error>(serde_json::from_str::<ForgeInstallerProfileV2>(&*contents)?)
}).await??;
let mut archive_clone = archive.clone();
let version_info = tokio::task::spawn_blocking(move || {
let mut install_profile = archive_clone.by_name("version.json")?;
let mut contents = String::new();
install_profile.read_to_string(&mut contents)?;
Ok::<PartialVersionInfo, Error>(serde_json::from_str::<PartialVersionInfo>(&*contents)?)
}).await??;
let mut libs : Vec<daedalus::minecraft::Library> = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library {
downloads: x.downloads,
extract: x.extract,
name: x.name,
url: x.url,
natives: x.natives,
rules: x.rules,
checksums: x.checksums,
include_in_classpath: false
})).collect();
let mut local_libs : HashMap<String, bytes::Bytes> = HashMap::new();
for lib in &libs {
if lib.downloads.as_ref().map(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).flatten().unwrap_or(false) {
let mut archive_clone = archive.clone();
let lib_name_clone = lib.name.clone();
let lib_bytes = tokio::task::spawn_blocking(move || {
let mut lib_file = archive_clone.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*lib_name_clone)?))?;
let mut lib_bytes = Vec::new();
lib_file.read_to_end(&mut lib_bytes)?;
Ok::<bytes::Bytes, Error>(bytes::Bytes::from(lib_bytes))
}).await??;
local_libs.insert(lib.name.clone(), lib_bytes);
}
} }
Ok::<Library, Error>(lib) let path = profile.path.clone();
})).await?; let version = profile.version.clone();
let elapsed = now.elapsed(); for entry in profile.data.values_mut() {
info!("Elapsed lib DL: {:.2?}", elapsed); if entry.client.starts_with('/') || entry.server.starts_with('/') {
macro_rules! read_data {
let new_profile = PartialVersionInfo {
id: profile.version_info.id,
inherits_from: profile.install.minecraft,
release_time: profile.version_info.release_time,
time: profile.version_info.time,
main_class: profile.version_info.main_class,
minecraft_arguments: profile.version_info.minecraft_arguments.clone(),
arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()),
libraries: libs,
type_: profile.version_info.type_,
data: None,
processors: None
};
let version_path = format!(
"forge/v{}/versions/{}.json",
daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
new_profile.id
);
upload_file_to_bucket(
version_path.clone(),
serde_json::to_vec(&new_profile)?,
Some("application/json".to_string()),
uploaded_files_mutex.as_ref()
).await?;
return Ok(Some(LoaderVersion {
id: loader_version_full,
url: format_url(&*version_path),
stable: false
}));
} else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) {
let mut archive_clone = archive.clone();
let mut profile = tokio::task::spawn_blocking(move || {
let mut install_profile = archive_clone.by_name("install_profile.json")?;
let mut contents = String::new();
install_profile.read_to_string(&mut contents)?;
Ok::<ForgeInstallerProfileV2, Error>(serde_json::from_str::<ForgeInstallerProfileV2>(&*contents)?)
}).await??;
let mut archive_clone = archive.clone();
let version_info = tokio::task::spawn_blocking(move || {
let mut install_profile = archive_clone.by_name("version.json")?;
let mut contents = String::new();
install_profile.read_to_string(&mut contents)?;
Ok::<PartialVersionInfo, Error>(serde_json::from_str::<PartialVersionInfo>(&*contents)?)
}).await??;
let mut libs : Vec<daedalus::minecraft::Library> = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library {
downloads: x.downloads,
extract: x.extract,
name: x.name,
url: x.url,
natives: x.natives,
rules: x.rules,
checksums: x.checksums,
include_in_classpath: false
})).collect();
let mut local_libs : HashMap<String, bytes::Bytes> = HashMap::new();
for lib in &libs {
if lib.downloads.as_ref().map(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).flatten().unwrap_or(false) {
let mut archive_clone = archive.clone();
let lib_name_clone = lib.name.clone();
let lib_bytes = tokio::task::spawn_blocking(move || {
let mut lib_file = archive_clone.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*lib_name_clone)?))?;
let mut lib_bytes = Vec::new();
lib_file.read_to_end(&mut lib_bytes)?;
Ok::<bytes::Bytes, Error>(bytes::Bytes::from(lib_bytes))
}).await??;
local_libs.insert(lib.name.clone(), lib_bytes);
}
}
let path = profile.path.clone();
let version = profile.version.clone();
for entry in profile.data.values_mut() {
if entry.client.starts_with('/') || entry.server.starts_with('/') {
macro_rules! read_data {
($value:expr) => { ($value:expr) => {
let mut archive_clone = archive.clone(); let mut archive_clone = archive.clone();
let value_clone = $value.clone(); let value_clone = $value.clone();
@ -302,131 +306,150 @@ pub async fn retrieve_data(
} }
} }
if entry.client.starts_with('/') { if entry.client.starts_with('/') {
read_data!(entry.client); read_data!(entry.client);
}
// Do we really need to support server installs? Keeping this here
// just in case
//
// if entry.server.starts_with('/') {
// read_data!(entry.server);
// }
}
}
let now = Instant::now();
let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async {
let artifact_path =
daedalus::get_path_from_artifact(&*lib.name)?;
{
let mut visited_assets = visited_assets.lock().await;
if visited_assets.contains(&lib.name) {
if let Some(ref mut downloads) = lib.downloads {
if let Some(ref mut artifact) = downloads.artifact {
artifact.url = format_url(&*format!("maven/{}", artifact_path));
}
} else if lib.url.is_some() {
lib.url = Some(format_url("maven/"));
} }
return Ok::<Library, Error>(lib); // Do we really need to support server installs? Keeping this here
} else { // just in case
visited_assets.push(lib.name.clone()) //
// if entry.server.starts_with('/') {
// read_data!(entry.server);
// }
} }
} }
let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { let now = Instant::now();
if let Some(ref mut artifact) = downloads.artifact { let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async {
let res = if artifact.url.is_empty() { let artifact_path =
daedalus::get_path_from_artifact(&*lib.name)?;
{
let mut visited_assets = visited_assets.lock().await;
if visited_assets.contains(&lib.name) {
if let Some(ref mut downloads) = lib.downloads {
if let Some(ref mut artifact) = downloads.artifact {
artifact.url = format_url(&*format!("maven/{}", artifact_path));
}
} else if lib.url.is_some() {
lib.url = Some(format_url("maven/"));
}
return Ok::<Library, Error>(lib);
} else {
visited_assets.push(lib.name.clone())
}
}
let artifact_bytes = if let Some(ref mut downloads) = lib.downloads {
if let Some(ref mut artifact) = downloads.artifact {
let res = if artifact.url.is_empty() {
local_libs.get(&lib.name).cloned()
} else {
Some(daedalus::download_file(
&*artifact.url,
Some(&*artifact.sha1),
)
.await?)
};
if res.is_some() {
artifact.url = format_url(&*format!("maven/{}", artifact_path));
}
res
} else { None }
} else if let Some(ref mut url) = lib.url {
let res = if url.is_empty() {
local_libs.get(&lib.name).cloned() local_libs.get(&lib.name).cloned()
} else { } else {
Some(daedalus::download_file( Some(daedalus::download_file(
&*artifact.url, url,
Some(&*artifact.sha1), None,
) )
.await?) .await?)
}; };
if res.is_some() { if res.is_some() {
artifact.url = format_url(&*format!("maven/{}", artifact_path)); lib.url = Some(format_url("maven/"));
} }
res res
} else { None } } else { None };
} else if let Some(ref mut url) = lib.url {
let res = if url.is_empty() {
local_libs.get(&lib.name).cloned()
} else {
Some(daedalus::download_file(
url,
None,
)
.await?)
};
if res.is_some() { if let Some(bytes) = artifact_bytes {
lib.url = Some(format_url("maven/")); upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
bytes.to_vec(),
Some("application/java-archive".to_string()),
uploaded_files_mutex.as_ref()
).await?;
} }
res Ok::<Library, Error>(lib)
} else { None }; })).await?;
if let Some(bytes) = artifact_bytes { let elapsed = now.elapsed();
upload_file_to_bucket( info!("Elapsed lib DL: {:.2?}", elapsed);
format!("{}/{}", "maven", artifact_path),
bytes.to_vec(),
Some("application/java-archive".to_string()),
uploaded_files_mutex.as_ref()
).await?;
}
Ok::<Library, Error>(lib) let new_profile = PartialVersionInfo {
})).await?; id: version_info.id,
inherits_from: version_info.inherits_from,
release_time: version_info.release_time,
time: version_info.time,
main_class: version_info.main_class,
minecraft_arguments: version_info.minecraft_arguments,
arguments: version_info.arguments,
libraries: libs,
type_: version_info.type_,
data: Some(profile.data),
processors: Some(profile.processors),
};
let elapsed = now.elapsed(); let version_path = format!(
info!("Elapsed lib DL: {:.2?}", elapsed); "forge/v{}/versions/{}.json",
daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
new_profile.id
);
let new_profile = PartialVersionInfo { upload_file_to_bucket(
id: version_info.id, version_path.clone(),
inherits_from: version_info.inherits_from, serde_json::to_vec(&new_profile)?,
release_time: version_info.release_time, Some("application/json".to_string()),
time: version_info.time, uploaded_files_mutex.as_ref()
main_class: version_info.main_class, ).await?;
minecraft_arguments: version_info.minecraft_arguments,
arguments: version_info.arguments,
libraries: libs,
type_: version_info.type_,
data: Some(profile.data),
processors: Some(profile.processors),
};
let version_path = format!( return Ok(Some(LoaderVersion {
"forge/v{}/versions/{}.json", id: loader_version_full,
daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, url: format_url(&*version_path),
new_profile.id stable: false
); }));
}
upload_file_to_bucket(
version_path.clone(),
serde_json::to_vec(&new_profile)?,
Some("application/json".to_string()),
uploaded_files_mutex.as_ref()
).await?;
return Ok(Some(LoaderVersion {
id: loader_version_full,
url: format_url(&*version_path),
stable: false
}));
} }
}
Ok(None) Ok(None)
}.await }.await
})).await?.into_iter().flatten().collect(); }).into_iter().peekable()/*.into_iter().flatten().collect()*/;
let mut chunk_index = 0;
while loaders_futures.peek().is_some() {
info!("Loader Chunk {} Start", chunk_index);
let now = Instant::now();
let chunk: Vec<_> = loaders_futures.by_ref().take(10).collect();
let res = futures::future::try_join_all(chunk).await?;
loaders_versions.extend(res.into_iter().flatten());
tokio::time::sleep(Duration::from_secs(1)).await;
chunk_index += 1;
let elapsed = now.elapsed();
info!("Loader Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
}
}
versions.lock().await.push(daedalus::modded::Version { versions.lock().await.push(daedalus::modded::Version {
id: minecraft_version, id: minecraft_version,
@ -444,7 +467,7 @@ pub async fn retrieve_data(
info!("Chunk {} Start", chunk_index); info!("Chunk {} Start", chunk_index);
let now = Instant::now(); let now = Instant::now();
let chunk: Vec<_> = versions_peek.by_ref().take(10).collect(); let chunk: Vec<_> = versions_peek.by_ref().take(1).collect();
futures::future::try_join_all(chunk).await?; futures::future::try_join_all(chunk).await?;
tokio::time::sleep(Duration::from_secs(1)).await; tokio::time::sleep(Duration::from_secs(1)).await;

39
daedalus_client/src/main.rs
View File

@ -48,34 +48,33 @@ async fn main() {
loop { loop {
timer.tick().await; timer.tick().await;
tokio::spawn(async {
let mut uploaded_files = Vec::new();
let versions = match minecraft::retrieve_data(&mut uploaded_files).await { let mut uploaded_files = Vec::new();
Ok(res) => Some(res),
Err(err) => {
error!("{:?}", err);
None let versions = match minecraft::retrieve_data(&mut uploaded_files).await {
} Ok(res) => Some(res),
}; Err(err) => {
error!("{:?}", err);
if let Some(manifest) = versions { None
match fabric::retrieve_data(&manifest, &mut uploaded_files).await {
Ok(..) => {}
Err(err) => error!("{:?}", err),
};
match forge::retrieve_data(&manifest, &mut uploaded_files).await {
Ok(..) => {}
Err(err) => error!("{:?}", err),
};
} }
};
match purge_digitalocean_cache(uploaded_files).await { if let Some(manifest) = versions {
// match fabric::retrieve_data(&manifest, &mut uploaded_files).await {
// Ok(..) => {}
// Err(err) => error!("{:?}", err),
// };
match forge::retrieve_data(&manifest, &mut uploaded_files).await {
Ok(..) => {} Ok(..) => {}
Err(err) => error!("{:?}", err), Err(err) => error!("{:?}", err),
}; };
}); }
match purge_digitalocean_cache(uploaded_files).await {
Ok(..) => {}
Err(err) => error!("{:?}", err),
};
} }
} }