From d4de1dc9a18ef962bffdfc53c5c7f12602448622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Gonz=C3=A1lez?= <7822554+AlexTMjugador@users.noreply.github.com> Date: Fri, 13 Jun 2025 22:52:57 +0200 Subject: [PATCH] fix(app): make instances with non-UTF8 text file encodings launcheable and importable (#3721) Previous to these changes, the app always assumed that Minecraft and other launchers always use UTF-8, which is not necessarily always true. --- Cargo.lock | 13 +++++ Cargo.toml | 2 + packages/app-lib/Cargo.toml | 2 + .../app-lib/src/api/pack/import/atlauncher.rs | 30 +++++++----- .../app-lib/src/api/pack/import/curseforge.rs | 32 ++++++------ .../app-lib/src/api/pack/import/gdlauncher.rs | 25 +++++----- packages/app-lib/src/api/pack/import/mmc.rs | 25 +++++----- packages/app-lib/src/launcher/mod.rs | 29 ++++++++--- packages/app-lib/src/util/io.rs | 49 ++++++++++--------- 9 files changed, 131 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c29be8c6..9cb8f0a1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1379,6 +1379,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chardetng" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b8f0b65b7b08ae3c8187e8d77174de20cb6777864c6b832d8ad365999cf1ea" +dependencies = [ + "cfg-if", + "encoding_rs", + "memchr", +] + [[package]] name = "chrono" version = "0.4.41" @@ -8865,6 +8876,7 @@ dependencies = [ "async_zip", "base64 0.22.1", "bytes", + "chardetng", "chrono", "daedalus", "dashmap", @@ -8872,6 +8884,7 @@ dependencies = [ "discord-rich-presence", "dunce", "either", + "encoding_rs", "enumset", "flate2", "fs4", diff --git a/Cargo.toml b/Cargo.toml index eabe40a95..2bd20f96c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ base64 = "0.22.1" bitflags = "2.9.1" bytes = "1.10.1" censor = "0.3.0" +chardetng = "0.1.17" chrono = "0.4.41" clap = "4.5.40" clickhouse = "0.13.3" @@ -50,6 +51,7 @@ dotenv-build = "0.1.1" dotenvy = "0.15.7" dunce = "1.0.5" either = "1.15.0" +encoding_rs = "0.8.35" enumset = "1.1.6" flate2 = "1.1.2" fs4 = { version = "0.13.1", default-features = false } diff --git a/packages/app-lib/Cargo.toml b/packages/app-lib/Cargo.toml index 275a31c06..78f004a73 100644 --- a/packages/app-lib/Cargo.toml +++ b/packages/app-lib/Cargo.toml @@ -20,6 +20,8 @@ tempfile.workspace = true dashmap = { workspace = true, features = ["serde"] } quick-xml = { workspace = true, features = ["async-tokio"] } enumset.workspace = true +chardetng.workspace = true +encoding_rs.workspace = true chrono = { workspace = true, features = ["serde"] } daedalus.workspace = true diff --git a/packages/app-lib/src/api/pack/import/atlauncher.rs b/packages/app-lib/src/api/pack/import/atlauncher.rs index f6dabca4c..1c8ba084d 100644 --- a/packages/app-lib/src/api/pack/import/atlauncher.rs +++ b/packages/app-lib/src/api/pack/import/atlauncher.rs @@ -97,12 +97,15 @@ pub struct ATLauncherMod { // Check if folder has a instance.json that parses pub async fn is_valid_atlauncher(instance_folder: PathBuf) -> bool { - let instance: String = - io::read_to_string(&instance_folder.join("instance.json")) - .await - .unwrap_or("".to_string()); - let instance: Result = - serde_json::from_str::(&instance); + let instance = serde_json::from_str::( + &io::read_any_encoding_to_string( + &instance_folder.join("instance.json"), + ) + .await + .unwrap_or(("".into(), encoding_rs::UTF_8)) + .0, + ); + if let Err(e) = instance { tracing::warn!( "Could not parse instance.json at {}: {}", @@ -124,14 +127,17 @@ pub async fn import_atlauncher( ) -> crate::Result<()> { let atlauncher_instance_path = atlauncher_base_path .join("instances") - .join(instance_folder.clone()); + .join(&instance_folder); // Load instance.json - let atinstance: String = - io::read_to_string(&atlauncher_instance_path.join("instance.json")) - .await?; - let atinstance: ATInstance = - serde_json::from_str::(&atinstance)?; + let atinstance = serde_json::from_str::( + &io::read_any_encoding_to_string( + &atlauncher_instance_path.join("instance.json"), + ) + .await + .unwrap_or(("".into(), encoding_rs::UTF_8)) + .0, + )?; // Icon path should be {instance_folder}/instance.png if it exists, // Second possibility is ATLauncher/configs/images/{safe_pack_name}.png (safe pack name is alphanumeric lowercase) diff --git a/packages/app-lib/src/api/pack/import/curseforge.rs b/packages/app-lib/src/api/pack/import/curseforge.rs index 1c0819e4b..6c5b983d2 100644 --- a/packages/app-lib/src/api/pack/import/curseforge.rs +++ b/packages/app-lib/src/api/pack/import/curseforge.rs @@ -36,13 +36,15 @@ pub struct InstalledModpack { // Check if folder has a minecraftinstance.json that parses pub async fn is_valid_curseforge(instance_folder: PathBuf) -> bool { - let minecraftinstance: String = - io::read_to_string(&instance_folder.join("minecraftinstance.json")) - .await - .unwrap_or("".to_string()); - let minecraftinstance: Result = - serde_json::from_str::(&minecraftinstance); - minecraftinstance.is_ok() + let minecraft_instance = serde_json::from_str::( + &io::read_any_encoding_to_string( + &instance_folder.join("minecraftinstance.json"), + ) + .await + .unwrap_or(("".into(), encoding_rs::UTF_8)) + .0, + ); + minecraft_instance.is_ok() } pub async fn import_curseforge( @@ -50,13 +52,15 @@ pub async fn import_curseforge( profile_path: &str, // path to profile ) -> crate::Result<()> { // Load minecraftinstance.json - let minecraft_instance: String = io::read_to_string( - &curseforge_instance_folder.join("minecraftinstance.json"), - ) - .await?; - let minecraft_instance: MinecraftInstance = - serde_json::from_str::(&minecraft_instance)?; - let override_title: Option = minecraft_instance.name.clone(); + let minecraft_instance = serde_json::from_str::( + &io::read_any_encoding_to_string( + &curseforge_instance_folder.join("minecraftinstance.json"), + ) + .await + .unwrap_or(("".into(), encoding_rs::UTF_8)) + .0, + )?; + let override_title = minecraft_instance.name; let backup_name = format!( "Curseforge-{}", curseforge_instance_folder diff --git a/packages/app-lib/src/api/pack/import/gdlauncher.rs b/packages/app-lib/src/api/pack/import/gdlauncher.rs index 307301014..5daf6cd28 100644 --- a/packages/app-lib/src/api/pack/import/gdlauncher.rs +++ b/packages/app-lib/src/api/pack/import/gdlauncher.rs @@ -25,12 +25,12 @@ pub struct GDLauncherLoader { // Check if folder has a config.json that parses pub async fn is_valid_gdlauncher(instance_folder: PathBuf) -> bool { - let config: String = - io::read_to_string(&instance_folder.join("config.json")) + let config = serde_json::from_str::( + &io::read_any_encoding_to_string(&instance_folder.join("config.json")) .await - .unwrap_or("".to_string()); - let config: Result = - serde_json::from_str::(&config); + .unwrap_or(("".into(), encoding_rs::UTF_8)) + .0, + ); config.is_ok() } @@ -39,12 +39,15 @@ pub async fn import_gdlauncher( profile_path: &str, // path to profile ) -> crate::Result<()> { // Load config.json - let config: String = - io::read_to_string(&gdlauncher_instance_folder.join("config.json")) - .await?; - let config: GDLauncherConfig = - serde_json::from_str::(&config)?; - let override_title: Option = config.loader.source_name.clone(); + let config = serde_json::from_str::( + &io::read_any_encoding_to_string( + &gdlauncher_instance_folder.join("config.json"), + ) + .await + .unwrap_or(("".into(), encoding_rs::UTF_8)) + .0, + )?; + let override_title = config.loader.source_name; let backup_name = format!( "GDLauncher-{}", gdlauncher_instance_folder diff --git a/packages/app-lib/src/api/pack/import/mmc.rs b/packages/app-lib/src/api/pack/import/mmc.rs index 94083be88..c3399276a 100644 --- a/packages/app-lib/src/api/pack/import/mmc.rs +++ b/packages/app-lib/src/api/pack/import/mmc.rs @@ -144,8 +144,8 @@ pub async fn is_valid_mmc(instance_folder: PathBuf) -> bool { let instance_cfg = instance_folder.join("instance.cfg"); let mmc_pack = instance_folder.join("mmc-pack.json"); - let mmc_pack = match io::read_to_string(&mmc_pack).await { - Ok(mmc_pack) => mmc_pack, + let mmc_pack = match io::read_any_encoding_to_string(&mmc_pack).await { + Ok((mmc_pack, _)) => mmc_pack, Err(_) => return false, }; @@ -155,7 +155,7 @@ pub async fn is_valid_mmc(instance_folder: PathBuf) -> bool { #[tracing::instrument] pub async fn get_instances_subpath(config: PathBuf) -> Option { - let launcher = io::read_to_string(&config).await.ok()?; + let launcher = io::read_any_encoding_to_string(&config).await.ok()?.0; let launcher: MMCLauncherEnum = serde_ini::from_str(&launcher).ok()?; match launcher { MMCLauncherEnum::General(p) => Some(p.general.instance_dir), @@ -165,10 +165,9 @@ pub async fn get_instances_subpath(config: PathBuf) -> Option { // Loading the INI (instance.cfg) file async fn load_instance_cfg(file_path: &Path) -> crate::Result { - let instance_cfg: String = io::read_to_string(file_path).await?; - let instance_cfg_enum: MMCInstanceEnum = - serde_ini::from_str::(&instance_cfg)?; - match instance_cfg_enum { + match serde_ini::from_str::( + &io::read_any_encoding_to_string(file_path).await?.0, + )? { MMCInstanceEnum::General(instance_cfg) => Ok(instance_cfg.general), MMCInstanceEnum::Instance(instance_cfg) => Ok(instance_cfg), } @@ -183,9 +182,13 @@ pub async fn import_mmc( let mmc_instance_path = mmc_base_path.join("instances").join(instance_folder); - let mmc_pack = - io::read_to_string(&mmc_instance_path.join("mmc-pack.json")).await?; - let mmc_pack: MMCPack = serde_json::from_str::(&mmc_pack)?; + let mmc_pack = serde_json::from_str::( + &io::read_any_encoding_to_string( + &mmc_instance_path.join("mmc-pack.json"), + ) + .await? + .0, + )?; let instance_cfg = load_instance_cfg(&mmc_instance_path.join("instance.cfg")).await?; @@ -243,7 +246,7 @@ pub async fn import_mmc( _ => return Err(crate::ErrorKind::InputError("Instance is managed, but managed pack type not specified in instance.cfg".to_string()).into()) } } else { - // Direclty import unmanaged pack + // Directly import unmanaged pack import_mmc_unmanaged( profile_path, minecraft_folder, diff --git a/packages/app-lib/src/launcher/mod.rs b/packages/app-lib/src/launcher/mod.rs index d3e9d90b9..fafcbee20 100644 --- a/packages/app-lib/src/launcher/mod.rs +++ b/packages/app-lib/src/launcher/mod.rs @@ -14,6 +14,7 @@ use chrono::Utc; use daedalus as d; use daedalus::minecraft::{LoggingSide, RuleAction, VersionInfo}; use daedalus::modded::LoaderVersion; +use regex::Regex; use serde::Deserialize; use st::Profile; use std::collections::HashMap; @@ -662,14 +663,29 @@ pub async fn launch_minecraft( // Overwrites the minecraft options.txt file with the settings from the profile // Uses 'a:b' syntax which is not quite yaml - use regex::Regex; - if !mc_set_options.is_empty() { let options_path = instance_path.join("options.txt"); - let mut options_string = String::new(); - if options_path.exists() { - options_string = io::read_to_string(&options_path).await?; + + let (mut options_string, input_encoding) = if options_path.exists() { + io::read_any_encoding_to_string(&options_path).await? + } else { + (String::new(), encoding_rs::UTF_8) + }; + + // UTF-16 encodings may be successfully detected and read, but we cannot encode + // them back, and it's technically possible that the game client strongly expects + // such encoding + if input_encoding != input_encoding.output_encoding() { + return Err(crate::ErrorKind::LauncherError(format!( + "The instance options.txt file uses an unsupported encoding: {}. \ + Please either turn off instance options that need to modify this file, \ + or convert the file to an encoding that both the game and this app support, \ + such as UTF-8.", + input_encoding.name() + )) + .into()); } + for (key, value) in mc_set_options { let re = Regex::new(&format!(r"(?m)^{}:.*$", regex::escape(key)))?; // check if the regex exists in the file @@ -684,7 +700,8 @@ pub async fn launch_minecraft( } } - io::write(&options_path, options_string).await?; + io::write(&options_path, input_encoding.encode(&options_string).0) + .await?; } crate::api::profile::edit(&profile.path, |prof| { diff --git a/packages/app-lib/src/util/io.rs b/packages/app-lib/src/util/io.rs index 9d82c24d3..0079f4809 100644 --- a/packages/app-lib/src/util/io.rs +++ b/packages/app-lib/src/util/io.rs @@ -35,7 +35,6 @@ impl IOError { } } -// dunce canonicalize pub fn canonicalize( path: impl AsRef, ) -> Result { @@ -46,7 +45,6 @@ pub fn canonicalize( }) } -// read_dir pub async fn read_dir( path: impl AsRef, ) -> Result { @@ -59,7 +57,6 @@ pub async fn read_dir( }) } -// create_dir pub async fn create_dir( path: impl AsRef, ) -> Result<(), IOError> { @@ -72,7 +69,6 @@ pub async fn create_dir( }) } -// create_dir_all pub async fn create_dir_all( path: impl AsRef, ) -> Result<(), IOError> { @@ -85,7 +81,6 @@ pub async fn create_dir_all( }) } -// remove_dir_all pub async fn remove_dir_all( path: impl AsRef, ) -> Result<(), IOError> { @@ -98,20 +93,37 @@ pub async fn remove_dir_all( }) } -// read_to_string -pub async fn read_to_string( +/// Reads a text file to a string, automatically detecting its encoding and +/// substituting any invalid characters with the Unicode replacement character. +/// +/// This function is best suited for reading Minecraft instance files, whose +/// encoding may vary depending on the platform, launchers, client versions +/// (older Minecraft versions tended to rely on the system's default codepage +/// more on Windows platforms), and mods used, while not being highly sensitive +/// to occasional occurrences of mojibake or character replacements. +pub async fn read_any_encoding_to_string( path: impl AsRef, -) -> Result { +) -> Result<(String, &'static encoding_rs::Encoding), IOError> { let path = path.as_ref(); - tokio::fs::read_to_string(path) - .await - .map_err(|e| IOError::IOPathError { - source: e, - path: path.to_string_lossy().to_string(), - }) + let file_bytes = + tokio::fs::read(path) + .await + .map_err(|e| IOError::IOPathError { + source: e, + path: path.to_string_lossy().to_string(), + })?; + + let file_encoding = { + let mut encoding_detector = chardetng::EncodingDetector::new(); + encoding_detector.feed(&file_bytes, true); + encoding_detector.guess(None, true) + }; + + let (file_string, actual_file_encoding, _) = + file_encoding.decode(&file_bytes); + Ok((file_string.to_string(), actual_file_encoding)) } -// read pub async fn read( path: impl AsRef, ) -> Result, IOError> { @@ -124,7 +136,6 @@ pub async fn read( }) } -// write pub async fn write( path: impl AsRef, data: impl AsRef<[u8]>, @@ -186,7 +197,6 @@ pub fn is_same_disk(old_dir: &Path, new_dir: &Path) -> Result { } } -// rename pub async fn rename_or_move( from: impl AsRef, to: impl AsRef, @@ -228,7 +238,6 @@ async fn move_recursive(from: &Path, to: &Path) -> Result<(), IOError> { Ok(()) } -// copy pub async fn copy( from: impl AsRef, to: impl AsRef, @@ -243,7 +252,6 @@ pub async fn copy( }) } -// remove file pub async fn remove_file( path: impl AsRef, ) -> Result<(), IOError> { @@ -256,7 +264,6 @@ pub async fn remove_file( }) } -// open file pub async fn open_file( path: impl AsRef, ) -> Result { @@ -269,7 +276,6 @@ pub async fn open_file( }) } -// remove dir pub async fn remove_dir( path: impl AsRef, ) -> Result<(), IOError> { @@ -282,7 +288,6 @@ pub async fn remove_dir( }) } -// metadata pub async fn metadata( path: impl AsRef, ) -> Result {