From 9551676cb9ad982865c12dede84c4ec646ef629c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Sj=C3=B6blom?= Date: Thu, 16 Oct 2025 23:56:20 +0200 Subject: [PATCH] Adding support for locales --- src/helpers.cpp | 67 ++++++++++++++++++++++-------- src/helpers.h | 1 + src/main.cpp | 24 ++++++++--- src/mpq.cpp | 106 ++++++++++++++++++++++++++++++++++-------------- src/mpq.h | 13 +++--- 5 files changed, 151 insertions(+), 60 deletions(-) diff --git a/src/helpers.cpp b/src/helpers.cpp index 77cfacd..e498dbd 100644 --- a/src/helpers.cpp +++ b/src/helpers.cpp @@ -1,10 +1,9 @@ #include -#include #include #include -#include #include #include +#include #include #ifdef _WIN32 @@ -15,7 +14,6 @@ #include #include "helpers.h" -#include "mpq.h" namespace fs = std::filesystem; @@ -36,22 +34,55 @@ std::string FileTimeToLsTime(int64_t fileTime) { return std::string(buf); } +namespace { + // Files in MPQs have locales with which they are associated. + // Multiple files can have the same file name if they have different locales. + // This function maps locales to language names. + // + // The mappings are from the Windows Language Code Identifier (LCID). + // They can be found, for example, here: + // https://winprotocoldoc.z19.web.core.windows.net/MS-LCID/%5bMS-LCID%5d.pdf + + // Define a bidirectional map for locale-language mappings + const std::map localeToLangMap = { + {0x000, "enUS"}, // Default - English (US) + {0x409, "enUS"}, // English (US) + {0x404, "zhTW"}, // Chinese (Taiwan) + {0x405, "csCZ"}, // Czech + {0x407, "deDE"}, // German + {0x40a, "esES"}, // Spanish (Spain) + {0x40c, "frFR"}, // French + {0x410, "itIT"}, // Italian + {0x411, "jaJP"}, // Japanese + {0x412, "koKR"}, // Korean + {0x415, "plPL"}, // Polish + {0x416, "ptPT"}, // Portuguese (Portugal) + {0x419, "ruRU"}, // Russian + {0x804, "zhCN"}, // Chinese (Simplified) + {0x809, "enGB"}, // English (UK) + {0x80A, "esMX"} // Spanish (Mexico) + }; + + // Create a reverse map for language to locale lookups + const std::map langToLocaleMap = []() { + std::map reverseMap; + for (const auto& [locale, lang] : localeToLangMap) { + if (locale != 0x000) { // Skip the default entry to avoid duplication + reverseMap[lang] = locale; + } + } + return reverseMap; + }(); +} + std::string LocaleToLang(uint16_t locale) { - switch (locale) { - case 0: return "enUS"; // English (US/GB) - case 1: return "koKR"; // Korean - case 2: return "frFR"; // French - case 3: return "deDE"; // German - case 4: return "zhCN"; // Chinese (Simplified) - case 5: return "zhTW"; // Chinese (Taiwan) - case 6: return "esES"; // Spanish (Spain) - case 7: return "esMX"; // Spanish (Mexico) - case 8: return "ruRU"; // Russian - case 9: return "jaJP"; // Japanese - case 10: return "ptPT"; // Portuguese (Portugal) - case 11: return "itIT"; // Italian - default: return ""; // Unknown locale - } + auto it = localeToLangMap.find(locale); + return it != localeToLangMap.end() ? it->second : ""; +} + +LCID LangToLocale(const std::string& lang) { + auto it = langToLocaleMap.find(lang); + return it != langToLocaleMap.end() ? it->second : 0; } std::string NormalizeFilePath(const fs::path &path) { diff --git a/src/helpers.h b/src/helpers.h index b38b35a..f1e1279 100644 --- a/src/helpers.h +++ b/src/helpers.h @@ -8,6 +8,7 @@ namespace fs = std::filesystem; std::string FileTimeToLsTime(int64_t fileTime); std::string LocaleToLang(uint16_t locale); +LCID LangToLocale(const std::string &lang); std::string NormalizeFilePath(const fs::path &path); std::string WindowsifyFilePath(const fs::path &path); int32_t CalculateMpqMaxFileValue(const std::string &directory); diff --git a/src/main.cpp b/src/main.cpp index e5d80cf..9419370 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -25,6 +25,7 @@ int main(int argc, char **argv) { std::string baseTarget = "default"; // all subcommands std::string baseFile = "default"; // add, remove, extract, read std::string basePath = "default"; // add + std::string baseLocale = "default"; // add, remove, extract, read std::string baseOutput = "default"; // create, extract std::string baseListfileName = "default"; // list, extract // CLI: info @@ -83,6 +84,7 @@ int main(int argc, char **argv) { ->required() ->check(CLI::ExistingFile); add->add_option("-p,--path", basePath, "Path within MPQ archive"); + add->add_option("--locale", baseLocale, "Locale to use for added file"); // Subcommand: Remove CLI::App *remove = app.add_subcommand("remove", "Remove file from an existing MPQ archive"); @@ -91,6 +93,7 @@ int main(int argc, char **argv) { remove->add_option("target", baseTarget, "Target MPQ archive") ->required() ->check(CLI::ExistingFile); + remove->add_option("--locale", baseLocale, "Locale of file to remove"); // Subcommand: List CLI::App *list = app.add_subcommand("list", "List files from the MPQ archive"); @@ -112,6 +115,7 @@ int main(int argc, char **argv) { extract->add_flag("-k,--keep", extractKeepFolderStructure, "Keep folder structure (default false)"); extract->add_option("-l,--listfile", baseListfileName, "File listing content of an MPQ archive") ->check(CLI::ExistingFile); + extract->add_option("--locale", baseLocale, "Preferred locale for extracted file"); // Subcommand: Read CLI::App* read = app.add_subcommand("read", "Read a file from an MPQ archive"); @@ -120,6 +124,7 @@ int main(int argc, char **argv) { read->add_option("target", baseTarget, "Target MPQ archive") ->required() ->check(CLI::ExistingFile); + read->add_option("--locale", baseLocale, "Preferred locale for read file"); // Subcommand: Verify CLI::App *verify = app.add_subcommand("verify", "Verify the MPQ archive"); @@ -187,7 +192,8 @@ int main(int argc, char **argv) { // Create the MPQ archive and add files HANDLE hArchive = CreateMpqArchive(outputFile, fileCount, createMpqVersion); if (hArchive) { - AddFiles(hArchive, baseTarget); + LCID locale = LangToLocale(baseLocale); + AddFiles(hArchive, baseTarget, locale); if (createSignArchive) { SignMpqArchive(hArchive); } @@ -221,7 +227,9 @@ int main(int argc, char **argv) { archivePath = WindowsifyFilePath(archiveFullPath.u8string()); } - AddFile(hArchive, baseFile, archivePath); + LCID locale = LangToLocale(baseLocale); + + AddFile(hArchive, baseFile, archivePath, locale); CloseMpqArchive(hArchive); } @@ -234,7 +242,9 @@ int main(int argc, char **argv) { return 1; } - RemoveFile(hArchive, baseFile); + LCID locale = LangToLocale(baseLocale); + + RemoveFile(hArchive, baseFile, locale); CloseMpqArchive(hArchive); } @@ -266,10 +276,11 @@ int main(int argc, char **argv) { return 1; } + LCID locale = LangToLocale(baseLocale); if (baseFile != "default") { - ExtractFile(hArchive, baseOutput, baseFile, extractKeepFolderStructure); + ExtractFile(hArchive, baseOutput, baseFile, extractKeepFolderStructure, locale); } else { - ExtractFiles(hArchive, baseOutput, baseListfileName); + ExtractFiles(hArchive, baseOutput, baseListfileName, locale); } } @@ -281,8 +292,9 @@ int main(int argc, char **argv) { return 1; } + LCID locale = LangToLocale(baseLocale); uint32_t fileSize; - char* fileContent = ReadFile(hArchive, baseFile.c_str(), &fileSize); + char* fileContent = ReadFile(hArchive, baseFile.c_str(), &fileSize, locale); if (fileContent == NULL) { return 1; } diff --git a/src/mpq.cpp b/src/mpq.cpp index d8915a3..2021ba8 100644 --- a/src/mpq.cpp +++ b/src/mpq.cpp @@ -4,8 +4,8 @@ #include #include #include +#include #include - #include #include "mpq.h" @@ -37,7 +37,8 @@ int SignMpqArchive(HANDLE hArchive) { return 1; } -int ExtractFiles(HANDLE hArchive, const std::string& output, const std::string& listfileName) { +int ExtractFiles(HANDLE hArchive, const std::string& output, const std::string& listfileName, LCID preferredLocale) { + SFileSetLocale(preferredLocale); // Check if the user provided a listfile input const char *listfile = (listfileName == "default") ? NULL : listfileName.c_str(); @@ -54,7 +55,8 @@ int ExtractFiles(HANDLE hArchive, const std::string& output, const std::string& hArchive, output, findData.cFileName, - true // Keep folder structure + true, // Keep folder structure + preferredLocale ); if (result != 0) { return result; @@ -66,7 +68,8 @@ int ExtractFiles(HANDLE hArchive, const std::string& output, const std::string& return 0; } -int ExtractFile(HANDLE hArchive, const std::string& output, const std::string& fileName, bool keepFolderStructure) { +int ExtractFile(HANDLE hArchive, const std::string& output, const std::string& fileName, bool keepFolderStructure, LCID preferredLocale) { + SFileSetLocale(preferredLocale); const char *szFileName = fileName.c_str(); if (!SFileHasFile(hArchive, szFileName)) { std::cerr << "[+] Failed: File doesn't exist: " << szFileName << std::endl; @@ -152,7 +155,7 @@ HANDLE CreateMpqArchive(std::string outputArchiveName, int32_t fileCount, int32_ return hMpq; } -int AddFiles(HANDLE hArchive, const std::string& target) { +int AddFiles(HANDLE hArchive, const std::string& target, LCID preferredLocale) { // We need to "clean" the target path to ensure it is a valid directory // and to strip any directory structure from the files we add fs::path targetPath = fs::path(target); @@ -165,14 +168,15 @@ int AddFiles(HANDLE hArchive, const std::string& target) { // Normalise path for MPQ std::string archiveFilePath = WindowsifyFilePath(inputFilePath.u8string()); - AddFile(hArchive, entry.path().u8string(), archiveFilePath); + AddFile(hArchive, entry.path().u8string(), archiveFilePath, preferredLocale); } } return 0; } -int AddFile(HANDLE hArchive, fs::path localFile, const std::string& archiveFilePath) { - std::cout << "[+] Adding file: " << archiveFilePath << std::endl; +int AddFile(HANDLE hArchive, fs::path localFile, const std::string& archiveFilePath, LCID locale) { + SFileSetLocale(locale); + std::cout << "[+] Adding file: " << archiveFilePath << " for locale " << locale << std::endl; // Return if file doesn't exist on disk if (!fs::exists(localFile)) { @@ -181,11 +185,15 @@ int AddFile(HANDLE hArchive, fs::path localFile, const std::string& archiveFileP } // Check if file exists in MPQ archive - bool hasFile = SFileHasFile(hArchive, archiveFilePath.c_str()); - if (hasFile) { - std::cerr << "[!] File already exists in MPQ archive: " << archiveFilePath << " Skipping..." << std::endl; - return -1; + HANDLE hFile; + if (SFileOpenFileEx(hArchive, archiveFilePath.c_str(), SFILE_OPEN_FROM_MPQ, &hFile)) { + int32_t fileLocale = GetFileInfo(hFile, SFileInfoLocale); + if (fileLocale == locale) { + std::cerr << "[!] File for locale " << locale << " already exists in MPQ archive: " << archiveFilePath << " - Skipping..." << std::endl; + return -1; + } } + SFileCloseFile(hFile); // Verify that we are not exceeding maxFile size of the archive, and if we do, increase it int32_t numberOfFiles = GetFileInfo(hArchive, SFileMpqNumberOfFiles); @@ -225,16 +233,17 @@ int AddFile(HANDLE hArchive, fs::path localFile, const std::string& archiveFileP return 0; } -int RemoveFile(HANDLE hArchive, const std::string& archiveFilePath) { - std::cout << "[+] Removing file: " << archiveFilePath << std::endl; +int RemoveFile(HANDLE hArchive, const std::string& archiveFilePath, LCID locale) { + SFileSetLocale(locale); + std::cout << "[-] Removing file: " << archiveFilePath << " for locale " << locale << std::endl; if (!SFileHasFile(hArchive, archiveFilePath.c_str())) { - std::cerr << "[+] Failed: File doesn't exist: " << archiveFilePath << std::endl; + std::cerr << "[!] Failed: File doesn't exist: " << archiveFilePath << " for locale " << locale << std::endl; return -1; } if (!SFileRemoveFile(hArchive, archiveFilePath.c_str(), 0)) { - std::cerr << "[+] Failed: File cannot be removed: " << archiveFilePath << std::endl; + std::cerr << "[!] Failed: File cannot be removed: " << archiveFilePath << " for locale " << locale << std::endl; return -1; } @@ -261,7 +270,8 @@ int ListFiles(HANDLE hArchive, const std::string& listfileName, bool listAll, bo "(attributes)" }; - // Loop through all files in MPQ archive + std::set seenFileNames; // Used to prevent printing the same file name multiple times + // Loop through all files in the MPQ archive do { // Skip special files unless user wants to list all (like ls -a) if (!listAll && std::find(specialFiles.begin(), specialFiles.end(), findData.cFileName) != specialFiles.end()) { @@ -277,20 +287,55 @@ int ListFiles(HANDLE hArchive, const std::string& listfileName, bool listAll, bo std::cerr << "[+] Failed to open file: " << findData.cFileName << std::endl; continue; // Skip to the next file } + SFileCloseFile(hFile); - int32_t fileSize = GetFileInfo(hFile, SFileInfoFileSize); - int32_t fileLocale = GetFileInfo(hFile, SFileInfoLocale); - std::string fileLocaleStr = LocaleToLang(fileLocale); - int64_t fileTime = GetFileInfo(hFile, SFileInfoFileTime); - std::string fileTimeStr = FileTimeToLsTime(fileTime); + if (seenFileNames.find(findData.cFileName) != seenFileNames.end()) { + // Filename has been seen before, and thus printed before. Skip over it. + continue; + } + seenFileNames.insert(findData.cFileName); - // Print the file details in a formatted way - std::cout << std::setw(11) << fileSize << " " // 4GB max size is 10 characters - << std::setw(5) << fileLocaleStr << " " // Locale is max 4 characters - << std::setw(18) << fileTimeStr << " " // File time is formatted as "MMM DD YYYY HH:MM" - << findData.cFileName << std::endl; - SFileCloseFile(hFile); + // Multiple files can be stored with identical filenames under different locales. + // Loop over all locales and print the file details for each locale. + DWORD maxLocales = 32; // This will be updated in the call to SFileEnumLocales + LCID * fileLocales = (LCID *)malloc(maxLocales * sizeof(LCID)); + + if (fileLocales == NULL) { + std::cerr << "[!] Unable to allocate memory for locales for file: " << findData.cFileName << std::endl; + continue; + } + DWORD result = SFileEnumLocales(hArchive, findData.cFileName, fileLocales, &maxLocales, 0); + + if (result == ERROR_INSUFFICIENT_BUFFER) { + std::cerr << "[!] There are more than " << maxLocales << " locales for the file: " << findData.cFileName << + ". Will only list the " << maxLocales << " first files." << std::endl; + } + + // Loop through all found locales + for (DWORD i = 0; i < maxLocales; i++) { + LCID locale = fileLocales[i]; + SFileSetLocale(locale); + if (!SFileOpenFileEx(hArchive, findData.cFileName, SFILE_OPEN_FROM_MPQ, &hFile)) { + std::cerr << "[!] Failed to open file: " << findData.cFileName << std::endl; + continue; // Skip to the next file + } + + int32_t fileSize = GetFileInfo(hFile, SFileInfoFileSize); + int32_t fileLocale = GetFileInfo(hFile, SFileInfoLocale); + std::string fileLocaleStr = LocaleToLang(fileLocale); + int64_t fileTime = GetFileInfo(hFile, SFileInfoFileTime); + std::string fileTimeStr = FileTimeToLsTime(fileTime); + + // Print the file details in a formatted way + std::cout << std::setw(11) << fileSize << " " // 4GB max size is 10 characters + << std::setw(5) << fileLocaleStr << " " // Locale is max 4 characters + << std::setw(19) << fileTimeStr << " " // File time is formatted as "YYYY-MM-DD HH:MM:SS" + << findData.cFileName << std::endl; + SFileCloseFile(hFile); + } + SFileSetLocale(defaultLocale); // Reset locale to default after changing it + free(fileLocales); } else { // Print just the filename (like default ls command output) std::cout << findData.cFileName << std::endl; @@ -301,7 +346,8 @@ int ListFiles(HANDLE hArchive, const std::string& listfileName, bool listAll, bo return 0; } -char* ReadFile(HANDLE hArchive, const char *szFileName, unsigned int *fileSize) { +char* ReadFile(HANDLE hArchive, const char *szFileName, unsigned int *fileSize, LCID preferredLocale) { + SFileSetLocale(preferredLocale); if (!SFileHasFile(hArchive, szFileName)) { std::cerr << "[+] Failed: File doesn't exist: " << szFileName << std::endl; return NULL; @@ -435,7 +481,7 @@ int32_t PrintMpqSignature(HANDLE hArchive, std::string target) { } else if (signatureType == SIGNATURE_TYPE_WEAK) { const char* szFileName = "(signature)"; uint32_t fileSize; - char* fileContent = ReadFile(hArchive, szFileName, &fileSize); + char* fileContent = ReadFile(hArchive, szFileName, &fileSize, defaultLocale); if (fileContent == NULL) { std::cerr << "[+] Failed to read weak signature file." << std::endl; diff --git a/src/mpq.h b/src/mpq.h index ee3226b..adc2735 100644 --- a/src/mpq.h +++ b/src/mpq.h @@ -7,18 +7,19 @@ #include namespace fs = std::filesystem; +const LCID defaultLocale = 0; int OpenMpqArchive(const std::string &filename, HANDLE *hArchive, int32_t flags); int CloseMpqArchive(HANDLE hArchive); int SignMpqArchive(HANDLE hArchive); -int ExtractFiles(HANDLE hArchive, const std::string& output, const std::string &listfileName); -int ExtractFile(HANDLE hArchive, const std::string& output, const std::string& fileName, bool keepFolderStructure); +int ExtractFiles(HANDLE hArchive, const std::string& output, const std::string &listfileName, LCID preferredLocale); +int ExtractFile(HANDLE hArchive, const std::string& output, const std::string& fileName, bool keepFolderStructure, LCID preferredLocale); HANDLE CreateMpqArchive(std::string outputArchiveName, int32_t fileCount, int32_t mpqVersion); -int AddFiles(HANDLE hArchive, const std::string& inputPath); -int AddFile(HANDLE hArchive, fs::path localFile, const std::string& archiveFilePath); -int RemoveFile(HANDLE hArchive, const std::string& archiveFilePath); +int AddFiles(HANDLE hArchive, const std::string& inputPath, LCID preferredLocale); +int AddFile(HANDLE hArchive, fs::path localFile, const std::string& archiveFilePath, LCID locale); +int RemoveFile(HANDLE hArchive, const std::string& archiveFilePath, LCID locale); int ListFiles(HANDLE hHandle, const std::string &listfileName, bool listAll, bool listDetailed); -char* ReadFile(HANDLE hArchive, const char *szFileName, unsigned int *fileSize); +char* ReadFile(HANDLE hArchive, const char *szFileName, unsigned int *fileSize, LCID preferredLocale); void PrintMpqInfo(HANDLE hArchive, const std::string& infoProperty); uint32_t VerifyMpqArchive(HANDLE hArchive); int32_t PrintMpqSignature(HANDLE hArchive, std::string target);