From a7ce92411ed3d697abb68b5b1a48d32f65f01679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Sj=C3=B6blom?= Date: Wed, 22 Oct 2025 08:16:05 +0200 Subject: [PATCH 1/2] Allowing users to specify which file properties to list --- README.md | 67 +++++++++++++++++++++++----- src/main.cpp | 37 ++++++++++++---- src/mpq.cpp | 108 ++++++++++++++++++++++++++++++++++++++++------ src/mpq.h | 2 +- test/test_list.py | 104 +++++++++++++++++++++++++++++++++++++++----- 5 files changed, 274 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index b7bdac5..0852d85 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ mpqcli remove fth.txt wow-patch.mpq Pretty simple, list files in an MPQ archive. Useful to "pipe" to other tools, such as `grep` (see below for examples). ``` -mpqcli list wow-patch.mpq +$ mpqcli list wow-patch.mpq BM_COKETENT01.BLP Blizzard_CraftUI.xml CreatureSoundData.dbc @@ -188,22 +188,67 @@ realmlist.wtf Similar to the `ls` command with the `-l` and `-a` options, additional detailed information can be included with the `list` subcommand. The `-a` option includes printing "special" files used in MPQ archives including: `(listfile)`, `(attributes)` and `(signature)`. ``` -mpqcli list -d -a wow-patch.mpq - 88604 enUS 2006-03-29 02:02:37 BM_COKETENT01.BLP - 243 enUS 2006-04-04 21:28:14 Blizzard_CraftUI.xml - 388 enUS 2006-03-29 19:32:46 CreatureSoundData.dbc - ... - 184 enUS 2006-04-04 21:28:14 Blizzard_CraftUI.lua - 44900 enUS 2006-03-29 02:01:02 30ee7bd3959906e358eff01332cf045e.blp - 68 enUS 2006-04-07 00:58:44 realmlist.wtf +$ mpqcli list -d -a wow-patch.mpq + 88604 enUS 2006-03-29 02:02:37 BM_COKETENT01.BLP + 243 enUS 2006-04-04 21:28:14 Blizzard_CraftUI.xml + 388 enUS 2006-03-29 19:32:46 CreatureSoundData.dbc + ... + 184 enUS 2006-04-04 21:28:14 Blizzard_CraftUI.lua + 44900 enUS 2006-03-29 02:01:02 30ee7bd3959906e358eff01332cf045e.blp + 68 enUS 2006-04-07 00:58:44 realmlist.wtf +``` + +### List specific properties + +The `list` subcommand supports listing the following properties: + +- `hash-index` - Index in the hash table where the file entry is. +- `name-hash1` - The first hash of the file name. +- `name-hash2` - The second hash of the file name. +- `name-hash3` - 64-bit Jenkins hash of the file name, used for searching in the HET table. +- `locale` - Locale info of the file. +- `file-index` - Index in the file table of the file. +- `byte-offset` - Offset of the file in the MPQ, relative to the MPQ header. +- `file-time` - Timestamp of the file. +- `file-size` - Uncompressed file size of the file, in bytes. +- `compressed-size` - Compressed file size of the file, in bytes. +- `encryption-key` - Encryption key for the file. +- `encryption-key-raw` - Encryption key for the file. +- `flags` - File flags for the file within MPQ: + * `i`: File is Imploded (By PKWARE Data Compression Library). + * `c`: File is Compressed (By any of multiple methods). + * `e`: File is Encrypted. + * `2`: File is Encrypted with key v2. + * `p`: File is a Patch file. + * `u`: File is stored as a single Unit, rather than split into sectors. + * `d`: File is a Deletion marker. Used in MPQ patches, indicating that the file no longer exists. + * `r`: File has Sector CRC checksums for each sector. This is ignored if the file is not compressed or imploded. + * `s`: Present on STANDARD.SNP\(signature). + * `x`: File exists; this is reset if the file is deleted. + * `m`: Mask for a file being compressed. + * `n`: Use default flags for internal files. + * `f`: Fix key; This is obsolete. + +You can use the `-p` or `--property` argument with the `list` subcommand to print the given properties. Many properties can be given, and they will be printed in the order given. + +``` +$ mpqcli list -d -a War2Patch.mpq -p hash-index -p locale -p flags + 3028 enUS iexmn Rez\gluchat.bin + 8926 enUS iexmn Rez\Gateways.txt + 9078 enUS iexmn Rez\mltiplay.bin +10329 enUS ce2xmnf (listfile) +14213 enUS iexmn Rez\mltiplay_ita.bin +14472 enUS iexmn Rez\mltiplay_esp.bin +15278 enUS iexmn Rez\mltiplay_fra.bin +15731 enUS iexmn Rez\mltiplay_deu.bin ``` ### List all files with an external listfile -Older MPQ archives do not contain (complete) file paths of their content. By providing an external listfile that lists the content of the MPQ archive, the listed files will have the correct paths. Listfiles can be downloaded on [Ladislav Zezula's site](http://www.zezula.net/en/mpq/download.html). +Older MPQ archives do not contain (complete) file paths of their content. By using the `-l` or `--listfile` argument, one can provide an external listfile that lists the content of the MPQ archive, so that the listed files will have the correct paths. Listfiles can be downloaded on [Ladislav Zezula's site](http://www.zezula.net/en/mpq/download.html). ``` -mpqcli list -l /path/to/listfile StarDat.mpq +$ mpqcli list -l /path/to/listfile StarDat.mpq ``` ### Extract all files from an MPQ archive diff --git a/src/main.cpp b/src/main.cpp index d6ec78e..4f941ce 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -28,7 +28,9 @@ int main(int argc, char **argv) { std::string baseOutput = "default"; // create, extract std::string baseListfileName = "default"; // list, extract // CLI: info - std::string infoProperty = "default"; + std::string infoProperty = "default"; + // CLI: list + std::vector listProperties; // CLI: extract bool extractKeepFolderStructure = false; // CLI: create @@ -41,13 +43,28 @@ int main(int argc, char **argv) { bool verifyPrintSignature = false; std::set validInfoProperties = { - "format-version", - "header-offset", - "header-size", - "archive-size", - "file-count", - "max-files", - "signature-type" + "format-version", + "header-offset", + "header-size", + "archive-size", + "file-count", + "max-files", + "signature-type", + }; + std::set validFileListProperties = { + "hash-index", + "name-hash1", + "name-hash2", + "name-hash3", + "locale", + "file-index", + "byte-offset", + "file-time", + "file-size", + "compressed-size", + "flags", + "encryption-key", + "encryption-key-raw", }; // Subcommand: Version @@ -101,6 +118,8 @@ int main(int argc, char **argv) { ->check(CLI::ExistingFile); list->add_flag("-d,--detailed", listDetailed, "File listing with additional columns (default false)"); list->add_flag("-a,--all", listAll, "File listing including hidden files (default true)"); + list->add_option("-p,--property", listProperties, "Prints only specific property values") + ->check(CLI::IsMember(validFileListProperties)); // Subcommand: Extract CLI::App *extract = app.add_subcommand("extract", "Extract files from the MPQ archive"); @@ -245,7 +264,7 @@ int main(int argc, char **argv) { std::cerr << "[!] Failed to open MPQ archive." << std::endl; return 1; } - ListFiles(hArchive, baseListfileName, listAll, listDetailed); + ListFiles(hArchive, baseListfileName, listAll, listDetailed, listProperties); } // Handle subcommand: Extract diff --git a/src/mpq.cpp b/src/mpq.cpp index 2253275..b5f08ae 100644 --- a/src/mpq.cpp +++ b/src/mpq.cpp @@ -241,7 +241,27 @@ int RemoveFile(HANDLE hArchive, const std::string& archiveFilePath) { return 0; } -int ListFiles(HANDLE hArchive, const std::string& listfileName, bool listAll, bool listDetailed) { +std::string GetFlagString(uint32_t flags) { + std::string result; + + if (flags & MPQ_FILE_IMPLODE) result += 'i'; + if (flags & MPQ_FILE_COMPRESS) result += 'c'; + if (flags & MPQ_FILE_ENCRYPTED) result += 'e'; + if (flags & MPQ_FILE_KEY_V2) result += '2'; + if (flags & MPQ_FILE_PATCH_FILE) result += 'p'; + if (flags & MPQ_FILE_SINGLE_UNIT) result += 'u'; + if (flags & MPQ_FILE_DELETE_MARKER) result += 'd'; + if (flags & MPQ_FILE_SECTOR_CRC) result += 'r'; + if (flags & MPQ_FILE_SIGNATURE) result += 's'; + if (flags & MPQ_FILE_EXISTS) result += 'x'; + if (flags & MPQ_FILE_COMPRESS_MASK) result += 'm'; + if (flags & MPQ_FILE_DEFAULT_INTERNAL) result += 'n'; + if (flags & MPQ_FILE_FIX_KEY) result += 'f'; + + return result; +} + +int ListFiles(HANDLE hArchive, const std::string& listfileName, bool listAll, bool listDetailed, std::vector& propertiesToPrint) { // Check if the user provided a listfile input const char *listfile = (listfileName == "default") ? NULL : listfileName.c_str(); @@ -253,6 +273,13 @@ int ListFiles(HANDLE hArchive, const std::string& listfileName, bool listAll, bo return -1; } + if (propertiesToPrint.empty()) { + propertiesToPrint = { + // Default properties, if the user didn't specify any + "file-size", "locale", "file-time", + }; + } + // "Special" files are base files used by MPQ file format // These are skipped, unless "-a" or "--all" are specified std::vector specialFiles = { @@ -278,17 +305,74 @@ int ListFiles(HANDLE hArchive, const std::string& listfileName, bool listAll, bo continue; // Skip to the next file } - int32_t fileSize = GetFileInfo(hFile, SFileInfoFileSize); - int32_t fileLocale = GetFileInfo(hFile, SFileInfoLocale); - std::string fileLocaleStr = LocaleToLang(fileLocale); - int64_t fileTime = GetFileInfo(hFile, SFileInfoFileTime); - std::string fileTimeStr = FileTimeToLsTime(fileTime); - - // Print the file details in a formatted way - std::cout << std::setw(11) << fileSize << " " // 4GB max size is 10 characters - << std::setw(5) << fileLocaleStr << " " // Locale is max 4 characters - << std::setw(19) << fileTimeStr << " " // File time is formatted as "YYYY-MM-DD HH:MM:SS" - << findData.cFileName << std::endl; + std::vector>> propertyActions = { + {"hash-index", [&]() { + std::cout << std::setw(5) << GetFileInfo(hFile, SFileInfoHashIndex) << " " ; + }}, + {"name-hash1", [&]() { + std::cout << std::setfill('0') << std::hex << std::setw(8) << + GetFileInfo(hFile, SFileInfoNameHash1) << + std::setfill(' ') << std::dec << " "; + }}, + {"name-hash2", [&]() { + std::cout << std::setfill('0') << std::hex << std::setw(8) << + GetFileInfo(hFile, SFileInfoNameHash2) << + std::setfill(' ') << std::dec << " "; + }}, + {"name-hash3", [&]() { + std::cout << std::setfill('0') << std::hex << std::setw(16) << + GetFileInfo(hFile, SFileInfoNameHash3) << + std::setfill(' ') << std::dec << " "; + }}, + {"locale", [&]() { + int32_t fileLocale = GetFileInfo(hFile, SFileInfoLocale); + std::string fileLocaleStr = LocaleToLang(fileLocale); + std::cout << std::setw(4) << fileLocaleStr << " "; + }}, + {"file-index", [&]() { + std::cout << std::setw(5) << GetFileInfo(hFile, SFileInfoFileIndex) << " "; + }}, + {"byte-offset", [&]() { + std::cout << std::hex << std::setw(8) << + GetFileInfo(hFile, SFileInfoByteOffset) << + std::dec << " "; + }}, + {"file-time", [&]() { + int64_t fileTime = GetFileInfo(hFile, SFileInfoFileTime); + std::string fileTimeStr = FileTimeToLsTime(fileTime); + std::cout << std::setw(19) << fileTimeStr << " "; + }}, + {"file-size", [&]() { + std::cout << std::setw(8) << GetFileInfo(hFile, SFileInfoFileSize) << " "; + }}, + {"compressed-size", [&]() { + std::cout << std::setw(8) << GetFileInfo(hFile, SFileInfoCompressedSize) << " "; + }}, + {"flags", [&]() { + int32_t flags = GetFileInfo(hFile, SFileInfoFlags); + std::cout << std::setw(8) << GetFlagString(flags) << " "; + }}, + {"encryption-key", [&]() { + std::cout << std::setfill('0') << std::hex << std::setw(8) << + GetFileInfo(hFile, SFileInfoEncryptionKey) << + std::setfill(' ') << std::dec << " "; + }}, + {"encryption-key-raw", [&]() { + std::cout << std::setfill('0') << std::hex << std::setw(8) << + GetFileInfo(hFile, SFileInfoEncryptionKeyRaw) << + std::setfill(' ') << std::dec << " "; + }}, + }; + + for (const auto& prop : propertiesToPrint) { + for (const auto &[key, action]: propertyActions) { + if (prop == key) { + action(); // Print property + } + } + } + + std::cout << " " << findData.cFileName << std::endl; SFileCloseFile(hFile); } else { diff --git a/src/mpq.h b/src/mpq.h index ee3226b..2dccb21 100644 --- a/src/mpq.h +++ b/src/mpq.h @@ -17,7 +17,7 @@ HANDLE CreateMpqArchive(std::string outputArchiveName, int32_t fileCount, int32_ int AddFiles(HANDLE hArchive, const std::string& inputPath); int AddFile(HANDLE hArchive, fs::path localFile, const std::string& archiveFilePath); int RemoveFile(HANDLE hArchive, const std::string& archiveFilePath); -int ListFiles(HANDLE hHandle, const std::string &listfileName, bool listAll, bool listDetailed); +int ListFiles(HANDLE hHandle, const std::string &listfileName, bool listAll, bool listDetailed, std::vector& propertiesToPrint); char* ReadFile(HANDLE hArchive, const char *szFileName, unsigned int *fileSize); void PrintMpqInfo(HANDLE hArchive, const std::string& infoProperty); uint32_t VerifyMpqArchive(HANDLE hArchive); diff --git a/test/test_list.py b/test/test_list.py index 6ce3141..563b8d3 100644 --- a/test/test_list.py +++ b/test/test_list.py @@ -4,6 +4,12 @@ def test_list_mpq_with_output_v1(binary_path): + """ + Test MPQ file listing with no parameters. + + This test checks: + - That running the list command with no parameters renders a list of the files inside, with no details. + """ script_dir = Path(__file__).parent test_file = script_dir / "data" / "mpq_with_output_v1.mpq" @@ -26,25 +32,30 @@ def test_list_mpq_with_output_v1(binary_path): assert output_lines == expected_output, f"Unexpected output: {output_lines}" -def test_list_mpq_with_detailed(binary_path): +def test_list_mpq_with_standard_details(binary_path): + """ + Test MPQ file listing with the standard details. + + This test checks: + - That the standard long listing parameters work as expected. + """ script_dir = Path(__file__).parent test_file = script_dir / "data" / "mpq_with_output_v1.mpq" - # Update expected_out to match long listing format expected_output = { - " 27 enUS (listfile)", - " 148 enUS (attributes)", - " 27 enUS 2025-07-29 14:31:00 dogs.txt", - " 8 enUS 2025-07-29 14:31:00 bytes", - " 27 enUS 2025-07-29 14:31:00 cats.txt", + " 27 enUS (listfile)", + " 148 enUS (attributes)", + " 27 enUS 2025-07-29 14:31:00 dogs.txt", + " 8 enUS 2025-07-29 14:31:00 bytes", + " 27 enUS 2025-07-29 14:31:00 cats.txt", } # Adjust filesize for Windows if platform.system() == "Windows": - expected_output.remove(" 27 enUS 2025-07-29 14:31:00 dogs.txt") - expected_output.add(" 28 enUS 2025-07-29 14:31:00 dogs.txt") - expected_output.remove(" 27 enUS 2025-07-29 14:31:00 cats.txt") - expected_output.add(" 28 enUS 2025-07-29 14:31:00 cats.txt") + expected_output.remove(" 27 enUS 2025-07-29 14:31:00 dogs.txt") + expected_output.add(" 28 enUS 2025-07-29 14:31:00 dogs.txt") + expected_output.remove(" 27 enUS 2025-07-29 14:31:00 cats.txt") + expected_output.add(" 28 enUS 2025-07-29 14:31:00 cats.txt") result = subprocess.run( [str(binary_path), "list", "-a", "-d", str(test_file)], @@ -59,7 +70,78 @@ def test_list_mpq_with_detailed(binary_path): assert output_lines == expected_output, f"Unexpected output: {output_lines}" +def test_list_mpq_with_specified_details(binary_path): + """ + Test MPQ file listing with specified details. + + This test checks: + - That providing parameters to be listed works as expected. + """ + script_dir = Path(__file__).parent + test_file = script_dir / "data" / "mpq_with_output_v1.mpq" + + expected_output = { + " 44 0fd58937 70ab788e 0000000000000000 1 43 35 cexmn 935a7772 935a7772 cats.txt", + " 0 eb30456b 48345fbb 0000000000000000 0 20 35 cexmn a073c614 a073c614 dogs.txt", + " 35 147178ed c99b9ee2 0000000000000000 2 66 16 cexmn eaa753f9 eaa753f9 bytes", + " 25 fd657910 4e9b98a7 0000000000000000 3 76 35 ce2xmnf 2d2f0a94 2d2f0b11 (listfile)", + " 14 d38437cb 07dfeaec 0000000000000000 4 99 123 ce2xmnf 50e314af 50e315dc (attributes)", + } + + result = subprocess.run( + [str(binary_path), "list", "-a", "-d", str(test_file), + "-p", "hash-index", "-p", "name-hash1", "-p", "name-hash2", "-p", "name-hash3", "-p", "file-index", + "-p", "byte-offset", "-p", "compressed-size", "-p", "flags", "-p", "encryption-key-raw", "-p", "encryption-key"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + output_lines = set(result.stdout.splitlines()) + + assert result.returncode == 0, f"mpqcli failed with error: {result.stderr}" + assert output_lines == expected_output, f"Unexpected output: {output_lines}" + + +def test_list_mpq_with_specified_details_but_no_detailed_flag(binary_path): + """ + Test MPQ file listing. + + This test checks: + - That providing parameters to be listed does nothing if the -d/--detailed flag is not provided. + """ + script_dir = Path(__file__).parent + test_file = script_dir / "data" / "mpq_with_output_v1.mpq" + + expected_output = { + "cats.txt", + "dogs.txt", + "bytes", + "(listfile)", + "(attributes)", + } + + result = subprocess.run( + [str(binary_path), "list", "-a", str(test_file), + "-p", "hash-index", "-p", "flags", "-p", "encryption-key-raw", "-p", "encryption-key"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + output_lines = set(result.stdout.splitlines()) + + assert result.returncode == 0, f"mpqcli failed with error: {result.stderr}" + assert output_lines == expected_output, f"Unexpected output: {output_lines}" + + def test_list_mpq_with_weak_signature(binary_path): + """ + Test MPQ file listing of MPQ with weak signature. + + This test checks: + - That handling MPQs with weak signatures generates the expected output. + """ script_dir = Path(__file__).parent test_file = script_dir / "data" / "mpq_with_weak_signature.mpq" From 4ef8dd86a700ae14e5f9c3a3bacac5c60d9c01d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Sj=C3=B6blom?= Date: Wed, 22 Oct 2025 09:34:44 +0200 Subject: [PATCH 2/2] Fixing test by not displaying platform specific properties and adjusting file size --- test/test_list.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/test/test_list.py b/test/test_list.py index 563b8d3..6483e19 100644 --- a/test/test_list.py +++ b/test/test_list.py @@ -43,19 +43,19 @@ def test_list_mpq_with_standard_details(binary_path): test_file = script_dir / "data" / "mpq_with_output_v1.mpq" expected_output = { - " 27 enUS (listfile)", - " 148 enUS (attributes)", - " 27 enUS 2025-07-29 14:31:00 dogs.txt", - " 8 enUS 2025-07-29 14:31:00 bytes", - " 27 enUS 2025-07-29 14:31:00 cats.txt", + " 27 enUS (listfile)", + " 148 enUS (attributes)", + " 27 enUS 2025-07-29 14:31:00 dogs.txt", + " 8 enUS 2025-07-29 14:31:00 bytes", + " 27 enUS 2025-07-29 14:31:00 cats.txt", } # Adjust filesize for Windows if platform.system() == "Windows": - expected_output.remove(" 27 enUS 2025-07-29 14:31:00 dogs.txt") - expected_output.add(" 28 enUS 2025-07-29 14:31:00 dogs.txt") - expected_output.remove(" 27 enUS 2025-07-29 14:31:00 cats.txt") - expected_output.add(" 28 enUS 2025-07-29 14:31:00 cats.txt") + expected_output.remove(" 27 enUS 2025-07-29 14:31:00 dogs.txt") + expected_output.add(" 28 enUS 2025-07-29 14:31:00 dogs.txt") + expected_output.remove(" 27 enUS 2025-07-29 14:31:00 cats.txt") + expected_output.add(" 28 enUS 2025-07-29 14:31:00 cats.txt") result = subprocess.run( [str(binary_path), "list", "-a", "-d", str(test_file)], @@ -81,17 +81,23 @@ def test_list_mpq_with_specified_details(binary_path): test_file = script_dir / "data" / "mpq_with_output_v1.mpq" expected_output = { - " 44 0fd58937 70ab788e 0000000000000000 1 43 35 cexmn 935a7772 935a7772 cats.txt", - " 0 eb30456b 48345fbb 0000000000000000 0 20 35 cexmn a073c614 a073c614 dogs.txt", - " 35 147178ed c99b9ee2 0000000000000000 2 66 16 cexmn eaa753f9 eaa753f9 bytes", - " 25 fd657910 4e9b98a7 0000000000000000 3 76 35 ce2xmnf 2d2f0a94 2d2f0b11 (listfile)", - " 14 d38437cb 07dfeaec 0000000000000000 4 99 123 ce2xmnf 50e314af 50e315dc (attributes)", + " 44 0fd58937 70ab788e 0000000000000000 35 cexmn 935a7772 cats.txt", + " 0 eb30456b 48345fbb 0000000000000000 35 cexmn a073c614 dogs.txt", + " 35 147178ed c99b9ee2 0000000000000000 16 cexmn eaa753f9 bytes", + " 25 fd657910 4e9b98a7 0000000000000000 35 ce2xmnf 2d2f0a94 (listfile)", + " 14 d38437cb 07dfeaec 0000000000000000 123 ce2xmnf 50e314af (attributes)", } + # Adjust filesize for Windows + if platform.system() == "Windows": + expected_output.remove(" 44 0fd58937 70ab788e 0000000000000000 35 cexmn 935a7772 cats.txt") + expected_output.add(" 44 0fd58937 70ab788e 0000000000000000 36 cexmn 935a7772 cats.txt") + expected_output.remove(" 0 eb30456b 48345fbb 0000000000000000 35 cexmn a073c614 dogs.txt") + expected_output.add(" 0 eb30456b 48345fbb 0000000000000000 36 cexmn a073c614 dogs.txt") result = subprocess.run( [str(binary_path), "list", "-a", "-d", str(test_file), - "-p", "hash-index", "-p", "name-hash1", "-p", "name-hash2", "-p", "name-hash3", "-p", "file-index", - "-p", "byte-offset", "-p", "compressed-size", "-p", "flags", "-p", "encryption-key-raw", "-p", "encryption-key"], + "-p", "hash-index", "-p", "name-hash1", "-p", "name-hash2", "-p", "name-hash3", + "-p", "compressed-size", "-p", "flags", "-p", "encryption-key-raw"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True