From af4cd0a64aed23ea6cc52f0f5b63043a76ca87dd Mon Sep 17 00:00:00 2001 From: MaxtorCoder Date: Sun, 14 Apr 2019 21:29:52 +0200 Subject: [PATCH 1/4] Fixed #23 TODO: Fix Entry adding in Database --- CASCEdit/Handlers/ArchiveGroupIndex.cs | 5 ++- CASCEdit/Handlers/ArchiveIndex.cs | 46 +++++++++++++------------- CASCEdit/Handlers/RootHandler.cs | 10 ++++-- CASCEdit/Structs/IndexStructs.cs | 3 +- CASCHost/CASCHost.csproj | 3 ++ CASCHost/appsettings.json | 4 +-- 6 files changed, 38 insertions(+), 33 deletions(-) diff --git a/CASCEdit/Handlers/ArchiveGroupIndex.cs b/CASCEdit/Handlers/ArchiveGroupIndex.cs index d8438d1..30aa607 100644 --- a/CASCEdit/Handlers/ArchiveGroupIndex.cs +++ b/CASCEdit/Handlers/ArchiveGroupIndex.cs @@ -72,12 +72,11 @@ public void Read() Footer = new IndexFooter() { - IndexBlockHash = br.ReadBytes(8), TOCHash = br.ReadBytes(8), Version = br.ReadByte(), _11 = br.ReadByte(), _12 = br.ReadByte(), - _13 = br.ReadByte(), + BlockSizeKb = br.ReadByte(), Offset = br.ReadByte(), Size = br.ReadByte(), KeySize = br.ReadByte(), @@ -155,7 +154,7 @@ public string Write() bw.Write(Footer.Version); bw.Write(Footer._11); bw.Write(Footer._12); - bw.Write(Footer._13); + bw.Write(Footer.BlockSizeKb); bw.Write(Footer.Offset); bw.Write(Footer.Size); bw.Write(Footer.KeySize); diff --git a/CASCEdit/Handlers/ArchiveIndex.cs b/CASCEdit/Handlers/ArchiveIndex.cs index c5c8b58..f34e6db 100644 --- a/CASCEdit/Handlers/ArchiveIndex.cs +++ b/CASCEdit/Handlers/ArchiveIndex.cs @@ -16,7 +16,7 @@ public class ArchiveIndexHandler public IndexFooter Footer = new IndexFooter(); public readonly string BaseFile; - const int CHUNK_SIZE = 0x1000; + const int CHUNK_SIZE = 4096; public ArchiveIndexHandler(string path = "") { @@ -65,18 +65,17 @@ private void Read() //Footer Footer = new IndexFooter() { - IndexBlockHash = br.ReadBytes(8), - TOCHash = br.ReadBytes(8), - Version = br.ReadByte(), + TOCHash = br.ReadBytes(8), + Version = br.ReadByte(), _11 = br.ReadByte(), _12 = br.ReadByte(), - _13 = br.ReadByte(), + BlockSizeKb = br.ReadByte(), Offset = br.ReadByte(), Size = br.ReadByte(), KeySize = br.ReadByte(), ChecksumSize = br.ReadByte(), EntryCount = br.ReadUInt32(), - FooterMD5 = br.ReadBytes(8) + FooterMD5 = br.ReadBytes(Footer.ChecksumSize) }; } } @@ -102,11 +101,11 @@ public string Write() if (pos + entry.EntrySize > CHUNK_SIZE) { - entryHashes.Add(Entries[i - 1].EKey.Value); //Last entry hash + entryHashes.Add(Entries[i - 1].EKey.Value); //Last entry hash - bw.Write(new byte[CHUNK_SIZE - pos]); - blockHashes.Add(GetBlockHash(bw, md5)); - pos = 0; + bw.Write(new byte[CHUNK_SIZE - pos]); + blockHashes.Add(GetBlockHash(bw, md5)); + pos = 0; } entry.Offset = offset; @@ -130,23 +129,23 @@ public string Write() //Block hashes if (blockHashes.Count > 0) { - bw.Write(blockHashes.SelectMany(x => x).ToArray()); - blockHashes.Clear(); + bw.Write(blockHashes.SelectMany(x => x).ToArray()); + blockHashes.Clear(); } //Footer Start long posFooterStart = bw.BaseStream.Position; - //Calculate IndexBlockHash - bw.BaseStream.Position = CHUNK_SIZE * (blockCount - 1); - byte[] indexBlockHashBytes = new byte[CHUNK_SIZE]; - bw.BaseStream.Read(indexBlockHashBytes, 0, indexBlockHashBytes.Length); - bw.BaseStream.Position = posFooterStart; - var lowerHash = md5.ComputeHash(indexBlockHashBytes).Take(8).ToArray(); - bw.Write(lowerHash); + //Calculate IndexBlockHash + bw.BaseStream.Position = CHUNK_SIZE * (blockCount - 1); + byte[] indexBlockHashBytes = new byte[CHUNK_SIZE]; + bw.BaseStream.Read(indexBlockHashBytes, 0, indexBlockHashBytes.Length); + bw.BaseStream.Position = posFooterStart; + var lowerHash = md5.ComputeHash(indexBlockHashBytes).Take(8).ToArray(); + bw.Write(lowerHash); - //Calculate TOCHash - bw.BaseStream.Position = CHUNK_SIZE * blockCount; + //Calculate TOCHash + bw.BaseStream.Position = CHUNK_SIZE * blockCount; byte[] tocHashBytes = new byte[8 + posFooterStart - bw.BaseStream.Position]; bw.BaseStream.Read(tocHashBytes, 0, tocHashBytes.Length); var upperHash = md5.ComputeHash(tocHashBytes).Take(8).ToArray(); @@ -156,7 +155,7 @@ public string Write() bw.Write(Footer.Version); bw.Write(Footer._11); bw.Write(Footer._12); - bw.Write(Footer._13); + bw.Write(Footer.BlockSizeKb); bw.Write(Footer.Offset); bw.Write(Footer.Size); bw.Write(Footer.KeySize); @@ -186,7 +185,8 @@ public string Write() CASContainer.CDNConfig["archives"].RemoveAll(x => x == Path.GetFileNameWithoutExtension(BaseFile)); CASContainer.CDNConfig["archives"].Add(filename); CASContainer.CDNConfig["archives"].Sort(new HashComparer()); - + CASContainer.Logger.LogInformation($"Index Hash Position: {CASContainer.CDNConfig["archives"].IndexOf(filename).ToString()} Index Size: {ms.Length}"); + CASContainer.CDNConfig["archives-index-size"].Insert(CASContainer.CDNConfig["archives"].IndexOf(filename) - 1, ms.Length.ToString()); return path; } } diff --git a/CASCEdit/Handlers/RootHandler.cs b/CASCEdit/Handlers/RootHandler.cs index 93d1004..0c7b6b7 100644 --- a/CASCEdit/Handlers/RootHandler.cs +++ b/CASCEdit/Handlers/RootHandler.cs @@ -120,7 +120,8 @@ public void AddEntry(string path, CASResult file) .SelectMany(chunk => chunk.Entries) // Flatten the array to get all entries within all matching chunks .Where(e => e.NameHash == namehash); - if (entries.Count() == 0) { // New file, we need to create an entry for it + if (entries.Count() == 0) + { // New file, we need to create an entry for it var cached = cache.Entries.FirstOrDefault(x => x.Path == path); var fileDataId = Math.Max(maxId + 1, minimumId); @@ -138,8 +139,11 @@ public void AddEntry(string path, CASResult file) GlobalRoot.Entries.Add(entry); // Insert into the Global Root maxId = Math.Max(entry.FileDataId, maxId); // Update the max id - } else { // Existing file, we just have to update the data hash - foreach (var entry in entries) { + } + else + { // Existing file, we just have to update the data hash + foreach (var entry in entries) + { entry.CEKey = file.CEKey; entry.Path = path; diff --git a/CASCEdit/Structs/IndexStructs.cs b/CASCEdit/Structs/IndexStructs.cs index 705d4c0..48f4ce2 100644 --- a/CASCEdit/Structs/IndexStructs.cs +++ b/CASCEdit/Structs/IndexStructs.cs @@ -26,12 +26,11 @@ public class IndexEntry public class IndexFooter { - public byte[] IndexBlockHash; // ChecksumSize public byte[] TOCHash; // ChecksumSize public byte Version = 1; public byte _11 = 0; public byte _12 = 0; - public byte _13 = 4; + public byte BlockSizeKb = 4; public byte Offset = 4; public byte Size = 4; public byte KeySize = 16; diff --git a/CASCHost/CASCHost.csproj b/CASCHost/CASCHost.csproj index eb43f2d..a8d6731 100644 --- a/CASCHost/CASCHost.csproj +++ b/CASCHost/CASCHost.csproj @@ -3,6 +3,9 @@ netcoreapp2.1 AnyCPU;x64 + + Exe + diff --git a/CASCHost/appsettings.json b/CASCHost/appsettings.json index eac9599..c929bd8 100644 --- a/CASCHost/appsettings.json +++ b/CASCHost/appsettings.json @@ -10,8 +10,8 @@ "PatchUrl": "http://us.patch.battle.net:1119/wow", "Locale": "enus", "DirectoryHash": [ - "b02eac153ad91adee3c24c9d80c114f9", - "68a88f97705094a9c5a3a6decf08b5ab" + "0aa8acdad03fa93a760d6a5824e1f0a3", + "89a7085aa643b8a200eef77c25d20439" ] } } \ No newline at end of file From ee6bcadf699e546388c46eed60c8e9b3b5170702 Mon Sep 17 00:00:00 2001 From: MaxtorCoder Date: Mon, 15 Apr 2019 09:48:33 +0200 Subject: [PATCH 2/4] Fixed footer reading. --- CASCEdit/Handlers/ArchiveIndex.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CASCEdit/Handlers/ArchiveIndex.cs b/CASCEdit/Handlers/ArchiveIndex.cs index f34e6db..68e58b3 100644 --- a/CASCEdit/Handlers/ArchiveIndex.cs +++ b/CASCEdit/Handlers/ArchiveIndex.cs @@ -16,7 +16,7 @@ public class ArchiveIndexHandler public IndexFooter Footer = new IndexFooter(); public readonly string BaseFile; - const int CHUNK_SIZE = 4096; + const int CHUNK_SIZE = 0x1000; public ArchiveIndexHandler(string path = "") { @@ -60,7 +60,7 @@ private void Read() br.BaseStream.Position += blockcount * 16; //Block hashes - lower_md5 all blocks except last - br.BaseStream.Position += (blockcount - 1) * 8; + br.BaseStream.Position += blockcount * 8; //Footer Footer = new IndexFooter() From 506cd990efd9ff8e9ce99a98e1343cc079f9e5c4 Mon Sep 17 00:00:00 2001 From: RussianBias Date: Mon, 26 Aug 2019 13:55:13 +0200 Subject: [PATCH 3/4] 8.2 root change handling --- CASCEdit/Handlers/RootHandler.cs | 78 ++++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 14 deletions(-) diff --git a/CASCEdit/Handlers/RootHandler.cs b/CASCEdit/Handlers/RootHandler.cs index 0c7b6b7..a23530b 100644 --- a/CASCEdit/Handlers/RootHandler.cs +++ b/CASCEdit/Handlers/RootHandler.cs @@ -23,8 +23,14 @@ public class RootHandler : IDisposable private uint maxId = 0; private readonly uint minimumId; private readonly EncodingMap encodingMap; - - public RootHandler() + private int namedFiles = 0; + private int allFiles = 0; + private int parsedFiles = 0; + private const int headerMagic = 0x4D465354; // MFST + private const string customFileDataDirectory = "filedatas"; // we generate a custom namehash for files without one ( 8.2 root change ), therefore we need allow some options. + private const string customFileDataPrefix = "filedata_"; + + public RootHandler() { GlobalRoot = new RootChunk() { ContentFlags = ContentFlags.None, LocaleFlags = LocaleFlags.All_WoW }; encodingMap = new EncodingMap(EncodingType.ZLib, 9); @@ -37,7 +43,20 @@ public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0) BinaryReader stream = new BinaryReader(data); - long length = stream.BaseStream.Length; + // 8.2 root change + int magic = stream.ReadInt32(); + bool newFormat = magic == headerMagic; + if (newFormat) + { + allFiles = stream.ReadInt32(); + namedFiles = stream.ReadInt32(); + } + else + { + stream.BaseStream.Position = 0; + } + + long length = stream.BaseStream.Length; while (stream.BaseStream.Position < length) { RootChunk chunk = new RootChunk() @@ -47,8 +66,10 @@ public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0) LocaleFlags = (LocaleFlags)stream.ReadUInt32(), }; - // set the global root - if (chunk.LocaleFlags == LocaleFlags.All_WoW && chunk.ContentFlags == ContentFlags.None) + parsedFiles += (int)chunk.Count; + + // set the global root + if (chunk.LocaleFlags == LocaleFlags.All_WoW && chunk.ContentFlags == ContentFlags.None) GlobalRoot = chunk; uint fileDataIndex = 0; @@ -66,14 +87,41 @@ public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0) chunk.Entries.Add(entry); } - foreach (var entry in chunk.Entries) - { - entry.CEKey = new MD5Hash(stream); - entry.NameHash = stream.ReadUInt64(); - maxId = Math.Max(maxId, entry.FileDataId); - } - - Chunks.Add(chunk); + if (newFormat) + { + foreach (var entry in chunk.Entries) + { + entry.CEKey = new MD5Hash(stream); + maxId = Math.Max(maxId, entry.FileDataId); + } + + if (parsedFiles > allFiles - namedFiles) + { + foreach (var entry in chunk.Entries) + { + entry.NameHash = stream.ReadUInt64(); + } + } + else // no namehash, so we generate a custom one + { + foreach (var entry in chunk.Entries) + { + entry.NameHash = new Jenkins96().ComputeHash($"{customFileDataDirectory}/{customFileDataPrefix}" + entry.FileDataId); + } + } + + } + else + { + foreach (var entry in chunk.Entries) + { + entry.CEKey = new MD5Hash(stream); + entry.NameHash = stream.ReadUInt64(); + maxId = Math.Max(maxId, entry.FileDataId); + } + } + + Chunks.Add(chunk); } if (GlobalRoot == null) @@ -139,7 +187,9 @@ public void AddEntry(string path, CASResult file) GlobalRoot.Entries.Add(entry); // Insert into the Global Root maxId = Math.Max(entry.FileDataId, maxId); // Update the max id - } + + cache?.AddOrUpdate(new CacheEntry(entry, file.EKey)); // If not done, sometimes files will not be added. + } else { // Existing file, we just have to update the data hash foreach (var entry in entries) From 0b4a5c3ffd5ad9ff9982c223f4978c1d80049443 Mon Sep 17 00:00:00 2001 From: RussianBias Date: Sat, 5 Oct 2019 16:37:44 +0200 Subject: [PATCH 4/4] Use listfile for files without hash names. --- CASCEdit/CASContainer.cs | 6 ++-- CASCEdit/Handlers/RootHandler.cs | 55 +++++++++++++++++++++++++++----- CASCEdit/Structs/RootStructs.cs | 2 +- CASCHost/AppSettings.cs | 3 +- CASCHost/DataWatcher.cs | 2 +- CASCHost/appsettings.json | 1 + 6 files changed, 55 insertions(+), 14 deletions(-) diff --git a/CASCEdit/CASContainer.cs b/CASCEdit/CASContainer.cs index 32329e5..6bf4714 100644 --- a/CASCEdit/CASContainer.cs +++ b/CASCEdit/CASContainer.cs @@ -193,7 +193,7 @@ public static void OpenEncoding() } } - public static void OpenRoot(LocaleFlags locale, uint minimumid = 0) + public static void OpenRoot(LocaleFlags locale, uint minimumid = 0, bool onlineListfile = false) { Logger.LogInformation("Loading Root..."); @@ -208,7 +208,7 @@ public static void OpenRoot(LocaleFlags locale, uint minimumid = 0) if (idxInfo != null) { var path = Path.Combine(BasePath, "Data", "data", string.Format("data.{0:D3}", idxInfo.Archive)); - RootHandler = new RootHandler(DataHandler.Read(path, idxInfo), locale, minimumid); + RootHandler = new RootHandler(DataHandler.Read(path, idxInfo), locale, minimumid, onlineListfile); } else { @@ -223,7 +223,7 @@ public static void OpenRoot(LocaleFlags locale, uint minimumid = 0) Logger.LogCritical($"Unable to download Root {key}."); } - RootHandler = new RootHandler(DataHandler.ReadDirect(path), locale, minimumid); + RootHandler = new RootHandler(DataHandler.ReadDirect(path), locale, minimumid, onlineListfile); } } diff --git a/CASCEdit/Handlers/RootHandler.cs b/CASCEdit/Handlers/RootHandler.cs index a23530b..85c85f4 100644 --- a/CASCEdit/Handlers/RootHandler.cs +++ b/CASCEdit/Handlers/RootHandler.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using System.Diagnostics; using CASCEdit.IO; +using System.Net; namespace CASCEdit.Handlers { @@ -23,12 +24,16 @@ public class RootHandler : IDisposable private uint maxId = 0; private readonly uint minimumId; private readonly EncodingMap encodingMap; + private int namedFiles = 0; private int allFiles = 0; private int parsedFiles = 0; private const int headerMagic = 0x4D465354; // MFST - private const string customFileDataDirectory = "filedatas"; // we generate a custom namehash for files without one ( 8.2 root change ), therefore we need allow some options. - private const string customFileDataPrefix = "filedata_"; + + + private Dictionary ListFile = new Dictionary(); + private WebClient ListFileClient = new WebClient(); + public RootHandler() { @@ -36,12 +41,19 @@ public RootHandler() encodingMap = new EncodingMap(EncodingType.ZLib, 9); } - public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0) + public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0, bool onlineListfile = false) { this.minimumId = minimumid; this.locale = locale; + string cdnPath = Helper.GetCDNPath("listfile.csv"); + + if (!(File.Exists(Path.Combine(CASContainer.Settings.OutputPath, cdnPath))) && onlineListfile) + { + CASContainer.Logger.LogInformation("Downloading listfile from WoW.Tools"); + ListFileClient.DownloadFile("https://wow.tools/casc/listfile/download/csv/unverified", cdnPath); + } - BinaryReader stream = new BinaryReader(data); + BinaryReader stream = new BinaryReader(data); // 8.2 root change int magic = stream.ReadInt32(); @@ -102,11 +114,11 @@ public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0) entry.NameHash = stream.ReadUInt64(); } } - else // no namehash, so we generate a custom one + else // no namehash { foreach (var entry in chunk.Entries) { - entry.NameHash = new Jenkins96().ComputeHash($"{customFileDataDirectory}/{customFileDataPrefix}" + entry.FileDataId); + entry.NameHash = 0; } } @@ -130,8 +142,35 @@ public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0) return; } - // set maxid from cache - maxId = Math.Max(Math.Max(maxId, minimumid), CASContainer.Settings.Cache?.MaxId ?? 0); + // use listfile to assign names + var listFileLines = File.ReadAllLines(cdnPath); + foreach (var listFileData in listFileLines) + { + var splitData = listFileData.Split(';'); + + if (splitData.Length != 2) + continue; + + if (!uint.TryParse(splitData[0], out uint listFileDataID)) + continue; + + ListFile[listFileDataID] = new Jenkins96().ComputeHash(splitData[1]); + } + + foreach (var chunk in Chunks) + { + foreach (var entry in chunk.Entries) + { + if (entry.NameHash == 0) + { + if (ListFile.ContainsKey(entry.FileDataId)) + entry.NameHash = ListFile[entry.FileDataId]; + } + } + } + + // set maxid from cache + maxId = Math.Max(Math.Max(maxId, minimumid), CASContainer.Settings.Cache?.MaxId ?? 0); // store encoding map encodingMap = (data as BLTEStream)?.EncodingMap.FirstOrDefault() ?? new EncodingMap(EncodingType.ZLib, 9); diff --git a/CASCEdit/Structs/RootStructs.cs b/CASCEdit/Structs/RootStructs.cs index ea5057f..7ee83b3 100644 --- a/CASCEdit/Structs/RootStructs.cs +++ b/CASCEdit/Structs/RootStructs.cs @@ -41,7 +41,7 @@ public enum ContentFlags : uint F00000008 = 0x8, // added in 7.2.0.23436 F00000010 = 0x10, // added in 7.2.0.23436 LowViolence = 0x80, // many models have this flag - F10000000 = 0x10000000, + NoNames = 0x10000000, F20000000 = 0x20000000, // added in 21737 Bundle = 0x40000000, NoCompression = 0x80000000 // sounds have this flag diff --git a/CASCHost/AppSettings.cs b/CASCHost/AppSettings.cs index e1a4e84..bfca81c 100644 --- a/CASCHost/AppSettings.cs +++ b/CASCHost/AppSettings.cs @@ -13,7 +13,8 @@ namespace CASCHost public class AppSettings { public uint MinimumFileDataId { get; set; } // the minimum file id for new files - public bool BNetAppSupport { get; set; } = false; // create install and download files? + public bool OnlineListFile { get; set; } = false; // fetchs from WoW.Tools the lastest listfile + public bool BNetAppSupport { get; set; } = false; // create install and download files? public bool StaticMode { get; set; } = false; // Build CDN file struct public string RebuildPassword { get; set; } = ""; diff --git a/CASCHost/DataWatcher.cs b/CASCHost/DataWatcher.cs index 16d9df8..961b9a3 100644 --- a/CASCHost/DataWatcher.cs +++ b/CASCHost/DataWatcher.cs @@ -145,7 +145,7 @@ private void UpdateCASCDirectory(object obj) CASContainer.Open(settings); CASContainer.OpenCdnIndices(false); CASContainer.OpenEncoding(); - CASContainer.OpenRoot(settings.Locale, Startup.Settings.MinimumFileDataId); + CASContainer.OpenRoot(settings.Locale, Startup.Settings.MinimumFileDataId, Startup.Settings.OnlineListFile); if(Startup.Settings.BNetAppSupport) // these are only needed by the bnet app launcher { diff --git a/CASCHost/appsettings.json b/CASCHost/appsettings.json index c929bd8..320998f 100644 --- a/CASCHost/appsettings.json +++ b/CASCHost/appsettings.json @@ -1,6 +1,7 @@ { "AppSettings": { "MinimumFileDataId": 2500000, + "OnlineListFile": false, "BNetAppSupport": false, "StaticMode": false, "RebuildPassword": "",