diff --git a/CASCEdit/CASContainer.cs b/CASCEdit/CASContainer.cs index 32329e5..6bf4714 100644 --- a/CASCEdit/CASContainer.cs +++ b/CASCEdit/CASContainer.cs @@ -193,7 +193,7 @@ public static void OpenEncoding() } } - public static void OpenRoot(LocaleFlags locale, uint minimumid = 0) + public static void OpenRoot(LocaleFlags locale, uint minimumid = 0, bool onlineListfile = false) { Logger.LogInformation("Loading Root..."); @@ -208,7 +208,7 @@ public static void OpenRoot(LocaleFlags locale, uint minimumid = 0) if (idxInfo != null) { var path = Path.Combine(BasePath, "Data", "data", string.Format("data.{0:D3}", idxInfo.Archive)); - RootHandler = new RootHandler(DataHandler.Read(path, idxInfo), locale, minimumid); + RootHandler = new RootHandler(DataHandler.Read(path, idxInfo), locale, minimumid, onlineListfile); } else { @@ -223,7 +223,7 @@ public static void OpenRoot(LocaleFlags locale, uint minimumid = 0) Logger.LogCritical($"Unable to download Root {key}."); } - RootHandler = new RootHandler(DataHandler.ReadDirect(path), locale, minimumid); + RootHandler = new RootHandler(DataHandler.ReadDirect(path), locale, minimumid, onlineListfile); } } diff --git a/CASCEdit/Handlers/ArchiveGroupIndex.cs b/CASCEdit/Handlers/ArchiveGroupIndex.cs index d8438d1..30aa607 100644 --- a/CASCEdit/Handlers/ArchiveGroupIndex.cs +++ b/CASCEdit/Handlers/ArchiveGroupIndex.cs @@ -72,12 +72,11 @@ public void Read() Footer = new IndexFooter() { - IndexBlockHash = br.ReadBytes(8), TOCHash = br.ReadBytes(8), Version = br.ReadByte(), _11 = br.ReadByte(), _12 = br.ReadByte(), - _13 = br.ReadByte(), + BlockSizeKb = br.ReadByte(), Offset = br.ReadByte(), Size = br.ReadByte(), KeySize = br.ReadByte(), @@ -155,7 +154,7 @@ public string Write() bw.Write(Footer.Version); bw.Write(Footer._11); bw.Write(Footer._12); - bw.Write(Footer._13); + bw.Write(Footer.BlockSizeKb); bw.Write(Footer.Offset); bw.Write(Footer.Size); bw.Write(Footer.KeySize); diff --git a/CASCEdit/Handlers/ArchiveIndex.cs b/CASCEdit/Handlers/ArchiveIndex.cs index c5c8b58..68e58b3 100644 --- a/CASCEdit/Handlers/ArchiveIndex.cs +++ b/CASCEdit/Handlers/ArchiveIndex.cs @@ -60,23 +60,22 @@ private void Read() br.BaseStream.Position += blockcount * 16; //Block hashes - lower_md5 all blocks except last - br.BaseStream.Position += (blockcount - 1) * 8; + br.BaseStream.Position += blockcount * 8; //Footer Footer = new IndexFooter() { - IndexBlockHash = br.ReadBytes(8), - TOCHash = br.ReadBytes(8), - Version = br.ReadByte(), + TOCHash = br.ReadBytes(8), + Version = br.ReadByte(), _11 = br.ReadByte(), _12 = br.ReadByte(), - _13 = br.ReadByte(), + BlockSizeKb = br.ReadByte(), Offset = br.ReadByte(), Size = br.ReadByte(), KeySize = br.ReadByte(), ChecksumSize = br.ReadByte(), EntryCount = br.ReadUInt32(), - FooterMD5 = br.ReadBytes(8) + FooterMD5 = br.ReadBytes(Footer.ChecksumSize) }; } } @@ -102,11 +101,11 @@ public string Write() if (pos + entry.EntrySize > CHUNK_SIZE) { - entryHashes.Add(Entries[i - 1].EKey.Value); //Last entry hash + entryHashes.Add(Entries[i - 1].EKey.Value); //Last entry hash - bw.Write(new byte[CHUNK_SIZE - pos]); - blockHashes.Add(GetBlockHash(bw, md5)); - pos = 0; + bw.Write(new byte[CHUNK_SIZE - pos]); + blockHashes.Add(GetBlockHash(bw, md5)); + pos = 0; } entry.Offset = offset; @@ -130,23 +129,23 @@ public string Write() //Block hashes if (blockHashes.Count > 0) { - bw.Write(blockHashes.SelectMany(x => x).ToArray()); - blockHashes.Clear(); + bw.Write(blockHashes.SelectMany(x => x).ToArray()); + blockHashes.Clear(); } //Footer Start long posFooterStart = bw.BaseStream.Position; - //Calculate IndexBlockHash - bw.BaseStream.Position = CHUNK_SIZE * (blockCount - 1); - byte[] indexBlockHashBytes = new byte[CHUNK_SIZE]; - bw.BaseStream.Read(indexBlockHashBytes, 0, indexBlockHashBytes.Length); - bw.BaseStream.Position = posFooterStart; - var lowerHash = md5.ComputeHash(indexBlockHashBytes).Take(8).ToArray(); - bw.Write(lowerHash); + //Calculate IndexBlockHash + bw.BaseStream.Position = CHUNK_SIZE * (blockCount - 1); + byte[] indexBlockHashBytes = new byte[CHUNK_SIZE]; + bw.BaseStream.Read(indexBlockHashBytes, 0, indexBlockHashBytes.Length); + bw.BaseStream.Position = posFooterStart; + var lowerHash = md5.ComputeHash(indexBlockHashBytes).Take(8).ToArray(); + bw.Write(lowerHash); - //Calculate TOCHash - bw.BaseStream.Position = CHUNK_SIZE * blockCount; + //Calculate TOCHash + bw.BaseStream.Position = CHUNK_SIZE * blockCount; byte[] tocHashBytes = new byte[8 + posFooterStart - bw.BaseStream.Position]; bw.BaseStream.Read(tocHashBytes, 0, tocHashBytes.Length); var upperHash = md5.ComputeHash(tocHashBytes).Take(8).ToArray(); @@ -156,7 +155,7 @@ public string Write() bw.Write(Footer.Version); bw.Write(Footer._11); bw.Write(Footer._12); - bw.Write(Footer._13); + bw.Write(Footer.BlockSizeKb); bw.Write(Footer.Offset); bw.Write(Footer.Size); bw.Write(Footer.KeySize); @@ -186,7 +185,8 @@ public string Write() CASContainer.CDNConfig["archives"].RemoveAll(x => x == Path.GetFileNameWithoutExtension(BaseFile)); CASContainer.CDNConfig["archives"].Add(filename); CASContainer.CDNConfig["archives"].Sort(new HashComparer()); - + CASContainer.Logger.LogInformation($"Index Hash Position: {CASContainer.CDNConfig["archives"].IndexOf(filename).ToString()} Index Size: {ms.Length}"); + CASContainer.CDNConfig["archives-index-size"].Insert(CASContainer.CDNConfig["archives"].IndexOf(filename) - 1, ms.Length.ToString()); return path; } } diff --git a/CASCEdit/Handlers/RootHandler.cs b/CASCEdit/Handlers/RootHandler.cs index 93d1004..85c85f4 100644 --- a/CASCEdit/Handlers/RootHandler.cs +++ b/CASCEdit/Handlers/RootHandler.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using System.Diagnostics; using CASCEdit.IO; +using System.Net; namespace CASCEdit.Handlers { @@ -24,20 +25,50 @@ public class RootHandler : IDisposable private readonly uint minimumId; private readonly EncodingMap encodingMap; - public RootHandler() + private int namedFiles = 0; + private int allFiles = 0; + private int parsedFiles = 0; + private const int headerMagic = 0x4D465354; // MFST + + + private Dictionary ListFile = new Dictionary(); + private WebClient ListFileClient = new WebClient(); + + + public RootHandler() { GlobalRoot = new RootChunk() { ContentFlags = ContentFlags.None, LocaleFlags = LocaleFlags.All_WoW }; encodingMap = new EncodingMap(EncodingType.ZLib, 9); } - public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0) + public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0, bool onlineListfile = false) { this.minimumId = minimumid; this.locale = locale; - - BinaryReader stream = new BinaryReader(data); - - long length = stream.BaseStream.Length; + string cdnPath = Helper.GetCDNPath("listfile.csv"); + + if (!(File.Exists(Path.Combine(CASContainer.Settings.OutputPath, cdnPath))) && onlineListfile) + { + CASContainer.Logger.LogInformation("Downloading listfile from WoW.Tools"); + ListFileClient.DownloadFile("https://wow.tools/casc/listfile/download/csv/unverified", cdnPath); + } + + BinaryReader stream = new BinaryReader(data); + + // 8.2 root change + int magic = stream.ReadInt32(); + bool newFormat = magic == headerMagic; + if (newFormat) + { + allFiles = stream.ReadInt32(); + namedFiles = stream.ReadInt32(); + } + else + { + stream.BaseStream.Position = 0; + } + + long length = stream.BaseStream.Length; while (stream.BaseStream.Position < length) { RootChunk chunk = new RootChunk() @@ -47,8 +78,10 @@ public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0) LocaleFlags = (LocaleFlags)stream.ReadUInt32(), }; - // set the global root - if (chunk.LocaleFlags == LocaleFlags.All_WoW && chunk.ContentFlags == ContentFlags.None) + parsedFiles += (int)chunk.Count; + + // set the global root + if (chunk.LocaleFlags == LocaleFlags.All_WoW && chunk.ContentFlags == ContentFlags.None) GlobalRoot = chunk; uint fileDataIndex = 0; @@ -66,14 +99,41 @@ public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0) chunk.Entries.Add(entry); } - foreach (var entry in chunk.Entries) - { - entry.CEKey = new MD5Hash(stream); - entry.NameHash = stream.ReadUInt64(); - maxId = Math.Max(maxId, entry.FileDataId); - } - - Chunks.Add(chunk); + if (newFormat) + { + foreach (var entry in chunk.Entries) + { + entry.CEKey = new MD5Hash(stream); + maxId = Math.Max(maxId, entry.FileDataId); + } + + if (parsedFiles > allFiles - namedFiles) + { + foreach (var entry in chunk.Entries) + { + entry.NameHash = stream.ReadUInt64(); + } + } + else // no namehash + { + foreach (var entry in chunk.Entries) + { + entry.NameHash = 0; + } + } + + } + else + { + foreach (var entry in chunk.Entries) + { + entry.CEKey = new MD5Hash(stream); + entry.NameHash = stream.ReadUInt64(); + maxId = Math.Max(maxId, entry.FileDataId); + } + } + + Chunks.Add(chunk); } if (GlobalRoot == null) @@ -82,8 +142,35 @@ public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0) return; } - // set maxid from cache - maxId = Math.Max(Math.Max(maxId, minimumid), CASContainer.Settings.Cache?.MaxId ?? 0); + // use listfile to assign names + var listFileLines = File.ReadAllLines(cdnPath); + foreach (var listFileData in listFileLines) + { + var splitData = listFileData.Split(';'); + + if (splitData.Length != 2) + continue; + + if (!uint.TryParse(splitData[0], out uint listFileDataID)) + continue; + + ListFile[listFileDataID] = new Jenkins96().ComputeHash(splitData[1]); + } + + foreach (var chunk in Chunks) + { + foreach (var entry in chunk.Entries) + { + if (entry.NameHash == 0) + { + if (ListFile.ContainsKey(entry.FileDataId)) + entry.NameHash = ListFile[entry.FileDataId]; + } + } + } + + // set maxid from cache + maxId = Math.Max(Math.Max(maxId, minimumid), CASContainer.Settings.Cache?.MaxId ?? 0); // store encoding map encodingMap = (data as BLTEStream)?.EncodingMap.FirstOrDefault() ?? new EncodingMap(EncodingType.ZLib, 9); @@ -120,7 +207,8 @@ public void AddEntry(string path, CASResult file) .SelectMany(chunk => chunk.Entries) // Flatten the array to get all entries within all matching chunks .Where(e => e.NameHash == namehash); - if (entries.Count() == 0) { // New file, we need to create an entry for it + if (entries.Count() == 0) + { // New file, we need to create an entry for it var cached = cache.Entries.FirstOrDefault(x => x.Path == path); var fileDataId = Math.Max(maxId + 1, minimumId); @@ -138,8 +226,13 @@ public void AddEntry(string path, CASResult file) GlobalRoot.Entries.Add(entry); // Insert into the Global Root maxId = Math.Max(entry.FileDataId, maxId); // Update the max id - } else { // Existing file, we just have to update the data hash - foreach (var entry in entries) { + + cache?.AddOrUpdate(new CacheEntry(entry, file.EKey)); // If not done, sometimes files will not be added. + } + else + { // Existing file, we just have to update the data hash + foreach (var entry in entries) + { entry.CEKey = file.CEKey; entry.Path = path; diff --git a/CASCEdit/Structs/IndexStructs.cs b/CASCEdit/Structs/IndexStructs.cs index 705d4c0..48f4ce2 100644 --- a/CASCEdit/Structs/IndexStructs.cs +++ b/CASCEdit/Structs/IndexStructs.cs @@ -26,12 +26,11 @@ public class IndexEntry public class IndexFooter { - public byte[] IndexBlockHash; // ChecksumSize public byte[] TOCHash; // ChecksumSize public byte Version = 1; public byte _11 = 0; public byte _12 = 0; - public byte _13 = 4; + public byte BlockSizeKb = 4; public byte Offset = 4; public byte Size = 4; public byte KeySize = 16; diff --git a/CASCEdit/Structs/RootStructs.cs b/CASCEdit/Structs/RootStructs.cs index ea5057f..7ee83b3 100644 --- a/CASCEdit/Structs/RootStructs.cs +++ b/CASCEdit/Structs/RootStructs.cs @@ -41,7 +41,7 @@ public enum ContentFlags : uint F00000008 = 0x8, // added in 7.2.0.23436 F00000010 = 0x10, // added in 7.2.0.23436 LowViolence = 0x80, // many models have this flag - F10000000 = 0x10000000, + NoNames = 0x10000000, F20000000 = 0x20000000, // added in 21737 Bundle = 0x40000000, NoCompression = 0x80000000 // sounds have this flag diff --git a/CASCHost/AppSettings.cs b/CASCHost/AppSettings.cs index e1a4e84..bfca81c 100644 --- a/CASCHost/AppSettings.cs +++ b/CASCHost/AppSettings.cs @@ -13,7 +13,8 @@ namespace CASCHost public class AppSettings { public uint MinimumFileDataId { get; set; } // the minimum file id for new files - public bool BNetAppSupport { get; set; } = false; // create install and download files? + public bool OnlineListFile { get; set; } = false; // fetchs from WoW.Tools the lastest listfile + public bool BNetAppSupport { get; set; } = false; // create install and download files? public bool StaticMode { get; set; } = false; // Build CDN file struct public string RebuildPassword { get; set; } = ""; diff --git a/CASCHost/CASCHost.csproj b/CASCHost/CASCHost.csproj index eb43f2d..a8d6731 100644 --- a/CASCHost/CASCHost.csproj +++ b/CASCHost/CASCHost.csproj @@ -3,6 +3,9 @@ netcoreapp2.1 AnyCPU;x64 + + Exe + diff --git a/CASCHost/DataWatcher.cs b/CASCHost/DataWatcher.cs index 16d9df8..961b9a3 100644 --- a/CASCHost/DataWatcher.cs +++ b/CASCHost/DataWatcher.cs @@ -145,7 +145,7 @@ private void UpdateCASCDirectory(object obj) CASContainer.Open(settings); CASContainer.OpenCdnIndices(false); CASContainer.OpenEncoding(); - CASContainer.OpenRoot(settings.Locale, Startup.Settings.MinimumFileDataId); + CASContainer.OpenRoot(settings.Locale, Startup.Settings.MinimumFileDataId, Startup.Settings.OnlineListFile); if(Startup.Settings.BNetAppSupport) // these are only needed by the bnet app launcher { diff --git a/CASCHost/appsettings.json b/CASCHost/appsettings.json index eac9599..320998f 100644 --- a/CASCHost/appsettings.json +++ b/CASCHost/appsettings.json @@ -1,6 +1,7 @@ { "AppSettings": { "MinimumFileDataId": 2500000, + "OnlineListFile": false, "BNetAppSupport": false, "StaticMode": false, "RebuildPassword": "", @@ -10,8 +11,8 @@ "PatchUrl": "http://us.patch.battle.net:1119/wow", "Locale": "enus", "DirectoryHash": [ - "b02eac153ad91adee3c24c9d80c114f9", - "68a88f97705094a9c5a3a6decf08b5ab" + "0aa8acdad03fa93a760d6a5824e1f0a3", + "89a7085aa643b8a200eef77c25d20439" ] } } \ No newline at end of file