diff --git a/TACTLib/Core/Product/Tank/AssetPackageManifest.cs b/TACTLib/Core/Product/Tank/AssetPackageManifest.cs index 804ac48..256f6e1 100644 --- a/TACTLib/Core/Product/Tank/AssetPackageManifest.cs +++ b/TACTLib/Core/Product/Tank/AssetPackageManifest.cs @@ -20,14 +20,14 @@ public struct APMHeader { public int m_entryCount; public uint m_checksum; // ? } - + [StructLayout(LayoutKind.Sequential, Pack = 4)] public struct Entry { public uint m_index; public ulong m_hashA; public ulong m_hashB; } - + [StructLayout(LayoutKind.Sequential, Pack = 4)] public struct PackageEntry { public ulong m_packageGUID; // 077 file @@ -35,30 +35,30 @@ public struct PackageEntry { public uint m_unknown1; public uint m_unknown2; } - + [StructLayout(LayoutKind.Sequential, Pack = 4)] public struct PackageHeader { - public long m_offsetRecords; // 0 + public long m_offsetRecords; // 0 public long m_offset8; // 8 public long m_offset16; // 16 public long m_offset24; // 24 - public long m_offsetBundles; // 32 + public long m_offsetBundles; // 32 public long m_offset40; // 40 - - public uint m_recordCount; // 48 + + public uint m_recordCount; // 48 public uint m_count52; // 52 - public uint m_count56; // 56 - public uint m_count60; // 60 - public uint m_count64; // 64 + public uint m_count56; // 56 + public uint m_count60; // 60 + public uint m_count64; // 64 public uint m_count68; // 68 - public uint m_bundleCount; // 72 + public uint m_bundleCount; // 72 } - [StructLayout(LayoutKind.Explicit, Pack = 1, Size = 12)] // size = 12 + [StructLayout(LayoutKind.Explicit, Pack = 1, Size = 12)] // size = 12 public struct PackageRecord { [FieldOffset(0)] public ulong m_GUID; [FieldOffset(8)] public uint m_flagsReal; - + [FieldOffset(8)] public short m_unknown1; [FieldOffset(10)] public byte m_unknown2; [FieldOffset(11)] public RecordFlags m_flags; @@ -77,25 +77,25 @@ public enum RecordFlags : byte { public PackageHeader[] m_packages; public PackageRecord[][] m_packageRecords; public ulong[][] m_packageBundles; - + public AssetPackageManifest(ClientHandler client, ProductHandler_Tank tankHandler, Stream stream, string name) { m_name = name; var cmf = tankHandler.m_rootContentManifest; - + using (BinaryReader reader = new BinaryReader(stream)) { m_header = reader.Read(); - if(m_header.m_buildVersion >= 12923648 || m_header.m_buildVersion < 52320) { + if (m_header.m_buildVersion >= 12923648 || m_header.m_buildVersion < 52320) { throw new NotSupportedException("Overwatch 1.29 or earlier is not supported"); } - + m_entries = reader.ReadArray(m_header.m_entryCount); m_packageEntries = reader.ReadArray(m_header.m_packageCount); if (!VerifyEntries(cmf)) { Logger.Debug("APM", "Entry hash invalid. IV may be wrong"); } - + m_packages = new PackageHeader[m_header.m_packageCount]; m_packageRecords = new PackageRecord[m_header.m_packageCount][]; m_packageBundles = new ulong[m_header.m_packageCount][]; @@ -108,15 +108,15 @@ public AssetPackageManifest(ClientHandler client, ProductHandler_Tank tankHandle c++; if (c % 1000 == 0) { if (!Console.IsOutputRedirected) { - Console.Out.Write($"Loading packages: {Math.Floor(c / (float)m_header.m_packageCount * 10000) / 100:F0}% ({c}/{m_header.m_packageCount})\r"); + Console.Out.Write($"Loading packages: {Math.Floor(c / (float) m_header.m_packageCount * 10000) / 100:F0}% ({c}/{m_header.m_packageCount})\r"); } } - + LoadPackage(i, tankHandler); }); - + if (!Console.IsOutputRedirected) { - Console.Write(new string(' ', Console.WindowWidth-1)+"\r"); + Console.Write(new string(' ', Console.WindowWidth - 1) + "\r"); } } } @@ -132,10 +132,10 @@ private void LoadPackage(int i, ProductHandler_Tank tankHandler) { m_packageRecords[i] = Array.Empty(); return; } - + if (package.m_bundleCount > 0) { packageStream.Position = package.m_offsetBundles; - m_packageBundles[i] = packageReader.ReadArray((int)package.m_bundleCount); + m_packageBundles[i] = packageReader.ReadArray((int) package.m_bundleCount); } packageStream.Position = package.m_offsetRecords; diff --git a/TACTLib/Core/Product/Tank/Bundle.cs b/TACTLib/Core/Product/Tank/Bundle.cs index 4a82d28..87b499b 100644 --- a/TACTLib/Core/Product/Tank/Bundle.cs +++ b/TACTLib/Core/Product/Tank/Bundle.cs @@ -14,35 +14,35 @@ public struct HeaderData { public int EntryCount; public int Flags; public byte OffsetSize; - + public HeaderData148 To148() => new HeaderData148 { EntryCount = EntryCount, OffsetSize = OffsetSize }; } - + [StructLayout(LayoutKind.Sequential, Pack = 1)] public struct HeaderData148 { public int EntryCount; public byte OffsetSize; } - + [StructLayout(LayoutKind.Sequential, Pack = 1, Size = 9)] public struct Entry1 { public ulong GUID; - public byte Offset; // 1 + public byte Offset; // 1 } [StructLayout(LayoutKind.Sequential, Pack = 1, Size = 10)] public struct Entry2 { public ulong GUID; - public ushort Offset; // 2 + public ushort Offset; // 2 } - + [StructLayout(LayoutKind.Sequential, Pack = 1, Size = 12)] public struct Entry4 { public ulong GUID; - public uint Offset; // 4 + public uint Offset; // 4 } public Bundle(Stream? stream, bool is148) { @@ -50,6 +50,7 @@ public Bundle(Stream? stream, bool is148) { Entries = Array.Empty(); return; } + using (BinaryReader reader = new BinaryReader(stream)) { if (is148) { Header = reader.Read(); @@ -75,6 +76,6 @@ public Bundle(Stream? stream, bool is148) { // this should never happen } } - } + } } -} +} \ No newline at end of file diff --git a/TACTLib/Core/Product/Tank/ContentManifestFile.cs b/TACTLib/Core/Product/Tank/ContentManifestFile.cs index afec6ae..504bd1d 100644 --- a/TACTLib/Core/Product/Tank/ContentManifestFile.cs +++ b/TACTLib/Core/Product/Tank/ContentManifestFile.cs @@ -14,12 +14,11 @@ public struct HashData : IComparable { // version 25? public byte Unknown; public CKey ContentKey; - public int CompareTo(HashData other) - { + public int CompareTo(HashData other) { return GUID.CompareTo(other.GUID); } } - + [StructLayout(LayoutKind.Sequential, Pack = 1)] public struct HashData24 { // version 24? public ulong GUID; @@ -60,7 +59,7 @@ public CMFHeader Upgrade() { } [StructLayout(LayoutKind.Sequential, Pack = 4)] - public struct CMFHeader { // 1.48+, version 26 + public struct CMFHeader { // 1.48+, version 26 public uint m_buildVersion; public uint m_unk04; public uint m_unk08; @@ -73,32 +72,29 @@ public struct CMFHeader { // 1.48+, version 26 public int m_entryPatchRecordCount; public int m_entryCount; public uint m_magic; - - public uint GetNonEncryptedMagic() - { - return (uint)(0x00666D63u | (GetVersion() << 24)); + + public uint GetNonEncryptedMagic() { + return (uint) (0x00666D63u | (GetVersion() << 24)); } - - public byte GetVersion() - { - return IsEncrypted() ? (byte)(m_magic & 0x000000FF) : (byte)((m_magic & 0xFF000000) >> 24); + + public byte GetVersion() { + return IsEncrypted() ? (byte) (m_magic & 0x000000FF) : (byte) ((m_magic & 0xFF000000) >> 24); } - - public bool IsEncrypted() - { + + public bool IsEncrypted() { return (m_magic >> 8) == ENCRYPTED_MAGIC; } } public string m_name; public CMFHeader m_header; - + public AssetPackageManifest.Entry[] m_entries = null!; public HashData[] m_hashList = null!; // ReSharper disable once InconsistentNaming public const int ENCRYPTED_MAGIC = 0x636D66; // todo: use the thingy again? - + public ContentManifestFile(ClientHandler client, Stream stream, string name) { m_name = name; using (BinaryReader reader = new BinaryReader(stream)) { @@ -107,8 +103,8 @@ public ContentManifestFile(ClientHandler client, Stream stream, string name) { stream.Position = 0; m_header = reader.Read().Upgrade(); } - - if(m_header.m_buildVersion >= 12923648 || m_header.m_buildVersion < 52320) { + + if (m_header.m_buildVersion >= 12923648 || m_header.m_buildVersion < 52320) { throw new NotSupportedException("Overwatch 1.29 or earlier is not supported"); } @@ -135,16 +131,16 @@ private void ParseEntries(BinaryReader reader) { } public bool TryGet(ulong guid, out HashData hashData) { - var speculativeEntry = new HashData - { + var speculativeEntry = new HashData { GUID = guid }; + var index = Array.BinarySearch(m_hashList, speculativeEntry); - if (index < 0 || index >= m_hashList.Length) - { + if (index < 0 || index >= m_hashList.Length) { hashData = default; return false; } + hashData = m_hashList[index]; return true; } @@ -157,6 +153,7 @@ public HashData GetHashData(ulong guid) { if (TryGet(guid, out var data)) { return data; } + throw new FileNotFoundException($"{guid:X16}"); } diff --git a/TACTLib/Core/Product/Tank/IManifestCrypto.cs b/TACTLib/Core/Product/Tank/IManifestCrypto.cs index 9d2445b..390572d 100644 --- a/TACTLib/Core/Product/Tank/IManifestCrypto.cs +++ b/TACTLib/Core/Product/Tank/IManifestCrypto.cs @@ -5,6 +5,6 @@ public interface IManifestCrypto { } public interface ICMFEncryptionProc : IManifestCrypto { } - + public interface ITRGEncryptionProc : IManifestCrypto { } -} +} \ No newline at end of file diff --git a/TACTLib/Core/Product/Tank/ManifestCryptoHandler.cs b/TACTLib/Core/Product/Tank/ManifestCryptoHandler.cs index 94b63ee..2b5c486 100644 --- a/TACTLib/Core/Product/Tank/ManifestCryptoHandler.cs +++ b/TACTLib/Core/Product/Tank/ManifestCryptoHandler.cs @@ -10,19 +10,20 @@ namespace TACTLib.Core.Product.Tank { public static class ManifestCryptoHandler { #region Helpers + // ReSharper disable once InconsistentNaming public const uint SHA1_DIGESTSIZE = 20; public static uint Constrain(long value) { - return (uint)(value % uint.MaxValue); + return (uint) (value % uint.MaxValue); } - public static int SignedMod(long p1, long p2) - { - var a = (int)p1; - var b = (int)p2; + public static int SignedMod(long p1, long p2) { + var a = (int) p1; + var b = (int) p2; return (a % b) < 0 ? (a % b + b) : (a % b); } + #endregion public static bool AttemptFallbackManifests = false; @@ -39,9 +40,11 @@ public static void GenerateKeyIV(string name, string manifestType, T header, if (!s_headerTypeToProviderType.TryGetValue(typeof(T), out var providerType)) { throw new InvalidDataException($"[Manifest]: Unable to get crypto provider for {typeof(T)}"); } + if (!Providers.TryGetValue(product, out var cryptoTypeMap)) { throw new InvalidDataException($"[Manifest]: {product} does not have any crypto providers?"); } + if (!cryptoTypeMap.TryGetValue(providerType, out var providerVersions)) { throw new InvalidDataException($"[Manifest]: {product} does not have any {providerType} providers?"); } @@ -106,6 +109,7 @@ public static Dictionary> GetManifestProvidersFor providerCryptoTypeMap = new Dictionary>(); Providers[product] = providerCryptoTypeMap; } + return providerCryptoTypeMap; } @@ -116,14 +120,14 @@ public static Dictionary GetManifestProvidersForProductAndInterfac typeVersionMap = new Dictionary(); providerCryptoTypeMap[t] = typeVersionMap; } + return typeVersionMap; } public static void AddProvider(TACTProduct product, Type @interface, object provider, uint buildVersion) { if (!s_headerTypeToProviderType.ContainsKey(@interface)) { - var thisInterface = @interface.GetInterfaces().First(x => - x.IsGenericType && - x.GetGenericTypeDefinition() == typeof(IManifestCrypto<>)); + var thisInterface = @interface.GetInterfaces().First(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(IManifestCrypto<>)); + s_headerTypeToProviderType[thisInterface.GetGenericArguments()[0]] = @interface; } @@ -139,7 +143,7 @@ public static void AddProvidersFromAssembly(Assembly asm) { var metadata = tt.GetCustomAttribute(); if (metadata == null) continue; - var provider = (T)Activator.CreateInstance(tt)!; + var provider = (T) Activator.CreateInstance(tt)!; if (metadata.AutoDetectVersion) { AddProvider(metadata.Product, typeof(T), provider, uint.Parse(tt.Name.Split('_')[1])); } diff --git a/TACTLib/Core/Product/Tank/ProductHandler_Tank.Bundles.cs b/TACTLib/Core/Product/Tank/ProductHandler_Tank.Bundles.cs index a7f3151..6d51ab4 100644 --- a/TACTLib/Core/Product/Tank/ProductHandler_Tank.Bundles.cs +++ b/TACTLib/Core/Product/Tank/ProductHandler_Tank.Bundles.cs @@ -11,14 +11,15 @@ private class BundleCache { public Dictionary m_offsets; public Memory m_buffer; } + private readonly Dictionary m_bundleCache = new Dictionary(); - + private readonly Dictionary? m_hackedBundleLookup; private readonly HashSet? m_hackedLookedUpBundles; private void DoBundleLookupHack() { if (!m_usingResourceGraph) return; - + foreach (var asset in m_assets) { if ((asset.Key & 0xFFF000000000000ul) != 0x0D90000000000000) continue; // bundles only if (m_hackedLookedUpBundles!.Contains(asset.Key)) continue; // already done @@ -39,10 +40,11 @@ private void DoBundleLookupHack() { if (bundle.Header.OffsetSize == 0) { throw new InvalidDataException($"failed to load bundle {asset.Key:X16}"); } - + foreach (var valuePair in bundle.Entries) { m_hackedBundleLookup![valuePair.GUID] = asset.Key; } + m_hackedLookedUpBundles.Add(asset.Key); } } @@ -61,20 +63,22 @@ private BundleCache GetBundleCache(ulong bundleGuid) { //using (Stream outStr = File.OpenWrite($"{bundleGuid:X16}.bndl")) { // bundleStream.CopyTo(outStr); //} - + bundle = new Bundle(bundleStream, m_usingResourceGraph); } + var offsetMap = bundle.Entries.ToDictionary(x => x.GUID, x => x.Offset); - + cache = new BundleCache { m_buffer = buf, m_offsets = offsetMap }; + m_bundleCache[bundleGuid] = cache; return cache; } } - + private Bundle OpenBundle(ulong bundleGuid) { using (var bundleStream = OpenFile(bundleGuid)) return new Bundle(bundleStream, m_usingResourceGraph); @@ -83,12 +87,12 @@ private Bundle OpenBundle(ulong bundleGuid) { private Stream? OpenFileFromBundle(ulong bundleGuid, ulong guid) { var cache = GetBundleCache(bundleGuid); if (!cache.m_offsets.TryGetValue(guid, out var offset)) return null; - + var hashData = GetHashData(guid); - var slice = cache.m_buffer.Slice((int)offset, (int)hashData.Size); + var slice = cache.m_buffer.Slice((int) offset, (int) hashData.Size); return new MemoryStream(slice.ToArray()); } - + /// /// Clears bundle cache /// @@ -98,4 +102,4 @@ public void WipeBundleCache() { } } } -} +} \ No newline at end of file diff --git a/TACTLib/Core/Product/Tank/ProductHandler_Tank.cs b/TACTLib/Core/Product/Tank/ProductHandler_Tank.cs index b4280ba..ca34168 100644 --- a/TACTLib/Core/Product/Tank/ProductHandler_Tank.cs +++ b/TACTLib/Core/Product/Tank/ProductHandler_Tank.cs @@ -24,7 +24,7 @@ public partial class ProductHandler_Tank : IProductHandler { public readonly ContentManifestFile? m_textContentManifest; public readonly ContentManifestFile? m_speechContentManifest; public readonly ResourceGraph? m_resourceGraph; - + public struct Asset { public int m_packageIdx; public int m_recordIdx; @@ -34,6 +34,7 @@ public Asset(int packageIdx, int recordIdx) { m_recordIdx = recordIdx; } } + public readonly ConcurrentDictionary m_assets = null!; private readonly bool m_usingResourceGraph; @@ -43,12 +44,12 @@ public Asset(int packageIdx, int recordIdx) { public const int PACKAGE_IDX_FAKE_TEXT_CMF = -1; public const int PACKAGE_IDX_FAKE_SPEECH_CMF = -2; public const int PACKAGE_IDX_FAKE_ROOT_CMF = -3; - + public const string REGION_DEV = "RDEV"; public const string REGION_CN = "RCN"; public const string SPEECH_MANIFEST_NAME = "speech"; public const string TEXT_MANIFEST_NAME = "text"; - + public static string? GetManifestLocale(string name) { var tag = name.Split('_').Reverse().SingleOrDefault(v => v[0] == 'L' && v.Length >= 4); return tag?.Substring(1); @@ -69,8 +70,8 @@ public ProductHandler_Tank(ClientHandler client, Stream stream) { if (!clientArgs.LoadManifest) return; var totalAssetCount = 0; - - foreach (var rootFile in m_rootFiles.Reverse()) { // cmf first, then apm + + foreach (var rootFile in m_rootFiles.Reverse()) { // cmf first, then apm var extension = Path.GetExtension(rootFile.FileName!); if (extension != ".cmf" && extension != ".apm" && extension != ".trg") { // not a manifest @@ -82,7 +83,7 @@ public ProductHandler_Tank(ClientHandler client, Stream stream) { if (!manifestName.Contains(clientArgs.ManifestRegion ?? REGION_DEV)) continue; // is a CN (china) CMF var manifestLocale = GetManifestLocale(manifestName); - + switch (extension) { case ".cmf": { var isSpeech = manifestName.Contains(SPEECH_MANIFEST_NAME); @@ -94,7 +95,7 @@ public ProductHandler_Tank(ClientHandler client, Stream stream) { // text or old root/text combo if (manifestLocale != client.CreateArgs.TextLanguage) continue; } - + ContentManifestFile cmf; try { using var cmfStream = client.OpenCKey(rootFile.MD5)!; @@ -107,8 +108,10 @@ public ProductHandler_Tank(ClientHandler client, Stream stream) { if (Debugger.IsAttached) { Debugger.Break(); } + throw; } + if (isSpeech) { m_speechContentManifest = cmf; } else if (isText) { @@ -116,12 +119,13 @@ public ProductHandler_Tank(ClientHandler client, Stream stream) { } else { m_rootContentManifest = cmf; } + totalAssetCount += cmf.m_header.m_dataCount; break; } case ".apm": { if (manifestLocale != client.CreateArgs.TextLanguage) break; // not relevant - + using var apmStream = client.OpenCKey(rootFile.MD5)!; m_packageManifest = new AssetPackageManifest(client, this, apmStream, manifestName); break; @@ -138,6 +142,7 @@ public ProductHandler_Tank(ClientHandler client, Stream stream) { if (Debugger.IsAttached) { Debugger.Break(); } + throw; } @@ -165,7 +170,7 @@ public ProductHandler_Tank(ClientHandler client, Stream stream) { } } } - + RegisterCMFAssets(m_rootContentManifest, PACKAGE_IDX_FAKE_ROOT_CMF); RegisterCMFAssets(m_textContentManifest, PACKAGE_IDX_FAKE_TEXT_CMF); RegisterCMFAssets(m_speechContentManifest, PACKAGE_IDX_FAKE_SPEECH_CMF); @@ -176,14 +181,15 @@ public ProductHandler_Tank(ClientHandler client, Stream stream) { DoBundleLookupHack(); } } - + public ContentManifestFile.HashData GetHashData(ulong guid) { if (!TryGetHashData(guid, out var hashData)) { throw new FileNotFoundException($"{guid:X16}"); } + return hashData; } - + public bool TryGetHashData(ulong guid, out ContentManifestFile.HashData hashData) { if (m_textContentManifest != null && m_textContentManifest.TryGet(guid, out hashData)) return true; if (m_speechContentManifest != null && m_speechContentManifest.TryGet(guid, out hashData)) return true; @@ -204,7 +210,7 @@ private void RegisterCMFAssets(ContentManifestFile? contentManifestFile, int fak var cmfAsset = contentManifestFile.m_hashList[j]; if (m_assets.ContainsKey(cmfAsset.GUID)) return; m_assets[cmfAsset.GUID] = new Asset(fakePackageIdx, j); - }); + }); } public Stream? OpenFile(ContentManifestFile.HashData hashData) { @@ -224,18 +230,18 @@ private void RegisterCMFAssets(ContentManifestFile? contentManifestFile, int fak var hashData = GetHashData(guid); return OpenFile(hashData)!; } - + if (!m_assets.TryGetValue(guid, out var asset)) throw new FileNotFoundException($"{guid:X16}"); if (m_hackedBundleLookup != null && m_hackedBundleLookup.TryGetValue(guid, out var bundleGUID)) { return OpenFileFromBundle(bundleGUID, guid); } - + var normalStream = OpenPackageFile(asset); // could be null here, if in encrypted bundle that the hack can't read return normalStream; } - + /// /// Open an /// @@ -248,7 +254,7 @@ private void RegisterCMFAssets(ContentManifestFile? contentManifestFile, int fak var hashData = GetHashData(record.m_GUID); return OpenFile(hashData); } - + var bundles = m_packageManifest!.m_packageBundles[asset.m_packageIdx]; foreach (var bundleGuid in bundles) { var foundStream = OpenFileFromBundle(bundleGuid, record.m_GUID); @@ -256,9 +262,10 @@ private void RegisterCMFAssets(ContentManifestFile? contentManifestFile, int fak return foundStream; } } + throw new Exception("bundle not found. :tim:"); } - + /// /// Unpacks asset indices to real data /// @@ -269,7 +276,7 @@ public void UnpackPackageAsset(Asset asset, out AssetPackageManifest.PackageReco record = m_packageManifest!.m_packageRecords[asset.m_packageIdx][asset.m_recordIdx]; return; } - + ContentManifestFile? contentManifest; if (asset.m_packageIdx == PACKAGE_IDX_FAKE_ROOT_CMF) { @@ -287,4 +294,4 @@ record = m_packageManifest!.m_packageRecords[asset.m_packageIdx][asset.m_recordI }; } } -} +} \ No newline at end of file diff --git a/TACTLib/Core/Product/Tank/ResourceGraph.cs b/TACTLib/Core/Product/Tank/ResourceGraph.cs index cb89807..9daa060 100644 --- a/TACTLib/Core/Product/Tank/ResourceGraph.cs +++ b/TACTLib/Core/Product/Tank/ResourceGraph.cs @@ -110,11 +110,11 @@ public struct TRGHeader { // version 11 public const uint ENCRYPTED_MAGIC = 0x677274; public uint GetNonEncryptedMagic() { - return (uint)(UNENCRYPTED_MAGIC | (GetVersion() << 24)); + return (uint) (UNENCRYPTED_MAGIC | (GetVersion() << 24)); } public byte GetVersion() { - return IsEncrypted() ? (byte)(m_footerMagic & 0x000000FF) : (byte)((m_footerMagic & 0xFF000000) >> 24); + return IsEncrypted() ? (byte) (m_footerMagic & 0x000000FF) : (byte) ((m_footerMagic & 0xFF000000) >> 24); } public bool IsEncrypted() { @@ -320,4 +320,4 @@ private void ParseBlocks(BinaryReader reader, string name) { } } } -} +} \ No newline at end of file