From b990b743749f71822be0fe0c27c56834fe7b72ae Mon Sep 17 00:00:00 2001 From: GoneUp Date: Tue, 21 Jul 2020 23:35:32 +0200 Subject: [PATCH] MassDumper Optimization Reduced Logging --- GPK_RePack/Forms/GUI.cs | 27 +++++++++++++++++++++------ GPK_RePack/IO/MassDumper.cs | 27 ++++++++++++++++++++++----- GPK_RePack/IO/Reader.cs | 11 +++++++---- GPK_RePack/IO/Writer.cs | 3 +++ GPK_RePack/Model/Payload/MipMap.cs | 16 +++++++++++++++- GPK_RePack/Model/Payload/Texture2D.cs | 17 ++++++++++++----- 6 files changed, 80 insertions(+), 21 deletions(-) diff --git a/GPK_RePack/Forms/GUI.cs b/GPK_RePack/Forms/GUI.cs index 2da66d7..0e2872f 100644 --- a/GPK_RePack/Forms/GUI.cs +++ b/GPK_RePack/Forms/GUI.cs @@ -245,7 +245,7 @@ private void openToolStripMenuItem_Click(object sender, EventArgs e) } //display info while loading - while (!Task.WaitAll(runningTasks.ToArray(), 500)) + while (!Task.WaitAll(runningTasks.ToArray(), 50)) { Application.DoEvents(); DisplayStatus(runningReaders, "Loading", start); @@ -344,9 +344,22 @@ private void DisplayStatus(List list, string tag, DateTime start) foreach (IProgress p in list) { Status stat = p.GetStatus(); - actual += stat.progress; - total += stat.totalobjects; - if (stat.finished) finished++; + if (stat.subGpkCount > 1) + { + //dont show actual objects, just the sub-file count + actual += stat.subGpkDone; + total += stat.subGpkCount; + if (actual == total) finished++; + } + else + { + //normal gpk + actual += stat.progress; + total += stat.totalobjects; + if (stat.finished) finished++; + } + + } if (finished < list.Count) @@ -403,6 +416,7 @@ public void DrawPackages() } treeMain.BeginUpdate(); treeMain.Nodes.Clear(); + var toAdd = new List(); for (int i = 0; i < gpkStore.LoadedGpkPackages.Count; i++) { @@ -471,11 +485,12 @@ public void DrawPackages() } - treeMain.Nodes.Add(nodeP); + toAdd.Add(nodeP); + } - + treeMain.Nodes.AddRange(toAdd.ToArray()); treeMain.Sort(); treeMain.EndUpdate(); } diff --git a/GPK_RePack/IO/MassDumper.cs b/GPK_RePack/IO/MassDumper.cs index fb0ffac..27c1318 100644 --- a/GPK_RePack/IO/MassDumper.cs +++ b/GPK_RePack/IO/MassDumper.cs @@ -1,9 +1,11 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.ServiceModel.Configuration; using System.Text; +using System.Threading; using System.Threading.Tasks; using System.Windows; using GPK_RePack.Editors; @@ -103,23 +105,34 @@ public static void DumpMassHeaders(String outfile, String[] gpkFiles) } } - + //in thread public static void DumpMassTextures(GpkStore store, String outdir, Dictionary> filterList) { logger.Info("Started dumping textures to " + outdir); Directory.CreateDirectory(outdir); - List runningTasks = new List(); + SynchronizedCollection runningTasks = new SynchronizedCollection(); + + int MAX_TASKS = 100; foreach (var file in filterList) { + //throttle thread creation + while (runningTasks.Count > MAX_TASKS) + { + Thread.Sleep(1000); + } + + + //create out dir var fileOutPath = string.Format("{0}\\{1}.gpk\\", outdir, file.Key); Directory.CreateDirectory(fileOutPath); //limit to 5 threads by default foreach (var entry in file.Value) { - Task newTask = new Task(() => + Task newTask = null; + newTask = new Task(() => { string path = string.Format("{0}\\{1}.gpk", store.BaseSearchPath, entry.SubGPKName); @@ -149,15 +162,19 @@ public static void DumpMassTextures(GpkStore store, String outdir, Dictionary ReadGpk(string path, bool skipExportData) var fileID = string.Format("{0}_{1}", Path.GetFileName(path), reader.BaseStream.Position); logger = LogManager.GetLogger("[Reader:" + fileID + "]"); - logger.Info("Reading Start"); + logger.Debug("Reading Start"); tmpGPK.Filename = fileID; tmpGPK.Path = path; @@ -74,15 +74,18 @@ public List ReadGpk(string path, bool skipExportData) //header is read, let us extract the data and read it List returnPackageList = new List(); + stat = new Status(); + stat.subGpkCount = tmpPackageList.Count; foreach (var subGPK in tmpPackageList) { - stat = new Status(); + reader.BaseStream.Seek(subGPK.CompositeStartOffset, SeekOrigin.Begin); var data = reader.ReadBytes((int)subGPK.OrginalSize); - var fullGpk = ReadSubGpkPackage(subGPK, data, skipExportData, stat); returnPackageList.Add(fullGpk); + + stat.subGpkDone++; } reader.Close(); @@ -253,7 +256,7 @@ private void ReadHeader(BinaryReader reader, GpkPackage package) if (package.x64) package.Header.Unk3 = reader.ReadBytes(12); stat.totalobjects = package.Header.NameCount + package.Header.ImportCount + package.Header.ExportCount * 3; //Export, Export Linking, ExportData = *3 - logger.Info("File Info: NameCount {0}, ImportCount {1}, ExportCount {2}", package.Header.NameCount, package.Header.ImportCount, package.Header.ExportCount); + logger.Debug("File Info: NameCount {0}, ImportCount {1}, ExportCount {2}", package.Header.NameCount, package.Header.ImportCount, package.Header.ExportCount); package.Header.FGUID = reader.ReadBytes(16); //logger.Info("FGUID " + package.Header.FGUID); diff --git a/GPK_RePack/IO/Writer.cs b/GPK_RePack/IO/Writer.cs index 27f7ac8..29bec9b 100644 --- a/GPK_RePack/IO/Writer.cs +++ b/GPK_RePack/IO/Writer.cs @@ -14,6 +14,9 @@ namespace GPK_RePack.IO { struct Status { + public int subGpkCount; + public int subGpkDone; + public int progress; public int totalobjects; public long time; diff --git a/GPK_RePack/Model/Payload/MipMap.cs b/GPK_RePack/Model/Payload/MipMap.cs index f2c6c1b..6041ec1 100644 --- a/GPK_RePack/Model/Payload/MipMap.cs +++ b/GPK_RePack/Model/Payload/MipMap.cs @@ -34,7 +34,7 @@ class MipMap public byte[] uncompressedData; - public string loadedFromTextureCache = ""; + public string textureCachePath = ""; public List blocks = new List(); internal int blockCount; @@ -47,6 +47,10 @@ public void generateBlocks() compressedSize = 0; for (int i = 0; i < blockCount; i++) { + //if data was not found, skip + if (uncompressedData == null) + continue; + int blockOffset = i * blocksize; int blockEnd = blockOffset + blocksize; if (blockEnd > uncompressedSize) @@ -73,6 +77,16 @@ public override string ToString() info.AppendFormat("Size: {0} x {1} {2}", sizeX, sizeY, Environment.NewLine); info.AppendLine("Compression: " + flags); if (((CompressionTypes)flags & NoOp) != 0) info.AppendLine("Data for this MipMap is stored external!"); + if (((CompressionTypes)flags & CompressionTypes.StoreInSeparatefile) != 0) + { + info.AppendLine("Data for this MipMap is stored in tfc"); + if (compressedSize == 0) + { + info.AppendLine("TFC data was not found. Searched for tfc file at " + textureCachePath); + } + } + + info.AppendLine("Compressed Size: " + compressedSize); info.AppendLine("Uncompressed Size: " + uncompressedSize); info.AppendLine("Blocks: " + blocks.Count); diff --git a/GPK_RePack/Model/Payload/Texture2D.cs b/GPK_RePack/Model/Payload/Texture2D.cs index a1764c6..5b74f15 100644 --- a/GPK_RePack/Model/Payload/Texture2D.cs +++ b/GPK_RePack/Model/Payload/Texture2D.cs @@ -136,11 +136,15 @@ public void ReadData(GpkPackage package, GpkExport export) { tgaPath = Reader.ReadString(reader, length); } - else + else if (length < 0) { inUnicode = true; tgaPath = Reader.ReadUnicodeString(reader, (length * -1) * 2); } + else + { + tgaPath = ""; + } int count = reader.ReadInt32(); @@ -169,13 +173,15 @@ public void ReadData(GpkPackage package, GpkExport export) //compChunkSize == size //TextureFileCacheName prop has name var txtProp = export.Properties.Find(t => ((GpkBaseProperty)t).name == "TextureFileCacheName"); + if (txtProp == null) + goto fail; String txtCacheFile = ((GpkNameProperty)txtProp).value; //assumption: cache in same dir, happens for cookedpc compositegpks - var path = $"{Path.GetDirectoryName(package.Path)}\\{txtCacheFile}.tfc"; - if (File.Exists(path)) + map.textureCachePath = $"{Path.GetDirectoryName(package.Path)}\\{txtCacheFile}.tfc"; + if (File.Exists(map.textureCachePath)) { - BinaryReader cacheReader = new BinaryReader(new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)); + BinaryReader cacheReader = new BinaryReader(new FileStream(map.textureCachePath, FileMode.Open, FileAccess.Read, FileShare.Read)); cacheReader.BaseStream.Seek(map.compChunkOffset, SeekOrigin.Begin); map.signature = cacheReader.ReadUInt32(); //0x9e2a83c1 @@ -189,7 +195,7 @@ public void ReadData(GpkPackage package, GpkExport export) } else { - logger.Warn("{0}, MipMap {1}, Cache {2}, CompressionTypes.StoreInSeparatefile, could not find tfc!!", export.ObjectName, i, txtCacheFile); + logger.Debug("{0}, MipMap {1}, Cache {2}, CompressionTypes.StoreInSeparatefile, could not find tfc!!", export.ObjectName, i, txtCacheFile); } } else if (((CompressionTypes)map.flags & CompressionTypes.SeperateData) != 0) @@ -207,6 +213,7 @@ public void ReadData(GpkPackage package, GpkExport export) logger.Trace("{0}, MipMap {0}, no data!!", export.ObjectName, i); } + fail: map.sizeX = reader.ReadInt32(); map.sizeY = reader.ReadInt32();