Add mechanism to handle completion of async res loads

Penumbra/Interop/Hooks/HookSettings.cs

@@ -100,6 +100,7 @@ public struct ResourceLoadingHooks
         public bool DecRef;
         public bool GetResourceSync;
         public bool GetResourceAsync;
+        public bool UpdateResourceState;
         public bool CheckFileState;
         public bool TexResourceHandleOnLoad;
         public bool LoadMdlFileExtern;

Penumbra/Interop/Hooks/ResourceLoading/ResourceLoader.cs

@@ -1,7 +1,9 @@
+using System.IO;
 using FFXIVClientStructs.FFXIV.Client.System.Resource;
 using OtterGui.Services;
 using Penumbra.Api.Enums;
 using Penumbra.Collections;
+using Penumbra.Interop.Hooks.Resources;
 using Penumbra.Interop.PathResolving;
 using Penumbra.Interop.SafeHandles;
 using Penumbra.Interop.Structs;
@@ -13,27 +15,38 @@ namespace Penumbra.Interop.Hooks.ResourceLoading;
 
 public unsafe class ResourceLoader : IDisposable, IService
 {
-    private readonly ResourceService _resources;
-    private readonly FileReadService _fileReadService;
-    private readonly RsfService   _rsfService;
-    private readonly PapHandler      _papHandler;
-    private readonly Configuration   _config;
+    private readonly ResourceService          _resources;
+    private readonly FileReadService          _fileReadService;
+    private readonly RsfService               _rsfService;
+    private readonly PapHandler               _papHandler;
+    private readonly Configuration            _config;
+    private readonly ResourceHandleDestructor _destructor;
+
+    private readonly ConcurrentDictionary<nint, Utf8GamePath> _ongoingLoads = [];
 
     private ResolveData                                        _resolvedData = ResolveData.Invalid;
     public event Action<Utf8GamePath, FullPath?, ResolveData>? PapRequested;
 
-    public ResourceLoader(ResourceService resources, FileReadService fileReadService, RsfService rsfService, Configuration config, PeSigScanner sigScanner)
+    public IReadOnlyDictionary<nint, Utf8GamePath> OngoingLoads
+        => _ongoingLoads;
+
+    public ResourceLoader(ResourceService resources, FileReadService fileReadService, RsfService rsfService, Configuration config, PeSigScanner sigScanner,
+        ResourceHandleDestructor destructor)
     {
         _resources       = resources;
         _fileReadService = fileReadService;
-        _rsfService   = rsfService;
+        _rsfService      = rsfService;
         _config          = config;
+        _destructor      = destructor;
         ResetResolvePath();
 
-        _resources.ResourceRequested    += ResourceHandler;
-        _resources.ResourceHandleIncRef += IncRefProtection;
-        _resources.ResourceHandleDecRef += DecRefProtection;
-        _fileReadService.ReadSqPack     += ReadSqPackDetour;
+        _resources.ResourceRequested     += ResourceHandler;
+        _resources.ResourceStateUpdating += ResourceStateUpdatingHandler;
+        _resources.ResourceStateUpdated  += ResourceStateUpdatedHandler;
+        _resources.ResourceHandleIncRef  += IncRefProtection;
+        _resources.ResourceHandleDecRef  += DecRefProtection;
+        _fileReadService.ReadSqPack      += ReadSqPackDetour;
+        _destructor.Subscribe(ResourceDestructorHandler, ResourceHandleDestructor.Priority.ResourceLoader);
 
         _papHandler = new PapHandler(sigScanner, PapResourceHandler);
         _papHandler.Enable();
@@ -109,12 +122,32 @@ public delegate void FileLoadedDelegate(ResourceHandle* resource, CiByteString p
     /// </summary>
     public event FileLoadedDelegate? FileLoaded;
 
+    public delegate void ResourceCompleteDelegate(ResourceHandle* resource, CiByteString path, Utf8GamePath originalPath,
+        ReadOnlySpan<byte> additionalData, bool isAsync);
+
+    /// <summary>
+    /// Event fired just before a resource finishes loading.
+    /// <see cref="ResourceHandle.LoadState"/> must be checked to know whether the load was successful or not.
+    /// AdditionalData is either empty or the part of the path inside the leading pipes.
+    /// </summary>
+    public event ResourceCompleteDelegate? BeforeResourceComplete;
+
+    /// <summary>
+    /// Event fired when a resource has finished loading.
+    /// <see cref="ResourceHandle.LoadState"/> must be checked to know whether the load was successful or not.
+    /// AdditionalData is either empty or the part of the path inside the leading pipes.
+    /// </summary>
+    public event ResourceCompleteDelegate? ResourceComplete;
+
     public void Dispose()
     {
-        _resources.ResourceRequested    -= ResourceHandler;
-        _resources.ResourceHandleIncRef -= IncRefProtection;
-        _resources.ResourceHandleDecRef -= DecRefProtection;
-        _fileReadService.ReadSqPack     -= ReadSqPackDetour;
+        _resources.ResourceRequested     -= ResourceHandler;
+        _resources.ResourceStateUpdating -= ResourceStateUpdatingHandler;
+        _resources.ResourceStateUpdated  -= ResourceStateUpdatedHandler;
+        _resources.ResourceHandleIncRef  -= IncRefProtection;
+        _resources.ResourceHandleDecRef  -= DecRefProtection;
+        _fileReadService.ReadSqPack      -= ReadSqPackDetour;
+        _destructor.Unsubscribe(ResourceDestructorHandler);
         _papHandler.Dispose();
     }
 
@@ -135,7 +168,8 @@ private void ResourceHandler(ref ResourceCategory category, ref ResourceType typ
 
         if (resolvedPath == null || !Utf8GamePath.FromByteString(resolvedPath.Value.InternalName, out var p))
         {
-            returnValue = _resources.GetOriginalResource(sync, category, type, hash, path.Path, parameters);
+            returnValue = _resources.GetOriginalResource(sync, category, type, hash, path.Path, parameters, original: original);
+            TrackResourceLoad(returnValue, original);
             ResourceLoaded?.Invoke(returnValue, path, resolvedPath, data);
             return;
         }
@@ -145,10 +179,57 @@ private void ResourceHandler(ref ResourceCategory category, ref ResourceType typ
         hash = ComputeHash(resolvedPath.Value.InternalName, parameters);
         var oldPath = path;
         path        = p;
-        returnValue = _resources.GetOriginalResource(sync, category, type, hash, path.Path, parameters);
+        returnValue = _resources.GetOriginalResource(sync, category, type, hash, path.Path, parameters, original: original);
+        TrackResourceLoad(returnValue, original);
         ResourceLoaded?.Invoke(returnValue, oldPath, resolvedPath.Value, data);
     }
 
+    private void TrackResourceLoad(ResourceHandle* handle, Utf8GamePath original)
+    {
+        if (handle->UnkState == 2 && handle->LoadState >= LoadState.Success)
+            return;
+
+        _ongoingLoads.TryAdd((nint)handle, original.Clone());
+    }
+
+    private void ResourceStateUpdatedHandler(ResourceHandle* handle, Utf8GamePath syncOriginal, (byte, LoadState) previousState, ref uint returnValue)
+    {
+        if (handle->UnkState != 2 || handle->LoadState < LoadState.Success || previousState.Item1 == 2 && previousState.Item2 >= LoadState.Success)
+            return;
+
+        if (!_ongoingLoads.TryRemove((nint)handle, out var asyncOriginal))
+            asyncOriginal = Utf8GamePath.Empty;
+
+        var path = handle->CsHandle.FileName;
+        if (!syncOriginal.IsEmpty && !asyncOriginal.IsEmpty && !syncOriginal.Equals(asyncOriginal))
+            Penumbra.Log.Warning($"[ResourceLoader] Resource original paths inconsistency: 0x{(nint)handle:X}, of path {path}, sync original {syncOriginal}, async original {asyncOriginal}.");
+        var original = !asyncOriginal.IsEmpty ? asyncOriginal : syncOriginal;
+
+        // Penumbra.Log.Information($"[ResourceLoader] Resource is complete: 0x{(nint)handle:X}, of path {path}, original {original}, state {previousState.Item1}:{previousState.Item2} -> {handle->UnkState}:{handle->LoadState}, sync: {asyncOriginal.IsEmpty}");
+        if (PathDataHandler.Split(path.AsSpan(), out var actualPath, out var additionalData))
+            ResourceComplete?.Invoke(handle, new CiByteString(actualPath), original, additionalData, !asyncOriginal.IsEmpty);
+        else
+            ResourceComplete?.Invoke(handle, path.AsByteString(), original, [], !asyncOriginal.IsEmpty);
+    }
+
+    private void ResourceStateUpdatingHandler(ResourceHandle* handle, Utf8GamePath syncOriginal)
+    {
+        if (handle->UnkState != 1 || handle->LoadState != LoadState.Success)
+            return;
+
+        if (!_ongoingLoads.TryGetValue((nint)handle, out var asyncOriginal))
+            asyncOriginal = Utf8GamePath.Empty;
+
+        var path     = handle->CsHandle.FileName;
+        var original = asyncOriginal.IsEmpty ? syncOriginal : asyncOriginal;
+
+        // Penumbra.Log.Information($"[ResourceLoader] Resource is about to be complete: 0x{(nint)handle:X}, of path {path}, original {original}");
+        if (PathDataHandler.Split(path.AsSpan(), out var actualPath, out var additionalData))
+            BeforeResourceComplete?.Invoke(handle, new CiByteString(actualPath), original, additionalData, !asyncOriginal.IsEmpty);
+        else
+            BeforeResourceComplete?.Invoke(handle, path.AsByteString(), original, [], !asyncOriginal.IsEmpty);
+    }
+
     private void ReadSqPackDetour(SeFileDescriptor* fileDescriptor, ref int priority, ref bool isSync, ref byte? returnValue)
     {
         if (fileDescriptor->ResourceHandle == null)
@@ -265,6 +346,11 @@ private static void DecRefProtection(ResourceHandle* handle, ref byte? returnVal
         returnValue = 1;
     }
 
+    private void ResourceDestructorHandler(ResourceHandle* handle)
+    {
+        _ongoingLoads.TryRemove((nint)handle, out _);
+    }
+
     /// <summary> Compute the CRC32 hash for a given path together with potential resource parameters. </summary>
     private static int ComputeHash(CiByteString path, GetResourceParameters* pGetResParams)
     {

Penumbra/Interop/Hooks/ResourceLoading/ResourceService.cs

@@ -19,6 +19,8 @@ public unsafe class ResourceService : IDisposable, IRequiredService
     private readonly PerformanceTracker     _performance;
     private readonly ResourceManagerService _resourceManager;
 
+    private readonly ThreadLocal<Utf8GamePath> _currentGetResourcePath = new(() => Utf8GamePath.Empty);
+
     public ResourceService(PerformanceTracker performance, ResourceManagerService resourceManager, IGameInteropProvider interop)
     {
         _performance     = performance;
@@ -34,6 +36,8 @@ public ResourceService(PerformanceTracker performance, ResourceManagerService re
             _getResourceSyncHook.Enable();
         if (!HookOverrides.Instance.ResourceLoading.GetResourceAsync)
             _getResourceAsyncHook.Enable();
+        if (!HookOverrides.Instance.ResourceLoading.UpdateResourceState)
+            _updateResourceStateHook.Enable();
         if (!HookOverrides.Instance.ResourceLoading.IncRef)
             _incRefHook.Enable();
         if (!HookOverrides.Instance.ResourceLoading.DecRef)
@@ -54,8 +58,10 @@ public void Dispose()
     {
         _getResourceSyncHook.Dispose();
         _getResourceAsyncHook.Dispose();
+        _updateResourceStateHook.Dispose();
         _incRefHook.Dispose();
         _decRefHook.Dispose();
+        _currentGetResourcePath.Dispose();
     }
 
     #region GetResource
@@ -112,28 +118,84 @@ public delegate void GetResourcePreDelegate(ref ResourceCategory category, ref R
                     unk9);
         }
 
+        var original = gamePath;
         ResourceHandle* returnValue = null;
-        ResourceRequested?.Invoke(ref *categoryId, ref *resourceType, ref *resourceHash, ref gamePath, gamePath, pGetResParams, ref isSync,
+        ResourceRequested?.Invoke(ref *categoryId, ref *resourceType, ref *resourceHash, ref gamePath, original, pGetResParams, ref isSync,
             ref returnValue);
         if (returnValue != null)
             return returnValue;
 
-        return GetOriginalResource(isSync, *categoryId, *resourceType, *resourceHash, gamePath.Path, pGetResParams, isUnk, unk8, unk9);
+        return GetOriginalResource(isSync, *categoryId, *resourceType, *resourceHash, gamePath.Path, pGetResParams, isUnk, unk8, unk9, original);
     }
 
     /// <summary> Call the original GetResource function. </summary>
     public ResourceHandle* GetOriginalResource(bool sync, ResourceCategory categoryId, ResourceType type, int hash, CiByteString path,
-        GetResourceParameters* resourceParameters = null, byte unk = 0, nint unk8 = 0, uint unk9 = 0)
-        => sync
-            ? _getResourceSyncHook.OriginalDisposeSafe(_resourceManager.ResourceManager, &categoryId, &type, &hash, path.Path,
-                resourceParameters, unk8, unk9)
-            : _getResourceAsyncHook.OriginalDisposeSafe(_resourceManager.ResourceManager, &categoryId, &type, &hash, path.Path,
-                resourceParameters, unk, unk8, unk9);
+        GetResourceParameters* resourceParameters = null, byte unk = 0, nint unk8 = 0, uint unk9 = 0, Utf8GamePath original = default)
+    {
+        if (original.Path is null) // i. e. if original is default
+            Utf8GamePath.FromByteString(path, out original);
+        var previous = _currentGetResourcePath.Value;
+        try
+        {
+            _currentGetResourcePath.Value = original;
+            return sync
+                ? _getResourceSyncHook.OriginalDisposeSafe(_resourceManager.ResourceManager, &categoryId, &type, &hash, path.Path,
+                    resourceParameters, unk8, unk9)
+                : _getResourceAsyncHook.OriginalDisposeSafe(_resourceManager.ResourceManager, &categoryId, &type, &hash, path.Path,
+                    resourceParameters, unk, unk8, unk9);
+        } finally
+        {
+            _currentGetResourcePath.Value = previous;
+        }
+    }
 
     #endregion
 
     private delegate nint ResourceHandlePrototype(ResourceHandle* handle);
 
+    #region UpdateResourceState
+
+    /// <summary> Invoked before a resource state is updated. </summary>
+    /// <param name="handle">The resource handle.</param>
+    /// <param name="syncOriginal">The original game path of the resource, if loaded synchronously.</param>
+    public delegate void ResourceStateUpdatingDelegate(ResourceHandle* handle, Utf8GamePath syncOriginal);
+
+    /// <summary> Invoked after a resource state is updated. </summary>
+    /// <param name="handle">The resource handle.</param>
+    /// <param name="syncOriginal">The original game path of the resource, if loaded synchronously.</param>
+    /// <param name="previousState">The previous state of the resource.</param>
+    /// <param name="returnValue">The return value to use.</param>
+    public delegate void ResourceStateUpdatedDelegate(ResourceHandle* handle, Utf8GamePath syncOriginal, (byte UnkState, LoadState LoadState) previousState, ref uint returnValue);
+
+    /// <summary>
+    /// <inheritdoc cref="ResourceStateUpdatingDelegate"/> <para/>
+    /// Subscribers should be exception-safe.
+    /// </summary>
+    public event ResourceStateUpdatingDelegate? ResourceStateUpdating;
+
+    /// <summary>
+    /// <inheritdoc cref="ResourceStateUpdatedDelegate"/> <para/>
+    /// Subscribers should be exception-safe.
+    /// </summary>
+    public event ResourceStateUpdatedDelegate? ResourceStateUpdated;
+
+    private delegate uint UpdateResourceStatePrototype(ResourceHandle* handle, byte offFileThread);
+
+    [Signature(Sigs.UpdateResourceState, DetourName = nameof(UpdateResourceStateDetour))]
+    private readonly Hook<UpdateResourceStatePrototype> _updateResourceStateHook = null!;
+
+    private uint UpdateResourceStateDetour(ResourceHandle* handle, byte offFileThread)
+    {
+        var previousState = (handle->UnkState, handle->LoadState);
+        var syncOriginal = _currentGetResourcePath.IsValueCreated ? _currentGetResourcePath.Value! : Utf8GamePath.Empty;
+        ResourceStateUpdating?.Invoke(handle, syncOriginal);
+        var ret = _updateResourceStateHook.OriginalDisposeSafe(handle, offFileThread);
+        ResourceStateUpdated?.Invoke(handle, syncOriginal, previousState, ref ret);
+        return ret;
+    }
+
+    #endregion
+
     #region IncRef
 
     /// <summary> Invoked before a resource handle reference count is incremented. </summary>

Penumbra/Interop/Hooks/Resources/ResourceHandleDestructor.cs

@@ -14,9 +14,12 @@ public enum Priority
         /// <seealso cref="PathResolving.SubfileHelper"/>
         SubfileHelper,
 
-        /// <seealso cref="ShaderReplacementFixer"/>
+        /// <seealso cref="PostProcessing.ShaderReplacementFixer"/>
         ShaderReplacementFixer,
 
+        /// <seealso cref="ResourceLoading.ResourceLoader"/>
+        ResourceLoader,
+
         /// <seealso cref="ResourceWatcher.OnResourceDestroyed"/>
         ResourceWatcher,
     }

Penumbra/Interop/Processing/FilePostProcessService.cs

@@ -4,6 +4,7 @@
 using Penumbra.Interop.Hooks.ResourceLoading;
 using Penumbra.Interop.Structs;
 using Penumbra.String;
+using Penumbra.String.Classes;
 
 namespace Penumbra.Interop.Processing;
 
@@ -20,20 +21,23 @@ public unsafe class FilePostProcessService : IRequiredService, IDisposable
 
     public FilePostProcessService(ResourceLoader resourceLoader, ServiceManager services)
     {
-        _resourceLoader            =  resourceLoader;
-        _processors                =  services.GetServicesImplementing<IFilePostProcessor>().ToFrozenDictionary(s => s.Type, s => s);
-        _resourceLoader.FileLoaded += OnFileLoaded;
+        _resourceLoader                        =  resourceLoader;
+        _processors                            =  services.GetServicesImplementing<IFilePostProcessor>().ToFrozenDictionary(s => s.Type, s => s);
+        _resourceLoader.BeforeResourceComplete += OnResourceComplete;
     }
 
     public void Dispose()
     {
-        _resourceLoader.FileLoaded -= OnFileLoaded;
+        _resourceLoader.BeforeResourceComplete -= OnResourceComplete;
     }
 
-    private void OnFileLoaded(ResourceHandle* resource, CiByteString path, bool returnValue, bool custom,
-        ReadOnlySpan<byte> additionalData)
+    private void OnResourceComplete(ResourceHandle* resource, CiByteString path, Utf8GamePath original,
+        ReadOnlySpan<byte> additionalData, bool isAsync)
     {
+        if (resource->LoadState != LoadState.Success)
+            return;
+
         if (_processors.TryGetValue(resource->FileType, out var processor))
-            processor.PostProcess(resource, path, additionalData);
+            processor.PostProcess(resource, original.Path, additionalData);
     }
 }

Penumbra/Interop/Structs/ResourceHandle.cs

@@ -24,10 +24,20 @@ public bool ChangeLod
 
 public enum LoadState : byte
 {
+    Constructing      = 0x00,
+    Constructed       = 0x01,
+    Async2            = 0x02,
+    AsyncRequested    = 0x03,
+    Async4            = 0x04,
+    AsyncLoading      = 0x05,
+    Async6            = 0x06,
     Success           = 0x07,
-    Async             = 0x03,
+    Unknown8          = 0x08,
     Failure           = 0x09,
     FailedSubResource = 0x0A,
+    FailureB          = 0x0B,
+    FailureC          = 0x0C,
+    FailureD          = 0x0D,
     None              = 0xFF,
 }
 
@@ -74,6 +84,9 @@ public readonly bool GamePath(out Utf8GamePath path)
     [FieldOffset(0x58)]
     public int FileNameLength;
 
+    [FieldOffset(0xA8)]
+    public byte UnkState;
+
     [FieldOffset(0xA9)]
     public LoadState LoadState;