Add ReShade peeler

Dalamud/Game/Internal/DXGI/SwapChainVtableResolver.cs

@@ -1,10 +1,10 @@
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Runtime.InteropServices;
-
 using Dalamud.Game.Internal.DXGI.Definitions;
+using Dalamud.ImGuiScene.Helpers;
+
 using FFXIVClientStructs.FFXIV.Client.Graphics.Kernel;
-using Serilog;
+
+using TerraFX.Interop.DirectX;
+using TerraFX.Interop.Windows;
 
 namespace Dalamud.Game.Internal.DXGI;
 
@@ -22,25 +22,18 @@ internal class SwapChainVtableResolver : BaseAddressResolver, ISwapChainAddressR
     /// <inheritdoc/>
     public IntPtr ResizeBuffers { get; set; }
 
-    /// <summary>
-    /// Gets a value indicating whether or not ReShade is loaded/used.
-    /// </summary>
-    public bool IsReshade { get; private set; }
-
     /// <inheritdoc/>
     protected override unsafe void Setup64Bit(ISigScanner sig)
     {
-        Device* kernelDev;
-        SwapChain* swapChain;
         void* dxgiSwapChain;
 
         while (true)
         {
-            kernelDev = Device.Instance();
+            var kernelDev = Device.Instance();
             if (kernelDev == null)
                 continue;
 
-            swapChain = kernelDev->SwapChain;
+            var swapChain = kernelDev->SwapChain;
             if (swapChain == null)
                 continue;
 
@@ -51,77 +44,10 @@ protected override unsafe void Setup64Bit(ISigScanner sig)
             break;
         }
 
-        var scVtbl = GetVTblAddresses(new IntPtr(dxgiSwapChain), Enum.GetValues(typeof(IDXGISwapChainVtbl)).Length);
-
-        this.Present = scVtbl[(int)IDXGISwapChainVtbl.Present];
-
-        var modules = Process.GetCurrentProcess().Modules;
-        foreach (ProcessModule processModule in modules)
-        {
-            if (processModule.FileName != null && processModule.FileName.EndsWith("game\\dxgi.dll"))
-            {
-                var fileInfo = FileVersionInfo.GetVersionInfo(processModule.FileName);
-
-                if (fileInfo.FileDescription == null)
-                    break;
-
-                if (!fileInfo.FileDescription.Contains("GShade") && !fileInfo.FileDescription.Contains("ReShade"))
-                    break;
-
-                // reshade master@4232872 RVA
-                // var p = processModule.BaseAddress + 0x82C7E0; // DXGISwapChain::Present
-                // var p = processModule.BaseAddress + 0x82FAC0; // DXGISwapChain::runtime_present
-
-                // DXGISwapChain::handle_device_loss => DXGISwapChain::Present => DXGISwapChain::runtime_present
-
-                var scanner = new SigScanner(processModule);
-                var runtimePresentSig = "F6 C2 01 0F 85 ?? ?? ?? ??";
-
-                try
-                {
-                    // Looks like this sig only works for GShade 4
-                    if (fileInfo.FileDescription?.Contains("GShade 4.") == true)
-                    {
-                        Log.Verbose("Hooking present for GShade 4");
-                        runtimePresentSig = "E8 ?? ?? ?? ?? 45 0F B6 5E ??";
-                    }
-                }
-                catch (Exception ex)
-                {
-                    Log.Error(ex, "Failed to get reshade version info - falling back to default DXGISwapChain::runtime_present signature");
-                }
-
-                try
-                {
-                    var p = scanner.ScanText(runtimePresentSig);
-                    Log.Information($"ReShade DLL: {processModule.FileName} with DXGISwapChain::runtime_present at {p:X}");
-
-                    this.Present = p;
-                    this.IsReshade = true;
-                    break;
-                }
-                catch (Exception ex)
-                {
-                    Log.Error(ex, "Could not find reshade DXGISwapChain::runtime_present offset!");
-                }
-            }
-        }
-
-        this.ResizeBuffers = scVtbl[(int)IDXGISwapChainVtbl.ResizeBuffers];
-    }
-
-    private static List<IntPtr> GetVTblAddresses(IntPtr pointer, int numberOfMethods)
-    {
-        return GetVTblAddresses(pointer, 0, numberOfMethods);
-    }
-
-    private static List<IntPtr> GetVTblAddresses(IntPtr pointer, int startIndex, int numberOfMethods)
-    {
-        var vtblAddresses = new List<IntPtr>();
-        var vTable = Marshal.ReadIntPtr(pointer);
-        for (var i = startIndex; i < startIndex + numberOfMethods; i++)
-            vtblAddresses.Add(Marshal.ReadIntPtr(vTable, i * IntPtr.Size)); // using IntPtr.Size allows us to support both 32 and 64-bit processes
+        using var sc = new ComPtr<IDXGISwapChain>((IDXGISwapChain*)dxgiSwapChain);
+        ReShadePeeler.PeelSwapChain(&sc);
 
-        return vtblAddresses;
+        this.Present = (nint)sc.Get()->lpVtbl[(int)IDXGISwapChainVtbl.Present];
+        this.ResizeBuffers = (nint)sc.Get()->lpVtbl[(int)IDXGISwapChainVtbl.ResizeBuffers];
     }
 }

Dalamud/ImGuiScene/Helpers/ReShadePeeler.cs

@@ -0,0 +1,134 @@
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.InteropServices;
+
+using TerraFX.Interop.DirectX;
+using TerraFX.Interop.Windows;
+
+using static TerraFX.Interop.Windows.Windows;
+
+namespace Dalamud.ImGuiScene.Helpers;
+
+/// <summary>
+/// Peels ReShade off stuff.
+/// </summary>
+[SuppressMessage(
+    "StyleCop.CSharp.LayoutRules",
+    "SA1519:Braces should not be omitted from multi-line child statement",
+    Justification = "Multiple fixed blocks")]
+internal static unsafe class ReShadePeeler
+{
+    /// <summary>
+    /// Peels <see cref="IDXGISwapChain"/> if it is wrapped by ReShade.
+    /// </summary>
+    /// <param name="comptr">[inout] The COM pointer to an instance of <see cref="IDXGISwapChain"/>.</param>
+    /// <typeparam name="T">A COM type that is or extends <see cref="IDXGISwapChain"/>.</typeparam>
+    /// <returns><c>true</c> if peeled.</returns>
+    public static bool PeelSwapChain<T>(ComPtr<T>* comptr)
+        where T : unmanaged, IDXGISwapChain.Interface =>
+        PeelIUnknown(comptr, 0x10);
+
+    /// <summary>
+    /// Peels <see cref="ID3D12Device"/> if it is wrapped by ReShade.
+    /// </summary>
+    /// <param name="comptr">[inout] The COM pointer to an instance of <see cref="ID3D12Device"/>.</param>
+    /// <typeparam name="T">A COM type that is or extends <see cref="ID3D12Device"/>.</typeparam>
+    /// <returns><c>true</c> if peeled.</returns>
+    public static bool PeelD3D12Device<T>(ComPtr<T>* comptr)
+        where T : unmanaged, ID3D12Device.Interface =>
+        PeelIUnknown(comptr, 0x10);
+
+    /// <summary>
+    /// Peels <see cref="ID3D12CommandQueue"/> if it is wrapped by ReShade.
+    /// </summary>
+    /// <param name="comptr">[inout] The COM pointer to an instance of <see cref="ID3D12CommandQueue"/>.</param>
+    /// <typeparam name="T">A COM type that is or extends <see cref="ID3D12CommandQueue"/>.</typeparam>
+    /// <returns><c>true</c> if peeled.</returns>
+    public static bool PeelD3D12CommandQueue<T>(ComPtr<T>* comptr)
+        where T : unmanaged, ID3D12CommandQueue.Interface =>
+        PeelIUnknown(comptr, 0x10);
+
+    private static bool PeelIUnknown<T>(ComPtr<T>* comptr, nint offset)
+        where T : unmanaged, IUnknown.Interface
+    {
+        if (comptr->Get() == null || !IsReShadedComObject(comptr->Get()))
+            return false;
+
+        var punk = new ComPtr<IUnknown>(*(IUnknown**)((nint)comptr->Get() + offset));
+        using var comptr2 = default(ComPtr<T>);
+        if (punk.As(&comptr2).FAILED)
+            return false;
+        comptr2.Swap(comptr);
+        return true;
+    }
+
+    private static bool BelongsInReShadeDll(nint ptr)
+    {
+        foreach (ProcessModule processModule in Process.GetCurrentProcess().Modules)
+        {
+            if (ptr < processModule.BaseAddress)
+                continue;
+
+            var dosh = (IMAGE_DOS_HEADER*)processModule.BaseAddress;
+            var nth = (IMAGE_NT_HEADERS64*)(processModule.BaseAddress + dosh->e_lfanew);
+            if (ptr >= processModule.BaseAddress + nth->OptionalHeader.SizeOfImage)
+                continue;
+
+            fixed (byte* pfn0 = "CreateDXGIFactory"u8)
+            fixed (byte* pfn1 = "D2D1CreateDevice"u8)
+            fixed (byte* pfn2 = "D3D10CreateDevice"u8)
+            fixed (byte* pfn3 = "D3D11CreateDevice"u8)
+            fixed (byte* pfn4 = "D3D12CreateDevice"u8)
+            fixed (byte* pfn5 = "glBegin"u8)
+            fixed (byte* pfn6 = "vkCreateDevice"u8)
+            {
+                if (GetProcAddress((HMODULE)dosh, (sbyte*)pfn0) == 0)
+                    continue;
+                if (GetProcAddress((HMODULE)dosh, (sbyte*)pfn1) == 0)
+                    continue;
+                if (GetProcAddress((HMODULE)dosh, (sbyte*)pfn2) == 0)
+                    continue;
+                if (GetProcAddress((HMODULE)dosh, (sbyte*)pfn3) == 0)
+                    continue;
+                if (GetProcAddress((HMODULE)dosh, (sbyte*)pfn4) == 0)
+                    continue;
+                if (GetProcAddress((HMODULE)dosh, (sbyte*)pfn5) == 0)
+                    continue;
+                if (GetProcAddress((HMODULE)dosh, (sbyte*)pfn6) == 0)
+                    continue;
+            }
+
+            var fileInfo = FileVersionInfo.GetVersionInfo(processModule.FileName);
+
+            if (fileInfo.FileDescription == null)
+                continue;
+
+            if (!fileInfo.FileDescription.Contains("GShade") && !fileInfo.FileDescription.Contains("ReShade"))
+                continue;
+
+            return true;
+        }
+
+        return false;
+    }
+
+    private static bool IsReShadedComObject<T>(T* obj)
+        where T : unmanaged, IUnknown.Interface
+    {
+        var vtbl = (nint*)((IUnknown*)obj)->lpVtbl;
+        try
+        {
+            for (var i = 0; i < 3; i++)
+            {
+                if (!BelongsInReShadeDll(Marshal.ReadIntPtr((nint)(&vtbl[i]))))
+                    return false;
+            }
+
+            return true;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+}

Dalamud/ImGuiScene/Implementations/Dx11Renderer.ViewportHandler.cs

@@ -210,6 +210,12 @@ public static ViewportData CreateDComposition(Dx11Renderer renderer, HWND hWnd)
                 null,
                 swapChain1.GetAddressOf()).ThrowHr();
 
+            if (ReShadePeeler.PeelSwapChain(&swapChain1))
+            {
+                swapChain1.Get()->ResizeBuffers(sd1.BufferCount, sd1.Width, sd1.Height, sd1.Format, sd1.Flags)
+                    .ThrowHr();
+            }
+
             using var dcTarget = default(ComPtr<IDCompositionTarget>);
             renderer.dcompDevice.Get()->CreateTargetForHwnd(hWnd, BOOL.TRUE, dcTarget.GetAddressOf());
 
@@ -258,6 +264,17 @@ public static ViewportData Create(Dx11Renderer renderer, HWND hWnd)
             dxgiFactory.Get()->CreateSwapChain((IUnknown*)renderer.device.Get(), &desc, swapChain.GetAddressOf())
                 .ThrowHr();
 
+            if (ReShadePeeler.PeelSwapChain(&swapChain))
+            {
+                swapChain.Get()->ResizeBuffers(
+                        desc.BufferCount,
+                        desc.BufferDesc.Width,
+                        desc.BufferDesc.Height,
+                        desc.BufferDesc.Format,
+                        desc.Flags)
+                    .ThrowHr();
+            }
+
             return Create(renderer, swapChain, null, null);
         }
 

Dalamud/ImGuiScene/Implementations/Dx11Win32Scene.cs

@@ -29,6 +29,7 @@ internal sealed unsafe class Dx11Win32Scene : IWin32Scene
     private readonly Win32InputHandler imguiInput;
     private readonly WicEasy wicEasy;
 
+    private ComPtr<IDXGISwapChain> swapChainPossiblyWrapped;
     private ComPtr<IDXGISwapChain> swapChain;
     private ComPtr<ID3D11Device> device;
     private ComPtr<ID3D11DeviceContext> deviceContext;
@@ -45,8 +46,10 @@ public Dx11Win32Scene(IDXGISwapChain* swapChain)
         this.wicEasy = new();
         try
         {
-            swapChain->AddRef();
-            this.swapChain.Attach(swapChain);
+            this.swapChainPossiblyWrapped = new(swapChain);
+            this.swapChain = new(swapChain);
+            fixed (ComPtr<IDXGISwapChain>* ppSwapChain = &this.swapChain)
+                ReShadePeeler.PeelSwapChain(ppSwapChain);
 
             fixed (Guid* guid = &IID.IID_ID3D11Device)
             fixed (ID3D11Device** pp = &this.device.GetPinnableReference())
@@ -232,7 +235,9 @@ public void SetTexturePipeline(IDalamudTextureWrap textureHandle, ITexturePipeli
     public bool IsImGuiCursor(nint cursorHandle) => this.imguiInput.IsImGuiCursor(cursorHandle);
 
     /// <inheritdoc/>
-    public bool IsAttachedToPresentationTarget(nint targetHandle) => this.swapChain.Get() == (void*)targetHandle;
+    public bool IsAttachedToPresentationTarget(nint targetHandle) =>
+        this.swapChain.Get() == (void*)targetHandle
+        || this.swapChainPossiblyWrapped.Get() == (void*)targetHandle;
 
     /// <inheritdoc/>
     public bool IsMainViewportFullScreen()
@@ -319,5 +324,6 @@ private void ReleaseUnmanagedResources()
         this.swapChain.Dispose();
         this.deviceContext.Dispose();
         this.device.Dispose();
+        this.swapChainPossiblyWrapped.Dispose();
     }
 }

Dalamud/ImGuiScene/Implementations/Dx12OnDx11Win32Scene.cs

@@ -13,8 +13,6 @@
 using TerraFX.Interop.DirectX;
 using TerraFX.Interop.Windows;
 
-using Win32 = TerraFX.Interop.Windows.Windows;
-
 namespace Dalamud.ImGuiScene.Implementations;
 
 /// <summary>
@@ -28,6 +26,7 @@ internal unsafe class Dx12OnDx11Win32Scene : IWin32Scene
 {
     private readonly Dx12Win32Scene scene12;
     private readonly ComPtr<ID3D11ShaderResourceView>[] shaderResourceViewsD3D11;
+    private ComPtr<IDXGISwapChain> swapChainPossiblyWrapped;
     private ComPtr<IDXGISwapChain> swapChain;
     private ComPtr<ID3D11Device1> device11;
     private ComPtr<ID3D11DeviceContext> deviceContext;
@@ -46,8 +45,10 @@ public Dx12OnDx11Win32Scene(IDXGISwapChain* swapChain)
     {
         try
         {
-            swapChain->AddRef();
-            this.swapChain.Attach(swapChain);
+            this.swapChainPossiblyWrapped = new(swapChain);
+            this.swapChain = new(swapChain);
+            fixed (ComPtr<IDXGISwapChain>* ppSwapChain = &this.swapChain)
+                ReShadePeeler.PeelSwapChain(ppSwapChain);
 
             fixed (Guid* guid = &IID.IID_ID3D11Device1)
             fixed (ID3D11Device1** pp = &this.device11.GetPinnableReference())
@@ -235,7 +236,9 @@ public void OnPostResize(int newWidth, int newHeight)
     public bool IsImGuiCursor(nint cursorHandle) => this.scene12.IsImGuiCursor(cursorHandle);
 
     /// <inheritdoc/>
-    public bool IsAttachedToPresentationTarget(nint targetHandle) => this.swapChain.Get() == (void*)targetHandle;
+    public bool IsAttachedToPresentationTarget(nint targetHandle) =>
+        this.swapChain.Get() == (void*)targetHandle
+        || this.swapChainPossiblyWrapped.Get() == (void*)targetHandle;
 
     /// <inheritdoc/>
     public bool IsMainViewportFullScreen()
@@ -293,6 +296,7 @@ private void ReleaseUnmanagedResources()
             s.Reset();
         this.device12.Reset();
         this.drawsOneSquare.Dispose();
+        this.swapChainPossiblyWrapped.Dispose();
     }
 
     private struct DrawsOneSquare : IDisposable

Dalamud/ImGuiScene/Implementations/Dx12Renderer.Queues.cs

@@ -2,6 +2,7 @@
 using System.Runtime.InteropServices;
 using System.Threading;
 
+using Dalamud.ImGuiScene.Helpers;
 using Dalamud.Utility;
 
 using TerraFX.Interop.DirectX;
@@ -133,6 +134,8 @@ public CommandQueueWrapper(ID3D12Device* device, ID3D12CommandQueue* queue, stri
             try
             {
                 this.queue = new(queue);
+                fixed (ComPtr<ID3D12CommandQueue>* ppQueue = &this.queue)
+                    ReShadePeeler.PeelD3D12CommandQueue(ppQueue);
                 fixed (void* pName = $"{debugName}:{nameof(this.queue)}")
                     this.queue.Get()->SetName((ushort*)pName).ThrowHr();