Rework DX12 overlay to use Present to render the overlay

overlay_experimental/windows/DX12_Hook.cpp

@@ -30,8 +30,7 @@ bool DX12_Hook::start_hook()
             std::make_pair<void**, void*>(&(PVOID&)DX12_Hook::Present, &DX12_Hook::MyPresent),
             std::make_pair<void**, void*>(&(PVOID&)DX12_Hook::ResizeTarget, &DX12_Hook::MyResizeTarget),
             std::make_pair<void**, void*>(&(PVOID&)DX12_Hook::ResizeBuffers, &DX12_Hook::MyResizeBuffers),
-            std::make_pair<void**, void*>(&(PVOID&)DX12_Hook::ExecuteCommandLists, &DX12_Hook::MyExecuteCommandLists),
+            std::make_pair<void**, void*>(&(PVOID&)DX12_Hook::ExecuteCommandLists, &DX12_Hook::MyExecuteCommandLists)
-            std::make_pair<void**, void*>(&(PVOID&)DX12_Hook::Close, &DX12_Hook::MyClose)
         );
         EndHook();
 
@@ -45,6 +44,11 @@ void DX12_Hook::resetRenderState()
     if (initialized)
     {
         pSrvDescHeap->Release();
+        for (UINT i = 0; i < bufferCount; ++i)
+            pCmdAlloc[i]->Release();
+        pRtvDescHeap->Release();
+        delete[]pMainRenderTargets;
+        delete[]pCmdAlloc;
 
         ImGui_ImplDX12_Shutdown();
         Windows_Hook::Inst()->resetRenderState();
@@ -57,14 +61,28 @@ void DX12_Hook::resetRenderState()
 // Try to make this function and overlay's proc as short as possible or it might affect game's fps.
 void DX12_Hook::prepareForOverlay(IDXGISwapChain* pSwapChain)
 {
-    pSwapChain->GetDesc(&sc_desc);
+    if (pCmdQueue == nullptr)
+        return;
+
+    ID3D12CommandQueue* pCmdQueue = this->pCmdQueue;
+
+    IDXGISwapChain3* pSwapChain3 = nullptr;
+    DXGI_SWAP_CHAIN_DESC sc_desc;
+    pSwapChain->QueryInterface(IID_PPV_ARGS(&pSwapChain3));
+    if (pSwapChain3 == nullptr)
+        return;
+
+    pSwapChain3->GetDesc(&sc_desc);
 
     if (!initialized)
     {
+        UINT bufferIndex = pSwapChain3->GetCurrentBackBufferIndex();
         ID3D12Device* pDevice;
-        if (pSwapChain->GetDevice(IID_PPV_ARGS(&pDevice)) != S_OK)
+        if (pSwapChain3->GetDevice(IID_PPV_ARGS(&pDevice)) != S_OK)
             return;
 
+        bufferCount = sc_desc.BufferCount;
+
         {
             D3D12_DESCRIPTOR_HEAP_DESC desc = {};
             desc.Type = D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV;
@@ -73,24 +91,108 @@ void DX12_Hook::prepareForOverlay(IDXGISwapChain* pSwapChain)
             if (pDevice->CreateDescriptorHeap(&desc, IID_PPV_ARGS(&pSrvDescHeap)) != S_OK)
             {
                 pDevice->Release();
+                pSwapChain3->Release();
                 return;
             }
         }
+        {
+            D3D12_DESCRIPTOR_HEAP_DESC desc = {};
+            desc.Type = D3D12_DESCRIPTOR_HEAP_TYPE_RTV;
+            desc.NumDescriptors = bufferCount;
+            desc.Flags = D3D12_DESCRIPTOR_HEAP_FLAG_NONE;
+            desc.NodeMask = 1;
+            if (pDevice->CreateDescriptorHeap(&desc, IID_PPV_ARGS(&pRtvDescHeap)) != S_OK)
+            {
+                pDevice->Release();
+                pSwapChain3->Release();
+                pSrvDescHeap->Release();
+                return;
+            }
+        
+            SIZE_T rtvDescriptorSize = pDevice->GetDescriptorHandleIncrementSize(D3D12_DESCRIPTOR_HEAP_TYPE_RTV);
+            D3D12_CPU_DESCRIPTOR_HANDLE rtvHandle = pRtvDescHeap->GetCPUDescriptorHandleForHeapStart();
+            pMainRenderTargets = new D3D12_CPU_DESCRIPTOR_HANDLE[bufferCount];
+            pCmdAlloc = new ID3D12CommandAllocator * [bufferCount];
+            for (int i = 0; i < bufferCount; ++i)
+            {
+                pMainRenderTargets[i] = rtvHandle;
+                rtvHandle.ptr += rtvDescriptorSize;
+            }
+        }
+
+        for (UINT i = 0; i < sc_desc.BufferCount; ++i)
+        {
+            if (pDevice->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(&pCmdAlloc[i])) != S_OK)
+            {
+                pDevice->Release();
+                pSwapChain3->Release();
+                pSrvDescHeap->Release();
+                for (UINT j = 0; j < i; ++j)
+                {
+                    pCmdAlloc[j]->Release();
+                }
+                pRtvDescHeap->Release();
+                delete[]pMainRenderTargets;
+                delete[]pCmdAlloc;
+                return;
+            }
+        }
+
+        if (pDevice->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, pCmdAlloc[0], NULL, IID_PPV_ARGS(&pCmdList)) != S_OK ||
+            pCmdList->Close() != S_OK)
+        {
+            pDevice->Release();
+            pSwapChain3->Release();
+            pSrvDescHeap->Release();
+            for (UINT i = 0; i < bufferCount; ++i)
+                pCmdAlloc[i]->Release();
+            pRtvDescHeap->Release();
+            delete[]pMainRenderTargets;
+            delete[]pCmdAlloc;
+            return;
+        }
+
+        for (UINT i = 0; i < bufferCount; i++)
+        {
+            ID3D12Resource* pBackBuffer = NULL;
+            pSwapChain3->GetBuffer(i, IID_PPV_ARGS(&pBackBuffer));
+            pDevice->CreateRenderTargetView(pBackBuffer, NULL, pMainRenderTargets[i]);
+        }
 
         ImGui::CreateContext();
         ImGuiIO& io = ImGui::GetIO();
         io.IniFilename = NULL;
     
-        ImGui_ImplDX12_Init(pDevice, 1, DXGI_FORMAT_R8G8B8A8_UNORM,
+        ImGui_ImplDX12_Init(pDevice, bufferCount, DXGI_FORMAT_R8G8B8A8_UNORM,
             pSrvDescHeap->GetCPUDescriptorHandleForHeapStart(),
             pSrvDescHeap->GetGPUDescriptorHandleForHeapStart());
         
-        pDevice->Release();
-
         get_steam_client()->steam_overlay->CreateFonts();
 
         initialized = true;
+
+        pDevice->Release();
     }
+    
+    ImGui_ImplDX12_NewFrame();
+    Windows_Hook::Inst()->prepareForOverlay(sc_desc.OutputWindow);
+
+    ImGui::NewFrame();
+
+    get_steam_client()->steam_overlay->OverlayProc();
+
+    UINT bufferIndex = pSwapChain3->GetCurrentBackBufferIndex();
+
+    pCmdList->Reset(pCmdAlloc[bufferIndex], NULL);
+    pCmdList->OMSetRenderTargets(1, &pMainRenderTargets[bufferIndex], FALSE, NULL);
+    pCmdList->SetDescriptorHeaps(1, &pSrvDescHeap);
+
+    ImGui::Render();
+    ImGui_ImplDX12_RenderDrawData(ImGui::GetDrawData(), pCmdList);
+    pCmdList->Close();
+    pCmdQueue->ExecuteCommandLists(1, (ID3D12CommandList**)&pCmdList);
+
+    pSwapChain3->Release();
 }
 
 HRESULT STDMETHODCALLTYPE DX12_Hook::MyPresent(IDXGISwapChain *_this, UINT SyncInterval, UINT Flags)
@@ -115,52 +217,24 @@ HRESULT STDMETHODCALLTYPE DX12_Hook::MyResizeBuffers(IDXGISwapChain* _this, UINT
 void STDMETHODCALLTYPE DX12_Hook::MyExecuteCommandLists(ID3D12CommandQueue *_this, UINT NumCommandLists, ID3D12CommandList* const* ppCommandLists)
 {
     DX12_Hook* me = DX12_Hook::Inst();
-    // ----------------------------------------------------- //
+    me->pCmdQueue = _this;
-    // \/\/\/ TODO: Find a cleaner way to do all this \/\/\/ //
-    //      I'd like to put it in IDXGISwapChain::Present    //
-    // ----------------------------------------------------- //
-    if (me->initialized)
-    {
-        static std::recursive_mutex render_mutex; // Sniper Elite 4 where I test this uses multiple thread on this. ImGui is not reentrant
-        std::lock_guard<std::recursive_mutex> lock(render_mutex);
-        for (int i = 0; i < NumCommandLists; ++i)
-        {
-            if (((ID3D12GraphicsCommandList*)ppCommandLists[i])->GetType() == D3D12_COMMAND_LIST_TYPE_DIRECT)
-            {
-                ImGui_ImplDX12_NewFrame();
-                Windows_Hook::Inst()->prepareForOverlay(me->sc_desc.OutputWindow);
-
-                ImGui::NewFrame();
-
-                get_steam_client()->steam_overlay->OverlayProc();
-
-                ((ID3D12GraphicsCommandList*)ppCommandLists[i])->SetDescriptorHeaps(1, &me->pSrvDescHeap);
-                ImGui::Render();
-                ImGui_ImplDX12_RenderDrawData(ImGui::GetDrawData(), (ID3D12GraphicsCommandList*)ppCommandLists[i]);
-                (((ID3D12GraphicsCommandList*)ppCommandLists[i])->*me->Close)();
-            }
-        }
-    }
     (_this->*DX12_Hook::Inst()->ExecuteCommandLists)(NumCommandLists, ppCommandLists);
 }
 
-HRESULT STDMETHODCALLTYPE DX12_Hook::MyClose(ID3D12GraphicsCommandList* _this)
-{
-    if (DX12_Hook::Inst()->initialized && _this->GetType() == D3D12_COMMAND_LIST_TYPE_DIRECT)
-        return S_OK;
-    else
-        return (_this->*DX12_Hook::Inst()->Close)();
-}
-
 DX12_Hook::DX12_Hook():
     initialized(false),
-    hooked(false),
+    pCmdQueue(nullptr),
+    bufferCount(0),
+    pMainRenderTargets(nullptr),
+    pCmdAlloc(nullptr),
     pSrvDescHeap(nullptr),
+    pCmdList(nullptr),
+    pRtvDescHeap(nullptr),
+    hooked(false),
     Present(nullptr),
     ResizeBuffers(nullptr),
     ResizeTarget(nullptr),
-    ExecuteCommandLists(nullptr),
+    ExecuteCommandLists(nullptr)
-    Close(nullptr)
 {
     _library = LoadLibrary(DLL_NAME);
 
@@ -197,7 +271,7 @@ const char* DX12_Hook::get_lib_name() const
     return DLL_NAME;
 }
 
-void DX12_Hook::loadFunctions(ID3D12CommandQueue* pCommandQueue, ID3D12GraphicsCommandList* pCommandList, IDXGISwapChain *pSwapChain)
+void DX12_Hook::loadFunctions(ID3D12CommandQueue* pCommandQueue, IDXGISwapChain *pSwapChain)
 {
     void** vTable;
     
@@ -206,11 +280,6 @@ void DX12_Hook::loadFunctions(ID3D12CommandQueue* pCommandQueue, ID3D12GraphicsC
     LOAD_FUNC(ExecuteCommandLists);
 #undef LOAD_FUNC
 
-    vTable = *reinterpret_cast<void***>(pCommandList);
-#define LOAD_FUNC(X) (void*&)X = vTable[(int)ID3D12GraphicsCommandListVTable::X]
-    LOAD_FUNC(Close);
-#undef LOAD_FUNC
-
     vTable = *reinterpret_cast<void***>(pSwapChain);
 #define LOAD_FUNC(X) (void*&)X = vTable[(int)IDXGISwapChainVTable::X]
     LOAD_FUNC(Present);

overlay_experimental/windows/DX12_Hook.h

@@ -20,8 +20,13 @@ private:
     bool hooked;
     bool initialized;
 
-    DXGI_SWAP_CHAIN_DESC sc_desc;
+    ID3D12CommandQueue* pCmdQueue;
+    UINT bufferCount;
+    D3D12_CPU_DESCRIPTOR_HANDLE* pMainRenderTargets;
+    ID3D12CommandAllocator** pCmdAlloc;
     ID3D12DescriptorHeap* pSrvDescHeap;
+    ID3D12GraphicsCommandList* pCmdList;
+    ID3D12DescriptorHeap* pRtvDescHeap;
 
     // Functions
     DX12_Hook();
@@ -34,13 +39,11 @@ private:
     static HRESULT STDMETHODCALLTYPE MyResizeTarget(IDXGISwapChain* _this, const DXGI_MODE_DESC* pNewTargetParameters);
     static HRESULT STDMETHODCALLTYPE MyResizeBuffers(IDXGISwapChain* _this, UINT BufferCount, UINT Width, UINT Height, DXGI_FORMAT NewFormat, UINT SwapChainFlags);
     static void STDMETHODCALLTYPE MyExecuteCommandLists(ID3D12CommandQueue *_this, UINT NumCommandLists, ID3D12CommandList* const* ppCommandLists);
-    static HRESULT STDMETHODCALLTYPE MyClose(ID3D12GraphicsCommandList* _this);
 
     decltype(&IDXGISwapChain::Present)       Present;
     decltype(&IDXGISwapChain::ResizeBuffers) ResizeBuffers;
     decltype(&IDXGISwapChain::ResizeTarget)  ResizeTarget;
     decltype(&ID3D12CommandQueue::ExecuteCommandLists) ExecuteCommandLists;
-    decltype(&ID3D12GraphicsCommandList::Close) Close;
 
 public:
     virtual ~DX12_Hook();
@@ -49,7 +52,7 @@ public:
     static DX12_Hook* Inst();
     virtual const char* get_lib_name() const;
 
-    void loadFunctions(ID3D12CommandQueue* pCommandQueue, ID3D12GraphicsCommandList* pCommandList, IDXGISwapChain* pSwapChain);
+    void loadFunctions(ID3D12CommandQueue* pCommandQueue, IDXGISwapChain* pSwapChain);
 };
 
 #endif//NO_OVERLAY