Refactor dx device creation

Find the game's window instead of using the foreground window.

overlay_experimental/DX10_Hook.cpp

@@ -17,6 +17,10 @@ bool DX10_Hook::start_hook()
         if (!Windows_Hook::Inst().start_hook())
             return false;
 
+        HWND hWnd = GetGameWindow();
+        if (!hWnd)
+            return false;
+
         IDXGISwapChain* pSwapChain;
         ID3D10Device* pDevice;
         DXGI_SWAP_CHAIN_DESC SwapChainDesc = {};
@@ -29,7 +33,7 @@ bool DX10_Hook::start_hook()
         SwapChainDesc.BufferDesc.RefreshRate.Numerator = 0;
         SwapChainDesc.BufferDesc.RefreshRate.Denominator = 0;
         SwapChainDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
-        SwapChainDesc.OutputWindow = GetForegroundWindow();
+        SwapChainDesc.OutputWindow = hWnd;
         SwapChainDesc.SampleDesc.Count = 1;
         SwapChainDesc.SampleDesc.Quality = 0;
         SwapChainDesc.Windowed = TRUE;

overlay_experimental/DX11_Hook.cpp

@@ -27,6 +27,10 @@ bool DX11_Hook::start_hook()
         if (!Windows_Hook::Inst().start_hook())
             return false;
 
+        HWND hWnd = GetGameWindow();
+        if (!hWnd)
+            return false;
+
         IDXGISwapChain* pSwapChain;
         ID3D11Device* pDevice;
         DXGI_SWAP_CHAIN_DESC SwapChainDesc = {};
@@ -39,7 +43,7 @@ bool DX11_Hook::start_hook()
         SwapChainDesc.BufferDesc.RefreshRate.Numerator = 0;
         SwapChainDesc.BufferDesc.RefreshRate.Denominator = 0;
         SwapChainDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
-        SwapChainDesc.OutputWindow = GetForegroundWindow();
+        SwapChainDesc.OutputWindow = hWnd;
         SwapChainDesc.SampleDesc.Count = 1;
         SwapChainDesc.SampleDesc.Quality = 0;
         SwapChainDesc.Windowed = TRUE;

overlay_experimental/DX12_Hook.cpp

@@ -19,6 +19,10 @@ bool DX12_Hook::start_hook()
         if (!Windows_Hook::Inst().start_hook())
             return false;
 
+        HWND hWnd = GetGameWindow();
+        if (!hWnd)
+            return false;
+
         IDXGIFactory4* pDXGIFactory = nullptr;
         IDXGISwapChain1* pSwapChain = nullptr;
         D3D12_COMMAND_QUEUE_DESC queueDesc = {};
@@ -50,7 +54,7 @@ bool DX12_Hook::start_hook()
             if (pCommandQueue)
             {
                 reinterpret_cast<decltype(CreateDXGIFactory1)*>(GetProcAddress(GetModuleHandle("dxgi.dll"), "CreateDXGIFactory1"))(IID_PPV_ARGS(&pDXGIFactory));
-                pDXGIFactory->CreateSwapChainForHwnd(pCommandQueue, GetForegroundWindow(), &SwapChainDesc, NULL, NULL, &pSwapChain);
+                pDXGIFactory->CreateSwapChainForHwnd(pCommandQueue, hWnd, &SwapChainDesc, NULL, NULL, &pSwapChain);
 
                 if (pSwapChain != nullptr)
                 {

overlay_experimental/DX9_Hook.cpp

@@ -19,6 +19,10 @@ bool DX9_Hook::start_hook()
         if (!Windows_Hook::Inst().start_hook())
             return false;
 
+        HWND hWnd = GetGameWindow();
+        if (!hWnd)
+            return false;
+
         IDirect3D9Ex* pD3D;
         IDirect3DDevice9Ex* pDeviceEx;
         decltype(Direct3DCreate9Ex)* Direct3DCreate9Ex = (decltype(Direct3DCreate9Ex))GetProcAddress(reinterpret_cast<HMODULE>(_library), "Direct3DCreate9Ex");
@@ -28,7 +32,7 @@ bool DX9_Hook::start_hook()
         D3DPRESENT_PARAMETERS params = {};
         params.BackBufferWidth = 1;
         params.BackBufferHeight = 1;
-        params.hDeviceWindow = GetForegroundWindow();
+        params.hDeviceWindow = hWnd;
 
         pD3D->CreateDeviceEx(D3DADAPTER_DEFAULT, D3DDEVTYPE_NULLREF, NULL, D3DCREATE_HARDWARE_VERTEXPROCESSING, &params, NULL, &pDeviceEx);
 

overlay_experimental/Hook_Manager.cpp

@@ -10,12 +10,15 @@
 #include "DX11_Hook.h"
 #include "DX10_Hook.h"
 #include "DX9_Hook.h"
+#include "Windows_Hook.h"
 #endif
 
 #include "OpenGL_Hook.h"
 
 #include <algorithm>
 
+constexpr int max_hook_retries = 500;
+
 #ifdef STEAM_WIN32
 static decltype(&IDXGISwapChain::Present) _IDXGISwapChain_Present;
 static decltype(&IDirect3DDevice9::Present) _IDirect3DDevice9_Present;
@@ -147,6 +150,10 @@ void Hook_Manager::hook_dx9()
 {
     if (!_dx9_hooked && !_renderer_found)
     {
+        HWND hWnd = GetGameWindow();
+        if (!hWnd)
+            return;
+
         IDirect3D9Ex* pD3D;
         IDirect3DDevice9Ex* pDeviceEx;
         decltype(Direct3DCreate9Ex)* Direct3DCreate9Ex = (decltype(Direct3DCreate9Ex))GetProcAddress(GetModuleHandle(DX9_Hook::DLL_NAME), "Direct3DCreate9Ex");
@@ -156,7 +163,7 @@ void Hook_Manager::hook_dx9()
         D3DPRESENT_PARAMETERS params = {};
         params.BackBufferWidth = 1;
         params.BackBufferHeight = 1;
-        params.hDeviceWindow = GetForegroundWindow();
+        params.hDeviceWindow = hWnd;
 
         pD3D->CreateDeviceEx(D3DADAPTER_DEFAULT, D3DDEVTYPE_NULLREF, NULL, D3DCREATE_HARDWARE_VERTEXPROCESSING, &params, NULL, &pDeviceEx);
 
@@ -181,6 +188,10 @@ void Hook_Manager::hook_dx10()
 {
     if (!_dxgi_hooked && !_renderer_found)
     {
+        HWND hWnd = GetGameWindow();
+        if (!hWnd)
+            return;
+
         IDXGISwapChain* pSwapChain;
         ID3D10Device* pDevice;
         DXGI_SWAP_CHAIN_DESC SwapChainDesc = {};
@@ -193,7 +204,7 @@ void Hook_Manager::hook_dx10()
         SwapChainDesc.BufferDesc.RefreshRate.Numerator = 0;
         SwapChainDesc.BufferDesc.RefreshRate.Denominator = 0;
         SwapChainDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
-        SwapChainDesc.OutputWindow = GetForegroundWindow();
+        SwapChainDesc.OutputWindow = hWnd;
         SwapChainDesc.SampleDesc.Count = 1;
         SwapChainDesc.SampleDesc.Quality = 0;
         SwapChainDesc.Windowed = TRUE;
@@ -219,6 +230,10 @@ void Hook_Manager::hook_dx11()
 {
     if (!_dxgi_hooked && !_renderer_found)
     {
+        HWND hWnd = GetGameWindow();
+        if (!hWnd)
+            return;
+
         IDXGISwapChain* pSwapChain;
         ID3D11Device* pDevice;
         DXGI_SWAP_CHAIN_DESC SwapChainDesc = {};
@@ -231,7 +246,7 @@ void Hook_Manager::hook_dx11()
         SwapChainDesc.BufferDesc.RefreshRate.Numerator = 0;
         SwapChainDesc.BufferDesc.RefreshRate.Denominator = 0;
         SwapChainDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
-        SwapChainDesc.OutputWindow = GetForegroundWindow();
+        SwapChainDesc.OutputWindow = hWnd;
         SwapChainDesc.SampleDesc.Count = 1;
         SwapChainDesc.SampleDesc.Quality = 0;
         SwapChainDesc.Windowed = TRUE;
@@ -258,6 +273,10 @@ void Hook_Manager::hook_dx12()
 {
     if (!_dxgi_hooked && !_renderer_found)
     {
+        HWND hWnd = GetGameWindow();
+        if (!hWnd)
+            return;
+
         IDXGIFactory4* pDXGIFactory = nullptr;
         IDXGISwapChain1* pSwapChain = nullptr;
         D3D12_COMMAND_QUEUE_DESC queueDesc = {};
@@ -289,7 +308,7 @@ void Hook_Manager::hook_dx12()
             if (pCommandQueue)
             {
                 reinterpret_cast<decltype(CreateDXGIFactory1)*>(GetProcAddress(GetModuleHandle("dxgi.dll"), "CreateDXGIFactory1"))(IID_PPV_ARGS(&pDXGIFactory));
-                pDXGIFactory->CreateSwapChainForHwnd(pCommandQueue, GetForegroundWindow(), &SwapChainDesc, NULL, NULL, &pSwapChain);
+                pDXGIFactory->CreateSwapChainForHwnd(pCommandQueue, hWnd, &SwapChainDesc, NULL, NULL, &pSwapChain);
                 if (pSwapChain != nullptr)
                 {
                     PRINT_DEBUG("Hooked IDXGISwapChain::Present to detect DX Version\n");
@@ -378,13 +397,10 @@ void Hook_Manager::create_hook(const char* libname)
 bool Hook_Manager::stop_retry()
 {
     // Retry 200 times, we look for rendering functions so its actually: "retry for 200 frames"
-    bool stop = ++_hook_retries >= 200;
+    bool stop = ++_hook_retries >= max_hook_retries;
 
     if (stop)
-    {
-        PRINT_DEBUG("We found a renderer but couldn't hook it, aborting overlay hook.\n");
         FoundRenderer(nullptr);
-    }
 
     return stop;
 }
@@ -457,6 +473,11 @@ void Hook_Manager::FoundRenderer(Base_Hook* hook)
     {
         _renderer_found = true;
 
+        if (hook == nullptr)
+            PRINT_DEBUG("We found a renderer but couldn't hook it, aborting overlay hook.\n");
+        else
+            PRINT_DEBUG("Hooked renderer in %d tries\n", _hook_retries);
+
         _hook_thread->join();
         delete _hook_thread;
 

overlay_experimental/Windows_Hook.cpp

@@ -7,6 +7,26 @@ extern LRESULT ImGui_ImplWin32_WndProcHandler(HWND hWnd, UINT msg, WPARAM wParam
 
 #include "../dll/dll.h"
 
+HWND GetGameWindow()
+{
+    HWND hWnd = FindWindow(NULL, NULL);
+    while (hWnd)
+    {
+        if (!GetParent(hWnd))
+        {
+#if defined(_WIN64)
+            if (GetModuleHandle(NULL) == (HINSTANCE)GetWindowLong(hWnd, GWLP_HINSTANCE))
+                break;
+#elif defined(_WIN32)
+            if (GetModuleHandle(NULL) == (HINSTANCE)GetWindowLong(hWnd, GWL_HINSTANCE))      
+                break;
+#endif
+        }
+        hWnd = GetWindow(hWnd, GW_HWNDNEXT);
+    }
+    return hWnd;
+}
+
 bool Windows_Hook::start_hook()
 {
     if (!_hooked)

overlay_experimental/Windows_Hook.h

@@ -42,5 +42,7 @@ public:
     static Windows_Hook& Inst();
 };
 
+HWND GetGameWindow();
+
 #endif//NO_OVERLAY
 #endif//__INCLUDED_WINDOWS_HOOK_H__