Added error handling when hook fails

overlay_experimental/DX10_Hook.cpp

@@ -12,7 +12,7 @@
 // This is created by DX10_Hook::Create, and deleted by the Hook_Manager if not used
 static DX10_Hook* hook;
 
-void DX10_Hook::start_hook()
+bool DX10_Hook::start_hook()
 {
     if (!_hooked)
     {
@@ -55,10 +55,12 @@ void DX10_Hook::start_hook()
         else
         {
             PRINT_DEBUG("Failed to hook DirectX 10\n");
+            return false;
         }
         if(pDevice)pDevice->Release();
         if(pSwapChain)pSwapChain->Release();
     }
+    return true;
 }
 
 void DX10_Hook::resetRenderState()

overlay_experimental/DX10_Hook.h

@@ -22,7 +22,6 @@ private:
     DX10_Hook();
     virtual ~DX10_Hook();
 
-    void start_hook();
     void resetRenderState();
     void prepareForOverlay(IDXGISwapChain *pSwapChain);
 
@@ -43,7 +42,8 @@ private:
     //decltype(D3D10CreateDeviceAndSwapChain)* D3D10CreateDeviceAndSwapChain;
 
 public:
-    static void Create(); // Initialize DX10 Hook.
+    bool start_hook();
+    static DX10_Hook* Inst();
     
     void loadFunctions(ID3D10Device *pDevice, IDXGISwapChain *pSwapChain);
 };

overlay_experimental/DX11_Hook.cpp

@@ -22,7 +22,7 @@ HRESULT GetDeviceAndCtxFromSwapchain(IDXGISwapChain* pSwapChain, ID3D11Device**
     return ret;
 }
 
-void DX11_Hook::start_hook()
+bool DX11_Hook::start_hook()
 {
     if (!_hooked)
     {
@@ -65,11 +65,13 @@ void DX11_Hook::start_hook()
         else
         {
             PRINT_DEBUG("Failed to hook DirectX 11\n");
+            return false;
         }
 
         if(pDevice) pDevice->Release();
         if(pSwapChain) pSwapChain->Release();
     }
+    return true;
 }
 
 void DX11_Hook::resetRenderState()

overlay_experimental/DX11_Hook.h

@@ -22,7 +22,6 @@ private:
     DX11_Hook();
     virtual ~DX11_Hook();
 
-    void start_hook();
     void resetRenderState();
     void prepareForOverlay(IDXGISwapChain* pSwapChain);
 
@@ -43,7 +42,8 @@ private:
     //decltype(D3D11CreateDeviceAndSwapChain)* D3D11CreateDeviceAndSwapChain;
 
 public:
-    static void Create(); // Initialize DX11 Hook.
+    bool start_hook();
+    static DX11_Hook* Inst();
     
     void loadFunctions(ID3D11Device *pDevice, IDXGISwapChain *pSwapChain);
 };

overlay_experimental/DX12_Hook.cpp

@@ -9,15 +9,14 @@
 #include <impls/imgui_impl_win32.h>
 #include <impls/imgui_impl_dx12.h>
 
-// This is created by DX12_Hook::Create, and deleted by the Hook_Manager if not used
-static DX12_Hook* hook;
-
-void DX12_Hook::start_hook()
+bool DX12_Hook::start_hook()
 {
     if (!_hooked)
     {
         PRINT_DEBUG("Hooked DirectX 12\n");
+        return false;
     }
+    return true;
 }
 
 void DX12_Hook::resetRenderState()

overlay_experimental/DX12_Hook.h

@@ -24,7 +24,6 @@ private:
     DX12_Hook();
     virtual ~DX12_Hook();
 
-    void start_hook();
     void resetRenderState();
     void prepareForOverlay(IDXGISwapChain* pSwapChain);
 
@@ -43,7 +42,8 @@ private:
     //decltype(D3D12CreateDevice)* D3D12CreateDevice;
 
 public:
-    static void Create(); // Initialize DX11 Hook.
+    bool start_hook();
+    static DX12_Hook* Inst();
     
     void loadFunctions(ID3D12Device *pDevice, IDXGISwapChain *pSwapChain);
 };

overlay_experimental/DX9_Hook.cpp

@@ -18,7 +18,7 @@ static DX9_Hook* hook;
 /////////                                                /////////
 //////////////////////////////////////////////////////////////////
 
-void DX9_Hook::start_hook()
+bool DX9_Hook::start_hook()
 {
     if (!_hooked)
     {
@@ -56,11 +56,13 @@ void DX9_Hook::start_hook()
         else
         {
             PRINT_DEBUG("Failed to DirectX 9\n");
+            return false;
         }
 
         if(pDeviceEx)pDeviceEx->Release();
         if(pD3D)pD3D->Release();
     }
+    return true;
 }
 
 void DX9_Hook::resetRenderState()

overlay_experimental/DX9_Hook.h

@@ -21,7 +21,6 @@ private:
     DX9_Hook();
     virtual ~DX9_Hook();
 
-    void start_hook();
     void resetRenderState();
     void prepareForOverlay(IDirect3DDevice9* pDevice);
 
@@ -44,7 +43,8 @@ private:
     //decltype(Direct3DCreate9Ex)* Direct3DCreate9Ex;
 
 public:
-    static void Create(); // Initialize DX9 Hook.
+    bool start_hook();
+    static DX9_Hook* Inst();
     
     void loadFunctions(IDirect3DDevice9Ex *pDeviceEx);
 };

overlay_experimental/Hook_Manager.cpp

@@ -34,7 +34,17 @@ HRESULT STDMETHODCALLTYPE Hook_Manager::MyIDXGISwapChain_Present(IDXGISwapChain*
     if (pDevice)
     {
         Hook_Manager::Inst().UnHookAllRendererDetector();
-        DX10_Hook::Create();
+        DX10_Hook* hook = DX10_Hook::Inst();
+        if (!hook->start_hook())
+        {
+            // Hook failed, start over
+            delete static_cast<Base_Hook*>(hook);
+            Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay());
+        }
+        else
+        {
+            Hook_Manager::Inst().AddHook(hook);
+        }
     }
     else
     {
@@ -42,15 +52,31 @@ HRESULT STDMETHODCALLTYPE Hook_Manager::MyIDXGISwapChain_Present(IDXGISwapChain*
         if (pDevice)
         {
             Hook_Manager::Inst().UnHookAllRendererDetector();
-            DX11_Hook::Create();
+            DX11_Hook* hook = DX11_Hook::Inst();
+            if (!hook->start_hook())
+            {
+                // Hook failed, start over
+                delete static_cast<Base_Hook*>(hook);
+                Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay());
+            }
+            else
+            {
+                Hook_Manager::Inst().AddHook(hook);
+            }
         }
         else
         {
             _this->GetDevice(__uuidof(ID3D12Device), (void**)& pDevice);
-            if (pDevice)
+            DX12_Hook* hook = DX12_Hook::Inst();
+            if (!hook->start_hook())
             {
-                Hook_Manager::Inst().UnHookAllRendererDetector();
-                DX12_Hook::Create();
+                // Hook failed, start over
+                delete static_cast<Base_Hook*>(hook);
+                Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay());
+            }
+            else
+            {
+                Hook_Manager::Inst().AddHook(hook);
             }
         }
     }
@@ -62,28 +88,60 @@ HRESULT STDMETHODCALLTYPE Hook_Manager::MyIDXGISwapChain_Present(IDXGISwapChain*
 HRESULT STDMETHODCALLTYPE Hook_Manager::MyPresent(IDirect3DDevice9* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion)
 {
     Hook_Manager::Inst().UnHookAllRendererDetector();
-    DX9_Hook::Create();
+    DX9_Hook* hook = DX9_Hook::Inst();
+    if (!hook->start_hook())
+    {
+        // Hook failed, start over
+        delete static_cast<Base_Hook*>(hook);
+        Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay());
+    }
+    else
+    {
+        Hook_Manager::Inst().AddHook(hook);
+    }
     return (_this->*_IDirect3DDevice9_Present)(pSourceRect, pDestRect, hDestWindowOverride, pDirtyRegion);
 }
 
 HRESULT STDMETHODCALLTYPE Hook_Manager::MyPresentEx(IDirect3DDevice9Ex* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion, DWORD dwFlags)
 {
     Hook_Manager::Inst().UnHookAllRendererDetector();
-    DX9_Hook::Create();
+    DX9_Hook* hook = DX9_Hook::Inst();
+    if (!hook->start_hook())
+    {
+        // Hook failed, start over
+        delete static_cast<Base_Hook*>(hook);
+        Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay());
+    }
+    else
+    {
+        Hook_Manager::Inst().AddHook(hook);
+    }
     return (_this->*_IDirect3DDevice9Ex_PresentEx)(pSourceRect, pDestRect, hDestWindowOverride, pDirtyRegion, dwFlags);
 }
 
 BOOL WINAPI Hook_Manager::MywglMakeCurrent(HDC hDC, HGLRC hGLRC)
 {
     Hook_Manager::Inst().UnHookAllRendererDetector();
-    OpenGL_Hook::Create();
-    return _wglMakeCurrent(hDC, hGLRC);;
+    OpenGL_Hook* hook = OpenGL_Hook::Inst();
+    if (!hook->start_hook())
+    {
+        // Hook failed, start over
+        delete static_cast<Base_Hook*>(hook);
+        Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay());
+    }
+    else
+    {
+        Hook_Manager::Inst().AddHook(hook);
+    }
+
+    return _wglMakeCurrent(hDC, hGLRC);
 }
 
 void Hook_Manager::HookDXGIPresent()
 {
     if (!_dxgi_hooked)
     {
+        _dxgi_hooked = true;
         rendererdetect_hook->BeginHook();
 
         rendererdetect_hook->HookFuncs(
@@ -98,6 +156,7 @@ void Hook_Manager::HookDX9Present()
 {
     if (!_dx9_hooked)
     {
+        _dx9_hooked = true;
         rendererdetect_hook->BeginHook();
 
         rendererdetect_hook->HookFuncs(
@@ -113,6 +172,7 @@ void Hook_Manager::HookwglMakeCurrent()
 {
     if (!_ogl_hooked)
     {
+        _ogl_hooked = true;
         rendererdetect_hook->BeginHook();
 
         rendererdetect_hook->HookFuncs(
@@ -238,7 +298,9 @@ void Hook_Manager::hook_dx12()
 {
     if (!_dxgi_hooked && !_renderer_found)
     {
-        DX12_Hook::Create();
+        DX12_Hook* hook = DX12_Hook::Inst();
+        hook->start_hook(); // TODO: Prints to error log about DX12 Implementation status
+        delete static_cast<Base_Hook*>(hook);
     }
 }
 
@@ -350,6 +412,7 @@ void Hook_Manager::UnHookAllRendererDetector()
         delete rendererdetect_hook;
         rendererdetect_hook = nullptr;
     }
+    _loadlibrary_hooked = false;
     _ogl_hooked = false;
 
 #ifdef STEAM_WIN32

overlay_experimental/Hook_Manager.h

@@ -40,7 +40,7 @@ protected:
     virtual ~Hook_Manager();
     
     void UnHookAllRendererDetector();
-    
+    // Setup opengl device
     void hook_opengl();
 
     void HookLoadLibrary();
@@ -52,13 +52,21 @@ protected:
     bool _dx9_hooked;           // DX9 Present and PresentEx Hooked ?
     bool _dxgi_hooked;          // DXGI Present is hooked ? (DX10, DX11, DX12)
 
+    // DXGIPresent will be used to detect if DX10, DX11 or DX12 should be used for overlay
     void HookDXGIPresent();
+    // DX9 Present and PresentEx will be used to detect if DX9 should be used for overlay
     void HookDX9Present();
+    // wglMakeCurrent will be used to detect if OpenGL3 should be used for overlay
     void HookwglMakeCurrent();
+    // Setup DX9 Device and get vtable
     void hook_dx9();
+    // Setup DX10 Device and get vtable
     void hook_dx10();
+    // Setup DX11 Device and get vtable
     void hook_dx11();
+    // Setup DX12 Device and get vtable
     void hook_dx12();
+
     void create_hookA(const char* libname);
     void create_hookW(const wchar_t* libname);
 
@@ -66,9 +74,12 @@ protected:
     static HMODULE WINAPI MyLoadLibraryW(LPCWSTR lpLibFileName);
     static HMODULE WINAPI MyLoadLibraryExA(LPCTSTR lpLibFileName, HANDLE hFile, DWORD dwFlags);
     static HMODULE WINAPI MyLoadLibraryExW(LPCWSTR lpLibFileName, HANDLE hFile, DWORD dwFlags);
+    // If this is called, then DX10, DX11 or DX12 will be used to render overlay
     static HRESULT STDMETHODCALLTYPE MyIDXGISwapChain_Present(IDXGISwapChain* _this, UINT SyncInterval, UINT Flags);
+    // If any of theses is called, then DX9 will be used to render overlay
     static HRESULT STDMETHODCALLTYPE MyPresent(IDirect3DDevice9* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion);
     static HRESULT STDMETHODCALLTYPE MyPresentEx(IDirect3DDevice9Ex* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion, DWORD dwFlags);
+    // If this is called, then OpenGL 3 will be used to render overlay
     static BOOL WINAPI MywglMakeCurrent(HDC hDC, HGLRC hGLRC);
 
 public:

overlay_experimental/OpenGL_Hook.cpp

@@ -13,10 +13,9 @@
 
 #include "steam_overlay.h"
 
-// This is created by OpenGL_Hook::Create, and deleted by the Hook_Manager if not used
-static OpenGL_Hook* hook;
+OpenGL_Hook* OpenGL_Hook::_inst = nullptr;
 
-void OpenGL_Hook::start_hook()
+bool OpenGL_Hook::start_hook()
 {
     if (!_hooked)
     {
@@ -40,8 +39,10 @@ void OpenGL_Hook::start_hook()
             PRINT_DEBUG("Failed to hook OpenGL\n");
             /* Problem: glewInit failed, something is seriously wrong. */
             PRINT_DEBUG("Error: %s\n", glewGetErrorString(err));
+            return false;
         }
     }
+    return true;
 }
 
 void OpenGL_Hook::resetRenderState()
@@ -106,8 +107,8 @@ void OpenGL_Hook::prepareForOverlay(HDC hDC)
 /////////////////////////////////////////////////////////////////////////////////////
 BOOL WINAPI OpenGL_Hook::MywglSwapBuffers(HDC hDC)
 {
-    hook->prepareForOverlay(hDC);
-    return hook->wglSwapBuffers(hDC);
+    OpenGL_Hook::Inst()->prepareForOverlay(hDC);
+    return OpenGL_Hook::Inst()->wglSwapBuffers(hDC);
 }
 
 OpenGL_Hook::OpenGL_Hook():
@@ -138,18 +139,15 @@ OpenGL_Hook::~OpenGL_Hook()
         ImGui::DestroyContext();
     }
 
-    hook = nullptr;
+    _inst = nullptr;
 }
 
-void OpenGL_Hook::Create()
+OpenGL_Hook* OpenGL_Hook::Inst()
 {
-    if (hook == nullptr)
-    {
-        hook = new OpenGL_Hook;
-        hook->start_hook();
-        // Register the hook to the Hook Manager
-        Hook_Manager::Inst().AddHook(hook);
-    }
+    if (_inst == nullptr)
+        _inst = new OpenGL_Hook;
+
+    return _inst;
 }
 
 #endif//NO_OVERLAY

overlay_experimental/OpenGL_Hook.h

@@ -13,6 +13,8 @@ public:
     //using wglMakeCurrent_t = BOOL(WINAPI*)(HDC, HGLRC);
 
 private:
+    static OpenGL_Hook* _inst;
+
     // Variables
     bool initialized;
 
@@ -20,7 +22,6 @@ private:
     OpenGL_Hook();
     virtual ~OpenGL_Hook();
 
-    void start_hook();
     void resetRenderState();
     void prepareForOverlay(HDC hDC);
 
@@ -35,7 +36,8 @@ private:
     //wglMakeCurrent_t wglMakeCurrent;
 
 public:
-    static void Create(); // Initialize OGL Hook.
+    bool start_hook();
+    inline static OpenGL_Hook* Inst();
 };
 
 #endif//NO_OVERLAY