Where to start for creating a D3D11 Hook to make a game overlay?

Question

I am currently working on my first D3D project, which involves creating an overlay for a game in order to present the user with real time information. The actual data acquisition has been handled, and will be sent in via a message system. Currently, I have the program working for borderless fullscreen mode, and am in the process of adapting it to scale proportionally when in windowed mode (to ensure that everything points to the right place, regardless of size). The app is parented to the game, however, everything falls apart when the game is put into true fullscreen mode. Research has indicated that I will, to my understanding, need to create a fake .dll, and insert it into the game's local directory. This .dll will be opened instead of the actual directX one that it is looking for, and I can do the needed graphics there.

To answer a few potential questions, I do not have access to the game directly, and, while this has been okay'd by the developers for game in question, I would rather it not be caught and punished by anti-cheat software. Additionally, I'd like to keep the performance hit to a minimum, so I currently have the FPS for the overlay quite low (~10fps).

Thanks in advance for any help!

Answer 1

This is a D3D11 Present x64 trampoline hook project I was just working with. You can draw inside hkPresent() before returning and whatever you draw will show up on the screen.

#include <Windows.h>
#include <d3d11.h>
#include <d3dcompiler.h>
#include <DirectXMath.h>
#pragma comment(lib, "d3d11.lib")
#pragma comment(lib, "d3dcompiler.lib")

#define SAFE_RELEASE(p) if (p) { p->Release(); p = nullptr; } 

void* Tramp64(void* src, void* dst, int len)
{
    int MinLen = 14;

    if (len < MinLen) return NULL;

    BYTE stub[] = {
    0xFF, 0x25, 0x00, 0x00, 0x00, 0x00, // jmp qword ptr [$+6]
    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // ptr
    };

    void* pTrampoline = VirtualAlloc(0, len + sizeof(stub), MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE);

    DWORD dwOld = 0;
    VirtualProtect(src, len, PAGE_EXECUTE_READWRITE, &dwOld);

    uintptr_t retto = (uintptr_t)src + len;

    // trampoline
    memcpy(stub + 6, &retto, 8);
    memcpy((void*)((uintptr_t)pTrampoline), src, len);
    memcpy((void*)((uintptr_t)pTrampoline + len), stub, sizeof(stub));

    // orig
    memcpy(stub + 6, &dst, 8);
    memcpy(src, stub, sizeof(stub));

    for (int i = MinLen; i < len; i++)
    {
        *(BYTE*)((uintptr_t)src + i) = 0x90;
    }

    VirtualProtect(src, len, dwOld, &dwOld);
    return (void*)((uintptr_t)pTrampoline);
}

bool GetD3D11SwapchainDeviceContext(void** pSwapchainTable, size_t Size_Swapchain, void** pDeviceTable, size_t Size_Device, void** pContextTable, size_t Size_Context)
{
    WNDCLASSEX wc{ 0 };
    wc.cbSize = sizeof(wc);
    wc.lpfnWndProc = DefWindowProc;
    wc.lpszClassName = TEXT("dummy class");

    if (!RegisterClassEx(&wc))
    {
        return false;
    }


    DXGI_SWAP_CHAIN_DESC swapChainDesc{ 0 };
    swapChainDesc.BufferCount = 1;
    swapChainDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
    swapChainDesc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
    swapChainDesc.OutputWindow = GetForegroundWindow();
    swapChainDesc.SampleDesc.Count = 1;
    swapChainDesc.BufferDesc.ScanlineOrdering = DXGI_MODE_SCANLINE_ORDER_UNSPECIFIED;
    swapChainDesc.BufferDesc.Scaling = DXGI_MODE_SCALING_UNSPECIFIED;
    swapChainDesc.SwapEffect = DXGI_SWAP_EFFECT_DISCARD;
    swapChainDesc.Windowed = TRUE;

    D3D_FEATURE_LEVEL featureLevel;

    IDXGISwapChain* pDummySwapChain = nullptr;
    ID3D11Device* pDummyDevice = nullptr;
    ID3D11DeviceContext* pDummyContext = nullptr;

    HRESULT hr = D3D11CreateDeviceAndSwapChain(nullptr, D3D_DRIVER_TYPE_REFERENCE, nullptr, 0, nullptr, 0, D3D11_SDK_VERSION, &swapChainDesc, &pDummySwapChain, &pDummyDevice, &featureLevel, nullptr);
    if (FAILED(hr))
    {
        DestroyWindow(swapChainDesc.OutputWindow);
        UnregisterClass(wc.lpszClassName, GetModuleHandle(nullptr));
        return false;
    }

    if (pSwapchainTable && pDummySwapChain)
    {
        memcpy(pSwapchainTable, *reinterpret_cast<void***>(pDummySwapChain), Size_Swapchain);
    }

    if (pDeviceTable && pDummyDevice)
    {
        memcpy(pDeviceTable, *reinterpret_cast<void***>(pDummyDevice), Size_Device);
    }

    if (pContextTable && pDummyContext)
    {
        memcpy(pContextTable, *reinterpret_cast<void***>(pDummyContext), Size_Context);
    }

    SAFE_RELEASE(pDummySwapChain);
    SAFE_RELEASE(pDummyDevice);
    SAFE_RELEASE(pDummyContext);

    DestroyWindow(swapChainDesc.OutputWindow);
    UnregisterClass(wc.lpszClassName, GetModuleHandle(nullptr));

    return true;
}

void* SwapChain[18];
void* Device[40];
void* Context[108];

typedef HRESULT(__fastcall* tPresent)(IDXGISwapChain* pThis, UINT SyncInterval, UINT Flags);
tPresent oPresent = nullptr;

HRESULT __fastcall hkPresent(IDXGISwapChain* pThis, UINT SyncInterval, UINT Flags)
{
    return oPresent(pThis, SyncInterval, Flags);
}

DWORD WINAPI MainThread(HMODULE hModule)
{
    if (GetD3D11SwapchainDeviceContext(SwapChain, sizeof(SwapChain), Device, sizeof(Device), Context, sizeof(Context)))
    {
        oPresent = (tPresent)Tramp64(SwapChain[8], hkPresent, 19);
    }

    return 0;
}

BOOL APIENTRY DllMain(HMODULE hModule, DWORD  ul_reason_for_call, LPVOID lpReserved)
{
    switch (ul_reason_for_call)
    {
    case DLL_PROCESS_ATTACH:
        CreateThread(nullptr, 0, (LPTHREAD_START_ROUTINE)MainThread, hModule, 0, nullptr);
    case DLL_THREAD_ATTACH:
    case DLL_THREAD_DETACH:
    case DLL_PROCESS_DETACH:
        break;
    }
    return TRUE;
}

This is a combination of code from several people: me, Broihon, Traxin & A200K