Detours 3.0 Hook Crashes MessageBoxA

Question

I'm trying to hook the MessageBoxA function with MS Detours 3.0 but when I try it my program crashes. I'm not sure what is causing the program to crash. When i run the test program and hit shift the message box appears, but when I inject the dll and hit shift my program crashes.

TEST PROGRAM

#include <Windows.h>

int main()
{
    for(;;)
    {
        if(GetAsyncKeyState(VK_SHIFT))
        {
            MessageBoxA(0,"NOT HOOKED","HOOK STATUS",0);
        }
    }
}

HOOK DLL

#include <Windows.h>
#include <detours.h>
#pragma comment(lib,"detours.lib")

BOOL (WINAPI* oMessageBoxA)(HWND,LPCTSTR,LPCTSTR,UINT);

BOOL WINAPI hMessageBoxA( HWND hWnd, LPCTSTR lpText, LPCTSTR lpCaption,UINT uType) 
{ 
        return oMessageBoxA(hWnd,"HOOKED",lpCaption,uType);
} 

void patch()
{
    HMODULE user32 = GetModuleHandle("user32.dll");
    if(user32 != NULL)
    {
        DWORD MessageBoxAddress = (DWORD)GetProcAddress(user32,"MessageBoxA");
        oMessageBoxA = (BOOL (WINAPI*)(HWND, LPCTSTR, LPCTSTR, UINT))MessageBoxAddress;
        DetourTransactionBegin();
        DetourUpdateThread(GetCurrentThread());
        DetourAttach(&(PVOID&)oMessageBoxA, hMessageBoxA);
        DetourTransactionCommit();
    }
}

BOOL WINAPI DllMain(HINSTANCE hinstDLL,DWORD fdwReason,LPVOID lpvReserved)
{
    if(fdwReason==DLL_PROCESS_ATTACH)
    {
        patch();
    }
}

Answer 1

You have declared the signature of MessageBoxA() incorrectly, and your use of DWORD MessageBoxAddress will not work in a 64bit DLL.

Try this DLL code instead:

#include <Windows.h>
#include <detours.h>
#pragma comment(lib,"detours.lib")

typedef int (WINAPI* LPFN_MBA)(HWND, LPCSTR, LPCSTR, UINT);
LPFN_MBA oMessageBoxA = NULL;

int WINAPI hMessageBoxA( HWND hWnd, LPCSTR lpText, LPCSTR lpCaption,UINT uType) 
{ 
    return oMessageBoxA(hWnd,"HOOKED",lpCaption,uType);
} 

void patch()
{
    HMODULE user32 = GetModuleHandle(TEXT("user32.dll"));
    if (user32 != NULL)
    {
        oMessageBoxA = (LPFN_MBA) GetProcAddress(user32, "MessageBoxA");
        if (oMessageBoxA != NULL)
        { 
            DetourTransactionBegin();
            DetourUpdateThread(GetCurrentThread());
            DetourAttach((PVOID*)&oMessageBoxA, hMessageBoxA);
            DetourTransactionCommit();
        } 
    }
}

void unpatch()
{
    if (oMessageBoxA != NULL)
    {
        DetourTransactionBegin();
        DetourUpdateThread(GetCurrentThread());
        DetourDetach((PVOID*)&oMessageBoxA, hMessageBoxA);
        DetourTransactionCommit();
    }
}


BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved)
{
    if (fdwReason == DLL_PROCESS_ATTACH)
    {
        DisableThreadLibraryCalls(hinstDLL);
        patch();
    }
    else if (fdwReason == DLL_PROCESS_DETACH)
    {
        unpatch();
    }
}

Read the following for more details:

API Hooking with MS Detours