SetWindowsHookEx() method fails with code: 1428 - dll injection

Question

I have the following code:

LRESULT __stdcall HookCallback(int code,
    WPARAM wParam,
    LPARAM lParam)
{
    ...

    return CallNextHookEx(_hook, code, wParam, lParam);
}

void SetHook()
{
    HMODULE hmod = GetModuleHandle(0);

    if (!(_hook = SetWindowsHookEx(WH_CALLWNDPROCRET, HookCallback, hmod, 0)))
    {
        OutputDebugString(TEXT("Failed to Install hook"));
    }

    OutputDebugString(TEXT("Exiting SETHOOK METHOD"));
}

void ReleaseHook()
{
    UnhookWindowsHookEx(_hook);
}

BOOL APIENTRY DllMain(HMODULE hModule,
    DWORD  ul_reason_for_call,
    LPVOID lpReserved
)
{
    OutputDebugString(TEXT("Entered DLL"));

    switch (ul_reason_for_call)
    {
    case DLL_PROCESS_ATTACH:
        SetHook();
        break;
    case DLL_THREAD_ATTACH:
    case DLL_THREAD_DETACH:
        break;
    case DLL_PROCESS_DETACH:
        ReleaseHook();
        break;
    }

    return TRUE;
}

I am trying to set up a hook to react to window manipulations. Mainly opening and closing of windows. I am very unexperienced in c++, so I'm having a very dificult time debugging the application and understanding what I'm doing wrong.

In my SetHook() method the SetWindowHookEx method always fails with code 1428. So the Callback method (not shown here) is never called.

What am I doing wrong?

PS: I'm injecting the .dll into the processes using registry: Path:Computer\HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Windows Name:AppInit_DLLS

and as far I as I can see this is working, because I get the debug message "Failed to Install hook" and "Exiting SETHOOK METHOD".

Answer 1

You are passing 0 (aka NULL) to GetModuleHandle(), so you are retrieving the HMODULE of the EXE file whose process your DLL is loaded into:

Parameters

lpModuleName

The name of the loaded module (either a .dll or .exe file) ...

If this parameter is NULL, GetModuleHandle returns a handle to the file used to create the calling process (.exe file).

You can't use that HMODULE to install a SetWindowsHookEx() hook callback that is in a DLL. You need to use the DLL's own HMODULE itself, ie the one that is provided as an input parameter to your DllMain(), eg:

HHOOK _hook = NULL;

LRESULT __stdcall HookCallback(int code,
    WPARAM wParam,
    LPARAM lParam)
{
    ...

    return CallNextHookEx(_hook, code, wParam, lParam);
}

void SetHook(HMODULE hModule) // <-- ADD THIS PARAM
{
    if (!_hook)
    {
        _hook = SetWindowsHookEx(WH_CALLWNDPROCRET, HookCallback, hModule, 0); // <-- USE IT HERE
        if (!_hook)
        {
            OutputDebugString(TEXT("Failed to Install hook"));
        }
    }

    OutputDebugString(TEXT("Exiting SETHOOK METHOD"));
}

void ReleaseHook()
{
    if (_hook)
    {
        UnhookWindowsHookEx(_hook);
        _hook = NULL;
    }
}

BOOL APIENTRY DllMain(HMODULE hModule,
    DWORD  ul_reason_for_call,
    LPVOID lpReserved
)
{
    OutputDebugString(TEXT("Entered DLL"));

    switch (ul_reason_for_call)
    {
    case DLL_PROCESS_ATTACH:
        DisableThreadLibraryCalls(hModule);
        SetHook(hModule); // <-- PASS IT HERE
        break;
    case DLL_PROCESS_DETACH:
        ReleaseHook();
        break;
    }

    return TRUE;
}

That being said, you are trying to install a hook globally (hMod != NULL and dwThreadId == 0), so you need only 1 call to SetWindowsHookEx(), it does not make sense to call it in every process that your DLL is loaded into. You should create your own EXE that loads your DLL into memory and calls SetWindowsHookEx() 1 time for it, eg:

HMODULE _hmod = NULL;
HHOOK _hook = NULL;

// be sure to export your hook functions from the DLL..

LRESULT __stdcall HookCallback(int code,
    WPARAM wParam,
    LPARAM lParam)
{
    ...

    return CallNextHookEx(_hook, code, wParam, lParam);
}

void SetHook()
{
    if (!_hook)
    {
        _hook = SetWindowsHookEx(WH_CALLWNDPROCRET, HookCallback, _hmod, 0);
        if (!_hook)
        {
            OutputDebugString(TEXT("Failed to Install hook"));
        }
    }

    OutputDebugString(TEXT("Exiting SETHOOK METHOD"));
}

void ReleaseHook()
{
    if (_hook)
    {
        UnhookWindowsHookEx(_hook);
        _hook = NULL;
    }
}

BOOL APIENTRY DllMain(HMODULE hModule,
    DWORD  ul_reason_for_call,
    LPVOID lpReserved
)
{
    _hmod = hModule;

    OutputDebugString(TEXT("Entered DLL"));

    switch (ul_reason_for_call)
    {
    case DLL_PROCESS_ATTACH:
        DisableThreadLibraryCalls(hModule);
        break;
    }

    return TRUE;
}

typedef void (*LPFN_SH)();
typedef void (*LPFN_RH)();

HMODULE hMod = LoadLibrary("my.dll");
LPFN_SH SetHook = (LPFN_SH) GetProcAddress(hMod, "SetHook");
LPFN_RH ReleaseHook = (LPFN_RH) GetProcAddress(hMod, "ReleaseHook");
...
SetHook();
...
ReleaseHook();
FreeLibrary(hMod);

Your DLL will then be injected automatically into every running process that matches your DLL's bitness (that means you need separate DLLs for hooking 32bit and 64bit processes) for the lifetime of your EXE process. No need to use the AppInit_DLLS Registry key at all.

---

Otherwise, if you really want to use AppInit_DLLS to inject your DLL into every bitness-matching process, then it would be better to have the DLL call SetWindowsHookEx() on a per-thread basis (hMod == NULL and dwThreadId != 0) instead of on a global basis, eg:

__declspec(thread) HHOOK _hook = NULL;

LRESULT __stdcall HookCallback(int code,
    WPARAM wParam,
    LPARAM lParam)
{
    ...

    return CallNextHookEx(_hook, code, wParam, lParam);
}

void SetHook()
{
    if (!_hook)
    {
        DWORD dwThreadId = GetCurrentThreadId();

        _hook = SetWindowsHookEx(WH_CALLWNDPROCRET, HookCallback, NULL, dwThreadId);
        if (!_hook)
        {
            OutputDebugString(TEXT("Failed to Install hook"));
        }
    }

    OutputDebugString(TEXT("Exiting SETHOOK METHOD"));
}

void ReleaseHook()
{
    if (_hook)
    {
        UnhookWindowsHookEx(_hook);
        _hook = NULL;
    }
}

BOOL APIENTRY DllMain(HMODULE hModule,
    DWORD  ul_reason_for_call,
    LPVOID lpReserved
)
{
    OutputDebugString(TEXT("Entered DLL"));

    switch (ul_reason_for_call)
    {
    case DLL_PROCESS_ATTACH:
    case DLL_THREAD_ATTACH:
        SetHook();
        break;
    case DLL_PROCESS_DETACH:
    case DLL_THREAD_DETACH:
        ReleaseHook();
        break;
    }

    return TRUE;
}

Then there won't be a need to use a separate EXE loader.