system-wide hook not run outside main program

Question

I writing Hook Mouse system-wide (when mouse over any components in screen, Handle, classname, wndproc of it will display in my program).

But system-wide hook in my program not run when move mouse outside main program

I create DLL file successfully, but dont understand why it ONLY run as local hook (it run good in local my program - SCREEN AREA of my program).

This is source of hookDLL.cpp (DLL file):

#include "stdafx.h"
#include "Win32_Iczelion_HookDLL.h"

#define WM_MOUSEHOOK WM_USER+10

HINSTANCE hInstance;
HHOOK hHook;
HWND hWnd;

EXPORT HHOOK CALLBACK InstallHook(HWND);
EXPORT void CALLBACK UninstallHook(void);
EXPORT LRESULT CALLBACK MouseProc(int, WPARAM, LPARAM);

BOOL APIENTRY DllMain( HMODULE hModule,
                       DWORD  ul_reason_for_call,
                       LPVOID lpReserved
                     )
{
    switch (ul_reason_for_call)
    {
    case DLL_PROCESS_ATTACH:
        hInstance = hModule;
        break;
    case DLL_THREAD_ATTACH:
    case DLL_THREAD_DETACH:
    case DLL_PROCESS_DETACH:
        break;
    }
    return TRUE;
}

EXPORT HHOOK CALLBACK InstallHook(HWND hwnd)
{
    hWnd = hwnd;
    hHook = SetWindowsHookEx (WH_MOUSE, (HOOKPROC)MouseProc, hInstance, NULL); 
    return hHook;
}

EXPORT void CALLBACK UninstallHook()
{
    UnhookWindowsHookEx(hHook);
}

EXPORT void test()
{
    MessageBox(NULL, L"yeah", L"yeah", MB_OK);
}

EXPORT LRESULT CALLBACK MouseProc(int nCode, WPARAM wParam, LPARAM lParam)
{
    CallNextHookEx(hHook, nCode, wParam, lParam);
    POINT pt = {0};
    GetCursorPos(&pt);
    HANDLE handle = WindowFromPoint(pt);
    PostMessage(hWnd, WM_MOUSEHOOK, (WPARAM)handle, 0);
    return 0;
}

And this is source of hook.cpp (main program):

#include "stdafx.h"
#include "Win32_Iczelion_Hook.h"
#include "Win32_Iczelion_HookDLL.h"

#define WM_MOUSEHOOK WM_USER+10

#define MAX_LOADSTRING 100

// Global Variables:
HINSTANCE hInst;                                // current instance
TCHAR szTitle[MAX_LOADSTRING];                  // The title bar text
TCHAR szWindowClass[MAX_LOADSTRING];            // the main window class name
HWND hWndEHandle;
HWND hWndEClassName;
HWND hWndEWndProc;
HWND hWndBHook;
HWND hWndBExit;
HHOOK hHook;
BOOL HookFlag = FALSE;

// Forward declarations of functions included in this code module:
ATOM                MyRegisterClass(HINSTANCE hInstance);
BOOL                InitInstance(HINSTANCE, int);
LRESULT CALLBACK    WndProc(HWND, UINT, WPARAM, LPARAM);
INT_PTR CALLBACK    About(HWND, UINT, WPARAM, LPARAM);

int APIENTRY _tWinMain(_In_ HINSTANCE hInstance,
                     _In_opt_ HINSTANCE hPrevInstance,
                     _In_ LPTSTR    lpCmdLine,
                     _In_ int       nCmdShow)
{
    UNREFERENCED_PARAMETER(hPrevInstance);
    UNREFERENCED_PARAMETER(lpCmdLine);

    // TODO: Place code here.
    MSG msg;
    HACCEL hAccelTable;

    // Initialize global strings
    LoadString(hInstance, IDS_APP_TITLE, szTitle, MAX_LOADSTRING);
    LoadString(hInstance, IDC_WIN32_ICZELION_HOOK, szWindowClass, MAX_LOADSTRING);
    MyRegisterClass(hInstance);

    // Perform application initialization:
    if (!InitInstance (hInstance, nCmdShow))
    {
        return FALSE;
    }

    hAccelTable = LoadAccelerators(hInstance, MAKEINTRESOURCE(IDC_WIN32_ICZELION_HOOK));

    // Main message loop:
    while (GetMessage(&msg, NULL, 0, 0))
    {
        if (!TranslateAccelerator(msg.hwnd, hAccelTable, &msg))
        {
            TranslateMessage(&msg);
            DispatchMessage(&msg);
        }
    }

    return (int) msg.wParam;
}

ATOM MyRegisterClass(HINSTANCE hInstance)
{
    WNDCLASSEX wcex;

    wcex.cbSize = sizeof(WNDCLASSEX);

    wcex.style          = CS_HREDRAW | CS_VREDRAW;
    wcex.lpfnWndProc    = WndProc;
    wcex.cbClsExtra     = 0;
    wcex.cbWndExtra     = 0;
    wcex.hInstance      = hInstance;
    wcex.hIcon          = LoadIcon(hInstance, MAKEINTRESOURCE(IDI_WIN32_ICZELION_HOOK));
    wcex.hCursor        = LoadCursor(NULL, IDC_ARROW);
    wcex.hbrBackground  = (HBRUSH)(COLOR_WINDOW+1);
    wcex.lpszMenuName   = MAKEINTRESOURCE(IDC_WIN32_ICZELION_HOOK);
    wcex.lpszClassName  = szWindowClass;
    wcex.hIconSm        = LoadIcon(wcex.hInstance, MAKEINTRESOURCE(IDI_SMALL));

    return RegisterClassEx(&wcex);
}

BOOL InitInstance(HINSTANCE hInstance, int nCmdShow)
{
   HWND hWnd;

   hInst = hInstance; // Store instance handle in our global variable

   hWnd = CreateWindow(szWindowClass, szTitle, WS_OVERLAPPEDWINDOW,
      CW_USEDEFAULT, 0, 600, 300, NULL, NULL, hInstance, NULL);

   hWndEHandle = CreateWindowEx(NULL, L"Edit", NULL,
                        WS_CHILD | WS_VISIBLE | WS_BORDER | ES_LEFT | ES_AUTOHSCROLL,
                        100,35,345,25,hWnd,(HMENU)IDC_HANDLE,hInstance,NULL);
   hWndEClassName = CreateWindowEx(NULL, L"Edit", NULL,
                        WS_CHILD | WS_VISIBLE | WS_BORDER | ES_LEFT | ES_AUTOHSCROLL,
                        100,65,200,25,hWnd,(HMENU)IDC_CLASSNAME,hInstance,NULL);
   hWndEWndProc = CreateWindowEx(NULL, L"Edit", NULL,
                        WS_CHILD | WS_VISIBLE | WS_BORDER | ES_LEFT | ES_AUTOHSCROLL,
                        100,95,200,25,hWnd,(HMENU)IDC_WNDPROC,hInstance,NULL);

   hWndBHook = CreateWindowEx(NULL, L"Button", L"Hook", 
                        WS_CHILD | WS_VISIBLE | BS_DEFPUSHBUTTON,
                        305,65,140,25,hWnd,(HMENU)IDC_HOOK,hInstance,NULL);
   hWndBExit = CreateWindowEx(NULL, L"Button", L"Exit", 
                        WS_CHILD | WS_VISIBLE | BS_DEFPUSHBUTTON,
                        305,95,140,25,hWnd,(HMENU)IDC_EXIT,hInstance,NULL);

   if (!hWnd)
   {
      return FALSE;
   }

   ShowWindow(hWnd, nCmdShow);
   UpdateWindow(hWnd);

   return TRUE;
}

LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
{
    int wmId, wmEvent;
    PAINTSTRUCT ps;
    HDC hdc;
    LPWSTR buffer = 0;
    RECT rect;

    switch (message)
    {
    case WM_CREATE:
        GetWindowRect(hWnd, &rect);
        SetWindowPos(hWnd, HWND_TOPMOST, rect.left, rect.top, rect.right, rect.bottom, SWP_SHOWWINDOW);
        break;
    case WM_CLOSE:
        if (HookFlag==TRUE)
            UninstallHook();
        DestroyWindow(hWnd);
    case WM_MOUSEHOOK:
    {
        wchar_t buffer[256];
        wsprintfW(buffer, L"%p", wParam);
        SetDlgItemText(hWnd, IDC_HANDLE, buffer);

        GetClassName((HWND)wParam, buffer, 128);
        SetDlgItemText(hWnd, IDC_CLASSNAME, buffer);

        DWORD buffer2 = GetClassLong((HWND)wParam,GCL_WNDPROC);
        wsprintfW(buffer, L"%p", buffer2);
        SetDlgItemText(hWnd, IDC_WNDPROC, buffer);
        break;
    }
    case WM_COMMAND:
    {
        wmId    = LOWORD(wParam);
        wmEvent = HIWORD(wParam);
        // Parse the menu selections:

        if (wmEvent==BN_CLICKED)
        {
            if (wmId==IDC_HOOK)
            {
                if (HookFlag==FALSE)
                {
                    if (InstallHook(hWnd))
                    {
                        HookFlag = TRUE;
                        SetDlgItemText(hWnd, IDC_HOOK, L"Un hook");
                    }
                }
                else
                {
                     UninstallHook();
                     SetDlgItemText(hWnd,IDC_HOOK,L"Hook");
                     HookFlag = FALSE;
                     SetDlgItemText(hWnd,IDC_CLASSNAME,NULL);
                     SetDlgItemText(hWnd,IDC_HANDLE,NULL);
                     SetDlgItemText(hWnd,IDC_WNDPROC,NULL);
                }
            }
            else if (wmId==IDC_EXIT)
            {
                SendMessage(hWnd,WM_CLOSE,0,0);
            }
        }
        switch (wmId)
        {
        case IDM_ABOUT:
            DialogBox(hInst, MAKEINTRESOURCE(IDD_ABOUTBOX), hWnd, About);
            break;
        case IDM_EXIT:
            DestroyWindow(hWnd);
            break;
        default:
            return DefWindowProc(hWnd, message, wParam, lParam);
        }
        break;
    }
    case WM_PAINT:
        hdc = BeginPaint(hWnd, &ps);
        // TODO: Add any drawing code here...
        EndPaint(hWnd, &ps);
        break;
    case WM_DESTROY:
        PostQuitMessage(0);
        break;
    default:
        return DefWindowProc(hWnd, message, wParam, lParam);
    }
    return 0;
}

Anyone can help me.

Answer 1

When DLL is loaded from another program (when you mouse over outside main program), hWnd is NULL, so you have to use FindWindow to get hWnd value of main program before PostMessage.

Answer 2

Last EDIT : corrected code.

The DLL is loaded, by Windows, in other processes, if they have the same bitness as your main process (and if they get the mouse). Consequence: the global hWnd variable in your Dll is always NULL and PostMessage fails.

Advices:

1 Always check the API's return code. You should have noticed that PostMessage failed.

2 When you are about to PostMessage in your DLL, if you see a NULL hWnd, use FindWindow to get the HWND and store it in hWnd for next time.

New version of the MouseProc DLL CallBack:

EXPORT LRESULT CALLBACK MouseProc(int nCode, WPARAM wParam, LPARAM lParam)
{
    if ( nCode >= 0 ) {
        PMOUSEHOOKSTRUCT pmhs = reinterpret_cast<PMOUSEHOOKSTRUCT>( lParam );
        if ( hWnd == NULL ) hWnd = FindWindow( L"WIN32_ICZELION_HOOK", NULL );
        if ( hWnd ) PostMessage(hWnd, WM_MOUSEHOOK, (WPARAM)pmhs->hwnd, 0);
    }
    return CallNextHookEx(0, nCode, wParam, lParam);
}

EDIT: make sure you run your progam (EXE) from a folder where the DLL has been rebuild). Create an empty solution (SLN) and add the 2 projects in it (vcxproj). Delete all "old" EXE and DLL. Rebuild.

EDIT added ScreenShot: The EXE showing the class name of the Explorer folder bar