Why doesn't this system wide CBT hook work properly?

Question

I am trying to create a system wide hook to monitor processes and terminate unwanted ones. I searched and found out I need to use CBT hooks, my first try failed and this is the second one, the former question can be found here though.

The following code builds just fine, but it seems the hooks are not even called, since I tried setting break point in the DllMain(), but I never reach there. Other functions seem to be accessible though!

Here are the code snippets:

dllmain.cpp

// dllmain.cpp : Defines the entry point for the DLL application.
#pragma once
#include "stdafx.h"
#include <iostream>
#include <fstream>
#include <Windows.h>
using namespace std;

 HINSTANCE currentProcessHandle;
 HOOKPROC hkprcSysMsg;
 HHOOK hookID;

BOOL APIENTRY DllMain( HMODULE hModule, DWORD  ul_reason_for_call, LPVOID lpReserved)
{
    std::ofstream outfile("test.txt");


    switch (ul_reason_for_call)
    {
    case DLL_PROCESS_ATTACH:
        currentProcessHandle = hModule;
        break;
    case DLL_THREAD_ATTACH:
    case DLL_THREAD_DETACH:
    case DLL_PROCESS_DETACH:
        break;
    }
    return TRUE;
}

LRESULT CALLBACK HookProcedure(int nCode, WPARAM wparam, LPARAM lparam)
{
    std::ofstream outfile("test.txt");
    if (nCode >= 0)
    {
        switch (nCode)
        {
        case HCBT_CREATEWND:
            outfile << L"Created!~";
            cout << "Created!~" << endl;
            break;
        case HCBT_DESTROYWND:
            outfile << L"Destroied!~";
            cout << "Destroied!~" << endl;
            break;
        default:
            cout << "sth else" << endl;
            break;
        }
    }
    else
    {
        return CallNextHookEx(hookID, nCode, wparam, lparam);
    }
    outfile.close();
}

__declspec(dllexport) void InstallHook()
{
    hookID = SetWindowsHookEx(WH_CBT, HookProcedure, currentProcessHandle, 0);
}

__declspec(dllexport) void UnistallHook()
{
    UnhookWindowsHookEx(hookID);
}

And this is the Consumer application

// Hook Executer.cpp : Defines the entry point for the console application.
//

#include "stdafx.h"
#include "..\Dll\dllmain.cpp"
#include <iostream>
using namespace std;

int _tmain(int argc, _TCHAR* argv[])
{
    int num = -1;
    cout << "1.Install Hook"<<endl
        << "2.Unistall Hook"<<endl
        << "0.Exit";
    do{
        cin >> num;
        if (num ==1)
        {
            InstallHook();

        }
        else
        {
            UnistallHook();
        }
        getchar();
        system("cls");
        cout << "1.Install Hook" << endl
            << "2.Unistall Hook" << endl
            << "0.Exit";
    } while (num != 0 && num < 3);


    return 0;
}

When I run the program there is no error, not even exceptions of any kind, it's as if there is no DLL or I have coded nothing inside that DLL. What's wrong with it?

Answer 1

implement your DLL code in a CPP file, not an header:

//dllmain.cpp
#include "stdafx.h" // include <Windows.h>
                    // and other std headers in stdafx.h, if not already done

HINSTANCE currentProcessHandle;
HHOOK hookID;

BOOL APIENTRY DllMain( HMODULE hModule, DWORD  ul_reason_for_call,
                       LPVOID lpReserved ) {

    if ( ul_reason_for_call == DLL_PROCESS_ATTACH )
        currentProcessHandle = hModule;
    return TRUE;
}

LRESULT CALLBACK HookProcedure( int nCode, WPARAM wparam, LPARAM lparam ) {

    if ( nCode < 0 ) return CallNextHookEx( NULL, nCode, wparam, lparam );

    std::ofstream outfile;
    outfile.open( "test.txt",          // replace with an absolute path
                  std::fstream::app ); // append mode
    if (nCode >= 0) {
        switch( nCode ) {
            case HCBT_CREATEWND:
                outfile << "Created!\n";
                break;
            case HCBT_DESTROYWND:
                outfile << "Destroyed!\n";
                break;
            default:
                break;
        }
    }
    outfile.close();
    return 0;
}

void InstallHook( void ) {
    hookID = SetWindowsHookEx( WH_CBT, HookProcedure, currentProcessHandle, 0 );
}

void UninstallHook( void ) { // NEW NAME
    UnhookWindowsHookEx( hookID );
}    

Declare the DLL APIs in a header file.

// dllapi.h
void InstallHook( void );
void UninstallHook( void ); // NEW NAME

Use a DEF file for exporting, add it to the DLL project

; Def file
EXPORTS
    InstallHook
    UninstallHook

In the EXE project, include ONLY the DLL header file

 #include "..\Dll\dllapi.h"

In the EXE project, go to properties->Linker->Input->Additional dependencies and add the lib file generated during the build of the DLL. Alternative: make the DLL a dependency of the EXE in the project dependencies of the Solution, and in the EXE properties, set Yes for Linker->general->Use Library dependency Inputs