C++ Output Parameter Causes Memory Leak

Question

I am working on a C++ project based on COM, My Code has a function with an out parameter which takes an object as input and assigns a new instance of a class into it. But when I used CRT Debugging I found some memory leak in the function, Here is the code of the function.

bool __stdcall FluentCompositor::CreateCompositionHost(HWND hwnd, ICompositionHost** compositionHost)
{
    if (compositionHost != nullptr)
    {
        *compositionHost = reinterpret_cast<ICompositionHost*>(new CompositionHost(hwnd));
        if (compositionHost != nullptr)
        {
            return true;
        }
    }
    else
    {
        return false;
    }
    return false;
}

This function takes an object of ICompositionHost and initialize it with CompositionHost object, Where should I need to free the memory to avoid memory leak.

I Call the function using ComPtr, but still there is memory leak

ComPtr<ICompositionHost> host;
compositor->CreateCompositionHost(hwnd,host.GetAddressOf());

The Complete Code:

FluentCompositor.cpp

#include "pch.h"
#include "ICompositionHost.h"
#include "IFluentCompositor.h"
#include "CompositionHost.h"
#include "FluentCompositor.h"

FluentCompositor::FluentCompositor() :ref(1)
{
}

ulong __stdcall FluentCompositor::AddRef()
{
    return (++ref);
}

ulong __stdcall FluentCompositor::Release()
{
    if (--ref == 0)
    {
        delete this;
        return 0;
    }
    return ref;
}

HRESULT __stdcall FluentCompositor::QueryInterface(REFIID iid, LPVOID* ppv)
{
    if (iid == IID_IFluentCompositor || iid == IID_IUnknown)
    {
        *ppv = (void*)this;
        AddRef();
    }
    else
    {
        *ppv = NULL;
    }
    return (*ppv == NULL) ? E_NOINTERFACE : S_OK;
}

HRESULT __stdcall CreateFluentCompositor(void** compositor)
{
    if (compositor != nullptr)
    {
        *compositor = reinterpret_cast<void*>(new FluentCompositor());
        if (compositor != nullptr)
        {
            return S_OK;
        }
    }
    else
    {
        return E_INVALIDARG;
    }
    return E_FAIL;
}

bool __stdcall FluentCompositor::CreateCompositionHost(HWND hwnd, ICompositionHost** compositionHost)
{
    if (compositionHost != nullptr)
    {
        *compositionHost = reinterpret_cast<ICompositionHost*>(new CompositionHost(hwnd));
        if (compositionHost != nullptr)
        {
            return true;
        }
    }
    else
    {
        return false;
    }
    return false;
}

MainWindow.cpp

#include "pch.h"
#include "MainWindow.h"

int __stdcall wWinMain(HINSTANCE, HINSTANCE, LPWSTR, int)
{
    _CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF);
    const wchar_t className[] = L"Fluent Compositor";

    WNDCLASS wc = {
        .lpfnWndProc = WndProc,
        .hInstance = reinterpret_cast<HINSTANCE>(&__ImageBase),
        .hCursor = LoadCursor(nullptr, IDC_ARROW),
        .lpszClassName = className,
    };

    RegisterClass(&wc);

    HWND hwnd = CreateWindowEx(WS_EX_NOREDIRECTIONBITMAP, className, L"Fluent Compositor Sample", WS_OVERLAPPEDWINDOW | WS_VISIBLE, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, nullptr, nullptr, wc.hInstance, nullptr);
    if (hwnd == nullptr)
    {
        return 0;
    }

    CreateCompositionEffect(hwnd);

    MSG msg = { };
    while (GetMessage(&msg, NULL, 0, 0))
    {
        TranslateMessage(&msg);
        DispatchMessage(&msg);
    }

    _CrtDumpMemoryLeaks();
    return 0;
}

bool CreateCompositionEffect(HWND hwnd)
{
    auto fluentCompositorLib = LoadLibrary(L"FluentCompositor.dll");
    if (!fluentCompositorLib)
    {
        return false;
    }

    CreateCompositor = (CreateFluentCompositor)GetProcAddress(fluentCompositorLib, "CreateFluentCompositor");
    if (!CreateCompositor)
    {
        return false;
    }

    CreateCompositor(&compositor);
    if (compositor == nullptr)
    {
        return false;
    }

    ComPtr<ICompositionHost> host;
    compositor->CreateCompositionHost(hwnd,&host);
    return true;
}

LRESULT __stdcall WndProc(HWND hwnd, UINT msg, WPARAM wparam, LPARAM lparam)
{
    switch (msg)
    {
        case WM_DESTROY:
        {
            PostQuitMessage(0);
            return 0;
        }
    }
    return DefWindowProc(hwnd, msg, wparam, lparam);
}

MainWindow.h

#pragma once
#include "IFluentCompositor.h"

using namespace Microsoft::WRL;

extern "C" IMAGE_DOS_HEADER __ImageBase;

ComPtr<IFluentCompositor> compositor;

typedef BOOL(__stdcall* CreateFluentCompositor)(IFluentCompositor** compositor);
CreateFluentCompositor CreateCompositor;

bool CreateCompositionEffect(HWND hwnd);
LRESULT __stdcall WndProc(HWND hwnd, UINT msg, WPARAM wparam, LPARAM lparam);

Answer 1

Your use of reinterpret_cast (and type-casts in general) is simply wrong. If your classes implement the correct interfaces, there is no need to cast them manually (except maybe in QueryInterface()), the compiler will perform the correct conversions implicitly for you.

Also, you are not checking the return value of new correctly. Or, for that matter, handling the possibility that new throws an exception on failure by default, not returns nullptr. If you want a nullptr on falilure, use the nothrow version of new instead.

Also, when using ComPtr, should should be using its overloaded operator& rather than its GetAddressOf() method. Especially if the ComPtr already holds an interface. GetAddressOf() will not release the interface (that is why there is a separate ReleaseAndGetAddressOf() method), but operator& will.

Try this instead:

#include <new>

HRESULT __stdcall FluentCompositor::QueryInterface(REFIID iid, LPVOID* ppv)
{
    if (!ppv) return E_POINTER;

    if (iid == IID_IFluentCompositor)
    {
        *ppv = static_cast<IFluentCompositor*>(this);
        /* alternatively:
        IFluentCompositor *comp = this;
        *ppv = comp;
        */
    }
    else if (iid == IID_IUnknown)
    {
        *ppv = static_cast<IUnknown*>(static_cast<IFluentCompositor*>(this));
        /* alternatively:
        IFluentCompositor *comp = this;
        IUnknown *unk = comp;
        *ppv = unk;
        */
    }
    else
    {
        *ppv = nullptr;
    }

    if (!*ppv)
        return E_NOINTERFACE;

    AddRef();
    return S_OK;
}

// similar for CompositionHost::QueryInterface() ...

HRESULT __stdcall CreateFluentCompositor(void** compositor)
{
    if (!compositor) return E_POINTER; // not E_INVALIDARG
    // make sure FluentCompositor has a refcount of 1 when created!
    *compositor = static_cast<IFluentCompositor*>(new(std::nothrow) FluentCompositor);
    return (*compositor) ? S_OK : E_FAIL;
}

bool __stdcall FluentCompositor::CreateCompositionHost(HWND hwnd, ICompositionHost** compositionHost)
{
    if (!compositionHost) return false;
    // make sure CompositionHost has a refcount of 1 when created!
    *compositionHost = new(std::nothrow) CompositionHost(hwnd);
    return (*compositionHost);
}

ComPtr<ICompositionHost> host;
compositor->CreateCompositionHost(hwnd, &host);

Alternatively, consider using ComPtr internally when creating your objects, eg:

#include <new>

HRESULT __stdcall CreateFluentCompositor(void** compositor)
{
    if (!compositor) return E_POINTER; // not E_INVALIDARG
    // make sure FluentCompositor has a refcount of 0 when created,
    // as the ComPtr constructor will increment it!
    ComPtr<IFluentCompositor> obj(new(std::nothrow) FluentCompositor);
    return (obj) ? obj->QueryInterface(IID_IFluentCompositor, compositor) : E_FAIL;
}

bool __stdcall FluentCompositor::CreateCompositionHost(HWND hwnd, ICompositionHost** compositionHost)
{
    // make sure CompositionHost has a refcount of 0 when created,
    // as the ComPtr constructor will increment it!
    ComPtr<ICompositionHost> obj(new(std::nothrow) CompositionHost(hwnd));
    return ((obj) && (obj->QueryInterface(IID_ICompositionHost, reinterpret_cast<void**>(compositionHost)) == S_OK));
}

It is a good idea to have your objects start with a reference count of 0 rather than 1, because they don't know if they are going to be used with interface pointers or object pointers (if they are even used with pointers at all). Don't increment an object's reference count unless it is actually assigned to an interface pointer that is AddRef()'ed and needs to be Release()'d.