How to avoid RuntimeWarning memory leak in callback function (with Python callback returning a string to C)?

Question

C++ library

CallbackTestLib.hpp

#pragma once

using callback_prototype = const char* __cdecl();

extern "C" __declspec(dllexport) int __cdecl do_something(callback_prototype*);

CallbackTestLib.cpp

#include "CallbackTestLib.hpp"
#include <iostream>

using namespace std;

__declspec(dllexport) auto __cdecl do_something(callback_prototype* cb) -> int
{
    if (!cb) { return 5678; }
    const auto* str = cb();
    cout << "Hello " << str << endl;
    return 1234;
}

Python script

CallbackTest.py

import os
import sys
from ctypes import CDLL, CFUNCTYPE, c_char_p, c_int32

assert sys.maxsize > 2 ** 32, "Python x64 required"
assert sys.version_info.major == 3 and sys.version_info.minor == 8 and sys.version_info.micro == 4, "Python 3.8.4 required"

callback_prototype = CFUNCTYPE(c_char_p)

@callback_prototype
def python_callback_func() -> bytes:
    return "from Python".encode("utf-8")

dll_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "CallbackTestLib.dll")
testlib = CDLL(dll_path)
testlib.do_something.restype = c_int32
testlib.do_something.argtypes = [callback_prototype]

null_return_code = testlib.do_something(callback_prototype())
assert null_return_code == 5678, "Null return code failed"

ok_return_code = testlib.do_something(python_callback_func)
assert ok_return_code == 1234, "Ok return code failed"

print("Done.")

Python output

d:/path/to/CallbackTest.py:22: RuntimeWarning: memory leak in callback function.
ok_return_code = testlib.do_something(python_callback_func)
Hello from Python
Done.

As the output shows, Python (somehow) seems to have detected a memory leak when python_callback_func is executed, which returns bytes (UTF-8 encoded string) back to C++ where the string is being printed out.
My question is all about this: what is going on around with this warning, how to avoid/solve it?

Answer 1

This is still somewhat vague/unclear to me. But here is a stupid fix (I am not happy with it), it makes the memleak warning message go away:

Version 1

CallbackTestLib.hpp

#pragma once

using callback_prototype = void* __cdecl(); // Changed 'const char*' to 'void*'.

extern "C" __declspec(dllexport) int __cdecl do_something(callback_prototype*);

CallbackTestLib.cpp

#include "CallbackTestLib.hpp"
#include <iostream>

using namespace std;

__declspec(dllexport) auto __cdecl do_something(callback_prototype* cb) -> int
{
    if (!cb) { return 5678; }
    const auto* str = cb();
    cout << "Hello " << static_cast<const char*>(str) << endl; // Added 'static_cast'.
    return 1234;
}

CallbackTest.py

import os
import sys
from ctypes import CDLL, CFUNCTYPE, cast, c_void_p, c_int32

assert sys.maxsize > 2 ** 32, "Python x64 required"
assert sys.version_info.major == 3 and sys.version_info.minor == 8 and sys.version_info.micro == 4, "Python 3.8.4 required"

callback_prototype = CFUNCTYPE(c_void_p)  # Changed restype to 'c_void_p'.

@callback_prototype
def python_callback_func():
    return cast("from Python :)".encode("utf-8"), c_void_p).value  # Added casting.

dll_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "CallbackTestLib.dll")
testlib = CDLL(dll_path)
testlib.do_something.restype = c_int32
testlib.do_something.argtypes = [callback_prototype]

null_return_code = testlib.do_something(callback_prototype())
assert null_return_code == 5678, "Null return code failed"

ok_return_code = testlib.do_something(python_callback_func)
assert ok_return_code == 1234, "Ok return code failed"

print("Done.")

Output

Hello from Python :)
Done.

Version 2

CallbackTestLib.hpp

#pragma once

using callback_prototype = void __cdecl();

static char* do_something_buffer;

extern "C" __declspec(dllexport) int __cdecl do_something(callback_prototype);

extern "C" __declspec(dllexport) void __cdecl receive_do_something_buffer(const char*);

CallbackTestLib.cpp

#include "CallbackTestLib.hpp"
#include <iostream>

using namespace std;

auto do_something(callback_prototype cb) -> int
{
    if (!cb) { return 5678; }
    cb();
    cout << "Hello " << do_something_buffer << endl;
    cb();
    cout << "Hello again " << do_something_buffer << endl;
    return 1234;
}

void receive_do_something_buffer(const char* str)
{
    // Create a copy of the given string and save it into buffer.
    if (do_something_buffer) { free(do_something_buffer); }
    do_something_buffer = _strdup(str);
}

CallbackTest.py

import os
import sys
from ctypes import CDLL, CFUNCTYPE, c_int32, c_char_p

assert sys.maxsize > 2 ** 32, "Python x64 required"
assert sys.version_info.major == 3 and sys.version_info.minor == 8 and sys.version_info.micro == 4, "Python 3.8.4 required"

callback_prototype = CFUNCTYPE(None)

dll_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "CallbackTestLib.dll")
testlib = CDLL(dll_path)
testlib.do_something.restype = c_int32
testlib.do_something.argtypes = [callback_prototype]

testlib.receive_do_something_buffer.restype = None
testlib.receive_do_something_buffer.argtypes = [c_char_p]

@callback_prototype
def python_callback_func() -> None:
    testlib.receive_do_something_buffer("from Python :D".encode("utf-8"))

null_return_code = testlib.do_something(callback_prototype())
assert null_return_code == 5678, "Null return code failed"

ok_return_code = testlib.do_something(python_callback_func)
assert ok_return_code == 1234, "Ok return code failed"

print("Done.")

Output

Hello from Python :D
Hello again from Python :D
Done.

Answer 2

Having your Python callback return a char* is similar to a C++ function returning:

char* callback() {
    return new char[10];
}

You've got a memory leak unless you free it. Since Python allocated the bytes object, C++ can't free it correctly, hence the leak.

Instead, pass a C++-managed buffer to the callback:

test.cpp

#include <iostream>
using namespace std;

typedef void (*CB)(char* buf,size_t len);

extern "C" __declspec(dllexport) int func(CB cb) {
    char buf[80];
    if(cb) {
        cb(buf,sizeof buf);
        cout << "Hello " << buf << endl;
        return 1234;
    }
    return 5678;
}

test.py

from ctypes import *

CALLBACK = CFUNCTYPE(None,POINTER(c_char),c_size_t)

@CALLBACK
def callback(buf,size):
    # Cast the pointer to a single char to a pointer to a sized array
    # so it can be accessed safely and correctly.
    arr = cast(buf,POINTER(c_char * size))
    arr.contents.value = b'world!'

dll = CDLL('./test')
dll.func.argtypes = CALLBACK,
dll.func.restype = c_int

print(dll.func(callback))

Output:

Hello world!
1234