How to store a pointer in a custom embedded Python object

Question

I have created a custom Python type in C as per the tutorial https://docs.python.org/2.7/extending/newtypes.html#the-basics. In my C I receive a pointer to a struct, I want to be able to get and set the values in the struct from Python without taking a copy of it. I.e.

a = myObject.x() # gets the x value in the struct.

or

myObject.x(255) # sets the x value in the struct.

However I cannot see how to store the pointer in the python object.

My current object definition is currently just the basic object implementation from the python website.

typedef struct {
    PyObject_HEAD
    myStruct *s;
} KeyObject;

static PyTypeObject KeyType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    "ckb.Key",             /* tp_name */
    sizeof(KeyObject), /* tp_basicsize */
    0,                         /* tp_itemsize */
    0,                         /* tp_dealloc */
    0,                         /* tp_print */
    0,                         /* tp_getattr */
    0,                         /* tp_setattr */
    0,                         /* tp_compare */
    0,                         /* tp_repr */
    0,                         /* tp_as_number */
    0,                         /* tp_as_sequence */
    0,                         /* tp_as_mapping */
    0,                         /* tp_hash */
    0,                         /* tp_call */
    0,                         /* tp_str */
    0,                         /* tp_getattro */
    0,                         /* tp_setattro */
    0,                         /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT,        /* tp_flags */
    "Key objects",           /* tp_doc */
};

static PyMethodDef key_methods[] = {
    {NULL}  /* Sentinel */
};

Answer 1

Here's an example (cbk.c), that can act as a backbone for this task:

#include "external.h"
#include "Python.h"

#define MOD_NAME "ckb"
#define KEY_CLASS_NAME "Key"


/*
typedef struct InnerStruct_tag {
    int x;
} InnerStruct;
//*/

typedef struct KeyObject_tag {
    PyObject_HEAD
    InnerStruct *inner;
} KeyObject;


static PyObject *Key_new(PyTypeObject *type, PyObject *args, PyObject *kwds) {
    KeyObject *self;
    self = (KeyObject*)type->tp_alloc(type, 0);
    if (self != NULL) {
        //self->inner = (InnerStruct*)calloc(1, sizeof(Key));
        self->inner = getExternalPtr(1234);  // Don't allocate here, get the pointer from external lib
        if (self->inner == NULL) {
            Py_DECREF(self);
            return NULL;
        }
    }
    return (PyObject*)self;
}

static void Key_dealloc(KeyObject *self) {
    //free(self->inner);
    delExternalPtr(self->inner);  // Use the external dellocation function (optional)
    Py_TYPE(self)->tp_free((PyObject*)self);
}


static PyObject *Key_getX(KeyObject *self, void *closure) {
    return PyInt_FromLong(self->inner->x);
}

static int Key_setX(KeyObject *self, PyObject *value, void *closure) {
    if (value == NULL) {
        PyErr_SetString(PyExc_TypeError, "Cannot delete 'x'");
        return -1;
    }
    if (!PyInt_Check(value)) {
        PyErr_SetString(PyExc_TypeError, "'x' value must be an int");
        return -1;
    }
    self->inner->x = ((PyIntObject*)value)->ob_ival;
    return 0;
}

static PyGetSetDef Key_getsets[] = {
    {"x", (getter)Key_getX, (setter)Key_setX, "x", NULL},
    {NULL}  // Sentinel
};


static PyTypeObject Key_Type = {
    PyVarObject_HEAD_INIT(NULL, 0)
    MOD_NAME"."KEY_CLASS_NAME, /* tp_name */
    sizeof(KeyObject),         /* tp_basicsize */
    0,                         /* tp_itemsize */
    (destructor)Key_dealloc,   /* tp_dealloc */
    0,                         /* tp_print */
    0,                         /* tp_getattr */
    0,                         /* tp_setattr */
    0,                         /* tp_compare */
    0,                         /* tp_repr */
    0,                         /* tp_as_number */
    0,                         /* tp_as_sequence */
    0,                         /* tp_as_mapping */
    0,                         /* tp_hash */
    0,                         /* tp_call */
    0,                         /* tp_str */
    0,                         /* tp_getattro */
    0,                         /* tp_setattro */
    0,                         /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT |
        Py_TPFLAGS_BASETYPE,   /* tp_flags */
    KEY_CLASS_NAME" objects",  /* tp_doc */
    0,                         /* tp_traverse */
    0,                         /* tp_clear */
    0,                         /* tp_richcompare */
    0,                         /* tp_weaklistoffset */
    0,                         /* tp_iter */
    0,                         /* tp_iternext */
    0,                         /* tp_methods */
    0,                         /* tp_members */
    Key_getsets,               /* tp_getset */
    0,                         /* tp_base */
    0,                         /* tp_dict */
    0,                         /* tp_descr_get */
    0,                         /* tp_descr_set */
    0,                         /* tp_dictoffset */
    0,                         /* tp_init */
    0,                         /* tp_alloc */
    Key_new,                   /* tp_new */
};

#define Key_CheckExact(op) ((op)->ob_type == &Key_Type)


static PyMethodDef module_methods[] = {
    {NULL}  // Sentinel
};

PyMODINIT_FUNC initckb(void) {
    PyObject* m;
    if (PyType_Ready(&Key_Type) < 0)
        return;
    m = Py_InitModule3(MOD_NAME, module_methods,
        MOD_NAME": Example module that creates an extension type ("KEY_CLASS_NAME").");
    Py_INCREF(&Key_Type);
    PyModule_AddObject(m, KEY_CLASS_NAME, (PyObject*)&Key_Type);
}

Notes:

• Existing structure names / members was renamed (reorganized), for clarity
• The inner structure only has one member (x), which is enough for making a point
• Everything relies solely on the [Python 2.7.Docs]: Defining New Types page (that was mentioned in the question as well)
• Since the wrapper object (Key) contains pointers (inner), in order to avoid checking them for NULL every time they are accessed:
  • A constructor (Key_new - equivalent to __new__ in Python) - that initializes them - was added
  • A destructor (Key_dealloc - equivalent __del__ in Python) - that does the exact opposite - was added as well, to avoid memory leaks (this is just a previous bullet consequence)
• Access to InnerStruct's x member is done via Key_getX, Key_setX functions (note they are referenced in Key_getsets):
  • From Python, the inner x member, will be accessed by key_instance.x, as it was a Key instance attribute
    • This way makes more sense than using a getter (get_x()) and a setter (set_x(value)), and is more Pythonic
    • If however the getter / setter way is preferred, Key_getX, Key_setX signatures should be slightly modified (I think removing the last argument would do), and they should be referenced in Key_methods - that should be specified to KeyType as tp_methods (also described in the above web page)
  • When adding new members to InnerStruct, only the stuff done for x needs to be replicated and adapted (of course if there will be functions that look too similar, code should be refactored - but this is outside current scope)
• The last part is pretty standard Python extension module code

---

Update #0

After the 1^st comment, it seems like the question is trickier than it seems. Not sure if I still get it wrong, because it doesn't seem such a big deal. The change is (as I understood), that the inner pointer should come from somewhere else (another library (.dll)), instead of being created in the constructor. Changed to the example to mimic the new (and hopefully expected) behavior:

• Since a InnerStruct pointer is returned by the external library (called it external.dll), the structure definition was moved in a header file belonging to that library - called it external.h (below), which is included by cbk.c
• Would make sense that a library exports some data via a function (also exported by the library): getExternalPtr which may take arguments - currently it only has (a dummy) one: dummyArg0
• Since getExternalPtr allocates memory inside, would make sense to have a corresponding function that deallocates it (delExternalPtr), in order to avoid memory leaks and Undefined Behavior (e.g. if memory is allocated in one place, deallocated in another, and the 2 places are deserved by different C runtimes). Any pointer returned by getExternalPtr should be passed to delExternalPtr exactly once
• The 2 above functions will now be called from Key_new and Key_dealloc. If this is still not OK, and the object needs to be modified after creation (although it may be possible that some race issues would arise), setting the member could be done like: ((KeyObject*)keyInstancePyObjectPtr)->inner = getExternalPtr(0); with just one catch:
  • keyInstancePyObjectPtr (which is a generic PyObject*) should be of type Key_Type. Key_CheckExact macro does exactly that check

• Now, the module depends (is linked to) on the external lib (not sure how things actually are), but that can be changed to Dynamic (DLL (SO)) loading (via [Man7]: DLOPEN(3) / [Man7]: DLSYM(3) or [MSDN]: LoadLibrary function) / [MSDN]: GetProcAddress function

  external library code:

  • external.h:

    #if defined (WIN32)
    #  if defined (EXTERNAL_DYNAMIC)
    #    if defined EXTERNAL_EXPORTS
    #      define EXTERNAL_EXPORT __declspec(dllexport)
    #    else
    #      define EXTERNAL_EXPORT __declspec(dllimport)
    #    endif
    #  else
    #    define EXTERNAL_EXPORT
    #  endif
    #else
    #  define EXTERNAL_EXPORT
    #endif
    
    
    typedef struct InnerStruct_tag {
        int x;
    } InnerStruct;
    
    
    #if defined (__cplusplus)
    extern "C" {
    #endif
    
    EXTERNAL_EXPORT InnerStruct *getExternalPtr(int dummyArg0);
    EXTERNAL_EXPORT void delExternalPtr(InnerStruct *ptr);
    
    #if defined (__cplusplus)
    }
    #endif
    

  • external.c:

    #include "external.h"
    #include <stdlib.h>
    
    
    InnerStruct *getExternalPtr(int dummyArg0) {
        InnerStruct *ret = (InnerStruct*)malloc(sizeof(InnerStruct));
        if (ret != NULL)
            ret->x = 1618;
        return ret;
    }
    
    void delExternalPtr(InnerStruct *ptr) {
        free(ptr);
    }
    

Test program (ckb_test.py):

import traceback
import ckb

print "\nModule:", ckb
print "Dir:", dir(ckb)

print "\nClass:", ckb.Key
print "Dir:", dir(ckb.Key)

key = ckb.Key()
print "\nInstance:", key
print "Dir:", dir(key)

print "\nKey.x (initial):", key.x
key.x = 123
print "Key.x (modified):", key.x

try:
    key.x = 1.0
except:
    traceback.print_exc()

del(key)
print "\nEnd"

Output:

c:\Work\Dev\StackOverflow\q46833364>set PATH=%PATH%;.\external\Win32-Release

c:\Work\Dev\StackOverflow\q46833364>set PYTHONPATH=%PYTHONPATH%;.\ckb\Win32-Release

c:\Work\Dev\StackOverflow\q46833364\>"c:\Install\x86\HPE\OPSWpython\2.7.10__00\python.exe" ckb_test.py

Module: <module 'ckb' from 'c:\Work\Dev\StackOverflow\q46833364\ckb\Win32-Release\ckb.pyd'>
Dir: ['Key', '__doc__', '__file__', '__name__', '__package__']

Class: <type 'ckb.Key'>
Dir: ['__class__', '__delattr__', '__doc__', '__format__', '__getattribute__', '__hash__', '__init__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'x']

Instance: <ckb.Key object at 0x027A7050>
Dir: ['__class__', '__delattr__', '__doc__', '__format__', '__getattribute__', '__hash__', '__init__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'x']

Key.x (initial): 1618
Key.x (modified): 123
Traceback (most recent call last):
  File "..\ckb_test.py", line 20, in <module>
    key.x = 1.0
TypeError: 'x' value must be an int

End