How to instantiate a custom object from within a C module?

Question

I'm having trouble creating an instance of a class (type) I wrote from within the C module. I've written a minimal, self-contained example that illustrates the point. Just copy-paste the three files spam.c, spamtest.py, and setup.py to a directory and run

$ python setup.py develop && python spamtest.py

I don't understand why the essential functions new() and init() are not called when instantiating a spam instance. Needless to say, this causes big time segfaults in the real application where those functions allocate dynamic memory for newly created objects.

Here's what happens when running spamtest.py under a debugging version of Python. Note that new() and init() are called when instantiating a spam object from within the Python interpreter, but not from C.

(pyenv36d) $ python spamtest.py 
---------
From Python
New Spam at 0x7fa7a5ca9280
Init Spam at 0x7fa7a5ca9280
Finalize Spam at 0x7fa7a5ca9280
---------
From C
Finalize Spam at 0x7fa7a5ca92c0
---------
* ob
object  : <refcnt 0 at 0x7fa7a5bb6670>
type    : bytes
refcount: 0
address : 0x7fa7a5bb6670
* op->_ob_prev->_ob_next
<NULL object>
* op->_ob_next->_ob_prev
object  : <refcnt 0 at 0x7fa7a5bb6670>
type    : bytes
refcount: 0
address : 0x7fa7a5bb6670
Fatal Python error: UNREF invalid object

Current thread 0x00007fa7a5ff8080 (most recent call first):
Aborted
(pyenv36d) $ 

The module:

#include <Python.h>

typedef struct {
     PyObject_HEAD
} Spam;

static PyObject *new(PyTypeObject *type,
         PyObject *args, PyObject *kw) {
     Spam *self;

     self = (Spam *) type->tp_alloc(type, 0);
     fprintf(stderr, "New Spam at %p\n", self);
     return (PyObject*)self;
}

static int init(Spam *self, PyObject *args, PyObject *kw) {
     fprintf(stderr, "Init Spam at %p\n", self);
     return 0;
}

static void finalize(PyObject *self) {
     fprintf(stderr, "Finalize Spam at %p\n", self);
}

static PyMethodDef spam_methods[] = {
     {NULL, NULL, 0, NULL},
};

static PyTypeObject spam_type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     .tp_name = "Spam",
     .tp_basicsize = sizeof(Spam),
     .tp_flags = 0
         | Py_TPFLAGS_DEFAULT
         | Py_TPFLAGS_BASETYPE,
     .tp_doc = "Spam object",
     .tp_methods = spam_methods,
     .tp_new = new,
     .tp_init = (initproc) init,
     .tp_dealloc = finalize,
};

/* To create a new Spam object directly from C */
PyObject *make_spam() {
     Spam *spam;
     if (PyType_Ready(&spam_type) != 0) {
         Py_RETURN_NONE;
     }
     spam = PyObject_New(Spam, &spam_type);
     PyObject_Init((PyObject *)spam, &spam_type);
     return (PyObject *) spam;
}

static PyMethodDef module_methods[] = {
     {"make_spam",  (PyCFunction)make_spam, METH_NOARGS,
      "Instantiate and return a new Spam object."},
     {NULL, NULL, 0, NULL}
};

static PyModuleDef spam_module = {
     PyModuleDef_HEAD_INIT,
     "spam",
     "Defines the Spam (time, value) class"
     ,
     -1,
     module_methods
};

PyMODINIT_FUNC PyInit_spam(void) {
     PyObject *m;
     m = PyModule_Create(&spam_module);
     if (PyType_Ready(&spam_type) < 0) {
         return NULL;
     }
     PyModule_AddObject(m, "Spam", (PyObject*)&spam_type);
     return m;
}

setup.py

from setuptools import setup, Extension

spam = Extension('spam', sources=['spam.c'])

setup (
    name = 'spam',
    version = '0.1',
    description = 'Trying to instantiate an object from C',
    ext_modules = [spam],
    packages = [],
)

spamtest.py:

from spam import Spam, make_spam

print("---------\nFrom Python")
s1 = Spam()
del s1

print("---------\nFrom C")
s2 = make_spam()
del s2
print("---------")

Answer 1

One answer is to construct your new object by calling the type object, in the same way as you would from Python itself.

That is, change make_spam() to do something more like:

/* To create a new Spam object directly from C */
PyObject *make_spam() {
     Spam *spam;
     if (PyType_Ready(&spam_type) != 0) {
         Py_RETURN_NONE;
     }
     spam = (Spam *)PyObject_CallObject((PyObject *)&spam_type, Py_BuildValue(""));
     /* your other init here, assuming you have some */
     return (PyObject *) spam;
}

This puzzles me too. I'd like to know why your original code doesn't work, and if there's a better way to do this "properly", but this approach is working for me.

Answer 2

One possibility is that this line:

 PyModule_AddObject(m, "Spam", (PyObject*)&spam_type);

should instead be this (as per the documentation):

Py_INCREF(&spam_type);
if (PyModule_AddObject(m, "Spam", (PyObject *) &spam_type) < 0) {
    Py_DECREF(&spam_type);
    Py_DECREF(m);
    return NULL;
}