"ValueError: call stack is not deep enough" when calling IPython.embed() method from C

Question

I would like to call a function defined in a Python module from the Python C API. For instance, I would like to call IPython.embed(). The following C code leads to a run-time error (with Miniconda 3 with IPython installed).

#include <Python.h>

int
main(int argc, char *argv[])
{
  Py_Initialize();
  PyObject *ipython = PyImport_ImportModule("IPython");
  if (ipython == NULL) {
    PyErr_Print();
    return 1;
  }
  PyObject *embed = PyUnicode_FromString("embed");
  if (embed == NULL) {
    PyErr_Print();
    return 1;
  }
  PyObject *result = PyObject_CallMethodObjArgs(ipython, embed);
  if (result == NULL) {
    PyErr_Print();
    return 1;
  }
  return 0;
}

Error observed:

Traceback (most recent call last):
  File "[...]/miniconda3/lib/python3.5/site-packages/IPython/terminal/embed.py", line 381, in embed
    frame = sys._getframe(1)
ValueError: call stack is not deep enough

However, the code above works fine if I replace IPython and embed by a reference to a simple test module with a mere hello function defined in it.

On the other hand, the following code works as intended (i.e., it runs the IPython REPL) but is not as flexible as the code above and is not suitable for my needs.

PyRun_SimpleString("\n\
  from IPython import embed\n\
  embed()\n\
");

The following code works also as intended, where the code that I have initially given now lives in a callback called with PyRun_SimpleString.

static PyObject *
callback_f(PyObject *obj, PyObject *args)
{
  PyObject *ipython = PyImport_ImportModule("IPython");
  if (ipython == NULL) {
    PyErr_Print();
    exit(1);
  }
  PyObject *embed = PyUnicode_FromString("embed");
  if (embed == NULL) {
    PyErr_Print();
    exit(1);
  }
  PyObject *result = PyObject_CallMethodObjArgs(ipython, embed);
  if (result == NULL) {
    PyErr_Print();
    exit(1);
  }
  Py_RETURN_NONE;
}

int
main(int argc, char *argv[])
{
  Py_Initialize();
  PyObject *module = PyImport_AddModule("test");
  if (module == NULL) {
    PyErr_Print();
    return 1;
  }
  PyMethodDef method_def;
  method_def.ml_name = "callback";
  method_def.ml_meth = callback_f;
  method_def.ml_flags = 1;
  method_def.ml_doc = NULL;
  PyObject *callback_obj = PyCFunction_New(&method_def, NULL);
  if (callback_obj == NULL) {
    PyErr_Print();
    return 1;
  }
  if (PyObject_SetAttrString(module, "callback", callback_obj) != 0) {
    PyErr_Print();
    return 1;
  }
  PyRun_SimpleString("\n\
  from test import callback\n\
  callback()\n\
");
}

I suppose therefore that PyRun_SimpleString performs an initialization related to the stack frame that is necessary to call IPython.embed(), but I don't see where it is documented.

Answer 1

A solution is to insert a new frame created with PyFrame_New, but it is outside the documented Python C API.

#include <Python.h>
#include <frameobject.h>

int
main(int argc, char *argv[])
{
  Py_Initialize();
  PyThreadState *tstate = PyThreadState_GET();
  if (tstate == NULL) {
    PyErr_Print();
    exit(1);
  }
  PyObject *main_module = PyImport_AddModule("__main__");
  if (main_module == NULL) {
    PyErr_Print();
    exit(1);
  }
  PyObject *main_dict = PyModule_GetDict(main_module);
  if (main_dict == NULL) {
    PyErr_Print();
    exit(1);
  }
  PyCodeObject *code_object = PyCode_NewEmpty("foo.py", "f", 0);
  if (code_object == NULL) {
    PyErr_Print();
    exit(1);
  }
  PyFrameObject *root_frame = PyFrame_New(tstate, code_object, main_dict, main_dict);
  if (root_frame == NULL) {
    PyErr_Print();
    exit(1);
  }
  tstate->frame = root_frame;
  PyObject *ipython = PyImport_ImportModule("IPython");
  if (ipython == NULL) {
    PyErr_Print();
    exit(1);
  }
  PyObject *embed = PyUnicode_FromString("embed");
  if (embed == NULL) {
    PyErr_Print();
    exit(1);
  }
  PyObject *result = PyObject_CallMethodObjArgs(ipython, embed);
  if (result == NULL) {
    PyErr_Print();
    exit(1);
  }
}