How do you use the python3 c api for a command line driven app?

Question

I've been using a custom build as a replacement for virtualenv for a while now, and it's brillant. It takes longer to build, but it actually works, and it never screws up.

Part of this in a simple python wrapper that adds some specific folders to the library path, which I've found very useful. The code for it is trivial:

#include <stdio.h>
#include <n/text/StringUtils.h>
#include <Python.h>

int main(int argc, char *argv[]) {

  /* Setup */
  Py_SetProgramName(argv[0]);
  Py_Initialize();
  PySys_SetArgv(argc, argv);

  /* Add local path */
  PyObject *sys = PyImport_ImportModule("sys");
  PyObject *path = PyObject_GetAttrString(sys, "path");

  /* Custom path */
  char *cwd = nrealpath(argv[0]);
  char *libdir = nstrpath(cwd, "python_lib", NULL);
  PyList_Append(path, PyString_FromString(libdir));
  free(cwd);
  free(libdir);

  /* Run the 'main' module */
  int rtn = Py_Main(argc, argv); // <-- Notice the command line arguments.
  Py_Finalize();

  return rtn;
}

So, moving to python3 is good right? So...

I dutifully replaced the call to PyString_FromString() with PyByte_FromString() and tried to recompile, but it raises errors:

/Users/doug/env/src/main.c:8:21: error: incompatible pointer types passing 'char *' to parameter of type 'wchar_t *' (aka 'int *')
      [-Werror,-Wincompatible-pointer-types]
  Py_SetProgramName(argv[0]);
                    ^~~~~~~
/Users/doug/projects/py-sdl2/py3/include/python3.3m/pythonrun.h:25:45: note: passing argument to parameter here
PyAPI_FUNC(void) Py_SetProgramName(wchar_t *);
                                            ^
/Users/doug/env/src/main.c:10:23: error: incompatible pointer types passing 'char **' to parameter of type 'wchar_t **' (aka 'int **')
      [-Werror,-Wincompatible-pointer-types]
  PySys_SetArgv(argc, argv);
                      ^~~~
/Users/doug/projects/py-sdl2/py3/include/python3.3m/sysmodule.h:12:47: note: passing argument to parameter here
PyAPI_FUNC(void) PySys_SetArgv(int, wchar_t **);
                                              ^
/Users/doug/env/src/main.c:24:27: error: incompatible pointer types passing 'char **' to parameter of type 'wchar_t **' (aka 'int **')
      [-Werror,-Wincompatible-pointer-types]
  int rtn = Py_Main(argc, argv);
                          ^~~~
/Users/doug/projects/py-sdl2/py3/include/python3.3m/pythonrun.h:148:45: note: passing argument to parameter 'argv' here
PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv);
                                            ^
3 errors generated.
make[2]: *** [CMakeFiles/python.dir/src/main.c.o] Error 1
make[1]: *** [CMakeFiles/python.dir/all] Error 2
make: *** [all] Error 2

As you can see from the error, wchar_t is used instead of char *.

How are you supposed to use this api?

I see there are a few examples of doing this, for example: http://svn.python.org/projects/python/tags/r32rc2/Python/frozenmain.c

seriously?

My 29 line program has to become a 110 line monster full of #ifdefs?

Am I misunderstanding, or has the python3 c api really become this ridiculously difficult to use?

Surely I'm missing some obvious convenience function which does this for you, in a simple, portable and cross platform way?

Answer 1

Py_DecodeLocale Return a pointer to a newly allocated wide character string, use PyMem_RawFree() to free the memory. So there is no need to define wchar_t** _argv = PyMem_Malloc(sizeof(wchar_t*)*argc); bellow is my code:

#include <iostream>
#include <string>
#include "hello-time.h"
#include "main/hello-greet.h"
#include "pybind11/embed.h"

namespace py = pybind11;

int main(int argc, char** argv) {
  std::string who = "world";
  if (argc > 1) {
    who = argv[1];
  }
  std::cout << get_greet(who) << std::endl;
  print_localtime();

  int py_argc = 9;
  py::scoped_interpreter guard{};
  wchar_t* wargv[py_argc];
  std::string py_argv[py_argc] = {"convert.py",
                                  "--input",
                                  "resnet_v1_50_inference.pb",
                                  "--inputs",
                                  "input:0",
                                  "--outputs",
                                  "resnet_v1_50/predictions/Reshape_1:0",
                                  "--output",
                                  "resnet50.onnx"};
  for (int i = 0; i < py_argc; i++) {
    wargv[i] = Py_DecodeLocale(py_argv[i].c_str(), nullptr);
    if (wargv[i] == nullptr) {
      fprintf(stderr, "Fatal error: cannot decode py_argv[%d]\n", i);
      exit(1);
    }
  }

  PySys_SetArgv(py_argc, wargv);

  py::module_ sys = py::module_::import("sys");
  py::print(sys.attr("path"));
  print_localtime();
  py::object convert = py::module_::import("tf2onnx.convert");
  py::object do_convert = convert.attr("main");
  do_convert();
  py::print(py::module::import("sys").attr("argv"));

  for (int i = 0; i < py_argc; i++) {
    PyMem_RawFree(wargv[i]);
    wargv[i] = nullptr;
  }
  return 0;
}

Answer 2

The following code contains C++ implementation of Hello World embedding Python.

HelloWorld.cpp:

#include <cstdlib>
#include <stdexcept>

#include <Python.h>

int main(int argc, char* argv[])
{
    wchar_t** wargv = new wchar_t*[argc];
    
    for(int i = 0; i < argc; i++)
    {
        wargv[i] = Py_DecodeLocale(argv[i], nullptr);
        if(wargv[i] == nullptr)
        {
            return EXIT_FAILURE;
        }
    }
    
    Py_SetProgramName(wargv[0]);

    Py_Initialize();
    
    PySys_SetArgv(argc, wargv);
    
    PyRun_SimpleString("import sys");
    PyRun_SimpleString("print('Hello World!', sys.argv)");
    
    Py_Finalize();
    
    for(int i = 0; i < argc; i++)
    {
        PyMem_RawFree(wargv[i]);
        wargv[i] = nullptr;
    }

    delete[] wargv;
    wargv = nullptr;
    
    return 0;
}

Install Prerequisites for Compilation*:

$ sudo apt install build-essential pkg-config python3 python3-dev 

Compiling the Source Code:

$ g++ HelloWorld.cpp `pkg-config python3-embed --libs --cflags` -o HelloWorld

Run:

$ ./HelloWorld
$ # Hello World! ['./HelloWorld']
$
$ ./HelloWorld hi
$ # Hello World! ['./HelloWorld', 'hi']

Answer 3

I spent a lot of time looking for the answer myself. I pieced together comments from other people, and built this snippet to convert char** argv to type wchar_t**:

wchar_t** _argv = PyMem_Malloc(sizeof(wchar_t*)*argc);
for (int i=0; i<argc; i++) {
  wchar_t* arg = Py_DecodeLocale(argv[i], NULL);
  _argv[i] = arg;
}
Py_Initialize();
PySys_SetArgv(argc, _argv);

So far, it works great. I've confirmed that the command line arguments are being received correctly by my embedded Python code.

Answer 4

Seems there's no easy way to do this.

The closest I've come to below. I'll leave the question open in the vague hopes someone will come along and show me the super easy and simple way to do this.

#include <stdio.h>
#include <Python.h>
#include <wchar.h>

int main(int argc, char *argv[]) {

  /* These have to be wchar_t */
  char *str_program_name = argv[0];
  char **str_argv = argv;

  /* For ever stupid reason, these don't need to be wchar_t * */
  char *_sys = "sys";
  char *_libdir = "lib";
  char *_path = "path";
  char *_dot = ".";

#if PY_MAJOR_VERSION >= 3
  wchar_t **_argv = nstrws_array(argc, str_argv);
  wchar_t *_program_name = nstrws_convert(str_program_name);
#else
  char **_argv = str_argv;
  char *_program_name = str_program_name;
#endif

  /* Setup */
  Py_SetProgramName(_program_name);
  Py_Initialize();

  /* Add local path */
#if PY_MAJOR_VERSION >= 3
  PyObject *sys = PyImport_ImportModule(_sys);
  PyObject *path = PyObject_GetAttrString(sys, _path);
  PyList_Append(path, PyBytes_FromString(_dot));
  PyList_Append(path, PyBytes_FromString(_libdir));
#else
  PyObject *sys = PyImport_ImportModule(_sys);
  PyObject *path = PyObject_GetAttrString(sys, _path);
  PyList_Append(path, PyString_FromString(_dot));
  PyList_Append(path, PyString_FromString(_libdir));
#endif

  /* Run the 'main' module */
  int rtn = Py_Main(argc, _argv);
  Py_Finalize();

#if PY_MAJOR_VERSION >= 3
  nstrws_dispose(argc, _argv);
  free(_program_name);
#endif

  return rtn;
}

Using:

/** Unix-like platform char * to wchar_t conversion. */
wchar_t *nstrws_convert(char *raw) {
  wchar_t *rtn = (wchar_t *) calloc(1, (sizeof(wchar_t) * (strlen(raw) + 1)));
  setlocale(LC_ALL,"en_US.UTF-8"); // Unless you do this python 3 crashes.
  mbstowcs(rtn, raw, strlen(raw));
  return rtn;
}

/** Dispose of an array of wchar_t * */
void nstrws_dispose(int count, wchar_t ** values) {
  for (int i = 0; i < count; i++) {
    free(values[i]);
  }
  free(values);
}

/** Convert an array of strings to wchar_t * all at once. */
wchar_t **nstrws_array(int argc, char *argv[]) {
  wchar_t **rtn = (wchar_t **) calloc(argc, sizeof(wchar_t *));
  for (int i = 0; i < argc; i++) {
    rtn[i] = nstrws_convert(argv[i]);
  }
  return rtn;
}

and for windows users, if required:

#include <windows.h>

/** Windows char * to wchar_t conversion. */
wchar_t *nstrws_convert(char *raw) {
  int size_needed = MultiByteToWideChar(CP_UTF8, 0, raw, -1, NULL, 0);
  wchar_t *rtn = (wchar_t *) calloc(1, size_needed * sizeof(wchar_t));
  MultiByteToWideChar(CP_UTF8, 0, raw, -1, rtn, size_needed);
  return rtn;
}

Answer 5

The official recommended way of converting from char to wchar_t is by using Py_DecodeLocale. Like this:

wchar_t *program = Py_DecodeLocale(argv[0], NULL);
Py_SetProgramName(program);

Answer 6

I've found that this works at converting the char* to wchar_t* in the main function:

wchar_t progname[FILENAME_MAX + 1];
mbstowcs(progname, argv[0], strlen(argv[0]) + 1);
Py_SetProgramName(progname);

If you are on unix use:

#include "sys/param.h"

Answer 7

This is probably the wrong way to do it, never the less:

Py_SetProgramName((wchar_t*)argv[0]);

This fix stopped my code from complaining, haven't tested it to know how it handles args, but at least it compiles..