Virtual override in boost python objects created by static method (or factory (?))

Question

I am trying to create a class in python that overrides a (pure) virtual function in a C++ class (using boost.python). The catch is that the C++ class is created via a static member function (all constructors are private or deleted). I have successfully created the class Base and a BaseWrap class that python "knows" about. I have also been able to create a pure virtual function that can be overridden in python. However, my problem is when a member function of Base calls the pure virtual function. When this happens, the class cannot find the python implementation and the program crashes.

Here is the C++ code:

#include <iostream>
#include <boost/python.hpp>
#include <boost/static_assert.hpp>

#define CREATE(NAME) \
  static std::shared_ptr<NAME> Create() { \
    std::cout << "STATIC BASE CREATE" << std::endl; \
    return std::make_shared<NAME>();  \
  }

class Base {
protected:
  Base() { std::cout << "BASE DEFAULT CONSTRUCTOR" << std::endl; }
private:

  std::string CallSay() {
    return Say(); 
  }

  virtual std::string Say() const = 0;
};

class BaseWrap : public Base, public boost::python::wrapper<Base> {
public:
  BaseWrap() : Base() { std::cout << "BASEWRAP DEFAULT CONSTRUCTOR" << std::endl; }

  virtual std::string Say() const override
  {
    std::cout << "C++ Say" << std::endl;
    return this->get_override("say") ();
  }

  CREATE(BaseWrap)
};

BOOST_PYTHON_MODULE(Example)
{
  namespace python = boost::python;

  // Expose Base.
  python::class_<BaseWrap, std::shared_ptr<BaseWrap>, boost::noncopyable>("Base", python::no_init)
    .def("__init__", python::make_constructor(&BaseWrap::Create))
    .def("Say", python::pure_virtual(&Base::Say))
    .def("CallSay", &Base::CallSay);
}

and the python code to test the issue:

import sys
import Example

class PythonDerived(Example.Base):
    def __init__(self):
        print "PYTHON DEFAULT CONSTRUCTOR"
        Example.Base.__init__(self)

    def Say(self):
         return "Python Say"

d = PythonDerived()
print d
print 
print d.Say()
print
print d.CallSay()

Which, when run, gives the output:

PYTHON DEFAULT CONSTRUCTOR
STATIC BASE CREATE
BASE DEFAULT CONSTRUCTOR
BASEWRAP DEFAULT CONSTRUCTOR
<__main__.PythonDerived object at 0x1091caf70>

Python Say

C++ Say
Traceback (most recent call last):
  File "test.py", line 20, in <module>
    print d.CallSay()
 TypeError: 'NoneType' object is not callable

It looks like the Base::CallSay method is finding the implementation of BaseWrap::Say but cannot find the python implementation. Does anyone know why or how make this work?

Thanks!

Answer 1

I realize that this is an almost decade old question but here is my take on it in case anyone is looking for an elegant solution by using custom policies.

template<typename HeldType, typename BasePolicies = default_call_policies, int iSelf = -1>
struct initialize_wrapper_policies : BasePolicies
{
    template<typename ArgumentPackage>
    static PyObject *postcall(const ArgumentPackage &args, PyObject *pResult)
    {
        PyObject *pSelf = boost::python::detail::get(boost::mpl::int_<iSelf>(), args);
        boost::python::detail::initialize_wrapper(
            pSelf,
            get_pointer((HeldType)extract<HeldType>(pSelf))
        );

        return BasePolicies::postcall(args, pResult);
    }
};

 

python::class_<BaseWrap, std::shared_ptr<BaseWrap>, boost::noncopyable>("Base", python::no_init)
    .def("__init__", 
        python::make_constructor(
            &BaseWrap::Create,
            initialize_wrapper_policies<std::shared_ptr<BaseWrap> >()
        )
    );

Answer 2

This looks as though it is a bug in Boost.Python.

The boost::python::wrapper hierarchy is not getting initialized in the functor returned from boost::python::make_constructor. With the wrapper hierarchy having no handle to the Python object, get_override() returns NoneType, and attempting to call NoneType raises the TypeError exception.

To resolve this, one can explicitly initialize the wrapper hierarchy. Below is a complete example that provides a generic way to accomplish this. Instead of using make_constructor(), one can use make_wrapper_constructor(). I have opted to not use C++11 features. As such, there will be some boilerplate code that could be reduced with variadic templates, but porting to C++11 should be fairly trivial.

#include <iostream>
#include <boost/function_types/components.hpp>
#include <boost/function_types/result_type.hpp>
#include <boost/make_shared.hpp>
#include <boost/mpl/insert.hpp>
#include <boost/python.hpp>

namespace detail {

/// @brief wrapper_constructor will force the initialization
///        of the wrapper hierarchy when a class is held by
///        another type and inherits from boost::python::wrapper.
template <typename Fn>
class wrapper_constructor
{
public:

  typedef typename boost::function_types::result_type<Fn>::type result_type;

public:

  /// @brief Constructor.
  wrapper_constructor(Fn fn)
    : constructor_(boost::python::make_constructor(fn))
  {}

  /// @brief Construct and initialize python object.
  result_type operator()(boost::python::object self)
  {
    constructor_(self);
    return initialize(self);
  }

  /// @brief Construct and initialize python object.
  template <typename A1>
  result_type operator()(boost::python::object self, A1 a1)
  {
    constructor_(self, a1);
    return initialize(self);
  }

  // ... overloads for arguments, or use variadic templates.

private:

  /// @brief Explicitly initialize the wrapper.
  static result_type initialize(boost::python::object self)
  {
    // Extract holder from self.
    result_type ptr = boost::python::extract<result_type>(self);

    // Explicitly initialize the boost::python::wrapper hierarchy.
    initialize_wrapper(self.ptr(),        // PyObject.
                       get_pointer(ptr)); // wrapper hierarchy.

    return ptr;
  }

private:
  boost::python::object constructor_;
};

} // namespace detail

/// @brief Makes a wrapper constructor (constructor that works with
///        classes inheriting from boost::python::wrapper).
template <typename Fn>
boost::python::object make_wrapper_constructor(Fn fn)
{
  // Python constructors take the instance/self argument as the first
  // argument.  Thus, inject the 'self' argument into the provided
  // constructor function type.
  typedef typename boost::function_types::components<Fn>::type
      components_type;
  typedef typename boost::mpl::begin<components_type>::type begin;
  typedef typename boost::mpl::next<begin>::type self_pos;
  typedef typename boost::mpl::insert<
    components_type, self_pos, boost::python::object>::type signature_type;

  // Create a callable python object that defers to the wrapper_constructor.
  return boost::python::make_function(
    detail::wrapper_constructor<Fn>(fn),
    boost::python::default_call_policies(),
    signature_type());
}

class Base
{
protected:
  Base(int x) : x(x) { std::cout << "BASE DEFAULT CONSTRUCTOR" << std::endl; }
  virtual ~Base() {}
  int x;
public:
  std::string CallSay() { return Say(); }
  virtual std::string Say() const = 0;
};

class BaseWrap:
  public Base,
  public boost::python::wrapper<Base>
{
public:
  BaseWrap(int x):
    Base(x)
  { std::cout << "BASEWRAP DEFAULT CONSTRUCTOR" << std::endl; }

  virtual std::string Say() const 
  {
    std::cout << "C++ Say: " << x << std::endl;
    return this->get_override("Say")();
  }

  static boost::shared_ptr<BaseWrap> Create(int x)
  {
    return boost::make_shared<BaseWrap>(x);
  }
};

BOOST_PYTHON_MODULE(example)
{
  namespace python = boost::python;

  // Expose Base.
  python::class_<BaseWrap, boost::shared_ptr<BaseWrap>,
                 boost::noncopyable>("Base", python::no_init)
    .def("__init__", make_wrapper_constructor(&BaseWrap::Create))
    .def("Say", python::pure_virtual(&Base::Say))
    .def("CallSay", &Base::CallSay)
    ;
}

And its usage:

>>> import example
>>> class PythonDerived(example.Base):
...     def __init__(self, x):
...         print "PYTHON DEFAULT CONSTRUCTOR"
...         example.Base.__init__(self, x)
...     def Say(self):
...          return "Python Say"
... 
>>> d = PythonDerived(5)
PYTHON DEFAULT CONSTRUCTOR
BASE DEFAULT CONSTRUCTOR
BASEWRAP DEFAULT CONSTRUCTOR
>>> d
<__main__.PythonDerived object at 0xb7e688ec>
>>> d.Say()
'Python Say'
>>> d.CallSay()
C++ Say: 5
'Python Say'

Answer 3

I found a work around that seems to address this issue. It is a bit "hacky" feeling so if anyone has a better solution it would be appreciated.

Basically I wrote a helper class so the C++ code becomes:

#include <iostream>
#include <boost/python.hpp>
#include <boost/python/module.hpp>
#include <boost/python/class.hpp>
#include <boost/python/manage_new_object.hpp>
#include <boost/python/return_value_policy.hpp>

#define CREATE(NAME)                                \
  static inline std::shared_ptr<NAME> Create()      \
  {                                                 \
    std::cout << "STATIC BASE CREATE" << std::endl; \
    return std::make_shared<NAME>();                \
  }

class Base {
protected:

  Base()
  {
    std::cout << "BASE DEFAULT CONSTRUCTOR" << std::endl;
  }

public:

  std::string CallSay()
  {
    return Say();
  }

  virtual std::string Say() const = 0;
};

class BaseHelper {
public:

  BaseHelper() {}

  virtual std::string eval() = 0;
};

class BaseHelperWrap : public BaseHelper, public boost::python::wrapper<BaseHelper> {
public:

  BaseHelperWrap() : BaseHelper() {}

  virtual std::string eval() override
  {
    return this->get_override("eval") ();
  }
};

class BaseWrap : public Base, public boost::python::wrapper<Base> {
public:

  BaseWrap() : Base()
  {
    std::cout << "BASEWRAP DEFAULT CONSTRUCTOR" << std::endl;
  }

  virtual std::string Say() const override
  {
    std::cout << "C++ Say" << std::endl;

    return func->eval();
  }

  CREATE(BaseWrap) 

  static std::shared_ptr<BaseWrap> PyCreate(std::shared_ptr<BaseHelper> const& f)
  {
    std::shared_ptr<BaseWrap> ptr = Create();
    ptr->set_func(f);
    return ptr;
  }

private:

  void set_func(std::shared_ptr<BaseHelper> const& f)
  {
    func = f;
  }

  std::shared_ptr<BaseHelper> func;
};

BOOST_PYTHON_MODULE(Example)
{
  namespace python = boost::python;

  python::def("make_foo", make_foo, python::return_value_policy<python::manage_new_object>());

  // Expose Base.
  python::class_<BaseWrap, std::shared_ptr<BaseWrap>, boost::noncopyable>("Base", python::no_init)
  .def("Create", &BaseWrap::PyCreate).staticmethod("Create") 
  .def("CallSay", &Base::CallSay);


  python::class_<BaseHelperWrap, std::shared_ptr<BaseHelperWrap>, boost::noncopyable>("BaseHelper", python::init<>())
    .def("eval", python::pure_virtual(&BaseHelper::eval));

  python::implicitly_convertible<std::shared_ptr<BaseHelperWrap>, std::shared_ptr<BaseHelper> >();
}

And the python code:

import sys
import Example

class PyBaseHelper(Example.BaseHelper):
    def eval(self):
        return "Python Say"

h = PyBaseHelper()
d = Example.Base.Create(h)

print 
print
print d.CallSay()

It works ... but isn't as elegant of a fix as I was hoping.