Use Boost python with libtorch

Question

I am trying to create a python package from C++ code with Boost python. However when including libtorchin the code, the resulting python package shows strange errors such as

Boost.Python.ArgumentError: Python argument types in Test.init(Test, int, str) did not match C++ signature: init(_object*, int, std::string)"

Creating a C++ executable from the code works fine and creating python packages with Boost python without libtorch dependencies works also fine, so the problem seems to come from libtorchand boost pythonnot working properly together. Has anybody encountered the same and found a solutoin to this?

Here is my example:

test.h

#ifndef TEST_H
#define TEST_H

#include <string>

class Test{

    int tensorSize;
    std::string label;

    public:
        Test(int i, std::string label);
    void printTensor();
};

#endif

test.cpp

#include <boost/python.hpp>
#include <string>
#include "test.h"
#include <torch/torch.h>
#include <torch/script.h>


#include <iostream>

Test::Test(int i, std::string lbl) {
    tensorSize = i;
    label = lbl;
}

void Test::printTensor(){
    auto x = torch::randn({10, tensorSize});

    std::cout << label << '\n';
    std::cout << "x:\n" << x << '\n';
}


using namespace boost::python;
BOOST_PYTHON_MODULE(test)
{
    class_<Test>("Test", init<int, std::string>())
    .def("printTensor", &Test::printTensor)
    ;
}

main.cpp (for the executable)

#include "test.h"

int main() {
    Test t(3, "test");
    t.printTensor();
    return 0;
}

and the test file for testing the python package

import test

t = test.Test(3, "test")
t.printTensor()

for compiling I use CMAKE, here the CMakeLists.txt

cmake_minimum_required(VERSION 3.20)
project(boostTest)

set(BOOST_MIN_VERSION 1.86.0)
find_package(Torch REQUIRED)
find_package(PythonLibs 3 REQUIRED)

find_package(
    Boost ${BOOST_MIN_VERSION} REQUIRED
    COMPONENTS python REQUIRED
)
#C++ executable
include_directories(${PYTHON_INCLUDE_PATH})
add_executable(test.out main.cpp test.cpp  test.h)
target_link_libraries(test.out ${PYTHON_LIBRARIES} ${TORCH_LIBRARIES} Boost::python)

#boost python
set(CMAKE_SHARED_MODULE_PREFIX "")
add_library(test MODULE test.cpp)
target_link_libraries(test ${Boost_LIBRARIES} ${PYTHON_LIBRARIES} ${TORCH_LIBRARIES} )
target_include_directories(test PRIVATE ${PYTHON_INCLUDE_DIRS})

I compiled the example above. The resulting executable works fine, the python package throws an error.

Answer 1

I tried to repro. All I ran into was the missing dependency on Boost Graph:

• File CMakeLists.txt

  cmake_minimum_required(VERSION 3.19)
  project(sotest)
  #set(CMAKE_CXX_COMPILER_LAUNCHER ccache)
  set(CMAKE_EXPORT_COMPILE_COMMANDS On)
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb ")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -march=native ")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic")
  
  find_package(Torch REQUIRED)
  link_libraries("${TORCH_LIBRARIES}")
  
  find_package(Threads REQUIRED)
  link_libraries(Threads::Threads)
  
  find_package(Boost CONFIG 1.64.0 COMPONENTS python graph)
  link_libraries(Boost::python Boost::graph)
  
  find_package(Python3 COMPONENTS Development)
  link_libraries(Python3::Python)
  
  add_executable(sotest test.cpp main.cpp)
  add_library(sopytorch SHARED test.cpp)
  
  #target_include_directories(sotest PUBLIC ${Python3_INCLUDE_DIRS})
  

• File test.h

  #include <string>
  
  class Test{
  
      int tensorSize;
      std::string label;
  
      public:
          Test(int i, std::string label);
      void printTensor();
  };
  

• File test.cpp

  #include "test.h"
  #include <boost/python.hpp>
  #include <string>
  #include <torch/script.h>
  #include <torch/torch.h>
  
  #include <iostream>
  
  Test::Test(int i, std::string lbl) {
      tensorSize = i;
      label = lbl;
  }
  
  void Test::printTensor(){
      auto x = torch::randn({10, tensorSize});
  
      std::cout << label << '\n';
      std::cout << "x:\n" << x << '\n';
  }
  
  
  using namespace boost::python;
  BOOST_PYTHON_MODULE(test)
  {
      class_<Test>("Test", init<int, std::string>())
      .def("printTensor", &Test::printTensor)
      ;
  }
  
  

• File main.cpp

  #include "test.h"
  
  int main() {
      Test t(3, "test");
      t.printTensor();
  }
  

• File test.py

  import test
  
  t = test.Test(3, "test")
  t.printTensor()
  

PS

Oh yeah, I forgot to mention a step, but I assume you know you need to copy the shared library so python can find it, e.g. with

ln -sfv build/libsopytorch.so test.so