Import Pybind11/C++ compiled module not working

Question

I'm fairly new to python setuptools and dist. I can't seem to get a c++ wrapper module to import so that I may use the functions.

The compiled .so file shows up in the installed-files.txt after pip install but yet importing the wrapper does not.

setup.py

import subprocess
import os
from pathlib import Path

from setuptools import setup, Extension, find_packages
from setuptools.command.build_ext import build_ext


class CMakeExtension(Extension):
    def __init__(self, name):
        Extension.__init__(self, name, sources=[])


class CMakeBuild(build_ext):
    def run(self):
        for ext in self.extensions:
            self.build_cmake(ext)
        super().run()

    def build_cmake(self, ext):
        try:
            subprocess.check_output(["cmake", "--version"])
        except OSError:
            raise RuntimeError(
                "CMake must be installed to build the following extensions: "
                + ", ".join(e.name for e in self.extensions)
            )

        cwd = Path().absolute()

        # these dirs will be created in build_py, so if you don't have
        # any python sources to bundle, the dirs will be missing
        build_temp = Path(self.build_temp)
        build_temp.mkdir(parents=True, exist_ok=True)
        extdir = Path(self.get_ext_fullpath(ext.name))
        extdir.mkdir(parents=True, exist_ok=True)

        pyenv_root = os.environ.get("PYENV_ROOT")

        cmake_args = [
            f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}",
            "-DCMAKE_BUILD_TYPE=Release",
            "-DTRANSIT_INCLUDE_TESTS:BOOL=OFF",
        ]

        if pyenv_root is not None:
            cmake_args += [f"-DPYTHON_EXECUTABLE={pyenv_root}/shims/python"]

        build_args = ["--config", "Release", "--", "-j2"]

        env = os.environ.copy()

        self.announce("Running CMake prepare", level=3)
        subprocess.check_call(["cmake", cwd] + cmake_args, cwd=build_temp, env=env)

        self.announce("Building extensions")
        cmake_cmd = ["cmake", "--build", "."] + build_args
        subprocess.check_call(cmake_cmd, cwd=build_temp)


setup(
    name="bgtfs_py_lib",
    version="3.2.2",
    long_description="",
    zip_safe=False,
    install_requires=[
        "redis==2.10.6",
        "cffi==1.11.5",
        "numpy==1.15.3",
        "patricia-trie==10",
        "PuLP==1.6.8",
        "py-lz4framed==0.13.0",
        "pycparser==2.19",
        "pyparsing==2.2.2",
        "pypng==0.0.18",
        "pyproj==1.9.5.1",
        "python-graph-core==1.8.2",
        "pytz==2018.6",
        "redis==2.10.6",
        "requests==2.21.0",
        "six==1.11.0",
        "tabulate==0.8.2",
        "unicodecsv==0.14.1",
        "Unidecode==1.0.22",
    ],
    ext_modules=[CMakeExtension("bgtfs_py_lib.bgtfs_py_lib_wrapper")],
    cmdclass=dict(build_ext=CMakeBuild),
    packages=find_packages(exclude=["tests"]),
    package_data={"": "*.so"},
)

CMakeLists.txt

cmake_minimum_required(VERSION 3.8)

project(bgtfs_py_lib_wrapper)
include(submodules/transitLib/bgtfs/bgtfsLib/TransitUtils/transit_shared.cmake)

# bgtfsPyLib
set(PYBIND11_CPP_STANDARD -std=c++14)
set(PYBIND11_PYTHON_VERSION 3.6)

add_subdirectory(submodules/transitLib transitLib)
add_subdirectory(pybind11)

include_directories(
    cpp
    submodules/transitLib/bgtfs/bgtfsLib/
    submodules/transitLib/bgtfs/bgtfsLib/bgtfsLib
)

pybind11_add_module(bgtfs_py_lib_wrapper MODULE NO_EXTRAS
    cpp/pybindCasts.cpp
    cpp/bgtfsPyLibWrapper.cpp
    cpp/BgtfsFeedHandler.cpp
)

target_link_libraries(bgtfs_py_lib_wrapper PRIVATE transitLib)
set_target_properties(bgtfs_py_lib_wrapper PROPERTIES
    LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/bgtfs_py_lib
    LIBRARY_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_LIST_DIR}/bgtfs_py_lib
    LIBRARY_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_LIST_DIR}/bgtfs_py_lib
)
target_compile_definitions(bgtfs_py_lib_wrapper PRIVATE TRANSIT_SERVER)
target_compile_definitions(transitLib PRIVATE TRANSIT_SERVER)
target_compile_definitions(bgtfsLib PRIVATE ENABLE_BACKWARD_FILE_COMPATIBILITY YES)
set_default_target_properties(bgtfs_py_lib_wrapper)

I am trying to use virtualenv in order to isolate the modules required to run in my project.

Here is the file struct:

.
|-- CMakeLists.txt
|-- README.md
|-- bgtfs_py_lib
|   |-- __init__.py
|   |-- bgtfs_handler
|-- cpp
|   |-- BgtfsFeedHandler.cpp
|   |-- BgtfsFeedHandler.h
|   |-- bgtfsPyLibWrapper.cpp
|   `-- pybindCasts.cpp
|-- deploy.sh
|-- make.sh
|-- pybind11
|-- setup.py
|-- submodules
|-- test.sh
`-- tests
    |-- __init__.py
    |-- __pycache__
    |-- fixtures
    |-- test.py
    `-- test_functions.py

The init.py file in bgtfs_py_lib looks like this. The functions of the wrapper are being exposed.

import bgtfs_py_lib_wrapper as _bgtfs_py_lib
from bgtfs_py_lib.bgtfs_handler.bgtfs_handler import BgtfsHandler

In the other project it is being pip installed using git+ssh and egg .

git+ssh://git@github.com/path/to/project.git@build/production/setup#egg=bgtfs_py_lib

When I ctrl+space in pyCharm the wrapper module is found and the Classes are present.
The module is located in Binary Skeletons directory but yet

import bgtfs_py_lib_wrapper as _bgtfs_py_lib just simply does not work and the following exception is thrown: ModuleNotFoundError: No module named 'bgtfs_py_lib_wrapper'

Can someone please help me figure out how to properly build C++/Pybind11 modules and use them in a pip installed package with virtualenv?

Import Pybind11/C++ compiled module not working

Answers (1)

Related Questions