Compiling .pyx to .so using setup.py

Question

I have a module that contains .pyx and their equivalent .cpp.

When I run, python setup.py develop, only the .cpp files get converted to .so. However, since only the .pyx file are readable enough for me to change them, I would like them to be converted to .so instead. This module is large, so compiling the .pyx files individually could be hectic.

Ideally, I would like python setup.py develop to convert the .pyx files directly to .so.

This is the setup file,

#! /usr/bin/env python
#
# Copyright (C) 2012 Mathieu Blondel

import sys
import os

DISTNAME = 'lightning'
DESCRIPTION = "Large-scale sparse linear classification, " + \
              "regression and ranking in Python"
LONG_DESCRIPTION = open('README.rst').read()
MAINTAINER = 'Mathieu Blondel'
MAINTAINER_EMAIL = 'mathieu@mblondel.org'
URL = 'https://github.com/mblondel/lightning'
LICENSE = 'new BSD'
DOWNLOAD_URL = 'https://github.com/mblondel/lightning'
VERSION = '0.1-git'

import setuptools  # we are using a setuptools namespace
from numpy.distutils.core import setup


def configuration(parent_package='', top_path=None):
    if os.path.exists('MANIFEST'):
        os.remove('MANIFEST')

    from numpy.distutils.misc_util import Configuration
    config = Configuration(None, parent_package, top_path)

    config.add_subpackage('lightning')

    return config

if __name__ == "__main__":

    old_path = os.getcwd()
    local_path = os.path.dirname(os.path.abspath(sys.argv[0]))

    os.chdir(local_path)
    sys.path.insert(0, local_path)

    setup(configuration=configuration,
          name=DISTNAME,
          maintainer=MAINTAINER,
          include_package_data=True,
          scripts=["bin/lightning_train",
                   "bin/lightning_predict"],
          maintainer_email=MAINTAINER_EMAIL,
          description=DESCRIPTION,
          license=LICENSE,
          url=URL,
          version=VERSION,
          download_url=DOWNLOAD_URL,
          long_description=LONG_DESCRIPTION,
          zip_safe=False, # the package can run out of an .egg file
          classifiers=[
              'Intended Audience :: Science/Research',
              'Intended Audience :: Developers',
              'License :: OSI Approved',
              'Programming Language :: C',
              'Programming Language :: Python',
              'Topic :: Software Development',
              'Topic :: Scientific/Engineering',
              'Operating System :: Microsoft :: Windows',
              'Operating System :: POSIX',
              'Operating System :: Unix',
              'Operating System :: MacOS'
             ]
    )

And this is the Make File,

PYTHON ?= python
CYTHON ?= cython
NOSETESTS ?= nosetests
DATADIR=$(HOME)/lightning_data

# Compilation...

CYTHONSRC= $(wildcard lightning/impl/*.pyx lightning/impl/randomkit/*.pyx)
CSRC= $(CYTHONSRC:.pyx=.cpp)

inplace:
    $(PYTHON) setup.py build_ext -i

all: cython inplace

cython: $(CSRC)

clean:
    rm -f lightning/impl/*.c lightning/impl/*.html
    rm -f `find lightning -name "*.pyc"`
    rm -f `find lightning -name "*.so"`

%.cpp: %.pyx
    $(CYTHON) --cplus $<

# Tests...
#
test-code: in
    $(NOSETESTS) -s lightning

test-coverage:
    $(NOSETESTS) -s --with-coverage --cover-html --cover-html-dir=coverage \
    --cover-package=lightning lightning

test: test-code test-doc

# Datasets...
#
datadir:
    mkdir -p $(DATADIR)

# regression
download-abalone: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/regression/abalone_scale
    mv abalone_scale $(DATADIR)

download-cadata: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/regression/cadata
    mv cadata $(DATADIR)

download-cpusmall: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/regression/cpusmall_scale
    mv cpusmall_scale $(DATADIR)

download-space_ga: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/regression/space_ga_scale
    mv space_ga_scale $(DATADIR)

download-YearPredictionMSD: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/regression/YearPredictionMSD.bz2
    bunzip2 YearPredictionMSD.bz2
    mv YearPredictionMSD $(DATADIR)
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/regression/YearPredictionMSD.t.bz2
    bunzip2 YearPredictionMSD.t.bz2
    mv YearPredictionMSD.t $(DATADIR)

# binary classification
download-adult: datadir
    ./download.sh http://leon.bottou.org/_media/papers/lasvm-adult.tar.bz2
    tar xvfj lasvm-adult.tar.bz2
    mv adult $(DATADIR)
    rm lasvm-adult.tar.bz2

download-banana: datadir
    ./download.sh http://leon.bottou.org/_media/papers/lasvm-banana.tar.bz2
    tar xvfj lasvm-banana.tar.bz2
    mv banana $(DATADIR)
    rm lasvm-banana.tar.bz2

download-covtype: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/covtype.libsvm.binary.scale.bz2
    bunzip2 covtype.libsvm.binary.scale.bz2
    mv covtype.libsvm.binary.scale $(DATADIR)

download-ijcnn: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/ijcnn1.tr.bz2
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/ijcnn1.t.bz2
    bunzip2 ijcnn1.tr.bz2
    bunzip2 ijcnn1.t.bz2
    mv ijcnn1* $(DATADIR)

download-real-sim: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/real-sim.bz2
    bunzip2 real-sim.bz2
    mv real-sim $(DATADIR)/realsim

download-reuters: datadir
    ./download.sh http://leon.bottou.org/_media/papers/lasvm-reuters.tar.bz2
    tar xvfj lasvm-reuters.tar.bz2
    mv reuters $(DATADIR)
    rm lasvm-reuters.tar.bz2

download-url: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/url_combined.bz2
    bunzip2 url_combined.bz2
    mv url_combined $(DATADIR)

download-waveform: datadir
    ./download.sh http://leon.bottou.org/_media/papers/lasvm-waveform.tar.bz2
    tar xvfj lasvm-waveform.tar.bz2
    mv waveform $(DATADIR)
    rm lasvm-waveform.tar.bz2

download-webspam: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/webspam_wc_normalized_unigram.svm.bz2
    bunzip2 webspam_wc_normalized_unigram.svm.bz2
    mv webspam_wc_normalized_unigram.svm $(DATADIR)/webspam

# multi-class

download-dna: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/dna.scale.tr
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/dna.scale.t
    mv dna* $(DATADIR)

download-letter: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/letter.scale.tr
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/letter.scale.t
    mv letter* $(DATADIR)

download-mnist: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/mnist.scale.bz2
    bunzip2 mnist.scale.bz2
    mv mnist.scale $(DATADIR)
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/mnist.scale.t.bz2
    bunzip2 mnist.scale.t.bz2
    mv mnist.scale.t $(DATADIR)

download-news20: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/news20.scale.bz2
    bunzip2 news20.scale.bz2
    mv news20.scale $(DATADIR)
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/news20.t.scale.bz2
    bunzip2 news20.t.scale.bz2
    mv news20.t.scale $(DATADIR)

download-pendigits: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/pendigits
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/pendigits.t
    mv pendigits* $(DATADIR)

download-protein: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/protein.tr.bz2
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/protein.t.bz2
    bunzip2 protein.tr.bz2
    bunzip2 protein.t.bz2
    mv protein* $(DATADIR)

download-rcv1: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/rcv1_train.multiclass.bz2
    bunzip2 rcv1_train.multiclass.bz2
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/rcv1_test.multiclass.bz2
    bunzip2 rcv1_test.multiclass.bz2
    mv rcv1* $(DATADIR)

download-satimage: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/satimage.scale.tr
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/satimage.scale.t
    mv satimage* $(DATADIR)

download-sector: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/sector/sector.scale.bz2
    bunzip2 sector.scale.bz2
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/sector/sector.t.scale.bz2
    bunzip2 sector.t.scale.bz2
    mv sector* $(DATADIR)

download-usps: datadir
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/usps.bz2
    bunzip2 usps.bz2
    mv usps $(DATADIR)
    ./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/usps.t.bz2
    bunzip2 usps.t.bz2
    mv usps.t $(DATADIR)

And during compilation, I get this output,

compile options: '-I/home/is/anaconda/lib/python2.7/site-packages/numpy/core/include -I/home/is/lightning/lightning/impl/randomkit -I/home/is/anaconda/lib/python2.7/site-packages/numpy/core/include -I/home/is/anaconda/include/python2.7 -c'
c++: svrg_fast.cpp

I would rather have the setup compile the .c code as I can easily convert .pyx to .c. Or is it possible to convert .pyx to C++ ?

Thanks in advance!

Compiling .pyx to .so using setup.py

Answers (1)

Related Questions