5

I'm trying to build a C++ library together with Cython bindings following the structure of https://bloerg.net/2012/11/10/cmake-and-distutils.html.

The issue is that during make install, the extension will be compiled twice. This double compilation does not happen when there is only one main CMakeLists.txt in the main folder (with paths adjusted). Here are the details:

My project structure is

.
├── CMakeLists.txt
├── python
│   ├── CMakeLists.txt
│   ├── a_py.pxd
│   ├── a_py.pyx
│   └── setup.py.in
└── src
    ├── A.cpp
    └── A.h

The top level CMakeLists.txt only contains add_subdirectory(python).

python/CMakeLists.txt is

IF(NOT ${PYTHON})
    find_program(PYTHON "python")
ENDIF()

set(SETUP_PY_IN "${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in")
set(SETUP_PY    "${CMAKE_CURRENT_BINARY_DIR}/setup.py")

set(PY_OUTPUT      "${CMAKE_CURRENT_BINARY_DIR}/build/pytimestamp")

configure_file(
    ${SETUP_PY_IN}
    ${SETUP_PY}
)

add_custom_command(OUTPUT "${PY_OUTPUT}"
                   COMMAND ${PYTHON} ${SETUP_PY} build_ext
                   COMMAND ${CMAKE_COMMAND} -E touch ${PY_OUTPUT}
               )

add_custom_target(a_py ALL DEPENDS ${PY_OUTPUT})

install(CODE "execute_process(COMMAND ${PYTHON} ${SETUP_PY} install)")

setup.py is:

from distutils.core import setup
from distutils.extension import Extension
from Cython.Distutils import build_ext
ext_modules = [
    Extension(
    name="a",
    sources=["${CMAKE_CURRENT_SOURCE_DIR}/a_py.pyx", "${CMAKE_CURRENT_SOURCE_DIR}/../src/A.cpp"],
    include_dirs = ['${CMAKE_CURRENT_SOURCE_DIR}/../src'],
    language="c++",
    ),
]
setup(
    name = 'a',
    version='${PROJECT_VERSION}',
    cmdclass = {'build_ext': build_ext},
    ext_modules = ext_modules,
    package_dir={ 'a': '${CMAKE_CURRENT_SOURCE_DIR}' },
)

In both cases (CMakeFile.txt in root or in python subfolder), first the build_ext step runs:

Scanning dependencies of target a_py
[100%] Generating build/pytimestamp
running build_ext

and compiles the generated a_py.cpp and A.cpp and links the library. During the install step, the compilation is run again only when CMakeFile.txt is in the python subfolder.

This is what happens during installation:

running build_ext
skipping '/Users/xxx/tmp/ctest/t08/python/a_py.cpp' Cython extension (up-to-date)
building 'a' extension
creating build

Note that the a_py.pyx does not get cythonized again, but the build directory gets recreated (it is the same between the build and install steps) and the files are compiled (with exactly the same compiler and linker invocations).

A full example can be found here: https://github.com/zeeMonkeez/cmakeCythonTest

zeeMonkeez
  • 5,057
  • 3
  • 33
  • 56

1 Answers1

2

I had the same problem. Apparently, in my case, during the install phase setup.py had trouble finding the directory where it put the compiled extensions in the build phase, so it recompiled them even with the --skip-build switch.

I solved by specifying the same directory path at both build and install phases with the --build-lib and --build-dir switches respectively:

    add_custom_command(OUTPUT ${PY_OUTPUT}
            COMMAND ${PYTHON} ${SETUP_PY} build_ext --build-lib ${CMAKE_CURRENT_BINARY_DIR}/mysoext
            COMMAND ${CMAKE_COMMAND} -E touch ${PY_OUTPUT}
            DEPENDS ${DEPS}
            )
...
        set(MYINSTCMD "\
            EXECUTE_PROCESS(COMMAND ${PYTHON} ${SETUP_PY} install_lib \
            --skip-build \
            --install-dir /my/install/dir \
            --build-dir ${CMAKE_CURRENT_BINARY_DIR}/mysoext)")
        install(CODE "${MYINSTCMD}")
PJ_Finnegan
  • 1,981
  • 1
  • 20
  • 17