Modularize libmambapy (#2960)

* Modularize libmambapy

* Add test-libmambapy to Taskfile

* Fix libmamba installation

* Fix standalone libmambapy configuration

* Change libmambapy layout

* Add scikit-build

* Fix libmambapy extension name

* Add submodules shims

* Fix libmambaoy tests

* Fix stubgen

* Adapt libmambapy tests to scikit-build

* Read version from Python

* Replace confusing names

.github/workflows/unix_impl.yml

@@ -39,7 +39,8 @@ jobs:
           cmake -B build/ -G Ninja \
             --preset mamba-unix-shared-${{ inputs.build_type }}  \
             -D CMAKE_CXX_COMPILER_LAUNCHER=sccache \
-            -D CMAKE_C_COMPILER_LAUNCHER=sccache
+            -D CMAKE_C_COMPILER_LAUNCHER=sccache \
+            -D BUILD_LIBMAMBAPY=OFF
           cmake --build build/ --parallel
       - name: Show build cache statistics
         run: sccache --show-stats
@@ -97,9 +98,9 @@ jobs:
           environment-name: build_env
       - name: Install libmambapy
         run: |
-          ln build/libmambapy/bindings* libmambapy/libmambapy/
           cmake --install build/ --prefix "${CONDA_PREFIX}"
-          python -m pip install ./libmambapy/
+          # TODO add some ccache and parallelism to builds
+          python -m pip install --no-deps --no-build-isolation ./libmambapy
       - name: Run libmamba Python bindings tests
         run: |
           python -m pytest libmambapy/tests/

.github/workflows/windows_impl.yml

@@ -41,7 +41,8 @@ jobs:
             --preset mamba-win-shared-${{ inputs.build_type }}  ^
             -D CMAKE_MSVC_RUNTIME_LIBRARY="MultiThreadedDLL" ^
             -D CMAKE_CXX_COMPILER_LAUNCHER=sccache ^
-            -D CMAKE_C_COMPILER_LAUNCHER=sccache
+            -D CMAKE_C_COMPILER_LAUNCHER=sccache ^
+            -D BUILD_LIBMAMBAPY=OFF
           if %errorlevel% neq 0 exit /b %errorlevel%
           cmake --build build/ --parallel
           if %errorlevel% neq 0 exit /b %errorlevel%
@@ -84,13 +85,6 @@ jobs:
           unset CONDARC  # Interferes with tests
           cd ./build/libmamba && ./tests/test_libmamba
 
-      # - name: Run libmamba tests
-      #   run: |
-      #     @REM Interferes with tests
-      #     set CONDARC=
-      #     @REM Move to directory of libmamba DLL for Windows to find it
-      #     cd "build\libmamba" && .\tests\test_libmamba
-
   libmambapy_tests_win:
     name: Test libmamba Python bindings
     needs: ["build_shared_win"]
@@ -111,11 +105,10 @@ jobs:
           environment-name: build_env
           init-shell: bash cmd.exe
       - name: Install libmambapy
-        shell: bash -elo pipefail {0}
         run: |
-          ln build/libmambapy/bindings*.pyd libmambapy/libmambapy/
-          cmake --install build/ --prefix "${CONDA_PREFIX}"
-          python -m pip install ./libmambapy/
+          cmake --install build/ --prefix %CONDA_PREFIX%
+          # TODO add some ccache and parallelism to builds
+          python -m pip install --no-deps --no-build-isolation ./libmambapy
       - name: Run libmamba Python bindings tests
         run: |
           python -m pytest libmambapy/tests/

.pre-commit-config.yaml

@@ -41,6 +41,7 @@ repos:
           - flake8-builtins==2.1.0
           - flake8-bugbear==23.9.16
           - flake8-isort==6.1.0
+        exclude: libmambapy/src
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: v16.0.6
     hooks:

Taskfile.dist.yml

@@ -155,14 +155,23 @@ tasks:
     cmds:
       - >-
         {{.TEST_RUN}} python -m pip install
-        --no-deps --no-build-isolation --ignore-installed --editable libmambapy/
+        --no-deps --no-build-isolation --ignore-installed ./libmambapy/
+
+  _test-libmambapy:
+    internal: true
+    deps: [install-py]
+    cmds:
+      - >-
+        {{.TEST_RUN}} python -m pytest libmambapy/tests/ {{.args}}
+  test-libmambapy:
+    cmds: [{task: _test-libmambapy, vars: {args: '{{.CLI_ARGS}}'}}]
 
   stubgen:
     deps: [install-py]
     cmds:
-      - '{{.TEST_RUN}} python -m pybind11_stubgen -o "{{.BUILD_DIR}}/stubs" libmambapy.bindings'
-      - cp "{{.BUILD_DIR}}/stubs/libmambapy/bindings-stubs/__init__.pyi" libmambapy/libmambapy/
-      - '{{.DEV_RUN}} pre-commit run --files libmambapy/libmambapy/__init__.pyi'
+      - '{{.TEST_RUN}} python -m pybind11_stubgen -o "{{.BUILD_DIR}}/stubs" libmambapy.core.bindings'
+      - cp "{{.BUILD_DIR}}/stubs/libmambapy/core/bindings-stubs/__init__.pyi" libmambapy/src/libmambapy/__init__.pyi
+      - '{{.DEV_RUN}} pre-commit run --files libmambapy/src/libmambapy/__init__.pyi'
 
   clean: 'rm -rf {{.BUILD_DIR}}'
 

dev/environment-dev.yml

@@ -38,6 +38,8 @@ dependencies:
   - pip:
     - securesystemslib
   # libmambapy build dependencies
+  - scikit-build
+  - build
   - pybind11-stubgen <1.0
   # libmambapy dependencies
   - python

libmamba/CMakeLists.txt

@@ -660,7 +660,7 @@ set(
 
 install(
     TARGETS ${libmamba_targets}
-    EXPORT ${PROJECT_NAME}-targets
+    EXPORT ${PROJECT_NAME}Targets
     ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
     RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
@@ -674,13 +674,6 @@ install(
     PATTERN "*.h"
 )
 
-# Makes the project importable from the build directory
-export(
-    EXPORT ${PROJECT_NAME}-targets
-    FILE "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Targets.cmake"
-    NAMESPACE mamba::
-)
-
 # Configure 'mambaConfig.cmake' for a build tree
 set(MAMBA_CONFIG_CODE "####### Expanded from \@MAMBA_CONFIG_CODE\@ #######\n")
 set(
@@ -711,8 +704,12 @@ install(
           ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
     DESTINATION ${LIBMAMBA_CMAKECONFIG_INSTALL_DIR}
 )
+# Need to install the FindLibsolv for the installed target to work
+install(FILES "../cmake/modules/FindLibsolv.cmake" DESTINATION ${LIBMAMBA_CMAKECONFIG_INSTALL_DIR})
+
 install(
-    EXPORT ${PROJECT_NAME}-targets
-    FILE ${PROJECT_NAME}Targets.cmake
+    EXPORT ${PROJECT_NAME}Targets
+    NAMESPACE mamba::
     DESTINATION ${LIBMAMBA_CMAKECONFIG_INSTALL_DIR}
+    COMPONENT Mamba_Development
 )

libmamba/libmambaConfig.cmake.in

@@ -22,25 +22,31 @@ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR};${CMAKE_MODULE_PATH}")
 @LIBMAMBA_CONFIG_CODE@
 
 include(CMakeFindDependencyMacro)
+find_dependency(CURL)
+find_dependency(LibArchive)
+find_dependency(zstd)
+find_dependency(BZip2)
+find_dependency(OpenSSL)
 find_dependency(fmt)
-find_dependency(nlohmann_json)
 find_dependency(spdlog)
-find_dependency(Threads)
 find_dependency(tl-expected)
-find_dependency(zstd)
-find_dependency(BZip2)
+find_dependency(nlohmann_json)
+find_dependency(simdjson)
 find_dependency(yaml-cpp)
+find_dependency(reproc)
+find_dependency(reproc++)
+find_dependency(Libsolv MODULE)
 
-if(NOT (TARGET libmamba OR TARGET libmamba-static))
+if(NOT (TARGET libmamba-dyn OR TARGET libmamba-static))
     include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
 
     if (TARGET mamba::libmamba-dyn)
-        get_target_property(@PROJECT_NAME@_INCLUDE_DIR libmamba INTERFACE_INCLUDE_DIRECTORIES)
-        get_target_property(@PROJECT_NAME@_LIBRARY libmamba LOCATION)
+        get_target_property(@PROJECT_NAME@_INCLUDE_DIR mamba::libmamba-dyn INTERFACE_INCLUDE_DIRECTORIES)
+        get_target_property(@PROJECT_NAME@_LIBRARY mamba::libmamba-dyn LOCATION)
     endif()
 
     if (TARGET mamba::libmamba-static)
-        get_target_property(@PROJECT_NAME@_INCLUDE_DIR libmamba-static INTERFACE_INCLUDE_DIRECTORIES)
-        get_target_property(@PROJECT_NAME@_STATIC_LIBRARY libmamba-static LOCATION)
+        get_target_property(@PROJECT_NAME@_INCLUDE_DIR mamba::libmamba-static INTERFACE_INCLUDE_DIRECTORIES)
+        get_target_property(@PROJECT_NAME@_STATIC_LIBRARY mamba::libmamba-static LOCATION)
     endif()
 endif()

libmambapy/CMakeLists.txt

@@ -5,6 +5,8 @@
 # The full license is in the file LICENSE, distributed with this software.
 cmake_minimum_required(VERSION 3.18.2)
 
+include("../cmake/CompilerWarnings.cmake")
+
 cmake_policy(SET CMP0025 NEW)
 cmake_policy(SET CMP0077 NEW)
 cmake_policy(SET CMP0057 NEW)
@@ -13,16 +15,45 @@ project(libmambapy)
 
 if(NOT TARGET mamba::libmamba)
     find_package(libmamba REQUIRED)
+    set(libmamba_target mamba::libmamba-dyn)
+else()
+    set(libmamba_target mamba::libmamba)
 endif()
 
 find_package(Python COMPONENTS Interpreter Development)
 find_package(pybind11 REQUIRED)
 
-pybind11_add_module(bindings src/main.cpp longpath.manifest)
+pybind11_add_module(
+    bindings
+    src/libmambapy/bindings/longpath.manifest
+    src/libmambapy/bindings/bindings.cpp
+    src/libmambapy/bindings/legacy.cpp
+)
 
-mamba_target_add_compile_warnings(bindings WARNING_AS_ERROR ${MAMBA_WARNING_AS_ERROR})
+target_include_directories(bindings PRIVATE src/libmambapy/bindings)
 
-target_link_libraries(bindings PRIVATE pybind11::pybind11 mamba::libmamba)
-set_property(TARGET bindings PROPERTY CXX_STANDARD 17)
+mamba_target_add_compile_warnings(bindings WARNING_AS_ERROR ${MAMBA_WARNING_AS_ERROR})
 
-install(TARGETS bindings LIBRARY DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/libmambapy/)
+target_link_libraries(bindings PRIVATE pybind11::pybind11 ${libmamba_target})
+target_compile_features(bindings PRIVATE cxx_std_17)
+
+# Installation
+
+if(SKBUILD)
+    install(TARGETS bindings DESTINATION ${MAMBA_INSTALL_PYTHON_EXT_LIBDIR})
+else()
+    # WARNING: this default should probably not be used as it is but set extranlly by a proper
+    # Python packager tool
+    set(
+        MAMBA_INSTALL_PYTHON_EXT_LIBDIR
+        "lib"
+        CACHE PATH "Installation directory for Python extension"
+    )
+
+    install(
+        TARGETS bindings
+        EXCLUDE_FROM_ALL
+        COMPONENT Mamba_Python_Extension
+        DESTINATION ${MAMBA_INSTALL_PYTHON_EXT_LIBDIR}
+    )
+endif()

libmambapy/pyproject.toml

@@ -0,0 +1,34 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel",
+    "scikit-build>=0.13",
+    "cmake>=3.18",
+    "ninja",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "libmambapy"
+authors = [
+    {name = "Wolf Vollprecht"},
+    {name = "Adrien Delsalle"},
+    {name = "Jonas Haag"},
+    {name = "QuantStack", email = "info@quantstack.net"},
+    {name = "Other contributors"},
+]
+maintainers = [
+    {name = "QuantStack", email = "info@quantstack.net"},
+]
+description = "A fast library to interact with the Conda package ecosystem"
+requires-python = ">=3.7"
+keywords = ["mamba", "conda", "packaging"]
+license = {text = "BSD-3-Clause"}
+dependencies = []
+dynamic = ["version"]
+[projet.url]
+Documentation = "https://mamba.readthedocs.io"
+Repository = "https://github.com/mamba-org/mamba/"
+
+[tool.setuptools]
+platforms = ["Windows", "Linux", "Mac OS X"]

libmambapy/setup.py

@@ -1,3 +1,37 @@
-import setuptools
+import importlib.util
+import os
+import pathlib
 
-setuptools.setup()
+import skbuild
+import skbuild.constants
+
+__dir__ = pathlib.Path(__file__).parent.absolute()
+
+
+def CMAKE_INSTALL_DIR():
+    """Where scikit-build configures CMAKE_INSTALL_PREFIX."""
+    return os.path.abspath(skbuild.constants.CMAKE_INSTALL_DIR())
+
+
+def libmambapy_version():
+    """Get the version of libmambapy from its version module."""
+    spec = importlib.util.spec_from_file_location(
+        "libmambapy_version", __dir__ / "src/libmambapy/version.py"
+    )
+    ver = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(ver)
+    return ver.__version__
+
+
+skbuild.setup(
+    version=libmambapy_version(),
+    packages=["libmambapy", "libmambapy.bindings"],
+    package_dir={"": "src"},
+    package_data={"libmambapy": ["py.typed", "__init__.pyi"]},
+    cmake_languages=["CXX"],
+    cmake_minimum_required_version="3.17",
+    cmake_install_dir="src/libmambapy",  # Must match package_dir layout
+    cmake_args=[
+        f"-DMAMBA_INSTALL_PYTHON_EXT_LIBDIR={CMAKE_INSTALL_DIR()}/src/libmambapy",
+    ],
+)

libmambapy/src/libmambapy/__init__.py

@@ -0,0 +1,5 @@
+import libmambapy.version
+from libmambapy.bindings.legacy import *  # Legacy which used to combine everything
+
+# Define top-level attributes
+__version__ = libmambapy.version.__version__