feat: ci

.github/workflows/buildwheels.yml

@@ -0,0 +1,85 @@
+name: Build
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build_sdist:
+    name: Build SDist
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+
+    - name: Build SDist
+      run: pipx run build --sdist
+
+    - name: Check metadata
+      run: pipx run twine check dist/*
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: dist-sdist
+        path: dist/*.tar.gz
+
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        # macos-13 is an intel runner, macos-14 is apple silicon
+        os: [ubuntu-latest, macos-14, windows-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # see https://github.com/llvm/llvm-project/issues/86250#issuecomment-2219995275
+      - name: install DIA SDK
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          set -x
+          mkdir -p "C:/Program Files (x86)/Microsoft Visual Studio/2019/Professional" || true
+          cp -rv "C:/Program Files/Microsoft Visual Studio/2022/Enterprise/DIA SDK" "C:/Program Files (x86)/Microsoft Visual Studio/2019/Professional/DIA SDK"
+
+      # - name: Set up QEMU
+      #   if: runner.os == 'Linux'
+      #   uses: docker/setup-qemu-action@v3
+      #   with:
+      #     platforms: all
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.19.2
+
+      - name: Upload wheels artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-${{ matrix.os }}
+          path: ./wheelhouse/*.whl
+
+
+  pypi-publish:
+    name: Upload to PyPI
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/llvmpym
+    permissions:
+      id-token: write
+
+    steps:
+    - uses: actions/setup-python@v5
+
+    - uses: actions/download-artifact@v4
+      with:
+        pattern: dist-*
+        path: dist
+        merge-multiple: true
+
+    - name: Display structure of downloaded files
+      run: ls -R ./dist
+
+    - name: Publish package distributions to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -2,6 +2,7 @@
 /dist
 /build
 /llvm-build
+/wheelhouse
 
 /pyrightconfig.json
 

CMakeLists.txt

@@ -1,6 +1,13 @@
 cmake_minimum_required(VERSION 3.20.0...3.29.6)
 
-project(${SKBUILD_PROJECT_NAME} LANGUAGES CXX)
+# default for both c and c++ (c is needed for MacOS build)
+project(${SKBUILD_PROJECT_NAME})
+
+# required for linking fmt library during cibuildwheels build on manylinux_2_28
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+if(UNIX)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+endif()
 
 include(FetchContent)
 FetchContent_Declare(
@@ -13,33 +20,14 @@ FetchContent_MakeAvailable(fmt)
 if (NOT SKBUILD)
   message(WARNING "\
   This CMake file is meant to be executed using 'scikit-build'. Running
-  it directly will almost certainly not produce the desired result. If
-  you are a user trying to install this package, please use the command
-  below, which will install all necessary build dependencies, compile
-  the package in an isolated environment, and then install it.
-  =====================================================================
-   $ pip install .
-  =====================================================================
-  If you are a software developer, and this is your own package, then
-  it is usually much more efficient to install the build dependencies
-  in your environment once and use the following command that avoids
-  a costly creation of a new virtual environment at every compilation:
-TODO outdated
-  =====================================================================
-   $ pip install nanobind scikit-build-core[pyproject]
-   $ pip install --no-build-isolation -ve .
-  =====================================================================
-  You may optionally add -Ceditable.rebuild=true to auto-rebuild when
-  the package is imported. Otherwise, you need to re-run the above
-  after editing C++ files.")
+  it directly will almost certainly not produce the desired result.")
 endif()
 
 if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
   set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
   set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
 endif()
 
-
 # Try to import all Python components potentially needed by nanobind
 find_package(Python 3.8
   REQUIRED COMPONENTS Interpreter Development.Module
@@ -56,7 +44,6 @@ message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}")
 message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
 
 include_directories(${CMAKE_SOURCE_DIR}/src/llvm)
-include_directories(${CMAKE_SOURCE_DIR}/vender)
 
 include_directories(${LLVM_INCLUDE_DIRS})
 separate_arguments(LLVM_DEFINITIONS_LIST NATIVE_COMMAND ${LLVM_DEFINITIONS})
@@ -70,12 +57,6 @@ nanobind_add_module(
   # does nothing on older Python versions
   STABLE_ABI
 
-  # Build libnanobind statically and merge it into the
-  # extension (which itself remains a shared library)
-  #
-  # If your project builds multiple extensions, you can
-  # replace this flag by NB_SHARED to conserve space by
-  # reusing a shared libnanobind across libraries
   NB_STATIC
 
   # Source code goes here
@@ -91,8 +72,14 @@ llvm_map_components_to_libnames(llvm_libs core transformutils)
 target_link_libraries(llvmpym_ext PRIVATE ${llvm_libs} fmt::fmt)
 # -flto and --exclude-libs allow us to remove those parts of LLVM we don't use
 # although we are going build a full llvm python binding :-
-set_property(TARGET llvmpym_ext APPEND_STRING PROPERTY LINK_FLAGS "-flto -Wl,--exclude-libs,ALL") # taken from llvmite
-
+# TODO for MacOS immitate llvmlite only expose certain symbols. Can it work?
+# https://github.com/numba/llvmlite/blob/78ebf9bf188379b2642112aff388480384306c6b/ffi/CMakeLists.txt#L76C53-L76C61
+if(UNIX)
+  set_property(TARGET llvmpym_ext APPEND_STRING PROPERTY LINK_FLAGS "-flto")
+  if(NOT APPLE)
+    set_property(TARGET llvmpym_ext APPEND_STRING PROPERTY LINK_FLAGS " -Wl,--exclude-libs,ALL")
+  endif()
+endif()
 
 # Stub Files
 nanobind_add_stub(

README.md

@@ -2,13 +2,54 @@
 
 Note this project is still in the very early stage, not even half completed.   
 
+## Installation
+
+``` shell
+pip install llvmpym
+```
+
+### Additional notes for NixOS User
+
+If you encounter issues like `ImportError: libz.so.1: cannot open shared object file: No such file or directory` when using this library  
+You can use the following command to inspect those shared libraries that are not founded.  
+
+``` shell
+ldd ./.venv/lib/python3.12/site-packages/llvmpym/llvmpym_ext.abi3.so
+```
+
+You can use the following command to give this library a quick run (if you miss `libz.so.1`
+and `libstdc++.so.6`):
+
+``` shell
+env LD_LIBRARY_PATH="$(nix path-info nixpkgs#zlib)/lib:$(nix path-info nixpkgs#stdenv.cc.cc.lib)/lib:$LD_LIBRARY_PATH" python ./main.py
+```
+
+Or using a `shellHook` in your `devShell`:
+``` nix
+devShells = forAllSystems ({ pkgs }: {
+  default = pkgs.mkShell {
+      packages = with pkgs; [
+        (python312.withPackages (ppkgs: with ppkgs; [
+          pip
+        ]))
+      ];
+      shellHook = ''
+        export LD_LIBRARY_PATH="${pkgs.zlib}/lib:${pkgs.stdenv.cc.cc.lib.outPath}/lib:$LD_LIBRARY_PATH"
+      '';
+    };
+});
+```
+
+Currently I haven't invested into how to solve the problem from its root. If you 
+know how to solve it, please give it a hand. It would really be appreciated.
+
 ## Build
 
 ### Requirements
 + A C++17 compiler
 + CMake 3.20.0+
 + ninja (optional, if you want to follow the guide)
-+ Python at least 3.8 (not tested. Try 3.12 if failed)
++ Python at least 3.12
 
 ### Manual Build of LLVM
 

flake.nix

@@ -15,6 +15,9 @@
             (python312.withPackages (ppkgs: with ppkgs; [
               pip
             ]))
+
+            llvm_18
+            cmake # need for finding ZLIB::ZLIB LLVM cmake dependency
           ];
         };
       });

justfile

@@ -18,7 +18,7 @@ install-normal:
     pip install .
 
 install-dev-requirements:
-    pip install nanobind scikit-build-core[pyproject] pytest build
+    pip install nanobind scikit-build-core[pyproject] pytest build cibuildwheel
 
 # Example: just find-components ./build/cp312-abi3-linux_x86_64/ LLVMCloneModule
 # Find which llvm archive file contains the desired symbol

note.typ

@@ -11,9 +11,9 @@
 - https://github.com/vovkos/llvm-package-windows
 - https://github.com/c3lang/win-llvm?tab=readme-ov-file
 == ci reference
+- https://github.com/makslevental/mlir-wheels
 - https://github.com/c3lang/c3c/blob/master/.github/workflows/main.yml
 - https://github.com/revng/llvmcpy/blob/master/.github/workflows/ci.yml
-- https://github.com/leapmotion/pyopticam/blob/main/src/pyopticam_ext.cpp
 
 
 = Misc

pyproject.toml

@@ -1,19 +1,16 @@
 [project]
 name = "llvmpym"
-version = "0.0.1"
+version = "0.0.2"
 description = "LLVM python binding"
 readme = "README.md"
-requires-python = ">=3.8"
+# can only be compiled for python >= 3.12
+requires-python = ">=3.12"
 license = {file = "LICENSE"}
 authors = [
   { name = "Meow King", email = "mr.meowking@anche.no" },
 ]
 classifiers = [
   "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.8",
-  "Programming Language :: Python :: 3.9",
-  "Programming Language :: Python :: 3.10",
-  "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
 ]
 
@@ -26,9 +23,34 @@ build-backend = "scikit_build_core.build"
 
 [tool.scikit-build]
 build-dir = "build/{wheel_tag}" # Setuptools-style build caching in a local directory
-wheel.py-api = "cp312" # Build stable ABI wheels for CPython 3.12+
+# since it can only be compiled for python >= 3.12, we here specify the version
+# so that it will target ABI3
+wheel.py-api = "cp312" 
 
 [tool.cibuildwheel]
 build-verbosity = 1
-test-command = "pytest {project}/tests" # Run pytest to ensure that the package was correctly built
-test-requires = "pytest"
+build-frontend = "build"
+archs = "auto64"
+skip = "*musllinux*" # doesn't find LLVM musl build
+before-all = "bash ./scripts/action/install_llvm.sh"
+
+# test-command = "pytest {project}/tests" # Run pytest to ensure that the package was correctly built
+# test-requires = "pytest"
+
+[tool.cibuildwheel.linux]
+# archs = ["auto64", "aarch64"]
+environment = { SKBUILD_CMAKE_ARGS="-DLLVM_DIR=$(pwd)/llvm-build/lib/cmake/llvm"}
+manylinux-x86_64-image = "manylinux_2_28"
+manylinux-aarch64-image = "manylinux_2_28"
+
+[tool.cibuildwheel.windows]
+environment = { SKBUILD_CMAKE_ARGS="-DLLVM_DIR=D:/a/llvmpym/llvmpym/llvm-build/lib/cmake/llvm" }
+
+[tool.cibuildwheel.macos.environment]
+# Needed for full C++17 support
+MACOSX_DEPLOYMENT_TARGET = "14.0"
+PATH="/opt/homebrew/opt/llvm/bin:$PATH"
+SKBUILD_CMAKE_ARGS="-DCMAKE_CXX_FLAGS=-I/opt/homebrew/opt/llvm/include;-DCMAKE_EXE_LINKER_FLAGS=-L/opt/homebrew/opt/llvm/lib;-DCMAKE_SHARED_LINKER_FLAGS=-L/opt/homebrew/opt/llvm/lib/c++;-DLDFLAGS=-L/opt/homebrew/opt/llvm/lib/c++ -Wl,-rpath,/opt/homebrew/opt/llvm/lib/c++"
+
+
+

scripts/action/install_llvm.sh

@@ -0,0 +1,44 @@
+sys="$(uname -o)"
+arch="$(uname -m)"
+
+
+# Linux
+if [[ "$sys" == "GNU/Linux" ]]; then 
+
+    if [[ "$arch" == "x86_64" ]]; then
+        url="https://github.com/llvm/llvm-project/releases/download/llvmorg-18.1.8/clang+llvm-18.1.8-x86_64-linux-gnu-ubuntu-18.04.tar.xz"
+        root_llvm_dir_name="clang+llvm-18.1.8-x86_64-linux-gnu-ubuntu-18.04"
+
+    elif [[ "$arch" == "aarch64" ]]; then
+        url="https://github.com/llvm/llvm-project/releases/download/llvmorg-18.1.8/clang+llvm-18.1.8-aarch64-linux-gnu.tar.xz"
+        root_llvm_dir_name="clang+llvm-18.1.8-aarch64-linux-gnu.tar.xz"
+    fi
+
+    # install libraries
+    DISTRO_NMAE=$(awk -F= '$1=="NAME" { print $2 ;}' /etc/os-release)
+    if [[ "$DISTRO_NMAE" == "\"AlmaLinux\"" ]]; then
+        dnf install -y ncurses-devel # Terminfo::terminfo required by LLVM
+    fi
+
+
+# Windows
+elif [[ "$sys" == "Msys" ]]; then
+    url="https://github.com/llvm/llvm-project/releases/download/llvmorg-18.1.8/clang+llvm-18.1.8-x86_64-pc-windows-msvc.tar.xz"
+    root_llvm_dir_name="clang+llvm-18.1.8-x86_64-pc-windows-msvc"
+
+    # install libraries
+    # vcpkg install zlib
+
+# MacOS
+elif [[ "$sys" == "Darwin" ]]; then # Mac (only for 14 or later, i.e. apple silicon)
+    # install library
+    brew install llvm@18
+    exit
+fi
+
+
+# Linux & Windows
+curl -L -o ./llvm-build.tar.xz "$url"
+tar -xf ./llvm-build.tar.xz
+mv "$root_llvm_dir_name" ./llvm-build
+