[ESI][Runtime] Build and publish wheels (#6763)

Make the publishing pipeline actually work to produce wheels for multiple python versions.

.github/workflows/esiRuntimePublish.yml

@@ -1,6 +1,11 @@
 name: Publish ESI Runtime
 
 on:
+  push:
+    tags:
+      - ESIRuntime-*
+  schedule:
+    - cron: 0 12 * * 1
   workflow_dispatch:
 
 jobs:
@@ -14,19 +19,21 @@ jobs:
         config:
           - os: ubuntu-20.04
             cibw_build: cp38-manylinux_x86_64
+          - os: ubuntu-20.04
+            cibw_build: cp39-manylinux_x86_64
           - os: ubuntu-20.04
             cibw_build: cp310-manylinux_x86_64
+          - os: ubuntu-20.04
+            cibw_build: cp311-manylinux_x86_64
+          - os: ubuntu-20.04
+            cibw_build: cp312-manylinux_x86_64
 
     steps:
       - name: Get CIRCT
         uses: actions/checkout@v3
         with:
-          fetch-depth: 2
-          submodules: "true"
-
-      - name: Unshallow CIRCT
-        run: |
-          git fetch --force --unshallow --tags --no-recurse-submodules
+          fetch-depth: 0
+          submodules: false
 
       - name: Setup Python
         uses: actions/setup-python@v3
@@ -38,8 +45,7 @@ jobs:
         run: python -m cibuildwheel --output-dir wheelhouse ./lib/Dialect/ESI/runtime
         env:
           CIBW_BUILD: ${{ matrix.config.cibw_build }}
-          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
-          CIBW_BUILD_FRONTEND: build
+          CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/circt/images/pycde-build
           SETUPTOOLS_SCM_DEBUG: True
 
       - name: Upload (stage) wheels as artifacts
@@ -49,3 +55,31 @@ jobs:
           path: ./wheelhouse/*.whl
           retention-days: 7
           if-no-files-found: error
+
+  push_wheels:
+    name: Push wheels (Tag or Weekly)
+    runs-on: ubuntu-20.04
+    if: github.repository == 'llvm/circt' && (github.ref_type == 'tag' || github.event_name == 'schedule')
+    needs: build_wheels
+    environment:
+      name: pypi
+      url: https://pypi.org/p/esiaccel
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download wheels
+        uses: actions/download-artifact@v3
+        with:
+          name: python-wheels
+          path: ./wheelhouse/
+
+      - name: List downloaded wheels
+        run: ls -laR
+        working-directory: ./wheelhouse/
+
+      - name: Upload wheels to pypi
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: wheelhouse/
+          verify-metadata: false

lib/Dialect/ESI/runtime/CMakeLists.txt

@@ -192,8 +192,8 @@ if (WHEEL_BUILD)
 endif()
 
 # Pybind11 is used to wrap the ESIRuntime APIs.
-find_package(Python COMPONENTS Interpreter Development)
-if(Python_FOUND)
+find_package(Python3 COMPONENTS Interpreter Development)
+if(Python3_FOUND)
   IF(MSVC)
     # Work around an issue with pybind11 and cmake incompatibility on Windows in debug mode.
     set_target_properties(Python::Module PROPERTIES
@@ -205,7 +205,7 @@ if(Python_FOUND)
   else()
     message(STATUS "Checking for pybind11 in python path...")
     execute_process(
-      COMMAND "${Python_EXECUTABLE}"
+      COMMAND "${Python3_EXECUTABLE}"
       -c "import pybind11;print(pybind11.get_cmake_dir(), end='')"
       WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
       RESULT_VARIABLE STATUS
@@ -222,6 +222,9 @@ if(Python_FOUND)
   find_package(pybind11 CONFIG)
   if (NOT pybind11_FOUND)
     message (STATUS "Could not find pybind11. Disabling Python API.")
+    if (WHEEL_BUILD)
+      message (FATAL_ERROR "pybind11 is required for a wheel build.")
+    endif()
   else()
     # Compile Pybind11 module and copy to the correct python directory.
     pybind11_add_module(esiCppAccel python/esiaccel/esiCppAccel.cpp)
@@ -290,4 +293,8 @@ if(Python_FOUND)
         esiCppAccel
     )
   endif()
+else() # Python not found.
+  if (WHEEL_BUILD)
+    message (FATAL_ERROR "python-dev is required for a wheel build.")
+  endif()
 endif()

lib/Dialect/ESI/runtime/pyproject.toml

@@ -3,7 +3,7 @@ requires = [
   "setuptools>=68",
   "setuptools_scm>=8.0",
   "wheel",
-  "cmake>=3.12",
+  "cmake>=3.20",
   "pybind11>=2.9",
 ]
 build-backend = "setuptools.build_meta"
@@ -17,7 +17,6 @@ git_describe_command = "git describe --dirty --tags --long --match ESIRuntime*"
 
 [tool.cibuildwheel]
 build-frontend = "build"
-manylinux-x86_64-image = "ghcr.io/circt/images/pycde-build"
 
 [project]
 name = "esiaccel"

lib/Dialect/ESI/runtime/setup.py

@@ -1,3 +1,7 @@
+#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#  See https://llvm.org/LICENSE.txt for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
 # Build/install the ESI runtime python package.
 #
 # To install:
@@ -13,9 +17,11 @@
 # This can be set with the PYCDE_CMAKE_BUILD_DIR env var.
 
 import os
+import platform
 import shutil
 import subprocess
 import sys
+import sysconfig
 
 from distutils.command.build import build as _build
 from setuptools import find_namespace_packages, setup, Extension
@@ -62,7 +68,7 @@ def run(self):
     cmake_args = [
         "-DCMAKE_BUILD_TYPE={}".format(cfg),  # not used on MSVC, but no harm
         "-DCAPNP_PATH={}".format(os.getenv("CAPNP_PATH")),
-        "-DPython_EXECUTABLE={}".format(sys.executable.replace("\\", "/")),
+        "-DPython3_EXECUTABLE={}".format(sys.executable.replace("\\", "/")),
         "-DWHEEL_BUILD=ON",
     ]
     cxx = os.getenv("CXX")
@@ -76,6 +82,25 @@ def run(self):
     if "CIRCT_EXTRA_CMAKE_ARGS" in os.environ:
       cmake_args += os.environ["CIRCT_EXTRA_CMAKE_ARGS"].split(" ")
 
+    # HACK: CMake fails to auto-detect static linked Python installations, which
+    # happens to be what exists on manylinux. We detect this and give it a dummy
+    # library file to reference (which is checks exists but never gets
+    # used).
+    if platform.system() == "Linux":
+      python_libdir = sysconfig.get_config_var('LIBDIR')
+      python_library = sysconfig.get_config_var('LIBRARY')
+      if python_libdir and not os.path.isabs(python_library):
+        python_library = os.path.join(python_libdir, python_library)
+      if python_library and not os.path.exists(python_library):
+        print("Detected static linked python. Faking a library for cmake.")
+        fake_libdir = os.path.join(cmake_build_dir, "fake_python", "lib")
+        os.makedirs(fake_libdir, exist_ok=True)
+        fake_library = os.path.join(fake_libdir,
+                                    sysconfig.get_config_var('LIBRARY'))
+        subprocess.check_call(["ar", "q", fake_library])
+        cmake_args.append("-DPython3_LIBRARY:PATH={}".format(fake_library))
+
+    # Finally run the cmake configure.
     subprocess.check_call(["cmake", src_dir] + cmake_args, cwd=cmake_build_dir)
 
     # Run the build.