Python wheels (G-Research#214)

Co-authored-by: Jonathan Giannuzzi <jonathan@giannuzzi.me>

.github/workflows/ci.yml

@@ -138,9 +138,9 @@ jobs:
       - name: Run ${{ matrix.api }} integration tests
         run: ./tests/integration/python/${{ matrix.api }}/test.sh
 
-  golang-build:
+  build:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
-    name: Build Go binary for ${{ matrix.os }}/${{ matrix.arch }}
+    name: Build software distribution for ${{ matrix.os }}/${{ matrix.arch }}
     strategy:
       matrix:
         os: [darwin, linux, windows]
@@ -164,6 +164,16 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.20"
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
       - name: Install arm64 cross-compilation toolchain on Linux
         if: matrix.os == 'linux' && matrix.arch == 'arm64'
         run: |
@@ -178,18 +188,24 @@ jobs:
           sudo apt-get install -y --no-install-recommends gcc-mingw-w64-x86-64-win32
           echo CC=x86_64-w64-mingw32-gcc >> $GITHUB_ENV
 
-      - name: Build and archive Go binary
-        run: make go-dist
+      - name: Build software distribution
+        run: make dist
         env:
           GOOS: ${{ matrix.os }}
           GOARCH: ${{ matrix.arch }}
 
-      - name: Upload artifact
+      - name: Upload binary artifact
         uses: actions/upload-artifact@v3
         with:
           name: fasttrackml-archives
           path: dist/*
 
+      - name: Upload wheels artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: fasttrackml-wheels
+          path: wheelhouse/*.whl
+
   build-image:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
     name: Build container image
@@ -254,7 +270,7 @@ jobs:
       - golang-unit-tests
       - golang-integration-tests
       - python-integration-tests
-      - golang-build
+      - build
       - build-image
     runs-on: ubuntu-latest
     steps:

.github/workflows/release.yml

@@ -49,6 +49,23 @@ jobs:
             exit 1
           fi
 
+  pypi-publish:
+    name: upload release to PyPI
+    needs: validate
+    runs-on: ubuntu-latest
+    environment: release
+    permissions:
+      id-token: write
+    steps:
+      - name: Download artifact
+        run: gh run download ${{ github.event.workflow_run.id }} --repo ${{ github.event.workflow_run.repository.full_name }} --name fasttrackml-wheels --dir wheelhouse
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: wheelhouse/
+
   github-release:
     name: Publish GitHub release
     needs: validate

.gitignore

@@ -25,4 +25,9 @@ mock_*
 
 # Optional workspace
 go.work
-*.code-workspace
+*.code-workspace
+
+#Python wheels
+python/build/
+python/*.egg-info/
+wheelhouse/

Makefile

@@ -81,6 +81,14 @@ go-dist: go-build ## archive app binary.
 	@if [ -f $(ARCHIVE_NAME) ]; then rm -f $(ARCHIVE_NAME); fi
 	@$(ARCHIVE_CMD) $(ARCHIVE_NAME) $(ARCHIVE_FILES)
 
+#
+# Python targets.
+#
+.PHONY: python-dist
+python-dist: go-build ## build python wheels.
+	@echo '>>> Building Python Wheels.'
+	@VERSION=$(VERSION) python3 -m pip wheel ./python --wheel-dir=wheelhouse --no-deps
+
 #
 # Tests targets.
 #
@@ -171,7 +179,7 @@ PHONY: build
 build: go-build ## build the go components
 
 PHONY: dist
-dist: go-dist ## archive the go components
+dist: go-dist python-dist ## build the software archives
 
 PHONY: run
 run: build ## run the FastTrackML server

python/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"

python/setup.py

@@ -0,0 +1,65 @@
+import os
+import subprocess
+
+from setuptools import find_packages, setup
+from wheel.bdist_wheel import bdist_wheel
+
+
+def get_long_description():
+    with open("../README.md", "r", encoding="utf-8") as f:
+        return f.read()
+
+
+def get_data_files():
+    os = subprocess.check_output(["go", "env", "GOOS"]).strip().decode("utf-8")
+    return [("Scripts", ["../fml.exe"])] if os == "windows" else [("bin", ["../fml"])]
+
+
+def get_version():
+    version = os.environ.get("VERSION")
+    return version.replace("-", "+", 1)
+
+
+def get_platform():
+    os = subprocess.check_output(["go", "env", "GOOS"]).strip().decode("utf-8")
+    arch = subprocess.check_output(["go", "env", "GOARCH"]).strip().decode("utf-8")
+    plat = f"{os}_{arch}"
+    if plat == "darwin_amd64":
+        return "macosx_10_13_x86_64"
+    elif plat == "darwin_arm64":
+        return "macosx_11_0_arm64"
+    elif plat == "linux_amd64":
+        return "manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64"
+    elif plat == "linux_arm64":
+        return "manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64"
+    elif plat == "windows_amd64":
+        return "win_amd64"
+    else:
+        raise ValueError("not supported platform.")
+
+
+class custom_bdist_wheel(bdist_wheel):
+    def finalize_options(self):
+        bdist_wheel.finalize_options(self)
+        # Mark us as not a pure python package
+        self.root_is_pure = False
+
+    def get_tag(self):
+        return "py3", "none", get_platform()
+
+
+setup(
+    name="fasttrackml",
+    version=get_version(),
+    description="Rewrite of the MLFlow tracking server with a focus on scalability.",
+    long_description=get_long_description(),
+    packages=find_packages(),
+    include_package_data=True,
+    data_files=get_data_files(),
+    python_requires=">=3.6",
+    zip_safe=False,
+    ext_modules=[],
+    cmdclass=dict(
+        bdist_wheel=custom_bdist_wheel,
+    ),
+)