Skip to content

Depthai Python CI/CD #2622

Depthai Python CI/CD

Depthai Python CI/CD #2622

Workflow file for this run

name: Depthai Python CI/CD
# Controls when the action will run. Triggers the workflow on push
###################################
# WARNING #########################
# as self-hosted runners are used, and security policy for them has not been yet determined by GitHub
# pay close attention to not enable workflows on pull_request events
# TLDR: do NOT add 'pull_request' here for the time being
on:
workflow_dispatch:
push:
branches:
- main
- develop
- gen2
- gen2_develop
tags:
- 'v*'
###################################
###################################
env:
CMAKE_WINDOWS_SDK_VERSION: '10.0.18362.0'
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# Job which builds docstrings for the rest of the wheel builds
build-docstrings:
runs-on: ubuntu-latest
steps:
- name: Cache .hunter folder
uses: actions/cache@v3
with:
path: ~/.hunter
key: hunter-ubuntu-latest
- name: List .hunter cache directory
run: ls -a -l ~/.hunter/_Base/ || true
- uses: actions/checkout@v3
with:
submodules: 'recursive'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Install dependencies
run: |
python -m pip install --upgrade pip
sudo apt install libusb-1.0-0-dev
python -m pip install clang==14.0 --force-reinstall
python -m pip install -r docs/requirements_mkdoc.txt
- name: Configure project
run: cmake -S . -B build -DDEPTHAI_PYTHON_FORCE_DOCSTRINGS=ON -DDEPTHAI_PYTHON_DOCSTRINGS_OUTPUT="$PWD/docstrings/depthai_python_docstring.hpp"
- name: Build target 'pybind11_mkdoc'
run: cmake --build build --target pybind11_mkdoc --parallel 4
- name: Upload docstring artifacts
uses: actions/upload-artifact@v3
with:
name: docstrings
path: docstrings/
retention-days: 1
# Build and test bindings
pytest:
needs: build-docstrings
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
runs-on: ${{ matrix.os }}
steps:
- name: Print home directory
run: echo Home directory inside container $HOME
- name: Cache .hunter folder
if: matrix.os != 'windows-latest'
uses: actions/cache@v3
with:
path: ~/.hunter/
key: hunter-pytest-${{ matrix.os }}
- name: Cache .hunter folder
if: matrix.os == 'windows-latest'
uses: actions/cache@v3
with:
path: C:/.hunter/
key: hunter-pytest-${{ matrix.os }}
- uses: actions/checkout@v3
with:
submodules: 'recursive'
- uses: actions/download-artifact@v3
with:
name: 'docstrings'
path: docstrings
- name: Specify docstring to use while building the wheel
run: echo "DEPTHAI_PYTHON_DOCSTRINGS_INPUT=$PWD/docstrings/depthai_python_docstring.hpp" >> $GITHUB_ENV
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install dependencies (Ubuntu)
if: matrix.os == 'ubuntu-latest'
run: |
python -m pip install --upgrade pip
sudo apt install libusb-1.0-0-dev
- name: Install dependencies (MacOS)
if: matrix.os == 'macos-latest'
run: |
python -m pip install --upgrade pip
brew install libusb
- name: Install pytest
run: |
python -m pip install pytest numpy
- name: Compile
run: |
cmake -S . -B build -D CMAKE_BUILD_TYPE=Release -D DEPTHAI_PYTHON_DOCSTRINGS_INPUT=$PWD/docstrings/depthai_python_docstring.hpp -D DEPTHAI_PYTHON_ENABLE_TESTS=ON
cmake --build build --parallel 4
- name: Test
run: |
cmake --build build --target pytest --config Release
# This job builds wheels for armhf arch (RPi)
build-linux-armhf:
needs: build-docstrings
strategy:
matrix:
rpi-os: [rpi-buster, rpi-bullseye]
runs-on: ${{ matrix.rpi-os }}
steps:
- name: Print home directory
run: echo Home directory inside container $HOME
- uses: actions/checkout@v3
with:
submodules: 'recursive'
- uses: actions/download-artifact@v3
with:
name: 'docstrings'
path: docstrings
- name: Specify docstring to use while building the wheel
run: echo "DEPTHAI_PYTHON_DOCSTRINGS_INPUT=$PWD/docstrings/depthai_python_docstring.hpp" >> $GITHUB_ENV
- name: Append build hash if not a tagged commit
if: startsWith(github.ref, 'refs/tags/v') != true
run: echo "BUILD_COMMIT_HASH=${{github.sha}}" >> $GITHUB_ENV
- name: Building wheel
run: python3 -m pip wheel . -w ./wheelhouse/ --verbose
- name: Auditing wheels and adding armv6l tag (Running on RPi, binaries compiled as armv6l)
run: |
python3 -m pip install -U wheel auditwheel
for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat linux_armv7l -w wheelhouse/preaudited/; done
for whl in wheelhouse/preaudited/*.whl; do python3 -m wheel tags --platform-tag +linux_armv6l "$whl"; done
mkdir -p wheelhouse/audited/
for whl in wheelhouse/preaudited/*linux_armv6l*.whl; do cp "$whl" wheelhouse/audited/$(basename $whl); done
- name: Archive wheel artifacts
uses: actions/upload-artifact@v3
with:
name: audited-wheels
path: wheelhouse/audited/
- name: Deploy wheels to artifactory (if not a release)
if: startsWith(github.ref, 'refs/tags/v') != true
run: bash ./ci/upload-artifactory.sh
env:
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
# This job builds wheels for Windows x86_64 arch
build-windows-x86_64:
needs: build-docstrings
runs-on: windows-latest
strategy:
matrix:
python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
python-architecture: [x64, x86]
fail-fast: false
steps:
- name: Cache .hunter folder
uses: actions/cache@v3
with:
path: C:/.hunter
key: hunter-msvc
- uses: actions/checkout@v3
with:
submodules: 'recursive'
- uses: actions/download-artifact@v3
with:
name: 'docstrings'
path: docstrings
- name: Specify docstring to use while building the wheel
run: echo "DEPTHAI_PYTHON_DOCSTRINGS_INPUT=$PWD/docstrings/depthai_python_docstring.hpp" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Select Windows SDK
run: echo "CMAKE_ARGS=-DCMAKE_SYSTEM_VERSION=${{ env.CMAKE_WINDOWS_SDK_VERSION }}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
architecture: ${{ matrix.python-architecture }}
- name: Append build hash if not a tagged commit
if: startsWith(github.ref, 'refs/tags/v') != true
run: echo "BUILD_COMMIT_HASH=${{github.sha}}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Install dependencies
run: |
python -m pip install --upgrade pip
- name: Building wheels
run: python -m pip wheel . -w ./wheelhouse/audited/ --verbose
- name: Archive wheel artifacts
uses: actions/upload-artifact@v3
with:
name: audited-wheels
path: wheelhouse/audited/
- name: Deploy wheels to artifactory (if not a release)
if: startsWith(github.ref, 'refs/tags/v') != true
run: bash ./ci/upload-artifactory.sh
env:
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
# This job builds wheels for macOS x86_64 arch
build-macos-x86_64:
needs: build-docstrings
runs-on: macos-latest
strategy:
matrix:
python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
fail-fast: false
steps:
- name: Cache .hunter folder
uses: actions/cache@v3
with:
path: ~/.hunter
key: hunter-macos-latest
- name: List .hunter cache directory
run: |
ls -a -l ~/.hunter/_Base/ || true
echo "PATH=$PATH"
- uses: actions/checkout@v3
with:
submodules: 'recursive'
- uses: actions/download-artifact@v3
with:
name: 'docstrings'
path: docstrings
- name: Specify docstring to use while building the wheel
run: echo "DEPTHAI_PYTHON_DOCSTRINGS_INPUT=$PWD/docstrings/depthai_python_docstring.hpp" >> $GITHUB_ENV
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Append build hash if not a tagged commit
if: startsWith(github.ref, 'refs/tags/v') != true
run: echo "BUILD_COMMIT_HASH=${{github.sha}}" >> $GITHUB_ENV
- name: Install dependencies
run: |
python -m pip install --upgrade pip
brew install libusb
python -m pip install delocate
- name: Set macos deployment target
run: echo "MACOSX_DEPLOYMENT_TARGET=10.9" >> $GITHUB_ENV
- name: Building wheels
run: python -m pip wheel . -w ./wheelhouse/ --verbose
- name: Auditing wheels
run: ci/repair-whl-macos.sh `pwd`/wheelhouse/* `pwd`/wheelhouse/audited
- name: Archive wheel artifacts
uses: actions/upload-artifact@v3
with:
name: audited-wheels
path: wheelhouse/audited/
- name: Deploy wheels to artifactory (if not a release)
if: startsWith(github.ref, 'refs/tags/v') != true
run: bash ./ci/upload-artifactory.sh
env:
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
# This job builds wheels for macOS arm64 arch
build-macos-arm64:
needs: build-docstrings
runs-on: [self-hosted, macOS, ARM64]
steps:
# Cached locally on runner
# - name: Cache .hunter folder
# uses: actions/cache@v3
# with:
# path: ~/.hunter
# key: hunter-macos-latest
- name: List .hunter cache directory
run: |
ls -a -l ~/.hunter/_Base/ || true
echo "PATH=$PATH"
- uses: actions/checkout@v3
with:
submodules: 'recursive'
- uses: actions/download-artifact@v3
with:
name: 'docstrings'
path: docstrings
- name: Specify docstring to use while building the wheel
run: echo "DEPTHAI_PYTHON_DOCSTRINGS_INPUT=$PWD/docstrings/depthai_python_docstring.hpp" >> $GITHUB_ENV
- name: Append build hash if not a tagged commit
if: startsWith(github.ref, 'refs/tags/v') != true
run: echo "BUILD_COMMIT_HASH=${{github.sha}}" >> $GITHUB_ENV
# - name: Build and install depthai-core
# run: |
# echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> $GITHUB_ENV
# cmake -S depthai-core/ -B build_core -D CMAKE_BUILD_TYPE=Release -D CMAKE_TOOLCHAIN_FILE=$PWD/cmake/toolchain/pic.cmake
# cmake --build build_core --target install --parallel 4
# echo "DEPTHAI_INSTALLATION_DIR=$PWD/build_core/install/" >> $GITHUB_ENV
- name: Build wheels
run: for PYBIN in {9..11}; do "python3.${PYBIN}" -m pip wheel . -w wheelhouse/ --verbose; done
- name: Auditing wheels
run: delocate-wheel -v -w wheelhouse/audited wheelhouse/*.whl
- name: Archive wheel artifacts
uses: actions/upload-artifact@v3
with:
name: audited-wheels
path: wheelhouse/audited/
- name: Deploy wheels to artifactory (if not a release)
if: startsWith(github.ref, 'refs/tags/v') != true
run: bash ./ci/upload-artifactory.sh
env:
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
# This job builds wheels for x86_64 arch
build-linux-x86_64:
needs: build-docstrings
runs-on: ubuntu-latest
container:
image: quay.io/pypa/manylinux2014_x86_64:2022-10-30-402504a
env:
PLAT: manylinux2014_x86_64
steps:
- name: Cache .hunter folder
uses: actions/cache@v3
with:
path: ~/.hunter
key: hunter-x86_64
- uses: actions/checkout@v3
with:
submodules: 'recursive'
- name: Installing libusb1-devel dependency
run: yum install -y --disableplugin=fastestmirror libusb1-devel
- name: Installing cmake dependency
run: |
/opt/python/cp38-cp38/bin/python3.8 -m pip install cmake
ln -s /opt/python/cp38-cp38/bin/cmake /bin/
- name: Create folder structure
run: mkdir -p wheelhouse/audited/
- uses: actions/download-artifact@v3
with:
name: 'docstrings'
path: docstrings
- name: Specify docstring to use while building the wheel
run: echo "DEPTHAI_PYTHON_DOCSTRINGS_INPUT=$PWD/docstrings/depthai_python_docstring.hpp" >> $GITHUB_ENV
- name: Build and install depthai-core
run: |
cmake -S depthai-core/ -B build_core -D CMAKE_BUILD_TYPE=Release -D CMAKE_TOOLCHAIN_FILE=$PWD/cmake/toolchain/pic.cmake
cmake --build build_core --target install --parallel 4
echo "DEPTHAI_INSTALLATION_DIR=$PWD/build_core/install/" >> $GITHUB_ENV
- name: Append build hash if not a tagged commit
if: startsWith(github.ref, 'refs/tags/v') != true
run: echo "BUILD_COMMIT_HASH=${{github.sha}}" >> $GITHUB_ENV
- name: Building source distribution
run: |
/opt/python/cp38-cp38/bin/python3.8 setup.py sdist --formats=gztar
mv dist/* wheelhouse/audited/
- name: Build wheels
run: for PYBIN in /opt/python/cp3{6..11}*/bin; do "${PYBIN}/pip" wheel . -w ./wheelhouse/ --verbose; done
- name: Audit wheels
run: for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat $PLAT -w wheelhouse/audited/; done
- name: Archive wheel artifacts
uses: actions/upload-artifact@v3
with:
name: audited-wheels
path: wheelhouse/audited/
- name: Deploy wheels to artifactory (if not a release)
if: startsWith(github.ref, 'refs/tags/v') != true
run: bash ./ci/upload-artifactory.sh
env:
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
# This job builds wheels for ARM64 arch
build-linux-arm64:
needs: build-docstrings
runs-on: [self-hosted, linux, ARM64]
container:
image: quay.io/pypa/manylinux2014_aarch64:2022-10-30-402504a
env:
PLAT: manylinux2014_aarch64
# Mount local hunter cache directory, instead of transfering to Github and back
volumes:
- /.hunter:/github/home/.hunter
steps:
- uses: actions/checkout@v3
with:
submodules: 'recursive'
- name: Installing libusb1-devel dependency
run: yum install -y --disableplugin=fastestmirror libusb1-devel
- name: Installing cmake dependency
run: |
/opt/python/cp38-cp38/bin/python3.8 -m pip install cmake
ln -s /opt/python/cp38-cp38/bin/cmake /bin/
- name: Create folder structure
run: mkdir -p wheelhouse/audited/
- uses: actions/download-artifact@v3
with:
name: 'docstrings'
path: docstrings
- name: Specify docstring to use while building the wheel
run: echo "DEPTHAI_PYTHON_DOCSTRINGS_INPUT=$PWD/docstrings/depthai_python_docstring.hpp" >> $GITHUB_ENV
- name: Build and install depthai-core
run: |
cmake -S depthai-core/ -B build_core -D CMAKE_BUILD_TYPE=Release -D CMAKE_TOOLCHAIN_FILE=$PWD/cmake/toolchain/pic.cmake
cmake --build build_core --target install --parallel 4
echo "DEPTHAI_INSTALLATION_DIR=$PWD/build_core/install/" >> $GITHUB_ENV
- name: Append build hash if not a tagged commit
if: startsWith(github.ref, 'refs/tags/v') != true
run: echo "BUILD_COMMIT_HASH=${{github.sha}}" >> $GITHUB_ENV
- name: Building wheels
run: for PYBIN in /opt/python/cp3{6..11}*/bin; do "${PYBIN}/pip" wheel . -w ./wheelhouse/ --verbose; done
- name: Auditing wheels
run: for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat $PLAT -w wheelhouse/audited/; done
- name: Archive wheel artifacts
uses: actions/upload-artifact@v3
with:
name: audited-wheels
path: wheelhouse/audited/
- name: Deploy wheels to artifactory (if not a release)
if: startsWith(github.ref, 'refs/tags/v') != true
run: bash ./ci/upload-artifactory.sh
env:
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
release:
if: startsWith(github.ref, 'refs/tags/v')
needs: [pytest, build-linux-armhf, build-windows-x86_64, build-macos-x86_64, build-macos-arm64, build-linux-x86_64, build-linux-arm64]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: 'recursive'
- uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Check if version matches
run: python3.8 -c 'import find_version as v; exit(0) if "${{ github.ref_name }}" == f"v{v.get_package_version()}" else exit(1)'
# Create GitHub release
- uses: actions/create-release@master
id: createRelease
name: Create ${{ github.ref_name }} depthai-python release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ github.ref }}
release_name: Release ${{ github.ref }}
body: |
## Features
## Bug fixes
## Misc
draft: true
# Deploy to PyPi and Artifactory. Only when a commit is tagged
deploy:
if: startsWith(github.ref, 'refs/tags/v')
needs: [release]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: 'recursive'
- uses: actions/download-artifact@v3
with:
name: audited-wheels
path: wheelhouse/audited/
- name: List files
run: ls -lah
- name: Run deploy to PyPi
run: bash ./ci/upload-pypi.sh
env:
PYPI_SERVER: ${{ secrets.PYPI_SERVER }}
PYPI_USER: ${{ secrets.PYPI_USER }}
PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
- name: Run deploy to Artifactory
run: bash ./ci/upload-artifactory-release.sh
env:
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
notify_robothub:
if: startsWith(github.ref, 'refs/tags/v')
needs: [release]
runs-on: ubuntu-latest
steps:
- name: Repository Dispatch
uses: peter-evans/repository-dispatch@v2
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
repository: luxonis/robothub-apps
event-type: depthai-python-release
client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}'
# notify_hil_workflow_linux_x86_64:
# needs: [build-linux-x86_64]
# runs-on: ubuntu-latest
# steps:
# - name: Repository Dispatch
# uses: peter-evans/repository-dispatch@v2
# with:
# token: ${{ secrets.HIL_CORE_DISPATCH_TOKEN }}
# repository: luxonis/depthai-core-hil-tests
# event-type: python-hil-event
# client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}'
notify_hil_workflow_linux_x86_64:
needs: [build-linux-x86_64]
runs-on: ubuntu-latest
steps:
- name: Dispatch an action and get the run ID
uses: codex-/return-dispatch@v1
id: return_dispatch
with:
token: ${{ secrets.HIL_CORE_DISPATCH_TOKEN }} # Note this is NOT GITHUB_TOKEN but a PAT
ref: main # or refs/heads/target_branch
repo: depthai-core-hil-tests
owner: luxonis
workflow: regression_test.yml
workflow_inputs: '{"commit": "${{ github.ref }}", "parent_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}'
workflow_timeout_seconds: 300 # was 120 Default: 300
- name: Release
run: echo "https://github.com/luxonis/depthai-core-hil-tests/actions/runs/${{steps.return_dispatch.outputs.run_id}}" >> $GITHUB_STEP_SUMMARY