diff --git a/.cirrus.star b/.cirrus.star new file mode 100644 index 0000000..7bb046a --- /dev/null +++ b/.cirrus.star @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright 2021, Siavash Ameli +# SPDX-License-Identifier: BSD-3-Clause +# SPDX-FileType: SOURCE +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the license found in the LICENSE.txt file in the root +# directory of this source tree. + + +load("cirrus", "env", "fs", "http") + +def main(ctx): + + file = '' + file += fs.read("tools/ci/cirrus_deploy_docker_manylinux2014_aarch64_cuda_12.3.yml") + # file += fs.read("tools/ci/cirrus_deploy_pypi_linux_aarch64.yml") + # file += fs.read("tools/ci/cirrus_deploy_pypi_macosx_arm64.yml") + # file += fs.read("tools/ci/cirrus_deploy_conda_linux_aarch64.yml") + # file += fs.read("tools/ci/cirrus_deploy_conda_macosx_arm64.yml") + + return file diff --git a/.github/workflows/deploy-docker-10-2.yml b/.github/workflows/deploy-docker-manylinux2014_x86_64_cuda_10.2.yml similarity index 86% rename from .github/workflows/deploy-docker-10-2.yml rename to .github/workflows/deploy-docker-manylinux2014_x86_64_cuda_10.2.yml index 71a5d60..8fdf680 100644 --- a/.github/workflows/deploy-docker-10-2.yml +++ b/.github/workflows/deploy-docker-manylinux2014_x86_64_cuda_10.2.yml @@ -1,4 +1,4 @@ -name: deploy-docker-10-2 +name: deploy-docker-manylinux2014_x86_64_cuda_10.2 on: push: @@ -9,7 +9,7 @@ on: - published jobs: - build-cuda-10-2: + build-manylinux_x86_64_cuda_10_2: runs-on: ubuntu-latest steps: - name: Checkout @@ -21,12 +21,12 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push Cuda-10.2 + - name: Build and push manylinux2014_x86_64_cuda-10.2 id: docker_build uses: docker/build-push-action@v3 with: push: true - file: ./docker/cuda-10.2/Dockerfile + file: ./docker/manylinux2014_x86_64_cuda_10.2/Dockerfile tags: sameli/manylinux2014_x86_64_cuda_10.2:latest - name: Image digest diff --git a/.github/workflows/deploy-docker-11-7.yml b/.github/workflows/deploy-docker-manylinux2014_x86_64_cuda_11.7.yml similarity index 86% rename from .github/workflows/deploy-docker-11-7.yml rename to .github/workflows/deploy-docker-manylinux2014_x86_64_cuda_11.7.yml index b6405e2..9bf8d0d 100644 --- a/.github/workflows/deploy-docker-11-7.yml +++ b/.github/workflows/deploy-docker-manylinux2014_x86_64_cuda_11.7.yml @@ -1,4 +1,4 @@ -name: deploy-docker-11-7 +name: deploy-docker-manylinux2014_x86_64_cuda_11.7 on: push: @@ -9,7 +9,7 @@ on: - published jobs: - build-cuda-11-7: + build-manylinux_x86_64_cuda_11_7: runs-on: ubuntu-latest steps: - name: Checkout @@ -21,12 +21,12 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push Cuda-11.7 + - name: Build and push manylinux2014_x86_64_cuda-11.7 id: docker_build uses: docker/build-push-action@v3 with: push: true - file: ./docker/cuda-11.7/Dockerfile + file: ./docker/manylinux2014_x86_64_cuda_11.7/Dockerfile tags: sameli/manylinux2014_x86_64_cuda_11.7:latest - name: Image digest diff --git a/.github/workflows/deploy-docker-11-8.yml b/.github/workflows/deploy-docker-manylinux2014_x86_64_cuda_11.8.yml similarity index 86% rename from .github/workflows/deploy-docker-11-8.yml rename to .github/workflows/deploy-docker-manylinux2014_x86_64_cuda_11.8.yml index e5248b0..f675a7e 100644 --- a/.github/workflows/deploy-docker-11-8.yml +++ b/.github/workflows/deploy-docker-manylinux2014_x86_64_cuda_11.8.yml @@ -1,4 +1,4 @@ -name: deploy-docker-11-8 +name: deploy-docker-manylinux2014_x86_64_cuda_11.8 on: push: @@ -9,7 +9,7 @@ on: - published jobs: - build-cuda-11-8: + build-manylinux_x86_64_cuda_11_8: runs-on: ubuntu-latest steps: - name: Checkout @@ -21,12 +21,12 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push Cuda-11.8 + - name: Build and push manylinux2014_x86_64_cuda-11.8 id: docker_build uses: docker/build-push-action@v3 with: push: true - file: ./docker/cuda-11.8/Dockerfile + file: ./docker/manylinux2014_x86_64_cuda_11.8/Dockerfile tags: sameli/manylinux2014_x86_64_cuda_11.8:latest - name: Image digest diff --git a/.github/workflows/deploy-docker-12-0.yml b/.github/workflows/deploy-docker-manylinux2014_x86_64_cuda_12.0.yml similarity index 86% rename from .github/workflows/deploy-docker-12-0.yml rename to .github/workflows/deploy-docker-manylinux2014_x86_64_cuda_12.0.yml index 4f28a02..93a0c04 100644 --- a/.github/workflows/deploy-docker-12-0.yml +++ b/.github/workflows/deploy-docker-manylinux2014_x86_64_cuda_12.0.yml @@ -1,4 +1,4 @@ -name: deploy-docker-12-0 +name: deploy-docker-manylinux2014_x86_64_cuda_12.0 on: push: @@ -9,7 +9,7 @@ on: - published jobs: - build-cuda-12-0: + build-manylinux_x86_64_cuda_12_0: runs-on: ubuntu-latest steps: - name: Checkout @@ -21,12 +21,12 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push Cuda-12.0 + - name: Build and push manylinux2014_x86_64_cuda-12.0 id: docker_build uses: docker/build-push-action@v3 with: push: true - file: ./docker/cuda-12.0/Dockerfile + file: ./docker/manylinux2014_x86_64_cuda_12.0/Dockerfile tags: sameli/manylinux2014_x86_64_cuda_12.0:latest - name: Image digest diff --git a/.github/workflows/deploy-docker-12-2.yml b/.github/workflows/deploy-docker-manylinux2014_x86_64_cuda_12.2.yml similarity index 86% rename from .github/workflows/deploy-docker-12-2.yml rename to .github/workflows/deploy-docker-manylinux2014_x86_64_cuda_12.2.yml index 72d0a50..4c81c46 100644 --- a/.github/workflows/deploy-docker-12-2.yml +++ b/.github/workflows/deploy-docker-manylinux2014_x86_64_cuda_12.2.yml @@ -1,4 +1,4 @@ -name: deploy-docker-12-2 +name: deploy-docker-manylinux2014_x86_64_cuda_12.2 on: push: @@ -9,7 +9,7 @@ on: - published jobs: - build-cuda-12-2: + build-manylinux_x86_64_cuda_12_2: runs-on: ubuntu-latest steps: - name: Checkout @@ -21,12 +21,12 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push Cuda-12.2 + - name: Build and push manylinux2014_x86_64_cuda-12.2 id: docker_build uses: docker/build-push-action@v3 with: push: true - file: ./docker/cuda-12.2/Dockerfile + file: ./docker/manylinux2014_x86_64_cuda_12.2/Dockerfile tags: sameli/manylinux2014_x86_64_cuda_12.2:latest - name: Image digest diff --git a/README.rst b/README.rst index 607ed2e..a1220a1 100644 --- a/README.rst +++ b/README.rst @@ -190,10 +190,26 @@ Push to Dockerhub: docker login docker-compose push +Troubleshooting +=============== + +When running the docker container in Github action, you may encounter this error: + +:: + + no space left on device. + +To resolve this, try clearing the Github's runner cache before executing the docker container: + +:: + + - name: Clear Cache + run: rm -rf /opt/hostedtoolcache + Other CUDA Versions =================== -To request a docker image for a specific CUDA version, feel free to create a `GitHub issue `_. +To request a docker image for a specific CUDA version, feel free to create a `GitHub issue `__. License ======= diff --git a/docker-compose.yml b/docker-compose.yml index 5a994f3..ac8e192 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,21 +1,21 @@ version: '3' services: - cuda_10.2: - build: ./docker/cuda-10.2 + manylinux2014_x86_64_cuda_10.2: + build: ./docker/manylinux2014_x86_64_cuda_10.2 image: sameli/manylinux2014_x86_64_cuda_10.2 - cuda_11.7: - build: ./docker/cuda-11.7 + manylinux2014_x86_64_cuda_11.7: + build: ./docker/manylinux2014_x86_64_cuda_11.7 image: sameli/manylinux2014_x86_64_cuda_11.7 - cuda_11.8: - build: ./docker/cuda-11.8 + manylinux2014_x86_64_cuda_11.8: + build: ./docker/manylinux2014_x86_64_cuda_11.8 image: sameli/manylinux2014_x86_64_cuda_11.8 - cuda_12.0: - build: ./docker/cuda-12.0 + manylinux2014_x86_64_cuda_12.0: + build: ./docker/manylinux2014_x86_64_cuda_12.0 image: sameli/manylinux2014_x86_64_cuda_12.0 - cuda_12.2: - build: ./docker/cuda-12.2 + manylinux2014_x86_64_cuda_12.2: + build: ./docker/manylinux2014_x86_64_cuda_12.2 image: sameli/manylinux2014_x86_64_cuda_12.2 diff --git a/docker/manylinux2014_aarch64_cuda_12.3/Dockerfile b/docker/manylinux2014_aarch64_cuda_12.3/Dockerfile new file mode 100644 index 0000000..c63335c --- /dev/null +++ b/docker/manylinux2014_aarch64_cuda_12.3/Dockerfile @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright 2021, Siavash Ameli +# SPDX-License-Identifier: BSD-3-Clause +# SPDX-FileType: SOURCE + +# ----------------------------------------------------------------------------- +# How to build +# $ docker build -t sameli/manylinux2014_aarch64_cuda_12.3 -f . +# +# How to run: +# $ docker run -it -v/host_dir:/image_dir --entrypoint /bin/bash \ +# manylinux2014_aarch64_cuda_12.3 +# ----------------------------------------------------------------------------- + +# ----------------- +# Choose base image +# ----------------- + +FROM quay.io/pypa/manylinux2014_aarch64 + +MAINTAINER Siavash Ameli +LABEL Description="manylinux2014_aarch64 with cuda 12.3" + +# ------------ +# Install cuda +# ------------ + +RUN dnf install -y dnf-utils +RUN dnf install -y wget + +RUN wget https://developer.download.nvidia.com/compute/cuda/12.3.1/local_installers/cuda-repo-rhel9-12-3-local-12.3.1_545.23.08-1.aarch64.rpm +RUN sudo rpm -i cuda-repo-rhel9-12-3-local-12.3.1_545.23.08-1.aarch64.rpm +RUN sudo dnf clean all +RUN sudo dnf -y install cuda-toolkit-12-3 +RUN rm cuda-repo-rhel9-12-3-local-12.3.1_545.23.08-1.aarch64.rpm + +# ------------------------- +# Set environment variables +# ------------------------- + +ENV PATH="/usr/local/cuda/bin:${PATH}" +ENV CUDA_HOME=/usr/local/cuda +ENV CUDA_ROOT=/usr/local/cuda +ENV CUDA_PATH=/usr/local/cuda +ENV CUDADIR=/usr/local/cuda +ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}" + +# -------- +# Commands +# -------- + +CMD ["/bin/bash"] diff --git a/docker/cuda-10.2/Dockerfile b/docker/manylinux2014_x86_64_cuda_10.2/Dockerfile similarity index 100% rename from docker/cuda-10.2/Dockerfile rename to docker/manylinux2014_x86_64_cuda_10.2/Dockerfile diff --git a/docker/cuda-11.7/Dockerfile b/docker/manylinux2014_x86_64_cuda_11.7/Dockerfile similarity index 100% rename from docker/cuda-11.7/Dockerfile rename to docker/manylinux2014_x86_64_cuda_11.7/Dockerfile diff --git a/docker/cuda-11.8/Dockerfile b/docker/manylinux2014_x86_64_cuda_11.8/Dockerfile similarity index 100% rename from docker/cuda-11.8/Dockerfile rename to docker/manylinux2014_x86_64_cuda_11.8/Dockerfile diff --git a/docker/cuda-12.0/Dockerfile b/docker/manylinux2014_x86_64_cuda_12.0/Dockerfile similarity index 100% rename from docker/cuda-12.0/Dockerfile rename to docker/manylinux2014_x86_64_cuda_12.0/Dockerfile diff --git a/docker/cuda-12.2/Dockerfile b/docker/manylinux2014_x86_64_cuda_12.2/Dockerfile similarity index 100% rename from docker/cuda-12.2/Dockerfile rename to docker/manylinux2014_x86_64_cuda_12.2/Dockerfile diff --git a/tools/ci/cirrus_deploy_conda_linux_aarch64.yml b/tools/ci/cirrus_deploy_conda_linux_aarch64.yml new file mode 100644 index 0000000..3a210b9 --- /dev/null +++ b/tools/ci/cirrus_deploy_conda_linux_aarch64.yml @@ -0,0 +1,129 @@ +# SPDX-FileCopyrightText: Copyright 2021, Siavash Ameli +# SPDX-License-Identifier: BSD-3-Clause +# SPDX-FileType: SOURCE +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the license found in the LICENSE.txt file in the root +# directory of this source tree. + + +# ======================== +# conda linux aarch64 cibw +# ======================== + +conda_linux_aarch64_cibw: &CONDA_LINUX_AARCH64_CIBW + install_cibuildwheel_script: + - python -m pip install cibuildwheel + cibuildwheel_script: + - cibuildwheel + conda_linux_aarch64_wheels_artifacts: + path: "wheelhouse/*.whl" + + +# ================================ +# conda build wheels linux aarch64 +# ================================ + +conda_build_wheels_linux_aarch64_task: + use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder-arm64 + architecture: arm64 + platform: linux + cpu: 1 + memory: 4G + matrix: + - env: + CIBW_BUILD: "cp39-manylinux_aarch64" + # EXPECT_CPU_FEATURES: NEON NEON_FP16 NEON_VFPV4 ASIMD ASIMDHP ASIMDDP ASIMDFHM + - env: + CIBW_BUILD: "cp310-manylinux_aarch64" + - env: + CIBW_BUILD: "cp311-manylinux_aarch64" + - env: + CIBW_BUILD: "cp312-manylinux_aarch64" + env: + CIBW_ARCHS: "aarch64" + # CIBW_BUILD: "*-manylinux_aarch64" + CIBW_SKIP: "pp* cp36-* cp37-* cp38-*" + CIBW_BUILD_VERBOSITY: "3" + MANYLINUX_AARCH64_IMAGE: "manylinux2014" + CIBW_ENVIRONMENT: > + USE_LONG_INT=0 + USE_UNSIGNED_LONG_INT=0 + USE_CBLAS=0 + + build_script: | + apt update + apt install -y python3-venv python-is-python3 + git fetch origin + which python + echo $CIRRUS_CHANGE_MESSAGE + + <<: *CONDA_LINUX_AARCH64_CIBW + + +# ====================================== +# conda build upload conda linux aarch64 +# ====================================== + +conda_build_upload_conda_linux_aarch64_task: + use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder-arm64 + architecture: arm64 + platform: linux + cpu: 1 + memory: 4G + depends_on: + - conda_build_wheels_linux_aarch64 + env: + # see setup.py. Avoids a bug when building with multiple python variants + CYTHON_BUILD_IN_SOURCE: '1' + ANACONDA_API_TOKEN: ENCRYPTED[!f177322643cd551d0cf3749cb4f6549ce7f7be107a0394385e5b5c8b7edd62aa4ac478cc0fa53564890549449dc2dce8!] + + build_script: | + # The name of the zip file is derived from the `wheels_artifact` line. + # If you change the artifact line to `myfile_artifact` then it would be + # called myfile.zip + curl https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/conda_linux_aarch64_wheels.zip --output wheels.zip + mkdir -p dist + unzip -j wheels.zip -d dist + + source ./tools/wheels/upload_wheels.sh + install_anaconda "linux_aarch64" + build_upload_anaconda "linux_aarch64" + + +# ======================== +# conda test linux aarch64 +# ======================== + +conda_test_linux_aarch64_task: + use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder-arm64 + architecture: arm64 + platform: linux + cpu: 1 + memory: 4G + depends_on: + - conda_build_upload_conda_linux_aarch64 + + test_script: | + source ./tools/wheels/upload_wheels.sh + install_anaconda "linux_aarch64" + set -x + CONDA_BIN_DIR=$(dirname `which conda`) + PYTHON=${CONDA_BIN_DIR}/python + uname -m + ${PYTHON} -c "import platform;print(platform.python_version());print(platform.system());print(platform.machine())" + mv detkit detkit_do_dot_import + conda install -c s-ameli detkit -y + ${PYTHON} -m pip install --upgrade pip + ${PYTHON} -m pip install numpy + ${PYTHON} -m pip install -r tests/requirements.txt + ${PYTHON} -m pytest diff --git a/tools/ci/cirrus_deploy_conda_macosx_arm64.yml b/tools/ci/cirrus_deploy_conda_macosx_arm64.yml new file mode 100644 index 0000000..4f2d7d7 --- /dev/null +++ b/tools/ci/cirrus_deploy_conda_macosx_arm64.yml @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: Copyright 2021, Siavash Ameli +# SPDX-License-Identifier: BSD-3-Clause +# SPDX-FileType: SOURCE +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the license found in the LICENSE.txt file in the root +# directory of this source tree. + + +# ======================= +# conda macosx arm64 cibw +# ======================= + +conda_macosx_arm64_cibw: &CONDA_MACOSX_ARM64_CIBW + install_cibuildwheel_script: + - python -m pip install cibuildwheel + cibuildwheel_script: + - cibuildwheel + conda_macosx_arm64_wheels_artifacts: + path: "wheelhouse/*.whl" + + +# =============================== +# conda build wheels macosx arm64 +# =============================== + +conda_build_wheels_macosx_arm64_task: + use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' + macos_instance: + image: ghcr.io/cirruslabs/macos-monterey-xcode:latest + cpu: 1 + memory: 4G + matrix: + - env: + CIBW_BUILD: "cp39-macosx_arm64" + - env: + CIBW_BUILD: "cp310-macosx_arm64" + - env: + CIBW_BUILD: "cp311-macosx_arm64" + - env: + CIBW_BUILD: "cp312-macosx_arm64" + env: + PATH: /opt/homebrew/opt/python@3.10/bin:/usr/local/lib:/usr/local/include:$PATH + # Specifying CIBW_ENVIRONMENT_MACOS overrides pyproject.toml, so include + # all the settings from there, otherwise they're lost. + # SDKROOT needs to be set for repackaged conda-forge gfortran compilers + # supplied by isuruf. + # Find out SDKROOT via `xcrun --sdk macosx --show-sdk-path` + CIBW_ENVIRONMENT_MACOS: > + SDKROOT=/Applications/Xcode-14.0.0.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.3.sdk + LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + CFLAGS="-I/opt/homebrew/opt/libomp/include $CFLAGS" + CXXFLAGS="-I/opt/homebrew/opt/libomp/include $CXXFLAGS" + LDFLAGS="-L/opt/homebrew/opt/libomp/lib $LDFLAGS" + LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$LIBRARY_PATH" + USE_LONG_INT=0 + USE_UNSIGNED_LONG_INT=0 + USE_CBLAS=0 + CIBW_ARCHS: "arm64" + # CIBW_BUILD: "*-macosx_arm64" + CIBW_SKIP: "pp* cp36-* cp37-* cp38-*" + CIBW_BUILD_VERBOSITY: "3" + CIBW_BEFORE_BUILD: "pip install delocate; brew install libomp" + CIBW_TEST_SKIP: "*_universal2:arm64" + + build_script: + - brew install python@3.10 + - ln -s python3 /opt/homebrew/opt/python@3.10/bin/python + - which python + - uname -m + - python -c "import platform;print(platform.python_version());print(platform.system());print(platform.machine())" + - clang --version + <<: *CONDA_MACOSX_ARM64_CIBW + + +# ===================================== +# conda build upload conda macosx arm64 +# ===================================== + +conda_build_upload_conda_macosx_arm64_task: + use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' + macos_instance: + image: ghcr.io/cirruslabs/macos-monterey-xcode:latest + cpu: 1 + memory: 4G + depends_on: + - conda_build_wheels_macosx_arm64 + env: + # see setup.py. Avoids a bug when building with multiple python variants + CYTHON_BUILD_IN_SOURCE: '1' + ANACONDA_API_TOKEN: ENCRYPTED[!f177322643cd551d0cf3749cb4f6549ce7f7be107a0394385e5b5c8b7edd62aa4ac478cc0fa53564890549449dc2dce8!] + + build_script: + # The name of the zip file is derived from the `wheels_artifact` line. + # If you change the artifact line to `myfile_artifact` then it would be + # called myfile.zip + - curl https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/conda_macosx_arm64_wheels.zip --output wheels.zip + - mkdir -p dist + - unzip -j wheels.zip -d dist + - source ./tools/wheels/upload_wheels.sh + - install_anaconda "macosx_arm64" + - build_upload_anaconda "macosx_arm64" + + +# ======================= +# conda test macosx arm64 +# ======================= + +conda_test_macosx_arm64_task: + use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' + macos_instance: + image: ghcr.io/cirruslabs/macos-monterey-xcode:latest + cpu: 1 + memory: 4G + depends_on: + - conda_build_upload_conda_macosx_arm64 + + test_script: + - source ./tools/wheels/upload_wheels.sh + - install_anaconda "macosx_arm64" + - set -x + - CONDA_BIN_DIR=$(dirname `which conda`) + - PYTHON=${CONDA_BIN_DIR}/python + - uname -m + - ${PYTHON} -c "import platform;print(platform.python_version());print(platform.system());print(platform.machine())" + - mv detkit detkit_do_dot_import + - conda install -c s-ameli detkit -y + - ${PYTHON} -m pip install --upgrade pip + - ${PYTHON} -m pip install numpy + - ${PYTHON} -m pip install -r tests/requirements.txt + - ${PYTHON} -m pytest diff --git a/tools/ci/cirrus_deploy_docker_manylinux2014_aarch64_cuda_12.3.yml b/tools/ci/cirrus_deploy_docker_manylinux2014_aarch64_cuda_12.3.yml new file mode 100644 index 0000000..86d5065 --- /dev/null +++ b/tools/ci/cirrus_deploy_docker_manylinux2014_aarch64_cuda_12.3.yml @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright 2021, Siavash Ameli +# SPDX-License-Identifier: BSD-3-Clause +# SPDX-FileType: SOURCE +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the license found in the LICENSE.txt file in the root +# directory of this source tree. + + +# ==================================== +# manylinux2014 aarch64 cuda 12.3 cibw +# ==================================== + +manylinux2014_aarch64_cuda_12_3_cibw: &MANYLINUX2014_AARCH64_CUDA_12_3_CIBW + install_cibuildwheel_script: + - python -m pip install cibuildwheel + cibuildwheel_script: + - cibuildwheel + conda_linux_aarch64_wheels_artifacts: + path: "wheelhouse/*.whl" + + +# ============================================ +# build docker manylinux2014 aarch64 cuda 12-3 +# ============================================ + +build_docker_manylinux2014_aarch64_cuda_12_3task: + use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder-arm64 + architecture: arm64 + platform: linux + cpu: 1 + memory: 8G + # matrix: + # - env: + # CIBW_BUILD: "cp39-manylinux_aarch64" + # # EXPECT_CPU_FEATURES: NEON NEON_FP16 NEON_VFPV4 ASIMD ASIMDHP ASIMDDP ASIMDFHM + # - env: + # CIBW_BUILD: "cp310-manylinux_aarch64" + # - env: + # CIBW_BUILD: "cp311-manylinux_aarch64" + # - env: + # CIBW_BUILD: "cp312-manylinux_aarch64" + env: + DOCKERHUB_TOKEN: ENCRYPTED[!05c76b305df7bb81c59faa33981e2cf4d08c4aa6f1c0313645815c5729f42b6d9db7f2829260ca8cb1e92637151670ad!] + + build_script: | + + # Install docker + source ./tools/scripts/docker_utils.sh + install_docker + + # Build source + docker build -t sameli/manylinux2014_aarch64_cuda_12.3 -f ./docker/manylinux2014_aarch64_cuda_12.3/Dockerfile . + + ls + + # Upload image to dockerhub + DOCKERHUB_USERNAME="sameli" + docker login --username ${DOCKERHUB_USERNAME} --password ${DOCKERHUB_TOKEN} + + +# ======================== +# conda test linux aarch64 +# ======================== + +# conda_test_linux_aarch64_task: +# use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' +# compute_engine_instance: +# image_project: cirrus-images +# image: family/docker-builder-arm64 +# architecture: arm64 +# platform: linux +# cpu: 1 +# memory: 4G +# depends_on: +# - conda_build_upload_conda_linux_aarch64 +# +# test_script: | +# source ./tools/wheels/upload_wheels.sh +# install_anaconda "linux_aarch64" +# set -x +# CONDA_BIN_DIR=$(dirname `which conda`) +# PYTHON=${CONDA_BIN_DIR}/python +# uname -m +# ${PYTHON} -c "import platform;print(platform.python_version());print(platform.system());print(platform.machine())" +# mv detkit detkit_do_dot_import +# conda install -c s-ameli detkit -y +# ${PYTHON} -m pip install --upgrade pip +# ${PYTHON} -m pip install numpy +# ${PYTHON} -m pip install -r tests/requirements.txt +# ${PYTHON} -m pytest diff --git a/tools/ci/cirrus_deploy_pypi_linux_aarch64.yml b/tools/ci/cirrus_deploy_pypi_linux_aarch64.yml new file mode 100644 index 0000000..3e36106 --- /dev/null +++ b/tools/ci/cirrus_deploy_pypi_linux_aarch64.yml @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: Copyright 2021, Siavash Ameli +# SPDX-License-Identifier: BSD-3-Clause +# SPDX-FileType: SOURCE +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the license found in the LICENSE.txt file in the root +# directory of this source tree. + + +# ======================= +# pypi linux aarch64 cibw +# ======================= + +pypi_linux_aarch64_cibw: &PYPI_LINUX_AARCH64_CIBW + install_cibuildwheel_script: + - python -m pip install cibuildwheel + cibuildwheel_script: + - cibuildwheel + pypi_linux_aarch64_wheels_artifacts: + path: "wheelhouse/*.whl" + + +# ============================== +# pypi build wheel linux aarch64 +# ============================== + +pypi_build_wheels_linux_aarch64_task: + use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder-arm64 + architecture: arm64 + platform: linux + cpu: 1 + memory: 4G + matrix: + - env: + CIBW_BUILD: "cp39-manylinux_aarch64" + # EXPECT_CPU_FEATURES: NEON NEON_FP16 NEON_VFPV4 ASIMD ASIMDHP ASIMDDP ASIMDFHM + - env: + CIBW_BUILD: "cp310-manylinux_aarch64" + - env: + CIBW_BUILD: "cp311-manylinux_aarch64" + - env: + CIBW_BUILD: "cp312-manylinux_aarch64" + env: + CIBW_ARCHS: "aarch64" + # CIBW_BUILD: "*-manylinux_aarch64" + CIBW_SKIP: "pp37-* cp36-* cp37-* cp38-*" + CIBW_BUILD_VERBOSITY: "3" + MANYLINUX_AARCH64_IMAGE: "manylinux2014" + CIBW_ENVIRONMENT: > + USE_LONG_INT=0 + USE_UNSIGNED_LONG_INT=0 + USE_CBLAS=0 + + build_script: | + apt update + apt install -y python3-venv python-is-python3 + git fetch origin + which python + echo $CIRRUS_CHANGE_MESSAGE + <<: *PYPI_LINUX_AARCH64_CIBW + + +# ================================= +# pypi upload wheels linux aarch 64 +# ================================= + +pypi_upload_wheels_linux_aarch64_task: + use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' + # Artifacts don't seem to be persistent from task to task. + # Rather than upload wheels at the end of each cibuildwheel run we do a + # final upload here. This is because a run may be on different OS for + # which bash, etc, may not be present. + depends_on: + - pypi_build_wheels_linux_aarch64 + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 4G + + env: + PYPI_PASSWORD: ENCRYPTED[!fd3bceb1c585064d6a6803ba64d3c76c83434af2fbe9b6002f3018c8e5c42344c702f1e8093bbc4dd6a6aaea464dae61!] + + upload_script: | + apt-get update + apt-get install -y curl wget + + # The name of the zip file is derived from the `wheels_artifact` line. + # If you change the artifact line to `myfile_artifact` then it would be + # called myfile.zip + curl https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/pypi_linux_aarch64_wheels.zip --output wheels.zip + mkdir -p dist + unzip -j wheels.zip -d dist + + source ./tools/wheels/upload_wheels.sh + install_anaconda "linux_x86_64" + upload_wheels_pypi + + +# ======================= +# pypi test linux aarch64 +# ======================= + +pypi_test_linux_aarch64_task: + use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder-arm64 + architecture: arm64 + platform: linux + cpu: 1 + memory: 4G + depends_on: + - pypi_upload_wheels_linux_aarch64 + + test_script: | + apt update + apt install -y python3 + set -x + which python3 + uname -m + python3 -c "import platform;print(platform.python_version());print(platform.system());print(platform.machine())" + mv detkit detkit_do_dot_import + python3 -m pip install --upgrade pip + python3 -m pip install --prefer-binary --upgrade detkit + python3 -m pip install numpy + python3 -m pip install -r tests/requirements.txt + python3 -m pytest diff --git a/tools/ci/cirrus_deploy_pypi_macosx_arm64.yml b/tools/ci/cirrus_deploy_pypi_macosx_arm64.yml new file mode 100644 index 0000000..2950fed --- /dev/null +++ b/tools/ci/cirrus_deploy_pypi_macosx_arm64.yml @@ -0,0 +1,146 @@ +# SPDX-FileCopyrightText: Copyright 2021, Siavash Ameli +# SPDX-License-Identifier: BSD-3-Clause +# SPDX-FileType: SOURCE +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the license found in the LICENSE.txt file in the root +# directory of this source tree. + + +# ====================== +# pypi macosx arm64 cibw +# ====================== + +pypi_macosx_arm64_cibw: &PYPI_MACOSX_ARM64_CIBW + install_cibuildwheel_script: + - python -m pip install cibuildwheel + cibuildwheel_script: + - cibuildwheel + pypi_macosx_arm64_wheels_artifacts: + path: "wheelhouse/*.whl" + + +# ============================== +# pypi build wheels macosx arm64 +# ============================== + +pypi_build_wheels_macosx_arm64_task: + use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' + macos_instance: + image: ghcr.io/cirruslabs/macos-monterey-xcode:latest + cpu: 1 + memory: 4G + matrix: + - env: + CIBW_BUILD: "cp39-macosx_arm64" + - env: + CIBW_BUILD: "cp310-macosx_arm64" + - env: + CIBW_BUILD: "cp311-macosx_arm64" + - env: + CIBW_BUILD: "cp312-macosx_arm64" + env: + PATH: /opt/homebrew/opt/python@3.10/bin:/usr/local/lib:/usr/local/include:$PATH + # Specifying CIBW_ENVIRONMENT_MACOS overrides pyproject.toml, so include + # all the settings from there, otherwise they're lost. + # SDKROOT needs to be set for repackaged conda-forge gfortran compilers + # supplied by isuruf. + # Find out SDKROOT via `xcrun --sdk macosx --show-sdk-path` + CIBW_ENVIRONMENT_MACOS: > + SDKROOT=/Applications/Xcode-14.0.0.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.3.sdk + LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + CFLAGS="-I/opt/homebrew/opt/libomp/include $CFLAGS" + CXXFLAGS="-I/opt/homebrew/opt/libomp/include $CXXFLAGS" + LDFLAGS="-L/opt/homebrew/opt/libomp/lib $LDFLAGS" + LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$LIBRARY_PATH" + USE_CBLAS=0 + CIBW_ARCHS: "arm64" + # CIBW_BUILD: "*-macosx_arm64" + CIBW_SKIP: "pp37-* cp36-* cp37-* cp38-*" + CIBW_BUILD_VERBOSITY: "3" + CIBW_BEFORE_BUILD: "pip install delocate; brew install libomp" + CIBW_TEST_SKIP: "*_universal2:arm64" + + build_script: + - brew install python@3.10 + - ln -s python3 /opt/homebrew/opt/python@3.10/bin/python + - which python + - uname -m + - python -c "import platform;print(platform.python_version());print(platform.system());print(platform.machine())" + - clang --version + <<: *PYPI_MACOSX_ARM64_CIBW + + +# =============================== +# pypi upload wheels macosx arm64 +# =============================== + +pypi_upload_wheels_macosx_arm64_task: + use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' + # Artifacts don't seem to be persistent from task to task. + # Rather than upload wheels at the end of each cibuildwheel run we do a + # final upload here. This is because a run may be on different OS for + # which bash, etc, may not be present. + depends_on: + - pypi_build_wheels_macosx_arm64 + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 4G + + env: + PYPI_PASSWORD: ENCRYPTED[!fd3bceb1c585064d6a6803ba64d3c76c83434af2fbe9b6002f3018c8e5c42344c702f1e8093bbc4dd6a6aaea464dae61!] + + upload_script: | + apt-get update + apt-get install -y curl wget + + # The name of the zip file is derived from the `wheels_artifact` line. + # If you change the artifact line to `myfile_artifact` then it would be + # called myfile.zip + curl https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/pypi_macosx_arm64_wheels.zip --output wheels.zip + mkdir -p dist + unzip -j wheels.zip -d dist + + source ./tools/wheels/upload_wheels.sh + install_anaconda "linux_x86_64" + upload_wheels_pypi + + +# ====================== +# pypi test macosx arm64 +# ====================== + +pypi_test_macosx_arm64_task: + use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' + macos_instance: + image: ghcr.io/cirruslabs/macos-monterey-xcode:latest + cpu: 1 + memory: 4G + depends_on: + - pypi_upload_wheels_macosx_arm64 + matrix: + - env: + PY_VER: "3.9" + - env: + PY_VER: "3.10" + # - env: + # PY_VER: "3.11" + # - env: + # PY_VER: "3.12" + + test_script: + - brew install python@${PY_VER} + - ln -s python3 /opt/homebrew/opt/python@${PY_VER}/bin/python + - set -x + - which python3 + - uname -m + - python3 -c "import platform;print(platform.python_version());print(platform.system());print(platform.machine())" + - mv detkit detkit_do_dot_import + - python3 -m pip install --upgrade pip + - python3 -m pip install --prefer-binary --upgrade detkit + - python3 -m pip install numpy + - python3 -m pip install -r tests/requirements.txt + - python3 -m pytest diff --git a/tools/scripts/docker_utils.sh b/tools/scripts/docker_utils.sh new file mode 100644 index 0000000..584aae9 --- /dev/null +++ b/tools/scripts/docker_utils.sh @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright 2021, Siavash Ameli +# SPDX-License-Identifier: BSD-3-Clause +# SPDX-FileType: SOURCE +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the license found in the LICENSE.txt file in the root +# directory of this source tree. + + +set -xe + +# ============== +# install docker +# ============== + +install_docker() { + + # Add Docker's official GPG key: + sudo apt-get update + sudo apt-get install ca-certificates curl gnupg + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + sudo chmod a+r /etc/apt/keyrings/docker.gpg + + # Add the repository to Apt sources: + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + + # Install latest version of docker + sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + + # Check docker with hello world example + sudo service docker start + sudo docker run hello-world + + # Configure to use docker without sudo + sudo groupadd docker + sudo usermod -aG docker $USER + + # Check docker works without sudo + docker run hello-world +} + + +# ================ +# install anaconda +# ================ + +install_anaconda() { + + # Usage: + # install_anaconda "linux_aarch64" + # install_anaconda "linux_x86_64" + # install_anaconda "macosx_arm64" + + MINICONDA_URL="https://repo.anaconda.com/miniconda/" + if [ "$1" == "linux_aarch64" ]; + then + URL=${MINICONDA_URL}"Miniconda3-latest-Linux-aarch64.sh" + elif [ "$1" == "linux_x86_64" ]; + then + URL=${MINICONDA_URL}"Miniconda3-latest-Linux-x86_64.sh" + elif [ "$1" == "macosx_arm64" ]; + then + URL=${MINICONDA_URL}"Miniconda3-latest-MacOSX-arm64.sh" + else + echo "OS or platform is invalid." + # return 1; + fi + + + # install miniconda in the temp directory, not in the current directory, + # because the source code of the package is in the current directory, and + # if we also install miniconda3 in the same directory of the package source + # code, when we use conda-build, it gives error as it also tries to build + # miniconda as well as the code. + TMP_DIR="/tmp" + + # install miniconda for uploading to anaconda + wget -q ${URL} -O ${TMP_DIR}/miniconda.sh + bash ${TMP_DIR}/miniconda.sh -b -p ${TMP_DIR}/miniconda3 + + # Conda directories + CONDA_BASE_DIR=${TMP_DIR}/miniconda3 + CONDA_BIN_DIR=${CONDA_BASE_DIR}/bin + + # Export PATH + export PATH=${CONDA_BASE_DIR}:$PATH + export PATH=${CONDA_BIN_DIR}:$PATH + + # Initialize conda + # ${CONDA_BIN_DIR}/conda init bash + # source ${CONDA_BIN_DIR}/activate +} + + +# ===================== +# build upload anaconda +# ===================== + +build_upload_anaconda() { + + # Usage: + # build_upload_anaconda "linux_aarch64" + # build_upload_anaconda "linux_x86_64" + # build_upload_anaconda "macosx_arm64" + + conda install -y anaconda-client conda-build + + # Anaconda account + ANACONDA_USERNAME="s-ameli" + + # Upload sdist + if compgen -G "./dist/*.gz"; then + anaconda -t ${ANACONDA_API_TOKEN} upload --force \ + -u ${ANACONDA_USERNAME} ./dist/*.gz + else + echo "Dist files do not exist" + fi + + # Determine the sub-directory where the conda builds the package depending + # on the operating system and platform + if [ "$1" == "linux_aarch64" ]; + then + BUILD_SUBDIR="linux-aarch64" + elif [ "$1" == "linux_x86_64" ]; + then + BUILD_SUBDIR="linux-64" + elif [ "$1" == "macosx_arm64" ]; + then + BUILD_SUBDIR="osx-arm64" + else + echo "OS or platform is invalid." + return 1; + fi + + # Build wheels and upload them automatically + if compgen -G "./dist/*.whl"; then + conda-build --output-folder conda-bld . + anaconda -t ${ANACONDA_API_TOKEN} upload --force \ + -u ${ANACONDA_USERNAME} conda-bld/${BUILD_SUBDIR}/*.tar.bz2 + else + echo "Wheel files do not exist" + return 1 + fi +} + + +# ================== +# upload wheels pypi +# ================== + +upload_wheels_pypi() { + + # Install pip and twine + python -m pip install --upgrade pip + python -m pip install twine + + PYPI_USERNAME="__token__" + + if [[ -z ${PYPI_PASSWORD} ]]; then + echo no pypi password set, not uploading + else + if compgen -G "./dist/*.whl"; then + echo "Found wheel" + twine upload ./dist/* -u ${PYPI_USERNAME} -p ${PYPI_PASSWORD} \ + --verbose + else + echo "Wheel files do not exist" + return 1 + fi + fi +}