This repository has been archived by the owner on Nov 1, 2024. It is now read-only.
TorchArrow Nightly Build and Test #949
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: TorchArrow Nightly Build and Test | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" | |
| workflow_call: | |
| secrets: | |
| AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: | |
| required: true | |
| AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: | |
| required: true | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| jobs: | |
| linux-container: | |
| runs-on: ubuntu-latest | |
| container: prestocpp/velox-sse-velox-torcharrow:kpai-20220711 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - 3.7 | |
| - 3.8 | |
| - 3.9 | |
| - "3.10" | |
| steps: | |
| - name: Check out source repository | |
| uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - name: Build the wheel | |
| run: | | |
| source /opt/conda/etc/profile.d/conda.sh | |
| PYTHON_VERSION=${{ matrix.python-version }} CPU_TARGET="sse" packaging/build_wheel.sh | |
| conda activate env${{ matrix.python-version }} | |
| pip install auditwheel | |
| auditwheel repair dist/*.whl -w fixed_dist --plat manylinux2014_x86_64 | |
| - name: Upload wheel to S3 Storage | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} | |
| S3_PATH: "s3://pytorch/whl/nightly/" | |
| run: | | |
| source /opt/conda/etc/profile.d/conda.sh | |
| conda activate env${{ matrix.python-version }} | |
| pip install --user awscli | |
| set -x | |
| if [[ -z "${AWS_ACCESS_KEY_ID}" ]]; then | |
| echo "AWS_ACCESS_KEY_ID is not set, exiting upload" | |
| exit 1 | |
| fi | |
| for pkg in fixed_dist/torcharrow*.whl; do | |
| ~/.local/bin/aws s3 cp "$pkg" "$S3_PATH" --acl public-read | |
| done | |
| # We only run this part on ubuntu wheel build for python 3.7, | |
| # since we only need to deploy the doc once. | |
| - name: Install TorchArrow and build docs | |
| if: matrix.python-version == '3.7' | |
| run: | | |
| source /opt/conda/etc/profile.d/conda.sh | |
| conda activate env${{ matrix.python-version }} | |
| pip install fixed_dist/*.whl | |
| cd ./docs | |
| pip install -r requirements.txt --user | |
| PATH=${PATH}:~/.local/bin | |
| make html | |
| cd .. | |
| - name: Deploy Docs on Push | |
| if: matrix.python-version == '3.7' | |
| uses: JamesIves/github-pages-deploy-action@v4.2.5 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: gh-pages # The branch the action should deploy to. | |
| folder: docs/build/html # The folder the action should deploy. | |
| target-folder: main | |
| macos-container: | |
| runs-on: macos-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - 3.7 | |
| - 3.8 | |
| - 3.9 | |
| - "3.10" | |
| steps: | |
| - name: Setup Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Check out source repository | |
| uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - name: Build the wheel | |
| run: | | |
| MACOSX_DEPLOYMENT_TARGET=10.15 CPU_TARGET="sse" ./csrc/velox/velox/scripts/setup-macos.sh | |
| pip install wheel | |
| CPU_TARGET="sse" ./packaging/build_wheel.sh | |
| pip install delocate | |
| delocate-wheel dist/*.whl -w fixed_dist | |
| - name: Upload wheel to S3 Storage | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} | |
| S3_PATH: "s3://pytorch/whl/nightly/" | |
| run: | | |
| pip install --user awscli | |
| set -x | |
| if [[ -z "${AWS_ACCESS_KEY_ID}" ]]; then | |
| echo "AWS_ACCESS_KEY_ID is not set, exiting upload" | |
| exit 1 | |
| fi | |
| for pkg in fixed_dist/torcharrow*.whl; do | |
| ~/.local/bin/aws s3 cp "$pkg" "$S3_PATH" --acl public-read | |
| done |