release-pypi.yml

name: PyPI

# https://help.github.com/en/actions/reference/events-that-trigger-workflows
on:  # Trigger the workflow on push or pull request, but only for the master branch
  push:
    branches: [master, "release/*"]
  release:
    types: [published]


jobs:
  # based on https://github.com/pypa/gh-action-pypi-publish
  build-package:
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-python@v2
      with:
        python-version: 3.9

    - name: Install dependencies
      run: >-
        python -m pip install --user --upgrade setuptools wheel

    - name: Build packages
      run: |
        python setup.py sdist bdist_wheel
        ls -lh dist/

    - uses: actions/upload-artifact@v2
      with:
        name: pypi-packages-${{ github.sha }}
        path: dist

  upload-package:
    runs-on: ubuntu-20.04
    needs: build-package
    if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
    steps:
    - uses: actions/checkout@v2
    - uses: actions/download-artifact@v2
      with:
        name: pypi-packages-${{ github.sha }}
        path: dist
    - run: ls -lh dist/

    - name: Upload to release
      uses: AButler/upload-release-assets@v2.0
      with:
        files: 'dist/*'
        repo-token: ${{ secrets.GITHUB_TOKEN }}

  publish-package:
    runs-on: ubuntu-20.04
    needs: build-package
    if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
    steps:
    - uses: actions/checkout@v2
    - uses: actions/download-artifact@v2
      with:
        name: pypi-packages-${{ github.sha }}
        path: dist
    - run: ls -lh dist/

    - name: Delay releasing
      uses: juliangruber/sleep-action@v1
      with:
        time: 10m

    # We do this, since failures on test.pypi aren't that bad
    - name: Publish to Test PyPI
      uses: pypa/gh-action-pypi-publish@v1.4.1
      with:
        user: __token__
        password: ${{ secrets.test_pypi_password }}
        repository_url: https://test.pypi.org/legacy/
        verbose: true

    - name: Publish distribution 📦 to PyPI
      uses: pypa/gh-action-pypi-publish@v1.4.1
      with:
        user: __token__
        password: ${{ secrets.pypi_password }}

  create-legacy-ckpt:
    runs-on: ubuntu-20.04
    needs: [build-package, publish-package]
    steps:
    - uses: actions/checkout@v2

    - uses: actions/setup-python@v2
      with:
        python-version: 3.9

    - name: Weekly reset caching
      run: echo "::set-output name=period::$(python -c 'import time ; days = time.time() / 60 / 60 / 24 ; print(int(days / 7))' 2>&1)"
      id: times

    # Note: This uses an internal pip API and may not always work
    # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
    - name: Cache pip
      uses: actions/cache@v2
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-td${{ steps.times.outputs.period }}-${{ hashFiles('requirements.txt') }}
        restore-keys: ${{ runner.os }}-pip-td${{ steps.times.outputs.period }}-

    - name: Install dependencies
      run: |
        pip install -r requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet
        pip install awscli

    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v1
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_ID }}
        aws-region: us-east-1

    - uses: actions/download-artifact@v2
      with:
        name: pypi-packages-${{ github.sha }}
        path: dist

    - name: Pull files from S3
      run: |
        aws s3 cp --recursive s3://pl-public-data/legacy/checkpoints/ legacy/checkpoints/ #  --acl public-read
        ls -l legacy/checkpoints/

    - name: Generate checkpoint
      run: |
        ls -lh dist/
        pip install dist/*.whl

        pl_ver=$(python -c "import pytorch_lightning as pl ; print(pl.__version__)" 2>&1)
        # generate checkpoint to this version
        bash legacy/generate_checkpoints.sh $pl_ver

    - name: Push files to S3
      working-directory: ./legacy
      run: |
        aws s3 sync legacy/checkpoints/ s3://pl-public-data/legacy/checkpoints/
        zip -r checkpoints.zip checkpoints
        aws s3 cp checkpoints.zip s3://pl-public-data/legacy/ --acl public-read