tests: misc logic updates - 3 #2045
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: wipac ci/cd | |
| on: [ push ] | |
| env: | |
| THIS_IMAGE_WITH_TAG: 'ghcr.io/wipacrepo/skydriver:vX.Y.Z' | |
| EWMS_PILOT_TASK_TIMEOUT: 999 | |
| SCAN_BACKLOG_RUNNER_SHORT_DELAY: 1 | |
| SCAN_BACKLOG_RUNNER_DELAY: 1 | |
| SCAN_BACKLOG_PENDING_ENTRY_TTL_REVIVE: 200 | |
| LOG_LEVEL: debug | |
| # mandatory env vars... | |
| EWMS_ADDRESS: http://localhost:8081 | |
| EWMS_TOKEN_URL: 65f3b929 | |
| EWMS_CLIENT_ID: b75a974d | |
| EWMS_CLIENT_SECRET: 411b16fe | |
| S3_URL: a4f92304 | |
| S3_ACCESS_KEY_ID: 36c5c849 | |
| S3_ACCESS_KEY_ID__K8S_SECRET_KEY: 230ec9dc | |
| S3_SECRET_KEY: 8dea68a1 | |
| S3_SECRET_KEY__K8S_SECRET_KEY: cdf7c60b | |
| S3_BUCKET: 72017610 | |
| K8S_SECRET_NAME: super-secrets | |
| MIN_SKYMAP_SCANNER_TAG: "v3.21.2" # TODO: remove once skyscan v4 is out (that's the real min) | |
| jobs: | |
| py-versions: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.versions.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: versions | |
| uses: WIPACrepo/wipac-dev-py-versions-action@v2.5 | |
| ############################################################################# | |
| # LINTERS | |
| ############################################################################# | |
| flake8: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| - uses: WIPACrepo/wipac-dev-flake8-action@v1.2 | |
| with: | |
| max-complexity: 10 | |
| mypy: | |
| needs: [ py-versions ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.py3 }} | |
| - uses: WIPACrepo/wipac-dev-mypy-action@v2.0 | |
| ############################################################################# | |
| # PACKAGING | |
| ############################################################################# | |
| writable-branch-detect: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| OKAY: ${{ steps.detect.outputs.OKAY }} | |
| steps: | |
| - name: is this a bot-writable branch? | |
| id: detect | |
| # dependabot can't access normal secrets | |
| # & don't run non-branch triggers (like tags) | |
| # & we don't want to trigger an update on PR's merge to main/master/default (which is a branch) | |
| run: | | |
| set -euo pipefail | |
| if [[ \ | |
| ${{github.actor}} != 'dependabot[bot]' && \ | |
| ${{github.ref_type}} == 'branch' && \ | |
| ${{format('refs/heads/{0}', github.event.repository.default_branch)}} != ${{github.ref}} \ | |
| ]]; then | |
| echo "OKAY=true" >> "$GITHUB_OUTPUT" | |
| echo "yes, this branch is compatible" | |
| else | |
| echo "OKAY=false" >> "$GITHUB_OUTPUT" | |
| echo "no, this branch is incompatible" | |
| fi | |
| py-setup: | |
| needs: [ writable-branch-detect ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - if: needs.writable-branch-detect.outputs.OKAY == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| - if: needs.writable-branch-detect.outputs.OKAY == 'true' | |
| uses: WIPACrepo/wipac-dev-py-setup-action@v3.1 | |
| with: | |
| base-keywords: WIPAC IceCube | |
| py-dependencies: | |
| needs: [ writable-branch-detect ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - if: needs.writable-branch-detect.outputs.OKAY == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| - if: needs.writable-branch-detect.outputs.OKAY == 'true' | |
| uses: docker/setup-buildx-action@v2 | |
| - if: needs.writable-branch-detect.outputs.OKAY == 'true' | |
| uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| tags: skydriver:py-dep-this | |
| load: true | |
| - if: needs.writable-branch-detect.outputs.OKAY == 'true' | |
| uses: WIPACrepo/wipac-dev-py-dependencies-action@v2.1 | |
| ############################################################################# | |
| # TESTS | |
| ############################################################################# | |
| unit-tests: | |
| needs: [ py-versions ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.py3 }} | |
| - name: pip install | |
| run: | | |
| set -euo pipefail | |
| pip install --upgrade pip wheel setuptools | |
| pip install .[tests] | |
| - name: test | |
| run: | | |
| set -euo pipefail | |
| pytest -vvv tests/unit --exitfirst | |
| - name: Dump logs | |
| if: always() | |
| run: | | |
| set -euo pipefail | |
| cat pytest.logs || true | |
| integration-tests: | |
| needs: [ py-versions ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }} | |
| services: | |
| mongo: | |
| image: bitnami/mongodb:4 | |
| ports: | |
| - 27017:27017 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-buildx-action@v2 | |
| - uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=min | |
| file: Dockerfile | |
| tags: wipac/skydriver:local | |
| load: true | |
| - name: test | |
| run: | | |
| set -euo pipefail | |
| pip install .[tests] | |
| python tests/integration/dummy_ewms.py &> ./dummy_ewms.out & | |
| export LATEST_TAG=$( \ | |
| curl -I https://github.com/icecube/skymap_scanner/releases/latest \ | |
| | awk -F '/' '/^location/ {print substr($NF, 1, length($NF)-1)}' \ | |
| | sed 's/v//' \ | |
| ) | |
| echo $LATEST_TAG # this tag may be off if there's a delay between GH release & docker hub | |
| # make test script | |
| DIR="test-script-dir" | |
| mkdir $DIR | |
| echo "#!/bin/bash" >> $DIR/test-script.sh | |
| echo "set -xe" >> $DIR/test-script.sh | |
| echo "pip install .[tests]" >> $DIR/test-script.sh | |
| echo "python -m pytest -vvv tests/integration --exitfirst" >> $DIR/test-script.sh | |
| chmod +x $DIR/test-script.sh | |
| cat $DIR/test-script.sh | |
| docker run --network="host" --rm -i --name test \ | |
| --env LATEST_TAG=$LATEST_TAG \ | |
| --env THIS_IMAGE_WITH_TAG=$THIS_IMAGE_WITH_TAG \ | |
| --env K8S_SECRET_NAME=$K8S_SECRET_NAME \ | |
| $(env | grep '^SKYSCAN_' | awk '$0="--env "$0') \ | |
| $(env | grep '^EWMS_' | awk '$0="--env "$0') \ | |
| $(env | grep '^S3_' | awk '$0="--env "$0') \ | |
| $(env | grep '^CI' | awk '$0="--env "$0') \ | |
| $(env | grep '^CI_' | awk '$0="--env "$0') \ | |
| $(env | grep '^SCAN_' | awk '$0="--env "$0') \ | |
| $(env | grep '^MIN_SKYMAP_SCANNER_TAG' | awk '$0="--env "$0') \ | |
| --mount type=bind,source=$(realpath $DIR),target=/local/$DIR \ | |
| wipac/skydriver:local \ | |
| /local/$DIR/test-script.sh | |
| # | |
| - name: dump test logs | |
| if: always() | |
| run: | | |
| set -euo pipefail | |
| docker logs test || true | |
| - name: dump dummy-ewms logs | |
| if: always() | |
| run: | | |
| set -euo pipefail | |
| cat ./dummy_ewms.out | |
| - name: dump mongo logs | |
| if: always() | |
| run: | | |
| set -euo pipefail | |
| docker logs "${{ job.services.mongo.id }}" || true | |
| test-build-docker: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-buildx-action@v2 | |
| - uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| tags: wipac/skydriver:local | |
| release: | |
| # only run on main/master/default | |
| if: format('refs/heads/{0}', github.event.repository.default_branch) == github.ref | |
| needs: [ flake8, mypy, py-setup, py-dependencies, unit-tests, integration-tests, test-build-docker ] | |
| runs-on: ubuntu-latest | |
| concurrency: release | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| - name: Python Semantic Release | |
| uses: python-semantic-release/python-semantic-release@v7.34.6 | |
| with: | |
| github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| # repository_username: __token__ | |
| # repository_password: ${{ secrets.PYPI_TOKEN }} |