From 2a0203138e450b7a274a22eb147e3c5ae2d45475 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 27 Oct 2023 13:40:28 -0400 Subject: [PATCH 1/2] Revert "add data workflow to retrieve WebbPSF cache and rename toxenvs (#223)" This reverts commit 4a809eb8d08934909ea4979e89e2849aad725be4. --- .github/workflows/ci.yml | 60 ++++++++++++++++++++------- .github/workflows/ci_cron.yml | 8 ++-- .github/workflows/data.yml | 77 ----------------------------------- 3 files changed, 51 insertions(+), 94 deletions(-) delete mode 100644 .github/workflows/data.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a71dc8a0..43cd6569 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,31 +23,63 @@ jobs: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: envs: | + - linux: check-style - linux: check-security + - linux: check-build test: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: envs: | - - linux: py39-oldestdeps-cov-xdist - - linux: py39-xdist - - linux: py310-xdist - - linux: py311-xdist - - macos: py311-xdist - - linux: py3-cov-xdist + - linux: test-oldestdeps-cov-xdist + python-version: 3.9 + - linux: test-xdist + python-version: '3.9' + - linux: test-xdist + python-version: '3.10' + - linux: test-xdist + python-version: '3.11' + - macos: test-xdist + python-version: '3.11' + - linux: test-cov-xdist coverage: 'codecov' data: - uses: ./.github/workflows/data.yml + name: retrieve data + runs-on: ubuntu-latest + outputs: + data_path: ${{ steps.data.outputs.path }} + data_hash: ${{ steps.data_hash.outputs.hash }} + steps: + # webbpsf: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - id: data + run: | + echo "path=/tmp/data" >> $GITHUB_OUTPUT + echo "webbpsf_url=https://stsci.box.com/shared/static/n1fealx9q0m6sdnass6wnyfikvxtc0zz.gz" >> $GITHUB_OUTPUT + - run: | + mkdir -p tmp/data/ + mkdir -p ${{ steps.data.outputs.path }} + - run: wget ${{ steps.data.outputs.webbpsf_url }} -O tmp/minimal-webbpsf-data.tar.gz + - run: tar -xzvf tmp/minimal-webbpsf-data.tar.gz -C tmp/data/ + - id: data_hash + run: echo "hash=${{ hashFiles( 'tmp/data' ) }}" >> $GITHUB_OUTPUT + - run: mv tmp/data/* ${{ steps.data.outputs.path }} + - uses: actions/cache@v3 + with: + path: ${{ steps.data.outputs.path }} + key: data-${{ steps.data_hash.outputs.hash }} test_downstream: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main needs: [ data ] with: setenv: | - WEBBPSF_PATH: ${{ needs.data.outputs.webbpsf_path }} - CRDS_PATH: ${{ needs.data.outputs.path }}/crds_cache + CRDS_PATH: /tmp/crds_cache CRDS_CLIENT_RETRY_COUNT: 3 - CRDS_CLIENT_RETRY_DELAY_SECONDS: 20 - cache-path: ${{ needs.data.outputs.webbpsf_path }} - cache-key: webbpsf-${{ needs.data.outputs.webbpsf_hash }} + CRDS_CLIENT_RETRY_DELAY_SECONDS: 20 + WEBBPSF_PATH: ${{ needs.data.outputs.data_path }}/webbpsf-data + cache-path: ${{ needs.data.outputs.data_path }} + cache-key: data-${{ needs.data.outputs.data_hash }} envs: | - - linux: py311-jwst-cov-xdist - - linux: py311-romancal-cov-xdist + - linux: test-jwst-cov-xdist + - linux: test-romancal-cov-xdist diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml index 6289032a..43ac5546 100644 --- a/.github/workflows/ci_cron.yml +++ b/.github/workflows/ci_cron.yml @@ -11,6 +11,8 @@ jobs: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: envs: | - - macos: py39-xdist - - macos: py310-xdist - - linux: py3-devdeps-xdist + - macos: test-xdist + python-version: 3.9 + - macos: test-xdist + python-version: 3.10 + - linux: test-devdeps-xdist diff --git a/.github/workflows/data.yml b/.github/workflows/data.yml deleted file mode 100644 index 66a0ec33..00000000 --- a/.github/workflows/data.yml +++ /dev/null @@ -1,77 +0,0 @@ -on: - workflow_call: - outputs: - path: - value: ${{ jobs.path.outputs.path }} - webbpsf_path: - value: ${{ jobs.webbpsf_path.outputs.path }} - webbpsf_hash: - value: ${{ jobs.webbpsf_data.outputs.hash }} - workflow_dispatch: - schedule: - - cron: "42 4 * * 3" - -env: - DATA_PATH: /tmp/data - -jobs: - path: - runs-on: ubuntu-latest - outputs: - path: ${{ steps.path.outputs.path }} - steps: - - id: path - run: echo "path=${{ env.DATA_PATH }}" >> $GITHUB_OUTPUT - webbpsf_path: - needs: [ path ] - runs-on: ubuntu-latest - outputs: - path: ${{ steps.path.outputs.path }} - steps: - - id: path - run: echo "path=${{ env.DATA_PATH }}/webbpsf-data" >> $GITHUB_OUTPUT - webbpsf_data: - if: (github.repository == 'spacetelescope/stcal' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'update webbpsf data'))) - needs: [ webbpsf_path ] - name: download and cache WebbPSF data - runs-on: ubuntu-latest - env: - WEBBPSF_DATA_URL: https://stsci.box.com/shared/static/qxpiaxsjwo15ml6m4pkhtk36c9jgj70k.gz - steps: - - run: mkdir -p tmp/data - - run: wget ${{ env.WEBBPSF_DATA_URL }} -O tmp/webbpsf-data.tar.gz - - id: data_hash - run: echo "hash=$( shasum tmp/webbpsf-data.tar.gz | cut -d ' ' -f 1 )" >> $GITHUB_OUTPUT - - id: cache_check - uses: actions/cache@v3 - with: - path: ${{ needs.webbpsf_path.outputs.path }} - key: webbpsf-${{ steps.data_hash.outputs.hash }} - - if: ${{ steps.cache_check.outputs.cache-hit != 'true' }} - run: mkdir -p ${{ env.DATA_PATH }} - - if: ${{ steps.cache_check.outputs.cache-hit != 'true' }} - run: tar -xzvf tmp/webbpsf-data.tar.gz -C ${{ env.DATA_PATH }} - webbpsf_hash: - needs: [ webbpsf_path, webbpsf_data ] - # run data job if webbpsf-data succeeds or is skipped. This allows - # this data job to always fetch the crds context even if the webbpsf data fetching - # was skipped (and an existing cache will be used for the webbpsf data). - if: always() && (needs.webbpsf_data.result == 'success' || needs.webbpsf_data.result == 'skipped') - name: retrieve latest data cache key - runs-on: ubuntu-latest - env: - GH_TOKEN: ${{ github.token }} - outputs: - hash: ${{ steps.hash.outputs.hash }} - steps: - - id: hash - run: | - # use actions/gh-actions-cache to allow filtering by key - gh extension install actions/gh-actions-cache - - RECENT=$(gh actions-cache list -R spacetelescope/stcal --key webbpsf- --sort created-at | cut -f 1 | head -n 1) - echo "RECENT=$RECENT" - HASH=$(echo $RECENT | cut -d '-' -f 2) - echo "HASH=$HASH" - echo "hash=$HASH" >> $GITHUB_OUTPUT - if [ "$HASH" == '' ]; then exit 1; fi From 538329f9729f5cf4e738ff75dafac83d27fc061c Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 27 Oct 2023 13:48:34 -0400 Subject: [PATCH 2/2] cleanup old webbpsf stuff --- .github/workflows/ci.yml | 52 ++++++----------------------------- .github/workflows/ci_cron.yml | 8 ++---- 2 files changed, 11 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43cd6569..b35acca3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,56 +30,20 @@ jobs: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: envs: | - - linux: test-oldestdeps-cov-xdist - python-version: 3.9 - - linux: test-xdist - python-version: '3.9' - - linux: test-xdist - python-version: '3.10' - - linux: test-xdist - python-version: '3.11' - - macos: test-xdist - python-version: '3.11' - - linux: test-cov-xdist + - linux: py39-oldestdeps-cov-xdist + - linux: py39-xdist + - linux: py310-xdist + - linux: py311-xdist + - macos: py311-xdist + - linux: py3-cov-xdist coverage: 'codecov' - data: - name: retrieve data - runs-on: ubuntu-latest - outputs: - data_path: ${{ steps.data.outputs.path }} - data_hash: ${{ steps.data_hash.outputs.hash }} - steps: - # webbpsf: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: data - run: | - echo "path=/tmp/data" >> $GITHUB_OUTPUT - echo "webbpsf_url=https://stsci.box.com/shared/static/n1fealx9q0m6sdnass6wnyfikvxtc0zz.gz" >> $GITHUB_OUTPUT - - run: | - mkdir -p tmp/data/ - mkdir -p ${{ steps.data.outputs.path }} - - run: wget ${{ steps.data.outputs.webbpsf_url }} -O tmp/minimal-webbpsf-data.tar.gz - - run: tar -xzvf tmp/minimal-webbpsf-data.tar.gz -C tmp/data/ - - id: data_hash - run: echo "hash=${{ hashFiles( 'tmp/data' ) }}" >> $GITHUB_OUTPUT - - run: mv tmp/data/* ${{ steps.data.outputs.path }} - - uses: actions/cache@v3 - with: - path: ${{ steps.data.outputs.path }} - key: data-${{ steps.data_hash.outputs.hash }} test_downstream: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main - needs: [ data ] with: setenv: | CRDS_PATH: /tmp/crds_cache CRDS_CLIENT_RETRY_COUNT: 3 CRDS_CLIENT_RETRY_DELAY_SECONDS: 20 - WEBBPSF_PATH: ${{ needs.data.outputs.data_path }}/webbpsf-data - cache-path: ${{ needs.data.outputs.data_path }} - cache-key: data-${{ needs.data.outputs.data_hash }} envs: | - - linux: test-jwst-cov-xdist - - linux: test-romancal-cov-xdist + - linux: py311-jwst-cov-xdist + - linux: py311-romancal-cov-xdist diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml index 43ac5546..6289032a 100644 --- a/.github/workflows/ci_cron.yml +++ b/.github/workflows/ci_cron.yml @@ -11,8 +11,6 @@ jobs: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: envs: | - - macos: test-xdist - python-version: 3.9 - - macos: test-xdist - python-version: 3.10 - - linux: test-devdeps-xdist + - macos: py39-xdist + - macos: py310-xdist + - linux: py3-devdeps-xdist