Make data fetching scalable with Dask #1333
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI tests Upstream | |
| # Daily tests with all un-pinned dependencies | |
| # Allows to be warned when argopy fail tests due to updates in dependencies | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" # Daily “At 00:00” UTC | |
| workflow_dispatch: # allows you to trigger the workflow run manually | |
| pull_request: | |
| types: [synchronize] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| detect-ci-trigger: | |
| name: detect upstream ci trigger | |
| runs-on: ubuntu-latest | |
| if: | | |
| github.repository == 'euroargodev/argopy' | |
| && (github.event_name == 'push' || github.event_name == 'pull_request') | |
| outputs: | |
| triggered: ${{ steps.detect-trigger.outputs.trigger-found }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| - uses: xarray-contrib/ci-trigger@v1 | |
| id: detect-trigger | |
| with: | |
| keyword: "[test-upstream]" | |
| detect-ci-skip: | |
| name: detect upstream ci skip | |
| runs-on: ubuntu-latest | |
| if: | | |
| github.repository == 'euroargodev/argopy' | |
| && (github.event_name == 'push' || github.event_name == 'pull_request') | |
| outputs: | |
| skipped: ${{ steps.detect-skip.outputs.trigger-found }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| - uses: xarray-contrib/ci-trigger@v1.2 | |
| id: detect-skip | |
| with: | |
| keyword: "[skip-ci]" | |
| core-free: | |
| # CI tests for environments with core requirements in free versions | |
| name: Core - Free - Py${{matrix.python-version}} - ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| needs: [detect-ci-trigger, detect-ci-skip] | |
| if: | | |
| always() | |
| && ( | |
| (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') | |
| || needs.detect-ci-trigger.outputs.triggered == 'true' | |
| || needs.detect-ci-skip.outputs.skipped == 'false' | |
| || contains(github.event.pull_request.labels.*.name, 'release') | |
| ) | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| # timeout-minutes: 45 | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| python-version: ["3.9", "3.10", "3.11", "3.12"] | |
| os: ["ubuntu-latest", "macos-latest", "windows-latest"] | |
| steps: | |
| - name: Energy Estimation - Initialize | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v3 | |
| with: | |
| task: start-measurement | |
| company-uuid: ${{ secrets.CARBONDB_COMPANY_UUID }} | |
| project-uuid: ${{ secrets.CARBONDB_PROJECT_UUID }} | |
| machine-uuid: ${{ secrets.CARBONDB_MACHINE_UUID_PYTESTS_UPSTREAM }} | |
| continue-on-error: true | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Fetch all history for all branches and tags. | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-region: us-west-1 | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ga-ci-tests-argopy-01 | |
| - name: Set environment variables | |
| run: | | |
| echo "CONDA_ENV_FILE=ci/requirements/py${{matrix.python-version}}-core-free.yml" >> $GITHUB_ENV | |
| echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV | |
| echo "LOG_FILE=argopy-tests-Core-Free-Py${{matrix.python-version}}-${{matrix.os}}.log" >> $GITHUB_ENV | |
| - name: Setup Micromamba ${{ matrix.python-version }} | |
| uses: mamba-org/setup-micromamba@v1 | |
| with: | |
| environment-name: argopy-tests | |
| environment-file: ${{ env.CONDA_ENV_FILE }} | |
| init-shell: bash | |
| cache-environment: false | |
| # cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" | |
| create-args: >- | |
| python=${{matrix.python-version}} | |
| - name: Install argopy | |
| run: | | |
| python -m pip install --no-deps -e . | |
| - name: Version info | |
| run: | | |
| micromamba info | |
| micromamba list | |
| - name: Energy Estimation - Measure Tests Setup | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v3 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'setup core-free' | |
| continue-on-error: true | |
| - name: Lint with flake8 | |
| run: | | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Test with pytest | |
| run: | | |
| pytest -ra -v -s -c argopy/tests/pytest.ini --durations=10 \ | |
| --report-log output-${{ matrix.python-version }}-log.jsonl | |
| - name: Energy Estimation - Measure Tests Exec | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v3 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'pytest core-free' | |
| continue-on-error: true | |
| - name: Generate and publish a report on failure | |
| if: | | |
| failure() | |
| && steps.status.outcome == 'failure' | |
| && github.repository_owner == 'euroargodev' | |
| uses: xarray-contrib/issue-from-pytest-log@main | |
| with: | |
| log-path: output-${{ matrix.python-version }}-log.jsonl | |
| - name: Save tests log as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Argopy-Tests-Core-${{matrix.python-version}}-${{matrix.os}} | |
| path: output-${{ matrix.python-version }}-log.jsonl | |
| retention-days: 90 | |
| - name: Energy Estimation - Show Results | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v3 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: display-results | |
| send-data: true | |
| continue-on-error: true | |
| all-free: | |
| # CI tests for environments with all possible requirements in free versions | |
| name: All - Free - Py${{matrix.python-version}} - ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| needs: [detect-ci-trigger, detect-ci-skip] | |
| if: | | |
| always() | |
| && ( | |
| (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') | |
| || needs.detect-ci-trigger.outputs.triggered == 'true' | |
| || needs.detect-ci-skip.outputs.skipped == 'false' | |
| || contains(github.event.pull_request.labels.*.name, 'release') | |
| ) | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| # timeout-minutes: 45 | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| python-version: ["3.9", "3.10", "3.11", "3.12"] | |
| os: ["ubuntu-latest", "macos-latest", "windows-latest"] | |
| steps: | |
| - name: Energy Estimation - Initialize | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v3 | |
| with: | |
| task: start-measurement | |
| company-uuid: ${{ secrets.CARBONDB_COMPANY_UUID }} | |
| project-uuid: ${{ secrets.CARBONDB_PROJECT_UUID }} | |
| machine-uuid: ${{ secrets.CARBONDB_MACHINE_UUID_PYTESTS_UPSTREAM }} | |
| continue-on-error: true | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Fetch all history for all branches and tags. | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-region: us-west-1 | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ga-ci-tests-argopy-01 | |
| - name: Set environment variables | |
| run: | | |
| echo "CONDA_ENV_FILE=ci/requirements/py${{matrix.python-version}}-all-free.yml" >> $GITHUB_ENV | |
| echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV | |
| - name: Setup Micromamba ${{ matrix.python-version }} | |
| uses: mamba-org/setup-micromamba@v1 | |
| with: | |
| environment-name: argopy-tests | |
| environment-file: ${{ env.CONDA_ENV_FILE }} | |
| init-shell: bash | |
| cache-environment: false | |
| # cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" | |
| create-args: >- | |
| python=${{matrix.python-version}} | |
| pytest-reportlog | |
| - name: Install argopy | |
| run: | | |
| python -m pip install --no-deps -e . | |
| - name: Version info | |
| run: | | |
| micromamba info | |
| micromamba list | |
| - name: Energy Estimation - Measure Tests Setup | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v3 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'setup all-free' | |
| continue-on-error: true | |
| - name: Lint with flake8 | |
| run: | | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Test with pytest | |
| run: | | |
| pytest -ra -v -s -c argopy/tests/pytest.ini --durations=10 \ | |
| --report-log output-${{ matrix.python-version }}-log.jsonl | |
| - name: Energy Estimation - Measure Tests Exec | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v3 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'pytest all-free' | |
| continue-on-error: true | |
| - name: Generate and publish a report on failure | |
| if: | | |
| failure() | |
| && steps.status.outcome == 'failure' | |
| && github.repository_owner == 'euroargodev' | |
| uses: xarray-contrib/issue-from-pytest-log@main | |
| with: | |
| log-path: output-${{ matrix.python-version }}-log.jsonl | |
| - name: Save tests log as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Argopy-Tests-All-${{matrix.python-version}}-${{matrix.os}} | |
| path: output-${{ matrix.python-version }}-log.jsonl | |
| retention-days: 90 | |
| - name: Energy Estimation - Show Results | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v3 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: display-results | |
| send-data: true | |
| continue-on-error: true |