Add a fetcher for AWS S3 Argo GDAC data #1154
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI tests | |
| # CI tests ran every time new code is pushed to the repo | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| branches-ignore: | |
| - 'dependabot*' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| detect-ci-skip: | |
| name: detect ci skip | |
| runs-on: ubuntu-latest | |
| if: | | |
| github.repository == 'euroargodev/argopy' | |
| && (github.event_name == 'push' || github.event_name == 'pull_request') | |
| outputs: | |
| skipped: ${{ steps.detect-skip.outputs.trigger-found }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| - uses: xarray-contrib/ci-trigger@v1.2 | |
| id: detect-skip | |
| with: | |
| keyword: "[skip-ci]" | |
| core-pinned: | |
| # CI tests for environments with core requirements in pinned versions | |
| # All commits must pass this | |
| name: Core - Pinned - Py${{matrix.python-version}} - ${{ matrix.os }} | |
| needs: detect-ci-skip | |
| runs-on: ${{ matrix.os }} | |
| if: needs.detect-ci-skip.outputs.skipped == 'false' | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| continue-on-error: ${{ matrix.experimental }} | |
| # timeout-minutes: 45 | |
| strategy: | |
| max-parallel: 12 | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11"] | |
| os: ["ubuntu-latest", "windows-latest", "macos-latest"] | |
| experimental: [false] | |
| steps: | |
| - name: Energy Estimation - Initialize | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 | |
| with: | |
| task: start-measurement | |
| company-uuid: ${{ secrets.CARBONDB_COMPANY_UUID }} | |
| project-uuid: ${{ secrets.CARBONDB_PROJECT_UUID }} | |
| machine-uuid: ${{ secrets.CARBONDB_MACHINE_UUID_PYTESTS }} | |
| continue-on-error: true | |
| - uses: actions/checkout@v4 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-region: us-west-1 | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ga-ci-tests-argopy-01 | |
| - name: Set environment variables | |
| run: | | |
| echo "CONDA_ENV_FILE=ci/requirements/py${{matrix.python-version}}-core-pinned.yml" >> $GITHUB_ENV | |
| echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV | |
| echo "LOG_FILE=argopy-tests-Core-Pinned-Py${{matrix.python-version}}-${{matrix.os}}.log" >> $GITHUB_ENV | |
| - name: Setup Micromamba ${{ matrix.python-version }} | |
| uses: mamba-org/setup-micromamba@v2 | |
| with: | |
| micromamba-version: '1.5.10-0' | |
| environment-name: argopy-tests | |
| environment-file: ${{ env.CONDA_ENV_FILE }} | |
| init-shell: bash | |
| cache-environment: true | |
| cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" | |
| create-args: >- | |
| python=${{matrix.python-version}} | |
| - name: Install argopy | |
| run: | | |
| python -m pip install --no-deps -e . | |
| - name: Version info | |
| run: | | |
| micromamba info | |
| micromamba list | |
| - name: Energy Estimation - Measure Tests Setup | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'setup core-pinned' | |
| continue-on-error: true | |
| - name: Lint with flake8 | |
| run: | | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Test with pytest | |
| run: | | |
| pytest -ra -v -s -c argopy/tests/pytest.ini --durations=10 --cov=./ --cov-config=.coveragerc --cov-report xml:cov.xml --cov-report term-missing --log-file=${{env.LOG_FILE}} --junitxml=junit.xml -o junit_family=legacy | |
| - name: Energy Estimation - Measure Tests Exec | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'pytest core-pinned' | |
| continue-on-error: true | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4.6.0 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: ./cov.xml | |
| flags: unittests, core, pinned, py${{matrix.python-version}} | |
| name: codecov-github | |
| fail_ci_if_error: false | |
| env_vars: RUNNER_OS,PYTHON_VERSION | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/test-results-action@v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Save tests log as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Argopy-Tests-Core-${{matrix.python-version}}-${{matrix.os}} | |
| path: ${{env.LOG_FILE}} | |
| retention-days: 90 | |
| - name: Energy Estimation - Show Results | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: display-results | |
| send-data: true | |
| continue-on-error: true | |
| all-pinned: | |
| # CI tests for environments with all requirements in pinned versions | |
| # All commits must pass this | |
| name: All - Pinned - Py${{matrix.python-version}} - ${{ matrix.os }} | |
| needs: detect-ci-skip | |
| runs-on: ${{ matrix.os }} | |
| if: needs.detect-ci-skip.outputs.skipped == 'false' | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| continue-on-error: ${{ matrix.experimental }} | |
| # timeout-minutes: 45 | |
| strategy: | |
| max-parallel: 12 | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11"] | |
| os: ["ubuntu-latest", "macos-latest", "windows-latest"] | |
| experimental: [false] | |
| steps: | |
| - name: Energy Estimation - Initialize | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 | |
| with: | |
| task: start-measurement | |
| company-uuid: ${{ secrets.CARBONDB_COMPANY_UUID }} | |
| project-uuid: ${{ secrets.CARBONDB_PROJECT_UUID }} | |
| machine-uuid: ${{ secrets.CARBONDB_MACHINE_UUID_PYTESTS }} | |
| continue-on-error: true | |
| - uses: actions/checkout@v4 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-region: us-west-1 | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ga-ci-tests-argopy-01 | |
| - name: Set environment variables | |
| run: | | |
| echo "CONDA_ENV_FILE=ci/requirements/py${{matrix.python-version}}-all-pinned.yml" >> $GITHUB_ENV | |
| echo "PYTHON_VERSION=${{matrix.python-version}}" >> $GITHUB_ENV | |
| echo "LOG_FILE=argopy-tests-All-Pinned-Py${{matrix.python-version}}-${{matrix.os}}.log" >> $GITHUB_ENV | |
| - name: Setup Micromamba ${{ matrix.python-version }} | |
| uses: mamba-org/setup-micromamba@v2 | |
| with: | |
| micromamba-version: '1.5.10-0' | |
| environment-name: argopy-tests | |
| environment-file: ${{ env.CONDA_ENV_FILE }} | |
| init-shell: bash | |
| cache-environment: true | |
| cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" | |
| create-args: >- | |
| python=${{matrix.python-version}} | |
| - name: Install argopy | |
| run: | | |
| python -m pip install --no-deps -e . | |
| - name: Version info | |
| run: | | |
| micromamba info | |
| micromamba list | |
| - name: Energy Estimation - Measure Tests Setup | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'setup all-pinned' | |
| continue-on-error: true | |
| - name: Lint with flake8 | |
| run: | | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Test with pytest | |
| run: | | |
| pytest -ra -v -s -c argopy/tests/pytest.ini --durations=10 --cov=./ --cov-config=.coveragerc --cov-report xml:cov.xml --cov-report term-missing --log-file=${{env.LOG_FILE}} --junitxml=junit.xml -o junit_family=legacy | |
| - name: Energy Estimation - Measure Tests Exec | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'pytest all-pinned' | |
| continue-on-error: true | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4.6.0 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: ./cov.xml | |
| flags: unittests, all, pinned, py${{matrix.python-version}} | |
| name: codecov-github | |
| fail_ci_if_error: false | |
| env_vars: RUNNER_OS,PYTHON_VERSION | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/test-results-action@v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload tests log to GA | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Argopy-Tests-All-${{matrix.python-version}}-${{matrix.os}} | |
| path: ${{env.LOG_FILE}} | |
| retention-days: 90 | |
| - name: Energy Estimation - Show Results | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: display-results | |
| send-data: true | |
| continue-on-error: true |