E2E Cross-version Tests #15
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Cross-version Tests | |
| on: | |
| schedule: | |
| - cron: '0 9 * * 1-5' | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| jobs: | |
| files-changed: | |
| name: Check which files changed | |
| # don't run this workflow on a cron for forks | |
| if: ${{ github.event_name != 'schedule' || github.repository == 'metabase/metabase' }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 3 | |
| outputs: | |
| e2e_cross_version: ${{ steps.changes.outputs.e2e_cross_version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Test which files changed | |
| uses: dorny/paths-filter@v3.0.0 | |
| id: changes | |
| with: | |
| token: ${{ github.token }} | |
| filters: .github/file-paths.yaml | |
| cross-version-testing: | |
| needs: files-changed | |
| if: ${{ needs.files-changed.outputs.e2e_cross_version == 'true' || github.event_name == 'schedule'}} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 20 | |
| strategy: | |
| matrix: | |
| version: [ | |
| # Major OSS upgrade | |
| { source: v0.42.2, target: v0.48.7 }, | |
| # OSS upgrade | |
| { source: v0.41.3.1, target: v0.42.2 }, | |
| # EE upgrade | |
| { source: v1.43.4, target: v1.44.0 }, | |
| # EE downgrade | |
| { source: v1.42.0, target: v1.41.3.1 }, | |
| # Cross-edition upgrade | |
| { source: v0.41.3.1, target: v1.42.2 } | |
| ] | |
| fail-fast: false | |
| env: | |
| CROSS_VERSION_SOURCE: ${{ matrix.version.source }} | |
| CROSS_VERSION_TARGET: ${{ matrix.version.target }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set source Docker image to OSS | |
| run: | | |
| echo "DOCKER_SOURCE_IMAGE=metabase/metabase:${{ matrix.version.source }}" >> $GITHUB_ENV | |
| if: startsWith(matrix.version.source, 'v0') | |
| - name: Set source Docker image to EE | |
| run: | | |
| echo "DOCKER_SOURCE_IMAGE=metabase/metabase-enterprise:${{ matrix.version.source }}" >> $GITHUB_ENV | |
| if: startsWith(matrix.version.source, 'v1') | |
| - name: Prepare front-end environment | |
| uses: ./.github/actions/prepare-frontend | |
| - name: Start Metabase ${{ matrix.version.source }} | |
| run: | | |
| docker run -d \ | |
| -v $PWD/my-metabase:/metabase.db \ | |
| -p 3000:3000 \ | |
| -e CYPRESS_ALL_FEATURES_TOKEN=${{ secrets.ENTERPRISE_TOKEN }} \ | |
| -e CYPRESS_NO_FEATURES_TOKEN=${{ secrets.E2E_STARTER_TOKEN }} \ | |
| --name metabase-${{ matrix.version.source }} \ | |
| ${{ env.DOCKER_SOURCE_IMAGE }} | |
| - name: Wait for Metabase to start on port 3000 | |
| run: while ! curl -s 'http://localhost:3000/api/health' | grep '{"status":"ok"}'; do sleep 1; done | |
| - name: Prepare Cypress environment | |
| id: cypress-prep | |
| uses: ./.github/actions/prepare-cypress | |
| - name: Run Cypress on the source | |
| run: | | |
| yarn cypress run \ | |
| --browser ${{ steps.cypress-prep.outputs.chrome-path }} \ | |
| --config-file e2e/test/scenarios/cross-version/source/shared/cross-version-source.config.js \ | |
| --spec e2e/test/scenarios/cross-version/source/**/*.cy.spec.js | |
| - name: Stop Metabase ${{ matrix.version.source }} | |
| run: docker stop metabase-${{ matrix.version.source }} | |
| - name: Set target Docker image to OSS | |
| run: | | |
| echo "DOCKER_TARGET_IMAGE=metabase/metabase:${{ matrix.version.source }}" >> $GITHUB_ENV | |
| if: startsWith(matrix.version.source, 'v0') | |
| - name: Set target Docker image to EE | |
| run: | | |
| echo "DOCKER_TARGET_IMAGE=metabase/metabase-enterprise:${{ matrix.version.source }}" >> $GITHUB_ENV | |
| if: startsWith(matrix.version.source, 'v1') | |
| - name: Start Metabase ${{ matrix.version.target }} | |
| run: | | |
| docker run -d \ | |
| -v $PWD/my-metabase:/metabase.db \ | |
| -p 3001:3000 \ | |
| -e CYPRESS_ALL_FEATURES_TOKEN=${{ secrets.ENTERPRISE_TOKEN }} \ | |
| -e CYPRESS_NO_FEATURES_TOKEN=${{ secrets.E2E_STARTER_TOKEN }} \ | |
| --name metabase-${{ matrix.version.target }} \ | |
| ${{ env.DOCKER_TARGET_IMAGE }} | |
| - name: Wait for Metabase to start on port 3001 | |
| run: while ! curl -s 'http://localhost:3001/api/health' | grep '{"status":"ok"}'; do sleep 1; done | |
| - name: Run Cypress on the target | |
| run: | | |
| yarn cypress run \ | |
| --browser ${{ steps.cypress-prep.outputs.chrome-path }} \ | |
| --config-file e2e/test/scenarios/cross-version/target/shared/cross-version-target.config.js \ | |
| --spec e2e/test/scenarios/cross-version/target/**/*.cy.spec.js | |
| - name: Upload Cypress Artifacts upon failure | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: cypress-artifacts-${{ matrix.version.source}}-to-${{ matrix.version.target }} | |
| path: | | |
| ./cypress | |
| if-no-files-found: ignore | |
| notify-on-failure: | |
| needs: cross-version-testing | |
| if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name == 'schedule' }} | |
| name: Cross-version test failure | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: lts/Hydrogen # 18.x.x | |
| - run: npm install @slack/web-api | |
| - name: notify slack of failure | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | # js | |
| const FAILED_RUN_URL = "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"; | |
| const REF_NAME = "${{ github.ref_name }}"; | |
| // notify slack of failure | |
| const { WebClient } = require('@slack/web-api'); | |
| const slack = new WebClient('${{ secrets.SLACK_BOT_TOKEN }}'); | |
| await slack.chat.postMessage({ | |
| channel: 'engineering-ci', | |
| text: 'Cross-version Tests Have Failed', | |
| blocks: [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": `:broken_heart: Cross-version tests are failing on ${REF_NAME}`, | |
| "emoji": true, | |
| } | |
| }, | |
| ], | |
| attachments: [{ | |
| color: "#BF40BF", | |
| blocks: [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": `See <${FAILED_RUN_URL}|failing run details>.` | |
| } | |
| }, | |
| ] | |
| }] | |
| }); |