Tweak specs to try to avoid flakiness #9587
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: System Tests | |
| on: | |
| push: | |
| branches: | |
| - "**" | |
| workflow_dispatch: {} | |
| schedule: | |
| - cron: '00 04 * * 2-6' | |
| env: | |
| REGISTRY: ghcr.io | |
| REPO: ghcr.io/datadog/dd-trace-rb | |
| ST_REF: main | |
| FORCE_TESTS: | |
| jobs: | |
| build-harness: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - name: runner | |
| internal: system_tests/runner:latest | |
| - name: agent | |
| internal: system_tests/agent:latest | |
| - name: proxy | |
| internal: datadog/system-tests:proxy-v1 | |
| runs-on: ubuntu-latest | |
| name: Build (${{ matrix.image.name }}) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| ref: ${{ env.ST_REF }} | |
| - name: Pull released image | |
| run: | | |
| if docker pull ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest; then | |
| docker tag ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest ${{ matrix.image.internal }} | |
| fi | |
| - name: Build image | |
| run: | | |
| cache_from=() | |
| for tag in latest; do | |
| cache_from+=(--cache-from "${{ env.REPO }}/system-tests/${{ matrix.image.name }}:${tag}") | |
| done | |
| echo "cache args: ${cache_from[*]}" | |
| ./build.sh --images ${{ matrix.image.name }} --docker --extra-docker-args "${cache_from[*]}" | |
| - name: List images | |
| run: | | |
| docker image list | |
| - name: Log in to the Container registry | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - name: Tag image for CI run | |
| run: | |
| docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Push image for CI run | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Tag image for commit | |
| run: | |
| docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:g${{ github.sha }} | |
| - name: Push image for commit | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:g${{ github.sha }} | |
| - name: Tag image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | |
| docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest | |
| - name: Push image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest | |
| build-apps: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - weblog | |
| library: | |
| - name: ruby | |
| repository: DataDog/dd-trace-rb | |
| path: dd-trace-rb | |
| app: | |
| - rack | |
| # - sinatra14 # DEV-2.0: Uncomment after https://github.com/DataDog/system-tests/pull/1882 is merged | |
| - sinatra20 | |
| - sinatra21 | |
| - rails50 | |
| - rails51 | |
| - rails52 | |
| - rails60 | |
| - rails61 | |
| - rails70 | |
| - rails71 | |
| runs-on: ubuntu-latest | |
| name: Build (${{ matrix.app }}) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| ref: ${{ env.ST_REF }} | |
| - name: Checkout ${{ matrix.library.repository }} | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: '${{ matrix.library.repository }}' | |
| path: 'binaries/${{ matrix.library.path }}' | |
| fetch-depth: 2 | |
| - name: Pull released image | |
| run: | | |
| if docker pull ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest; then | |
| docker tag ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest system_tests/${{ matrix.image }}:latest | |
| fi | |
| parents="$(cd 'binaries/${{ matrix.library.path }}' && git rev-list --parents -n 1 ${{ github.sha }})" | |
| for sha in $parents; do | |
| docker pull "${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${sha}" || true | |
| done | |
| - name: Log in to the Container registry | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - name: Build | |
| run: | | |
| cache_from=() | |
| for tag in latest; do | |
| cache_from+=(--cache-from "${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:${tag}") | |
| done | |
| parents="$(cd 'binaries/${{ matrix.library.path }}' && git rev-list --parents -n 1 ${{ github.sha }})" | |
| for sha in $parents; do | |
| cache_from+=(--cache-from ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${sha}) | |
| done | |
| echo "cache args: ${cache_from[*]}" | |
| ./build.sh --library ${{ matrix.library.name }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }} --extra-docker-args "${cache_from[*]}" | |
| - name: Tag image for CI run | |
| run: | |
| docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Push image for CI run | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Tag image for commit | |
| run: | |
| docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${{ github.sha }} | |
| - name: Push image for commit | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${{ github.sha }} | |
| - name: Tag image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | |
| docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest | |
| - name: Push image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest | |
| test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| library: | |
| - ruby | |
| app: | |
| - rack | |
| # - sinatra14 # DEV-2.0: Uncomment after https://github.com/DataDog/system-tests/pull/1882 is merged | |
| - sinatra20 | |
| - sinatra21 | |
| - rails50 | |
| - rails51 | |
| - rails52 | |
| - rails60 | |
| - rails61 | |
| - rails70 | |
| - rails71 | |
| scenario: | |
| - DEFAULT | |
| - APPSEC_DISABLED | |
| - APPSEC_BLOCKING_FULL_DENYLIST | |
| - APPSEC_REQUEST_BLOCKING | |
| include: | |
| - library: ruby | |
| app: rack | |
| scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD | |
| - library: ruby | |
| app: rack | |
| scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES | |
| - library: ruby | |
| app: rack | |
| scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE | |
| - library: ruby | |
| app: rack | |
| scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_CUSTOM_RULES | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_MISSING_RULES | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_CORRUPTED_RULES | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_LOW_WAF_TIMEOUT | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_CUSTOM_OBFUSCATION | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_RATE_LIMITER | |
| - library: ruby | |
| app: rails70 | |
| scenario: APPSEC_AUTO_EVENTS_EXTENDED | |
| - library: ruby | |
| app: rails70 | |
| scenario: APPSEC_API_SECURITY | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_RULES_MONITORING_WITH_ERRORS | |
| - library: ruby | |
| app: rack | |
| scenario: SAMPLING | |
| - library: ruby | |
| app: rack | |
| scenario: PROFILING | |
| - library: ruby | |
| app: rack | |
| scenario: TELEMETRY_APP_STARTED_PRODUCTS_DISABLED | |
| - library: ruby | |
| app: rack | |
| scenario: TELEMETRY_DEPENDENCY_LOADED_TEST_FOR_DEPENDENCY_COLLECTION_DISABLED | |
| - library: ruby | |
| app: rack | |
| scenario: TELEMETRY_LOG_GENERATION_DISABLED | |
| - library: ruby | |
| app: rack | |
| scenario: TELEMETRY_METRIC_GENERATION_DISABLED | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-harness | |
| - build-apps | |
| name: Test (${{ matrix.app }}, ${{ matrix.scenario }}) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| ref: ${{ env.ST_REF }} | |
| - name: Pull runner image | |
| run: | | |
| docker pull ${{ env.REPO }}/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }} | |
| docker tag ${{ env.REPO }}/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }} system_tests/runner:latest | |
| - name: Pull proxy image | |
| run: | | |
| docker pull ${{ env.REPO }}/system-tests/proxy:gha${{ github.run_id }}-g${{ github.sha }} | |
| docker tag ${{ env.REPO }}/system-tests/proxy:gha${{ github.run_id }}-g${{ github.sha }} datadog/system-tests:proxy-v1 | |
| - name: Pull agent image | |
| run: | | |
| docker pull ${{ env.REPO }}/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} | |
| docker tag ${{ env.REPO }}/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} system_tests/agent:latest | |
| - name: Pull app image | |
| run: | | |
| docker pull ${{ env.REPO }}/system-tests/${{ matrix.library }}/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| docker tag ${{ env.REPO }}/system-tests/${{ matrix.library }}/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} system_tests/weblog:latest | |
| - name: List images | |
| run: | | |
| docker image list | |
| - name: Run scenario | |
| run: | | |
| ./run.sh ++docker ${{ matrix.scenario }} ${{ env.FORCE_TESTS }} | |
| env: | |
| DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }} | |
| - name: Archive logs | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ always() }} | |
| with: | |
| name: system-tests-${{ matrix.library }}-${{ matrix.app }}-${{ matrix.scenario }}-logs-gha${{ github.run_id }}-g${{ github.sha }} | |
| path: logs* | |
| aggregate: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| library: | |
| - ruby | |
| app: | |
| - rack | |
| # - sinatra14 # DEV-2.0: Uncomment after https://github.com/DataDog/system-tests/pull/1882 is merged | |
| - sinatra20 | |
| - sinatra21 | |
| - rails50 | |
| - rails51 | |
| - rails52 | |
| - rails60 | |
| - rails61 | |
| - rails70 | |
| - rails71 | |
| runs-on: ubuntu-latest | |
| needs: | |
| - test | |
| if: ${{ always() }} | |
| name: Aggregate (${{ matrix.app }}) | |
| steps: | |
| - name: Setup python 3.9 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| ref: ${{ env.ST_REF }} | |
| - name: Retrieve logs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: system-tests-${{ matrix.library }}-${{ matrix.app }}-*-logs-gha${{ github.run_id }}-g${{ github.sha }} | |
| merge-multiple: true | |
| path: . | |
| - name: Print fancy log report | |
| run: | | |
| find logs* | |
| python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY | |
| cleanup: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - runner | |
| - agent | |
| - weblog-rack | |
| # - weblog-sinatra14 # DEV-2.0: Uncomment after https://github.com/DataDog/system-tests/pull/1882 is merged | |
| - weblog-sinatra20 | |
| - weblog-sinatra21 | |
| - weblog-rails50 | |
| - weblog-rails51 | |
| - weblog-rails52 | |
| - weblog-rails60 | |
| - weblog-rails61 | |
| - weblog-rails70 | |
| runs-on: ubuntu-latest | |
| needs: | |
| - test | |
| if: ${{ always() }} | |
| name: Cleanup (${{ matrix.image }}) | |
| steps: | |
| - name: Log in to the Container registry | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - uses: actions/delete-package-versions@v4 | |
| with: | |
| package-version-ids: 'gha${{ github.run_id }}-g${{ github.sha }}' | |
| package-name: 'system-tests/${{ matrix.image }}' | |
| package-type: 'container' | |
| continue-on-error: true |