push-tests.yml

name: Push Tests

on:
  pull_request:
    branches:
      - "*"
  push:
    branches:
      - "main"

  workflow_call:

env:
  DEPLOYMENT_STAGE: test
  # Force using BuildKit instead of normal Docker, required so that metadata
  # is written/read to allow us to use layers of previous builds as cache.
  DOCKER_BUILDKIT: 1
  COMPOSE_DOCKER_CLI_BUILD: 1
  DOCKER_REPO: ${{ secrets.ECR_REPO }}/

permissions:
  id-token: write
  contents: read

jobs:
  lint:
    if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
    runs-on: ubuntu-24.04
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-west-2
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          role-duration-seconds: 1800
      - name: Login to ECR
        uses: docker/login-action@v3
        with:
          registry: ${{ secrets.ECR_REPO }}
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - uses: actions/setup-python@v5

      - name: check backend
        uses: pre-commit/action@v3.0.1
      - uses: actions/setup-node@v4
        with:
          node-version-file: "frontend/.nvmrc"
          cache: "npm"
          cache-dependency-path: "frontend/package-lock.json"
      - name: check frontend
        run: |
          cd frontend
          npm ci
          cp src/configs/local.js src/configs/configs.js
          npm run lint
      - uses: 8398a7/action-slack@v3
        with:
          status: ${{ job.status }}
          fields: repo,commit,author,eventName,workflow,job,mention
          mention: "here"
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
        if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')

  backend-unit-test:
    if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-west-2
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          role-duration-seconds: 1800
      - name: Login to ECR
        uses: docker/login-action@v3
        with:
          registry: ${{ secrets.ECR_REPO }}
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v45.0.5
        with:
          files: |
            Dockerfile*
            **/Dockerfile
            requirements*.txt
            **/requirements*.txt
      - name: Check if containers need to be rebuilt
        if: steps.changed-files.outputs.any_changed == 'true'
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-rebuild-backend
      - name: Run unit tests
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-unit-test-backend

      - name: Upload coverage results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: coverage-backennd
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
          retention-days: 3
          include-hidden-files: true

      - name: Upload Allure results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: allure-results-backed
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
          retention-days: 20

      - uses: 8398a7/action-slack@v3.16.2
        with:
          status: ${{ job.status }}
          fields: repo,commit,author,eventName,workflow,job,mention
          mention: "here"
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
        if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')

  backend-wmg-unit-test:
    if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-west-2
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          role-duration-seconds: 1800
      - name: Login to ECR
        uses: docker/login-action@v3
        with:
          registry: ${{ secrets.ECR_REPO }}
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v45.0.5
        with:
          files: |
            Dockerfile*
            **/Dockerfile
            requirements*.txt
            **/requirements*.txt
      - name: Check if containers need to be rebuilt
        if: steps.changed-files.outputs.any_changed == 'true'
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-rebuild-wmg-backend
      - name: Run unit tests
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-unit-test-wmg-backend

      - name: Upload coverage results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: coverage-wmg-backend
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
          retention-days: 3
          include-hidden-files: true

      - name: Upload Allure results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: allure-results-wmg-backend
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
          retention-days: 20

      - uses: 8398a7/action-slack@v3.16.2
        with:
          status: ${{ job.status }}
          fields: repo,commit,author,eventName,workflow,job,mention
          mention: "here"
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
        if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')

  backend-de-unit-test:
    if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-west-2
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          role-duration-seconds: 1800
      - name: Login to ECR
        uses: docker/login-action@v3
        with:
          registry: ${{ secrets.ECR_REPO }}
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v45.0.5
        with:
          files: |
            Dockerfile*
            **/Dockerfile
            requirements*.txt
            **/requirements*.txt
      - name: Check if containers need to be rebuilt
        if: steps.changed-files.outputs.any_changed == 'true'
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-rebuild-de-backend
      - name: Run unit tests
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-unit-test-de-backend

      - name: Upload coverage results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: coverage-de-backend
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
          retention-days: 3
          include-hidden-files: true

      - name: Upload Allure results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: allure-results-de-backend
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
          retention-days: 20

      - uses: 8398a7/action-slack@v3.16.2
        with:
          status: ${{ job.status }}
          fields: repo,commit,author,eventName,workflow,job,mention
          mention: "here"
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
        if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')

  processing-unit-test:
    if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-west-2
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          role-duration-seconds: 1800
      - name: Login to ECR
        uses: docker/login-action@v3
        with:
          registry: ${{ secrets.ECR_REPO }}
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v45.0.5
        with:
          files: |
            Dockerfile*
            **/Dockerfile*
            requirements*.txt
            **/requirements*.txt
      - name: Check if containers need to be rebuilt
        if: steps.changed-files.outputs.any_changed == 'true'
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-rebuild-processing
      - name: Run tests in docker-compose
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-unit-test-processing

      - name: Upload coverage results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: coverage-processing
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
          retention-days: 3
          include-hidden-files: true

      - name: Upload Allure results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: allure-results-processing
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
          retention-days: 20
          include-hidden-files: true

      - uses: 8398a7/action-slack@v3.16.2
        with:
          status: ${{ job.status }}
          fields: repo,commit,author,eventName,workflow,job,mention
          mention: "here"
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
        if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')

  wmg-processing-unit-test:
    if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-west-2
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          role-duration-seconds: 1800
      - name: Login to ECR
        uses: docker/login-action@v3
        with:
          registry: ${{ secrets.ECR_REPO }}
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v45.0.5
        with:
          files: |
            Dockerfile*
            **/Dockerfile*
            requirements*.txt
            **/requirements*.txt
      - name: Check if containers need to be rebuilt
        if: steps.changed-files.outputs.any_changed == 'true'
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-rebuild-wmg-processing
      - name: Run tests in docker-compose
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-unit-test-wmg-processing
      - name: Upload coverage results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: coverage-wmg-processing
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
          retention-days: 3
          include-hidden-files: true
      - name: Upload Allure results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: allure-results-wmg-processing
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
          retention-days: 20
      - uses: 8398a7/action-slack@v3.16.2
        with:
          status: ${{ job.status }}
          fields: repo,commit,author,eventName,workflow,job,mention
          mention: "here"
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
        if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')

  cellguide-pipeline-unit-test:
    if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-west-2
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          role-duration-seconds: 1800
      - name: Login to ECR
        uses: docker/login-action@v3
        with:
          registry: ${{ secrets.ECR_REPO }}
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v45.0.5
        with:
          files: |
            Dockerfile*
            **/Dockerfile*
            requirements*.txt
            **/requirements*.txt
      - name: Check if containers need to be rebuilt
        if: steps.changed-files.outputs.any_changed == 'true'
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-rebuild-cellguide-pipeline
      - name: Run tests in docker-compose
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-unit-test-cellguide-pipeline
      - name: Upload coverage results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: coverage-cellguide-pipeline
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
          retention-days: 3
          include-hidden-files: true
      - name: Upload Allure results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: allure-results-cellguide-pipeline
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
          retention-days: 20
      - uses: 8398a7/action-slack@v3.16.2
        with:
          status: ${{ job.status }}
          fields: repo,commit,author,eventName,workflow,job,mention
          mention: "here"
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
        if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')

  cxg-admin-unit-test:
    if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-west-2
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          role-duration-seconds: 1800
      - name: Login to ECR
        uses: docker/login-action@v3
        with:
          registry: ${{ secrets.ECR_REPO }}
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v45.0.5
        with:
          files: |
            Dockerfile*
            **/Dockerfile
            requirements*.txt
            **/requirements*.txt
      - name: Check if containers need to be rebuilt
        if: steps.changed-files.outputs.any_changed == 'true'
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-rebuild-backend
      - name: Run unit tests
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-unit-test-cxg-admin

      - name: Upload coverage results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: coverage-cxg-admin
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
          retention-days: 3
          include-hidden-files: true

      - name: Upload Allure results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: allure-results-cxg-admin
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
          retention-days: 20

      - uses: 8398a7/action-slack@v3.16.2
        with:
          status: ${{ job.status }}
          fields: repo,commit,author,eventName,workflow,job,mention
          mention: "here"
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
        if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')

  backend-integration-test:
    if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-west-2
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          role-duration-seconds: 1800
      - name: Login to ECR
        uses: docker/login-action@v3
        with:
          registry: ${{ secrets.ECR_REPO }}
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v45.0.5
        with:
          files: |
            Dockerfile*
            **/Dockerfile
            requirements*.txt
            **/requirements*.txt
      - name: Check if containers need to be rebuilt
        if: steps.changed-files.outputs.any_changed == 'true'
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-rebuild-backend
      - name: Run tests against postgres instance
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make local-integration-test-backend
      - name: Upload coverage results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: coverage-backend-integration
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
          retention-days: 3
          include-hidden-files: true

      - name: Upload Allure results as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: allure-results-backend-integration
          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
          retention-days: 20
      - uses: 8398a7/action-slack@v3.16.2
        with:
          status: ${{ job.status }}
          fields: repo,commit,author,eventName,workflow,job,mention
          mention: "here"
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
        if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')

  submit-codecoverage:
    if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
    needs:
      - backend-unit-test
      - backend-integration-test
      - backend-wmg-unit-test
      - wmg-processing-unit-test
      - processing-unit-test
      - cellguide-pipeline-unit-test
    runs-on: ubuntu-24.04
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-west-2
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          role-duration-seconds: 1800
      - name: Login to ECR
        uses: docker/login-action@v3
        with:
          registry: ${{ secrets.ECR_REPO }}
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - uses: actions/download-artifact@v4
        with:
          pattern: coverage-* # Download all coverage artifacts
          merge-multiple: true
          path: .
      - name: coverage report
        run: |
          echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
          make coverage/report-xml
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v5
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          env_vars: OS,PYTHON
          files: ./coverage.xml
          flags: unittests
          name: codecov-umbrella

  publish-test-report:
    if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
    needs:
      - backend-unit-test
      - backend-wmg-unit-test
      - backend-integration-test
      - wmg-processing-unit-test
      - processing-unit-test
      - cellguide-pipeline-unit-test
    runs-on: ubuntu-24.04
    permissions:
      contents: write
    steps:
      - uses: actions/download-artifact@v4
        with:
          pattern: allure-results-* # Download all allure-results artifacts
          path: allure-results
          merge-multiple: true

      # Checkout gh-pages branch and get Allure result history
      - name: Get Allure history
        uses: actions/checkout@v4
        if: always()
        continue-on-error: true
        with:
          ref: gh-pages
          path: gh-pages
          fetch-depth: 2

      # Create test results history as an Allure report
      - name: Create test results history as an Allure report
        uses: simple-elf/allure-report-action@master
        if: always()
        id: allure-report
        with:
          allure_results: allure-results
          gh_pages: gh-pages
          allure_report: allure-report
          allure_history: allure-history

      # Deploy Allure report to github pages
      - name: Deploy report to Github Pages
        if: always()
        uses: peaceiris/actions-gh-pages@v4
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_branch: gh-pages
          publish_dir: allure-history

      # Output the link to the github page for this current run
      - name: Post the link to the report in summary
        run: "echo 'Test Results Dashboard (Run #${{ github.run_number }}): https://chanzuckerberg.github.io/single-cell-data-portal/${{ github.run_number }}' >> $GITHUB_STEP_SUMMARY"