Update ci-cd.yml

Author: Zahed-Riyaz

.github/workflows/ci-cd.yml

@@ -13,6 +13,8 @@ env:
   DISPLAY: :99.0
   PYTHON_VERSION: '3.9'
   AWSCLI_VERSION: '1.18.66'
+  REGISTRY: ${{ secrets.DOCKER_USERNAME }}
+  IMAGE_TAG: ${{ github.sha }}
 
 jobs:
   quality-check:
@@ -24,7 +26,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Python and dependencies
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip'
@@ -64,9 +66,11 @@ jobs:
         run: |
           docker compose run -e DJANGO_SETTINGS_MODULE=settings.dev django python manage.py makemigrations --check --dry-run
 
-  build-docker:
-    name: Build Docker Images
+  build-and-push:
+    name: Build & Push Docker Images
     runs-on: ubuntu-latest
+    outputs:
+      image-digest: ${{ steps.build.outputs.digest }}
     steps:
       - uses: actions/checkout@v4
 
@@ -75,49 +79,58 @@ jobs:
         with:
           version: latest
 
-      - name: Setup Docker environment
-        run: |
-          docker buildx create --use
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-buildx-
 
       - name: Docker login
-        if: github.event_name == 'push'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Build Docker images
+      - name: Build and push Docker images
+        id: build
         run: |
+          # Build and push images with unique tags
+          docker buildx build \
+            --cache-from type=local,src=/tmp/.buildx-cache \
+            --cache-to type=local,dest=/tmp/.buildx-cache-new,mode=max \
+            --push \
+            --tag ${{ env.REGISTRY }}/evalai:${{ env.IMAGE_TAG }} \
+            --tag ${{ env.REGISTRY }}/evalai:latest \
+            .
+          
+          # Build compose services and push
+          export COMPOSE_DOCKER_CLI_BUILD=1
+          export DOCKER_BUILDKIT=1
+          
           docker compose --profile worker_py3_7 --profile worker_py3_8 --profile worker_py3_9 --profile statsd build ${{ env.COMPOSE_BAKE_ARGS }}
-
-      - name: Save Docker images
-        run: |
-          docker save $(docker images --format "table {{.Repository}}:{{.Tag}}" | grep -v REPOSITORY | tr '\n' ' ') -o /tmp/docker-images.tar
-
-      - name: Upload Docker images
-        uses: actions/upload-artifact@v4
-        with:
-          name: docker-images
-          path: /tmp/docker-images.tar
-          retention-days: 1
+          docker compose --profile worker_py3_7 --profile worker_py3_8 --profile worker_py3_9 --profile statsd push
+          
+          # Clean up cache
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
   test:
     name: Run Tests
     runs-on: ubuntu-latest
-    needs: build-docker
+    strategy:
+      matrix:
+        test-type: [frontend, backend]
+      fail-fast: false
     steps:
       - uses: actions/checkout@v4
 
       - name: Setup Python and dependencies
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip'
@@ -127,40 +140,47 @@ jobs:
         with:
           version: latest
 
+      - name: Docker login
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
       - name: Install test dependencies
         run: pip install awscli==${{ env.AWSCLI_VERSION }} coveralls
 
-      - name: Download Docker images
-        uses: actions/download-artifact@v3
-        with:
-          name: docker-images
-          path: /tmp
-
-      - name: Load Docker images
+      - name: Pull Docker images
         run: |
-          docker load -i /tmp/docker-images.tar
+          # Pull the latest images (they might be building in parallel)
+          timeout 300 bash -c 'until docker pull ${{ env.REGISTRY }}/evalai:${{ env.IMAGE_TAG }} 2>/dev/null; do echo "Waiting for image..."; sleep 10; done'
+          docker compose pull
 
       - name: Setup display for frontend tests
+        if: matrix.test-type == 'frontend'
         run: |
           sudo apt-get update
           sudo apt-get install -y xvfb chromium-browser
           Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
 
       - name: Run frontend tests
+        if: matrix.test-type == 'frontend'
         run: |
           docker compose run nodejs bash -c "gulp dev && karma start --single-run && gulp staging"
 
       - name: Run backend tests
+        if: matrix.test-type == 'backend'
         run: |
           docker compose run -e DJANGO_SETTINGS_MODULE=settings.test django python manage.py flush --noinput
           docker compose run -e DJANGO_SETTINGS_MODULE=settings.test django pytest --cov . --cov-config .coveragerc
 
       - name: Upload coverage to Codecov
+        if: matrix.test-type == 'backend'
         uses: codecov/codecov-action@v3
         with:
           fail_ci_if_error: false
 
       - name: Upload coverage to Coveralls
+        if: matrix.test-type == 'backend'
         continue-on-error: true
         env:
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
@@ -170,14 +190,14 @@ jobs:
   deploy:
     name: Deploy & Notify
     runs-on: ubuntu-latest
-    needs: [test, quality-check]
+    needs: [build-and-push, test, quality-check]
     if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main')
     environment: production
     steps:
       - uses: actions/checkout@v4
 
       - name: Setup Python and dependencies
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip'