Separate builder and runner image

erikzaadi · erikzaadi · commit 95003b9ecb48 · 2024-10-27T10:29:03.000+02:00
diff --git a/.github/workflows/actions/build-docker-image/action.yml b/.github/workflows/actions/build-docker-image/action.yml
@@ -0,0 +1,81 @@
+name: Build docker images
+description: Build Docker Images
+# NOTE: In composite actions, all parameters are strings,
+# thus flags are simply checked by being non empty strings,
+# where there the default is an empty string
+inputs:
+  dockerfile:
+    description: Dockerfile to build
+    required: true
+  tags:
+    description: Docker tags to publish
+    required: true
+  platforms:
+    description: Platforms to build (csv)
+    required: false
+    default: 'linux/arm64,linux/amd64'
+  test:
+    description: Test command to run on the created image (Optional)
+    required: false
+    default: ''
+  build-args:
+    description: Explicit docker build-args
+    required: false
+    default: ''
+  skip-init:
+    description: Skip docker init (if ran after another invocation of this action)
+    required: false
+    default: ''
+  docker-user:
+    required: true
+    description: Docker Hub User
+  docker-password:
+    required: true
+    description: Docker Hub User
+  skip-push:
+    required: false
+    description: Optionally skip push
+    default: ''
+  load-created-image:
+    required: false
+    description: Optionally load created docker image
+    default: ''
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+      if: ${{ inputs.skip-init == '' }}
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      if: ${{ inputs.skip-init == '' }}
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v3
+      if: ${{ inputs.skip-init == '' }}
+      with:
+        registry: ghcr.io
+        username: ${{ inputs.docker-user }}
+        password: ${{ inputs.docker-password }}
+
+    - name: Build Runner Image
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        file: ${{ inputs.dockerfile }}
+        platforms: ${{ inputs.platforms }}
+        push: ${{ inputs.skip-push == '' }}
+        load: ${{ inputs.test != '' || inputs.load-created-image != '' }}
+        tags: ${{ inputs.tags }}
+        build-args: |
+          ${{ inputs.build-args }}
+
+    - name: Verify Built Image
+      shell: bash
+      if: ${{ inputs.test != '' }}
+      run: |
+        SINGLE_TAG=$(echo "${{ inputs.tags }}" | awk -F ',' '{print $1};' )
+        SINGLE_PLATFORM=$(echo "${{ inputs.platforms }}" | awk -F ',' '{print $1};' )
+        docker run --platform "${SINGLE_PLATFORM}" --rm --entrypoint bash "${SINGLE_TAG}" -c '${{ inputs.test }}'
diff --git a/.github/workflows/build-infra-images.yml b/.github/workflows/build-infra-images.yml
@@ -0,0 +1,34 @@
+name: Build infra images
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  detect-changes:
+    uses: ./.github/workflows/detect-changes-matrix.yml
+  build-infra:
+    runs-on: 'ubuntu-latest'
+    needs: detect-changes
+    if: ${{ needs.detect-changes.outputs.infra == 'true' }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Build Docker Image
+        uses: ./.github/workflows/actions/build-docker-image
+        with:
+          dockerfile: ./integrations/_infra/Dockerfile.base.builder
+          platforms: linux/amd64,linux/arm64
+          tags: ghcr.io/port-labs/port-ocean-base-builder:latest
+          docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
+          docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
+
+      - name: Build Docker Image
+        uses: ./.github/workflows/actions/build-docker-image
+        with:
+          dockerfile: ./integrations/_infra/Dockerfile.base.runner
+          platforms: linux/amd64,linux/arm64
+          tags: ghcr.io/port-labs/port-ocean-base-runner:latest
+          docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
+          docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
+          skip-init: 'true'
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -37,10 +37,8 @@ jobs:
           echo $(echo ${integrations_to_build[@]} | jq -R -c 'split(" ")')
           echo "INTEGRATIONS_MATRIX=$(echo ${integrations_to_build[@]} | jq -R -c 'split(" ")')" >> $GITHUB_OUTPUT
 
-
   build-integration:
-    # runs-on: ${{ matrix.platform == 'linux/arm64' && 'macos-13' || 'ubuntu-latest' }}
-    runs-on: 'ubuntu-latest'
+    runs-on: ubuntu-latest
     if: needs.prepare-matrix.outputs.matrix != '[]'
     outputs:
       is_dev_version: ${{ steps.prepare_tags.outputs.is_dev_version }}
@@ -58,20 +56,6 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v4
 
-      # - name: Setup docker (missing on MacOS)
-      #   if: matrix.platform == 'linux/arm64'
-      #   uses: douglascamata/setup-docker-macos-action@v1-alpha
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: ${{ matrix.platform }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          platforms: ${{ matrix.platform }}
-
       - name: Prepare Docker images tags
         id: prepare_tags
         run: |
@@ -106,37 +90,15 @@ jobs:
             echo "is_dev_version=false" >> $GITHUB_OUTPUT
           fi
 
-      - name: Get used docker base image
-        id: get-docker-image
-        run: |
-          echo "base_image=$(cat ${{ steps.prepare_tags.outputs.dockerfile_path }} | head -n 1 | awk -F '=' '{print $2}' )" >> $GITHUB_OUTPUT
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.DOCKER_MACHINE_USER }}
-          password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
-
-      # - name: Cache Docker images
-      #   uses: ScribeMD/docker-cache@0.5.0
-      #   with:
-      #     key: docker-${{ matrix.integration }}-${{ steps.get-docker-image.outputs.base_image }}-${{ matrix.platform }}
-
-      - name: Build
-        uses: docker/build-push-action@v6
+      - name: Build Docker Image
+        uses: ./.github/workflows/actions/build-docker-image
         with:
-          context: .
-          file: ${{ steps.prepare_tags.outputs.dockerfile_path }}
+          dockerfile: ${{ steps.prepare_tags.outputs.dockerfile_path }}
           platforms: ${{ matrix.platform }}
-          push: false
-          load: true
           tags: ${{ steps.prepare_tags.outputs.tags }}
           build-args: |
             BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }}
             INTEGRATION_VERSION=${{ steps.prepare_tags.outputs.version }}
-
-      - name: Verify Built Image
-        run: |
-          SINGLE_TAG=$(echo "${{ steps.prepare_tags.outputs.tags }}" | awk -F ',' '{print $1};' )
-          docker run --platform ${{ matrix.platform }} --rm --entrypoint bash "${SINGLE_TAG}" -c 'ocean version'
+          docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
+          docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
+          skip-push: 'true'
diff --git a/.github/workflows/detect-changes-matrix.yml b/.github/workflows/detect-changes-matrix.yml
@@ -11,6 +11,9 @@ on:
       core:
         value: ${{ jobs.detect-changes.outputs.core }}
         description: "Determine if any core changes per git commit changes"
+      infra:
+        value: ${{ jobs.detect-changes.outputs.infra }}
+        description: "Determine if any changes to docker infra"
 
 jobs:
   detect-changes:
@@ -20,6 +23,7 @@ jobs:
       matrix: ${{ steps.set-all-matrix.outputs.matrix }}
       integrations: ${{ steps.set-all-matrix.outputs.integrations }}
       core: ${{ steps.set-all-matrix.outputs.core }}
+      infra: ${{ steps.set-all-matrix.outputs.infra}}
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4
@@ -42,6 +46,8 @@ jobs:
               - 'integrations/**'
               - '!integrations/**/*.md'
               - '!integrations/_infra/*'
+            infra:
+              - 'integrations/_infra/*'
 
       - name: Set integrations and all matrix
         id: set-all-matrix
@@ -50,8 +56,11 @@ jobs:
           HAS_CORE=${{ steps.changed-files.outputs.core_all_changed_files != '[]' }}
           echo "Core changes : ${HAS_CORE}"
           MATRIX=$(node -e "integrations=${INTEGRATIONS}; hasCore=${HAS_CORE}; console.log(JSON.stringify(hasCore ? integrations.concat(['.']) : integrations))")
+          HAS_INFRA=${{ steps.changed-files.outputs.infra_all_changed_files != '[]' }}
+          echo "Infra changes : ${HAS_INFRA}"
           echo "Integration changes : ${INTEGRATIONS}"
           echo "All changes : ${MATRIX}"
           echo "core=${HAS_CORE}" >> $GITHUB_OUTPUT
           echo "integrations=${INTEGRATIONS}" >> $GITHUB_OUTPUT
           echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT
+          echo "infra=${HAS_INFRA}" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/docker-images-security-scan.yml b/.github/workflows/docker-images-security-scan.yml
@@ -8,33 +8,34 @@ on:
         # This is a bit annoying, there's no real way to display the integrations dynamically in a dropdown for the action dispatcher
         options:
           - all
+          - argocd
           - aws
+          - azure
           - azure-devops
+          - backstage
+          - datadog
           - dynatrace
           - fake-integration
-          - gcp
-          - jenkins
-          - kafka
-          - launchdarkly
-          - newrelic
-          - opencost
-          - pagerduty
-          - servicenow
-          - sonarqube
-          - terraform-cloud
-          - argocd
-          - azure
-          - datadog
           - firehydrant
+          - gcp
           - gitlab
+          - jenkins
           - jira
+          - kafka
           - kubecost
+          - launchdarkly
           - linear
+          - newrelic
           - octopus
+          - opencost
           - opsgenie
+          - pagerduty
           - sentry
+          - servicenow
           - snyk
+          - sonarqube
           - statuspage
+          - terraform-cloud
           - wiz
 
 jobs:
@@ -77,14 +78,6 @@ jobs:
       - name: Checkout Repo
         uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: linux/amd64,linux/arm64
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
       - name: Extract version and image tag
         id: enrich-version
         run: |
@@ -97,17 +90,16 @@ jobs:
           echo "identifier=${IDENTIFIER}" >> ${GITHUB_OUTPUT}
           echo "image_tag=${IMAGE_FULL_TAG}" >> ${GITHUB_OUTPUT}
 
-      - name: Build
-        uses: docker/build-push-action@v6
+      - name: Build Docker Image
+        uses: ./.github/workflows/actions/build-docker-image
         with:
-          context: .
-          file: ./integrations/_infra/Dockerfile
+          dockerfile: ./integrations/_infra/Dockerfile
           platforms: linux/amd64
-          push: false
+          skip-push: 'true'
           tags: ${{ steps.enrich-version.outputs.image_tag }}
-          load: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          load-created-image: 'true'
+          docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
+          docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
           build-args: |
             BUILD_CONTEXT=./integrations/${{ steps.enrich-version.outputs.integration }}
             INTEGRATION_VERSION=${{ steps.enrich-version.outputs.version }}
diff --git a/.github/workflows/release-integrations.yml b/.github/workflows/release-integrations.yml
@@ -13,12 +13,7 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.DOCKER_MACHINE_USER }}
-          password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
+
       - name: Prepare matrix
         id: prepare-matrix
         run: |
@@ -57,24 +52,13 @@ jobs:
     needs: [prepare-matrix]
     strategy:
       # limit the number of parallel jobs to avoid hitting the ghcr.io rate limit
-      max-parallel: 10
+      max-parallel: 5
       matrix:
         integration: ${{fromJson(needs.prepare-matrix.outputs.matrix)}}
     steps:
       - name: Check out code
         uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: linux/amd64,linux/arm64
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.DOCKER_MACHINE_USER }}
-          password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
+
       - name: Prepare Docker images tags
         id: prepare_tags
         run: |
@@ -108,17 +92,17 @@ jobs:
             echo "is_dev_version=false" >> $GITHUB_OUTPUT
           fi
 
-      - name: Build and push
-        uses: docker/build-push-action@v6
+      - name: Build Docker Image
+        uses: ./.github/workflows/actions/build-docker-image
         with:
-          context: .
-          file: ${{ steps.prepare_tags.outputs.dockerfile_path }}
+          dockerfile: ${{ steps.prepare_tags.outputs.dockerfile_path }}
           platforms: linux/amd64,linux/arm64
-          push: true
           tags: ${{ steps.prepare_tags.outputs.tags }}
           build-args: |
             BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }}
             INTEGRATION_VERSION=${{ steps.prepare_tags.outputs.version }}
+          docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
+          docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
 
   upload-specs:
     runs-on: ubuntu-latest
diff --git a/integrations/_infra/Dockerfile.Deb b/integrations/_infra/Dockerfile.Deb
diff --git a/integrations/_infra/Dockerfile.base.builder b/integrations/_infra/Dockerfile.base.builder
diff --git a/integrations/_infra/Dockerfile.base.runner b/integrations/_infra/Dockerfile.base.runner
