Separate builder and runner image

port-labs · Oct 21, 2024 · f0c9371 · f0c9371
1 parent ac745b3
commit f0c9371
Show file tree

Hide file tree

Showing 7 changed files with 170 additions and 67 deletions.
diff --git a/.github/workflows/actions/build-docker-image/action.yml b/.github/workflows/actions/build-docker-image/action.yml
@@ -0,0 +1,78 @@
+name: Build docker images
+description: Build Docker Images
+inputs:
+  dockerfile:
+    description: Dockerfile to build
+    required: true
+  tags:
+    description: Docker tags to publish
+    required: true
+  platforms:
+    description: Platforms to build (csv)
+    required: false
+    default: 'linux/arm64,linux/amd64'
+  test:
+    description: Test command to run on the created image (Optional)
+    required: false
+    default: ''
+  build-args:
+    description: Explicit docker build-args
+    required: false
+    default: ''
+  skip-init:
+    description: Skip docker init (if ran after another invocation of this action)
+    required: false
+    default: ''
+  docker-user:
+    required: true
+    description: Docker Hub User
+  docker-password:
+    required: true
+    description: Docker Hub User
+  skip-push:
+    required: false
+    description: Optionally skip push
+    default: ''
+
+runs:
+  using: 'composite'
+  steps:
+    # - name: Setup docker (missing on MacOS)
+    #   if: matrix.platform == 'linux/arm64'
+    #   uses: douglascamata/setup-docker-macos-action@v1-alpha
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+      if: ${{ inputs.skip-init == '' }}
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      if: ${{ inputs.skip-init == '' }}
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v3
+      if: ${{ inputs.skip-init == '' }}
+      with:
+        registry: ghcr.io
+        username: ${{ inputs.docker-user }}
+        password: ${{ inputs.docker-password }}
+
+    - name: Build Runner Image
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        file: ${{ inputs.dockerfile }}
+        platforms: ${{ inputs.platforms }}
+        push: ${{ inputs.skip-push == '' }}
+        load: ${{ inputs.test != '' }}
+        tags: ${{ inputs.tags }}
+        build-args: |
+          ${{ inputs.build-args }}
+
+    - name: Verify Built Image
+      shell: bash
+      if: ${{ inputs.test != '' }}
+      run: |
+        SINGLE_TAG=$(echo "${{ inputs.tags }}" | awk -F ',' '{print $1};' )
+        SINGLE_PLATFORM=$(echo "${{ inputs.platforms }}" | awk -F ',' '{print $1};' )
+        docker run --platform "${SINGLE_PLATFORM}" --rm --entrypoint bash "${SINGLE_TAG}" -c '${{ inputs.test }}'
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,7 +1,6 @@
 name: Build integration images
 on:
   pull_request:
-  workflow_dispatch:
 
 jobs:
   prepare-matrix:
@@ -39,7 +38,6 @@ jobs:
 
 
   build-integration:
-    # runs-on: ${{ matrix.platform == 'linux/arm64' && 'macos-13' || 'ubuntu-latest' }}
     runs-on: 'ubuntu-latest'
     if: needs.prepare-matrix.outputs.matrix != '[]'
     outputs:
@@ -58,20 +56,6 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v4
 
-      # - name: Setup docker (missing on MacOS)
-      #   if: matrix.platform == 'linux/arm64'
-      #   uses: douglascamata/setup-docker-macos-action@v1-alpha
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: ${{ matrix.platform }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          platforms: ${{ matrix.platform }}
-
       - name: Prepare Docker images tags
         id: prepare_tags
         run: |
@@ -106,37 +90,15 @@ jobs:
             echo "is_dev_version=false" >> $GITHUB_OUTPUT
           fi
 
-      - name: Get used docker base image
-        id: get-docker-image
-        run: |
-          echo "base_image=$(cat ${{ steps.prepare_tags.outputs.dockerfile_path }} | head -n 1 | awk -F '=' '{print $2}' )" >> $GITHUB_OUTPUT
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
+      - name: Build Docker Image
+        uses: ./.github/workflows/actions/build-docker-image
         with:
-          registry: ghcr.io
-          username: ${{ secrets.DOCKER_MACHINE_USER }}
-          password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
-
-      # - name: Cache Docker images
-      #   uses: ScribeMD/docker-cache@0.5.0
-      #   with:
-      #     key: docker-${{ matrix.integration }}-${{ steps.get-docker-image.outputs.base_image }}-${{ matrix.platform }}
-
-      - name: Build
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          file: ${{ steps.prepare_tags.outputs.dockerfile_path }}
+          dockerfile: ${{ steps.prepare_tags.outputs.dockerfile_path }}
           platforms: ${{ matrix.platform }}
-          push: false
-          load: true
           tags: ${{ steps.prepare_tags.outputs.tags }}
           build-args: |
             BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }}
             INTEGRATION_VERSION=${{ steps.prepare_tags.outputs.version }}
-
-      - name: Verify Built Image
-        run: |
-          SINGLE_TAG=$(echo "${{ steps.prepare_tags.outputs.tags }}" | awk -F ',' '{print $1};' )
-          docker run --platform ${{ matrix.platform }} --rm --entrypoint bash "${SINGLE_TAG}" -c 'ocean version'
+          docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
+          docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
+          skip-push: 'yupp'
diff --git a/.github/workflows/detect-changes-matrix.yml b/.github/workflows/detect-changes-matrix.yml
@@ -11,6 +11,9 @@ on:
       core:
         value: ${{ jobs.detect-changes.outputs.core }}
         description: "Determine if any core changes per git commit changes"
+      infra:
+        value: ${{ jobs.detect-changes.outputs.infra }}
+        description: "Determine if any changes to docker infra"
 
 jobs:
   detect-changes:
@@ -20,6 +23,7 @@ jobs:
       matrix: ${{ steps.set-all-matrix.outputs.matrix }}
       integrations: ${{ steps.set-all-matrix.outputs.integrations }}
       core: ${{ steps.set-all-matrix.outputs.core }}
+      infra: ${{ steps.set-all-matrix.outputs.infra}}
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4
@@ -42,6 +46,8 @@ jobs:
               - 'integrations/**'
               - '!integrations/**/*.md'
               - '!integrations/_infra/*'
+            infra:
+              - 'integrations/_infra/*'
 
       - name: Set integrations and all matrix
         id: set-all-matrix
@@ -50,8 +56,11 @@ jobs:
           HAS_CORE=${{ steps.changed-files.outputs.core_all_changed_files != '[]' }}
           echo "Core changes : ${HAS_CORE}"
           MATRIX=$(node -e "integrations=${INTEGRATIONS}; hasCore=${HAS_CORE}; console.log(JSON.stringify(hasCore ? integrations.concat(['.']) : integrations))")
+          HAS_INFRA=${{ steps.changed-files.outputs.infra_all_changed_files != '[]' }}
+          echo "Infra changes : ${HAS_INFRA}"
           echo "Integration changes : ${INTEGRATIONS}"
           echo "All changes : ${MATRIX}"
           echo "core=${HAS_CORE}" >> $GITHUB_OUTPUT
           echo "integrations=${INTEGRATIONS}" >> $GITHUB_OUTPUT
           echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT
+          echo "infra=${HAS_INFRA}" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/infra.yml b/.github/workflows/infra.yml
@@ -0,0 +1,36 @@
+name: Build infra images
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  detect-changes:
+    uses: ./.github/workflows/detect-changes-matrix.yml
+  build-infra:
+    # runs-on: ${{ matrix.platform == 'linux/arm64' && 'macos-13' || 'ubuntu-latest' }}
+    runs-on: 'ubuntu-latest'
+    needs: detect-changes
+    if: ${{ needs.detect-changes.outputs.infra == 'true' }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Build Docker Image
+        uses: ./.github/workflows/actions/build-docker-image
+        with:
+          dockerfile: ./integrations/_infra/Dockerfile.base.builder
+          platforms: linux/amd64,linux/arm64
+          tags: ghcr.io/port-labs/port-ocean-base-builder:latest
+          docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
+          docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
+
+
+      - name: Build Docker Image
+        uses: ./.github/workflows/actions/build-docker-image
+        with:
+          dockerfile: ./integrations/_infra/Dockerfile.base.runner
+          platforms: linux/amd64,linux/arm64
+          tags: ghcr.io/port-labs/port-ocean-base-runner:latest
+          docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
+          docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
+          skip-init: 'yupp'
diff --git a/integrations/_infra/Dockerfile.Deb b/integrations/_infra/Dockerfile.Deb
@@ -1,6 +1,7 @@
-ARG BASE_PYTHON_IMAGE=debian:trixie-slim
+ARG BASE_BUILDER_PYTHON_IMAGE=ghcr.io/port-labs/port-ocean-base-builder:latest
+ARG BASE_RUNNER_PYTHON_IMAGE=ghcr.io/port-labs/port-ocean-base-runner:latest
 
-FROM ${BASE_PYTHON_IMAGE} AS base
+FROM ${BASE_BUILDER_PYTHON_IMAGE} AS base
 
 ARG BUILD_CONTEXT
 ARG BUILDPLATFORM
@@ -10,29 +11,13 @@ ENV LIBRDKAFKA_VERSION=1.9.2 \
     POETRY_VIRTUALENVS_IN_PROJECT=1 \
     PIP_ROOT_USER_ACTION=ignore
 
-RUN apt-get update \
-    && apt-get install -y \
-        --no-install-recommends \
-        wget \
-        g++ \
-        libssl-dev \
-        autoconf \
-        automake \
-        libtool \
-        curl \
-        librdkafka-dev \
-        python3 \
-        python3-pip \
-        python3-poetry \
-    && apt-get clean
-
 WORKDIR /app
 
 COPY ./${BUILD_CONTEXT}/pyproject.toml ./${BUILD_CONTEXT}/poetry.lock /app/
 
 RUN poetry install --without dev --no-root --no-interaction --no-ansi --no-cache
 
-FROM ${BASE_PYTHON_IMAGE} AS prod
+FROM ${BASE_RUNNER_PYTHON_IMAGE} AS prod
 
 ARG INTEGRATION_VERSION
 ARG BUILD_CONTEXT
@@ -45,10 +30,6 @@ LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean
 
 ENV PIP_ROOT_USER_ACTION=ignore
 
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends librdkafka-dev python3 \
-    && apt-get clean
-
 WORKDIR /app
 
 # Copy the application code

diff --git a/integrations/_infra/Dockerfile.base.builder b/integrations/_infra/Dockerfile.base.builder
@@ -0,0 +1,25 @@
+ARG BASE_PYTHON_IMAGE=debian:trixie-slim
+FROM ${BASE_PYTHON_IMAGE}
+
+LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean
+
+ENV LIBRDKAFKA_VERSION=1.9.2 \
+    PYTHONUNBUFFERED=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    PIP_ROOT_USER_ACTION=ignore
+
+RUN apt-get update \
+    && apt-get install -y \
+        --no-install-recommends \
+        wget \
+        g++ \
+        libssl-dev \
+        autoconf \
+        automake \
+        libtool \
+        curl \
+        librdkafka-dev \
+        python3 \
+        python3-pip \
+        python3-poetry \
+    && apt-get clean
diff --git a/integrations/_infra/Dockerfile.base.runner b/integrations/_infra/Dockerfile.base.runner
@@ -0,0 +1,12 @@
+ARG BASE_PYTHON_IMAGE=debian:trixie-slim
+FROM ${BASE_PYTHON_IMAGE}
+
+LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean
+
+ENV LIBRDKAFKA_VERSION=1.9.2
+
+ENV PIP_ROOT_USER_ACTION=ignore
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends librdkafka-dev python3 \
+    && apt-get clean