From f0c93711c680a8d4d92c8ee80e0d78735c0736d9 Mon Sep 17 00:00:00 2001 From: erikzaadi Date: Sun, 20 Oct 2024 12:42:13 +0300 Subject: [PATCH] Separate builder and runner image --- .../actions/build-docker-image/action.yml | 78 +++++++++++++++++++ .github/workflows/ci.yml | 50 ++---------- .github/workflows/detect-changes-matrix.yml | 9 +++ .github/workflows/infra.yml | 36 +++++++++ integrations/_infra/Dockerfile.Deb | 27 +------ integrations/_infra/Dockerfile.base.builder | 25 ++++++ integrations/_infra/Dockerfile.base.runner | 12 +++ 7 files changed, 170 insertions(+), 67 deletions(-) create mode 100644 .github/workflows/actions/build-docker-image/action.yml create mode 100644 .github/workflows/infra.yml create mode 100644 integrations/_infra/Dockerfile.base.builder create mode 100644 integrations/_infra/Dockerfile.base.runner diff --git a/.github/workflows/actions/build-docker-image/action.yml b/.github/workflows/actions/build-docker-image/action.yml new file mode 100644 index 0000000000..5c8a19ba71 --- /dev/null +++ b/.github/workflows/actions/build-docker-image/action.yml @@ -0,0 +1,78 @@ +name: Build docker images +description: Build Docker Images +inputs: + dockerfile: + description: Dockerfile to build + required: true + tags: + description: Docker tags to publish + required: true + platforms: + description: Platforms to build (csv) + required: false + default: 'linux/arm64,linux/amd64' + test: + description: Test command to run on the created image (Optional) + required: false + default: '' + build-args: + description: Explicit docker build-args + required: false + default: '' + skip-init: + description: Skip docker init (if ran after another invocation of this action) + required: false + default: '' + docker-user: + required: true + description: Docker Hub User + docker-password: + required: true + description: Docker Hub User + skip-push: + required: false + description: Optionally skip push + default: '' + +runs: + using: 'composite' + steps: + # - name: Setup docker (missing on MacOS) + # if: matrix.platform == 'linux/arm64' + # uses: douglascamata/setup-docker-macos-action@v1-alpha + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + if: ${{ inputs.skip-init == '' }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + if: ${{ inputs.skip-init == '' }} + + - name: Login to Docker Hub + uses: docker/login-action@v3 + if: ${{ inputs.skip-init == '' }} + with: + registry: ghcr.io + username: ${{ inputs.docker-user }} + password: ${{ inputs.docker-password }} + + - name: Build Runner Image + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ inputs.dockerfile }} + platforms: ${{ inputs.platforms }} + push: ${{ inputs.skip-push == '' }} + load: ${{ inputs.test != '' }} + tags: ${{ inputs.tags }} + build-args: | + ${{ inputs.build-args }} + + - name: Verify Built Image + shell: bash + if: ${{ inputs.test != '' }} + run: | + SINGLE_TAG=$(echo "${{ inputs.tags }}" | awk -F ',' '{print $1};' ) + SINGLE_PLATFORM=$(echo "${{ inputs.platforms }}" | awk -F ',' '{print $1};' ) + docker run --platform "${SINGLE_PLATFORM}" --rm --entrypoint bash "${SINGLE_TAG}" -c '${{ inputs.test }}' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3ede6d70b..f1bf2cb5c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,6 @@ name: Build integration images on: pull_request: - workflow_dispatch: jobs: prepare-matrix: @@ -39,7 +38,6 @@ jobs: build-integration: - # runs-on: ${{ matrix.platform == 'linux/arm64' && 'macos-13' || 'ubuntu-latest' }} runs-on: 'ubuntu-latest' if: needs.prepare-matrix.outputs.matrix != '[]' outputs: @@ -58,20 +56,6 @@ jobs: - name: Check out code uses: actions/checkout@v4 - # - name: Setup docker (missing on MacOS) - # if: matrix.platform == 'linux/arm64' - # uses: douglascamata/setup-docker-macos-action@v1-alpha - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: ${{ matrix.platform }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - platforms: ${{ matrix.platform }} - - name: Prepare Docker images tags id: prepare_tags run: | @@ -106,37 +90,15 @@ jobs: echo "is_dev_version=false" >> $GITHUB_OUTPUT fi - - name: Get used docker base image - id: get-docker-image - run: | - echo "base_image=$(cat ${{ steps.prepare_tags.outputs.dockerfile_path }} | head -n 1 | awk -F '=' '{print $2}' )" >> $GITHUB_OUTPUT - - - name: Login to Docker Hub - uses: docker/login-action@v3 + - name: Build Docker Image + uses: ./.github/workflows/actions/build-docker-image with: - registry: ghcr.io - username: ${{ secrets.DOCKER_MACHINE_USER }} - password: ${{ secrets.DOCKER_MACHINE_TOKEN }} - - # - name: Cache Docker images - # uses: ScribeMD/docker-cache@0.5.0 - # with: - # key: docker-${{ matrix.integration }}-${{ steps.get-docker-image.outputs.base_image }}-${{ matrix.platform }} - - - name: Build - uses: docker/build-push-action@v6 - with: - context: . - file: ${{ steps.prepare_tags.outputs.dockerfile_path }} + dockerfile: ${{ steps.prepare_tags.outputs.dockerfile_path }} platforms: ${{ matrix.platform }} - push: false - load: true tags: ${{ steps.prepare_tags.outputs.tags }} build-args: | BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }} INTEGRATION_VERSION=${{ steps.prepare_tags.outputs.version }} - - - name: Verify Built Image - run: | - SINGLE_TAG=$(echo "${{ steps.prepare_tags.outputs.tags }}" | awk -F ',' '{print $1};' ) - docker run --platform ${{ matrix.platform }} --rm --entrypoint bash "${SINGLE_TAG}" -c 'ocean version' + docker-user: ${{ secrets.DOCKER_MACHINE_USER }} + docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }} + skip-push: 'yupp' diff --git a/.github/workflows/detect-changes-matrix.yml b/.github/workflows/detect-changes-matrix.yml index 91dd9c31fc..4973fe16c9 100644 --- a/.github/workflows/detect-changes-matrix.yml +++ b/.github/workflows/detect-changes-matrix.yml @@ -11,6 +11,9 @@ on: core: value: ${{ jobs.detect-changes.outputs.core }} description: "Determine if any core changes per git commit changes" + infra: + value: ${{ jobs.detect-changes.outputs.infra }} + description: "Determine if any changes to docker infra" jobs: detect-changes: @@ -20,6 +23,7 @@ jobs: matrix: ${{ steps.set-all-matrix.outputs.matrix }} integrations: ${{ steps.set-all-matrix.outputs.integrations }} core: ${{ steps.set-all-matrix.outputs.core }} + infra: ${{ steps.set-all-matrix.outputs.infra}} steps: - name: Checkout Repo uses: actions/checkout@v4 @@ -42,6 +46,8 @@ jobs: - 'integrations/**' - '!integrations/**/*.md' - '!integrations/_infra/*' + infra: + - 'integrations/_infra/*' - name: Set integrations and all matrix id: set-all-matrix @@ -50,8 +56,11 @@ jobs: HAS_CORE=${{ steps.changed-files.outputs.core_all_changed_files != '[]' }} echo "Core changes : ${HAS_CORE}" MATRIX=$(node -e "integrations=${INTEGRATIONS}; hasCore=${HAS_CORE}; console.log(JSON.stringify(hasCore ? integrations.concat(['.']) : integrations))") + HAS_INFRA=${{ steps.changed-files.outputs.infra_all_changed_files != '[]' }} + echo "Infra changes : ${HAS_INFRA}" echo "Integration changes : ${INTEGRATIONS}" echo "All changes : ${MATRIX}" echo "core=${HAS_CORE}" >> $GITHUB_OUTPUT echo "integrations=${INTEGRATIONS}" >> $GITHUB_OUTPUT echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT + echo "infra=${HAS_INFRA}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/infra.yml b/.github/workflows/infra.yml new file mode 100644 index 0000000000..fe4fac875c --- /dev/null +++ b/.github/workflows/infra.yml @@ -0,0 +1,36 @@ +name: Build infra images +on: + pull_request: + workflow_dispatch: + +jobs: + detect-changes: + uses: ./.github/workflows/detect-changes-matrix.yml + build-infra: + # runs-on: ${{ matrix.platform == 'linux/arm64' && 'macos-13' || 'ubuntu-latest' }} + runs-on: 'ubuntu-latest' + needs: detect-changes + if: ${{ needs.detect-changes.outputs.infra == 'true' }} + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Build Docker Image + uses: ./.github/workflows/actions/build-docker-image + with: + dockerfile: ./integrations/_infra/Dockerfile.base.builder + platforms: linux/amd64,linux/arm64 + tags: ghcr.io/port-labs/port-ocean-base-builder:latest + docker-user: ${{ secrets.DOCKER_MACHINE_USER }} + docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }} + + + - name: Build Docker Image + uses: ./.github/workflows/actions/build-docker-image + with: + dockerfile: ./integrations/_infra/Dockerfile.base.runner + platforms: linux/amd64,linux/arm64 + tags: ghcr.io/port-labs/port-ocean-base-runner:latest + docker-user: ${{ secrets.DOCKER_MACHINE_USER }} + docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }} + skip-init: 'yupp' diff --git a/integrations/_infra/Dockerfile.Deb b/integrations/_infra/Dockerfile.Deb index eebce4cda5..5cd9cfba0e 100644 --- a/integrations/_infra/Dockerfile.Deb +++ b/integrations/_infra/Dockerfile.Deb @@ -1,6 +1,7 @@ -ARG BASE_PYTHON_IMAGE=debian:trixie-slim +ARG BASE_BUILDER_PYTHON_IMAGE=ghcr.io/port-labs/port-ocean-base-builder:latest +ARG BASE_RUNNER_PYTHON_IMAGE=ghcr.io/port-labs/port-ocean-base-runner:latest -FROM ${BASE_PYTHON_IMAGE} AS base +FROM ${BASE_BUILDER_PYTHON_IMAGE} AS base ARG BUILD_CONTEXT ARG BUILDPLATFORM @@ -10,29 +11,13 @@ ENV LIBRDKAFKA_VERSION=1.9.2 \ POETRY_VIRTUALENVS_IN_PROJECT=1 \ PIP_ROOT_USER_ACTION=ignore -RUN apt-get update \ - && apt-get install -y \ - --no-install-recommends \ - wget \ - g++ \ - libssl-dev \ - autoconf \ - automake \ - libtool \ - curl \ - librdkafka-dev \ - python3 \ - python3-pip \ - python3-poetry \ - && apt-get clean - WORKDIR /app COPY ./${BUILD_CONTEXT}/pyproject.toml ./${BUILD_CONTEXT}/poetry.lock /app/ RUN poetry install --without dev --no-root --no-interaction --no-ansi --no-cache -FROM ${BASE_PYTHON_IMAGE} AS prod +FROM ${BASE_RUNNER_PYTHON_IMAGE} AS prod ARG INTEGRATION_VERSION ARG BUILD_CONTEXT @@ -45,10 +30,6 @@ LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean ENV PIP_ROOT_USER_ACTION=ignore -RUN apt-get update \ - && apt-get install -y --no-install-recommends librdkafka-dev python3 \ - && apt-get clean - WORKDIR /app # Copy the application code diff --git a/integrations/_infra/Dockerfile.base.builder b/integrations/_infra/Dockerfile.base.builder new file mode 100644 index 0000000000..4121f52e31 --- /dev/null +++ b/integrations/_infra/Dockerfile.base.builder @@ -0,0 +1,25 @@ +ARG BASE_PYTHON_IMAGE=debian:trixie-slim +FROM ${BASE_PYTHON_IMAGE} + +LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean + +ENV LIBRDKAFKA_VERSION=1.9.2 \ + PYTHONUNBUFFERED=1 \ + POETRY_VIRTUALENVS_IN_PROJECT=1 \ + PIP_ROOT_USER_ACTION=ignore + +RUN apt-get update \ + && apt-get install -y \ + --no-install-recommends \ + wget \ + g++ \ + libssl-dev \ + autoconf \ + automake \ + libtool \ + curl \ + librdkafka-dev \ + python3 \ + python3-pip \ + python3-poetry \ + && apt-get clean diff --git a/integrations/_infra/Dockerfile.base.runner b/integrations/_infra/Dockerfile.base.runner new file mode 100644 index 0000000000..48061584de --- /dev/null +++ b/integrations/_infra/Dockerfile.base.runner @@ -0,0 +1,12 @@ +ARG BASE_PYTHON_IMAGE=debian:trixie-slim +FROM ${BASE_PYTHON_IMAGE} + +LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean + +ENV LIBRDKAFKA_VERSION=1.9.2 + +ENV PIP_ROOT_USER_ACTION=ignore + +RUN apt-get update \ + && apt-get install -y --no-install-recommends librdkafka-dev python3 \ + && apt-get clean