From 9f8d7a23d78eec662e17447d73b396ac4380b128 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 17 Dec 2024 20:33:47 +0100 Subject: [PATCH] revert docker build changes --- .github/workflows/docker-image.yml | 443 +++++------------------------ 1 file changed, 65 insertions(+), 378 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 0c4955c92561d..1df6422d7876a 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1,11 +1,9 @@ env: REGISTRY: ghcr.io IMAGE_NAME: - ${{ github.event_name != 'pull_request' && github.event_name != - 'workflow_dispatch' && github.repository || 'windmill-labs/windmill-test' }} + ${{ github.event_name != 'pull_request' && github.event_name != 'workflow_dispatch' && github.repository || 'windmill-labs/windmill-test' }} DEV_SHA: - ${{ github.event_name != 'pull_request' && github.event_name != - 'workflow_dispatch' && 'dev' || github.event.inputs.tag || 'test' }} + ${{ github.event_name != 'pull_request' && github.event_name != 'workflow_dispatch' && 'dev' || github.event.inputs.tag }} name: Build windmill:main on: @@ -19,14 +17,14 @@ on: workflow_dispatch: inputs: ee: - description: "Build EE image (true, false)" + description: 'Build EE image (true, false)' required: false default: false type: boolean tag: - description: "Tag the image" + description: 'Tag the image' required: true - default: "test" + default: 'test' concurrency: group: ${{ github.ref }} @@ -36,71 +34,8 @@ permissions: write-all jobs: build: - runs-on: ubicloud-standard-30 - if: (github.event_name != 'workflow_dispatch') || (inputs.ee == false) - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Read EE repo commit hash - run: | - echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV" - - - uses: actions/checkout@v4 - with: - repository: windmill-labs/windmill-ee-private - path: ./windmill-ee-private - ref: ${{ env.ee_repo_ref }} - token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} - fetch-depth: 0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - # - uses: depot/setup-action@v1 - - - name: Login to registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Substitute EE code (EE logic is behind feature flag) - run: | - ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private - - - name: Docker meta - id: meta-public - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - flavor: | - latest=false - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - - - name: Build and push publicly - uses: docker/build-push-action@v6 - with: - context: . - push: true - build-args: | - features=embedding,parquet,openidconnect,jemalloc,deno_core,license,http_trigger,zip,oauth2,dind,php,mysql,mssql,bigquery,websocket,python,smtp,csharp,static_frontend,rust - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} - ${{ steps.meta-public.outputs.tags }} - labels: | - ${{ steps.meta-public.outputs.labels }} - org.opencontainers.image.licenses=AGPLv3 - cache-from: type=gha - cache-to: type=gha - - build_arm: - runs-on: ubicloud-standard-30-arm - if: (github.event_name != 'workflow_dispatch') || (inputs.ee == false) + runs-on: ubicloud + if: (github.event_name != 'workflow_dispatch') || (github.event.inputs && !github.event.inputs.ee) steps: - uses: actions/checkout@v4 with: @@ -118,9 +53,9 @@ jobs: token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} fetch-depth: 0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - # - uses: depot/setup-action@v1 + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + - uses: depot/setup-action@v1 - name: Login to registry uses: docker/login-action@v3 @@ -146,9 +81,10 @@ jobs: type=semver,pattern={{major}}.{{minor}} - name: Build and push publicly - uses: docker/build-push-action@v6 + uses: depot/build-push-action@v1 with: context: . + platforms: linux/amd64,linux/arm64 push: true build-args: | features=embedding,parquet,openidconnect,jemalloc,deno_core,license,http_trigger,zip,oauth2,dind,php,mysql,mssql,bigquery,websocket,python,smtp,csharp,static_frontend,rust @@ -158,12 +94,11 @@ jobs: labels: | ${{ steps.meta-public.outputs.labels }} org.opencontainers.image.licenses=AGPLv3 - cache-from: type=gha - cache-to: type=gha build_ee: runs-on: ubicloud - if: (github.event_name != 'workflow_dispatch') || (inputs.ee == true) + if: + (github.event_name != 'workflow_dispatch') || (github.event.inputs.ee) steps: - uses: actions/checkout@v4 with: @@ -181,9 +116,9 @@ jobs: token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} fetch-depth: 0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - # - uses: depot/setup-action@v1 + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + - uses: depot/setup-action@v1 - name: Docker meta id: meta-ee-public @@ -209,10 +144,10 @@ jobs: ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private - name: Build and push publicly ee - uses: docker/build-push-action@v6 + uses: depot/build-push-action@v1 with: context: . - # platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64 push: true build-args: | features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,otel,dind,php,mysql,mssql,bigquery,websocket,python,smtp,csharp,static_frontend,rust @@ -222,72 +157,7 @@ jobs: labels: | ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License - cache-from: type=gha - cache-to: type=gha - - build_ee_arm: - runs-on: ubicloud - if: (github.event_name != 'workflow_dispatch') || (inputs.ee == true) - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Read EE repo commit hash - run: | - echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV" - - - uses: actions/checkout@v4 - with: - repository: windmill-labs/windmill-ee-private - path: ./windmill-ee-private - ref: ${{ env.ee_repo_ref }} - token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} - fetch-depth: 0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - # - uses: depot/setup-action@v1 - - - name: Docker meta - id: meta-ee-public - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee - flavor: | - latest=false - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - - - name: Login to registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Substitute EE code - run: | - ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private - - - name: Build and push publicly ee - uses: docker/build-push-action@v6 - with: - context: . - # platforms: linux/amd64,linux/arm64 - push: true - build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,otel,dind,php,mysql,mssql,bigquery,websocket,python,smtp,csharp,static_frontend,rust - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} - ${{ steps.meta-ee-public.outputs.tags }} - labels: | - ${{ steps.meta-ee-public.outputs.labels }} - org.opencontainers.image.licenses=Windmill-Enterprise-License - cache-from: type=gha - cache-to: type=gha build_ee_312: runs-on: ubicloud if: ${{ startsWith(github.ref, 'refs/tags/v') }} @@ -308,9 +178,9 @@ jobs: token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} fetch-depth: 0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - # - uses: depot/setup-action@v1 + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + - uses: depot/setup-action@v1 - name: Docker meta id: meta-ee-public-py312 @@ -336,10 +206,10 @@ jobs: ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private - name: Build and push publicly ee - uses: docker/build-push-action@v1 + uses: depot/build-push-action@v1 with: context: . - # platforms: linux/amd64 + platforms: linux/amd64 push: true build-args: | features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,otel,dind,php,mysql,mssql,bigquery,websocket,python,smtp,csharp,static_frontend,rust @@ -349,8 +219,6 @@ jobs: labels: | ${{ steps.meta-ee-public-py312.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License - cache-from: type=gha - cache-to: type=gha # disabled until we make it 100% reliable and add more meaningful tests # playwright: @@ -474,7 +342,7 @@ jobs: # ${{ steps.extract-ee.outputs.destination }}/* run_integration_test: - runs-on: ubicloud-standard-8 + runs-on: ubicloud needs: [build_ee] steps: - uses: actions/checkout@v4 @@ -485,7 +353,7 @@ jobs: run: cd integration_tests && ./build.sh - name: Test run if: ${{ ! startsWith(github.ref, 'refs/tags/v') }} - timeout-minutes: 16 + timeout-minutes: 15 env: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} run: cd integration_tests && ./run.sh @@ -585,68 +453,16 @@ jobs: needs: [build_ee] runs-on: ubicloud if: - (github.event_name != 'pull_request') && (github.event_name != - 'workflow_dispatch') - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - # - uses: depot/setup-action@v1 - - - name: Docker meta - id: meta-ee-public - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-nsjail - flavor: | - latest=false - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=sha,enable=true,priority=100,prefix=,suffix=,format=short - type=ref,event=branch - type=ref,event=pr - - - name: Login to registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push publicly ee - uses: docker/build-push-action@v6 - with: - context: . - # platforms: linux/amd64,linux/arm64 - push: true - file: "./docker/DockerfileNsjail" - tags: | - ${{ steps.meta-ee-public.outputs.tags }} - labels: | - ${{ steps.meta-ee-public.outputs.labels }} - org.opencontainers.image.licenses=Windmill-Enterprise-License - - build_ee_nsjail_arm: - needs: [build_ee_arm] - runs-on: ubicloud - if: - (github.event_name != 'pull_request') && (github.event_name != - 'workflow_dispatch') + (github.event_name != 'pull_request') && (github.event_name != 'workflow_dispatch') steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 - # - uses: depot/setup-action@v1 + - uses: depot/setup-action@v1 - name: Docker meta id: meta-ee-public @@ -671,10 +487,10 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee - uses: docker/build-push-action@v6 + uses: depot/build-push-action@v1 with: context: . - # platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64 push: true file: "./docker/DockerfileNsjail" tags: | @@ -691,54 +507,10 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - # - uses: depot/setup-action@v1 - - - name: Login to registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker meta - id: meta-ee-public - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-reports - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=sha,enable=true,priority=100,prefix=,suffix=,format=short - - - name: Build and push publicly ee reports - uses: docker/build-push-action@v6 - with: - context: . - # platforms: linux/amd64,linux/arm64 - push: true - file: "./docker/DockerfileReports" - tags: | - ${{ steps.meta-ee-public.outputs.tags }} - labels: | - ${{ steps.meta-ee-public.outputs.labels }} - org.opencontainers.image.licenses=Windmill-Enterprise-License - - build_ee_reports_privately_arm: - needs: [build_ee_nsjail_arm] - runs-on: ubicloud - if: github.event_name != 'pull_request' - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 - # - uses: depot/setup-action@v1 + - uses: depot/setup-action@v1 - name: Login to registry uses: docker/login-action@v3 @@ -759,10 +531,10 @@ jobs: type=sha,enable=true,priority=100,prefix=,suffix=,format=short - name: Build and push publicly ee reports - uses: docker/build-push-action@v6 + uses: depot/build-push-action@v1 with: context: . - # platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64 push: true file: "./docker/DockerfileReports" tags: | @@ -772,11 +544,9 @@ jobs: org.opencontainers.image.licenses=Windmill-Enterprise-License publish_ecr_s3: - needs: [build_ee_nsjail_arm] + needs: [build_ee_nsjail] runs-on: ubicloud-standard-2-arm - if: - (github.event_name != 'pull_request') && (github.event_name != - 'workflow_dispatch') + if: (github.event_name != 'pull_request') && (github.event_name != 'workflow_dispatch') env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -820,16 +590,16 @@ jobs: build_ee_cuda: if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build_ee] - runs-on: ubicloud-standard-16 + runs-on: ubicloud steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 - # - uses: depot/setup-action@v1 + - uses: depot/setup-action@v1 - name: Docker meta id: meta-ee-public @@ -849,10 +619,10 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee - uses: docker/build-push-action@v6 + uses: depot/build-push-action@v1 with: context: . - # platforms: linux/amd64 + platforms: linux/amd64 push: true file: "./docker/DockerfileCuda" tags: | @@ -870,10 +640,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 - # - uses: depot/setup-action@v1 + - uses: depot/setup-action@v1 - name: Docker meta id: meta-ee-public @@ -893,10 +663,10 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee - uses: docker/build-push-action@v6 + uses: depot/build-push-action@v1 with: context: . - # platforms: linux/amd64 + platforms: linux/amd64 push: true file: "./docker/DockerfileSlim" tags: | @@ -913,10 +683,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 - # - uses: depot/setup-action@v1 + - uses: depot/setup-action@v1 - name: Docker meta id: meta-ee-public @@ -936,10 +706,10 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee - uses: docker/build-push-action@v1 + uses: depot/build-push-action@v1 with: context: . - # platforms: linux/amd64 + platforms: linux/amd64 push: true file: "./docker/DockerfileSlimEe" tags: | @@ -957,53 +727,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - # - uses: depot/setup-action@v1 - - - name: Docker meta - id: meta-public - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-full - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - - - name: Login to registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push publicly - uses: docker/build-push-action@v6 - with: - context: . - # platforms: linux/amd64,linux/arm64 - push: true - file: "./docker/DockerfileFull" - tags: | - ${{ steps.meta-public.outputs.tags }} - labels: | - ${{ steps.meta-public.outputs.labels }} - - build_full_arm: - if: ${{ startsWith(github.ref, 'refs/tags/v') }} - needs: [build_arm] - runs-on: ubicloud - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 - # - uses: depot/setup-action@v1 + - uses: depot/setup-action@v1 - name: Docker meta id: meta-public @@ -1023,10 +750,10 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly - uses: docker/build-push-action@v6 + uses: depot/build-push-action@v1 with: context: . - # platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64 push: true file: "./docker/DockerfileFull" tags: | @@ -1043,50 +770,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Docker meta - id: meta-ee-public - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-full - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 - - name: Login to registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push publicly ee - uses: docker/build-push-action@v6 - with: - context: . - # platforms: linux/amd64,linux/arm64 - push: true - file: "./docker/DockerfileFullEe" - tags: | - ${{ steps.meta-ee-public.outputs.tags }} - labels: | - ${{ steps.meta-ee-public.outputs.labels }} - org.opencontainers.image.licenses=Windmill-Enterprise-License - - build_ee_full_arm: - if: ${{ startsWith(github.ref, 'refs/tags/v') }} - needs: [build_ee_arm] - runs-on: ubicloud - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + - uses: depot/setup-action@v1 - name: Docker meta id: meta-ee-public @@ -1106,10 +793,10 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee - uses: docker/build-push-action@v6 + uses: depot/build-push-action@v1 with: context: . - # platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64 push: true file: "./docker/DockerfileFullEe" tags: |