update intermediate docker workflow (#1296)

Co-authored-by: johnsonjie <xiaojie@scroll.io> Co-authored-by: sebastien.baizet <sebastien.baizet@gmail.com> Co-authored-by: georgehao <haohongfan@gmail.com>
scroll-tech · Jun 12, 2024 · 6dd09fe · 6dd09fe
1 parent 75c81d5
commit 6dd09fe
Show file tree

Hide file tree

Showing 2 changed files with 83 additions and 183 deletions.
diff --git a/.github/workflows/intermediate-docker.yml b/.github/workflows/intermediate-docker.yml
@@ -4,113 +4,65 @@ on:
   workflow_dispatch:
     inputs:
       GO_VERSION:
-        description: 'Go version'
+        description: "Go version"
         required: true
-        type: string
-        default: '1.21'
+        type: choice
+        options:
+          - "1.20"
+          - "1.21"
+          - "1.22"
+          - "1.23"
+        default: "1.21"
       RUST_VERSION:
-        description: 'Rust toolchain version'
+        description: "Rust toolchain version"
         required: true
-        type: string
-        default: 'nightly-2023-12-03'
+        type: choice
+        options:
+          - nightly-2023-12-03
+          - nightly-2022-12-10
+        default: "nightly-2023-12-03"
       PYTHON_VERSION:
-        description: 'Python version'
+        description: "Python version"
         required: false
-        type: string
-        default: '3.10'
+        type: choice
+        options:
+          - "3.10"
+        default: "3.10"
       CUDA_VERSION:
-        description: 'Cuda version'
+        description: "Cuda version"
         required: false
-        type: string
-        default: '11.7.1'
+        type: choice
+        options:
+          - "11.7.1"
+          - "12.2.2"
+        default: "11.7.1"
+      CARGO_CHEF_TAG:
+        description: "Cargo chef version"
+        required: true
+        default: "0.1.41"
+        type: choice
+        options:
+          - 0.1.41
+      BASE_IMAGE:
+        description: "which intermediate image you want to update"
+        required: true
+        default: "go-alpine-builder"
+        type: choice
+        options:
+          - cuda-go-rust-builder
+          - go-rust-builder
+          - go-alpine-builder
+          - rust-builder
+          - rust-alpine-builder
+          - go-rust-alpine-builder
+          - py-runner
 
 defaults:
   run:
-    working-directory: 'build/dockerfiles/intermediate'
+    working-directory: "build/dockerfiles/intermediate"
 
 jobs:
-  build-and-publish-cuda-go-rust-builder:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build image
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          file: build/dockerfiles/intermediate/cuda-go-rust-builder.Dockerfile
-          tags: scrolltech/cuda-go-rust-builder:cuda-${{ github.event.inputs.CUDA_VERSION }}-go-${{ github.event.inputs.GO_VERSION }}-rust-${{ github.event.inputs.RUST_VERSION }}
-          build-args: |
-            CUDA_VERSION: ${{ github.event.inputs.CUDA_VERSION }}
-            GO_VERSION: ${{ github.event.inputs.GO_VERSION }}
-            RUST_VERSION: ${{ github.event.inputs.RUST_VERSION }}
-
-  build-and-publish-go-rust-builder:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build image
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          file: build/dockerfiles/intermediate/go-rust-builder.Dockerfile
-          platforms: linux/amd64,linux/arm64
-          tags: scrolltech/go-rust-builder:go-${{ github.event.inputs.GO_VERSION }}-rust-${{ github.event.inputs.RUST_VERSION }}
-          build-args: |
-            GO_VERSION: ${{ github.event.inputs.GO_VERSION }}
-            RUST_VERSION: ${{ github.event.inputs.RUST_VERSION }}
-
-  build-and-publish-go-alpine-builder:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build image
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          file: build/dockerfiles/intermediate/go-alpine-builder.Dockerfile
-          platforms: linux/amd64,linux/arm64
-          tags: scrolltech/go-alpine-builder:${{ github.event.inputs.GO_VERSION }}
-          build-args: |
-            GO_VERSION: ${{ github.event.inputs.GO_VERSION }}
-            RUST_VERSION: ${{ github.event.inputs.RUST_VERSION }}
-
-  build-and-publish-rust-builder:
+  build-and-publish-intermediate:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -124,96 +76,37 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: set tag env
+        run: |
+          if [ ${{github.event.inputs.BASE_IMAGE}} == "cuda-go-rust-builder" ]; then
+            echo "TAG=cuda-${{ github.event.inputs.CUDA_VERSION }}-go-${{ github.event.inputs.GO_VERSION }}-rust-${{ github.event.inputs.RUST_VERSION }}" >> $GITHUB_ENV
+          elif [ ${{github.event.inputs.BASE_IMAGE}} == "go-rust-builder" ]; then
+            echo "TAG=go-${{ github.event.inputs.GO_VERSION }}-rust-${{ github.event.inputs.RUST_VERSION }}" >> $GITHUB_ENV
+          elif [ ${{github.event.inputs.BASE_IMAGE}} == "go-alpine-builder" ]; then
+            echo "TAG=${{ github.event.inputs.GO_VERSION }}" >> $GITHUB_ENV
+          elif [ ${{github.event.inputs.BASE_IMAGE}} == "rust-builder" ]; then
+            echo "TAG=${{ github.event.inputs.RUST_VERSION }}" >> $GITHUB_ENV
+          elif [ ${{github.event.inputs.BASE_IMAGE}} == "rust-alpine-builder" ]; then
+            echo "TAG=${{ github.event.inputs.RUST_VERSION }}" >> $GITHUB_ENV
+          elif [ ${{github.event.inputs.BASE_IMAGE}} == "go-rust-alpine-builder" ]; then
+            echo "TAG=go-${{ github.event.inputs.GO_VERSION }}-rust-${{ github.event.inputs.RUST_VERSION }}" >> $GITHUB_ENV
+          elif [ ${{github.event.inputs.BASE_IMAGE}} == "py-runner" ]; then
+            echo "TAG=${{ github.event.inputs.PYTHON_VERSION }}" >> $GITHUB_ENV
+          else
+            echo "no BASE_IMAGE match"
+          fi
       - name: Build image
         id: build
         uses: docker/build-push-action@v5
         with:
           context: .
           push: true
-          file: build/dockerfiles/intermediate/rust-builder.Dockerfile
+          file: build/dockerfiles/intermediate/${{ github.event.inputs.BASE_IMAGE }}.Dockerfile
           platforms: linux/amd64,linux/arm64
-          tags: scrolltech/rust-builder:${{ github.event.inputs.RUST_VERSION }}
+          tags: scrolltech/${{ github.event.inputs.BASE_IMAGE }}:${{ env.TAG }}
           build-args: |
-            RUST_VERSION: ${{ github.event.inputs.RUST_VERSION }}
-
-  build-and-publish-rust-alpine-builder:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build image
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          file: build/dockerfiles/intermediate/rust-alpine-builder.Dockerfile
-          platforms: linux/amd64,linux/arm64
-          tags: scrolltech/rust-alpine-builder:${{ github.event.inputs.RUST_VERSION }}
-          build-args: |
-            RUST_VERSION: ${{ github.event.inputs.RUST_VERSION }}
-
-  build-and-publish-go-rust-alpine-builder:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build image
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          file: build/dockerfiles/intermediate/go-rust-alpine-builder.Dockerfile
-          platforms: linux/amd64,linux/arm64
-          tags: scrolltech/go-rust-alpine-builder:go-${{ github.event.inputs.GO_VERSION }}-rust-${{ github.event.inputs.RUST_VERSION }}
-          build-args: |
-            GO_VERSION: ${{ github.event.inputs.GO_VERSION }}
-            RUST_VERSION: ${{ github.event.inputs.RUST_VERSION }}
-
-  build-and-publish-py-runner:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build image
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          file: build/dockerfiles/intermediate/py-runner.Dockerfile
-          platforms: linux/amd64,linux/arm64
-          tags: scrolltech/py-runner:${{ github.event.inputs.PYTHON_VERSION }}
-          build-args: |
-            CUDA_VERSION: ${{ github.event.inputs.CUDA_VERSION }}
-            GO_VERSION: ${{ github.event.inputs.GO_VERSION }}
-            RUST_VERSION: ${{ github.event.inputs.RUST_VERSION }}
-
+            CUDA_VERSION=${{ github.event.inputs.CUDA_VERSION }}
+            GO_VERSION=${{ github.event.inputs.GO_VERSION }}
+            RUST_VERSION=${{ github.event.inputs.RUST_VERSION }}
+            PYTHON_VERSION=${{ github.event.inputs.PYTHON_VERSION }}
+            CARGO_CHEF_TAG=${{ github.event.inputs.CARGO_CHEF_TAG }}
diff --git a/build/dockerfiles/intermediate/go-rust-builder.Dockerfile b/build/dockerfiles/intermediate/go-rust-builder.Dockerfile
@@ -25,7 +25,14 @@ RUN cargo install cargo-chef --locked --version ${CARGO_CHEF_TAG} \
 # Install Go
 ARG GO_VERSION
 RUN rm -rf /usr/local/go
-RUN wget https://go.dev/dl/go${GO_VERSION}.1.linux-amd64.tar.gz
-RUN tar -C /usr/local -xzf go${GO_VERSION}.1.linux-amd64.tar.gz
-RUN rm go${GO_VERSION}.1.linux-amd64.tar.gz
+RUN if [ "$(uname -m)" = "x86_64" ]; then \
+    echo amd64 >/tmp/arch; \
+    elif [ "$(uname -m)" = "aarch64" ]; then \
+    echo arm64 >/tmp/arch; \
+    else \
+    echo "Unsupported architecture"; exit 1; \
+    fi
+RUN wget https://go.dev/dl/go${GO_VERSION}.1.linux-$(cat /tmp/arch).tar.gz
+RUN tar -C /usr/local -xzf go${GO_VERSION}.1.linux-$(cat /tmp/arch).tar.gz
+RUN rm go${GO_VERSION}.1.linux-$(cat /tmp/arch).tar.gz && rm /tmp/arch
 ENV PATH="/usr/local/go/bin:${PATH}"