From 0c610cc5978972fbab8c9d27fbdfaefd7fea2dbd Mon Sep 17 00:00:00 2001 From: mickamy Date: Mon, 8 Jan 2024 11:30:04 +0900 Subject: [PATCH] ci: run build and push images after benchmark --- .github/workflows/bench_and_build.yml | 165 ++++++++++++++++++++ .github/workflows/benchmark.yml | 52 ------ .github/workflows/build_and_push_images.yml | 112 ------------- 3 files changed, 165 insertions(+), 164 deletions(-) create mode 100644 .github/workflows/bench_and_build.yml delete mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/bench_and_build.yml b/.github/workflows/bench_and_build.yml new file mode 100644 index 0000000..97fa2bb --- /dev/null +++ b/.github/workflows/bench_and_build.yml @@ -0,0 +1,165 @@ +name: Benchmark + +on: + workflow_dispatch: + push: + +concurrency: + group: ${{ github.workflow }}-${{ github.sha }} + cancel-in-progress: true + +jobs: + benchmark: + name: Run benchmark + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + language: + - "crystal" + - "go" + - "nodejs" + - "php" + - "python" + - "ruby" + env: + ISHOCON_APP_LANG: ${{ matrix.language }} + UNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + steps: + - uses: actions/checkout@v4 + - name: Replace base image in docker-compose.yml with github actor name + run: | + make change-lang + sed -i 's/ishocon2-app-base/${{ env.UNAME }}\/ishocon2-app-base/g' ./docker-compose.yml + sed -i 's/ishocon2-bench/${{ env.UNAME }}\/ishocon2-bench/g' ./docker-compose.yml + sed -i 's/ishocon2-app-${{ env.ISHOCON_APP_LANG }}/${{ env.UNAME }}\/ishocon2-app-${{ env.ISHOCON_APP_LANG }}/g' ./docker-compose.yml + cat ./docker-compose.yml + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.UNAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + - name: Build images + run: | + make pull || true + make build + timeout-minutes: 20 + - run: make up bench-with-db-init + timeout-minutes: 10 + - name: Dump docker logs + uses: jwalton/gh-docker-logs@v2 + if: ${{ always() }} + build-base-and-bench-images: + name: Build and push base and bench images + needs: benchmark + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + UNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + steps: + - run: echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV + - uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.UNAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + - name: Cache docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Build and push bench image + uses: docker/build-push-action@v5 + with: + context: . + push: true + file: ./docker/benchmarker/Dockerfile + tags: ${{ env.UNAME }}/ishocon2-bench:latest,${{ env.UNAME }}/ishocon2-bench:${{ env.DATE }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + platforms: linux/amd64,linux/arm64/v8 + - name: Build and push base image + uses: docker/build-push-action@v5 + with: + context: . + push: true + file: ./docker/app/base/Dockerfile + tags: ${{ env.UNAME }}/ishocon2-app-base:latest,${{ env.UNAME }}/ishocon2-app-base:${{ env.DATE }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + platforms: linux/amd64,linux/arm64/v8 + - name: Move new cache to the place where to be cached + run: | + echo "Temporary fix for cleaning up old cache." + echo "See isssues: + - https://github.com/docker/build-push-action/issues/252 + - https://github.com/moby/buildkit/issues/1896 + " + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + build-app-images: + name: Build app images + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: build-base-and-bench-images + strategy: + fail-fast: false + matrix: + language: + - "crystal" + - "go" + - "nodejs" + - "php" + - "python" + - "ruby" + env: + ISHOCON_APP_LANG: ${{ matrix.language }} + UNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + steps: + - run: echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV + - uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.UNAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + - name: Cache docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ matrix.language }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-${{ matrix.language }} + ${{ runner.os }}-buildx + - name: Build and push app image + uses: docker/build-push-action@v5 + with: + context: . + push: true + file: ./docker/app/${{ env.ISHOCON_APP_LANG }}/Dockerfile + tags: ${{ env.UNAME }}/ishocon2-app-${{ env.ISHOCON_APP_LANG }}:latest,${{ env.UNAME }}/ishocon2-app-${{ env.ISHOCON_APP_LANG }}:${{ env.DATE }} + build-args: BASE_IMAGE=${{ env.UNAME }}/ishocon2-app-base:latest + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + platforms: linux/amd64,linux/arm64/v8 + - name: Move new cache to the place where to be cached + run: | + echo "Temporary fix for cleaning up old cache." + echo "See isssues: + - https://github.com/docker/build-push-action/issues/252 + - https://github.com/moby/buildkit/issues/1896 + " + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index 3f4e913..0000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Benchmark - -on: - workflow_dispatch: - push: - -concurrency: - group: ${{ github.workflow }}-${{ github.sha }} - cancel-in-progress: true - -jobs: - benchmark: - name: Run benchmark - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - language: - - "crystal" - - "go" - - "nodejs" - - "php" - - "python" - - "ruby" - env: - ISHOCON_APP_LANG: ${{ matrix.language }} - UNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - steps: - - uses: actions/checkout@v4 - - name: Replace base image in docker-compose.yml with github actor name - run: | - make change-lang - sed -i 's/ishocon2-app-base/${{ env.UNAME }}\/ishocon2-app-base/g' ./docker-compose.yml - sed -i 's/ishocon2-bench/${{ env.UNAME }}\/ishocon2-bench/g' ./docker-compose.yml - sed -i 's/ishocon2-app-${{ env.ISHOCON_APP_LANG }}/${{ env.UNAME }}\/ishocon2-app-${{ env.ISHOCON_APP_LANG }}/g' ./docker-compose.yml - cat ./docker-compose.yml - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ env.UNAME }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: Build images - run: | - make pull || true - make build - timeout-minutes: 20 - - run: make up bench-with-db-init - timeout-minutes: 10 - - name: Dump docker logs - uses: jwalton/gh-docker-logs@v2 - if: ${{ always() }} diff --git a/.github/workflows/build_and_push_images.yml b/.github/workflows/build_and_push_images.yml index fc63140..cdad2bb 100644 --- a/.github/workflows/build_and_push_images.yml +++ b/.github/workflows/build_and_push_images.yml @@ -11,115 +11,3 @@ concurrency: cancel-in-progress: true jobs: - build-and-push-base-and-bench-images: - name: Build and push base and bench images - runs-on: ubuntu-latest - timeout-minutes: 60 - env: - UNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - steps: - - run: echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV - - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ env.UNAME }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: Cache docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Build and push bench image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: ./docker/benchmarker/Dockerfile - tags: ${{ env.UNAME }}/ishocon2-bench:latest,${{ env.UNAME }}/ishocon2-bench:${{ env.DATE }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - platforms: linux/amd64,linux/arm64/v8 - - name: Build and push base image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: ./docker/app/base/Dockerfile - tags: ${{ env.UNAME }}/ishocon2-app-base:latest,${{ env.UNAME }}/ishocon2-app-base:${{ env.DATE }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - platforms: linux/amd64,linux/arm64/v8 - - name: Move new cache to the place where to be cached - run: | - echo "Temporary fix for cleaning up old cache." - echo "See isssues: - - https://github.com/docker/build-push-action/issues/252 - - https://github.com/moby/buildkit/issues/1896 - " - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache - build-app-images: - name: Build app images - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: build-and-push-base-and-bench-images - strategy: - fail-fast: false - matrix: - language: - - "crystal" - - "go" - - "nodejs" - - "php" - - "python" - - "ruby" - env: - ISHOCON_APP_LANG: ${{ matrix.language }} - UNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - steps: - - run: echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV - - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ env.UNAME }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: Cache docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ matrix.language }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-${{ matrix.language }} - ${{ runner.os }}-buildx - - name: Build and push app image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: ./docker/app/${{ env.ISHOCON_APP_LANG }}/Dockerfile - tags: ${{ env.UNAME }}/ishocon2-app-${{ env.ISHOCON_APP_LANG }}:latest,${{ env.UNAME }}/ishocon2-app-${{ env.ISHOCON_APP_LANG }}:${{ env.DATE }} - build-args: BASE_IMAGE=${{ env.UNAME }}/ishocon2-app-base:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - platforms: linux/amd64,linux/arm64/v8 - - name: Move new cache to the place where to be cached - run: | - echo "Temporary fix for cleaning up old cache." - echo "See isssues: - - https://github.com/docker/build-push-action/issues/252 - - https://github.com/moby/buildkit/issues/1896 - " - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache