Merge branch 'main' into ci-throttler-grpc

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
vitessio · Oct 30, 2024 · 79d2ac0 · 79d2ac0
2 parents a34b8d9 + 9c9a893
commit 79d2ac0
Show file tree

Hide file tree

Showing 285 changed files with 44,939 additions and 20,714 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -77,7 +77,7 @@ go.sum @ajm188 @deepthi @harshit-gangal @mattlord @rohit-nayak-ps @systay @froui
 /go/vt/vttablet/*tmclient* @ajm188 @GuptaManan100 @rohit-nayak-ps @shlomi-noach
 /go/vt/vttablet/vexec @mattlord @rohit-nayak-ps @shlomi-noach
 /go/vt/wrangler @deepthi @mattlord @rohit-nayak-ps
-/go/vt/workflow @mattlord @rohit-nayak-ps
+/go/vt/vtctl/workflow @mattlord @rohit-nayak-ps
 /proto/ @deepthi @harshit-gangal
 /proto/vtadmin.proto @ajm188 @notfelineit @mattlord
 /proto/vtctldata.proto @ajm188 @notfelineit @mattlord

diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
@@ -16,24 +16,18 @@ jobs:
     permissions:
       contents: write
     steps:
+    - name: Check out code
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
     - name: Set up Go
       uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
       with:
         go-version-file: go.mod
 
-    - name: Setup node
-      uses: actions/setup-node@v4
-      with:
-        node-version: '20.12.2'
-
     - name: Tune the OS
       run: |
         sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535"
 
-    - name: Check out code
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
-
     - name: Get dependencies
       run: |
         sudo apt-get update

diff --git a/.github/workflows/docker_build_images.yml b/.github/workflows/docker_build_images.yml
@@ -1,4 +1,4 @@
-name: Docker Build Images
+name: Build Docker Images
 on:
   push:
     branches:
@@ -8,14 +8,68 @@ on:
   workflow_dispatch:
 
 concurrency:
-  group: format('{0}-{1}', ${{ github.ref }}, 'Docker Build Images (v20+)')
+  group: format('{0}-{1}', ${{ github.ref }}, 'Build Docker Images')
   cancel-in-progress: true
 
 permissions: read-all
 
 jobs:
+  build_and_push_vttestserver:
+    name: Build and push vttestserver
+    runs-on: gh-hosted-runners-16cores-1
+    if: github.repository == 'vitessio/vitess'
+
+    strategy:
+      fail-fast: true
+      matrix:
+        branch: [ mysql80 ]
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set Dockerfile path
+        run: |
+          echo "DOCKERFILE=./docker/vttestserver/Dockerfile.${{ matrix.branch }}" >> $GITHUB_ENV
+
+      - name: Build and push on main
+        if: startsWith(github.ref, 'refs/tags/') == false
+        uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0
+        with:
+          context: .
+          file: ${{ env.DOCKERFILE }}
+          push: true
+          tags: vitess/vttestserver:${{ matrix.branch }}
+
+      ######
+      # All code below only applies to new tags
+      ######
+      - name: Get the Git tag
+        if: startsWith(github.ref, 'refs/tags/')
+        run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+
+      - name: Set Docker tag name
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          echo "DOCKER_TAG=vitess/vttestserver:${TAG_NAME}-${{ matrix.branch }}" >> $GITHUB_ENV
+
+      - name: Build and push on tags
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0
+        with:
+          context: .
+          file: ${{ env.DOCKERFILE }}
+          push: true
+          tags: ${{ env.DOCKER_TAG }}
+
   build_and_push_lite:
-    name: Build and push vitess/lite Docker images
+    name: Build and push lite
     runs-on: ubuntu-latest
     if: github.repository == 'vitessio/vitess'
 
@@ -26,10 +80,10 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -43,8 +97,8 @@ jobs:
           fi
 
       - name: Build and push on main
-        if: github.ref == 'refs/heads/main'
-        uses: docker/build-push-action@v5
+        if: startsWith(github.ref, 'refs/tags/') == false
+        uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0
         with:
           context: .
           file: ${{ env.DOCKERFILE }}
@@ -69,31 +123,32 @@ jobs:
 
       - name: Build and push on tags
         if: startsWith(github.ref, 'refs/tags/')
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0
         with:
           context: .
           file: ${{ env.DOCKERFILE }}
           push: true
           tags: ${{ env.DOCKER_TAG }}
 
   build_and_push_components:
-    name: Build and push vitess components Docker images
-    needs: build_and_push_lite
+    name: Build and push
     runs-on: gh-hosted-runners-16cores-1
-    if: github.repository == 'vitessio/vitess'
+    if: github.repository == 'vitessio/vitess' && needs.build_and_push_lite.result == 'success'
+    needs:
+      - build_and_push_lite
 
     strategy:
       fail-fast: true
       matrix:
         debian: [ bullseye, bookworm ]
-        component: [ vtadmin, vtorc, vtgate, vttablet, mysqlctld, mysqlctl, vtctl, vtctlclient, vtctld, logrotate, logtail, vtbackup, vtexplain ]
+        component: [ vtadmin, vtorc, vtgate, vttablet, mysqlctld, mysqlctl, vtctl, vtctlclient, vtctld, vtctldclient, logrotate, logtail, vtbackup, vtexplain ]
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -103,8 +158,8 @@ jobs:
           echo "DOCKER_CTX=./docker/binaries/${{ matrix.component }}" >> $GITHUB_ENV
 
       - name: Build and push on main latest tag
-        if: github.ref == 'refs/heads/main' && matrix.debian == 'bookworm'
-        uses: docker/build-push-action@v5
+        if: startsWith(github.ref, 'refs/tags/') == false && matrix.debian == 'bookworm'
+        uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0
         with:
           context: ${{ env.DOCKER_CTX }}
           push: true
@@ -114,8 +169,8 @@ jobs:
             DEBIAN_VER=${{ matrix.debian }}-slim
 
       - name: Build and push on main debian specific tag
-        if: github.ref == 'refs/heads/main'
-        uses: docker/build-push-action@v5
+        if: startsWith(github.ref, 'refs/tags/') == false
+        uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0
         with:
           context: ${{ env.DOCKER_CTX }}
           push: true
@@ -147,7 +202,7 @@ jobs:
       # Build and Push component image to DOCKER_TAG, applies to both debian version
       - name: Build and push on tags using Debian extension
         if: startsWith(github.ref, 'refs/tags/')
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0
         with:
           context: ${{ env.DOCKER_CTX }}
           push: true
@@ -160,11 +215,29 @@ jobs:
       # It is fine to build a second time here when "matrix.debian == 'bookworm'" as we have cached the first build already
       - name: Build and push on tags without Debian extension
         if: startsWith(github.ref, 'refs/tags/') && matrix.debian == 'bookworm'
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0
         with:
           context: ${{ env.DOCKER_CTX }}
           push: true
           tags: ${{ env.DOCKER_TAG_DEFAULT_DEBIAN }}
           build-args: |
             VT_BASE_VER=${{ env.TAG_NAME }}
-            DEBIAN_VER=${{ matrix.debian }}-slim
+            DEBIAN_VER=${{ matrix.debian }}-slim
+
+  slack_notification:
+    name: Slack Notification if failed
+    runs-on: ubuntu-latest
+    needs:
+      - build_and_push_vttestserver
+      - build_and_push_lite
+      - build_and_push_components
+    if: ${{ failure() }}
+    steps:
+    - name: Slack Workflow Notification
+      uses: Gamesight/slack-workflow-status@68bf00d0dbdbcb206c278399aa1ef6c14f74347a # v1.3.0
+      with:
+        repo_token: ${{secrets.GITHUB_TOKEN}}
+        slack_webhook_url: ${{secrets.SLACK_WEBHOOK_URL}}
+        channel: '#docker-build-notifications'
+        name: 'Docker Build Notification'
+        icon_url: https://avatars.githubusercontent.com/u/33043890?s=96&v=4
diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml
@@ -15,7 +15,7 @@ jobs:
       pull-requests: write
     strategy:
       matrix:
-        branch: [ main, release-20.0, release-19.0, release-18.0 ]
+        branch: [ main, release-21.0, release-20.0, release-19.0, release-18.0 ]
     name: Update Golang Version
     runs-on: ubuntu-latest
     steps:

diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml
@@ -117,43 +117,46 @@ jobs:
         # install JUnit report formatter
         go install github.com/vitessio/go-junit-report@HEAD
 
-    # Build current commit's binaries
-    - name: Get dependencies for this commit
+    # Checkout to the next release of Vitess
+    - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }})
+      if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      with:
+        ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }}
+
+    - name: Get dependencies for the next release
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
         go mod download
 
-    - name: Building the binaries for this commit
+    - name: Building next release's binaries
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       timeout-minutes: 10
       run: |
         source build.env
         NOVTADMINBUILD=1 make build
-        mkdir -p /tmp/vitess-build-current/
-        cp -R bin /tmp/vitess-build-current/
+        mkdir -p /tmp/vitess-build-other/
+        cp -R bin /tmp/vitess-build-other/
         rm -Rf bin/*
 
-    # Checkout to the next release of Vitess
-    - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }})
+    # Checkout to this build's commit
+    - name: Check out commit's code
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
-      with:
-        ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }}
 
-    - name: Get dependencies for the next release
+    - name: Get dependencies for this commit
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
         go mod download
 
-    - name: Building next release's binaries
+    - name: Building the binaries for this commit
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       timeout-minutes: 10
       run: |
         source build.env
         NOVTADMINBUILD=1 make build
-        mkdir -p /tmp/vitess-build-other/
-        cp -R bin /tmp/vitess-build-other/
-        rm -Rf bin/*
+        mkdir -p /tmp/vitess-build-current/
+        cp -R bin /tmp/vitess-build-current/
 
     - name: Convert ErrorContains checks to Error checks
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
@@ -166,8 +169,6 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
         source build.env
-
-        cp -r /tmp/vitess-build-current/bin/* $PWD/bin/
         rm -f $PWD/bin/vtgate
         cp /tmp/vitess-build-other/bin/vtgate $PWD/bin/vtgate
         vtgate --version

diff --git a/.github/workflows/vitess_tester_vtgate.yml b/.github/workflows/vitess_tester_vtgate.yml
@@ -112,7 +112,7 @@ jobs:
         go install github.com/vitessio/go-junit-report@HEAD
         
         # install vitess tester
-        go install github.com/vitessio/vitess-tester/go/vt@374fd9f495c1afd3b6bea9d4ec7728119714055
+        go install github.com/vitessio/vt/go/vt@e43009309f599378504905d4b804460f47822ac5
 
     - name: Setup launchable dependencies
       if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main'
@@ -142,7 +142,7 @@ jobs:
         i=1
         for dir in ./go/test/endtoend/vtgate/vitess_tester/*/; do 
           # We go over all the directories in the given path.
-          # If there is a vschema file there, we use it, otherwise we let vitess-tester autogenerate it.
+          # If there is a vschema file there, we use it, otherwise we let vt tester autogenerate it.
           if [ -f $dir/vschema.json ]; then
             vt tester --xunit --vschema "$dir"vschema.json $dir/*.test
           else