From 3bcd6c6ac1baeb0ce8738b898fc31e0a6e4ef6b9 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 9 Jul 2024 15:46:06 +0530 Subject: [PATCH 1/5] test: add test with workflow that verifies that downgrading from v20 to v19 is broken Signed-off-by: Manan Gupta --- .../upgrade_downgrade_test_semi_sync.yml | 164 ++++++++++++++++++ .../reparent/semisync/semi_sync_test.go | 64 +++++++ 2 files changed, 228 insertions(+) create mode 100644 .github/workflows/upgrade_downgrade_test_semi_sync.yml create mode 100644 go/test/endtoend/reparent/semisync/semi_sync_test.go diff --git a/.github/workflows/upgrade_downgrade_test_semi_sync.yml b/.github/workflows/upgrade_downgrade_test_semi_sync.yml new file mode 100644 index 00000000000..ffbd9449f61 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_semi_sync.yml @@ -0,0 +1,164 @@ +name: Semi Sync Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Semi Sync Upgrade Downgrade Testing') + cancel-in-progress: true + +permissions: read-all + +jobs: + upgrade_downgrade_test_e2e: + timeout-minutes: 60 + name: Run Semi Sync Upgrade Downgrade Test + runs-on: gh-hosted-runners-16cores-1 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + if: steps.skip-workflow.outputs.skip-workflow == 'false' + id: output-previous-release-ref + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_semi_sync.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v5 + with: + go-version: 1.22.5 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v5 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release enable-only tools + sudo apt-get update + sudo apt-get install -y percona-xtrabackup-80 + + # Checkout to the last release of Vitess + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@v4 + with: + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + + - name: Get dependencies for the last release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building last release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@v4 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + # Copy last releases vttablet + - name: Copy last release's VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttabletold + vttabletold --version + + - name: Run semi sync tests + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + set -x + source build.env + go test -v -count=1 -run="" ./go/test/endtoend/reparent/semisync -alsologtostderr diff --git a/go/test/endtoend/reparent/semisync/semi_sync_test.go b/go/test/endtoend/reparent/semisync/semi_sync_test.go new file mode 100644 index 00000000000..4be5d71f622 --- /dev/null +++ b/go/test/endtoend/reparent/semisync/semi_sync_test.go @@ -0,0 +1,64 @@ +package semisync + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/reparent/utils" +) + +func TestSemiSyncUpgradeDowngrade(t *testing.T) { + defer cluster.PanicHandler(t) + clusterInstance := utils.SetupReparentCluster(t, "semi_sync") + defer utils.TeardownCluster(clusterInstance) + tablets := clusterInstance.Keyspaces[0].Shards[0].Vttablets + + // Verify that replication is running as intended. + utils.ConfirmReplication(t, tablets[0], []*cluster.Vttablet{tablets[1], tablets[2], tablets[3]}) + + replica := tablets[1] + // Verify we are using the correct vttablet version. + verifyVttabletVersion(t, replica, 20) + // Check the plugin loaded in vttablet. + require.EqualValues(t, mysql.SemiSyncTypeSource, semiSyncExtensionLoaded(t, replica)) + + // change vttablet binary and downgrade it. + changeVttabletBinary(t, replica, "vttabletold") + // Verify we are using the older vttablet version. + verifyVttabletVersion(t, replica, 19) +} + +// semiSyncExtensionLoaded checks if the semisync extension has been loaded. +// It should work for both MariaDB and MySQL. +func semiSyncExtensionLoaded(t *testing.T, replica *cluster.Vttablet) mysql.SemiSyncType { + qr := utils.RunSQL(context.Background(), t, `SHOW VARIABLES LIKE 'rpl_semi_sync_%_enabled'`, replica) + for _, row := range qr.Rows { + if row[0].ToString() == "rpl_semi_sync_source_enabled" { + return mysql.SemiSyncTypeSource + } + if row[0].ToString() == "rpl_semi_sync_master_enabled" { + return mysql.SemiSyncTypeMaster + } + } + return mysql.SemiSyncTypeOff +} + +func changeVttabletBinary(t *testing.T, replica *cluster.Vttablet, binary string) { + t.Helper() + err := replica.VttabletProcess.TearDown() + require.NoError(t, err) + replica.VttabletProcess.Binary = binary + err = replica.VttabletProcess.Setup() + require.NoError(t, err) +} + +func verifyVttabletVersion(t *testing.T, replica *cluster.Vttablet, version int) { + t.Helper() + verGot, err := cluster.GetMajorVersion(replica.VttabletProcess.Binary) + require.NoError(t, err) + require.EqualValues(t, version, verGot) +} From 73d262765a9bf3e4c87b10426c59c5399e0af338 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 9 Jul 2024 16:08:19 +0530 Subject: [PATCH 2/5] feat: use the old plugins in v20 to fix the upgrade downgrade problem Signed-off-by: Manan Gupta --- config/mycnf/mysql8026.cnf | 6 +++--- config/mycnf/mysql84.cnf | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/mycnf/mysql8026.cnf b/config/mycnf/mysql8026.cnf index c7755be488f..2488db029b9 100644 --- a/config/mycnf/mysql8026.cnf +++ b/config/mycnf/mysql8026.cnf @@ -20,13 +20,13 @@ default_authentication_plugin = mysql_native_password # # VTTablet will enable semi-sync at the proper time when replication is set up, # or when a primary is promoted or demoted based on the durability policy configured. -plugin-load = rpl_semi_sync_source=semisync_source.so;rpl_semi_sync_replica=semisync_replica.so +plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so # MySQL 8.0.26 and later will not load plugins during --initialize # which makes these options unknown. Prefixing with --loose # tells the server it's fine if they are not understood. -loose_rpl_semi_sync_source_timeout = 1000000000000000000 -loose_rpl_semi_sync_source_wait_no_replica = 1 +loose_rpl_semi_sync_master_timeout = 1000000000000000000 +loose_rpl_semi_sync_master_wait_no_slave = 1 # In order to protect against any errand GTIDs we will start the mysql instance # in super-read-only mode. diff --git a/config/mycnf/mysql84.cnf b/config/mycnf/mysql84.cnf index 90d7a535602..db51a76d19f 100644 --- a/config/mycnf/mysql84.cnf +++ b/config/mycnf/mysql84.cnf @@ -22,13 +22,13 @@ default_authentication_plugin = mysql_native_password # # VTTablet will enable semi-sync at the proper time when replication is set up, # or when a primary is promoted or demoted based on the durability policy configured. -plugin-load = rpl_semi_sync_source=semisync_source.so;rpl_semi_sync_replica=semisync_replica.so +plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so # MySQL 8.0.26 and later will not load plugins during --initialize # which makes these options unknown. Prefixing with --loose # tells the server it's fine if they are not understood. -loose_rpl_semi_sync_source_timeout = 1000000000000000000 -loose_rpl_semi_sync_source_wait_no_replica = 1 +loose_rpl_semi_sync_master_timeout = 1000000000000000000 +loose_rpl_semi_sync_master_wait_no_slave = 1 # In order to protect against any errand GTIDs we will start the mysql instance # in super-read-only mode. From caec7d65a800a0be6bf523072a23d3b5289a1c9d Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 9 Jul 2024 16:08:53 +0530 Subject: [PATCH 3/5] test: update test to also verify upgrades work fine and new plugins are still supported in v20 Signed-off-by: Manan Gupta --- .../reparent/semisync/semi_sync_test.go | 44 ++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/go/test/endtoend/reparent/semisync/semi_sync_test.go b/go/test/endtoend/reparent/semisync/semi_sync_test.go index 4be5d71f622..63d11daf2b6 100644 --- a/go/test/endtoend/reparent/semisync/semi_sync_test.go +++ b/go/test/endtoend/reparent/semisync/semi_sync_test.go @@ -24,12 +24,46 @@ func TestSemiSyncUpgradeDowngrade(t *testing.T) { // Verify we are using the correct vttablet version. verifyVttabletVersion(t, replica, 20) // Check the plugin loaded in vttablet. - require.EqualValues(t, mysql.SemiSyncTypeSource, semiSyncExtensionLoaded(t, replica)) + require.EqualValues(t, mysql.SemiSyncTypeMaster, semiSyncExtensionLoaded(t, replica)) - // change vttablet binary and downgrade it. - changeVttabletBinary(t, replica, "vttabletold") - // Verify we are using the older vttablet version. - verifyVttabletVersion(t, replica, 19) + t.Run("Downgrade to v19", func(t *testing.T) { + // change vttablet binary and downgrade it. + changeVttabletBinary(t, replica, "vttabletold") + // Verify we are using the older vttablet version. + verifyVttabletVersion(t, replica, 19) + // Verify that replication is running as intended. + utils.ConfirmReplication(t, tablets[0], []*cluster.Vttablet{tablets[1], tablets[2], tablets[3]}) + // Check the plugin loaded in vttablet. + require.EqualValues(t, mysql.SemiSyncTypeMaster, semiSyncExtensionLoaded(t, replica)) + }) + + t.Run("Upgrade to v19", func(t *testing.T) { + // change vttablet binary and downgrade it. + changeVttabletBinary(t, replica, "vttablet") + // Verify we are using the older vttablet version. + verifyVttabletVersion(t, replica, 20) + // Verify that replication is running as intended. + utils.ConfirmReplication(t, tablets[0], []*cluster.Vttablet{tablets[1], tablets[2], tablets[3]}) + // Check the plugin loaded in vttablet. + require.EqualValues(t, mysql.SemiSyncTypeMaster, semiSyncExtensionLoaded(t, replica)) + }) + + t.Run("Change the semi-sync plugin", func(t *testing.T) { + // Change MySQL plugins loaded. + utils.RunSQLs(context.Background(), t, []string{ + `SET GLOBAL READ_ONLY=OFF`, + `STOP REPLICA;`, + `UNINSTALL PLUGIN rpl_semi_sync_master;`, + `UNINSTALL PLUGIN rpl_semi_sync_slave;`, + `INSTALL PLUGIN rpl_semi_sync_source SONAME 'semisync_source.so';`, + `INSTALL PLUGIN rpl_semi_sync_replica SONAME 'semisync_replica.so';`, + `START REPLICA;`, + }, replica) + // Check the plugin loaded in vttablet. + require.EqualValues(t, mysql.SemiSyncTypeSource, semiSyncExtensionLoaded(t, replica)) + // Verify that replication is running as intended. + utils.ConfirmReplication(t, tablets[0], []*cluster.Vttablet{tablets[1], tablets[2], tablets[3]}) + }) } // semiSyncExtensionLoaded checks if the semisync extension has been loaded. From 56787f94b5d172c5271bf76ef19dd19d9b597058 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 11 Jul 2024 10:29:35 +0530 Subject: [PATCH 4/5] add license header Signed-off-by: Manan Gupta --- .../endtoend/reparent/semisync/semi_sync_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/go/test/endtoend/reparent/semisync/semi_sync_test.go b/go/test/endtoend/reparent/semisync/semi_sync_test.go index 63d11daf2b6..b0cb9cc1aec 100644 --- a/go/test/endtoend/reparent/semisync/semi_sync_test.go +++ b/go/test/endtoend/reparent/semisync/semi_sync_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package semisync import ( From f71e5e0fcba9c140509a55f29bba08778fb5ee9d Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 12 Jul 2024 20:22:46 +0530 Subject: [PATCH 5/5] revert changes to mysql84 cnf Signed-off-by: Manan Gupta --- config/mycnf/mysql84.cnf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/mycnf/mysql84.cnf b/config/mycnf/mysql84.cnf index db51a76d19f..90d7a535602 100644 --- a/config/mycnf/mysql84.cnf +++ b/config/mycnf/mysql84.cnf @@ -22,13 +22,13 @@ default_authentication_plugin = mysql_native_password # # VTTablet will enable semi-sync at the proper time when replication is set up, # or when a primary is promoted or demoted based on the durability policy configured. -plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so +plugin-load = rpl_semi_sync_source=semisync_source.so;rpl_semi_sync_replica=semisync_replica.so # MySQL 8.0.26 and later will not load plugins during --initialize # which makes these options unknown. Prefixing with --loose # tells the server it's fine if they are not understood. -loose_rpl_semi_sync_master_timeout = 1000000000000000000 -loose_rpl_semi_sync_master_wait_no_slave = 1 +loose_rpl_semi_sync_source_timeout = 1000000000000000000 +loose_rpl_semi_sync_source_wait_no_replica = 1 # In order to protect against any errand GTIDs we will start the mysql instance # in super-read-only mode.