resolved conflict

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>

Author: shlomi-noach

.github/workflows/cluster_endtoend_onlineddl_vrepl.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_onlineddl_vrepl_bench.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_bench | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_stress_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_schemadiff_vrepl.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard schemadiff_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_across_db_versions | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_basic.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_basic | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_cellalias.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_cellalias | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_copy_parallel | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_foreign_key_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_mariadb_to_mysql | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_migrate.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_migrate | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_multi_tenant | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_partial_movetables_and_materialize | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_v2.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_v2 | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_vtctldclient_vdiff2_movetables_tz | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_evalengine_mysql57.yml

@@ -163,6 +163,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="1"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql57"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_evalengine_mysql80.yml

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="1"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql80"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_evalengine_mysql84.yml

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="1"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql84"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_mysql57.yml

@@ -163,6 +163,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="0"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql57"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_mysql80.yml

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="0"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql80"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_mysql84.yml

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="0"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql84"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

changelog/22.0/22.0.0/summary.md

@@ -8,9 +8,10 @@
   - **[RPC Changes](#rpc-changes)**
   - **[Prefer not promoting a replica that is currently taking a backup](#reparents-prefer-not-backing-up)**
   - **[VTOrc Config File Changes](#vtorc-config-file-changes)**
+  - **[Support for More Efficient JSON Replication](#efficient-json-replication)**
 - **[Minor Changes](#minor-changes)**
   - **[VTTablet Flags](#flags-vttablet)**
-
+  - **[Topology read concurrency behaviour changes](#topo-read-concurrency-changes)**
 
 ## <a id="major-changes"/>Major Changes</a>
 
@@ -59,6 +60,12 @@ The following fields can be dynamically changed -
 
 To upgrade to the newer version of the configuration file, first switch to using the flags in your current deployment before upgrading. Then you can switch to using the configuration file in the newer release.
 
+### <a id="efficient-json-replication"/>Support for More Efficient JSON Replication</a>
+
+In [#7345](https://github.com/vitessio/vitess/pull/17345) we added support for [`--binlog-row-value-options=PARTIAL_JSON`](https://dev.mysql.com/doc/refman/en/replication-options-binary-log.html#sysvar_binlog_row_value_options). You can read more about [this feature added to MySQL 8.0 here](https://dev.mysql.com/blog-archive/efficient-json-replication-in-mysql-8-0/).
+
+If you are using MySQL 8.0 or later and using JSON columns, you can now enable this MySQL feature across your Vitess cluster(s) to lower the disk space needed for binary logs and improve the CPU and memory usage in both `mysqld` (standard intrashard MySQL replication) and `vttablet` ([VReplication](https://vitess.io/docs/reference/vreplication/vreplication/)) without losing any capabilities or features.
+
 
 ## <a id="minor-changes"/>Minor Changes</a>
 
@@ -67,3 +74,9 @@ To upgrade to the newer version of the configuration file, first switch to using
 - `twopc_abandon_age` flag now supports values in the time.Duration format (e.g., 1s, 2m, 1h). 
 While the flag will continue to accept float values (interpreted as seconds) for backward compatibility, 
 **float inputs are deprecated** and will be removed in a future release.
+
+### <a id="topo-read-concurrency-changes"/>`--topo_read_concurrency` behaviour changes
+
+The `--topo_read_concurrency` flag was added to all components that access the topology and the provided limit is now applied separately for each global or local cell _(default `32`)_.
+
+All topology read calls _(`Get`, `GetVersion`, `List` and `ListDir`)_ now respect this per-cell limit. Previous to this version a single limit was applied to all cell calls and it was not respected by many topology calls.

config/mycnf/default.cnf

@@ -16,10 +16,6 @@ secure-file-priv = {{.SecureFilePriv}}
 
 server-id = {{.ServerID}}
 
-# all db instances should skip starting replication threads - that way we can do any
-# additional configuration (like enabling semi-sync) before we connect to
-# the source.
-skip_slave_start
 socket = {{.SocketFile}}
 tmpdir = {{.TmpDir}}
 

config/mycnf/mariadb10.cnf

@@ -1,5 +1,10 @@
 # This file is auto-included when MariaDB 10 is detected.
 
+# all db instances should skip starting replication threads - that way we can do any
+# additional configuration (like enabling semi-sync) before we connect to
+# the source.
+skip_slave_start
+
 # Semi-sync replication is required for automated unplanned failover
 # (when the primary goes away). Here we just load the plugin so it's
 # available if desired, but it's disabled at startup.

config/mycnf/mysql57.cnf

@@ -1,5 +1,10 @@
 # This file is auto-included when MySQL 5.7 is detected.
 
+# all db instances should skip starting replication threads - that way we can do any
+# additional configuration (like enabling semi-sync) before we connect to
+# the source.
+skip_slave_start
+
 # MySQL 5.7 does not enable the binary log by default, and 
 # info repositories default to file
 

config/mycnf/mysql80.cnf

@@ -1,5 +1,10 @@
 # This file is auto-included when MySQL 8.0 is detected.
 
+# all db instances should skip starting replication threads - that way we can do any
+# additional configuration (like enabling semi-sync) before we connect to
+# the source.
+skip_slave_start
+
 # MySQL 8.0 enables binlog by default with sync_binlog and TABLE info repositories
 # It does not enable GTIDs or enforced GTID consistency
 

config/mycnf/mysql8026.cnf

@@ -1,5 +1,10 @@
 # This file is auto-included when MySQL 8.0.26 or later is detected.
 
+# all db instances should skip starting replication threads - that way we can do any
+# additional configuration (like enabling semi-sync) before we connect to
+# the source.
+skip_replica_start
+
 # MySQL 8.0 enables binlog by default with sync_binlog and TABLE info repositories
 # It does not enable GTIDs or enforced GTID consistency
 

config/mycnf/mysql84.cnf

@@ -1,5 +1,10 @@
 # This file is auto-included when MySQL 8.4.0 or later is detected.
 
+# all db instances should skip starting replication threads - that way we can do any
+# additional configuration (like enabling semi-sync) before we connect to
+# the source.
+skip_replica_start
+
 # MySQL 8.0 enables binlog by default with sync_binlog and TABLE info repositories
 # It does not enable GTIDs or enforced GTID consistency