Merge branch 'main' into vtorc-debug-health-fix

Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com>

Author: timvaillancourt

.github/CODEOWNERS

@@ -28,6 +28,7 @@ go.sum @ajm188 @deepthi @harshit-gangal @mattlord @rohit-nayak-ps @systay @froui
 /go/test/endtoend/onlineddl @rohit-nayak-ps @shlomi-noach
 /go/test/endtoend/messaging @mattlord @rohit-nayak-ps @derekperkins
 /go/test/endtoend/schemadiff @shlomi-noach @mattlord
+/go/test/endtoend/transaction @harshit-gangal @systay @frouioui @GuptaManan100
 /go/test/endtoend/*throttler* @shlomi-noach @mattlord @timvaillancourt
 /go/test/endtoend/vtgate @harshit-gangal @systay @frouioui
 /go/test/endtoend/vtorc @deepthi @shlomi-noach @GuptaManan100 @timvaillancourt

.github/workflows/vtop_example.yml

@@ -0,0 +1,97 @@
+name: vtop_example
+on: [push, pull_request]
+concurrency:
+  group: format('{0}-{1}', ${{ github.ref }}, 'vtop_example')
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: VTop Example
+    runs-on: self-hosted
+
+    steps:
+      - name: Skip CI
+        run: |
+          if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then
+            echo "skipping CI due to the 'Skip CI' label"
+            exit 1
+          fi
+
+      - name: Check if workflow needs to be skipped
+        id: skip-workflow
+        run: |
+          skip='false'
+          if [[ "${{github.event.pull_request}}" ==  "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then
+            skip='true'
+          fi
+          echo Skip ${skip}
+          echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT
+          
+          PR_DATA=$(curl -s\
+            -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Accept: application/vnd.github.v3+json" \
+            "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}")
+          draft=$(echo "$PR_DATA" | jq .draft -r)
+          echo "is_draft=${draft}" >> $GITHUB_OUTPUT
+
+      - name: Check out code
+        if: steps.skip-workflow.outputs.skip-workflow == 'false'
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Check for changes in relevant files
+        if: steps.skip-workflow.outputs.skip-workflow == 'false'
+        uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1
+        id: changes
+        with:
+          token: ''
+          filters: |
+            end_to_end:
+              - 'go/**/*.go'
+              - 'go/vt/sidecardb/**/*.sql'
+              - 'test.go'
+              - 'Makefile'
+              - 'build.env'
+              - 'go.[sumod]'
+              - 'proto/*.proto'
+              - 'tools/**'
+              - 'config/**'
+              - 'bootstrap.sh'
+              - 'examples/**'
+              - 'test/**'
+              - '.github/workflows/vtop_example.yml'
+
+      - name: Set up Go
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          go-version-file: go.mod
+
+      - name: Tune the OS
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        run: |
+          echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+
+      - name: Get dependencies
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        run: |
+          # Install everything we need, and configure
+          sudo apt-get install -y eatmydata make
+          go mod download
+
+      # needed for vtctldclient
+      - name: Build vitess
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        run: |
+          make build
+
+      - name: Install kubectl & kind
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        run: |
+          make install_kubectl_kind
+
+      - name: vtop_example
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        timeout-minutes: 60
+        run: |
+          source build.env
+          eatmydata -- go run test.go -docker=false -skip-build -print-log -follow -retry=1 -timeout=60m vtop_example

changelog/19.0/19.0.8/changelog.md

@@ -0,0 +1,29 @@
+# Changelog of Vitess v19.0.8
+
+### Bug fixes 
+#### Topology
+ * [release-19.0] Close zookeeper topo connection on disconnect (#17136) [#17191](https://github.com/vitessio/vitess/pull/17191) 
+#### VTTablet
+ * [release-19.0] Fix deadlock in messager and health streamer (#17230) [#17233](https://github.com/vitessio/vitess/pull/17233)
+ * [release-19.0] Fix potential deadlock in health streamer (#17261) [#17268](https://github.com/vitessio/vitess/pull/17268)
+### CI/Build 
+#### Build/CI
+ * [release-19.0] Specify Ubuntu 24.04 for all jobs (#17278) [#17280](https://github.com/vitessio/vitess/pull/17280) 
+#### Cluster management
+ * [release-19.0] Fix flakiness in `TestListenerShutdown` (#17024) [#17187](https://github.com/vitessio/vitess/pull/17187) 
+#### General
+ * [release-19.0] Upgrade the Golang version to `go1.22.9` [#17214](https://github.com/vitessio/vitess/pull/17214)
+### Enhancement 
+#### Query Serving
+ * [release-19.0] Fix to prevent stopping buffering prematurely (#17013) [#17203](https://github.com/vitessio/vitess/pull/17203)
+### Internal Cleanup 
+#### Build/CI
+ * [release-19.0] Change the name of the vitess-tester repository (#16917) [#17028](https://github.com/vitessio/vitess/pull/17028)
+### Release 
+#### General
+ * [release-19.0] Bump to `v19.0.8-SNAPSHOT` after the `v19.0.7` release [#17158](https://github.com/vitessio/vitess/pull/17158)
+ * [release-19.0] Code Freeze for `v19.0.8` [#17310](https://github.com/vitessio/vitess/pull/17310)
+### Testing 
+#### Build/CI
+ * [release-19.0] Flakes: Address flakiness in TestZkConnClosedOnDisconnect (#17194) [#17195](https://github.com/vitessio/vitess/pull/17195)
+

changelog/19.0/19.0.8/release_notes.md

@@ -0,0 +1,7 @@
+# Release of Vitess v19.0.8
+The entire changelog for this release can be found [here](https://github.com/vitessio/vitess/blob/main/changelog/19.0/19.0.8/changelog.md).
+
+The release includes 11 merged Pull Requests.
+
+Thanks to all our contributors: @app/vitess-bot, @frouioui, @vitess-bot
+

changelog/19.0/README.md

@@ -1,4 +1,8 @@
 ## v19.0
+* **[19.0.8](19.0.8)**
+	* [Changelog](19.0.8/changelog.md)
+	* [Release Notes](19.0.8/release_notes.md)
+
 * **[19.0.7](19.0.7)**
 	* [Changelog](19.0.7/changelog.md)
 	* [Release Notes](19.0.7/release_notes.md)

changelog/20.0/20.0.4/changelog.md

@@ -0,0 +1,27 @@
+# Changelog of Vitess v20.0.4
+
+### Bug fixes 
+#### Query Serving
+ * [release-20.0] Use proper keyspace when updating the query graph of a reference DML (#17226) [#17257](https://github.com/vitessio/vitess/pull/17257) 
+#### Topology
+ * [release-20.0] Close zookeeper topo connection on disconnect (#17136) [#17192](https://github.com/vitessio/vitess/pull/17192) 
+#### VTTablet
+ * [release-20.0] Fix deadlock in messager and health streamer (#17230) [#17234](https://github.com/vitessio/vitess/pull/17234)
+ * [release-20.0] Fix potential deadlock in health streamer (#17261) [#17269](https://github.com/vitessio/vitess/pull/17269)
+### CI/Build 
+#### Build/CI
+ * [release-20.0] Specify Ubuntu 24.04 for all jobs (#17278) [#17281](https://github.com/vitessio/vitess/pull/17281) 
+#### Cluster management
+ * [release-20.0] Fix flakiness in `TestListenerShutdown` (#17024) [#17188](https://github.com/vitessio/vitess/pull/17188) 
+#### General
+ * [release-20.0] Upgrade the Golang version to `go1.22.9` [#17212](https://github.com/vitessio/vitess/pull/17212)
+### Enhancement 
+#### Query Serving
+ * [release-20.0] Fix to prevent stopping buffering prematurely (#17013) [#17204](https://github.com/vitessio/vitess/pull/17204)
+### Internal Cleanup 
+#### Build/CI
+ * [release-20.0] Change the name of the vitess-tester repository (#16917) [#17029](https://github.com/vitessio/vitess/pull/17029)
+### Testing 
+#### Build/CI
+ * [release-20.0] Flakes: Address flakiness in TestZkConnClosedOnDisconnect (#17194) [#17196](https://github.com/vitessio/vitess/pull/17196)
+

changelog/20.0/20.0.4/release_notes.md

@@ -0,0 +1,7 @@
+# Release of Vitess v20.0.4
+The entire changelog for this release can be found [here](https://github.com/vitessio/vitess/blob/main/changelog/20.0/20.0.4/changelog.md).
+
+The release includes 10 merged Pull Requests.
+
+Thanks to all our contributors: @app/vitess-bot, @frouioui
+

changelog/20.0/README.md

@@ -1,4 +1,8 @@
 ## v20.0
+* **[20.0.4](20.0.4)**
+	* [Changelog](20.0.4/changelog.md)
+	* [Release Notes](20.0.4/release_notes.md)
+
 * **[20.0.3](20.0.3)**
 	* [Changelog](20.0.3/changelog.md)
 	* [Release Notes](20.0.3/release_notes.md)

changelog/21.0/21.0.0/summary.md

@@ -9,6 +9,7 @@
         - [Deprecated VTTablet Flags](#vttablet-flags)
         - [Deletion of deprecated metrics](#metric-deletion)
         - [Deprecated Metrics](#deprecations-metrics)
+    - **[RPC Changes](#rpc-changes)**
     - **[Traffic Mirroring](#traffic-mirroring)**
     - **[Atomic Distributed Transaction Support](#atomic-transaction)**
     - **[New VTGate Shutdown Behavior](#new-vtgate-shutdown-behavior)**
@@ -77,6 +78,12 @@ The following metrics are now deprecated and will be deleted in a future release
 | `vttablet` |   `QueryCacheHits`    |   `QueryEnginePlanCacheHits`    |
 | `vttablet` |  `QueryCacheMisses`   |  `QueryEnginePlanCacheMisses`   |
 
+### <a id="rpc-changes"/>RPC Changes</a>
+
+These are the RPC changes made in this release - 
+1. `ReadReparentJournalInfo` RPC has been added in TabletManagerClient interface, that is going to be used in EmergencyReparentShard for better errant GTID detection.
+2. `PrimaryStatus` RPC in TabletManagerClient interface has been updated to also return the server UUID of the primary. This is going to be used in the vttablets so that they can do their own errant GTID detection in `SetReplicationSource`.
+
 ### <a id="traffic-mirroring"/>Traffic Mirroring</a>
 
 Traffic mirroring is intended to help reduce some of the uncertainty inherent to `MoveTables SwitchTraffic`. When

changelog/21.0/21.0.1/changelog.md

@@ -0,0 +1,57 @@
+# Changelog of Vitess v21.0.1
+
+### Bug fixes 
+#### Backup and Restore
+ * [release-21.0] Fix how we cancel the context in the builtin backup engine (#17285) [#17291](https://github.com/vitessio/vitess/pull/17291)
+ * [release-21.0] S3: optional endpoint resolver and correct retrier [#17307](https://github.com/vitessio/vitess/pull/17307) 
+#### Cluster management
+ * [release-21.0] Fix panic in vttablet when closing topo server twice (#17094) [#17122](https://github.com/vitessio/vitess/pull/17122) 
+#### Online DDL
+ * [release-21.0] Online DDL: fix defer function, potential connection pool exhaustion (#17207) [#17210](https://github.com/vitessio/vitess/pull/17210) 
+#### Query Serving
+ * [release-21.0] bugfix: treat EXPLAIN like SELECT (#17054) [#17058](https://github.com/vitessio/vitess/pull/17058)
+ * [release-21.0] Delegate Column Availability Checks to MySQL for Single-Route Queries (#17077) [#17087](https://github.com/vitessio/vitess/pull/17087)
+ * [release-21.0] bugfix: Handle CTEs with columns named in the CTE def (#17179) [#17181](https://github.com/vitessio/vitess/pull/17181)
+ * [release-21.0] Use proper keyspace when updating the query graph of a reference DML (#17226) [#17258](https://github.com/vitessio/vitess/pull/17258) 
+#### Topology
+ * [release-21.0] Close zookeeper topo connection on disconnect (#17136) [#17193](https://github.com/vitessio/vitess/pull/17193) 
+#### VReplication
+ * [release-21.0] VReplication: Qualify and SQL escape tables in created AutoIncrement VSchema definitions (#17174) [#17176](https://github.com/vitessio/vitess/pull/17176) 
+#### VTTablet
+ * [release-21.0] Fix deadlock in messager and health streamer (#17230) [#17235](https://github.com/vitessio/vitess/pull/17235)
+ * [release-21.0] Fix potential deadlock in health streamer (#17261) [#17270](https://github.com/vitessio/vitess/pull/17270)
+### CI/Build 
+#### Build/CI
+ * [release-21.0] Specify Ubuntu 24.04 for all jobs (#17278) [#17282](https://github.com/vitessio/vitess/pull/17282) 
+#### Cluster management
+ * [release-21.0] Fix flakiness in `TestListenerShutdown` (#17024) [#17189](https://github.com/vitessio/vitess/pull/17189) 
+#### General
+ * [release-21.0] Upgrade the Golang version to `go1.23.3` [#17211](https://github.com/vitessio/vitess/pull/17211)
+### Dependencies 
+#### Java
+ * [release-21.0] java package updates for grpc and protobuf and release plugins (#17100) [#17105](https://github.com/vitessio/vitess/pull/17105)
+### Documentation 
+#### Documentation
+ * [Direct PR][release-21.0] Add RPC changes segment in the summary doc [#17034](https://github.com/vitessio/vitess/pull/17034)
+### Enhancement 
+#### Online DDL
+ * [release-21.0] Improve Schema Engine's TablesWithSize80 query (#17066) [#17091](https://github.com/vitessio/vitess/pull/17091) 
+#### Query Serving
+ * [release-21.0] Fix to prevent stopping buffering prematurely (#17013) [#17205](https://github.com/vitessio/vitess/pull/17205) 
+#### VReplication
+ * [release-21.0] Binlog: Improve ZstdInMemoryDecompressorMaxSize management (#17220) [#17241](https://github.com/vitessio/vitess/pull/17241)
+### Internal Cleanup 
+#### Build/CI
+ * [release-21.0] Change the name of the vitess-tester repository (#16917) [#17030](https://github.com/vitessio/vitess/pull/17030)
+### Regression 
+#### Backup and Restore
+ * [release-21.0] Fix unreachable errors when taking a backup (#17062) [#17112](https://github.com/vitessio/vitess/pull/17112)
+### Release 
+#### General
+ * [release-21.0] Bump to `v21.0.1-SNAPSHOT` after the `v21.0.0` release [#17098](https://github.com/vitessio/vitess/pull/17098)
+### Testing 
+#### Build/CI
+ * [release-21.0] Flakes: Address flakiness in TestZkConnClosedOnDisconnect (#17194) [#17197](https://github.com/vitessio/vitess/pull/17197) 
+#### Query Serving
+ * [release-21.0] fix: flaky test on twopc transaction (#17068) [#17070](https://github.com/vitessio/vitess/pull/17070)
+

changelog/21.0/21.0.1/release_notes.md

@@ -0,0 +1,7 @@
+# Release of Vitess v21.0.1
+The entire changelog for this release can be found [here](https://github.com/vitessio/vitess/blob/main/changelog/21.0/21.0.1/changelog.md).
+
+The release includes 25 merged Pull Requests.
+
+Thanks to all our contributors: @GuptaManan100, @app/vitess-bot, @frouioui, @vitess-bot
+

changelog/21.0/README.md

@@ -1,4 +1,8 @@
 ## v21.0
+* **[21.0.1](21.0.1)**
+	* [Changelog](21.0.1/changelog.md)
+	* [Release Notes](21.0.1/release_notes.md)
+
 * **[21.0.0](21.0.0)**
 	* [Changelog](21.0.0/changelog.md)
 	* [Release Notes](21.0.0/release_notes.md)

changelog/22.0/22.0.0/summary.md

@@ -5,6 +5,7 @@
 - **[Major Changes](#major-changes)**
   - **[RPC Changes](#rpc-changes)**
   - **[Prefer not promoting a replica that is currently taking a backup](#reparents-prefer-not-backing-up)**
+  - **[VTOrc Config File Changes](#vtorc-config-file-changes)**
 
 
 ## <a id="major-changes"/>Major Changes</a>
@@ -25,4 +26,25 @@ For planned reparents, hosts taking backups with a backup engine other than `bui
 valid candidates. This means they will never get promoted - not even if there's no other candidates.
 
 Note that behavior for `builtin` backups remains unchanged: a replica that is currently taking a `builtin` backup will
-never be promoted, neither by planned nor by emergency reparents.
+never be promoted, neither by planned nor by emergency reparents.
+
+### <a id="vtorc-config-file-changes"/>VTOrc Config File Changes</a>
+
+The configuration file for VTOrc has been updated to now support dynamic fields. The old `--config` parameter has been removed. The alternative is to use the `--config-file` parameter. The configuration can now be provided in json, yaml or any other format that [viper](https://github.com/spf13/viper) supports.
+
+The following fields can be dynamically changed - 
+1. `instance-poll-time`
+2. `prevent-cross-cell-failover`
+3. `snapshot-topology-interval`
+4. `reasonable-replication-lag`
+5. `audit-to-backend`
+6. `audit-to-syslog`
+7. `audit-purge-duration`
+8. `wait-replicas-timeout`
+9. `tolerable-replication-lag`
+10. `topo-information-refresh-duration`
+11. `recovery-poll-duration`
+12. `allow-emergency-reparent`
+13. `change-tablets-with-errant-gtid-to-drained`
+
+To upgrade to the newer version of the configuration file, first switch to using the flags in your current deployment before upgrading. Then you can switch to using the configuration file in the newer release.

examples/common/scripts/vtctld-up.sh

@@ -33,6 +33,7 @@ vtctld \
  --port $vtctld_web_port \
  --grpc_port $grpc_port \
  --pid_file $VTDATAROOT/tmp/vtctld.pid \
+ --pprof-http \
   > $VTDATAROOT/tmp/vtctld.out 2>&1 &
 
 for _ in {0..300}; do

examples/common/scripts/vtgate-up.sh

@@ -41,6 +41,7 @@ vtgate \
   --pid_file $VTDATAROOT/tmp/vtgate.pid \
   --enable_buffer \
   --mysql_auth_server_impl none \
+  --pprof-http \
   > $VTDATAROOT/tmp/vtgate.out 2>&1 &
 
 # Block waiting for vtgate to be listening

examples/common/scripts/vtorc-up.sh

@@ -11,7 +11,9 @@ vtorc \
   $TOPOLOGY_FLAGS \
   --logtostderr \
   --alsologtostderr \
-  --config="${script_dir}/../vtorc/config.json" \
+  --config-path="${script_dir}/../vtorc/" \
+  --config-name="config.yaml" \
+  --config-type="yml" \
   --port $port \
   > "${log_dir}/vtorc.out" 2>&1 &
 

examples/common/scripts/vttablet-up.sh

@@ -54,6 +54,7 @@ vttablet \
  --service_map 'grpc-queryservice,grpc-tabletmanager,grpc-updatestream' \
  --pid_file $VTDATAROOT/$tablet_dir/vttablet.pid \
  --heartbeat_on_demand_duration=5s \
+ --pprof-http \
  > $VTDATAROOT/$tablet_dir/vttablet.out 2>&1 &
 
 # Block waiting for the tablet to be listening

examples/common/vtorc/config.json

@@ -0,0 +1,13 @@
+instance-poll-time: 1s
+prevent-cross-cell-failover: false
+snapshot-topology-interval: 0h
+reasonable-replication-lag: 10s
+audit-to-backend: false
+audit-to-syslog: false
+audit-purge-duration: 168h
+wait-replicas-timeout: 30s
+tolerable-replication-lag: 0s
+topo-information-refresh-duration: 15s
+recovery-poll-duration: 1s
+allow-emergency-reparent: true
+change-tablets-with-errant-gtid-to-drained: false