Merge remote-tracking branch 'upstream/main' into log-only-errors

Signed-off-by: Harshit Gangal <harshit@planetscale.com>

Author: harshit-gangal

changelog/22.0/22.0.0/summary.md

@@ -12,6 +12,7 @@
   - **[Support for More Efficient JSON Replication](#efficient-json-replication)**
   - **[Support for LAST_INSERT_ID(x)](#last-insert-id)**
   - **[Support for Maximum Idle Connections in the Pool](#max-idle-connections)**
+  - **[Stalled Disk Recovery in VTOrc](#stall-disk-recovery)**
   - **[Support for Filtering Query logs on Error](#query-logs)**
 - **[Minor Changes](#minor-changes)**
   - **[VTTablet Flags](#flags-vttablet)**
@@ -101,6 +102,11 @@ You can control idle connection retention for the query server’s query pool, s
 
 This feature ensures that, during traffic spikes, idle connections are available for faster responses, while minimizing overhead in low-traffic periods by limiting the number of idle connections retained. It helps strike a balance between performance, efficiency, and cost.
 
+### <a id="stall-disk-recovery"/>Stalled Disk Recovery in VTOrc</a>
+VTOrc can now identify and recover from stalled disk errors. VTTablets test whether the disk is writable and they send this information in the full status output to VTOrc. If the disk is not writable on the primary tablet, VTOrc will attempt to recover the cluster by promoting a new primary. This is useful in scenarios where the disk is stalled and the primary vttablet is unable to accept writes because of it.
+
+To opt into this feature, `--enable-primary-disk-stalled-recovery` flag has to be specified on VTOrc, and `--disk-write-dir` flag has to be specified on the vttablets. `--disk-write-interval` and `--disk-write-timeout` flags can be used to configure the polling interval and timeout respectively. 
+
 ### <a id="query-logs"/>Support for Filtering Query logs on Error</a>
 
 The `querylog-mode` setting can be configured to `error` to log only queries that result in errors. This option is supported in both VTGate and VTTablet.

go/cmd/vtctldclient/command/vreplication/vdiff/vdiff.go

@@ -436,7 +436,7 @@ State:        {{.State}}
 RowsCompared: {{.RowsCompared}}
 HasMismatch:  {{.HasMismatch}}
 StartedAt:    {{.StartedAt}}
-{{if (eq .State "started")}}Progress:     {{printf "%.2f" .Progress.Percentage}}%%{{if .Progress.ETA}}, ETA: {{.Progress.ETA}}{{end}}{{end}}
+{{if (eq .State "started")}}Progress:     {{printf "%.2f" .Progress.Percentage}}%{{if .Progress.ETA}}, ETA: {{.Progress.ETA}}{{end}}{{end}}
 {{if .CompletedAt}}CompletedAt:  {{.CompletedAt}}{{end}}
 {{range $table := .TableSummaryMap}} 
 Table {{$table.TableName}}:

go/cmd/vtctldclient/command/vreplication/vdiff/vdiff_test.go

@@ -680,6 +680,75 @@ func TestVDiffSharded(t *testing.T) {
 	}
 }
 
+func TestVDiffTextTemplate(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	env := newTestVDiffEnv(t, ctx, []string{"0"}, []string{"0"}, "", nil)
+	defer env.close()
+
+	now := time.Now()
+	UUID := uuid.New().String()
+	req := &tabletmanagerdatapb.VDiffRequest{
+		Keyspace:  env.targetKeyspace,
+		Workflow:  env.workflow,
+		Action:    string(vdiff.ShowAction),
+		ActionArg: UUID,
+	}
+	starttime := now.UTC().Format(vdiff.TimestampFormat)
+
+	testCases := []struct {
+		id     string
+		res    *sqltypes.Result
+		report string
+	}{{
+		id: "1",
+		res: sqltypes.MakeTestResult(fields,
+			"started||t1|"+UUID+"|started|300|"+starttime+"|30||0|"+
+				`{"TableName": "t1", "MatchingRows": 30, "ProcessedRows": 30, "MismatchedRows": 0, "ExtraRowsSource": 0, `+
+				`"ExtraRowsTarget": 0}`),
+		report: fmt.Sprintf(`
+VDiff Summary for targetks.vdiffTest (%s)
+State:        started
+RowsCompared: 30
+HasMismatch:  false
+StartedAt:    %s
+Progress:     10.00%%, ETA: %s
+ 
+Table t1:
+	State:            started
+	ProcessedRows:    30
+	MatchingRows:     30
+ 
+Use "--format=json" for more detailed output.
+
+`, UUID, starttime, starttime),
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.id, func(t *testing.T) {
+			res := &tabletmanagerdatapb.VDiffResponse{
+				Id:     1,
+				Output: sqltypes.ResultToProto3(tc.res),
+			}
+			env.tmc.setVDResults(env.tablets[200].tablet, req, res)
+			req := &vtctldatapb.VDiffShowRequest{
+				TargetKeyspace: env.targetKeyspace,
+				Workflow:       env.workflow,
+				Arg:            UUID,
+			}
+
+			resp, err := env.ws.VDiffShow(context.Background(), req)
+			require.NoError(t, err)
+			vds, err := displayShowSingleSummary(env.out, "text", env.targetKeyspace, env.workflow, UUID, resp, true)
+			require.NoError(t, err)
+			require.Equal(t, vdiff.StartedState, vds)
+
+			require.Equal(t, tc.report, env.getOutput())
+			env.resetOutput()
+		})
+	}
+}
+
 func TestGetStructNames(t *testing.T) {
 	type s struct {
 		A string

go/flags/endtoend/vtcombo.txt

@@ -102,6 +102,9 @@ Flags:
       --ddl_strategy string                                              Set default strategy for DDL statements. Override with @@ddl_strategy session variable (default "direct")
       --default_tablet_type topodatapb.TabletType                        The default tablet type to set for queries, when one is not explicitly selected. (default PRIMARY)
       --degraded_threshold duration                                      replication lag after which a replica is considered degraded (default 30s)
+      --disk-write-dir string                                            if provided, tablet will attempt to write a file to this directory to check if the disk is stalled
+      --disk-write-interval duration                                     how often to write to the disk to check whether it is stalled (default 5s)
+      --disk-write-timeout duration                                      if writes exceed this duration, the disk is considered stalled (default 30s)
       --emit_stats                                                       If set, emit stats to push-based monitoring and stats backends
       --enable-consolidator                                              Synonym to -enable_consolidator (default true)
       --enable-consolidator-replicas                                     Synonym to -enable_consolidator_replicas

go/flags/endtoend/vtorc.txt

@@ -33,6 +33,7 @@ Flags:
       --config-type string                                          Config file type (omit to infer config type from file extension).
       --consul_auth_static_file string                              JSON File to read the topos/tokens from.
       --emit_stats                                                  If set, emit stats to push-based monitoring and stats backends
+      --enable-primary-disk-stalled-recovery                        Whether VTOrc should detect a stalled disk on the primary and failover
       --grpc-dial-concurrency-limit int                             Maximum concurrency of grpc dial operations. This should be less than the golang max thread limit of 10000. (default 1024)
       --grpc_auth_static_client_creds string                        When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server.
       --grpc_compression string                                     Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy

go/flags/endtoend/vttablet.txt

@@ -133,6 +133,9 @@ Flags:
       --dba_idle_timeout duration                                        Idle timeout for dba connections (default 1m0s)
       --dba_pool_size int                                                Size of the connection pool for dba connections (default 20)
       --degraded_threshold duration                                      replication lag after which a replica is considered degraded (default 30s)
+      --disk-write-dir string                                            if provided, tablet will attempt to write a file to this directory to check if the disk is stalled
+      --disk-write-interval duration                                     how often to write to the disk to check whether it is stalled (default 5s)
+      --disk-write-timeout duration                                      if writes exceed this duration, the disk is considered stalled (default 30s)
       --emit_stats                                                       If set, emit stats to push-based monitoring and stats backends
       --enable-consolidator                                              Synonym to -enable_consolidator (default true)
       --enable-consolidator-replicas                                     Synonym to -enable_consolidator_replicas

go/mysql/capabilities/capability.go

@@ -31,25 +31,26 @@ var (
 type FlavorCapability int
 
 const (
-	NoneFlavorCapability                        FlavorCapability = iota // default placeholder
-	FastDropTableFlavorCapability                                       // supported in MySQL 8.0.23 and above: https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-23.html
-	TransactionalGtidExecutedFlavorCapability                           //
-	InstantDDLFlavorCapability                                          // ALGORITHM=INSTANT general support
-	InstantAddLastColumnFlavorCapability                                //
-	InstantAddDropVirtualColumnFlavorCapability                         //
-	InstantAddDropColumnFlavorCapability                                // Adding/dropping column in any position/ordinal.
-	InstantChangeColumnDefaultFlavorCapability                          //
-	InstantExpandEnumCapability                                         //
-	InstantChangeColumnVisibilityCapability                             //
-	MySQLUpgradeInServerFlavorCapability                                //
-	DynamicRedoLogCapacityFlavorCapability                              // supported in MySQL 8.0.30 and above: https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-30.html
-	DisableRedoLogFlavorCapability                                      // supported in MySQL 8.0.21 and above: https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-21.html
-	CheckConstraintsCapability                                          // supported in MySQL 8.0.16 and above: https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-16.html
-	PerformanceSchemaDataLocksTableCapability                           // supported in MySQL 8.0.1 and above: https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-1.html
-	InstantDDLXtrabackupCapability                                      // Supported in 8.0.32 and above, solving a MySQL-vs-Xtrabackup bug starting 8.0.29
-	ReplicaTerminologyCapability                                        // Supported in 8.0.26 and above, using SHOW REPLICA STATUS and all variations.
-	BinaryLogStatus                                                     // Supported in 8.2.0 and above, uses SHOW BINARY LOG STATUS
-	RestrictFKOnNonStandardKey                                          // Supported in 8.4.0 and above, restricts usage of non-standard indexes for foreign keys.
+	NoneFlavorCapability                          FlavorCapability = iota // default placeholder
+	FastDropTableFlavorCapability                                         // supported in MySQL 8.0.23 and above: https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-23.html
+	TransactionalGtidExecutedFlavorCapability                             //
+	InstantDDLFlavorCapability                                            // ALGORITHM=INSTANT general support
+	InstantAddLastColumnFlavorCapability                                  //
+	InstantAddDropVirtualColumnFlavorCapability                           //
+	InstantAddDropColumnFlavorCapability                                  // Adding/dropping column in any position/ordinal.
+	InstantChangeColumnDefaultFlavorCapability                            //
+	InstantExpandEnumCapability                                           //
+	InstantChangeColumnVisibilityCapability                               //
+	MySQLUpgradeInServerFlavorCapability                                  //
+	DynamicRedoLogCapacityFlavorCapability                                // supported in MySQL 8.0.30 and above: https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-30.html
+	DisableRedoLogFlavorCapability                                        // supported in MySQL 8.0.21 and above: https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-21.html
+	CheckConstraintsCapability                                            // supported in MySQL 8.0.16 and above: https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-16.html
+	PerformanceSchemaDataLocksTableCapability                             // supported in MySQL 8.0.1 and above: https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-1.html
+	PerformanceSchemaMetadataLocksTableCapability                         // supported in MySQL 8.0.2 and above: https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-2.html
+	InstantDDLXtrabackupCapability                                        // Supported in 8.0.32 and above, solving a MySQL-vs-Xtrabackup bug starting 8.0.29
+	ReplicaTerminologyCapability                                          // Supported in 8.0.26 and above, using SHOW REPLICA STATUS and all variations.
+	BinaryLogStatus                                                       // Supported in 8.2.0 and above, uses SHOW BINARY LOG STATUS
+	RestrictFKOnNonStandardKey                                            // Supported in 8.4.0 and above, restricts usage of non-standard indexes for foreign keys.
 )
 
 type CapableOf func(capability FlavorCapability) (bool, error)
@@ -97,6 +98,8 @@ func MySQLVersionHasCapability(serverVersion string, capability FlavorCapability
 		return atLeast(8, 0, 0)
 	case PerformanceSchemaDataLocksTableCapability:
 		return atLeast(8, 0, 1)
+	case PerformanceSchemaMetadataLocksTableCapability:
+		return atLeast(8, 0, 2)
 	case MySQLUpgradeInServerFlavorCapability:
 		return atLeast(8, 0, 16)
 	case CheckConstraintsCapability:

go/mysql/capabilities/capability_test.go

@@ -218,6 +218,25 @@ func TestMySQLVersionCapableOf(t *testing.T) {
 			version:    "8.0.20",
 			capability: PerformanceSchemaDataLocksTableCapability,
 			isCapable:  true,
+		}, {
+			version:    "5.7.38",
+			capability: PerformanceSchemaMetadataLocksTableCapability,
+			isCapable:  false,
+		},
+		{
+			version:    "8.0",
+			capability: PerformanceSchemaMetadataLocksTableCapability,
+			isCapable:  false,
+		},
+		{
+			version:    "8.0.1",
+			capability: PerformanceSchemaMetadataLocksTableCapability,
+			isCapable:  false,
+		},
+		{
+			version:    "8.0.2",
+			capability: PerformanceSchemaMetadataLocksTableCapability,
+			isCapable:  true,
 		},
 		{
 			version:    "8.0.29",

go/mysql/collations/colldata/cached_size.go

@@ -102,6 +102,15 @@ func TestServerVersionCapableOf(t *testing.T) {
 			version:    "8.0.20",
 			capability: capabilities.PerformanceSchemaDataLocksTableCapability,
 			isCapable:  true,
+		}, {
+			version:    "5.7.38",
+			capability: capabilities.PerformanceSchemaMetadataLocksTableCapability,
+			isCapable:  false,
+		},
+		{
+			version:    "8.0.20",
+			capability: capabilities.PerformanceSchemaMetadataLocksTableCapability,
+			isCapable:  true,
 		},
 		{
 			// Some ridiculous version

go/mysql/flavor_test.go

@@ -714,6 +714,89 @@ func testScheduler(t *testing.T) {
 				}
 			})
 		})
+		t.Run("force_cutover mdl", func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), extendedWaitTime*5)
+			defer cancel()
+
+			t1uuid = testOnlineDDLStatement(t, createParams(trivialAlterT1Statement, ddlStrategy+" --postpone-completion", "vtgate", "", "", true)) // skip wait
+
+			t.Run("wait for t1 running", func(t *testing.T) {
+				status := onlineddl.WaitForMigrationStatus(t, &vtParams, shards, t1uuid, normalWaitTime, schema.OnlineDDLStatusRunning)
+				fmt.Printf("# Migration status (for debug purposes): <%s>\n", status)
+			})
+			t.Run("wait for t1 ready to complete", func(t *testing.T) {
+				// Waiting for 'running', above, is not enough. We want to let vreplication a chance to start running, or else
+				// we attempt the cut-over too early. Specifically in this test, we're going to lock rows FOR UPDATE, which,
+				// if vreplication does not get the chance to start, will prevent it from doing anything at all.
+				// ready_to_complete is a great signal for us that vreplication is healthy and up to date.
+				waitForReadyToComplete(t, t1uuid, true)
+			})
+
+			conn, err := primaryTablet.VttabletProcess.TabletConn(keyspaceName, true)
+			require.NoError(t, err)
+			defer conn.Close()
+
+			unlockTables := func() error {
+				_, err := conn.ExecuteFetch("unlock tables", 0, false)
+				return err
+			}
+			t.Run("locking table", func(t *testing.T) {
+				_, err := conn.ExecuteFetch("lock tables t1_test write", 0, false)
+				require.NoError(t, err)
+			})
+			defer unlockTables()
+			t.Run("injecting heartbeats asynchronously", func(t *testing.T) {
+				go func() {
+					ticker := time.NewTicker(time.Second)
+					defer ticker.Stop()
+					for {
+						throttler.CheckThrottler(clusterInstance, primaryTablet, throttlerapp.OnlineDDLName, nil)
+						select {
+						case <-ticker.C:
+						case <-ctx.Done():
+							return
+						}
+					}
+				}()
+			})
+			t.Run("check no force_cutover", func(t *testing.T) {
+				rs := onlineddl.ReadMigrations(t, &vtParams, t1uuid)
+				require.NotNil(t, rs)
+				for _, row := range rs.Named().Rows {
+					forceCutOver := row.AsInt64("force_cutover", 0)
+					assert.Equal(t, int64(0), forceCutOver) // disabled
+				}
+			})
+			t.Run("attempt to complete", func(t *testing.T) {
+				onlineddl.CheckCompleteMigration(t, &vtParams, shards, t1uuid, true)
+			})
+			t.Run("cut-over fail due to timeout", func(t *testing.T) {
+				waitForMessage(t, t1uuid, "(errno 3024) (sqlstate HY000): Query execution was interrupted, maximum statement execution time exceeded")
+				status := onlineddl.WaitForMigrationStatus(t, &vtParams, shards, t1uuid, normalWaitTime, schema.OnlineDDLStatusComplete, schema.OnlineDDLStatusFailed, schema.OnlineDDLStatusRunning)
+				fmt.Printf("# Migration status (for debug purposes): <%s>\n", status)
+				onlineddl.CheckMigrationStatus(t, &vtParams, shards, t1uuid, schema.OnlineDDLStatusRunning)
+			})
+			t.Run("force_cutover", func(t *testing.T) {
+				onlineddl.CheckForceMigrationCutOver(t, &vtParams, shards, t1uuid, true)
+			})
+			t.Run("check force_cutover", func(t *testing.T) {
+				rs := onlineddl.ReadMigrations(t, &vtParams, t1uuid)
+				require.NotNil(t, rs)
+				for _, row := range rs.Named().Rows {
+					forceCutOver := row.AsInt64("force_cutover", 0)
+					assert.Equal(t, int64(1), forceCutOver) // enabled
+				}
+			})
+			t.Run("expect completion", func(t *testing.T) {
+				status := onlineddl.WaitForMigrationStatus(t, &vtParams, shards, t1uuid, normalWaitTime, schema.OnlineDDLStatusComplete, schema.OnlineDDLStatusFailed)
+				fmt.Printf("# Migration status (for debug purposes): <%s>\n", status)
+				onlineddl.CheckMigrationStatus(t, &vtParams, shards, t1uuid, schema.OnlineDDLStatusComplete)
+			})
+			t.Run("expect unlock failure", func(t *testing.T) {
+				err := unlockTables()
+				assert.ErrorContains(t, err, "broken pipe")
+			})
+		})
 	}
 	t.Run("ALTER both tables non-concurrent", func(t *testing.T) {
 		t1uuid = testOnlineDDLStatement(t, createParams(trivialAlterT1Statement, ddlStrategy, "vtgate", "", "", true)) // skip wait