diff --git a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go index cfe961c4558..84765a7841d 100644 --- a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go +++ b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go @@ -320,9 +320,9 @@ func TestPopulateTable(t *testing.T) { func generateRenameStatement(newFormat bool, fromTableName string, state schema.TableGCState, tm time.Time) (statement string, toTableName string, err error) { if newFormat { - return schema.GenerateRenameStatementNewFormat(fromTableName, state, tm) + return schema.GenerateRenameStatement(fromTableName, state, tm) } - return schema.GenerateRenameStatement(fromTableName, state, tm) + return schema.GenerateRenameStatementOldFormat(fromTableName, state, tm) } func TestHold(t *testing.T) { @@ -448,7 +448,7 @@ func TestPurge(t *testing.T) { func TestPurgeView(t *testing.T) { populateTable(t) - query, tableName, err := schema.GenerateRenameStatement("v1", schema.PurgeTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) + query, tableName, err := generateRenameStatement(true, "v1", schema.PurgeTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) require.NoError(t, err) _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) diff --git a/go/vt/schema/name.go b/go/vt/schema/name.go index 9a3f038c477..c9754129c39 100644 --- a/go/vt/schema/name.go +++ b/go/vt/schema/name.go @@ -17,6 +17,7 @@ limitations under the License. package schema import ( + "fmt" "regexp" "strings" "time" @@ -32,8 +33,23 @@ const ( InternalTableNameExpression string = `^_vt_([a-zA-Z0-9]{3})_([0-f]{32})_([0-9]{14})_$` ) +type InternalTableHint string + +const ( + InternalTableUnknownHint InternalTableHint = "nil" + InternalTableGCHoldHint InternalTableHint = "hld" + InternalTableGCPurgeHint InternalTableHint = "prg" + InternalTableGCEvacHint InternalTableHint = "evc" + InternalTableGCDropHint InternalTableHint = "drp" + InternalTableVreplicationHint InternalTableHint = "vrp" +) + +func (h InternalTableHint) String() string { + return string(h) +} + var ( - // internalTableNameRegexp parses new intrnal table name format, e.g. _vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_ + // internalTableNameRegexp parses new internal table name format, e.g. _vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_ internalTableNameRegexp = regexp.MustCompile(InternalTableNameExpression) ) @@ -65,6 +81,44 @@ func ToReadableTimestamp(t time.Time) string { return t.Format(readableTimeFormat) } +// ReadableTimestamp returns the current timestamp, in seconds resolution, that is human readable +func ReadableTimestamp() string { + return ToReadableTimestamp(time.Now()) +} + +func condenseUUID(uuid string) string { + uuid = strings.ReplaceAll(uuid, "-", "") + uuid = strings.ReplaceAll(uuid, "_", "") + return uuid +} + +// isCondensedUUID answers 'true' when the given string is a condensed UUID, e.g.: +// a0638f6bec7b11ea9bf8000d3a9b8a9a +func isCondensedUUID(uuid string) bool { + return condensedUUIDRegexp.MatchString(uuid) +} + +// generateGCTableName creates an internal table name, based on desired hint and time, and with optional preset UUID. +// If uuid is given, then it must be in condensed-UUID format. If empty, the function auto-generates a UUID. +func GenerateInternalTableName(hint string, uuid string, t time.Time) (tableName string, err error) { + if len(hint) != 3 { + return "", fmt.Errorf("Invalid hint: %s, expected 3 characters", hint) + } + if uuid == "" { + uuid, err = CreateUUIDWithDelimiter("") + } else { + uuid = condenseUUID(uuid) + } + if err != nil { + return "", err + } + if !isCondensedUUID(uuid) { + return "", fmt.Errorf("Invalid UUID: %s, expected condensed 32 hexadecimals", uuid) + } + timestamp := ToReadableTimestamp(t) + return fmt.Sprintf("_vt_%s_%s_%s_", hint, uuid, timestamp), nil +} + // IsInternalOperationTableName answers 'true' when the given table name stands for an internal Vitess // table used for operations such as: // - Online DDL (gh-ost, pt-online-schema-change) diff --git a/go/vt/schema/name_test.go b/go/vt/schema/name_test.go index 24571b91b9f..7d1d086cc45 100644 --- a/go/vt/schema/name_test.go +++ b/go/vt/schema/name_test.go @@ -21,6 +21,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestNameIsGCTableName(t *testing.T) { @@ -165,10 +166,61 @@ func TestAnalyzeInternalTableName(t *testing.T) { assert.Equal(t, ts.isInternal, isInternal) if ts.isInternal { assert.NoError(t, err) - assert.True(t, IsGCUUID(uuid)) + assert.True(t, isCondensedUUID(uuid)) assert.Equal(t, ts.hint, hint) assert.Equal(t, ts.t, tm) } }) } } + +func TestToReadableTimestamp(t *testing.T) { + ti, err := time.Parse(time.UnixDate, "Wed Feb 25 11:06:39 PST 2015") + assert.NoError(t, err) + + readableTimestamp := ToReadableTimestamp(ti) + assert.Equal(t, "20150225110639", readableTimestamp) +} + +func TestGenerateInternalTableName(t *testing.T) { + ti, err := time.Parse(time.UnixDate, "Wed Feb 25 11:06:39 PST 2015") + assert.NoError(t, err) + + { + uuid := "6ace8bcef73211ea87e9f875a4d24e90" + tableName, err := GenerateInternalTableName(InternalTableGCPurgeHint.String(), uuid, ti) + require.NoError(t, err) + assert.Equal(t, "_vt_prg_6ace8bcef73211ea87e9f875a4d24e90_20150225110639_", tableName) + assert.True(t, IsInternalOperationTableName(tableName)) + } + { + uuid := "4e5dcf80_354b_11eb_82cd_f875a4d24e90" + tableName, err := GenerateInternalTableName(InternalTableGCPurgeHint.String(), uuid, ti) + require.NoError(t, err) + assert.Equal(t, "_vt_prg_4e5dcf80354b11eb82cdf875a4d24e90_20150225110639_", tableName) + assert.True(t, IsInternalOperationTableName(tableName)) + } + { + uuid := "4e5dcf80-354b-11eb-82cd-f875a4d24e90" + tableName, err := GenerateInternalTableName(InternalTableGCPurgeHint.String(), uuid, ti) + require.NoError(t, err) + assert.Equal(t, "_vt_prg_4e5dcf80354b11eb82cdf875a4d24e90_20150225110639_", tableName) + assert.True(t, IsInternalOperationTableName(tableName)) + } + { + uuid := "" + tableName, err := GenerateInternalTableName(InternalTableGCPurgeHint.String(), uuid, ti) + require.NoError(t, err) + assert.True(t, IsInternalOperationTableName(tableName)) + } + { + uuid := "4e5dcf80_354b_11eb_82cd_f875a4d24e90_00001111" + _, err := GenerateInternalTableName(InternalTableGCPurgeHint.String(), uuid, ti) + require.ErrorContains(t, err, "Invalid UUID") + } + { + uuid := "6ace8bcef73211ea87e9f875a4d24e90" + _, err := GenerateInternalTableName("abcdefg", uuid, ti) + require.ErrorContains(t, err, "Invalid hint") + } +} diff --git a/go/vt/schema/online_ddl_test.go b/go/vt/schema/online_ddl_test.go index f73984bcf20..c443f6b28ce 100644 --- a/go/vt/schema/online_ddl_test.go +++ b/go/vt/schema/online_ddl_test.go @@ -59,7 +59,7 @@ func TestGetGCUUID(t *testing.T) { onlineDDL, err := NewOnlineDDL("ks", "tbl", "alter table t drop column c", NewDDLStrategySetting(DDLStrategyDirect, ""), "", "", parser) assert.NoError(t, err) gcUUID := onlineDDL.GetGCUUID() - assert.True(t, IsGCUUID(gcUUID)) + assert.True(t, isCondensedUUID(gcUUID)) uuids[gcUUID] = true } assert.Equal(t, count, len(uuids)) diff --git a/go/vt/schema/tablegc.go b/go/vt/schema/tablegc.go index 2f102085f00..fc1b8361fb4 100644 --- a/go/vt/schema/tablegc.go +++ b/go/vt/schema/tablegc.go @@ -44,73 +44,66 @@ const ( TableDroppedGCState TableGCState = "" ) +func (s TableGCState) TableHint() InternalTableHint { + if hint, ok := gcStatesTableHints[s]; ok { + return hint + } + return InternalTableUnknownHint +} + const ( - GCTableNameExpression string = `^_vt_(HOLD|PURGE|EVAC|DROP)_([0-f]{32})_([0-9]{14})$` - // NewGCTableNameExpression parses new intrnal table name format, e.g. _vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_ - NewGCTableNameExpression string = `^_vt_(hld|prg|evc|drp)_([0-f]{32})_([0-9]{14})_$` + OldGCTableNameExpression string = `^_vt_(HOLD|PURGE|EVAC|DROP)_([0-f]{32})_([0-9]{14})$` + // GCTableNameExpression parses new internal table name format, e.g. _vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_ + GCTableNameExpression string = `^_vt_(hld|prg|evc|drp)_([0-f]{32})_([0-9]{14})_$` ) var ( - gcUUIDRegexp = regexp.MustCompile(`^[0-f]{32}$`) - gcTableNameRegexp = regexp.MustCompile(GCTableNameExpression) - - gcStates = map[string]TableGCState{ - string(HoldTableGCState): HoldTableGCState, - "hld": HoldTableGCState, - string(PurgeTableGCState): PurgeTableGCState, - "prg": PurgeTableGCState, - string(EvacTableGCState): EvacTableGCState, - "evc": EvacTableGCState, - string(DropTableGCState): DropTableGCState, - "drp": DropTableGCState, - } + condensedUUIDRegexp = regexp.MustCompile(`^[0-f]{32}$`) + oldGCTableNameRegexp = regexp.MustCompile(OldGCTableNameExpression) + + gcStates = map[string]TableGCState{} + gcStatesTableHints = map[TableGCState]InternalTableHint{} ) -// IsGCUUID answers 'true' when the given string is an GC UUID, e.g.: -// a0638f6bec7b11ea9bf8000d3a9b8a9a -func IsGCUUID(uuid string) bool { - return gcUUIDRegexp.MatchString(uuid) +func init() { + gcStatesTableHints[HoldTableGCState] = InternalTableGCHoldHint + gcStatesTableHints[PurgeTableGCState] = InternalTableGCPurgeHint + gcStatesTableHints[EvacTableGCState] = InternalTableGCEvacHint + gcStatesTableHints[DropTableGCState] = InternalTableGCDropHint + for _, gcState := range []TableGCState{HoldTableGCState, PurgeTableGCState, EvacTableGCState, DropTableGCState} { + gcStates[string(gcState)] = gcState + gcStates[gcState.TableHint().String()] = gcState + } } // generateGCTableName creates a GC table name, based on desired state and time, and with optional preset UUID. // If uuid is given, then it must be in GC-UUID format. If empty, the function auto-generates a UUID. -func generateGCTableName(state TableGCState, uuid string, t time.Time) (tableName string, err error) { +func generateGCTableNameOldFormat(state TableGCState, uuid string, t time.Time) (tableName string, err error) { if uuid == "" { uuid, err = CreateUUIDWithDelimiter("") } if err != nil { return "", err } - if !IsGCUUID(uuid) { + if !isCondensedUUID(uuid) { return "", fmt.Errorf("Not a valid GC UUID format: %s", uuid) } timestamp := ToReadableTimestamp(t) return fmt.Sprintf("_vt_%s_%s_%s", state, uuid, timestamp), nil } -// generateGCTableNameNewFormat creates a GC table name, based on desired state and time, and with optional preset UUID. +// generateGCTableName creates a GC table name, based on desired state and time, and with optional preset UUID. // If uuid is given, then it must be in GC-UUID format. If empty, the function auto-generates a UUID. -func generateGCTableNameNewFormat(state TableGCState, uuid string, t time.Time) (tableName string, err error) { - if uuid == "" { - uuid, err = CreateUUIDWithDelimiter("") - } - if err != nil { - return "", err - } - if !IsGCUUID(uuid) { - return "", fmt.Errorf("Not a valid GC UUID format: %s", uuid) - } - timestamp := ToReadableTimestamp(t) - var hint string +func generateGCTableName(state TableGCState, uuid string, t time.Time) (tableName string, err error) { for k, v := range gcStates { if v != state { continue } if len(k) == 3 && k != string(state) { // the "new" format - hint = k + return GenerateInternalTableName(k, uuid, t) } } - return fmt.Sprintf("_vt_%s_%s_%s_", hint, uuid, timestamp), nil + return "", fmt.Errorf("Unknown GC state: %v", state) } // GenerateGCTableName creates a GC table name, based on desired state and time, and with random UUID @@ -118,11 +111,6 @@ func GenerateGCTableName(state TableGCState, t time.Time) (tableName string, err return generateGCTableName(state, "", t) } -// GenerateGCTableNameNewFormat creates a GC table name, based on desired state and time, and with random UUID -func GenerateGCTableNameNewFormat(state TableGCState, t time.Time) (tableName string, err error) { - return generateGCTableNameNewFormat(state, "", t) -} - // AnalyzeGCTableName analyzes a given table name to see if it's a GC table, and if so, parse out // its state, uuid, and timestamp func AnalyzeGCTableName(tableName string) (isGCTable bool, state TableGCState, uuid string, t time.Time, err error) { @@ -134,7 +122,7 @@ func AnalyzeGCTableName(tableName string) (isGCTable bool, state TableGCState, u } // Try old naming formats. These names will not be generated in v20. // TODO(shlomi): the code below should be remvoed in v21 - submatch := gcTableNameRegexp.FindStringSubmatch(tableName) + submatch := oldGCTableNameRegexp.FindStringSubmatch(tableName) if len(submatch) == 0 { return false, state, uuid, t, nil } @@ -165,8 +153,8 @@ func GenerateRenameStatementWithUUID(fromTableName string, state TableGCState, u } // GenerateRenameStatementWithUUIDNewFormat generates a "RENAME TABLE" statement, where a table is renamed to a GC table, with preset UUID -func GenerateRenameStatementWithUUIDNewFormat(fromTableName string, state TableGCState, uuid string, t time.Time) (statement string, toTableName string, err error) { - toTableName, err = generateGCTableNameNewFormat(state, uuid, t) +func generateRenameStatementWithUUIDOldFormat(fromTableName string, state TableGCState, uuid string, t time.Time) (statement string, toTableName string, err error) { + toTableName, err = generateGCTableNameOldFormat(state, uuid, t) if err != nil { return "", "", err } @@ -179,8 +167,8 @@ func GenerateRenameStatement(fromTableName string, state TableGCState, t time.Ti } // GenerateRenameStatement generates a "RENAME TABLE" statement, where a table is renamed to a GC table. -func GenerateRenameStatementNewFormat(fromTableName string, state TableGCState, t time.Time) (statement string, toTableName string, err error) { - return GenerateRenameStatementWithUUIDNewFormat(fromTableName, state, "", t) +func GenerateRenameStatementOldFormat(fromTableName string, state TableGCState, t time.Time) (statement string, toTableName string, err error) { + return generateRenameStatementWithUUIDOldFormat(fromTableName, state, "", t) } // ParseGCLifecycle parses a comma separated list of gc states and returns a map of indicated states diff --git a/go/vt/schema/tablegc_test.go b/go/vt/schema/tablegc_test.go index fb9795d4978..3f4e4e7bc09 100644 --- a/go/vt/schema/tablegc_test.go +++ b/go/vt/schema/tablegc_test.go @@ -25,20 +25,37 @@ import ( "github.com/stretchr/testify/require" ) +func TestGCStates(t *testing.T) { + // These are all hard coded + require.Equal(t, HoldTableGCState, gcStates["hld"]) + require.Equal(t, HoldTableGCState, gcStates["HOLD"]) + require.Equal(t, PurgeTableGCState, gcStates["prg"]) + require.Equal(t, PurgeTableGCState, gcStates["PURGE"]) + require.Equal(t, EvacTableGCState, gcStates["evc"]) + require.Equal(t, EvacTableGCState, gcStates["EVAC"]) + require.Equal(t, DropTableGCState, gcStates["drp"]) + require.Equal(t, DropTableGCState, gcStates["DROP"]) + _, ok := gcStates["purge"] + require.False(t, ok) + _, ok = gcStates["vrp"] + require.False(t, ok) + require.Equal(t, 2*4, len(gcStates)) // 4 states, 2 forms each +} + func TestIsGCTableName(t *testing.T) { tm := time.Now() states := []TableGCState{HoldTableGCState, PurgeTableGCState, EvacTableGCState, DropTableGCState} for _, state := range states { for i := 0; i < 10; i++ { - tableName, err := generateGCTableName(state, "", tm) + tableName, err := generateGCTableNameOldFormat(state, "", tm) assert.NoError(t, err) - assert.True(t, IsGCTableName(tableName)) + assert.Truef(t, IsGCTableName(tableName), "table name: %s", tableName) - tableName, err = generateGCTableNameNewFormat(state, "6ace8bcef73211ea87e9f875a4d24e90", tm) + tableName, err = generateGCTableName(state, "6ace8bcef73211ea87e9f875a4d24e90", tm) assert.NoError(t, err) assert.Truef(t, IsGCTableName(tableName), "table name: %s", tableName) - tableName, err = GenerateGCTableNameNewFormat(state, tm) + tableName, err = GenerateGCTableName(state, tm) assert.NoError(t, err) assert.Truef(t, IsGCTableName(tableName), "table name: %s", tableName) } @@ -77,7 +94,7 @@ func TestIsGCTableName(t *testing.T) { t.Run("explicit regexp", func(t *testing.T) { // NewGCTableNameExpression regexp is used externally by vreplication. Its a redundant form of // InternalTableNameExpression, but is nonetheless required. We verify it works correctly - re := regexp.MustCompile(NewGCTableNameExpression) + re := regexp.MustCompile(GCTableNameExpression) t.Run("accept", func(t *testing.T) { names := []string{ "_vt_hld_6ace8bcef73211ea87e9f875a4d24e90_20200915120410_", @@ -173,7 +190,7 @@ func TestAnalyzeGCTableName(t *testing.T) { assert.Equal(t, ts.isGC, isGC) if ts.isGC { assert.NoError(t, err) - assert.True(t, IsGCUUID(uuid)) + assert.True(t, isCondensedUUID(uuid)) assert.Equal(t, ts.state, state) assert.Equal(t, ts.t, tm) } diff --git a/go/vt/vttablet/onlineddl/executor.go b/go/vt/vttablet/onlineddl/executor.go index dcebc0b9bd4..30bb465a1f0 100644 --- a/go/vt/vttablet/onlineddl/executor.go +++ b/go/vt/vttablet/onlineddl/executor.go @@ -1438,7 +1438,10 @@ func (e *Executor) initVreplicationOriginalMigration(ctx context.Context, online return v, err } - vreplTableName := fmt.Sprintf("_%s_%s_vrepl", onlineDDL.UUID, ReadableTimestamp()) + vreplTableName, err := schema.GenerateInternalTableName(schema.InternalTableVreplicationHint.String(), onlineDDL.UUID, time.Now()) + if err != nil { + return v, err + } if err := e.updateArtifacts(ctx, onlineDDL.UUID, vreplTableName); err != nil { return v, err } @@ -1757,7 +1760,7 @@ exit $exit_code runGhost := func(execute bool) error { alterOptions := e.parseAlterOptions(ctx, onlineDDL) - forceTableNames := fmt.Sprintf("%s_%s", onlineDDL.UUID, ReadableTimestamp()) + forceTableNames := fmt.Sprintf("%s_%s", onlineDDL.UUID, schema.ReadableTimestamp()) if err := e.updateArtifacts(ctx, onlineDDL.UUID, fmt.Sprintf("_%s_gho", forceTableNames), @@ -1988,7 +1991,7 @@ export MYSQL_PWD runPTOSC := func(execute bool) error { os.Setenv("MYSQL_PWD", onlineDDLPassword) - newTableName := fmt.Sprintf("_%s_%s_new", onlineDDL.UUID, ReadableTimestamp()) + newTableName := fmt.Sprintf("_%s_%s_new", onlineDDL.UUID, schema.ReadableTimestamp()) if err := e.updateArtifacts(ctx, onlineDDL.UUID, fmt.Sprintf("_%s_old", onlineDDL.Table), diff --git a/go/vt/vttablet/onlineddl/util.go b/go/vt/vttablet/onlineddl/util.go index 305b01c057f..3d06e6df60e 100644 --- a/go/vt/vttablet/onlineddl/util.go +++ b/go/vt/vttablet/onlineddl/util.go @@ -26,15 +26,10 @@ import ( "os/exec" "path/filepath" "strings" - "time" "vitess.io/vitess/go/vt/log" ) -const ( - readableTimeFormat = "20060102150405" -) - // execCmd searches the PATH for a command and runs it, logging the output. // If input is not nil, pipe it to the command's stdin. func execCmd(name string, args, env []string, dir string, input io.Reader, output io.Writer) (cmd *exec.Cmd, err error) { @@ -89,17 +84,3 @@ func RandomHash() string { hasher.Write(rb) return hex.EncodeToString(hasher.Sum(nil)) } - -// ToReadableTimestamp returns a timestamp, in seconds resolution, that is human readable -// (as opposed to unix timestamp which is just a number) -// Example: for Aug 25 2020, 16:04:25 we return "20200825160425" -func ToReadableTimestamp(t time.Time) string { - return t.Format(readableTimeFormat) -} - -// ReadableTimestamp returns a timestamp, in seconds resolution, that is human readable -// (as opposed to unix timestamp which is just a number), and which corresponds to the time now. -// Example: for Aug 25 2020, 16:04:25 we return "20200825160425" -func ReadableTimestamp() string { - return ToReadableTimestamp(time.Now()) -} diff --git a/go/vt/vttablet/onlineddl/util_test.go b/go/vt/vttablet/onlineddl/util_test.go index 707e321c6f5..4beb154c0ae 100644 --- a/go/vt/vttablet/onlineddl/util_test.go +++ b/go/vt/vttablet/onlineddl/util_test.go @@ -18,7 +18,6 @@ package onlineddl import ( "testing" - "time" "github.com/stretchr/testify/assert" ) @@ -31,11 +30,3 @@ func TestRandomHash(t *testing.T) { assert.Equal(t, len(h2), 64) assert.NotEqual(t, h1, h2) } - -func TestToReadableTimestamp(t *testing.T) { - ti, err := time.Parse(time.UnixDate, "Wed Feb 25 11:06:39 PST 2015") - assert.NoError(t, err) - - readableTimestamp := ToReadableTimestamp(ti) - assert.Equal(t, readableTimestamp, "20150225110639") -} diff --git a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go index 11633d95f33..67f4815b2a1 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go @@ -330,8 +330,8 @@ func (vr *vreplicator) buildColInfoMap(ctx context.Context) (map[string][]*Colum req := &tabletmanagerdatapb.GetSchemaRequest{ Tables: []string{"/.*/"}, ExcludeTables: []string{ + "/" + schema.OldGCTableNameExpression + "/", "/" + schema.GCTableNameExpression + "/", - "/" + schema.NewGCTableNameExpression + "/", }, } schema, err := vr.mysqld.GetSchema(ctx, vr.dbClient.DBName(), req)