From 31229d9da8a3b907fe62b0b8a8e7cbae153ecf15 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 29 Oct 2024 16:54:14 +0100 Subject: [PATCH 01/12] test wip: run plan tests end2end Signed-off-by: Andres Taylor --- .../vtgate/queries/plan_tests/plan_test.go | 162 ++++++++++++++++++ go/vt/vtgate/planbuilder/plan_test.go | 21 +-- go/vt/vtgate/planbuilder/test_helper.go | 27 +++ .../planbuilder/testdata/vschemas/schema.json | 48 +++--- .../planbuilder/testdata/vschemas/schema.sql | 92 ++++++++++ 5 files changed, 314 insertions(+), 36 deletions(-) create mode 100644 go/test/endtoend/vtgate/queries/plan_tests/plan_test.go create mode 100644 go/vt/vtgate/planbuilder/test_helper.go create mode 100644 go/vt/vtgate/planbuilder/testdata/vschemas/schema.sql diff --git a/go/test/endtoend/vtgate/queries/plan_tests/plan_test.go b/go/test/endtoend/vtgate/queries/plan_tests/plan_test.go new file mode 100644 index 00000000000..f629916fdb3 --- /dev/null +++ b/go/test/endtoend/vtgate/queries/plan_tests/plan_test.go @@ -0,0 +1,162 @@ +package plan_tests + +import ( + _ "embed" + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" + "vitess.io/vitess/go/vt/vtgate/planbuilder" +) + +func readJSONTests(filename string) []planbuilder.PlanTest { + var output []planbuilder.PlanTest + file, err := os.Open(locateFile(filename)) + if err != nil { + panic(err) + } + dec := json.NewDecoder(file) + err = dec.Decode(&output) + if err != nil { + panic(err) + } + return output +} + +func locateFile(name string) string { + return "../../../../../vt/vtgate/planbuilder/testdata/" + name +} + +func TestPlan(t *testing.T) { + mcmp, closer := start(t) + defer closer() + tests := readJSONTests("select_cases.json") + for _, test := range tests { + mcmp.Run(test.Query, func(mcmp *utils.MySQLCompare) { + mcmp.Exec(test.Query) + }) + } +} + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + mysqlParams mysql.ConnParams + keyspaceName = "user" + cell = "test_aggr" +) + +func readFile(filename string) string { + schema, err := os.ReadFile(locateFile(filename)) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + return string(schema) +} + +func start(t *testing.T) (utils.MySQLCompare, func()) { + mcmp, err := utils.NewMySQLCompare(t, vtParams, mysqlParams) + require.NoError(t, err) + return mcmp, func() { + mcmp.Close() + } +} + +func extractUserKS(jsonString string) string { + var result map[string]any + if err := json.Unmarshal([]byte(jsonString), &result); err != nil { + panic(err.Error()) + } + + keyspaces, ok := result["keyspaces"].(map[string]any) + if !ok { + panic("Keyspaces not found") + } + + user, ok := keyspaces["user"].(map[string]any) + if !ok { + panic("User keyspaces not found") + } + + tables, ok := user["tables"].(map[string]any) + if !ok { + panic("Tables not found") + } + + userTbl, ok := tables["user"].(map[string]any) + if !ok { + panic("User table not found") + } + + delete(userTbl, "auto_increment") // TODO: we should have an unsharded keyspace where this could live + + // Marshal the inner part back to JSON string + userJson, err := json.Marshal(user) + if err != nil { + panic(err.Error()) + } + + return string(userJson) +} + +func TestMain(m *testing.M) { + defer cluster.PanicHandler(nil) + + schemaSQL := readFile("vschemas/schema.sql") + vschema := extractUserKS(readFile("vschemas/schema.json")) + + exitCode := func() int { + clusterInstance = cluster.NewCluster(cell, "localhost") + defer clusterInstance.Teardown() + + // Start topo server + err := clusterInstance.StartTopo() + if err != nil { + return 1 + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + SchemaSQL: schemaSQL, + VSchema: vschema, + } + err = clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 0, false) + if err != nil { + return 1 + } + + // TODO: (@GuptaManan100/@systay): Also run the tests with normalizer on. + clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs, + "--normalize_queries=false", + "--schema_change_signal=false", + ) + + // Start vtgate + err = clusterInstance.StartVtgate() + if err != nil { + return 1 + } + + vtParams = clusterInstance.GetVTParams(keyspaceName) + + // create mysql instance and connection parameters + conn, closer, err := utils.NewMySQL(clusterInstance, keyspaceName, schemaSQL) + if err != nil { + fmt.Println(err) + return 1 + } + defer closer() + mysqlParams = conn + + return m.Run() + }() + os.Exit(exitCode) +} diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 9cf92a91ddf..36ed3cdc3f5 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -659,21 +659,12 @@ func createFkDefinition(childCols []string, parentTableName string, parentCols [ } } -type ( - planTest struct { - Comment string `json:"comment,omitempty"` - Query string `json:"query,omitempty"` - Plan json.RawMessage `json:"plan,omitempty"` - Skip bool `json:"skip,omitempty"` - } -) - func (s *planTestSuite) testFile(filename string, vschema *vschemawrapper.VSchemaWrapper, render bool) { opts := jsondiff.DefaultConsoleOptions() s.T().Run(filename, func(t *testing.T) { failed := false - var expected []planTest + var expected []PlanTest for _, tcase := range readJSONTests(filename) { testName := tcase.Comment if testName == "" { @@ -682,7 +673,7 @@ func (s *planTestSuite) testFile(filename string, vschema *vschemawrapper.VSchem if tcase.Query == "" { continue } - current := planTest{ + current := PlanTest{ Comment: testName, Query: tcase.Query, } @@ -730,8 +721,8 @@ func (s *planTestSuite) testFile(filename string, vschema *vschemawrapper.VSchem }) } -func readJSONTests(filename string) []planTest { - var output []planTest +func readJSONTests(filename string) []PlanTest { + var output []PlanTest file, err := os.Open(locateFile(filename)) if err != nil { panic(err) @@ -745,7 +736,7 @@ func readJSONTests(filename string) []planTest { return output } -func getPlanOutput(tcase planTest, vschema *vschemawrapper.VSchemaWrapper, render bool) (out string) { +func getPlanOutput(tcase PlanTest, vschema *vschemawrapper.VSchemaWrapper, render bool) (out string) { defer func() { if r := recover(); r != nil { out = fmt.Sprintf("panicked: %v\n%s", r, string(debug.Stack())) @@ -887,7 +878,7 @@ func BenchmarkBaselineVsMirrored(b *testing.B) { }) } -func benchmarkPlanner(b *testing.B, version plancontext.PlannerVersion, testCases []planTest, vschema *vschemawrapper.VSchemaWrapper) { +func benchmarkPlanner(b *testing.B, version plancontext.PlannerVersion, testCases []PlanTest, vschema *vschemawrapper.VSchemaWrapper) { b.ReportAllocs() for n := 0; n < b.N; n++ { for _, tcase := range testCases { diff --git a/go/vt/vtgate/planbuilder/test_helper.go b/go/vt/vtgate/planbuilder/test_helper.go new file mode 100644 index 00000000000..25d6b7306d1 --- /dev/null +++ b/go/vt/vtgate/planbuilder/test_helper.go @@ -0,0 +1,27 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package planbuilder + +import "encoding/json" + +type PlanTest struct { + Comment string `json:"comment,omitempty"` + Query string `json:"query,omitempty"` + Plan json.RawMessage `json:"plan,omitempty"` + Skip bool `json:"skip,omitempty"` + SkipE2E bool `json:"skip_e2e,omitempty"` +} diff --git a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json index 4fe275f2398..aab90c3b578 100644 --- a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json +++ b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json @@ -58,34 +58,34 @@ "sharded": true, "vindexes": { "user_index": { - "type": "hash_test", + "type": "hash", "owner": "user" }, "kid_index": { - "type": "hash_test", + "type": "hash", "owner": "multicolvin" }, "user_md5_index": { "type": "unicode_loose_md5" }, "music_user_map": { - "type": "lookup_test", + "type": "lookup_unique", "owner": "music" }, "cola_map": { - "type": "lookup_test", + "type": "lookup_unique", "owner": "multicolvin" }, "colb_colc_map": { - "type": "lookup_test", + "type": "lookup_unique", "owner": "multicolvin" }, "cola_kid_map": { - "type": "lookup_test", + "type": "lookup_unique", "owner": "overlap_vindex" }, "name_user_map": { - "type": "name_lkp_test", + "type": "lookup_hash", "owner": "user", "params": { "table": "name_user_vdx", @@ -94,41 +94,44 @@ } }, "email_user_map": { - "type": "lookup_test", + "type": "lookup_unique", "owner": "user_metadata" }, "address_user_map": { - "type": "lookup_test", + "type": "lookup_unique", "owner": "user_metadata" }, "costly_map": { - "type": "costly", + "type": "lookup_unique", "owner": "user" }, "hash_dup": { - "type": "hash_test", + "type": "hash", "owner": "user" }, "vindex1": { - "type": "hash_test", + "type": "hash", "owner": "samecolvin" }, "vindex2": { - "type": "lookup_test", + "type": "lookup_unique", "owner": "samecolvin" }, "cfc": { "type": "cfc" }, "multicolIdx": { - "type": "multiCol_test" + "type": "multicol", + "params": { + "column_count": "2" + } }, "colc_map": { - "type": "lookup_test", + "type": "lookup_unique", "owner": "multicol_tbl" }, "name_muticoltbl_map": { - "type": "name_lkp_test", + "type": "lookup_hash", "owner": "multicol_tbl" }, "non_planable_user_map": { @@ -141,7 +144,7 @@ "owner": "user_metadata" }, "lkp_shard_map": { - "type": "name_lkp_test", + "type": "lookup_hash", "owner": "mixed_tbl", "params": { "table": "lkp_shard_vdx", @@ -153,7 +156,7 @@ "type": "xxhash" }, "unq_lkp_bf_vdx": { - "type": "unq_lkp_test", + "type": "lookup_unique", "owner": "customer", "params": { "table": "unq_lkp_idx", @@ -164,7 +167,7 @@ } }, "unq_lkp_vdx": { - "type": "unq_lkp_test", + "type": "lookup_unique", "owner": "customer", "params": { "table": "unq_lkp_idx", @@ -174,7 +177,7 @@ } }, "lkp_bf_vdx": { - "type": "name_lkp_test", + "type": "lookup_hash", "owner": "customer", "params": { "table": "lkp_shard_vdx", @@ -641,7 +644,10 @@ "type": "hash_test" }, "multicolIdx": { - "type": "multiCol_test" + "type": "multicol", + "params": { + "column_count": "3" + } } }, "tables": { diff --git a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.sql b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.sql new file mode 100644 index 00000000000..c54943f9ecd --- /dev/null +++ b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.sql @@ -0,0 +1,92 @@ +CREATE TABLE user +( + id INT PRIMARY KEY, + col BIGINT, + predef1 VARCHAR(255), + predef2 VARCHAR(255), + textcol1 VARCHAR(255), + intcol BIGINT, + textcol2 VARCHAR(255) +); + +CREATE TABLE user_metadata +( + user_id INT, + email VARCHAR(255), + address VARCHAR(255), + md5 VARCHAR(255), + non_planable VARCHAR(255), + PRIMARY KEY (user_id) +); + +CREATE TABLE music +( + user_id INT, + id INT, + PRIMARY KEY (user_id) +); + +CREATE TABLE samecolvin +( + col VARCHAR(255), + PRIMARY KEY (col) +); + +CREATE TABLE multicolvin +( + kid INT, + column_a VARCHAR(255), + column_b VARCHAR(255), + column_c VARCHAR(255), + PRIMARY KEY (kid) +); + +CREATE TABLE customer +( + id INT, + email VARCHAR(255), + phone VARCHAR(255), + PRIMARY KEY (id) +); + +CREATE TABLE multicol_tbl +( + cola VARCHAR(255), + colb VARCHAR(255), + colc VARCHAR(255), + name VARCHAR(255), + PRIMARY KEY (cola, colb) +); + +CREATE TABLE mixed_tbl +( + shard_key VARCHAR(255), + lkp_key VARCHAR(255), + PRIMARY KEY (shard_key) +); + +CREATE TABLE pin_test +( + id INT PRIMARY KEY +); + +CREATE TABLE cfc_vindex_col +( + c1 VARCHAR(255), + c2 VARCHAR(255), + PRIMARY KEY (c1) +); + +CREATE TABLE unq_lkp_idx +( + unq_key INT PRIMARY KEY, + keyspace_id VARCHAR(255) +); + +CREATE TABLE t1 +( + c1 INT, + c2 INT, + c3 INT, + PRIMARY KEY (c1) +); \ No newline at end of file From ff829935e3908ca864b6817ad5a9269dd646aa35 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Tue, 29 Oct 2024 10:30:32 -0600 Subject: [PATCH 02/12] Add skip_e2e to the plan tests Signed-off-by: Florent Poinsard --- go/test/endtoend/utils/cmp.go | 12 + .../vtgate/queries/plan_tests/plan_test.go | 42 ++- .../planbuilder/testdata/select_cases.json | 339 ++++++++++++------ 3 files changed, 278 insertions(+), 115 deletions(-) diff --git a/go/test/endtoend/utils/cmp.go b/go/test/endtoend/utils/cmp.go index 3b47e1f68dc..7d94c181abd 100644 --- a/go/test/endtoend/utils/cmp.go +++ b/go/test/endtoend/utils/cmp.go @@ -215,6 +215,18 @@ func (mcmp *MySQLCompare) Exec(query string) *sqltypes.Result { return vtQr } +// ExecAssert is the same as Exec, but it only does assertions, it won't FailNow +func (mcmp *MySQLCompare) ExecAssert(query string) *sqltypes.Result { + mcmp.t.Helper() + vtQr, err := mcmp.VtConn.ExecuteFetch(query, 1000, true) + assert.NoError(mcmp.t, err, "[Vitess Error] for query: "+query) + + mysqlQr, err := mcmp.MySQLConn.ExecuteFetch(query, 1000, true) + assert.NoError(mcmp.t, err, "[MySQL Error] for query: "+query) + compareVitessAndMySQLResults(mcmp.t, query, mcmp.VtConn, vtQr, mysqlQr, CompareOptions{}) + return vtQr +} + // ExecNoCompare executes the query on vitess and mysql but does not compare the result with each other. func (mcmp *MySQLCompare) ExecNoCompare(query string) (*sqltypes.Result, *sqltypes.Result) { mcmp.t.Helper() diff --git a/go/test/endtoend/vtgate/queries/plan_tests/plan_test.go b/go/test/endtoend/vtgate/queries/plan_tests/plan_test.go index f629916fdb3..c82175308b7 100644 --- a/go/test/endtoend/vtgate/queries/plan_tests/plan_test.go +++ b/go/test/endtoend/vtgate/queries/plan_tests/plan_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package plan_tests import ( @@ -21,6 +37,7 @@ func readJSONTests(filename string) []planbuilder.PlanTest { if err != nil { panic(err) } + defer file.Close() dec := json.NewDecoder(file) err = dec.Decode(&output) if err != nil { @@ -36,12 +53,30 @@ func locateFile(name string) string { func TestPlan(t *testing.T) { mcmp, closer := start(t) defer closer() - tests := readJSONTests("select_cases.json") + tests := readJSONTests("select_cases_new.json") for _, test := range tests { + if test.SkipE2E { + continue + } mcmp.Run(test.Query, func(mcmp *utils.MySQLCompare) { mcmp.Exec(test.Query) + // if mcmp.AsT().Failed() { + // tests[i].SkipE2E = true + // } }) } + + // file, err := os.Create(locateFile("select_cases_new.json")) + // if err != nil { + // panic(err) + // } + // defer file.Close() + // + // enc := json.NewEncoder(file) + // enc.SetEscapeHTML(false) + // enc.SetIndent("", " ") + // err = enc.Encode(tests) + // require.NoError(t, err) } var ( @@ -119,6 +154,7 @@ func TestMain(m *testing.M) { // Start topo server err := clusterInstance.StartTopo() if err != nil { + fmt.Println(err.Error()) return 1 } @@ -130,6 +166,7 @@ func TestMain(m *testing.M) { } err = clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 0, false) if err != nil { + fmt.Println(err.Error()) return 1 } @@ -142,6 +179,7 @@ func TestMain(m *testing.M) { // Start vtgate err = clusterInstance.StartVtgate() if err != nil { + fmt.Println(err.Error()) return 1 } @@ -150,7 +188,7 @@ func TestMain(m *testing.M) { // create mysql instance and connection parameters conn, closer, err := utils.NewMySQL(clusterInstance, keyspaceName, schemaSQL) if err != nil { - fmt.Println(err) + fmt.Println(err.Error()) return 1 } defer closer() diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.json b/go/vt/vtgate/planbuilder/testdata/select_cases.json index ab69df2cc47..68cd5a5bdfc 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.json @@ -92,7 +92,8 @@ "user.user", "user.user_metadata" ] - } + }, + "skip_e2e": true }, { "comment": "select with timeout directive sets QueryTimeout in the route", @@ -197,7 +198,8 @@ "TablesUsed": [ "main.unsharded" ] - } + }, + "skip_e2e": true }, { "comment": "select with partial scatter directive", @@ -375,7 +377,8 @@ "TablesUsed": [ "user.authoritative" ] - } + }, + "skip_e2e": true }, { "comment": "select * from join of authoritative tables", @@ -397,12 +400,14 @@ "TablesUsed": [ "user.authoritative" ] - } + }, + "skip_e2e": true }, { "comment": "test table lookup failure for authoritative code path", "query": "select a.* from authoritative", - "plan": "Unknown table 'a'" + "plan": "Unknown table 'a'", + "skip_e2e": true }, { "comment": "select * from qualified authoritative table", @@ -424,7 +429,8 @@ "TablesUsed": [ "user.authoritative" ] - } + }, + "skip_e2e": true }, { "comment": "select * from intermixing of authoritative table with non-authoritative results in no expansion", @@ -447,7 +453,8 @@ "user.authoritative", "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "select authoritative.* with intermixing still expands", @@ -470,7 +477,8 @@ "user.authoritative", "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "auto-resolve anonymous columns for simple route", @@ -493,7 +501,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "json_arrayagg in single sharded query", @@ -519,7 +528,8 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "json_objectagg in single sharded query", @@ -545,17 +555,20 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "unsupported json aggregation expressions in scatter query", "query": "select count(1) from user where cola = 'abc' group by n_id having json_arrayagg(a_id) = '[]'", - "plan": "VT12001: unsupported: in scatter query: aggregation function 'json_arrayagg(a_id)'" + "plan": "VT12001: unsupported: in scatter query: aggregation function 'json_arrayagg(a_id)'", + "skip_e2e": true }, { "comment": "Cannot auto-resolve for cross-shard joins", "query": "select col from user join user_extra", - "plan": "Column 'col' in field list is ambiguous" + "plan": "Column 'col' in field list is ambiguous", + "skip_e2e": true }, { "comment": "Auto-resolve should work if unique vindex columns are referenced", @@ -597,7 +610,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "database calls should be substituted", @@ -619,7 +633,8 @@ "TablesUsed": [ "main.dual" ] - } + }, + "skip_e2e": true }, { "comment": "last_insert_id for unsharded route", @@ -641,7 +656,8 @@ "TablesUsed": [ "main.unsharded" ] - } + }, + "skip_e2e": true }, { "comment": "select from dual on unqualified keyspace", @@ -694,7 +710,8 @@ { "comment": "prefixing dual with a keyspace should not work", "query": "select 1 from user.dual", - "plan": "table dual not found" + "plan": "table dual not found", + "skip_e2e": true }, { "comment": "RHS route referenced", @@ -736,7 +753,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "Both routes referenced", @@ -778,7 +796,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "Expression with single-route reference", @@ -820,7 +839,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "subquery with an aggregation in order by that can be merged into a single route", @@ -847,7 +867,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "Jumbled references", @@ -889,7 +910,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "Comments", @@ -931,7 +953,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "for update", @@ -973,7 +996,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "Field query should work for joins select bind vars", @@ -1018,7 +1042,8 @@ "main.unsharded", "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "Case preservation", @@ -1060,12 +1085,14 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "syntax error", "query": "the quick brown fox", - "plan": "syntax error at position 4 near 'the'" + "plan": "syntax error at position 4 near 'the'", + "skip_e2e": true }, { "comment": "Hex number is not treated as a simple value", @@ -1113,7 +1140,8 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "Selection but make the planner explicitly use a vindex", @@ -1164,12 +1192,14 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "Vindex hint on a non-existing vindex", "query": "select * from user use vindex (does_not_exist) where id = 1", - "plan": "VT09021: Vindex 'does_not_exist' does not exist in table 'user.user'" + "plan": "VT09021: Vindex 'does_not_exist' does not exist in table 'user.user'", + "skip_e2e": true }, { "comment": "sharded limit offset", @@ -1231,7 +1261,8 @@ "TablesUsed": [ "user.music" ] - } + }, + "skip_e2e": true }, { "comment": "Sharding Key Condition in Parenthesis", @@ -1257,7 +1288,8 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "Multiple parenthesized expressions", @@ -1283,7 +1315,8 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "Multiple parenthesized expressions", @@ -1309,7 +1342,8 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "Column Aliasing with Table.Column", @@ -1387,7 +1421,8 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "Column as boolean-ish", @@ -1413,7 +1448,8 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "PK as fake boolean, and column as boolean-ish", @@ -1439,7 +1475,8 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "top level subquery in select", @@ -1484,7 +1521,8 @@ "main.unsharded", "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "sub-expression subquery in select", @@ -1529,7 +1567,8 @@ "main.unsharded", "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "select * from derived table expands specific columns", @@ -1571,17 +1610,20 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "duplicate columns not allowed in derived table", "query": "select * from (select user.id, user_extra.id from user join user_extra) as t", - "plan": "Duplicate column name 'id'" + "plan": "Duplicate column name 'id'", + "skip_e2e": true }, { "comment": "non-existent symbol in cross-shard derived table", "query": "select t.col from (select user.id from user join user_extra) as t", - "plan": "column 't.col' not found" + "plan": "column 't.col' not found", + "skip_e2e": true }, { "comment": "union with the same target shard", @@ -1608,7 +1650,8 @@ "user.music", "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "union with the same target shard last_insert_id", @@ -1635,7 +1678,8 @@ "user.music", "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "unsharded union in derived table", @@ -1657,7 +1701,8 @@ "TablesUsed": [ "main.unsharded" ] - } + }, + "skip_e2e": true }, { "comment": "unsharded union in subquery", @@ -1679,7 +1724,8 @@ "TablesUsed": [ "main.unsharded" ] - } + }, + "skip_e2e": true }, { "comment": "(select id from unsharded) union (select id from unsharded_auto) order by id limit 5", @@ -1702,7 +1748,8 @@ "main.unsharded", "main.unsharded_auto" ] - } + }, + "skip_e2e": true }, { "comment": "unsharded union", @@ -1725,7 +1772,8 @@ "main.unsharded", "main.unsharded_auto" ] - } + }, + "skip_e2e": true }, { "comment": "unsharded nested union", @@ -1748,7 +1796,8 @@ "main.unsharded", "main.unsharded_auto" ] - } + }, + "skip_e2e": true }, { "comment": "unsharded nested union with limit", @@ -1770,7 +1819,8 @@ "TablesUsed": [ "main.unsharded" ] - } + }, + "skip_e2e": true }, { "comment": "routing rules: ensure directives are not lost", @@ -1793,7 +1843,8 @@ "TablesUsed": [ "main.unsharded" ] - } + }, + "skip_e2e": true }, { "comment": "routing table on music", @@ -1815,7 +1866,8 @@ "TablesUsed": [ "user.music" ] - } + }, + "skip_e2e": true }, { "comment": "testing SingleRow Projection", @@ -1962,7 +2014,8 @@ "main.unsharded_a", "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "Complex expression in a subquery used in NOT IN clause of an aggregate query", @@ -2015,7 +2068,8 @@ "main.unsharded_a", "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "testing SingleRow Projection with arithmetics", @@ -2218,12 +2272,14 @@ { "comment": "sql_calc_found_rows in sub queries", "query": "select * from music where user_id IN (select sql_calc_found_rows * from music limit 10)", - "plan": "Incorrect usage/placement of 'SQL_CALC_FOUND_ROWS'" + "plan": "Incorrect usage/placement of 'SQL_CALC_FOUND_ROWS'", + "skip_e2e": true }, { "comment": "sql_calc_found_rows in derived table", "query": "select sql_calc_found_rows * from (select sql_calc_found_rows * from music limit 10) t limit 1", - "plan": "Incorrect usage/placement of 'SQL_CALC_FOUND_ROWS'" + "plan": "Incorrect usage/placement of 'SQL_CALC_FOUND_ROWS'", + "skip_e2e": true }, { "comment": "select from unsharded keyspace into dumpfile", @@ -2245,7 +2301,8 @@ "TablesUsed": [ "main.unsharded" ] - } + }, + "skip_e2e": true }, { "comment": "select from unsharded keyspace into outfile", @@ -2267,7 +2324,8 @@ "TablesUsed": [ "main.unsharded" ] - } + }, + "skip_e2e": true }, { "comment": "select from unsharded keyspace into outfile s3", @@ -2289,7 +2347,8 @@ "TablesUsed": [ "main.unsharded" ] - } + }, + "skip_e2e": true }, { "comment": "left join with a dual table on left - merge-able", @@ -2500,17 +2559,20 @@ { "comment": "Union after into outfile is incorrect", "query": "select id from user into outfile 'out_file_name' union all select id from music", - "plan": "syntax error at position 55 near 'union'" + "plan": "syntax error at position 55 near 'union'", + "skip_e2e": true }, { "comment": "Into outfile s3 in derived table is incorrect", "query": "select id from (select id from user into outfile s3 'inner_outfile') as t2", - "plan": "syntax error at position 41 near 'into'" + "plan": "syntax error at position 41 near 'into'", + "skip_e2e": true }, { "comment": "Into outfile s3 in derived table with union incorrect", "query": "select id from (select id from user into outfile s3 'inner_outfile' union select 1) as t2", - "plan": "syntax error at position 41 near 'into'" + "plan": "syntax error at position 41 near 'into'", + "skip_e2e": true }, { "comment": "select (select u.id from user as u where u.id = 1), a.id from user as a where a.id = 1", @@ -2579,7 +2641,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "((((select 1))))", @@ -2624,7 +2687,8 @@ "main.dual", "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "subquery in select expression of derived table", @@ -2694,7 +2758,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "ORDER BY subquery", @@ -2764,7 +2829,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "plan test for a natural character set string", @@ -2831,7 +2897,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "Straight Join ensures specific ordering of joins", @@ -2876,7 +2943,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "Dual query should be handled on the vtgate even with a LIMIT", @@ -2950,7 +3018,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "Straight Join preserved in MySQL query", @@ -2973,7 +3042,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "correlated subquery in exists clause", @@ -3031,7 +3101,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "correlated subquery in exists clause with an order by", @@ -3090,7 +3161,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "correlated subquery having dependencies on two tables", @@ -3163,7 +3235,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "correlated subquery using a column twice", @@ -3220,7 +3293,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "correlated subquery that is dependent on one side of a join, fully mergeable", @@ -3271,7 +3345,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "union as a derived table", @@ -3310,7 +3385,8 @@ "main.unsharded", "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "use output column containing data from both sides of the join", @@ -3360,7 +3436,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "mergeable derived table with order by and limit", @@ -3382,7 +3459,8 @@ "TablesUsed": [ "main.unsharded" ] - } + }, + "skip_e2e": true }, { "comment": "mergeable derived table with group by and limit", @@ -3404,7 +3482,8 @@ "TablesUsed": [ "main.unsharded" ] - } + }, + "skip_e2e": true }, { "comment": "select user.id, trim(leading 'x' from user.name) from user", @@ -3426,7 +3505,8 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "json utility functions", @@ -3448,7 +3528,8 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "dual query with exists clause", @@ -3546,7 +3627,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "yeah, it does not make sense, but it's valid", @@ -3639,7 +3721,8 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "groupe by with non aggregated columns and table alias", @@ -3661,7 +3744,8 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "Functions that return JSON value attributes", @@ -3866,7 +3950,8 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "gtid functions", @@ -3934,7 +4019,8 @@ "user.user_extra", "user.user_metadata" ] - } + }, + "skip_e2e": true }, { "comment": "Join across multiple tables, with conditions on different vindexes, but mergeable through join predicates", @@ -3962,7 +4048,8 @@ "user.music_extra", "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "SQL_CALC_FOUND_ROWS with vindex lookup", @@ -4073,7 +4160,8 @@ "TablesUsed": [ "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "`None` route being merged with another route via join predicate on Vindex columns", @@ -4200,7 +4288,8 @@ "TablesUsed": [ "user.music" ] - } + }, + "skip_e2e": true }, { "comment": "Subquery with `IN` condition using columns with matching lookup vindexes", @@ -4314,7 +4403,8 @@ "TablesUsed": [ "user.music" ] - } + }, + "skip_e2e": true }, { "comment": "Mergeable scatter subquery with `GROUP BY` on unique vindex column", @@ -4336,7 +4426,8 @@ "TablesUsed": [ "user.music" ] - } + }, + "skip_e2e": true }, { "comment": "Unmergeable scatter subquery with `GROUP BY` on-non vindex column", @@ -4394,7 +4485,8 @@ "TablesUsed": [ "user.music" ] - } + }, + "skip_e2e": true }, { "comment": "Unmergeable scatter subquery with LIMIT", @@ -4449,7 +4541,8 @@ "TablesUsed": [ "user.music" ] - } + }, + "skip_e2e": true }, { "comment": "Mergeable subquery with `MAX` aggregate and grouped by unique vindex", @@ -4502,7 +4595,8 @@ "TablesUsed": [ "user.music" ] - } + }, + "skip_e2e": true }, { "comment": "Unmergeable subquery with `MAX` aggregate", @@ -4562,7 +4656,8 @@ "TablesUsed": [ "user.music" ] - } + }, + "skip_e2e": true }, { "comment": "Mergeable subquery with `MAX` aggregate with `EqualUnique` route operator", @@ -4615,7 +4710,8 @@ "TablesUsed": [ "user.music" ] - } + }, + "skip_e2e": true }, { "comment": "Mergeable subquery with `LIMIT` due to `EqualUnique` route", @@ -4668,7 +4764,8 @@ "TablesUsed": [ "user.music" ] - } + }, + "skip_e2e": true }, { "comment": "Mergeable subquery with multiple levels of derived statements", @@ -4779,7 +4876,8 @@ "TablesUsed": [ "user.music" ] - } + }, + "skip_e2e": true }, { "comment": "Unmergeable subquery with multiple levels of derived statements", @@ -4834,7 +4932,8 @@ "TablesUsed": [ "user.music" ] - } + }, + "skip_e2e": true }, { "comment": "`None` subquery as top level predicate - outer query changes from `Scatter` to `None` on merge", @@ -5033,7 +5132,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "select user.a, t.b from user join (select id, count(*) b, req from user_extra group by req, id) as t on user.id = t.id", @@ -5097,7 +5197,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "SELECT music.id FROM (SELECT MAX(id) as maxt FROM music WHERE music.user_id = 5) other JOIN music ON other.maxt = music.id", @@ -5220,7 +5321,8 @@ "main.dual", "main.unsharded_a" ] - } + }, + "skip_e2e": true }, { "comment": "subquery having join table on clause, using column reference of outer select table", @@ -5269,7 +5371,8 @@ "main.unsharded", "main.unsharded_a" ] - } + }, + "skip_e2e": true }, { "comment": "ALL modifier on unsharded table works well", @@ -5292,7 +5395,8 @@ "main.unsharded", "main.unsharded_a" ] - } + }, + "skip_e2e": true }, { "comment": "allow last_insert_id with argument", @@ -5337,7 +5441,8 @@ "user.music_extra", "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "Query with non-plannable lookup vindex", @@ -5363,7 +5468,8 @@ "TablesUsed": [ "user.user_metadata" ] - } + }, + "skip_e2e": true }, { "comment": "join query with lookup and join on different vindex column", @@ -5415,7 +5521,8 @@ "user.user", "user.user_metadata" ] - } + }, + "skip_e2e": true }, { "comment": "pick email as vindex lookup", @@ -5510,7 +5617,8 @@ "TablesUsed": [ "user.customer" ] - } + }, + "skip_e2e": true }, { "comment": "email vindex is costly than phone vindex - but phone vindex is backfiling hence ignored", @@ -5634,7 +5742,8 @@ "TablesUsed": [ "user.samecolvin" ] - } + }, + "skip_e2e": true }, { "comment": "column with qualifier is correctly used", @@ -5677,7 +5786,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "Derived tables going to a single shard still need to expand derived table columns", @@ -5722,7 +5832,8 @@ "main.unsharded", "user.user" ] - } + }, + "skip_e2e": true }, { "comment": "column name aliases in outer join queries", @@ -5777,7 +5888,8 @@ "user.user", "user.user_extra" ] - } + }, + "skip_e2e": true }, { "comment": "Over clause works for unsharded tables", @@ -5799,7 +5911,8 @@ "TablesUsed": [ "main.unsharded_a" ] - } + }, + "skip_e2e": true }, { "comment": "join with derived table with alias and join condition - merge into route", From 7864e3adb0e54d62a7bd5aabb3e697d25faf8e60 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Fri, 29 Nov 2024 17:35:52 +0530 Subject: [PATCH 03/12] add plan equality with tracing Signed-off-by: Harshit Gangal --- go/test/endtoend/utils/utils.go | 14 ++ .../vtgate/queries/plan_tests/main_test.go | 191 ++++++++++++++++++ .../vtgate/queries/plan_tests/plan_test.go | 175 +--------------- go/vt/vtgate/engine/plan_description.go | 126 ++++++++++++ 4 files changed, 338 insertions(+), 168 deletions(-) create mode 100644 go/test/endtoend/vtgate/queries/plan_tests/main_test.go diff --git a/go/test/endtoend/utils/utils.go b/go/test/endtoend/utils/utils.go index 35404981164..baa82821306 100644 --- a/go/test/endtoend/utils/utils.go +++ b/go/test/endtoend/utils/utils.go @@ -32,6 +32,7 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/vt/vtgate/engine" ) // AssertContains ensures the given query result contains the expected results. @@ -160,6 +161,19 @@ func Exec(t testing.TB, conn *mysql.Conn, query string) *sqltypes.Result { return qr } +// ExecTrace executes the given query with trace using the given connection. The trace result is returned. +// The test fails if the query produces an error. +func ExecTrace(t testing.TB, conn *mysql.Conn, query string) engine.PrimitiveDescription { + t.Helper() + qr, err := conn.ExecuteFetch(fmt.Sprintf("vexplain trace %s", query), 10000, false) + require.NoError(t, err, "for query: "+query) + + // Extract the trace result and format it with indentation for pretty printing + pd, err := engine.PrimitiveDescriptionFromString(qr.Rows[0][0].ToString()) + require.NoError(t, err) + return pd +} + // ExecMulti executes the given (potential multi) queries using the given connection. // The test fails if any of the queries produces an error func ExecMulti(t testing.TB, conn *mysql.Conn, query string) error { diff --git a/go/test/endtoend/vtgate/queries/plan_tests/main_test.go b/go/test/endtoend/vtgate/queries/plan_tests/main_test.go new file mode 100644 index 00000000000..e8018ed2582 --- /dev/null +++ b/go/test/endtoend/vtgate/queries/plan_tests/main_test.go @@ -0,0 +1,191 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan_tests + +import ( + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" + "vitess.io/vitess/go/vt/vtgate/engine" + "vitess.io/vitess/go/vt/vtgate/planbuilder" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + mysqlParams mysql.ConnParams + keyspaceName = "user" + cell = "test_aggr" +) + +func TestMain(m *testing.M) { + defer cluster.PanicHandler(nil) + + schemaSQL := readFile("vschemas/schema.sql") + vschema := extractUserKS(readFile("vschemas/schema.json")) + + exitCode := func() int { + clusterInstance = cluster.NewCluster(cell, "localhost") + defer clusterInstance.Teardown() + + // Start topo server + err := clusterInstance.StartTopo() + if err != nil { + fmt.Println(err.Error()) + return 1 + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + SchemaSQL: schemaSQL, + VSchema: vschema, + } + err = clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 0, false) + if err != nil { + fmt.Println(err.Error()) + return 1 + } + + // TODO: (@GuptaManan100/@systay): Also run the tests with normalizer on. + clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs, + "--normalize_queries=false", + "--schema_change_signal=false", + ) + + // Start vtgate + err = clusterInstance.StartVtgate() + if err != nil { + fmt.Println(err.Error()) + return 1 + } + + vtParams = clusterInstance.GetVTParams(keyspaceName) + + // create mysql instance and connection parameters + conn, closer, err := utils.NewMySQL(clusterInstance, keyspaceName, schemaSQL) + if err != nil { + fmt.Println(err.Error()) + return 1 + } + defer closer() + mysqlParams = conn + + return m.Run() + }() + os.Exit(exitCode) +} + +func readFile(filename string) string { + schema, err := os.ReadFile(locateFile(filename)) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + return string(schema) +} + +func start(t *testing.T) (utils.MySQLCompare, func()) { + mcmp, err := utils.NewMySQLCompare(t, vtParams, mysqlParams) + require.NoError(t, err) + return mcmp, func() { + mcmp.Close() + } +} + +func readJSONTests(filename string) []planbuilder.PlanTest { + var output []planbuilder.PlanTest + file, err := os.Open(locateFile(filename)) + if err != nil { + panic(err) + } + defer file.Close() + dec := json.NewDecoder(file) + err = dec.Decode(&output) + if err != nil { + panic(err) + } + return output +} + +func locateFile(name string) string { + return "../../../../../vt/vtgate/planbuilder/testdata/" + name +} + +// verifyTestExpectations verifies the expectations of the test. +func verifyTestExpectations(t *testing.T, pd engine.PrimitiveDescription, test planbuilder.PlanTest) { + // 1. Verify that the Join primitive sees atleast 1 row on the left side. + engine.WalkPrimitiveDescription(pd, func(description engine.PrimitiveDescription) { + if description.OperatorType == "Join" { + require.NotZero(t, description.Inputs[0].RowsReceived[0]) + } + }) + + // 2. Verify that the plan description matches the expected plan description. + planBytes, err := test.Plan.MarshalJSON() + require.NoError(t, err) + mp := make(map[string]any) + err = json.Unmarshal(planBytes, &mp) + require.NoError(t, err) + pdExpected, err := engine.PrimitiveDescriptionFromMap(mp["Instructions"].(map[string]any)) + require.NoError(t, err) + require.Empty(t, pdExpected.Equals(pd), "Expected: %v\nGot: %v", string(planBytes), pd) +} + +func extractUserKS(jsonString string) string { + var result map[string]any + if err := json.Unmarshal([]byte(jsonString), &result); err != nil { + panic(err.Error()) + } + + keyspaces, ok := result["keyspaces"].(map[string]any) + if !ok { + panic("Keyspaces not found") + } + + user, ok := keyspaces["user"].(map[string]any) + if !ok { + panic("User keyspaces not found") + } + + tables, ok := user["tables"].(map[string]any) + if !ok { + panic("Tables not found") + } + + userTbl, ok := tables["user"].(map[string]any) + if !ok { + panic("User table not found") + } + + delete(userTbl, "auto_increment") // TODO: we should have an unsharded keyspace where this could live + + // Marshal the inner part back to JSON string + userJson, err := json.Marshal(user) + if err != nil { + panic(err.Error()) + } + + return string(userJson) +} diff --git a/go/test/endtoend/vtgate/queries/plan_tests/plan_test.go b/go/test/endtoend/vtgate/queries/plan_tests/plan_test.go index c82175308b7..6065a2514ee 100644 --- a/go/test/endtoend/vtgate/queries/plan_tests/plan_test.go +++ b/go/test/endtoend/vtgate/queries/plan_tests/plan_test.go @@ -17,184 +17,23 @@ limitations under the License. package plan_tests import ( - _ "embed" - "encoding/json" - "fmt" - "os" "testing" - "github.com/stretchr/testify/require" - - "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/test/endtoend/utils" - "vitess.io/vitess/go/vt/vtgate/planbuilder" ) -func readJSONTests(filename string) []planbuilder.PlanTest { - var output []planbuilder.PlanTest - file, err := os.Open(locateFile(filename)) - if err != nil { - panic(err) - } - defer file.Close() - dec := json.NewDecoder(file) - err = dec.Decode(&output) - if err != nil { - panic(err) - } - return output -} - -func locateFile(name string) string { - return "../../../../../vt/vtgate/planbuilder/testdata/" + name -} - -func TestPlan(t *testing.T) { +func TestSelectCases(t *testing.T) { mcmp, closer := start(t) defer closer() - tests := readJSONTests("select_cases_new.json") + tests := readJSONTests("select_cases.json") for _, test := range tests { - if test.SkipE2E { - continue - } mcmp.Run(test.Query, func(mcmp *utils.MySQLCompare) { + if test.SkipE2E { + mcmp.AsT().Skip(test.Query) + } mcmp.Exec(test.Query) - // if mcmp.AsT().Failed() { - // tests[i].SkipE2E = true - // } + pd := utils.ExecTrace(mcmp.AsT(), mcmp.VtConn, test.Query) + verifyTestExpectations(mcmp.AsT(), pd, test) }) } - - // file, err := os.Create(locateFile("select_cases_new.json")) - // if err != nil { - // panic(err) - // } - // defer file.Close() - // - // enc := json.NewEncoder(file) - // enc.SetEscapeHTML(false) - // enc.SetIndent("", " ") - // err = enc.Encode(tests) - // require.NoError(t, err) -} - -var ( - clusterInstance *cluster.LocalProcessCluster - vtParams mysql.ConnParams - mysqlParams mysql.ConnParams - keyspaceName = "user" - cell = "test_aggr" -) - -func readFile(filename string) string { - schema, err := os.ReadFile(locateFile(filename)) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - return string(schema) -} - -func start(t *testing.T) (utils.MySQLCompare, func()) { - mcmp, err := utils.NewMySQLCompare(t, vtParams, mysqlParams) - require.NoError(t, err) - return mcmp, func() { - mcmp.Close() - } -} - -func extractUserKS(jsonString string) string { - var result map[string]any - if err := json.Unmarshal([]byte(jsonString), &result); err != nil { - panic(err.Error()) - } - - keyspaces, ok := result["keyspaces"].(map[string]any) - if !ok { - panic("Keyspaces not found") - } - - user, ok := keyspaces["user"].(map[string]any) - if !ok { - panic("User keyspaces not found") - } - - tables, ok := user["tables"].(map[string]any) - if !ok { - panic("Tables not found") - } - - userTbl, ok := tables["user"].(map[string]any) - if !ok { - panic("User table not found") - } - - delete(userTbl, "auto_increment") // TODO: we should have an unsharded keyspace where this could live - - // Marshal the inner part back to JSON string - userJson, err := json.Marshal(user) - if err != nil { - panic(err.Error()) - } - - return string(userJson) -} - -func TestMain(m *testing.M) { - defer cluster.PanicHandler(nil) - - schemaSQL := readFile("vschemas/schema.sql") - vschema := extractUserKS(readFile("vschemas/schema.json")) - - exitCode := func() int { - clusterInstance = cluster.NewCluster(cell, "localhost") - defer clusterInstance.Teardown() - - // Start topo server - err := clusterInstance.StartTopo() - if err != nil { - fmt.Println(err.Error()) - return 1 - } - - // Start keyspace - keyspace := &cluster.Keyspace{ - Name: keyspaceName, - SchemaSQL: schemaSQL, - VSchema: vschema, - } - err = clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 0, false) - if err != nil { - fmt.Println(err.Error()) - return 1 - } - - // TODO: (@GuptaManan100/@systay): Also run the tests with normalizer on. - clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs, - "--normalize_queries=false", - "--schema_change_signal=false", - ) - - // Start vtgate - err = clusterInstance.StartVtgate() - if err != nil { - fmt.Println(err.Error()) - return 1 - } - - vtParams = clusterInstance.GetVTParams(keyspaceName) - - // create mysql instance and connection parameters - conn, closer, err := utils.NewMySQL(clusterInstance, keyspaceName, schemaSQL) - if err != nil { - fmt.Println(err.Error()) - return 1 - } - defer closer() - mysqlParams = conn - - return m.Run() - }() - os.Exit(exitCode) } diff --git a/go/vt/vtgate/engine/plan_description.go b/go/vt/vtgate/engine/plan_description.go index dfcad4e5e6b..9f14d03803f 100644 --- a/go/vt/vtgate/engine/plan_description.go +++ b/go/vt/vtgate/engine/plan_description.go @@ -126,6 +126,132 @@ func (pd PrimitiveDescription) MarshalJSON() ([]byte, error) { return buf.Bytes(), nil } +// PrimitiveDescriptionFromString creates primitive description out of a data string. +func PrimitiveDescriptionFromString(data string) (pd PrimitiveDescription, err error) { + resultMap := make(map[string]any) + err = json.Unmarshal([]byte(data), &resultMap) + if err != nil { + return PrimitiveDescription{}, err + } + return PrimitiveDescriptionFromMap(resultMap) +} + +// PrimitiveDescriptionFromMap populates the fields of a PrimitiveDescription from a map representation. +func PrimitiveDescriptionFromMap(data map[string]any) (pd PrimitiveDescription, err error) { + if opType, isPresent := data["OperatorType"]; isPresent { + pd.OperatorType = opType.(string) + } + if variant, isPresent := data["Variant"]; isPresent { + pd.Variant = variant.(string) + } + if ksMap, isPresent := data["Keyspace"]; isPresent { + ksMap := ksMap.(map[string]any) + pd.Keyspace = &vindexes.Keyspace{ + Name: ksMap["Name"].(string), + Sharded: ksMap["Sharded"].(bool), + } + } + if ttt, isPresent := data["TargetTabletType"]; isPresent { + pd.TargetTabletType = topodatapb.TabletType(ttt.(int)) + } + if other, isPresent := data["Other"]; isPresent { + pd.Other = other.(map[string]any) + } + if inpName, isPresent := data["InputName"]; isPresent { + pd.InputName = inpName.(string) + } + if avgRows, isPresent := data["AvgNumberOfRows"]; isPresent { + pd.RowsReceived = RowsReceived{ + int(avgRows.(float64)), + } + } + if sq, isPresent := data["ShardsQueried"]; isPresent { + sq := int(sq.(float64)) + pd.ShardsQueried = (*ShardsQueried)(&sq) + } + if inputs, isPresent := data["Inputs"]; isPresent { + inputs := inputs.([]any) + for _, input := range inputs { + inputMap := input.(map[string]any) + inp, err := PrimitiveDescriptionFromMap(inputMap) + if err != nil { + return PrimitiveDescription{}, err + } + pd.Inputs = append(pd.Inputs, inp) + } + } + return pd, nil +} + +// WalkPrimitiveDescription walks the primitive description. +func WalkPrimitiveDescription(pd PrimitiveDescription, f func(PrimitiveDescription)) { + f(pd) + for _, child := range pd.Inputs { + WalkPrimitiveDescription(child, f) + } +} + +func (pd PrimitiveDescription) Equals(other PrimitiveDescription) string { + if pd.Variant != other.Variant { + return fmt.Sprintf("Variant: %v != %v", pd.Variant, other.Variant) + } + + if pd.OperatorType != other.OperatorType { + return fmt.Sprintf("OperatorType: %v != %v", pd.OperatorType, other.OperatorType) + } + + switch { + case pd.Keyspace == nil && other.Keyspace == nil: + // do nothing + case pd.Keyspace != nil && other.Keyspace != nil: + if pd.Keyspace.Name != other.Keyspace.Name { + return fmt.Sprintf("Keyspace.Name: %v != %v", pd.Keyspace.Name, other.Keyspace.Name) + } + default: + return "Keyspace is nil in one of the descriptions" + } + + switch { + case pd.TargetDestination == nil && other.TargetDestination == nil: + // do nothing + case pd.TargetDestination != nil && other.TargetDestination != nil: + if pd.TargetDestination.String() != other.TargetDestination.String() { + return fmt.Sprintf("TargetDestination: %v != %v", pd.TargetDestination, other.TargetDestination) + } + default: + return "TargetDestination is nil in one of the descriptions" + } + + if pd.TargetTabletType != other.TargetTabletType { + return fmt.Sprintf("TargetTabletType: %v != %v", pd.TargetTabletType, other.TargetTabletType) + } + + switch { + case pd.Other == nil && other.Other == nil: + // do nothing + case pd.Other != nil && other.Other != nil: + if len(pd.Other) != len(other.Other) { + return fmt.Sprintf("Other length did not match: %v != %v", pd.Other, other.Other) + } + for ky, val := range pd.Other { + if other.Other[ky] != val { + return fmt.Sprintf("Other[%v]: %v != %v", ky, val, other.Other[ky]) + } + } + default: + return "Other is nil in one of the descriptions" + } + if len(pd.Inputs) != len(other.Inputs) { + return fmt.Sprintf("Inputs length did not match: %v != %v", len(pd.Inputs), len(other.Inputs)) + } + for idx, input := range pd.Inputs { + if diff := input.Equals(other.Inputs[idx]); diff != "" { + return diff + } + } + return "" +} + func average(nums []int) float64 { total := 0 for _, num := range nums { From a51815740b210dd1bc6749f71e286e15119dd147 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 2 Dec 2024 10:42:28 +0530 Subject: [PATCH 04/12] added global test Signed-off-by: Harshit Gangal --- .../{vschemas/schema.sql => schemas/sschema.sql} | 0 .../planbuilder/testdata/schemas/uschema.sql | 0 go/vt/vtgate/vindexes/vschema_test.go | 15 +++++++++++++++ 3 files changed, 15 insertions(+) rename go/vt/vtgate/planbuilder/testdata/{vschemas/schema.sql => schemas/sschema.sql} (100%) create mode 100644 go/vt/vtgate/planbuilder/testdata/schemas/uschema.sql diff --git a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.sql b/go/vt/vtgate/planbuilder/testdata/schemas/sschema.sql similarity index 100% rename from go/vt/vtgate/planbuilder/testdata/vschemas/schema.sql rename to go/vt/vtgate/planbuilder/testdata/schemas/sschema.sql diff --git a/go/vt/vtgate/planbuilder/testdata/schemas/uschema.sql b/go/vt/vtgate/planbuilder/testdata/schemas/uschema.sql new file mode 100644 index 00000000000..e69de29bb2d diff --git a/go/vt/vtgate/vindexes/vschema_test.go b/go/vt/vtgate/vindexes/vschema_test.go index 25f8e135698..f9bcf43ddaa 100644 --- a/go/vt/vtgate/vindexes/vschema_test.go +++ b/go/vt/vtgate/vindexes/vschema_test.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "os" "reflect" "strings" "testing" @@ -3551,6 +3552,20 @@ func TestFindTableWithSequences(t *testing.T) { } } +func TestGlobalTables(t *testing.T) { + input, err := os.ReadFile("../planbuilder/testdata/vschemas/schema.json") + require.NoError(t, err) + + var vs vschemapb.SrvVSchema + err = json2.UnmarshalPB(input, &vs) + require.NoError(t, err) + + got := BuildVSchema(&vs, sqlparser.NewTestParser()) + tbl, err := got.findGlobalTable("user", false) + require.NoError(t, err) + assert.NotNil(t, tbl) +} + func vindexNames(vindexes []*ColumnVindex) (result []string) { for _, vindex := range vindexes { result = append(result, vindex.Name) From 6be7923c443d4194aa0d64b7ff4a05776967e63b Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 2 Dec 2024 11:54:29 +0530 Subject: [PATCH 05/12] added unsharded keyspace to the test Signed-off-by: Harshit Gangal --- .../vtgate/queries/plan_tests/main_test.go | 67 +++++++++++++++---- .../planbuilder/testdata/schemas/uschema.sql | 12 ++++ 2 files changed, 66 insertions(+), 13 deletions(-) diff --git a/go/test/endtoend/vtgate/queries/plan_tests/main_test.go b/go/test/endtoend/vtgate/queries/plan_tests/main_test.go index e8018ed2582..4ceefb774f5 100644 --- a/go/test/endtoend/vtgate/queries/plan_tests/main_test.go +++ b/go/test/endtoend/vtgate/queries/plan_tests/main_test.go @@ -35,15 +35,19 @@ var ( clusterInstance *cluster.LocalProcessCluster vtParams mysql.ConnParams mysqlParams mysql.ConnParams - keyspaceName = "user" - cell = "test_aggr" + uks = "main" + sks = "user" + cell = "plantests" ) func TestMain(m *testing.M) { defer cluster.PanicHandler(nil) - schemaSQL := readFile("vschemas/schema.sql") - vschema := extractUserKS(readFile("vschemas/schema.json")) + vschema := readFile("vschemas/schema.json") + userVs := extractUserKS(vschema) + mainVs := extractMainKS(vschema) + sSQL := readFile("schemas/sschema.sql") + uSQL := readFile("schemas/uschema.sql") exitCode := func() int { clusterInstance = cluster.NewCluster(cell, "localhost") @@ -56,13 +60,25 @@ func TestMain(m *testing.M) { return 1 } - // Start keyspace - keyspace := &cluster.Keyspace{ - Name: keyspaceName, - SchemaSQL: schemaSQL, - VSchema: vschema, + // Start unsharded keyspace + uKeyspace := &cluster.Keyspace{ + Name: uks, + SchemaSQL: uSQL, + VSchema: mainVs, } - err = clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 0, false) + err = clusterInstance.StartUnshardedKeyspace(*uKeyspace, 0, false) + if err != nil { + fmt.Println(err.Error()) + return 1 + } + + // Start sharded keyspace + skeyspace := &cluster.Keyspace{ + Name: sks, + SchemaSQL: sSQL, + VSchema: userVs, + } + err = clusterInstance.StartKeyspace(*skeyspace, []string{"-80", "80-"}, 0, false) if err != nil { fmt.Println(err.Error()) return 1 @@ -81,10 +97,10 @@ func TestMain(m *testing.M) { return 1 } - vtParams = clusterInstance.GetVTParams(keyspaceName) + vtParams = clusterInstance.GetVTParams(sks) // create mysql instance and connection parameters - conn, closer, err := utils.NewMySQL(clusterInstance, keyspaceName, schemaSQL) + conn, closer, err := utils.NewMySQL(clusterInstance, sks, sSQL, uSQL) if err != nil { fmt.Println(err.Error()) return 1 @@ -166,7 +182,7 @@ func extractUserKS(jsonString string) string { user, ok := keyspaces["user"].(map[string]any) if !ok { - panic("User keyspaces not found") + panic("User keyspace not found") } tables, ok := user["tables"].(map[string]any) @@ -189,3 +205,28 @@ func extractUserKS(jsonString string) string { return string(userJson) } + +func extractMainKS(jsonString string) string { + var result map[string]any + if err := json.Unmarshal([]byte(jsonString), &result); err != nil { + panic(err.Error()) + } + + keyspaces, ok := result["keyspaces"].(map[string]any) + if !ok { + panic("Keyspaces not found") + } + + main, ok := keyspaces["main"].(map[string]any) + if !ok { + panic("main keyspace not found") + } + + // Marshal the inner part back to JSON string + mainJson, err := json.Marshal(main) + if err != nil { + panic(err.Error()) + } + + return string(mainJson) +} diff --git a/go/vt/vtgate/planbuilder/testdata/schemas/uschema.sql b/go/vt/vtgate/planbuilder/testdata/schemas/uschema.sql index e69de29bb2d..8c15b99218c 100644 --- a/go/vt/vtgate/planbuilder/testdata/schemas/uschema.sql +++ b/go/vt/vtgate/planbuilder/testdata/schemas/uschema.sql @@ -0,0 +1,12 @@ +CREATE TABLE `unsharded` ( + `id` INT NOT NULL PRIMARY KEY, + `col1` VARCHAR(255) DEFAULT NULL, + `col2` VARCHAR(255) DEFAULT NULL, + `name` VARCHAR(255) DEFAULT NULL +); + +CREATE TABLE `unsharded_auto` ( + `id` INT NOT NULL PRIMARY KEY, + `col1` VARCHAR(255) DEFAULT NULL, + `col2` VARCHAR(255) DEFAULT NULL +); \ No newline at end of file From bc938cdb4c32c4208f93c982066b86efd3055804 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Fri, 6 Dec 2024 23:47:14 +0530 Subject: [PATCH 06/12] plan test to use valid vindexes Signed-off-by: Harshit Gangal --- .../planbuilder/testdata/aggr_cases.json | 70 +- .../planbuilder/testdata/filter_cases.json | 140 +++- .../planbuilder/testdata/from_cases.json | 35 +- .../testdata/memory_sort_cases.json | 70 +- .../testdata/postprocess_cases.json | 140 +++- .../planbuilder/testdata/select_cases.json | 625 ++++++++++-------- .../planbuilder/testdata/union_cases.json | 70 +- .../planbuilder/testdata/vschemas/schema.json | 77 ++- go/vt/vtgate/vindexes/lookup_cost.go | 70 ++ 9 files changed, 941 insertions(+), 356 deletions(-) create mode 100644 go/vt/vtgate/vindexes/lookup_cost.go diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index 8b268e367dd..49a03a8f05a 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -940,19 +940,44 @@ "Table": "`user`, user_extra" }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.`name` from music where 1 != 1", - "Query": "select music.`name` from music where music.id = :user_id", - "Table": "music", "Values": [ ":user_id" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.`name` from music where 1 != 1", + "Query": "select music.`name` from music where music.id = :user_id", + "Table": "music" + } + ] } ] } @@ -2992,19 +3017,44 @@ ] }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from music as m where 1 != 1", - "Query": "select 1 from music as m where m.id = :u2_val2", - "Table": "music", "Values": [ ":u2_val2" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from music as m where 1 != 1", + "Query": "select 1 from music as m where m.id = :u2_val2", + "Table": "music" + } + ] } ] } diff --git a/go/vt/vtgate/planbuilder/testdata/filter_cases.json b/go/vt/vtgate/planbuilder/testdata/filter_cases.json index 4194a369bd6..edce4ebd0cb 100644 --- a/go/vt/vtgate/planbuilder/testdata/filter_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/filter_cases.json @@ -3404,19 +3404,44 @@ "QueryType": "SELECT", "Original": "select * from multicolvin where column_b = 1", "Instructions": { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select * from multicolvin where 1 != 1", - "Query": "select * from multicolvin where column_b = 1", - "Table": "multicolvin", "Values": [ "1" ], - "Vindex": "colb_colc_map" + "Vindex": "colb_colc_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select colb, keyspace_id from colb_colc_map where 1 != 1", + "Query": "select colb, keyspace_id from colb_colc_map where colb in ::__vals", + "Table": "colb_colc_map", + "Values": [ + "::colb" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select * from multicolvin where 1 != 1", + "Query": "select * from multicolvin where column_b = 1", + "Table": "multicolvin" + } + ] }, "TablesUsed": [ "user.multicolvin" @@ -3430,19 +3455,44 @@ "QueryType": "SELECT", "Original": "select * from multicolvin where column_b = 1 and column_c = 2", "Instructions": { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select * from multicolvin where 1 != 1", - "Query": "select * from multicolvin where column_b = 1 and column_c = 2", - "Table": "multicolvin", "Values": [ "1" ], - "Vindex": "colb_colc_map" + "Vindex": "colb_colc_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select colb, keyspace_id from colb_colc_map where 1 != 1", + "Query": "select colb, keyspace_id from colb_colc_map where colb in ::__vals", + "Table": "colb_colc_map", + "Values": [ + "::colb" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select * from multicolvin where 1 != 1", + "Query": "select * from multicolvin where column_b = 1 and column_c = 2", + "Table": "multicolvin" + } + ] }, "TablesUsed": [ "user.multicolvin" @@ -3456,19 +3506,44 @@ "QueryType": "SELECT", "Original": "select * from multicolvin where column_b = 1 and column_c = 2 and column_a = 3", "Instructions": { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select * from multicolvin where 1 != 1", - "Query": "select * from multicolvin where column_b = 1 and column_c = 2 and column_a = 3", - "Table": "multicolvin", "Values": [ "1" ], - "Vindex": "colb_colc_map" + "Vindex": "colb_colc_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select colb, keyspace_id from colb_colc_map where 1 != 1", + "Query": "select colb, keyspace_id from colb_colc_map where colb in ::__vals", + "Table": "colb_colc_map", + "Values": [ + "::colb" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select * from multicolvin where 1 != 1", + "Query": "select * from multicolvin where column_b = 1 and column_c = 2 and column_a = 3", + "Table": "multicolvin" + } + ] }, "TablesUsed": [ "user.multicolvin" @@ -3482,19 +3557,44 @@ "QueryType": "SELECT", "Original": "select * from multicolvin where column_a = 3 and column_b = 1", "Instructions": { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select * from multicolvin where 1 != 1", - "Query": "select * from multicolvin where column_a = 3 and column_b = 1", - "Table": "multicolvin", "Values": [ "1" ], - "Vindex": "colb_colc_map" + "Vindex": "colb_colc_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select colb, keyspace_id from colb_colc_map where 1 != 1", + "Query": "select colb, keyspace_id from colb_colc_map where colb in ::__vals", + "Table": "colb_colc_map", + "Values": [ + "::colb" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select * from multicolvin where 1 != 1", + "Query": "select * from multicolvin where column_a = 3 and column_b = 1", + "Table": "multicolvin" + } + ] }, "TablesUsed": [ "user.multicolvin" diff --git a/go/vt/vtgate/planbuilder/testdata/from_cases.json b/go/vt/vtgate/planbuilder/testdata/from_cases.json index 2e0fe429c1f..bec64fd7b1e 100644 --- a/go/vt/vtgate/planbuilder/testdata/from_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/from_cases.json @@ -4709,19 +4709,44 @@ ] }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from music as m where 1 != 1", - "Query": "select 1 from music as m where m.user_id = 5 and m.id = 20 and m.col = :u_col /* INT16 */", - "Table": "music", "Values": [ "20" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from music as m where 1 != 1", + "Query": "select 1 from music as m where m.user_id = 5 and m.id = 20 and m.col = :u_col /* INT16 */", + "Table": "music" + } + ] } ] }, diff --git a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json index 060f073a366..a35949cd4c1 100644 --- a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json @@ -318,19 +318,44 @@ "Vindex": "user_index" }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3 as c, weight_string(music.col3) from music where 1 != 1", - "Query": "select music.col3 as c, weight_string(music.col3) from music where music.id = :user_id", - "Table": "music", "Values": [ ":user_id" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.col3 as c, weight_string(music.col3) from music where 1 != 1", + "Query": "select music.col3 as c, weight_string(music.col3) from music where music.id = :user_id", + "Table": "music" + } + ] } ] } @@ -379,19 +404,44 @@ "Vindex": "user_index" }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3, weight_string(music.col3) from music where 1 != 1", - "Query": "select music.col3, weight_string(music.col3) from music where music.id = :user_id", - "Table": "music", "Values": [ ":user_id" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.col3, weight_string(music.col3) from music where 1 != 1", + "Query": "select music.col3, weight_string(music.col3) from music where music.id = :user_id", + "Table": "music" + } + ] } ] } diff --git a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json index 36f1472007d..6a8e94c0241 100644 --- a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json @@ -544,19 +544,44 @@ "Vindex": "user_index" }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3 from music where 1 != 1", - "Query": "select music.col3 from music where music.id = :user_id", - "Table": "music", "Values": [ ":user_id" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.col3 from music where 1 != 1", + "Query": "select music.col3 from music where music.id = :user_id", + "Table": "music" + } + ] } ] }, @@ -597,19 +622,44 @@ "Vindex": "user_index" }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3 from music where 1 != 1", - "Query": "select music.col3 from music where music.id = :user_id", - "Table": "music", "Values": [ ":user_id" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.col3 from music where 1 != 1", + "Query": "select music.col3 from music where music.id = :user_id", + "Table": "music" + } + ] } ] }, @@ -650,19 +700,44 @@ "Vindex": "user_index" }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3 from music where 1 != 1", - "Query": "select music.col3 from music where music.id = :user_id", - "Table": "music", "Values": [ ":user_id" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.col3 from music where 1 != 1", + "Query": "select music.col3 from music where music.id = :user_id", + "Table": "music" + } + ] } ] }, @@ -770,19 +845,44 @@ "Vindex": "user_index" }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3 from music where 1 != 1", - "Query": "select music.col3 from music where music.id = :user_id", - "Table": "music", "Values": [ ":user_id" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.col3 from music where 1 != 1", + "Query": "select music.col3 from music where music.id = :user_id", + "Table": "music" + } + ] } ] }, diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.json b/go/vt/vtgate/planbuilder/testdata/select_cases.json index 68cd5a5bdfc..ae37d4b10e6 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.json @@ -92,8 +92,7 @@ "user.user", "user.user_metadata" ] - }, - "skip_e2e": true + } }, { "comment": "select with timeout directive sets QueryTimeout in the route", @@ -198,8 +197,7 @@ "TablesUsed": [ "main.unsharded" ] - }, - "skip_e2e": true + } }, { "comment": "select with partial scatter directive", @@ -377,8 +375,7 @@ "TablesUsed": [ "user.authoritative" ] - }, - "skip_e2e": true + } }, { "comment": "select * from join of authoritative tables", @@ -400,14 +397,12 @@ "TablesUsed": [ "user.authoritative" ] - }, - "skip_e2e": true + } }, { "comment": "test table lookup failure for authoritative code path", "query": "select a.* from authoritative", - "plan": "Unknown table 'a'", - "skip_e2e": true + "plan": "Unknown table 'a'" }, { "comment": "select * from qualified authoritative table", @@ -429,8 +424,7 @@ "TablesUsed": [ "user.authoritative" ] - }, - "skip_e2e": true + } }, { "comment": "select * from intermixing of authoritative table with non-authoritative results in no expansion", @@ -453,8 +447,7 @@ "user.authoritative", "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "select authoritative.* with intermixing still expands", @@ -477,8 +470,7 @@ "user.authoritative", "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "auto-resolve anonymous columns for simple route", @@ -501,8 +493,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "json_arrayagg in single sharded query", @@ -528,8 +519,7 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "json_objectagg in single sharded query", @@ -555,20 +545,17 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "unsupported json aggregation expressions in scatter query", "query": "select count(1) from user where cola = 'abc' group by n_id having json_arrayagg(a_id) = '[]'", - "plan": "VT12001: unsupported: in scatter query: aggregation function 'json_arrayagg(a_id)'", - "skip_e2e": true + "plan": "VT12001: unsupported: in scatter query: aggregation function 'json_arrayagg(a_id)'" }, { "comment": "Cannot auto-resolve for cross-shard joins", "query": "select col from user join user_extra", - "plan": "Column 'col' in field list is ambiguous", - "skip_e2e": true + "plan": "Column 'col' in field list is ambiguous" }, { "comment": "Auto-resolve should work if unique vindex columns are referenced", @@ -610,8 +597,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "database calls should be substituted", @@ -633,8 +619,7 @@ "TablesUsed": [ "main.dual" ] - }, - "skip_e2e": true + } }, { "comment": "last_insert_id for unsharded route", @@ -656,8 +641,7 @@ "TablesUsed": [ "main.unsharded" ] - }, - "skip_e2e": true + } }, { "comment": "select from dual on unqualified keyspace", @@ -710,8 +694,7 @@ { "comment": "prefixing dual with a keyspace should not work", "query": "select 1 from user.dual", - "plan": "table dual not found", - "skip_e2e": true + "plan": "table dual not found" }, { "comment": "RHS route referenced", @@ -753,8 +736,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "Both routes referenced", @@ -796,8 +778,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "Expression with single-route reference", @@ -839,8 +820,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "subquery with an aggregation in order by that can be merged into a single route", @@ -867,8 +847,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "Jumbled references", @@ -910,8 +889,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "Comments", @@ -953,8 +931,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "for update", @@ -996,8 +973,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "Field query should work for joins select bind vars", @@ -1042,8 +1018,7 @@ "main.unsharded", "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "Case preservation", @@ -1085,14 +1060,12 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "syntax error", "query": "the quick brown fox", - "plan": "syntax error at position 4 near 'the'", - "skip_e2e": true + "plan": "syntax error at position 4 near 'the'" }, { "comment": "Hex number is not treated as a simple value", @@ -1140,8 +1113,7 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "Selection but make the planner explicitly use a vindex", @@ -1192,14 +1164,12 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "Vindex hint on a non-existing vindex", "query": "select * from user use vindex (does_not_exist) where id = 1", - "plan": "VT09021: Vindex 'does_not_exist' does not exist in table 'user.user'", - "skip_e2e": true + "plan": "VT09021: Vindex 'does_not_exist' does not exist in table 'user.user'" }, { "comment": "sharded limit offset", @@ -1261,8 +1231,7 @@ "TablesUsed": [ "user.music" ] - }, - "skip_e2e": true + } }, { "comment": "Sharding Key Condition in Parenthesis", @@ -1288,8 +1257,7 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "Multiple parenthesized expressions", @@ -1315,8 +1283,7 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "Multiple parenthesized expressions", @@ -1342,8 +1309,7 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "Column Aliasing with Table.Column", @@ -1421,8 +1387,7 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "Column as boolean-ish", @@ -1448,8 +1413,7 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "PK as fake boolean, and column as boolean-ish", @@ -1475,8 +1439,7 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "top level subquery in select", @@ -1521,8 +1484,7 @@ "main.unsharded", "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "sub-expression subquery in select", @@ -1567,8 +1529,7 @@ "main.unsharded", "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "select * from derived table expands specific columns", @@ -1610,20 +1571,17 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "duplicate columns not allowed in derived table", "query": "select * from (select user.id, user_extra.id from user join user_extra) as t", - "plan": "Duplicate column name 'id'", - "skip_e2e": true + "plan": "Duplicate column name 'id'" }, { "comment": "non-existent symbol in cross-shard derived table", "query": "select t.col from (select user.id from user join user_extra) as t", - "plan": "column 't.col' not found", - "skip_e2e": true + "plan": "column 't.col' not found" }, { "comment": "union with the same target shard", @@ -1650,8 +1608,7 @@ "user.music", "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "union with the same target shard last_insert_id", @@ -1678,8 +1635,7 @@ "user.music", "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "unsharded union in derived table", @@ -1701,8 +1657,7 @@ "TablesUsed": [ "main.unsharded" ] - }, - "skip_e2e": true + } }, { "comment": "unsharded union in subquery", @@ -1724,8 +1679,7 @@ "TablesUsed": [ "main.unsharded" ] - }, - "skip_e2e": true + } }, { "comment": "(select id from unsharded) union (select id from unsharded_auto) order by id limit 5", @@ -1748,8 +1702,7 @@ "main.unsharded", "main.unsharded_auto" ] - }, - "skip_e2e": true + } }, { "comment": "unsharded union", @@ -1772,8 +1725,7 @@ "main.unsharded", "main.unsharded_auto" ] - }, - "skip_e2e": true + } }, { "comment": "unsharded nested union", @@ -1796,8 +1748,7 @@ "main.unsharded", "main.unsharded_auto" ] - }, - "skip_e2e": true + } }, { "comment": "unsharded nested union with limit", @@ -1819,8 +1770,7 @@ "TablesUsed": [ "main.unsharded" ] - }, - "skip_e2e": true + } }, { "comment": "routing rules: ensure directives are not lost", @@ -1843,8 +1793,7 @@ "TablesUsed": [ "main.unsharded" ] - }, - "skip_e2e": true + } }, { "comment": "routing table on music", @@ -1866,8 +1815,7 @@ "TablesUsed": [ "user.music" ] - }, - "skip_e2e": true + } }, { "comment": "testing SingleRow Projection", @@ -2014,8 +1962,7 @@ "main.unsharded_a", "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "Complex expression in a subquery used in NOT IN clause of an aggregate query", @@ -2068,8 +2015,7 @@ "main.unsharded_a", "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "testing SingleRow Projection with arithmetics", @@ -2272,14 +2218,12 @@ { "comment": "sql_calc_found_rows in sub queries", "query": "select * from music where user_id IN (select sql_calc_found_rows * from music limit 10)", - "plan": "Incorrect usage/placement of 'SQL_CALC_FOUND_ROWS'", - "skip_e2e": true + "plan": "Incorrect usage/placement of 'SQL_CALC_FOUND_ROWS'" }, { "comment": "sql_calc_found_rows in derived table", "query": "select sql_calc_found_rows * from (select sql_calc_found_rows * from music limit 10) t limit 1", - "plan": "Incorrect usage/placement of 'SQL_CALC_FOUND_ROWS'", - "skip_e2e": true + "plan": "Incorrect usage/placement of 'SQL_CALC_FOUND_ROWS'" }, { "comment": "select from unsharded keyspace into dumpfile", @@ -2301,8 +2245,7 @@ "TablesUsed": [ "main.unsharded" ] - }, - "skip_e2e": true + } }, { "comment": "select from unsharded keyspace into outfile", @@ -2324,8 +2267,7 @@ "TablesUsed": [ "main.unsharded" ] - }, - "skip_e2e": true + } }, { "comment": "select from unsharded keyspace into outfile s3", @@ -2347,8 +2289,7 @@ "TablesUsed": [ "main.unsharded" ] - }, - "skip_e2e": true + } }, { "comment": "left join with a dual table on left - merge-able", @@ -2559,20 +2500,17 @@ { "comment": "Union after into outfile is incorrect", "query": "select id from user into outfile 'out_file_name' union all select id from music", - "plan": "syntax error at position 55 near 'union'", - "skip_e2e": true + "plan": "syntax error at position 55 near 'union'" }, { "comment": "Into outfile s3 in derived table is incorrect", "query": "select id from (select id from user into outfile s3 'inner_outfile') as t2", - "plan": "syntax error at position 41 near 'into'", - "skip_e2e": true + "plan": "syntax error at position 41 near 'into'" }, { "comment": "Into outfile s3 in derived table with union incorrect", "query": "select id from (select id from user into outfile s3 'inner_outfile' union select 1) as t2", - "plan": "syntax error at position 41 near 'into'", - "skip_e2e": true + "plan": "syntax error at position 41 near 'into'" }, { "comment": "select (select u.id from user as u where u.id = 1), a.id from user as a where a.id = 1", @@ -2641,8 +2579,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "((((select 1))))", @@ -2687,8 +2624,7 @@ "main.dual", "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "subquery in select expression of derived table", @@ -2758,8 +2694,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "ORDER BY subquery", @@ -2829,8 +2764,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "plan test for a natural character set string", @@ -2897,8 +2831,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "Straight Join ensures specific ordering of joins", @@ -2943,8 +2876,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "Dual query should be handled on the vtgate even with a LIMIT", @@ -3018,8 +2950,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "Straight Join preserved in MySQL query", @@ -3042,8 +2973,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "correlated subquery in exists clause", @@ -3101,8 +3031,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "correlated subquery in exists clause with an order by", @@ -3161,8 +3090,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "correlated subquery having dependencies on two tables", @@ -3235,8 +3163,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "correlated subquery using a column twice", @@ -3293,8 +3220,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "correlated subquery that is dependent on one side of a join, fully mergeable", @@ -3345,8 +3271,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "union as a derived table", @@ -3385,8 +3310,7 @@ "main.unsharded", "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "use output column containing data from both sides of the join", @@ -3436,8 +3360,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "mergeable derived table with order by and limit", @@ -3459,8 +3382,7 @@ "TablesUsed": [ "main.unsharded" ] - }, - "skip_e2e": true + } }, { "comment": "mergeable derived table with group by and limit", @@ -3482,8 +3404,7 @@ "TablesUsed": [ "main.unsharded" ] - }, - "skip_e2e": true + } }, { "comment": "select user.id, trim(leading 'x' from user.name) from user", @@ -3505,8 +3426,7 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "json utility functions", @@ -3528,8 +3448,7 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "dual query with exists clause", @@ -3627,8 +3546,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "yeah, it does not make sense, but it's valid", @@ -3721,8 +3639,7 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "groupe by with non aggregated columns and table alias", @@ -3744,8 +3661,7 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "Functions that return JSON value attributes", @@ -3950,8 +3866,7 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "gtid functions", @@ -4019,8 +3934,7 @@ "user.user_extra", "user.user_metadata" ] - }, - "skip_e2e": true + } }, { "comment": "Join across multiple tables, with conditions on different vindexes, but mergeable through join predicates", @@ -4048,8 +3962,7 @@ "user.music_extra", "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "SQL_CALC_FOUND_ROWS with vindex lookup", @@ -4160,8 +4073,7 @@ "TablesUsed": [ "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "`None` route being merged with another route via join predicate on Vindex columns", @@ -4288,8 +4200,7 @@ "TablesUsed": [ "user.music" ] - }, - "skip_e2e": true + } }, { "comment": "Subquery with `IN` condition using columns with matching lookup vindexes", @@ -4403,8 +4314,7 @@ "TablesUsed": [ "user.music" ] - }, - "skip_e2e": true + } }, { "comment": "Mergeable scatter subquery with `GROUP BY` on unique vindex column", @@ -4426,8 +4336,7 @@ "TablesUsed": [ "user.music" ] - }, - "skip_e2e": true + } }, { "comment": "Unmergeable scatter subquery with `GROUP BY` on-non vindex column", @@ -4466,27 +4375,51 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, "TablesUsed": [ "user.music" ] - }, - "skip_e2e": true + } }, { "comment": "Unmergeable scatter subquery with LIMIT", @@ -4522,27 +4455,51 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, "TablesUsed": [ "user.music" ] - }, - "skip_e2e": true + } }, { "comment": "Mergeable subquery with `MAX` aggregate and grouped by unique vindex", @@ -4576,27 +4533,51 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, "TablesUsed": [ "user.music" ] - }, - "skip_e2e": true + } }, { "comment": "Unmergeable subquery with `MAX` aggregate", @@ -4637,27 +4618,51 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, "TablesUsed": [ "user.music" ] - }, - "skip_e2e": true + } }, { "comment": "Mergeable subquery with `MAX` aggregate with `EqualUnique` route operator", @@ -4691,27 +4696,51 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, "TablesUsed": [ "user.music" ] - }, - "skip_e2e": true + } }, { "comment": "Mergeable subquery with `LIMIT` due to `EqualUnique` route", @@ -4745,27 +4774,51 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, "TablesUsed": [ "user.music" ] - }, - "skip_e2e": true + } }, { "comment": "Mergeable subquery with multiple levels of derived statements", @@ -4857,27 +4910,51 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, "TablesUsed": [ "user.music" ] - }, - "skip_e2e": true + } }, { "comment": "Unmergeable subquery with multiple levels of derived statements", @@ -4913,27 +4990,51 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, "TablesUsed": [ "user.music" ] - }, - "skip_e2e": true + } }, { "comment": "`None` subquery as top level predicate - outer query changes from `Scatter` to `None` on merge", @@ -5132,8 +5233,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "select user.a, t.b from user join (select id, count(*) b, req from user_extra group by req, id) as t on user.id = t.id", @@ -5197,8 +5297,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "SELECT music.id FROM (SELECT MAX(id) as maxt FROM music WHERE music.user_id = 5) other JOIN music ON other.maxt = music.id", @@ -5321,8 +5420,7 @@ "main.dual", "main.unsharded_a" ] - }, - "skip_e2e": true + } }, { "comment": "subquery having join table on clause, using column reference of outer select table", @@ -5371,8 +5469,7 @@ "main.unsharded", "main.unsharded_a" ] - }, - "skip_e2e": true + } }, { "comment": "ALL modifier on unsharded table works well", @@ -5395,8 +5492,7 @@ "main.unsharded", "main.unsharded_a" ] - }, - "skip_e2e": true + } }, { "comment": "allow last_insert_id with argument", @@ -5441,8 +5537,7 @@ "user.music_extra", "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "Query with non-plannable lookup vindex", @@ -5468,8 +5563,7 @@ "TablesUsed": [ "user.user_metadata" ] - }, - "skip_e2e": true + } }, { "comment": "join query with lookup and join on different vindex column", @@ -5521,8 +5615,7 @@ "user.user", "user.user_metadata" ] - }, - "skip_e2e": true + } }, { "comment": "pick email as vindex lookup", @@ -5532,7 +5625,7 @@ "Original": "select * from customer where email = 'a@mail.com'", "Instructions": { "OperatorType": "VindexLookup", - "Variant": "Equal", + "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true @@ -5617,8 +5710,7 @@ "TablesUsed": [ "user.customer" ] - }, - "skip_e2e": true + } }, { "comment": "email vindex is costly than phone vindex - but phone vindex is backfiling hence ignored", @@ -5628,7 +5720,7 @@ "Original": "select * from customer where email = 'a@mail.com' and phone = 123456", "Instructions": { "OperatorType": "VindexLookup", - "Variant": "Equal", + "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true @@ -5679,7 +5771,7 @@ "Original": "select * from customer where phone = 123456 and email = 'a@mail.com'", "Instructions": { "OperatorType": "VindexLookup", - "Variant": "Equal", + "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true @@ -5742,8 +5834,7 @@ "TablesUsed": [ "user.samecolvin" ] - }, - "skip_e2e": true + } }, { "comment": "column with qualifier is correctly used", @@ -5786,8 +5877,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "Derived tables going to a single shard still need to expand derived table columns", @@ -5832,8 +5922,7 @@ "main.unsharded", "user.user" ] - }, - "skip_e2e": true + } }, { "comment": "column name aliases in outer join queries", @@ -5888,8 +5977,7 @@ "user.user", "user.user_extra" ] - }, - "skip_e2e": true + } }, { "comment": "Over clause works for unsharded tables", @@ -5911,8 +5999,7 @@ "TablesUsed": [ "main.unsharded_a" ] - }, - "skip_e2e": true + } }, { "comment": "join with derived table with alias and join condition - merge into route", diff --git a/go/vt/vtgate/planbuilder/testdata/union_cases.json b/go/vt/vtgate/planbuilder/testdata/union_cases.json index 7feabb0a698..2927c1c6093 100644 --- a/go/vt/vtgate/planbuilder/testdata/union_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/union_cases.json @@ -447,34 +447,84 @@ "OperatorType": "Concatenate", "Inputs": [ { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from music where 1 != 1", - "Query": "select distinct 1 from music where id = 1", - "Table": "music", "Values": [ "1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from music where 1 != 1", + "Query": "select distinct 1 from music where id = 1", + "Table": "music" + } + ] }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from music where 1 != 1", - "Query": "select distinct 1 from music where id = 2", - "Table": "music", "Values": [ "2" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from music where 1 != 1", + "Query": "select distinct 1 from music where id = 2", + "Table": "music" + } + ] } ] } diff --git a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json index aab90c3b578..a5de9d3697e 100644 --- a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json +++ b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json @@ -65,27 +65,45 @@ "type": "hash", "owner": "multicolvin" }, + "hash": { + "type": "hash" + }, "user_md5_index": { "type": "unicode_loose_md5" }, "music_user_map": { "type": "lookup_unique", - "owner": "music" + "owner": "music", + "params": { + "table": "name_user_vdx", + "from": "name", + "to": "keyspace_id" + } }, "cola_map": { "type": "lookup_unique", - "owner": "multicolvin" + "owner": "multicolvin", + "params": { + "table": "cola_map", + "from": "cola", + "to": "keyspace_id" + } }, "colb_colc_map": { "type": "lookup_unique", - "owner": "multicolvin" + "owner": "multicolvin", + "params": { + "table": "colb_colc_map", + "from": "colb,colc", + "to": "keyspace_id" + } }, "cola_kid_map": { "type": "lookup_unique", "owner": "overlap_vindex" }, "name_user_map": { - "type": "lookup_hash", + "type": "lookup", "owner": "user", "params": { "table": "name_user_vdx", @@ -102,8 +120,14 @@ "owner": "user_metadata" }, "costly_map": { - "type": "lookup_unique", - "owner": "user" + "type": "lookup_cost", + "owner": "user", + "params": { + "table": "costly_map", + "from": "costly", + "to": "keyspace_id", + "cost": "100" + } }, "hash_dup": { "type": "hash", @@ -131,8 +155,13 @@ "owner": "multicol_tbl" }, "name_muticoltbl_map": { - "type": "lookup_hash", - "owner": "multicol_tbl" + "type": "lookup", + "owner": "multicol_tbl", + "params": { + "table": "name_user_vdx", + "from": "name", + "to": "keyspace_id" + } }, "non_planable_user_map": { "type": "lookup_unicodeloosemd5_hash", @@ -144,7 +173,7 @@ "owner": "user_metadata" }, "lkp_shard_map": { - "type": "lookup_hash", + "type": "lookup_unique", "owner": "mixed_tbl", "params": { "table": "lkp_shard_vdx", @@ -160,7 +189,7 @@ "owner": "customer", "params": { "table": "unq_lkp_idx", - "from": " ", + "from": "unq_key", "to": "keyspace_id", "cost": "100", "write_only": "true" @@ -177,11 +206,11 @@ } }, "lkp_bf_vdx": { - "type": "lookup_hash", + "type": "lookup_unique", "owner": "customer", "params": { "table": "lkp_shard_vdx", - "from": " ", + "from": "unq_key", "to": "keyspace_id", "write_only": "true" } @@ -355,6 +384,22 @@ } ] }, + "cola_map": { + "column_vindexes": [ + { + "column": "cola", + "name": "hash" + } + ] + }, + "colb_colc_map": { + "column_vindexes": [ + { + "column": "colb", + "name": "hash" + } + ] + }, "overlap_vindex": { "column_vindexes": [ { @@ -465,6 +510,14 @@ } ] }, + "costly_map": { + "column_vindexes": [ + { + "column": "name", + "name": "user_md5_index" + } + ] + }, "mixed_tbl": { "column_vindexes": [ { diff --git a/go/vt/vtgate/vindexes/lookup_cost.go b/go/vt/vtgate/vindexes/lookup_cost.go new file mode 100644 index 00000000000..6556032cea5 --- /dev/null +++ b/go/vt/vtgate/vindexes/lookup_cost.go @@ -0,0 +1,70 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vindexes + +import ( + "strconv" +) + +var ( + _ SingleColumn = (*LookupCost)(nil) + _ Lookup = (*LookupCost)(nil) + _ LookupPlanable = (*LookupCost)(nil) +) + +func init() { + Register("lookup_cost", newLookupCost) +} + +const defaultCost = 5 + +// LookupCost defines a test vindex that uses the cost provided by the user. +// This is a test vindex. +type LookupCost struct { + *LookupNonUnique + cost int +} + +// Cost returns the cost of this vindex as provided. +func (lc *LookupCost) Cost() int { + return lc.cost +} + +func newLookupCost(name string, m map[string]string) (Vindex, error) { + lookup, err := newLookup(name, m) + if err != nil { + return nil, err + } + cost := getInt(m, "cost", defaultCost) + return &LookupCost{ + LookupNonUnique: lookup.(*LookupNonUnique), + cost: cost, + }, nil + +} + +func getInt(m map[string]string, key string, defaultVal int) int { + val, ok := m[key] + if !ok { + return defaultVal + } + intVal, err := strconv.Atoi(val) + if err != nil { + return defaultVal + } + return intVal +} From 25b73debaecf3d72b994e55845eb594b2a92d88d Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Fri, 6 Dec 2024 23:48:08 +0530 Subject: [PATCH 07/12] refactor: test file rename Signed-off-by: Harshit Gangal --- .../{plan_test.go => plan_e2e_test.go} | 0 go/vt/vtgate/engine/plan_description.go | 20 +++++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) rename go/test/endtoend/vtgate/queries/plan_tests/{plan_test.go => plan_e2e_test.go} (100%) diff --git a/go/test/endtoend/vtgate/queries/plan_tests/plan_test.go b/go/test/endtoend/vtgate/queries/plan_tests/plan_e2e_test.go similarity index 100% rename from go/test/endtoend/vtgate/queries/plan_tests/plan_test.go rename to go/test/endtoend/vtgate/queries/plan_tests/plan_e2e_test.go diff --git a/go/vt/vtgate/engine/plan_description.go b/go/vt/vtgate/engine/plan_description.go index 9f14d03803f..59c60c39dc7 100644 --- a/go/vt/vtgate/engine/plan_description.go +++ b/go/vt/vtgate/engine/plan_description.go @@ -200,16 +200,16 @@ func (pd PrimitiveDescription) Equals(other PrimitiveDescription) string { return fmt.Sprintf("OperatorType: %v != %v", pd.OperatorType, other.OperatorType) } - switch { - case pd.Keyspace == nil && other.Keyspace == nil: - // do nothing - case pd.Keyspace != nil && other.Keyspace != nil: - if pd.Keyspace.Name != other.Keyspace.Name { - return fmt.Sprintf("Keyspace.Name: %v != %v", pd.Keyspace.Name, other.Keyspace.Name) - } - default: - return "Keyspace is nil in one of the descriptions" - } + // switch { + // case pd.Keyspace == nil && other.Keyspace == nil: + // // do nothing + // case pd.Keyspace != nil && other.Keyspace != nil: + // if pd.Keyspace.Name != other.Keyspace.Name { + // return fmt.Sprintf("Keyspace.Name: %v != %v", pd.Keyspace.Name, other.Keyspace.Name) + // } + // default: + // return "Keyspace is nil in one of the descriptions" + // } switch { case pd.TargetDestination == nil && other.TargetDestination == nil: From 672b503f09abb1093927c7d5e1564b1e92d3c227 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Sat, 7 Dec 2024 11:06:25 +0530 Subject: [PATCH 08/12] run plantests in different shard CI Signed-off-by: Harshit Gangal --- .../cluster_endtoend_vtgate_plantests.yml | 166 ++++++++++++++++++ .../{queries => }/plan_tests/main_test.go | 0 .../{queries => }/plan_tests/plan_e2e_test.go | 0 test/ci_workflow_gen.go | 1 + test/config.json | 9 + 5 files changed, 176 insertions(+) create mode 100644 .github/workflows/cluster_endtoend_vtgate_plantests.yml rename go/test/endtoend/vtgate/{queries => }/plan_tests/main_test.go (100%) rename go/test/endtoend/vtgate/{queries => }/plan_tests/plan_e2e_test.go (100%) diff --git a/.github/workflows/cluster_endtoend_vtgate_plantests.yml b/.github/workflows/cluster_endtoend_vtgate_plantests.yml new file mode 100644 index 00000000000..93ed6a55f05 --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_plantests.yml @@ -0,0 +1,166 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_plantests) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vtgate_plantests)') + cancel-in-progress: true + +permissions: read-all + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (vtgate_plantests) + runs-on: ubuntu-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl -s\ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'go/vt/sidecardb/**/*.sql' + - 'go/test/endtoend/onlineddl/vrepl_suite/**' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_vtgate_plantests.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + + # Install everything else we need, and configure + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --no-commit-collection --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -exo pipefail + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vtgate_plantests | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable if PR is not a draft + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + fi + + # print test output + cat output.txt + + - name: Test Summary + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 + with: + paths: "report.xml" + show: "fail" diff --git a/go/test/endtoend/vtgate/queries/plan_tests/main_test.go b/go/test/endtoend/vtgate/plan_tests/main_test.go similarity index 100% rename from go/test/endtoend/vtgate/queries/plan_tests/main_test.go rename to go/test/endtoend/vtgate/plan_tests/main_test.go diff --git a/go/test/endtoend/vtgate/queries/plan_tests/plan_e2e_test.go b/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go similarity index 100% rename from go/test/endtoend/vtgate/queries/plan_tests/plan_e2e_test.go rename to go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index 7956c491408..30580f8b8c8 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -103,6 +103,7 @@ var ( "vtgate_vindex_heavy", "vtgate_vschema", "vtgate_queries", + "vtgate_plantests", "vtgate_schema_tracker", "vtgate_foreignkey_stress", "vtorc", diff --git a/test/config.json b/test/config.json index 1e278546c7a..091164e0c42 100644 --- a/test/config.json +++ b/test/config.json @@ -887,6 +887,15 @@ "RetryMax": 1, "Tags": [] }, + "vtgate_plantests": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/plan_tests"], + "Command": [], + "Manual": false, + "Shard": "vtgate_plantests", + "RetryMax": 1, + "Tags": [] + }, "vtgate_unsharded": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/unsharded"], From e4d0edd37cd1adb0508c530860214235faac6a44 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Sat, 7 Dec 2024 11:19:11 +0530 Subject: [PATCH 09/12] added authoritative table Signed-off-by: Harshit Gangal --- go/test/endtoend/vtgate/plan_tests/main_test.go | 6 +++--- go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go | 3 +++ .../testdata/schemas/{uschema.sql => main.sql} | 0 .../testdata/schemas/{sschema.sql => user.sql} | 10 +++++++++- 4 files changed, 15 insertions(+), 4 deletions(-) rename go/vt/vtgate/planbuilder/testdata/schemas/{uschema.sql => main.sql} (100%) rename go/vt/vtgate/planbuilder/testdata/schemas/{sschema.sql => user.sql} (90%) diff --git a/go/test/endtoend/vtgate/plan_tests/main_test.go b/go/test/endtoend/vtgate/plan_tests/main_test.go index 4ceefb774f5..59f95247bfb 100644 --- a/go/test/endtoend/vtgate/plan_tests/main_test.go +++ b/go/test/endtoend/vtgate/plan_tests/main_test.go @@ -46,8 +46,8 @@ func TestMain(m *testing.M) { vschema := readFile("vschemas/schema.json") userVs := extractUserKS(vschema) mainVs := extractMainKS(vschema) - sSQL := readFile("schemas/sschema.sql") - uSQL := readFile("schemas/uschema.sql") + sSQL := readFile("schemas/user.sql") + uSQL := readFile("schemas/main.sql") exitCode := func() int { clusterInstance = cluster.NewCluster(cell, "localhost") @@ -146,7 +146,7 @@ func readJSONTests(filename string) []planbuilder.PlanTest { } func locateFile(name string) string { - return "../../../../../vt/vtgate/planbuilder/testdata/" + name + return "../../../../vt/vtgate/planbuilder/testdata/" + name } // verifyTestExpectations verifies the expectations of the test. diff --git a/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go b/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go index 6065a2514ee..5dee5b73781 100644 --- a/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go +++ b/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go @@ -34,6 +34,9 @@ func TestSelectCases(t *testing.T) { mcmp.Exec(test.Query) pd := utils.ExecTrace(mcmp.AsT(), mcmp.VtConn, test.Query) verifyTestExpectations(mcmp.AsT(), pd, test) + if mcmp.VtConn.IsClosed() { + mcmp.AsT().Fatal("vtgate connection is closed") + } }) } } diff --git a/go/vt/vtgate/planbuilder/testdata/schemas/uschema.sql b/go/vt/vtgate/planbuilder/testdata/schemas/main.sql similarity index 100% rename from go/vt/vtgate/planbuilder/testdata/schemas/uschema.sql rename to go/vt/vtgate/planbuilder/testdata/schemas/main.sql diff --git a/go/vt/vtgate/planbuilder/testdata/schemas/sschema.sql b/go/vt/vtgate/planbuilder/testdata/schemas/user.sql similarity index 90% rename from go/vt/vtgate/planbuilder/testdata/schemas/sschema.sql rename to go/vt/vtgate/planbuilder/testdata/schemas/user.sql index c54943f9ecd..55f4078557a 100644 --- a/go/vt/vtgate/planbuilder/testdata/schemas/sschema.sql +++ b/go/vt/vtgate/planbuilder/testdata/schemas/user.sql @@ -89,4 +89,12 @@ CREATE TABLE t1 c2 INT, c3 INT, PRIMARY KEY (c1) -); \ No newline at end of file +); + +CREATE TABLE authoritative +( + user_id bigint NOT NULL, + col1 VARCHAR(255), + col2 bigint, + PRIMARY KEY (user_id) +) ENGINE=InnoDB; \ No newline at end of file From 431e946c367db45a6409185b8671b78714b919e2 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Sat, 7 Dec 2024 11:42:06 +0530 Subject: [PATCH 10/12] make test green, skip the failing ones Signed-off-by: Harshit Gangal --- .../vtgate/plan_tests/plan_e2e_test.go | 2 +- .../planbuilder/testdata/select_cases.json | 101 ++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go b/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go index 5dee5b73781..1594e9b392c 100644 --- a/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go +++ b/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go @@ -27,7 +27,7 @@ func TestSelectCases(t *testing.T) { defer closer() tests := readJSONTests("select_cases.json") for _, test := range tests { - mcmp.Run(test.Query, func(mcmp *utils.MySQLCompare) { + mcmp.Run(test.Comment, func(mcmp *utils.MySQLCompare) { if test.SkipE2E { mcmp.AsT().Skip(test.Query) } diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.json b/go/vt/vtgate/planbuilder/testdata/select_cases.json index ae37d4b10e6..9b03571fc5c 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.json @@ -68,6 +68,7 @@ { "comment": "join on sharding column with limit - should be a simple scatter query and limit on top with non resolved columns", "query": "select * from user u join user_metadata um on u.id = um.user_id where foo=41 limit 20", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user u join user_metadata um on u.id = um.user_id where foo=41 limit 20", @@ -179,6 +180,7 @@ { "comment": "select limit with timeout directive sets QueryTimeout in the route", "query": "select /*vt+ QUERY_TIMEOUT_MS=1000 */ * from main.unsharded limit 10", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select /*vt+ QUERY_TIMEOUT_MS=1000 */ * from main.unsharded limit 10", @@ -402,6 +404,7 @@ { "comment": "test table lookup failure for authoritative code path", "query": "select a.* from authoritative", + "skip_e2e": true, "plan": "Unknown table 'a'" }, { @@ -452,6 +455,7 @@ { "comment": "select authoritative.* with intermixing still expands", "query": "select user.id, a.*, user.col1 from authoritative a join user on a.user_id=user.id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.id, a.*, user.col1 from authoritative a join user on a.user_id=user.id", @@ -475,6 +479,7 @@ { "comment": "auto-resolve anonymous columns for simple route", "query": "select anon_col from user join user_extra on user.id = user_extra.user_id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select anon_col from user join user_extra on user.id = user_extra.user_id", @@ -498,6 +503,7 @@ { "comment": "json_arrayagg in single sharded query", "query": "select count(1) from user where id = 'abc' group by n_id having json_arrayagg(a_id) = '[]'", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select count(1) from user where id = 'abc' group by n_id having json_arrayagg(a_id) = '[]'", @@ -524,6 +530,7 @@ { "comment": "json_objectagg in single sharded query", "query": "select count(1) from user where id = 'abc' group by n_id having json_objectagg(a_id, b_id) = '[]'", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select count(1) from user where id = 'abc' group by n_id having json_objectagg(a_id, b_id) = '[]'", @@ -550,16 +557,19 @@ { "comment": "unsupported json aggregation expressions in scatter query", "query": "select count(1) from user where cola = 'abc' group by n_id having json_arrayagg(a_id) = '[]'", + "skip_e2e": true, "plan": "VT12001: unsupported: in scatter query: aggregation function 'json_arrayagg(a_id)'" }, { "comment": "Cannot auto-resolve for cross-shard joins", "query": "select col from user join user_extra", + "skip_e2e": true, "plan": "Column 'col' in field list is ambiguous" }, { "comment": "Auto-resolve should work if unique vindex columns are referenced", "query": "select id, user_id from user join user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select id, user_id from user join user_extra", @@ -602,6 +612,7 @@ { "comment": "database calls should be substituted", "query": "select database() from dual", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select database() from dual", @@ -624,6 +635,7 @@ { "comment": "last_insert_id for unsharded route", "query": "select last_insert_id() as x from main.unsharded", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select last_insert_id() as x from main.unsharded", @@ -694,11 +706,13 @@ { "comment": "prefixing dual with a keyspace should not work", "query": "select 1 from user.dual", + "skip_e2e": true, "plan": "table dual not found" }, { "comment": "RHS route referenced", "query": "select user_extra.id from user join user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user_extra.id from user join user_extra", @@ -741,6 +755,7 @@ { "comment": "Both routes referenced", "query": "select user.col, user_extra.id from user join user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col, user_extra.id from user join user_extra", @@ -783,6 +798,7 @@ { "comment": "Expression with single-route reference", "query": "select user.col, user_extra.id + user_extra.col from user join user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col, user_extra.id + user_extra.col from user join user_extra", @@ -825,6 +841,7 @@ { "comment": "subquery with an aggregation in order by that can be merged into a single route", "query": "select col, trim((select user_name from user where id = 3)) val from user_extra where user_id = 3 group by col order by val", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select col, trim((select user_name from user where id = 3)) val from user_extra where user_id = 3 group by col order by val", @@ -852,6 +869,7 @@ { "comment": "Jumbled references", "query": "select user.col, user_extra.id, user.col2 from user join user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col, user_extra.id, user.col2 from user join user_extra", @@ -894,6 +912,7 @@ { "comment": "Comments", "query": "select /* comment */ user.col from user join user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select /* comment */ user.col from user join user_extra", @@ -936,6 +955,7 @@ { "comment": "for update", "query": "select user.col from user join user_extra for update", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col from user join user_extra for update", @@ -978,6 +998,7 @@ { "comment": "Field query should work for joins select bind vars", "query": "select user.id, (select user.id+outm.m+unsharded.m from unsharded) from user join unsharded outm", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.id, (select user.id+outm.m+unsharded.m from unsharded) from user join unsharded outm", @@ -1023,6 +1044,7 @@ { "comment": "Case preservation", "query": "select user.Col, user_extra.Id from user join user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.Col, user_extra.Id from user join user_extra", @@ -1065,6 +1087,7 @@ { "comment": "syntax error", "query": "the quick brown fox", + "skip_e2e": true, "plan": "syntax error at position 4 near 'the'" }, { @@ -1096,6 +1119,7 @@ { "comment": "Selection but explicitly ignore a vindex", "query": "select * from user ignore vindex (user_index) where id = 1", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user ignore vindex (user_index) where id = 1", @@ -1118,6 +1142,7 @@ { "comment": "Selection but make the planner explicitly use a vindex", "query": "select intcol, id from user use vindex (name_user_map) where costly = 'aa' and name = 'bb' and id = 3", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select intcol, id from user use vindex (name_user_map) where costly = 'aa' and name = 'bb' and id = 3", @@ -1169,6 +1194,7 @@ { "comment": "Vindex hint on a non-existing vindex", "query": "select * from user use vindex (does_not_exist) where id = 1", + "skip_e2e": true, "plan": "VT09021: Vindex 'does_not_exist' does not exist in table 'user.user'" }, { @@ -1205,6 +1231,7 @@ { "comment": "sharded limit offset with arguments", "query": "select user_id from music order by user_id limit :limit, :offset", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user_id from music order by user_id limit :limit, :offset", @@ -1236,6 +1263,7 @@ { "comment": "Sharding Key Condition in Parenthesis", "query": "select * from user where name ='abc' AND (id = 4) limit 5", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user where name ='abc' AND (id = 4) limit 5", @@ -1262,6 +1290,7 @@ { "comment": "Multiple parenthesized expressions", "query": "select * from user where (id = 4) AND (name ='abc') limit 5", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user where (id = 4) AND (name ='abc') limit 5", @@ -1288,6 +1317,7 @@ { "comment": "Multiple parenthesized expressions", "query": "select * from user where (id = 4 and name ='abc') limit 5", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user where (id = 4 and name ='abc') limit 5", @@ -1366,6 +1396,7 @@ { "comment": "Booleans and parenthesis", "query": "select * from user where (id = 1) AND name = true limit 5", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user where (id = 1) AND name = true limit 5", @@ -1392,6 +1423,7 @@ { "comment": "Column as boolean-ish", "query": "select * from user where (id = 1) AND name limit 5", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user where (id = 1) AND name limit 5", @@ -1418,6 +1450,7 @@ { "comment": "PK as fake boolean, and column as boolean-ish", "query": "select * from user where (id = 5) AND name = true limit 5", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user where (id = 5) AND name = true limit 5", @@ -1444,6 +1477,7 @@ { "comment": "top level subquery in select", "query": "select a, (select col from user) from unsharded", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select a, (select col from user) from unsharded", @@ -1489,6 +1523,7 @@ { "comment": "sub-expression subquery in select", "query": "select a, 1+(select col from user) from unsharded", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select a, 1+(select col from user) from unsharded", @@ -1534,6 +1569,7 @@ { "comment": "select * from derived table expands specific columns", "query": "select * from (select user.id id1, user_extra.id id2 from user join user_extra) as t", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from (select user.id id1, user_extra.id id2 from user join user_extra) as t", @@ -1576,16 +1612,19 @@ { "comment": "duplicate columns not allowed in derived table", "query": "select * from (select user.id, user_extra.id from user join user_extra) as t", + "skip_e2e": true, "plan": "Duplicate column name 'id'" }, { "comment": "non-existent symbol in cross-shard derived table", "query": "select t.col from (select user.id from user join user_extra) as t", + "skip_e2e": true, "plan": "column 't.col' not found" }, { "comment": "union with the same target shard", "query": "select * from music where user_id = 1 union select * from user where id = 1", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from music where user_id = 1 union select * from user where id = 1", @@ -1613,6 +1652,7 @@ { "comment": "union with the same target shard last_insert_id", "query": "select *, last_insert_id() from music where user_id = 1 union select * from user where id = 1", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select *, last_insert_id() from music where user_id = 1 union select * from user where id = 1", @@ -1775,6 +1815,7 @@ { "comment": "routing rules: ensure directives are not lost", "query": "select /*vt+ QUERY_TIMEOUT_MS=1000 */ * from route2", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select /*vt+ QUERY_TIMEOUT_MS=1000 */ * from route2", @@ -1798,6 +1839,7 @@ { "comment": "routing table on music", "query": "select * from second_user.bar where id > 2", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from second_user.bar where id > 2", @@ -1914,6 +1956,7 @@ { "comment": "Complex expression in a subquery used in IN clause of an aggregate query", "query": "select count(*) from user where user.id = 2 or user.id in (select id from unsharded_a where colb = 2)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select count(*) from user where user.id = 2 or user.id in (select id from unsharded_a where colb = 2)", @@ -1967,6 +2010,7 @@ { "comment": "Complex expression in a subquery used in NOT IN clause of an aggregate query", "query": "select count(*) from user where user.id = 2 or user.id not in (select id from unsharded_a where colb = 2)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select count(*) from user where user.id = 2 or user.id not in (select id from unsharded_a where colb = 2)", @@ -2218,16 +2262,19 @@ { "comment": "sql_calc_found_rows in sub queries", "query": "select * from music where user_id IN (select sql_calc_found_rows * from music limit 10)", + "skip_e2e": true, "plan": "Incorrect usage/placement of 'SQL_CALC_FOUND_ROWS'" }, { "comment": "sql_calc_found_rows in derived table", "query": "select sql_calc_found_rows * from (select sql_calc_found_rows * from music limit 10) t limit 1", + "skip_e2e": true, "plan": "Incorrect usage/placement of 'SQL_CALC_FOUND_ROWS'" }, { "comment": "select from unsharded keyspace into dumpfile", "query": "select * from main.unsharded into Dumpfile 'x.txt'", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from main.unsharded into Dumpfile 'x.txt'", @@ -2250,6 +2297,7 @@ { "comment": "select from unsharded keyspace into outfile", "query": "select * from main.unsharded into outfile 'x.txt' character set binary fields terminated by 'term' optionally enclosed by 'c' escaped by 'e' lines starting by 'a' terminated by '\n'", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from main.unsharded into outfile 'x.txt' character set binary fields terminated by 'term' optionally enclosed by 'c' escaped by 'e' lines starting by 'a' terminated by '\n'", @@ -2272,6 +2320,7 @@ { "comment": "select from unsharded keyspace into outfile s3", "query": "select * from main.unsharded into outfile s3 'out_file_name' character set binary format csv header fields terminated by 'term' optionally enclosed by 'c' escaped by 'e' lines starting by 'a' terminated by '\n' manifest on overwrite off", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from main.unsharded into outfile s3 'out_file_name' character set binary format csv header fields terminated by 'term' optionally enclosed by 'c' escaped by 'e' lines starting by 'a' terminated by '\n' manifest on overwrite off", @@ -2500,16 +2549,19 @@ { "comment": "Union after into outfile is incorrect", "query": "select id from user into outfile 'out_file_name' union all select id from music", + "skip_e2e": true, "plan": "syntax error at position 55 near 'union'" }, { "comment": "Into outfile s3 in derived table is incorrect", "query": "select id from (select id from user into outfile s3 'inner_outfile') as t2", + "skip_e2e": true, "plan": "syntax error at position 41 near 'into'" }, { "comment": "Into outfile s3 in derived table with union incorrect", "query": "select id from (select id from user into outfile s3 'inner_outfile' union select 1) as t2", + "skip_e2e": true, "plan": "syntax error at position 41 near 'into'" }, { @@ -2541,6 +2593,7 @@ { "comment": "Add two tables with the same column in a join", "query": "select t.id, s.id from user t join user_extra s on t.id = s.user_id join unsharded", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select t.id, s.id from user t join user_extra s on t.id = s.user_id join unsharded", @@ -2606,6 +2659,7 @@ { "comment": "Merging dual with user", "query": "select 42, id from dual, user", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 42, id from dual, user", @@ -2629,6 +2683,7 @@ { "comment": "subquery in select expression of derived table", "query": "select t.a from (select (select col from user limit 1) as a from user join user_extra) t", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select t.a from (select (select col from user limit 1) as a from user join user_extra) t", @@ -2699,6 +2754,7 @@ { "comment": "ORDER BY subquery", "query": "select (select col from user limit 1) as a from user join user_extra order by a", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select (select col from user limit 1) as a from user join user_extra order by a", @@ -2791,6 +2847,7 @@ { "comment": "select expression having dependencies on both sides of a join", "query": "select user.id * user_id as amount from user, user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.id * user_id as amount from user, user_extra", @@ -2836,6 +2893,7 @@ { "comment": "Straight Join ensures specific ordering of joins", "query": "select user.id, user_extra.user_id from user straight_join user_extra where user.id = user_extra.foo", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.id, user_extra.user_id from user straight_join user_extra where user.id = user_extra.foo", @@ -2903,6 +2961,7 @@ { "comment": "PullOut subquery with an aggregation that should be typed in the final output", "query": "select (select sum(col) from user) from user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select (select sum(col) from user) from user_extra", @@ -2955,6 +3014,7 @@ { "comment": "Straight Join preserved in MySQL query", "query": "select user.id, user_extra.user_id from user straight_join user_extra where user.id = user_extra.user_id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.id, user_extra.user_id from user straight_join user_extra where user.id = user_extra.user_id", @@ -2978,6 +3038,7 @@ { "comment": "correlated subquery in exists clause", "query": "select col from user where exists(select user_id from user_extra where user_id = 3 and user_id < user.id)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select col from user where exists(select user_id from user_extra where user_id = 3 and user_id < user.id)", @@ -3036,6 +3097,7 @@ { "comment": "correlated subquery in exists clause with an order by", "query": "select col from user where exists(select user_id from user_extra where user_id = 3 and user_id < user.id) order by col", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select col from user where exists(select user_id from user_extra where user_id = 3 and user_id < user.id) order by col", @@ -3095,6 +3157,7 @@ { "comment": "correlated subquery having dependencies on two tables", "query": "select 1 from user u1, user u2 where exists (select 1 from user_extra ue where ue.col = u1.col and ue.col = u2.col)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from user u1, user u2 where exists (select 1 from user_extra ue where ue.col = u1.col and ue.col = u2.col)", @@ -3168,6 +3231,7 @@ { "comment": "correlated subquery using a column twice", "query": "select 1 from user u where exists (select 1 from user_extra ue where ue.col = u.col and u.col = ue.col2)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from user u where exists (select 1 from user_extra ue where ue.col = u.col and u.col = ue.col2)", @@ -3252,6 +3316,7 @@ { "comment": "Complex join with multiple conditions merged into single route", "query": "select 0 from user as u join user_extra as s on u.id = s.user_id join music as m on m.user_id = u.id and (s.foo or m.bar)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 0 from user as u join user_extra as s on u.id = s.user_id join music as m on m.user_id = u.id and (s.foo or m.bar)", @@ -3315,6 +3380,7 @@ { "comment": "use output column containing data from both sides of the join", "query": "select user_extra.col + user.col from user join user_extra on user.id = user_extra.id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user_extra.col + user.col from user join user_extra on user.id = user_extra.id", @@ -3365,6 +3431,7 @@ { "comment": "mergeable derived table with order by and limit", "query": "select 1 from (select col from main.unsharded order by main.unsharded.col1 desc limit 12 offset 0) as f left join unsharded as u on f.col = u.id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from (select col from main.unsharded order by main.unsharded.col1 desc limit 12 offset 0) as f left join unsharded as u on f.col = u.id", @@ -3387,6 +3454,7 @@ { "comment": "mergeable derived table with group by and limit", "query": "select 1 from (select col, count(*) as a from main.unsharded group by col having a > 0 limit 12 offset 0) as f left join unsharded as u on f.col = u.id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from (select col, count(*) as a from main.unsharded group by col having a > 0 limit 12 offset 0) as f left join unsharded as u on f.col = u.id", @@ -3409,6 +3477,7 @@ { "comment": "select user.id, trim(leading 'x' from user.name) from user", "query": "select user.id, trim(leading 'x' from user.name) from user", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.id, trim(leading 'x' from user.name) from user", @@ -3431,6 +3500,7 @@ { "comment": "json utility functions", "query": "select jcol, JSON_STORAGE_SIZE(jcol), JSON_STORAGE_FREE(jcol), JSON_PRETTY(jcol) from user", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select jcol, JSON_STORAGE_SIZE(jcol), JSON_STORAGE_FREE(jcol), JSON_PRETTY(jcol) from user", @@ -3499,6 +3569,7 @@ { "comment": "select (select id from user order by id limit 1) from user_extra", "query": "select (select id from user order by id limit 1) from user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select (select id from user order by id limit 1) from user_extra", @@ -3622,6 +3693,7 @@ { "comment": "Json extract and json unquote shorthands", "query": "SELECT a->\"$[4]\", a->>\"$[3]\" FROM user", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT a->\"$[4]\", a->>\"$[3]\" FROM user", @@ -3644,6 +3716,7 @@ { "comment": "groupe by with non aggregated columns and table alias", "query": "select u.id, u.age from user u group by u.id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select u.id, u.age from user u group by u.id", @@ -3849,6 +3922,7 @@ { "comment": "insert function using column names as arguments", "query": "select insert(tcol1, id, 3, tcol2) from user", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select insert(tcol1, id, 3, tcol2) from user", @@ -3893,6 +3967,7 @@ { "comment": "Predicate in apply join which is merged", "query": "select user.col, user_metadata.user_id from user join user_extra on user.col = user_extra.col join user_metadata on user_extra.user_id = user_metadata.user_id where user.textcol1 = 'alice@gmail.com'", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col, user_metadata.user_id from user join user_extra on user.col = user_extra.col join user_metadata on user_extra.user_id = user_metadata.user_id where user.textcol1 = 'alice@gmail.com'", @@ -3939,6 +4014,7 @@ { "comment": "Join across multiple tables, with conditions on different vindexes, but mergeable through join predicates", "query": "SELECT user.id FROM user INNER JOIN music_extra ON user.id = music_extra.user_id INNER JOIN music ON music_extra.user_id = music.user_id WHERE user.id = 123 and music.id = 456", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT user.id FROM user INNER JOIN music_extra ON user.id = music_extra.user_id INNER JOIN music ON music_extra.user_id = music.user_id WHERE user.id = 123 and music.id = 456", @@ -3967,6 +4043,7 @@ { "comment": "SQL_CALC_FOUND_ROWS with vindex lookup", "query": "select SQL_CALC_FOUND_ROWS id, name from user where name = 'aa' order by id limit 2", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select SQL_CALC_FOUND_ROWS id, name from user where name = 'aa' order by id limit 2", @@ -4101,6 +4178,7 @@ { "comment": "Treating single value tuples as `EqualUnique` routes", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.user_id IN (5)) AND music.user_id = 5", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.user_id IN (5)) AND music.user_id = 5", @@ -4179,6 +4257,7 @@ { "comment": "Subquery with `IN` condition using columns with matching lookup vindexes, with inner scatter query", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.foo = 'bar') AND music.user_id IN (3, 4, 5)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.foo = 'bar') AND music.user_id IN (3, 4, 5)", @@ -4297,6 +4376,7 @@ { "comment": "Mergeable scatter subquery", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop')", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop')", @@ -4319,6 +4399,7 @@ { "comment": "Mergeable scatter subquery with `GROUP BY` on unique vindex column", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop' GROUP BY music.id)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop' GROUP BY music.id)", @@ -4341,6 +4422,7 @@ { "comment": "Unmergeable scatter subquery with `GROUP BY` on-non vindex column", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop' GROUP BY music.genre)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop' GROUP BY music.genre)", @@ -4424,6 +4506,7 @@ { "comment": "Unmergeable scatter subquery with LIMIT", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop' LIMIT 10)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop' LIMIT 10)", @@ -4504,6 +4587,7 @@ { "comment": "Mergeable subquery with `MAX` aggregate and grouped by unique vindex", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT MAX(music.id) FROM music WHERE music.user_id IN (5, 6) GROUP BY music.user_id)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT MAX(music.id) FROM music WHERE music.user_id IN (5, 6) GROUP BY music.user_id)", @@ -4745,6 +4829,7 @@ { "comment": "Mergeable subquery with `LIMIT` due to `EqualUnique` route", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT MAX(music.id) FROM music WHERE music.user_id = 5 LIMIT 10)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT MAX(music.id) FROM music WHERE music.user_id = 5 LIMIT 10)", @@ -4875,6 +4960,7 @@ { "comment": "Unmergeable subquery with multiple levels of derived statements, using a multi value `IN` predicate", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT * FROM (SELECT * FROM (SELECT music.id FROM music WHERE music.user_id IN (5, 6) LIMIT 10) subquery_for_limit) subquery_for_limit)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT * FROM (SELECT * FROM (SELECT music.id FROM music WHERE music.user_id IN (5, 6) LIMIT 10) subquery_for_limit) subquery_for_limit)", @@ -4959,6 +5045,7 @@ { "comment": "Unmergeable subquery with multiple levels of derived statements", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT * FROM (SELECT * FROM (SELECT music.id FROM music LIMIT 10) subquery_for_limit) subquery_for_limit)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT * FROM (SELECT * FROM (SELECT music.id FROM music LIMIT 10) subquery_for_limit) subquery_for_limit)", @@ -5183,6 +5270,7 @@ { "comment": "limit on the vtgate has to be executed on the LHS of a join", "query": "select id from user join (select user_id from user_extra limit 10) ue on user.id = ue.user_id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select id from user join (select user_id from user_extra limit 10) ue on user.id = ue.user_id", @@ -5238,6 +5326,7 @@ { "comment": "select user.a, t.b from user join (select id, count(*) b, req from user_extra group by req, id) as t on user.id = t.id", "query": "select user.a, t.b from user join (select id, count(*) b, req from user_extra group by req, id) as t on user.id = t.id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.a, t.b from user join (select id, count(*) b, req from user_extra group by req, id) as t on user.id = t.id", @@ -5402,6 +5491,7 @@ { "comment": "query with a derived table and dual table in unsharded keyspace", "query": "SELECT * FROM unsharded_a AS t1 JOIN (SELECT trim((SELECT MAX(name) FROM unsharded_a)) AS name) AS t2 WHERE t1.name >= t2.name ORDER BY t1.name ASC LIMIT 1;", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT * FROM unsharded_a AS t1 JOIN (SELECT trim((SELECT MAX(name) FROM unsharded_a)) AS name) AS t2 WHERE t1.name >= t2.name ORDER BY t1.name ASC LIMIT 1;", @@ -5451,6 +5541,7 @@ { "comment": "SOME modifier on unsharded table works well", "query": "select 1 from unsharded where foo = SOME (select 1 from unsharded_a where foo = 1)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from unsharded where foo = SOME (select 1 from unsharded_a where foo = 1)", @@ -5474,6 +5565,7 @@ { "comment": "ALL modifier on unsharded table works well", "query": "select 1 from unsharded where foo = ALL (select 1 from unsharded_a where foo = 1)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from unsharded where foo = ALL (select 1 from unsharded_a where foo = 1)", @@ -5519,6 +5611,7 @@ { "comment": "merge subquery using MAX and join into single route", "query": "select 1 from user join music_extra on user.id = music_extra.user_id where music_extra.music_id = (select max(music_id) from music_extra where user_id = user.id)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from user join music_extra on user.id = music_extra.user_id where music_extra.music_id = (select max(music_id) from music_extra where user_id = user.id)", @@ -5542,6 +5635,7 @@ { "comment": "Query with non-plannable lookup vindex", "query": "SELECT * FROM user_metadata WHERE user_metadata.non_planable = 'foo'", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT * FROM user_metadata WHERE user_metadata.non_planable = 'foo'", @@ -5568,6 +5662,7 @@ { "comment": "join query with lookup and join on different vindex column", "query": "select u.id from user u, user_metadata um where u.name = 'foo' and u.id = um.user_id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select u.id from user u, user_metadata um where u.name = 'foo' and u.id = um.user_id", @@ -5693,6 +5788,7 @@ { "comment": "name is in backfill vindex - not selected for vindex lookup", "query": "select * from customer where name = 'x'", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from customer where name = 'x'", @@ -5817,6 +5913,7 @@ { "comment": "invisible column is not expanded, but valid in predicate", "query": "select * from samecolvin where secret = 12", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from samecolvin where secret = 12", @@ -5839,6 +5936,7 @@ { "comment": "column with qualifier is correctly used", "query": "select u.foo, ue.foo as apa from user u, user_extra ue order by foo ", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select u.foo, ue.foo as apa from user u, user_extra ue order by foo ", @@ -5882,6 +5980,7 @@ { "comment": "Derived tables going to a single shard still need to expand derived table columns", "query": "SELECT c.column_name FROM user c JOIN (SELECT table_name FROM unsharded LIMIT 1) AS tables ON tables.table_name = c.table_name", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT c.column_name FROM user c JOIN (SELECT table_name FROM unsharded LIMIT 1) AS tables ON tables.table_name = c.table_name", @@ -5927,6 +6026,7 @@ { "comment": "column name aliases in outer join queries", "query": "select name as t0, name as t1 from user left outer join user_extra on user.cola = user_extra.cola", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select name as t0, name as t1 from user left outer join user_extra on user.cola = user_extra.cola", @@ -5982,6 +6082,7 @@ { "comment": "Over clause works for unsharded tables", "query": "SELECT val, CUME_DIST() OVER w, ROW_NUMBER() OVER w, DENSE_RANK() OVER w, PERCENT_RANK() OVER w, RANK() OVER w AS 'cd' FROM unsharded_a", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT val, CUME_DIST() OVER w, ROW_NUMBER() OVER w, DENSE_RANK() OVER w, PERCENT_RANK() OVER w, RANK() OVER w AS 'cd' FROM unsharded_a", From cc9d853daa573da272aca0da40f112416962a618 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Sat, 7 Dec 2024 11:43:58 +0530 Subject: [PATCH 11/12] added a todo comment Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/plan_description.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/vt/vtgate/engine/plan_description.go b/go/vt/vtgate/engine/plan_description.go index 59c60c39dc7..e8e763c1ee1 100644 --- a/go/vt/vtgate/engine/plan_description.go +++ b/go/vt/vtgate/engine/plan_description.go @@ -200,6 +200,7 @@ func (pd PrimitiveDescription) Equals(other PrimitiveDescription) string { return fmt.Sprintf("OperatorType: %v != %v", pd.OperatorType, other.OperatorType) } + // TODO (harshit): enable this to compare keyspace as well // switch { // case pd.Keyspace == nil && other.Keyspace == nil: // // do nothing From c476d58c6edb296ccaa788b7cea416f09cac4756 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 9 Dec 2024 12:00:19 +0530 Subject: [PATCH 12/12] codegen Signed-off-by: Harshit Gangal --- go/vt/vtgate/vindexes/cached_size.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/go/vt/vtgate/vindexes/cached_size.go b/go/vt/vtgate/vindexes/cached_size.go index a97411a6ac8..eeadb69b532 100644 --- a/go/vt/vtgate/vindexes/cached_size.go +++ b/go/vt/vtgate/vindexes/cached_size.go @@ -175,6 +175,18 @@ func (cached *Keyspace) CachedSize(alloc bool) int64 { size += hack.RuntimeAllocSize(int64(len(cached.Name))) return size } +func (cached *LookupCost) CachedSize(alloc bool) int64 { + if cached == nil { + return int64(0) + } + size := int64(0) + if alloc { + size += int64(16) + } + // field LookupNonUnique *vitess.io/vitess/go/vt/vtgate/vindexes.LookupNonUnique + size += cached.LookupNonUnique.CachedSize(true) + return size +} func (cached *LookupHash) CachedSize(alloc bool) int64 { if cached == nil { return int64(0)