From 2c2b8a48ee4ce83bc75592e04ddd7e9a88b1d3ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Taylor?= Date: Tue, 10 Dec 2024 06:38:26 +0100 Subject: [PATCH] Run plan tests in end to end configuration (#17117) Signed-off-by: Andres Taylor Signed-off-by: Florent Poinsard Signed-off-by: Harshit Gangal Co-authored-by: Florent Poinsard Co-authored-by: Harshit Gangal --- .../cluster_endtoend_vtgate_plantests.yml | 166 ++++++++ go/test/endtoend/utils/cmp.go | 12 + go/test/endtoend/utils/utils.go | 14 + .../endtoend/vtgate/plan_tests/main_test.go | 232 +++++++++++ .../vtgate/plan_tests/plan_e2e_test.go | 42 ++ go/vt/vtgate/engine/plan_description.go | 127 ++++++ go/vt/vtgate/planbuilder/plan_test.go | 21 +- go/vt/vtgate/planbuilder/test_helper.go | 27 ++ .../planbuilder/testdata/aggr_cases.json | 70 +++- .../planbuilder/testdata/filter_cases.json | 140 ++++++- .../planbuilder/testdata/from_cases.json | 35 +- .../testdata/memory_sort_cases.json | 70 +++- .../testdata/postprocess_cases.json | 140 ++++++- .../planbuilder/testdata/schemas/main.sql | 12 + .../planbuilder/testdata/schemas/user.sql | 100 +++++ .../planbuilder/testdata/select_cases.json | 387 ++++++++++++++++-- .../planbuilder/testdata/union_cases.json | 70 +++- .../planbuilder/testdata/vschemas/schema.json | 115 ++++-- go/vt/vtgate/vindexes/cached_size.go | 12 + go/vt/vtgate/vindexes/lookup_cost.go | 70 ++++ go/vt/vtgate/vindexes/vschema_test.go | 15 + test/ci_workflow_gen.go | 1 + test/config.json | 9 + 23 files changed, 1726 insertions(+), 161 deletions(-) create mode 100644 .github/workflows/cluster_endtoend_vtgate_plantests.yml create mode 100644 go/test/endtoend/vtgate/plan_tests/main_test.go create mode 100644 go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go create mode 100644 go/vt/vtgate/planbuilder/test_helper.go create mode 100644 go/vt/vtgate/planbuilder/testdata/schemas/main.sql create mode 100644 go/vt/vtgate/planbuilder/testdata/schemas/user.sql create mode 100644 go/vt/vtgate/vindexes/lookup_cost.go diff --git a/.github/workflows/cluster_endtoend_vtgate_plantests.yml b/.github/workflows/cluster_endtoend_vtgate_plantests.yml new file mode 100644 index 00000000000..93ed6a55f05 --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_plantests.yml @@ -0,0 +1,166 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_plantests) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vtgate_plantests)') + cancel-in-progress: true + +permissions: read-all + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (vtgate_plantests) + runs-on: ubuntu-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl -s\ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'go/vt/sidecardb/**/*.sql' + - 'go/test/endtoend/onlineddl/vrepl_suite/**' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_vtgate_plantests.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + + # Install everything else we need, and configure + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --no-commit-collection --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -exo pipefail + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vtgate_plantests | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable if PR is not a draft + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + fi + + # print test output + cat output.txt + + - name: Test Summary + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 + with: + paths: "report.xml" + show: "fail" diff --git a/go/test/endtoend/utils/cmp.go b/go/test/endtoend/utils/cmp.go index 3b47e1f68dc..7d94c181abd 100644 --- a/go/test/endtoend/utils/cmp.go +++ b/go/test/endtoend/utils/cmp.go @@ -215,6 +215,18 @@ func (mcmp *MySQLCompare) Exec(query string) *sqltypes.Result { return vtQr } +// ExecAssert is the same as Exec, but it only does assertions, it won't FailNow +func (mcmp *MySQLCompare) ExecAssert(query string) *sqltypes.Result { + mcmp.t.Helper() + vtQr, err := mcmp.VtConn.ExecuteFetch(query, 1000, true) + assert.NoError(mcmp.t, err, "[Vitess Error] for query: "+query) + + mysqlQr, err := mcmp.MySQLConn.ExecuteFetch(query, 1000, true) + assert.NoError(mcmp.t, err, "[MySQL Error] for query: "+query) + compareVitessAndMySQLResults(mcmp.t, query, mcmp.VtConn, vtQr, mysqlQr, CompareOptions{}) + return vtQr +} + // ExecNoCompare executes the query on vitess and mysql but does not compare the result with each other. func (mcmp *MySQLCompare) ExecNoCompare(query string) (*sqltypes.Result, *sqltypes.Result) { mcmp.t.Helper() diff --git a/go/test/endtoend/utils/utils.go b/go/test/endtoend/utils/utils.go index 35404981164..baa82821306 100644 --- a/go/test/endtoend/utils/utils.go +++ b/go/test/endtoend/utils/utils.go @@ -32,6 +32,7 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/vt/vtgate/engine" ) // AssertContains ensures the given query result contains the expected results. @@ -160,6 +161,19 @@ func Exec(t testing.TB, conn *mysql.Conn, query string) *sqltypes.Result { return qr } +// ExecTrace executes the given query with trace using the given connection. The trace result is returned. +// The test fails if the query produces an error. +func ExecTrace(t testing.TB, conn *mysql.Conn, query string) engine.PrimitiveDescription { + t.Helper() + qr, err := conn.ExecuteFetch(fmt.Sprintf("vexplain trace %s", query), 10000, false) + require.NoError(t, err, "for query: "+query) + + // Extract the trace result and format it with indentation for pretty printing + pd, err := engine.PrimitiveDescriptionFromString(qr.Rows[0][0].ToString()) + require.NoError(t, err) + return pd +} + // ExecMulti executes the given (potential multi) queries using the given connection. // The test fails if any of the queries produces an error func ExecMulti(t testing.TB, conn *mysql.Conn, query string) error { diff --git a/go/test/endtoend/vtgate/plan_tests/main_test.go b/go/test/endtoend/vtgate/plan_tests/main_test.go new file mode 100644 index 00000000000..59f95247bfb --- /dev/null +++ b/go/test/endtoend/vtgate/plan_tests/main_test.go @@ -0,0 +1,232 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan_tests + +import ( + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/utils" + "vitess.io/vitess/go/vt/vtgate/engine" + "vitess.io/vitess/go/vt/vtgate/planbuilder" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + mysqlParams mysql.ConnParams + uks = "main" + sks = "user" + cell = "plantests" +) + +func TestMain(m *testing.M) { + defer cluster.PanicHandler(nil) + + vschema := readFile("vschemas/schema.json") + userVs := extractUserKS(vschema) + mainVs := extractMainKS(vschema) + sSQL := readFile("schemas/user.sql") + uSQL := readFile("schemas/main.sql") + + exitCode := func() int { + clusterInstance = cluster.NewCluster(cell, "localhost") + defer clusterInstance.Teardown() + + // Start topo server + err := clusterInstance.StartTopo() + if err != nil { + fmt.Println(err.Error()) + return 1 + } + + // Start unsharded keyspace + uKeyspace := &cluster.Keyspace{ + Name: uks, + SchemaSQL: uSQL, + VSchema: mainVs, + } + err = clusterInstance.StartUnshardedKeyspace(*uKeyspace, 0, false) + if err != nil { + fmt.Println(err.Error()) + return 1 + } + + // Start sharded keyspace + skeyspace := &cluster.Keyspace{ + Name: sks, + SchemaSQL: sSQL, + VSchema: userVs, + } + err = clusterInstance.StartKeyspace(*skeyspace, []string{"-80", "80-"}, 0, false) + if err != nil { + fmt.Println(err.Error()) + return 1 + } + + // TODO: (@GuptaManan100/@systay): Also run the tests with normalizer on. + clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs, + "--normalize_queries=false", + "--schema_change_signal=false", + ) + + // Start vtgate + err = clusterInstance.StartVtgate() + if err != nil { + fmt.Println(err.Error()) + return 1 + } + + vtParams = clusterInstance.GetVTParams(sks) + + // create mysql instance and connection parameters + conn, closer, err := utils.NewMySQL(clusterInstance, sks, sSQL, uSQL) + if err != nil { + fmt.Println(err.Error()) + return 1 + } + defer closer() + mysqlParams = conn + + return m.Run() + }() + os.Exit(exitCode) +} + +func readFile(filename string) string { + schema, err := os.ReadFile(locateFile(filename)) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + return string(schema) +} + +func start(t *testing.T) (utils.MySQLCompare, func()) { + mcmp, err := utils.NewMySQLCompare(t, vtParams, mysqlParams) + require.NoError(t, err) + return mcmp, func() { + mcmp.Close() + } +} + +func readJSONTests(filename string) []planbuilder.PlanTest { + var output []planbuilder.PlanTest + file, err := os.Open(locateFile(filename)) + if err != nil { + panic(err) + } + defer file.Close() + dec := json.NewDecoder(file) + err = dec.Decode(&output) + if err != nil { + panic(err) + } + return output +} + +func locateFile(name string) string { + return "../../../../vt/vtgate/planbuilder/testdata/" + name +} + +// verifyTestExpectations verifies the expectations of the test. +func verifyTestExpectations(t *testing.T, pd engine.PrimitiveDescription, test planbuilder.PlanTest) { + // 1. Verify that the Join primitive sees atleast 1 row on the left side. + engine.WalkPrimitiveDescription(pd, func(description engine.PrimitiveDescription) { + if description.OperatorType == "Join" { + require.NotZero(t, description.Inputs[0].RowsReceived[0]) + } + }) + + // 2. Verify that the plan description matches the expected plan description. + planBytes, err := test.Plan.MarshalJSON() + require.NoError(t, err) + mp := make(map[string]any) + err = json.Unmarshal(planBytes, &mp) + require.NoError(t, err) + pdExpected, err := engine.PrimitiveDescriptionFromMap(mp["Instructions"].(map[string]any)) + require.NoError(t, err) + require.Empty(t, pdExpected.Equals(pd), "Expected: %v\nGot: %v", string(planBytes), pd) +} + +func extractUserKS(jsonString string) string { + var result map[string]any + if err := json.Unmarshal([]byte(jsonString), &result); err != nil { + panic(err.Error()) + } + + keyspaces, ok := result["keyspaces"].(map[string]any) + if !ok { + panic("Keyspaces not found") + } + + user, ok := keyspaces["user"].(map[string]any) + if !ok { + panic("User keyspace not found") + } + + tables, ok := user["tables"].(map[string]any) + if !ok { + panic("Tables not found") + } + + userTbl, ok := tables["user"].(map[string]any) + if !ok { + panic("User table not found") + } + + delete(userTbl, "auto_increment") // TODO: we should have an unsharded keyspace where this could live + + // Marshal the inner part back to JSON string + userJson, err := json.Marshal(user) + if err != nil { + panic(err.Error()) + } + + return string(userJson) +} + +func extractMainKS(jsonString string) string { + var result map[string]any + if err := json.Unmarshal([]byte(jsonString), &result); err != nil { + panic(err.Error()) + } + + keyspaces, ok := result["keyspaces"].(map[string]any) + if !ok { + panic("Keyspaces not found") + } + + main, ok := keyspaces["main"].(map[string]any) + if !ok { + panic("main keyspace not found") + } + + // Marshal the inner part back to JSON string + mainJson, err := json.Marshal(main) + if err != nil { + panic(err.Error()) + } + + return string(mainJson) +} diff --git a/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go b/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go new file mode 100644 index 00000000000..1594e9b392c --- /dev/null +++ b/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go @@ -0,0 +1,42 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan_tests + +import ( + "testing" + + "vitess.io/vitess/go/test/endtoend/utils" +) + +func TestSelectCases(t *testing.T) { + mcmp, closer := start(t) + defer closer() + tests := readJSONTests("select_cases.json") + for _, test := range tests { + mcmp.Run(test.Comment, func(mcmp *utils.MySQLCompare) { + if test.SkipE2E { + mcmp.AsT().Skip(test.Query) + } + mcmp.Exec(test.Query) + pd := utils.ExecTrace(mcmp.AsT(), mcmp.VtConn, test.Query) + verifyTestExpectations(mcmp.AsT(), pd, test) + if mcmp.VtConn.IsClosed() { + mcmp.AsT().Fatal("vtgate connection is closed") + } + }) + } +} diff --git a/go/vt/vtgate/engine/plan_description.go b/go/vt/vtgate/engine/plan_description.go index dfcad4e5e6b..e8e763c1ee1 100644 --- a/go/vt/vtgate/engine/plan_description.go +++ b/go/vt/vtgate/engine/plan_description.go @@ -126,6 +126,133 @@ func (pd PrimitiveDescription) MarshalJSON() ([]byte, error) { return buf.Bytes(), nil } +// PrimitiveDescriptionFromString creates primitive description out of a data string. +func PrimitiveDescriptionFromString(data string) (pd PrimitiveDescription, err error) { + resultMap := make(map[string]any) + err = json.Unmarshal([]byte(data), &resultMap) + if err != nil { + return PrimitiveDescription{}, err + } + return PrimitiveDescriptionFromMap(resultMap) +} + +// PrimitiveDescriptionFromMap populates the fields of a PrimitiveDescription from a map representation. +func PrimitiveDescriptionFromMap(data map[string]any) (pd PrimitiveDescription, err error) { + if opType, isPresent := data["OperatorType"]; isPresent { + pd.OperatorType = opType.(string) + } + if variant, isPresent := data["Variant"]; isPresent { + pd.Variant = variant.(string) + } + if ksMap, isPresent := data["Keyspace"]; isPresent { + ksMap := ksMap.(map[string]any) + pd.Keyspace = &vindexes.Keyspace{ + Name: ksMap["Name"].(string), + Sharded: ksMap["Sharded"].(bool), + } + } + if ttt, isPresent := data["TargetTabletType"]; isPresent { + pd.TargetTabletType = topodatapb.TabletType(ttt.(int)) + } + if other, isPresent := data["Other"]; isPresent { + pd.Other = other.(map[string]any) + } + if inpName, isPresent := data["InputName"]; isPresent { + pd.InputName = inpName.(string) + } + if avgRows, isPresent := data["AvgNumberOfRows"]; isPresent { + pd.RowsReceived = RowsReceived{ + int(avgRows.(float64)), + } + } + if sq, isPresent := data["ShardsQueried"]; isPresent { + sq := int(sq.(float64)) + pd.ShardsQueried = (*ShardsQueried)(&sq) + } + if inputs, isPresent := data["Inputs"]; isPresent { + inputs := inputs.([]any) + for _, input := range inputs { + inputMap := input.(map[string]any) + inp, err := PrimitiveDescriptionFromMap(inputMap) + if err != nil { + return PrimitiveDescription{}, err + } + pd.Inputs = append(pd.Inputs, inp) + } + } + return pd, nil +} + +// WalkPrimitiveDescription walks the primitive description. +func WalkPrimitiveDescription(pd PrimitiveDescription, f func(PrimitiveDescription)) { + f(pd) + for _, child := range pd.Inputs { + WalkPrimitiveDescription(child, f) + } +} + +func (pd PrimitiveDescription) Equals(other PrimitiveDescription) string { + if pd.Variant != other.Variant { + return fmt.Sprintf("Variant: %v != %v", pd.Variant, other.Variant) + } + + if pd.OperatorType != other.OperatorType { + return fmt.Sprintf("OperatorType: %v != %v", pd.OperatorType, other.OperatorType) + } + + // TODO (harshit): enable this to compare keyspace as well + // switch { + // case pd.Keyspace == nil && other.Keyspace == nil: + // // do nothing + // case pd.Keyspace != nil && other.Keyspace != nil: + // if pd.Keyspace.Name != other.Keyspace.Name { + // return fmt.Sprintf("Keyspace.Name: %v != %v", pd.Keyspace.Name, other.Keyspace.Name) + // } + // default: + // return "Keyspace is nil in one of the descriptions" + // } + + switch { + case pd.TargetDestination == nil && other.TargetDestination == nil: + // do nothing + case pd.TargetDestination != nil && other.TargetDestination != nil: + if pd.TargetDestination.String() != other.TargetDestination.String() { + return fmt.Sprintf("TargetDestination: %v != %v", pd.TargetDestination, other.TargetDestination) + } + default: + return "TargetDestination is nil in one of the descriptions" + } + + if pd.TargetTabletType != other.TargetTabletType { + return fmt.Sprintf("TargetTabletType: %v != %v", pd.TargetTabletType, other.TargetTabletType) + } + + switch { + case pd.Other == nil && other.Other == nil: + // do nothing + case pd.Other != nil && other.Other != nil: + if len(pd.Other) != len(other.Other) { + return fmt.Sprintf("Other length did not match: %v != %v", pd.Other, other.Other) + } + for ky, val := range pd.Other { + if other.Other[ky] != val { + return fmt.Sprintf("Other[%v]: %v != %v", ky, val, other.Other[ky]) + } + } + default: + return "Other is nil in one of the descriptions" + } + if len(pd.Inputs) != len(other.Inputs) { + return fmt.Sprintf("Inputs length did not match: %v != %v", len(pd.Inputs), len(other.Inputs)) + } + for idx, input := range pd.Inputs { + if diff := input.Equals(other.Inputs[idx]); diff != "" { + return diff + } + } + return "" +} + func average(nums []int) float64 { total := 0 for _, num := range nums { diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index ccbc9821170..89f0b5ed01b 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -649,21 +649,12 @@ func createFkDefinition(childCols []string, parentTableName string, parentCols [ } } -type ( - planTest struct { - Comment string `json:"comment,omitempty"` - Query string `json:"query,omitempty"` - Plan json.RawMessage `json:"plan,omitempty"` - Skip bool `json:"skip,omitempty"` - } -) - func (s *planTestSuite) testFile(filename string, vschema *vschemawrapper.VSchemaWrapper, render bool) { opts := jsondiff.DefaultConsoleOptions() s.T().Run(filename, func(t *testing.T) { failed := false - var expected []planTest + var expected []PlanTest for _, tcase := range readJSONTests(filename) { testName := tcase.Comment if testName == "" { @@ -672,7 +663,7 @@ func (s *planTestSuite) testFile(filename string, vschema *vschemawrapper.VSchem if tcase.Query == "" { continue } - current := planTest{ + current := PlanTest{ Comment: testName, Query: tcase.Query, } @@ -720,8 +711,8 @@ func (s *planTestSuite) testFile(filename string, vschema *vschemawrapper.VSchem }) } -func readJSONTests(filename string) []planTest { - var output []planTest +func readJSONTests(filename string) []PlanTest { + var output []PlanTest file, err := os.Open(locateFile(filename)) if err != nil { panic(err) @@ -735,7 +726,7 @@ func readJSONTests(filename string) []planTest { return output } -func getPlanOutput(tcase planTest, vschema *vschemawrapper.VSchemaWrapper, render bool) (out string) { +func getPlanOutput(tcase PlanTest, vschema *vschemawrapper.VSchemaWrapper, render bool) (out string) { defer func() { if r := recover(); r != nil { out = fmt.Sprintf("panicked: %v\n%s", r, string(debug.Stack())) @@ -867,7 +858,7 @@ func BenchmarkBaselineVsMirrored(b *testing.B) { }) } -func benchmarkPlanner(b *testing.B, version plancontext.PlannerVersion, testCases []planTest, vschema *vschemawrapper.VSchemaWrapper) { +func benchmarkPlanner(b *testing.B, version plancontext.PlannerVersion, testCases []PlanTest, vschema *vschemawrapper.VSchemaWrapper) { b.ReportAllocs() for n := 0; n < b.N; n++ { for _, tcase := range testCases { diff --git a/go/vt/vtgate/planbuilder/test_helper.go b/go/vt/vtgate/planbuilder/test_helper.go new file mode 100644 index 00000000000..25d6b7306d1 --- /dev/null +++ b/go/vt/vtgate/planbuilder/test_helper.go @@ -0,0 +1,27 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package planbuilder + +import "encoding/json" + +type PlanTest struct { + Comment string `json:"comment,omitempty"` + Query string `json:"query,omitempty"` + Plan json.RawMessage `json:"plan,omitempty"` + Skip bool `json:"skip,omitempty"` + SkipE2E bool `json:"skip_e2e,omitempty"` +} diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index 8b268e367dd..49a03a8f05a 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -940,19 +940,44 @@ "Table": "`user`, user_extra" }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.`name` from music where 1 != 1", - "Query": "select music.`name` from music where music.id = :user_id", - "Table": "music", "Values": [ ":user_id" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.`name` from music where 1 != 1", + "Query": "select music.`name` from music where music.id = :user_id", + "Table": "music" + } + ] } ] } @@ -2992,19 +3017,44 @@ ] }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from music as m where 1 != 1", - "Query": "select 1 from music as m where m.id = :u2_val2", - "Table": "music", "Values": [ ":u2_val2" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from music as m where 1 != 1", + "Query": "select 1 from music as m where m.id = :u2_val2", + "Table": "music" + } + ] } ] } diff --git a/go/vt/vtgate/planbuilder/testdata/filter_cases.json b/go/vt/vtgate/planbuilder/testdata/filter_cases.json index 4194a369bd6..edce4ebd0cb 100644 --- a/go/vt/vtgate/planbuilder/testdata/filter_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/filter_cases.json @@ -3404,19 +3404,44 @@ "QueryType": "SELECT", "Original": "select * from multicolvin where column_b = 1", "Instructions": { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select * from multicolvin where 1 != 1", - "Query": "select * from multicolvin where column_b = 1", - "Table": "multicolvin", "Values": [ "1" ], - "Vindex": "colb_colc_map" + "Vindex": "colb_colc_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select colb, keyspace_id from colb_colc_map where 1 != 1", + "Query": "select colb, keyspace_id from colb_colc_map where colb in ::__vals", + "Table": "colb_colc_map", + "Values": [ + "::colb" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select * from multicolvin where 1 != 1", + "Query": "select * from multicolvin where column_b = 1", + "Table": "multicolvin" + } + ] }, "TablesUsed": [ "user.multicolvin" @@ -3430,19 +3455,44 @@ "QueryType": "SELECT", "Original": "select * from multicolvin where column_b = 1 and column_c = 2", "Instructions": { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select * from multicolvin where 1 != 1", - "Query": "select * from multicolvin where column_b = 1 and column_c = 2", - "Table": "multicolvin", "Values": [ "1" ], - "Vindex": "colb_colc_map" + "Vindex": "colb_colc_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select colb, keyspace_id from colb_colc_map where 1 != 1", + "Query": "select colb, keyspace_id from colb_colc_map where colb in ::__vals", + "Table": "colb_colc_map", + "Values": [ + "::colb" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select * from multicolvin where 1 != 1", + "Query": "select * from multicolvin where column_b = 1 and column_c = 2", + "Table": "multicolvin" + } + ] }, "TablesUsed": [ "user.multicolvin" @@ -3456,19 +3506,44 @@ "QueryType": "SELECT", "Original": "select * from multicolvin where column_b = 1 and column_c = 2 and column_a = 3", "Instructions": { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select * from multicolvin where 1 != 1", - "Query": "select * from multicolvin where column_b = 1 and column_c = 2 and column_a = 3", - "Table": "multicolvin", "Values": [ "1" ], - "Vindex": "colb_colc_map" + "Vindex": "colb_colc_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select colb, keyspace_id from colb_colc_map where 1 != 1", + "Query": "select colb, keyspace_id from colb_colc_map where colb in ::__vals", + "Table": "colb_colc_map", + "Values": [ + "::colb" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select * from multicolvin where 1 != 1", + "Query": "select * from multicolvin where column_b = 1 and column_c = 2 and column_a = 3", + "Table": "multicolvin" + } + ] }, "TablesUsed": [ "user.multicolvin" @@ -3482,19 +3557,44 @@ "QueryType": "SELECT", "Original": "select * from multicolvin where column_a = 3 and column_b = 1", "Instructions": { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select * from multicolvin where 1 != 1", - "Query": "select * from multicolvin where column_a = 3 and column_b = 1", - "Table": "multicolvin", "Values": [ "1" ], - "Vindex": "colb_colc_map" + "Vindex": "colb_colc_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select colb, keyspace_id from colb_colc_map where 1 != 1", + "Query": "select colb, keyspace_id from colb_colc_map where colb in ::__vals", + "Table": "colb_colc_map", + "Values": [ + "::colb" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select * from multicolvin where 1 != 1", + "Query": "select * from multicolvin where column_a = 3 and column_b = 1", + "Table": "multicolvin" + } + ] }, "TablesUsed": [ "user.multicolvin" diff --git a/go/vt/vtgate/planbuilder/testdata/from_cases.json b/go/vt/vtgate/planbuilder/testdata/from_cases.json index 2e0fe429c1f..bec64fd7b1e 100644 --- a/go/vt/vtgate/planbuilder/testdata/from_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/from_cases.json @@ -4709,19 +4709,44 @@ ] }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from music as m where 1 != 1", - "Query": "select 1 from music as m where m.user_id = 5 and m.id = 20 and m.col = :u_col /* INT16 */", - "Table": "music", "Values": [ "20" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from music as m where 1 != 1", + "Query": "select 1 from music as m where m.user_id = 5 and m.id = 20 and m.col = :u_col /* INT16 */", + "Table": "music" + } + ] } ] }, diff --git a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json index 060f073a366..a35949cd4c1 100644 --- a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json @@ -318,19 +318,44 @@ "Vindex": "user_index" }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3 as c, weight_string(music.col3) from music where 1 != 1", - "Query": "select music.col3 as c, weight_string(music.col3) from music where music.id = :user_id", - "Table": "music", "Values": [ ":user_id" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.col3 as c, weight_string(music.col3) from music where 1 != 1", + "Query": "select music.col3 as c, weight_string(music.col3) from music where music.id = :user_id", + "Table": "music" + } + ] } ] } @@ -379,19 +404,44 @@ "Vindex": "user_index" }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3, weight_string(music.col3) from music where 1 != 1", - "Query": "select music.col3, weight_string(music.col3) from music where music.id = :user_id", - "Table": "music", "Values": [ ":user_id" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.col3, weight_string(music.col3) from music where 1 != 1", + "Query": "select music.col3, weight_string(music.col3) from music where music.id = :user_id", + "Table": "music" + } + ] } ] } diff --git a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json index 36f1472007d..6a8e94c0241 100644 --- a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json @@ -544,19 +544,44 @@ "Vindex": "user_index" }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3 from music where 1 != 1", - "Query": "select music.col3 from music where music.id = :user_id", - "Table": "music", "Values": [ ":user_id" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.col3 from music where 1 != 1", + "Query": "select music.col3 from music where music.id = :user_id", + "Table": "music" + } + ] } ] }, @@ -597,19 +622,44 @@ "Vindex": "user_index" }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3 from music where 1 != 1", - "Query": "select music.col3 from music where music.id = :user_id", - "Table": "music", "Values": [ ":user_id" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.col3 from music where 1 != 1", + "Query": "select music.col3 from music where music.id = :user_id", + "Table": "music" + } + ] } ] }, @@ -650,19 +700,44 @@ "Vindex": "user_index" }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3 from music where 1 != 1", - "Query": "select music.col3 from music where music.id = :user_id", - "Table": "music", "Values": [ ":user_id" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.col3 from music where 1 != 1", + "Query": "select music.col3 from music where music.id = :user_id", + "Table": "music" + } + ] } ] }, @@ -770,19 +845,44 @@ "Vindex": "user_index" }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3 from music where 1 != 1", - "Query": "select music.col3 from music where music.id = :user_id", - "Table": "music", "Values": [ ":user_id" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.col3 from music where 1 != 1", + "Query": "select music.col3 from music where music.id = :user_id", + "Table": "music" + } + ] } ] }, diff --git a/go/vt/vtgate/planbuilder/testdata/schemas/main.sql b/go/vt/vtgate/planbuilder/testdata/schemas/main.sql new file mode 100644 index 00000000000..8c15b99218c --- /dev/null +++ b/go/vt/vtgate/planbuilder/testdata/schemas/main.sql @@ -0,0 +1,12 @@ +CREATE TABLE `unsharded` ( + `id` INT NOT NULL PRIMARY KEY, + `col1` VARCHAR(255) DEFAULT NULL, + `col2` VARCHAR(255) DEFAULT NULL, + `name` VARCHAR(255) DEFAULT NULL +); + +CREATE TABLE `unsharded_auto` ( + `id` INT NOT NULL PRIMARY KEY, + `col1` VARCHAR(255) DEFAULT NULL, + `col2` VARCHAR(255) DEFAULT NULL +); \ No newline at end of file diff --git a/go/vt/vtgate/planbuilder/testdata/schemas/user.sql b/go/vt/vtgate/planbuilder/testdata/schemas/user.sql new file mode 100644 index 00000000000..55f4078557a --- /dev/null +++ b/go/vt/vtgate/planbuilder/testdata/schemas/user.sql @@ -0,0 +1,100 @@ +CREATE TABLE user +( + id INT PRIMARY KEY, + col BIGINT, + predef1 VARCHAR(255), + predef2 VARCHAR(255), + textcol1 VARCHAR(255), + intcol BIGINT, + textcol2 VARCHAR(255) +); + +CREATE TABLE user_metadata +( + user_id INT, + email VARCHAR(255), + address VARCHAR(255), + md5 VARCHAR(255), + non_planable VARCHAR(255), + PRIMARY KEY (user_id) +); + +CREATE TABLE music +( + user_id INT, + id INT, + PRIMARY KEY (user_id) +); + +CREATE TABLE samecolvin +( + col VARCHAR(255), + PRIMARY KEY (col) +); + +CREATE TABLE multicolvin +( + kid INT, + column_a VARCHAR(255), + column_b VARCHAR(255), + column_c VARCHAR(255), + PRIMARY KEY (kid) +); + +CREATE TABLE customer +( + id INT, + email VARCHAR(255), + phone VARCHAR(255), + PRIMARY KEY (id) +); + +CREATE TABLE multicol_tbl +( + cola VARCHAR(255), + colb VARCHAR(255), + colc VARCHAR(255), + name VARCHAR(255), + PRIMARY KEY (cola, colb) +); + +CREATE TABLE mixed_tbl +( + shard_key VARCHAR(255), + lkp_key VARCHAR(255), + PRIMARY KEY (shard_key) +); + +CREATE TABLE pin_test +( + id INT PRIMARY KEY +); + +CREATE TABLE cfc_vindex_col +( + c1 VARCHAR(255), + c2 VARCHAR(255), + PRIMARY KEY (c1) +); + +CREATE TABLE unq_lkp_idx +( + unq_key INT PRIMARY KEY, + keyspace_id VARCHAR(255) +); + +CREATE TABLE t1 +( + c1 INT, + c2 INT, + c3 INT, + PRIMARY KEY (c1) +); + +CREATE TABLE authoritative +( + user_id bigint NOT NULL, + col1 VARCHAR(255), + col2 bigint, + PRIMARY KEY (user_id) +) ENGINE=InnoDB; \ No newline at end of file diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.json b/go/vt/vtgate/planbuilder/testdata/select_cases.json index ab69df2cc47..9b03571fc5c 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.json @@ -68,6 +68,7 @@ { "comment": "join on sharding column with limit - should be a simple scatter query and limit on top with non resolved columns", "query": "select * from user u join user_metadata um on u.id = um.user_id where foo=41 limit 20", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user u join user_metadata um on u.id = um.user_id where foo=41 limit 20", @@ -179,6 +180,7 @@ { "comment": "select limit with timeout directive sets QueryTimeout in the route", "query": "select /*vt+ QUERY_TIMEOUT_MS=1000 */ * from main.unsharded limit 10", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select /*vt+ QUERY_TIMEOUT_MS=1000 */ * from main.unsharded limit 10", @@ -402,6 +404,7 @@ { "comment": "test table lookup failure for authoritative code path", "query": "select a.* from authoritative", + "skip_e2e": true, "plan": "Unknown table 'a'" }, { @@ -452,6 +455,7 @@ { "comment": "select authoritative.* with intermixing still expands", "query": "select user.id, a.*, user.col1 from authoritative a join user on a.user_id=user.id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.id, a.*, user.col1 from authoritative a join user on a.user_id=user.id", @@ -475,6 +479,7 @@ { "comment": "auto-resolve anonymous columns for simple route", "query": "select anon_col from user join user_extra on user.id = user_extra.user_id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select anon_col from user join user_extra on user.id = user_extra.user_id", @@ -498,6 +503,7 @@ { "comment": "json_arrayagg in single sharded query", "query": "select count(1) from user where id = 'abc' group by n_id having json_arrayagg(a_id) = '[]'", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select count(1) from user where id = 'abc' group by n_id having json_arrayagg(a_id) = '[]'", @@ -524,6 +530,7 @@ { "comment": "json_objectagg in single sharded query", "query": "select count(1) from user where id = 'abc' group by n_id having json_objectagg(a_id, b_id) = '[]'", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select count(1) from user where id = 'abc' group by n_id having json_objectagg(a_id, b_id) = '[]'", @@ -550,16 +557,19 @@ { "comment": "unsupported json aggregation expressions in scatter query", "query": "select count(1) from user where cola = 'abc' group by n_id having json_arrayagg(a_id) = '[]'", + "skip_e2e": true, "plan": "VT12001: unsupported: in scatter query: aggregation function 'json_arrayagg(a_id)'" }, { "comment": "Cannot auto-resolve for cross-shard joins", "query": "select col from user join user_extra", + "skip_e2e": true, "plan": "Column 'col' in field list is ambiguous" }, { "comment": "Auto-resolve should work if unique vindex columns are referenced", "query": "select id, user_id from user join user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select id, user_id from user join user_extra", @@ -602,6 +612,7 @@ { "comment": "database calls should be substituted", "query": "select database() from dual", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select database() from dual", @@ -624,6 +635,7 @@ { "comment": "last_insert_id for unsharded route", "query": "select last_insert_id() as x from main.unsharded", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select last_insert_id() as x from main.unsharded", @@ -694,11 +706,13 @@ { "comment": "prefixing dual with a keyspace should not work", "query": "select 1 from user.dual", + "skip_e2e": true, "plan": "table dual not found" }, { "comment": "RHS route referenced", "query": "select user_extra.id from user join user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user_extra.id from user join user_extra", @@ -741,6 +755,7 @@ { "comment": "Both routes referenced", "query": "select user.col, user_extra.id from user join user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col, user_extra.id from user join user_extra", @@ -783,6 +798,7 @@ { "comment": "Expression with single-route reference", "query": "select user.col, user_extra.id + user_extra.col from user join user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col, user_extra.id + user_extra.col from user join user_extra", @@ -825,6 +841,7 @@ { "comment": "subquery with an aggregation in order by that can be merged into a single route", "query": "select col, trim((select user_name from user where id = 3)) val from user_extra where user_id = 3 group by col order by val", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select col, trim((select user_name from user where id = 3)) val from user_extra where user_id = 3 group by col order by val", @@ -852,6 +869,7 @@ { "comment": "Jumbled references", "query": "select user.col, user_extra.id, user.col2 from user join user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col, user_extra.id, user.col2 from user join user_extra", @@ -894,6 +912,7 @@ { "comment": "Comments", "query": "select /* comment */ user.col from user join user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select /* comment */ user.col from user join user_extra", @@ -936,6 +955,7 @@ { "comment": "for update", "query": "select user.col from user join user_extra for update", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col from user join user_extra for update", @@ -978,6 +998,7 @@ { "comment": "Field query should work for joins select bind vars", "query": "select user.id, (select user.id+outm.m+unsharded.m from unsharded) from user join unsharded outm", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.id, (select user.id+outm.m+unsharded.m from unsharded) from user join unsharded outm", @@ -1023,6 +1044,7 @@ { "comment": "Case preservation", "query": "select user.Col, user_extra.Id from user join user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.Col, user_extra.Id from user join user_extra", @@ -1065,6 +1087,7 @@ { "comment": "syntax error", "query": "the quick brown fox", + "skip_e2e": true, "plan": "syntax error at position 4 near 'the'" }, { @@ -1096,6 +1119,7 @@ { "comment": "Selection but explicitly ignore a vindex", "query": "select * from user ignore vindex (user_index) where id = 1", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user ignore vindex (user_index) where id = 1", @@ -1118,6 +1142,7 @@ { "comment": "Selection but make the planner explicitly use a vindex", "query": "select intcol, id from user use vindex (name_user_map) where costly = 'aa' and name = 'bb' and id = 3", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select intcol, id from user use vindex (name_user_map) where costly = 'aa' and name = 'bb' and id = 3", @@ -1169,6 +1194,7 @@ { "comment": "Vindex hint on a non-existing vindex", "query": "select * from user use vindex (does_not_exist) where id = 1", + "skip_e2e": true, "plan": "VT09021: Vindex 'does_not_exist' does not exist in table 'user.user'" }, { @@ -1205,6 +1231,7 @@ { "comment": "sharded limit offset with arguments", "query": "select user_id from music order by user_id limit :limit, :offset", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user_id from music order by user_id limit :limit, :offset", @@ -1236,6 +1263,7 @@ { "comment": "Sharding Key Condition in Parenthesis", "query": "select * from user where name ='abc' AND (id = 4) limit 5", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user where name ='abc' AND (id = 4) limit 5", @@ -1262,6 +1290,7 @@ { "comment": "Multiple parenthesized expressions", "query": "select * from user where (id = 4) AND (name ='abc') limit 5", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user where (id = 4) AND (name ='abc') limit 5", @@ -1288,6 +1317,7 @@ { "comment": "Multiple parenthesized expressions", "query": "select * from user where (id = 4 and name ='abc') limit 5", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user where (id = 4 and name ='abc') limit 5", @@ -1366,6 +1396,7 @@ { "comment": "Booleans and parenthesis", "query": "select * from user where (id = 1) AND name = true limit 5", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user where (id = 1) AND name = true limit 5", @@ -1392,6 +1423,7 @@ { "comment": "Column as boolean-ish", "query": "select * from user where (id = 1) AND name limit 5", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user where (id = 1) AND name limit 5", @@ -1418,6 +1450,7 @@ { "comment": "PK as fake boolean, and column as boolean-ish", "query": "select * from user where (id = 5) AND name = true limit 5", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user where (id = 5) AND name = true limit 5", @@ -1444,6 +1477,7 @@ { "comment": "top level subquery in select", "query": "select a, (select col from user) from unsharded", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select a, (select col from user) from unsharded", @@ -1489,6 +1523,7 @@ { "comment": "sub-expression subquery in select", "query": "select a, 1+(select col from user) from unsharded", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select a, 1+(select col from user) from unsharded", @@ -1534,6 +1569,7 @@ { "comment": "select * from derived table expands specific columns", "query": "select * from (select user.id id1, user_extra.id id2 from user join user_extra) as t", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from (select user.id id1, user_extra.id id2 from user join user_extra) as t", @@ -1576,16 +1612,19 @@ { "comment": "duplicate columns not allowed in derived table", "query": "select * from (select user.id, user_extra.id from user join user_extra) as t", + "skip_e2e": true, "plan": "Duplicate column name 'id'" }, { "comment": "non-existent symbol in cross-shard derived table", "query": "select t.col from (select user.id from user join user_extra) as t", + "skip_e2e": true, "plan": "column 't.col' not found" }, { "comment": "union with the same target shard", "query": "select * from music where user_id = 1 union select * from user where id = 1", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from music where user_id = 1 union select * from user where id = 1", @@ -1613,6 +1652,7 @@ { "comment": "union with the same target shard last_insert_id", "query": "select *, last_insert_id() from music where user_id = 1 union select * from user where id = 1", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select *, last_insert_id() from music where user_id = 1 union select * from user where id = 1", @@ -1775,6 +1815,7 @@ { "comment": "routing rules: ensure directives are not lost", "query": "select /*vt+ QUERY_TIMEOUT_MS=1000 */ * from route2", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select /*vt+ QUERY_TIMEOUT_MS=1000 */ * from route2", @@ -1798,6 +1839,7 @@ { "comment": "routing table on music", "query": "select * from second_user.bar where id > 2", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from second_user.bar where id > 2", @@ -1914,6 +1956,7 @@ { "comment": "Complex expression in a subquery used in IN clause of an aggregate query", "query": "select count(*) from user where user.id = 2 or user.id in (select id from unsharded_a where colb = 2)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select count(*) from user where user.id = 2 or user.id in (select id from unsharded_a where colb = 2)", @@ -1967,6 +2010,7 @@ { "comment": "Complex expression in a subquery used in NOT IN clause of an aggregate query", "query": "select count(*) from user where user.id = 2 or user.id not in (select id from unsharded_a where colb = 2)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select count(*) from user where user.id = 2 or user.id not in (select id from unsharded_a where colb = 2)", @@ -2218,16 +2262,19 @@ { "comment": "sql_calc_found_rows in sub queries", "query": "select * from music where user_id IN (select sql_calc_found_rows * from music limit 10)", + "skip_e2e": true, "plan": "Incorrect usage/placement of 'SQL_CALC_FOUND_ROWS'" }, { "comment": "sql_calc_found_rows in derived table", "query": "select sql_calc_found_rows * from (select sql_calc_found_rows * from music limit 10) t limit 1", + "skip_e2e": true, "plan": "Incorrect usage/placement of 'SQL_CALC_FOUND_ROWS'" }, { "comment": "select from unsharded keyspace into dumpfile", "query": "select * from main.unsharded into Dumpfile 'x.txt'", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from main.unsharded into Dumpfile 'x.txt'", @@ -2250,6 +2297,7 @@ { "comment": "select from unsharded keyspace into outfile", "query": "select * from main.unsharded into outfile 'x.txt' character set binary fields terminated by 'term' optionally enclosed by 'c' escaped by 'e' lines starting by 'a' terminated by '\n'", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from main.unsharded into outfile 'x.txt' character set binary fields terminated by 'term' optionally enclosed by 'c' escaped by 'e' lines starting by 'a' terminated by '\n'", @@ -2272,6 +2320,7 @@ { "comment": "select from unsharded keyspace into outfile s3", "query": "select * from main.unsharded into outfile s3 'out_file_name' character set binary format csv header fields terminated by 'term' optionally enclosed by 'c' escaped by 'e' lines starting by 'a' terminated by '\n' manifest on overwrite off", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from main.unsharded into outfile s3 'out_file_name' character set binary format csv header fields terminated by 'term' optionally enclosed by 'c' escaped by 'e' lines starting by 'a' terminated by '\n' manifest on overwrite off", @@ -2500,16 +2549,19 @@ { "comment": "Union after into outfile is incorrect", "query": "select id from user into outfile 'out_file_name' union all select id from music", + "skip_e2e": true, "plan": "syntax error at position 55 near 'union'" }, { "comment": "Into outfile s3 in derived table is incorrect", "query": "select id from (select id from user into outfile s3 'inner_outfile') as t2", + "skip_e2e": true, "plan": "syntax error at position 41 near 'into'" }, { "comment": "Into outfile s3 in derived table with union incorrect", "query": "select id from (select id from user into outfile s3 'inner_outfile' union select 1) as t2", + "skip_e2e": true, "plan": "syntax error at position 41 near 'into'" }, { @@ -2541,6 +2593,7 @@ { "comment": "Add two tables with the same column in a join", "query": "select t.id, s.id from user t join user_extra s on t.id = s.user_id join unsharded", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select t.id, s.id from user t join user_extra s on t.id = s.user_id join unsharded", @@ -2606,6 +2659,7 @@ { "comment": "Merging dual with user", "query": "select 42, id from dual, user", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 42, id from dual, user", @@ -2629,6 +2683,7 @@ { "comment": "subquery in select expression of derived table", "query": "select t.a from (select (select col from user limit 1) as a from user join user_extra) t", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select t.a from (select (select col from user limit 1) as a from user join user_extra) t", @@ -2699,6 +2754,7 @@ { "comment": "ORDER BY subquery", "query": "select (select col from user limit 1) as a from user join user_extra order by a", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select (select col from user limit 1) as a from user join user_extra order by a", @@ -2791,6 +2847,7 @@ { "comment": "select expression having dependencies on both sides of a join", "query": "select user.id * user_id as amount from user, user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.id * user_id as amount from user, user_extra", @@ -2836,6 +2893,7 @@ { "comment": "Straight Join ensures specific ordering of joins", "query": "select user.id, user_extra.user_id from user straight_join user_extra where user.id = user_extra.foo", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.id, user_extra.user_id from user straight_join user_extra where user.id = user_extra.foo", @@ -2903,6 +2961,7 @@ { "comment": "PullOut subquery with an aggregation that should be typed in the final output", "query": "select (select sum(col) from user) from user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select (select sum(col) from user) from user_extra", @@ -2955,6 +3014,7 @@ { "comment": "Straight Join preserved in MySQL query", "query": "select user.id, user_extra.user_id from user straight_join user_extra where user.id = user_extra.user_id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.id, user_extra.user_id from user straight_join user_extra where user.id = user_extra.user_id", @@ -2978,6 +3038,7 @@ { "comment": "correlated subquery in exists clause", "query": "select col from user where exists(select user_id from user_extra where user_id = 3 and user_id < user.id)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select col from user where exists(select user_id from user_extra where user_id = 3 and user_id < user.id)", @@ -3036,6 +3097,7 @@ { "comment": "correlated subquery in exists clause with an order by", "query": "select col from user where exists(select user_id from user_extra where user_id = 3 and user_id < user.id) order by col", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select col from user where exists(select user_id from user_extra where user_id = 3 and user_id < user.id) order by col", @@ -3095,6 +3157,7 @@ { "comment": "correlated subquery having dependencies on two tables", "query": "select 1 from user u1, user u2 where exists (select 1 from user_extra ue where ue.col = u1.col and ue.col = u2.col)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from user u1, user u2 where exists (select 1 from user_extra ue where ue.col = u1.col and ue.col = u2.col)", @@ -3168,6 +3231,7 @@ { "comment": "correlated subquery using a column twice", "query": "select 1 from user u where exists (select 1 from user_extra ue where ue.col = u.col and u.col = ue.col2)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from user u where exists (select 1 from user_extra ue where ue.col = u.col and u.col = ue.col2)", @@ -3252,6 +3316,7 @@ { "comment": "Complex join with multiple conditions merged into single route", "query": "select 0 from user as u join user_extra as s on u.id = s.user_id join music as m on m.user_id = u.id and (s.foo or m.bar)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 0 from user as u join user_extra as s on u.id = s.user_id join music as m on m.user_id = u.id and (s.foo or m.bar)", @@ -3315,6 +3380,7 @@ { "comment": "use output column containing data from both sides of the join", "query": "select user_extra.col + user.col from user join user_extra on user.id = user_extra.id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user_extra.col + user.col from user join user_extra on user.id = user_extra.id", @@ -3365,6 +3431,7 @@ { "comment": "mergeable derived table with order by and limit", "query": "select 1 from (select col from main.unsharded order by main.unsharded.col1 desc limit 12 offset 0) as f left join unsharded as u on f.col = u.id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from (select col from main.unsharded order by main.unsharded.col1 desc limit 12 offset 0) as f left join unsharded as u on f.col = u.id", @@ -3387,6 +3454,7 @@ { "comment": "mergeable derived table with group by and limit", "query": "select 1 from (select col, count(*) as a from main.unsharded group by col having a > 0 limit 12 offset 0) as f left join unsharded as u on f.col = u.id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from (select col, count(*) as a from main.unsharded group by col having a > 0 limit 12 offset 0) as f left join unsharded as u on f.col = u.id", @@ -3409,6 +3477,7 @@ { "comment": "select user.id, trim(leading 'x' from user.name) from user", "query": "select user.id, trim(leading 'x' from user.name) from user", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.id, trim(leading 'x' from user.name) from user", @@ -3431,6 +3500,7 @@ { "comment": "json utility functions", "query": "select jcol, JSON_STORAGE_SIZE(jcol), JSON_STORAGE_FREE(jcol), JSON_PRETTY(jcol) from user", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select jcol, JSON_STORAGE_SIZE(jcol), JSON_STORAGE_FREE(jcol), JSON_PRETTY(jcol) from user", @@ -3499,6 +3569,7 @@ { "comment": "select (select id from user order by id limit 1) from user_extra", "query": "select (select id from user order by id limit 1) from user_extra", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select (select id from user order by id limit 1) from user_extra", @@ -3622,6 +3693,7 @@ { "comment": "Json extract and json unquote shorthands", "query": "SELECT a->\"$[4]\", a->>\"$[3]\" FROM user", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT a->\"$[4]\", a->>\"$[3]\" FROM user", @@ -3644,6 +3716,7 @@ { "comment": "groupe by with non aggregated columns and table alias", "query": "select u.id, u.age from user u group by u.id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select u.id, u.age from user u group by u.id", @@ -3849,6 +3922,7 @@ { "comment": "insert function using column names as arguments", "query": "select insert(tcol1, id, 3, tcol2) from user", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select insert(tcol1, id, 3, tcol2) from user", @@ -3893,6 +3967,7 @@ { "comment": "Predicate in apply join which is merged", "query": "select user.col, user_metadata.user_id from user join user_extra on user.col = user_extra.col join user_metadata on user_extra.user_id = user_metadata.user_id where user.textcol1 = 'alice@gmail.com'", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col, user_metadata.user_id from user join user_extra on user.col = user_extra.col join user_metadata on user_extra.user_id = user_metadata.user_id where user.textcol1 = 'alice@gmail.com'", @@ -3939,6 +4014,7 @@ { "comment": "Join across multiple tables, with conditions on different vindexes, but mergeable through join predicates", "query": "SELECT user.id FROM user INNER JOIN music_extra ON user.id = music_extra.user_id INNER JOIN music ON music_extra.user_id = music.user_id WHERE user.id = 123 and music.id = 456", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT user.id FROM user INNER JOIN music_extra ON user.id = music_extra.user_id INNER JOIN music ON music_extra.user_id = music.user_id WHERE user.id = 123 and music.id = 456", @@ -3967,6 +4043,7 @@ { "comment": "SQL_CALC_FOUND_ROWS with vindex lookup", "query": "select SQL_CALC_FOUND_ROWS id, name from user where name = 'aa' order by id limit 2", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select SQL_CALC_FOUND_ROWS id, name from user where name = 'aa' order by id limit 2", @@ -4101,6 +4178,7 @@ { "comment": "Treating single value tuples as `EqualUnique` routes", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.user_id IN (5)) AND music.user_id = 5", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.user_id IN (5)) AND music.user_id = 5", @@ -4179,6 +4257,7 @@ { "comment": "Subquery with `IN` condition using columns with matching lookup vindexes, with inner scatter query", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.foo = 'bar') AND music.user_id IN (3, 4, 5)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.foo = 'bar') AND music.user_id IN (3, 4, 5)", @@ -4297,6 +4376,7 @@ { "comment": "Mergeable scatter subquery", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop')", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop')", @@ -4319,6 +4399,7 @@ { "comment": "Mergeable scatter subquery with `GROUP BY` on unique vindex column", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop' GROUP BY music.id)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop' GROUP BY music.id)", @@ -4341,6 +4422,7 @@ { "comment": "Unmergeable scatter subquery with `GROUP BY` on-non vindex column", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop' GROUP BY music.genre)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop' GROUP BY music.genre)", @@ -4375,19 +4457,44 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, @@ -4399,6 +4506,7 @@ { "comment": "Unmergeable scatter subquery with LIMIT", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop' LIMIT 10)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT music.id FROM music WHERE music.genre = 'pop' LIMIT 10)", @@ -4430,19 +4538,44 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, @@ -4454,6 +4587,7 @@ { "comment": "Mergeable subquery with `MAX` aggregate and grouped by unique vindex", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT MAX(music.id) FROM music WHERE music.user_id IN (5, 6) GROUP BY music.user_id)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT MAX(music.id) FROM music WHERE music.user_id IN (5, 6) GROUP BY music.user_id)", @@ -4483,19 +4617,44 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, @@ -4543,19 +4702,44 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, @@ -4596,19 +4780,44 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, @@ -4620,6 +4829,7 @@ { "comment": "Mergeable subquery with `LIMIT` due to `EqualUnique` route", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT MAX(music.id) FROM music WHERE music.user_id = 5 LIMIT 10)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT MAX(music.id) FROM music WHERE music.user_id = 5 LIMIT 10)", @@ -4649,19 +4859,44 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, @@ -4725,6 +4960,7 @@ { "comment": "Unmergeable subquery with multiple levels of derived statements, using a multi value `IN` predicate", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT * FROM (SELECT * FROM (SELECT music.id FROM music WHERE music.user_id IN (5, 6) LIMIT 10) subquery_for_limit) subquery_for_limit)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT * FROM (SELECT * FROM (SELECT music.id FROM music WHERE music.user_id IN (5, 6) LIMIT 10) subquery_for_limit) subquery_for_limit)", @@ -4760,19 +4996,44 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, @@ -4784,6 +5045,7 @@ { "comment": "Unmergeable subquery with multiple levels of derived statements", "query": "SELECT music.id FROM music WHERE music.id IN (SELECT * FROM (SELECT * FROM (SELECT music.id FROM music LIMIT 10) subquery_for_limit) subquery_for_limit)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT music.id FROM music WHERE music.id IN (SELECT * FROM (SELECT * FROM (SELECT music.id FROM music LIMIT 10) subquery_for_limit) subquery_for_limit)", @@ -4815,19 +5077,44 @@ }, { "InputName": "Outer", - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "IN", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music where 1 != 1", - "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", - "Table": "music", "Values": [ "::__sq1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music where :__sq_has_values and music.id in ::__vals", + "Table": "music" + } + ] } ] }, @@ -4983,6 +5270,7 @@ { "comment": "limit on the vtgate has to be executed on the LHS of a join", "query": "select id from user join (select user_id from user_extra limit 10) ue on user.id = ue.user_id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select id from user join (select user_id from user_extra limit 10) ue on user.id = ue.user_id", @@ -5038,6 +5326,7 @@ { "comment": "select user.a, t.b from user join (select id, count(*) b, req from user_extra group by req, id) as t on user.id = t.id", "query": "select user.a, t.b from user join (select id, count(*) b, req from user_extra group by req, id) as t on user.id = t.id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.a, t.b from user join (select id, count(*) b, req from user_extra group by req, id) as t on user.id = t.id", @@ -5202,6 +5491,7 @@ { "comment": "query with a derived table and dual table in unsharded keyspace", "query": "SELECT * FROM unsharded_a AS t1 JOIN (SELECT trim((SELECT MAX(name) FROM unsharded_a)) AS name) AS t2 WHERE t1.name >= t2.name ORDER BY t1.name ASC LIMIT 1;", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT * FROM unsharded_a AS t1 JOIN (SELECT trim((SELECT MAX(name) FROM unsharded_a)) AS name) AS t2 WHERE t1.name >= t2.name ORDER BY t1.name ASC LIMIT 1;", @@ -5251,6 +5541,7 @@ { "comment": "SOME modifier on unsharded table works well", "query": "select 1 from unsharded where foo = SOME (select 1 from unsharded_a where foo = 1)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from unsharded where foo = SOME (select 1 from unsharded_a where foo = 1)", @@ -5274,6 +5565,7 @@ { "comment": "ALL modifier on unsharded table works well", "query": "select 1 from unsharded where foo = ALL (select 1 from unsharded_a where foo = 1)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from unsharded where foo = ALL (select 1 from unsharded_a where foo = 1)", @@ -5319,6 +5611,7 @@ { "comment": "merge subquery using MAX and join into single route", "query": "select 1 from user join music_extra on user.id = music_extra.user_id where music_extra.music_id = (select max(music_id) from music_extra where user_id = user.id)", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from user join music_extra on user.id = music_extra.user_id where music_extra.music_id = (select max(music_id) from music_extra where user_id = user.id)", @@ -5342,6 +5635,7 @@ { "comment": "Query with non-plannable lookup vindex", "query": "SELECT * FROM user_metadata WHERE user_metadata.non_planable = 'foo'", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT * FROM user_metadata WHERE user_metadata.non_planable = 'foo'", @@ -5368,6 +5662,7 @@ { "comment": "join query with lookup and join on different vindex column", "query": "select u.id from user u, user_metadata um where u.name = 'foo' and u.id = um.user_id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select u.id from user u, user_metadata um where u.name = 'foo' and u.id = um.user_id", @@ -5425,7 +5720,7 @@ "Original": "select * from customer where email = 'a@mail.com'", "Instructions": { "OperatorType": "VindexLookup", - "Variant": "Equal", + "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true @@ -5493,6 +5788,7 @@ { "comment": "name is in backfill vindex - not selected for vindex lookup", "query": "select * from customer where name = 'x'", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from customer where name = 'x'", @@ -5520,7 +5816,7 @@ "Original": "select * from customer where email = 'a@mail.com' and phone = 123456", "Instructions": { "OperatorType": "VindexLookup", - "Variant": "Equal", + "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true @@ -5571,7 +5867,7 @@ "Original": "select * from customer where phone = 123456 and email = 'a@mail.com'", "Instructions": { "OperatorType": "VindexLookup", - "Variant": "Equal", + "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true @@ -5617,6 +5913,7 @@ { "comment": "invisible column is not expanded, but valid in predicate", "query": "select * from samecolvin where secret = 12", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from samecolvin where secret = 12", @@ -5639,6 +5936,7 @@ { "comment": "column with qualifier is correctly used", "query": "select u.foo, ue.foo as apa from user u, user_extra ue order by foo ", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select u.foo, ue.foo as apa from user u, user_extra ue order by foo ", @@ -5682,6 +5980,7 @@ { "comment": "Derived tables going to a single shard still need to expand derived table columns", "query": "SELECT c.column_name FROM user c JOIN (SELECT table_name FROM unsharded LIMIT 1) AS tables ON tables.table_name = c.table_name", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT c.column_name FROM user c JOIN (SELECT table_name FROM unsharded LIMIT 1) AS tables ON tables.table_name = c.table_name", @@ -5727,6 +6026,7 @@ { "comment": "column name aliases in outer join queries", "query": "select name as t0, name as t1 from user left outer join user_extra on user.cola = user_extra.cola", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select name as t0, name as t1 from user left outer join user_extra on user.cola = user_extra.cola", @@ -5782,6 +6082,7 @@ { "comment": "Over clause works for unsharded tables", "query": "SELECT val, CUME_DIST() OVER w, ROW_NUMBER() OVER w, DENSE_RANK() OVER w, PERCENT_RANK() OVER w, RANK() OVER w AS 'cd' FROM unsharded_a", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT val, CUME_DIST() OVER w, ROW_NUMBER() OVER w, DENSE_RANK() OVER w, PERCENT_RANK() OVER w, RANK() OVER w AS 'cd' FROM unsharded_a", diff --git a/go/vt/vtgate/planbuilder/testdata/union_cases.json b/go/vt/vtgate/planbuilder/testdata/union_cases.json index 7feabb0a698..2927c1c6093 100644 --- a/go/vt/vtgate/planbuilder/testdata/union_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/union_cases.json @@ -447,34 +447,84 @@ "OperatorType": "Concatenate", "Inputs": [ { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from music where 1 != 1", - "Query": "select distinct 1 from music where id = 1", - "Table": "music", "Values": [ "1" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from music where 1 != 1", + "Query": "select distinct 1 from music where id = 1", + "Table": "music" + } + ] }, { - "OperatorType": "Route", + "OperatorType": "VindexLookup", "Variant": "EqualUnique", "Keyspace": { "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from music where 1 != 1", - "Query": "select distinct 1 from music where id = 2", - "Table": "music", "Values": [ "2" ], - "Vindex": "music_user_map" + "Vindex": "music_user_map", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `name`, keyspace_id from name_user_vdx where 1 != 1", + "Query": "select `name`, keyspace_id from name_user_vdx where `name` in ::__vals", + "Table": "name_user_vdx", + "Values": [ + "::name" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "ByDestination", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from music where 1 != 1", + "Query": "select distinct 1 from music where id = 2", + "Table": "music" + } + ] } ] } diff --git a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json index 4fe275f2398..a5de9d3697e 100644 --- a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json +++ b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json @@ -58,34 +58,52 @@ "sharded": true, "vindexes": { "user_index": { - "type": "hash_test", + "type": "hash", "owner": "user" }, "kid_index": { - "type": "hash_test", + "type": "hash", "owner": "multicolvin" }, + "hash": { + "type": "hash" + }, "user_md5_index": { "type": "unicode_loose_md5" }, "music_user_map": { - "type": "lookup_test", - "owner": "music" + "type": "lookup_unique", + "owner": "music", + "params": { + "table": "name_user_vdx", + "from": "name", + "to": "keyspace_id" + } }, "cola_map": { - "type": "lookup_test", - "owner": "multicolvin" + "type": "lookup_unique", + "owner": "multicolvin", + "params": { + "table": "cola_map", + "from": "cola", + "to": "keyspace_id" + } }, "colb_colc_map": { - "type": "lookup_test", - "owner": "multicolvin" + "type": "lookup_unique", + "owner": "multicolvin", + "params": { + "table": "colb_colc_map", + "from": "colb,colc", + "to": "keyspace_id" + } }, "cola_kid_map": { - "type": "lookup_test", + "type": "lookup_unique", "owner": "overlap_vindex" }, "name_user_map": { - "type": "name_lkp_test", + "type": "lookup", "owner": "user", "params": { "table": "name_user_vdx", @@ -94,42 +112,56 @@ } }, "email_user_map": { - "type": "lookup_test", + "type": "lookup_unique", "owner": "user_metadata" }, "address_user_map": { - "type": "lookup_test", + "type": "lookup_unique", "owner": "user_metadata" }, "costly_map": { - "type": "costly", - "owner": "user" + "type": "lookup_cost", + "owner": "user", + "params": { + "table": "costly_map", + "from": "costly", + "to": "keyspace_id", + "cost": "100" + } }, "hash_dup": { - "type": "hash_test", + "type": "hash", "owner": "user" }, "vindex1": { - "type": "hash_test", + "type": "hash", "owner": "samecolvin" }, "vindex2": { - "type": "lookup_test", + "type": "lookup_unique", "owner": "samecolvin" }, "cfc": { "type": "cfc" }, "multicolIdx": { - "type": "multiCol_test" + "type": "multicol", + "params": { + "column_count": "2" + } }, "colc_map": { - "type": "lookup_test", + "type": "lookup_unique", "owner": "multicol_tbl" }, "name_muticoltbl_map": { - "type": "name_lkp_test", - "owner": "multicol_tbl" + "type": "lookup", + "owner": "multicol_tbl", + "params": { + "table": "name_user_vdx", + "from": "name", + "to": "keyspace_id" + } }, "non_planable_user_map": { "type": "lookup_unicodeloosemd5_hash", @@ -141,7 +173,7 @@ "owner": "user_metadata" }, "lkp_shard_map": { - "type": "name_lkp_test", + "type": "lookup_unique", "owner": "mixed_tbl", "params": { "table": "lkp_shard_vdx", @@ -153,18 +185,18 @@ "type": "xxhash" }, "unq_lkp_bf_vdx": { - "type": "unq_lkp_test", + "type": "lookup_unique", "owner": "customer", "params": { "table": "unq_lkp_idx", - "from": " ", + "from": "unq_key", "to": "keyspace_id", "cost": "100", "write_only": "true" } }, "unq_lkp_vdx": { - "type": "unq_lkp_test", + "type": "lookup_unique", "owner": "customer", "params": { "table": "unq_lkp_idx", @@ -174,11 +206,11 @@ } }, "lkp_bf_vdx": { - "type": "name_lkp_test", + "type": "lookup_unique", "owner": "customer", "params": { "table": "lkp_shard_vdx", - "from": " ", + "from": "unq_key", "to": "keyspace_id", "write_only": "true" } @@ -352,6 +384,22 @@ } ] }, + "cola_map": { + "column_vindexes": [ + { + "column": "cola", + "name": "hash" + } + ] + }, + "colb_colc_map": { + "column_vindexes": [ + { + "column": "colb", + "name": "hash" + } + ] + }, "overlap_vindex": { "column_vindexes": [ { @@ -462,6 +510,14 @@ } ] }, + "costly_map": { + "column_vindexes": [ + { + "column": "name", + "name": "user_md5_index" + } + ] + }, "mixed_tbl": { "column_vindexes": [ { @@ -641,7 +697,10 @@ "type": "hash_test" }, "multicolIdx": { - "type": "multiCol_test" + "type": "multicol", + "params": { + "column_count": "3" + } } }, "tables": { diff --git a/go/vt/vtgate/vindexes/cached_size.go b/go/vt/vtgate/vindexes/cached_size.go index a97411a6ac8..eeadb69b532 100644 --- a/go/vt/vtgate/vindexes/cached_size.go +++ b/go/vt/vtgate/vindexes/cached_size.go @@ -175,6 +175,18 @@ func (cached *Keyspace) CachedSize(alloc bool) int64 { size += hack.RuntimeAllocSize(int64(len(cached.Name))) return size } +func (cached *LookupCost) CachedSize(alloc bool) int64 { + if cached == nil { + return int64(0) + } + size := int64(0) + if alloc { + size += int64(16) + } + // field LookupNonUnique *vitess.io/vitess/go/vt/vtgate/vindexes.LookupNonUnique + size += cached.LookupNonUnique.CachedSize(true) + return size +} func (cached *LookupHash) CachedSize(alloc bool) int64 { if cached == nil { return int64(0) diff --git a/go/vt/vtgate/vindexes/lookup_cost.go b/go/vt/vtgate/vindexes/lookup_cost.go new file mode 100644 index 00000000000..6556032cea5 --- /dev/null +++ b/go/vt/vtgate/vindexes/lookup_cost.go @@ -0,0 +1,70 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vindexes + +import ( + "strconv" +) + +var ( + _ SingleColumn = (*LookupCost)(nil) + _ Lookup = (*LookupCost)(nil) + _ LookupPlanable = (*LookupCost)(nil) +) + +func init() { + Register("lookup_cost", newLookupCost) +} + +const defaultCost = 5 + +// LookupCost defines a test vindex that uses the cost provided by the user. +// This is a test vindex. +type LookupCost struct { + *LookupNonUnique + cost int +} + +// Cost returns the cost of this vindex as provided. +func (lc *LookupCost) Cost() int { + return lc.cost +} + +func newLookupCost(name string, m map[string]string) (Vindex, error) { + lookup, err := newLookup(name, m) + if err != nil { + return nil, err + } + cost := getInt(m, "cost", defaultCost) + return &LookupCost{ + LookupNonUnique: lookup.(*LookupNonUnique), + cost: cost, + }, nil + +} + +func getInt(m map[string]string, key string, defaultVal int) int { + val, ok := m[key] + if !ok { + return defaultVal + } + intVal, err := strconv.Atoi(val) + if err != nil { + return defaultVal + } + return intVal +} diff --git a/go/vt/vtgate/vindexes/vschema_test.go b/go/vt/vtgate/vindexes/vschema_test.go index 25f8e135698..f9bcf43ddaa 100644 --- a/go/vt/vtgate/vindexes/vschema_test.go +++ b/go/vt/vtgate/vindexes/vschema_test.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "os" "reflect" "strings" "testing" @@ -3551,6 +3552,20 @@ func TestFindTableWithSequences(t *testing.T) { } } +func TestGlobalTables(t *testing.T) { + input, err := os.ReadFile("../planbuilder/testdata/vschemas/schema.json") + require.NoError(t, err) + + var vs vschemapb.SrvVSchema + err = json2.UnmarshalPB(input, &vs) + require.NoError(t, err) + + got := BuildVSchema(&vs, sqlparser.NewTestParser()) + tbl, err := got.findGlobalTable("user", false) + require.NoError(t, err) + assert.NotNil(t, tbl) +} + func vindexNames(vindexes []*ColumnVindex) (result []string) { for _, vindex := range vindexes { result = append(result, vindex.Name) diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index 9253e16c43f..b24db1154fb 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -103,6 +103,7 @@ var ( "vtgate_vindex_heavy", "vtgate_vschema", "vtgate_queries", + "vtgate_plantests", "vtgate_schema_tracker", "vtgate_foreignkey_stress", "vtorc", diff --git a/test/config.json b/test/config.json index 1ed73e98fba..4bcf616aa3b 100644 --- a/test/config.json +++ b/test/config.json @@ -896,6 +896,15 @@ "RetryMax": 1, "Tags": [] }, + "vtgate_plantests": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/plan_tests"], + "Command": [], + "Manual": false, + "Shard": "vtgate_plantests", + "RetryMax": 1, + "Tags": [] + }, "vtgate_unsharded": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/unsharded"],