From 1a69fc223618815a25505ba727cb4ec9fc9c4950 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 9 Aug 2024 10:08:13 +0200 Subject: [PATCH 01/11] make sure to plan offsets on aggregator before adding additional columns Signed-off-by: Andres Taylor --- .../planbuilder/operators/aggregator.go | 14 ++- .../vtgate/planbuilder/operators/ordering.go | 14 ++- .../planbuilder/testdata/aggr_cases.json | 103 ++++++++---------- .../testdata/memory_sort_cases.json | 23 ++-- .../testdata/postprocess_cases.json | 8 +- .../planbuilder/testdata/tpch_cases.json | 17 ++- 6 files changed, 89 insertions(+), 90 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/aggregator.go b/go/vt/vtgate/planbuilder/operators/aggregator.go index bb969912f4f..875386061d3 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregator.go +++ b/go/vt/vtgate/planbuilder/operators/aggregator.go @@ -151,6 +151,8 @@ func (a *Aggregator) checkOffset(offset int) { } func (a *Aggregator) AddColumn(ctx *plancontext.PlanningContext, reuse bool, groupBy bool, ae *sqlparser.AliasedExpr) (offset int) { + a.planOffsets(ctx) + defer func() { a.checkOffset(offset) }() @@ -199,6 +201,10 @@ func (a *Aggregator) AddColumn(ctx *plancontext.PlanningContext, reuse bool, gro } func (a *Aggregator) AddWSColumn(ctx *plancontext.PlanningContext, offset int, underRoute bool) int { + if !underRoute { + a.planOffsets(ctx) + } + if len(a.Columns) <= offset { panic(vterrors.VT13001("offset out of range")) } @@ -221,7 +227,7 @@ func (a *Aggregator) AddWSColumn(ctx *plancontext.PlanningContext, offset int, u } if expr == nil { - for _, aggr := range a.Aggregations { + for i, aggr := range a.Aggregations { if aggr.ColOffset != offset { continue } @@ -230,9 +236,13 @@ func (a *Aggregator) AddWSColumn(ctx *plancontext.PlanningContext, offset int, u return aggr.WSOffset } - panic(vterrors.VT13001("expected to find a weight string for aggregation")) + a.Aggregations[i].WSOffset = len(a.Columns) + expr = a.Columns[offset].Expr + break } + } + if expr == nil { panic(vterrors.VT13001("could not find expression at offset")) } diff --git a/go/vt/vtgate/planbuilder/operators/ordering.go b/go/vt/vtgate/planbuilder/operators/ordering.go index 94c4f3dd846..c8f4ccdf853 100644 --- a/go/vt/vtgate/planbuilder/operators/ordering.go +++ b/go/vt/vtgate/planbuilder/operators/ordering.go @@ -82,17 +82,25 @@ func (o *Ordering) GetOrdering(*plancontext.PlanningContext) []OrderBy { } func (o *Ordering) planOffsets(ctx *plancontext.PlanningContext) Operator { + var weightStrings []*OrderBy + for _, order := range o.Order { offset := o.Source.AddColumn(ctx, true, false, aeWrap(order.SimplifiedExpr)) o.Offset = append(o.Offset, offset) if !ctx.NeedsWeightString(order.SimplifiedExpr) { - o.WOffset = append(o.WOffset, -1) + weightStrings = append(weightStrings, nil) continue } + weightStrings = append(weightStrings, &order) + } - wsExpr := &sqlparser.WeightStringFuncExpr{Expr: order.SimplifiedExpr} - offset = o.Source.AddColumn(ctx, true, false, aeWrap(wsExpr)) + for i, order := range weightStrings { + if order == nil { + o.WOffset = append(o.WOffset, -1) + continue + } + offset := o.Source.AddWSColumn(ctx, o.Offset[i], false) o.WOffset = append(o.WOffset, offset) } return nil diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index 628a959af1d..33e8de55def 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -5846,16 +5846,16 @@ { "OperatorType": "Aggregate", "Variant": "Ordered", - "Aggregates": "group_concat(1) AS group_concat(u.bar), any_value(2) AS baz, any_value(4)", + "Aggregates": "group_concat(1) AS group_concat(u.bar), any_value(2|4) AS baz", "GroupBy": "(0|3)", "ResultColumns": 5, "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,L:1,L:2,L:3,L:4", + "JoinColumnIndexes": "L:0,L:1,L:2,L:3,L:5", "JoinVars": { - "u_col": 5 + "u_col": 4 }, "TableName": "`user`_music", "Inputs": [ @@ -5866,9 +5866,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select u.foo, u.bar, u.baz, weight_string(u.foo), weight_string(u.baz), u.col from `user` as u where 1 != 1", + "FieldQuery": "select u.foo, u.bar, u.baz, weight_string(u.foo), u.col, weight_string(u.baz) from `user` as u where 1 != 1", "OrderBy": "(0|3) ASC", - "Query": "select u.foo, u.bar, u.baz, weight_string(u.foo), weight_string(u.baz), u.col from `user` as u order by u.foo asc", + "Query": "select u.foo, u.bar, u.baz, weight_string(u.foo), u.col, weight_string(u.baz) from `user` as u order by u.foo asc", "Table": "`user`" }, { @@ -6663,70 +6663,55 @@ "OrderBy": "(4|6) ASC, (5|7) ASC", "Inputs": [ { - "OperatorType": "Projection", - "Expressions": [ - "count(*) as count(*)", - "count(*) as count(*)", - "`user`.col as col", - "ue.col as col", - "`user`.foo as foo", - "ue.bar as bar", - "weight_string(`user`.foo) as weight_string(`user`.foo)", - "weight_string(ue.bar) as weight_string(ue.bar)" - ], + "OperatorType": "Join", + "Variant": "HashLeftJoin", + "Collation": "binary", + "ComparisonType": "INT16", + "JoinColumnIndexes": "-1,1,-2,2,-3,3,-3,4", + "Predicate": "`user`.col = ue.col", + "TableName": "`user`_user_extra", "Inputs": [ { - "OperatorType": "Join", - "Variant": "HashLeftJoin", - "Collation": "binary", - "ComparisonType": "INT16", - "JoinColumnIndexes": "-1,1,-2,2,-3,3,-3,3", - "Predicate": "`user`.col = ue.col", - "TableName": "`user`_user_extra", + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select count(*), `user`.col, `user`.foo from `user` where 1 != 1 group by `user`.col, `user`.foo", + "Query": "select count(*), `user`.col, `user`.foo from `user` group by `user`.col, `user`.foo", + "Table": "`user`" + }, + { + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "count_star(0)", + "GroupBy": "1, (2|3)", + "ResultColumns": 4, "Inputs": [ { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select count(*), `user`.col, `user`.foo from `user` where 1 != 1 group by `user`.col, `user`.foo", - "Query": "select count(*), `user`.col, `user`.foo from `user` group by `user`.col, `user`.foo", - "Table": "`user`" - }, - { - "OperatorType": "Aggregate", - "Variant": "Ordered", - "Aggregates": "count_star(0)", - "GroupBy": "1, (2|3)", - "ResultColumns": 3, + "OperatorType": "SimpleProjection", + "Columns": "2,0,1,3", "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "2,0,1,3", + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "0 ASC, (1|3) ASC", "Inputs": [ { - "OperatorType": "Sort", - "Variant": "Memory", - "OrderBy": "0 ASC, (1|3) ASC", + "OperatorType": "Limit", + "Count": "10", "Inputs": [ { - "OperatorType": "Limit", - "Count": "10", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select ue.col, ue.bar, 1, weight_string(ue.bar) from (select col, bar from user_extra where 1 != 1) as ue where 1 != 1", - "Query": "select ue.col, ue.bar, 1, weight_string(ue.bar) from (select col, bar from user_extra) as ue limit 10", - "Table": "user_extra" - } - ] + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select ue.col, ue.bar, 1, weight_string(ue.bar) from (select col, bar from user_extra where 1 != 1) as ue where 1 != 1", + "Query": "select ue.col, ue.bar, 1, weight_string(ue.bar) from (select col, bar from user_extra) as ue limit 10", + "Table": "user_extra" } ] } diff --git a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json index f26b160ef69..3ce182b51b4 100644 --- a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json @@ -14,7 +14,7 @@ { "OperatorType": "Aggregate", "Variant": "Ordered", - "Aggregates": "any_value(1) AS b, sum_count_star(2) AS count(*), any_value(4)", + "Aggregates": "any_value(1|4) AS b, sum_count_star(2) AS count(*)", "GroupBy": "(0|3)", "ResultColumns": 5, "Inputs": [ @@ -25,9 +25,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*), weight_string(a), weight_string(`user`.b) from `user` where 1 != 1 group by a, weight_string(a)", + "FieldQuery": "select dt.c0 as a, dt.c1 as b, dt.c2 as `count(*)`, dt.c3 as `weight_string(a)`, weight_string(dt.c1) from (select a, b, count(*), weight_string(a) from `user` where 1 != 1 group by a, weight_string(a)) as dt(c0, c1, c2, c3) where 1 != 1", "OrderBy": "(0|3) ASC", - "Query": "select a, b, count(*), weight_string(a), weight_string(`user`.b) from `user` group by a, weight_string(a) order by a asc", + "Query": "select dt.c0 as a, dt.c1 as b, dt.c2 as `count(*)`, dt.c3 as `weight_string(a)`, weight_string(dt.c1) from (select a, b, count(*), weight_string(a) from `user` group by a, weight_string(a) order by a asc) as dt(c0, c1, c2, c3)", "Table": "`user`" } ] @@ -93,7 +93,7 @@ { "OperatorType": "Aggregate", "Variant": "Ordered", - "Aggregates": "any_value(1) AS b, sum_count_star(2) AS k, any_value(4)", + "Aggregates": "any_value(1|4) AS b, sum_count_star(2) AS k", "GroupBy": "(0|3)", "ResultColumns": 5, "Inputs": [ @@ -104,9 +104,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k, weight_string(a), weight_string(`user`.b) from `user` where 1 != 1 group by a, weight_string(a)", + "FieldQuery": "select dt.c0 as a, dt.c1 as b, dt.c2 as k, dt.c3 as `weight_string(a)`, weight_string(dt.c1) from (select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a, weight_string(a)) as dt(c0, c1, c2, c3) where 1 != 1", "OrderBy": "(0|3) ASC", - "Query": "select a, b, count(*) as k, weight_string(a), weight_string(`user`.b) from `user` group by a, weight_string(a) order by a asc", + "Query": "select dt.c0 as a, dt.c1 as b, dt.c2 as k, dt.c3 as `weight_string(a)`, weight_string(dt.c1) from (select a, b, count(*) as k, weight_string(a) from `user` group by a, weight_string(a) order by a asc) as dt(c0, c1, c2, c3)", "Table": "`user`" } ] @@ -173,14 +173,13 @@ "OperatorType": "Sort", "Variant": "Memory", "OrderBy": "(0|3) ASC, 2 ASC", - "ResultColumns": 3, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "any_value(1) AS b, sum_count_star(2) AS k", "GroupBy": "(0|3)", - "ResultColumns": 4, + "ResultColumns": 3, "Inputs": [ { "OperatorType": "Route", @@ -360,9 +359,9 @@ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,L:1,R:0,L:2,R:1,L:3", + "JoinColumnIndexes": "L:0,L:1,R:0,L:3,R:1,L:4", "JoinVars": { - "user_id": 4 + "user_id": 2 }, "TableName": "`user`_music", "Inputs": [ @@ -373,8 +372,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select `user`.col1 as a, `user`.col2, weight_string(`user`.col1), weight_string(`user`.col2), `user`.id from `user` where 1 != 1", - "Query": "select `user`.col1 as a, `user`.col2, weight_string(`user`.col1), weight_string(`user`.col2), `user`.id from `user` where `user`.id = 1", + "FieldQuery": "select `user`.col1 as a, `user`.col2, `user`.id, weight_string(`user`.col1), weight_string(`user`.col2) from `user` where 1 != 1", + "Query": "select `user`.col1 as a, `user`.col2, `user`.id, weight_string(`user`.col1), weight_string(`user`.col2) from `user` where `user`.id = 1", "Table": "`user`", "Values": [ "1" diff --git a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json index 74e5229016a..fd8ee10713b 100644 --- a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json @@ -1554,7 +1554,7 @@ { "OperatorType": "Aggregate", "Variant": "Scalar", - "Aggregates": "sum_count(0) AS count(id), any_value(1) AS num, any_value(2)", + "Aggregates": "sum_count(0) AS count(id), any_value(1|2) AS num", "Inputs": [ { "OperatorType": "Route", @@ -1729,7 +1729,7 @@ { "OperatorType": "Aggregate", "Variant": "Ordered", - "Aggregates": "sum_count_star(1) AS count(*), any_value(2) AS c1, any_value(3)", + "Aggregates": "sum_count_star(1) AS count(*), any_value(2|3) AS c1", "GroupBy": "0", "Inputs": [ { @@ -1739,9 +1739,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select col, count(*), c1, weight_string(c1) from `user` where 1 != 1 group by col", + "FieldQuery": "select dt.c0 as col, dt.c1 as `count(*)`, dt.c2 as c1, weight_string(dt.c2) from (select col, count(*), c1 from `user` where 1 != 1 group by col) as dt(c0, c1, c2) where 1 != 1", "OrderBy": "0 ASC", - "Query": "select col, count(*), c1, weight_string(c1) from `user` group by col order by col asc", + "Query": "select dt.c0 as col, dt.c1 as `count(*)`, dt.c2 as c1, weight_string(dt.c2) from (select col, count(*), c1 from `user` group by col order by col asc) as dt(c0, c1, c2)", "Table": "`user`" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json index 442f4d6b8b6..6fda1620e15 100644 --- a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json @@ -67,14 +67,13 @@ "OperatorType": "Sort", "Variant": "Memory", "OrderBy": "1 DESC COLLATE utf8mb4_0900_ai_ci, (2|5) ASC", - "ResultColumns": 4, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum(1) AS revenue", "GroupBy": "(0|4), (2|5), (3|6)", - "ResultColumns": 6, + "ResultColumns": 4, "Inputs": [ { "OperatorType": "Projection", @@ -1130,12 +1129,12 @@ { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "(0|6) ASC, (4|7) ASC", + "OrderBy": "(0|8) ASC, (4|7) ASC", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,R:0,R:1,R:2,R:3,R:4,L:2,R:5", + "JoinColumnIndexes": "L:0,R:0,R:1,R:2,R:3,R:4,L:2,R:5,L:3", "JoinVars": { "o_orderkey": 1 }, @@ -1148,8 +1147,8 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select profit.o_year, profit.o_orderkey, weight_string(profit.o_year) from (select extract(year from o_orderdate) as o_year, o_orderkey as o_orderkey from orders where 1 != 1) as profit where 1 != 1", - "Query": "select profit.o_year, profit.o_orderkey, weight_string(profit.o_year) from (select extract(year from o_orderdate) as o_year, o_orderkey as o_orderkey from orders) as profit", + "FieldQuery": "select profit.o_year, profit.o_orderkey, weight_string(profit.o_year), weight_string(extract(year from o_orderdate)) from (select extract(year from o_orderdate) as o_year, o_orderkey as o_orderkey from orders where 1 != 1) as profit where 1 != 1", + "Query": "select profit.o_year, profit.o_orderkey, weight_string(profit.o_year), weight_string(extract(year from o_orderdate)) from (select extract(year from o_orderdate) as o_year, o_orderkey as o_orderkey from orders) as profit", "Table": "orders" }, { @@ -2099,14 +2098,13 @@ "OperatorType": "Sort", "Variant": "Memory", "OrderBy": "3 DESC, (0|4) ASC, (1|5) ASC, (2|6) ASC", - "ResultColumns": 4, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "count_distinct(3|7) AS supplier_cnt", "GroupBy": "(0|4), (1|5), (2|6)", - "ResultColumns": 7, + "ResultColumns": 4, "Inputs": [ { "OperatorType": "Sort", @@ -2437,14 +2435,13 @@ "OperatorType": "Sort", "Variant": "Memory", "OrderBy": "1 DESC, (0|2) ASC", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum_count_star(1) AS numwait", "GroupBy": "(0|2)", - "ResultColumns": 3, + "ResultColumns": 2, "Inputs": [ { "OperatorType": "Projection", From 7ab5cb266ea113e4bdbf25d4d8d312ff9df8edd2 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 9 Aug 2024 10:22:04 +0200 Subject: [PATCH 02/11] use addWSColumn instead of addColumn Signed-off-by: Andres Taylor --- .../planbuilder/operators/aggregator.go | 15 +++++++++-- .../planbuilder/testdata/aggr_cases.json | 27 +++++++++---------- .../planbuilder/testdata/cte_cases.json | 9 +++---- .../planbuilder/testdata/from_cases.json | 1 - .../planbuilder/testdata/tpch_cases.json | 18 ++++++------- 5 files changed, 37 insertions(+), 33 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/aggregator.go b/go/vt/vtgate/planbuilder/operators/aggregator.go index 875386061d3..624a0a297ba 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregator.go +++ b/go/vt/vtgate/planbuilder/operators/aggregator.go @@ -525,7 +525,7 @@ func (a *Aggregator) pushRemainingGroupingColumnsAndWeightStrings(ctx *planconte continue } - offset := a.internalAddColumn(ctx, aeWrap(weightStringFor(gb.Inner)), false) + offset := a.internalAddWSColumn(ctx, a.Grouping[idx].ColOffset, aeWrap(weightStringFor(gb.Inner))) a.Grouping[idx].WSOffset = offset } for idx, aggr := range a.Aggregations { @@ -534,11 +534,22 @@ func (a *Aggregator) pushRemainingGroupingColumnsAndWeightStrings(ctx *planconte } arg := aggr.getPushColumn() - offset := a.internalAddColumn(ctx, aeWrap(weightStringFor(arg)), false) + offset := a.internalAddWSColumn(ctx, aggr.ColOffset, aeWrap(weightStringFor(arg))) + a.Aggregations[idx].WSOffset = offset } } +func (a *Aggregator) internalAddWSColumn(ctx *plancontext.PlanningContext, inOffset int, aliasedExpr *sqlparser.AliasedExpr) int { + offset := a.Source.AddWSColumn(ctx, inOffset, false) + + if offset == len(a.Columns) { + // if we get an offset at the end of our current column list, it means we added a new column + a.Columns = append(a.Columns, aliasedExpr) + } + return offset +} + func (a *Aggregator) setTruncateColumnCount(offset int) { a.ResultColumns = offset } diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index 33e8de55def..c11877e2b66 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -2918,16 +2918,16 @@ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,L:1,L:2", + "JoinColumnIndexes": "L:0,L:1,L:3", "JoinVars": { - "u2_val2": 3 + "u2_val2": 2 }, "TableName": "`user`_`user`_music", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,L:1,L:2,R:0", + "JoinColumnIndexes": "L:0,L:1,R:0,L:2", "JoinVars": { "u_val2": 1 }, @@ -5681,7 +5681,6 @@ "OperatorType": "Aggregate", "Variant": "Ordered", "GroupBy": "0, (1|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "SimpleProjection", @@ -5690,7 +5689,7 @@ { "OperatorType": "Aggregate", "Variant": "Scalar", - "Aggregates": "any_value(0) AS id, sum_count_star(1) AS a, any_value(2)", + "Aggregates": "any_value(0|2) AS id, sum_count_star(1) AS a", "Inputs": [ { "OperatorType": "Route", @@ -5699,9 +5698,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id, count(*) as a, weight_string(id) from `user` where 1 != 1", - "OrderBy": "1 ASC, (0|2) ASC", - "Query": "select id, count(*) as a, weight_string(id) from `user` order by count(*) asc, id asc", + "FieldQuery": "select dt.c0 as id, dt.c1 as a, weight_string(dt.c0), weight_string(dt.c0) from (select id, count(*) as a from `user` where 1 != 1) as dt(c0, c1) where 1 != 1", + "OrderBy": "1 ASC, (0|3) ASC", + "Query": "select dt.c0 as id, dt.c1 as a, weight_string(dt.c0), weight_string(dt.c0) from (select id, count(*) as a from `user` order by count(*) asc, id asc) as dt(c0, c1)", "Table": "`user`" } ] @@ -5848,14 +5847,13 @@ "Variant": "Ordered", "Aggregates": "group_concat(1) AS group_concat(u.bar), any_value(2|4) AS baz", "GroupBy": "(0|3)", - "ResultColumns": 5, "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,L:1,L:2,L:3,L:5", + "JoinColumnIndexes": "L:0,L:1,L:2,L:4,L:5", "JoinVars": { - "u_col": 4 + "u_col": 3 }, "TableName": "`user`_music", "Inputs": [ @@ -5866,9 +5864,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select u.foo, u.bar, u.baz, weight_string(u.foo), u.col, weight_string(u.baz) from `user` as u where 1 != 1", - "OrderBy": "(0|3) ASC", - "Query": "select u.foo, u.bar, u.baz, weight_string(u.foo), u.col, weight_string(u.baz) from `user` as u order by u.foo asc", + "FieldQuery": "select u.foo, u.bar, u.baz, u.col, weight_string(u.foo), weight_string(u.baz) from `user` as u where 1 != 1", + "OrderBy": "(0|4) ASC", + "Query": "select u.foo, u.bar, u.baz, u.col, weight_string(u.foo), weight_string(u.baz) from `user` as u order by u.foo asc", "Table": "`user`" }, { @@ -6687,7 +6685,6 @@ "Variant": "Ordered", "Aggregates": "count_star(0)", "GroupBy": "1, (2|3)", - "ResultColumns": 4, "Inputs": [ { "OperatorType": "SimpleProjection", diff --git a/go/vt/vtgate/planbuilder/testdata/cte_cases.json b/go/vt/vtgate/planbuilder/testdata/cte_cases.json index 1fd398012e3..0b801aa1a62 100644 --- a/go/vt/vtgate/planbuilder/testdata/cte_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/cte_cases.json @@ -565,7 +565,6 @@ "OperatorType": "Aggregate", "Variant": "Ordered", "GroupBy": "0, (1|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "SimpleProjection", @@ -574,7 +573,7 @@ { "OperatorType": "Aggregate", "Variant": "Scalar", - "Aggregates": "any_value(0) AS id, sum_count_star(1) AS a, any_value(2)", + "Aggregates": "any_value(0|2) AS id, sum_count_star(1) AS a", "Inputs": [ { "OperatorType": "Route", @@ -583,9 +582,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id, count(*) as a, weight_string(id) from `user` where 1 != 1", - "OrderBy": "1 ASC, (0|2) ASC", - "Query": "select id, count(*) as a, weight_string(id) from `user` order by count(*) asc, id asc", + "FieldQuery": "select dt.c0 as id, dt.c1 as a, weight_string(dt.c0), weight_string(dt.c0) from (select id, count(*) as a from `user` where 1 != 1) as dt(c0, c1) where 1 != 1", + "OrderBy": "1 ASC, (0|3) ASC", + "Query": "select dt.c0 as id, dt.c1 as a, weight_string(dt.c0), weight_string(dt.c0) from (select id, count(*) as a from `user` order by count(*) asc, id asc) as dt(c0, c1)", "Table": "`user`" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/from_cases.json b/go/vt/vtgate/planbuilder/testdata/from_cases.json index ca94f4ee866..19388aec78e 100644 --- a/go/vt/vtgate/planbuilder/testdata/from_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/from_cases.json @@ -788,7 +788,6 @@ "OperatorType": "Aggregate", "Variant": "Ordered", "GroupBy": "(0|1)", - "ResultColumns": 1, "Inputs": [ { "OperatorType": "SimpleProjection", diff --git a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json index 6fda1620e15..b0540b011e5 100644 --- a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json @@ -831,7 +831,6 @@ "Variant": "Ordered", "Aggregates": "sum(0) AS sum(case when nation = 'BRAZIL' then volume else 0 end), sum(1) AS sum(volume)", "GroupBy": "(2|3)", - "ResultColumns": 3, "Inputs": [ { "OperatorType": "Join", @@ -1111,12 +1110,11 @@ "Variant": "Ordered", "Aggregates": "sum(0) AS sum_profit", "GroupBy": "(1|3), (2|4)", - "ResultColumns": 4, "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "R:0,L:0,L:4,L:6,L:7", + "JoinColumnIndexes": "R:0,L:0,L:4,L:6,L:8", "JoinVars": { "l_discount": 2, "l_extendedprice": 1, @@ -1129,12 +1127,12 @@ { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "(0|8) ASC, (4|7) ASC", + "OrderBy": "(0|6) ASC, (4|8) ASC", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,R:0,R:1,R:2,R:3,R:4,L:2,R:5,L:3", + "JoinColumnIndexes": "L:0,R:0,R:1,R:2,R:3,R:4,L:3,R:5", "JoinVars": { "o_orderkey": 1 }, @@ -2098,13 +2096,13 @@ "OperatorType": "Sort", "Variant": "Memory", "OrderBy": "3 DESC, (0|4) ASC, (1|5) ASC, (2|6) ASC", + "ResultColumns": 4, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "count_distinct(3|7) AS supplier_cnt", "GroupBy": "(0|4), (1|5), (2|6)", - "ResultColumns": 4, "Inputs": [ { "OperatorType": "Sort", @@ -2114,9 +2112,9 @@ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "R:0,R:1,R:2,L:0,R:3,R:4,R:5,L:1", + "JoinColumnIndexes": "R:0,R:1,R:2,L:0,R:3,R:4,R:5,L:2", "JoinVars": { - "ps_partkey": 2, + "ps_partkey": 1, "ps_suppkey": 0 }, "TableName": "partsupp_part", @@ -2149,8 +2147,8 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select ps_suppkey, weight_string(ps_suppkey), ps_partkey from partsupp where 1 != 1", - "Query": "select ps_suppkey, weight_string(ps_suppkey), ps_partkey from partsupp where not :__sq_has_values or ps_suppkey not in ::__sq1", + "FieldQuery": "select ps_suppkey, ps_partkey, weight_string(ps_suppkey) from partsupp where 1 != 1", + "Query": "select ps_suppkey, ps_partkey, weight_string(ps_suppkey) from partsupp where not :__sq_has_values or ps_suppkey not in ::__sq1", "Table": "partsupp" } ] From d78361797e50817d1ec5844569a6d9ff7ddab3dc Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 9 Aug 2024 10:28:28 +0200 Subject: [PATCH 03/11] only add bindvars once Signed-off-by: Andres Taylor --- go/vt/vtgate/planbuilder/operators/apply_join.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/go/vt/vtgate/planbuilder/operators/apply_join.go b/go/vt/vtgate/planbuilder/operators/apply_join.go index ef36f6a6765..0900384dc5e 100644 --- a/go/vt/vtgate/planbuilder/operators/apply_join.go +++ b/go/vt/vtgate/planbuilder/operators/apply_join.go @@ -302,12 +302,18 @@ func (aj *ApplyJoin) planOffsets(ctx *plancontext.PlanningContext) Operator { for _, col := range aj.JoinPredicates.columns { for _, lhsExpr := range col.LHSExprs { + if _, found := aj.Vars[lhsExpr.Name]; found { + continue + } offset := aj.LHS.AddColumn(ctx, true, false, aeWrap(lhsExpr.Expr)) aj.Vars[lhsExpr.Name] = offset } } for _, lhsExpr := range aj.ExtraLHSVars { + if _, found := aj.Vars[lhsExpr.Name]; found { + continue + } offset := aj.LHS.AddColumn(ctx, true, false, aeWrap(lhsExpr.Expr)) aj.Vars[lhsExpr.Name] = offset } From fac9cc998525b997311fa6edf14c8316df1f9abf Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 9 Aug 2024 10:32:43 +0200 Subject: [PATCH 04/11] make method an instance method Signed-off-by: Andres Taylor --- .../planbuilder/operators/join_merging.go | 29 ++++++++----------- .../planbuilder/operators/route_planning.go | 3 +- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/join_merging.go b/go/vt/vtgate/planbuilder/operators/join_merging.go index 6f2af8b5ff9..c124cefd73c 100644 --- a/go/vt/vtgate/planbuilder/operators/join_merging.go +++ b/go/vt/vtgate/planbuilder/operators/join_merging.go @@ -18,7 +18,6 @@ package operators import ( "fmt" - "reflect" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" @@ -28,7 +27,7 @@ import ( // mergeJoinInputs checks whether two operators can be merged into a single one. // If they can be merged, a new operator with the merged routing is returned // If they cannot be merged, nil is returned. -func mergeJoinInputs(ctx *plancontext.PlanningContext, lhs, rhs Operator, joinPredicates []sqlparser.Expr, m *joinMerger) *Route { +func (jm *joinMerger) mergeJoinInputs(ctx *plancontext.PlanningContext, lhs, rhs Operator, joinPredicates []sqlparser.Expr) *Route { lhsRoute, rhsRoute, routingA, routingB, a, b, sameKeyspace := prepareInputRoutes(lhs, rhs) if lhsRoute == nil { return nil @@ -44,39 +43,39 @@ func mergeJoinInputs(ctx *plancontext.PlanningContext, lhs, rhs Operator, joinPr rhsClone.AddPredicate(ctx, predicate) } } - if !m.joinType.IsInner() && !rhsClone.Routing.OpCode().IsSingleShard() { + if !jm.joinType.IsInner() && !rhsClone.Routing.OpCode().IsSingleShard() { return nil } - return m.merge(ctx, lhsRoute, rhsClone, rhsClone.Routing) + return jm.merge(ctx, lhsRoute, rhsClone, rhsClone.Routing) // If a dual is on the right side. case b == dual: - return m.merge(ctx, lhsRoute, rhsRoute, routingA) + return jm.merge(ctx, lhsRoute, rhsRoute, routingA) // As both are reference route. We need to merge the alternates as well. case a == anyShard && b == anyShard && sameKeyspace: - newrouting := mergeAnyShardRoutings(ctx, routingA.(*AnyShardRouting), routingB.(*AnyShardRouting), joinPredicates, m.joinType) - return m.merge(ctx, lhsRoute, rhsRoute, newrouting) + newrouting := mergeAnyShardRoutings(ctx, routingA.(*AnyShardRouting), routingB.(*AnyShardRouting), joinPredicates, jm.joinType) + return jm.merge(ctx, lhsRoute, rhsRoute, newrouting) // an unsharded/reference route can be merged with anything going to that keyspace case a == anyShard && sameKeyspace: - return m.merge(ctx, lhsRoute, rhsRoute, routingB) + return jm.merge(ctx, lhsRoute, rhsRoute, routingB) case b == anyShard && sameKeyspace: - return m.merge(ctx, lhsRoute, rhsRoute, routingA) + return jm.merge(ctx, lhsRoute, rhsRoute, routingA) // None routing can always be merged, as long as we are aiming for the same keyspace case a == none && sameKeyspace: - return m.merge(ctx, lhsRoute, rhsRoute, routingA) + return jm.merge(ctx, lhsRoute, rhsRoute, routingA) case b == none && sameKeyspace: - return m.merge(ctx, lhsRoute, rhsRoute, routingB) + return jm.merge(ctx, lhsRoute, rhsRoute, routingB) // infoSchema routing is complex, so we handle it in a separate method case a == infoSchema && b == infoSchema: - return tryMergeInfoSchemaRoutings(ctx, routingA, routingB, m, lhsRoute, rhsRoute) + return tryMergeInfoSchemaRoutings(ctx, routingA, routingB, jm, lhsRoute, rhsRoute) // sharded routing is complex, so we handle it in a separate method case a == sharded && b == sharded: - return tryMergeShardedRouting(ctx, lhsRoute, rhsRoute, m, joinPredicates) + return tryMergeShardedRouting(ctx, lhsRoute, rhsRoute, jm, joinPredicates) default: return nil @@ -188,10 +187,6 @@ func getRoutesOrAlternates(lhsRoute, rhsRoute *Route) (*Route, *Route, Routing, return lhsRoute, rhsRoute, routingA, routingB, sameKeyspace } -func getTypeName(myvar interface{}) string { - return reflect.TypeOf(myvar).String() -} - func getRoutingType(r Routing) routingType { switch r.(type) { case *InfoSchemaRouting: diff --git a/go/vt/vtgate/planbuilder/operators/route_planning.go b/go/vt/vtgate/planbuilder/operators/route_planning.go index 22db69f287b..6a242649725 100644 --- a/go/vt/vtgate/planbuilder/operators/route_planning.go +++ b/go/vt/vtgate/planbuilder/operators/route_planning.go @@ -288,7 +288,8 @@ func requiresSwitchingSides(ctx *plancontext.PlanningContext, op Operator) (requ } func mergeOrJoin(ctx *plancontext.PlanningContext, lhs, rhs Operator, joinPredicates []sqlparser.Expr, joinType sqlparser.JoinType) (Operator, *ApplyResult) { - newPlan := mergeJoinInputs(ctx, lhs, rhs, joinPredicates, newJoinMerge(joinPredicates, joinType)) + jm := newJoinMerge(joinPredicates, joinType) + newPlan := jm.mergeJoinInputs(ctx, lhs, rhs, joinPredicates) if newPlan != nil { return newPlan, Rewrote("merge routes into single operator") } From 988ae6d4f6061fd94b4e5fe0ad28ec4fdca9016f Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 9 Aug 2024 10:34:29 +0200 Subject: [PATCH 05/11] extract type from method and move method to different file Signed-off-by: Andres Taylor --- .../planbuilder/operators/offset_planning.go | 54 ++----------------- go/vt/vtgate/planbuilder/operators/phases.go | 44 +++++++++++++++ 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/offset_planning.go b/go/vt/vtgate/planbuilder/operators/offset_planning.go index e8301c18823..adf71a47f24 100644 --- a/go/vt/vtgate/planbuilder/operators/offset_planning.go +++ b/go/vt/vtgate/planbuilder/operators/offset_planning.go @@ -25,13 +25,13 @@ import ( "vitess.io/vitess/go/vt/vtgate/semantics" ) +type offsettable interface { + Operator + planOffsets(ctx *plancontext.PlanningContext) Operator +} + // planOffsets will walk the tree top down, adding offset information to columns in the tree for use in further optimization, func planOffsets(ctx *plancontext.PlanningContext, root Operator) Operator { - type offsettable interface { - Operator - planOffsets(ctx *plancontext.PlanningContext) Operator - } - visitor := func(in Operator, _ semantics.TableSet, _ bool) (Operator, *ApplyResult) { switch op := in.(type) { case *Horizon: @@ -120,50 +120,6 @@ func findAggregatorInSource(op Operator) *Aggregator { } } -// addColumnsToInput adds columns needed by an operator to its input. -// This happens only when the filter expression can be retrieved as an offset from the underlying mysql. -func addColumnsToInput(ctx *plancontext.PlanningContext, root Operator) Operator { - - addColumnsNeededByFilter := func(in Operator, _ semantics.TableSet, _ bool) (Operator, *ApplyResult) { - addedCols := false - filter, ok := in.(*Filter) - if !ok { - return in, NoRewrite - } - - var neededAggrs []sqlparser.Expr - extractAggrs := func(cursor *sqlparser.CopyOnWriteCursor) { - node := cursor.Node() - if ctx.IsAggr(node) { - neededAggrs = append(neededAggrs, node.(sqlparser.Expr)) - } - } - - for _, expr := range filter.Predicates { - _ = sqlparser.CopyOnRewrite(expr, dontEnterSubqueries, extractAggrs, nil) - } - - if neededAggrs == nil { - return in, NoRewrite - } - - aggregator := findAggregatorInSource(filter.Source) - for _, aggr := range neededAggrs { - if aggregator.FindCol(ctx, aggr, false) == -1 { - aggregator.addColumnWithoutPushing(ctx, aeWrap(aggr), false) - addedCols = true - } - } - - if addedCols { - return in, Rewrote("added columns because filter needs it") - } - return in, NoRewrite - } - - return TopDown(root, TableID, addColumnsNeededByFilter, stopAtRoute) -} - // isolateDistinctFromUnion will pull out the distinct from a union operator func isolateDistinctFromUnion(_ *plancontext.PlanningContext, root Operator) Operator { visitor := func(in Operator, _ semantics.TableSet, isRoot bool) (Operator, *ApplyResult) { diff --git a/go/vt/vtgate/planbuilder/operators/phases.go b/go/vt/vtgate/planbuilder/operators/phases.go index 9f2178bae05..68eb0a6559e 100644 --- a/go/vt/vtgate/planbuilder/operators/phases.go +++ b/go/vt/vtgate/planbuilder/operators/phases.go @@ -210,6 +210,50 @@ func enableDelegateAggregation(ctx *plancontext.PlanningContext, op Operator) Op return addColumnsToInput(ctx, op) } +// addColumnsToInput adds columns needed by an operator to its input. +// This happens only when the filter expression can be retrieved as an offset from the underlying mysql. +func addColumnsToInput(ctx *plancontext.PlanningContext, root Operator) Operator { + + addColumnsNeededByFilter := func(in Operator, _ semantics.TableSet, _ bool) (Operator, *ApplyResult) { + addedCols := false + filter, ok := in.(*Filter) + if !ok { + return in, NoRewrite + } + + var neededAggrs []sqlparser.Expr + extractAggrs := func(cursor *sqlparser.CopyOnWriteCursor) { + node := cursor.Node() + if ctx.IsAggr(node) { + neededAggrs = append(neededAggrs, node.(sqlparser.Expr)) + } + } + + for _, expr := range filter.Predicates { + _ = sqlparser.CopyOnRewrite(expr, dontEnterSubqueries, extractAggrs, nil) + } + + if neededAggrs == nil { + return in, NoRewrite + } + + aggregator := findAggregatorInSource(filter.Source) + for _, aggr := range neededAggrs { + if aggregator.FindCol(ctx, aggr, false) == -1 { + aggregator.addColumnWithoutPushing(ctx, aeWrap(aggr), false) + addedCols = true + } + } + + if addedCols { + return in, Rewrote("added columns because filter needs it") + } + return in, NoRewrite + } + + return TopDown(root, TableID, addColumnsNeededByFilter, stopAtRoute) +} + // addOrderingForAllAggregations is run we have pushed down Aggregators as far down as possible. func addOrderingForAllAggregations(ctx *plancontext.PlanningContext, root Operator) Operator { visitor := func(in Operator, _ semantics.TableSet, isRoot bool) (Operator, *ApplyResult) { From 8abec03b41f9d0e4c69bb6a327a01b1a8cd59824 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 9 Aug 2024 10:42:16 +0200 Subject: [PATCH 06/11] don't push down literal evaluation Signed-off-by: Andres Taylor --- .../planbuilder/operators/expressions.go | 12 +++ go/vt/vtgate/planbuilder/operators/phases.go | 2 +- .../planbuilder/operators/projection.go | 14 ++- .../planbuilder/testdata/aggr_cases.json | 66 ++++++++----- .../planbuilder/testdata/cte_cases.json | 67 ++++++++----- .../planbuilder/testdata/from_cases.json | 15 +-- .../planbuilder/testdata/tpch_cases.json | 94 +++++++++++-------- 7 files changed, 168 insertions(+), 102 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/expressions.go b/go/vt/vtgate/planbuilder/operators/expressions.go index 17b4bc7c3f1..4e920d4312c 100644 --- a/go/vt/vtgate/planbuilder/operators/expressions.go +++ b/go/vt/vtgate/planbuilder/operators/expressions.go @@ -59,3 +59,15 @@ func breakExpressionInLHSandRHS( col.Original = expr return } + +// nothingNeedsFetching will return true if all the nodes in the expression are constant +func nothingNeedsFetching(ctx *plancontext.PlanningContext, expr sqlparser.Expr) (constant bool) { + constant = true + _ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { + if mustFetchFromInput(ctx, node) { + constant = false + } + return true, nil + }, expr) + return +} diff --git a/go/vt/vtgate/planbuilder/operators/phases.go b/go/vt/vtgate/planbuilder/operators/phases.go index 68eb0a6559e..bf8e96372bc 100644 --- a/go/vt/vtgate/planbuilder/operators/phases.go +++ b/go/vt/vtgate/planbuilder/operators/phases.go @@ -334,7 +334,7 @@ func addLiteralGroupingToRHS(in *ApplyJoin) (Operator, *ApplyResult) { return nil } if len(aggr.Grouping) == 0 { - gb := sqlparser.NewIntLiteral(".0") + gb := sqlparser.NewFloatLiteral(".0") aggr.Grouping = append(aggr.Grouping, NewGroupBy(gb)) } return nil diff --git a/go/vt/vtgate/planbuilder/operators/projection.go b/go/vt/vtgate/planbuilder/operators/projection.go index d8ede63b612..616731bcfb7 100644 --- a/go/vt/vtgate/planbuilder/operators/projection.go +++ b/go/vt/vtgate/planbuilder/operators/projection.go @@ -383,8 +383,18 @@ func (p *Projection) addColumn( return p.addProjExpr(pe) } - // we need to push down this column to our input - inputOffset := p.Source.AddColumn(ctx, true, addToGroupBy, ae) + var inputOffset int + if nothingNeedsFetching(ctx, expr) { + // if we don't need to fetch anything, we could just evaluate it in the projection + // we still check if it's there - if it is, we can, we should use it + inputOffset = p.Source.FindCol(ctx, expr, false) + if inputOffset < 0 { + return p.addProjExpr(pe) + } + } else { + // we need to push down this column to our input + inputOffset = p.Source.AddColumn(ctx, true, addToGroupBy, ae) + } pe.Info = Offset(inputOffset) // since we already know the offset, let's save the information return p.addProjExpr(pe) diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index c11877e2b66..cd5f052deeb 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -3621,8 +3621,10 @@ "Aggregates": "count_star(0) AS count(*)", "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "3", + "OperatorType": "Projection", + "Expressions": [ + "1 as 1" + ], "Inputs": [ { "OperatorType": "Limit", @@ -3635,8 +3637,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select x.phone, x.id, x.city, 1 from (select phone, id, city from `user` where 1 != 1) as x where 1 != 1", - "Query": "select x.phone, x.id, x.city, 1 from (select phone, id, city from `user` where id > 12) as x limit 10", + "FieldQuery": "select 1 from (select phone, id, city from `user` where 1 != 1) as x where 1 != 1", + "Query": "select 1 from (select phone, id, city from `user` where id > 12) as x limit 10", "Table": "`user`" } ] @@ -3734,8 +3736,12 @@ "ResultColumns": 2, "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "1,2,3", + "OperatorType": "Projection", + "Expressions": [ + ":1 as val1", + "1 as 1", + ":2 as weight_string(val1)" + ], "Inputs": [ { "OperatorType": "Limit", @@ -3748,9 +3754,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select x.id, x.val1, 1, weight_string(x.val1) from (select id, val1 from `user` where 1 != 1) as x where 1 != 1", - "OrderBy": "(1|3) ASC", - "Query": "select x.id, x.val1, 1, weight_string(x.val1) from (select id, val1 from `user` where val2 < 4) as x order by x.val1 asc limit 2", + "FieldQuery": "select x.id, x.val1, weight_string(x.val1) from (select id, val1 from `user` where 1 != 1) as x where 1 != 1", + "OrderBy": "(1|2) ASC", + "Query": "select x.id, x.val1, weight_string(x.val1) from (select id, val1 from `user` where val2 < 4) as x order by x.val1 asc limit 2", "Table": "`user`" } ] @@ -6105,15 +6111,18 @@ "ResultColumns": 1, "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "2,1", + "OperatorType": "Projection", + "Expressions": [ + "1 as 1", + "0 as .0" + ], "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", - "Aggregates": "sum_count_star(0) AS count(*), any_value(2)", + "Aggregates": "sum_count_star(0) AS count(*)", "GroupBy": "1", - "ResultColumns": 3, + "ResultColumns": 1, "Inputs": [ { "OperatorType": "Route", @@ -6122,8 +6131,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select count(*), .0, 1 from `user` where 1 != 1 group by .0", - "Query": "select count(*), .0, 1 from `user` group by .0", + "FieldQuery": "select count(*), .0 from `user` where 1 != 1 group by .0", + "Query": "select count(*), .0 from `user` group by .0", "Table": "`user`" } ] @@ -6687,13 +6696,18 @@ "GroupBy": "1, (2|3)", "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "2,0,1,3", + "OperatorType": "Projection", + "Expressions": [ + "1 as 1", + ":0 as col", + ":1 as bar", + ":2 as weight_string(ue.bar)" + ], "Inputs": [ { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "0 ASC, (1|3) ASC", + "OrderBy": "0 ASC, (1|2) ASC", "Inputs": [ { "OperatorType": "Limit", @@ -6706,8 +6720,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select ue.col, ue.bar, 1, weight_string(ue.bar) from (select col, bar from user_extra where 1 != 1) as ue where 1 != 1", - "Query": "select ue.col, ue.bar, 1, weight_string(ue.bar) from (select col, bar from user_extra) as ue limit 10", + "FieldQuery": "select ue.col, ue.bar, weight_string(ue.bar) from (select col, bar from user_extra where 1 != 1) as ue where 1 != 1", + "Query": "select ue.col, ue.bar, weight_string(ue.bar) from (select col, bar from user_extra) as ue limit 10", "Table": "user_extra" } ] @@ -7253,8 +7267,10 @@ "Aggregates": "count_star(0) AS count(*)", "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "2", + "OperatorType": "Projection", + "Expressions": [ + "1 as 1" + ], "Inputs": [ { "OperatorType": "Limit", @@ -7268,9 +7284,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select subquery_for_count.one, subquery_for_count.id, 1, weight_string(subquery_for_count.id) from (select 1 as one, id from `user` where 1 != 1) as subquery_for_count where 1 != 1", - "OrderBy": "(1|3) DESC", - "Query": "select subquery_for_count.one, subquery_for_count.id, 1, weight_string(subquery_for_count.id) from (select 1 as one, id from `user` where `user`.is_not_deleted = true) as subquery_for_count order by subquery_for_count.id desc limit 25", + "FieldQuery": "select subquery_for_count.one, subquery_for_count.id, weight_string(subquery_for_count.id) from (select 1 as one, id from `user` where 1 != 1) as subquery_for_count where 1 != 1", + "OrderBy": "(1|2) DESC", + "Query": "select subquery_for_count.one, subquery_for_count.id, weight_string(subquery_for_count.id) from (select 1 as one, id from `user` where `user`.is_not_deleted = true) as subquery_for_count order by subquery_for_count.id desc limit 25", "Table": "`user`" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/cte_cases.json b/go/vt/vtgate/planbuilder/testdata/cte_cases.json index 0b801aa1a62..82987fb52f4 100644 --- a/go/vt/vtgate/planbuilder/testdata/cte_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/cte_cases.json @@ -223,8 +223,10 @@ "Aggregates": "count_star(0) AS count(*)", "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "3", + "OperatorType": "Projection", + "Expressions": [ + "1 as 1" + ], "Inputs": [ { "OperatorType": "Limit", @@ -237,8 +239,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select x.phone, x.id, x.city, 1 from (select phone, id, city from `user` where 1 != 1) as x where 1 != 1", - "Query": "select x.phone, x.id, x.city, 1 from (select phone, id, city from `user` where id > 12) as x limit 10", + "FieldQuery": "select 1 from (select phone, id, city from `user` where 1 != 1) as x where 1 != 1", + "Query": "select 1 from (select phone, id, city from `user` where id > 12) as x limit 10", "Table": "`user`" } ] @@ -336,8 +338,12 @@ "ResultColumns": 2, "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "1,2,3", + "OperatorType": "Projection", + "Expressions": [ + ":1 as val1", + "1 as 1", + ":2 as weight_string(val1)" + ], "Inputs": [ { "OperatorType": "Limit", @@ -350,9 +356,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select x.id, x.val1, 1, weight_string(x.val1) from (select id, val1 from `user` where 1 != 1) as x where 1 != 1", - "OrderBy": "(1|3) ASC", - "Query": "select x.id, x.val1, 1, weight_string(x.val1) from (select id, val1 from `user` where val2 < 4) as x order by x.val1 asc limit 2", + "FieldQuery": "select x.id, x.val1, weight_string(x.val1) from (select id, val1 from `user` where 1 != 1) as x where 1 != 1", + "OrderBy": "(1|2) ASC", + "Query": "select x.id, x.val1, weight_string(x.val1) from (select id, val1 from `user` where val2 < 4) as x order by x.val1 asc limit 2", "Table": "`user`" } ] @@ -717,13 +723,15 @@ "Aggregates": "count_star(0) AS count(*)", "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "1", + "OperatorType": "Projection", + "Expressions": [ + "1 as 1" + ], "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Scalar", - "Aggregates": "sum_count_star(0) AS count(*), any_value(1)", + "Aggregates": "sum_count_star(0) AS count(*)", "Inputs": [ { "OperatorType": "Route", @@ -732,8 +740,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select count(*), 1 from `user` where 1 != 1", - "Query": "select count(*), 1 from `user`", + "FieldQuery": "select count(*) from `user` where 1 != 1", + "Query": "select count(*) from `user`", "Table": "`user`" } ] @@ -1998,8 +2006,11 @@ "Aggregates": "any_value(0), sum(1) AS sum(num)", "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "1,0", + "OperatorType": "Projection", + "Expressions": [ + "1000 as 1000", + ":0 as num" + ], "Inputs": [ { "OperatorType": "Concatenate", @@ -2011,11 +2022,13 @@ { "OperatorType": "Aggregate", "Variant": "Scalar", - "Aggregates": "count_star(0) AS num, any_value(1)", + "Aggregates": "count_star(0) AS num", "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "1,2", + "OperatorType": "Projection", + "Expressions": [ + "1 as 1" + ], "Inputs": [ { "OperatorType": "Limit", @@ -2028,8 +2041,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select t.id, 1, 1000 from (select `user`.id from `user` where 1 != 1) as t where 1 != 1", - "Query": "select t.id, 1, 1000 from (select `user`.id from `user` where `user`.textcol1 = 'open' and `user`.intcol = 1) as t limit :__upper_limit", + "FieldQuery": "select 1 from (select `user`.id from `user` where 1 != 1) as t where 1 != 1", + "Query": "select 1 from (select `user`.id from `user` where `user`.textcol1 = 'open' and `user`.intcol = 1) as t limit :__upper_limit", "Table": "`user`" } ] @@ -2047,11 +2060,13 @@ { "OperatorType": "Aggregate", "Variant": "Scalar", - "Aggregates": "count_star(0) AS num, any_value(1)", + "Aggregates": "count_star(0) AS num", "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "1,2", + "OperatorType": "Projection", + "Expressions": [ + "1 as 1" + ], "Inputs": [ { "OperatorType": "Limit", @@ -2064,8 +2079,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select t.id, 1, 1000 from (select `user`.id from `user` where 1 != 1) as t where 1 != 1", - "Query": "select t.id, 1, 1000 from (select `user`.id from `user` where `user`.textcol1 = 'closed' and `user`.intcol = 1) as t limit :__upper_limit", + "FieldQuery": "select 1 from (select `user`.id from `user` where 1 != 1) as t where 1 != 1", + "Query": "select 1 from (select `user`.id from `user` where `user`.textcol1 = 'closed' and `user`.intcol = 1) as t limit :__upper_limit", "Table": "`user`" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/from_cases.json b/go/vt/vtgate/planbuilder/testdata/from_cases.json index 19388aec78e..da869bb5e69 100644 --- a/go/vt/vtgate/planbuilder/testdata/from_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/from_cases.json @@ -4593,20 +4593,21 @@ "Aggregates": "count_star(0) AS count(*)", "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "1", + "OperatorType": "Projection", + "Expressions": [ + "1 as 1" + ], "Inputs": [ { "OperatorType": "Distinct", "Collations": [ - "(0:2)", - "1" + "(0:1)" ], "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "R:0,L:1,R:1", + "JoinColumnIndexes": "R:0,R:1", "JoinVars": { "m_id": 0 }, @@ -4619,8 +4620,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select subquery_for_count.`m.id`, 1 from (select m.id as `m.id` from music as m where 1 != 1) as subquery_for_count where 1 != 1", - "Query": "select distinct subquery_for_count.`m.id`, 1 from (select m.id as `m.id` from music as m) as subquery_for_count", + "FieldQuery": "select subquery_for_count.`m.id` from (select m.id as `m.id` from music as m where 1 != 1) as subquery_for_count where 1 != 1", + "Query": "select distinct subquery_for_count.`m.id` from (select m.id as `m.id` from music as m) as subquery_for_count", "Table": "music" }, { diff --git a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json index b0540b011e5..2f8a927d0c8 100644 --- a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json @@ -1889,23 +1889,25 @@ "GroupBy": "0", "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": "1,3", + "OperatorType": "Projection", + "Expressions": [ + ":1 as c_count", + "1 as 1" + ], "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", - "Aggregates": "sum_count(1) AS count(o_orderkey), any_value(3)", + "Aggregates": "sum_count(1) AS count(o_orderkey)", "GroupBy": "(0|2)", - "ResultColumns": 4, + "ResultColumns": 2, "Inputs": [ { "OperatorType": "Projection", "Expressions": [ ":2 as c_custkey", "count(*) * count(o_orderkey) as count(o_orderkey)", - ":3 as weight_string(c_custkey)", - ":4 as 1" + ":3 as weight_string(c_custkey)" ], "Inputs": [ { @@ -1916,7 +1918,7 @@ { "OperatorType": "Join", "Variant": "LeftJoin", - "JoinColumnIndexes": "R:0,L:0,L:1,L:2,L:3", + "JoinColumnIndexes": "R:0,L:0,L:1,L:2", "JoinVars": { "c_custkey": 1 }, @@ -1929,9 +1931,9 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select count(*), c_custkey, weight_string(c_custkey), 1 from customer where 1 != 1 group by c_custkey, weight_string(c_custkey)", + "FieldQuery": "select count(*), c_custkey, weight_string(c_custkey) from customer where 1 != 1 group by c_custkey, weight_string(c_custkey)", "OrderBy": "(1|2) ASC", - "Query": "select count(*), c_custkey, weight_string(c_custkey), 1 from customer group by c_custkey, weight_string(c_custkey) order by c_custkey asc", + "Query": "select count(*), c_custkey, weight_string(c_custkey) from customer group by c_custkey, weight_string(c_custkey) order by c_custkey asc", "Table": "customer" }, { @@ -1983,41 +1985,51 @@ "Aggregates": "any_value(0), sum(1) AS sum(case when p_type like 'PROMO%' then l_extendedprice * (1 - l_discount) else 0 end), sum(2) AS sum(l_extendedprice * (1 - l_discount))", "Inputs": [ { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "L:0,R:0,L:3", - "JoinVars": { - "l_discount": 2, - "l_extendedprice": 1, - "l_partkey": 4 - }, - "TableName": "lineitem_part", + "OperatorType": "Projection", + "Expressions": [ + "100.00 as 100.00", + ":0 as case when p_type like 'PROMO%' then l_extendedprice * (1 - l_discount) else 0 end", + ":1 as l_extendedprice * (1 - l_discount)" + ], "Inputs": [ { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select 100.00, l_extendedprice, l_discount, l_extendedprice * (1 - l_discount), l_partkey from lineitem where 1 != 1", - "Query": "select 100.00, l_extendedprice, l_discount, l_extendedprice * (1 - l_discount), l_partkey from lineitem where l_shipdate >= date('1995-09-01') and l_shipdate < date('1995-09-01') + interval '1' month", - "Table": "lineitem" - }, - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:2", + "JoinVars": { + "l_discount": 1, + "l_extendedprice": 0, + "l_partkey": 3 }, - "FieldQuery": "select case when p_type like 'PROMO%' then :l_extendedprice * (1 - :l_discount) else 0 end from part where 1 != 1", - "Query": "select case when p_type like 'PROMO%' then :l_extendedprice * (1 - :l_discount) else 0 end from part where p_partkey = :l_partkey", - "Table": "part", - "Values": [ - ":l_partkey" - ], - "Vindex": "hash" + "TableName": "lineitem_part", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select l_extendedprice, l_discount, l_extendedprice * (1 - l_discount), l_partkey from lineitem where 1 != 1", + "Query": "select l_extendedprice, l_discount, l_extendedprice * (1 - l_discount), l_partkey from lineitem where l_shipdate >= date('1995-09-01') and l_shipdate < date('1995-09-01') + interval '1' month", + "Table": "lineitem" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select case when p_type like 'PROMO%' then :l_extendedprice * (1 - :l_discount) else 0 end from part where 1 != 1", + "Query": "select case when p_type like 'PROMO%' then :l_extendedprice * (1 - :l_discount) else 0 end from part where p_partkey = :l_partkey", + "Table": "part", + "Values": [ + ":l_partkey" + ], + "Vindex": "hash" + } + ] } ] } From 535fa3915d13665f1519a3bcaa87aef111a5b5fa Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 9 Aug 2024 10:58:25 +0200 Subject: [PATCH 07/11] refactor test Signed-off-by: Andres Taylor --- go/vt/vtgate/endtoend/aggr_test.go | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/go/vt/vtgate/endtoend/aggr_test.go b/go/vt/vtgate/endtoend/aggr_test.go index b37697a72f4..402cecc0c6d 100644 --- a/go/vt/vtgate/endtoend/aggr_test.go +++ b/go/vt/vtgate/endtoend/aggr_test.go @@ -21,37 +21,35 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/test/utils" + "vitess.io/vitess/go/mysql" ) func TestAggregateTypes(t *testing.T) { ctx := context.Background() conn, err := mysql.Connect(ctx, &vtParams) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer conn.Close() exec(t, conn, "insert into aggr_test(id, val1, val2) values(1,'a',1), (2,'A',1), (3,'b',1), (4,'c',3), (5,'c',4)") exec(t, conn, "insert into aggr_test(id, val1, val2) values(6,'d',null), (7,'e',null), (8,'E',1)") qr := exec(t, conn, "select val1, count(distinct val2), count(*) from aggr_test group by val1") - if got, want := fmt.Sprintf("%v", qr.Rows), `[[VARCHAR("a") INT64(1) INT64(2)] [VARCHAR("b") INT64(1) INT64(1)] [VARCHAR("c") INT64(2) INT64(2)] [VARCHAR("d") INT64(0) INT64(1)] [VARCHAR("e") INT64(1) INT64(2)]]`; got != want { - t.Errorf("select:\n%v want\n%v", got, want) - } + want := `[[VARCHAR("a") INT64(1) INT64(2)] [VARCHAR("b") INT64(1) INT64(1)] [VARCHAR("c") INT64(2) INT64(2)] [VARCHAR("d") INT64(0) INT64(1)] [VARCHAR("e") INT64(1) INT64(2)]]` + utils.MustMatch(t, want, fmt.Sprintf("%v", qr.Rows)) qr = exec(t, conn, "select val1, sum(distinct val2), sum(val2) from aggr_test group by val1") - if got, want := fmt.Sprintf("%v", qr.Rows), `[[VARCHAR("a") DECIMAL(1) DECIMAL(2)] [VARCHAR("b") DECIMAL(1) DECIMAL(1)] [VARCHAR("c") DECIMAL(7) DECIMAL(7)] [VARCHAR("d") NULL NULL] [VARCHAR("e") DECIMAL(1) DECIMAL(1)]]`; got != want { - t.Errorf("select:\n%v want\n%v", got, want) - } + want = `[[VARCHAR("a") DECIMAL(1) DECIMAL(2)] [VARCHAR("b") DECIMAL(1) DECIMAL(1)] [VARCHAR("c") DECIMAL(7) DECIMAL(7)] [VARCHAR("d") NULL NULL] [VARCHAR("e") DECIMAL(1) DECIMAL(1)]]` + utils.MustMatch(t, want, fmt.Sprintf("%v", qr.Rows)) qr = exec(t, conn, "select val1, count(distinct val2) k, count(*) from aggr_test group by val1 order by k desc, val1") - if got, want := fmt.Sprintf("%v", qr.Rows), `[[VARCHAR("c") INT64(2) INT64(2)] [VARCHAR("a") INT64(1) INT64(2)] [VARCHAR("b") INT64(1) INT64(1)] [VARCHAR("e") INT64(1) INT64(2)] [VARCHAR("d") INT64(0) INT64(1)]]`; got != want { - t.Errorf("select:\n%v want\n%v", got, want) - } + want = `[[VARCHAR("c") INT64(2) INT64(2)] [VARCHAR("a") INT64(1) INT64(2)] [VARCHAR("b") INT64(1) INT64(1)] [VARCHAR("e") INT64(1) INT64(2)] [VARCHAR("d") INT64(0) INT64(1)]]` + utils.MustMatch(t, want, fmt.Sprintf("%v", qr.Rows)) qr = exec(t, conn, "select val1, count(distinct val2) k, count(*) from aggr_test group by val1 order by k desc, val1 limit 4") - if got, want := fmt.Sprintf("%v", qr.Rows), `[[VARCHAR("c") INT64(2) INT64(2)] [VARCHAR("a") INT64(1) INT64(2)] [VARCHAR("b") INT64(1) INT64(1)] [VARCHAR("e") INT64(1) INT64(2)]]`; got != want { - t.Errorf("select:\n%v want\n%v", got, want) - } + want = `[[VARCHAR("c") INT64(2) INT64(2)] [VARCHAR("a") INT64(1) INT64(2)] [VARCHAR("b") INT64(1) INT64(1)] [VARCHAR("e") INT64(1) INT64(2)]]` + utils.MustMatch(t, want, fmt.Sprintf("%v", qr.Rows)) } From 83f560c8ef625e095e324a21824e78d6c7ca3258 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 9 Aug 2024 11:05:30 +0200 Subject: [PATCH 08/11] update test expectations Signed-off-by: Andres Taylor --- go/vt/vtgate/executor_select_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index 6e3bcdd3eda..51158818ec5 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -3913,15 +3913,15 @@ func TestSelectAggregationNoData(t *testing.T) { { sql: `select count(*) from (select col1, col2 from user limit 2) x`, sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2|1", "int64|int64|int64")), - expSandboxQ: "select x.col1, x.col2, 1 from (select col1, col2 from `user`) as x limit 2", - expField: `[name:"count(*)" type:INT64]`, + expSandboxQ: "select 1 from (select col1, col2 from `user`) as x limit 2", + expField: `[name:"count(*)" type:INT64 charset:63 flags:32769]`, expRow: `[[INT64(0)]]`, }, { sql: `select col2, count(*) from (select col1, col2 from user limit 2) x group by col2`, sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2|1|weight_string(col2)", "int64|int64|int64|varbinary")), - expSandboxQ: "select x.col1, x.col2, 1, weight_string(x.col2) from (select col1, col2 from `user`) as x limit 2", - expField: `[name:"col2" type:INT64 name:"count(*)" type:INT64]`, + expSandboxQ: "select x.col1, x.col2, weight_string(x.col2) from (select col1, col2 from `user`) as x limit 2", + expField: `[name:"col2" type:INT64 charset:63 flags:32768 name:"count(*)" type:INT64 charset:63 flags:32769]`, expRow: `[]`, }, } @@ -4005,15 +4005,15 @@ func TestSelectAggregationData(t *testing.T) { { sql: `select count(*) from (select col1, col2 from user limit 2) x`, sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2|1", "int64|int64|int64"), "100|200|1", "200|300|1"), - expSandboxQ: "select x.col1, x.col2, 1 from (select col1, col2 from `user`) as x limit 2", - expField: `[name:"count(*)" type:INT64]`, + expSandboxQ: "select 1 from (select col1, col2 from `user`) as x limit 2", + expField: `[name:"count(*)" type:INT64 charset:63 flags:32769]`, expRow: `[[INT64(2)]]`, }, { sql: `select col2, count(*) from (select col1, col2 from user limit 9) x group by col2`, sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2|1|weight_string(col2)", "int64|int64|int64|varbinary"), "100|3|1|NULL", "200|2|1|NULL"), - expSandboxQ: "select x.col1, x.col2, 1, weight_string(x.col2) from (select col1, col2 from `user`) as x limit 9", - expField: `[name:"col2" type:INT64 name:"count(*)" type:INT64]`, + expSandboxQ: "select x.col1, x.col2, weight_string(x.col2) from (select col1, col2 from `user`) as x limit 9", + expField: `[name:"col2" type:INT64 charset:63 flags:32768 name:"count(*)" type:INT64 charset:63 flags:32769]`, expRow: `[[INT64(2) INT64(4)] [INT64(3) INT64(5)]]`, }, { From 703dfa4d9996911822b58416c10c30d0653154bc Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 13 Aug 2024 15:25:55 +0200 Subject: [PATCH 09/11] handle derived projection pushing better Signed-off-by: Andres Taylor --- .../planbuilder/operators/apply_join.go | 23 +++----- .../planbuilder/operators/projection.go | 2 +- .../operators/projection_pushing.go | 53 ++++++++++++++----- .../planbuilder/testdata/from_cases.json | 4 +- .../planbuilder/testdata/tpch_cases.json | 10 ++-- 5 files changed, 54 insertions(+), 38 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/apply_join.go b/go/vt/vtgate/planbuilder/operators/apply_join.go index 0900384dc5e..f7bd5b131b8 100644 --- a/go/vt/vtgate/planbuilder/operators/apply_join.go +++ b/go/vt/vtgate/planbuilder/operators/apply_join.go @@ -69,11 +69,10 @@ type ( // so they can be used for the result of this expression that is using data from both sides. // All fields will be used for these applyJoinColumn struct { - Original sqlparser.Expr // this is the original expression being passed through - LHSExprs []BindVarExpr // These are the expressions we are pushing to the left hand side which we'll receive as bind variables - RHSExpr sqlparser.Expr // This the expression that we'll evaluate on the right hand side. This is nil, if the right hand side has nothing. - DTColName *sqlparser.ColName // This is the output column name that the parent of JOIN will be seeing. If this is unset, then the colname is the String(Original). We set this when we push Projections with derived tables underneath a Join. - GroupBy bool // if this is true, we need to push this down to our inputs with addToGroupBy set to true + Original sqlparser.Expr // this is the original expression being passed through + LHSExprs []BindVarExpr // These are the expressions we are pushing to the left hand side which we'll receive as bind variables + RHSExpr sqlparser.Expr // This the expression that we'll evaluate on the right hand side. This is nil, if the right hand side has nothing. + GroupBy bool // if this is true, we need to push this down to our inputs with addToGroupBy set to true } // BindVarExpr is an expression needed from one side of a join/subquery, and the argument name for it. @@ -225,8 +224,7 @@ func (aj *ApplyJoin) getJoinColumnFor(ctx *plancontext.PlanningContext, orig *sq func applyJoinCompare(ctx *plancontext.PlanningContext, expr sqlparser.Expr) func(e applyJoinColumn) bool { return func(e applyJoinColumn) bool { - // e.DTColName is how the outside world will be using this expression. So we should check for an equality with that too. - return ctx.SemTable.EqualsExprWithDeps(e.Original, expr) || ctx.SemTable.EqualsExprWithDeps(e.DTColName, expr) + return ctx.SemTable.EqualsExprWithDeps(e.Original, expr) } } @@ -447,11 +445,8 @@ func (jc applyJoinColumn) String() string { lhs := slice.Map(jc.LHSExprs, func(e BindVarExpr) string { return sqlparser.String(e.Expr) }) - if jc.DTColName == nil { - return fmt.Sprintf("[%s | %s | %s]", strings.Join(lhs, ", "), rhs, sqlparser.String(jc.Original)) - } - return fmt.Sprintf("[%s | %s | %s | %s]", strings.Join(lhs, ", "), rhs, sqlparser.String(jc.Original), sqlparser.String(jc.DTColName)) + return fmt.Sprintf("[%s | %s | %s]", strings.Join(lhs, ", "), rhs, sqlparser.String(jc.Original)) } func (jc applyJoinColumn) IsPureLeft() bool { @@ -467,16 +462,10 @@ func (jc applyJoinColumn) IsMixedLeftAndRight() bool { } func (jc applyJoinColumn) GetPureLeftExpr() sqlparser.Expr { - if jc.DTColName != nil { - return jc.DTColName - } return jc.LHSExprs[0].Expr } func (jc applyJoinColumn) GetRHSExpr() sqlparser.Expr { - if jc.DTColName != nil { - return jc.DTColName - } return jc.RHSExpr } diff --git a/go/vt/vtgate/planbuilder/operators/projection.go b/go/vt/vtgate/planbuilder/operators/projection.go index 616731bcfb7..95ebeadaeb7 100644 --- a/go/vt/vtgate/planbuilder/operators/projection.go +++ b/go/vt/vtgate/planbuilder/operators/projection.go @@ -87,7 +87,7 @@ type ( ProjExpr struct { Original *sqlparser.AliasedExpr // this is the expression the user asked for. should only be used to decide on the column alias - EvalExpr sqlparser.Expr // EvalExpr is the expression that will be evaluated at runtime + EvalExpr sqlparser.Expr // EvalExpr represents the expression evaluated at runtime or used when the ProjExpr is pushed under a route ColExpr sqlparser.Expr // ColExpr is used during planning to figure out which column this ProjExpr is representing Info ExprInfo // Here we store information about evalengine, offsets or subqueries } diff --git a/go/vt/vtgate/planbuilder/operators/projection_pushing.go b/go/vt/vtgate/planbuilder/operators/projection_pushing.go index b5c6a10bb78..21d2a058a86 100644 --- a/go/vt/vtgate/planbuilder/operators/projection_pushing.go +++ b/go/vt/vtgate/planbuilder/operators/projection_pushing.go @@ -320,7 +320,7 @@ func splitSubqueryExpression( alias string, ) applyJoinColumn { col := join.getJoinColumnFor(ctx, pe.Original, pe.ColExpr, false) - return pushDownSplitJoinCol(col, lhs, pe, alias, rhs) + return pushDownSplitJoinCol(col, lhs, rhs, pe, alias) } func splitUnexploredExpression( @@ -334,23 +334,50 @@ func splitUnexploredExpression( original := sqlparser.Clone(pe.Original) expr := pe.ColExpr - var colName *sqlparser.ColName - if dt != nil { - if !pe.isSameInAndOut(ctx) { - panic(vterrors.VT13001("derived table columns must be the same in and out")) - } - colName = sqlparser.NewColNameWithQualifier(pe.Original.ColumnName(), sqlparser.NewTableName(dt.Alias)) - ctx.SemTable.CopySemanticInfo(expr, colName) - } - // Get a applyJoinColumn for the current expression. col := join.getJoinColumnFor(ctx, original, expr, false) - col.DTColName = colName - return pushDownSplitJoinCol(col, lhs, pe, alias, rhs) + if dt == nil { + return pushDownSplitJoinCol(col, lhs, rhs, pe, alias) + } + + if !pe.isSameInAndOut(ctx) { + panic(vterrors.VT13001("derived table columns must be the same in and out")) + } + // we are pushing a derived projection through a join. that means that after this rewrite, we are on top of the + // derived table divider, and can only see the projected columns, not the underlying expressions + colName := sqlparser.NewColNameWithQualifier(pe.Original.ColumnName(), sqlparser.NewTableName(dt.Alias)) + ctx.SemTable.CopySemanticInfo(expr, colName) + col.Original = colName + if alias == "" { + alias = pe.Original.ColumnName() + } + + // Update the left and right child columns and names based on the applyJoinColumn type. + switch { + case col.IsPureLeft(): + lhs.add(pe, alias) + col.LHSExprs[0].Expr = colName + case col.IsPureRight(): + rhs.add(pe, alias) + col.RHSExpr = colName + case col.IsMixedLeftAndRight(): + for _, lhsExpr := range col.LHSExprs { + ae := aeWrap(lhsExpr.Expr) + columnName := ae.ColumnName() + ae.As = sqlparser.NewIdentifierCI(columnName) + lhs.add(newProjExpr(ae), columnName) + } + innerPE := newProjExprWithInner(pe.Original, col.RHSExpr) + innerPE.ColExpr = col.RHSExpr + col.RHSExpr = colName + innerPE.Info = pe.Info + rhs.add(innerPE, alias) + } + return col } -func pushDownSplitJoinCol(col applyJoinColumn, lhs *projector, pe *ProjExpr, alias string, rhs *projector) applyJoinColumn { +func pushDownSplitJoinCol(col applyJoinColumn, lhs, rhs *projector, pe *ProjExpr, alias string) applyJoinColumn { // Update the left and right child columns and names based on the applyJoinColumn type. switch { case col.IsPureLeft(): diff --git a/go/vt/vtgate/planbuilder/testdata/from_cases.json b/go/vt/vtgate/planbuilder/testdata/from_cases.json index da869bb5e69..9a0ffa37f0c 100644 --- a/go/vt/vtgate/planbuilder/testdata/from_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/from_cases.json @@ -4803,8 +4803,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select dt.foo from (select u.foo from `user` as u where 1 != 1) as dt where 1 != 1", - "Query": "select dt.foo from (select u.foo from `user` as u) as dt", + "FieldQuery": "select dt.foo from (select u.foo as foo from `user` as u where 1 != 1) as dt where 1 != 1", + "Query": "select dt.foo from (select u.foo as foo from `user` as u) as dt", "Table": "`user`" }, { diff --git a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json index 2f8a927d0c8..6e67d4f9e8d 100644 --- a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json @@ -1132,7 +1132,7 @@ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,R:0,R:1,R:2,R:3,R:4,L:3,R:5", + "JoinColumnIndexes": "L:0,R:0,R:1,R:2,R:3,R:4,L:2,R:5", "JoinVars": { "o_orderkey": 1 }, @@ -1145,8 +1145,8 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select profit.o_year, profit.o_orderkey, weight_string(profit.o_year), weight_string(extract(year from o_orderdate)) from (select extract(year from o_orderdate) as o_year, o_orderkey as o_orderkey from orders where 1 != 1) as profit where 1 != 1", - "Query": "select profit.o_year, profit.o_orderkey, weight_string(profit.o_year), weight_string(extract(year from o_orderdate)) from (select extract(year from o_orderdate) as o_year, o_orderkey as o_orderkey from orders) as profit", + "FieldQuery": "select profit.o_year, profit.o_orderkey, weight_string(profit.o_year) from (select extract(year from o_orderdate) as o_year, o_orderkey as o_orderkey from orders where 1 != 1) as profit where 1 != 1", + "Query": "select profit.o_year, profit.o_orderkey, weight_string(profit.o_year) from (select extract(year from o_orderdate) as o_year, o_orderkey as o_orderkey from orders) as profit", "Table": "orders" }, { @@ -1192,8 +1192,8 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select profit.l_extendedprice, profit.l_discount, profit.l_quantity, profit.l_suppkey, profit.l_partkey, weight_string(profit.l_suppkey) from (select l_extendedprice, l_discount, l_quantity, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem where 1 != 1) as profit where 1 != 1", - "Query": "select profit.l_extendedprice, profit.l_discount, profit.l_quantity, profit.l_suppkey, profit.l_partkey, weight_string(profit.l_suppkey) from (select l_extendedprice, l_discount, l_quantity, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem where l_orderkey = :o_orderkey) as profit", + "FieldQuery": "select profit.l_extendedprice, profit.l_discount, profit.l_quantity, profit.l_suppkey, profit.l_partkey, weight_string(profit.l_suppkey) from (select l_extendedprice as l_extendedprice, l_discount as l_discount, l_quantity as l_quantity, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem where 1 != 1) as profit where 1 != 1", + "Query": "select profit.l_extendedprice, profit.l_discount, profit.l_quantity, profit.l_suppkey, profit.l_partkey, weight_string(profit.l_suppkey) from (select l_extendedprice as l_extendedprice, l_discount as l_discount, l_quantity as l_quantity, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem where l_orderkey = :o_orderkey) as profit", "Table": "lineitem" } ] From 131b98d2496835960de3246da1119ee5fea33c42 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Wed, 14 Aug 2024 10:22:04 +0200 Subject: [PATCH 10/11] limit result columns only when needed Signed-off-by: Andres Taylor --- .../planbuilder/operators/aggregator.go | 11 ++++++++- .../vtgate/planbuilder/operators/ast_to_op.go | 12 ++++++++-- go/vt/vtgate/planbuilder/operators/horizon.go | 2 ++ .../operators/horizon_expanding.go | 14 +++++++---- .../planbuilder/testdata/aggr_cases.json | 24 ++++++------------- .../planbuilder/testdata/cte_cases.json | 2 -- .../planbuilder/testdata/from_cases.json | 1 - .../testdata/memory_sort_cases.json | 8 +++---- .../testdata/postprocess_cases.json | 3 +-- .../planbuilder/testdata/select_cases.json | 4 ---- .../planbuilder/testdata/tpch_cases.json | 14 ++++------- 11 files changed, 47 insertions(+), 48 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/aggregator.go b/go/vt/vtgate/planbuilder/operators/aggregator.go index 624a0a297ba..cfeaf1c8cdc 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregator.go +++ b/go/vt/vtgate/planbuilder/operators/aggregator.go @@ -57,6 +57,9 @@ type ( // This is used to truncate the columns in the final result ResultColumns int + // Truncate is set to true if the columns produced by this operator should be truncated if we added any additional columns + Truncate bool + QP *QueryProjection DT *DerivedTable @@ -541,6 +544,12 @@ func (a *Aggregator) pushRemainingGroupingColumnsAndWeightStrings(ctx *planconte } func (a *Aggregator) internalAddWSColumn(ctx *plancontext.PlanningContext, inOffset int, aliasedExpr *sqlparser.AliasedExpr) int { + if a.ResultColumns == 0 && a.Truncate { + // if we need to use `internalAddColumn`, it means we are adding columns that are not part of the original list, + // so we need to set the ResultColumns to the current length of the columns list + a.ResultColumns = len(a.Columns) + } + offset := a.Source.AddWSColumn(ctx, inOffset, false) if offset == len(a.Columns) { @@ -559,7 +568,7 @@ func (a *Aggregator) getTruncateColumnCount() int { } func (a *Aggregator) internalAddColumn(ctx *plancontext.PlanningContext, aliasedExpr *sqlparser.AliasedExpr, addToGroupBy bool) int { - if a.ResultColumns == 0 { + if a.ResultColumns == 0 && a.Truncate { // if we need to use `internalAddColumn`, it means we are adding columns that are not part of the original list, // so we need to set the ResultColumns to the current length of the columns list a.ResultColumns = len(a.Columns) diff --git a/go/vt/vtgate/planbuilder/operators/ast_to_op.go b/go/vt/vtgate/planbuilder/operators/ast_to_op.go index 0d838610866..f017f77d6a3 100644 --- a/go/vt/vtgate/planbuilder/operators/ast_to_op.go +++ b/go/vt/vtgate/planbuilder/operators/ast_to_op.go @@ -148,8 +148,8 @@ func createOperatorFromUnion(ctx *plancontext.PlanningContext, node *sqlparser.U if isRHSUnion { panic(vterrors.VT12001("nesting of UNIONs on the right-hand side")) } - opLHS := translateQueryToOp(ctx, node.Left) - opRHS := translateQueryToOp(ctx, node.Right) + opLHS := translateQueryToOpForUnion(ctx, node.Left) + opRHS := translateQueryToOpForUnion(ctx, node.Right) lexprs := ctx.SemTable.SelectExprs(node.Left) rexprs := ctx.SemTable.SelectExprs(node.Right) @@ -158,6 +158,14 @@ func createOperatorFromUnion(ctx *plancontext.PlanningContext, node *sqlparser.U return newHorizon(union, node) } +func translateQueryToOpForUnion(ctx *plancontext.PlanningContext, node sqlparser.SelectStatement) Operator { + op := translateQueryToOp(ctx, node) + if hz, ok := op.(*Horizon); ok { + hz.Truncate = true + } + return op +} + // createOpFromStmt creates an operator from the given statement. It takes in two additional arguments— // 1. verifyAllFKs: For this given statement, do we need to verify validity of all the foreign keys on the vtgate level. // 2. fkToIgnore: The foreign key constraint to specifically ignore while planning the statement. This field is used in UPDATE CASCADE planning, wherein while planning the child update diff --git a/go/vt/vtgate/planbuilder/operators/horizon.go b/go/vt/vtgate/planbuilder/operators/horizon.go index 7539704c2a9..70186c062b9 100644 --- a/go/vt/vtgate/planbuilder/operators/horizon.go +++ b/go/vt/vtgate/planbuilder/operators/horizon.go @@ -49,6 +49,8 @@ type Horizon struct { // Columns needed to feed other plans Columns []*sqlparser.ColName ColumnsOffset []int + + Truncate bool } func newHorizon(src Operator, query sqlparser.SelectStatement) *Horizon { diff --git a/go/vt/vtgate/planbuilder/operators/horizon_expanding.go b/go/vt/vtgate/planbuilder/operators/horizon_expanding.go index fd3d992d0ca..29c1b1033f1 100644 --- a/go/vt/vtgate/planbuilder/operators/horizon_expanding.go +++ b/go/vt/vtgate/planbuilder/operators/horizon_expanding.go @@ -208,7 +208,7 @@ func createProjectionFromSelect(ctx *plancontext.PlanningContext, horizon *Horiz } if qp.NeedsAggregation() { - return createProjectionWithAggr(ctx, qp, dt, horizon.src()) + return createProjectionWithAggr(ctx, qp, dt, horizon) } projX := createProjectionWithoutAggr(ctx, qp, horizon.src()) @@ -216,8 +216,9 @@ func createProjectionFromSelect(ctx *plancontext.PlanningContext, horizon *Horiz return projX } -func createProjectionWithAggr(ctx *plancontext.PlanningContext, qp *QueryProjection, dt *DerivedTable, src Operator) Operator { +func createProjectionWithAggr(ctx *plancontext.PlanningContext, qp *QueryProjection, dt *DerivedTable, horizon *Horizon) Operator { aggregations, complexAggr := qp.AggregationExpressions(ctx, true) + src := horizon.Source aggrOp := &Aggregator{ Source: src, Original: true, @@ -239,7 +240,11 @@ func createProjectionWithAggr(ctx *plancontext.PlanningContext, qp *QueryProject if complexAggr { return createProjectionForComplexAggregation(aggrOp, qp) } - return createProjectionForSimpleAggregation(ctx, aggrOp, qp) + + addAllColumnsToAggregator(ctx, aggrOp, qp) + aggrOp.Truncate = horizon.Truncate + + return aggrOp } func pullOutValueSubqueries(ctx *plancontext.PlanningContext, aggr Aggr, sqc *SubQueryBuilder, outerID semantics.TableSet) Aggr { @@ -261,7 +266,7 @@ func pullOutValueSubqueries(ctx *plancontext.PlanningContext, aggr Aggr, sqc *Su return aggr } -func createProjectionForSimpleAggregation(ctx *plancontext.PlanningContext, a *Aggregator, qp *QueryProjection) Operator { +func addAllColumnsToAggregator(ctx *plancontext.PlanningContext, a *Aggregator, qp *QueryProjection) { outer: for colIdx, expr := range qp.SelectExprs { ae, err := expr.GetAliasedExpr() @@ -292,7 +297,6 @@ outer: } panic(vterrors.VT13001(fmt.Sprintf("Could not find the %s in aggregation in the original query", sqlparser.String(ae)))) } - return a } func createProjectionForComplexAggregation(a *Aggregator, qp *QueryProjection) Operator { diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index cd5f052deeb..343159bccaa 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -689,13 +689,13 @@ "OperatorType": "Sort", "Variant": "Memory", "OrderBy": "1 ASC", + "ResultColumns": 2, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "count_distinct(1|3) AS k", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Route", @@ -2149,13 +2149,13 @@ "Instructions": { "OperatorType": "Filter", "Predicate": "count(*) <= 10", + "ResultColumns": 2, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum_count_star(1) AS a", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Route", @@ -2187,13 +2187,13 @@ "Instructions": { "OperatorType": "Filter", "Predicate": "count(*) = 1.00", + "ResultColumns": 2, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum_count_star(0) AS a", "GroupBy": "(1|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Route", @@ -3097,13 +3097,13 @@ "Instructions": { "OperatorType": "Filter", "Predicate": "count(*) = 3", + "ResultColumns": 2, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum_count_star(1) AS count(*)", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Route", @@ -3135,13 +3135,13 @@ "Instructions": { "OperatorType": "Filter", "Predicate": "sum(foo) + sum(bar) = 42", + "ResultColumns": 3, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum(1) AS sum(foo), sum(2) AS sum(bar)", "GroupBy": "(0|3)", - "ResultColumns": 3, "Inputs": [ { "OperatorType": "Route", @@ -3173,13 +3173,13 @@ "Instructions": { "OperatorType": "Filter", "Predicate": "sum(`user`.foo) + sum(bar) = 42", + "ResultColumns": 3, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum(1) AS fooSum, sum(2) AS barSum", "GroupBy": "(0|3)", - "ResultColumns": 3, "Inputs": [ { "OperatorType": "Route", @@ -3218,7 +3218,6 @@ "Variant": "Ordered", "Aggregates": "sum_count_star(1) AS count(*)", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Route", @@ -3257,7 +3256,6 @@ "Variant": "Ordered", "Aggregates": "sum_count(1) AS count(u.`name`)", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Projection", @@ -3362,7 +3360,6 @@ "Variant": "Ordered", "Aggregates": "sum_count_star(1) AS count(*)", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Projection", @@ -4118,7 +4115,6 @@ "Variant": "Ordered", "Aggregates": "max(1|3) AS bazo", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Route", @@ -4157,7 +4153,6 @@ "Variant": "Ordered", "Aggregates": "sum_count(1) AS bazo", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Route", @@ -5005,13 +5000,13 @@ "Collations": [ "0" ], + "ResultColumns": 1, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum_count_star(0) AS count(*)", "GroupBy": "1", - "ResultColumns": 1, "Inputs": [ { "OperatorType": "Route", @@ -6108,7 +6103,6 @@ "Variant": "Ordered", "Aggregates": "count_star(0)", "GroupBy": "1", - "ResultColumns": 1, "Inputs": [ { "OperatorType": "Projection", @@ -6122,7 +6116,6 @@ "Variant": "Ordered", "Aggregates": "sum_count_star(0) AS count(*)", "GroupBy": "1", - "ResultColumns": 1, "Inputs": [ { "OperatorType": "Route", @@ -6296,7 +6289,6 @@ "Variant": "Ordered", "Aggregates": "sum(1) AS avg(id), sum_count(2) AS count(foo), sum_count(3) AS count(id)", "GroupBy": "(0|4)", - "ResultColumns": 4, "Inputs": [ { "OperatorType": "Route", @@ -6346,7 +6338,6 @@ "Variant": "Ordered", "Aggregates": "sum(1) AS avg(id), sum_count(2) AS count(foo), sum_count(3) AS count(id)", "GroupBy": "(0|4)", - "ResultColumns": 4, "Inputs": [ { "OperatorType": "Route", @@ -6530,7 +6521,6 @@ "OperatorType": "Aggregate", "Variant": "Scalar", "Aggregates": "min(0|2) AS min_id, max(1|3) AS max_id", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Route", diff --git a/go/vt/vtgate/planbuilder/testdata/cte_cases.json b/go/vt/vtgate/planbuilder/testdata/cte_cases.json index 82987fb52f4..d6647681103 100644 --- a/go/vt/vtgate/planbuilder/testdata/cte_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/cte_cases.json @@ -485,7 +485,6 @@ "Variant": "Ordered", "Aggregates": "max(1|3) AS bazo", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Route", @@ -524,7 +523,6 @@ "Variant": "Ordered", "Aggregates": "sum_count(1) AS bazo", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Route", diff --git a/go/vt/vtgate/planbuilder/testdata/from_cases.json b/go/vt/vtgate/planbuilder/testdata/from_cases.json index 9a0ffa37f0c..138dc8b045e 100644 --- a/go/vt/vtgate/planbuilder/testdata/from_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/from_cases.json @@ -798,7 +798,6 @@ "Variant": "Ordered", "Aggregates": "sum_count_star(1) AS count", "GroupBy": "(0|2)", - "ResultColumns": 3, "Inputs": [ { "OperatorType": "Route", diff --git a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json index 3ce182b51b4..060f073a366 100644 --- a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json @@ -16,7 +16,6 @@ "Variant": "Ordered", "Aggregates": "any_value(1|4) AS b, sum_count_star(2) AS count(*)", "GroupBy": "(0|3)", - "ResultColumns": 5, "Inputs": [ { "OperatorType": "Route", @@ -49,13 +48,13 @@ "OperatorType": "Sort", "Variant": "Memory", "OrderBy": "2 ASC", + "ResultColumns": 3, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "any_value(1) AS b, sum_count_star(2) AS k", "GroupBy": "(0|3)", - "ResultColumns": 3, "Inputs": [ { "OperatorType": "Route", @@ -95,7 +94,6 @@ "Variant": "Ordered", "Aggregates": "any_value(1|4) AS b, sum_count_star(2) AS k", "GroupBy": "(0|3)", - "ResultColumns": 5, "Inputs": [ { "OperatorType": "Route", @@ -132,13 +130,13 @@ "OperatorType": "Sort", "Variant": "Memory", "OrderBy": "2 DESC", + "ResultColumns": 3, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "any_value(1) AS b, sum_count_star(2) AS k", "GroupBy": "(0|3)", - "ResultColumns": 3, "Inputs": [ { "OperatorType": "Route", @@ -173,13 +171,13 @@ "OperatorType": "Sort", "Variant": "Memory", "OrderBy": "(0|3) ASC, 2 ASC", + "ResultColumns": 3, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "any_value(1) AS b, sum_count_star(2) AS k", "GroupBy": "(0|3)", - "ResultColumns": 3, "Inputs": [ { "OperatorType": "Route", diff --git a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json index fd8ee10713b..36f1472007d 100644 --- a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json @@ -1618,13 +1618,13 @@ "OperatorType": "Sort", "Variant": "Memory", "OrderBy": "0 ASC", + "ResultColumns": 2, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum_count(0) AS count(id)", "GroupBy": "(1|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Route", @@ -2090,7 +2090,6 @@ "Variant": "Ordered", "Aggregates": "min(1|3) AS min(a.id)", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Join", diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.json b/go/vt/vtgate/planbuilder/testdata/select_cases.json index 1099f62175f..f06a6a50d45 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.json @@ -1836,7 +1836,6 @@ "Variant": "Ordered", "Aggregates": "sum(0) AS avg_col, sum_count(3) AS count(intcol)", "GroupBy": "1 COLLATE latin1_swedish_ci, (2|4) COLLATE ", - "ResultColumns": 4, "Inputs": [ { "OperatorType": "Route", @@ -4330,7 +4329,6 @@ "Variant": "Ordered", "Aggregates": "any_value(0) AS id", "GroupBy": "(1|2)", - "ResultColumns": 1, "Inputs": [ { "OperatorType": "Route", @@ -4496,7 +4494,6 @@ "OperatorType": "Aggregate", "Variant": "Scalar", "Aggregates": "max(0|1) AS max(music.id)", - "ResultColumns": 1, "Inputs": [ { "OperatorType": "Route", @@ -5033,7 +5030,6 @@ "Variant": "Ordered", "Aggregates": "sum_count_star(1) AS b", "GroupBy": "(2|3), (0|4)", - "ResultColumns": 3, "Inputs": [ { "OperatorType": "Route", diff --git a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json index 6e67d4f9e8d..58ec909104a 100644 --- a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json @@ -25,7 +25,6 @@ "Variant": "Ordered", "Aggregates": "sum(2) AS sum_qty, sum(3) AS sum_base_price, sum(4) AS sum_disc_price, sum(5) AS sum_charge, sum(6) AS avg_qty, sum(7) AS avg_price, sum(8) AS avg_disc, sum_count_star(9) AS count_order, sum_count(10) AS count(l_quantity), sum_count(11) AS count(l_extendedprice), sum_count(12) AS count(l_discount)", "GroupBy": "(0|13), (1|14)", - "ResultColumns": 13, "Inputs": [ { "OperatorType": "Route", @@ -67,13 +66,13 @@ "OperatorType": "Sort", "Variant": "Memory", "OrderBy": "1 DESC COLLATE utf8mb4_0900_ai_ci, (2|5) ASC", + "ResultColumns": 4, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum(1) AS revenue", "GroupBy": "(0|4), (2|5), (3|6)", - "ResultColumns": 4, "Inputs": [ { "OperatorType": "Projection", @@ -278,13 +277,13 @@ "OperatorType": "Sort", "Variant": "Memory", "OrderBy": "1 DESC COLLATE utf8mb4_0900_ai_ci", + "ResultColumns": 2, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum(1) AS revenue", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Projection", @@ -801,7 +800,6 @@ "Variant": "Ordered", "Aggregates": "sum(1) AS sum(case when nation = 'BRAZIL' then volume else 0 end), sum(2) AS sum(volume)", "GroupBy": "(0|3)", - "ResultColumns": 3, "Inputs": [ { "OperatorType": "Projection", @@ -1346,13 +1344,13 @@ "OperatorType": "Sort", "Variant": "Memory", "OrderBy": "2 DESC COLLATE utf8mb4_0900_ai_ci", + "ResultColumns": 8, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum(2) AS revenue", "GroupBy": "(0|8), (1|9), (3|10), (6|11), (4|12), (5|13), (7|14)", - "ResultColumns": 8, "Inputs": [ { "OperatorType": "Projection", @@ -1657,6 +1655,7 @@ "InputName": "Outer", "OperatorType": "Filter", "Predicate": "sum(ps_supplycost * ps_availqty) > :__sq1", + "ResultColumns": 2, "Inputs": [ { "OperatorType": "Sort", @@ -1668,7 +1667,6 @@ "Variant": "Ordered", "Aggregates": "sum(1) AS value", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Projection", @@ -1900,7 +1898,6 @@ "Variant": "Ordered", "Aggregates": "sum_count(1) AS count(o_orderkey)", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Projection", @@ -2061,7 +2058,6 @@ "OperatorType": "Aggregate", "Variant": "Scalar", "Aggregates": "max(0|1) AS max(total_revenue)", - "ResultColumns": 1, "Inputs": [ { "OperatorType": "Route", @@ -2445,13 +2441,13 @@ "OperatorType": "Sort", "Variant": "Memory", "OrderBy": "1 DESC, (0|2) ASC", + "ResultColumns": 2, "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum_count_star(1) AS numwait", "GroupBy": "(0|2)", - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Projection", From 4451af63c4c92dc6f5bc984c2e6ba6ea0bf2d621 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Thu, 15 Aug 2024 08:37:34 +0200 Subject: [PATCH 11/11] review comments Signed-off-by: Andres Taylor --- go/vt/vtgate/planbuilder/operators/projection_pushing.go | 3 +-- go/vt/vtgate/planbuilder/testdata/from_cases.json | 4 ++-- go/vt/vtgate/planbuilder/testdata/tpch_cases.json | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/projection_pushing.go b/go/vt/vtgate/planbuilder/operators/projection_pushing.go index 21d2a058a86..1d1a5a04501 100644 --- a/go/vt/vtgate/planbuilder/operators/projection_pushing.go +++ b/go/vt/vtgate/planbuilder/operators/projection_pushing.go @@ -361,11 +361,10 @@ func splitUnexploredExpression( case col.IsPureRight(): rhs.add(pe, alias) col.RHSExpr = colName - case col.IsMixedLeftAndRight(): + default: for _, lhsExpr := range col.LHSExprs { ae := aeWrap(lhsExpr.Expr) columnName := ae.ColumnName() - ae.As = sqlparser.NewIdentifierCI(columnName) lhs.add(newProjExpr(ae), columnName) } innerPE := newProjExprWithInner(pe.Original, col.RHSExpr) diff --git a/go/vt/vtgate/planbuilder/testdata/from_cases.json b/go/vt/vtgate/planbuilder/testdata/from_cases.json index 138dc8b045e..2e0fe429c1f 100644 --- a/go/vt/vtgate/planbuilder/testdata/from_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/from_cases.json @@ -4802,8 +4802,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select dt.foo from (select u.foo as foo from `user` as u where 1 != 1) as dt where 1 != 1", - "Query": "select dt.foo from (select u.foo as foo from `user` as u) as dt", + "FieldQuery": "select dt.foo from (select u.foo from `user` as u where 1 != 1) as dt where 1 != 1", + "Query": "select dt.foo from (select u.foo from `user` as u) as dt", "Table": "`user`" }, { diff --git a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json index 58ec909104a..6b3f84d01d6 100644 --- a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json @@ -1190,8 +1190,8 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select profit.l_extendedprice, profit.l_discount, profit.l_quantity, profit.l_suppkey, profit.l_partkey, weight_string(profit.l_suppkey) from (select l_extendedprice as l_extendedprice, l_discount as l_discount, l_quantity as l_quantity, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem where 1 != 1) as profit where 1 != 1", - "Query": "select profit.l_extendedprice, profit.l_discount, profit.l_quantity, profit.l_suppkey, profit.l_partkey, weight_string(profit.l_suppkey) from (select l_extendedprice as l_extendedprice, l_discount as l_discount, l_quantity as l_quantity, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem where l_orderkey = :o_orderkey) as profit", + "FieldQuery": "select profit.l_extendedprice, profit.l_discount, profit.l_quantity, profit.l_suppkey, profit.l_partkey, weight_string(profit.l_suppkey) from (select l_extendedprice, l_discount, l_quantity, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem where 1 != 1) as profit where 1 != 1", + "Query": "select profit.l_extendedprice, profit.l_discount, profit.l_quantity, profit.l_suppkey, profit.l_partkey, weight_string(profit.l_suppkey) from (select l_extendedprice, l_discount, l_quantity, l_suppkey as l_suppkey, l_partkey as l_partkey from lineitem where l_orderkey = :o_orderkey) as profit", "Table": "lineitem" } ]