Skip to content

Commit dd0d323

Browse files
authored
[fix] (mtmv) Fix topN rewrite by materialized view fail and cause rewrite result wrong (#58406)
### What problem does this PR solve? This PR addresses two issues: 1 it stabilizes the regression tests for `topN_rewrite` and `limit_rewrite`, and 2 it fixes a bug affecting negative test cases. For example, the following case should not be rewritten successfully but was incorrectly rewritten CREATE MATERIALIZED VIEW mv BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS select o_orderdate, ps_partkey, l_orderkey from orders left join lineitem on l_orderkey = o_orderkey left join partsupp on ps_partkey = l_partkey and l_suppkey = ps_suppkey where o_orderdate > '2025-01-02' order by o_orderdate, l_orderkey, ps_partkey limit 8 offset 1; the query as following should not rewrite successfully select o_orderdate, ps_partkey, l_orderkey from orders left join lineitem on l_orderkey = o_orderkey left join partsupp on ps_partkey = l_partkey and l_suppkey = ps_suppkey order by o_orderdate, l_orderkey limit 4 offset 2;
1 parent 0b5c2c9 commit dd0d323

17 files changed

+228
-15
lines changed

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1066,9 +1066,9 @@ protected Plan tryRewriteTopN(LogicalTopN<Plan> queryTopNode, LogicalTopN<Plan>
10661066
// check the order keys of TopN between query and view is consistent
10671067
List<OrderKey> queryOrderKeys = queryTopNode.getOrderKeys();
10681068
List<OrderKey> viewOrderKeys = viewTopNode.getOrderKeys();
1069-
if (queryOrderKeys.size() != viewOrderKeys.size()) {
1069+
if (queryOrderKeys.size() > viewOrderKeys.size()) {
10701070
materializationContext.recordFailReason(queryStructInfo,
1071-
"query topN order keys size is not consistent with view topN order keys size",
1071+
"query topN order keys size is bigger than view topN order keys size",
10721072
() -> String.format("query topN order keys = %s,\n view topN order keys = %s,\n",
10731073
queryOrderKeys, viewOrderKeys));
10741074
return null;
@@ -1096,7 +1096,7 @@ protected Plan tryRewriteTopN(LogicalTopN<Plan> queryTopNode, LogicalTopN<Plan>
10961096
viewShuttledOrderKeys.add(new OrderKey(viewOrderByExpressionsQueryBasedSet.get(j), viewOrderKey.isAsc(),
10971097
viewOrderKey.isNullFirst()));
10981098
}
1099-
if (!queryShuttledOrderKeys.equals(viewShuttledOrderKeys)) {
1099+
if (!MaterializedViewUtils.isPrefixSameFromStart(queryShuttledOrderKeys, viewShuttledOrderKeys)) {
11001100
materializationContext.recordFailReason(queryStructInfo,
11011101
"view topN order key doesn't match query order key",
11021102
() -> String.format("queryShuttledOrderKeys = %s,\n viewShuttledOrderKeys = %s,\n",

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,8 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
9090
protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) {
9191
PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET);
9292
return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext)
93-
&& !checkContext.isContainsTopAggregate();
93+
&& !checkContext.isContainsTopAggregate()
94+
&& !checkContext.isContainsTopLimit() && !checkContext.isContainsTopTopN()
95+
&& !checkContext.isContainsTopWindow();
9496
}
9597
}

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,13 +82,17 @@ protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesCon
8282
// any check result of join or scan is true, then return true
8383
PlanCheckContext joinCheckContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET);
8484
boolean joinCheckResult = structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, joinCheckContext)
85-
&& !joinCheckContext.isContainsTopAggregate();
85+
&& !joinCheckContext.isContainsTopAggregate()
86+
&& !joinCheckContext.isContainsTopLimit() && !joinCheckContext.isContainsTopTopN()
87+
&& !joinCheckContext.isContainsTopWindow();
8688
if (joinCheckResult) {
8789
return true;
8890
}
8991
PlanCheckContext scanCheckContext = PlanCheckContext.of(ImmutableSet.of());
9092
return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, scanCheckContext)
91-
&& !scanCheckContext.isContainsTopAggregate();
93+
&& !scanCheckContext.isContainsTopAggregate()
94+
&& !joinCheckContext.isContainsTopLimit() && !joinCheckContext.isContainsTopTopN()
95+
&& !joinCheckContext.isContainsTopWindow();
9296
}
9397

9498
@Override

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitAggregateRule.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
7070
return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext)
7171
&& checkContext.isContainsTopAggregate() && checkContext.getTopAggregateNum() == 1
7272
&& !checkContext.isContainsTopTopN()
73+
&& !checkContext.isContainsTopWindow()
7374
&& checkContext.isContainsTopLimit() && checkContext.getTopLimitNum() == 1;
7475
}
7576

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitJoinRule.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
7070
return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext)
7171
&& !checkContext.isContainsTopAggregate()
7272
&& !checkContext.isContainsTopTopN()
73+
&& !checkContext.isContainsTopWindow()
7374
&& checkContext.isContainsTopLimit() && checkContext.getTopLimitNum() == 1;
7475
}
7576

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitScanRule.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,15 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
7171
return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext)
7272
&& !checkContext.isContainsTopAggregate()
7373
&& !checkContext.isContainsTopTopN()
74+
&& !checkContext.isContainsTopWindow()
7475
&& checkContext.isContainsTopLimit() && checkContext.getTopLimitNum() == 1;
7576
}
7677

78+
@Override
79+
protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) {
80+
return checkQueryPattern(structInfo, cascadesContext);
81+
}
82+
7783
@Override
7884
public List<Rule> buildRules() {
7985
return ImmutableList.of(

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNAggregateRule.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
6767
return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext)
6868
&& checkContext.isContainsTopAggregate() && checkContext.getTopAggregateNum() == 1
6969
&& !checkContext.isContainsTopLimit()
70+
&& !checkContext.isContainsTopWindow()
7071
&& checkContext.isContainsTopTopN() && checkContext.getTopTopNNum() == 1;
7172
}
7273

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNJoinRule.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
6868
return accept
6969
&& !checkContext.isContainsTopAggregate()
7070
&& !checkContext.isContainsTopLimit()
71+
&& !checkContext.isContainsTopWindow()
7172
&& checkContext.isContainsTopTopN() && checkContext.getTopTopNNum() == 1;
7273
}
7374

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNScanRule.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,15 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
6969
return accept
7070
&& !checkContext.isContainsTopAggregate()
7171
&& !checkContext.isContainsTopLimit()
72+
&& !checkContext.isContainsTopWindow()
7273
&& checkContext.isContainsTopTopN() && checkContext.getTopTopNNum() == 1;
7374
}
7475

76+
@Override
77+
protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) {
78+
return checkQueryPattern(structInfo, cascadesContext);
79+
}
80+
7581
@Override
7682
public List<Rule> buildRules() {
7783
return ImmutableList.of(

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.doris.nereids.StatementContext;
2525
import org.apache.doris.nereids.memo.Group;
2626
import org.apache.doris.nereids.memo.StructInfoMap;
27+
import org.apache.doris.nereids.properties.OrderKey;
2728
import org.apache.doris.nereids.rules.RuleType;
2829
import org.apache.doris.nereids.rules.analysis.BindRelation;
2930
import org.apache.doris.nereids.rules.exploration.mv.PartitionIncrementMaintainer.PartitionIncrementCheckContext;
@@ -651,4 +652,23 @@ public Boolean visit(Plan plan, Void context) {
651652
return false;
652653
}
653654
}
655+
656+
/**
657+
* Check the prefix of two order key list is same from start
658+
*/
659+
public static boolean isPrefixSameFromStart(List<OrderKey> queryShuttledOrderKeys,
660+
List<OrderKey> viewShuttledOrderKeys) {
661+
if (queryShuttledOrderKeys == null || viewShuttledOrderKeys == null) {
662+
return false;
663+
}
664+
if (queryShuttledOrderKeys.size() > viewShuttledOrderKeys.size()) {
665+
return false;
666+
}
667+
for (int i = 0; i < queryShuttledOrderKeys.size(); i++) {
668+
if (!java.util.Objects.equals(queryShuttledOrderKeys.get(i), viewShuttledOrderKeys.get(i))) {
669+
return false;
670+
}
671+
}
672+
return true;
673+
}
654674
}

0 commit comments

Comments
 (0)