diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 94cbdbf6e78330..7723c11d8497bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -111,8 +111,7 @@ public class MTMVPlanUtil { RuleType.ELIMINATE_JOIN_BY_UK, RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, RuleType.ELIMINATE_GROUP_BY, - RuleType.SALT_JOIN, - RuleType.AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION + RuleType.SALT_JOIN ); // The rules should be disabled when run MTMV task public static final List DISABLE_RULES_WHEN_RUN_MTMV_TASK = DISABLE_RULES_WHEN_GENERATE_MTMV_CACHE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index f283aaae139368..b3026026e942c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -228,6 +228,8 @@ public class Rewriter extends AbstractBatchJobExecutor { // so there may be two filters we need to merge them new MergeFilters() ), + custom(RuleType.AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION, + AggScalarSubQueryToWindowFunction::new), bottomUp( new EliminateUselessPlanUnderApply(), // CorrelateApplyToUnCorrelateApply and ApplyToJoin @@ -455,6 +457,7 @@ public class Rewriter extends AbstractBatchJobExecutor { // so there may be two filters we need to merge them new MergeFilters() ), + // query rewrite support window, so add this rule here custom(RuleType.AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION, AggScalarSubQueryToWindowFunction::new), bottomUp( new EliminateUselessPlanUnderApply(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java index b367d7c064cfa3..5eb4f82f0e3a8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java @@ -69,7 +69,6 @@ public class PreMaterializedViewRewriter { NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.PROCESS_SCALAR_AGG_MUST_USE_MULTI_DISTINCT.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.SALT_JOIN.ordinal()); - NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION.ordinal()); } /** diff --git a/regression-test/suites/mv_p0/query_in_different_db/query_in_different_db.groovy b/regression-test/suites/mv_p0/query_in_different_db/query_in_different_db.groovy index 3f7580e7ccbca5..d9b4eaa9a58211 100644 --- a/regression-test/suites/mv_p0/query_in_different_db/query_in_different_db.groovy +++ b/regression-test/suites/mv_p0/query_in_different_db/query_in_different_db.groovy @@ -48,12 +48,13 @@ suite ("query_in_different_db") { sql "insert into d_table select -4,-4,-4,'d';" sql "insert into d_table select -4,-4,-4,'d';" + sql "analyze table d_table with sync;" + sql """alter table d_table modify column k1 set stats ('row_count'='12');""" + create_sync_mv(db, "d_table", "mv_in_${db}", """ select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1 """) - sql "analyze table d_table with sync;" - sql """alter table d_table modify column k1 set stats ('row_count'='12');""" // use another db, mv rewrite should be correct sql """drop database IF EXISTS test_query_in_different_db""" diff --git a/regression-test/suites/nereids_rules_p0/mv/limit/limit_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/limit/limit_rewrite.groovy index 3af7a56cd9588c..5971de6301767e 100644 --- a/regression-test/suites/nereids_rules_p0/mv/limit/limit_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/limit/limit_rewrite.groovy @@ -580,7 +580,7 @@ suite("limit_rewrite") { where o_orderdate > '2023-12-08' limit 2 offset 5; """ - async_mv_rewrite_fail(db, mv5_1, query5_1, "mv5_1") + async_mv_rewrite_success(db, mv5_1, query5_1, "mv5_1") sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_1""" @@ -606,7 +606,7 @@ suite("limit_rewrite") { where o_orderdate > '2023-12-09' limit 4 offset 2; """ - async_mv_rewrite_fail(db, mv5_2, query5_2, "mv5_2") + async_mv_rewrite_success(db, mv5_2, query5_2, "mv5_2") sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_2""" @@ -657,7 +657,7 @@ suite("limit_rewrite") { orders limit 2 offset 5; """ - async_mv_rewrite_fail(db, mv6_1, query6_1, "mv6_1") + async_mv_rewrite_success(db, mv6_1, query6_1, "mv6_1") sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_1""" } diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy index ab7102684548a1..d0f8d94150be88 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy @@ -902,8 +902,8 @@ select * from ( sql "SET enable_materialized_view_nest_rewrite = true" // DP Hyper can not use pre materialized view rewrite sql """SET enable_dphyp_optimizer = false""" - mv_rewrite_all_success(query2_0, ["mv_all_6_a", "mv_all_6_b", "mv_all_6_c", "mv_all_6_d"], - true, [TRY_IN_RBO, FORCE_IN_RBO]) + mv_rewrite_all_success_without_check_chosen(query2_0, ["mv_all_6_a", "mv_all_6_b", "mv_all_6_c", "mv_all_6_d"], + [TRY_IN_RBO, FORCE_IN_RBO]) mv_rewrite_all_fail(query2_0, ["mv_all_6_a", "mv_all_6_b", "mv_all_6_c", "mv_all_6_d"], [NOT_IN_RBO]) // Compare result when before and after mv rewrite diff --git a/regression-test/suites/nereids_rules_p0/mv/pre_rewrite/limit/query_with_limit.groovy b/regression-test/suites/nereids_rules_p0/mv/pre_rewrite/limit/query_with_limit.groovy index 35056eb08f33ca..5e29a5ef2afbce 100644 --- a/regression-test/suites/nereids_rules_p0/mv/pre_rewrite/limit/query_with_limit.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/pre_rewrite/limit/query_with_limit.groovy @@ -326,8 +326,11 @@ suite("query_with_limit") { o_orderkey, l_partkey, l_suppkey + order by l_orderkey limit 2 offset 3; """ + // because query explain memo is too big without order by, so add order by + // if query explain memo is too big fixed, then remove order by l_orderkey async_mv_rewrite_success(db, mv1_5, query1_5, "mv1_5", [TRY_IN_RBO, FORCE_IN_RBO]) async_mv_rewrite_fail(db, mv1_5, query1_5, "mv1_5", [NOT_IN_RBO]) sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_5""" @@ -958,7 +961,7 @@ suite("query_with_limit") { orders on l_orderkey = o_orderkey limit 2 offset 1; """ - async_mv_rewrite_success(db, mv3_1, query3_1, "mv3_1") + async_mv_rewrite_success(db, mv3_1, query3_1, "mv3_1", [TRY_IN_RBO, FORCE_IN_RBO]) sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_1""" diff --git a/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy index 5212a7112e70eb..54dcab2dcd0ae3 100644 --- a/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy @@ -593,7 +593,7 @@ suite("topN_rewrite") { limit 2 offset 5; """ order_qt_query5_1_before "${query5_1}" - async_mv_rewrite_fail(db, mv5_1, query5_1, "mv5_1") + async_mv_rewrite_success(db, mv5_1, query5_1, "mv5_1") order_qt_query5_1_after "${query5_1}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_1""" @@ -624,7 +624,7 @@ suite("topN_rewrite") { limit 4 offset 2; """ order_qt_query5_2_before "${query5_2}" - async_mv_rewrite_fail(db, mv5_2, query5_2, "mv5_2") + async_mv_rewrite_success(db, mv5_2, query5_2, "mv5_2") order_qt_query5_2_after "${query5_2}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_2""" @@ -683,7 +683,7 @@ suite("topN_rewrite") { limit 2 offset 5; """ order_qt_query6_1_before "${query6_1}" - async_mv_rewrite_fail(db, mv6_1, query6_1, "mv6_1") + async_mv_rewrite_success(db, mv6_1, query6_1, "mv6_1") order_qt_query6_1_after "${query6_1}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_1""" diff --git a/regression-test/suites/nereids_rules_p0/mv/tpch/mv_tpch_test.groovy b/regression-test/suites/nereids_rules_p0/mv/tpch/mv_tpch_test.groovy index 40a3960a7901e7..335ecb7e6446e7 100644 --- a/regression-test/suites/nereids_rules_p0/mv/tpch/mv_tpch_test.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/tpch/mv_tpch_test.groovy @@ -1231,12 +1231,13 @@ suite("mv_tpch_test") { l_partkey = p_partkey ) """ - // contains subquery, doesn't support now - async_mv_rewrite_success(db, mv17, query17, "mv17") + // agg under join should rewrite successfully, + // but because AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION rule + // would rewrite to agg-window-join, so now doesn't support + async_mv_rewrite_fail(db, mv17, query17, "mv17") order_qt_query17_after "${query17}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv17""" - def mv18 = """ SELECT c_name,