This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 92dc8ed718f781901ff9a97647c77f0bce81b3ac Author: seawinde <[email protected]> AuthorDate: Sat Apr 27 21:55:35 2024 +0800 [opt](mtmv) Add enable materialized view nest rewrite switch (#34197) * [opt](mtmv) Add enable materialized view nest rewrite switch * fix ut * fix ut2 --- .../java/org/apache/doris/nereids/memo/Memo.java | 6 ++++-- .../apache/doris/nereids/memo/StructInfoMap.java | 21 +++++--------------- .../mv/AbstractMaterializedViewRule.java | 3 ++- .../exploration/mv/MaterializedViewUtils.java | 11 +++++------ .../java/org/apache/doris/qe/SessionVariable.java | 12 +++++++++++ .../doris/nereids/memo/StructInfoMapTest.java | 5 ++++- .../mv/nested/nested_materialized_view.out | 23 +++++++++++++++++++++- .../mv/nested/nested_materialized_view.groovy | 16 +++++++++++++++ .../mv/nested_mtmv/nested_mtmv.groovy | 1 + 9 files changed, 71 insertions(+), 27 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index 1e6b285187d..d7d46ecc15e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -36,6 +36,7 @@ import org.apache.doris.nereids.trees.plans.LeafPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.algebra.SetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -72,7 +73,6 @@ public class Memo { EventChannel.getDefaultChannel().addConsumers(new LogConsumer(GroupMergeEvent.class, EventChannel.LOG))); private static long stateId = 0; private final ConnectContext connectContext; - private final Set<Long> needRefreshTableIdSet = new HashSet<>(); private final AtomicLong refreshVersion = new AtomicLong(1); private final IdGenerator<GroupId> groupIdGenerator = GroupId.createGenerator(); private final Map<GroupId, Group> groups = Maps.newLinkedHashMap(); @@ -416,7 +416,9 @@ public class Memo { throw new IllegalStateException("Insert a plan into targetGroup but differ in logicalproperties"); } // TODO Support sync materialized view in the future - if (plan instanceof LogicalPlan && plan instanceof CatalogRelation + if (connectContext != null + && connectContext.getSessionVariable().isEnableMaterializedViewNestRewrite() + && plan instanceof LogicalCatalogRelation && ((CatalogRelation) plan).getTable() instanceof MTMV && !plan.getGroupExpression().isPresent()) { refreshVersion.incrementAndGet(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index ae07c2043ab..df41eb3eb48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -48,12 +48,10 @@ public class StructInfoMap { * get struct info according to table map * * @param tableMap the original table map - * @param foldTableMap the fold table map * @param group the group that the mv matched * @return struct info or null if not found */ - public @Nullable StructInfo getStructInfo(Memo memo, BitSet tableMap, BitSet foldTableMap, - Group group, Plan originPlan) { + public @Nullable StructInfo getStructInfo(Memo memo, BitSet tableMap, Group group, Plan originPlan) { StructInfo structInfo = infoMap.get(tableMap); if (structInfo != null) { return structInfo; @@ -84,10 +82,6 @@ public class StructInfoMap { return groupExpressionMap.get(tableMap); } - public long getRefreshVersion() { - return refreshVersion; - } - public void setRefreshVersion(long refreshVersion) { this.refreshVersion = refreshVersion; } @@ -119,7 +113,8 @@ public class StructInfoMap { * */ public void refresh(Group group, long memoVersion) { - if (memoVersion == group.getstructInfoMap().refreshVersion) { + StructInfoMap structInfoMap = group.getstructInfoMap(); + if (!structInfoMap.getTableMaps().isEmpty() && memoVersion == structInfoMap.refreshVersion) { return; } Set<Integer> refreshedGroup = new HashSet<>(); @@ -152,8 +147,7 @@ public class StructInfoMap { } // if cumulative child table map is different from current // or current group expression map is empty, should update the groupExpressionMap currently - Collection<Pair<BitSet, List<BitSet>>> bitSetWithChildren = cartesianProduct(childrenTableMap, - new BitSet()); + Collection<Pair<BitSet, List<BitSet>>> bitSetWithChildren = cartesianProduct(childrenTableMap); for (Pair<BitSet, List<BitSet>> bitSetWithChild : bitSetWithChildren) { groupExpressionMap.putIfAbsent(bitSetWithChild.first, Pair.of(groupExpression, bitSetWithChild.second)); @@ -173,8 +167,7 @@ public class StructInfoMap { return tableMap; } - private Collection<Pair<BitSet, List<BitSet>>> cartesianProduct(List<Set<BitSet>> childrenTableMap, - BitSet targetBitSet) { + private Collection<Pair<BitSet, List<BitSet>>> cartesianProduct(List<Set<BitSet>> childrenTableMap) { Set<List<BitSet>> cartesianLists = Sets.cartesianProduct(childrenTableMap); List<Pair<BitSet, List<BitSet>>> resultPairSet = new LinkedList<>(); for (List<BitSet> bitSetList : cartesianLists) { @@ -182,10 +175,6 @@ public class StructInfoMap { for (BitSet b : bitSetList) { bitSet.or(b); } - // filter the useless bitset which targetBitSet not contains, avoid exponential expansion - if (!targetBitSet.isEmpty() && !StructInfo.containsAll(targetBitSet, bitSet)) { - continue; - } resultPairSet.add(Pair.of(bitSet, bitSetList)); } return resultPairSet; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 3405942c3a8..6d0f6027066 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -107,7 +107,6 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac if (checkIfRewritten(queryPlan, context)) { continue; } - context.tryReGenerateMvScanPlan(cascadesContext); // check mv plan is valid or not if (!checkPattern(context.getStructInfo())) { context.recordFailReason(context.getStructInfo(), @@ -321,6 +320,8 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac continue; } recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext); + // if rewrite successfully, try to regenerate mv scan because it maybe used again + materializationContext.tryReGenerateMvScanPlan(cascadesContext); rewriteResults.add(rewrittenPlan); } return rewriteResults; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 0fbcf6b7d63..6863a7e01b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -148,11 +148,10 @@ public class MaterializedViewUtils { if (plan.getGroupExpression().isPresent()) { Group ownerGroup = plan.getGroupExpression().get().getOwnerGroup(); StructInfoMap structInfoMap = ownerGroup.getstructInfoMap(); - if (cascadesContext.getMemo().getRefreshVersion() != structInfoMap.getRefreshVersion() - || structInfoMap.getTableMaps().isEmpty()) { - structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion()); - structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion()); - } + // Refresh struct info in current level plan from top to bottom + structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion()); + structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion()); + Set<BitSet> queryTableSets = structInfoMap.getTableMaps(); ImmutableList.Builder<StructInfo> structInfosBuilder = ImmutableList.builder(); if (!queryTableSets.isEmpty()) { @@ -163,7 +162,7 @@ public class MaterializedViewUtils { continue; } StructInfo structInfo = structInfoMap.getStructInfo(cascadesContext.getMemo(), - queryTableSet, queryTableSet, ownerGroup, plan); + queryTableSet, ownerGroup, plan); if (structInfo != null) { structInfosBuilder.add(structInfo); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 480d153ba5a..207da09a430 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -518,6 +518,9 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_MATERIALIZED_VIEW_UNION_REWRITE = "enable_materialized_view_union_rewrite"; + public static final String ENABLE_MATERIALIZED_VIEW_NEST_REWRITE + = "enable_materialized_view_nest_rewrite"; + public static final String CREATE_TABLE_PARTITION_MAX_NUM = "create_table_partition_max_num"; @@ -1630,6 +1633,11 @@ public class SessionVariable implements Serializable, Writable { + "respond to the query"}) public boolean enableMaterializedViewUnionRewrite = false; + @VariableMgr.VarAttr(name = ENABLE_MATERIALIZED_VIEW_NEST_REWRITE, needForward = true, + description = {"是否允许嵌套物化视图改写", + "Whether enable materialized view nest rewrite"}) + public boolean enableMaterializedViewNestRewrite = false; + @VariableMgr.VarAttr(name = CREATE_TABLE_PARTITION_MAX_NUM, needForward = true, description = {"建表时创建分区的最大数量", "The maximum number of partitions created during table creation"}) @@ -3656,6 +3664,10 @@ public class SessionVariable implements Serializable, Writable { return enableMaterializedViewUnionRewrite; } + public boolean isEnableMaterializedViewNestRewrite() { + return enableMaterializedViewNestRewrite; + } + public int getCreateTablePartitionMaxNum() { return createTablePartitionMaxNum; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java index 217cbf2f1a2..d644ce2b890 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java @@ -59,6 +59,7 @@ class StructInfoMapTest extends SqlTestBase { } }; connectContext.getSessionVariable().enableMaterializedViewRewrite = true; + connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + " PROPERTIES ('replication_num' = '1') \n" @@ -107,6 +108,7 @@ class StructInfoMapTest extends SqlTestBase { } }; connectContext.getSessionVariable().enableMaterializedViewRewrite = true; + connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + " PROPERTIES ('replication_num' = '1') \n" @@ -146,6 +148,7 @@ class StructInfoMapTest extends SqlTestBase { } }; connectContext.getSessionVariable().enableMaterializedViewRewrite = true; + connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + " PROPERTIES ('replication_num' = '1') \n" @@ -168,7 +171,7 @@ class StructInfoMapTest extends SqlTestBase { BitSet mvMap = structInfoMap.getTableMaps().stream() .filter(b -> b.cardinality() == 2) .collect(Collectors.toList()).get(0); - StructInfo structInfo = structInfoMap.getStructInfo(c1.getMemo(), mvMap, mvMap, root, null); + StructInfo structInfo = structInfoMap.getStructInfo(c1.getMemo(), mvMap, root, null); System.out.println(structInfo.getOriginalPlan().treeString()); BitSet bitSet = new BitSet(); structInfo.getRelations().forEach(r -> bitSet.set((int) r.getTable().getId())); diff --git a/regression-test/data/nereids_rules_p0/mv/nested/nested_materialized_view.out b/regression-test/data/nereids_rules_p0/mv/nested/nested_materialized_view.out index 09c11bcee6a..2647a9f7d56 100644 --- a/regression-test/data/nereids_rules_p0/mv/nested/nested_materialized_view.out +++ b/regression-test/data/nereids_rules_p0/mv/nested/nested_materialized_view.out @@ -17,4 +17,25 @@ 6 6 6 -6 \ No newline at end of file +6 + +-- !query1_1_before -- +4 +4 +4 +4 +6 +6 +6 +6 + +-- !query1_1_after -- +4 +4 +4 +4 +6 +6 +6 +6 + diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy index 6ad175e85ce..5a5ddc5b3c7 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy @@ -23,6 +23,7 @@ suite("nested_materialized_view") { sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=true" sql "SET enable_nereids_timeout = false" + sql "SET enable_materialized_view_nest_rewrite = true" def create_mtmv = { db_name, mv_name, mv_sql -> sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" @@ -179,4 +180,19 @@ suite("nested_materialized_view") { order_qt_query1_0_after "${query1_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0_inner_mv""" sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + + + sql "SET enable_materialized_view_nest_rewrite = false" + + order_qt_query1_1_before "${query1_0}" + create_mtmv(db, "mv1_0_inner_mv", mv1_0_inner_mv) + check_mv_rewrite_fail(db, mv1_0, query1_0, "mv1_0") + + explain { + sql("${query1_0}") + contains("mv1_0_inner_mv(mv1_0_inner_mv)") + } + + order_qt_query1_1_after "${query1_0}" + } diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy index 4a0f513f0fa..988275c489a 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy @@ -22,6 +22,7 @@ suite("nested_mtmv") { sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=true" sql "SET enable_nereids_timeout = false" + sql "SET enable_materialized_view_nest_rewrite = true" sql """ drop table if exists orders_1 --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
