This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit d5bf20c96ec31ae65b11c46efe38c18ed6317007 Author: seawinde <[email protected]> AuthorDate: Fri Mar 8 23:32:30 2024 +0800 [improvement](mtmv) Improve the performance for query rewritting by materialized view (#31886) - Limit the number of times for the query rewritting to the group - Remove the unnecessary log and explain detail info in query --- .../query-async-materialized-view.md | 20 +-- .../query-async-materialized-view.md | 16 ++- .../main/java/org/apache/doris/catalog/Column.java | 6 +- .../org/apache/doris/nereids/NereidsPlanner.java | 2 +- .../mv/AbstractMaterializedViewAggregateRule.java | 54 ++++---- .../mv/AbstractMaterializedViewJoinRule.java | 20 +-- .../mv/AbstractMaterializedViewRule.java | 146 ++++++++++++--------- .../mv/InitMaterializationContextHook.java | 2 +- .../exploration/mv/MaterializationContext.java | 65 +++++---- .../java/org/apache/doris/qe/SessionVariable.java | 13 +- .../agg_with_roll_up/aggregate_with_roll_up.groovy | 4 +- .../mv/dimension/dimension_1.groovy | 8 +- .../mv/dimension/dimension_2_4.groovy | 4 +- 13 files changed, 204 insertions(+), 156 deletions(-) diff --git a/docs/en/docs/query-acceleration/async-materialized-view/query-async-materialized-view.md b/docs/en/docs/query-acceleration/async-materialized-view/query-async-materialized-view.md index 0e29d3b415f..e6ff886bc5b 100644 --- a/docs/en/docs/query-acceleration/async-materialized-view/query-async-materialized-view.md +++ b/docs/en/docs/query-acceleration/async-materialized-view/query-async-materialized-view.md @@ -426,13 +426,14 @@ It will display a concise overview of the transparent rewriting process. | Names: mv1 ``` -**MaterializedViewRewriteFail**: Lists transparent rewrite failures and summarizes the reasons. +**MaterializedViewRewriteSuccessAndChose**: Transparent rewrite succeeded, and the materialized view names list +chosen by the CBO. -**MaterializedViewRewriteSuccessButNotChose**: Transparent rewrite succeeded, but the final CBO did not choose the +**MaterializedViewRewriteSuccessButNotChose**: Transparent rewrite succeeded, but the final CBO did not choose the materialized view names list. -**MaterializedViewRewriteSuccessAndChose**: Transparent rewrite succeeded, and the materialized view names list -chosen by the CBO. +**MaterializedViewRewriteFail**: Lists transparent rewrite failures and summarizes the reasons. + If you want to know the detailed information about materialized view candidates, rewriting, and the final selection process, @@ -442,11 +443,12 @@ you can execute the following statement. It will provide a detailed breakdown of ## Relevant Environment Variables -| Switch | Description | -|---------------------------------------------------------------------------|----------------------------------------| -| SET enable_nereids_planner = true; | Asynchronous materialized views are only supported under the new optimizer, so the new optimizer needs to be enabled. | -| SET enable_materialized_view_rewrite = true; | Enable or disable query transparent rewriting, default is disabled | -| SET materialized_view_rewrite_enable_contain_external_table = true; | Whether materialized views participating in transparent rewriting are allowed to contain external tables, default is not allowed | +| Switch | Description | +|---------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------| +| SET enable_nereids_planner = true; | Asynchronous materialized views are only supported under the new optimizer, so the new optimizer needs to be enabled. | +| SET enable_materialized_view_rewrite = true; | Enable or disable query transparent rewriting, default is disabled | +| SET materialized_view_rewrite_enable_contain_external_table = true; | Whether materialized views participating in transparent rewriting are allowed to contain external tables, default is not allowed | +| SET materialized_view_rewrite_success_candidate_num = 3; | Transparently rewrites the successful result set, allowing the maximum number of CBO candidates to participate, the default is 3 | ## Limitations diff --git a/docs/zh-CN/docs/query-acceleration/async-materialized-view/query-async-materialized-view.md b/docs/zh-CN/docs/query-acceleration/async-materialized-view/query-async-materialized-view.md index ab6a7692c08..8e379153ffd 100644 --- a/docs/zh-CN/docs/query-acceleration/async-materialized-view/query-async-materialized-view.md +++ b/docs/zh-CN/docs/query-acceleration/async-materialized-view/query-async-materialized-view.md @@ -385,11 +385,12 @@ WHERE o_orderkey > 5 AND o_orderkey <= 10; | MaterializedViewRewriteSuccessAndChose: | | Names: mv1 ``` -**MaterializedViewRewriteFail**：列举透明改写失败及原因摘要。 +**MaterializedViewRewriteSuccessAndChose**：透明改写成功，并且CBO选择的物化视图名称列表。 **MaterializedViewRewriteSuccessButNotChose**：透明改写成功，但是最终CBO没有选择的物化视图名称列表。 -**MaterializedViewRewriteSuccessAndChose**：透明改写成功，并且CBO选择的物化视图名称列表。 +**MaterializedViewRewriteFail**：列举透明改写失败及原因摘要。 + 如果想知道物化视图候选，改写和最终选择情况的过程详细信息，可以执行如下语句，会展示透明改写过程详细的信息。 @@ -397,11 +398,12 @@ WHERE o_orderkey > 5 AND o_orderkey <= 10; ## 相关环境变量 -| 开关 | 说明 | -|------------------------------------------------------------------------|----------------------------------------| -| SET enable_nereids_planner = true; | 异步物化视图只有在新优化器下才支持，所以需要开启新优化器 | -| SET enable_materialized_view_rewrite = true; | 开启或者关闭查询透明改写，默认关闭 | -| SET materialized_view_rewrite_enable_contain_external_table = true; | 参与透明改写的物化视图是否允许包含外表，默认不允许 | +| 开关 | 说明 | +|---------------------------------------------------------------------|-----------------------------------| +| SET enable_nereids_planner = true; | 异步物化视图只有在新优化器下才支持，所以需要开启新优化器 | +| SET enable_materialized_view_rewrite = true; | 开启或者关闭查询透明改写，默认关闭 | +| SET materialized_view_rewrite_enable_contain_external_table = true; | 参与透明改写的物化视图是否允许包含外表，默认不允许 | +| SET materialized_view_rewrite_success_candidate_num = 3; | 透明改写成功的结果集合，允许参与到CBO候选的最大数量，默认是3个 | ## 限制 diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index a51426745f9..920b21c0202 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -884,14 +884,14 @@ public class Column implements Writable, GsonPostProcessable { && Objects.equals(realDefaultValue, other.realDefaultValue) && clusterKeyId == other.clusterKeyId; - if (!ok) { - LOG.info("this column: name {} default value {} aggregationType {} isAggregationTypeImplicit {} " + if (!ok && LOG.isDebugEnabled()) { + LOG.debug("this column: name {} default value {} aggregationType {} isAggregationTypeImplicit {} " + "isKey {}, isAllowNull {}, datatype {}, strlen {}, precision {}, scale {}, visible {} " + "children {}, realDefaultValue {}, clusterKeyId {}", name, getDefaultValue(), aggregationType, isAggregationTypeImplicit, isKey, isAllowNull, getDataType(), getStrLen(), getPrecision(), getScale(), visible, children, realDefaultValue, clusterKeyId); - LOG.info("other column: name {} default value {} aggregationType {} isAggregationTypeImplicit {} " + LOG.debug("other column: name {} default value {} aggregationType {} isAggregationTypeImplicit {} " + "isKey {}, isAllowNull {}, datatype {}, strlen {}, precision {}, scale {}, visible {}, " + "children {}, realDefaultValue {}, clusterKeyId {}", other.name, other.getDefaultValue(), other.aggregationType, other.isAggregationTypeImplicit, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 57048d345de..eedc77e9df7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -433,7 +433,7 @@ public class NereidsPlanner extends Planner { + "\n\n========== OPTIMIZED PLAN ==========\n" + optimizedPlan.treeString() + "\n\n========== MATERIALIZATIONS ==========\n" - + MaterializationContext.toString(cascadesContext.getMaterializationContexts()); + + MaterializationContext.toDetailString(cascadesContext.getMaterializationContexts()); break; case ALL_PLAN: plan = "========== PARSED PLAN " diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 24ecb41295b..995dbfb6b0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -103,16 +103,16 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate // get view and query aggregate and top plan correspondingly Pair<Plan, LogicalAggregate<Plan>> viewTopPlanAndAggPair = splitToTopPlanAndAggregate(viewStructInfo); if (viewTopPlanAndAggPair == null) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Split view to top plan and agg fail, view doesn't not contain aggregate", - String.format("view plan = %s\n", viewStructInfo.getOriginalPlan().treeString()))); + materializationContext.recordFailReason(queryStructInfo, + "Split view to top plan and agg fail, view doesn't not contain aggregate", + () -> String.format("view plan = %s\n", viewStructInfo.getOriginalPlan().treeString())); return null; } Pair<Plan, LogicalAggregate<Plan>> queryTopPlanAndAggPair = splitToTopPlanAndAggregate(queryStructInfo); if (queryTopPlanAndAggPair == null) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Split query to top plan and agg fail", - String.format("query plan = %s\n", queryStructInfo.getOriginalPlan().treeString()))); + materializationContext.recordFailReason(queryStructInfo, + "Split query to top plan and agg fail", + () -> String.format("query plan = %s\n", queryStructInfo.getOriginalPlan().treeString())); return null; } // Firstly,if group by expression between query and view is equals, try to rewrite expression directly @@ -131,13 +131,13 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate } // if fails, record the reason and then try to roll up aggregate function - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Can not rewrite expression when no roll up", - String.format("expressionToWrite = %s,\n mvExprToMvScanExprMapping = %s,\n" - + "viewToQuerySlotMapping = %s", - queryTopPlan.getOutput(), - materializationContext.getMvExprToMvScanExprMapping(), - viewToQuerySlotMapping))); + materializationContext.recordFailReason(queryStructInfo, + "Can not rewrite expression when no roll up", + () -> String.format("expressionToWrite = %s,\n mvExprToMvScanExprMapping = %s,\n" + + "viewToQuerySlotMapping = %s", + queryTopPlan.getOutput(), + materializationContext.getMvExprToMvScanExprMapping(), + viewToQuerySlotMapping)); } // if view is scalar aggregate but query is not. Or if query is scalar aggregate but view is not // Should not rewrite @@ -145,12 +145,12 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate List<Expression> viewGroupByExpressions = viewTopPlanAndAggPair.value().getGroupByExpressions(); if ((queryGroupByExpressions.isEmpty() && !viewGroupByExpressions.isEmpty()) || (!queryGroupByExpressions.isEmpty() && viewGroupByExpressions.isEmpty())) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("only one the of query or view is scalar aggregate and " - + "can not rewrite expression meanwhile", - String.format("query aggregate = %s,\n view aggregate = %s,\n", - queryTopPlanAndAggPair.value().treeString(), - viewTopPlanAndAggPair.value().treeString()))); + materializationContext.recordFailReason(queryStructInfo, + "only one the of query or view is scalar aggregate and " + + "can not rewrite expression meanwhile", + () -> String.format("query aggregate = %s,\n view aggregate = %s,\n", + queryTopPlanAndAggPair.value().treeString(), + viewTopPlanAndAggPair.value().treeString())); return null; } // try to roll up. @@ -178,10 +178,10 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate Expression rollupedExpression = queryFunctionShuttled.accept(AGGREGATE_EXPRESSION_REWRITER, context); if (!context.isValid()) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Query function roll up fail", - String.format("queryFunctionShuttled = %s,\n mvExprToMvScanExprQueryBased = %s", - queryFunctionShuttled, mvExprToMvScanExprQueryBased))); + materializationContext.recordFailReason(queryStructInfo, + "Query function roll up fail", + () -> String.format("queryFunctionShuttled = %s,\n mvExprToMvScanExprQueryBased = %s", + queryFunctionShuttled, mvExprToMvScanExprQueryBased)); return null; } finalOutputExpressions.add(new Alias(rollupedExpression)); @@ -196,10 +196,10 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate context); if (!context.isValid()) { // group expr can not rewrite by view - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("View dimensions doesn't not cover the query dimensions", - String.format("mvExprToMvScanExprQueryBased is %s,\n queryGroupShuttledExpr is %s", - mvExprToMvScanExprQueryBased, queryGroupShuttledExpr))); + materializationContext.recordFailReason(queryStructInfo, + "View dimensions doesn't not cover the query dimensions", + () -> String.format("mvExprToMvScanExprQueryBased is %s,\n queryGroupShuttledExpr is %s", + mvExprToMvScanExprQueryBased, queryGroupShuttledExpr)); return null; } NamedExpression groupByExpression = rewrittenGroupByExpression instanceof NamedExpression diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index dba0afb4925..df58345405a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.rules.exploration.mv; -import org.apache.doris.common.Pair; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.expressions.Alias; @@ -52,21 +51,14 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali ); // Can not rewrite, bail out if (expressionsRewritten.isEmpty()) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Rewrite expressions by view in join fail", - String.format("expressionToRewritten is %s,\n mvExprToMvScanExprMapping is %s,\n" - + "targetToSourceMapping = %s", - queryStructInfo.getExpressions(), - materializationContext.getMvExprToMvScanExprMapping(), - targetToSourceMapping))); + materializationContext.recordFailReason(queryStructInfo, + "Rewrite expressions by view in join fail", + () -> String.format("expressionToRewritten is %s,\n mvExprToMvScanExprMapping is %s,\n" + + "targetToSourceMapping = %s", queryStructInfo.getExpressions(), + materializationContext.getMvExprToMvScanExprMapping(), + targetToSourceMapping)); return null; } - // record the group id in materializationContext, and when rewrite again in - // the same group, bail out quickly. - if (queryStructInfo.getOriginalPlan().getGroupExpression().isPresent()) { - materializationContext.addMatchedGroup( - queryStructInfo.getOriginalPlan().getGroupExpression().get().getOwnerGroup().getGroupId()); - } return new LogicalProject<>( expressionsRewritten.stream() .map(expression -> expression instanceof NamedExpression ? expression : new Alias(expression)) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index e001359574e..32e90a67efa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -22,13 +22,11 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.TableIf; -import org.apache.doris.common.Pair; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVPartitionInfo; import org.apache.doris.mtmv.MTMVRewriteUtil; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.executor.Rewriter; -import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.exploration.ExplorationRuleFactory; import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; @@ -45,7 +43,6 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.JoinType; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -94,10 +91,13 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac // TODO Just support only one query struct info, support multi later. StructInfo queryStructInfo = queryStructInfos.get(0); try { - rewrittenPlans.addAll(doRewrite(queryStructInfo, cascadesContext, context)); + if (rewrittenPlans.size() < cascadesContext.getConnectContext() + .getSessionVariable().getMaterializedViewRewriteSuccessCandidateNum()) { + rewrittenPlans.addAll(doRewrite(queryStructInfo, cascadesContext, context)); + } } catch (Exception exception) { - context.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Materialized view rule exec fail", exception.toString())); + context.recordFailReason(queryStructInfo, + "Materialized view rule exec fail", exception::toString); } } return rewrittenPlans; @@ -117,10 +117,9 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac StructInfo queryStructInfo = queryStructInfos.get(0); if (!checkPattern(queryStructInfo)) { cascadesContext.getMaterializationContexts().forEach(ctx -> - ctx.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Query struct info is invalid", - String.format("queryPlan is %s", queryPlan.treeString()))) - ); + ctx.recordFailReason(queryStructInfo, "Query struct info is invalid", + () -> String.format("queryPlan is %s", queryPlan.treeString()) + )); return validQueryStructInfos; } validQueryStructInfos.add(queryStructInfo); @@ -138,64 +137,70 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac materializationContext.getMvPlan(), cascadesContext); if (viewStructInfos.size() > 1) { // view struct info should only have one - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("The num of view struct info is more then one", - String.format("mv plan is %s", materializationContext.getMvPlan().treeString()))); + materializationContext.recordFailReason(queryStructInfo, + "The num of view struct info is more then one", + () -> String.format("mv plan is %s", materializationContext.getMvPlan().treeString())); return rewriteResults; } StructInfo viewStructInfo = viewStructInfos.get(0); if (!checkPattern(viewStructInfo)) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("View struct info is invalid", - String.format(", view plan is %s", viewStructInfo.getOriginalPlan().treeString()))); + materializationContext.recordFailReason(queryStructInfo, + "View struct info is invalid", + () -> String.format(", view plan is %s", viewStructInfo.getOriginalPlan().treeString())); return rewriteResults; } MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations()); if (MatchMode.COMPLETE != matchMode) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Match mode is invalid", String.format("matchMode is %s", matchMode))); + materializationContext.recordFailReason(queryStructInfo, "Match mode is invalid", + () -> String.format("matchMode is %s", matchMode)); return rewriteResults; } List<RelationMapping> queryToViewTableMappings = RelationMapping.generate(queryStructInfo.getRelations(), viewStructInfo.getRelations()); // if any relation in query and view can not map, bail out. if (queryToViewTableMappings == null) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Query to view table mapping is null", "")); + materializationContext.recordFailReason(queryStructInfo, + "Query to view table mapping is null", () -> ""); return rewriteResults; } for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) { SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); if (queryToViewSlotMapping == null) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Query to view slot mapping is null", "")); + materializationContext.recordFailReason(queryStructInfo, + "Query to view slot mapping is null", () -> ""); continue; } SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse(); + // check the column used in query is in mv or not + if (!checkColumnUsedValid(queryStructInfo, viewStructInfo, queryToViewSlotMapping)) { + materializationContext.recordFailReason(queryStructInfo, + "The columns used by query are not in view", + () -> String.format("query struct info is %s, view struct info is %s", + queryStructInfo.getTopPlan().treeString(), + viewStructInfo.getTopPlan().treeString())); + continue; + } LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( queryToViewTableMapping, queryToViewSlotMapping, queryStructInfo, viewStructInfo); ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, compatibilityContext); if (comparisonResult.isInvalid()) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("The graph logic between query and view is not consistent", - comparisonResult.getErrorMessage())); + materializationContext.recordFailReason(queryStructInfo, + "The graph logic between query and view is not consistent", + comparisonResult::getErrorMessage); continue; } SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo, viewToQuerySlotMapping, comparisonResult, cascadesContext); // Can not compensate, bail out if (compensatePredicates.isInvalid()) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Predicate compensate fail", - String.format("query predicates = %s,\n query equivalenceClass = %s, \n" - + "view predicates = %s,\n query equivalenceClass = %s\n" - + "comparisonResult = %s ", - queryStructInfo.getPredicates(), - queryStructInfo.getEquivalenceClass(), - viewStructInfo.getPredicates(), - viewStructInfo.getEquivalenceClass(), - comparisonResult))); + materializationContext.recordFailReason(queryStructInfo, + "Predicate compensate fail", + () -> String.format("query predicates = %s,\n query equivalenceClass = %s, \n" + + "view predicates = %s,\n query equivalenceClass = %s\n" + + "comparisonResult = %s ", queryStructInfo.getPredicates(), + queryStructInfo.getEquivalenceClass(), viewStructInfo.getPredicates(), + viewStructInfo.getEquivalenceClass(), comparisonResult)); continue; } Plan rewrittenPlan; @@ -209,13 +214,12 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac originalPlan, materializationContext.getMvExprToMvScanExprMapping(), viewToQuerySlotMapping, true); if (rewriteCompensatePredicates.isEmpty()) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Rewrite compensate predicate by view fail", String.format( - "compensatePredicates = %s,\n mvExprToMvScanExprMapping = %s,\n" + materializationContext.recordFailReason(queryStructInfo, + "Rewrite compensate predicate by view fail", + () -> String.format("compensatePredicates = %s,\n mvExprToMvScanExprMapping = %s,\n" + "viewToQuerySlotMapping = %s", - compensatePredicates, - materializationContext.getMvExprToMvScanExprMapping(), - viewToQuerySlotMapping))); + compensatePredicates, materializationContext.getMvExprToMvScanExprMapping(), + viewToQuerySlotMapping)); continue; } rewrittenPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan); @@ -226,37 +230,55 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac if (rewrittenPlan == null) { continue; } - rewrittenPlan = rewriteByRules(cascadesContext, rewrittenPlan, originalPlan); - if (!isOutputValid(originalPlan, rewrittenPlan)) { - ObjectId planObjId = originalPlan.getGroupExpression().map(GroupExpression::getId) - .orElseGet(() -> new ObjectId(-1)); - materializationContext.recordFailReason(planObjId, Pair.of( + final Plan finalRewrittenPlan = rewriteByRules(cascadesContext, rewrittenPlan, originalPlan); + if (!isOutputValid(originalPlan, finalRewrittenPlan)) { + materializationContext.recordFailReason(queryStructInfo, "RewrittenPlan output logical properties is different with target group", - String.format("planOutput logical properties = %s,\n" - + "groupOutput logical properties = %s", rewrittenPlan.getLogicalProperties(), - originalPlan.getLogicalProperties()))); + () -> String.format("planOutput logical" + + " properties = %s,\n groupOutput logical properties = %s", + finalRewrittenPlan.getLogicalProperties(), originalPlan.getLogicalProperties())); continue; } // check the partitions used by rewritten plan is valid or not Set<Long> invalidPartitionsQueryUsed = - calcInvalidPartitions(rewrittenPlan, materializationContext, cascadesContext); + calcInvalidPartitions(finalRewrittenPlan, materializationContext, cascadesContext); if (!invalidPartitionsQueryUsed.isEmpty()) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Check partition query used validation fail", - String.format("the partition used by query is invalid by materialized view," - + "invalid partition info query used is %s", - materializationContext.getMTMV().getPartitions().stream() - .filter(partition -> - invalidPartitionsQueryUsed.contains(partition.getId())) - .collect(Collectors.toSet())))); + materializationContext.recordFailReason(queryStructInfo, + "Check partition query used validation fail", + () -> String.format("the partition used by query is invalid by materialized view," + + "invalid partition info query used is %s", + materializationContext.getMTMV().getPartitions().stream() + .filter(partition -> + invalidPartitionsQueryUsed.contains(partition.getId())) + .collect(Collectors.toSet()))); continue; } recordIfRewritten(originalPlan, materializationContext); - rewriteResults.add(rewrittenPlan); + rewriteResults.add(finalRewrittenPlan); } return rewriteResults; } + /** + * Check the column used by query is in materialized view output or not + */ + protected boolean checkColumnUsedValid(StructInfo queryInfo, StructInfo mvInfo, + SlotMapping queryToViewSlotMapping) { + Set<ExprId> queryUsedSlotSetViewBased = ExpressionUtils.shuttleExpressionWithLineage( + queryInfo.getTopPlan().getOutput(), queryInfo.getTopPlan()).stream() + .flatMap(expr -> ExpressionUtils.replace(expr, queryToViewSlotMapping.toSlotReferenceMap()) + .collectToSet(each -> each instanceof Slot).stream()) + .map(each -> ((Slot) each).getExprId()) + .collect(Collectors.toSet()); + + Set<ExprId> viewUsedSlotSet = ExpressionUtils.shuttleExpressionWithLineage(mvInfo.getTopPlan().getOutput(), + mvInfo.getTopPlan()).stream() + .flatMap(expr -> expr.collectToSet(each -> each instanceof Slot).stream()) + .map(each -> ((Slot) each).getExprId()) + .collect(Collectors.toSet()); + return viewUsedSlotSet.containsAll(queryUsedSlotSetViewBased); + } + /** * Rewrite by rules and try to make output is the same after optimize by rules */ @@ -324,7 +346,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac } // get mv valid partitions Set<Long> mvDataValidPartitionIdSet = MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv, - cascadesContext.getConnectContext(), System.currentTimeMillis()).stream() + cascadesContext.getConnectContext(), System.currentTimeMillis()).stream() .map(Partition::getId) .collect(Collectors.toSet()); Set<Long> queryUsedPartitionIdSet = rewrittenPlan.collectToList(node -> node instanceof LogicalOlapScan @@ -521,8 +543,8 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac .collect(Collectors.toSet()); // query pulledUp predicates should have null reject predicates and contains any require noNullable slot return !queryPulledUpPredicates.containsAll(nullRejectPredicates) - && requireNoNullableViewSlot.stream().noneMatch( - set -> Sets.intersection(set, queryUsedNeedRejectNullSlotsViewBased).isEmpty()); + && requireNoNullableViewSlot.stream().noneMatch(set -> + Sets.intersection(set, queryUsedNeedRejectNullSlotsViewBased).isEmpty()); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index ab8f4d69d45..8046639d1ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -95,7 +95,7 @@ public class InitMaterializationContextHook implements PlannerHook { // todo should force keep consistency to mv sql plan output Plan projectScan = new LogicalProject<Plan>(mvProjects, mvScan); cascadesContext.addMaterializationContext( - MaterializationContext.fromMaterializedView(materializedView, projectScan)); + MaterializationContext.fromMaterializedView(materializedView, projectScan, cascadesContext)); }); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index e64e1290954..b3de7443652 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -17,15 +17,18 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Pair; import org.apache.doris.mtmv.MTMVCache; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupId; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; @@ -38,6 +41,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -64,16 +68,20 @@ public class MaterializationContext { // if rewrite by mv fail, record the reason, if success the failReason should be empty. // The key is the query belonged group expression objectId, the value is the fail reason private final Map<ObjectId, Pair<String, String>> failReason = new HashMap<>(); + private boolean enableRecordFailureDetail = false; /** * MaterializationContext, this contains necessary info for query rewriting by mv */ - public MaterializationContext(MTMV mtmv, Plan mvScanPlan, List<Table> baseTables, List<Table> baseViews) { + public MaterializationContext(MTMV mtmv, Plan mvScanPlan, List<Table> baseTables, List<Table> baseViews, + CascadesContext cascadesContext) { this.mtmv = mtmv; this.mvScanPlan = mvScanPlan; this.baseTables = baseTables; this.baseViews = baseViews; - + StatementBase parsedStatement = cascadesContext.getStatementContext().getParsedStatement(); + this.enableRecordFailureDetail = parsedStatement != null && parsedStatement.isExplain() + && ExplainLevel.MEMO_PLAN == parsedStatement.getExplainOptions().getExplainLevel(); MTMVCache mtmvCache = null; try { mtmvCache = mtmv.getOrGenerateCache(); @@ -94,10 +102,6 @@ public class MaterializationContext { this.mvPlan = mtmvCache.getLogicalPlan(); } - public Set<GroupId> getMatchedGroups() { - return matchedGroups; - } - public boolean alreadyRewrite(GroupId groupId) { return this.matchedGroups.contains(groupId); } @@ -138,6 +142,10 @@ public class MaterializationContext { return failReason; } + public boolean isEnableRecordFailureDetail() { + return enableRecordFailureDetail; + } + public void setSuccess(boolean success) { this.success = success; this.failReason.clear(); @@ -146,13 +154,18 @@ public class MaterializationContext { /** * recordFailReason */ - public void recordFailReason(ObjectId objectId, Pair<String, String> summaryAndReason) { + public void recordFailReason(StructInfo structInfo, String summary, Supplier<String> failureReasonSupplier) { + // record it's rewritten + if (structInfo.getTopPlan().getGroupExpression().isPresent()) { + this.addMatchedGroup(structInfo.getTopPlan().getGroupExpression().get().getOwnerGroup().getGroupId()); + } // once success, do not record the fail reason if (this.success) { return; } this.success = false; - this.failReason.put(objectId, summaryAndReason); + this.failReason.put(structInfo.getOriginalPlanId(), + Pair.of(summary, this.isEnableRecordFailureDetail() ? failureReasonSupplier.get() : "")); } public boolean isSuccess() { @@ -178,7 +191,7 @@ public class MaterializationContext { /** * toString, this contains summary and detail info. */ - public static String toString(List<MaterializationContext> materializationContexts) { + public static String toDetailString(List<MaterializationContext> materializationContexts) { StringBuilder builder = new StringBuilder(); builder.append("materializationContexts:").append("\n"); for (MaterializationContext ctx : materializationContexts) { @@ -200,6 +213,22 @@ public class MaterializationContext { .collect(Collectors.toSet()); StringBuilder builder = new StringBuilder(); builder.append("\nMaterializedView"); + // rewrite success and chosen + builder.append("\nMaterializedViewRewriteSuccessAndChose:\n"); + if (!materializationChosenNameSet.isEmpty()) { + builder.append(" Names: ").append(String.join(", ", materializationChosenNameSet)); + } + // rewrite success but not chosen + builder.append("\nMaterializedViewRewriteSuccessButNotChose:\n"); + Set<String> rewriteSuccessButNotChoseNameSet = materializationContexts.stream() + .filter(materializationContext -> materializationContext.isSuccess() + && !materializationChosenNameSet.contains(materializationContext.getMTMV().getName())) + .map(materializationContext -> materializationContext.getMTMV().getName()) + .collect(Collectors.toSet()); + if (!rewriteSuccessButNotChoseNameSet.isEmpty()) { + builder.append(" Names: ").append(String.join(", ", rewriteSuccessButNotChoseNameSet)); + } + // rewrite fail builder.append("\nMaterializedViewRewriteFail:"); for (MaterializationContext ctx : materializationContexts) { if (!ctx.isSuccess()) { @@ -211,25 +240,15 @@ public class MaterializationContext { .append(" FailSummary: ").append(String.join(", ", failReasonSet)); } } - builder.append("\nMaterializedViewRewriteSuccessButNotChose:\n"); - builder.append(" Names: ").append(materializationContexts.stream() - .filter(materializationContext -> materializationContext.isSuccess() - && !materializationChosenNameSet.contains(materializationContext.getMTMV().getName())) - .map(materializationContext -> materializationContext.getMTMV().getName()) - .collect(Collectors.joining(", "))); - builder.append("\nMaterializedViewRewriteSuccessAndChose:\n"); - builder.append(" Names: ").append(String.join(", ", materializationChosenNameSet)); return builder.toString(); } /** * MaterializationContext fromMaterializedView */ - public static MaterializationContext fromMaterializedView(MTMV materializedView, Plan mvScanPlan) { - return new MaterializationContext( - materializedView, - mvScanPlan, - ImmutableList.of(), - ImmutableList.of()); + public static MaterializationContext fromMaterializedView(MTMV materializedView, Plan mvScanPlan, + CascadesContext cascadesContext) { + return new MaterializationContext(materializedView, mvScanPlan, ImmutableList.of(), ImmutableList.of(), + cascadesContext); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index f36e831a5c4..634904202ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -498,6 +498,9 @@ public class SessionVariable implements Serializable, Writable { public static final String MATERIALIZED_VIEW_REWRITE_ENABLE_CONTAIN_EXTERNAL_TABLE = "materialized_view_rewrite_enable_contain_external_table"; + public static final String MATERIALIZED_VIEW_REWRITE_SUCCESS_CANDIDATE_NUM + = "materialized_view_rewrite_success_candidate_num"; + public static final String CREATE_TABLE_PARTITION_MAX_NUM = "create_table_partition_max_num"; @@ -1582,9 +1585,13 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = MATERIALIZED_VIEW_REWRITE_ENABLE_CONTAIN_EXTERNAL_TABLE, needForward = true, description = {"基于结构信息的透明改写，是否使用包含外表的物化视图", - "whether to use a materialized view that contains the foreign table " + "Whether to use a materialized view that contains the foreign table " + "when using rewriting based on struct info"}) public boolean materializedViewRewriteEnableContainExternalTable = false; + @VariableMgr.VarAttr(name = MATERIALIZED_VIEW_REWRITE_SUCCESS_CANDIDATE_NUM, needForward = true, + description = {"异步物化视图透明改写成功的结果集合，允许参与到CBO候选的最大数量", + "The max candidate num which participate in CBO when using asynchronous materialized views"}) + public int materializedViewRewriteSuccessCandidateNum = 3; @VariableMgr.VarAttr(name = CREATE_TABLE_PARTITION_MAX_NUM, needForward = true, description = {"建表时创建分区的最大数量", @@ -3432,6 +3439,10 @@ public class SessionVariable implements Serializable, Writable { return materializedViewRewriteEnableContainExternalTable; } + public int getMaterializedViewRewriteSuccessCandidateNum() { + return materializedViewRewriteSuccessCandidateNum; + } + public int getCreateTablePartitionMaxNum() { return createTablePartitionMaxNum; } diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy index 0b0c4aa107b..b314927636b 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy @@ -168,8 +168,8 @@ suite("aggregate_with_roll_up") { explain { sql("${query_sql}") check {result -> - def splitResult = result.split("MaterializedViewRewriteSuccessButNotChose") - splitResult.length == 2 ? splitResult[1].contains(mv_name) : false + def splitResult = result.split("MaterializedViewRewriteFail") + splitResult.length == 2 ? splitResult[0].contains(mv_name) : false } } } diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy index 988bbdbd803..2b81dfd8aea 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy @@ -520,9 +520,9 @@ suite("partition_mv_rewrite_dimension_1") { o_comment """ def agg_sql_explain_2 = sql """explain ${agg_sql_2};""" - def mv_index_1 = agg_sql_explain_2.toString().indexOf("MaterializedViewRewriteSuccessButNotChose:") + def mv_index_1 = agg_sql_explain_2.toString().indexOf("MaterializedViewRewriteFail:") assert(mv_index_1 != -1) - assert(agg_sql_explain_2.toString().substring(mv_index_1).indexOf(agg_mv_name_2) != -1) + assert(agg_sql_explain_2.toString().substring(0, mv_index_1).indexOf(agg_mv_name_2) != -1) sql """DROP MATERIALIZED VIEW IF EXISTS ${agg_mv_name_2};""" // agg + with group by + with agg function @@ -559,9 +559,9 @@ suite("partition_mv_rewrite_dimension_1") { o_comment """ def agg_sql_explain_3 = sql """explain ${agg_sql_3};""" - def mv_index_2 = agg_sql_explain_3.toString().indexOf("MaterializedViewRewriteSuccessButNotChose:") + def mv_index_2 = agg_sql_explain_3.toString().indexOf("MaterializedViewRewriteFail:") assert(mv_index_2 != -1) - assert(agg_sql_explain_3.toString().substring(mv_index_2).indexOf(agg_mv_name_3) != -1) + assert(agg_sql_explain_3.toString().substring(0, mv_index_2).indexOf(agg_mv_name_3) != -1) sql """DROP MATERIALIZED VIEW IF EXISTS ${agg_mv_name_3};""" diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy index 60a3aed84ac..180cf9b9095 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy @@ -564,9 +564,9 @@ suite("partition_mv_rewrite_dimension_2_4") { o_comment """ def agg_sql_explain_1 = sql """explain ${sql_stmt_16};""" - def mv_index_1 = agg_sql_explain_1.toString().indexOf("MaterializedViewRewriteSuccessButNotChose:") + def mv_index_1 = agg_sql_explain_1.toString().indexOf("MaterializedViewRewriteFail:") assert(mv_index_1 != -1) - assert(agg_sql_explain_1.toString().substring(mv_index_1).indexOf(mv_name_16) != -1) + assert(agg_sql_explain_1.toString().substring(0, mv_index_1).indexOf(mv_name_16) != -1) compare_res(sql_stmt_16 + " order by 1,2,3") sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name_16};""" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
