This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new b76cfcd007b [refactor](mtmv) Materialization context and mtmv
decoupling (#34093) (#34916)
b76cfcd007b is described below
commit b76cfcd007b1548bb8303acb50d4a36a16ad9c6e
Author: seawinde <[email protected]>
AuthorDate: Fri May 17 22:54:21 2024 +0800
[refactor](mtmv) Materialization context and mtmv decoupling (#34093)
(#34916)
Decoupling the MTMV from the materialization context.
Change MaterializationContext to abstract which is the materialization desc.
It now has AsyncMaterializationContext sub class, can also has other type
of MaterializationContext such as
SyncMaterializationContext and so on.
---
.../main/java/org/apache/doris/mtmv/MTMVCache.java | 5 +-
.../org/apache/doris/nereids/NereidsPlanner.java | 16 +-
.../mv/AbstractMaterializedViewRule.java | 82 +++----
.../mv/AsyncMaterializationContext.java | 140 ++++++++++++
.../mv/InitMaterializationContextHook.java | 19 +-
.../exploration/mv/MaterializationContext.java | 250 +++++++++++----------
.../exploration/mv/MaterializedViewUtils.java | 3 +
.../nereids/rules/exploration/mv/StructInfo.java | 7 +-
8 files changed, 351 insertions(+), 171 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
index 154bd4ec7f1..8bd87e2e149 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
@@ -42,9 +42,10 @@ import com.google.common.collect.ImmutableList;
*/
public class MTMVCache {
- // the materialized view plan which should be optimized by the same rules
to query
+ // The materialized view plan which should be optimized by the same rules
to query
+ // and will remove top sink and unused sort
private final Plan logicalPlan;
- // for stable output order, we should use original plan
+ // The original plan of mv def sql
private final Plan originalPlan;
public MTMVCache(Plan logicalPlan, Plan originalPlan) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 6f65ee9538c..2a354eb9c4b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -23,7 +23,6 @@ import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
-import org.apache.doris.catalog.MTMV;
import org.apache.doris.common.NereidsException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.profile.SummaryProfile;
@@ -53,7 +52,6 @@ import org.apache.doris.nereids.trees.plans.Plan;
import
org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache;
-import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
@@ -467,11 +465,16 @@ public class NereidsPlanner extends Planner {
plan = optimizedPlan.shape("");
break;
case MEMO_PLAN:
+ StringBuilder materializationStringBuilder = new
StringBuilder();
+
materializationStringBuilder.append("materializationContexts:").append("\n");
+ for (MaterializationContext ctx :
cascadesContext.getMaterializationContexts()) {
+
materializationStringBuilder.append("\n").append(ctx).append("\n");
+ }
plan = cascadesContext.getMemo().toString()
+ "\n\n========== OPTIMIZED PLAN ==========\n"
+ optimizedPlan.treeString()
+ "\n\n========== MATERIALIZATIONS ==========\n"
- +
MaterializationContext.toDetailString(cascadesContext.getMaterializationContexts());
+ + materializationStringBuilder;
break;
case ALL_PLAN:
plan = "========== PARSED PLAN "
@@ -488,14 +491,9 @@ public class NereidsPlanner extends Planner {
+ optimizedPlan.treeString();
break;
default:
- List<MTMV> materializationListChosenByCbo =
this.getPhysicalPlan()
- .collectToList(node -> node instanceof
PhysicalCatalogRelation
- && ((PhysicalCatalogRelation) node).getTable()
instanceof MTMV).stream()
- .map(node -> (MTMV) ((PhysicalCatalogRelation)
node).getTable())
- .collect(Collectors.toList());
plan = super.getExplainString(explainOptions)
+
MaterializationContext.toSummaryString(cascadesContext.getMaterializationContexts(),
- materializationListChosenByCbo);
+ this.getPhysicalPlan());
}
if (statementContext != null &&
!statementContext.getHints().isEmpty()) {
String hint = getHintExplainString(statementContext.getHints());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index 7a08797ca55..dcff9db69db 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -237,14 +237,14 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
// Rewrite query by view
rewrittenPlan = rewriteQueryByView(matchMode, queryStructInfo,
viewStructInfo, viewToQuerySlotMapping,
rewrittenPlan, materializationContext);
- if (rewrittenPlan == null) {
- continue;
- }
rewrittenPlan =
MaterializedViewUtils.rewriteByRules(cascadesContext,
childContext -> {
Rewriter.getWholeTreeRewriter(childContext).execute();
return childContext.getRewritePlan();
}, rewrittenPlan, queryPlan);
+ if (rewrittenPlan == null) {
+ continue;
+ }
// check the partitions used by rewritten plan is valid or not
Multimap<Pair<MTMVPartitionInfo, PartitionInfo>, Partition>
invalidPartitionsQueryUsed =
calcUsedInvalidMvPartitions(rewrittenPlan,
materializationContext, cascadesContext);
@@ -287,9 +287,10 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
}
// For rewrittenPlan which contains materialized view should
remove invalid partition ids
List<Plan> children = Lists.newArrayList(
- rewrittenPlan.accept(new InvalidPartitionRemover(),
Pair.of(materializationContext.getMTMV(),
- invalidPartitionsQueryUsed.values().stream()
-
.map(Partition::getId).collect(Collectors.toSet()))),
+ rewrittenPlan.accept(new InvalidPartitionRemover(),
Pair.of(
+
materializationContext.getMaterializationQualifier(),
+
invalidPartitionsQueryUsed.values().stream().map(Partition::getId)
+ .collect(Collectors.toSet()))),
StructInfo.addFilterOnTableScan(queryPlan,
filterOnOriginPlan, cascadesContext));
// Union query materialized view and source table
rewrittenPlan = new LogicalUnion(Qualifier.ALL,
@@ -387,40 +388,43 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
Plan rewrittenPlan,
MaterializationContext materializationContext,
CascadesContext cascadesContext) {
- // check partition is valid or not
- MTMV mtmv = materializationContext.getMTMV();
- PartitionInfo mvPartitionInfo = mtmv.getPartitionInfo();
- if (PartitionType.UNPARTITIONED.equals(mvPartitionInfo.getType())) {
- // if not partition, if rewrite success, it means mv is available
- return ImmutableMultimap.of();
- }
- // check mv related table partition is valid or not
- MTMVPartitionInfo mvCustomPartitionInfo = mtmv.getMvPartitionInfo();
- BaseTableInfo relatedPartitionTable =
mvCustomPartitionInfo.getRelatedTableInfo();
- if (relatedPartitionTable == null) {
- return ImmutableMultimap.of();
+ if (materializationContext instanceof AsyncMaterializationContext) {
+ // check partition is valid or not
+ MTMV mtmv = ((AsyncMaterializationContext)
materializationContext).getMtmv();
+ PartitionInfo mvPartitionInfo = mtmv.getPartitionInfo();
+ if (PartitionType.UNPARTITIONED.equals(mvPartitionInfo.getType()))
{
+ // if not partition, if rewrite success, it means mv is
available
+ return ImmutableMultimap.of();
+ }
+ // check mv related table partition is valid or not
+ MTMVPartitionInfo mvCustomPartitionInfo =
mtmv.getMvPartitionInfo();
+ BaseTableInfo relatedPartitionTable =
mvCustomPartitionInfo.getRelatedTableInfo();
+ if (relatedPartitionTable == null) {
+ return ImmutableMultimap.of();
+ }
+ // get mv valid partitions
+ Set<Long> mvDataValidPartitionIdSet =
MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv,
+ cascadesContext.getConnectContext(),
System.currentTimeMillis()).stream()
+ .map(Partition::getId)
+ .collect(Collectors.toSet());
+ // get partitions query used
+ Set<Long> mvPartitionSetQueryUsed =
rewrittenPlan.collectToList(node -> node instanceof LogicalOlapScan
+ && Objects.equals(((CatalogRelation)
node).getTable().getName(), mtmv.getName()))
+ .stream()
+ .map(node -> ((LogicalOlapScan)
node).getSelectedPartitionIds())
+ .flatMap(Collection::stream)
+ .collect(Collectors.toSet());
+ // get invalid partition ids
+ Set<Long> invalidMvPartitionIdSet = new
HashSet<>(mvPartitionSetQueryUsed);
+ invalidMvPartitionIdSet.removeAll(mvDataValidPartitionIdSet);
+ ImmutableMultimap.Builder<Pair<MTMVPartitionInfo, PartitionInfo>,
Partition> invalidPartitionMapBuilder =
+ ImmutableMultimap.builder();
+ Pair<MTMVPartitionInfo, PartitionInfo> partitionInfo =
Pair.of(mvCustomPartitionInfo, mvPartitionInfo);
+ invalidMvPartitionIdSet.forEach(invalidPartitionId ->
+ invalidPartitionMapBuilder.put(partitionInfo,
mtmv.getPartition(invalidPartitionId)));
+ return invalidPartitionMapBuilder.build();
}
- // get mv valid partitions
- Set<Long> mvDataValidPartitionIdSet =
MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv,
- cascadesContext.getConnectContext(),
System.currentTimeMillis()).stream()
- .map(Partition::getId)
- .collect(Collectors.toSet());
- // get partitions query used
- Set<Long> mvPartitionSetQueryUsed = rewrittenPlan.collectToList(node
-> node instanceof LogicalOlapScan
- && Objects.equals(((CatalogRelation)
node).getTable().getName(), mtmv.getName()))
- .stream()
- .map(node -> ((LogicalOlapScan)
node).getSelectedPartitionIds())
- .flatMap(Collection::stream)
- .collect(Collectors.toSet());
- // get invalid partition ids
- Set<Long> invalidMvPartitionIdSet = new
HashSet<>(mvPartitionSetQueryUsed);
- invalidMvPartitionIdSet.removeAll(mvDataValidPartitionIdSet);
- ImmutableMultimap.Builder<Pair<MTMVPartitionInfo, PartitionInfo>,
Partition> invalidPartitionMapBuilder =
- ImmutableMultimap.builder();
- Pair<MTMVPartitionInfo, PartitionInfo> partitionInfo =
Pair.of(mvCustomPartitionInfo, mvPartitionInfo);
- invalidMvPartitionIdSet.forEach(invalidPartitionId ->
- invalidPartitionMapBuilder.put(partitionInfo,
mtmv.getPartition(invalidPartitionId)));
- return invalidPartitionMapBuilder.build();
+ return ImmutableMultimap.of();
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java
new file mode 100644
index 00000000000..1c0d854dd90
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java
@@ -0,0 +1,140 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.catalog.MTMV;
+import org.apache.doris.catalog.Table;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
+import org.apache.doris.nereids.trees.plans.ObjectId;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.Relation;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation;
+import org.apache.doris.nereids.util.Utils;
+
+import com.google.common.collect.Multimap;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Async context for query rewrite by materialized view
+ */
+public class AsyncMaterializationContext extends MaterializationContext {
+
+ private static final Logger LOG =
LogManager.getLogger(AsyncMaterializationContext.class);
+ private final MTMV mtmv;
+
+ /**
+ * MaterializationContext, this contains necessary info for query
rewriting by mv
+ */
+ public AsyncMaterializationContext(MTMV mtmv, Plan mvPlan, Plan
mvOriginalPlan, List<Table> baseTables,
+ List<Table> baseViews, CascadesContext cascadesContext) {
+ super(mvPlan, mvOriginalPlan,
MaterializedViewUtils.generateMvScanPlan(mtmv, cascadesContext),
cascadesContext);
+ this.mtmv = mtmv;
+ }
+
+ public MTMV getMtmv() {
+ return mtmv;
+ }
+
+ @Override
+ Plan doGenerateMvPlan(CascadesContext cascadesContext) {
+ return MaterializedViewUtils.generateMvScanPlan(this.mtmv,
cascadesContext);
+ }
+
+ @Override
+ List<String> getMaterializationQualifier() {
+ return this.mtmv.getFullQualifiers();
+ }
+
+ @Override
+ String getStringInfo() {
+ StringBuilder failReasonBuilder = new StringBuilder("[").append("\n");
+ for (Map.Entry<ObjectId, Collection<Pair<String, String>>> reasonEntry
: this.failReason.asMap().entrySet()) {
+ failReasonBuilder
+ .append("\n")
+ .append("ObjectId :
").append(reasonEntry.getKey()).append(".\n");
+ for (Pair<String, String> reason : reasonEntry.getValue()) {
+ failReasonBuilder.append("Summary :
").append(reason.key()).append(".\n")
+ .append("Reason :
").append(reason.value()).append(".\n");
+ }
+ }
+ failReasonBuilder.append("\n").append("]");
+ return Utils.toSqlString("MaterializationContext[" +
getMaterializationQualifier() + "]",
+ "rewriteSuccess", this.success,
+ "failReason", failReasonBuilder.toString());
+ }
+
+ @Override
+ boolean isFinalChosen(Relation relation) {
+ if (!(relation instanceof PhysicalCatalogRelation)) {
+ return false;
+ }
+ return ((PhysicalCatalogRelation) relation).getTable() instanceof MTMV;
+ }
+
+ public Plan getMvScanPlan() {
+ return mvScanPlan;
+ }
+
+ public List<Table> getBaseTables() {
+ return baseTables;
+ }
+
+ public List<Table> getBaseViews() {
+ return baseViews;
+ }
+
+ public ExpressionMapping getMvExprToMvScanExprMapping() {
+ return mvExprToMvScanExprMapping;
+ }
+
+ public boolean isAvailable() {
+ return available;
+ }
+
+ public Plan getMvPlan() {
+ return mvPlan;
+ }
+
+ public Multimap<ObjectId, Pair<String, String>> getFailReason() {
+ return failReason;
+ }
+
+ public boolean isEnableRecordFailureDetail() {
+ return enableRecordFailureDetail;
+ }
+
+ public void setSuccess(boolean success) {
+ this.success = success;
+ this.failReason.clear();
+ }
+
+ public StructInfo getStructInfo() {
+ return structInfo;
+ }
+
+ public boolean isSuccess() {
+ return success;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index 08c312b4737..311932fa1b7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -20,7 +20,9 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.AnalysisException;
import org.apache.doris.mtmv.BaseTableInfo;
+import org.apache.doris.mtmv.MTMVCache;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.PlannerHook;
@@ -29,6 +31,7 @@ import
org.apache.doris.nereids.trees.plans.visitor.TableCollector;
import
org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -75,10 +78,18 @@ public class InitMaterializationContextHook implements
PlannerHook {
return;
}
for (MTMV materializedView : availableMTMVs) {
- cascadesContext.addMaterializationContext(
-
MaterializationContext.fromMaterializedView(materializedView,
-
MaterializedViewUtils.generateMvScanPlan(materializedView, cascadesContext),
- cascadesContext));
+ MTMVCache mtmvCache = null;
+ try {
+ mtmvCache = materializedView.getOrGenerateCache();
+ } catch (AnalysisException e) {
+ LOG.warn("MaterializationContext init mv cache generate fail",
e);
+ }
+ if (mtmvCache == null) {
+ continue;
+ }
+ cascadesContext.addMaterializationContext(new
AsyncMaterializationContext(materializedView,
+ mtmvCache.getLogicalPlan(), mtmvCache.getOriginalPlan(),
ImmutableList.of(), ImmutableList.of(),
+ cascadesContext));
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
index 1af63dab21f..261e3bb85f9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
@@ -18,102 +18,97 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.analysis.StatementBase;
-import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Table;
-import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
-import org.apache.doris.mtmv.MTMVCache;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.GroupId;
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.ObjectId;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.Relation;
import
org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
+import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
import org.apache.doris.nereids.util.ExpressionUtils;
-import org.apache.doris.nereids.util.Utils;
-import com.google.common.collect.ImmutableList;
+import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.BitSet;
import java.util.HashSet;
-import java.util.LinkedHashMap;
import java.util.List;
-import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Collectors;
/**
- * Maintain the context for query rewrite by materialized view
+ * Abstract context for query rewrite by materialized view
*/
-public class MaterializationContext {
-
+public abstract class MaterializationContext {
private static final Logger LOG =
LogManager.getLogger(MaterializationContext.class);
-
- private final MTMV mtmv;
- private final List<Table> baseTables;
- private final List<Table> baseViews;
- // Group ids that are rewritten by this mv to reduce rewrite times
- private final Set<GroupId> matchedFailGroups = new HashSet<>();
- private final Set<GroupId> matchedSuccessGroups = new HashSet<>();
- // if rewrite by mv fail, record the reason, if success the failReason
should be empty.
- // The key is the query belonged group expression objectId, the value is
the fail reason
- private final Map<ObjectId, Pair<String, String>> failReason = new
LinkedHashMap<>();
+ protected List<Table> baseTables;
+ protected List<Table> baseViews;
+ // The plan of mv def sql
+ protected Plan mvPlan;
+ // The original plan of mv def sql
+ protected Plan originalMvPlan;
// Should regenerate when materialization is already rewritten
successfully because one query may hit repeatedly
// make sure output is different in multi using
- private Plan mvScanPlan;
- // generated expressions form mv scan plan
- private ExpressionMapping mvExprToMvScanExprMapping;
- private List<? extends Expression> mvPlanOutputShuttledExpressions;
- private boolean available = true;
- // the mv plan from cache at present, record it to make sure query rewrite
by mv is right when cache change.
- private Plan mvPlan;
- // mark rewrite success or not
- private boolean success = false;
- private boolean enableRecordFailureDetail = false;
- private StructInfo structInfo;
+ protected Plan mvScanPlan;
+ // The mvPlan output shuttled expression, this is used by generate field
mvExprToMvScanExprMapping
+ protected List<? extends Expression> mvPlanOutputShuttledExpressions;
+ // Generated mapping from mv plan out shuttled expr to mv scan plan out
slot mapping, this is used for later used
+ protected ExpressionMapping mvExprToMvScanExprMapping;
+ // This mark the materialization context is available or not,
+ // will not be used in query transparent rewritten if false
+ protected boolean available = true;
+ // Mark the mv plan in the context is already rewritten successfully or not
+ protected boolean success = false;
+ // Mark enable record failure detail info or not, because record failure
detail info is performance-depleting
+ protected final boolean enableRecordFailureDetail;
+ // The mv plan struct info
+ protected final StructInfo structInfo;
+ // Group id set that are rewritten unsuccessfully by this mv for reducing
rewrite times
+ protected final Set<GroupId> matchedFailGroups = new HashSet<>();
+ // Group id set that are rewritten successfully by this mv for reducing
rewrite times
+ protected final Set<GroupId> matchedSuccessGroups = new HashSet<>();
+ // Record the reason, if rewrite by mv fail. The failReason should be
empty if success.
+ // The key is the query belonged group expression objectId, the value is
the fail reasons because
+ // for one materialization query may be multi when nested materialized
view.
+ protected final Multimap<ObjectId, Pair<String, String>> failReason =
HashMultimap.create();
/**
* MaterializationContext, this contains necessary info for query
rewriting by mv
*/
- public MaterializationContext(MTMV mtmv, Plan mvScanPlan, List<Table>
baseTables, List<Table> baseViews,
- CascadesContext cascadesContext) {
- this.mtmv = mtmv;
+ public MaterializationContext(Plan mvPlan, Plan originalMvPlan, Plan
mvScanPlan, CascadesContext cascadesContext) {
+ this.mvPlan = mvPlan;
+ this.originalMvPlan = originalMvPlan;
this.mvScanPlan = mvScanPlan;
- this.baseTables = baseTables;
- this.baseViews = baseViews;
+
StatementBase parsedStatement =
cascadesContext.getStatementContext().getParsedStatement();
this.enableRecordFailureDetail = parsedStatement != null &&
parsedStatement.isExplain()
&& ExplainLevel.MEMO_PLAN ==
parsedStatement.getExplainOptions().getExplainLevel();
- MTMVCache mtmvCache = null;
- try {
- mtmvCache = mtmv.getOrGenerateCache();
- } catch (AnalysisException e) {
- LOG.warn("MaterializationContext init mv cache generate fail", e);
- }
- if (mtmvCache == null) {
- this.available = false;
- return;
- }
+
this.mvPlanOutputShuttledExpressions =
ExpressionUtils.shuttleExpressionWithLineage(
- mtmvCache.getOriginalPlan().getOutput(),
- mtmvCache.getOriginalPlan(),
+ originalMvPlan.getOutput(),
+ originalMvPlan,
new BitSet());
// mv output expression shuttle, this will be used to expression
rewrite
this.mvExprToMvScanExprMapping =
ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions,
- this.mvScanPlan.getExpressions());
+ this.mvScanPlan.getOutput());
// copy the plan from cache, which the plan in cache may change
- this.mvPlan = mtmvCache.getLogicalPlan();
List<StructInfo> viewStructInfos =
MaterializedViewUtils.extractStructInfo(
- mtmvCache.getLogicalPlan(), cascadesContext, new BitSet());
+ mvPlan, cascadesContext, new BitSet());
if (viewStructInfos.size() > 1) {
// view struct info should only have one, log error and use the
first struct info
- LOG.warn(String.format("view strut info is more than one, mv name
is %s, mv plan is %s",
- mtmv.getName(), mvPlan.treeString()));
+ LOG.warn(String.format("view strut info is more than one,
materialization name is %s, mv plan is %s",
+ getMaterializationQualifier(), getMvPlan().treeString()));
}
this.structInfo = viewStructInfos.get(0);
}
@@ -131,22 +126,49 @@ public class MaterializationContext {
}
/**
- * Try to generate scan plan for materialized view
- * if MaterializationContext is already rewritten by materialized view,
then should generate in real time
- * when query rewrite, because one plan may hit the materialized view
repeatedly and the mv scan output
+ * Try to generate scan plan for materialization
+ * if MaterializationContext is already rewritten successfully, then
should generate new scan plan in later
+ * query rewrite, because one plan may hit the materialized view
repeatedly and the mv scan output
* should be different
*/
public void tryReGenerateMvScanPlan(CascadesContext cascadesContext) {
if (!this.matchedSuccessGroups.isEmpty()) {
- this.mvScanPlan =
MaterializedViewUtils.generateMvScanPlan(this.mtmv, cascadesContext);
+ this.mvScanPlan = doGenerateMvPlan(cascadesContext);
// mv output expression shuttle, this will be used to expression
rewrite
this.mvExprToMvScanExprMapping =
ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions,
this.mvScanPlan.getExpressions());
}
}
- public MTMV getMTMV() {
- return mtmv;
+ /**
+ * Try to generate scan plan for materialization
+ * if MaterializationContext is already rewritten successfully, then
should generate new scan plan in later
+ * query rewrite, because one plan may hit the materialized view
repeatedly and the mv scan output
+ * should be different
+ */
+ abstract Plan doGenerateMvPlan(CascadesContext cascadesContext);
+
+ /**
+ * Get materialization unique qualifier which identify it
+ */
+ abstract List<String> getMaterializationQualifier();
+
+ /**
+ * Get String info which is used for to string
+ */
+ abstract String getStringInfo();
+
+ /**
+ * Calc the relation is chosen finally or not
+ */
+ abstract boolean isFinalChosen(Relation relation);
+
+ public Plan getMvPlan() {
+ return mvPlan;
+ }
+
+ public Plan getOriginalMvPlan() {
+ return originalMvPlan;
}
public Plan getMvScanPlan() {
@@ -169,11 +191,7 @@ public class MaterializationContext {
return available;
}
- public Plan getMvPlan() {
- return mvPlan;
- }
-
- public Map<ObjectId, Pair<String, String>> getFailReason() {
+ public Multimap<ObjectId, Pair<String, String>> getFailReason() {
return failReason;
}
@@ -183,6 +201,7 @@ public class MaterializationContext {
public void setSuccess(boolean success) {
this.success = success;
+ // TODO clear the fail message by according planId ?
this.failReason.clear();
}
@@ -190,8 +209,12 @@ public class MaterializationContext {
return structInfo;
}
+ public boolean isSuccess() {
+ return success;
+ }
+
/**
- * recordFailReason
+ * Record fail reason when in rewriting
*/
public void recordFailReason(StructInfo structInfo, String summary,
Supplier<String> failureReasonSupplier) {
// record it's rewritten
@@ -207,65 +230,54 @@ public class MaterializationContext {
Pair.of(summary, this.isEnableRecordFailureDetail() ?
failureReasonSupplier.get() : ""));
}
- public boolean isSuccess() {
- return success;
- }
-
@Override
public String toString() {
- StringBuilder failReasonBuilder = new StringBuilder("[").append("\n");
- for (Map.Entry<ObjectId, Pair<String, String>> reason :
this.failReason.entrySet()) {
- failReasonBuilder
- .append("\n")
- .append("ObjectId :
").append(reason.getKey()).append(".\n")
- .append("Summary :
").append(reason.getValue().key()).append(".\n")
- .append("Reason :
").append(reason.getValue().value()).append(".\n");
- }
- failReasonBuilder.append("\n").append("]");
- return Utils.toSqlString("MaterializationContext[" + mtmv.getName() +
"]",
- "rewriteSuccess", this.success,
- "failReason", failReasonBuilder.toString());
- }
-
- /**
- * toString, this contains summary and detail info.
- */
- public static String toDetailString(List<MaterializationContext>
materializationContexts) {
- StringBuilder builder = new StringBuilder();
- builder.append("materializationContexts:").append("\n");
- for (MaterializationContext ctx : materializationContexts) {
- builder.append("\n").append(ctx).append("\n");
- }
- return builder.toString();
+ return getStringInfo();
}
/**
- * toSummaryString, this contains only summary info.
+ * ToSummaryString, this contains only summary info.
*/
public static String toSummaryString(List<MaterializationContext>
materializationContexts,
- List<MTMV> chosenMaterializationNames) {
+ PhysicalPlan physicalPlan) {
if (materializationContexts.isEmpty()) {
return "";
}
- Set<String> materializationChosenNameSet =
chosenMaterializationNames.stream()
- .map(MTMV::getName)
+ Set<MaterializationContext> rewrittenSuccessMaterializationSet =
materializationContexts.stream()
+ .filter(MaterializationContext::isSuccess)
.collect(Collectors.toSet());
+ Set<List<String>> chosenMaterializationQualifiers = new HashSet<>();
+ physicalPlan.accept(new DefaultPlanVisitor<Void, Void>() {
+ @Override
+ public Void visitPhysicalRelation(PhysicalRelation
physicalRelation, Void context) {
+ for (MaterializationContext rewrittenContext :
rewrittenSuccessMaterializationSet) {
+ if (rewrittenContext.isFinalChosen(physicalRelation)) {
+
chosenMaterializationQualifiers.add(rewrittenContext.getMaterializationQualifier());
+ }
+ }
+ return null;
+ }
+ }, null);
+
StringBuilder builder = new StringBuilder();
builder.append("\nMaterializedView");
// rewrite success and chosen
builder.append("\nMaterializedViewRewriteSuccessAndChose:\n");
- if (!materializationChosenNameSet.isEmpty()) {
- builder.append(" Names: ").append(String.join(", ",
materializationChosenNameSet));
+ if (!chosenMaterializationQualifiers.isEmpty()) {
+ builder.append(" Names: ");
+ chosenMaterializationQualifiers.forEach(materializationQualifier ->
+
builder.append(generateQualifierName(materializationQualifier)).append(", "));
}
// rewrite success but not chosen
builder.append("\nMaterializedViewRewriteSuccessButNotChose:\n");
- Set<String> rewriteSuccessButNotChoseNameSet =
materializationContexts.stream()
- .filter(materializationContext ->
materializationContext.isSuccess()
- &&
!materializationChosenNameSet.contains(materializationContext.getMTMV().getName()))
- .map(materializationContext ->
materializationContext.getMTMV().getName())
+ Set<List<String>> rewriteSuccessButNotChoseQualifiers =
rewrittenSuccessMaterializationSet.stream()
+ .map(MaterializationContext::getMaterializationQualifier)
+ .filter(materializationQualifier ->
!chosenMaterializationQualifiers.contains(materializationQualifier))
.collect(Collectors.toSet());
- if (!rewriteSuccessButNotChoseNameSet.isEmpty()) {
- builder.append(" Names: ").append(String.join(", ",
rewriteSuccessButNotChoseNameSet));
+ if (!rewriteSuccessButNotChoseQualifiers.isEmpty()) {
+ builder.append(" Names: ");
+
rewriteSuccessButNotChoseQualifiers.forEach(materializationQualifier ->
+
builder.append(generateQualifierName(materializationQualifier)).append(", "));
}
// rewrite fail
builder.append("\nMaterializedViewRewriteFail:");
@@ -274,7 +286,7 @@ public class MaterializationContext {
Set<String> failReasonSet =
ctx.getFailReason().values().stream().map(Pair::key).collect(ImmutableSet.toImmutableSet());
builder.append("\n")
- .append(" Name: ").append(ctx.getMTMV().getName())
+ .append(" Name:
").append(generateQualifierName(ctx.getMaterializationQualifier()))
.append("\n")
.append(" FailSummary: ").append(String.join(", ",
failReasonSet));
}
@@ -282,12 +294,24 @@ public class MaterializationContext {
return builder.toString();
}
- /**
- * MaterializationContext fromMaterializedView
- */
- public static MaterializationContext fromMaterializedView(MTMV
materializedView, Plan mvScanPlan,
- CascadesContext cascadesContext) {
- return new MaterializationContext(materializedView, mvScanPlan,
ImmutableList.of(), ImmutableList.of(),
- cascadesContext);
+ private static String generateQualifierName(List<String> qualifiers) {
+ return String.join("#", qualifiers);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ MaterializationContext context = (MaterializationContext) o;
+ return
getMaterializationQualifier().equals(context.getMaterializationQualifier());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getMaterializationQualifier());
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index 6863a7e01b1..608f3b5f5f0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -203,6 +203,9 @@ public class MaterializedViewUtils {
CascadesContext cascadesContext,
Function<CascadesContext, Plan> planRewriter,
Plan rewrittenPlan, Plan originPlan) {
+ if (originPlan == null || rewrittenPlan == null) {
+ return null;
+ }
if (originPlan.getOutputSet().size() !=
rewrittenPlan.getOutputSet().size()) {
return rewrittenPlan;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
index 7f6b5f0d291..1ba0c9964eb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -17,7 +17,6 @@
package org.apache.doris.nereids.rules.exploration.mv;
-import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.TableIf;
@@ -630,11 +629,11 @@ public class StructInfo {
/**
* Add predicates on base table when materialized view scan contains
invalid partitions
*/
- public static class InvalidPartitionRemover extends
DefaultPlanRewriter<Pair<MTMV, Set<Long>>> {
+ public static class InvalidPartitionRemover extends
DefaultPlanRewriter<Pair<List<String>, Set<Long>>> {
// materialized view scan is always LogicalOlapScan, so just handle
LogicalOlapScan
@Override
- public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Pair<MTMV,
Set<Long>> context) {
- if (olapScan.getTable().getName().equals(context.key().getName()))
{
+ public Plan visitLogicalOlapScan(LogicalOlapScan olapScan,
Pair<List<String>, Set<Long>> context) {
+ if (olapScan.getTable().getFullQualifiers().equals(context.key()))
{
List<Long> selectedPartitionIds =
olapScan.getSelectedPartitionIds();
return
olapScan.withSelectedPartitionIds(selectedPartitionIds.stream()
.filter(partitionId ->
!context.value().contains(partitionId))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
