This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 71caf88ec1563e01329fa0202f2a7cf854d94f6d
Author: seawinde <149132972+seawi...@users.noreply.github.com>
AuthorDate: Wed May 15 13:57:28 2024 +0800
[opt](mtmv) Optimize the logic of slot mapping generate for performance
(#34597)
Slot mapping is used for materialized view rewritting
given the relation mapping, the slot mapping is the same
Optimize the slot mapping genarate logic
Cache the slot mapping in materialization context by realation mapping key
---
.../mv/AbstractMaterializedViewRule.java | 9 +++-
.../mv/LogicalCompatibilityContext.java | 7 +--
.../exploration/mv/MaterializationContext.java | 13 ++++++
.../exploration/mv/mapping/ExpressionMapping.java | 23 ----------
.../rules/exploration/mv/mapping/Mapping.java | 24 +++++++----
.../exploration/mv/mapping/RelationMapping.java | 18 ++++++++
.../rules/exploration/mv/mapping/SlotMapping.java | 50 +++++++++++++---------
7 files changed, 86 insertions(+), 58 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index 41157a6c863..a88362e6697 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -179,7 +179,12 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
return rewriteResults;
}
for (RelationMapping queryToViewTableMapping :
queryToViewTableMappings) {
- SlotMapping queryToViewSlotMapping =
SlotMapping.generate(queryToViewTableMapping);
+ SlotMapping queryToViewSlotMapping =
+
materializationContext.getSlotMappingFromCache(queryToViewTableMapping);
+ if (queryToViewSlotMapping == null) {
+ queryToViewSlotMapping =
SlotMapping.generate(queryToViewTableMapping);
+
materializationContext.addSlotMappingToCache(queryToViewTableMapping,
queryToViewSlotMapping);
+ }
if (queryToViewSlotMapping == null) {
materializationContext.recordFailReason(queryStructInfo,
"Query to view slot mapping is null", () -> "");
@@ -187,7 +192,7 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
}
SlotMapping viewToQuerySlotMapping =
queryToViewSlotMapping.inverse();
LogicalCompatibilityContext compatibilityContext =
LogicalCompatibilityContext.from(
- queryToViewTableMapping, queryToViewSlotMapping,
queryStructInfo, viewStructInfo);
+ queryToViewTableMapping, viewToQuerySlotMapping,
queryStructInfo, viewStructInfo);
ComparisonResult comparisonResult =
StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo,
compatibilityContext);
if (comparisonResult.isInvalid()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
index b4ed509f300..25bafeb64c1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
@@ -128,7 +128,7 @@ public class LogicalCompatibilityContext {
* this make expression mapping between query and view by relation and the
slot in relation mapping
*/
public static LogicalCompatibilityContext from(RelationMapping
relationMapping,
- SlotMapping queryToViewSlotMapping,
+ SlotMapping viewToQuerySlotMapping,
StructInfo queryStructInfo,
StructInfo viewStructInfo) {
// init node mapping
@@ -147,11 +147,8 @@ public class LogicalCompatibilityContext {
queryToViewNodeMapping.put(queryStructInfoNode,
viewStructInfoNode);
}
}
- // init expression mapping
- Map<SlotReference, SlotReference> viewToQuerySlotMapping =
queryToViewSlotMapping.inverse()
- .toSlotReferenceMap();
return new LogicalCompatibilityContext(queryToViewNodeMapping,
- viewToQuerySlotMapping,
+ viewToQuerySlotMapping.toSlotReferenceMap(),
queryStructInfo,
viewStructInfo);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
index 261e3bb85f9..2f0d04e1141 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
@@ -23,6 +23,8 @@ import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.GroupId;
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.ObjectId;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -40,8 +42,10 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.BitSet;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Supplier;
@@ -52,6 +56,7 @@ import java.util.stream.Collectors;
*/
public abstract class MaterializationContext {
private static final Logger LOG =
LogManager.getLogger(MaterializationContext.class);
+ public final Map<RelationMapping, SlotMapping> queryToMvSlotMappingCache =
new HashMap<>();
protected List<Table> baseTables;
protected List<Table> baseViews;
// The plan of mv def sql
@@ -140,6 +145,14 @@ public abstract class MaterializationContext {
}
}
+ public void addSlotMappingToCache(RelationMapping relationMapping,
SlotMapping slotMapping) {
+ queryToMvSlotMappingCache.put(relationMapping, slotMapping);
+ }
+
+ public SlotMapping getSlotMappingFromCache(RelationMapping
relationMapping) {
+ return queryToMvSlotMappingCache.get(relationMapping);
+ }
+
/**
* Try to generate scan plan for materialization
* if MaterializationContext is already rewritten successfully, then
should generate new scan plan in later
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java
index 5a5bfedfe17..8c77eacfaf0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java
@@ -23,7 +23,6 @@ import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
@@ -32,7 +31,6 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
/**
* Expression mapping, maybe one expression map to multi expression
@@ -104,27 +102,6 @@ public class ExpressionMapping extends Mapping {
return new ExpressionMapping(expressionMultiMap);
}
- @Override
- public Mapping chainedFold(Mapping target) {
-
- ImmutableMultimap.Builder<Expression, Expression> foldedMappingBuilder
=
- ImmutableMultimap.builder();
-
- Multimap<Expression, Expression> targetMapping
- = ((ExpressionMapping) target).getExpressionMapping();
- for (Entry<Expression, ? extends Collection<Expression>> exprMapping :
- this.getExpressionMapping().asMap().entrySet()) {
- Collection<? extends Expression> valueExpressions =
exprMapping.getValue();
- valueExpressions.forEach(valueExpr -> {
- if (targetMapping.containsKey(valueExpr)) {
- targetMapping.get(valueExpr).forEach(
- targetValue ->
foldedMappingBuilder.put(exprMapping.getKey(), targetValue));
- }
- });
- }
- return new ExpressionMapping(foldedMappingBuilder.build());
- }
-
@Override
public String toString() {
return Utils.toSqlString("ExpressionMapping", "expressionMapping",
expressionMapping);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java
index 18fa282267c..e14b79a52ea 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java
@@ -22,6 +22,8 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Objects;
import javax.annotation.Nullable;
@@ -38,10 +40,15 @@ public abstract class Mapping {
public final RelationId relationId;
public final CatalogRelation belongedRelation;
+ // Generate eagerly, will be used to generate slot mapping
+ private final Map<String, Slot> slotNameToSlotMap = new HashMap<>();
public MappedRelation(RelationId relationId, CatalogRelation
belongedRelation) {
this.relationId = relationId;
this.belongedRelation = belongedRelation;
+ for (Slot slot : belongedRelation.getOutput()) {
+ slotNameToSlotMap.put(slot.getName(), slot);
+ }
}
public static MappedRelation of(RelationId relationId, CatalogRelation
belongedRelation) {
@@ -56,6 +63,10 @@ public abstract class Mapping {
return belongedRelation;
}
+ public Map<String, Slot> getSlotNameToSlotMap() {
+ return slotNameToSlotMap;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -72,6 +83,11 @@ public abstract class Mapping {
public int hashCode() {
return Objects.hash(relationId);
}
+
+ @Override
+ public String toString() {
+ return "MappedRelation{" + "relationId=" + relationId + ",
slotNameToSlotMap=" + slotNameToSlotMap + '}';
+ }
}
/**
@@ -141,12 +157,4 @@ public abstract class Mapping {
return "MappedSlot{" + "slot=" + slot + '}';
}
}
-
- /** Chain fold tow mapping, such as this mapping is {[a -> b]}, the target
mapping is
- * {[b -> c]} after chain fold, this result will be {[a -> c]}, if the
value side in this mapping
- * can get the key in the target mapping, will lose the mapping
- */
- protected Mapping chainedFold(Mapping target) {
- return null;
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
index b5494c01b1b..eb53923da53 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
@@ -32,6 +32,7 @@ import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
/**
@@ -143,4 +144,21 @@ public class RelationMapping extends Mapping {
private static Long getTableQualifier(TableIf tableIf) {
return tableIf.getId();
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ RelationMapping that = (RelationMapping) o;
+ return Objects.equals(mappedRelationMap, that.mappedRelationMap);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(mappedRelationMap);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java
index f95bcedd2fa..8384b3e094f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java
@@ -22,10 +22,11 @@ import
org.apache.doris.nereids.trees.expressions.SlotReference;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.util.HashMap;
import java.util.Map;
-import java.util.stream.Collectors;
import javax.annotation.Nullable;
/**
@@ -33,11 +34,15 @@ import javax.annotation.Nullable;
*/
public class SlotMapping extends Mapping {
+ public static final Logger LOG = LogManager.getLogger(SlotMapping.class);
+
private final BiMap<MappedSlot, MappedSlot> relationSlotMap;
private Map<SlotReference, SlotReference> slotReferenceMap;
- public SlotMapping(BiMap<MappedSlot, MappedSlot> relationSlotMap) {
+ public SlotMapping(BiMap<MappedSlot, MappedSlot> relationSlotMap,
+ Map<SlotReference, SlotReference> slotReferenceMap) {
this.relationSlotMap = relationSlotMap;
+ this.slotReferenceMap = slotReferenceMap;
}
public BiMap<MappedSlot, MappedSlot> getRelationSlotMap() {
@@ -45,11 +50,12 @@ public class SlotMapping extends Mapping {
}
public SlotMapping inverse() {
- return SlotMapping.of(relationSlotMap.inverse());
+ return SlotMapping.of(relationSlotMap.inverse(), null);
}
- public static SlotMapping of(BiMap<MappedSlot, MappedSlot>
relationSlotMap) {
- return new SlotMapping(relationSlotMap);
+ public static SlotMapping of(BiMap<MappedSlot, MappedSlot> relationSlotMap,
+ Map<SlotReference, SlotReference> slotReferenceMap) {
+ return new SlotMapping(relationSlotMap, slotReferenceMap);
}
/**
@@ -58,26 +64,31 @@ public class SlotMapping extends Mapping {
@Nullable
public static SlotMapping generate(RelationMapping relationMapping) {
BiMap<MappedSlot, MappedSlot> relationSlotMap = HashBiMap.create();
+ Map<SlotReference, SlotReference> slotReferenceMap = new HashMap<>();
BiMap<MappedRelation, MappedRelation> mappedRelationMap =
relationMapping.getMappedRelationMap();
for (Map.Entry<MappedRelation, MappedRelation> mappedRelationEntry :
mappedRelationMap.entrySet()) {
- Map<String, Slot> targetNameSlotMap =
-
mappedRelationEntry.getValue().getBelongedRelation().getOutput().stream()
- .collect(Collectors.toMap(Slot::getName, slot ->
slot));
- for (Slot sourceSlot :
mappedRelationEntry.getKey().getBelongedRelation().getOutput()) {
- Slot targetSlot = targetNameSlotMap.get(sourceSlot.getName());
+ MappedRelation sourceRelation = mappedRelationEntry.getKey();
+ Map<String, Slot> sourceSlotNameToSlotMap =
sourceRelation.getSlotNameToSlotMap();
+
+ MappedRelation targetRelation = mappedRelationEntry.getValue();
+ Map<String, Slot> targetSlotNameSlotMap =
targetRelation.getSlotNameToSlotMap();
+
+ for (String sourceSlotName : sourceSlotNameToSlotMap.keySet()) {
+ Slot targetSlot = targetSlotNameSlotMap.get(sourceSlotName);
// source slot can not map from target, bail out
if (targetSlot == null) {
+ LOG.warn(String.format("SlotMapping generate is null,
source relation is %s, "
+ + "target relation is %s", sourceRelation,
targetRelation));
return null;
}
- relationSlotMap.put(MappedSlot.of(sourceSlot,
mappedRelationEntry.getKey().getBelongedRelation()),
- MappedSlot.of(targetSlot,
mappedRelationEntry.getValue().getBelongedRelation()));
+ Slot sourceSlot = sourceSlotNameToSlotMap.get(sourceSlotName);
+ relationSlotMap.put(MappedSlot.of(sourceSlot,
+ sourceRelation.getBelongedRelation()),
+ MappedSlot.of(targetSlot,
targetRelation.getBelongedRelation()));
+ slotReferenceMap.put((SlotReference) sourceSlot,
(SlotReference) targetSlot);
}
}
- return SlotMapping.of(relationSlotMap);
- }
-
- public Map<MappedSlot, MappedSlot> toMappedSlotMap() {
- return (Map) this.getRelationSlotMap();
+ return SlotMapping.of(relationSlotMap, slotReferenceMap);
}
/**
@@ -87,12 +98,11 @@ public class SlotMapping extends Mapping {
if (this.slotReferenceMap != null) {
return this.slotReferenceMap;
}
- Map<SlotReference, SlotReference> slotReferenceSlotReferenceMap = new
HashMap<>();
+ this.slotReferenceMap = new HashMap<>();
for (Map.Entry<MappedSlot, MappedSlot> entry :
this.getRelationSlotMap().entrySet()) {
- slotReferenceSlotReferenceMap.put((SlotReference)
entry.getKey().getSlot(),
+ this.slotReferenceMap.put((SlotReference) entry.getKey().getSlot(),
(SlotReference) entry.getValue().getSlot());
}
- this.slotReferenceMap = slotReferenceSlotReferenceMap;
return this.slotReferenceMap;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
