This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit bfd875eae396c10d7c615e9f0071602a189ee809
Author: 谢健 <jianx...@gmail.com>
AuthorDate: Fri May 17 15:06:49 2024 +0800
[opt](nereids) lazy get expression map when comparing hypergraph (#34753)
---
.../rules/exploration/mv/HyperGraphComparator.java | 114 +++++++++++++--------
.../mv/LogicalCompatibilityContext.java | 31 ++----
2 files changed, 80 insertions(+), 65 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
index 0c9dc9c33de..d180254c7fe 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
@@ -46,6 +46,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
@@ -120,30 +121,37 @@ public class HyperGraphComparator {
}
// 3 try to construct a map which can be mapped from edge to edge
- Map<Edge, Edge> queryToView = constructQueryToViewMapWithExpr();
- if (!makeViewJoinCompatible(queryToView)) {
+ Map<Edge, Edge> queryToViewJoinEdge =
constructQueryToViewJoinMapWithExpr();
+ if (!makeViewJoinCompatible(queryToViewJoinEdge)) {
return ComparisonResult.newInvalidResWithErrorMessage("Join types
are not compatible\n");
}
refreshViewEdges();
// 4 compare them by expression and nodes. Note compare edges after
inferring for nodes
- boolean matchNodes = queryToView.entrySet().stream()
+ boolean matchNodes = queryToViewJoinEdge.entrySet().stream()
+ .allMatch(e -> compareEdgeWithNode(e.getKey(), e.getValue()));
+ if (!matchNodes) {
+ return ComparisonResult.newInvalidResWithErrorMessage("Join nodes
are not compatible\n");
+ }
+ Map<Edge, Edge> queryToViewFilterEdge =
constructQueryToViewFilterMapWithExpr();
+ matchNodes = queryToViewFilterEdge.entrySet().stream()
.allMatch(e -> compareEdgeWithNode(e.getKey(), e.getValue()));
if (!matchNodes) {
return ComparisonResult.newInvalidResWithErrorMessage("Join nodes
are not compatible\n");
}
- queryToView.forEach(this::compareEdgeWithExpr);
+ queryToViewJoinEdge.forEach(this::compareJoinEdgeWithExpr);
+ queryToViewFilterEdge.forEach(this::compareFilterEdgeWithExpr);
// 5 process residual edges
- Sets.difference(getQueryJoinEdgeSet(), queryToView.keySet())
+ Sets.difference(getQueryJoinEdgeSet(), queryToViewJoinEdge.keySet())
.forEach(e -> pullUpQueryExprWithEdge.put(e,
e.getExpressions()));
- Sets.difference(getQueryFilterEdgeSet(), queryToView.keySet())
+ Sets.difference(getQueryFilterEdgeSet(),
queryToViewFilterEdge.keySet())
.forEach(e -> pullUpQueryExprWithEdge.put(e,
e.getExpressions()));
- Sets.difference(getViewJoinEdgeSet(),
Sets.newHashSet(queryToView.values()))
+ Sets.difference(getViewJoinEdgeSet(),
Sets.newHashSet(queryToViewJoinEdge.values()))
.stream()
.filter(e -> !LongBitmap.isOverlap(e.getReferenceNodes(),
eliminateViewNodesMap))
.forEach(e -> pullUpViewExprWithEdge.put(e,
e.getExpressions()));
- Sets.difference(getViewFilterEdgeSet(),
Sets.newHashSet(queryToView.values()))
+ Sets.difference(getViewFilterEdgeSet(),
Sets.newHashSet(queryToViewFilterEdge.values()))
.stream()
.filter(e -> !LongBitmap.isOverlap(e.getReferenceNodes(),
eliminateViewNodesMap))
.forEach(e -> pullUpViewExprWithEdge.put(e,
e.getExpressions()));
@@ -238,7 +246,7 @@ public class HyperGraphComparator {
int size = queryExprSetList.size();
for (int i = 0; i < size; i++) {
Set<Expression> mappingQueryExprSet =
queryExprSetList.get(i).stream()
- .map(e ->
logicalCompatibilityContext.getQueryToViewAllExpressionMapping().get(e))
+ .map(logicalCompatibilityContext::getViewNodeExprFromQuery)
.collect(Collectors.toSet());
if (!mappingQueryExprSet.equals(viewExprSetList.get(i))) {
return false;
@@ -350,12 +358,6 @@ public class HyperGraphComparator {
return ImmutableSet.copyOf(queryHyperGraph.getFilterEdges());
}
- private List<Edge> getQueryEdges() {
- return ImmutableList.<Edge>builder()
- .addAll(getQueryJoinEdges())
- .addAll(getQueryFilterEdges()).build();
- }
-
private boolean makeViewJoinCompatible(Map<Edge, Edge> queryToView) {
for (Entry<Edge, Edge> entry : queryToView.entrySet()) {
if (entry.getKey() instanceof JoinEdge && entry.getValue()
instanceof JoinEdge) {
@@ -384,37 +386,50 @@ public class HyperGraphComparator {
return viewHyperGraph.getFilterEdges();
}
- private List<Edge> getViewEdges() {
- return ImmutableList.<Edge>builder()
- .addAll(getViewJoinEdges())
- .addAll(getViewFilterEdges()).build();
- }
-
- private Map<Expression, Expression> getQueryToViewExprMap() {
- return
logicalCompatibilityContext.getQueryToViewAllExpressionMapping();
- }
-
private Map<Integer, Integer> getQueryToViewNodeIdMap() {
return logicalCompatibilityContext.getQueryToViewNodeIDMapping();
}
- private Map<Edge, Edge> constructQueryToViewMapWithExpr() {
- Map<Expression, Edge> viewExprToEdge = getViewEdges().stream()
+ private Map<Edge, Edge> constructQueryToViewJoinMapWithExpr() {
+ Map<Expression, Edge> viewExprToEdge = getViewJoinEdges().stream()
.flatMap(e -> e.getExpressions().stream().map(expr ->
Pair.of(expr, e)))
.collect(ImmutableMap.toImmutableMap(p -> p.first, p ->
p.second));
- Map<Expression, Edge> queryExprToEdge = getQueryEdges().stream()
+ Map<Expression, Edge> queryExprToEdge = getQueryJoinEdges().stream()
.flatMap(e -> e.getExpressions().stream().map(expr ->
Pair.of(expr, e)))
.collect(ImmutableMap.toImmutableMap(p -> p.first, p ->
p.second));
- return queryExprToEdge.entrySet().stream()
- .filter(entry ->
viewExprToEdge.containsKey(getViewExprFromQueryExpr(entry.getKey())))
- .map(entry -> Pair.of(entry.getValue(),
-
viewExprToEdge.get(getViewExprFromQueryExpr(entry.getKey()))))
- .distinct()
- .collect(ImmutableMap.toImmutableMap(p -> p.first, p ->
p.second));
+
+ HashMap<Edge, Edge> edgeMap = new HashMap<>();
+ for (Entry<Expression, Edge> entry : queryExprToEdge.entrySet()) {
+ if (edgeMap.containsKey(entry.getValue())) {
+ continue;
+ }
+ Expression viewExpr =
logicalCompatibilityContext.getViewJoinExprFromQuery(entry.getKey());
+ if (viewExprToEdge.containsKey(viewExpr)) {
+ edgeMap.put(entry.getValue(),
Objects.requireNonNull(viewExprToEdge.get(viewExpr)));
+ }
+ }
+ return edgeMap;
}
- private Expression getViewExprFromQueryExpr(Expression query) {
- return
logicalCompatibilityContext.getQueryToViewAllExpressionMapping().get(query);
+ private Map<Edge, Edge> constructQueryToViewFilterMapWithExpr() {
+ Map<Expression, Edge> viewExprToEdge = getViewFilterEdges().stream()
+ .flatMap(e -> e.getExpressions().stream().map(expr ->
Pair.of(expr, e)))
+ .collect(ImmutableMap.toImmutableMap(p -> p.first, p ->
p.second));
+ Map<Expression, Edge> queryExprToEdge = getQueryFilterEdges().stream()
+ .flatMap(e -> e.getExpressions().stream().map(expr ->
Pair.of(expr, e)))
+ .collect(ImmutableMap.toImmutableMap(p -> p.first, p ->
p.second));
+
+ HashMap<Edge, Edge> edgeMap = new HashMap<>();
+ for (Entry<Expression, Edge> entry : queryExprToEdge.entrySet()) {
+ if (edgeMap.containsKey(entry.getValue())) {
+ continue;
+ }
+ Expression viewExpr =
logicalCompatibilityContext.getViewFilterExprFromQuery(entry.getKey());
+ if (viewExprToEdge.containsKey(viewExpr)) {
+ edgeMap.put(entry.getValue(),
Objects.requireNonNull(viewExprToEdge.get(viewExpr)));
+ }
+ }
+ return edgeMap;
}
private void refreshViewEdges() {
@@ -510,16 +525,35 @@ public class HyperGraphComparator {
return newBitmap;
}
- private void compareEdgeWithExpr(Edge query, Edge view) {
+ private void compareJoinEdgeWithExpr(Edge query, Edge view) {
+ Set<? extends Expression> queryExprSet = query.getExpressionSet();
+ Set<? extends Expression> viewExprSet = view.getExpressionSet();
+
+ Set<Expression> exprMappedOfView = new HashSet<>();
+ List<Expression> residualQueryExpr = new ArrayList<>();
+ for (Expression queryExpr : queryExprSet) {
+ Expression viewExpr =
logicalCompatibilityContext.getViewJoinExprFromQuery(queryExpr);
+ if (viewExprSet.contains(viewExpr)) {
+ exprMappedOfView.add(viewExpr);
+ } else {
+ residualQueryExpr.add(queryExpr);
+ }
+ }
+ List<Expression> residualViewExpr =
ImmutableList.copyOf(Sets.difference(viewExprSet, exprMappedOfView));
+ pullUpQueryExprWithEdge.put(query, residualQueryExpr);
+ pullUpViewExprWithEdge.put(query, residualViewExpr);
+ }
+
+ private void compareFilterEdgeWithExpr(Edge query, Edge view) {
Set<? extends Expression> queryExprSet = query.getExpressionSet();
Set<? extends Expression> viewExprSet = view.getExpressionSet();
Set<Expression> exprMappedOfView = new HashSet<>();
List<Expression> residualQueryExpr = new ArrayList<>();
for (Expression queryExpr : queryExprSet) {
- if (getQueryToViewExprMap().containsKey(queryExpr) &&
viewExprSet.contains(
- getQueryToViewExprMap().get(queryExpr))) {
- exprMappedOfView.add(getQueryToViewExprMap().get(queryExpr));
+ Expression viewExpr =
logicalCompatibilityContext.getViewFilterExprFromQuery(queryExpr);
+ if (viewExprSet.contains(viewExpr)) {
+ exprMappedOfView.add(viewExpr);
} else {
residualQueryExpr.add(queryExpr);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
index 25bafeb64c1..ca13c9701da 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
@@ -92,35 +92,16 @@ public class LogicalCompatibilityContext {
return queryToViewNodeIDMapping;
}
- /**
- * Get all expression mapping in query to view
- */
- @Deprecated
- public BiMap<Expression, Expression> getQueryToViewAllExpressionMapping() {
- if (queryToViewAllExpressionMapping != null) {
- return queryToViewAllExpressionMapping;
- }
- queryToViewAllExpressionMapping = HashBiMap.create();
-
queryToViewAllExpressionMapping.putAll(getQueryToViewJoinEdgeExpressionMapping());
-
queryToViewAllExpressionMapping.putAll(getQueryToViewNodeExpressionMapping());
-
queryToViewAllExpressionMapping.putAll(getQueryToViewFilterEdgeExpressionMapping());
- return queryToViewAllExpressionMapping;
- }
-
- public BiMap<Expression, Expression>
getQueryToViewJoinEdgeExpressionMapping() {
- return queryToViewJoinEdgeExpressionMappingSupplier.get();
- }
-
- public BiMap<Expression, Expression> getQueryToViewNodeExpressionMapping()
{
- return queryToViewNodeExpressionMappingSupplier.get();
+ public Expression getViewJoinExprFromQuery(Expression queryJoinExpr) {
+ return
queryToViewJoinEdgeExpressionMappingSupplier.get().get(queryJoinExpr);
}
- public BiMap<Expression, Expression>
getQueryToViewFilterEdgeExpressionMapping() {
- return queryToViewFilterEdgeExpressionMappingSupplier.get();
+ public Expression getViewFilterExprFromQuery(Expression queryJoinExpr) {
+ return
queryToViewFilterEdgeExpressionMappingSupplier.get().get(queryJoinExpr);
}
- public ObjectId getPlanNodeId() {
- return planNodeId;
+ public Expression getViewNodeExprFromQuery(Expression queryJoinExpr) {
+ return
queryToViewNodeExpressionMappingSupplier.get().get(queryJoinExpr);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
