This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch vector-index-dev
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/vector-index-dev by this push:
new 9421e5afdce [opt](vector) only extract l2_distance from filter
9421e5afdce is described below
commit 9421e5afdce9314a42357584f322c33cacaf1b4d
Author: morrySnow <[email protected]>
AuthorDate: Tue May 20 19:46:09 2025 +0800
[opt](vector) only extract l2_distance from filter
---
.../org/apache/doris/analysis/SlotDescriptor.java | 2 +-
.../post/CommonSubExpressionCollector.java | 3 +-
.../nereids/processor/post/PlanPostProcessors.java | 8 +-
.../PushDownVirtualColumnsIntoOlapScan.java | 118 +++++++++++----------
4 files changed, 67 insertions(+), 64 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
index 5af4b378274..5d0ec929ad6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
@@ -345,7 +345,7 @@ public class SlotDescriptor {
.add("type", typeStr).add("materialized",
isMaterialized).add("byteSize", byteSize)
.add("byteOffset", byteOffset).add("slotIdx",
slotIdx).add("nullable", getIsNullable())
.add("isAutoIncrement", isAutoInc).add("subColPath",
subColPath)
- .add("virtualColumn", virtualColumn.toSql()).toString();
+ .add("virtualColumn", virtualColumn == null ? null :
virtualColumn.toSql()).toString();
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/CommonSubExpressionCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/CommonSubExpressionCollector.java
index 29c70b7ed3f..520902c0439 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/CommonSubExpressionCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/CommonSubExpressionCollector.java
@@ -61,7 +61,8 @@ public class CommonSubExpressionCollector extends
ExpressionVisitor<Integer, Boo
// ArrayItemSlot and ArrayItemReference could not be common expressions
// TODO: could not extract common expression when expression contains
same lambda expression
// because ArrayItemSlot in Lambda are not same.
- if (!(inLambda && expr.containsType(ArrayItemSlot.class,
ArrayItemReference.class))) {
+ if (expressions.contains(expr)
+ && !(inLambda && expr.containsType(ArrayItemSlot.class,
ArrayItemReference.class))) {
Set<Expression> commonExpression =
getExpressionsFromDepthMap(depth, commonExprByDepth);
commonExpression.add(expr);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
index 86801dc1b26..a8654e27291 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
@@ -62,10 +62,10 @@ public class PlanPostProcessors {
builder.add(new RemoveUselessProjectPostProcessor());
builder.add(new MergeProjectPostProcessor());
builder.add(new RecomputeLogicalPropertiesProcessor());
- // if
(cascadesContext.getConnectContext().getSessionVariable().enableAggregateCse) {
- // builder.add(new ProjectAggregateExpressionsForCse());
- // }
- // builder.add(new CommonSubExpressionOpt());
+ if
(cascadesContext.getConnectContext().getSessionVariable().enableAggregateCse) {
+ builder.add(new ProjectAggregateExpressionsForCse());
+ }
+ builder.add(new CommonSubExpressionOpt());
// DO NOT replace PLAN NODE from here
if
(cascadesContext.getConnectContext().getSessionVariable().pushTopnToAgg) {
builder.add(new PushTopnToAgg());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java
index 56ee140169c..536f3e79f35 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java
@@ -17,84 +17,86 @@
package org.apache.doris.nereids.rules.rewrite;
-import org.apache.doris.nereids.processor.post.CommonSubExpressionCollector;
-import
org.apache.doris.nereids.processor.post.CommonSubExpressionOpt.ExpressionReplacer;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.Slot;
-import org.apache.doris.nereids.trees.expressions.WhenClause;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.L2Distance;
+import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.util.ExpressionUtils;
-import com.google.common.collect.Lists;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
/**
* extract virtual column from filter and push down them into olap scan.
*/
-public class PushDownVirtualColumnsIntoOlapScan extends OneRewriteRuleFactory {
+public class PushDownVirtualColumnsIntoOlapScan implements RewriteRuleFactory {
@Override
- public Rule build() {
- return logicalProject(logicalFilter(logicalOlapScan()
- .when(s -> s.getVirtualColumns().isEmpty())))
- .then(project -> {
- // 1. extract filter common expr
- // 2. generate virtual column from common expr and add
them to scan
- // 3. replace filter
- // 4. replace project
- LogicalFilter<LogicalOlapScan> filter = project.child();
- LogicalOlapScan logicalOlapScan = filter.child();
- CommonSubExpressionCollector collector = new
CommonSubExpressionCollector();
- for (Expression expr : filter.getConjuncts()) {
- collector.collect(expr);
- }
- Map<Expression, Alias> aliasMap = new LinkedHashMap<>();
- if (!collector.commonExprByDepth.isEmpty()) {
- for (int i = 1; i <=
collector.commonExprByDepth.size(); i++) {
- Set<Expression> exprsInDepth =
CommonSubExpressionCollector
- .getExpressionsFromDepthMap(i,
collector.commonExprByDepth);
- exprsInDepth.forEach(expr -> {
- if (!(expr instanceof WhenClause)) {
- // case whenClause1 whenClause2 END
- // whenClause should not be regarded as
common-sub-expression, because
- // cse will be replaced by a slot, after
rewrite the case clause becomes:
- // 'case slot whenClause2 END'
- // This is illegal.
- Expression rewritten =
expr.accept(ExpressionReplacer.INSTANCE, aliasMap);
- // if rewritten is already alias, use it
directly,
- // because in materialized view rewriting
- // Should keep out slot immutably after
rewritten successfully
- aliasMap.put(expr, rewritten instanceof
Alias
- ? (Alias) rewritten : new
Alias(rewritten));
- }
- });
- }
- }
- List<NamedExpression> virtualColumns =
Lists.newArrayList();
- Map<Expression, Slot> replaceMap = Maps.newHashMap();
- for (Map.Entry<Expression, Alias> entry :
aliasMap.entrySet()) {
- Alias alias = entry.getValue();
- replaceMap.put(entry.getKey(), alias.toSlot());
- virtualColumns.add(alias);
- }
- logicalOlapScan =
logicalOlapScan.withVirtualColumns(virtualColumns);
- Set<Expression> conjuncts =
ExpressionUtils.replace(filter.getConjuncts(), replaceMap);
- List<NamedExpression> projections =
ExpressionUtils.replace(
- (List) project.getProjects(), replaceMap);
- LogicalFilter<?> newFilter =
filter.withConjunctsAndChild(conjuncts, logicalOlapScan);
- LogicalProject<?> newProject =
project.withProjectsAndChild(projections, newFilter);
- return newProject;
- }).toRule(RuleType.PUSH_DOWN_VIRTUAL_COLUMNS_INTO_OLAP_SCAN);
+ public List<Rule> buildRules() {
+ return ImmutableList.of(
+ logicalProject(logicalFilter(logicalOlapScan()
+ .when(s -> s.getVirtualColumns().isEmpty())))
+ .then(project -> {
+ LogicalFilter<LogicalOlapScan> filter =
project.child();
+ LogicalOlapScan scan = filter.child();
+ return pushDown(filter, scan,
Optional.of(project));
+
}).toRule(RuleType.PUSH_DOWN_VIRTUAL_COLUMNS_INTO_OLAP_SCAN),
+ logicalFilter(logicalOlapScan()
+ .when(s -> s.getVirtualColumns().isEmpty()))
+ .then(filter -> {
+ LogicalOlapScan scan = filter.child();
+ return pushDown(filter, scan, Optional.empty());
+
}).toRule(RuleType.PUSH_DOWN_VIRTUAL_COLUMNS_INTO_OLAP_SCAN)
+
+ );
+ }
+
+ private Plan pushDown(LogicalFilter<LogicalOlapScan> filter,
LogicalOlapScan logicalOlapScan,
+ Optional<LogicalProject<?>> optionalProject) {
+ // 1. extract filter l2_distance
+ // 2. generate virtual column from l2_distance and add them to scan
+ // 3. replace filter
+ // 4. replace project
+ Map<Expression, Expression> replaceMap = Maps.newHashMap();
+ for (Expression conjunct : filter.getConjuncts()) {
+ Set<Expression> l2Distances =
conjunct.collect(L2Distance.class::isInstance);
+ for (Expression l2Distance : l2Distances) {
+ if (replaceMap.containsKey(l2Distance)) {
+ continue;
+ }
+ Alias alias = new Alias(l2Distance);
+ replaceMap.put(l2Distance, alias.toSlot());
+ }
+ }
+ if (replaceMap.isEmpty()) {
+ return null;
+ }
+ ImmutableList.Builder<NamedExpression> virtualColumnsBuilder =
ImmutableList.builder();
+ for (Expression expression : replaceMap.values()) {
+ virtualColumnsBuilder.add((NamedExpression) expression);
+ }
+ logicalOlapScan =
logicalOlapScan.withVirtualColumns(virtualColumnsBuilder.build());
+ Set<Expression> conjuncts =
ExpressionUtils.replace(filter.getConjuncts(), replaceMap);
+ Plan plan = filter.withConjunctsAndChild(conjuncts, logicalOlapScan);
+ if (optionalProject.isPresent()) {
+ LogicalProject<?> project = optionalProject.get();
+ List<NamedExpression> projections = ExpressionUtils.replace(
+ (List) project.getProjects(), replaceMap);
+ plan = project.withProjectsAndChild(projections, plan);
+ } else {
+ plan = new LogicalProject<>((List) filter.getOutput(), plan);
+ }
+ return plan;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
