This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 7095636156e branch-4.2: [fix](nereids) Merge existing filters when
adding predicates (70409) (#70620)
7095636156e is described below
commit 7095636156e2d29f3d131dbc33884178194742a9
Author: 924060929 <[email protected]>
AuthorDate: Thu Nov 19 15:29:12 2026 +0800
branch-4.2: [fix](nereids) Merge existing filters when adding predicates
(70409) (#70620)
cherry pick from #70409
---
.../org/apache/doris/nereids/util/PlanUtils.java | 24 ++++++
.../nereids/rules/rewrite/InferAggNotNullTest.java | 92 ++++++++++++++++++++++
.../apache/doris/nereids/util/PlanUtilsTest.java | 71 +++++++++++++++++
3 files changed, 187 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java
index a83d4bb6247..6d11c7fd459 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java
@@ -83,10 +83,34 @@ import java.util.stream.Collectors;
* Util for plan
*/
public class PlanUtils {
+ /** Create a filter, merging predicates into a directly adjacent logical
filter. */
public static Optional<LogicalFilter<? extends Plan>>
filter(Set<Expression> predicates, Plan plan) {
if (predicates.isEmpty()) {
return Optional.empty();
}
+ if (plan instanceof LogicalFilter) {
+ LogicalFilter<?> filter = (LogicalFilter<?>) plan;
+ Map<Expression, Expression> conjuncts =
Maps.newLinkedHashMapWithExpectedSize(
+ filter.getConjuncts().size() + predicates.size());
+ for (Expression predicate : filter.getConjuncts()) {
+ conjuncts.put(predicate, predicate);
+ }
+ boolean changed = false;
+ for (Expression predicate : predicates) {
+ Expression existing = conjuncts.get(predicate);
+ // Original SQL predicates take precedence over equal inferred
predicates.
+ if (existing == null || (ExpressionUtils.isInferred(existing)
+ && !ExpressionUtils.isInferred(predicate))) {
+ conjuncts.put(predicate, predicate);
+ changed = true;
+ }
+ }
+ if (!changed) {
+ return Optional.of(filter);
+ }
+ predicates = ImmutableSet.copyOf(conjuncts.values());
+ plan = filter.child();
+ }
return Optional.of(new LogicalFilter<>(predicates, plan));
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferAggNotNullTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferAggNotNullTest.java
index f0e709dc1b7..d01ac6ccbef 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferAggNotNullTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferAggNotNullTest.java
@@ -23,9 +23,15 @@ import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.Type;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.jobs.rewrite.TopDownVisitorRewriteJob;
+import org.apache.doris.nereids.rules.FilteredRules;
+import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.Multiply;
import org.apache.doris.nereids.trees.expressions.Not;
import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.ArrayAgg;
@@ -42,10 +48,15 @@ import
org.apache.doris.nereids.trees.expressions.functions.agg.Min;
import
org.apache.doris.nereids.trees.expressions.functions.agg.NullIgnoringAggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
import org.apache.doris.nereids.trees.expressions.functions.agg.Sum0;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.If;
+import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.util.LogicalPlanBuilder;
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
import org.apache.doris.nereids.util.MemoTestUtils;
@@ -54,6 +65,7 @@ import org.apache.doris.nereids.util.PlanConstructor;
import org.apache.doris.thrift.TStorageType;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -64,6 +76,86 @@ import java.util.Set;
class InferAggNotNullTest implements MemoPatternMatchSupported {
private final LogicalOlapScan scan1 =
PlanConstructor.newLogicalOlapScan(0, "t1", 0);
+ @Test
+ void testMergeInferredNotNullWithExistingFilter() {
+ LogicalOlapScan scan = newWideNullableScan(2);
+ Expression a = scan.getOutput().get(0);
+ Expression b = scan.getOutput().get(1);
+ Expression existingPredicate = new GreaterThan(a, new
IntegerLiteral(0));
+ for (boolean generated : ImmutableList.of(false, true)) {
+ Expression bNotNull = new Not(new IsNull(b), generated);
+ LogicalAggregate<?> aggregate = new
LogicalAggregate<>(ImmutableList.of(),
+ ImmutableList.of(new Alias(new Sum(new Multiply(a, b)),
"total")),
+ new LogicalFilter<>(ImmutableSet.of(bNotNull,
existingPredicate), scan));
+ CascadesContext context =
MemoTestUtils.createCascadesContext(aggregate);
+ Rule rule = new InferAggNotNull().build();
+
+ LogicalAggregate<?> rewritten = (LogicalAggregate<?>)
rule.transform(aggregate, context).get(0);
+ LogicalFilter<?> filter = (LogicalFilter<?>) rewritten.child();
+ Assertions.assertSame(scan, filter.child());
+ Assertions.assertEquals(ImmutableSet.of(bNotNull,
existingPredicate,
+ new Not(new IsNull(a), true)), filter.getConjuncts());
+ Assertions.assertSame(rewritten, rule.transform(rewritten,
context).get(0));
+ }
+ }
+
+ @Test
+ void testInferMultipleInputsWithoutFilter() {
+ LogicalOlapScan scan = newWideNullableScan(2);
+ Expression a = scan.getOutput().get(0);
+ Expression b = scan.getOutput().get(1);
+ LogicalAggregate<?> aggregate = new
LogicalAggregate<>(ImmutableList.of(),
+ ImmutableList.of(new Alias(new Sum(new Multiply(a, b)),
"total")), scan);
+ CascadesContext context =
MemoTestUtils.createCascadesContext(aggregate);
+ Rule rule = new InferAggNotNull().build();
+
+ LogicalAggregate<?> rewritten = (LogicalAggregate<?>)
rule.transform(aggregate, context).get(0);
+ LogicalFilter<?> filter = (LogicalFilter<?>) rewritten.child();
+ Assertions.assertSame(scan, filter.child());
+ Assertions.assertEquals(ImmutableSet.of(new Not(new IsNull(a), true),
+ new Not(new IsNull(b), true)), filter.getConjuncts());
+ Assertions.assertSame(rewritten, rule.transform(rewritten,
context).get(0));
+ }
+
+ @Test
+ void testKeepFilterWithAllInferredNotNullPredicates() {
+ LogicalOlapScan scan = newWideNullableScan(2);
+ Expression a = scan.getOutput().get(0);
+ Expression b = scan.getOutput().get(1);
+ LogicalAggregate<?> aggregate = new
LogicalAggregate<>(ImmutableList.of(),
+ ImmutableList.of(new Alias(new Sum(new Multiply(a, b)),
"total")),
+ new LogicalFilter<>(ImmutableSet.of(new Not(new IsNull(a)),
new Not(new IsNull(b))), scan));
+ CascadesContext context =
MemoTestUtils.createCascadesContext(aggregate);
+
+ Assertions.assertSame(aggregate, new
InferAggNotNull().build().transform(aggregate, context).get(0));
+ }
+
+ @Test
+ void testInferAfterEliminateAggCaseWhen() {
+ LogicalOlapScan scan = newWideNullableScan(2);
+ Expression a = scan.getOutput().get(0);
+ Expression b = scan.getOutput().get(1);
+ Expression bNotNull = new Not(new IsNull(b));
+ If conditional = new If(bNotNull, new Multiply(a, b), new
NullLiteral(IntegerType.INSTANCE));
+ LogicalAggregate<?> aggregate = new
LogicalAggregate<>(ImmutableList.of(),
+ ImmutableList.of(new Alias(new Sum(conditional), "total")),
scan);
+ CascadesContext context =
MemoTestUtils.createCascadesContext(aggregate);
+ Rule eliminateCase = new EliminateAggCaseWhen().build();
+ Rule inferNotNull = new InferAggNotNull().build();
+
+ // Check one rewrite first so a regression fails without entering an
unbounded visitor loop.
+ LogicalAggregate<?> withoutCase = (LogicalAggregate<?>)
eliminateCase.transform(aggregate, context).get(0);
+ LogicalAggregate<?> rewritten = (LogicalAggregate<?>)
inferNotNull.transform(withoutCase, context).get(0);
+ LogicalFilter<?> filter = (LogicalFilter<?>) rewritten.child();
+ Assertions.assertSame(scan, filter.child());
+ Assertions.assertEquals(ImmutableSet.of(bNotNull, new Not(new
IsNull(a), true)), filter.getConjuncts());
+ Assertions.assertSame(rewritten, inferNotNull.transform(rewritten,
context).get(0));
+
+ new TopDownVisitorRewriteJob(new
FilteredRules(ImmutableList.of(eliminateCase, inferNotNull)), plan -> true)
+ .execute(context.getCurrentJobContext());
+ Assertions.assertTrue(rewritten.deepEquals(context.getRewritePlan()));
+ }
+
@Test
void testInfer() {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanUtilsTest.java
index d5b8557e948..80628f14e05 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanUtilsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanUtilsTest.java
@@ -22,19 +22,90 @@ import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.load.NereidsLoadUtils;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.GreaterThan;
+import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.OriginStatement;
import org.apache.doris.utframe.UtFrameUtils;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
class PlanUtilsTest {
+ @Test
+ void filterMergesConjuncts() {
+ LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0);
+ Expression aNotNull = new Not(new IsNull(scan.getOutput().get(0)),
true);
+ Expression bNotNull = new Not(new IsNull(scan.getOutput().get(1)));
+ Expression generatedBNotNull = new Not(new
IsNull(scan.getOutput().get(1)), true);
+ Expression existingPredicate = new
GreaterThan(scan.getOutput().get(0), new IntegerLiteral(0));
+ ImmutableSet<Expression> existingPredicates =
ImmutableSet.of(bNotNull, existingPredicate);
+ LogicalFilter<?> original = new LogicalFilter<>(existingPredicates,
scan);
+
+ LogicalFilter<?> merged = PlanUtils.filter(ImmutableSet.of(aNotNull,
generatedBNotNull), original).get();
+
+ Assertions.assertSame(scan, merged.child());
+ Assertions.assertEquals(ImmutableSet.of(aNotNull, bNotNull,
generatedBNotNull, existingPredicate),
+ merged.getConjuncts());
+ Assertions.assertEquals(existingPredicates, original.getConjuncts());
+ }
+
+ @Test
+ void filterReusesExistingPredicates() {
+ LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0);
+ Expression aNotNull = new Not(new IsNull(scan.getOutput().get(0)),
true);
+ Expression bNotNull = new Not(new IsNull(scan.getOutput().get(1)),
true);
+ ImmutableSet<Expression> predicates = ImmutableSet.of(aNotNull,
bNotNull);
+ LogicalFilter<?> filter = new LogicalFilter<>(predicates, scan);
+
+ Assertions.assertSame(filter, PlanUtils.filter(predicates,
filter).get());
+ Assertions.assertSame(filter,
PlanUtils.filter(ImmutableSet.of(aNotNull), filter).get());
+ Assertions.assertSame(filter,
PlanUtils.filterOrSelf(ImmutableSet.of(bNotNull), filter));
+ Assertions.assertSame(filter,
PlanUtils.filterOrSelf(ImmutableSet.of(), filter));
+ Assertions.assertFalse(PlanUtils.filter(ImmutableSet.of(),
filter).isPresent());
+ }
+
+ @Test
+ void filterPreservesOriginalPredicate() {
+ LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0);
+ Expression originalPredicate = new
GreaterThan(scan.getOutput().get(0), new IntegerLiteral(0));
+ Expression inferredPredicate = originalPredicate.withInferred(true);
+ Assertions.assertEquals(originalPredicate, inferredPredicate);
+
+ LogicalFilter<?> inferredFilter = new
LogicalFilter<>(ImmutableSet.of(inferredPredicate), scan);
+ LogicalFilter<?> merged =
PlanUtils.filter(ImmutableSet.of(originalPredicate), inferredFilter).get();
+ Assertions.assertSame(scan, merged.child());
+ Assertions.assertSame(originalPredicate,
merged.getConjuncts().iterator().next());
+
Assertions.assertTrue(inferredFilter.getConjuncts().iterator().next().isInferred());
+
+ LogicalFilter<?> originalFilter = new
LogicalFilter<>(ImmutableSet.of(originalPredicate), scan);
+ Assertions.assertSame(originalFilter,
PlanUtils.filter(ImmutableSet.of(inferredPredicate), originalFilter).get());
+ }
+
+ @Test
+ void filterDoesNotMergeThroughProject() {
+ LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0);
+ Expression aNotNull = new Not(new IsNull(scan.getOutput().get(0)),
true);
+ Expression bNotNull = new Not(new IsNull(scan.getOutput().get(1)),
true);
+ LogicalFilter<?> original = new
LogicalFilter<>(ImmutableSet.of(bNotNull), scan);
+ LogicalProject<?> project = new
LogicalProject<>(ImmutableList.copyOf(scan.getOutput()), original);
+
+ LogicalFilter<?> filter = PlanUtils.filter(ImmutableSet.of(aNotNull),
project).get();
+
+ Assertions.assertSame(project, filter.child());
+ Assertions.assertEquals(ImmutableSet.of(aNotNull),
filter.getConjuncts());
+ Assertions.assertSame(original, project.child());
+ }
+
@Test
void filterOrSelf() {
LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]