This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3d4c1d391b3 [fix](join) Bound outer join null-reject inference to
nullable outputs (#65250)
3d4c1d391b3 is described below
commit 3d4c1d391b37d60900c405f0713c565e32c3e6e8
Author: foxtail463 <[email protected]>
AuthorDate: Sat Aug 1 11:56:58 2026 +0800
[fix](join) Bound outer join null-reject inference to nullable outputs
(#65250)
related PR: #63318
Problem Summary:
Outer join elimination uses fold-based null-reject inference to decide
whether
nullable-side rows from the current outer join can be filtered away. This
check
only needs slots from that join's nullable-side outputs, but the previous
flow
could also test unrelated predicate inputs. When a filter predicate
references a
mark slot produced by another join, folding that slot is useless for
eliminating
the current outer join and can be costly when the fixed-point rewrite batch
revisits the same predicate multiple times.
Solution:
Add an API to infer null-rejecting slots only for a given target slot set,
and
use the current join's nullable-side outputs as that target in
EliminateOuterJoin. Skip mark-join slots as inference targets, preserve the
existing expression complexity limits, and avoid rewriting when the join
type
does not change.
---------
Co-authored-by: yangtao555 <[email protected]>
---
.../nereids/rules/rewrite/EliminateOuterJoin.java | 35 +++---
.../nereids/rules/rewrite/InferAggNotNull.java | 3 +-
.../apache/doris/nereids/trees/plans/JoinType.java | 14 +++
.../nereids/trees/plans/logical/LogicalJoin.java | 16 +++
.../apache/doris/nereids/util/ExpressionUtils.java | 87 ++++++---------
.../rules/rewrite/EliminateOuterJoinTest.java | 122 +++++++++++++++++++++
.../nereids/rules/rewrite/InferAggNotNullTest.java | 55 ++++++++++
.../filter_push_down/push_filter_through.out | 8 +-
.../eliminate_outer_join/eliminate_outer_join.out | 19 ++--
.../eliminate_outer_join.groovy | 2 +-
10 files changed, 273 insertions(+), 88 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java
index 435d280ccf6..ac03ef50514 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java
@@ -28,17 +28,14 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.JoinUtils;
-import org.apache.doris.nereids.util.TypeUtils;
import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSet.Builder;
import com.google.common.collect.Sets;
import java.util.Collection;
-import java.util.HashSet;
-import java.util.Optional;
import java.util.Set;
/**
@@ -50,19 +47,11 @@ public class EliminateOuterJoin extends
OneRewriteRuleFactory {
public Rule build() {
return logicalFilter(
logicalJoin().when(join -> join.getJoinType().isOuterJoin() ||
join.getJoinType().isAsofOuterJoin())
- ).then(filter -> {
- LogicalJoin<Plan, Plan> join = filter.child();
+ ).thenApply(ctx -> {
+ LogicalJoin<Plan, Plan> join = ctx.root.child();
- Builder<Expression> conjunctsBuilder = ImmutableSet.builder();
- Set<Slot> notNullSlots = new HashSet<>();
- for (Expression predicate : filter.getConjuncts()) {
- Optional<Slot> notNullSlot = TypeUtils.isNotNull(predicate);
- if (notNullSlot.isPresent()) {
- notNullSlots.add(notNullSlot.get());
- } else {
- conjunctsBuilder.add(predicate);
- }
- }
+ Set<Slot> notNullSlots = ExpressionUtils.inferNotNullSlots(
+ ctx.root.getConjuncts(), join.getNullableSideOutput(),
ctx.cascadesContext);
boolean canFilterLeftNull =
Utils.isIntersecting(join.left().getOutputSet(), notNullSlots);
boolean canFilterRightNull =
Utils.isIntersecting(join.right().getOutputSet(), notNullSlots);
if (!canFilterRightNull && !canFilterLeftNull) {
@@ -70,8 +59,14 @@ public class EliminateOuterJoin extends
OneRewriteRuleFactory {
}
JoinType newJoinType = tryEliminateOuterJoin(join.getJoinType(),
canFilterLeftNull, canFilterRightNull);
+ // Nothing changed: avoid adding redundant generated `IS NOT NULL`
markers and
+ // returning a structurally-equivalent plan, which would otherwise
cause pointless
+ // re-rewrite churn inside the PUSH_DOWN_FILTERS fixed-point.
+ if (newJoinType == join.getJoinType()) {
+ return null;
+ }
Set<Expression> conjuncts = Sets.newHashSet();
- conjuncts.addAll(filter.getConjuncts());
+ conjuncts.addAll(ctx.root.getConjuncts());
boolean conjunctsChanged = false;
if (!notNullSlots.isEmpty()) {
for (Slot slot : notNullSlots) {
@@ -86,8 +81,6 @@ public class EliminateOuterJoin extends OneRewriteRuleFactory
{
* by which the left outer join could be eliminated. Finally,
the join transformed to
* (A join B on A.a=B.b) join C on B.x=C.x.
* This elimination can be processed recursively.
- *
- * TODO: is_not_null can also be inferred from A < B and so on
*/
conjunctsChanged |= join.getEqualToConjuncts().stream()
.map(EqualTo.class::cast)
@@ -105,10 +98,10 @@ public class EliminateOuterJoin extends
OneRewriteRuleFactory {
.anyMatch(equalTo ->
createIsNotNullIfNecessary(equalTo, conjuncts));
}
if (conjunctsChanged) {
- return
filter.withConjuncts(conjuncts.stream().collect(ImmutableSet.toImmutableSet()))
+ return
ctx.root.withConjuncts(conjuncts.stream().collect(ImmutableSet.toImmutableSet()))
.withChildren(join.withJoinTypeAndContext(newJoinType,
join.getJoinReorderContext()));
}
- return
filter.withChildren(join.withJoinTypeAndContext(newJoinType,
join.getJoinReorderContext()));
+ return
ctx.root.withChildren(join.withJoinTypeAndContext(newJoinType,
join.getJoinReorderContext()));
}).toRule(RuleType.ELIMINATE_OUTER_JOIN);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferAggNotNull.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferAggNotNull.java
index 4daf320811a..e27a7b8cfad 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferAggNotNull.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferAggNotNull.java
@@ -108,8 +108,7 @@ public class InferAggNotNull extends OneRewriteRuleFactory {
private boolean canInferFunctionNotNull(AggregateFunction
aggregateFunction) {
return isSupportedAggregateFunction(aggregateFunction)
- && !aggregateFunction.children().isEmpty()
- &&
ExpressionUtils.isCheapEnoughToInferNotNull(aggregateFunction.children());
+ && !aggregateFunction.children().isEmpty();
}
private boolean isSupportedAggregateFunction(AggregateFunction
aggregateFunction) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java
index b1fcf9bd460..9d91d3fc87a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java
@@ -253,6 +253,20 @@ public enum JoinType {
return this == ASOF_LEFT_OUTER_JOIN || this == ASOF_RIGHT_OUTER_JOIN;
}
+ /**
+ * Whether this join may null-extend slots from its left child.
+ */
+ public final boolean isLeftSideNullable() {
+ return isRightOuterJoin() || isAsofRightOuterJoin() ||
isFullOuterJoin();
+ }
+
+ /**
+ * Whether this join may null-extend slots from its right child.
+ */
+ public final boolean isRightSideNullable() {
+ return isLeftOuterJoin() || isAsofLeftOuterJoin() || isFullOuterJoin();
+ }
+
public final boolean isAsofLeftJoin() {
return this == ASOF_LEFT_INNER_JOIN || this == ASOF_LEFT_OUTER_JOIN;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
index 666ac5b95dd..5321b9c097a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
@@ -287,6 +287,22 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan,
RIGHT_CHILD_TYPE extends
return joinReorderContext;
}
+ /**
+ * Output slots of the null-extended side(s) of an outer join, i.e. the
slots that
+ * this join may turn into NULL. Rejecting NULL on them lets an outer join
be eliminated
+ * or weakened. Returns empty for non-outer joins.
+ */
+ public Set<Slot> getNullableSideOutput() {
+ ImmutableSet.Builder<Slot> nullableSide = ImmutableSet.builder();
+ if (joinType.isLeftSideNullable()) {
+ nullableSide.addAll(left().getOutputSet());
+ }
+ if (joinType.isRightSideNullable()) {
+ nullableSide.addAll(right().getOutputSet());
+ }
+ return nullableSide.build();
+ }
+
@Override
public List<Slot> computeOutput() {
return ImmutableList.<Slot>builder()
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index fd7e761bc6e..474909db546 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -752,62 +752,59 @@ public class ExpressionUtils {
* infer notNulls slot from predicate
*/
public static Set<Slot> inferNotNullSlots(Set<Expression> predicates,
CascadesContext cascadesContext) {
- ImmutableSet.Builder<Slot> notNullSlots =
ImmutableSet.builderWithExpectedSize(predicates.size());
- for (Expression predicate :
filterCheapPredicatesForNotNull(predicates)) {
+ Set<Slot> targetSlots = new HashSet<>();
+ for (Expression predicate : predicates) {
for (Slot slot : predicate.getInputSlots()) {
- Map<Expression, Expression> replaceMap = new HashMap<>();
- Literal nullLiteral = new NullLiteral(slot.getDataType());
- replaceMap.put(slot, nullLiteral);
- Expression evalExpr = FoldConstantRule.evaluate(
- ExpressionUtils.replace(predicate, replaceMap),
- new ExpressionRewriteContext(cascadesContext));
- if (evalExpr.isNullLiteral() ||
BooleanLiteral.FALSE.equals(evalExpr)) {
- notNullSlots.add(slot);
+ if (!(slot instanceof MarkJoinSlotReference)) {
+ targetSlots.add(slot);
}
}
}
- return notNullSlots.build();
+ return inferNotNullSlots(predicates, targetSlots, cascadesContext);
}
/**
- * Return whether all predicates are cheap enough for not-null inference.
+ * infer notNulls slot from predicate but these slots must be in the given
target slots.
*/
- public static boolean isCheapEnoughToInferNotNull(Collection<? extends
Expression> predicates) {
+ public static Set<Slot> inferNotNullSlots(Set<Expression> predicates,
Set<Slot> targetSlots,
+ CascadesContext cascadesContext) {
+ ImmutableSet.Builder<Slot> notNullSlots =
ImmutableSet.builderWithExpectedSize(targetSlots.size());
Set<Slot> inputSlots = new HashSet<>();
for (Expression predicate : predicates) {
- Optional<Set<Slot>> mergedInputSlots =
mergeInputSlotsIfCheap(predicate, inputSlots);
- if (!mergedInputSlots.isPresent()) {
- return false;
+ if (predicate.getWidth() > MAX_INFER_NOT_NULL_EXPR_WIDTH
+ || predicate.getDepth() > MAX_INFER_NOT_NULL_EXPR_DEPTH) {
+ continue;
}
- inputSlots = mergedInputSlots.get();
- }
- return true;
- }
-
- /**
- * Filter predicates that are cheap enough for not-null inference.
- */
- public static Set<Expression> filterCheapPredicatesForNotNull(
- Collection<? extends Expression> predicates) {
- Set<Slot> inputSlots = new HashSet<>();
- Set<Expression> cheapPredicates = Sets.newLinkedHashSet();
- for (Expression predicate : predicates) {
- Optional<Set<Slot>> mergedInputSlots =
mergeInputSlotsIfCheap(predicate, inputSlots);
+ Set<Slot> predicateInputSlots = predicate.getInputSlots();
+ Set<Slot> candidateSlots = Sets.intersection(predicateInputSlots,
targetSlots);
+ if (candidateSlots.isEmpty()) {
+ continue;
+ }
+ Optional<Set<Slot>> mergedInputSlots =
mergeInputSlotsWithinLimit(inputSlots, predicateInputSlots);
if (!mergedInputSlots.isPresent()) {
continue;
}
inputSlots = mergedInputSlots.get();
- cheapPredicates.add(predicate);
+ for (Slot slot : candidateSlots) {
+ if (isNullRejecting(predicate, slot, cascadesContext)) {
+ notNullSlots.add(slot);
+ }
+ }
}
- return cheapPredicates;
+ return notNullSlots.build();
}
- private static Optional<Set<Slot>> mergeInputSlotsIfCheap(Expression
predicate, Set<Slot> inputSlots) {
- if (predicate.getWidth() > MAX_INFER_NOT_NULL_EXPR_WIDTH
- || predicate.getDepth() > MAX_INFER_NOT_NULL_EXPR_DEPTH) {
- return Optional.empty();
- }
- Set<Slot> predicateInputSlots = predicate.getInputSlots();
+ private static boolean isNullRejecting(Expression predicate, Slot slot,
CascadesContext cascadesContext) {
+ Map<Expression, Expression> replaceMap = new HashMap<>();
+ Literal nullLiteral = new NullLiteral(slot.getDataType());
+ replaceMap.put(slot, nullLiteral);
+ Expression evalExpr = FoldConstantRule.evaluate(
+ ExpressionUtils.replace(predicate, replaceMap),
+ new ExpressionRewriteContext(cascadesContext));
+ return evalExpr.isNullLiteral() ||
BooleanLiteral.FALSE.equals(evalExpr);
+ }
+
+ private static Optional<Set<Slot>> mergeInputSlotsWithinLimit(Set<Slot>
inputSlots, Set<Slot> predicateInputSlots) {
if (predicateInputSlots.size() > MAX_INFER_NOT_NULL_INPUT_SLOTS) {
return Optional.empty();
}
@@ -830,20 +827,6 @@ public class ExpressionUtils {
return newPredicates.build();
}
- /**
- * infer notNulls slot from predicate but these slots must be in the given
slots.
- */
- public static Set<Expression> inferNotNull(Set<Expression> predicates,
Set<Slot> slots,
- CascadesContext cascadesContext) {
- ImmutableSet.Builder<Expression> newPredicates =
ImmutableSet.builderWithExpectedSize(predicates.size());
- for (Slot slot : inferNotNullSlots(predicates, cascadesContext)) {
- if (slots.contains(slot)) {
- newPredicates.add(new Not(new IsNull(slot), true));
- }
- }
- return newPredicates.build();
- }
-
public static boolean isGeneratedNotNull(Expression expression) {
return expression instanceof Not
&& ((Not) expression).isGeneratedIsNotNull()
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java
index 850c31a14ab..c16fbd870f1 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java
@@ -20,7 +20,11 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.trees.expressions.And;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
+import org.apache.doris.nereids.trees.expressions.InPredicate;
+import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
+import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.JoinType;
@@ -32,6 +36,7 @@ import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.nereids.util.PlanConstructor;
+import com.google.common.collect.ImmutableList;
import org.junit.jupiter.api.Test;
import java.util.Objects;
@@ -53,6 +58,24 @@ class EliminateOuterJoinTest implements
MemoPatternMatchSupported {
testEliminateLeftHelper(JoinType.ASOF_LEFT_OUTER_JOIN);
}
+ @Test
+ void testEliminateLeftByInPredicateOrFalse() {
+ LogicalPlan plan = new LogicalPlanBuilder(scan1)
+ .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) //
t1.id = t2.id
+ .filter(new Or(new InPredicate(scan2.getOutput().get(0),
+ ImmutableList.of(new IntegerLiteral(1), new
IntegerLiteral(2))), BooleanLiteral.FALSE))
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new EliminateOuterJoin())
+ .applyTopDown(new EliminateNotNull())
+ .matchesFromRoot(
+ logicalFilter(
+ logicalJoin().when(join ->
join.getJoinType().isInnerJoin()))
+ .when(filter -> filter.getConjuncts().size()
== 1)
+ );
+ }
+
private void testEliminateLeftHelper(JoinType joinType) {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id
@@ -119,4 +142,103 @@ class EliminateOuterJoinTest implements
MemoPatternMatchSupported {
).when(filter -> filter.getConjuncts().size() == 2)
);
}
+
+ /**
+ * FULL OUTER -> LEFT OUTER when only the left side is null-rejecting.
+ */
+ @Test
+ void testFullOuterDegradeToLeft() {
+ LogicalPlan plan = new LogicalPlanBuilder(scan1)
+ .join(scan2, JoinType.FULL_OUTER_JOIN, Pair.of(0, 0))
+ .filter(new GreaterThan(scan1.getOutput().get(0), new
IntegerLiteral(1)))
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new EliminateOuterJoin())
+ .matchesFromRoot(
+ logicalFilter(
+ logicalJoin().when(join ->
join.getJoinType().isLeftOuterJoin()))
+ );
+ }
+
+ /**
+ * FULL OUTER -> RIGHT OUTER when only the right side is null-rejecting.
+ */
+ @Test
+ void testFullOuterDegradeToRight() {
+ LogicalPlan plan = new LogicalPlanBuilder(scan1)
+ .join(scan2, JoinType.FULL_OUTER_JOIN, Pair.of(0, 0))
+ .filter(new GreaterThan(scan2.getOutput().get(0), new
IntegerLiteral(1)))
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new EliminateOuterJoin())
+ .matchesFromRoot(
+ logicalFilter(
+ logicalJoin().when(join ->
join.getJoinType().isRightOuterJoin()))
+ );
+ }
+
+ /**
+ * LEFT OUTER stays LEFT OUTER when the filter only constrains the
preserved (left) side.
+ * Without the early-return guard, the rule would still rewrite the plan
with redundant
+ * generated IS NOT NULL markers and trigger pointless re-rewrite churn.
+ */
+ @Test
+ void testLeftOuterNotChangedByLeftSidePredicate() {
+ LogicalPlan plan = new LogicalPlanBuilder(scan1)
+ .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0))
+ .filter(new GreaterThan(scan1.getOutput().get(0), new
IntegerLiteral(1)))
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new EliminateOuterJoin())
+ .matchesFromRoot(
+ logicalFilter(
+ logicalJoin().when(join ->
join.getJoinType().isLeftOuterJoin()))
+ .when(filter -> filter.getConjuncts().size()
== 1)
+ );
+ }
+
+ /**
+ * `t2.b IS NULL OR t2.b > 0` is NOT null-rejecting on t2.b (the IS NULL
branch keeps NULLs),
+ * so LEFT OUTER must stay LEFT OUTER.
+ */
+ @Test
+ void testLeftOuterNotEliminatedByIsNullOr() {
+ LogicalPlan plan = new LogicalPlanBuilder(scan1)
+ .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0))
+ .filter(new Or(
+ new IsNull(scan2.getOutput().get(0)),
+ new GreaterThan(scan2.getOutput().get(0), new
IntegerLiteral(0))))
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new EliminateOuterJoin())
+ .matchesFromRoot(
+ logicalFilter(
+ logicalJoin().when(join ->
join.getJoinType().isLeftOuterJoin()))
+ );
+ }
+
+ /**
+ * A predicate that mixes both sides via OR (`t1.a > 0 OR t2.b > 0`) is
NOT null-rejecting
+ * on either side individually, because one branch may stay TRUE when the
other side is NULL.
+ */
+ @Test
+ void testLeftOuterNotEliminatedByCrossSideOr() {
+ LogicalPlan plan = new LogicalPlanBuilder(scan1)
+ .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0))
+ .filter(new Or(
+ new GreaterThan(scan1.getOutput().get(0), new
IntegerLiteral(0)),
+ new GreaterThan(scan2.getOutput().get(0), new
IntegerLiteral(0))))
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new EliminateOuterJoin())
+ .matchesFromRoot(
+ logicalFilter(
+ logicalJoin().when(join ->
join.getJoinType().isLeftOuterJoin()))
+ );
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferAggNotNullTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferAggNotNullTest.java
index 23b108c8347..feea96e0f4d 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferAggNotNullTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferAggNotNullTest.java
@@ -17,12 +17,21 @@
package org.apache.doris.nereids.rules.rewrite;
+import org.apache.doris.catalog.AggregateType;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.KeysType;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.PartitionInfo;
+import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.Not;
import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.Avg;
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
+import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
@@ -31,11 +40,14 @@ import
org.apache.doris.nereids.util.MemoPatternMatchSupported;
import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.nereids.util.PlanConstructor;
+import org.apache.doris.thrift.TStorageType;
import com.google.common.collect.ImmutableList;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Set;
class InferAggNotNullTest implements MemoPatternMatchSupported {
@@ -130,6 +142,37 @@ class InferAggNotNullTest implements
MemoPatternMatchSupported {
);
}
+ @Test
+ void testInferPartialWhenArgsExceedSlotLimit() {
+ // count(distinct c0..c32) has 33 nullable arguments. inferNotNull
merges input slots up to
+ // a 32-slot limit, so the first 32 arguments get a generated IS NOT
NULL and the 33rd is
+ // skipped. This partial-inference path is only reachable after the
all-children cheapness
+ // gate was removed from InferAggNotNull.
+ LogicalOlapScan wideScan = newWideNullableScan(33);
+ List<Expression> args = new ArrayList<>(wideScan.getOutput());
+
+ LogicalPlan plan = new LogicalPlanBuilder(wideScan)
+ .aggGroupUsingIndex(ImmutableList.of(),
+ ImmutableList.of(new Alias(
+ new Count(true, args.get(0), args.subList(1,
33).toArray(new Expression[0])),
+ "cnt")))
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new InferAggNotNull())
+ .matches(
+ logicalAggregate(
+ logicalFilter().when(filter -> {
+ Set<Expression> conjuncts =
filter.getConjuncts();
+ return conjuncts.size() == 32
+ && conjuncts.stream().allMatch(e
-> e instanceof Not
+ && ((Not)
e).isGeneratedIsNotNull()
+ && ((Not) e).child()
instanceof IsNull);
+ })
+ )
+ );
+ }
+
@Test
void testGetAggregateFunctionsStopsAtAggregateFunction() {
// Use different agg function types for inner (Avg) and outer (Count),
@@ -147,4 +190,16 @@ class InferAggNotNullTest implements
MemoPatternMatchSupported {
Assertions.assertTrue(aggregateFunctions.stream().allMatch(f -> f
instanceof Count),
"should collect only the outer Count, got: " +
aggregateFunctions);
}
+
+ private LogicalOlapScan newWideNullableScan(int columnCount) {
+ List<Column> columns = new ArrayList<>(columnCount);
+ for (int i = 0; i < columnCount; i++) {
+ columns.add(new Column("c" + i, Type.INT, false,
AggregateType.NONE, true, "", ""));
+ }
+ OlapTable table = new OlapTable(100L, "wide", columns,
+ KeysType.DUP_KEYS, new PartitionInfo(), null);
+ table.setIndexMeta(-1, "wide", table.getFullSchema(), 0, 0, (short) 0,
+ TStorageType.COLUMN, KeysType.DUP_KEYS);
+ return new LogicalOlapScan(RelationId.createGenerator().getNextId(),
table, ImmutableList.of("db"));
+ }
}
diff --git
a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out
b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out
index d04e95189da..f7d67dc0b87 100644
---
a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out
+++
b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out
@@ -169,11 +169,13 @@ PhysicalResultSink
-- !filter_mixed_inner_left --
PhysicalResultSink
---filter((t3.id = 2))
-----hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.id = t3.id)) otherCondition=()
-------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t3.id)) otherCondition=()
+----hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
+------filter(( not t1.id IS NULL) and (t1.id = 2))
--------PhysicalOlapScan[t1]
+------filter((t2.id = 2))
--------PhysicalOlapScan[t2]
+----filter(( not t3.id IS NULL) and (t3.id = 2))
------PhysicalOlapScan[t3]
-- !filter_multi_left --
diff --git
a/regression-test/data/query_p0/eliminate_outer_join/eliminate_outer_join.out
b/regression-test/data/query_p0/eliminate_outer_join/eliminate_outer_join.out
index 6a94a1cf100..51b6ca56bc1 100644
---
a/regression-test/data/query_p0/eliminate_outer_join/eliminate_outer_join.out
+++
b/regression-test/data/query_p0/eliminate_outer_join/eliminate_outer_join.out
@@ -20,11 +20,12 @@ SyntaxError:
-- !2 --
PhysicalResultSink
--hashJoin[RIGHT_OUTER_JOIN] hashCondition=((partsupp.ps_suppkey =
supplier.s_suppkey)) otherCondition=()
-----filter((supplier.s_suppkey > 1))
-------hashJoin[FULL_OUTER_JOIN] hashCondition=((nation.n_nationkey =
supplier.s_suppkey)) otherCondition=()
---------hashJoin[FULL_OUTER_JOIN] hashCondition=((region.r_regionkey =
nation.n_regionkey)) otherCondition=()
-----------PhysicalOlapScan[region]
+----hashJoin[RIGHT_OUTER_JOIN] hashCondition=((nation.n_nationkey =
supplier.s_suppkey)) otherCondition=()
+------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((region.r_regionkey =
nation.n_regionkey)) otherCondition=()
+--------PhysicalOlapScan[region]
+--------filter(( not nation.n_nationkey IS NULL) and (nation.n_nationkey > 1))
----------PhysicalOlapScan[nation]
+------filter(( not supplier.s_suppkey IS NULL) and (supplier.s_suppkey > 1))
--------PhysicalOlapScan[supplier]
----filter(( not partsupp.ps_suppkey IS NULL) and (partsupp.ps_suppkey > 1))
------PhysicalOlapScan[partsupp]
@@ -37,11 +38,12 @@ SyntaxError:
-- !3 --
PhysicalResultSink
--hashJoin[RIGHT_OUTER_JOIN] hashCondition=((partsupp.ps_suppkey =
supplier.s_suppkey)) otherCondition=()
-----filter((supplier.s_suppkey > 1))
-------hashJoin[LEFT_OUTER_JOIN] hashCondition=((nation.n_nationkey =
supplier.s_suppkey)) otherCondition=()
---------hashJoin[FULL_OUTER_JOIN] hashCondition=((region.r_regionkey =
nation.n_regionkey)) otherCondition=()
-----------PhysicalOlapScan[region]
+----hashJoin[INNER_JOIN] hashCondition=((nation.n_nationkey =
supplier.s_suppkey)) otherCondition=()
+------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((region.r_regionkey =
nation.n_regionkey)) otherCondition=()
+--------PhysicalOlapScan[region]
+--------filter(( not nation.n_nationkey IS NULL) and (nation.n_nationkey > 1))
----------PhysicalOlapScan[nation]
+------filter(( not supplier.s_suppkey IS NULL) and (supplier.s_suppkey > 1))
--------PhysicalOlapScan[supplier]
----filter(( not partsupp.ps_suppkey IS NULL) and (partsupp.ps_suppkey > 1))
------PhysicalOlapScan[partsupp]
@@ -143,4 +145,3 @@ Hint log:
Used: [broadcast]_1
UnUsed:
SyntaxError:
-
diff --git
a/regression-test/suites/query_p0/eliminate_outer_join/eliminate_outer_join.groovy
b/regression-test/suites/query_p0/eliminate_outer_join/eliminate_outer_join.groovy
index 9f83b850fe8..177496d1a00 100644
---
a/regression-test/suites/query_p0/eliminate_outer_join/eliminate_outer_join.groovy
+++
b/regression-test/suites/query_p0/eliminate_outer_join/eliminate_outer_join.groovy
@@ -123,7 +123,7 @@ suite("eliminate_outer_join") {
where ps_suppkey > 1
'''
- // full join ps => right join ps, other outer joins are not eliminated
+ // The filter on partsupp null-rejects the right side of each cascading
full join.
qt_2 '''
explain shape plan
select *
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
