This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 653ff1e1cc1 [Fix](nereids) fix partition_prune or expression evaluate
wrongly (#38897)
653ff1e1cc1 is described below
commit 653ff1e1cc15c2d266e851a14abb037a20844f69
Author: feiniaofeiafei <53502832+feiniaofeia...@users.noreply.github.com>
AuthorDate: Tue Aug 6 19:40:57 2024 +0800
[Fix](nereids) fix partition_prune or expression evaluate wrongly (#38897)
introduced by #38025
Or expression evaluate wrongly in #38025. For example, partition column
is a, b, predicate is a=a or b>1 , partition column range is min<a<5, b
is infinite. The evaluate result of or expression should be b is
infinite(because when a is not null, a=a is true, there is no restrict
for b, e.g. a=1, b=0 satisfy a=a or b>1). #38025 pr has wrongly evaluate
result b >1
---
.../rules/OneRangePartitionEvaluator.java | 50 ++++++++--------------
.../partition_prune/test_date_trunc_prune.out | 10 +++++
.../partition_prune/test_date_trunc_prune.groovy | 41 ++++++++++++++++++
3 files changed, 68 insertions(+), 33 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
index 446faffe0f5..6d034431545 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
@@ -409,17 +409,8 @@ public class OneRangePartitionEvaluator
@Override
public EvaluateRangeResult visitAnd(And and, EvaluateRangeInput context) {
EvaluateRangeResult result = evaluateChildrenThenThis(and, context);
-
result = mergeRanges(result.result, result.childrenResult.get(0),
result.childrenResult.get(1),
- (leftRange, rightRange) -> {
- if (leftRange == null) {
- return rightRange;
- }
- if (rightRange == null) {
- return leftRange;
- }
- return leftRange.intersect(rightRange);
- });
+ (leftRange, rightRange) -> leftRange.intersect(rightRange));
result = returnFalseIfExistEmptyRange(result);
if (result.result.equals(BooleanLiteral.FALSE)) {
@@ -435,18 +426,20 @@ public class OneRangePartitionEvaluator
@Override
public EvaluateRangeResult visitOr(Or or, EvaluateRangeInput context) {
EvaluateRangeResult result = evaluateChildrenThenThis(or, context);
-
+ if (result.result.equals(BooleanLiteral.FALSE)) {
+ return result;
+ } else if
(result.childrenResult.get(0).result.equals(BooleanLiteral.FALSE)) {
+ // false or a<1 -> return range a<1
+ return new EvaluateRangeResult(result.result,
result.childrenResult.get(1).columnRanges,
+ result.childrenResult);
+ } else if
(result.childrenResult.get(1).result.equals(BooleanLiteral.FALSE)) {
+ // a<1 or false -> return range a<1
+ return new EvaluateRangeResult(result.result,
result.childrenResult.get(0).columnRanges,
+ result.childrenResult);
+ }
result = mergeRanges(result.result, result.childrenResult.get(0),
result.childrenResult.get(1),
- (leftRange, rightRange) -> {
- if (leftRange == null) {
- return rightRange;
- }
- if (rightRange == null) {
- return leftRange;
- }
- return leftRange.union(rightRange);
- });
- return removeEmptyRange(result);
+ (leftRange, rightRange) -> leftRange.union(rightRange));
+ return returnFalseIfExistEmptyRange(result);
}
@Override
@@ -596,7 +589,9 @@ public class OneRangePartitionEvaluator
.build();
Map<Expression, ColumnRange> mergedRange = exprs.stream()
- .map(expr -> Pair.of(expr,
mergeFunction.apply(leftRanges.get(expr), rightRanges.get(expr))))
+ .map(expr -> Pair.of(expr, mergeFunction.apply(
+ leftRanges.containsKey(expr) ? leftRanges.get(expr) :
rangeMap.get(expr),
+ rightRanges.containsKey(expr) ? rightRanges.get(expr)
: rangeMap.get(expr))))
.collect(ImmutableMap.toImmutableMap(Pair::key, Pair::value));
return new EvaluateRangeResult(originResult, mergedRange,
ImmutableList.of(left, right));
}
@@ -846,17 +841,6 @@ public class OneRangePartitionEvaluator
return onePartitionInputs;
}
- private EvaluateRangeResult removeEmptyRange(EvaluateRangeResult result) {
- ImmutableMap.Builder<Expression, ColumnRange> builder =
ImmutableMap.builder();
- for (Map.Entry<Expression, ColumnRange> entry :
result.columnRanges.entrySet()) {
- if (entry.getValue().isEmptyRange()) {
- continue;
- }
- builder.put(entry);
- }
- return new EvaluateRangeResult(result.result, builder.build(),
result.childrenResult);
- }
-
private EvaluateRangeResult
computeMonotonicFunctionRange(EvaluateRangeResult result) {
Monotonic func = (Monotonic) result.result;
if (rangeMap.containsKey(func)) {
diff --git
a/regression-test/data/nereids_rules_p0/partition_prune/test_date_trunc_prune.out
b/regression-test/data/nereids_rules_p0/partition_prune/test_date_trunc_prune.out
new file mode 100644
index 00000000000..ee8cabc4451
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/partition_prune/test_date_trunc_prune.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !or_evaluate --
+0
+3
+5
+6
+7
+8
+9
+
diff --git
a/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy
b/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy
index c4769b12b12..4ad3afbb9db 100644
---
a/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy
+++
b/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy
@@ -293,4 +293,45 @@ suite("test_date_trunc_prune") {
sql "select * from mal_test_partition_range2_two_date_int where
date_trunc(dt,'month') > '2017-2-1 00:00:00' and id>100;"
contains("partitions=2/3 (p201702_2000,p201703_all)")
}
+
+ sql "drop table if exists test_partition_prune_or;"
+ sql """create table test_partition_prune_or (
+ col_int_undef_signed int ,
+ col_date_undef_signed date ,
+ pk int,
+ col_int_undef_signed2 int ,
+ col_date_undef_signed2 date ,
+ col_varchar_10__undef_signed varchar(10) ,
+ col_varchar_1024__undef_signed varchar(1024)
+ ) engine=olap
+ DUPLICATE KEY(col_int_undef_signed, col_date_undef_signed, pk)
+ PARTITION BY RANGE(col_int_undef_signed,
col_date_undef_signed) (
+ PARTITION p0 VALUES LESS THAN ('4', '2023-12-11'),
+ PARTITION p1 VALUES LESS THAN ('6', '2023-12-15'),
+ PARTITION p2 VALUES LESS THAN ('7', '2023-12-16'),
+ PARTITION p3 VALUES LESS THAN ('8', '2023-12-25'),
+ PARTITION p4 VALUES LESS THAN ('8', '2024-01-18'),
+ PARTITION p5 VALUES LESS THAN ('10', '2024-02-18'),
+ PARTITION p6 VALUES LESS THAN ('1147483647', '2056-12-31'),
+ PARTITION p100 VALUES LESS THAN ('2147483647',
'9999-12-31')
+ )
+
+ distributed by hash(pk) buckets 10
+ properties("replication_num" = "1");"""
+
+ sql """insert into
test_partition_prune_or(pk,col_int_undef_signed,col_int_undef_signed2,col_date_undef_signed,col_date_undef_signed2,col_varchar_10__undef_signed,col_varchar_1024__undef_signed)
+ values (0,9,8,'2026-02-18','2027-01-09',null,null),
+
(1,6,6,'2023-12-10','2023-12-14','v','x'),(2,null,1,'2025-06-18','2027-01-16','o','m'),(3,2,null,'2024-01-19','2024-01-17','m','q'),(4,5,5,'2023-12-17','2023-12-18','q','o'),(5,3,2,'2023-12-15','2023-12-16','e','m'),(6,2,null,'2025-06-18','2025-02-18',null,'p'),(7,7,3,'2024-01-08','2024-02-18','j','f'),(8,8,9,'2024-01-31','2026-01-18','x','t'),(9,7,0,'2023-12-10','2025-02-18','p','b'),(10,2,4,'2023-12-12','2023-12-16','e','p'),(11,5,0,'2025-06-18','2025-06-18','n','v'),(12,0,4,'2024-
[...]
+ """
+ explain {
+ sql """SELECT table1 . `col_int_undef_signed` AS field1 FROM
test_partition_prune_or AS table1 WHERE
+ ( NOT ( ( table1 . `col_int_undef_signed` != table1 .
`col_int_undef_signed` ) AND table1 . `col_date_undef_signed` <= '2025-06-18' )
AND
+ table1 . `col_date_undef_signed` IN ( '2023-12-20' ) ) GROUP BY
field1 ORDER BY field1 LIMIT 1000;
+ """
+ contains("partitions=5/8 (p0,p1,p2,p3,p5)")
+ }
+ qt_or_evaluate """SELECT table1 . `col_int_undef_signed` AS field1 FROM
test_partition_prune_or AS table1 WHERE
+ ( NOT ( ( table1 . `col_int_undef_signed` != table1 .
`col_int_undef_signed` ) AND table1 . `col_date_undef_signed` <= '2025-06-18' )
AND
+ table1 . `col_date_undef_signed` IN ( '2023-12-20' ) ) GROUP BY field1
ORDER BY field1 LIMIT 1000;
+ """
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
