This is an automated email from the ASF dual-hosted git repository.
mrhhsg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 761162dd22d [test](nereids) Cover query columns captured inside nested
lambda bodies (#67713)
761162dd22d is described below
commit 761162dd22d78eba857b9d7caec53a0256055963
Author: Jerry Hu <[email protected]>
AuthorDate: Wed Sep 23 18:40:41 2026 +0800
[test](nereids) Cover query columns captured inside nested lambda bodies
(#67713)
### What problem does this PR solve?
Issue Number: None
Related PR: #68260
Problem Summary:
A lambda body may reference the columns of the enclosing query, but this
only worked for a single level of high-order function. As soon as the
lambda was nested inside another lambda, the reference failed to bind:
```sql
WITH t2 AS (SELECT c IS NULL AS dict_flag, ARRAY('A') AS arr1, ARRAY('a')
AS arr2 FROM t)
SELECT ARRAY_MAP(arg1 -> ARRAY_SUM(ARRAY_MAP(arg2 -> IF(dict_flag, 1, 0),
arr2)), arr1) FROM t2;
-- ERROR 1105 (HY000): Unknown lambda slot 'dict_flag in lambda
arguments[arg2]
```
The same happened with `array_sortby`, `array_filter` and any other
high-order function used as the inner lambda.
Root cause: `ExpressionAnalyzer` analyzed every lambda body with a
nested analyzer whose scope only held the lambda arguments and whose
outer scope was the enclosing scope. Slot binding looks at most one
level up, so with two nested lambdas the query columns were out of
reach for the inner lambda body.
#68260 has since changed the lambda analyzer so that every name that is
not a lambda argument is resolved by the enclosing analyzer, which
recurses through the enclosing lambdas up to the query analyzer. That
already makes the query columns bindable at any nesting depth, and an
unknown name is now reported by the query analyzer as
`Unknown column 'x' in 'table list' ...`. No production change is left;
this PR keeps the coverage so the behavior cannot regress silently:
- `BindFunctionTest`: query-column capture in nested
`array_map`/`array_sortby`/three-level lambdas, visibility of all
enclosing lambda arguments, same-name shadowing, and the unknown
column error.
- `test_nested_array_map`: nested lambdas that capture query columns
(plain column, CTE alias, together with the enclosing lambda
argument, three levels deep) and the unknown column error.
### Release note
None
### Check List (For Author)
- Test:
- Unit Test: `BindFunctionTest` (new cases pass on current master)
- Regression test: `test_nested_array_map` (passes on current master)
- Behavior changed: No
- Does this need documentation: No
https://claude.ai/code/session_01FfTgMuMpYGGFWuckWxeGNw
---
.../nereids/rules/analysis/BindFunctionTest.java | 76 +++++++++++++++++++++-
.../array_functions/test_nested_array_map.out | 31 +++++++++
.../array_functions/test_nested_array_map.groovy | 56 ++++++++++++++++
3 files changed, 162 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindFunctionTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindFunctionTest.java
index e176cbaa7d8..3697af015dc 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindFunctionTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindFunctionTest.java
@@ -18,15 +18,27 @@
package org.apache.doris.nereids.rules.analysis;
import org.apache.doris.common.Config;
+import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.parser.NereidsParser;
+import
org.apache.doris.nereids.trees.expressions.ArrayItemReference.ArrayItemSlot;
+import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.LessThan;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
+import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.utframe.TestWithFeService;
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
public class BindFunctionTest extends TestWithFeService implements
MemoPatternMatchSupported {
private final NereidsParser parser = new NereidsParser();
@@ -40,7 +52,10 @@ public class BindFunctionTest extends TestWithFeService
implements MemoPatternMa
"CREATE TABLE t1 (col1 date, col2 int) DISTRIBUTED BY
HASH(col2)\n" + "BUCKETS 1\n" + "PROPERTIES(\n"
+ " \"replication_num\"=\"1\"\n" + ");",
"CREATE TABLE t2 (col1 date, col2 int) DISTRIBUTED BY
HASH(col2)\n" + "BUCKETS 1\n" + "PROPERTIES(\n"
- + " \"replication_num\"=\"1\"\n" + ");"
+ + " \"replication_num\"=\"1\"\n" + ");",
+ "CREATE TABLE t_arr (id int, flag boolean, arr1
array<varchar(10)>, arr2 array<int>)\n"
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1\n"
+ + "PROPERTIES(\n \"replication_num\"=\"1\"\n);"
);
}
@@ -74,4 +89,63 @@ public class BindFunctionTest extends TestWithFeService
implements MemoPatternMa
).when(join -> join.getHashJoinConjuncts().size() == 1)
);
}
+
+ @Test
+ void testNestedLambdaCapturesOuterColumn() {
+ List<String> sqls = ImmutableList.of(
+ "SELECT array_map(a -> array_sum(array_map(b -> if(flag, b,
0), arr2)), arr1) FROM t_arr",
+ "SELECT array_map(a -> array_sum(array_sortby(b -> if(flag, b,
0), arr2)), arr1) FROM t_arr",
+ "SELECT array_map(a -> array_map(b -> array_map(c -> if(flag,
b + c, id), arr2), arr2), arr1)"
+ + " FROM t_arr"
+ );
+ for (String sql : sqls) {
+ Lambda innermostLambda =
innermostLambda(PlanChecker.from(connectContext).analyze(sql).getPlan());
+ // getInputSlots() never contains the lambda argument slots
+ Set<String> capturedColumns =
innermostLambda.getLambdaFunction().getInputSlots().stream()
+ .map(Slot::getName)
+ .collect(Collectors.toSet());
+ Assertions.assertTrue(capturedColumns.contains("flag"), sql);
+ }
+ }
+
+ @Test
+ void testNestedLambdaSeesAllEnclosingLambdaArguments() {
+ String sql = "SELECT array_map(a -> array_map(b -> array_map(c ->
concat(a, b, c), arr1), arr1), arr1)"
+ + " FROM t_arr";
+ Lambda innermostLambda =
innermostLambda(PlanChecker.from(connectContext).analyze(sql).getPlan());
+ Set<ArrayItemSlot> lambdaArguments =
innermostLambda.getLambdaFunction()
+ .collect(ArrayItemSlot.class::isInstance);
+ Assertions.assertEquals(ImmutableList.of("a", "b", "c"),
+
lambdaArguments.stream().map(Slot::getName).sorted().collect(Collectors.toList()));
+ }
+
+ @Test
+ void testNestedLambdaArgumentShadowsEnclosingArgument() {
+ String sql = "SELECT array_map(x -> array_map(x -> x + 1, arr2), arr1)
FROM t_arr";
+ Lambda innermostLambda =
innermostLambda(PlanChecker.from(connectContext).analyze(sql).getPlan());
+ Set<ArrayItemSlot> bodySlots =
innermostLambda.getLambdaFunction().collect(ArrayItemSlot.class::isInstance);
+ Assertions.assertEquals(1, bodySlots.size());
+
Assertions.assertEquals(innermostLambda.getLambdaArgument(0).getExprId(),
+ bodySlots.iterator().next().getExprId());
+
Assertions.assertTrue(innermostLambda.getLambdaFunction().getInputSlots().isEmpty());
+ }
+
+ @Test
+ void testNestedLambdaUnknownColumn() {
+ String sql = "SELECT array_map(a -> array_map(b -> unknown_col + b,
arr2), arr1) FROM t_arr";
+ AnalysisException exception =
Assertions.assertThrows(AnalysisException.class,
+ () -> PlanChecker.from(connectContext).analyze(sql));
+ Assertions.assertTrue(exception.getMessage().contains("Unknown column
'unknown_col'"),
+ exception.getMessage());
+ }
+
+ private static Lambda innermostLambda(Plan plan) {
+ List<Lambda> innermostLambdas = plan.<Plan>collect(node ->
true).stream()
+ .flatMap(node -> node.getExpressions().stream())
+ .flatMap(expression ->
expression.<Lambda>collect(Lambda.class::isInstance).stream())
+ .filter(lambda ->
lambda.getLambdaFunction().<Expression>collect(Lambda.class::isInstance).isEmpty())
+ .collect(Collectors.toList());
+ Assertions.assertEquals(1, innermostLambdas.size());
+ return innermostLambdas.get(0);
+ }
}
diff --git
a/regression-test/data/query_p0/sql_functions/array_functions/test_nested_array_map.out
b/regression-test/data/query_p0/sql_functions/array_functions/test_nested_array_map.out
index 32404ea64dc..d8353e74ef7 100644
---
a/regression-test/data/query_p0/sql_functions/array_functions/test_nested_array_map.out
+++
b/regression-test/data/query_p0/sql_functions/array_functions/test_nested_array_map.out
@@ -11,3 +11,34 @@
-- !select_same_name_shadow --
[[2, 3], [4, 5]]
+
+-- !nested_capture_column --
+1 [6, 6]
+2 [0]
+3 [0, 0, 0]
+4 []
+
+-- !nested_capture_column_sortby --
+1 [[3, 2, 1], [3, 2, 1]]
+2 [[4, 5]]
+3 [[6], [6], [6]]
+4 []
+
+-- !nested_capture_column_and_outer_argument --
+1 [["A1y", "A2y", "A3y"], ["B1y", "B2y", "B3y"]]
+2 [["A4n", "A5n"]]
+3 [["A6n"], ["B6n"], ["C6n"]]
+4 []
+
+-- !nested_capture_three_levels --
+1 [[[2, 3, 4], [3, 4, 5], [4, 5, 6]], [[2, 3, 4], [3, 4, 5], [4, 5, 6]]]
+2 [[[2, 2], [2, 2]]]
+3 [[[3]], [[3]], [[3]]]
+4 []
+
+-- !nested_capture_cte_alias --
+1 2
+2 1
+3 3
+4 0
+
diff --git
a/regression-test/suites/query_p0/sql_functions/array_functions/test_nested_array_map.groovy
b/regression-test/suites/query_p0/sql_functions/array_functions/test_nested_array_map.groovy
index 7741ce47ff2..d66e3c53f4a 100644
---
a/regression-test/suites/query_p0/sql_functions/array_functions/test_nested_array_map.groovy
+++
b/regression-test/suites/query_p0/sql_functions/array_functions/test_nested_array_map.groovy
@@ -87,4 +87,60 @@ suite("test_nested_array_map") {
qt_select_same_name_shadow """
select array_map(x -> array_map(x -> x + 1, x), [[1, 2], [3, 4]]);
"""
+
+ // the nested lambda body captures the columns of the query and the
arguments of the enclosing lambda
+ sql "DROP TABLE IF EXISTS test_nested_array_map_capture"
+ sql """
+ CREATE TABLE test_nested_array_map_capture (
+ id INT,
+ flag BOOLEAN,
+ arr1 ARRAY<VARCHAR(10)>,
+ arr2 ARRAY<INT>
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ )
+ """
+ sql """
+ INSERT INTO test_nested_array_map_capture VALUES
+ (1, true, ['A', 'B'], [1, 2, 3]),
+ (2, false, ['A'], [4, 5]),
+ (3, NULL, ['A', 'B', 'C'], [6]),
+ (4, true, [], [7, 8])
+ """
+
+ order_qt_nested_capture_column """
+ select id, array_map(a -> array_sum(array_map(b -> if(flag, b, 0),
arr2)), arr1)
+ from test_nested_array_map_capture
+ """
+
+ order_qt_nested_capture_column_sortby """
+ select id, array_map(a -> array_sortby(b -> if(flag, -b, b), arr2),
arr1)
+ from test_nested_array_map_capture
+ """
+
+ order_qt_nested_capture_column_and_outer_argument """
+ select id, array_map(a -> array_map(b -> concat(a, b, if(flag, 'y',
'n')), arr2), arr1)
+ from test_nested_array_map_capture
+ """
+
+ order_qt_nested_capture_three_levels """
+ select id, array_map(a -> array_map(b -> array_map(c -> if(flag, b +
c, id), arr2), arr2), arr1)
+ from test_nested_array_map_capture
+ """
+
+ order_qt_nested_capture_cte_alias """
+ with t2 as (
+ select id, flag is null as dict_flag, arr1, arr2 from
test_nested_array_map_capture
+ )
+ select id, array_size(array_map(a -> array_sum(array_map(b ->
if(dict_flag, 1, 0), arr2)), arr1)) as l
+ from t2
+ """
+
+ test {
+ sql "select array_map(a -> array_map(b -> unknown_col + b, arr2),
arr1) from test_nested_array_map_capture"
+ exception "Unknown column 'unknown_col'"
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
