This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 7a21b136291 branch-4.1: [fix](variant) Keep integer element_at indexes
out of variant sub-column paths #67955 (#68041)
7a21b136291 is described below
commit 7a21b1362910e71cc8cd8eadcb4dfc0f64b186bf
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Sep 16 16:17:27 2026 +0800
branch-4.1: [fix](variant) Keep integer element_at indexes out of variant
sub-column paths #67955 (#68041)
---
.../rules/rewrite/VariantSubPathPruning.java | 10 ++-
.../rules/rewrite/PruneNestedColumnTest.java | 17 +++++
.../rules/rewrite/VariantPruningLogicTest.java | 4 +-
.../data/variant_p0/element_function.out | 34 ++++++++++
.../suites/variant_p0/element_function.groovy | 73 ++++++++++++++++++++++
5 files changed, 131 insertions(+), 7 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
index 5b081ce2c24..82f71b9db29 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
@@ -32,7 +32,6 @@ import
org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
-import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import
org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
@@ -790,14 +789,13 @@ public class VariantSubPathPruning implements
CustomRewriter {
if (!(elementAt.left() instanceof ElementAt || elementAt.left()
instanceof SlotReference)) {
return null;
}
+ // Storage sub-paths address object keys only. An integer index
selects an array element of the
+ // VARIANT value, so the sub-path stops before it and the index is
applied to the extracted value.
Expression key = elementAt.right();
- if (key instanceof StringLikeLiteral) {
- subPath.add(((StringLikeLiteral) key).getStringValue());
- } else if (key instanceof Literal &&
key.getDataType().isIntegerLikeType()) {
- subPath.add(((Literal) key).getStringValue());
- } else {
+ if (!(key instanceof StringLikeLiteral)) {
return null;
}
+ subPath.add(((StringLikeLiteral) key).getStringValue());
if (elementAt.left() instanceof SlotReference) {
// ElementAt's left child is SlotReference
// reverse subPath because we put them by reverse order
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
index 4d7a8a4488e..b94b21de7a0 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -186,6 +186,23 @@ public class PruneNestedColumnTest extends
TestWithFeService implements MemoPatt
));
}
+ @Test
+ public void testVariantIntegerIndexStopsSubColumnPath() throws Exception {
+ // DORIS-28435: an integer index selects an array element, so only the
object-key prefix becomes a storage
+ // sub-column; a sub-column items.1 would look up a missing object key
and return NULL.
+ assertVariantSubColumnSlots("select element_at(element_at(v, 'items'),
1), v['items'][-1] from variant_tbl",
+ ImmutableList.of(ImmutableList.of("items")));
+ }
+
+ @Test
+ public void testVariantIntegerIndexPredicateStopsSubColumnPath() throws
Exception {
+ // Filters stop at the integer index too, also when a string key
follows it; the object-key sub-column
+ // items.1, spelled like the index, stays a separate slot.
+ assertVariantSubColumnSlots("select id from variant_tbl"
+ + " where v['items'][1] = 2 and v['items'][-1]['k'] =
1 and v['items']['1'] = 'x'",
+ ImmutableList.of(ImmutableList.of("items"),
ImmutableList.of("items", "1")));
+ }
+
@Test
public void testVariantPredicateAccessPath() throws Exception {
assertColumn("select 1 from variant_tbl where v['k'] is not null",
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java
index 65a61f8a0ff..2b17d366bbd 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java
@@ -61,10 +61,12 @@ public class VariantPruningLogicTest extends
TestWithFeService {
@Test
public void testVariantNumericIndexSubPath() throws Exception {
+ // A numeric index selects an array element, not an object key, so the
storage sub-column stops at arr and
+ // [0]['x'] is applied to it at runtime; the access path still names
the whole expression.
assertVariantSubColumnSlots(
"select v['arr'][0]['x'] from variant_tbl",
ImmutableList.of(
- ImmutableList.of("arr", "0", "x")
+ ImmutableList.of("arr")
)
);
assertAllAccessPathsContain(
diff --git a/regression-test/data/variant_p0/element_function.out
b/regression-test/data/variant_p0/element_function.out
index 095c7b20356..007761a663b 100644
--- a/regression-test/data/variant_p0/element_function.out
+++ b/regression-test/data/variant_p0/element_function.out
@@ -2,3 +2,37 @@
-- !sql --
1
+-- !nested_integer_index --
+1 2 4 \N \N 3 \N
+2 [5,6] 7 \N \N s \N
+3 \N \N \N \N \N key one
+4 \N \N \N \N \N \N
+5 {"k":1} {"k":2} \N \N {"k":2} \N
+6 2 4 \N \N null \N
+
+-- !nested_integer_index_const --
+2 4
+
+-- !nested_integer_index_filter --
+1
+6
+
+-- !nested_integer_index_filter_negative --
+1 3
+6 null
+
+-- !nested_integer_index_filter_null --
+3
+4
+
+-- !nested_integer_index_filter_object --
+5
+
+-- !nested_integer_index_filter_with_key --
+1
+3
+6
+
+-- !nested_integer_index_filter_topn --
+1 [2,3,4]
+
diff --git a/regression-test/suites/variant_p0/element_function.groovy
b/regression-test/suites/variant_p0/element_function.groovy
index ed4cd1503ea..e0c65ec6a48 100644
--- a/regression-test/suites/variant_p0/element_function.groovy
+++ b/regression-test/suites/variant_p0/element_function.groovy
@@ -59,4 +59,77 @@ suite("regression_test_variant_element_at", "p0") {
def obj = sql """select sort_json_object_keys(json_extract(
cast(${variantV2Function}('{"o":{"name":"john"}}') as json),
'\$.o'))"""
assertEquals('{"name":"john"}', obj[0][0])
+
+ // The cases below need Variant V2. On the legacy V1 path element_at
resolves an integer index as an
+ // object key, so indexing an array stored in a VARIANT column returns
NULL there.
+ if (!getFeConfig("enable_variant_v2").toBoolean()) {
+ return
+ }
+
+ // DORIS-28435: an integer index on a VARIANT array that was itself
extracted with element_at is 1-based like
+ // ARRAY element_at and counts from the end when negative; 0, out-of-range
indexes, non-array values and
+ // missing paths give NULL, while a string index still reads object keys.
On a stored column the planner
+ // must not turn the integer index into the storage sub-path items.1.
+ sql "DROP TABLE IF EXISTS element_at_nested_index_test"
+ sql """
+ CREATE TABLE element_at_nested_index_test (
+ id INT,
+ json_variant VARIANT
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """INSERT INTO element_at_nested_index_test VALUES
+ (1, ${variantV2Function}('{"items": [2, 3, 4]}')),
+ (2, ${variantV2Function}('{"items": [[5, 6], "s", 7]}')),
+ (3, ${variantV2Function}('{"items": {"1": "key one"}}')),
+ (4, ${variantV2Function}('{"other": 1}')),
+ (5, ${variantV2Function}('{"items": [{"k": 1}, {"k": 2}]}')),
+ (6, ${variantV2Function}('{"items": [2, null, 4]}'))"""
+ order_qt_nested_integer_index """
+ SELECT id,
+ element_at(element_at(json_variant, 'items'), 1),
+ element_at(element_at(json_variant, 'items'), -1),
+ element_at(element_at(json_variant, 'items'), 0),
+ element_at(element_at(json_variant, 'items'), 4),
+ json_variant['items'][2],
+ element_at(element_at(json_variant, 'items'), '1')
+ FROM element_at_nested_index_test
+ """
+ qt_nested_integer_index_const """
+ SELECT element_at(element_at(${variantV2Function}('{"items": [2, 3,
4]}'), 'items'), 1),
+ element_at(element_at(${variantV2Function}('{"items": [2, 3,
4]}'), 'items'), -1)
+ """
+
+ // Filters read items and apply the index the same way: as a pushed-down
predicate, after a string key that
+ // follows the index, next to the object-key sub-column items.1 in one
filter, and under a TopN.
+ order_qt_nested_integer_index_filter """
+ SELECT id FROM element_at_nested_index_test
+ WHERE CAST(element_at(element_at(json_variant, 'items'), 1) AS INT) = 2
+ """
+ order_qt_nested_integer_index_filter_negative """
+ SELECT id, json_variant['items'][2] FROM element_at_nested_index_test
+ WHERE CAST(json_variant['items'][-1] AS INT) = 4
+ """
+ order_qt_nested_integer_index_filter_null """
+ SELECT id FROM element_at_nested_index_test
+ WHERE element_at(element_at(json_variant, 'items'), 1) IS NULL
+ """
+ order_qt_nested_integer_index_filter_object """
+ SELECT id FROM element_at_nested_index_test WHERE
CAST(json_variant['items'][1]['k'] AS INT) = 1
+ """
+ order_qt_nested_integer_index_filter_with_key """
+ SELECT id FROM element_at_nested_index_test
+ WHERE CAST(json_variant['items']['1'] AS STRING) = 'key one' OR
CAST(json_variant['items'][1] AS INT) = 2
+ """
+ qt_nested_integer_index_filter_topn """
+ SELECT id, CAST(json_variant['items'] AS STRING) FROM
element_at_nested_index_test
+ WHERE CAST(json_variant['items'][1] AS INT) = 2 ORDER BY id LIMIT 1
+ """
+ // MATCH needs a storage column, and a path with an integer index is not
one.
+ test {
+ sql "SELECT id FROM element_at_nested_index_test WHERE
json_variant['items'][2] MATCH_ANY 's'"
+ exception "Only support match left operand is SlotRef"
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
