This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new f6702049866 branch-4.2: [fix](nereids) Match nested access path roots
case-insensitively (#68360)
f6702049866 is described below
commit f6702049866b35b86d86611b8c2d7b0e602e09c4
Author: daidai <[email protected]>
AuthorDate: Wed Sep 23 08:51:29 2026 +0800
branch-4.2: [fix](nereids) Match nested access path roots
case-insensitively (#68360)
### What problem does this PR solve?
Issue Number: None
Problem Summary:
Nested access paths are collected with a lower-cased column name, while
a whole-column access path is rebuilt from the name the catalog stores.
When a column name is not all lower case, the prefix check that decides
whether the all access paths already cover a predicate access path
compares the two spellings case-sensitively and reports "not covered",
so the sub-field path is added next to the whole-column path:
```sql
CREATE TABLE t (id INT, S STRUCT<City: STRING, Zip: INT>);
SELECT S FROM t WHERE struct_element(S, 'City') = 'x';
-- all access paths: [S], [s.city]
```
BE does not accept both for the same slot.
`ColumnIterator::_get_sub_access_paths` consumes the whole-column path
as the "read this column" marker and removes it from the list, so the
remaining `[city]` is treated as the only sub-column to read and
`StructFileColumnIterator::set_access_paths` marks the siblings
`SKIP_READING`. The projected column then comes back without those
fields, and no error is reported:
| | before | after |
| --- | --- | --- |
| all access paths | `[S]`, `[s.city]` | `[S]` |
| `SELECT S ... WHERE struct_element(S, 'City') = 'x'` | `Zip` is empty
| full struct |
Compare the path components case-insensitively, which is what BE does
when it matches the root (`StringCaseEqual`) and the struct fields
(`to_lower`). A column name that is all lower case was never affected.
Only OLAP scans lose data. External and TVF file scans read the whole
column either way, because `AccessPathParser` marks a one-component path
as `project_all` and then ignores the finer paths.
### Release note
None
### Check List (For Author)
- Test
- [x] Regression test
- [x] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason
- Behavior changed:
- [ ] No.
- [x] Yes. A sub-field predicate path is no longer added next to the
whole-column path.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../nereids/rules/rewrite/NestedColumnPruning.java | 20 +++++++-
.../rules/rewrite/PruneNestedColumnTest.java | 22 ++++++++
.../upper_case_nested_column_pruning.groovy | 59 ++++++++++++++++++++++
3 files changed, 100 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
index d60686c2ec6..b45cbc62f0f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
@@ -897,13 +897,31 @@ public class NestedColumnPruning implements
CustomRewriter {
TColumnAccessPath predicatePath, List<TColumnAccessPath> allPaths)
{
for (TColumnAccessPath allPath : allPaths) {
if (allPath.getType() == predicatePath.getType()
- && pathCoversPrefix(getAccessPathList(allPath),
getAccessPathList(predicatePath))) {
+ && coversPrefixIgnoreCase(getAccessPathList(allPath),
getAccessPathList(predicatePath))) {
return true;
}
}
return false;
}
+ /**
+ * Whether {@code path} is a prefix of {@code longerPath}, ignoring case.
The collected paths
+ * lower-case every component, but a whole-column path is rebuilt from the
catalog column name,
+ * so {@code [S]} still has to cover {@code [s, city]}. A case-sensitive
comparison would append
+ * {@code [s, city]} to the all paths, and BE then reads that field only
and skips its siblings.
+ */
+ private static boolean coversPrefixIgnoreCase(List<String> path,
List<String> longerPath) {
+ if (path.size() > longerPath.size()) {
+ return false;
+ }
+ for (int i = 0; i < path.size(); i++) {
+ if (!path.get(i).equalsIgnoreCase(longerPath.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
private static boolean hasStrictPrefix(List<String> path, List<String>
prefix) {
return path.size() > prefix.size() && path.subList(0,
prefix.size()).equals(prefix);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
index f3aa4bf42de..297d906c3b5 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -89,6 +89,11 @@ public class PruneNestedColumnTest extends TestWithFeService
implements MemoPatt
+ ">)\n"
+ "properties ('replication_num'='1')");
+ createTable("create table upper_case_tbl(\n"
+ + " id int,\n"
+ + " S struct<City: string, Zip: int>)\n"
+ + "properties ('replication_num'='1')");
+
createTable("create table tbl2(\n"
+ " id2 int,\n"
+ " value int,\n"
@@ -287,6 +292,23 @@ public class PruneNestedColumnTest extends
TestWithFeService implements MemoPatt
Assertions.assertEquals(ImmutableList.of(path("s", "city")),
ImmutableList.copyOf(predicateAccessPaths));
}
+ @Test
+ public void testWholeUpperCaseColumnOutputKeepsSubFieldPredicatePath()
throws Exception {
+ // The collected paths are lower-cased, the whole-column path keeps
the catalog name. The
+ // predicate path must still count as covered, otherwise it is added
to the all paths and
+ // BE reads that one field and skips its siblings.
+ Pair<PhysicalPlan, List<SlotDescriptor>> result = collectComplexSlots(
+ "select S from upper_case_tbl where struct_element(S, 'City')
= 'x'");
+ TreeSet<TColumnAccessPath> allAccessPaths = new TreeSet<>();
+ TreeSet<TColumnAccessPath> predicateAccessPaths = new TreeSet<>();
+ for (SlotDescriptor slotDescriptor : result.second) {
+ allAccessPaths.addAll(slotDescriptor.getAllAccessPaths());
+
predicateAccessPaths.addAll(slotDescriptor.getPredicateAccessPaths());
+ }
+ Assertions.assertEquals(ImmutableList.of(path("S")),
ImmutableList.copyOf(allAccessPaths));
+ Assertions.assertEquals(ImmutableList.of(path("s", "city")),
ImmutableList.copyOf(predicateAccessPaths));
+ }
+
@Test
public void testVariantAccessPath() throws Exception {
assertColumn("select v['a']['B'] from variant_tbl",
diff --git
a/regression-test/suites/nereids_rules_p0/column_pruning/upper_case_nested_column_pruning.groovy
b/regression-test/suites/nereids_rules_p0/column_pruning/upper_case_nested_column_pruning.groovy
new file mode 100644
index 00000000000..8c27cc6f3ae
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/column_pruning/upper_case_nested_column_pruning.groovy
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Access paths are collected with a lower-cased column name, while a
whole-column access path
+// keeps the name as the catalog stores it. When a column name is not all
lower case, the two
+// spellings must still be recognized as the same column, otherwise the
sub-field path of the
+// predicate is added next to the whole-column path and the BE reads that
field only, leaving the
+// sibling fields of the projected column empty.
+
+suite("upper_case_nested_column_pruning") {
+ sql """ SET enable_prune_nested_column = true """
+ sql """ DROP TABLE IF EXISTS ucnp_tbl """
+ sql """
+ CREATE TABLE ucnp_tbl (
+ id INT,
+ S STRUCT<City: STRING, Zip: INT> NULL,
+ M MAP<STRING, INT> NULL
+ ) ENGINE = OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+
+ sql """
+ INSERT INTO ucnp_tbl VALUES
+ (1, named_struct('city', 'beijing', 'zip', 10001), {'a': 1, 'b':
2})
+ """
+
+ // The whole struct is projected and the predicate reads one field of it.
+ def structRows = sql """ SELECT S FROM ucnp_tbl WHERE struct_element(S,
'City') = 'beijing' """
+ assertEquals(1, structRows.size())
+ assertTrue(structRows[0][0].toString().contains("beijing"),
+ "projected struct lost City: ${structRows[0][0]}")
+ assertTrue(structRows[0][0].toString().contains("10001"),
+ "projected struct lost the sibling field Zip: ${structRows[0][0]}")
+
+ // Same shape on a map column: every key and value must survive a
predicate on the values.
+ def mapRows = sql """ SELECT M FROM ucnp_tbl WHERE
array_contains(map_values(M), 1) """
+ assertEquals(1, mapRows.size())
+ def mapValue = mapRows[0][0].toString()
+ for (String entry : ["a", "b", "1", "2"]) {
+ assertTrue(mapValue.contains(entry),
+ "projected map lost ${entry}: ${mapValue}")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
