This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 7ce01f07987 [fix](be) Keep lazy map child columns consistent (#65355)
(#69309)
7ce01f07987 is described below
commit 7ce01f0798773c16b5861303c7265bdce8dcd439
Author: Jerry Hu <[email protected]>
AuthorDate: Wed Oct 21 10:01:21 2026 +0800
[fix](be) Keep lazy map child columns consistent (#65355) (#69309)
### What problem does this PR solve?
Issue Number: None
Related PR: #65355
Problem Summary: This is the branch-4.2 backport of #65355. With nested
column pruning enabled, a map nested inside a struct could be lazily
materialized with only its keys or only its values after a sibling
struct field was used for filtering. The map placeholder is cleared
during the lazy phase, but the skipped key/value counterpart stayed
empty, so the ColumnMap offsets no longer matched both child column
sizes. On branch-4.2 this surfaces as a
`ColumnVector<T>::insert_range_from` out-of-range error. This change
fills only the truly skipped counterparts with defaults during lazy map
materialization, keeps the child data already read in the predicate
phase, and still avoids reading pruned data pages.
### Release note
None
### Check List (For Author)
- Test: Regression test and unit test
- Unit Test:
ColumnReaderTest.MapLazyReadByRowidsFillsSkippedKeysForValuesOnlyPath
and
ColumnReaderTest.MapLazyReadByRowidsPreservesPredicateKeysForValuesOnlyPath
- Regression test:
regression-test/suites/nereids_rules_p0/column_pruning/struct_map_pruning_lazy_read.groovy
- Behavior changed: No
- Does this need documentation: No
### Test results
- BE unit tests (ASAN build, `-j 64`): `./run-be-ut.sh --run
--filter=ColumnReaderTest.MapLazyReadByRowidsFillsSkippedKeysForValuesOnlyPath:ColumnReaderTest.MapLazyReadByRowidsPreservesPredicateKeysForValuesOnlyPath`
Result: PASSED 2 tests. The same filter FAILS 2 tests on the unfixed
source (skipped key column size 0 vs 1; predicate key re-seeked),
confirming both tests cover the bug.
- Full build (BE + FE, Release): `DISABLE_BE_JAVA_EXTENSIONS=OFF
doris-local-regression build`
Result: "Successfully build Doris".
- Regression test: `doris-local-regression run -d
nereids_rules_p0/column_pruning -s struct_map_pruning_lazy_read`
Result: "All suites success", 1 suite passed / 0 failed. The
expected-output file was generated with `-forceGenOut` and then verified
again without it.
- Formatting: `build-support/check-format.sh` (clang-format 16) reports
no file to format; `git diff --check` is clean.
---
be/src/storage/segment/column_reader.cpp | 84 +++++++++-----
be/test/storage/segment/column_reader_test.cpp | 127 +++++++++++++++++++++
.../struct_map_pruning_lazy_read.out | 7 ++
.../struct_map_pruning_lazy_read.groovy | 55 +++++++++
4 files changed, 242 insertions(+), 31 deletions(-)
diff --git a/be/src/storage/segment/column_reader.cpp
b/be/src/storage/segment/column_reader.cpp
index 330b75c307a..73a2b52a5cc 100644
--- a/be/src/storage/segment/column_reader.cpp
+++ b/be/src/storage/segment/column_reader.cpp
@@ -1493,12 +1493,22 @@ Status MapFileColumnIterator::next_batch(size_t* n,
MutableColumnPtr& dst, bool*
key_ptr->insert_many_defaults(num_items);
val_ptr->insert_many_defaults(num_items);
} else {
- size_t num_read = num_items;
- bool key_has_null = false;
- bool val_has_null = false;
- RETURN_IF_ERROR(_key_iterator->next_batch(&num_read, key_ptr,
&key_has_null));
- RETURN_IF_ERROR(_val_iterator->next_batch(&num_read, val_ptr,
&val_has_null));
- DCHECK(num_read == num_items);
+ auto read_or_fill_child = [&](ColumnIterator* iterator,
+ MutableColumnPtr& column) -> Status {
+ if (_read_phase == ReadPhase::LAZY && read_meta_columns &&
+ iterator->read_requirement() == ReadRequirement::SKIP) {
+ column->insert_many_defaults(num_items);
+ return Status::OK();
+ }
+
+ bool dummy_has_null = false;
+ size_t num_read = num_items;
+ RETURN_IF_ERROR(iterator->next_batch(&num_read, column,
&dummy_has_null));
+ DCHECK(num_read == num_items);
+ return Status::OK();
+ };
+ RETURN_IF_ERROR(read_or_fill_child(_key_iterator.get(), key_ptr));
+ RETURN_IF_ERROR(read_or_fill_child(_val_iterator.get(), val_ptr));
}
}
@@ -1692,6 +1702,35 @@ Status MapFileColumnIterator::read_by_rowids(const
rowid_t* rowids, const size_t
auto vals_ptr = IColumn::mutate(std::move(column_map.get_values_ptr()));
Defer defer_keys {[&] { column_map.get_keys_ptr() = std::move(keys_ptr);
}};
Defer defer_values {[&] { column_map.get_values_ptr() =
std::move(vals_ptr); }};
+ // In lazy phase with read_meta_columns=true, this map was only a
placeholder during
+ // predicate evaluation and is cleared before the lazy read. If only KEYS
or VALUES is
+ // requested, fill the skipped counterpart with defaults to keep
ColumnMap's
+ // key/value/offset sizes consistent without reading unnecessary data
pages.
+ const bool fill_lazy_skipped_keys = _read_phase == ReadPhase::LAZY &&
read_meta_columns &&
+ _key_iterator->read_requirement() ==
ReadRequirement::SKIP;
+ const bool fill_lazy_skipped_values =
+ _read_phase == ReadPhase::LAZY && read_meta_columns &&
+ _val_iterator->read_requirement() == ReadRequirement::SKIP;
+ auto read_or_fill_range = [&](ColumnIterator* iterator, MutableColumnPtr&
column,
+ ordinal_t start_idx, size_t num_items,
+ bool fill_lazy_skipped_child) -> Status {
+ if (num_items == 0) {
+ return Status::OK();
+ }
+ if (fill_lazy_skipped_child) {
+ column->insert_many_defaults(num_items);
+ return Status::OK();
+ }
+ if (_read_phase == ReadPhase::LAZY && !iterator->need_to_read()) {
+ return Status::OK();
+ }
+ size_t n = num_items;
+ bool dummy_has_null = false;
+ RETURN_IF_ERROR(iterator->seek_to_ordinal(start_idx));
+ RETURN_IF_ERROR(iterator->next_batch(&n, column, &dummy_has_null));
+ DCHECK(n == num_items);
+ return Status::OK();
+ };
size_t this_run = sizes[0];
auto start_idx = starts_data[0];
@@ -1703,19 +1742,10 @@ Status MapFileColumnIterator::read_by_rowids(const
rowid_t* rowids, const size_t
}
auto start = static_cast<ordinal_t>(starts_data[i]);
if (start != last_idx) {
- size_t n = this_run;
- bool dummy_has_null = false;
-
- if (this_run != 0) {
- RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx));
- RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr,
&dummy_has_null));
- DCHECK(n == this_run);
-
- n = this_run;
- RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx));
- RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr,
&dummy_has_null));
- DCHECK(n == this_run);
- }
+ RETURN_IF_ERROR(read_or_fill_range(_key_iterator.get(), keys_ptr,
start_idx, this_run,
+ fill_lazy_skipped_keys));
+ RETURN_IF_ERROR(read_or_fill_range(_val_iterator.get(), vals_ptr,
start_idx, this_run,
+ fill_lazy_skipped_values));
start_idx = start;
this_run = sz;
last_idx = start + sz;
@@ -1726,18 +1756,10 @@ Status MapFileColumnIterator::read_by_rowids(const
rowid_t* rowids, const size_t
last_idx += sz;
}
- size_t n = this_run;
- bool dummy_has_null = false;
- if (this_run != 0) {
- RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx));
- RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr,
&dummy_has_null));
- DCHECK(n == this_run);
-
- n = this_run;
- RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx));
- RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr,
&dummy_has_null));
- DCHECK(n == this_run);
- }
+ RETURN_IF_ERROR(read_or_fill_range(_key_iterator.get(), keys_ptr,
start_idx, this_run,
+ fill_lazy_skipped_keys));
+ RETURN_IF_ERROR(read_or_fill_range(_val_iterator.get(), vals_ptr,
start_idx, this_run,
+ fill_lazy_skipped_values));
return Status::OK();
}
diff --git a/be/test/storage/segment/column_reader_test.cpp
b/be/test/storage/segment/column_reader_test.cpp
index d19158e1a06..6fc281ee59f 100644
--- a/be/test/storage/segment/column_reader_test.cpp
+++ b/be/test/storage/segment/column_reader_test.cpp
@@ -268,6 +268,42 @@ private:
ordinal_t _current_ordinal = 0;
};
+class RowidOffsetFileColumnIterator final : public FileColumnIterator {
+public:
+ RowidOffsetFileColumnIterator() :
FileColumnIterator(create_test_reader(false, 10)) {}
+
+ Status seek_to_ordinal(ordinal_t ord) override {
+ _current_ordinal = ord;
+ return Status::OK();
+ }
+
+ Status next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null)
override {
+ auto& offsets = assert_cast<ColumnOffset64&,
TypeCheckOnRelease::DISABLE>(*dst);
+ for (size_t i = 0; i < *n; ++i) {
+ offsets.insert_value(_current_ordinal + i);
+ }
+ _current_ordinal += *n;
+ if (has_null != nullptr) {
+ *has_null = false;
+ }
+ return Status::OK();
+ }
+
+ Status read_by_rowids(const rowid_t* rowids, const size_t count,
+ MutableColumnPtr& dst) override {
+ auto& offsets = assert_cast<ColumnOffset64&,
TypeCheckOnRelease::DISABLE>(*dst);
+ for (size_t i = 0; i < count; ++i) {
+ offsets.insert_value(rowids[i]);
+ }
+ return Status::OK();
+ }
+
+ ordinal_t get_current_ordinal() const override { return _current_ordinal; }
+
+private:
+ ordinal_t _current_ordinal = 0;
+};
+
class NullMapOnlyFileColumnIterator final : public FileColumnIterator {
public:
explicit NullMapOnlyFileColumnIterator(std::shared_ptr<ColumnReader>
reader)
@@ -3061,6 +3097,97 @@ TEST_F(ColumnReaderTest,
MapReadByRowidsSkipReadingAppendsPlaceholders) {
ASSERT_TRUE(st.ok()) << "second read_by_rowids failed: " << st.to_string();
ASSERT_EQ(2 * count, dst->size());
}
+
+TEST_F(ColumnReaderTest, MapLazyReadByRowidsFillsSkippedKeysForValuesOnlyPath)
{
+ auto map_reader = create_test_reader(false, 10);
+ auto offsets_iter = std::make_unique<OffsetFileColumnIterator>(
+ std::make_unique<RowidOffsetFileColumnIterator>());
+ auto key_iter = std::make_unique<TrackingColumnIterator>();
+ key_iter->set_column_name("key");
+ auto* key_ptr = key_iter.get();
+ auto val_iter = std::make_unique<TrackingColumnIterator>();
+ val_iter->set_column_name("value");
+ auto* val_ptr = val_iter.get();
+
+ MapFileColumnIterator map_iter(map_reader, nullptr,
std::move(offsets_iter),
+ std::move(key_iter), std::move(val_iter));
+ map_iter.set_column_name("map_col");
+
+ TColumnAccessPaths all_access_paths {
+ create_data_access_path({"map_col",
ColumnIterator::ACCESS_MAP_VALUES})};
+ auto st = map_iter.set_access_paths(all_access_paths, {});
+ ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string();
+ ASSERT_EQ(key_ptr->read_requirement(),
ColumnIterator::ReadRequirement::SKIP);
+ ASSERT_EQ(val_ptr->read_requirement(),
ColumnIterator::ReadRequirement::LAZY_OUTPUT);
+
+ // Simulate the top-level struct reader marking this map as a lazy output
branch.
+
map_iter.set_read_requirement_self(ColumnIterator::ReadRequirement::LAZY_OUTPUT);
+ map_iter.set_read_phase(ColumnIterator::ReadPhase::LAZY);
+
+ MutableColumnPtr dst = ColumnMap::create(ColumnInt32::create(),
ColumnInt32::create(),
+
ColumnArray::ColumnOffsets::create());
+ const rowid_t rowids[] = {1};
+ st = map_iter.read_by_rowids(rowids, std::size(rowids), dst);
+ ASSERT_TRUE(st.ok()) << "lazy map read_by_rowids failed: " <<
st.to_string();
+
+ const auto& column_map = assert_cast<const ColumnMap&,
TypeCheckOnRelease::DISABLE>(*dst);
+ EXPECT_EQ(1, dst->size());
+ EXPECT_EQ(1, column_map.get_keys().size());
+ EXPECT_EQ(1, column_map.get_values().size());
+ EXPECT_TRUE(key_ptr->next_batch_sizes.empty());
+ EXPECT_THAT(val_ptr->seek_ordinals, ::testing::ElementsAre(1));
+ EXPECT_THAT(val_ptr->next_batch_sizes, ::testing::ElementsAre(1));
+}
+
+TEST_F(ColumnReaderTest,
MapLazyReadByRowidsPreservesPredicateKeysForValuesOnlyPath) {
+ auto map_reader = create_test_reader(false, 10);
+ auto offsets_iter = std::make_unique<OffsetFileColumnIterator>(
+ std::make_unique<RowidOffsetFileColumnIterator>());
+ auto key_iter = std::make_unique<TrackingColumnIterator>();
+ key_iter->set_column_name("key");
+ auto* key_ptr = key_iter.get();
+ auto val_iter = std::make_unique<TrackingColumnIterator>();
+ val_iter->set_column_name("value");
+ auto* val_ptr = val_iter.get();
+
+ MapFileColumnIterator map_iter(map_reader, nullptr,
std::move(offsets_iter),
+ std::move(key_iter), std::move(val_iter));
+ map_iter.set_column_name("map_col");
+
+ TColumnAccessPaths all_access_paths {
+ create_data_access_path({"map_col",
ColumnIterator::ACCESS_MAP_VALUES})};
+ TColumnAccessPaths predicate_access_paths {
+ create_data_access_path({"map_col",
ColumnIterator::ACCESS_MAP_KEYS})};
+ auto st = map_iter.set_access_paths(all_access_paths,
predicate_access_paths);
+ ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string();
+ ASSERT_EQ(key_ptr->read_requirement(),
ColumnIterator::ReadRequirement::PREDICATE);
+ ASSERT_EQ(val_ptr->read_requirement(),
ColumnIterator::ReadRequirement::LAZY_OUTPUT);
+
+ map_iter.set_read_phase(ColumnIterator::ReadPhase::LAZY);
+ ASSERT_TRUE(map_iter.need_to_read());
+ ASSERT_FALSE(map_iter.need_to_read_meta_columns());
+
+ MutableColumnPtr dst = ColumnMap::create(ColumnInt32::create(),
ColumnInt32::create(),
+
ColumnArray::ColumnOffsets::create());
+ auto& column_map = assert_cast<ColumnMap&,
TypeCheckOnRelease::DISABLE>(*dst);
+ auto keys_ptr = IColumn::mutate(std::move(column_map.get_keys_ptr()));
+ keys_ptr->insert_many_defaults(1);
+ column_map.get_keys_ptr() = std::move(keys_ptr);
+ column_map.get_offsets().push_back(1);
+
+ const rowid_t rowids[] = {1};
+ st = map_iter.read_by_rowids(rowids, std::size(rowids), dst);
+ ASSERT_TRUE(st.ok()) << "lazy map read_by_rowids failed: " <<
st.to_string();
+
+ EXPECT_EQ(1, dst->size());
+ EXPECT_EQ(1, column_map.get_keys().size());
+ EXPECT_EQ(1, column_map.get_values().size());
+ EXPECT_TRUE(key_ptr->seek_ordinals.empty());
+ EXPECT_TRUE(key_ptr->next_batch_sizes.empty());
+ EXPECT_THAT(val_ptr->seek_ordinals, ::testing::ElementsAre(1));
+ EXPECT_THAT(val_ptr->next_batch_sizes, ::testing::ElementsAre(1));
+}
+
TEST_F(ColumnReaderTest, MapAccessAllWithOffsetDoesNotPropagateOffsetToKey) {
// Regression test: when the access path is [map_col, *, OFFSET]
// (e.g. length(map_col['some_key'])), the key column must be fully read
diff --git
a/regression-test/data/nereids_rules_p0/column_pruning/struct_map_pruning_lazy_read.out
b/regression-test/data/nereids_rules_p0/column_pruning/struct_map_pruning_lazy_read.out
new file mode 100644
index 00000000000..255b0ba3aea
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/column_pruning/struct_map_pruning_lazy_read.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !map_values --
+2 ["val3"]
+
+-- !map_keys --
+2 [3]
+
diff --git
a/regression-test/suites/nereids_rules_p0/column_pruning/struct_map_pruning_lazy_read.groovy
b/regression-test/suites/nereids_rules_p0/column_pruning/struct_map_pruning_lazy_read.groovy
new file mode 100644
index 00000000000..e92cd0388d6
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/column_pruning/struct_map_pruning_lazy_read.groovy
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("struct_map_pruning_lazy_read") {
+ sql "set enable_prune_nested_column = true"
+ sql "DROP TABLE IF EXISTS struct_map_pruning_lazy_read_tbl"
+ sql """
+ CREATE TABLE struct_map_pruning_lazy_read_tbl (
+ id INT,
+ heavy_struct STRUCT<
+ small_int: INT,
+ big_string: VARCHAR(2048),
+ nested_map: MAP<INT, VARCHAR(100)>
+ >,
+ standalone_map MAP<VARCHAR(50), INT>
+ )
+ UNIQUE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+ sql """
+ INSERT INTO struct_map_pruning_lazy_read_tbl VALUES
+ (1, struct(100, repeat('A', 2048), map(1, 'val1', 2, 'val2')),
map('key_x', 1, 'key_y', 2)),
+ (2, struct(200, repeat('B', 2048), map(3, 'val3')), map('key_z', 3)),
+ (3, struct(300, repeat('C', 2048), map()), map('empty', 0))
+ """
+
+ order_qt_map_values """
+ SELECT id, map_values(struct_element(heavy_struct, 'nested_map'))
+ FROM struct_map_pruning_lazy_read_tbl
+ WHERE struct_element(heavy_struct, 'big_string') LIKE 'B%'
+ ORDER BY id
+ """
+
+ order_qt_map_keys """
+ SELECT id, map_keys(struct_element(heavy_struct, 'nested_map'))
+ FROM struct_map_pruning_lazy_read_tbl
+ WHERE struct_element(heavy_struct, 'small_int') = 200
+ ORDER BY id
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
