This is an automated email from the ASF dual-hosted git repository.
mrhhsg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 36bc996f449 [fix](be) Keep lazy map child columns consistent (#65355)
36bc996f449 is described below
commit 36bc996f449f82d76c71489d9e9ff547d53d6516
Author: Jerry Hu <[email protected]>
AuthorDate: Tue Jul 14 10:53:59 2026 +0800
[fix](be) Keep lazy map child columns consistent (#65355)
### What problem does this PR solve?
Issue Number: None
Problem Summary: Nested column pruning can lazily read only map keys or
only map values after a sibling struct field is used for filtering. The
map placeholder is cleared in the lazy phase, but the skipped key/value
counterpart was left empty, so ColumnMap offsets no longer matched both
child column sizes. This PR fills only truly skipped counterparts with
defaults during lazy map materialization while preserving
predicate-phase child data and still avoiding unnecessary child data
reads.
### Release note
None
### Check List (For Author)
- Test:
- Unit Test: ./run-be-ut.sh --run
--filter=ColumnReaderTest.MapLazyReadByRowidsFillsSkippedKeysForValuesOnlyPath
- Unit Test: ./run-be-ut.sh --run
--filter=ColumnReaderTest.MapLazyReadByRowidsFillsSkippedKeysForValuesOnlyPath:ColumnReaderTest.MapLazyReadByRowidsPreservesPredicateKeysForValuesOnlyPath
- Regression test: doris-local-regression --network 10.26.20.3/24 all -d
nereids_rules_p0/column_pruning -s struct_map_pruning_lazy_read
-forceGenOut
- Regression test: doris-local-regression --network 10.26.20.3/24 all -d
nereids_rules_p0/column_pruning -s struct_map_pruning_lazy_read
- Format: build-support/clang-format.sh
be/src/storage/segment/column_reader.cpp
be/test/storage/segment/column_reader_test.cpp
- Static Check: git diff --check
- Static Check: build-support/check-format.sh
- Static Check: build-support/run-clang-tidy.sh --build-dir
be/ut_build_ASAN --base HEAD (failed on existing unrelated clang-tidy
analysis errors in included/current files)
- Behavior changed: No
- Does this need documentation: No
---
be/src/storage/segment/column_reader.cpp | 84 +++++++++-----
be/test/storage/segment/column_reader_test.cpp | 126 +++++++++++++++++++++
.../struct_map_pruning_lazy_read.out | 6 +
.../struct_map_pruning_lazy_read.groovy | 55 +++++++++
4 files changed, 240 insertions(+), 31 deletions(-)
diff --git a/be/src/storage/segment/column_reader.cpp
b/be/src/storage/segment/column_reader.cpp
index 2021793be20..25886d7515d 100644
--- a/be/src/storage/segment/column_reader.cpp
+++ b/be/src/storage/segment/column_reader.cpp
@@ -1171,12 +1171,22 @@ Status MapFileColumnIterator::next_batch(size_t* n,
MutableColumnPtr& dst, bool*
key_ptr->insert_many_defaults(num_items);
val_ptr->insert_many_defaults(num_items);
} else {
- size_t num_read = num_items;
- bool key_has_null = false;
- bool val_has_null = false;
- RETURN_IF_ERROR(_key_iterator->next_batch(&num_read, key_ptr,
&key_has_null));
- RETURN_IF_ERROR(_val_iterator->next_batch(&num_read, val_ptr,
&val_has_null));
- DCHECK(num_read == num_items);
+ auto read_or_fill_child = [&](ColumnIterator* iterator,
+ MutableColumnPtr& column) -> Status {
+ if (_read_phase == ReadPhase::LAZY && read_meta_columns &&
+ iterator->read_requirement() == ReadRequirement::SKIP) {
+ column->insert_many_defaults(num_items);
+ return Status::OK();
+ }
+
+ bool dummy_has_null = false;
+ size_t num_read = num_items;
+ RETURN_IF_ERROR(iterator->next_batch(&num_read, column,
&dummy_has_null));
+ DCHECK(num_read == num_items);
+ return Status::OK();
+ };
+ RETURN_IF_ERROR(read_or_fill_child(_key_iterator.get(), key_ptr));
+ RETURN_IF_ERROR(read_or_fill_child(_val_iterator.get(), val_ptr));
}
}
@@ -1366,6 +1376,35 @@ Status MapFileColumnIterator::read_by_rowids(const
rowid_t* rowids, const size_t
auto vals_ptr = IColumn::mutate(std::move(column_map.get_values_ptr()));
Defer defer_keys {[&] { column_map.get_keys_ptr() = std::move(keys_ptr);
}};
Defer defer_values {[&] { column_map.get_values_ptr() =
std::move(vals_ptr); }};
+ // In lazy phase with read_meta_columns=true, this map was only a
placeholder during
+ // predicate evaluation and is cleared before the lazy read. If only KEYS
or VALUES is
+ // requested, fill the skipped counterpart with defaults to keep
ColumnMap's
+ // key/value/offset sizes consistent without reading unnecessary data
pages.
+ const bool fill_lazy_skipped_keys = _read_phase == ReadPhase::LAZY &&
read_meta_columns &&
+ _key_iterator->read_requirement() ==
ReadRequirement::SKIP;
+ const bool fill_lazy_skipped_values =
+ _read_phase == ReadPhase::LAZY && read_meta_columns &&
+ _val_iterator->read_requirement() == ReadRequirement::SKIP;
+ auto read_or_fill_range = [&](ColumnIterator* iterator, MutableColumnPtr&
column,
+ ordinal_t start_idx, size_t num_items,
+ bool fill_lazy_skipped_child) -> Status {
+ if (num_items == 0) {
+ return Status::OK();
+ }
+ if (fill_lazy_skipped_child) {
+ column->insert_many_defaults(num_items);
+ return Status::OK();
+ }
+ if (_read_phase == ReadPhase::LAZY && !iterator->need_to_read()) {
+ return Status::OK();
+ }
+ size_t n = num_items;
+ bool dummy_has_null = false;
+ RETURN_IF_ERROR(iterator->seek_to_ordinal(start_idx));
+ RETURN_IF_ERROR(iterator->next_batch(&n, column, &dummy_has_null));
+ DCHECK(n == num_items);
+ return Status::OK();
+ };
size_t this_run = sizes[0];
auto start_idx = starts_data[0];
@@ -1377,19 +1416,10 @@ Status MapFileColumnIterator::read_by_rowids(const
rowid_t* rowids, const size_t
}
auto start = static_cast<ordinal_t>(starts_data[i]);
if (start != last_idx) {
- size_t n = this_run;
- bool dummy_has_null = false;
-
- if (this_run != 0) {
- RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx));
- RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr,
&dummy_has_null));
- DCHECK(n == this_run);
-
- n = this_run;
- RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx));
- RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr,
&dummy_has_null));
- DCHECK(n == this_run);
- }
+ RETURN_IF_ERROR(read_or_fill_range(_key_iterator.get(), keys_ptr,
start_idx, this_run,
+ fill_lazy_skipped_keys));
+ RETURN_IF_ERROR(read_or_fill_range(_val_iterator.get(), vals_ptr,
start_idx, this_run,
+ fill_lazy_skipped_values));
start_idx = start;
this_run = sz;
last_idx = start + sz;
@@ -1400,18 +1430,10 @@ Status MapFileColumnIterator::read_by_rowids(const
rowid_t* rowids, const size_t
last_idx += sz;
}
- size_t n = this_run;
- bool dummy_has_null = false;
- if (this_run != 0) {
- RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx));
- RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr,
&dummy_has_null));
- DCHECK(n == this_run);
-
- n = this_run;
- RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx));
- RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr,
&dummy_has_null));
- DCHECK(n == this_run);
- }
+ RETURN_IF_ERROR(read_or_fill_range(_key_iterator.get(), keys_ptr,
start_idx, this_run,
+ fill_lazy_skipped_keys));
+ RETURN_IF_ERROR(read_or_fill_range(_val_iterator.get(), vals_ptr,
start_idx, this_run,
+ fill_lazy_skipped_values));
return Status::OK();
}
diff --git a/be/test/storage/segment/column_reader_test.cpp
b/be/test/storage/segment/column_reader_test.cpp
index 3ee511a5980..8dfe640808e 100644
--- a/be/test/storage/segment/column_reader_test.cpp
+++ b/be/test/storage/segment/column_reader_test.cpp
@@ -233,6 +233,42 @@ private:
ordinal_t _current_ordinal = 0;
};
+class RowidOffsetFileColumnIterator final : public FileColumnIterator {
+public:
+ RowidOffsetFileColumnIterator() :
FileColumnIterator(create_test_reader(false, 10)) {}
+
+ Status seek_to_ordinal(ordinal_t ord) override {
+ _current_ordinal = ord;
+ return Status::OK();
+ }
+
+ Status next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null)
override {
+ auto& offsets = assert_cast<ColumnOffset64&,
TypeCheckOnRelease::DISABLE>(*dst);
+ for (size_t i = 0; i < *n; ++i) {
+ offsets.insert_value(_current_ordinal + i);
+ }
+ _current_ordinal += *n;
+ if (has_null != nullptr) {
+ *has_null = false;
+ }
+ return Status::OK();
+ }
+
+ Status read_by_rowids(const rowid_t* rowids, const size_t count,
+ MutableColumnPtr& dst) override {
+ auto& offsets = assert_cast<ColumnOffset64&,
TypeCheckOnRelease::DISABLE>(*dst);
+ for (size_t i = 0; i < count; ++i) {
+ offsets.insert_value(rowids[i]);
+ }
+ return Status::OK();
+ }
+
+ ordinal_t get_current_ordinal() const override { return _current_ordinal; }
+
+private:
+ ordinal_t _current_ordinal = 0;
+};
+
class NullMapOnlyFileColumnIterator final : public FileColumnIterator {
public:
explicit NullMapOnlyFileColumnIterator(std::shared_ptr<ColumnReader>
reader)
@@ -2383,6 +2419,96 @@ TEST_F(ColumnReaderTest,
MapReadByRowidsSkipReadingResizesDestination) {
ASSERT_TRUE(st.ok()) << "read_by_rowids failed: " << st.to_string();
ASSERT_EQ(count, dst->size());
}
+
+TEST_F(ColumnReaderTest, MapLazyReadByRowidsFillsSkippedKeysForValuesOnlyPath)
{
+ auto map_reader = create_test_reader(false, 10);
+ auto offsets_iter = std::make_unique<OffsetFileColumnIterator>(
+ std::make_unique<RowidOffsetFileColumnIterator>());
+ auto key_iter = std::make_unique<TrackingColumnIterator>();
+ key_iter->set_column_name("key");
+ auto* key_ptr = key_iter.get();
+ auto val_iter = std::make_unique<TrackingColumnIterator>();
+ val_iter->set_column_name("value");
+ auto* val_ptr = val_iter.get();
+
+ MapFileColumnIterator map_iter(map_reader, nullptr,
std::move(offsets_iter),
+ std::move(key_iter), std::move(val_iter));
+ map_iter.set_column_name("map_col");
+
+ TColumnAccessPaths all_access_paths {
+ create_access_path({"map_col",
ColumnIterator::ACCESS_MAP_VALUES})};
+ auto st = map_iter.set_access_paths(all_access_paths, {});
+ ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string();
+ ASSERT_EQ(key_ptr->read_requirement(),
ColumnIterator::ReadRequirement::SKIP);
+ ASSERT_EQ(val_ptr->read_requirement(),
ColumnIterator::ReadRequirement::LAZY_OUTPUT);
+
+ // Simulate the top-level struct reader marking this map as a lazy output
branch.
+
map_iter.set_read_requirement_self(ColumnIterator::ReadRequirement::LAZY_OUTPUT);
+ map_iter.set_read_phase(ColumnIterator::ReadPhase::LAZY);
+
+ MutableColumnPtr dst = ColumnMap::create(ColumnInt32::create(),
ColumnInt32::create(),
+
ColumnArray::ColumnOffsets::create());
+ const rowid_t rowids[] = {1};
+ st = map_iter.read_by_rowids(rowids, std::size(rowids), dst);
+ ASSERT_TRUE(st.ok()) << "lazy map read_by_rowids failed: " <<
st.to_string();
+
+ const auto& column_map = assert_cast<const ColumnMap&,
TypeCheckOnRelease::DISABLE>(*dst);
+ EXPECT_EQ(1, dst->size());
+ EXPECT_EQ(1, column_map.get_keys().size());
+ EXPECT_EQ(1, column_map.get_values().size());
+ EXPECT_TRUE(key_ptr->next_batch_sizes.empty());
+ EXPECT_THAT(val_ptr->seek_ordinals, ::testing::ElementsAre(1));
+ EXPECT_THAT(val_ptr->next_batch_sizes, ::testing::ElementsAre(1));
+}
+
+TEST_F(ColumnReaderTest,
MapLazyReadByRowidsPreservesPredicateKeysForValuesOnlyPath) {
+ auto map_reader = create_test_reader(false, 10);
+ auto offsets_iter = std::make_unique<OffsetFileColumnIterator>(
+ std::make_unique<RowidOffsetFileColumnIterator>());
+ auto key_iter = std::make_unique<TrackingColumnIterator>();
+ key_iter->set_column_name("key");
+ auto* key_ptr = key_iter.get();
+ auto val_iter = std::make_unique<TrackingColumnIterator>();
+ val_iter->set_column_name("value");
+ auto* val_ptr = val_iter.get();
+
+ MapFileColumnIterator map_iter(map_reader, nullptr,
std::move(offsets_iter),
+ std::move(key_iter), std::move(val_iter));
+ map_iter.set_column_name("map_col");
+
+ TColumnAccessPaths all_access_paths {
+ create_access_path({"map_col",
ColumnIterator::ACCESS_MAP_VALUES})};
+ TColumnAccessPaths predicate_access_paths {
+ create_access_path({"map_col", ColumnIterator::ACCESS_MAP_KEYS})};
+ auto st = map_iter.set_access_paths(all_access_paths,
predicate_access_paths);
+ ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string();
+ ASSERT_EQ(key_ptr->read_requirement(),
ColumnIterator::ReadRequirement::PREDICATE);
+ ASSERT_EQ(val_ptr->read_requirement(),
ColumnIterator::ReadRequirement::LAZY_OUTPUT);
+
+ map_iter.set_read_phase(ColumnIterator::ReadPhase::LAZY);
+ ASSERT_TRUE(map_iter.need_to_read());
+ ASSERT_FALSE(map_iter.need_to_read_meta_columns());
+
+ MutableColumnPtr dst = ColumnMap::create(ColumnInt32::create(),
ColumnInt32::create(),
+
ColumnArray::ColumnOffsets::create());
+ auto& column_map = assert_cast<ColumnMap&,
TypeCheckOnRelease::DISABLE>(*dst);
+ auto keys_ptr = IColumn::mutate(std::move(column_map.get_keys_ptr()));
+ keys_ptr->insert_many_defaults(1);
+ column_map.get_keys_ptr() = std::move(keys_ptr);
+ column_map.get_offsets().push_back(1);
+
+ const rowid_t rowids[] = {1};
+ st = map_iter.read_by_rowids(rowids, std::size(rowids), dst);
+ ASSERT_TRUE(st.ok()) << "lazy map read_by_rowids failed: " <<
st.to_string();
+
+ EXPECT_EQ(1, dst->size());
+ EXPECT_EQ(1, column_map.get_keys().size());
+ EXPECT_EQ(1, column_map.get_values().size());
+ EXPECT_TRUE(key_ptr->seek_ordinals.empty());
+ EXPECT_TRUE(key_ptr->next_batch_sizes.empty());
+ EXPECT_THAT(val_ptr->seek_ordinals, ::testing::ElementsAre(1));
+ EXPECT_THAT(val_ptr->next_batch_sizes, ::testing::ElementsAre(1));
+}
TEST_F(ColumnReaderTest, MapAccessAllWithOffsetDoesNotPropagateOffsetToKey) {
// Regression test: when the access path is [map_col, *, OFFSET]
// (e.g. length(map_col['some_key'])), the key column must be fully read
diff --git
a/regression-test/data/nereids_rules_p0/column_pruning/struct_map_pruning_lazy_read.out
b/regression-test/data/nereids_rules_p0/column_pruning/struct_map_pruning_lazy_read.out
new file mode 100644
index 00000000000..65051648946
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/column_pruning/struct_map_pruning_lazy_read.out
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !map_values --
+2 ["val3"]
+
+-- !map_keys --
+2 [3]
diff --git
a/regression-test/suites/nereids_rules_p0/column_pruning/struct_map_pruning_lazy_read.groovy
b/regression-test/suites/nereids_rules_p0/column_pruning/struct_map_pruning_lazy_read.groovy
new file mode 100644
index 00000000000..e92cd0388d6
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/column_pruning/struct_map_pruning_lazy_read.groovy
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("struct_map_pruning_lazy_read") {
+ sql "set enable_prune_nested_column = true"
+ sql "DROP TABLE IF EXISTS struct_map_pruning_lazy_read_tbl"
+ sql """
+ CREATE TABLE struct_map_pruning_lazy_read_tbl (
+ id INT,
+ heavy_struct STRUCT<
+ small_int: INT,
+ big_string: VARCHAR(2048),
+ nested_map: MAP<INT, VARCHAR(100)>
+ >,
+ standalone_map MAP<VARCHAR(50), INT>
+ )
+ UNIQUE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+ sql """
+ INSERT INTO struct_map_pruning_lazy_read_tbl VALUES
+ (1, struct(100, repeat('A', 2048), map(1, 'val1', 2, 'val2')),
map('key_x', 1, 'key_y', 2)),
+ (2, struct(200, repeat('B', 2048), map(3, 'val3')), map('key_z', 3)),
+ (3, struct(300, repeat('C', 2048), map()), map('empty', 0))
+ """
+
+ order_qt_map_values """
+ SELECT id, map_values(struct_element(heavy_struct, 'nested_map'))
+ FROM struct_map_pruning_lazy_read_tbl
+ WHERE struct_element(heavy_struct, 'big_string') LIKE 'B%'
+ ORDER BY id
+ """
+
+ order_qt_map_keys """
+ SELECT id, map_keys(struct_element(heavy_struct, 'nested_map'))
+ FROM struct_map_pruning_lazy_read_tbl
+ WHERE struct_element(heavy_struct, 'small_int') = 200
+ ORDER BY id
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
