This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 296d2a29964 [fix](be) Fix lazy reads for default value columns (#65603)
296d2a29964 is described below
commit 296d2a29964cc6e28d4033ac9f3532cf55492920
Author: Chenyang Sun <[email protected]>
AuthorDate: Wed Jul 15 15:13:41 2026 +0800
[fix](be) Fix lazy reads for default value columns (#65603)
### What problem does this PR solve?
Fix queries that lazily materialize nested fields added by light schema
change.
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [x] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/storage/segment/column_reader.cpp | 12 +++
be/test/storage/segment/column_reader_test.cpp | 120 +++++++++++++++++++++
...test_light_schema_change_lazy_pruned_struct.out | 13 +++
...t_light_schema_change_lazy_pruned_struct.groovy | 95 ++++++++++++++++
4 files changed, 240 insertions(+)
diff --git a/be/src/storage/segment/column_reader.cpp
b/be/src/storage/segment/column_reader.cpp
index 25886d7515d..db8f2d6819b 100644
--- a/be/src/storage/segment/column_reader.cpp
+++ b/be/src/storage/segment/column_reader.cpp
@@ -3037,6 +3037,12 @@ Status DefaultValueColumnIterator::init(const
ColumnIteratorOptions& opts) {
}
Status DefaultValueColumnIterator::next_batch(size_t* n, MutableColumnPtr&
dst, bool* has_null) {
+ if (!need_to_read()) {
+ _convert_to_place_holder_column(dst, *n);
+ return Status::OK();
+ }
+
+ _recovery_from_place_holder_column(dst);
*has_null = _default_value_field.is_null();
_insert_many_default(dst, *n);
return Status::OK();
@@ -3044,6 +3050,12 @@ Status DefaultValueColumnIterator::next_batch(size_t* n,
MutableColumnPtr& dst,
Status DefaultValueColumnIterator::read_by_rowids(const rowid_t* rowids, const
size_t count,
MutableColumnPtr& dst) {
+ if (!need_to_read()) {
+ _convert_to_place_holder_column(dst, count);
+ return Status::OK();
+ }
+
+ _recovery_from_place_holder_column(dst);
_insert_many_default(dst, count);
return Status::OK();
}
diff --git a/be/test/storage/segment/column_reader_test.cpp
b/be/test/storage/segment/column_reader_test.cpp
index 8dfe640808e..3dfe42ac7cd 100644
--- a/be/test/storage/segment/column_reader_test.cpp
+++ b/be/test/storage/segment/column_reader_test.cpp
@@ -629,6 +629,126 @@ TEST_F(ColumnReaderTest,
PlaceHolderRecoveryAfterColumnReplacement) {
EXPECT_EQ(2, dst->size());
}
+namespace {
+void check_default_value_lazy_output(bool read_by_rowids) {
+ SCOPED_TRACE(read_by_rowids ? "read_by_rowids" : "next_batch");
+
+ DefaultValueColumnIterator iterator(true, "7", false,
FieldType::OLAP_FIELD_TYPE_INT, 0, 0,
+ sizeof(int32_t));
+ ColumnIteratorOptions iter_opts;
+ ASSERT_TRUE(iterator.init(iter_opts).ok());
+
iterator.set_read_requirement(ColumnIterator::ReadRequirement::LAZY_OUTPUT);
+ iterator.set_read_phase(ColumnIterator::ReadPhase::PREDICATE);
+
+ MutableColumnPtr dst = ColumnInt32::create();
+ size_t rows = 3;
+ bool has_null = false;
+ ASSERT_TRUE(iterator.next_batch(&rows, dst, &has_null).ok());
+ EXPECT_TRUE(iterator._has_place_holder_column);
+
+ IColumn::Filter filter;
+ filter.resize(3);
+ filter[0] = 1;
+ filter[1] = 0;
+ filter[2] = 1;
+ dst = IColumn::mutate(dst->filter(filter, 2));
+ ASSERT_EQ(2, dst->size());
+
+ iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY);
+ if (read_by_rowids) {
+ const rowid_t rowids[] = {2, 8};
+ ASSERT_TRUE(iterator.read_by_rowids(rowids, std::size(rowids),
dst).ok());
+ } else {
+ rows = 2;
+ ASSERT_TRUE(iterator.next_batch(&rows, dst, &has_null).ok());
+ }
+
+ ASSERT_EQ(2, dst->size());
+ EXPECT_FALSE(iterator._has_place_holder_column);
+ const auto& result = assert_cast<const ColumnInt32&>(*dst);
+ EXPECT_EQ(7, result.get_element(0));
+ EXPECT_EQ(7, result.get_element(1));
+}
+
+void check_default_value_predicate_not_read_again(bool read_by_rowids) {
+ SCOPED_TRACE(read_by_rowids ? "read_by_rowids" : "next_batch");
+
+ DefaultValueColumnIterator iterator(true, "7", false,
FieldType::OLAP_FIELD_TYPE_INT, 0, 0,
+ sizeof(int32_t));
+ ColumnIteratorOptions iter_opts;
+ ASSERT_TRUE(iterator.init(iter_opts).ok());
+ iterator.set_read_requirement(ColumnIterator::ReadRequirement::PREDICATE);
+ iterator.set_read_phase(ColumnIterator::ReadPhase::PREDICATE);
+
+ MutableColumnPtr dst = ColumnInt32::create();
+ size_t rows = 3;
+ bool has_null = true;
+ ASSERT_TRUE(iterator.next_batch(&rows, dst, &has_null).ok());
+ EXPECT_FALSE(has_null);
+ EXPECT_FALSE(iterator._has_place_holder_column);
+
+ IColumn::Filter filter;
+ filter.resize(3);
+ filter[0] = 1;
+ filter[1] = 0;
+ filter[2] = 1;
+ dst = IColumn::mutate(dst->filter(filter, 2));
+ ASSERT_EQ(2, dst->size());
+
+ iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY);
+ if (read_by_rowids) {
+ const rowid_t rowids[] = {2, 8};
+ ASSERT_TRUE(iterator.read_by_rowids(rowids, std::size(rowids),
dst).ok());
+ } else {
+ rows = 2;
+ ASSERT_TRUE(iterator.next_batch(&rows, dst, &has_null).ok());
+ }
+
+ ASSERT_EQ(2, dst->size());
+ const auto& result = assert_cast<const ColumnInt32&>(*dst);
+ EXPECT_EQ(7, result.get_element(0));
+ EXPECT_EQ(7, result.get_element(1));
+}
+} // namespace
+
+TEST_F(ColumnReaderTest, DefaultValueLazyOutputRecoversFilteredPlaceholder) {
+ check_default_value_lazy_output(false);
+ check_default_value_lazy_output(true);
+}
+
+TEST_F(ColumnReaderTest, DefaultValueLazyOutputFinalizesEmptySelection) {
+ DefaultValueColumnIterator iterator(true, "7", false,
FieldType::OLAP_FIELD_TYPE_INT, 0, 0,
+ sizeof(int32_t));
+ ColumnIteratorOptions iter_opts;
+ ASSERT_TRUE(iterator.init(iter_opts).ok());
+
iterator.set_read_requirement(ColumnIterator::ReadRequirement::LAZY_OUTPUT);
+ iterator.set_read_phase(ColumnIterator::ReadPhase::PREDICATE);
+
+ MutableColumnPtr dst = ColumnInt32::create();
+ size_t rows = 3;
+ bool has_null = false;
+ ASSERT_TRUE(iterator.next_batch(&rows, dst, &has_null).ok());
+ EXPECT_TRUE(iterator._has_place_holder_column);
+
+ IColumn::Filter filter;
+ filter.resize(3);
+ filter[0] = 0;
+ filter[1] = 0;
+ filter[2] = 0;
+ dst = IColumn::mutate(dst->filter(filter, 0));
+ ASSERT_EQ(0, dst->size());
+
+ iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY);
+ iterator.finalize_lazy_phase(dst);
+ EXPECT_EQ(0, dst->size());
+ EXPECT_FALSE(iterator._has_place_holder_column);
+}
+
+TEST_F(ColumnReaderTest, DefaultValuePredicateIsNotReadAgainInLazyPhase) {
+ check_default_value_predicate_not_read_again(false);
+ check_default_value_predicate_not_read_again(true);
+}
+
TEST_F(ColumnReaderTest, SetReadRequirementPropagatesToNestedIterators) {
auto null_iter =
std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>());
std::vector<ColumnIteratorUPtr> struct_sub_iters;
diff --git
a/regression-test/data/datatype_p0/complex_types/test_light_schema_change_lazy_pruned_struct.out
b/regression-test/data/datatype_p0/complex_types/test_light_schema_change_lazy_pruned_struct.out
new file mode 100644
index 00000000000..55de0e75c37
--- /dev/null
+++
b/regression-test/data/datatype_p0/complex_types/test_light_schema_change_lazy_pruned_struct.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !default_value_as_lazy_output --
+\N
+\N
+\N
+90
+
+-- !default_value_as_predicate --
+0
+1
+2
+7
+
diff --git
a/regression-test/suites/datatype_p0/complex_types/test_light_schema_change_lazy_pruned_struct.groovy
b/regression-test/suites/datatype_p0/complex_types/test_light_schema_change_lazy_pruned_struct.groovy
new file mode 100644
index 00000000000..9797453e901
--- /dev/null
+++
b/regression-test/suites/datatype_p0/complex_types/test_light_schema_change_lazy_pruned_struct.groovy
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_light_schema_change_lazy_pruned_struct") {
+ sql "set batch_size = 4"
+ sql "set enable_prune_nested_column = true"
+
+ sql "DROP TABLE IF EXISTS test_light_schema_change_lazy_pruned_struct"
+ sql """
+ CREATE TABLE test_light_schema_change_lazy_pruned_struct (
+ id INT,
+ s STRUCT<b:INT, c:VARCHAR(20)>,
+ b INT,
+ a INT
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "true"
+ )
+ """
+
+ // Keep these rows in the old physical schema. After the ALTER, s.a is
read by
+ // DefaultValueColumnIterator for this rowset.
+ sql """
+ INSERT INTO test_light_schema_change_lazy_pruned_struct VALUES
+ (1, named_struct('b', 1, 'c', 'abc'), 21, 11),
+ (2, named_struct('b', 2, 'c', 'abc'), 22, 12),
+ (3, named_struct('b', 0, 'c', 'abc'), 23, 13),
+ (4, named_struct('b', 4, 'c', 'def'), 24, 14),
+ (5, named_struct('b', 5, 'c', 'abc'), 20, 15),
+ (6, named_struct('b', 6, 'c', 'abc'), 26, 10),
+ (7, named_struct('b', 7, 'c', 'abc'), 27, 17),
+ (8, named_struct('b', 8, 'c', 'def'), 28, 18)
+ """
+
+ sql """
+ ALTER TABLE test_light_schema_change_lazy_pruned_struct
+ MODIFY COLUMN s STRUCT<b:INT, c:VARCHAR(20), a:INT>
+ """
+ waitForSchemaChangeDone {
+ sql """
+ SHOW ALTER TABLE COLUMN
+ WHERE TableName='test_light_schema_change_lazy_pruned_struct'
+ ORDER BY CreateTime DESC LIMIT 1
+ """
+ time 600
+ }
+
+ // Also keep a rowset written with the new physical schema in the same
scan.
+ sql """
+ INSERT INTO test_light_schema_change_lazy_pruned_struct VALUES
+ (9, named_struct('b', 9, 'c', 'abc', 'a', 90), 29, 19),
+ (10, named_struct('b', 0, 'c', 'abc', 'a', 100), 30, 20)
+ """
+
+ // s.a is a lazy output column, while s.b and s.c are predicate columns.
+ // On old rowsets s.a must replace its predicate-phase placeholders
instead of
+ // appending defaults to them during lazy materialization.
+ order_qt_default_value_as_lazy_output """
+ SELECT s.a
+ FROM test_light_schema_change_lazy_pruned_struct
+ WHERE s.b > 0
+ AND s.c = 'abc'
+ AND b > 20
+ AND a > 10
+ """
+
+ // Exercise the inverse phase assignment: the default-backed child is used
by
+ // a predicate and an old physical child is materialized lazily.
+ order_qt_default_value_as_predicate """
+ SELECT s.b
+ FROM test_light_schema_change_lazy_pruned_struct
+ WHERE s.a IS NULL
+ AND s.c = 'abc'
+ AND b > 20
+ AND a > 10
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
