This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 7dee84810cf branch-4.2: [fix](be) Preserve pruned complex types in
merge blocks (#69696)
7dee84810cf is described below
commit 7dee84810cf2eed5454fa8e136b6955a26ed7ab5
Author: Chenyang Sun <[email protected]>
AuthorDate: Wed Oct 28 09:26:27 2026 +0800
branch-4.2: [fix](be) Preserve pruned complex types in merge blocks (#69696)
Backport the merge-block portion of #65396 to branch-4.2: use the
FE-pruned type by column unique ID when constructing merge blocks,
retaining the physical type for unpruned columns.
---
be/src/storage/iterator/vgeneric_iterators.cpp | 7 +-
be/src/storage/iterator/vgeneric_iterators.h | 18 +++
be/src/storage/tablet/tablet_schema.h | 4 +
be/test/exec/scan/vgeneric_iterators_test.cpp | 151 +++++++++++++++++++++
...est_topn_pruned_struct_overlapping_segments.out | 49 +++++++
..._topn_pruned_struct_overlapping_segments.groovy | 147 ++++++++++++++++++++
6 files changed, 375 insertions(+), 1 deletion(-)
diff --git a/be/src/storage/iterator/vgeneric_iterators.cpp
b/be/src/storage/iterator/vgeneric_iterators.cpp
index 99793ecefe5..60ffb79e18a 100644
--- a/be/src/storage/iterator/vgeneric_iterators.cpp
+++ b/be/src/storage/iterator/vgeneric_iterators.cpp
@@ -111,7 +111,7 @@ Status VMergeIteratorContext::block_reset(const
std::shared_ptr<Block>& block) {
const auto& column_ids = _output_schema->column_ids();
for (size_t i = 0; i < _output_schema->num_column_ids(); ++i) {
auto column_desc = _output_schema->column(column_ids[i]);
- auto data_type = Schema::get_data_type_ptr(*column_desc);
+ auto data_type = _data_type_maybe_pruned(*column_desc);
if (data_type == nullptr) {
return Status::RuntimeError("invalid data type");
}
@@ -293,6 +293,11 @@ Status VAutoIncrementIterator::init(const
StorageReadOptions& opts) {
Status VMergeIteratorContext::init(const StorageReadOptions& opts) {
_block_row_max = opts.block_row_max;
_record_rowids = opts.record_rowids;
+ // For query reads that prune nested sub-columns, block_reset() must build
the
+ // merge block with the FE-pruned complex types recorded on the tablet
schema.
+ if (opts.tablet_schema != nullptr &&
opts.tablet_schema->has_pruned_columns()) {
+ _pruned_columns_data_type =
&opts.tablet_schema->pruned_columns_data_type();
+ }
RETURN_IF_ERROR(_load_next_block());
if (valid()) {
RETURN_IF_ERROR(advance());
diff --git a/be/src/storage/iterator/vgeneric_iterators.h
b/be/src/storage/iterator/vgeneric_iterators.h
index 449d5cb1606..b5dbb92590d 100644
--- a/be/src/storage/iterator/vgeneric_iterators.h
+++ b/be/src/storage/iterator/vgeneric_iterators.h
@@ -184,6 +184,20 @@ private:
// Load next block into _block
Status _load_next_block();
+ // Data type used to materialize `column` in the merge block: the FE-pruned
+ // type when the query pruned this complex column's sub-columns (so the
block's
+ // struct children line up with the pruned sub-column iterators),
otherwise the
+ // full storage type.
+ DataTypePtr _data_type_maybe_pruned(const TabletColumn& column) const {
+ if (_pruned_columns_data_type) {
+ auto it = _pruned_columns_data_type->find(column.unique_id());
+ if (it != _pruned_columns_data_type->end()) {
+ return it->second;
+ }
+ }
+ return Schema::get_data_type_ptr(column);
+ }
+
RowwiseIteratorUPtr _iter;
int _sequence_id_idx = -1;
@@ -203,6 +217,10 @@ private:
// block_reset() uses _output_schema to build _block, and copy_rows()
iterates over
// _output_schema->num_column_ids() columns to copy from _block to the
destination.
const SchemaSPtr _output_schema;
+ // Column unique id -> FE-pruned type; null when no columns are pruned.
The owned
+ // segment iterator retains opts.tablet_schema through its
StorageReadOptions.
+ // block_reset() uses this map to match the pruned sub-column iterators.
+ const std::map<int32_t, DataTypePtr>* _pruned_columns_data_type = nullptr;
int _num_key_columns;
std::vector<uint32_t>* _compare_columns;
std::vector<RowLocation> _block_row_locations;
diff --git a/be/src/storage/tablet/tablet_schema.h
b/be/src/storage/tablet/tablet_schema.h
index 272bbe14622..4051d2849f5 100644
--- a/be/src/storage/tablet/tablet_schema.h
+++ b/be/src/storage/tablet/tablet_schema.h
@@ -781,6 +781,10 @@ public:
bool has_pruned_columns() const { return
!_pruned_columns_data_type.empty(); }
+ const std::map<int32_t, DataTypePtr>& pruned_columns_data_type() const {
+ return _pruned_columns_data_type;
+ }
+
private:
friend bool operator==(const TabletSchema& a, const TabletSchema& b);
friend bool operator!=(const TabletSchema& a, const TabletSchema& b);
diff --git a/be/test/exec/scan/vgeneric_iterators_test.cpp
b/be/test/exec/scan/vgeneric_iterators_test.cpp
index af4a5893d6e..134c5e24c3a 100644
--- a/be/test/exec/scan/vgeneric_iterators_test.cpp
+++ b/be/test/exec/scan/vgeneric_iterators_test.cpp
@@ -661,4 +661,155 @@ TEST(VGenericIteratorsTest, MergeWithSeqColumn) {
EXPECT_EQ(seg_iter_num - 1, actual_value);
}
+// Schema with a smallint key c1 and a struct column c2 = struct<f1:int,
f2:int, f3:int>.
+static Schema create_struct_schema(int32_t struct_unique_id) {
+ std::vector<TabletColumnPtr> col_schemas;
+ auto c1 =
std::make_shared<TabletColumn>(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE,
+
FieldType::OLAP_FIELD_TYPE_SMALLINT, true);
+ c1->set_is_key(true);
+ col_schemas.emplace_back(c1);
+
+ // The (agg, type, nullable) constructor does not support complex types, so
+ // build the struct column with the default constructor plus setters.
+ auto c2 = std::make_shared<TabletColumn>();
+ c2->set_type(FieldType::OLAP_FIELD_TYPE_STRUCT);
+
c2->set_aggregation_method(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE);
+ c2->set_is_nullable(true);
+ c2->set_name("c2");
+ c2->set_unique_id(struct_unique_id);
+ for (const auto* sub_name : {"f1", "f2", "f3"}) {
+ TabletColumn sub(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE,
+ FieldType::OLAP_FIELD_TYPE_INT, true);
+ sub.set_name(sub_name);
+ c2->add_sub_column(sub);
+ }
+ col_schemas.emplace_back(c2);
+
+ std::vector<ColumnId> column_ids(col_schemas.size());
+ for (uint32_t cid = 0; cid < column_ids.size(); ++cid) {
+ column_ids[cid] = cid;
+ }
+
+ Schema schema(col_schemas, column_ids);
+ return schema;
+}
+
+// Fills whatever block it is given: ascending keys (key_start + i *
key_stride) so
+// several inputs interleave under merge, and default values for the struct
column.
+// Also checks that the merge context built its block with the expected struct
type.
+class PrunedStructUtIterator : public RowwiseIterator {
+public:
+ PrunedStructUtIterator(const Schema& schema, size_t num_rows, int16_t
key_start,
+ int16_t key_stride, DataTypePtr
expected_struct_type)
+ : _schema(schema),
+ _num_rows(num_rows),
+ _key(key_start),
+ _key_stride(key_stride),
+ _expected_struct_type(std::move(expected_struct_type)) {}
+ ~PrunedStructUtIterator() override = default;
+
+ Status init(const StorageReadOptions& opts) override { return
Status::OK(); }
+
+ Status next_batch(Block* block) override {
+ if (_rows_returned >= _num_rows) {
+ return Status::EndOfFile("End of PrunedStructUtIterator");
+ }
+ // The merge block must carry the FE-pruned struct type (not the full
storage
+ // type) so that its struct children line up with the pruned
sub-column readers.
+
EXPECT_TRUE(block->get_by_position(1).type->equals(*_expected_struct_type));
+ // Return several blocks so the merge context also reuses a cleared
pruned block.
+ for (size_t i = 0; i < 16 && _rows_returned < _num_rows; ++i) {
+ ColumnWithTypeAndName& key_col = block->get_by_position(0);
+
key_col.column->assert_mutable()->insert_data(reinterpret_cast<const
char*>(&_key),
+ sizeof(_key));
+ ColumnWithTypeAndName& struct_col = block->get_by_position(1);
+ struct_col.column->assert_mutable()->insert_default();
+ _key = static_cast<int16_t>(_key + _key_stride);
+ ++_rows_returned;
+ }
+ return Status::OK();
+ }
+
+ const Schema& schema() const override { return _schema; }
+
+private:
+ const Schema& _schema;
+ size_t _num_rows;
+ size_t _rows_returned = 0;
+ int16_t _key;
+ int16_t _key_stride;
+ DataTypePtr _expected_struct_type;
+};
+
+// When the query prunes struct sub-columns, StorageReadOptions carries a
tablet schema
+// whose pruned-columns map records the FE-pruned type for the column's unique
id. The
+// merge iterator must build its internal blocks with that pruned type (see
+// VMergeIteratorContext::_data_type_maybe_pruned); otherwise the block's full
struct
+// children would be paired with the pruned sub-column iterators by bare index.
+TEST(VGenericIteratorsTest, MergeWithPrunedStructColumn) {
+ constexpr int32_t struct_unique_id = 10;
+ auto schema = create_struct_schema(struct_unique_id);
+ auto output_schema = std::make_shared<Schema>(schema);
+
+ // The FE-pruned type of c2 keeps only {f1, f3} out of {f1, f2, f3}.
+ TabletColumn pruned_c2;
+ pruned_c2.set_type(FieldType::OLAP_FIELD_TYPE_STRUCT);
+
pruned_c2.set_aggregation_method(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE);
+ pruned_c2.set_is_nullable(true);
+ pruned_c2.set_name("c2");
+ pruned_c2.set_unique_id(struct_unique_id);
+ for (const auto* sub_name : {"f1", "f3"}) {
+ TabletColumn sub(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE,
+ FieldType::OLAP_FIELD_TYPE_INT, true);
+ sub.set_name(sub_name);
+ pruned_c2.add_sub_column(sub);
+ }
+ auto pruned_type = Schema::get_data_type_ptr(pruned_c2);
+ auto full_type = Schema::get_data_type_ptr(*schema.column(1));
+ EXPECT_FALSE(pruned_type->equals(*full_type));
+
+ auto tablet_schema = std::make_shared<TabletSchema>();
+ tablet_schema->add_pruned_columns_data_type(struct_unique_id, pruned_type);
+
+ std::vector<RowwiseIteratorUPtr> inputs;
+ inputs.push_back(std::make_unique<PrunedStructUtIterator>(schema, 100, 0,
2, pruned_type));
+ inputs.push_back(std::make_unique<PrunedStructUtIterator>(schema, 100, 1,
2, pruned_type));
+
+ auto iter = new_merge_iterator(std::move(inputs), -1, false, false,
nullptr, output_schema);
+ StorageReadOptions opts;
+ opts.tablet_schema = tablet_schema;
+ opts.block_row_max = 16;
+ auto st = iter->init(opts);
+ ASSERT_TRUE(st.ok()) << st;
+
+ // The destination block mirrors a query-level block: the struct column is
created
+ // with the pruned type, matching what the merge context copies into it.
+ Block block;
+ auto key_type = Schema::get_data_type_ptr(*schema.column(0));
+ auto key_column = key_type->create_column();
+ block.insert(ColumnWithTypeAndName(std::move(key_column), key_type, "c1"));
+ auto struct_column = pruned_type->create_column();
+ block.insert(ColumnWithTypeAndName(std::move(struct_column), pruned_type,
"c2"));
+ std::vector<bool> row_is_same;
+ BlockWithSameBit block_with_same_bit {.block = &block, .same_bit =
row_is_same};
+
+ // Keys 0..199 in order: the two inputs (0,2,4,... and 1,3,5,...)
interleaved.
+ size_t rows_read = 0;
+ do {
+ block.clear_column_data();
+ row_is_same.clear();
+ st = iter->next_batch(&block_with_same_bit);
+ ASSERT_TRUE(st.ok() || st.is<END_OF_FILE>()) << st;
+ auto c0 = block.get_by_position(0).column;
+ for (size_t i = 0; i < block.rows(); ++i) {
+ EXPECT_EQ(rows_read + i, (*c0)[i].get<TYPE_SMALLINT>());
+ }
+ // The struct column keeps the pruned 2-child layout through block
reuse.
+ EXPECT_TRUE(block.get_by_position(1).type->equals(*pruned_type));
+ EXPECT_EQ(block.get_by_position(1).column->size(), block.rows());
+ rows_read += block.rows();
+ } while (st.ok());
+ EXPECT_EQ(rows_read, 200);
+}
+
} // namespace doris
diff --git
a/regression-test/data/fault_injection_p0/test_topn_pruned_struct_overlapping_segments.out
b/regression-test/data/fault_injection_p0/test_topn_pruned_struct_overlapping_segments.out
new file mode 100644
index 00000000000..213bd11c1fc
--- /dev/null
+++
b/regression-test/data/fault_injection_p0/test_topn_pruned_struct_overlapping_segments.out
@@ -0,0 +1,49 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !dict_default_session --
+1 s1
+2 s2
+3 s3
+4 s4
+5 s5
+6 s6
+7 s7
+8 s8
+9 s9
+10 s10
+
+-- !dict_inline_topn --
+1 s1
+2 s2
+3 s3
+4 s4
+5 s5
+6 s6
+7 s7
+8 s8
+9 s9
+10 s10
+
+-- !dict_inline_topn_single_child --
+s1
+s2
+s3
+s4
+s5
+s6
+s7
+s8
+s9
+s10
+
+-- !int_inline_topn_last_child --
+3000001
+3000002
+3000003
+3000004
+3000005
+3000006
+3000007
+3000008
+3000009
+3000010
+
diff --git
a/regression-test/suites/fault_injection_p0/test_topn_pruned_struct_overlapping_segments.groovy
b/regression-test/suites/fault_injection_p0/test_topn_pruned_struct_overlapping_segments.groovy
new file mode 100644
index 00000000000..2edf9ff301f
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/test_topn_pruned_struct_overlapping_segments.groovy
@@ -0,0 +1,147 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Regression case for the pruned-struct read misalignment on the inline topn
scan path.
+//
+// Root cause: when a topn ordered read (order by key limit N) hits a rowset
with
+// multiple OVERLAPPING segments, BetaRowsetReader picks VMergeIterator, whose
+// VMergeIteratorContext::init() eagerly loads the first block of every
segment.
+// That block is created from the storage Schema (FULL struct type, all
children),
+// while the struct sub-column iterators have been filtered down to the pruned
+// subset by set_access_paths()/remove_pruned_sub_iterators(). The pairing
loop in
+// StructFileColumnIterator::next_batch() indexes _sub_column_iterators[i] by
the
+// dst tuple position, so the two ordinal spaces disagree:
+// - text/dict child data decoded into a TINYINT child column
+// -> [E-3110] insert_many_dict_data is not supported for TINYINT
+// - same-typed children shift silently, and the loop runs past the pruned
+// iterator vector -> out-of-bounds -> BE SIGSEGV
+//
+// The FE TopN lazy materialization (VMaterializeNode) usually shields this
path
+// on master; setting topn_lazy_materialization_threshold=-1 (or any state
where
+// the rule bails, e.g. light_schema_change=false tables) exposes it.
+//
+// Layout construction: MemTable.need_flush debug point forces one segment per
+// inserted block (batch_size=500, 2000 rows -> 4 segments, below the
+// segcompaction_batch_size=10 threshold so segcompaction never merges them),
and
+// the NULL k1 rows in every block make every segment's key lower bound NULL,
so
+// the segments overlap and the rowset is marked OVERLAPPING.
+suite("test_topn_pruned_struct_overlapping_segments", "nonConcurrent") {
+ sql "DROP TABLE IF EXISTS test_topn_pruned_struct_ovlp_dict"
+ sql "DROP TABLE IF EXISTS test_topn_pruned_struct_ovlp_int"
+ sql """
+ CREATE TABLE test_topn_pruned_struct_ovlp_dict (
+ k1 BIGINT,
+ c ARRAY<ARRAY<STRUCT<col1:INT, col2:TINYINT, col17:TEXT>>>
+ ) DUPLICATE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES('replication_num' = '1', 'disable_auto_compaction' = 'true')
+ """
+ sql """
+ CREATE TABLE test_topn_pruned_struct_ovlp_int (
+ k1 BIGINT,
+ c ARRAY<ARRAY<STRUCT<col1:INT, col2:INT, col3:INT>>>
+ ) DUPLICATE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES('replication_num' = '1', 'disable_auto_compaction' = 'true')
+ """
+
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush")
+ // 4 blocks of 500 rows -> 4 segments per tablet; k1 NULL every 20
rows so
+ // every segment starts at NULL -> overlapping key ranges ->
OVERLAPPING rowset.
+ sql "SET batch_size = 500"
+ sql """
+ INSERT INTO test_topn_pruned_struct_ovlp_dict
+ SELECT if(number % 20 = 0, NULL, number),
+ array(array(named_struct(
+ 'col1', CAST(number AS INT),
+ 'col2', CAST(number % 127 AS TINYINT),
+ 'col17', concat('s', number))))
+ FROM numbers('number' = '2000')
+ """
+ sql """
+ INSERT INTO test_topn_pruned_struct_ovlp_int
+ SELECT if(number % 20 = 0, NULL, number),
+ array(array(named_struct(
+ 'col1', CAST(number + 1000000 AS INT),
+ 'col2', CAST(number + 2000000 AS INT),
+ 'col3', CAST(number + 3000000 AS INT))))
+ FROM numbers('number' = '2000')
+ """
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs("MemTable.need_flush")
+ }
+
+ // Self-check the storage layout: the test is void unless the rowset
really is
+ // a multi-segment OVERLAPPING one. BetaRowsetReader::is_merge_iterator()
needs
+ // both the OVERLAPPING flag and num_segments > 1, so assert both.
+ for (def tbl : ["test_topn_pruned_struct_ovlp_dict",
"test_topn_pruned_struct_ovlp_int"]) {
+ def tablets = sql_return_maparray("SHOW TABLETS FROM ${tbl}")
+ def (code, out, err) = curl("GET", tablets[0].MetaUrl)
+ assertEquals(0, code)
+ def jsonMeta = parseJson(out.trim())
+ def hasMergeLayout = jsonMeta.rs_metas.any { meta ->
+ meta.segments_overlap_pb == "OVERLAPPING" && meta.num_segments > 1
+ }
+ assertTrue(hasMergeLayout,
+ "expected an OVERLAPPING rowset with num_segments > 1 in
${tbl}, "
+ + "check the MemTable.need_flush debug point")
+ }
+
+ // 1. Default session: on master this plans VMaterializeNode (TopN lazy
+ // materialization + rowid fetch); guards that path stays correct too.
+ // Expected: k1 1..10 -> col1 = k1, col17 = 's' + k1.
+ qt_dict_default_session """
+ SELECT element_at(c[1][1], 1), element_at(c[1][1], 'col17')
+ FROM test_topn_pruned_struct_ovlp_dict WHERE k1 IS NOT NULL ORDER BY
k1 LIMIT 10
+ """
+
+ // 2. Force the inline topn scan path (no VMaterializeNode). This is the
exact
+ // path that used to fail with
+ // "[E-3110] Method insert_many_dict_data is not supported for TINYINT".
+ sql "SET topn_lazy_materialization_threshold = -1"
+ // The storage ordered-key read (read_orderby_key -> need_ordered_result ->
+ // VMergeIterator) is additionally gated by enable_segment_limit_pushdown
in
+ // OlapScanner. The variable is randomized in fuzzy regression runs; pin
it so
+ // this section always exercises the merge path this fix targets.
+ sql "SET enable_segment_limit_pushdown = true"
+ // Runtime filters disable storage TopN, and parallel scans can split the
+ // overlapping segments into separate scanners that do not need a merge.
+ sql "SET topn_filter_ratio = 0"
+ sql "SET enable_parallel_scan = false"
+
+ qt_dict_inline_topn """
+ SELECT element_at(c[1][1], 1), element_at(c[1][1], 'col17')
+ FROM test_topn_pruned_struct_ovlp_dict WHERE k1 IS NOT NULL ORDER BY
k1 LIMIT 10
+ """
+
+ // Single pruned child: the misalignment shifts col17 data onto col1 (INT),
+ // which used to fail with "... is not supported for INT".
+ qt_dict_inline_topn_single_child """
+ SELECT element_at(c[1][1], 'col17')
+ FROM test_topn_pruned_struct_ovlp_dict WHERE k1 IS NOT NULL ORDER BY
k1 LIMIT 10
+ """
+
+ // Same-typed children: the shifted decode does not error out, so the
pairing
+ // loop used to run past the pruned iterator vector and SIGSEGV the BE; a
+ // partial regression could also return col1's values (1000001..) instead
of
+ // col3's (3000001..) here.
+ qt_int_inline_topn_last_child """
+ SELECT element_at(c[1][1], 'col3')
+ FROM test_topn_pruned_struct_ovlp_int WHERE k1 IS NOT NULL ORDER BY k1
LIMIT 10
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
