This is an automated email from the ASF dual-hosted git repository.
hubgeter pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f57e825d98c [fix](rowid-fetch) fix TopN lazy materialization crash
with duplicate projected columns (#64372)
f57e825d98c is described below
commit f57e825d98c52d8750b75434aa6fb44b1e316d02
Author: York Cao <[email protected]>
AuthorDate: Thu Aug 20 20:53:24 2026 +0800
[fix](rowid-fetch) fix TopN lazy materialization crash with duplicate
projected columns (#64372)
### What problem does this PR solve?
Problem Summary:
When a SQL query projects the same physical column multiple times with
different aliases (e.g., `SELECT col1 a, col1 b ... ORDER BY col3 DESC
LIMIT 50`), the TopN lazy materialization row-id fetch path crashes or
returns 0 rows with error:
```
*** Query id: eb262350ed06459c-a8ab4307823a1e5a ***
*** is nereids: 1 ***
*** tablet id: 0 ***
*** Aborted at 1780931790 (unix time) try "date -d @1780931790" if you are
using GNU date ***
*** Current BE git commitID: 1ccfbcc4e0 ***
*** SIGSEGV invalid permissions for mapped object (@0x7fa77d496000)
received by PID 513 (TID 211603 OR 0x7f9df864d700) from PID 2101960704; stack
trace: ***
0# doris::signal::(anonymous namespace)::FailureSignalHandler(int,
siginfo_t*, void*) in /etc/doris/be/lib/doris_be
1# PosixSignals::chained_handler(int, siginfo*, void*) [clone .part.0] in
/software/servers/jdk/lib/server/libjvm.so
2# JVM_handle_linux_signal in /software/servers/jdk/lib/server/libjvm.so
3# 0x00007FB56B00CC20 in /lib64/libpthread.so.0
4# memcpy in /etc/doris/be/lib/doris_be
5# doris::vectorized::ColumnStr<unsigned
int>::insert_from(doris::vectorized::IColumn const&, unsigned long) in
/etc/doris/be/lib/doris_be
6# void
doris::vectorized::IColumn::insert_from_multi_column_impl<doris::vectorized::ColumnNullable>(std::vector<doris::vectorized::IColumn
const*, std::allocator<doris::vectorized::IColumn const*> > const&,
std::vector<unsigned long, std::allocator<unsigned long> > const&) in
/etc/doris/be/lib/doris_be
7# doris::RowIdStorageReader::read_batch_external_row(unsigned long,
doris::PRequestBlockDesc const&, std::shared_ptr<doris::IdFileMap>,
std::vector<doris::SlotDescriptor, std::allocator<doris::SlotDescriptor> >&,
std::shared_ptr<doris::FileMapping>, doris::TUniqueId const&,
doris::vectorized::Block&, doris::PRuntimeProfileTree*, long*, long*, unsigned
long*) in /etc/doris/be/lib/doris_be
8# doris::RowIdStorageReader::read_by_rowids(doris::PMultiGetRequestV2
const&, doris::PMultiGetResponseV2*) in /etc/doris/be/lib/doris_be
9# std::_Function_handler<bool (),
doris::PInternalService::multiget_data_v2(google::protobuf::RpcController*,
doris::PMultiGetRequestV2 const*, doris::PMultiGetResponseV2*,
google::protobuf::Closure*)::$_0>::_M_invoke(std::_Any_data const&) in
/etc/doris/be/lib/doris_be
10#
doris::vectorized::TaskExecutorSimplifiedScanScheduler::submit_scan_task(doris::vectorized::SimplifiedScanTask,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
> const&)::{lambda()#2}::operator()() const in /etc/doris/be/lib/doris_be
11#
doris::vectorized::ScannerSplitRunner::process_for(std::chrono::duration<long,
std::ratio<1l, 1000000000l> >) in /etc/doris/be/lib/doris_be
12# doris::vectorized::PrioritizedSplitRunner::process() in
/etc/doris/be/lib/doris_be
13# doris::vectorized::TimeSharingTaskExecutor::_dispatch_thread() in
/etc/doris/be/lib/doris_be
14# doris::Thread::supervise_thread(void*) in /etc/doris/be/lib/doris_be
15# start_thread in /lib64/libpthread.so.0
16# __clone in /lib64/libc.so.6
```
Root cause: The row-id fetch code rebuilds scanner params using
name-keyed maps (`colname_to_slot_id`, `slot_name_to_schema_pos`). When
a column like `advert_name` appears twice, `emplace()` silently drops
the second entry, while `required_slots` and the result block still
contain both logical output slots. This creates an inconsistent schema
between the scanner and the result block.
Fix:
- Build a deduplicated `scan_slots` list with a
`source_column_to_scan_idx` map keyed by (col_name, column_idx,
col_unique_id, column_paths, access_paths), so the scanner sees each
physical column only once.
- Use a `result_column_to_scan_column` remap table so both result
columns copy from the same fetched scan column.
### Release note
Fixed a bug where TopN lazy materialization with duplicate projected
columns (same physical column aliased multiple times) caused the row-id
fetch path to return 0 rows or crash.
---------
Co-authored-by: caokaihua1 <[email protected]>
Co-authored-by: daidai <[email protected]>
---
be/src/exec/rowid_fetcher.cpp | 241 ++++-
be/src/exec/rowid_fetcher.h | 18 +-
be/test/exec/rowid_fetcher_test.cpp | 216 ++++
.../hive/test_hive_topn_lazy_mat.out | 1048 ++++++++++++++++++++
.../tvf/test_tvf_topn_lazy_mat.out | 72 ++
.../hive/test_hive_topn_lazy_mat.groovy | 50 +-
.../tvf/test_tvf_topn_lazy_mat.groovy | 7 +
7 files changed, 1605 insertions(+), 47 deletions(-)
diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp
index 95544b4422e..05a62e70e13 100644
--- a/be/src/exec/rowid_fetcher.cpp
+++ b/be/src/exec/rowid_fetcher.cpp
@@ -439,18 +439,22 @@ const std::string
RowIdStorageReader::TopNLazyMaterializationSecondPhaseSegments
Status RowIdStorageReader::read_external_row_from_file_mapping(
size_t idx, const std::multimap<segment_v2::rowid_t, size_t>& row_ids,
- const std::shared_ptr<FileMapping>& file_mapping, const
std::vector<SlotDescriptor>& slots,
- const TUniqueId& query_id, const std::shared_ptr<RuntimeState>&
runtime_state,
- std::vector<Block>& scan_blocks, std::vector<std::pair<size_t,
size_t>>& row_id_block_idx,
+ const std::shared_ptr<FileMapping>& file_mapping,
+ const std::vector<SlotDescriptor>& scan_slots, const TUniqueId&
query_id,
+ const std::shared_ptr<RuntimeState>& runtime_state,
std::vector<Block>& scan_blocks,
+ std::vector<std::pair<size_t, size_t>>& row_id_block_idx,
std::vector<RowIdStorageReader::ExternalFetchStatistics>&
fetch_statistics,
const TFileScanRangeParams& rpc_scan_params,
const std::unordered_map<std::string, int>& colname_to_slot_id,
- std::atomic<int>& producer_count, size_t scan_rows_count,
- std::counting_semaphore<>& semaphore, std::condition_variable& cv,
std::mutex& mtx,
- TupleDescriptor& tuple_desc) {
+ std::counting_semaphore<>& semaphore, TupleDescriptor& tuple_desc) {
SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->rowid_storage_reader_tracker());
signal::set_signal_task_id(query_id);
+ // Release the concurrency permit on every exit path (including error
returns
+ // and exceptions). Completion accounting and status publishing are owned
by
+ // the caller, so the status is always published before the waiter is
woken.
+ Defer defer([&] { semaphore.release(); });
+
std::list<int64_t> read_ids;
//Generate an ordered list with the help of the orderliness of the map.
for (const auto& [row_id, result_block_idx] : row_ids) {
@@ -460,7 +464,7 @@ Status
RowIdStorageReader::read_external_row_from_file_mapping(
row_id_block_idx[result_block_idx] = std::make_pair(idx,
read_ids.size() - 1);
}
- scan_blocks[idx] = Block(slots, read_ids.size());
+ scan_blocks[idx] = Block(scan_slots, read_ids.size());
auto& external_info = file_mapping->get_external_file_info();
auto& scan_range_desc = external_info.scan_range_desc;
@@ -484,6 +488,23 @@ Status
RowIdStorageReader::read_external_row_from_file_mapping(
&fetch_statistics[idx].init_reader_ms,
&fetch_statistics[idx].get_block_ms));
}
+ if (scan_blocks[idx].rows() != read_ids.size()) {
+ return Status::InternalError(
+ "Row id fetch scan row count mismatch, "
+ "query_id={}, path={}, expected_rows={}, actual_rows={}",
+ print_id(query_id), scan_range_desc.path, read_ids.size(),
scan_blocks[idx].rows());
+ }
+ for (size_t column_id = 0; column_id < scan_blocks[idx].columns();
++column_id) {
+ const auto& column = scan_blocks[idx].get_by_position(column_id);
+ if (column.column->size() != read_ids.size()) {
+ return Status::InternalError(
+ "Row id fetch scan column row count mismatch, "
+ "query_id={}, path={}, column={}, expected_rows={},
actual_rows={}",
+ print_id(query_id), scan_range_desc.path, column.name,
read_ids.size(),
+ column.column->size());
+ }
+ }
+
auto file_read_bytes_counter =
sub_runtime_profile->get_counter(FileScanner::FileReadBytesProfile);
@@ -499,14 +520,95 @@ Status
RowIdStorageReader::read_external_row_from_file_mapping(
file_read_times_counter->value(),
file_read_times_counter->type());
}
- semaphore.release();
- if (++producer_count == scan_rows_count) {
- std::lock_guard<std::mutex> lock(mtx);
- cv.notify_one();
- }
return Status::OK();
}
+std::string RowIdStorageReader::source_column_key(const SlotDescriptor& slot,
uint32_t column_idx) {
+ fmt::memory_buffer key;
+ // Length-prefix each component so distinct sequences cannot alias, e.g.
+ // paths ["a", "b"] -> "1:a1:b" while ["a:b"] -> "3:a:b".
+ auto append = [&key](std::string_view component) {
+ fmt::format_to(key, "{}:", component.size());
+ key.append(component.data(), component.data() + component.size());
+ };
+ append(slot.col_name());
+ append(std::to_string(column_idx));
+ append(std::to_string(slot.col_unique_id()));
+ append(std::to_string(slot.column_paths().size()));
+ for (const auto& path : slot.column_paths()) {
+ append(path);
+ }
+ append(std::to_string(slot.all_access_paths().size()));
+ // Encode each optional sub-path's presence bit separately from its element
+ // count so an absent path ("0") never aliases a present-but-empty path
+ // ("1" + size "0").
+ auto append_optional_path = [&append](bool is_set, const
std::vector<std::string>& items) {
+ append(is_set ? "1" : "0");
+ if (is_set) {
+ append(std::to_string(items.size()));
+ for (const auto& item : items) {
+ append(item);
+ }
+ }
+ };
+ for (const auto& path : slot.all_access_paths()) {
+ append(fmt::format("{}", path.type));
+ append_optional_path(path.__isset.data_access_path,
path.data_access_path.path);
+ append_optional_path(path.__isset.meta_access_path,
path.meta_access_path.path);
+ }
+ return fmt::to_string(key);
+}
+
+Status RowIdStorageReader::submit_external_scan_tasks(
+ ScannerScheduler* scheduler, std::counting_semaphore<>& semaphore,
size_t task_count,
+ const std::function<std::string(size_t)>& make_task_id,
+ const std::function<Status(size_t)>& run_task) {
+ // `completed_count` is a plain counter guarded by `mtx`; the same mutex
guards
+ // the wait predicate below, so a worker can never notify between the
waiter's
+ // predicate check and its wait.
+ AtomicStatus scan_status;
+ std::condition_variable cv;
+ std::mutex mtx;
+ size_t completed_count = 0;
+
+ // Only tasks the scheduler actually accepted are waited for. If a
submission
+ // fails we stop submitting, but still wait for the already-accepted tasks
so
+ // their workers cannot outlive the locals they capture by reference.
+ size_t submitted_count = 0;
+ for (size_t idx = 0; idx < task_count; ++idx) {
+ semaphore.acquire();
+ auto run_one_task = [&, idx]() -> bool {
+ Status task_status = Status::OK();
+ // Publish the status before the completion signal wakes the
waiter, on every
+ // path. A scanner that throws would otherwise leave scan_status
OK while this
+ // Defer still counts the task as finished, and the caller would
report success
+ // over a half-filled result block.
+ Defer complete([&] {
+ scan_status.update(task_status);
+ std::lock_guard<std::mutex> lock(mtx);
+ ++completed_count;
+ cv.notify_one();
+ });
+ ASSIGN_STATUS_IF_CATCH_EXCEPTION(task_status = run_task(idx),
task_status);
+ return true;
+ };
+ Status submit_st = scheduler->submit_scan_task(
+ SimplifiedScanTask(run_one_task, nullptr, nullptr),
make_task_id(idx));
+ if (!submit_st.ok()) {
+ scan_status.update(submit_st);
+ semaphore.release();
+ break;
+ }
+ ++submitted_count;
+ }
+
+ {
+ std::unique_lock<std::mutex> lock(mtx);
+ cv.wait(lock, [&] { return completed_count == submitted_count; });
+ }
+ return scan_status.ok() ? Status::OK() : scan_status.status();
+}
+
Status RowIdStorageReader::read_batch_external_row(
const uint64_t workload_group_id, const PRequestBlockDesc&
request_block_desc,
std::shared_ptr<IdFileMap> id_file_map, std::vector<SlotDescriptor>&
slots,
@@ -517,12 +619,21 @@ Status RowIdStorageReader::read_batch_external_row(
TupleDescriptor tuple_desc(request_block_desc.desc(), false);
std::unordered_map<std::string, int> colname_to_slot_id;
std::shared_ptr<RuntimeState> runtime_state = nullptr;
+ std::vector<SlotDescriptor> scan_slots;
+ std::vector<size_t> result_column_to_scan_column;
+ std::vector<uint32_t> scan_column_idxs;
int max_file_scanners = 0;
{
if (result_block.is_empty_column()) [[likely]] {
result_block = Block(slots, request_block_desc.row_id_size());
}
+ if (request_block_desc.column_idxs_size() != slots.size()) {
+ return Status::InternalError(
+ "Row id fetch request has mismatched slots and column
indexes, "
+ "query_id={}, slots={}, column_idxs={}",
+ print_id(query_id), slots.size(),
request_block_desc.column_idxs_size());
+ }
auto& external_info = first_file_mapping->get_external_file_info();
int plan_node_id = external_info.plan_node_id;
@@ -538,13 +649,32 @@ Status RowIdStorageReader::read_batch_external_row(
std::set
partition_name_set(first_scan_range_desc.columns_from_path_keys.begin(),
first_scan_range_desc.columns_from_path_keys.end());
+
+ std::unordered_map<std::string, size_t> source_column_to_scan_idx;
+
+ result_column_to_scan_column.reserve(slots.size());
+ scan_slots.reserve(slots.size());
+ scan_column_idxs.reserve(slots.size());
for (auto slot_idx = 0; slot_idx < slots.size(); ++slot_idx) {
- auto& slot = slots[slot_idx];
+ const auto& slot = slots[slot_idx];
+ const auto column_idx = request_block_desc.column_idxs(slot_idx);
+ const auto key = source_column_key(slot, column_idx);
+ auto [it, inserted] =
+ source_column_to_scan_idx.emplace(key,
source_column_to_scan_idx.size());
+ result_column_to_scan_column.emplace_back(it->second);
+ if (inserted) {
+ scan_slots.emplace_back(slot);
+ scan_column_idxs.emplace_back(column_idx);
+ }
+ }
+
+ for (auto slot_idx = 0; slot_idx < scan_slots.size(); ++slot_idx) {
+ auto& slot = scan_slots[slot_idx];
tuple_desc.add_slot(&slot);
- colname_to_slot_id.emplace(slot.col_name(), slot.id());
+ colname_to_slot_id[slot.col_name()] = slot.id();
TFileScanSlotInfo slot_info;
slot_info.slot_id = slot.id();
- auto column_idx = request_block_desc.column_idxs(slot_idx);
+ auto column_idx = scan_column_idxs[slot_idx];
if (partition_name_set.contains(slot.col_name())) {
//This is partition column.
slot_info.is_file_slot = false;
@@ -637,42 +767,69 @@ Status RowIdStorageReader::read_batch_external_row(
int64_t scan_running_time = 0;
RETURN_IF_ERROR(scope_timer_run(
[&]() -> Status {
- // Make sure to insert data into result_block only after all
scan tasks have been executed.
- std::atomic<int> producer_count {0};
- std::condition_variable cv;
- std::mutex mtx;
-
//semaphore: Limit the number of scan tasks submitted at one
time
std::counting_semaphore semaphore {max_file_scanners};
- size_t idx = 0;
+ std::vector<std::pair<std::multimap<segment_v2::rowid_t,
size_t>,
+ std::shared_ptr<FileMapping>>>
+ scan_info_list;
+ scan_info_list.reserve(scan_rows.size());
for (const auto& [_, scan_info] : scan_rows) {
- semaphore.acquire();
- RETURN_IF_ERROR(remote_scan_sched->submit_scan_task(
- SimplifiedScanTask(
- [&, idx, scan_info]() -> Status {
- const auto& [row_ids, file_mapping] =
scan_info;
- return
read_external_row_from_file_mapping(
- idx, row_ids, file_mapping,
slots, query_id,
- runtime_state, scan_blocks,
row_id_block_idx,
- fetch_statistics,
rpc_scan_params,
- colname_to_slot_id,
producer_count,
- scan_rows.size(), semaphore,
cv, mtx, tuple_desc);
- },
- nullptr, nullptr),
- fmt::format("{}-read_batch_external_row-{}",
print_id(query_id), idx)));
- idx++;
+ scan_info_list.emplace_back(scan_info);
}
- {
- std::unique_lock<std::mutex> lock(mtx);
- cv.wait(lock, [&] { return producer_count ==
scan_rows.size(); });
- }
- return Status::OK();
+ return submit_external_scan_tasks(
+ remote_scan_sched, semaphore, scan_rows.size(),
+ [&](size_t idx) {
+ return
fmt::format("{}-read_batch_external_row-{}", print_id(query_id),
+ idx);
+ },
+ [&](size_t idx) -> Status {
+ const auto& [row_ids, file_mapping] =
scan_info_list[idx];
+ return read_external_row_from_file_mapping(
+ idx, row_ids, file_mapping, scan_slots,
query_id, runtime_state,
+ scan_blocks, row_id_block_idx,
fetch_statistics,
+ rpc_scan_params, colname_to_slot_id,
semaphore, tuple_desc);
+ });
},
&scan_running_time));
- scatter_scan_blocks_to_result_block(row_id_block_idx, scan_blocks,
result_block);
+ // Insert the read data into result_block. Use insert_indices_from()
instead of
+ // scatter_scan_blocks_to_result_block()/insert_from_multi_column(),
because
+ // scan_blocks may have fewer columns than result_block when duplicate
physical columns
+ // are deduplicated, and insert_from_multi_column() cannot handle
ColumnString
+ // cross-type (32/64) copies safely.
+ const size_t result_column_count = result_block.columns();
+ for (size_t column_id = 0; column_id < result_column_count; column_id++) {
+ auto dst_col_guard = result_block.mutate_column_scoped(column_id);
+ MutableColumnPtr& dst_col = dst_col_guard.mutable_column();
+
+ bool dst_is_nullable = dst_col->is_nullable();
+ std::vector<ColumnPtr> nullable_src_columns(scan_blocks.size());
+ auto scan_column_id = result_column_to_scan_column[column_id];
+ for (const auto& [pos_block, block_idx] : row_id_block_idx) {
+ DCHECK_GT(scan_blocks.size(), pos_block);
+ DCHECK_GT(scan_blocks[pos_block].columns(), scan_column_id);
+ const auto& src_column_ptr =
+
scan_blocks[pos_block].get_by_position(scan_column_id).column;
+ const auto* src_col = src_column_ptr.get();
+ if (dst_is_nullable && !src_col->is_nullable()) {
+ if (!nullable_src_columns[pos_block]) {
+ nullable_src_columns[pos_block] =
make_nullable(src_column_ptr);
+ }
+ src_col = nullable_src_columns[pos_block].get();
+ }
+ if (block_idx >= src_col->size()) {
+ return Status::InternalError(
+ "Row id fetch source index out of range, query_id={},
column={}, "
+ "source_block={}, source_rows={}, row_index={}",
+ print_id(query_id),
result_block.get_by_position(column_id).name, pos_block,
+ src_col->size(), block_idx);
+ }
+ uint32_t scan_position = cast_set<uint32_t>(block_idx);
+ dst_col->insert_indices_from(*src_col, &scan_position,
&scan_position + 1);
+ }
+ }
// Statistical runtime profile information.
std::unique_ptr<RuntimeProfile> runtime_profile =
diff --git a/be/src/exec/rowid_fetcher.h b/be/src/exec/rowid_fetcher.h
index 55bab866ff1..7c2dd3d1378 100644
--- a/be/src/exec/rowid_fetcher.h
+++ b/be/src/exec/rowid_fetcher.h
@@ -20,6 +20,7 @@
#include <gen_cpp/internal_service.pb.h>
#include <condition_variable>
+#include <functional>
#include <memory>
#include <mutex>
#include <semaphore>
@@ -35,6 +36,7 @@ namespace doris {
class RuntimeState;
class TupleDescriptor;
+class ScannerScheduler;
namespace io {
enum class FileCacheMissPolicy : uint8_t;
}
@@ -108,15 +110,23 @@ private:
static Status read_external_row_from_file_mapping(
size_t idx, const std::multimap<segment_v2::rowid_t, size_t>&
row_ids,
const std::shared_ptr<FileMapping>& file_mapping,
- const std::vector<SlotDescriptor>& slots, const TUniqueId&
query_id,
+ const std::vector<SlotDescriptor>& scan_slots, const TUniqueId&
query_id,
const std::shared_ptr<RuntimeState>& runtime_state,
std::vector<Block>& scan_blocks,
std::vector<std::pair<size_t, size_t>>& row_id_block_idx,
std::vector<ExternalFetchStatistics>& fetch_statistics,
const TFileScanRangeParams& rpc_scan_params,
const std::unordered_map<std::string, int>& colname_to_slot_id,
- std::atomic<int>& producer_count, size_t scan_rows_count,
- std::counting_semaphore<>& semaphore, std::condition_variable& cv,
std::mutex& mtx,
- TupleDescriptor& tuple_desc);
+ std::counting_semaphore<>& semaphore, TupleDescriptor& tuple_desc);
+
+ static std::string source_column_key(const SlotDescriptor& slot, uint32_t
column_idx);
+
+ friend class RowIdStorageReaderTest;
+
+ static Status submit_external_scan_tasks(ScannerScheduler* scheduler,
+ std::counting_semaphore<>&
semaphore,
+ size_t task_count,
+ const
std::function<std::string(size_t)>& make_task_id,
+ const
std::function<Status(size_t)>& run_task);
struct ExternalFetchStatistics {
int64_t init_reader_ms = 0;
diff --git a/be/test/exec/rowid_fetcher_test.cpp
b/be/test/exec/rowid_fetcher_test.cpp
new file mode 100644
index 00000000000..b2fcbbac236
--- /dev/null
+++ b/be/test/exec/rowid_fetcher_test.cpp
@@ -0,0 +1,216 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/rowid_fetcher.h"
+
+#include <gtest/gtest.h>
+
+#include <string>
+#include <vector>
+
+#include "runtime/descriptor_helper.h"
+#include "runtime/descriptors.h"
+
+namespace doris {
+
+// source_column_key() decides which projected slots collapse onto a single
scanned
+// column. Two slots that name the same physical column must produce the same
key, and
+// any two slots that do not must produce different ones -- a false merge
drops a column
+// the result block still expects, which is the crash this keying was
introduced to fix.
+class RowIdStorageReaderTest : public testing::Test {
+public:
+ static std::string key_of(const SlotDescriptor& slot, uint32_t column_idx)
{
+ return RowIdStorageReader::source_column_key(slot, column_idx);
+ }
+
+protected:
+ struct SlotSpec {
+ std::string col_name = "c";
+ int32_t col_unique_id = 1;
+ std::vector<std::string> column_paths = {};
+ TColumnAccessPaths access_paths = {};
+ };
+
+ static SlotDescriptor make_slot(const SlotSpec& spec) {
+ TSlotDescriptor tdesc = TSlotDescriptorBuilder()
+ .type(TYPE_INT)
+ .nullable(true)
+ .column_name(spec.col_name)
+ .column_pos(0)
+ .build();
+ tdesc.__set_col_unique_id(spec.col_unique_id);
+ tdesc.__set_column_paths(spec.column_paths);
+ if (!spec.access_paths.empty()) {
+ tdesc.__set_all_access_paths(spec.access_paths);
+ }
+ return SlotDescriptor(tdesc);
+ }
+
+ static TColumnAccessPath data_path(const std::vector<std::string>& path) {
+ TColumnAccessPath access_path;
+ access_path.type = TAccessPathType::DATA;
+ TDataAccessPath data;
+ data.__set_path(path);
+ access_path.__set_data_access_path(data);
+ return access_path;
+ }
+
+ static TColumnAccessPath bare_path() {
+ TColumnAccessPath access_path;
+ access_path.type = TAccessPathType::DATA;
+ return access_path;
+ }
+};
+
+TEST_F(RowIdStorageReaderTest, SameSourceColumnSharesKey) {
+ // The bug case: one physical column projected twice must dedup onto one
scan column.
+ const SlotDescriptor first = make_slot({});
+ const SlotDescriptor second = make_slot({});
+ EXPECT_EQ(key_of(first, 3), key_of(second, 3));
+}
+
+TEST_F(RowIdStorageReaderTest, ColumnIndexSeparatesKeys) {
+ const SlotDescriptor slot = make_slot({});
+ EXPECT_NE(key_of(slot, 3), key_of(slot, 4));
+}
+
+TEST_F(RowIdStorageReaderTest, ColumnNameSeparatesKeys) {
+ EXPECT_NE(key_of(make_slot({.col_name = "a"}), 0),
key_of(make_slot({.col_name = "b"}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, UniqueIdSeparatesKeys) {
+ EXPECT_NE(key_of(make_slot({.col_unique_id = 1}), 0),
+ key_of(make_slot({.col_unique_id = 2}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, NameAndIndexBoundaryIsNotAmbiguous) {
+ // Without length prefixes, "a" + idx 12 and "a1" + idx 2 both flatten to
"a12".
+ EXPECT_NE(key_of(make_slot({.col_name = "a"}), 12),
key_of(make_slot({.col_name = "a1"}), 2));
+}
+
+TEST_F(RowIdStorageReaderTest, PathComponentBoundaryIsNotAmbiguous) {
+ // The concatenation hazard the length prefix exists for: ["a", "b"] and
["a:b"] are
+ // different nested columns but share the naive ':'-joined spelling.
+ EXPECT_NE(key_of(make_slot({.column_paths = {"a", "b"}}), 0),
+ key_of(make_slot({.column_paths = {"a:b"}}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, EmptyPathIsNotTheSameAsNoPath) {
+ EXPECT_NE(key_of(make_slot({.column_paths = {}}), 0),
+ key_of(make_slot({.column_paths = {""}}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, PathOrderMatters) {
+ EXPECT_NE(key_of(make_slot({.column_paths = {"a", "b"}}), 0),
+ key_of(make_slot({.column_paths = {"b", "a"}}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, EqualPathsShareKey) {
+ EXPECT_EQ(key_of(make_slot({.column_paths = {"a", "b"}}), 0),
+ key_of(make_slot({.column_paths = {"a", "b"}}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, AccessPathSeparatesKeys) {
+ EXPECT_NE(key_of(make_slot({.access_paths = {data_path({"a"})}}), 0),
+ key_of(make_slot({.access_paths = {data_path({"b"})}}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, AbsentAccessPathIsNotAnEmptyOne) {
+ // The presence bit: an unset data_access_path must not collide with one
that is set
+ // but carries no components.
+ EXPECT_NE(key_of(make_slot({.access_paths = {bare_path()}}), 0),
+ key_of(make_slot({.access_paths = {data_path({})}}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, AccessPathCountSeparatesKeys) {
+ EXPECT_NE(key_of(make_slot({.access_paths = {data_path({"a"})}}), 0),
+ key_of(make_slot({.access_paths = {data_path({"a"}),
data_path({"b"})}}), 0));
+}
+
+// Runs every submitted task on the submitting thread. The point of these
cases is which
+// status reaches the caller, not the threading, and inline execution keeps
them
+// deterministic.
+class InlineScanScheduler : public ScannerScheduler {
+public:
+ Status start(int, int, int, int) override { return Status::OK(); }
+ void stop() override {}
+ Status submit_scan_task(SimplifiedScanTask scan_task) override {
+ scan_task.scan_func();
+ return Status::OK();
+ }
+ Status submit_scan_task(SimplifiedScanTask scan_task, const std::string&)
override {
+ scan_task.scan_func();
+ return Status::OK();
+ }
+ void reset_thread_num(int, int, int) override {}
+ int get_queue_size() override { return 0; }
+ int get_active_threads() override { return 0; }
+ std::vector<int> thread_debug_info() override { return {}; }
+ Status schedule_scan_task(std::shared_ptr<ScannerContext>,
std::shared_ptr<ScanTask>,
+ std::unique_lock<std::mutex>&) override {
+ return Status::OK();
+ }
+};
+
+// submit_external_scan_tasks() signals completion from a Defer, so a worker
that leaves
+// without publishing its status would still wake the waiter and the caller
would report
+// success over a partially filled result block.
+class SubmitExternalScanTasksTest : public RowIdStorageReaderTest {
+protected:
+ static constexpr size_t kTaskCount = 3;
+
+ static Status run_tasks(const std::function<Status(size_t)>& run_task) {
+ InlineScanScheduler scheduler;
+ std::counting_semaphore<> semaphore {kTaskCount};
+ return RowIdStorageReader::submit_external_scan_tasks(
+ &scheduler, semaphore, kTaskCount,
+ [](size_t idx) { return fmt::format("task-{}", idx); },
run_task);
+ }
+};
+
+TEST_F(SubmitExternalScanTasksTest, AllTasksSucceedingReturnsOk) {
+ size_t ran = 0;
+ EXPECT_TRUE(run_tasks([&](size_t) -> Status {
+ ++ran;
+ return Status::OK();
+ }).ok());
+ EXPECT_EQ(ran, kTaskCount);
+}
+
+TEST_F(SubmitExternalScanTasksTest, ReturnedErrorReachesTheCaller) {
+ const Status result = run_tasks([](size_t idx) -> Status {
+ return idx == kTaskCount - 1 ? Status::InternalError("scanner returned
an error")
+ : Status::OK();
+ });
+ EXPECT_FALSE(result.ok());
+ EXPECT_NE(result.to_string().find("scanner returned an error"),
std::string::npos);
+}
+
+TEST_F(SubmitExternalScanTasksTest, ThrownExceptionReachesTheCaller) {
+ // The last task is the interesting one: it is the completion that
releases the
+ // waiter, so a status lost here is a status the caller never sees.
+ const Status result = run_tasks([](size_t idx) -> Status {
+ if (idx == kTaskCount - 1) {
+ throw Exception(ErrorCode::INTERNAL_ERROR, "scanner threw");
+ }
+ return Status::OK();
+ });
+ EXPECT_FALSE(result.ok());
+ EXPECT_NE(result.to_string().find("scanner threw"), std::string::npos);
+}
+
+} // namespace doris
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out
b/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out
index 0d4e28fd9f0..4f1a0c6987e 100644
--- a/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out
+++ b/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out
@@ -47,6 +47,54 @@
17.0 user17 17 2
18.0 user18 18 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+user4 user4
+user5 user5
+user6 user6
+user7 user7
+user8 user8
+user9 user9
+user10 user10
+
+-- !dup_col_thrice --
+user1 user1 user1
+user2 user2 user2
+user3 user3 user3
+user4 user4 user4
+user5 user5 user5
+user6 user6 user6
+user7 user7 user7
+user8 user8 user8
+user9 user9 user9
+user10 user10 user10
+
+-- !dup_col_mixed --
+1 user1 0.5 user1
+2 user2 1 user2
+3 user3 1.5 user3
+4 user4 2 user4
+5 user5 2.5 user5
+6 user6 3 user6
+7 user7 3.5 user7
+8 user8 4 user8
+9 user9 4.5 user9
+10 user10 5 user10
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+3 3
+3.5 3.5
+4 4
+4.5 4.5
+5 5
+
-- !test_basic --
1 user1 1.0 false 0.5 1
2 user2 2.0 true 1.0 1
@@ -293,6 +341,54 @@
17.0 user17 17 2
18.0 user18 18 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+user4 user4
+user5 user5
+user6 user6
+user7 user7
+user8 user8
+user9 user9
+user10 user10
+
+-- !dup_col_thrice --
+user1 user1 user1
+user2 user2 user2
+user3 user3 user3
+user4 user4 user4
+user5 user5 user5
+user6 user6 user6
+user7 user7 user7
+user8 user8 user8
+user9 user9 user9
+user10 user10 user10
+
+-- !dup_col_mixed --
+1 user1 0.5 user1
+2 user2 1 user2
+3 user3 1.5 user3
+4 user4 2 user4
+5 user5 2.5 user5
+6 user6 3 user6
+7 user7 3.5 user7
+8 user8 4 user8
+9 user9 4.5 user9
+10 user10 5 user10
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+3 3
+3.5 3.5
+4 4
+4.5 4.5
+5 5
+
-- !test_basic --
1 user1 1.0 false 0.5 1
2 user2 2.0 true 1.0 1
@@ -721,6 +817,11 @@
1 user1 1.0 false 0.5 1 8 user8 8.0 true
4.0 1
1 user1 1.0 false 0.5 1 9 user9 9.0 false
4.5 1
+-- !test_join_dup_cols --
+user1 user1 1 1
+user2 user2 2 2
+user3 user3 3 3
+
-- !test_join1 --
2 user2 2.0 1.0
4 user4 4.0 2.0
@@ -966,6 +1067,16 @@
1 user1 1.0 false 0.5 1 8 user8 8.0 true
4.0 1
1 user1 1.0 false 0.5 1 9 user9 9.0 false
4.5 1
+-- !test_join_dup_cols --
+user1 user1 1 1
+user2 user2 2 2
+user3 user3 3 3
+user4 user4 4 4
+user5 user5 5 5
+user6 user6 6 6
+user7 user7 7 7
+user8 user8 8 8
+
-- !test_join1 --
2 user2 2.0 1.0
4 user4 4.0 2.0
@@ -1240,6 +1351,35 @@
1 user1 1.0 false 0.5 1 8 user8 8.0 true
4.0 1
1 user1 1.0 false 0.5 1 9 user9 9.0 false
4.5 1
+-- !test_join_dup_cols --
+user1 user1 1 1
+user2 user2 2 2
+user3 user3 3 3
+user4 user4 4 4
+user5 user5 5 5
+user6 user6 6 6
+user7 user7 7 7
+user8 user8 8 8
+user9 user9 9 9
+user10 user10 10 10
+user11 user11 11 11
+user12 user12 12 12
+user13 user13 13 13
+user14 user14 14 14
+user15 user15 15 15
+user16 user16 16 16
+user17 user17 17 17
+user18 user18 18 18
+user19 user19 19 19
+user20 user20 20 20
+user21 user21 21 21
+user22 user22 22 22
+user23 user23 23 23
+user24 user24 24 24
+user25 user25 25 25
+user26 user26 26 26
+user27 user27 27 27
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1",
null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -1301,6 +1441,31 @@ text_22 {22:["a", null], 23:["b", "c"], 24:["dd",
null]} {"a":22, "b":[4, 5]}
3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"],
3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1",
"v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null],
2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1",
null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"],
4:null, 5:["z1", "z2"]} text_3
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1",
null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -1407,6 +1572,56 @@ text_17 {17:["t", null], 18:["u1", "u2"], 19:null}
{"a":null, "b":[17, 18]}
13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"],
15:["f1", null]}
15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"],
17:null}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"],
3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1",
"v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null],
5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null],
7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"],
10:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null],
2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1",
null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"],
4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1",
null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"],
6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1",
null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null],
9:["d1", "d2"], 10:null} text_8
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1",
null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -1597,6 +1812,123 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]}
{"a":1, "b":[1, 2]}
21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null],
23:["k1", "k2"]}
23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"],
25:["tt", null]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+{"a":9, "b":[9, 10]} {"a":9, "b":[9, 10]}
+{"a":10, "b":null} {"a":10, "b":null}
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]}
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]}
+{"a":13, "b":null} {"a":13, "b":null}
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]}
+{"a":null, "b":null} {"a":null, "b":null}
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]}
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]}
+{"a":18, "b":null} {"a":18, "b":null}
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":21, "b":null} {"a":21, "b":null}
+{"a":22, "b":[4, 5]} {"a":22, "b":[4, 5]}
+{"a":null, "b":[6, 7]} {"a":null, "b":[6, 7]}
+{"a":24, "b":null} {"a":24, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"],
3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1",
"v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null],
5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null],
7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"],
10:null}
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {9:["aa", null],
10:["bb", "cc"], 11:["mm", null]}
+{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {10:["q1", null],
11:["w1", "w2"], 12:["x", null]}
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {11:["x", null],
12:["u1", "u2"], 13:["l", null]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {12:["p", null], 13:["k1",
"k2"], 14:null}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {13:["d", null],
14:["e1", "e2"], 15:["f1", null]}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {14:["f", null],
15:["g1", "g2"], 16:["h1", null]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {15:["z", null], 16:["y1",
"y2"], 17:null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {16:["i", null],
17:["j1", "j2"], 18:["k", null]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {17:["t", null], 18:["u1",
"u2"], 19:null}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {18:["v", null],
19:["w1", "w2"], 20:["aa", null]}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {19:["x1", null],
20:["x2", "x3"], 21:["pp", null]}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {20:["y1", null], 21:null,
22:["qq", "rr"]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {21:["z1", "z2"],
22:["m1", null], 23:["k1", "k2"]}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {22:["a", null],
23:["b", "c"], 24:["dd", null]}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {23:["k", null],
24:["q1", "q2"], 25:["tt", null]}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {24:["u1", null], 25:["u2",
"u3"], 26:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+text_9 text_9
+text_10 text_10
+text_11 text_11
+text_12 text_12
+text_13 text_13
+text_14 text_14
+text_15 text_15
+text_16 text_16
+text_17 text_17
+text_18 text_18
+text_19 text_19
+text_20 text_20
+text_21 text_21
+text_22 text_22
+text_23 text_23
+text_24 text_24
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null],
2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1",
null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"],
4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1",
null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"],
6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1",
null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null],
9:["d1", "d2"], 10:null} text_8
+9 {"a":9, "b":[9, 10]} text_9 {"a":9, "b":[9, 10]} {9:["aa",
null], 10:["bb", "cc"], 11:["mm", null]} text_9
+10 {"a":10, "b":null} text_10 {"a":10, "b":null} {10:["q1",
null], 11:["w1", "w2"], 12:["x", null]} text_10
+11 {"a":11, "b":[11, 12]} text_11 {"a":11, "b":[11, 12]} {11:["x",
null], 12:["u1", "u2"], 13:["l", null]} text_11
+12 {"a":null, "b":[12, 13]} text_12 {"a":null, "b":[12, 13]}
{12:["p", null], 13:["k1", "k2"], 14:null} text_12
+13 {"a":13, "b":null} text_13 {"a":13, "b":null} {13:["d",
null], 14:["e1", "e2"], 15:["f1", null]} text_13
+14 {"a":14, "b":[14, 15]} text_14 {"a":14, "b":[14, 15]} {14:["f",
null], 15:["g1", "g2"], 16:["h1", null]} text_14
+15 {"a":null, "b":null} text_15 {"a":null, "b":null} {15:["z",
null], 16:["y1", "y2"], 17:null} text_15
+16 {"a":16, "b":[16, 17]} text_16 {"a":16, "b":[16, 17]} {16:["i",
null], 17:["j1", "j2"], 18:["k", null]} text_16
+17 {"a":null, "b":[17, 18]} text_17 {"a":null, "b":[17, 18]}
{17:["t", null], 18:["u1", "u2"], 19:null} text_17
+18 {"a":18, "b":null} text_18 {"a":18, "b":null} {18:["v",
null], 19:["w1", "w2"], 20:["aa", null]} text_18
+19 {"a":19, "b":[1, null]} text_19 {"a":19, "b":[1, null]} {19:["x1",
null], 20:["x2", "x3"], 21:["pp", null]} text_19
+20 {"a":null, "b":[2, 3]} text_20 {"a":null, "b":[2, 3]} {20:["y1",
null], 21:null, 22:["qq", "rr"]} text_20
+21 {"a":21, "b":null} text_21 {"a":21, "b":null} {21:["z1",
"z2"], 22:["m1", null], 23:["k1", "k2"]} text_21
+22 {"a":22, "b":[4, 5]} text_22 {"a":22, "b":[4, 5]} {22:["a",
null], 23:["b", "c"], 24:["dd", null]} text_22
+23 {"a":null, "b":[6, 7]} text_23 {"a":null, "b":[6, 7]} {23:["k",
null], 24:["q1", "q2"], 25:["tt", null]} text_23
+24 {"a":24, "b":null} text_24 {"a":24, "b":null} {24:["u1",
null], 25:["u2", "u3"], 26:null} text_24
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} text_17
+{"a":18, "b":null} {"a":18, "b":null} text_18
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} text_19
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1",
null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -1658,6 +1990,31 @@ text_22 {22:["a", null], 23:["b", "c"], 24:["dd",
null]} {"a":22, "b":[4, 5]}
3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"],
3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1",
"v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null],
2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1",
null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"],
4:null, 5:["z1", "z2"]} text_3
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1",
null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -1764,6 +2121,56 @@ text_17 {17:["t", null], 18:["u1", "u2"], 19:null}
{"a":null, "b":[17, 18]}
13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"],
15:["f1", null]}
15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"],
17:null}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"],
3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1",
"v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null],
5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null],
7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"],
10:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null],
2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1",
null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"],
4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1",
null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"],
6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1",
null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null],
9:["d1", "d2"], 10:null} text_8
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1",
null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -1954,6 +2361,123 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]}
{"a":1, "b":[1, 2]}
21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null],
23:["k1", "k2"]}
23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"],
25:["tt", null]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+{"a":9, "b":[9, 10]} {"a":9, "b":[9, 10]}
+{"a":10, "b":null} {"a":10, "b":null}
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]}
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]}
+{"a":13, "b":null} {"a":13, "b":null}
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]}
+{"a":null, "b":null} {"a":null, "b":null}
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]}
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]}
+{"a":18, "b":null} {"a":18, "b":null}
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":21, "b":null} {"a":21, "b":null}
+{"a":22, "b":[4, 5]} {"a":22, "b":[4, 5]}
+{"a":null, "b":[6, 7]} {"a":null, "b":[6, 7]}
+{"a":24, "b":null} {"a":24, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"],
3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1",
"v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null],
5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null],
7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"],
10:null}
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {9:["aa", null],
10:["bb", "cc"], 11:["mm", null]}
+{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {10:["q1", null],
11:["w1", "w2"], 12:["x", null]}
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {11:["x", null],
12:["u1", "u2"], 13:["l", null]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {12:["p", null], 13:["k1",
"k2"], 14:null}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {13:["d", null],
14:["e1", "e2"], 15:["f1", null]}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {14:["f", null],
15:["g1", "g2"], 16:["h1", null]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {15:["z", null], 16:["y1",
"y2"], 17:null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {16:["i", null],
17:["j1", "j2"], 18:["k", null]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {17:["t", null], 18:["u1",
"u2"], 19:null}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {18:["v", null],
19:["w1", "w2"], 20:["aa", null]}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {19:["x1", null],
20:["x2", "x3"], 21:["pp", null]}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {20:["y1", null], 21:null,
22:["qq", "rr"]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {21:["z1", "z2"],
22:["m1", null], 23:["k1", "k2"]}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {22:["a", null],
23:["b", "c"], 24:["dd", null]}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {23:["k", null],
24:["q1", "q2"], 25:["tt", null]}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {24:["u1", null], 25:["u2",
"u3"], 26:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+text_9 text_9
+text_10 text_10
+text_11 text_11
+text_12 text_12
+text_13 text_13
+text_14 text_14
+text_15 text_15
+text_16 text_16
+text_17 text_17
+text_18 text_18
+text_19 text_19
+text_20 text_20
+text_21 text_21
+text_22 text_22
+text_23 text_23
+text_24 text_24
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null],
2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1",
null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"],
4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1",
null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"],
6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1",
null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null],
9:["d1", "d2"], 10:null} text_8
+9 {"a":9, "b":[9, 10]} text_9 {"a":9, "b":[9, 10]} {9:["aa",
null], 10:["bb", "cc"], 11:["mm", null]} text_9
+10 {"a":10, "b":null} text_10 {"a":10, "b":null} {10:["q1",
null], 11:["w1", "w2"], 12:["x", null]} text_10
+11 {"a":11, "b":[11, 12]} text_11 {"a":11, "b":[11, 12]} {11:["x",
null], 12:["u1", "u2"], 13:["l", null]} text_11
+12 {"a":null, "b":[12, 13]} text_12 {"a":null, "b":[12, 13]}
{12:["p", null], 13:["k1", "k2"], 14:null} text_12
+13 {"a":13, "b":null} text_13 {"a":13, "b":null} {13:["d",
null], 14:["e1", "e2"], 15:["f1", null]} text_13
+14 {"a":14, "b":[14, 15]} text_14 {"a":14, "b":[14, 15]} {14:["f",
null], 15:["g1", "g2"], 16:["h1", null]} text_14
+15 {"a":null, "b":null} text_15 {"a":null, "b":null} {15:["z",
null], 16:["y1", "y2"], 17:null} text_15
+16 {"a":16, "b":[16, 17]} text_16 {"a":16, "b":[16, 17]} {16:["i",
null], 17:["j1", "j2"], 18:["k", null]} text_16
+17 {"a":null, "b":[17, 18]} text_17 {"a":null, "b":[17, 18]}
{17:["t", null], 18:["u1", "u2"], 19:null} text_17
+18 {"a":18, "b":null} text_18 {"a":18, "b":null} {18:["v",
null], 19:["w1", "w2"], 20:["aa", null]} text_18
+19 {"a":19, "b":[1, null]} text_19 {"a":19, "b":[1, null]} {19:["x1",
null], 20:["x2", "x3"], 21:["pp", null]} text_19
+20 {"a":null, "b":[2, 3]} text_20 {"a":null, "b":[2, 3]} {20:["y1",
null], 21:null, 22:["qq", "rr"]} text_20
+21 {"a":21, "b":null} text_21 {"a":21, "b":null} {21:["z1",
"z2"], 22:["m1", null], 23:["k1", "k2"]} text_21
+22 {"a":22, "b":[4, 5]} text_22 {"a":22, "b":[4, 5]} {22:["a",
null], 23:["b", "c"], 24:["dd", null]} text_22
+23 {"a":null, "b":[6, 7]} text_23 {"a":null, "b":[6, 7]} {23:["k",
null], 24:["q1", "q2"], 25:["tt", null]} text_23
+24 {"a":24, "b":null} text_24 {"a":24, "b":null} {24:["u1",
null], 25:["u2", "u3"], 26:null} text_24
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} text_17
+{"a":18, "b":null} {"a":18, "b":null} text_18
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} text_19
+
-- !1 --
1 user1 1.0 false 0.5 1
2 user2 2.0 true 1.0 1
@@ -2002,6 +2526,54 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]}
{"a":1, "b":[1, 2]}
17.0 user17 17 2
18.0 user18 18 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+user4 user4
+user5 user5
+user6 user6
+user7 user7
+user8 user8
+user9 user9
+user10 user10
+
+-- !dup_col_thrice --
+user1 user1 user1
+user2 user2 user2
+user3 user3 user3
+user4 user4 user4
+user5 user5 user5
+user6 user6 user6
+user7 user7 user7
+user8 user8 user8
+user9 user9 user9
+user10 user10 user10
+
+-- !dup_col_mixed --
+1 user1 0.5 user1
+2 user2 1 user2
+3 user3 1.5 user3
+4 user4 2 user4
+5 user5 2.5 user5
+6 user6 3 user6
+7 user7 3.5 user7
+8 user8 4 user8
+9 user9 4.5 user9
+10 user10 5 user10
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+3 3
+3.5 3.5
+4 4
+4.5 4.5
+5 5
+
-- !test_basic --
1 user1 1.0 false 0.5 1
2 user2 2.0 true 1.0 1
@@ -2248,6 +2820,54 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]}
{"a":1, "b":[1, 2]}
17.0 user17 17 2
18.0 user18 18 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+user4 user4
+user5 user5
+user6 user6
+user7 user7
+user8 user8
+user9 user9
+user10 user10
+
+-- !dup_col_thrice --
+user1 user1 user1
+user2 user2 user2
+user3 user3 user3
+user4 user4 user4
+user5 user5 user5
+user6 user6 user6
+user7 user7 user7
+user8 user8 user8
+user9 user9 user9
+user10 user10 user10
+
+-- !dup_col_mixed --
+1 user1 0.5 user1
+2 user2 1 user2
+3 user3 1.5 user3
+4 user4 2 user4
+5 user5 2.5 user5
+6 user6 3 user6
+7 user7 3.5 user7
+8 user8 4 user8
+9 user9 4.5 user9
+10 user10 5 user10
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+3 3
+3.5 3.5
+4 4
+4.5 4.5
+5 5
+
-- !test_basic --
1 user1 1.0 false 0.5 1
2 user2 2.0 true 1.0 1
@@ -2676,6 +3296,11 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]}
{"a":1, "b":[1, 2]}
1 user1 1.0 false 0.5 1 8 user8 8.0 true
4.0 1
1 user1 1.0 false 0.5 1 9 user9 9.0 false
4.5 1
+-- !test_join_dup_cols --
+user1 user1 1 1
+user2 user2 2 2
+user3 user3 3 3
+
-- !test_join1 --
2 user2 2.0 1.0
4 user4 4.0 2.0
@@ -2921,6 +3546,16 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]}
{"a":1, "b":[1, 2]}
1 user1 1.0 false 0.5 1 8 user8 8.0 true
4.0 1
1 user1 1.0 false 0.5 1 9 user9 9.0 false
4.5 1
+-- !test_join_dup_cols --
+user1 user1 1 1
+user2 user2 2 2
+user3 user3 3 3
+user4 user4 4 4
+user5 user5 5 5
+user6 user6 6 6
+user7 user7 7 7
+user8 user8 8 8
+
-- !test_join1 --
2 user2 2.0 1.0
4 user4 4.0 2.0
@@ -3195,6 +3830,35 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]}
{"a":1, "b":[1, 2]}
1 user1 1.0 false 0.5 1 8 user8 8.0 true
4.0 1
1 user1 1.0 false 0.5 1 9 user9 9.0 false
4.5 1
+-- !test_join_dup_cols --
+user1 user1 1 1
+user2 user2 2 2
+user3 user3 3 3
+user4 user4 4 4
+user5 user5 5 5
+user6 user6 6 6
+user7 user7 7 7
+user8 user8 8 8
+user9 user9 9 9
+user10 user10 10 10
+user11 user11 11 11
+user12 user12 12 12
+user13 user13 13 13
+user14 user14 14 14
+user15 user15 15 15
+user16 user16 16 16
+user17 user17 17 17
+user18 user18 18 18
+user19 user19 19 19
+user20 user20 20 20
+user21 user21 21 21
+user22 user22 22 22
+user23 user23 23 23
+user24 user24 24 24
+user25 user25 25 25
+user26 user26 26 26
+user27 user27 27 27
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1",
null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -3256,6 +3920,31 @@ text_22 {22:["a", null], 23:["b", "c"], 24:["dd",
null]} {"a":22, "b":[4, 5]}
3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"],
3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1",
"v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null],
2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1",
null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"],
4:null, 5:["z1", "z2"]} text_3
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1",
null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -3362,6 +4051,56 @@ text_17 {17:["t", null], 18:["u1", "u2"], 19:null}
{"a":null, "b":[17, 18]}
13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"],
15:["f1", null]}
15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"],
17:null}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"],
3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1",
"v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null],
5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null],
7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"],
10:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null],
2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1",
null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"],
4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1",
null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"],
6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1",
null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null],
9:["d1", "d2"], 10:null} text_8
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1",
null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -3552,6 +4291,123 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]}
{"a":1, "b":[1, 2]}
21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null],
23:["k1", "k2"]}
23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"],
25:["tt", null]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+{"a":9, "b":[9, 10]} {"a":9, "b":[9, 10]}
+{"a":10, "b":null} {"a":10, "b":null}
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]}
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]}
+{"a":13, "b":null} {"a":13, "b":null}
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]}
+{"a":null, "b":null} {"a":null, "b":null}
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]}
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]}
+{"a":18, "b":null} {"a":18, "b":null}
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":21, "b":null} {"a":21, "b":null}
+{"a":22, "b":[4, 5]} {"a":22, "b":[4, 5]}
+{"a":null, "b":[6, 7]} {"a":null, "b":[6, 7]}
+{"a":24, "b":null} {"a":24, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"],
3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1",
"v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null],
5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null],
7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"],
10:null}
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {9:["aa", null],
10:["bb", "cc"], 11:["mm", null]}
+{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {10:["q1", null],
11:["w1", "w2"], 12:["x", null]}
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {11:["x", null],
12:["u1", "u2"], 13:["l", null]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {12:["p", null], 13:["k1",
"k2"], 14:null}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {13:["d", null],
14:["e1", "e2"], 15:["f1", null]}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {14:["f", null],
15:["g1", "g2"], 16:["h1", null]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {15:["z", null], 16:["y1",
"y2"], 17:null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {16:["i", null],
17:["j1", "j2"], 18:["k", null]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {17:["t", null], 18:["u1",
"u2"], 19:null}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {18:["v", null],
19:["w1", "w2"], 20:["aa", null]}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {19:["x1", null],
20:["x2", "x3"], 21:["pp", null]}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {20:["y1", null], 21:null,
22:["qq", "rr"]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {21:["z1", "z2"],
22:["m1", null], 23:["k1", "k2"]}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {22:["a", null],
23:["b", "c"], 24:["dd", null]}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {23:["k", null],
24:["q1", "q2"], 25:["tt", null]}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {24:["u1", null], 25:["u2",
"u3"], 26:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+text_9 text_9
+text_10 text_10
+text_11 text_11
+text_12 text_12
+text_13 text_13
+text_14 text_14
+text_15 text_15
+text_16 text_16
+text_17 text_17
+text_18 text_18
+text_19 text_19
+text_20 text_20
+text_21 text_21
+text_22 text_22
+text_23 text_23
+text_24 text_24
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null],
2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1",
null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"],
4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1",
null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"],
6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1",
null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null],
9:["d1", "d2"], 10:null} text_8
+9 {"a":9, "b":[9, 10]} text_9 {"a":9, "b":[9, 10]} {9:["aa",
null], 10:["bb", "cc"], 11:["mm", null]} text_9
+10 {"a":10, "b":null} text_10 {"a":10, "b":null} {10:["q1",
null], 11:["w1", "w2"], 12:["x", null]} text_10
+11 {"a":11, "b":[11, 12]} text_11 {"a":11, "b":[11, 12]} {11:["x",
null], 12:["u1", "u2"], 13:["l", null]} text_11
+12 {"a":null, "b":[12, 13]} text_12 {"a":null, "b":[12, 13]}
{12:["p", null], 13:["k1", "k2"], 14:null} text_12
+13 {"a":13, "b":null} text_13 {"a":13, "b":null} {13:["d",
null], 14:["e1", "e2"], 15:["f1", null]} text_13
+14 {"a":14, "b":[14, 15]} text_14 {"a":14, "b":[14, 15]} {14:["f",
null], 15:["g1", "g2"], 16:["h1", null]} text_14
+15 {"a":null, "b":null} text_15 {"a":null, "b":null} {15:["z",
null], 16:["y1", "y2"], 17:null} text_15
+16 {"a":16, "b":[16, 17]} text_16 {"a":16, "b":[16, 17]} {16:["i",
null], 17:["j1", "j2"], 18:["k", null]} text_16
+17 {"a":null, "b":[17, 18]} text_17 {"a":null, "b":[17, 18]}
{17:["t", null], 18:["u1", "u2"], 19:null} text_17
+18 {"a":18, "b":null} text_18 {"a":18, "b":null} {18:["v",
null], 19:["w1", "w2"], 20:["aa", null]} text_18
+19 {"a":19, "b":[1, null]} text_19 {"a":19, "b":[1, null]} {19:["x1",
null], 20:["x2", "x3"], 21:["pp", null]} text_19
+20 {"a":null, "b":[2, 3]} text_20 {"a":null, "b":[2, 3]} {20:["y1",
null], 21:null, 22:["qq", "rr"]} text_20
+21 {"a":21, "b":null} text_21 {"a":21, "b":null} {21:["z1",
"z2"], 22:["m1", null], 23:["k1", "k2"]} text_21
+22 {"a":22, "b":[4, 5]} text_22 {"a":22, "b":[4, 5]} {22:["a",
null], 23:["b", "c"], 24:["dd", null]} text_22
+23 {"a":null, "b":[6, 7]} text_23 {"a":null, "b":[6, 7]} {23:["k",
null], 24:["q1", "q2"], 25:["tt", null]} text_23
+24 {"a":24, "b":null} text_24 {"a":24, "b":null} {24:["u1",
null], 25:["u2", "u3"], 26:null} text_24
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} text_17
+{"a":18, "b":null} {"a":18, "b":null} text_18
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} text_19
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1",
null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -3613,6 +4469,31 @@ text_22 {22:["a", null], 23:["b", "c"], 24:["dd",
null]} {"a":22, "b":[4, 5]}
3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"],
3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1",
"v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null],
2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1",
null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"],
4:null, 5:["z1", "z2"]} text_3
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1",
null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -3719,6 +4600,56 @@ text_17 {17:["t", null], 18:["u1", "u2"], 19:null}
{"a":null, "b":[17, 18]}
13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"],
15:["f1", null]}
15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"],
17:null}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"],
3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1",
"v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null],
5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null],
7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"],
10:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null],
2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1",
null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"],
4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1",
null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"],
6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1",
null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null],
9:["d1", "d2"], 10:null} text_8
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1",
null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -3908,3 +4839,120 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]}
{"a":1, "b":[1, 2]}
19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"],
21:["pp", null]}
21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null],
23:["k1", "k2"]}
23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"],
25:["tt", null]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+{"a":9, "b":[9, 10]} {"a":9, "b":[9, 10]}
+{"a":10, "b":null} {"a":10, "b":null}
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]}
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]}
+{"a":13, "b":null} {"a":13, "b":null}
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]}
+{"a":null, "b":null} {"a":null, "b":null}
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]}
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]}
+{"a":18, "b":null} {"a":18, "b":null}
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":21, "b":null} {"a":21, "b":null}
+{"a":22, "b":[4, 5]} {"a":22, "b":[4, 5]}
+{"a":null, "b":[6, 7]} {"a":null, "b":[6, 7]}
+{"a":24, "b":null} {"a":24, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"],
3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1",
"v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null],
5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null],
7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"],
10:null}
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {9:["aa", null],
10:["bb", "cc"], 11:["mm", null]}
+{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {10:["q1", null],
11:["w1", "w2"], 12:["x", null]}
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {11:["x", null],
12:["u1", "u2"], 13:["l", null]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {12:["p", null], 13:["k1",
"k2"], 14:null}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {13:["d", null],
14:["e1", "e2"], 15:["f1", null]}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {14:["f", null],
15:["g1", "g2"], 16:["h1", null]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {15:["z", null], 16:["y1",
"y2"], 17:null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {16:["i", null],
17:["j1", "j2"], 18:["k", null]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {17:["t", null], 18:["u1",
"u2"], 19:null}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {18:["v", null],
19:["w1", "w2"], 20:["aa", null]}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {19:["x1", null],
20:["x2", "x3"], 21:["pp", null]}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {20:["y1", null], 21:null,
22:["qq", "rr"]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {21:["z1", "z2"],
22:["m1", null], 23:["k1", "k2"]}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {22:["a", null],
23:["b", "c"], 24:["dd", null]}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {23:["k", null],
24:["q1", "q2"], 25:["tt", null]}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {24:["u1", null], 25:["u2",
"u3"], 26:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+text_9 text_9
+text_10 text_10
+text_11 text_11
+text_12 text_12
+text_13 text_13
+text_14 text_14
+text_15 text_15
+text_16 text_16
+text_17 text_17
+text_18 text_18
+text_19 text_19
+text_20 text_20
+text_21 text_21
+text_22 text_22
+text_23 text_23
+text_24 text_24
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null],
2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1",
null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"],
4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1",
null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"],
6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1",
null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null],
8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null],
9:["d1", "d2"], 10:null} text_8
+9 {"a":9, "b":[9, 10]} text_9 {"a":9, "b":[9, 10]} {9:["aa",
null], 10:["bb", "cc"], 11:["mm", null]} text_9
+10 {"a":10, "b":null} text_10 {"a":10, "b":null} {10:["q1",
null], 11:["w1", "w2"], 12:["x", null]} text_10
+11 {"a":11, "b":[11, 12]} text_11 {"a":11, "b":[11, 12]} {11:["x",
null], 12:["u1", "u2"], 13:["l", null]} text_11
+12 {"a":null, "b":[12, 13]} text_12 {"a":null, "b":[12, 13]}
{12:["p", null], 13:["k1", "k2"], 14:null} text_12
+13 {"a":13, "b":null} text_13 {"a":13, "b":null} {13:["d",
null], 14:["e1", "e2"], 15:["f1", null]} text_13
+14 {"a":14, "b":[14, 15]} text_14 {"a":14, "b":[14, 15]} {14:["f",
null], 15:["g1", "g2"], 16:["h1", null]} text_14
+15 {"a":null, "b":null} text_15 {"a":null, "b":null} {15:["z",
null], 16:["y1", "y2"], 17:null} text_15
+16 {"a":16, "b":[16, 17]} text_16 {"a":16, "b":[16, 17]} {16:["i",
null], 17:["j1", "j2"], 18:["k", null]} text_16
+17 {"a":null, "b":[17, 18]} text_17 {"a":null, "b":[17, 18]}
{17:["t", null], 18:["u1", "u2"], 19:null} text_17
+18 {"a":18, "b":null} text_18 {"a":18, "b":null} {18:["v",
null], 19:["w1", "w2"], 20:["aa", null]} text_18
+19 {"a":19, "b":[1, null]} text_19 {"a":19, "b":[1, null]} {19:["x1",
null], 20:["x2", "x3"], 21:["pp", null]} text_19
+20 {"a":null, "b":[2, 3]} text_20 {"a":null, "b":[2, 3]} {20:["y1",
null], 21:null, 22:["qq", "rr"]} text_20
+21 {"a":21, "b":null} text_21 {"a":21, "b":null} {21:["z1",
"z2"], 22:["m1", null], 23:["k1", "k2"]} text_21
+22 {"a":22, "b":[4, 5]} text_22 {"a":22, "b":[4, 5]} {22:["a",
null], 23:["b", "c"], 24:["dd", null]} text_22
+23 {"a":null, "b":[6, 7]} text_23 {"a":null, "b":[6, 7]} {23:["k",
null], 24:["q1", "q2"], 25:["tt", null]} text_23
+24 {"a":24, "b":null} text_24 {"a":24, "b":null} {24:["u1",
null], 25:["u2", "u3"], 26:null} text_24
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} text_17
+{"a":18, "b":null} {"a":18, "b":null} text_18
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} text_19
+
diff --git
a/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out
b/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out
index a104af3c757..f9647a18b3e 100644
--- a/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out
+++ b/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out
@@ -21,6 +21,24 @@
1.0 user1 1
2.0 user2 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+
+-- !dup_col_mixed --
+1 user1 0.5 user1 0.5
+2 user2 1 user2 1
+3 user3 1.5 user3 1.5
+4 user4 2 user4 2
+
-- !test_basic --
1 user1 1.0 false 0.5
2 user2 2.0 true 1.0
@@ -111,6 +129,24 @@
1.0 user1 1
2.0 user2 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+
+-- !dup_col_mixed --
+1 user1 0.5 user1 0.5
+2 user2 1 user2 1
+3 user3 1.5 user3 1.5
+4 user4 2 user4 2
+
-- !test_basic --
1 user1 1.0 false 0.5
2 user2 2.0 true 1.0
@@ -235,6 +271,24 @@
1.0 user1 1
2.0 user2 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+
+-- !dup_col_mixed --
+1 user1 0.5 user1 0.5
+2 user2 1 user2 1
+3 user3 1.5 user3 1.5
+4 user4 2 user4 2
+
-- !test_basic --
1 user1 1.0 false 0.5
2 user2 2.0 true 1.0
@@ -325,6 +379,24 @@
1.0 user1 1
2.0 user2 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+
+-- !dup_col_mixed --
+1 user1 0.5 user1 0.5
+2 user2 1 user2 1
+3 user3 1.5 user3 1.5
+4 user4 2 user4 2
+
-- !test_basic --
1 user1 1.0 false 0.5
2 user2 2.0 true 1.0
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy
index 2ef76dc8bd0..2e2cbfcce7e 100644
---
a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy
+++
b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy
@@ -34,6 +34,15 @@ suite("test_hive_topn_lazy_mat", "p0,external") {
qt_3 """ select score, value, active,name from ${table} order by
id,file_id limit 10; """
qt_4 """ select value,name,id,file_id from ${table} order by name
limit 10; """
+ // Duplicate projected column - same column twice (the core bug
from rowid_fetcher fix)
+ qt_dup_col_twice """ select name a, name b from ${table} order by
id limit 10; """
+ // Duplicate projected column - same column three times
+ qt_dup_col_thrice """ select name a, name b, name c from ${table}
order by id limit 10; """
+ // Duplicate column mixed with other columns
+ qt_dup_col_mixed """ select id, name a, score, name b from
${table} order by id limit 10; """
+ // Duplicate nullable column
+ qt_dup_col_nullable """ select score x, score y from ${table}
order by id limit 10; """
+
for (int limit : limitValues) {
// Basic query
@@ -151,10 +160,18 @@ suite("test_hive_topn_lazy_mat", "p0,external") {
"""
qt_test_join6 """
- select * from parquet_topn_lazy_mat_table as a join
orc_topn_lazy_mat_table as b
+ select * from parquet_topn_lazy_mat_table as a join
orc_topn_lazy_mat_table as b
where a.file_id = 1 and b.file_id = 1 and a.id = 1
order by a.id,b.id limit 100;
"""
+
+ // Duplicate columns from both sides of join
+ qt_test_join_dup_cols """
+ select a.name, a.name, b.id, b.id
+ from parquet_topn_lazy_mat_table as a
+ join orc_topn_lazy_mat_table as b on a.id = b.id
+ order by a.id limit ${limit};
+ """
}
for (String table : ["parquet_topn_lazy_complex_table",
"parquet_topn_lazy_complex_table_multi_pages"]) {
@@ -175,6 +192,15 @@ suite("test_hive_topn_lazy_mat", "p0,external") {
qt_complex_12 """ select * from ${table} where id%2 = 0 order
by id limit ${limit}; """
qt_complex_13 """ select * from ${table} where id%2 = 1 order
by id limit ${limit}; """
+
+ // Duplicate projected columns over nested types. The scalar
cases above only
+ // cover text/double/boolean; a STRUCT or MAP column reaches
the row-id fetch as a
+ // different column shape, so dedup and the result remap need
coverage here too.
+ qt_complex_dup_struct """ select col2 x, col2 y from ${table}
order by id limit ${limit}; """
+ qt_complex_dup_map """ select col3 m, col3 n from ${table}
order by id limit ${limit}; """
+ qt_complex_dup_string """ select col1 a, col1 b from ${table}
order by id limit ${limit}; """
+ qt_complex_dup_mixed """ select id, col2 a, col1 b, col2 c,
col3 d, col1 e from ${table} order by id limit ${limit}; """
+ qt_complex_dup_pred """ select col2 x, col2 y, col1 z from
${table} where col1 like 'text_1%' order by id limit ${limit}; """
}
}
}
@@ -232,6 +258,28 @@ suite("test_hive_topn_lazy_mat", "p0,external") {
contains("row_ids:
[__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]")
}
+ // The duplicate-column regressions below only prove the fix if their
queries really
+ // enter phase-2 materialization. Pin the plan: two result slots must
point at one
+ // physical column index, which is the input that used to drop a
column.
+ explain {
+ sql "select name a, name b from orc_topn_lazy_mat_table order by
id limit 10;"
+ contains("VMaterializeNode")
+ contains("projectList:[a, b]")
+ contains("column_descs_lists[[`name` text NULL, `name` text
NULL]]")
+ contains("locations: [[1, 2]]")
+ contains("column_idxs_lists: [[1, 1]]")
+ contains("row_ids:
[__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]")
+ }
+
+ explain {
+ sql "select col2 x, col2 y from parquet_topn_lazy_complex_table
order by id limit 10;"
+ contains("VMaterializeNode")
+ contains("column_descs_lists[[`col2` struct<a:int,b:array<int>>
NULL, `col2` struct<a:int,b:array<int>> NULL]]")
+ contains("locations: [[1, 2]]")
+ contains("column_idxs_lists: [[2, 2]]")
+ contains("row_ids:
[__DORIS_GLOBAL_ROWID_COL__parquet_topn_lazy_complex_table]")
+ }
+
explain {
sql """ select a.name,length(a.name),a.value,b.*,a.* from
parquet_topn_lazy_mat_table as a
join orc_topn_lazy_mat_table as b on a.id = b.id order by a.name
limit 10 """
diff --git
a/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy
b/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy
index f9963b0f937..2dc8ad2a4b1 100644
--- a/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy
@@ -49,6 +49,13 @@ suite("test_tvf_topn_lazy_mat", "p0,external") {
qt_3 """ select score, value, active,name from ${table} order by
value limit 4; """
qt_4 """ select value,name,id from ${table} order by name,score
limit 2; """
+ // Duplicate projected columns: the TVF path shares
read_batch_external_row() with the
+ // catalog path, so the same physical column projected twice must
be fetched once and
+ // copied into both result columns.
+ qt_dup_col_twice """ select name a, name b from ${table} order by
id limit 3; """
+ qt_dup_col_nullable """ select score x, score y from ${table}
order by id limit 5; """
+ qt_dup_col_mixed """ select id, name a, score, name b, score c
from ${table} order by id limit 4; """
+
for (int limit : limitValues) {
// Basic query
qt_test_basic """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
