This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-4.2 in repository https://gitbox.apache.org/repos/asf/doris.git
commit b621170f2822300d05f0bf03a7f80961b08b6358 Author: daidai <[email protected]> AuthorDate: Thu Oct 8 19:10:22 2026 +0800 branch-4.1:[fix](rowid-fetch) fix TopN lazy materialization crash with duplicateprojected columns (#64372) (#67563) ### What problem does this PR solve? Problem Summary: bp #64372 ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [ ] No. - [ ] Yes. <!-- Add document PR link here. eg: https://github.com/apache/doris-website/pull/1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into --> Co-authored-by: York Cao <[email protected]> Co-authored-by: caokaihua1 <[email protected]> --- be/src/exec/rowid_fetcher.cpp | 267 +++-- be/src/exec/rowid_fetcher.h | 15 +- be/test/exec/rowid_fetcher_test.cpp | 167 ++++ .../hive/test_hive_topn_lazy_mat.out | 1047 ++++++++++++++++++++ .../tvf/test_tvf_topn_lazy_mat.out | 72 ++ .../hive/test_hive_topn_lazy_mat.groovy | 50 +- .../tvf/test_tvf_topn_lazy_mat.groovy | 7 + 7 files changed, 1552 insertions(+), 73 deletions(-) diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp index 92f8de5c2e6..0219ac0b778 100644 --- a/be/src/exec/rowid_fetcher.cpp +++ b/be/src/exec/rowid_fetcher.cpp @@ -32,6 +32,7 @@ #include <cstdint> #include <limits> #include <memory> +#include <mutex> #include <ostream> #include <string> #include <unordered_map> @@ -907,16 +908,22 @@ Status RowIdStorageReader::read_lance_rows_by_row_ids( Status RowIdStorageReader::read_external_row_from_file_mapping( size_t idx, const std::multimap<uint64_t, size_t>& row_ids, - const std::shared_ptr<FileMapping>& file_mapping, const std::vector<SlotDescriptor>& slots, - const TUniqueId& query_id, const std::shared_ptr<RuntimeState>& runtime_state, - std::vector<Block>& scan_blocks, std::vector<std::pair<size_t, size_t>>& row_id_block_idx, + const std::shared_ptr<FileMapping>& file_mapping, + const std::vector<SlotDescriptor>& scan_slots, const TUniqueId& query_id, + const std::shared_ptr<RuntimeState>& runtime_state, std::vector<Block>& scan_blocks, + std::vector<std::pair<size_t, size_t>>& row_id_block_idx, std::vector<RowIdStorageReader::ExternalFetchStatistics>& fetch_statistics, const TFileScanRangeParams& rpc_scan_params, const std::unordered_map<std::string, int>& colname_to_slot_id, - TupleDescriptor& tuple_desc) { + std::counting_semaphore<>& semaphore, TupleDescriptor& tuple_desc) { SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->rowid_storage_reader_tracker()); signal::set_signal_task_id(query_id); + // Release the concurrency permit on every exit path (including error returns + // and exceptions). Completion accounting and status publishing are owned by + // the caller, so the status is always published before the waiter is woken. + Defer defer([&] { semaphore.release(); }); + std::vector<uint64_t> read_ids; // Generate an ordered, deduplicated list with the help of the multimap ordering. for (const auto& [row_id, result_block_idx] : row_ids) { @@ -926,7 +933,7 @@ Status RowIdStorageReader::read_external_row_from_file_mapping( row_id_block_idx[result_block_idx] = std::make_pair(idx, read_ids.size() - 1); } - scan_blocks[idx] = Block(slots, read_ids.size()); + scan_blocks[idx] = Block(scan_slots, read_ids.size()); auto& external_info = file_mapping->get_external_file_info(); auto scan_range_desc = build_external_fetch_range(external_info.scan_range_desc); @@ -936,9 +943,10 @@ Status RowIdStorageReader::read_external_row_from_file_mapping( const auto format_type = scan_range_desc.__isset.format_type ? scan_range_desc.format_type : rpc_scan_params.format_type; if (format_type == TFileFormatType::FORMAT_LANCE) { - RETURN_IF_ERROR(read_lance_rows_by_row_ids( - scan_range_desc, read_ids, slots, runtime_state.get(), sub_runtime_profile.get(), - rpc_scan_params, &scan_blocks[idx], &fetch_statistics[idx])); + RETURN_IF_ERROR(read_lance_rows_by_row_ids(scan_range_desc, read_ids, scan_slots, + runtime_state.get(), sub_runtime_profile.get(), + rpc_scan_params, &scan_blocks[idx], + &fetch_statistics[idx])); } else { // Parquet/ORC IDs remain signed physical file positions; Lance keeps native uint64 IDs. std::list<int64_t> legacy_read_ids; @@ -970,6 +978,23 @@ Status RowIdStorageReader::read_external_row_from_file_mapping( } } + if (scan_blocks[idx].rows() != read_ids.size()) { + return Status::InternalError( + "Row id fetch scan row count mismatch, " + "query_id={}, path={}, expected_rows={}, actual_rows={}", + print_id(query_id), scan_range_desc.path, read_ids.size(), scan_blocks[idx].rows()); + } + for (size_t column_id = 0; column_id < scan_blocks[idx].columns(); ++column_id) { + const auto& column = scan_blocks[idx].get_by_position(column_id); + if (column.column->size() != read_ids.size()) { + return Status::InternalError( + "Row id fetch scan column row count mismatch, " + "query_id={}, path={}, column={}, expected_rows={}, actual_rows={}", + print_id(query_id), scan_range_desc.path, column.name, read_ids.size(), + column.column->size()); + } + } + auto file_read_bytes_counter = sub_runtime_profile->get_counter(FileScannerV2::FileReadBytesProfile); @@ -988,6 +1013,92 @@ Status RowIdStorageReader::read_external_row_from_file_mapping( return Status::OK(); } +std::string RowIdStorageReader::source_column_key(const SlotDescriptor& slot, uint32_t column_idx) { + fmt::memory_buffer key; + // Length-prefix each component so distinct sequences cannot alias, e.g. + // paths ["a", "b"] -> "1:a1:b" while ["a:b"] -> "3:a:b". + auto append = [&key](std::string_view component) { + fmt::format_to(key, "{}:", component.size()); + key.append(component.data(), component.data() + component.size()); + }; + append(slot.col_name()); + append(std::to_string(column_idx)); + append(std::to_string(slot.col_unique_id())); + append(std::to_string(slot.column_paths().size())); + for (const auto& path : slot.column_paths()) { + append(path); + } + append(std::to_string(slot.all_access_paths().size())); + // Encode each optional sub-path's presence bit separately from its element + // count so an absent path ("0") never aliases a present-but-empty path + // ("1" + size "0"). + auto append_optional_path = [&append](bool is_set, const std::vector<std::string>& items) { + append(is_set ? "1" : "0"); + if (is_set) { + append(std::to_string(items.size())); + for (const auto& item : items) { + append(item); + } + } + }; + for (const auto& path : slot.all_access_paths()) { + append(fmt::format("{}", path.type)); + append_optional_path(path.__isset.data_access_path, path.data_access_path.path); + append_optional_path(path.__isset.meta_access_path, path.meta_access_path.path); + } + return fmt::to_string(key); +} + +Status RowIdStorageReader::submit_external_scan_tasks( + ScannerScheduler* scheduler, std::counting_semaphore<>& semaphore, size_t task_count, + const std::function<std::string(size_t)>& make_task_id, + const std::function<Status(size_t)>& run_task) { + // `completed_count` is a plain counter guarded by `mtx`; the same mutex guards + // the wait predicate below, so a worker can never notify between the waiter's + // predicate check and its wait. + AtomicStatus scan_status; + std::condition_variable cv; + std::mutex mtx; + size_t completed_count = 0; + + // Only tasks the scheduler actually accepted are waited for. If a submission + // fails we stop submitting, but still wait for the already-accepted tasks so + // their workers cannot outlive the locals they capture by reference. + size_t submitted_count = 0; + for (size_t idx = 0; idx < task_count; ++idx) { + semaphore.acquire(); + auto run_one_task = [&, idx]() -> bool { + Status task_status = Status::OK(); + // Publish the status before the completion signal wakes the waiter, on every + // path. A scanner that throws would otherwise leave scan_status OK while this + // Defer still counts the task as finished, and the caller would report success + // over a half-filled result block. + Defer complete([&] { + scan_status.update(task_status); + std::lock_guard<std::mutex> lock(mtx); + ++completed_count; + cv.notify_one(); + }); + ASSIGN_STATUS_IF_CATCH_EXCEPTION(task_status = run_task(idx), task_status); + return true; + }; + Status submit_st = scheduler->submit_scan_task( + SimplifiedScanTask(run_one_task, nullptr, nullptr), make_task_id(idx)); + if (!submit_st.ok()) { + scan_status.update(submit_st); + semaphore.release(); + break; + } + ++submitted_count; + } + + { + std::unique_lock<std::mutex> lock(mtx); + cv.wait(lock, [&] { return completed_count == submitted_count; }); + } + return scan_status.ok() ? Status::OK() : scan_status.status(); +} + Status RowIdStorageReader::read_batch_external_row( const uint64_t workload_group_id, const PRequestBlockDesc& request_block_desc, std::shared_ptr<IdFileMap> id_file_map, std::vector<SlotDescriptor>& slots, @@ -998,12 +1109,21 @@ Status RowIdStorageReader::read_batch_external_row( TupleDescriptor tuple_desc(request_block_desc.desc(), false); std::unordered_map<std::string, int> colname_to_slot_id; std::shared_ptr<RuntimeState> runtime_state = nullptr; + std::vector<SlotDescriptor> scan_slots; + std::vector<size_t> result_column_to_scan_column; + std::vector<uint32_t> scan_column_idxs; int max_file_scanners = 0; { if (result_block.is_empty_column()) [[likely]] { result_block = Block(slots, request_block_desc.row_id_size()); } + if (request_block_desc.column_idxs_size() != slots.size()) { + return Status::InternalError( + "Row id fetch request has mismatched slots and column indexes, " + "query_id={}, slots={}, column_idxs={}", + print_id(query_id), slots.size(), request_block_desc.column_idxs_size()); + } auto& external_info = first_file_mapping->get_external_file_info(); int plan_node_id = external_info.plan_node_id; @@ -1028,16 +1148,30 @@ Status RowIdStorageReader::read_batch_external_row( } } - std::vector<uint32_t> scan_column_idxs; + std::unordered_map<std::string, size_t> source_column_to_scan_idx; + + result_column_to_scan_column.reserve(slots.size()); + scan_slots.reserve(slots.size()); scan_column_idxs.reserve(slots.size()); - for (int slot_idx = 0; slot_idx < slots.size(); ++slot_idx) { - auto& slot = slots[slot_idx]; + for (auto slot_idx = 0; slot_idx < slots.size(); ++slot_idx) { + const auto& slot = slots[slot_idx]; + const auto column_idx = request_block_desc.column_idxs(slot_idx); + const auto key = source_column_key(slot, column_idx); + auto [it, inserted] = + source_column_to_scan_idx.emplace(key, source_column_to_scan_idx.size()); + result_column_to_scan_column.emplace_back(it->second); + if (inserted) { + scan_slots.emplace_back(slot); + scan_column_idxs.emplace_back(column_idx); + } + } + + for (auto& slot : scan_slots) { tuple_desc.add_slot(&slot); - colname_to_slot_id.emplace(slot.col_name(), slot.id()); - scan_column_idxs.emplace_back(request_block_desc.column_idxs(slot_idx)); + colname_to_slot_id[slot.col_name()] = slot.id(); } - rpc_scan_params = build_external_scan_params(rpc_scan_params, first_scan_range_desc, slots, - scan_column_idxs); + rpc_scan_params = build_external_scan_params(rpc_scan_params, first_scan_range_desc, + scan_slots, scan_column_idxs); const auto& query_options = id_file_map->get_query_options(); const auto& query_globals = id_file_map->get_query_globals(); @@ -1131,75 +1265,68 @@ Status RowIdStorageReader::read_batch_external_row( int64_t scan_running_time = 0; RETURN_IF_ERROR(scope_timer_run( [&]() -> Status { - // Make sure to insert data into result_block only after all scan tasks have been executed. - std::atomic<int> producer_count {0}; - std::condition_variable cv; - std::mutex mtx; - Status scan_status = Status::OK(); - //semaphore: Limit the number of scan tasks submitted at one time std::counting_semaphore semaphore {max_file_scanners}; - size_t idx = 0; + std::vector< + std::pair<std::multimap<uint64_t, size_t>, std::shared_ptr<FileMapping>>> + scan_info_list; + scan_info_list.reserve(scan_rows.size()); for (const auto& [_, scan_info] : scan_rows) { - semaphore.acquire(); - RETURN_IF_ERROR(remote_scan_sched->submit_scan_task( - SimplifiedScanTask( - [&, idx, scan_info]() -> bool { - Defer complete_task {[&]() { - semaphore.release(); - if (++producer_count == scan_rows.size()) { - std::lock_guard<std::mutex> lock(mtx); - cv.notify_one(); - } - }}; - const auto& [row_ids, file_mapping] = scan_info; - auto status = read_external_row_from_file_mapping( - idx, row_ids, file_mapping, slots, query_id, - runtime_state, scan_blocks, row_id_block_idx, - fetch_statistics, rpc_scan_params, - colname_to_slot_id, tuple_desc); - if (!status.ok()) { - std::lock_guard<std::mutex> lock(mtx); - if (scan_status.ok()) { - scan_status = status; - } - } - // The return value indicates whether this one-shot - // scheduler task has completed, not whether the fetch - // succeeded. The fetch status is propagated by scan_status. - return true; - }, - nullptr, nullptr), - fmt::format("{}-read_batch_external_row-{}", print_id(query_id), idx))); - idx++; + scan_info_list.emplace_back(scan_info); } - { - std::unique_lock<std::mutex> lock(mtx); - cv.wait(lock, [&] { return producer_count == scan_rows.size(); }); - } - RETURN_IF_ERROR(scan_status); - return Status::OK(); + return submit_external_scan_tasks( + remote_scan_sched, semaphore, scan_rows.size(), + [&](size_t idx) { + return fmt::format("{}-read_batch_external_row-{}", print_id(query_id), + idx); + }, + [&](size_t idx) -> Status { + const auto& [row_ids, file_mapping] = scan_info_list[idx]; + return read_external_row_from_file_mapping( + idx, row_ids, file_mapping, scan_slots, query_id, runtime_state, + scan_blocks, row_id_block_idx, fetch_statistics, + rpc_scan_params, colname_to_slot_id, semaphore, tuple_desc); + }); }, &scan_running_time)); - // Insert the read data into result_block. - for (size_t column_id = 0; column_id < result_block.columns(); column_id++) { + // Insert the read data into result_block. Use insert_indices_from() instead of + // scatter_scan_blocks_to_result_block()/insert_from_multi_column(), because + // scan_blocks may have fewer columns than result_block when duplicate physical columns + // are deduplicated, and insert_from_multi_column() cannot handle ColumnString + // cross-type (32/64) copies safely. + const size_t result_column_count = result_block.columns(); + for (size_t column_id = 0; column_id < result_column_count; column_id++) { auto dst_col_guard = result_block.mutate_column_scoped(column_id); MutableColumnPtr& dst_col = dst_col_guard.mutable_column(); - std::vector<const IColumn*> scan_src_columns; - scan_src_columns.reserve(row_id_block_idx.size()); - std::vector<size_t> scan_positions; - scan_positions.reserve(row_id_block_idx.size()); + bool dst_is_nullable = dst_col->is_nullable(); + std::vector<ColumnPtr> nullable_src_columns(scan_blocks.size()); + auto scan_column_id = result_column_to_scan_column[column_id]; for (const auto& [pos_block, block_idx] : row_id_block_idx) { - DCHECK(scan_blocks.size() > pos_block); - DCHECK(scan_blocks[pos_block].get_columns().size() > column_id); - scan_src_columns.emplace_back(scan_blocks[pos_block].get_columns()[column_id].get()); - scan_positions.emplace_back(block_idx); + DCHECK_GT(scan_blocks.size(), pos_block); + DCHECK_GT(scan_blocks[pos_block].columns(), scan_column_id); + const auto& src_column_ptr = + scan_blocks[pos_block].get_by_position(scan_column_id).column; + const auto* src_col = src_column_ptr.get(); + if (dst_is_nullable && !src_col->is_nullable()) { + if (!nullable_src_columns[pos_block]) { + nullable_src_columns[pos_block] = make_nullable(src_column_ptr); + } + src_col = nullable_src_columns[pos_block].get(); + } + if (block_idx >= src_col->size()) { + return Status::InternalError( + "Row id fetch source index out of range, query_id={}, column={}, " + "source_block={}, source_rows={}, row_index={}", + print_id(query_id), result_block.get_by_position(column_id).name, pos_block, + src_col->size(), block_idx); + } + uint32_t scan_position = cast_set<uint32_t>(block_idx); + dst_col->insert_indices_from(*src_col, &scan_position, &scan_position + 1); } - dst_col->insert_from_multi_column(scan_src_columns, scan_positions); } // Statistical runtime profile information. diff --git a/be/src/exec/rowid_fetcher.h b/be/src/exec/rowid_fetcher.h index fed8e5ccd52..ca1f8b50690 100644 --- a/be/src/exec/rowid_fetcher.h +++ b/be/src/exec/rowid_fetcher.h @@ -22,6 +22,8 @@ #include <gen_cpp/DataSinks_types.h> #include <gen_cpp/internal_service.pb.h> +#include <condition_variable> +#include <functional> #include <map> #include <memory> #include <semaphore> @@ -42,6 +44,7 @@ class RuntimeProfile; class RuntimeState; class TQueryOptions; class TupleDescriptor; +class ScannerScheduler; namespace io { enum class FileCacheMissPolicy : uint8_t; } @@ -172,13 +175,21 @@ private: static Status read_external_row_from_file_mapping( size_t idx, const std::multimap<uint64_t, size_t>& row_ids, const std::shared_ptr<FileMapping>& file_mapping, - const std::vector<SlotDescriptor>& slots, const TUniqueId& query_id, + const std::vector<SlotDescriptor>& scan_slots, const TUniqueId& query_id, const std::shared_ptr<RuntimeState>& runtime_state, std::vector<Block>& scan_blocks, std::vector<std::pair<size_t, size_t>>& row_id_block_idx, std::vector<ExternalFetchStatistics>& fetch_statistics, const TFileScanRangeParams& rpc_scan_params, const std::unordered_map<std::string, int>& colname_to_slot_id, - TupleDescriptor& tuple_desc); + std::counting_semaphore<>& semaphore, TupleDescriptor& tuple_desc); + + static std::string source_column_key(const SlotDescriptor& slot, uint32_t column_idx); + + static Status submit_external_scan_tasks(ScannerScheduler* scheduler, + std::counting_semaphore<>& semaphore, + size_t task_count, + const std::function<std::string(size_t)>& make_task_id, + const std::function<Status(size_t)>& run_task); struct ExternalFetchStatistics { int64_t init_reader_ms = 0; diff --git a/be/test/exec/rowid_fetcher_test.cpp b/be/test/exec/rowid_fetcher_test.cpp index 2a092a20f70..3e8fd3e9517 100644 --- a/be/test/exec/rowid_fetcher_test.cpp +++ b/be/test/exec/rowid_fetcher_test.cpp @@ -33,13 +33,24 @@ namespace doris { +// source_column_key() decides which projected slots collapse onto a single scanned +// column. Two slots that name the same physical column must produce the same key, and +// any two slots that do not must produce different ones -- a false merge drops a column +// the result block still expects, which is the crash this keying was introduced to fix. class RowIdStorageReaderTest : public testing::Test { +public: + static std::string key_of(const SlotDescriptor& slot, uint32_t column_idx) { + return RowIdStorageReader::source_column_key(slot, column_idx); + } + protected: struct SlotSpec { std::string col_name = "c"; int32_t slot_id = 0; PrimitiveType type = TYPE_INT; int32_t col_unique_id = 1; + std::vector<std::string> column_paths = {}; + TColumnAccessPaths access_paths = {}; }; static SlotDescriptor make_slot(const SlotSpec& spec) { @@ -51,8 +62,27 @@ protected: .build(); tdesc.__set_id(spec.slot_id); tdesc.__set_col_unique_id(spec.col_unique_id); + tdesc.__set_column_paths(spec.column_paths); + if (!spec.access_paths.empty()) { + tdesc.__set_all_access_paths(spec.access_paths); + } return SlotDescriptor(tdesc); } + + static TColumnAccessPath data_path(const std::vector<std::string>& path) { + TColumnAccessPath access_path; + access_path.type = TAccessPathType::DATA; + TDataAccessPath data; + data.__set_path(path); + access_path.__set_data_access_path(data); + return access_path; + } + + static TColumnAccessPath bare_path() { + TColumnAccessPath access_path; + access_path.type = TAccessPathType::DATA; + return access_path; + } }; TEST_F(RowIdStorageReaderTest, ExternalScannerSelectionRespectsRolloutOption) { @@ -256,4 +286,141 @@ TEST_F(RowIdStorageReaderTest, ExternalFetchPartitionSlotsPreserveHivePositionMa } } +TEST_F(RowIdStorageReaderTest, SameSourceColumnSharesKey) { + // The bug case: one physical column projected twice must dedup onto one scan column. + const SlotDescriptor first = make_slot({}); + const SlotDescriptor second = make_slot({}); + EXPECT_EQ(key_of(first, 3), key_of(second, 3)); +} + +TEST_F(RowIdStorageReaderTest, ColumnIndexSeparatesKeys) { + const SlotDescriptor slot = make_slot({}); + EXPECT_NE(key_of(slot, 3), key_of(slot, 4)); +} + +TEST_F(RowIdStorageReaderTest, ColumnNameSeparatesKeys) { + EXPECT_NE(key_of(make_slot({.col_name = "a"}), 0), key_of(make_slot({.col_name = "b"}), 0)); +} + +TEST_F(RowIdStorageReaderTest, UniqueIdSeparatesKeys) { + EXPECT_NE(key_of(make_slot({.col_unique_id = 1}), 0), + key_of(make_slot({.col_unique_id = 2}), 0)); +} + +TEST_F(RowIdStorageReaderTest, NameAndIndexBoundaryIsNotAmbiguous) { + // Without length prefixes, "a" + idx 12 and "a1" + idx 2 both flatten to "a12". + EXPECT_NE(key_of(make_slot({.col_name = "a"}), 12), key_of(make_slot({.col_name = "a1"}), 2)); +} + +TEST_F(RowIdStorageReaderTest, PathComponentBoundaryIsNotAmbiguous) { + // The concatenation hazard the length prefix exists for: ["a", "b"] and ["a:b"] are + // different nested columns but share the naive ':'-joined spelling. + EXPECT_NE(key_of(make_slot({.column_paths = {"a", "b"}}), 0), + key_of(make_slot({.column_paths = {"a:b"}}), 0)); +} + +TEST_F(RowIdStorageReaderTest, EmptyPathIsNotTheSameAsNoPath) { + EXPECT_NE(key_of(make_slot({.column_paths = {}}), 0), + key_of(make_slot({.column_paths = {""}}), 0)); +} + +TEST_F(RowIdStorageReaderTest, PathOrderMatters) { + EXPECT_NE(key_of(make_slot({.column_paths = {"a", "b"}}), 0), + key_of(make_slot({.column_paths = {"b", "a"}}), 0)); +} + +TEST_F(RowIdStorageReaderTest, EqualPathsShareKey) { + EXPECT_EQ(key_of(make_slot({.column_paths = {"a", "b"}}), 0), + key_of(make_slot({.column_paths = {"a", "b"}}), 0)); +} + +TEST_F(RowIdStorageReaderTest, AccessPathSeparatesKeys) { + EXPECT_NE(key_of(make_slot({.access_paths = {data_path({"a"})}}), 0), + key_of(make_slot({.access_paths = {data_path({"b"})}}), 0)); +} + +TEST_F(RowIdStorageReaderTest, AbsentAccessPathIsNotAnEmptyOne) { + // The presence bit: an unset data_access_path must not collide with one that is set + // but carries no components. + EXPECT_NE(key_of(make_slot({.access_paths = {bare_path()}}), 0), + key_of(make_slot({.access_paths = {data_path({})}}), 0)); +} + +TEST_F(RowIdStorageReaderTest, AccessPathCountSeparatesKeys) { + EXPECT_NE(key_of(make_slot({.access_paths = {data_path({"a"})}}), 0), + key_of(make_slot({.access_paths = {data_path({"a"}), data_path({"b"})}}), 0)); +} + +// Runs every submitted task on the submitting thread. The point of these cases is which +// status reaches the caller, not the threading, and inline execution keeps them +// deterministic. +class InlineScanScheduler : public ScannerScheduler { +public: + Status start(int, int, int, int) override { return Status::OK(); } + void stop() override {} + Status submit_scan_task(SimplifiedScanTask scan_task) override { + scan_task.scan_func(); + return Status::OK(); + } + Status submit_scan_task(SimplifiedScanTask scan_task, const std::string&) override { + scan_task.scan_func(); + return Status::OK(); + } + void reset_thread_num(int, int, int) override {} + int get_queue_size() override { return 0; } + int get_active_threads() override { return 0; } + std::vector<int> thread_debug_info() override { return {}; } + Status schedule_scan_task(std::shared_ptr<ScannerContext>, std::shared_ptr<ScanTask>, + std::unique_lock<std::mutex>&) override { + return Status::OK(); + } +}; + +// submit_external_scan_tasks() signals completion from a Defer, so a worker that leaves +// without publishing its status would still wake the waiter and the caller would report +// success over a partially filled result block. +class SubmitExternalScanTasksTest : public RowIdStorageReaderTest { +protected: + static constexpr size_t kTaskCount = 3; + + static Status run_tasks(const std::function<Status(size_t)>& run_task) { + InlineScanScheduler scheduler; + std::counting_semaphore<> semaphore {kTaskCount}; + return RowIdStorageReader::submit_external_scan_tasks( + &scheduler, semaphore, kTaskCount, + [](size_t idx) { return fmt::format("task-{}", idx); }, run_task); + } +}; + +TEST_F(SubmitExternalScanTasksTest, AllTasksSucceedingReturnsOk) { + size_t ran = 0; + EXPECT_TRUE(run_tasks([&](size_t) -> Status { + ++ran; + return Status::OK(); + }).ok()); + EXPECT_EQ(ran, kTaskCount); +} + +TEST_F(SubmitExternalScanTasksTest, ReturnedErrorReachesTheCaller) { + const Status result = run_tasks([](size_t idx) -> Status { + return idx == kTaskCount - 1 ? Status::InternalError("scanner returned an error") + : Status::OK(); + }); + EXPECT_FALSE(result.ok()); + EXPECT_NE(result.to_string().find("scanner returned an error"), std::string::npos); +} + +TEST_F(SubmitExternalScanTasksTest, ThrownExceptionReachesTheCaller) { + // The last task is the interesting one: it is the completion that releases the + // waiter, so a status lost here is a status the caller never sees. + const Status result = run_tasks([](size_t idx) -> Status { + if (idx == kTaskCount - 1) { + throw Exception(ErrorCode::INTERNAL_ERROR, "scanner threw"); + } + return Status::OK(); + }); + EXPECT_FALSE(result.ok()); + EXPECT_NE(result.to_string().find("scanner threw"), std::string::npos); +} + } // namespace doris diff --git a/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out b/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out index 0d4e28fd9f0..c6533162a15 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out +++ b/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out @@ -47,6 +47,54 @@ 17.0 user17 17 2 18.0 user18 18 2 +-- !dup_col_twice -- +user1 user1 +user2 user2 +user3 user3 +user4 user4 +user5 user5 +user6 user6 +user7 user7 +user8 user8 +user9 user9 +user10 user10 + +-- !dup_col_thrice -- +user1 user1 user1 +user2 user2 user2 +user3 user3 user3 +user4 user4 user4 +user5 user5 user5 +user6 user6 user6 +user7 user7 user7 +user8 user8 user8 +user9 user9 user9 +user10 user10 user10 + +-- !dup_col_mixed -- +1 user1 0.5 user1 +2 user2 1 user2 +3 user3 1.5 user3 +4 user4 2 user4 +5 user5 2.5 user5 +6 user6 3 user6 +7 user7 3.5 user7 +8 user8 4 user8 +9 user9 4.5 user9 +10 user10 5 user10 + +-- !dup_col_nullable -- +0.5 0.5 +1 1 +1.5 1.5 +2 2 +2.5 2.5 +3 3 +3.5 3.5 +4 4 +4.5 4.5 +5 5 + -- !test_basic -- 1 user1 1.0 false 0.5 1 2 user2 2.0 true 1.0 1 @@ -293,6 +341,54 @@ 17.0 user17 17 2 18.0 user18 18 2 +-- !dup_col_twice -- +user1 user1 +user2 user2 +user3 user3 +user4 user4 +user5 user5 +user6 user6 +user7 user7 +user8 user8 +user9 user9 +user10 user10 + +-- !dup_col_thrice -- +user1 user1 user1 +user2 user2 user2 +user3 user3 user3 +user4 user4 user4 +user5 user5 user5 +user6 user6 user6 +user7 user7 user7 +user8 user8 user8 +user9 user9 user9 +user10 user10 user10 + +-- !dup_col_mixed -- +1 user1 0.5 user1 +2 user2 1 user2 +3 user3 1.5 user3 +4 user4 2 user4 +5 user5 2.5 user5 +6 user6 3 user6 +7 user7 3.5 user7 +8 user8 4 user8 +9 user9 4.5 user9 +10 user10 5 user10 + +-- !dup_col_nullable -- +0.5 0.5 +1 1 +1.5 1.5 +2 2 +2.5 2.5 +3 3 +3.5 3.5 +4 4 +4.5 4.5 +5 5 + -- !test_basic -- 1 user1 1.0 false 0.5 1 2 user2 2.0 true 1.0 1 @@ -721,6 +817,11 @@ 1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1 1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1 +-- !test_join_dup_cols -- +user1 user1 1 1 +user2 user2 2 2 +user3 user3 3 3 + -- !test_join1 -- 2 user2 2.0 1.0 4 user4 4.0 2.0 @@ -966,6 +1067,16 @@ 1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1 1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1 +-- !test_join_dup_cols -- +user1 user1 1 1 +user2 user2 2 2 +user3 user3 3 3 +user4 user4 4 4 +user5 user5 5 5 +user6 user6 6 6 +user7 user7 7 7 +user8 user8 8 8 + -- !test_join1 -- 2 user2 2.0 1.0 4 user4 4.0 2.0 @@ -1240,6 +1351,35 @@ 1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1 1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1 +-- !test_join_dup_cols -- +user1 user1 1 1 +user2 user2 2 2 +user3 user3 3 3 +user4 user4 4 4 +user5 user5 5 5 +user6 user6 6 6 +user7 user7 7 7 +user8 user8 8 8 +user9 user9 9 9 +user10 user10 10 10 +user11 user11 11 11 +user12 user12 12 12 +user13 user13 13 13 +user14 user14 14 14 +user15 user15 15 15 +user16 user16 16 16 +user17 user17 17 17 +user18 user18 18 18 +user19 user19 19 19 +user20 user20 20 20 +user21 user21 21 21 +user22 user22 22 22 +user23 user23 23 23 +user24 user24 24 24 +user25 user25 25 25 +user26 user26 26 26 +user27 user27 27 27 + -- !complex_1 -- 1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} 2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} @@ -1301,6 +1441,31 @@ text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} 3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} 5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +-- !complex_dup_struct -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":3, "b":null} {"a":3, "b":null} + +-- !complex_dup_map -- +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]} + +-- !complex_dup_string -- +text_1 text_1 +text_2 text_2 +text_3 text_3 + +-- !complex_dup_mixed -- +1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1 +2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2 +3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3 + +-- !complex_dup_pred -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1 +{"a":10, "b":null} {"a":10, "b":null} text_10 +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11 + -- !complex_1 -- 1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} 2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} @@ -1407,6 +1572,56 @@ text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} 13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} 15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +-- !complex_dup_struct -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":3, "b":null} {"a":3, "b":null} +{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]} +{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]} +{"a":6, "b":null} {"a":6, "b":null} +{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]} +{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]} + +-- !complex_dup_map -- +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null} + +-- !complex_dup_string -- +text_1 text_1 +text_2 text_2 +text_3 text_3 +text_4 text_4 +text_5 text_5 +text_6 text_6 +text_7 text_7 +text_8 text_8 + +-- !complex_dup_mixed -- +1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1 +2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2 +3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3 +4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4 +5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5 +6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6 +7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7 +8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_dup_pred -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1 +{"a":10, "b":null} {"a":10, "b":null} text_10 +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11 +{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12 +{"a":13, "b":null} {"a":13, "b":null} text_13 +{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14 +{"a":null, "b":null} {"a":null, "b":null} text_15 +{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16 + -- !complex_1 -- 1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} 2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} @@ -1597,6 +1812,123 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} 21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} 23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +-- !complex_dup_struct -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":3, "b":null} {"a":3, "b":null} +{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]} +{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]} +{"a":6, "b":null} {"a":6, "b":null} +{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]} +{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]} +{"a":9, "b":[9, 10]} {"a":9, "b":[9, 10]} +{"a":10, "b":null} {"a":10, "b":null} +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} +{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} +{"a":13, "b":null} {"a":13, "b":null} +{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} +{"a":null, "b":null} {"a":null, "b":null} +{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} +{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} +{"a":18, "b":null} {"a":18, "b":null} +{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":21, "b":null} {"a":21, "b":null} +{"a":22, "b":[4, 5]} {"a":22, "b":[4, 5]} +{"a":null, "b":[6, 7]} {"a":null, "b":[6, 7]} +{"a":24, "b":null} {"a":24, "b":null} + +-- !complex_dup_map -- +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null} +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +{12:["p", null], 13:["k1", "k2"], 14:null} {12:["p", null], 13:["k1", "k2"], 14:null} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +{15:["z", null], 16:["y1", "y2"], 17:null} {15:["z", null], 16:["y1", "y2"], 17:null} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +{17:["t", null], 18:["u1", "u2"], 19:null} {17:["t", null], 18:["u1", "u2"], 19:null} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +{20:["y1", null], 21:null, 22:["qq", "rr"]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +{22:["a", null], 23:["b", "c"], 24:["dd", null]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +{24:["u1", null], 25:["u2", "u3"], 26:null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_dup_string -- +text_1 text_1 +text_2 text_2 +text_3 text_3 +text_4 text_4 +text_5 text_5 +text_6 text_6 +text_7 text_7 +text_8 text_8 +text_9 text_9 +text_10 text_10 +text_11 text_11 +text_12 text_12 +text_13 text_13 +text_14 text_14 +text_15 text_15 +text_16 text_16 +text_17 text_17 +text_18 text_18 +text_19 text_19 +text_20 text_20 +text_21 text_21 +text_22 text_22 +text_23 text_23 +text_24 text_24 + +-- !complex_dup_mixed -- +1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1 +2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2 +3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3 +4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4 +5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5 +6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6 +7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7 +8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8 +9 {"a":9, "b":[9, 10]} text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} text_9 +10 {"a":10, "b":null} text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} text_10 +11 {"a":11, "b":[11, 12]} text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} text_11 +12 {"a":null, "b":[12, 13]} text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} text_12 +13 {"a":13, "b":null} text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} text_13 +14 {"a":14, "b":[14, 15]} text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} text_14 +15 {"a":null, "b":null} text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} text_15 +16 {"a":16, "b":[16, 17]} text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} text_16 +17 {"a":null, "b":[17, 18]} text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} text_17 +18 {"a":18, "b":null} text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} text_18 +19 {"a":19, "b":[1, null]} text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} text_19 +20 {"a":null, "b":[2, 3]} text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} text_20 +21 {"a":21, "b":null} text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} text_21 +22 {"a":22, "b":[4, 5]} text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} text_22 +23 {"a":null, "b":[6, 7]} text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} text_23 +24 {"a":24, "b":null} text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} text_24 + +-- !complex_dup_pred -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1 +{"a":10, "b":null} {"a":10, "b":null} text_10 +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11 +{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12 +{"a":13, "b":null} {"a":13, "b":null} text_13 +{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14 +{"a":null, "b":null} {"a":null, "b":null} text_15 +{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16 +{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} text_17 +{"a":18, "b":null} {"a":18, "b":null} text_18 +{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} text_19 + -- !complex_1 -- 1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} 2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} @@ -1658,6 +1990,31 @@ text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} 3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} 5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +-- !complex_dup_struct -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":3, "b":null} {"a":3, "b":null} + +-- !complex_dup_map -- +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]} + +-- !complex_dup_string -- +text_1 text_1 +text_2 text_2 +text_3 text_3 + +-- !complex_dup_mixed -- +1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1 +2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2 +3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3 + +-- !complex_dup_pred -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1 +{"a":10, "b":null} {"a":10, "b":null} text_10 +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11 + -- !complex_1 -- 1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} 2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} @@ -1764,6 +2121,56 @@ text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} 13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} 15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +-- !complex_dup_struct -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":3, "b":null} {"a":3, "b":null} +{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]} +{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]} +{"a":6, "b":null} {"a":6, "b":null} +{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]} +{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]} + +-- !complex_dup_map -- +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null} + +-- !complex_dup_string -- +text_1 text_1 +text_2 text_2 +text_3 text_3 +text_4 text_4 +text_5 text_5 +text_6 text_6 +text_7 text_7 +text_8 text_8 + +-- !complex_dup_mixed -- +1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1 +2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2 +3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3 +4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4 +5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5 +6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6 +7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7 +8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_dup_pred -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1 +{"a":10, "b":null} {"a":10, "b":null} text_10 +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11 +{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12 +{"a":13, "b":null} {"a":13, "b":null} text_13 +{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14 +{"a":null, "b":null} {"a":null, "b":null} text_15 +{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16 + -- !complex_1 -- 1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} 2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} @@ -1954,6 +2361,123 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} 21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} 23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +-- !complex_dup_struct -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":3, "b":null} {"a":3, "b":null} +{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]} +{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]} +{"a":6, "b":null} {"a":6, "b":null} +{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]} +{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]} +{"a":9, "b":[9, 10]} {"a":9, "b":[9, 10]} +{"a":10, "b":null} {"a":10, "b":null} +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} +{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} +{"a":13, "b":null} {"a":13, "b":null} +{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} +{"a":null, "b":null} {"a":null, "b":null} +{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} +{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} +{"a":18, "b":null} {"a":18, "b":null} +{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":21, "b":null} {"a":21, "b":null} +{"a":22, "b":[4, 5]} {"a":22, "b":[4, 5]} +{"a":null, "b":[6, 7]} {"a":null, "b":[6, 7]} +{"a":24, "b":null} {"a":24, "b":null} + +-- !complex_dup_map -- +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null} +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +{12:["p", null], 13:["k1", "k2"], 14:null} {12:["p", null], 13:["k1", "k2"], 14:null} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +{15:["z", null], 16:["y1", "y2"], 17:null} {15:["z", null], 16:["y1", "y2"], 17:null} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +{17:["t", null], 18:["u1", "u2"], 19:null} {17:["t", null], 18:["u1", "u2"], 19:null} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +{20:["y1", null], 21:null, 22:["qq", "rr"]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +{22:["a", null], 23:["b", "c"], 24:["dd", null]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +{24:["u1", null], 25:["u2", "u3"], 26:null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_dup_string -- +text_1 text_1 +text_2 text_2 +text_3 text_3 +text_4 text_4 +text_5 text_5 +text_6 text_6 +text_7 text_7 +text_8 text_8 +text_9 text_9 +text_10 text_10 +text_11 text_11 +text_12 text_12 +text_13 text_13 +text_14 text_14 +text_15 text_15 +text_16 text_16 +text_17 text_17 +text_18 text_18 +text_19 text_19 +text_20 text_20 +text_21 text_21 +text_22 text_22 +text_23 text_23 +text_24 text_24 + +-- !complex_dup_mixed -- +1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1 +2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2 +3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3 +4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4 +5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5 +6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6 +7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7 +8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8 +9 {"a":9, "b":[9, 10]} text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} text_9 +10 {"a":10, "b":null} text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} text_10 +11 {"a":11, "b":[11, 12]} text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} text_11 +12 {"a":null, "b":[12, 13]} text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} text_12 +13 {"a":13, "b":null} text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} text_13 +14 {"a":14, "b":[14, 15]} text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} text_14 +15 {"a":null, "b":null} text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} text_15 +16 {"a":16, "b":[16, 17]} text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} text_16 +17 {"a":null, "b":[17, 18]} text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} text_17 +18 {"a":18, "b":null} text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} text_18 +19 {"a":19, "b":[1, null]} text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} text_19 +20 {"a":null, "b":[2, 3]} text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} text_20 +21 {"a":21, "b":null} text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} text_21 +22 {"a":22, "b":[4, 5]} text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} text_22 +23 {"a":null, "b":[6, 7]} text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} text_23 +24 {"a":24, "b":null} text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} text_24 + +-- !complex_dup_pred -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1 +{"a":10, "b":null} {"a":10, "b":null} text_10 +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11 +{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12 +{"a":13, "b":null} {"a":13, "b":null} text_13 +{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14 +{"a":null, "b":null} {"a":null, "b":null} text_15 +{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16 +{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} text_17 +{"a":18, "b":null} {"a":18, "b":null} text_18 +{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} text_19 + -- !1 -- 1 user1 1.0 false 0.5 1 2 user2 2.0 true 1.0 1 @@ -2002,6 +2526,54 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} 17.0 user17 17 2 18.0 user18 18 2 +-- !dup_col_twice -- +user1 user1 +user2 user2 +user3 user3 +user4 user4 +user5 user5 +user6 user6 +user7 user7 +user8 user8 +user9 user9 +user10 user10 + +-- !dup_col_thrice -- +user1 user1 user1 +user2 user2 user2 +user3 user3 user3 +user4 user4 user4 +user5 user5 user5 +user6 user6 user6 +user7 user7 user7 +user8 user8 user8 +user9 user9 user9 +user10 user10 user10 + +-- !dup_col_mixed -- +1 user1 0.5 user1 +2 user2 1 user2 +3 user3 1.5 user3 +4 user4 2 user4 +5 user5 2.5 user5 +6 user6 3 user6 +7 user7 3.5 user7 +8 user8 4 user8 +9 user9 4.5 user9 +10 user10 5 user10 + +-- !dup_col_nullable -- +0.5 0.5 +1 1 +1.5 1.5 +2 2 +2.5 2.5 +3 3 +3.5 3.5 +4 4 +4.5 4.5 +5 5 + -- !test_basic -- 1 user1 1.0 false 0.5 1 2 user2 2.0 true 1.0 1 @@ -2248,6 +2820,54 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} 17.0 user17 17 2 18.0 user18 18 2 +-- !dup_col_twice -- +user1 user1 +user2 user2 +user3 user3 +user4 user4 +user5 user5 +user6 user6 +user7 user7 +user8 user8 +user9 user9 +user10 user10 + +-- !dup_col_thrice -- +user1 user1 user1 +user2 user2 user2 +user3 user3 user3 +user4 user4 user4 +user5 user5 user5 +user6 user6 user6 +user7 user7 user7 +user8 user8 user8 +user9 user9 user9 +user10 user10 user10 + +-- !dup_col_mixed -- +1 user1 0.5 user1 +2 user2 1 user2 +3 user3 1.5 user3 +4 user4 2 user4 +5 user5 2.5 user5 +6 user6 3 user6 +7 user7 3.5 user7 +8 user8 4 user8 +9 user9 4.5 user9 +10 user10 5 user10 + +-- !dup_col_nullable -- +0.5 0.5 +1 1 +1.5 1.5 +2 2 +2.5 2.5 +3 3 +3.5 3.5 +4 4 +4.5 4.5 +5 5 + -- !test_basic -- 1 user1 1.0 false 0.5 1 2 user2 2.0 true 1.0 1 @@ -2676,6 +3296,11 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} 1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1 1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1 +-- !test_join_dup_cols -- +user1 user1 1 1 +user2 user2 2 2 +user3 user3 3 3 + -- !test_join1 -- 2 user2 2.0 1.0 4 user4 4.0 2.0 @@ -2921,6 +3546,16 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} 1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1 1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1 +-- !test_join_dup_cols -- +user1 user1 1 1 +user2 user2 2 2 +user3 user3 3 3 +user4 user4 4 4 +user5 user5 5 5 +user6 user6 6 6 +user7 user7 7 7 +user8 user8 8 8 + -- !test_join1 -- 2 user2 2.0 1.0 4 user4 4.0 2.0 @@ -3195,6 +3830,35 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} 1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1 1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1 +-- !test_join_dup_cols -- +user1 user1 1 1 +user2 user2 2 2 +user3 user3 3 3 +user4 user4 4 4 +user5 user5 5 5 +user6 user6 6 6 +user7 user7 7 7 +user8 user8 8 8 +user9 user9 9 9 +user10 user10 10 10 +user11 user11 11 11 +user12 user12 12 12 +user13 user13 13 13 +user14 user14 14 14 +user15 user15 15 15 +user16 user16 16 16 +user17 user17 17 17 +user18 user18 18 18 +user19 user19 19 19 +user20 user20 20 20 +user21 user21 21 21 +user22 user22 22 22 +user23 user23 23 23 +user24 user24 24 24 +user25 user25 25 25 +user26 user26 26 26 +user27 user27 27 27 + -- !complex_1 -- 1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} 2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} @@ -3256,6 +3920,31 @@ text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} 3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} 5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +-- !complex_dup_struct -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":3, "b":null} {"a":3, "b":null} + +-- !complex_dup_map -- +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]} + +-- !complex_dup_string -- +text_1 text_1 +text_2 text_2 +text_3 text_3 + +-- !complex_dup_mixed -- +1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1 +2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2 +3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3 + +-- !complex_dup_pred -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1 +{"a":10, "b":null} {"a":10, "b":null} text_10 +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11 + -- !complex_1 -- 1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} 2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} @@ -3362,6 +4051,56 @@ text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} 13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} 15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +-- !complex_dup_struct -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":3, "b":null} {"a":3, "b":null} +{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]} +{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]} +{"a":6, "b":null} {"a":6, "b":null} +{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]} +{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]} + +-- !complex_dup_map -- +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null} + +-- !complex_dup_string -- +text_1 text_1 +text_2 text_2 +text_3 text_3 +text_4 text_4 +text_5 text_5 +text_6 text_6 +text_7 text_7 +text_8 text_8 + +-- !complex_dup_mixed -- +1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1 +2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2 +3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3 +4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4 +5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5 +6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6 +7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7 +8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_dup_pred -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1 +{"a":10, "b":null} {"a":10, "b":null} text_10 +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11 +{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12 +{"a":13, "b":null} {"a":13, "b":null} text_13 +{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14 +{"a":null, "b":null} {"a":null, "b":null} text_15 +{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16 + -- !complex_1 -- 1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} 2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} @@ -3552,6 +4291,123 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} 21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} 23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +-- !complex_dup_struct -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":3, "b":null} {"a":3, "b":null} +{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]} +{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]} +{"a":6, "b":null} {"a":6, "b":null} +{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]} +{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]} +{"a":9, "b":[9, 10]} {"a":9, "b":[9, 10]} +{"a":10, "b":null} {"a":10, "b":null} +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} +{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} +{"a":13, "b":null} {"a":13, "b":null} +{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} +{"a":null, "b":null} {"a":null, "b":null} +{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} +{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} +{"a":18, "b":null} {"a":18, "b":null} +{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":21, "b":null} {"a":21, "b":null} +{"a":22, "b":[4, 5]} {"a":22, "b":[4, 5]} +{"a":null, "b":[6, 7]} {"a":null, "b":[6, 7]} +{"a":24, "b":null} {"a":24, "b":null} + +-- !complex_dup_map -- +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null} +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +{12:["p", null], 13:["k1", "k2"], 14:null} {12:["p", null], 13:["k1", "k2"], 14:null} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +{15:["z", null], 16:["y1", "y2"], 17:null} {15:["z", null], 16:["y1", "y2"], 17:null} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +{17:["t", null], 18:["u1", "u2"], 19:null} {17:["t", null], 18:["u1", "u2"], 19:null} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +{20:["y1", null], 21:null, 22:["qq", "rr"]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +{22:["a", null], 23:["b", "c"], 24:["dd", null]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +{24:["u1", null], 25:["u2", "u3"], 26:null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_dup_string -- +text_1 text_1 +text_2 text_2 +text_3 text_3 +text_4 text_4 +text_5 text_5 +text_6 text_6 +text_7 text_7 +text_8 text_8 +text_9 text_9 +text_10 text_10 +text_11 text_11 +text_12 text_12 +text_13 text_13 +text_14 text_14 +text_15 text_15 +text_16 text_16 +text_17 text_17 +text_18 text_18 +text_19 text_19 +text_20 text_20 +text_21 text_21 +text_22 text_22 +text_23 text_23 +text_24 text_24 + +-- !complex_dup_mixed -- +1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1 +2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2 +3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3 +4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4 +5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5 +6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6 +7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7 +8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8 +9 {"a":9, "b":[9, 10]} text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} text_9 +10 {"a":10, "b":null} text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} text_10 +11 {"a":11, "b":[11, 12]} text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} text_11 +12 {"a":null, "b":[12, 13]} text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} text_12 +13 {"a":13, "b":null} text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} text_13 +14 {"a":14, "b":[14, 15]} text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} text_14 +15 {"a":null, "b":null} text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} text_15 +16 {"a":16, "b":[16, 17]} text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} text_16 +17 {"a":null, "b":[17, 18]} text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} text_17 +18 {"a":18, "b":null} text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} text_18 +19 {"a":19, "b":[1, null]} text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} text_19 +20 {"a":null, "b":[2, 3]} text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} text_20 +21 {"a":21, "b":null} text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} text_21 +22 {"a":22, "b":[4, 5]} text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} text_22 +23 {"a":null, "b":[6, 7]} text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} text_23 +24 {"a":24, "b":null} text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} text_24 + +-- !complex_dup_pred -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1 +{"a":10, "b":null} {"a":10, "b":null} text_10 +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11 +{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12 +{"a":13, "b":null} {"a":13, "b":null} text_13 +{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14 +{"a":null, "b":null} {"a":null, "b":null} text_15 +{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16 +{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} text_17 +{"a":18, "b":null} {"a":18, "b":null} text_18 +{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} text_19 + -- !complex_1 -- 1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} 2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} @@ -3613,6 +4469,31 @@ text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} 3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} 5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +-- !complex_dup_struct -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":3, "b":null} {"a":3, "b":null} + +-- !complex_dup_map -- +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]} + +-- !complex_dup_string -- +text_1 text_1 +text_2 text_2 +text_3 text_3 + +-- !complex_dup_mixed -- +1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1 +2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2 +3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3 + +-- !complex_dup_pred -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1 +{"a":10, "b":null} {"a":10, "b":null} text_10 +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11 + -- !complex_1 -- 1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} 2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} @@ -3719,6 +4600,56 @@ text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} 13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} 15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +-- !complex_dup_struct -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":3, "b":null} {"a":3, "b":null} +{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]} +{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]} +{"a":6, "b":null} {"a":6, "b":null} +{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]} +{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]} + +-- !complex_dup_map -- +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null} + +-- !complex_dup_string -- +text_1 text_1 +text_2 text_2 +text_3 text_3 +text_4 text_4 +text_5 text_5 +text_6 text_6 +text_7 text_7 +text_8 text_8 + +-- !complex_dup_mixed -- +1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1 +2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2 +3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3 +4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4 +5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5 +6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6 +7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7 +8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_dup_pred -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1 +{"a":10, "b":null} {"a":10, "b":null} text_10 +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11 +{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12 +{"a":13, "b":null} {"a":13, "b":null} text_13 +{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14 +{"a":null, "b":null} {"a":null, "b":null} text_15 +{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16 + -- !complex_1 -- 1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} 2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} @@ -3908,3 +4839,119 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} 19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} 21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} 23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +-- !complex_dup_struct -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":3, "b":null} {"a":3, "b":null} +{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]} +{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]} +{"a":6, "b":null} {"a":6, "b":null} +{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]} +{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]} +{"a":9, "b":[9, 10]} {"a":9, "b":[9, 10]} +{"a":10, "b":null} {"a":10, "b":null} +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} +{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} +{"a":13, "b":null} {"a":13, "b":null} +{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} +{"a":null, "b":null} {"a":null, "b":null} +{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} +{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} +{"a":18, "b":null} {"a":18, "b":null} +{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} +{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]} +{"a":21, "b":null} {"a":21, "b":null} +{"a":22, "b":[4, 5]} {"a":22, "b":[4, 5]} +{"a":null, "b":[6, 7]} {"a":null, "b":[6, 7]} +{"a":24, "b":null} {"a":24, "b":null} + +-- !complex_dup_map -- +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null} +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +{12:["p", null], 13:["k1", "k2"], 14:null} {12:["p", null], 13:["k1", "k2"], 14:null} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +{15:["z", null], 16:["y1", "y2"], 17:null} {15:["z", null], 16:["y1", "y2"], 17:null} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +{17:["t", null], 18:["u1", "u2"], 19:null} {17:["t", null], 18:["u1", "u2"], 19:null} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +{20:["y1", null], 21:null, 22:["qq", "rr"]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +{22:["a", null], 23:["b", "c"], 24:["dd", null]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +{24:["u1", null], 25:["u2", "u3"], 26:null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_dup_string -- +text_1 text_1 +text_2 text_2 +text_3 text_3 +text_4 text_4 +text_5 text_5 +text_6 text_6 +text_7 text_7 +text_8 text_8 +text_9 text_9 +text_10 text_10 +text_11 text_11 +text_12 text_12 +text_13 text_13 +text_14 text_14 +text_15 text_15 +text_16 text_16 +text_17 text_17 +text_18 text_18 +text_19 text_19 +text_20 text_20 +text_21 text_21 +text_22 text_22 +text_23 text_23 +text_24 text_24 + +-- !complex_dup_mixed -- +1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1 +2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2 +3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3 +4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4 +5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5 +6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6 +7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7 +8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8 +9 {"a":9, "b":[9, 10]} text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} text_9 +10 {"a":10, "b":null} text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} text_10 +11 {"a":11, "b":[11, 12]} text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} text_11 +12 {"a":null, "b":[12, 13]} text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} text_12 +13 {"a":13, "b":null} text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} text_13 +14 {"a":14, "b":[14, 15]} text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} text_14 +15 {"a":null, "b":null} text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} text_15 +16 {"a":16, "b":[16, 17]} text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} text_16 +17 {"a":null, "b":[17, 18]} text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} text_17 +18 {"a":18, "b":null} text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} text_18 +19 {"a":19, "b":[1, null]} text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} text_19 +20 {"a":null, "b":[2, 3]} text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} text_20 +21 {"a":21, "b":null} text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} text_21 +22 {"a":22, "b":[4, 5]} text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} text_22 +23 {"a":null, "b":[6, 7]} text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} text_23 +24 {"a":24, "b":null} text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} text_24 + +-- !complex_dup_pred -- +{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1 +{"a":10, "b":null} {"a":10, "b":null} text_10 +{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11 +{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12 +{"a":13, "b":null} {"a":13, "b":null} text_13 +{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14 +{"a":null, "b":null} {"a":null, "b":null} text_15 +{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16 +{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} text_17 +{"a":18, "b":null} {"a":18, "b":null} text_18 +{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} text_19 diff --git a/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out b/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out index a104af3c757..f9647a18b3e 100644 --- a/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out +++ b/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out @@ -21,6 +21,24 @@ 1.0 user1 1 2.0 user2 2 +-- !dup_col_twice -- +user1 user1 +user2 user2 +user3 user3 + +-- !dup_col_nullable -- +0.5 0.5 +1 1 +1.5 1.5 +2 2 +2.5 2.5 + +-- !dup_col_mixed -- +1 user1 0.5 user1 0.5 +2 user2 1 user2 1 +3 user3 1.5 user3 1.5 +4 user4 2 user4 2 + -- !test_basic -- 1 user1 1.0 false 0.5 2 user2 2.0 true 1.0 @@ -111,6 +129,24 @@ 1.0 user1 1 2.0 user2 2 +-- !dup_col_twice -- +user1 user1 +user2 user2 +user3 user3 + +-- !dup_col_nullable -- +0.5 0.5 +1 1 +1.5 1.5 +2 2 +2.5 2.5 + +-- !dup_col_mixed -- +1 user1 0.5 user1 0.5 +2 user2 1 user2 1 +3 user3 1.5 user3 1.5 +4 user4 2 user4 2 + -- !test_basic -- 1 user1 1.0 false 0.5 2 user2 2.0 true 1.0 @@ -235,6 +271,24 @@ 1.0 user1 1 2.0 user2 2 +-- !dup_col_twice -- +user1 user1 +user2 user2 +user3 user3 + +-- !dup_col_nullable -- +0.5 0.5 +1 1 +1.5 1.5 +2 2 +2.5 2.5 + +-- !dup_col_mixed -- +1 user1 0.5 user1 0.5 +2 user2 1 user2 1 +3 user3 1.5 user3 1.5 +4 user4 2 user4 2 + -- !test_basic -- 1 user1 1.0 false 0.5 2 user2 2.0 true 1.0 @@ -325,6 +379,24 @@ 1.0 user1 1 2.0 user2 2 +-- !dup_col_twice -- +user1 user1 +user2 user2 +user3 user3 + +-- !dup_col_nullable -- +0.5 0.5 +1 1 +1.5 1.5 +2 2 +2.5 2.5 + +-- !dup_col_mixed -- +1 user1 0.5 user1 0.5 +2 user2 1 user2 1 +3 user3 1.5 user3 1.5 +4 user4 2 user4 2 + -- !test_basic -- 1 user1 1.0 false 0.5 2 user2 2.0 true 1.0 diff --git a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy index 4eb1abf8675..2aaf7cbe8de 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy @@ -34,6 +34,15 @@ suite("test_hive_topn_lazy_mat", "p0,external,hive,external_docker,external_dock qt_3 """ select score, value, active,name from ${table} order by id,file_id limit 10; """ qt_4 """ select value,name,id,file_id from ${table} order by name limit 10; """ + // Duplicate projected column - same column twice (the core bug from rowid_fetcher fix) + qt_dup_col_twice """ select name a, name b from ${table} order by id limit 10; """ + // Duplicate projected column - same column three times + qt_dup_col_thrice """ select name a, name b, name c from ${table} order by id limit 10; """ + // Duplicate column mixed with other columns + qt_dup_col_mixed """ select id, name a, score, name b from ${table} order by id limit 10; """ + // Duplicate nullable column + qt_dup_col_nullable """ select score x, score y from ${table} order by id limit 10; """ + for (int limit : limitValues) { // Basic query @@ -151,10 +160,18 @@ suite("test_hive_topn_lazy_mat", "p0,external,hive,external_docker,external_dock """ qt_test_join6 """ - select * from parquet_topn_lazy_mat_table as a join orc_topn_lazy_mat_table as b + select * from parquet_topn_lazy_mat_table as a join orc_topn_lazy_mat_table as b where a.file_id = 1 and b.file_id = 1 and a.id = 1 order by a.id,b.id limit 100; """ + + // Duplicate columns from both sides of join + qt_test_join_dup_cols """ + select a.name, a.name, b.id, b.id + from parquet_topn_lazy_mat_table as a + join orc_topn_lazy_mat_table as b on a.id = b.id + order by a.id limit ${limit}; + """ } for (String table : ["parquet_topn_lazy_complex_table", "parquet_topn_lazy_complex_table_multi_pages"]) { @@ -175,6 +192,15 @@ suite("test_hive_topn_lazy_mat", "p0,external,hive,external_docker,external_dock qt_complex_12 """ select * from ${table} where id%2 = 0 order by id limit ${limit}; """ qt_complex_13 """ select * from ${table} where id%2 = 1 order by id limit ${limit}; """ + + // Duplicate projected columns over nested types. The scalar cases above only + // cover text/double/boolean; a STRUCT or MAP column reaches the row-id fetch as a + // different column shape, so dedup and the result remap need coverage here too. + qt_complex_dup_struct """ select col2 x, col2 y from ${table} order by id limit ${limit}; """ + qt_complex_dup_map """ select col3 m, col3 n from ${table} order by id limit ${limit}; """ + qt_complex_dup_string """ select col1 a, col1 b from ${table} order by id limit ${limit}; """ + qt_complex_dup_mixed """ select id, col2 a, col1 b, col2 c, col3 d, col1 e from ${table} order by id limit ${limit}; """ + qt_complex_dup_pred """ select col2 x, col2 y, col1 z from ${table} where col1 like 'text_1%' order by id limit ${limit}; """ } } } @@ -232,6 +258,28 @@ suite("test_hive_topn_lazy_mat", "p0,external,hive,external_docker,external_dock contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]") } + // The duplicate-column regressions below only prove the fix if their queries really + // enter phase-2 materialization. Pin the plan: two result slots must point at one + // physical column index, which is the input that used to drop a column. + explain { + sql "select name a, name b from orc_topn_lazy_mat_table order by id limit 10;" + contains("VMaterializeNode") + contains("projectList:[a, b]") + contains("column_descs_lists[[`name` text NULL, `name` text NULL]]") + contains("locations: [[1, 2]]") + contains("column_idxs_lists: [[1, 1]]") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]") + } + + explain { + sql "select col2 x, col2 y from parquet_topn_lazy_complex_table order by id limit 10;" + contains("VMaterializeNode") + contains("column_descs_lists[[`col2` struct<a:int,b:array<int>> NULL, `col2` struct<a:int,b:array<int>> NULL]]") + contains("locations: [[1, 2]]") + contains("column_idxs_lists: [[2, 2]]") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__parquet_topn_lazy_complex_table]") + } + explain { sql """ select a.name,length(a.name),a.value,b.*,a.* from parquet_topn_lazy_mat_table as a join orc_topn_lazy_mat_table as b on a.id = b.id order by a.name limit 10 """ diff --git a/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy index abc920df359..41f5d235c13 100644 --- a/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy @@ -49,6 +49,13 @@ suite("test_tvf_topn_lazy_mat","external,hive,tvf,external_docker") { qt_3 """ select score, value, active,name from ${table} order by value limit 4; """ qt_4 """ select value,name,id from ${table} order by name,score limit 2; """ + // Duplicate projected columns: the TVF path shares read_batch_external_row() with the + // catalog path, so the same physical column projected twice must be fetched once and + // copied into both result columns. + qt_dup_col_twice """ select name a, name b from ${table} order by id limit 3; """ + qt_dup_col_nullable """ select score x, score y from ${table} order by id limit 5; """ + qt_dup_col_mixed """ select id, name a, score, name b, score c from ${table} order by id limit 4; """ + for (int limit : limitValues) { // Basic query qt_test_basic """ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
