yiguolei commented on code in PR #28103:
URL: https://github.com/apache/doris/pull/28103#discussion_r1435904739
##########
be/src/vec/exec/scan/new_olap_scan_node.cpp:
##########
@@ -508,48 +503,103 @@ Status
NewOlapScanNode::_init_scanners(std::list<VScannerSPtr>* scanners) {
std::vector<std::pair<BaseTabletSPtr, int64_t /* version */>>
tablets_to_scan;
tablets_to_scan.reserve(_scan_ranges.size());
+ std::vector<TabletWithVersion> tablets;
+
for (auto&& scan_range : _scan_ranges) {
auto tablet = DORIS_TRY(ExecEnv::get_tablet(scan_range->tablet_id));
int64_t version = 0;
std::from_chars(scan_range->version.data(),
scan_range->version.data() +
scan_range->version.size(), version);
+ tablets.emplace_back(
+ TabletWithVersion {std::dynamic_pointer_cast<Tablet>(tablet),
version});
tablets_to_scan.emplace_back(std::move(tablet), version);
}
+ bool is_dup_mow_key = true;
+ bool unique_keys_with_mor = false;
+ bool enable_parallel_scan = _state->enable_parallel_scan();
+
// Split tablet segment by scanner, only use in pipeline in duplicate key
// 1. if tablet count lower than scanner thread num, count segment num of
all tablet ready for scan
// TODO: some tablet may do not have segment, may need split segment all
case
if (_shared_scan_opt && _scan_ranges.size() <
config::doris_scanner_thread_pool_thread_num) {
for (auto&& [tablet, version] : tablets_to_scan) {
- is_dup_mow_key =
+ unique_keys_with_mor |= tablet->keys_type() == UNIQUE_KEYS &&
+
!tablet->enable_unique_key_merge_on_write();
+ is_dup_mow_key &=
tablet->keys_type() == DUP_KEYS || (tablet->keys_type() ==
UNIQUE_KEYS &&
tablet->enable_unique_key_merge_on_write());
- if (!is_dup_mow_key) {
- break;
- }
+ if (is_dup_mow_key) {
+ auto& read_source = tablets_read_source.emplace_back();
+ {
+ std::shared_lock rdlock(tablet->get_header_lock());
+ auto st =
+ tablet->capture_rs_readers({0, version},
&read_source.rs_splits, false);
+ if (!st.ok()) {
+ LOG(WARNING) << "fail to init reader.res=" << st;
+ return Status::InternalError(
+ "failed to initialize storage reader.
tablet_id={} : {}",
+ tablet->tablet_id(), st.to_string());
+ }
+ }
+ if (!_state->skip_delete_predicate()) {
+ read_source.fill_delete_predicates();
+ }
- auto& read_source = tablets_read_source.emplace_back();
- {
- std::shared_lock rdlock(tablet->get_header_lock());
- auto st = tablet->capture_rs_readers({0, version},
&read_source.rs_splits, false);
- if (!st.ok()) {
- LOG(WARNING) << "fail to init reader.res=" << st;
- return Status::InternalError(
- "failed to initialize storage reader. tablet_id={}
: {}",
- tablet->tablet_id(), st.to_string());
+ auto& rs_seg_count = tablet_rs_seg_count.emplace_back();
+ for (const auto& rowset_splits : read_source.rs_splits) {
+ auto num_segments =
rowset_splits.rs_reader->rowset()->num_segments();
+ rs_seg_count.emplace_back(num_segments);
+ segment_count += num_segments;
}
}
- if (!_state->skip_delete_predicate()) {
- read_source.fill_delete_predicates();
- }
+ }
+ } else {
+ // To avoid each instance of ScanNode generating too many scanners,
+ // only enable parallel scan when `_shared_scan_opt` is true.
+ enable_parallel_scan = false;
+ is_dup_mow_key = false;
+ }
- auto& rs_seg_count = tablet_rs_seg_count.emplace_back();
- for (const auto& rowset_splits : read_source.rs_splits) {
- auto num_segments =
rowset_splits.rs_reader->rowset()->num_segments();
- rs_seg_count.emplace_back(num_segments);
- segment_count += num_segments;
+ bool has_cpu_limit = _state->query_options().__isset.resource_limit &&
+
_state->query_options().resource_limit.__isset.cpu_limit;
+ if (!has_cpu_limit && !should_run_serial() && !unique_keys_with_mor &&
enable_parallel_scan &&
Review Comment:
这里unique key mor的就不支持了吗？
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
