This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fc1b82283cb [opt](be) Cache segment row bitmap cardinality (#66742)
fc1b82283cb is described below
commit fc1b82283cbbab0ef4e23a8d9249dc3c83d71411
Author: Mryange <[email protected]>
AuthorDate: Thu Oct 29 16:21:53 2026 +0800
[opt](be) Cache segment row bitmap cardinality (#66742)
`SegmentIterator` recalculates the row bitmap cardinality for every
output batch. For large segments, this repeatedly traverses all Roaring
bitmap containers and adds significant scan overhead. This change caches
the cardinality after all bitmap filters are applied during lazy
initialization and reuses it for column reservation and batch reads.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/storage/segment/segment_iterator.cpp | 11 +++++++----
be/src/storage/segment/segment_iterator.h | 2 ++
2 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/be/src/storage/segment/segment_iterator.cpp
b/be/src/storage/segment/segment_iterator.cpp
index 58a1980c228..23af33be523 100644
--- a/be/src/storage/segment/segment_iterator.cpp
+++ b/be/src/storage/segment/segment_iterator.cpp
@@ -584,6 +584,10 @@ Status SegmentIterator::_lazy_init(Block* block) {
RETURN_IF_ERROR(_apply_ann_topn_predicate());
+ // _row_bitmap is immutable after _range_iter is created. Cache its
cardinality to avoid
+ // traversing all roaring containers for every batch.
+ _row_bitmap_cardinality = _row_bitmap.cardinality();
+
if (_opts.read_orderby_key_reverse) {
_range_iter.reset(new BackwardBitmapRangeIterator(_row_bitmap));
} else {
@@ -594,8 +598,7 @@ Status SegmentIterator::_lazy_init(Block* block) {
// prediction) because the predictor may increase block_row_max on
subsequent batches
// up to this ceiling. Using the current (possibly reduced)
_opts.block_row_max would
// cause heap-buffer-overflow if a later prediction is larger.
- auto nrows_reserve_limit =
- std::min(_row_bitmap.cardinality(),
uint64_t(_initial_block_row_max));
+ auto nrows_reserve_limit = std::min(_row_bitmap_cardinality,
uint64_t(_initial_block_row_max));
_current_columns.resize(_schema->num_read_columns());
for (size_t i = 0; i < _schema->num_read_columns(); i++) {
@@ -619,7 +622,7 @@ Status SegmentIterator::_lazy_init(Block* block) {
// shortcut only changes how fast those default rows are produced.
_count_emit_shortcut = _should_engage_count_emit_shortcut(block);
if (_count_emit_shortcut) {
- _count_emit_rows_remaining = _row_bitmap.cardinality();
+ _count_emit_rows_remaining = _row_bitmap_cardinality;
}
return Status::OK();
@@ -3041,7 +3044,7 @@ Status SegmentIterator::_next_batch_internal(Block*
block) {
// If the row bitmap size is smaller than nrows_read_limit, there's no
need to reserve that many column rows.
uint32_t nrows_read_limit =
- std::min(cast_set<uint32_t>(_row_bitmap.cardinality()),
_opts.block_row_max);
+ std::min(cast_set<uint32_t>(_row_bitmap_cardinality),
_opts.block_row_max);
if (_can_opt_limit_reads()) {
// No SegmentIterator-side conjunct remains to be evaluated, so LIMIT
is equivalent before
// and after filtering. Cap the first read directly; this is the
no-conjunct fast path that
diff --git a/be/src/storage/segment/segment_iterator.h
b/be/src/storage/segment/segment_iterator.h
index fb1caf9d157..2f65200c217 100644
--- a/be/src/storage/segment/segment_iterator.h
+++ b/be/src/storage/segment/segment_iterator.h
@@ -368,6 +368,8 @@ private:
std::vector<std::unique_ptr<IndexIterator>> _index_iterators;
// after init(), `_row_bitmap` contains all rowid to scan
roaring::Roaring _row_bitmap;
+ // Cardinality cached after all row bitmap filters are applied in
_lazy_init().
+ uint64_t _row_bitmap_cardinality = 0;
// an iterator for `_row_bitmap` that can be used to extract row range to
scan
std::unique_ptr<BitmapRangeIterator> _range_iter;
// the next rowid to read
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
