github-actions[bot] commented on code in PR #69438:
URL: https://github.com/apache/doris/pull/69438#discussion_r4366633033
##########
be/src/storage/index/inverted/reader/collection_statistics.cpp:
##########
@@ -89,13 +114,171 @@ void add_term_doc_frequency(
(*logical_frequencies)[field][logical_term] += doc_frequency;
}
+Status sum_non_null_norm_lengths(index_query::IndexSource& source,
+ std::span<const uint32_t> null_docs,
uint64_t* out) {
+ if (!null_docs.empty() && null_docs.back() >= source.doc_count()) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_FILE_CORRUPTED>(
+ "NULL row {} is beyond the {} documents of the index",
null_docs.back(),
+ source.doc_count());
+ }
+ uint64_t all_lengths = 0;
+ RETURN_IF_ERROR(source.sum_norm_lengths(&all_lengths));
+ uint64_t null_lengths = 0;
+ RETURN_IF_ERROR(source.sum_norm_lengths_of(null_docs, &null_lengths));
+ DORIS_CHECK(null_lengths <= all_lengths);
+ *out = all_lengths - null_lengths;
+ return Status::OK();
+}
+
+Status exclude_hidden_docs(index_query::IndexSource& source, std::span<const
uint32_t> hidden_docs,
+ const roaring::Roaring& null_docs, uint64_t
non_null_length_sum,
+ index_query::IndexScoringStats* stats) {
+ return exclude_hidden_docs(
+ hidden_docs, null_docs, non_null_length_sum,
+ [&source](std::span<const uint32_t> docs, uint64_t* out) {
+ return source.sum_norm_lengths_of(docs, out);
+ },
+ stats);
+}
+
+Status exclude_hidden_docs(std::span<const uint32_t> hidden_docs, const
roaring::Roaring& null_docs,
+ uint64_t non_null_length_sum,
+ const SumNormLengthsOf& sum_norm_lengths_of,
+ index_query::IndexScoringStats* stats) {
+ DORIS_CHECK(stats != nullptr);
+ if (!hidden_docs.empty() && hidden_docs.back() >= stats->doc_count) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_FILE_CORRUPTED>(
+ "hidden row {} is beyond the {} documents of the index",
hidden_docs.back(),
+ stats->doc_count);
+ }
+ stats->doc_count -= hidden_docs.size();
+ if (!stats->has_norms) {
+ // Without norms every document stands for one token.
+ stats->token_count = stats->doc_count;
+ return Status::OK();
+ }
+ // The exact token count of the hidden rows is not stored anywhere, and
norms hold lengths
+ // only approximately, so count the lengths the norms of the returned rows
stand for. A
+ // replica that has compacted the hidden rows away counts the same rows
with the same norms.
+ std::vector<uint32_t> hidden_non_null_docs;
+ hidden_non_null_docs.reserve(hidden_docs.size());
Review Comment:
[P2] Charge the materialized row-ID buffers to query memory
For a MoW segment with 10 million overwritten non-NULL rows,
`collect_hidden_docs` first expands the compressed delete bitmap into a roughly
40 MB `std::vector`, then this reserve allocates another roughly 40 MB while
that first vector is still alive. Both buffers bypass Doris allocation tracking
(the malloc hooks only forward to jemalloc), and neither has a manual
MemTracker charge or reservation. This happens during statistics collection
before normal scan pruning, so even a selective `score()` query can exceed its
memory limit by tens of megabytes per concurrently collected tablet. The
NULL-ID vectors have the same problem. Please use allocator-aware row-ID
buffers with admission/reservation, or process the bitmaps in bounded batches.
This is separate from the existing retained-norm finding: these are temporary
`uint32_t` ID arrays.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
