This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 5f512d84c64 branch-4.2: [fix](be) Prevent score leakage across Boolean
search branches #69053 (#69265)
5f512d84c64 is described below
commit 5f512d84c649202868b39a86aabf8a80428e3323
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Oct 22 09:45:59 2026 +0800
branch-4.2: [fix](be) Prevent score leakage across Boolean search branches
#69053 (#69265)
Cherry-picked from #69053
---------
Co-authored-by: Jack <[email protected]>
---
be/src/exprs/vcompound_pred.h | 33 ++-
.../query/scoring/collection_similarity.cpp | 9 +
.../inverted/query/scoring/collection_similarity.h | 11 +
be/test/exprs/vcompound_pred_score_test.cpp | 232 +++++++++++++++++++++
.../search/test_search_score_boolean_branches.out | 60 ++++++
.../test_search_score_boolean_branches.groovy | 105 ++++++++++
6 files changed, 448 insertions(+), 2 deletions(-)
diff --git a/be/src/exprs/vcompound_pred.h b/be/src/exprs/vcompound_pred.h
index 9fe4386fd30..42e5f1f772c 100644
--- a/be/src/exprs/vcompound_pred.h
+++ b/be/src/exprs/vcompound_pred.h
@@ -20,6 +20,7 @@
#include <algorithm>
#include <cstdint>
+#include <memory>
#include "common/logging.h"
#include "common/status.h"
@@ -29,7 +30,9 @@
#include "exprs/vectorized_fn_call.h"
#include "exprs/vexpr_context.h"
#include "exprs/vexpr_fwd.h"
+#include "storage/index/inverted/query/scoring/collection_similarity.h"
#include "storage/index/zone_map/zonemap_eval_context.h"
+#include "util/defer_op.h"
#include "util/simd/bits.h"
namespace doris {
@@ -271,9 +274,24 @@ public:
}
}
+ //
NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity):
Keep each operator's bitmap and score updates together.
Status evaluate_inverted_index(VExprContext* context, uint32_t
segment_num_rows) override {
segment_v2::InvertedIndexResultBitmap res;
bool all_pass = true;
+ auto index_context = context->get_index_context();
+ auto query_context = index_context->get_index_query_context();
+ auto parent_similarity =
+ query_context != nullptr ?
query_context->collection_similarity : nullptr;
+ CollectionSimilarityPtr branch_similarity;
+ if (parent_similarity != nullptr) {
+ branch_similarity = std::make_shared<CollectionSimilarity>();
+ query_context->collection_similarity = branch_similarity;
+ }
+ Defer restore_similarity {[&]() {
+ if (branch_similarity != nullptr) {
+ query_context->collection_similarity = parent_similarity;
+ }
+ }};
switch (_op) {
case TExprOpcode::COMPOUND_OR: {
@@ -297,7 +315,7 @@ public:
} else {
res |= *index_result;
}
- if (inverted_index_context->get_score_runtime() ==
nullptr) {
+ if (branch_similarity == nullptr) {
if (res.get_data_bitmap()->cardinality() ==
segment_num_rows) {
break; // Early exit if result is full
}
@@ -363,8 +381,19 @@ public:
"Compound operator must be AND, OR, or NOT to execute with
inverted index.");
}
+ if (branch_similarity != nullptr && _op != TExprOpcode::COMPOUND_NOT) {
+ auto scores = branch_similarity->release_scores();
+ if (!scores.empty() && (!all_pass || res.is_empty())) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
+ "Cannot score a compound predicate with incomplete
index evaluation");
+ }
+ if (all_pass && !res.is_empty()) {
+ DORIS_CHECK(res.get_data_bitmap() != nullptr);
+ parent_similarity->merge_scores(scores,
*res.get_data_bitmap());
+ }
+ }
if (all_pass && !res.is_empty()) {
- context->get_index_context()->set_index_result_for_expr(this, res);
+ index_context->set_index_result_for_expr(this, res);
}
return Status::OK();
}
diff --git
a/be/src/storage/index/inverted/query/scoring/collection_similarity.cpp
b/be/src/storage/index/inverted/query/scoring/collection_similarity.cpp
index 5486bee823d..b1a664c6826 100644
--- a/be/src/storage/index/inverted/query/scoring/collection_similarity.cpp
+++ b/be/src/storage/index/inverted/query/scoring/collection_similarity.cpp
@@ -27,6 +27,15 @@ void CollectionSimilarity::collect(segment_v2::rowid_t
row_id, float score) {
_bm25_scores[row_id] += score;
}
+void CollectionSimilarity::merge_scores(const ScoreMap& scores,
+ const roaring::Roaring& matched_rows) {
+ for (const auto& [row_id, score] : scores) {
+ if (matched_rows.contains(row_id)) {
+ collect(row_id, score);
+ }
+ }
+}
+
void CollectionSimilarity::get_bm25_scores(roaring::Roaring* row_bitmap,
IColumn::MutablePtr& scores,
std::unique_ptr<std::vector<uint64_t>>& row_ids,
diff --git
a/be/src/storage/index/inverted/query/scoring/collection_similarity.h
b/be/src/storage/index/inverted/query/scoring/collection_similarity.h
index 97256be13c8..5ec1687f765 100644
--- a/be/src/storage/index/inverted/query/scoring/collection_similarity.h
+++ b/be/src/storage/index/inverted/query/scoring/collection_similarity.h
@@ -50,6 +50,17 @@ public:
void collect(segment_v2::rowid_t row_id, float score);
+ void merge_scores(const ScoreMap& scores, const roaring::Roaring&
matched_rows);
+
+ // Moves a Boolean branch's scores to its parent without copying the
per-row map.
+ ScoreMap release_scores() {
+ ScoreMap released = std::move(_bm25_scores);
+ // A moved-from flat_hash_map is valid but unspecified, not guaranteed
empty, so make the
+ // "leaves this instance empty" half of the contract true rather than
merely likely.
+ _bm25_scores.clear();
+ return released;
+ }
+
void get_bm25_scores(roaring::Roaring* row_bitmap, IColumn::MutablePtr&
scores,
std::unique_ptr<std::vector<uint64_t>>& row_ids,
const ScoreRangeFilterPtr& filter = nullptr) const;
diff --git a/be/test/exprs/vcompound_pred_score_test.cpp
b/be/test/exprs/vcompound_pred_score_test.cpp
new file mode 100644
index 00000000000..60c44b0dd16
--- /dev/null
+++ b/be/test/exprs/vcompound_pred_score_test.cpp
@@ -0,0 +1,232 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <initializer_list>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "core/data_type/primitive_type.h"
+#include "exprs/score_runtime.h"
+#include "exprs/vcompound_pred.h"
+#include "exprs/vexpr_context.h"
+#include "storage/index/index_iterator.h"
+#include "storage/index/index_query_context.h"
+#include "storage/index/inverted/query/scoring/collection_similarity.h"
+#include "storage/schema.h"
+
+namespace doris {
+namespace {
+
+class ScoredBitmapExpr final : public VExpr {
+public:
+ ScoredBitmapExpr(std::initializer_list<uint32_t> rows, float score) :
_score(score) {
+ set_node_type(TExprNodeType::MATCH_PRED);
+ for (uint32_t row : rows) {
+ _rows.add(row);
+ }
+ }
+
+ const std::string& expr_name() const override {
+ static const std::string name = "ScoredBitmapExpr";
+ return name;
+ }
+
+ Status execute(VExprContext*, Block*, int*) const override { return
Status::OK(); }
+
+ Status execute_column_impl(VExprContext*, const Block*, const Selector*,
size_t,
+ ColumnPtr&) const override {
+ return Status::OK();
+ }
+
+ Status evaluate_inverted_index(VExprContext* context, uint32_t) override {
+ auto index_context = context->get_index_context();
+ auto query_context = index_context->get_index_query_context();
+ for (uint32_t row : _rows) {
+ query_context->collection_similarity->collect(row, _score);
+ }
+ index_context->set_index_result_for_expr(
+ this,
+
segment_v2::InvertedIndexResultBitmap(std::make_shared<roaring::Roaring>(_rows),
+
std::make_shared<roaring::Roaring>()));
+ return Status::OK();
+ }
+
+private:
+ roaring::Roaring _rows;
+ float _score;
+};
+
+class UnindexedExpr final : public VExpr {
+public:
+ const std::string& expr_name() const override {
+ static const std::string name = "UnindexedExpr";
+ return name;
+ }
+
+ Status execute(VExprContext*, Block*, int*) const override { return
Status::OK(); }
+
+ Status execute_column_impl(VExprContext*, const Block*, const Selector*,
size_t,
+ ColumnPtr&) const override {
+ return Status::OK();
+ }
+
+ Status evaluate_inverted_index(VExprContext*, uint32_t) override { return
Status::OK(); }
+};
+
+std::shared_ptr<VCompoundPred> make_compound(TExprOpcode::type op, VExprSPtr
left,
+ VExprSPtr right) {
+ TExprNode node;
+ node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
+ node.__set_node_type(TExprNodeType::COMPOUND_PRED);
+ node.__set_opcode(op);
+ node.__set_num_children(2);
+ node.__set_is_nullable(false);
+ auto expr = VCompoundPred::create_shared(node);
+ expr->add_child(left);
+ expr->add_child(right);
+ return expr;
+}
+
+std::shared_ptr<VCompoundPred> make_not(VExprSPtr child) {
+ TExprNode node;
+ node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
+ node.__set_node_type(TExprNodeType::COMPOUND_PRED);
+ node.__set_opcode(TExprOpcode::COMPOUND_NOT);
+ node.__set_num_children(1);
+ node.__set_is_nullable(false);
+ auto expr = VCompoundPred::create_shared(node);
+ expr->add_child(child);
+ return expr;
+}
+
+TEST(VCompoundPredScoreTest, FailedAndBranchDoesNotContributeToOrScore) {
+ auto first = make_compound(
+ TExprOpcode::COMPOUND_AND,
+ std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t>
{1, 2, 4}, 1.0F),
+ std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t>
{1, 2, 5}, 2.0F));
+ auto second = make_compound(
+ TExprOpcode::COMPOUND_AND,
+ std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t>
{1, 2, 3, 4, 5},
+ 3.0F),
+ std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t>
{1, 3, 4, 5}, 4.0F));
+ auto root = make_compound(TExprOpcode::COMPOUND_OR, first, second);
+
+ const std::vector<ColumnId> column_ids;
+ std::vector<std::unique_ptr<segment_v2::IndexIterator>> iterators;
+ std::vector<IndexFieldNameAndTypePair> storage_types;
+ std::unordered_map<ColumnId, std::unordered_map<const VExpr*, bool>>
status_map;
+ segment_v2::ColumnIteratorOptions iterator_options;
+ auto score_runtime = std::make_shared<ScoreRuntime>(nullptr, false, 0);
+ auto index_context = std::make_shared<IndexExecContext>(
+ column_ids, iterators, storage_types, status_map, score_runtime,
nullptr,
+ iterator_options,
+ std::make_shared<Schema>(std::vector<TabletColumnPtr> {},
column_ids));
+ auto query_context = std::make_shared<segment_v2::IndexQueryContext>();
+ query_context->collection_similarity =
std::make_shared<CollectionSimilarity>();
+ auto root_similarity = query_context->collection_similarity;
+ index_context->set_index_query_context(query_context);
+ auto expr_context = VExprContext::create_shared(root);
+ expr_context->set_index_context(index_context);
+
+ ASSERT_TRUE(root->evaluate_inverted_index(expr_context.get(), 6).ok());
+ const auto* result = index_context->get_index_result_for_expr(root.get());
+ ASSERT_NE(result, nullptr);
+ EXPECT_EQ(result->get_data_bitmap()->cardinality(), 5);
+ EXPECT_EQ(query_context->collection_similarity, root_similarity);
+
+ auto scores = root_similarity->release_scores();
+ ASSERT_EQ(scores.size(), 5);
+ EXPECT_FLOAT_EQ(scores.at(1), 10.0F);
+ EXPECT_FLOAT_EQ(scores.at(2), 3.0F);
+ EXPECT_FLOAT_EQ(scores.at(3), 7.0F);
+ EXPECT_FLOAT_EQ(scores.at(4), 7.0F);
+ EXPECT_FLOAT_EQ(scores.at(5), 7.0F);
+}
+
+TEST(VCompoundPredScoreTest, NotBranchDoesNotContributeToOrScore) {
+ auto negative = make_not(
+ std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t>
{1, 2}, 1.0F));
+ auto positive =
+ std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t>
{1, 3}, 5.0F);
+ auto root = make_compound(TExprOpcode::COMPOUND_OR, negative, positive);
+
+ const std::vector<ColumnId> column_ids;
+ std::vector<std::unique_ptr<segment_v2::IndexIterator>> iterators;
+ std::vector<IndexFieldNameAndTypePair> storage_types;
+ std::unordered_map<ColumnId, std::unordered_map<const VExpr*, bool>>
status_map;
+ segment_v2::ColumnIteratorOptions iterator_options;
+ auto score_runtime = std::make_shared<ScoreRuntime>(nullptr, false, 0);
+ auto index_context = std::make_shared<IndexExecContext>(
+ column_ids, iterators, storage_types, status_map, score_runtime,
nullptr,
+ iterator_options,
+ std::make_shared<Schema>(std::vector<TabletColumnPtr> {},
column_ids));
+ auto query_context = std::make_shared<segment_v2::IndexQueryContext>();
+ query_context->collection_similarity =
std::make_shared<CollectionSimilarity>();
+ index_context->set_index_query_context(query_context);
+ auto expr_context = VExprContext::create_shared(root);
+ expr_context->set_index_context(index_context);
+
+ ASSERT_TRUE(root->evaluate_inverted_index(expr_context.get(), 4).ok());
+ const auto* result = index_context->get_index_result_for_expr(root.get());
+ ASSERT_NE(result, nullptr);
+ EXPECT_EQ(result->get_data_bitmap()->cardinality(), 3);
+ EXPECT_TRUE(result->get_data_bitmap()->contains(0));
+ EXPECT_TRUE(result->get_data_bitmap()->contains(1));
+ EXPECT_TRUE(result->get_data_bitmap()->contains(3));
+
+ auto scores = query_context->collection_similarity->release_scores();
+ ASSERT_EQ(scores.size(), 2);
+ EXPECT_FLOAT_EQ(scores.at(1), 5.0F);
+ EXPECT_FLOAT_EQ(scores.at(3), 5.0F);
+}
+
+TEST(VCompoundPredScoreTest,
IncompleteScoredBranchReturnsErrorWithoutPublishingScores) {
+ auto root = make_compound(
+ TExprOpcode::COMPOUND_AND,
+ std::make_shared<ScoredBitmapExpr>(std::initializer_list<uint32_t>
{1}, 1.0F),
+ std::make_shared<UnindexedExpr>());
+
+ const std::vector<ColumnId> column_ids;
+ std::vector<std::unique_ptr<segment_v2::IndexIterator>> iterators;
+ std::vector<IndexFieldNameAndTypePair> storage_types;
+ std::unordered_map<ColumnId, std::unordered_map<const VExpr*, bool>>
status_map;
+ segment_v2::ColumnIteratorOptions iterator_options;
+ auto score_runtime = std::make_shared<ScoreRuntime>(nullptr, false, 0);
+ auto index_context = std::make_shared<IndexExecContext>(
+ column_ids, iterators, storage_types, status_map, score_runtime,
nullptr,
+ iterator_options,
+ std::make_shared<Schema>(std::vector<TabletColumnPtr> {},
column_ids));
+ auto query_context = std::make_shared<segment_v2::IndexQueryContext>();
+ query_context->collection_similarity =
std::make_shared<CollectionSimilarity>();
+ auto parent_similarity = query_context->collection_similarity;
+ index_context->set_index_query_context(query_context);
+ auto expr_context = VExprContext::create_shared(root);
+ expr_context->set_index_context(index_context);
+
+ auto status = root->evaluate_inverted_index(expr_context.get(), 2);
+ EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED);
+ EXPECT_EQ(query_context->collection_similarity, parent_similarity);
+ EXPECT_TRUE(parent_similarity->release_scores().empty());
+}
+
+} // namespace
+} // namespace doris
diff --git a/regression-test/data/search/test_search_score_boolean_branches.out
b/regression-test/data/search/test_search_score_boolean_branches.out
new file mode 100644
index 00000000000..ba8d7a888cb
--- /dev/null
+++ b/regression-test/data/search/test_search_score_boolean_branches.out
@@ -0,0 +1,60 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !search --
+1 1.3714
+10 1.2833
+2 0.8781
+4 0.4932
+5 0.4932
+6 1.5112
+7 0.4932
+9 1.5553
+
+-- !match --
+1 1.3714
+10 1.2833
+2 0.8781
+4 0.4932
+5 0.4932
+6 1.5112
+7 0.4932
+9 1.5553
+
+-- !match_warm --
+1 1.3714
+10 1.2833
+2 0.8781
+4 0.4932
+5 0.4932
+6 1.5112
+7 0.4932
+9 1.5553
+
+-- !failed_and --
+1 0.4932
+10 0.4052
+4 0.4932
+5 0.4932
+6 0.4932
+7 0.4932
+9 0.4932
+
+-- !failed_and_reversed --
+1 0.4932
+10 0.4052
+4 0.4932
+5 0.4932
+6 0.4932
+7 0.4932
+9 0.4932
+
+-- !not_branch --
+1 0.4932
+10 0.4052
+3 0
+4 0.4932
+5 0.4932
+6 0.4932
+7 0.4932
+8 0
+9 0.4932
+
diff --git
a/regression-test/suites/search/test_search_score_boolean_branches.groovy
b/regression-test/suites/search/test_search_score_boolean_branches.groovy
new file mode 100644
index 00000000000..87bfb8f4e2c
--- /dev/null
+++ b/regression-test/suites/search/test_search_score_boolean_branches.groovy
@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_search_score_boolean_branches", "p0") {
+ sql "DROP TABLE IF EXISTS test_search_score_multi_column_mixed"
+ sql """
+ CREATE TABLE test_search_score_multi_column_mixed (
+ id INT NOT NULL,
+ title STRING NULL,
+ content STRING NULL,
+ tags STRING NULL,
+ INDEX idx_title(title) USING INVERTED PROPERTIES("parser" =
"english"),
+ INDEX idx_content(content) USING INVERTED PROPERTIES("parser" =
"english"),
+ INDEX idx_tags(tags) USING INVERTED PROPERTIES("parser" =
"english")
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_allocation" = "tag.location.default:1")
+ """
+ sql """
+ INSERT INTO test_search_score_multi_column_mixed VALUES
+ (1, 'apple banana', 'search engine', 'database analytics'),
+ (2, 'apple guide', 'search basics', 'database'),
+ (3, 'banana manual', 'engine tuning', 'analytics'),
+ (4, 'apple banana', 'storage engine', 'database analytics'),
+ (5, 'pear guide', 'search engine', 'database analytics'),
+ (6, 'apple', 'engine search', 'analytics database'),
+ (7, 'banana', 'search', 'database analytics'),
+ (8, 'grape', 'storage', 'archive'),
+ (9, 'apple banana apple', 'search engine search', 'database
analytics'),
+ (10, 'apple docs', 'search engine', 'database analytics archive')
+ """
+
+ order_qt_search """
+ SELECT id, ROUND(score(), 4) AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE search('(title:apple AND content:search) OR (tags:database AND
tags:analytics)',
+
'{"default_operator":"AND","mode":"lucene","minimum_should_match":0}')
+ ORDER BY score() DESC LIMIT 100
+ """
+
+ order_qt_match """
+ SELECT id, ROUND(score(), 4) AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE (title MATCH_ANY 'apple' AND content MATCH_ANY 'search')
+ OR tags MATCH_ALL 'database analytics'
+ ORDER BY score() DESC LIMIT 100
+ """
+
+ order_qt_match_warm """
+ SELECT id, ROUND(score(), 4) AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE (title MATCH_ANY 'apple' AND content MATCH_ANY 'search')
+ OR tags MATCH_ALL 'database analytics'
+ ORDER BY score() DESC LIMIT 100
+ """
+
+ order_qt_failed_and """
+ SELECT id, ROUND(score(), 4) AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE (title MATCH_ANY 'apple' AND content MATCH_ANY 'missing_token')
+ OR tags MATCH_ALL 'database analytics'
+ ORDER BY score() DESC LIMIT 100
+ """
+
+ order_qt_failed_and_reversed """
+ SELECT id, ROUND(score(), 4) AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE (content MATCH_ANY 'missing_token' AND title MATCH_ANY 'apple')
+ OR tags MATCH_ALL 'database analytics'
+ ORDER BY score() DESC LIMIT 100
+ """
+
+ order_qt_not_branch """
+ SELECT id, ROUND(score(), 4) AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE NOT (title MATCH_ANY 'apple') OR tags MATCH_ALL 'database
analytics'
+ ORDER BY score() DESC LIMIT 100
+ """
+
+ test {
+ sql """
+ SELECT id, score() AS bm25_score
+ FROM test_search_score_multi_column_mixed
+ WHERE (title MATCH_ANY 'apple' AND id > 0)
+ OR tags MATCH_ALL 'database analytics'
+ ORDER BY bm25_score DESC LIMIT 100
+ """
+ exception "Cannot score a compound predicate with incomplete index
evaluation"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
