This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 9800bee743f branch-4.1: [fix](be) Support BM25 scoring on keyword
SEARCH indexes #69051 (#69159)
9800bee743f is described below
commit 9800bee743ffe2368402dc9d873be5b1034820fb
Author: Jack <[email protected]>
AuthorDate: Fri Oct 16 23:23:02 2026 +0800
branch-4.1: [fix](be) Support BM25 scoring on keyword SEARCH indexes #69051
(#69159)
### What problem does this PR solve?
Related PR: #69051
Problem Summary: Backport #69051 to `branch-4.1`.
Legacy keyword SEARCH indexes lack norms and total-term-frequency
metadata. Scoring could use a zero average length or zero norm and
return invalid scores. Preserve whole keyword values, use unit document
length when metadata is absent, and keep WAND upper bounds conservative.
Numeric BKD scoring remains unsupported.
#### Branch adaptations
Backport the final net change from
f22f16113644b4bbe254765329c92bdf8095593e. Keep the branch-4.1
collection-statistics and predicate-collector paths and TermInfo
representation. Select the same reader for plain-field statistics and
execution when keyword and analyzed indexes coexist; retain the
branch-specific variant binding path. The branch has no SNII, so this PR
includes V2/V3 coverage and omits SNII-only changes.
### Release note
SEARCH on legacy keyword indexes can return valid BM25 scores.
### Check List (For Author)
- Test
- [x] Regression test
- [x] Unit Test
- [ ] Manual test
- Behavior changed:
- [x] Yes. SEARCH on legacy keyword indexes can return valid BM25
scores.
- Does this need documentation?
- [x] No.
### Validation
- PASS: Clang 16 across the complete candidate merged with
`origin/branch-4.1` (`0993d5e5512b3d27649e6c44f38b13836e03ee97`).
- PASS: English code check for the complete PR diff.
- PASS: 69 targeted ASAN BE unit tests from 5 suites on this backport.
- PASS: ASAN BE/FE build with `./build.sh --be --fe -j48`.
- PASS: `test_search_score_raw_index` and `test_search_score_cache`,
each replayed on this backport with one suite executed and zero
failures, fatal scripts, or skips.
- PASS: the BE binary requires GLIBC no newer than 2.10 (baseline 2.17).
- PASS: the two backport commits merge together without conflicts.
Validation completed on the committed backport head. The temporary test
instances are shut down after the run.
The unit run selected relevant test sources and existing repository
helpers in a temporary local CMake manifest. The original manifest has
been restored and is not included in this PR.
---
.../storage/compaction/collection_statistics.cpp | 5 +-
.../index/inverted/query_v2/segment_postings.h | 31 +++--
be/src/storage/index/inverted/query_v2/weight.h | 8 +-
.../index/inverted/similarity/bm25_similarity.h | 10 +-
be/src/storage/predicate_collector.cpp | 49 +++++---
.../compaction/collection_statistics_test.cpp | 78 ++++++++++++
.../inverted/query_v2/segment_postings_test.cpp | 22 ++++
.../data/search/test_search_score_raw_index.out | 136 +++++++++++++++++++++
.../search/test_search_score_raw_index.groovy | 108 ++++++++++++++++
9 files changed, 407 insertions(+), 40 deletions(-)
diff --git a/be/src/storage/compaction/collection_statistics.cpp
b/be/src/storage/compaction/collection_statistics.cpp
index 8f21b18e34f..3aeb129d93f 100644
--- a/be/src/storage/compaction/collection_statistics.cpp
+++ b/be/src/storage/compaction/collection_statistics.cpp
@@ -209,7 +209,8 @@ Status CollectionStatistics::process_segment(const
RowsetSharedPtr& rowset, int3
// them, together with the field's token count, in the norms. A
segment written without
// norms would feed a zero avgdl, or rank its rows as zero-length
documents next to the
// segments that have norms, so refuse to score the collection. An
index that is not
- // analyzed never writes norms and is left as it is.
+ // analyzed never writes norms; use an average length of one when its
legacy
+ // metadata does not include the total term frequency.
const auto token_count =
index_reader->sumTotalTermFreq(ws_field_name.c_str());
if (!token_count.has_value() &&
segment_v2::inverted_index::InvertedIndexAnalyzer::should_analyzer(
@@ -220,7 +221,7 @@ Status CollectionStatistics::process_segment(const
RowsetSharedPtr& rowset, int3
"for a variant path, when
inverted_index_skip_norms_for_variant is on",
seg_path, StringHelper::to_string(ws_field_name));
}
- _total_num_tokens[ws_field_name] += token_count.value_or(0);
+ _total_num_tokens[ws_field_name] +=
token_count.value_or(index_reader->maxDoc());
for (const auto& term_info : collect_info.term_infos) {
auto iter = TermIterator::create(io_ctx, false, index_reader,
ws_field_name,
diff --git a/be/src/storage/index/inverted/query_v2/segment_postings.h
b/be/src/storage/index/inverted/query_v2/segment_postings.h
index 316d81d66d4..3d6268bb05f 100644
--- a/be/src/storage/index/inverted/query_v2/segment_postings.h
+++ b/be/src/storage/index/inverted/query_v2/segment_postings.h
@@ -49,9 +49,11 @@ class SegmentPostings : public Postings {
public:
using IterVariant = std::variant<std::monostate, TermDocsPtr,
TermPositionsPtr>;
- explicit SegmentPostings(TermDocsPtr iter, bool enable_scoring,
SimilarityPtr similarity)
+ explicit SegmentPostings(TermDocsPtr iter, bool enable_scoring,
SimilarityPtr similarity,
+ bool has_norms = true)
: _iter(std::move(iter)),
_enable_scoring(enable_scoring),
+ _has_norms(has_norms),
_similarity(std::move(similarity)) {
if (auto* p = std::get_if<TermDocsPtr>(&_iter)) {
_raw_iter = p->get();
@@ -59,9 +61,11 @@ public:
_init_doc();
}
- explicit SegmentPostings(TermPositionsPtr iter, bool enable_scoring,
SimilarityPtr similarity)
+ explicit SegmentPostings(TermPositionsPtr iter, bool enable_scoring,
SimilarityPtr similarity,
+ bool has_norms = true)
: _iter(std::move(iter)),
_enable_scoring(enable_scoring),
+ _has_norms(has_norms),
_has_positions(true),
_similarity(std::move(similarity)) {
if (auto* p = std::get_if<TermPositionsPtr>(&_iter)) {
@@ -120,7 +124,7 @@ public:
}
uint32_t norm() const override {
- if (!_enable_scoring || !_block.norm_many || _cursor == 0) {
+ if (!_enable_scoring || !_has_norms || !_block.norm_many || _cursor ==
0) {
return 1;
}
return (*_block.norm_many)[_cursor - 1];
@@ -194,24 +198,28 @@ public:
return _block_max_score_cache;
}
int32_t max_block_freq = _raw_iter->getMaxBlockFreq();
- int32_t max_block_norm = _raw_iter->getMaxBlockNorm();
+ int32_t max_block_norm = _has_norms ? _raw_iter->getMaxBlockNorm() : 1;
if (max_block_freq >= 0 && max_block_norm >= 0) {
_block_max_score_cache =
_similarity->score(static_cast<float>(max_block_freq),
static_cast<int64_t>(max_block_norm));
return _block_max_score_cache;
}
- return _similarity->max_score();
+ return max_score();
}
float max_score() const {
if (!_enable_scoring || !_similarity) {
return std::numeric_limits<float>::max();
}
+ // Missing norms have a fixed length of one, including for WAND bounds.
+ if (!_has_norms) {
+ return _similarity->score(std::numeric_limits<float>::max(), 1);
+ }
return _similarity->max_score();
}
int32_t max_block_freq() const { return _raw_iter->getMaxBlockFreq(); }
- int32_t max_block_norm() const { return _raw_iter->getMaxBlockNorm(); }
+ int32_t max_block_norm() const { return _has_norms ?
_raw_iter->getMaxBlockNorm() : 1; }
private:
bool _refill() {
@@ -241,6 +249,7 @@ private:
lucene::index::TermDocs* _raw_iter = nullptr;
uint32_t _doc = TERMINATED;
bool _enable_scoring = false;
+ bool _has_norms = true;
bool _has_positions = false;
DocRange _block;
@@ -253,13 +262,15 @@ private:
using SegmentPostingsPtr = std::shared_ptr<SegmentPostings>;
inline SegmentPostingsPtr make_segment_postings(TermDocsPtr iter, bool
enable_scoring,
- SimilarityPtr similarity) {
- return std::make_shared<SegmentPostings>(std::move(iter), enable_scoring,
similarity);
+ SimilarityPtr similarity, bool
has_norms = true) {
+ return std::make_shared<SegmentPostings>(std::move(iter), enable_scoring,
similarity,
+ has_norms);
}
inline SegmentPostingsPtr make_segment_postings(TermPositionsPtr iter, bool
enable_scoring,
- SimilarityPtr similarity) {
- return std::make_shared<SegmentPostings>(std::move(iter), enable_scoring,
similarity);
+ SimilarityPtr similarity, bool
has_norms = true) {
+ return std::make_shared<SegmentPostings>(std::move(iter), enable_scoring,
similarity,
+ has_norms);
}
} // namespace doris::segment_v2::inverted_index::query_v2
\ No newline at end of file
diff --git a/be/src/storage/index/inverted/query_v2/weight.h
b/be/src/storage/index/inverted/query_v2/weight.h
index 22d7b513030..4d694551549 100644
--- a/be/src/storage/index/inverted/query_v2/weight.h
+++ b/be/src/storage/index/inverted/query_v2/weight.h
@@ -151,7 +151,9 @@ protected:
const io::IOContext* io_ctx) const {
auto t = make_term_ptr(field.c_str(), term.c_str());
auto iter = make_term_doc_ptr(reader, t.get(), enable_scoring, io_ctx);
- return iter ? make_segment_postings(std::move(iter), enable_scoring,
similarity) : nullptr;
+ return iter ? make_segment_postings(std::move(iter), enable_scoring,
similarity,
+ reader->hasNorms(field.c_str()))
+ : nullptr;
}
SegmentPostingsPtr create_position_posting(lucene::index::IndexReader*
reader,
@@ -168,7 +170,9 @@ protected:
const io::IOContext* io_ctx)
const {
auto t = make_term_ptr(field.c_str(), term.c_str());
auto iter = make_term_positions_ptr(reader, t.get(), enable_scoring,
io_ctx);
- return iter ? make_segment_postings(std::move(iter), enable_scoring,
similarity) : nullptr;
+ return iter ? make_segment_postings(std::move(iter), enable_scoring,
similarity,
+ reader->hasNorms(field.c_str()))
+ : nullptr;
}
};
diff --git a/be/src/storage/index/inverted/similarity/bm25_similarity.h
b/be/src/storage/index/inverted/similarity/bm25_similarity.h
index e3bfcf17802..d22857962d8 100644
--- a/be/src/storage/index/inverted/similarity/bm25_similarity.h
+++ b/be/src/storage/index/inverted/similarity/bm25_similarity.h
@@ -22,14 +22,8 @@
namespace doris::segment_v2 {
#include "common/compile_check_begin.h"
-/**
- * BM25 similarity scoring implementation for inverted index queries.
- *
- * Note: Currently, BM25 scoring is only applied to tokenized indexes (e.g.,
standard, chinese analyzers)
- * and not to non-tokenized indexes (e.g., keyword analyzer). This differs
from Elasticsearch,
- * which applies scoring to keyword indexes as well. In Doris,
keyword/non-tokenized indexes
- * are primarily used for exact matching without relevance scoring.
- */
+// BM25 scoring for analyzed and keyword indexes. Keyword indexes without norms
+// use the Lucene default encoded norm of one.
class BM25Similarity : public Similarity {
public:
BM25Similarity();
diff --git a/be/src/storage/predicate_collector.cpp
b/be/src/storage/predicate_collector.cpp
index 7ab04fa2c6c..e588b1df707 100644
--- a/be/src/storage/predicate_collector.cpp
+++ b/be/src/storage/predicate_collector.cpp
@@ -382,27 +382,40 @@ Status SearchPredicateCollector::collect_from_leaf(const
TSearchClause& clause,
return Status::OK();
}
+ if (!column.is_extracted_column()) {
+ // Plain fields must use the same reader for statistics and SEARCH
execution.
+ // Variant paths retain the branch-specific binding resolution.
+ auto query_type = InvertedIndexQueryType::MATCH_ANY_QUERY;
+ if (clause_type == "EXACT") {
+ query_type = InvertedIndexQueryType::EQUAL_QUERY;
+ } else if (clause_type == "PHRASE") {
+ query_type = InvertedIndexQueryType::MATCH_PHRASE_QUERY;
+ } else if (clause_type == "ALL") {
+ query_type = InvertedIndexQueryType::MATCH_ALL_QUERY;
+ }
+ const auto* selected =
+ DORIS_TRY(select_index_meta(index_metas, column.type(),
query_type, "", ""));
+ index_metas = {selected};
+ }
+
ClauseTypeCategory category = get_clause_type_category(clause_type);
for (const auto* index_meta : index_metas) {
+ if (infer_inverted_index_reader_type(column.type(),
index_meta->properties()) ==
+ InvertedIndexReaderType::BKD) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
+ "Index statistics collection failed: BM25 scoring does not
support numeric BKD "
+ "search field={}",
+ field_name);
+ }
std::set<TermInfo, TermInfoComparer> term_infos;
-
- if (category == ClauseTypeCategory::TOKENIZED) {
- if
(InvertedIndexAnalyzer::should_analyzer(index_meta->properties())) {
- auto analyzed_terms =
- DORIS_TRY(analyze_plain_query(value,
index_meta->properties()));
- term_infos.insert(analyzed_terms.begin(),
analyzed_terms.end());
- } else {
- term_infos.insert(TermInfo(value));
- }
- } else if (category == ClauseTypeCategory::NON_TOKENIZED) {
- if (clause_type == "TERM" &&
-
InvertedIndexAnalyzer::should_analyzer(index_meta->properties())) {
- auto analyzed_terms =
- DORIS_TRY(analyze_plain_query(value,
index_meta->properties()));
- term_infos.insert(analyzed_terms.begin(),
analyzed_terms.end());
- } else {
- term_infos.insert(TermInfo(value));
- }
+ if (!InvertedIndexAnalyzer::should_analyzer(index_meta->properties())
||
+ clause_type == "MATCH" ||
+ (category == ClauseTypeCategory::NON_TOKENIZED && clause_type !=
"TERM")) {
+ // Keyword indexes store the whole value, including whitespace and
case.
+ term_infos.insert(TermInfo(value));
+ } else {
+ auto analyzed_terms = DORIS_TRY(analyze_plain_query(value,
index_meta->properties()));
+ term_infos.insert(analyzed_terms.begin(), analyzed_terms.end());
}
std::string lucene_field_name =
diff --git a/be/test/storage/compaction/collection_statistics_test.cpp
b/be/test/storage/compaction/collection_statistics_test.cpp
index 81248beb9fb..617f8dfe439 100644
--- a/be/test/storage/compaction/collection_statistics_test.cpp
+++ b/be/test/storage/compaction/collection_statistics_test.cpp
@@ -32,6 +32,7 @@
#include "exprs/vexpr_context.h"
#include "exprs/vliteral.h"
#include "exprs/vmatch_predicate.h"
+#include "exprs/vsearch.h"
#include "exprs/vslot_ref.h"
#include "io/fs/local_file_system.h"
#include "runtime/exec_env.h"
@@ -1443,4 +1444,81 @@ TEST_F(CollectionStatisticsTest,
ScoringAnalyzerContextReportsWrongFamilyCompone
EXPECT_TRUE(analyzer_context_from_properties(valid).has_value());
}
+namespace {
+VExprSPtr make_scoring_search(const std::string& clause_type, const
std::string& value) {
+ TSearchClause clause;
+ clause.__set_clause_type(clause_type);
+ clause.__set_field_name("content");
+ clause.__set_value(value);
+ TSearchParam param;
+ param.__set_root(clause);
+ TExprNode node;
+ node.__set_node_type(TExprNodeType::SEARCH_EXPR);
+ node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
+ node.__set_search_param(param);
+ return VSearchExpr::create_shared(node);
+}
+} // namespace
+
+TEST_F(CollectionStatisticsTest, SearchKeywordClausesKeepWholeValue) {
+ auto schema = std::make_shared<TabletSchema>();
+ TabletColumn column;
+ column.set_unique_id(1);
+ column.set_name("content");
+ column.set_type(FieldType::OLAP_FIELD_TYPE_STRING);
+ schema->append_column(column);
+ TabletIndex index;
+ index._index_id = 10;
+ index._index_type = IndexType::INVERTED;
+ index._col_unique_ids.push_back(1);
+ schema->append_index(std::move(index));
+
+ SearchPredicateCollector collector;
+ for (const auto* clause_type : {"TERM", "PHRASE", "EXACT", "MATCH", "ANY",
"ALL"}) {
+ CollectInfoMap infos;
+ auto expr = make_scoring_search(clause_type, "Apple banana");
+ ASSERT_TRUE(collector.collect(runtime_state_.get(), schema, expr,
&infos).ok());
+ ASSERT_EQ(infos.size(), 1U);
+ ASSERT_EQ(infos.begin()->second.term_infos.size(), 1U);
+ EXPECT_EQ(infos.begin()->second.term_infos.begin()->get_single_term(),
"Apple banana");
+ }
+}
+
+TEST_F(CollectionStatisticsTest,
SearchExactSelectsKeywordStatisticsWithDualIndexes) {
+ auto schema = create_tablet_schema_with_inverted_index();
+ TabletIndex keyword;
+ keyword._index_id = 10;
+ keyword._index_type = IndexType::INVERTED;
+ keyword._col_unique_ids.push_back(1);
+ schema->append_index(std::move(keyword));
+ SearchPredicateCollector collector;
+ CollectInfoMap infos;
+ auto expr = make_scoring_search("EXACT", "Apple banana");
+ ASSERT_TRUE(collector.collect(runtime_state_.get(), schema, expr,
&infos).ok());
+ ASSERT_EQ(infos.size(), 1U);
+ EXPECT_EQ(infos.begin()->second.index_meta->index_id(), 10);
+ ASSERT_EQ(infos.begin()->second.term_infos.size(), 1U);
+ EXPECT_EQ(infos.begin()->second.term_infos.begin()->get_single_term(),
"Apple banana");
+}
+
+TEST_F(CollectionStatisticsTest, SearchRejectsNumericBkdScoring) {
+ auto schema = std::make_shared<TabletSchema>();
+ TabletColumn column;
+ column.set_unique_id(1);
+ column.set_name("content");
+ column.set_type(FieldType::OLAP_FIELD_TYPE_INT);
+ schema->append_column(column);
+ TabletIndex index;
+ index._index_id = 10;
+ index._index_type = IndexType::INVERTED;
+ index._col_unique_ids.push_back(1);
+ schema->append_index(std::move(index));
+ SearchPredicateCollector collector;
+ CollectInfoMap infos;
+ auto expr = make_scoring_search("EXACT", "7");
+ const auto status = collector.collect(runtime_state_.get(), schema, expr,
&infos);
+ EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED);
+ EXPECT_TRUE(infos.empty());
+}
+
} // namespace doris
diff --git a/be/test/storage/index/inverted/query_v2/segment_postings_test.cpp
b/be/test/storage/index/inverted/query_v2/segment_postings_test.cpp
index 0b8e4649b3a..048c6d17ae3 100644
--- a/be/test/storage/index/inverted/query_v2/segment_postings_test.cpp
+++ b/be/test/storage/index/inverted/query_v2/segment_postings_test.cpp
@@ -23,8 +23,10 @@
#include <vector>
#include "CLucene/index/DocRange.h"
+#include "storage/index/inverted/similarity/bm25_similarity.h"
namespace doris::segment_v2::inverted_index::query_v2 {
+namespace {
class MockTermDocs : public lucene::index::TermDocs {
public:
@@ -69,6 +71,8 @@ public:
void close() override {}
lucene::index::TermPositions* __asTermPositions() override { return
nullptr; }
int32_t docFreq() override { return _doc_freq; }
+ int32_t getMaxBlockFreq() override { return -1; }
+ int32_t getMaxBlockNorm() override { return -1; }
protected:
std::vector<uint32_t> _docs;
@@ -155,6 +159,8 @@ private:
bool _read_done = false;
};
+} // namespace
+
class SegmentPostingsTest : public testing::Test {};
TEST_F(SegmentPostingsTest, test_postings_positions_with_offset) {
@@ -185,6 +191,22 @@ TEST_F(SegmentPostingsTest,
test_segment_postings_base_constructor_next_true) {
EXPECT_EQ(base.norm(), 1);
}
+TEST_F(SegmentPostingsTest, KeywordPostingsUseUnitNorm) {
+ auto similarity = std::make_shared<segment_v2::BM25Similarity>(1.0F, 1.0F);
+ TermDocsPtr iter(new MockTermDocs({1, 3}, {1, 7}, {0, 0}, 2));
+ SegmentPostings postings(std::move(iter), true, similarity, false);
+ EXPECT_EQ(postings.doc(), 1);
+ EXPECT_EQ(postings.norm(), 1);
+ EXPECT_FLOAT_EQ(similarity->score(postings.freq(), postings.norm()), 1.0F);
+ EXPECT_GE(postings.block_max_score(), similarity->score(postings.freq(),
postings.norm()));
+ EXPECT_EQ(postings.advance(), 3);
+ EXPECT_EQ(postings.norm(), 1);
+ EXPECT_NEAR(similarity->score(postings.freq(), postings.norm()), 7.0F *
2.2F / 8.2F, 1e-6F);
+ EXPECT_GE(postings.block_max_score(), similarity->score(postings.freq(),
postings.norm()));
+ EXPECT_GE(postings.max_score(), similarity->score(postings.freq(),
postings.norm()));
+ EXPECT_EQ(postings.max_block_norm(), 1);
+}
+
TEST_F(SegmentPostingsTest, test_segment_postings_base_constructor_next_false)
{
TermDocsPtr ptr(new MockTermDocs({}, {}, {}, 0));
SegmentPostings base(std::move(ptr), true, nullptr);
diff --git a/regression-test/data/search/test_search_score_raw_index.out
b/regression-test/data/search/test_search_score_raw_index.out
new file mode 100644
index 00000000000..19dc1db0988
--- /dev/null
+++ b/regression-test/data/search/test_search_score_raw_index.out
@@ -0,0 +1,136 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !V2_unscored --
+1
+3
+9
+
+-- !V2_true_term --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V2_true_phrase --
+5 1.568616 true
+10 1.568616 true
+
+-- !V2_true_exact --
+5 1.568616 true
+10 1.568616 true
+
+-- !V2_true_or_top_two --
+5 1.568616 true
+10 1.568616 true
+
+-- !V2_true_top_one --
+6 2.079442 true
+
+-- !V2_true_ascending --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V2_true_case --
+6 2.079442 true
+
+-- !V2_true_missing --
+
+-- !V2_false_term --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V2_false_phrase --
+5 1.568616 true
+10 1.568616 true
+
+-- !V2_false_exact --
+5 1.568616 true
+10 1.568616 true
+
+-- !V2_false_or_top_two --
+5 1.568616 true
+10 1.568616 true
+
+-- !V2_false_top_one --
+6 2.079442 true
+
+-- !V2_false_ascending --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V2_false_case --
+6 2.079442 true
+
+-- !V2_false_missing --
+
+-- !V2_dual_exact --
+1 0.980829 true
+
+-- !V3_unscored --
+1
+3
+9
+
+-- !V3_true_term --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V3_true_phrase --
+5 1.568616 true
+10 1.568616 true
+
+-- !V3_true_exact --
+5 1.568616 true
+10 1.568616 true
+
+-- !V3_true_or_top_two --
+5 1.568616 true
+10 1.568616 true
+
+-- !V3_true_top_one --
+6 2.079442 true
+
+-- !V3_true_ascending --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V3_true_case --
+6 2.079442 true
+
+-- !V3_true_missing --
+
+-- !V3_false_term --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V3_false_phrase --
+5 1.568616 true
+10 1.568616 true
+
+-- !V3_false_exact --
+5 1.568616 true
+10 1.568616 true
+
+-- !V3_false_or_top_two --
+5 1.568616 true
+10 1.568616 true
+
+-- !V3_false_top_one --
+6 2.079442 true
+
+-- !V3_false_ascending --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V3_false_case --
+6 2.079442 true
+
+-- !V3_false_missing --
+
+-- !V3_dual_exact --
+1 0.980829 true
diff --git a/regression-test/suites/search/test_search_score_raw_index.groovy
b/regression-test/suites/search/test_search_score_raw_index.groovy
new file mode 100644
index 00000000000..fb80fcf2637
--- /dev/null
+++ b/regression-test/suites/search/test_search_score_raw_index.groovy
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_search_score_raw_index", "p0") {
+ // Score TopN pushdown accepts one ordering key. Sort ties outside the
bounded query.
+ def scoredQuery = { hint, table, predicate, direction, limit ->
+ """
+ SELECT id, ROUND(bm25_score, 6), bm25_score > 0 AND bm25_score <
1000000
+ FROM (
+ SELECT ${hint} id, score() AS bm25_score FROM ${table}
+ WHERE ${predicate}
+ ORDER BY bm25_score ${direction} LIMIT ${limit}
+ ) hits
+ ORDER BY bm25_score ${direction}, id
+ """
+ }
+ for (def storageFormat : ["V2", "V3"]) {
+ sql "DROP TABLE IF EXISTS test_search_score_raw_index"
+ sql """
+ CREATE TABLE test_search_score_raw_index (
+ id INT NOT NULL,
+ raw_index_content STRING NULL,
+ INDEX idx_raw_content(raw_index_content) USING INVERTED
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default:1",
+ "inverted_index_storage_format" = "${storageFormat}"
+ )
+ """
+ // Separate inserts exercise statistics across multiple segments.
+ sql """INSERT INTO test_search_score_raw_index VALUES
+ (1, 'apple'), (2, 'banana'), (3, 'apple'), (4, 'orange'),
+ (5, 'apple banana'), (6, 'Apple'), (7, NULL), (8, '')
+ """
+ sql "sync"
+ sql """INSERT INTO test_search_score_raw_index VALUES
+ (9, 'apple'), (10, 'apple banana'), (11, 'banana')
+ """
+ sql "sync"
+
+ def term = "search('raw_index_content:apple', '{\"mode\":\"lucene\"}')"
+ def phrase = "search('raw_index_content:\"apple banana\"',
'{\"mode\":\"lucene\"}')"
+ def exact = "search('raw_index_content:EXACT(apple banana)',
'{\"mode\":\"standard\"}')"
+ def rankedOr = "search('raw_index_content:apple OR
raw_index_content:\"apple banana\"', '{\"mode\":\"lucene\"}')"
+ def uniqueTop = "search('raw_index_content:apple OR
raw_index_content:Apple', '{\"mode\":\"lucene\"}')"
+ def caseSensitive = "search('raw_index_content:Apple',
'{\"mode\":\"lucene\"}')"
+ def missing = "search('raw_index_content:pear',
'{\"mode\":\"lucene\"}')"
+ quickTest("${storageFormat}_unscored", """
+ SELECT id FROM test_search_score_raw_index WHERE ${term} ORDER BY
id
+ """)
+ for (def wand : [true, false]) {
+ def hint = "/*+SET_VAR(enable_inverted_index_wand_query=${wand})
*/"
+ for (def entry : [
+ ["term", term, "DESC", 100],
+ ["phrase", phrase, "DESC", 100],
+ ["exact", exact, "DESC", 100],
+ ["or_top_two", rankedOr, "DESC", 2],
+ ["top_one", uniqueTop, "DESC", 1],
+ ["ascending", term, "ASC", 100],
+ ["case", caseSensitive, "DESC", 100],
+ ["missing", missing, "DESC", 100]]) {
+ quickTest("${storageFormat}_${wand}_${entry[0]}",
+ scoredQuery(hint, "test_search_score_raw_index",
entry[1],
+ entry[2], entry[3]))
+ }
+ }
+ sql "DROP TABLE IF EXISTS test_search_score_dual_index"
+ sql """
+ CREATE TABLE test_search_score_dual_index (
+ id INT NOT NULL,
+ content STRING NULL,
+ INDEX idx_raw(content) USING INVERTED,
+ INDEX idx_analyzed(content) USING INVERTED
+ PROPERTIES("parser"="standard", "support_phrase"="true")
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation"="tag.location.default:1",
+ "inverted_index_storage_format"="${storageFormat}"
+ )
+ """
+ sql """INSERT INTO test_search_score_dual_index VALUES
+ (1, 'apple banana'), (2, 'apple'), (3, 'apple banana orange')
+ """
+ sql "sync"
+ quickTest("${storageFormat}_dual_exact",
+ scoredQuery("", "test_search_score_dual_index",
+ "search('content:EXACT(apple banana)',
'{\"mode\":\"standard\"}')",
+ "DESC", 10))
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
