This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 4735ebc17fb branch-4.1: [fix](be) Score regexp and wildcard patterns
with a constant like Elasticsearch (#69825)
4735ebc17fb is described below
commit 4735ebc17fbbcdcc02abbbadb74550acf93640ea
Author: Jack <[email protected]>
AuthorDate: Fri Oct 30 18:29:46 2026 +0800
branch-4.1: [fix](be) Score regexp and wildcard patterns with a constant
like Elasticsearch (#69825)
### What problem does this PR solve?
Issue Number: None
Related PR: #69815, #69269
Problem Summary: score() of pattern queries disagrees with Elasticsearch
and with itself on branch-4.1:
-- 'Inverted indexes are suitable for full text search',
-- 'The parser splits text into multiple terms', ...
SELECT score() FROM document WHERE SEARCH('content:/.*ext/'); -- BM25
sum
SELECT score() FROM document WHERE SEARCH('content:*ext'); -- BM25 sum
SELECT score() FROM document WHERE content MATCH_REGEXP 'ext'; -- 0
Elasticsearch scores a query_string /regexp/, wildcard or prefix, and a
regexp, wildcard or prefix query, with a constant 1 per matching row
(default rewrite constant_score_blended); measured on 9.5.5 with the
rows above: 1.0 for both matching rows. Only an explicit
"rewrite": "scoring_boolean" sums the BM25 of the expanded terms. Next
to a scored clause in a bool must, a regexp adds exactly 1.
#69269 (the branch-4.1 pick of #69052) made SEARCH regexp and wildcard
sum the BM25 of every expanded term, i.e. scoring_boolean always on,
with idf and avgdl taken from the current segment rather than the tablet
statistics MATCH uses, so identical rows in two segments scored
differently. MATCH_REGEXP never requested a scoring context, so it
scored 0 like a filter.
Fix, the same as #69815 on master:
1. Revert #69269: RegexpWeight scores every pattern with the constant
scorer again; its suite that pinned the BM25 sums is removed.
2. MATCH_REGEXP requests scoring on every index, and the legacy
RegexpQuery records 1 for every matching row when score() is
requested. The MATCH statistics collector skips it, since a constant
reads no statistics and the pattern is not text to analyze.
A MATCH_REGEXP on a keyword (untokenized) index also scores 1 once the
keyword reader requests scoring (#69791); until then it keeps 0.
### Release note
score() of SEARCH regexp and wildcard clauses is a constant 1 per
matching row again, like prefix clauses and Elasticsearch's default.
score() of MATCH_REGEXP on a tokenized index is 1 instead of 0.
### Check List (For Author)
- Test: Regression test / Unit Test / Manual test
- Regression test: new search/test_search_score_pattern_constant
(V2/V3, two segments); all 18 blocks equal the master PR's V2/V3
blocks; 16 existing score suites in inverted_index_p0 and search
pass unchanged
- Unit Test: 130 tests (IndexReaderHelperTest, CollectionStatistics*,
RegexpQueryTest and the query_v2 regexp/wildcard tests restored by
the revert)
- Manual test: Elasticsearch 9.5.5 on the same rows; a two-segment
table now scores 1 for every pattern hit on a branch-4.1 ASAN build
- Behavior changed: Yes. SEARCH regexp/wildcard scores and MATCH_REGEXP
scores change as described in the release note.
- Does this need documentation: No
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
be/src/storage/index/index_reader_helper.h | 10 +
.../storage/index/inverted/query/regexp_query.cpp | 9 +
be/src/storage/index/inverted/query/regexp_query.h | 2 +
.../query_v2/regexp_query/regexp_weight.cpp | 50 ---
.../inverted/query_v2/regexp_query/regexp_weight.h | 6 +-
be/src/storage/predicate_collector.cpp | 5 +
.../compaction/collection_statistics_test.cpp | 46 +++
.../index/inverted/query/regexp_query_test.cpp | 34 +++
.../index/inverted/query_v2/regexp_query_test.cpp | 61 +---
.../inverted/query_v2/wildcard_query_test.cpp | 72 +----
.../storage/segment/index_reader_helper_test.cpp | 26 ++
.../search/test_search_score_pattern_constant.out | 75 +++++
.../search/test_search_score_regexp_wildcard.out | 337 ---------------------
.../test_search_score_pattern_constant.groovy | 79 +++++
.../test_search_score_regexp_wildcard.groovy | 80 -----
15 files changed, 301 insertions(+), 591 deletions(-)
diff --git a/be/src/storage/index/index_reader_helper.h
b/be/src/storage/index/index_reader_helper.h
index 988db5c73ec..4b436feed5f 100644
--- a/be/src/storage/index/index_reader_helper.h
+++ b/be/src/storage/index/index_reader_helper.h
@@ -106,6 +106,12 @@ public:
static bool is_need_similarity_score(InvertedIndexQueryType query_type,
const TabletIndex* index_meta) {
+ // A regexp scores a constant 1 per matching row, as an Elasticsearch
regexp query does in
+ // query context. The constant reads no statistics, norms or
positions, so every index can
+ // score it.
+ if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY) {
+ return true;
+ }
if (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY ||
query_type == InvertedIndexQueryType::MATCH_ALL_QUERY ||
query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY ||
@@ -122,6 +128,10 @@ public:
static bool is_need_similarity_score(TExprOpcode::type query_type,
const TabletIndex* index_meta) {
+ // See the InvertedIndexQueryType overload: a regexp scores a constant
on every index.
+ if (query_type == TExprOpcode::MATCH_REGEXP) {
+ return true;
+ }
if (query_type == TExprOpcode::MATCH_ANY || query_type ==
TExprOpcode::MATCH_ALL ||
query_type == TExprOpcode::MATCH_PHRASE ||
query_type == TExprOpcode::MATCH_PHRASE_PREFIX) {
diff --git a/be/src/storage/index/inverted/query/regexp_query.cpp
b/be/src/storage/index/inverted/query/regexp_query.cpp
index bb35a9275a2..f12b93a177f 100644
--- a/be/src/storage/index/inverted/query/regexp_query.cpp
+++ b/be/src/storage/index/inverted/query/regexp_query.cpp
@@ -39,6 +39,7 @@ void RegexpQuery::add(const InvertedIndexQueryInfo&
query_info) {
if (query_info.term_infos.empty()) {
throw Exception(ErrorCode::INVALID_ARGUMENT, "term_infos cannot be
empty");
}
+ _constant_score = _context->collection_similarity &&
query_info.is_similarity_score;
const std::string& pattern = query_info.term_infos[0].get_single_term();
if (is_hyperscan_regexp_expensive(pattern)) {
@@ -84,6 +85,14 @@ void RegexpQuery::add(const InvertedIndexQueryInfo&
query_info) {
void RegexpQuery::search(roaring::Roaring& roaring) {
_query.search(roaring);
+ // A regexp scores a constant 1 per matching row, as an Elasticsearch
regexp query does in query
+ // context, so body MATCH 'apple' AND body MATCH_REGEXP 'ap.*' adds 1 to
the apple score. The
+ // expanded terms' own BM25 would depend on how many terms the pattern
happens to match.
+ if (_constant_score) {
+ for (const auto row_id : roaring) {
+ _context->collection_similarity->collect(row_id, 1.0F);
+ }
+ }
}
std::optional<std::string> RegexpQuery::get_regex_prefix(const std::string&
pattern) {
diff --git a/be/src/storage/index/inverted/query/regexp_query.h
b/be/src/storage/index/inverted/query/regexp_query.h
index c0d56ffbeae..55a37299dbe 100644
--- a/be/src/storage/index/inverted/query/regexp_query.h
+++ b/be/src/storage/index/inverted/query/regexp_query.h
@@ -50,6 +50,8 @@ private:
int32_t _max_expansions = 50;
DisjunctionQuery _query;
+ // Set when score() is requested; every matching row then scores the
constant 1.
+ bool _constant_score = false;
friend class RegexpQueryTest;
};
diff --git
a/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.cpp
b/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.cpp
index e2576724ab4..5ecfad0d3ae 100644
--- a/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.cpp
+++ b/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.cpp
@@ -34,18 +34,12 @@
#include <gen_cpp/PaloBrokerService_types.h>
#include <algorithm>
-#include <cmath>
#include <string>
-#include <vector>
#include "storage/index/inverted/query_v2/bit_set_query/bit_set_scorer.h"
#include
"storage/index/inverted/query_v2/const_score_query/const_score_scorer.h"
#include "storage/index/inverted/query_v2/nullable_scorer.h"
-#include "storage/index/inverted/query_v2/score_combiner.h"
#include "storage/index/inverted/query_v2/segment_postings.h"
-#include "storage/index/inverted/query_v2/term_query/term_scorer.h"
-#include "storage/index/inverted/query_v2/union/buffered_union.h"
-#include "storage/index/inverted/similarity/bm25_similarity.h"
#include "storage/index/inverted/util/string_helper.h"
#include "util/hyperscan_util.h"
@@ -116,10 +110,6 @@ ScorerPtr RegexpWeight::regexp_scorer(const
QueryExecutionContext& context,
}
auto reader = lookup_reader(_field, context, binding_key);
- if (_enable_scoring) {
- return scored_terms_scorer(reader.get(), context, binding_key,
matching_terms);
- }
-
auto doc_bitset = std::make_shared<roaring::Roaring>();
for (const auto& term : matching_terms) {
auto segment_postings =
@@ -136,46 +126,6 @@ ScorerPtr RegexpWeight::regexp_scorer(const
QueryExecutionContext& context,
return const_score;
}
-ScorerPtr RegexpWeight::scored_terms_scorer(lucene::index::IndexReader* reader,
- const QueryExecutionContext&
context,
- const std::string& binding_key,
- const std::vector<std::wstring>&
matching_terms) {
- // Expanded terms are not collected in collection statistics, so use
reader statistics.
- const auto num_docs = reader->maxDoc();
- DORIS_CHECK_GT(num_docs, 0);
- const auto total_term_freq = reader->sumTotalTermFreq(_field.c_str());
- // Without norms, postings use a document length of one.
- const float avgdl = static_cast<float>(total_term_freq.value_or(num_docs))
/ num_docs;
- DORIS_CHECK_GT(avgdl, 0.0F);
-
- std::vector<ScorerPtr> term_scorers;
- term_scorers.reserve(matching_terms.size());
- auto logical_field = logical_field_or_fallback(context, binding_key,
_field);
- for (const auto& term : matching_terms) {
- Term lucene_term(_field.c_str(), term.c_str());
- const int32_t doc_freq = reader->docFreq(&lucene_term);
- if (doc_freq == 0) {
- continue;
- }
- const auto idf = static_cast<float>(
- std::log1p((static_cast<double>(num_docs) - doc_freq + 0.5) /
(doc_freq + 0.5)));
- auto similarity = std::make_shared<BM25Similarity>(idf, avgdl);
- auto posting =
- create_term_posting(reader, _field, term, true, similarity,
_context->io_ctx);
- if (posting) {
- term_scorers.emplace_back(std::make_shared<TermScorer>(
- std::move(posting), std::move(similarity), logical_field));
- }
- }
- if (term_scorers.empty()) {
- return std::make_shared<EmptyScorer>();
- }
- if (term_scorers.size() == 1) {
- return std::move(term_scorers.front());
- }
- return make_buffered_union(term_scorers, std::make_shared<SumCombiner>());
-}
-
std::optional<std::string> RegexpWeight::get_regex_prefix(const std::string&
pattern) {
DBUG_EXECUTE_IF("RegexpQuery.get_regex_prefix", { return std::nullopt; });
diff --git
a/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.h
b/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.h
index 6ef13c71fa5..f8b1a0ba9fa 100644
--- a/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.h
+++ b/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.h
@@ -35,10 +35,6 @@ public:
private:
ScorerPtr regexp_scorer(const QueryExecutionContext& context, const
std::string& binding_key);
- ScorerPtr scored_terms_scorer(lucene::index::IndexReader* reader,
- const QueryExecutionContext& context,
- const std::string& binding_key,
- const std::vector<std::wstring>&
matching_terms);
std::optional<std::string> get_regex_prefix(const std::string& pattern);
void collect_matching_terms(const QueryExecutionContext& context,
@@ -50,7 +46,7 @@ private:
std::wstring _field;
std::string _pattern;
- bool _enable_scoring = false;
+ [[maybe_unused]] bool _enable_scoring = false;
bool _nullable = true;
// Set to 0 to disable limit (ES has no default limit for prefix queries)
// The limit prevents collecting too many terms, but can cause incorrect
results
diff --git a/be/src/storage/predicate_collector.cpp
b/be/src/storage/predicate_collector.cpp
index b79ab47652e..cfebb513f50 100644
--- a/be/src/storage/predicate_collector.cpp
+++ b/be/src/storage/predicate_collector.cpp
@@ -295,6 +295,11 @@ Status MatchPredicateCollector::collect(RuntimeState*
state, const TabletSchemaS
if (!IndexReaderHelper::is_need_similarity_score(expr->op(),
index_meta)) {
continue;
}
+ // A regexp scores a constant and reads no statistics. Its value is a
pattern, not text, so
+ // analyzing it would collect statistics for made-up terms: 'ap.*'
would become "ap".
+ if (expr->op() == TExprOpcode::MATCH_REGEXP) {
+ continue;
+ }
auto options = DataTypeSerDe::get_default_format_options();
options.timezone = &state->timezone_obj();
diff --git a/be/test/storage/compaction/collection_statistics_test.cpp
b/be/test/storage/compaction/collection_statistics_test.cpp
index 7fa78f1e6a6..705e2bb3b21 100644
--- a/be/test/storage/compaction/collection_statistics_test.cpp
+++ b/be/test/storage/compaction/collection_statistics_test.cpp
@@ -1809,6 +1809,52 @@ TEST_F(CollectionStatisticsTest,
CollectSkipsIndexWithoutAnalyzer) {
// L2: Index whose analyzer is set (should_analyzer returns true) but does
// not declare "support_phrase=true". MockVExpr drives MATCH_PHRASE opcode,
// so is_need_similarity_score returns false and the index is skipped.
+// A regexp scores a constant, so it collects no statistics even on an index
whose MATCH_ANY does:
+// analyzing its pattern 'ap.*' would register the made-up term "ap".
+TEST_F(CollectionStatisticsTest, RegexpCollectsNoStatistics) {
+ auto tablet_schema = std::make_shared<TabletSchema>();
+
+ constexpr int32_t kColUid = 1360;
+ TabletColumn col;
+ col.set_unique_id(kColUid);
+ col.set_name("body");
+ col.set_type(FieldType::OLAP_FIELD_TYPE_STRING);
+ tablet_schema->append_column(col);
+
+ TabletIndexPB index_pb;
+ index_pb.set_index_id(2360);
+ index_pb.set_index_name("body_idx");
+ index_pb.set_index_type(IndexType::INVERTED);
+ index_pb.add_col_unique_id(kColUid);
+ (*index_pb.mutable_properties())["parser"] = "english";
+ (*index_pb.mutable_properties())["support_phrase"] = "true";
+ TabletIndex index;
+ index.init_from_pb(index_pb);
+ tablet_schema->append_index(std::move(index));
+
+ constexpr int kSlotId = 86;
+ runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId),
kColUid, "body", {});
+
+ auto match_expr =
std::make_shared<collection_statistics::MockVExpr>(TExprNodeType::MATCH_PRED);
+ match_expr->_children.push_back(
+ std::make_shared<collection_statistics::MockVSlotRef>("body",
SlotId(kSlotId)));
+
match_expr->_children.push_back(std::make_shared<collection_statistics::MockVLiteral>("ap.*"));
+
+ MatchPredicateCollector collector;
+ std::unordered_map<std::wstring, CollectInfo> collect_infos;
+ match_expr->_opcode = TExprOpcode::MATCH_ANY;
+ auto status =
+ collector.collect(runtime_state_.get(), tablet_schema, match_expr,
&collect_infos);
+ ASSERT_TRUE(status.ok()) << status.msg();
+ EXPECT_EQ(collect_infos.size(), 1U);
+
+ collect_infos.clear();
+ match_expr->_opcode = TExprOpcode::MATCH_REGEXP;
+ status = collector.collect(runtime_state_.get(), tablet_schema,
match_expr, &collect_infos);
+ ASSERT_TRUE(status.ok()) << status.msg();
+ EXPECT_TRUE(collect_infos.empty());
+}
+
TEST_F(CollectionStatisticsTest, CollectSkipsIndexWithoutSimilarityScore) {
auto tablet_schema = std::make_shared<TabletSchema>();
diff --git a/be/test/storage/index/inverted/query/regexp_query_test.cpp
b/be/test/storage/index/inverted/query/regexp_query_test.cpp
index 40287b9dff1..a53e7672be2 100644
--- a/be/test/storage/index/inverted/query/regexp_query_test.cpp
+++ b/be/test/storage/index/inverted/query/regexp_query_test.cpp
@@ -25,6 +25,7 @@
#include "io/fs/local_file_system.h"
#include "runtime/runtime_state.h"
+#include "storage/compaction/collection_similarity.h"
#include "storage/index/inverted/query/query.h"
#include "storage/index/inverted/query/query_info.h"
@@ -298,6 +299,39 @@ TEST_F(RegexpQueryTest, SearchWithEmptyTerms) {
EXPECT_TRUE(result.isEmpty());
}
+// With score() a regexp scores the constant 1 for every matching row, and
without it publishes
+// no score. The matching rows stand in for what the expanded terms' postings
would return.
+TEST_F(RegexpQueryTest, ScoresConstantOnePerMatchingRow) {
+ std::shared_ptr<lucene::search::IndexSearcher> searcher = nullptr;
+ OlapReaderStatistics stats;
+ RuntimeState runtime_state;
+ io::IOContext io_ctx;
+ for (const bool constant_score : {false, true}) {
+ SCOPED_TRACE(constant_score);
+ auto context = std::make_shared<IndexQueryContext>();
+ context->runtime_state = &runtime_state;
+ context->stats = &stats;
+ context->io_ctx = &io_ctx;
+ context->collection_similarity =
std::make_shared<CollectionSimilarity>();
+ RegexpQuery regexp_query(searcher, context);
+ regexp_query._constant_score = constant_score;
+
+ roaring::Roaring result;
+ result.add(1);
+ result.add(5);
+ regexp_query.search(result);
+
+ auto scores = context->collection_similarity->release_scores();
+ if (constant_score) {
+ ASSERT_EQ(scores.size(), 2U);
+ EXPECT_FLOAT_EQ(scores[1], 1.0F);
+ EXPECT_FLOAT_EQ(scores[5], 1.0F);
+ } else {
+ EXPECT_TRUE(scores.empty());
+ }
+ }
+}
+
TEST_F(RegexpQueryTest, GetRegexPrefixWithDebugPoint) {
// Test the debug point that forces get_regex_prefix to return nullopt
// This covers the debug execute if code path
diff --git a/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp
b/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp
index ed02ac578d3..aea4ed627ed 100644
--- a/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp
+++ b/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp
@@ -28,7 +28,6 @@
#include "storage/index/index_query_context.h"
#include "storage/index/inverted/analyzer/custom_analyzer.h"
#include "storage/index/inverted/query_v2/regexp_query/regexp_weight.h"
-#include "storage/index/inverted/query_v2/term_query/term_query.h"
#include "storage/index/inverted/util/string_helper.h"
CL_NS_USE(search)
@@ -59,13 +58,10 @@ public:
private:
void create_test_index(const std::string& field_name, const std::string&
dir) {
std::vector<std::string> test_data = {
- "apple123", "apple456", "banana789",
- "test123abc", "pattern456", "regex999",
- "match123", "search456", "prefix123",
- "suffix789", "apple_test", "banana_data",
- "test_pattern", "data_regex", "apple_banana",
- "test_match", "pattern_test", "prefix_suffix",
- "abc123xyz", "def456ghi", "apple789 apple789 apple456"};
+ "apple123", "apple456", "banana789", "test123abc",
"pattern456",
+ "regex999", "match123", "search456", "prefix123",
"suffix789",
+ "apple_test", "banana_data", "test_pattern", "data_regex",
"apple_banana",
+ "test_match", "pattern_test", "prefix_suffix", "abc123xyz",
"def456ghi"};
CustomAnalyzerConfig::Builder builder;
builder.with_tokenizer_config("standard", {});
@@ -171,53 +167,6 @@ TEST_F(RegexpQueryV2Test, test_regexp_query_with_scoring) {
ASSERT_NE(regexp_weight, nullptr);
}
-TEST_F(RegexpQueryV2Test, test_regexp_query_scores_matching_terms) {
- auto context = std::make_shared<IndexQueryContext>();
- context->collection_statistics = std::make_shared<CollectionStatistics>();
- context->collection_similarity = std::make_shared<CollectionSimilarity>();
-
- auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
- auto reader_holder =
make_shared_reader(lucene::index::IndexReader::open(dir, true));
- ASSERT_NE(reader_holder, nullptr);
-
- std::wstring field = StringHelper::to_wstring("content");
- auto query = std::make_shared<query_v2::RegexpQuery>(context, field,
"apple.*");
- auto weight = query->weight(true);
-
- query_v2::QueryExecutionContext exec_ctx;
- exec_ctx.segment_num_rows = reader_holder->maxDoc();
- exec_ctx.readers = {reader_holder};
- exec_ctx.field_reader_bindings.emplace(field, reader_holder);
-
- auto statistics = context->collection_statistics;
- statistics->_total_num_docs = reader_holder->maxDoc();
- statistics->_total_num_tokens[field] =
-
reader_holder->sumTotalTermFreq(field.c_str()).value_or(reader_holder->maxDoc());
- for (const auto* term : {L"apple123", L"apple789", L"apple456"}) {
- Term lucene_term(field.c_str(), term);
- statistics->_term_doc_freqs[field][term] =
reader_holder->docFreq(&lucene_term);
- }
- auto term_score = [&](const wchar_t* term, uint32_t doc_id) {
- auto term_query = std::make_shared<query_v2::TermQuery>(context,
field, term);
- auto term_scorer = term_query->weight(true)->scorer(exec_ctx);
- EXPECT_EQ(term_scorer->seek(doc_id), doc_id);
- return term_scorer->score();
- };
-
- auto scorer = weight->scorer(exec_ctx);
- ASSERT_EQ(scorer->seek(0), 0);
- float single_term_score = scorer->score();
- EXPECT_NE(single_term_score, 1.0F);
- EXPECT_FLOAT_EQ(single_term_score, term_score(L"apple123", 0));
- uint32_t last_doc = reader_holder->maxDoc() - 1;
- ASSERT_EQ(scorer->seek(last_doc), last_doc);
- EXPECT_GT(scorer->score(), single_term_score);
- EXPECT_NEAR(scorer->score(),
- term_score(L"apple789", last_doc) + term_score(L"apple456",
last_doc), 1e-5F);
-
- _CLDECDELETE(dir);
-}
-
// Test regexp query execution
TEST_F(RegexpQueryV2Test, test_regexp_query_execution) {
auto context = std::make_shared<IndexQueryContext>();
@@ -625,7 +574,7 @@ TEST_F(RegexpQueryV2Test,
test_make_exact_match_wildcard_pattern) {
doc = scorer->advance();
}
- EXPECT_EQ(result.cardinality(), 21);
+ EXPECT_EQ(result.cardinality(), 20);
_CLDECDELETE(dir);
}
diff --git a/be/test/storage/index/inverted/query_v2/wildcard_query_test.cpp
b/be/test/storage/index/inverted/query_v2/wildcard_query_test.cpp
index 3807a2787db..af119cdd260 100644
--- a/be/test/storage/index/inverted/query_v2/wildcard_query_test.cpp
+++ b/be/test/storage/index/inverted/query_v2/wildcard_query_test.cpp
@@ -27,7 +27,6 @@
#include "common/status.h"
#include "storage/index/index_query_context.h"
#include "storage/index/inverted/analyzer/custom_analyzer.h"
-#include "storage/index/inverted/query_v2/term_query/term_query.h"
#include "storage/index/inverted/query_v2/wildcard_query/wildcard_weight.h"
#include "storage/index/inverted/util/string_helper.h"
@@ -58,20 +57,14 @@ public:
private:
void create_test_index(const std::string& field_name, const std::string&
dir) {
- std::vector<std::string> test_data = {"apple", "application",
"apply",
- "apricot", "banana",
"band",
- "bandana", "can",
"candy",
- "cat", "catalog",
"dog",
- "document", "door",
"hello",
- "help", "helpful",
"test",
- "testing", "testcase",
"wildcard",
- "wild", "wilderness",
"card",
- "cardboard", "abc",
"abcd",
- "abcde", "prefix123",
"123suffix",
- "pre123", "123suf",
"both123both",
- "star", "start",
"started",
- "starter", "question",
"quest",
- "query", "queries",
"test test testing"};
+ std::vector<std::string> test_data = {
+ "apple", "application", "apply", "apricot",
"banana", "band",
+ "bandana", "can", "candy", "cat",
"catalog", "dog",
+ "document", "door", "hello", "help",
"helpful", "test",
+ "testing", "testcase", "wildcard", "wild",
"wilderness", "card",
+ "cardboard", "abc", "abcd", "abcde",
"prefix123", "123suffix",
+ "pre123", "123suf", "both123both", "star", "start",
"started",
+ "starter", "question", "quest", "query",
"queries"};
CustomAnalyzerConfig::Builder builder;
builder.with_tokenizer_config("standard", {});
@@ -162,53 +155,6 @@ TEST_F(WildcardQueryV2Test,
test_wildcard_query_with_scoring) {
ASSERT_NE(wildcard_weight, nullptr);
}
-TEST_F(WildcardQueryV2Test, test_wildcard_query_scores_matching_terms) {
- auto context = std::make_shared<IndexQueryContext>();
- context->collection_statistics = std::make_shared<CollectionStatistics>();
- context->collection_similarity = std::make_shared<CollectionSimilarity>();
-
- auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
- auto reader_holder =
make_shared_reader(lucene::index::IndexReader::open(dir, true));
- ASSERT_NE(reader_holder, nullptr);
-
- std::wstring field = StringHelper::to_wstring("field");
- auto query = std::make_shared<query_v2::WildcardQuery>(context, field,
"test*");
- auto weight = query->weight(true);
-
- query_v2::QueryExecutionContext exec_ctx;
- exec_ctx.segment_num_rows = reader_holder->maxDoc();
- exec_ctx.readers = {reader_holder};
- exec_ctx.field_reader_bindings.emplace(field, reader_holder);
-
- auto statistics = context->collection_statistics;
- statistics->_total_num_docs = reader_holder->maxDoc();
- statistics->_total_num_tokens[field] =
-
reader_holder->sumTotalTermFreq(field.c_str()).value_or(reader_holder->maxDoc());
- for (const auto* term : {L"test", L"testing"}) {
- Term lucene_term(field.c_str(), term);
- statistics->_term_doc_freqs[field][term] =
reader_holder->docFreq(&lucene_term);
- }
- auto term_score = [&](const wchar_t* term, uint32_t doc_id) {
- auto term_query = std::make_shared<query_v2::TermQuery>(context,
field, term);
- auto term_scorer = term_query->weight(true)->scorer(exec_ctx);
- EXPECT_EQ(term_scorer->seek(doc_id), doc_id);
- return term_scorer->score();
- };
-
- auto scorer = weight->scorer(exec_ctx);
- ASSERT_EQ(scorer->seek(17), 17);
- float single_term_score = scorer->score();
- EXPECT_NE(single_term_score, 1.0F);
- EXPECT_FLOAT_EQ(single_term_score, term_score(L"test", 17));
- uint32_t last_doc = reader_holder->maxDoc() - 1;
- ASSERT_EQ(scorer->seek(last_doc), last_doc);
- EXPECT_GT(scorer->score(), single_term_score);
- EXPECT_NEAR(scorer->score(), term_score(L"test", last_doc) +
term_score(L"testing", last_doc),
- 1e-5F);
-
- _CLDECDELETE(dir);
-}
-
// Test wildcard query with asterisk prefix pattern
TEST_F(WildcardQueryV2Test, test_wildcard_query_prefix_pattern) {
auto context = std::make_shared<IndexQueryContext>();
@@ -578,4 +524,4 @@ TEST_F(WildcardQueryV2Test,
test_wildcard_query_move_semantics) {
ASSERT_NE(weight2, nullptr);
}
-} // namespace doris::segment_v2
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/test/storage/segment/index_reader_helper_test.cpp
b/be/test/storage/segment/index_reader_helper_test.cpp
index ca7a1ea5c80..352d6853249 100644
--- a/be/test/storage/segment/index_reader_helper_test.cpp
+++ b/be/test/storage/segment/index_reader_helper_test.cpp
@@ -439,6 +439,32 @@ TEST_F(IndexReaderHelperTest,
NormalizerOnlyPhraseAndScoringNeedNoPositions) {
InvertedIndexQueryType::MATCH_ANY_QUERY, &custom_meta));
}
+TEST_F(IndexReaderHelperTest, RegexpScoresConstantOnEveryIndex) {
+ // A regexp scores a constant 1 and reads no statistics, so neither the
analyzer nor phrase
+ // support decides whether it scores.
+ for (const auto& properties : std::vector<std::map<std::string,
std::string>> {
+ {},
+ {{"parser", "none"}},
+ {{"parser", "english"}},
+ {{"parser", "english"}, {"support_phrase", "true"}},
+ {{"normalizer", "split_norm"}}}) {
+ SCOPED_TRACE(::testing::PrintToString(properties));
+ TabletIndexPB pb;
+ pb.set_index_id(1);
+ pb.set_index_name("regexp_index");
+ pb.set_index_type(IndexType::INVERTED);
+ pb.add_col_unique_id(1);
+ for (const auto& [key, value] : properties) {
+ (*pb.mutable_properties())[key] = value;
+ }
+ TabletIndex index;
+ index.init_from_pb(pb);
+
EXPECT_TRUE(IndexReaderHelper::is_need_similarity_score(TExprOpcode::MATCH_REGEXP,
&index));
+ EXPECT_TRUE(IndexReaderHelper::is_need_similarity_score(
+ InvertedIndexQueryType::MATCH_REGEXP_QUERY, &index));
+ }
+}
+
TEST_F(IndexReaderHelperTest, IsNeedSimilarityScoreWithTExprOpcodeTest) {
TabletIndex index_meta;
auto index_meta_pb = std::make_unique<TabletIndexPB>();
diff --git a/regression-test/data/search/test_search_score_pattern_constant.out
b/regression-test/data/search/test_search_score_pattern_constant.out
new file mode 100644
index 00000000000..f2a3c30f31d
--- /dev/null
+++ b/regression-test/data/search/test_search_score_pattern_constant.out
@@ -0,0 +1,75 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !V2_search_regexp --
+2 1
+3 1
+6 1
+
+-- !V2_search_wildcard --
+2 1
+3 1
+6 1
+
+-- !V2_search_prefix --
+2 1
+3 1
+6 1
+
+-- !V2_search_keyword_regexp --
+6 1
+7 1
+
+-- !V2_match_regexp --
+2 1
+3 1
+6 1
+
+-- !V2_match_and_match_regexp --
+2 2.023573
+
+-- !V2_search_and_regexp --
+2 2.023573
+
+-- !V2_match_only --
+2 1.023573
+5 1.163151
+
+-- !V2_match_and_equality --
+2 1.023573
+
+-- !V3_search_regexp --
+2 1
+3 1
+6 1
+
+-- !V3_search_wildcard --
+2 1
+3 1
+6 1
+
+-- !V3_search_prefix --
+2 1
+3 1
+6 1
+
+-- !V3_search_keyword_regexp --
+6 1
+7 1
+
+-- !V3_match_regexp --
+2 1
+3 1
+6 1
+
+-- !V3_match_and_match_regexp --
+2 2.023573
+
+-- !V3_search_and_regexp --
+2 2.023573
+
+-- !V3_match_only --
+2 1.023573
+5 1.163151
+
+-- !V3_match_and_equality --
+2 1.023573
+
diff --git a/regression-test/data/search/test_search_score_regexp_wildcard.out
b/regression-test/data/search/test_search_score_regexp_wildcard.out
deleted file mode 100644
index 393a5781dab..00000000000
--- a/regression-test/data/search/test_search_score_regexp_wildcard.out
+++ /dev/null
@@ -1,337 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !V2_standard_unscored_regexp --
-1
-2
-3
-4
-7
-8
-
--- !V2_standard_unscored_wildcard --
-1
-2
-3
-4
-7
-8
-
--- !V2_standard_false_regexp --
-8 2.087193
-4 1.312985
-7 1.248584
-3 1.120813
-2 0.655108
-1 0.584433
-
--- !V2_standard_false_wildcard --
-8 2.087193
-4 1.312985
-7 1.248584
-3 1.120813
-2 0.655108
-1 0.584433
-
--- !V2_standard_false_terms --
-8 2.087193
-4 1.312985
-7 1.248584
-3 1.120813
-2 0.655108
-1 0.584433
-
--- !V2_standard_false_ascending --
-1 0.584433
-2 0.655108
-3 1.120813
-7 1.248584
-4 1.312985
-8 2.087193
-
--- !V2_standard_false_missing --
-
--- !V2_standard_true_regexp --
-8 2.087193
-4 1.312985
-7 1.248584
-3 1.120813
-2 0.655108
-1 0.584433
-
--- !V2_standard_true_wildcard --
-8 2.087193
-4 1.312985
-7 1.248584
-3 1.120813
-2 0.655108
-1 0.584433
-
--- !V2_standard_true_terms --
-8 2.087193
-4 1.312985
-7 1.248584
-3 1.120813
-2 0.655108
-1 0.584433
-
--- !V2_standard_true_ascending --
-1 0.584433
-2 0.655108
-3 1.120813
-7 1.248584
-4 1.312985
-8 2.087193
-
--- !V2_standard_true_missing --
-
--- !V2_none_unscored_regexp --
-1
-2
-3
-4
-7
-8
-
--- !V2_none_unscored_wildcard --
-1
-2
-3
-4
-7
-8
-
--- !V2_none_false_regexp --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V2_none_false_wildcard --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V2_none_false_terms --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V2_none_false_ascending --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V2_none_false_missing --
-
--- !V2_none_true_regexp --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V2_none_true_wildcard --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V2_none_true_terms --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V2_none_true_ascending --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V2_none_true_missing --
-
--- !V3_standard_unscored_regexp --
-1
-2
-3
-4
-7
-8
-
--- !V3_standard_unscored_wildcard --
-1
-2
-3
-4
-7
-8
-
--- !V3_standard_false_regexp --
-8 2.087193
-4 1.312985
-7 1.248584
-3 1.120813
-2 0.655108
-1 0.584433
-
--- !V3_standard_false_wildcard --
-8 2.087193
-4 1.312985
-7 1.248584
-3 1.120813
-2 0.655108
-1 0.584433
-
--- !V3_standard_false_terms --
-8 2.087193
-4 1.312985
-7 1.248584
-3 1.120813
-2 0.655108
-1 0.584433
-
--- !V3_standard_false_ascending --
-1 0.584433
-2 0.655108
-3 1.120813
-7 1.248584
-4 1.312985
-8 2.087193
-
--- !V3_standard_false_missing --
-
--- !V3_standard_true_regexp --
-8 2.087193
-4 1.312985
-7 1.248584
-3 1.120813
-2 0.655108
-1 0.584433
-
--- !V3_standard_true_wildcard --
-8 2.087193
-4 1.312985
-7 1.248584
-3 1.120813
-2 0.655108
-1 0.584433
-
--- !V3_standard_true_terms --
-8 2.087193
-4 1.312985
-7 1.248584
-3 1.120813
-2 0.655108
-1 0.584433
-
--- !V3_standard_true_ascending --
-1 0.584433
-2 0.655108
-3 1.120813
-7 1.248584
-4 1.312985
-8 2.087193
-
--- !V3_standard_true_missing --
-
--- !V3_none_unscored_regexp --
-1
-2
-3
-4
-7
-8
-
--- !V3_none_unscored_wildcard --
-1
-2
-3
-4
-7
-8
-
--- !V3_none_false_regexp --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V3_none_false_wildcard --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V3_none_false_terms --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V3_none_false_ascending --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V3_none_false_missing --
-
--- !V3_none_true_regexp --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V3_none_true_wildcard --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V3_none_true_terms --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V3_none_true_ascending --
-1 1.791759
-2 1.791759
-3 1.791759
-4 1.791759
-7 1.791759
-8 1.791759
-
--- !V3_none_true_missing --
-
diff --git
a/regression-test/suites/search/test_search_score_pattern_constant.groovy
b/regression-test/suites/search/test_search_score_pattern_constant.groovy
new file mode 100644
index 00000000000..430f313b10f
--- /dev/null
+++ b/regression-test/suites/search/test_search_score_pattern_constant.groovy
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// score() of pattern queries follows Elasticsearch's default rewrite: a
regexp, wildcard or
+// prefix in SEARCH and a MATCH_REGEXP score a constant 1 per matching row,
whatever segment the
+// row is in, and add exactly 1 next to a scored clause. An equality predicate
adds nothing.
+suite("test_search_score_pattern_constant", "p0") {
+ // SEARCH and MATCH need index execution even when CI fuzzes these session
defaults.
+ sql "set enable_common_expr_pushdown = true"
+ sql "set enable_inverted_index_query = true"
+
+ for (def storageFormat : ["V2", "V3"]) {
+ sql "DROP TABLE IF EXISTS test_search_score_pattern_constant"
+ sql """
+ CREATE TABLE test_search_score_pattern_constant (
+ id BIGINT NOT NULL,
+ content STRING NULL,
+ tag STRING NULL,
+ INDEX idx_content(content) USING INVERTED
+ PROPERTIES("parser" = "english", "support_phrase" =
"true"),
+ INDEX idx_tag(tag) USING INVERTED
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default:1",
+ "inverted_index_storage_format" = "${storageFormat}",
+ "disable_auto_compaction" = "true"
+ )
+ """
+ // Two inserts make two segments with different term statistics; a
constant score must not
+ // depend on them.
+ sql """INSERT INTO test_search_score_pattern_constant VALUES
+ (1, 'Apache Doris is a high performance analytical database',
'doris'),
+ (2, 'Inverted indexes are suitable for full text search', 'index'),
+ (3, 'The parser splits text into multiple terms', 'parser'),
+ (4, 'Doris supports fast real time data analysis', 'doris'),
+ (5, 'Users can quickly search product names', 'search')
+ """
+ sql """INSERT INTO test_search_score_pattern_constant VALUES
+ (6, 'text text text', 'text'), (7, 'plain words only', 'textual')
+ """
+ sql "sync"
+
+ for (def query : [
+ ["search_regexp", "SEARCH('content:/.*ext/')"],
+ ["search_wildcard", "SEARCH('content:*ext')"],
+ ["search_prefix", "SEARCH('content:te*')"],
+ ["search_keyword_regexp", "SEARCH('tag:/text.*/')"],
+ ["match_regexp", "content MATCH_REGEXP 'ext'"],
+ // A pattern next to a scored clause adds exactly 1, as an
Elasticsearch bool must
+ // with a regexp clause does.
+ ["match_and_match_regexp", "content MATCH 'search' AND content
MATCH_REGEXP 'ext'"],
+ ["search_and_regexp", "SEARCH('content:search AND
content:/.*ext/')"],
+ ["match_only", "content MATCH 'search'"],
+ ["match_and_equality", "content MATCH 'search' AND tag =
'index'"]]) {
+ quickTest("${storageFormat}_${query[0]}", """
+ SELECT id, ROUND(s, 6) FROM (
+ SELECT id, score() AS s FROM
test_search_score_pattern_constant
+ WHERE ${query[1]} ORDER BY s DESC LIMIT 10
+ ) hits ORDER BY id
+ """, true)
+ }
+ }
+}
diff --git
a/regression-test/suites/search/test_search_score_regexp_wildcard.groovy
b/regression-test/suites/search/test_search_score_regexp_wildcard.groovy
deleted file mode 100644
index 01cd72c70a7..00000000000
--- a/regression-test/suites/search/test_search_score_regexp_wildcard.groovy
+++ /dev/null
@@ -1,80 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("test_search_score_regexp_wildcard", "p0") {
- sql "set enable_common_expr_pushdown = true"
- sql "set enable_inverted_index_query = true"
-
- def scoredQuery = { predicate, direction, limit ->
- """
- SELECT id, ROUND(bm25_score, 6)
- FROM (
- SELECT id, score() AS bm25_score
- FROM test_search_score_regexp_wildcard
- WHERE ${predicate}
- ORDER BY bm25_score ${direction} LIMIT ${limit}
- ) hits
- ORDER BY bm25_score ${direction}, id
- """
- }
- for (String storageFormat : ["V2", "V3"]) {
- for (String parser : ["standard", "none"]) {
- sql "DROP TABLE IF EXISTS test_search_score_regexp_wildcard"
- sql """
- CREATE TABLE test_search_score_regexp_wildcard (
- id INT NOT NULL,
- content STRING NULL,
- INDEX idx_content(content) USING INVERTED
PROPERTIES("parser" = "${parser}")
- ) ENGINE=OLAP
- DUPLICATE KEY(id)
- DISTRIBUTED BY HASH(id) BUCKETS 1
- PROPERTIES (
- "replication_allocation" = "tag.location.default:1",
- "inverted_index_storage_format" = "${storageFormat}"
- )
- """
- sql """
- INSERT INTO test_search_score_regexp_wildcard VALUES
- (1, 'apple'), (2, 'apple apple apple'), (3, 'application'),
- (4, 'apple application'), (5, 'banana'), (6, NULL),
- (7, 'apple application apple'), (8, 'apply apple')
- """
- sql "sync"
- String regexp = "search('content:/app.*/',
'{\"mode\":\"lucene\"}')"
- String wildcard = "search('content:*pp*', '{\"mode\":\"lucene\"}')"
- String terms = parser == "standard"
- ? "search('content:apple OR content:application OR
content:apply', '{\"mode\":\"lucene\"}')"
- : "search('content:EXACT(apple) OR content:EXACT(apple
apple apple) OR content:EXACT(application) OR content:EXACT(apple application)
OR content:EXACT(apple application apple) OR content:EXACT(apply apple)',
'{\"mode\":\"standard\"}')"
- String label = "${storageFormat}_${parser}"
- quickTest("${label}_unscored_regexp", """
- SELECT id FROM test_search_score_regexp_wildcard WHERE
${regexp} ORDER BY id
- """)
- quickTest("${label}_unscored_wildcard", """
- SELECT id FROM test_search_score_regexp_wildcard WHERE
${wildcard} ORDER BY id
- """)
- for (boolean wand : [false, true]) {
- sql "set enable_inverted_index_wand_query = ${wand}"
- quickTest("${label}_${wand}_regexp", scoredQuery(regexp,
"DESC", 100))
- quickTest("${label}_${wand}_wildcard", scoredQuery(wildcard,
"DESC", 100))
- quickTest("${label}_${wand}_terms", scoredQuery(terms, "DESC",
100))
- quickTest("${label}_${wand}_ascending", scoredQuery(regexp,
"ASC", 100))
- quickTest("${label}_${wand}_missing", scoredQuery(
- "search('content:/missing.*/',
'{\"mode\":\"lucene\"}')", "DESC", 100))
- }
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
