airborne12 commented on code in PR #69791:
URL: https://github.com/apache/doris/pull/69791#discussion_r4422147825
##########
be/src/storage/index/index_reader_helper.h:
##########
@@ -106,33 +107,34 @@ class IndexReaderHelper {
static bool is_need_similarity_score(InvertedIndexQueryType query_type,
const TabletIndex* index_meta) {
- if (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY ||
- query_type == InvertedIndexQueryType::MATCH_ALL_QUERY ||
- query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY ||
- query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY) {
- const auto& properties = index_meta->properties();
- if (get_parser_phrase_support_string_from_properties(properties) ==
- INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES ||
- is_normalizer_only(properties)) {
- return true;
- }
- }
- return false;
+ return (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY ||
+ query_type == InvertedIndexQueryType::MATCH_ALL_QUERY ||
+ query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY ||
+ query_type ==
InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY) &&
+ can_score_match(index_meta);
}
static bool is_need_similarity_score(TExprOpcode::type query_type,
const TabletIndex* index_meta) {
- if (query_type == TExprOpcode::MATCH_ANY || query_type ==
TExprOpcode::MATCH_ALL ||
- query_type == TExprOpcode::MATCH_PHRASE ||
- query_type == TExprOpcode::MATCH_PHRASE_PREFIX) {
- const auto& properties = index_meta->properties();
- if (get_parser_phrase_support_string_from_properties(properties) ==
- INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES ||
- is_normalizer_only(properties)) {
- return true;
- }
+ return (query_type == TExprOpcode::MATCH_ANY || query_type ==
TExprOpcode::MATCH_ALL ||
+ query_type == TExprOpcode::MATCH_PHRASE ||
+ query_type == TExprOpcode::MATCH_PHRASE_PREFIX) &&
+ can_score_match(index_meta);
+ }
+
+private:
+ static bool can_score_match(const TabletIndex* index_meta) {
+ const auto& properties = index_meta->properties();
+ // A keyword index keeps each value as one term, without positions or
norms, and its
+ // postings score with unit frequency and length whatever
support_phrase says, as an
+ // Elasticsearch keyword field does: over tags (x, x, y, z), tag MATCH
'x' scores
+ // idf(x) = ln(2) for both matching rows, the same as SEARCH('tag:x').
+ if
(!inverted_index::InvertedIndexAnalyzer::should_analyzer(properties)) {
Review Comment:
Confirmed and fixed in 79bc425dde. On this branch it was worse than a failed
query: `IndexReader::open` has no try/catch here, so the `CLuceneError` aborted
the BE, and a cached BKD searcher aborted it through `std::get`.
Fixed where the BKD index is opened rather than in this gate, because a
VARIANT path reaches the same code with a schema type that says text: the path
is typed per segment. A BKD index now adds no statistics, as a segment without
the path does: the cache miss checks for the BKD data file before opening, the
cache hit uses `get_if`. Those rows match without the index and score 0.
`test_match_keyword_score` covers the numeric cast, VARIANT numeric, text
and sparse paths, and the score query after a numeric predicate, on V2/V3; with
the old `std::get` the suite fails with the same abort.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
