airborne12 commented on code in PR #68758: URL: https://github.com/apache/doris/pull/68758#discussion_r4254992644
########## be/src/storage/index/inverted/query/engine/phrase_query/phrase_weight.cpp: ########## @@ -0,0 +1,1065 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/query/engine/phrase_query/phrase_weight.h" + +#include <algorithm> +#include <functional> +#include <iterator> +#include <limits> +#include <memory> +#include <optional> +#include <span> +#include <utility> + +#include "common/check.h" +#include "common/exception.h" +#include "storage/index/inverted/common/string_helper.h" +#include "storage/index/inverted/query/engine/bit_set_query/bit_set_scorer.h" +#include "storage/index/inverted/query/engine/const_score_query/const_score_scorer.h" +#include "storage/index/inverted/query/engine/nullable_scorer.h" +#include "storage/index/inverted/query/engine/phrase_query/phrase_scorer.h" +#include "storage/index/inverted/query/engine/postings/listed_walk.h" +#include "storage/index/inverted/query/engine/scored_rows_scorer.h" +#include "storage/index/inverted/query/engine/segment_postings.h" +#include "storage/index/inverted/query/exec/cursor_chained_postings.h" +#include "storage/index/inverted/query/exec/term_pattern.h" +#include "storage/index/inverted/query/exec/term_waves.h" +#include "storage/index/inverted/query/phrase/exact_phrase_matcher.h" +#include "storage/index/inverted/query/phrase/phrase_verifier.h" +#include "storage/index/inverted/query/phrase/position_span.h" +#include "storage/index/inverted/query/spi/postings_cursor.h" + +namespace doris::segment_v2::inverted_index::query_v2 { + +SlotPhraseWeight::SlotPhraseWeight(std::wstring field, index_query::PhraseQueryOptions options, + index_query::ScoringContextPtr<float> similarity, + bool enable_scoring, bool nullable) + : _field(std::move(field)), + _options(options), + _similarity(std::move(similarity)), + _enable_scoring(enable_scoring), + _nullable(nullable) {} + +index_query::IndexSourcePtr SlotPhraseWeight::_source(const QueryExecutionContext& ctx, + const std::string& binding_key) const { + auto source = lookup_source(_field, ctx, binding_key); + if (!source) { + throw Exception(ErrorCode::NOT_FOUND, "Reader not found for field '{}'", + StringHelper::to_string(_field)); + } + return source; +} + +ScorerPtr SlotPhraseWeight::scorer(const QueryExecutionContext& ctx, + const std::string& binding_key) { + auto source = _source(ctx, binding_key); + ScorerPtr scorer = _lists(*source) ? _listed_scorer(*source, _options.candidates) + : _streamed_scorer(*source, ctx.segment_num_rows); + if (_nullable) { + auto logical_field = logical_field_or_fallback(ctx, binding_key, _field); + return make_nullable_scorer(scorer, logical_field, ctx.null_resolver); + } + return scorer; +} + +// A scored phrase needs a scorer to retain its phrase frequencies. +bool SlotPhraseWeight::lists_rows(const QueryExecutionContext& ctx, + const std::string& binding_key) const { + auto source = lookup_source(_field, ctx, binding_key); + return source != nullptr && !_enable_scoring && _lists(*source); +} + +index_query::TruthSet SlotPhraseWeight::listed_rows(const QueryExecutionContext& ctx, + const std::string& binding_key, + const roaring::Roaring* candidates) { + auto source = _source(ctx, binding_key); + DORIS_CHECK(_lists(*source)); + // The scan's candidates narrow the conjunction's for the rows listed; the UNKNOWN rows + // are the field's among the conjunction's candidates, as the streamed phrase reports them. + const roaring::Roaring* listed = candidates; + std::optional<roaring::Roaring> narrowed; + if (_options.candidates != nullptr) { + if (candidates == nullptr) { + listed = _options.candidates; + } else { + narrowed = *candidates & *_options.candidates; + listed = &*narrowed; + } + } + index_query::TruthSet result; + std::vector<uint32_t> matched; + _list_matches(*source, listed, &matched, nullptr); + result.true_rows.addMany(matched.size(), matched.data()); + if (_nullable) { + auto nulls = FieldNullBitmapFetcher::fetch( + ctx.null_resolver, logical_field_or_fallback(ctx, binding_key, _field)); + if (nulls != nullptr) { + result.null_rows = candidates == nullptr ? *nulls : *nulls & *candidates; + } + } + return result; +} + +PhraseWeight::PhraseWeight(std::wstring field, std::vector<TermInfo> term_infos, + index_query::PhraseQueryOptions options, + index_query::ScoringContextPtr<float> similarity, bool enable_scoring, + bool nullable) + : SlotPhraseWeight(std::move(field), options, std::move(similarity), enable_scoring, + nullable), + _term_infos(std::move(term_infos)) {} + +std::vector<PhraseSlot> PhraseWeight::_slots() const { + std::vector<PhraseSlot> slots; + slots.reserve(_term_infos.size()); + for (const auto& term_info : _term_infos) { + slots.push_back({.offset = static_cast<uint32_t>(term_info.position), + .terms = {term_info.get_single_term()}}); + } + return slots; +} + +ScorerPtr PhraseWeight::_streamed_scorer(index_query::IndexSource& source, uint32_t num_docs) { + std::vector<std::pair<size_t, SegmentPostingsPtr>> term_postings_list; + const uint32_t first_doc = _options.candidates != nullptr && !_options.candidates->isEmpty() + ? _options.candidates->minimum() + : 0; + for (const auto& term_info : _term_infos) { + size_t offset = term_info.position; + auto posting = open_postings(source, term_info.get_single_term(), Review Comment: Confirmed and fixed in fd861bd5014. The streamed phrase weight now opens one posting per distinct term and passes the same pointer to every clause that repeats it, so `PhraseScorer` sees the repeat. The multi-phrase weight had the same problem and now shares a posting per identical slot. `PhraseQueryV2Test.ARepeatedTermNeedsAnOccurrencePerClause` runs `lazy lazy` on the real CLucene test index. Before the fix, slop 1 matched rows 0, 3, 4, 10, 15, 18 and 19; now no row matches at slop 0 or 1, and only row 4, which holds `lazy` twice, matches at slop 3. `AMultiPhraseRepeatedSlotNeedsAnOccurrencePerClause` covers multi-phrase on streamed and listed sources; it failed on the streamed path before the fix. ########## be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp: ########## @@ -255,6 +256,56 @@ std::vector<TermInfo> InvertedIndexAnalyzer::get_analyse_result( return get_analyse_result(reader, analyzer.get()); } +Status InvertedIndexAnalyzer::analyze(std::string_view value, const InvertedIndexAnalyzerCtx* ctx, + const std::map<std::string, std::string>& properties, + std::vector<TermInfo>* out) { + try { + if (ctx == nullptr) { + *out = get_analyse_result(value, properties); + } else if (!ctx->requires_analysis()) { + // Every string, the empty one included, is a term of an untokenized index. + *out = {TermInfo {.term = std::string(value)}}; + } else if (const auto analyzer = ctx->get_analyzer(); analyzer != nullptr) { + auto reader = create_reader(ctx->char_filter_map); + reader->init(value.data(), static_cast<int32_t>(value.size()), true); + *out = get_analyse_result(reader, analyzer.get()); + } else { + *out = get_analyse_result(value, properties); + } + } catch (const CLuceneError& e) { + return Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>("analyzing '{}' failed: {}", + value, e.what()); + } catch (const Exception& e) { + return Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>("analyzing '{}' failed: {}", + value, e.what()); + } + return Status::OK(); +} + +std::string InvertedIndexAnalyzer::normalize(const std::string& value, + const std::map<std::string, std::string>& properties) { + const std::string analyzer_name = get_analyzer_name_from_properties(properties); + if (analyzer_name.empty() || is_builtin_analyzer(analyzer_name)) { + // The index writers lowercase a builtin analyzer's terms unless lower_case is false. + return get_parser_lowercase_from_properties<true>(properties) == INVERTED_INDEX_PARSER_TRUE + ? to_lower(value) Review Comment: Confirmed and fixed in 2c83e70372e. Builtin normalization now applies each parser's own term normalization after the character filter. The ICU, basic and kuromoji parsers lowercase with ICU (root locale, the mapping `LowerCaseFilter` uses, now shared with kuromoji through `unicode_lower`); the standard, unicode, english and chinese tokenizers lowercase ASCII bytes only and keep doing so. kuromoji and IK also fold full-width ASCII first, as their tokenizers do. `lower_case=false` still skips lowercasing. `FunctionSearchTest.TestSniiNativePrefixLowercasesLikeTheParser` runs `ÉClair*` on SNII fields with the ICU and standard parsers and expects the stems `éclair` and `Éclair`; the ICU case failed before the fix. `AnalyzerTest.TestNormalizeMatchesTheTermsOfBuiltinParsers` compares normalize, with and without lowercasing, with the terms the real standard, unicode, english, basic, ICU and kuromoji analyzers write, and checks IK's terms directly because its dictionary is a process-wide singleton in the test binary. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
