airborne12 commented on code in PR #68758:
URL: https://github.com/apache/doris/pull/68758#discussion_r4254991613
##########
be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp:
##########
@@ -255,6 +256,56 @@ std::vector<TermInfo>
InvertedIndexAnalyzer::get_analyse_result(
return get_analyse_result(reader, analyzer.get());
}
+Status InvertedIndexAnalyzer::analyze(std::string_view value, const
InvertedIndexAnalyzerCtx* ctx,
+ const std::map<std::string,
std::string>& properties,
+ std::vector<TermInfo>* out) {
+ try {
+ if (ctx == nullptr) {
+ *out = get_analyse_result(value, properties);
+ } else if (!ctx->requires_analysis()) {
+ // Every string, the empty one included, is a term of an
untokenized index.
+ *out = {TermInfo {.term = std::string(value)}};
+ } else if (const auto analyzer = ctx->get_analyzer(); analyzer !=
nullptr) {
+ auto reader = create_reader(ctx->char_filter_map);
+ reader->init(value.data(), static_cast<int32_t>(value.size()),
true);
+ *out = get_analyse_result(reader, analyzer.get());
+ } else {
+ *out = get_analyse_result(value, properties);
+ }
+ } catch (const CLuceneError& e) {
+ return
Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>("analyzing '{}' failed:
{}",
+ value,
e.what());
+ } catch (const Exception& e) {
+ return
Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>("analyzing '{}' failed:
{}",
+ value,
e.what());
+ }
+ return Status::OK();
+}
+
+std::string InvertedIndexAnalyzer::normalize(const std::string& value,
+ const std::map<std::string,
std::string>& properties) {
+ const std::string analyzer_name =
get_analyzer_name_from_properties(properties);
+ if (analyzer_name.empty() || is_builtin_analyzer(analyzer_name)) {
Review Comment:
Confirmed and fixed in 2c83e70372e. For a builtin parser,
`InvertedIndexAnalyzer::normalize` now reads the pattern through the index's
character filter, using the same `create_reader(char_filter_map)` as the
writers, before the parser's own lowercasing.
`FunctionSearchTest.TestSniiNativePrefixAppliesTheIndexCharacterFilter` runs
`A-B*` on an SNII field that replaces `-` with `x` and expects the stem `axb`;
it was `a-b` before the fix.
`AnalyzerTest.TestNormalizeMatchesTheTermsOfBuiltinParsers` also checks,
through the real analyzers, that normalize returns the term each builtin parser
writes, including this char filter case.
##########
be/src/exprs/function/search/search_lowering.cpp:
##########
@@ -0,0 +1,450 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/query/logical/search_lowering.h"
+
+#include <algorithm>
+#include <cctype>
+#include <charconv>
+#include <iterator>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+
+namespace doris::index_query::logical {
+
+using segment_v2::InvertedIndexQueryType;
+
+namespace {
+
+bool is_compound(const std::string& clause_type) {
+ return clause_type == "AND" || clause_type == "OR" || clause_type == "NOT"
||
+ clause_type == "OCCUR_BOOLEAN";
+}
+
+bool is_pattern_clause(const std::string& clause_type) {
+ return clause_type == "WILDCARD" || clause_type == "REGEXP" || clause_type
== "PREFIX";
+}
+
+// Clause types whose value is analyzed on a tokenized index.
+bool is_analyzed_clause(const std::string& clause_type) {
+ return clause_type == "TERM" || clause_type == "PHRASE" || clause_type ==
"MATCH" ||
+ clause_type == "ANY" || clause_type == "ALL";
+}
+
+Occur to_occur(TSearchOccur::type occur) {
+ switch (occur) {
+ case TSearchOccur::SHOULD:
+ return Occur::kShould;
+ case TSearchOccur::MUST_NOT:
+ return Occur::kMustNot;
+ case TSearchOccur::MUST:
+ default:
+ return Occur::kMust;
+ }
+}
+
+void append_terms(Token token, std::vector<std::string>* terms) {
+ if (token.is_single_term()) {
+ terms->push_back(std::move(std::get<std::string>(token.term)));
+ } else {
+ auto& many = std::get<std::vector<std::string>>(token.term);
+ terms->insert(terms->end(), std::make_move_iterator(many.begin()),
+ std::make_move_iterator(many.end()));
+ }
+}
+
+// Groups tokens that share a position into one slot, in place.
+std::vector<Token> group_by_position(std::vector<Token> tokens) {
+ size_t slots = 0;
+ for (size_t i = 0; i < tokens.size(); ++i) {
+ if (slots > 0 && tokens[slots - 1].position == tokens[i].position) {
+ Token& slot = tokens[slots - 1];
+ if (slot.is_single_term()) {
+ slot.term = std::vector<std::string>
{std::move(std::get<std::string>(slot.term))};
+ }
+ append_terms(std::move(tokens[i]),
&std::get<std::vector<std::string>>(slot.term));
+ continue;
+ }
+ if (slots != i) {
+ tokens[slots] = std::move(tokens[i]);
+ }
+ ++slots;
+ }
+ tokens.resize(slots);
+ return tokens;
+}
+
+std::vector<std::string> flatten(std::vector<Token> tokens) {
+ std::vector<std::string> terms;
+ terms.reserve(tokens.size());
+ for (auto& token : tokens) {
+ append_terms(std::move(token), &terms);
+ }
+ return terms;
+}
+
+// Sets `out` to `phrase` over `tokens`, or to any of their terms when they
take one position.
+void lower_phrase(Phrase phrase, std::vector<Token> tokens, Node* out) {
+ phrase.slots = group_by_position(std::move(tokens));
+ if (phrase.slots.size() > 1) {
+ out->value = std::move(phrase);
+ return;
+ }
+ out->value =
+ TermSet {.field = std::move(phrase.field), .terms =
flatten(std::move(phrase.slots))};
+}
+
+// Sets `out` to a phrase whose last slot is a prefix; a single token is just
that prefix.
+void lower_phrase_prefix(std::vector<Token> tokens, Node* out) {
+ std::vector<Token> slots = group_by_position(std::move(tokens));
+ if (slots.size() == 1 && slots.front().is_single_term()) {
+ out->value = Expand {.field = {},
+ .kind = ExpandKind::kPrefix,
+ .pattern =
std::move(std::get<std::string>(slots.front().term))};
+ return;
+ }
+ out->value = Phrase {.field = {}, .slots = std::move(slots), .prefix =
true};
+}
+
+// Sets `out` to a phrase whose first slot takes the terms that end with its
token and whose last
+// slot the terms that start with its token; a single token takes every term
that contains it.
+void lower_phrase_edge(std::vector<Token> tokens, Node* out) {
+ std::vector<Token> slots = group_by_position(std::move(tokens));
+ if (slots.size() == 1 && slots.front().is_single_term()) {
+ out->value = Expand {.field = {},
+ .kind = ExpandKind::kContains,
+ .pattern =
std::move(std::get<std::string>(slots.front().term))};
+ return;
+ }
+ out->value = Phrase {.field = {}, .slots = std::move(slots), .prefix =
true, .suffix = true};
+}
+
+// Removes a trailing " ~N" or " ~N+" and returns its slop and ordering rule.
+std::pair<int32_t, bool> take_slop(std::string_view* value) {
+ const size_t space = value->find_last_of(' ');
+ if (space == std::string_view::npos || value->substr(space + 1, 1) != "~")
{
+ return {};
+ }
+ std::string_view digits = value->substr(space + 2);
+ const bool ordered = digits.size() > 1 && digits.back() == '+';
+ if (ordered) {
+ digits.remove_suffix(1);
+ }
+ int32_t slop = 0;
+ if (digits.empty() ||
+ !std::ranges::all_of(digits, [](unsigned char c) { return
std::isdigit(c) != 0; }) ||
+ std::from_chars(digits.data(), digits.data() + digits.size(), slop).ec
!= std::errc()) {
+ return {};
+ }
+ *value = value->substr(0, space);
+ return {slop, ordered};
+}
+
+NodePtr lower_direct_index_leaf(const std::string& clause_type, FieldRef field,
+ const std::string& value) {
+ if (clause_type == "TERM" || clause_type == "EXACT") {
+ return make_node(
+ Compare {.field = std::move(field), .op = CompareOp::kEqual,
.value = value});
+ }
+ return make_node(Unknown {.field = std::move(field)});
+}
+
+// WILDCARD, REGEXP and PREFIX match dictionary terms against a pattern. As in
Elasticsearch's
+// query_string, a prefix or a glob is normalized the way the index normalizes
its terms and is
+// never analyzed, and a regular expression is taken as written.
+Status lower_pattern_leaf(const std::string& clause_type, const FieldProps&
props,
+ FieldCatalog& catalog, FieldRef field, const
std::string& value,
+ NodePtr* out) {
+ if (clause_type == "REGEXP") {
+ // SEARCH matches a regular expression against whole terms on every
format.
+ *out = make_node(Expand {.field = std::move(field),
+ .kind = ExpandKind::kRegexp,
+ .pattern = "^(" + value + ")$"});
+ return Status::OK();
+ }
+ if (clause_type == "WILDCARD" && value == "*") {
+ *out = make_node(Exists {.field = std::move(field)});
+ return Status::OK();
+ }
+ const bool prefix = clause_type == "PREFIX";
+ // The DSL keeps a PREFIX value's trailing '*'.
+ std::string pattern =
+ prefix && value.ends_with('*') ? value.substr(0, value.size() - 1)
: value;
+ if (props.analyzed) {
+ std::string normalized;
+ RETURN_IF_ERROR(catalog.normalize(props, pattern, &normalized));
Review Comment:
Confirmed and fixed in 2c83e70372e. Lowering now normalizes only the literal
text between a WILDCARD pattern's `*` and `?` and keeps the operators as
written. The wildcard matcher has no escape syntax, as both characters are
always operators, so nothing else needs restoring. PREFIX still normalizes its
whole stem.
`FunctionSearchTest.TestSniiNativeWildcardNormalizesOnlyItsLiterals` uses a
policy whose char filter replaces `*` and `?` with `x`. For `A*B` and `A?B` it
expects the pattern to stay a wildcard, so the source enumerates terms from the
normalized literal prefix `a`. Before the fix both patterns became the literal
`axb`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
