airborne12 commented on code in PR #68758:
URL: https://github.com/apache/doris/pull/68758#discussion_r4278849749
##########
be/src/exprs/function/search/function_search.cpp:
##########
@@ -355,12 +402,8 @@ Status
FunctionSearch::evaluate_inverted_index_with_search_param_unguarded(
// so score() queries must execute scorers to populate
CollectionSimilarity.
const bool enable_scoring =
index_query_context != nullptr &&
index_query_context->collection_similarity != nullptr;
- // Also bypass the DSL cache when any referenced field is served by an
SNII reader.
auto* dsl_cache =
- enable_cache && !enable_scoring &&
-
!referenced_fields_contain_snii_reader(search_param.root, iterators)
- ? InvertedIndexQueryCache::instance()
- : nullptr;
+ enable_cache && !enable_scoring ?
InvertedIndexQueryCache::instance() : nullptr;
Review Comment:
Fixed in 750d2f248e6. A cached SEARCH result now needs both of its entries:
a lookup that finds the TRUE rows without their UNKNOWN entry is a miss, and
the search computes both again (`lookup_dsl_result`). The UNKNOWN entry is also
inserted before the TRUE rows (`insert_dsl_result`). This covers both the
publication window and an independent eviction.
Test: `FunctionSearchNullTest.DslCacheEntryWithoutItsNullRowsIsAMiss` caches
only the TRUE rows of `NOT body:foo`. Before the change the hit returned no
UNKNOWN rows; now the search recomputes, returns row 3 as UNKNOWN, and the next
search hits both entries.
##########
be/src/storage/index/inverted/query/engine/term_query/term_weight.h:
##########
@@ -62,29 +63,31 @@ class TermWeight : public Weight {
std::move(result));
}
+ const std::wstring& field() const { return _field; }
+ const std::string& term() const { return _term; }
+ const index_query::ScoringContextPtr<float>& similarity() const { return
_similarity; }
+ bool scores() const { return _enable_scoring; }
+
private:
TermOrEmptyScorer specialized_scorer(const QueryExecutionContext& ctx,
const std::string& binding_key) {
- auto reader = lookup_reader(_field, ctx, binding_key);
+ auto source = lookup_source(_field, ctx, binding_key);
auto logical_field = logical_field_or_fallback(ctx, binding_key,
_field);
- if (!reader) {
+ if (!source) {
return std::make_shared<EmptyScorer>();
}
- SegmentPostingsPtr segment_postings;
- segment_postings = create_term_posting(reader.get(), _field, _term,
_enable_scoring,
- _similarity, _context->io_ctx);
+ SegmentPostingsPtr segment_postings =
+ open_postings(*source, _term, /*positions=*/false,
_enable_scoring, _similarity);
if (segment_postings) {
return std::make_shared<TermScorer>(segment_postings, _similarity,
logical_field);
}
return std::make_shared<EmptyScorer>();
Review Comment:
Fixed in 750d2f248e6. An absent term now yields an empty scorer that keeps
its field's NULL rows (`TermWeight::scorer`). The grouped paths do the same: a
listed conjunction or disjunction with absent terms keeps the field's UNKNOWN
rows, scored or not, and a conjunction with an absent term only short-circuits
when its field has no NULL rows.
Test: `FunctionSearchNullTest.AbsentTermKeepsTheFieldNullRows` runs `NOT
body:missing`, `NOT (body:missing AND body:foo)` and `NOT (body:missing OR
body:absent)` on batching and streaming sources, scored and unscored; each now
leaves the NULL row UNKNOWN.
`ListedTermsTest.AnAbsentTermMakesTheConjunctionFalseEverywhere` asserted the
old behavior and is now `AnAbsentTermLeavesOnlyTheNullRowsUnknown`.
##########
be/src/storage/index/inverted/query/engine/boolean_query/listed_terms.cpp:
##########
@@ -0,0 +1,304 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/query/engine/boolean_query/listed_terms.h"
+
+#include <parallel_hashmap/phmap.h>
+
+#include <algorithm>
+#include <bit>
+#include <optional>
+#include <span>
+#include <utility>
+
+#include "common/check.h"
+#include "common/exception.h"
+#include "storage/index/inverted/query/engine/postings/listed_walk.h"
+#include "storage/index/inverted/query/engine/scored_rows_scorer.h"
+#include "storage/index/inverted/query/exec/block_doc_set.h"
+#include "storage/index/inverted/query/exec/cursor_chained_postings.h"
+#include "storage/index/inverted/query/exec/roaring_docid_sink.h"
+#include "storage/index/inverted/query/exec/term_waves.h"
+#include "storage/index/inverted/query/phrase/position_span.h"
+
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+ListedTerms::ListedTerms(index_query::IndexSourcePtr source,
+ std::shared_ptr<roaring::Roaring> nulls)
+ : _source(std::move(source)), _nulls(std::move(nulls)) {}
+
+void ListedTerms::add(size_t clause, std::string term,
+ index_query::ScoringContextPtr<float> similarity) {
+ DCHECK(_clauses.empty() || _clauses.back() < clause);
+ _clauses.push_back(clause);
+ _terms.push_back(std::move(term));
+ _similarities.push_back(std::move(similarity));
+}
+
+bool ListedTerms::holds(size_t clause) const {
+ return std::ranges::binary_search(_clauses, clause);
+}
+
+void ListedTerms::open(bool conjunctive, bool scoring) {
+ if (!conjunctive) {
+ return;
+ }
+ for (const std::string& term : _terms) {
+ bool held = false;
+ THROW_IF_ERROR(_source->may_hold(term, &held));
+ if (!held) {
+ _cursors.resize(_terms.size());
+ return;
+ }
+ }
+ // A scored conjunction reads the positions of the rows it lists.
+ THROW_IF_ERROR(
+ _source->open_terms(_terms, /*positions=*/scoring,
/*scoring=*/false, &_cursors));
+}
+
+bool ListedTerms::has_absent_term() const {
+ return std::ranges::any_of(_cursors, [](const auto& cursor) { return
cursor == nullptr; });
+}
+
+uint64_t ListedTerms::cheapest_doc_freq() const {
+ uint64_t cheapest = UINT64_MAX;
+ for (const auto& cursor : _cursors) {
+ cheapest = std::min<uint64_t>(cheapest, cursor == nullptr ? 0 :
cursor->doc_freq());
+ }
+ return cheapest;
+}
+
+index_query::TruthSet ListedTerms::conjunction(const std::vector<uint32_t>*
candidates) {
+ index_query::TruthSet result;
+ if (has_absent_term()) {
+ return result;
+ }
+ std::vector<uint32_t> docs;
+ THROW_IF_ERROR(index_query::chain_cursors(_cursors, candidates, &docs));
+ result.true_rows.addMany(docs.size(), docs.data());
+ if (_nulls != nullptr) {
+ result.null_rows = *_nulls;
+ }
+ return result;
+}
+
+index_query::TruthSet ListedTerms::disjunction() {
+ index_query::TruthSet result;
+ bool any_present = false;
+ index_query::RoaringDocIdSink sink(result.true_rows);
+ THROW_IF_ERROR(_source->collect_terms(_terms, sink, &any_present));
+ if (any_present && _nulls != nullptr) {
+ result.null_rows = *_nulls;
+ }
+ return result;
+}
+
+namespace {
+
+// Merges a term's postings into the rows scored so far, both ascending: a row
both hold sums
+// the scores, a row either holds alone keeps its own.
+void merge_term(index_query::BlockDocSet& docs,
index_query::ScoringContext<float>& similarity,
+ std::vector<uint32_t>* rows, std::vector<float>* scores) {
+ // Reading the rows so far through spans lets the loop keep their bounds
in registers, which
+ // it cannot do through vectors a caller may hold by reference.
+ const std::span<const uint32_t> old_rows(*rows);
+ const std::span<const float> old_scores(*scores);
+ std::vector<uint32_t> merged_rows;
+ std::vector<float> merged_scores;
+ merged_rows.reserve(old_rows.size() + docs.size_hint());
+ merged_scores.reserve(old_rows.size() + docs.size_hint());
+ size_t i = 0;
+ for (; !docs.exhausted(); docs.advance()) {
+ const uint32_t doc = docs.doc();
+ for (; i < old_rows.size() && old_rows[i] < doc; ++i) {
+ merged_rows.push_back(old_rows[i]);
+ merged_scores.push_back(old_scores[i]);
+ }
+ float score = similarity.score(static_cast<float>(docs.freq()),
docs.norm());
+ if (i < old_rows.size() && old_rows[i] == doc) {
+ score += old_scores[i];
+ ++i;
+ }
+ merged_rows.push_back(doc);
+ merged_scores.push_back(score);
+ }
+ merged_rows.insert(merged_rows.end(), old_rows.begin() +
static_cast<std::ptrdiff_t>(i),
+ old_rows.end());
+ merged_scores.insert(merged_scores.end(), old_scores.begin() +
static_cast<std::ptrdiff_t>(i),
+ old_scores.end());
+ rows->swap(merged_rows);
+ scores->swap(merged_scores);
+}
+
+// The rows a disjunction holds and their scores, one slot per row of the
segment: a term adds
+// into the slots in one pass where merging would copy every row scored so far.
+class RowSlots {
+public:
+ RowSlots(uint32_t doc_count, std::span<const uint32_t> rows,
std::span<const float> scores)
+ : _scores(doc_count, 0.0F), _held((doc_count + 63) / 64, 0) {
+ for (size_t i = 0; i < rows.size(); ++i) {
+ _scores[rows[i]] = scores[i];
+ hold(rows[i]);
+ }
+ }
+
+ void add_term(index_query::BlockDocSet& docs,
index_query::ScoringContext<float>& similarity) {
+ for (; !docs.exhausted(); docs.advance()) {
+ const uint32_t doc = docs.doc();
+ DCHECK_LT(doc, _scores.size());
+ _scores[doc] += similarity.score(static_cast<float>(docs.freq()),
docs.norm());
+ hold(doc);
+ }
+ }
+
+ // The rows held, ascending, and their scores.
+ void list(std::vector<uint32_t>* rows, std::vector<float>* scores) const {
+ size_t count = 0;
+ for (const uint64_t word : _held) {
+ count += std::popcount(word);
+ }
+ rows->clear();
+ scores->clear();
+ rows->reserve(count);
+ scores->reserve(count);
+ for (size_t word = 0; word < _held.size(); ++word) {
+ for (uint64_t bits = _held[word]; bits != 0; bits &= bits - 1) {
+ const auto row = static_cast<uint32_t>(word * 64 +
std::countr_zero(bits));
+ rows->push_back(row);
+ scores->push_back(_scores[row]);
+ }
+ }
+ }
+
+private:
+ void hold(uint32_t row) { _held[row / 64] |= uint64_t {1} << (row % 64); }
+
+ std::vector<float> _scores;
+ std::vector<uint64_t> _held;
+};
+
+} // namespace
+
+// The chain lists the rows, then the terms read their positions a wave at a
time, one round
+// each, and score each row on the positions it holds there and the source's
norm.
+ScorerPtr ListedTerms::scored_conjunction() {
+ if (has_absent_term()) {
+ return std::make_shared<EmptyScorer>();
+ }
+ std::vector<uint32_t> rows;
+ THROW_IF_ERROR(index_query::chain_cursors(_cursors, nullptr, &rows));
+ if (rows.empty()) {
Review Comment:
Fixed in 750d2f248e6. When the chain lists no row, or a term is absent,
`ListedTerms::scored_conjunction` now returns an empty scored result that
carries the field's NULL rows, as the unscored conjunction does.
Test: `FunctionSearchNullTest.EmptyScoredConjunctionKeepsTheFieldNullRows`
runs `NOT body:foo/bar` with `default_operator=and` on a batching source,
scored and unscored. Both now leave the NULL row UNKNOWN; before the change the
scored run returned it.
##########
be/src/storage/index/inverted/query/engine/boolean_query/listed_terms.cpp:
##########
@@ -0,0 +1,304 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/query/engine/boolean_query/listed_terms.h"
+
+#include <parallel_hashmap/phmap.h>
+
+#include <algorithm>
+#include <bit>
+#include <optional>
+#include <span>
+#include <utility>
+
+#include "common/check.h"
+#include "common/exception.h"
+#include "storage/index/inverted/query/engine/postings/listed_walk.h"
+#include "storage/index/inverted/query/engine/scored_rows_scorer.h"
+#include "storage/index/inverted/query/exec/block_doc_set.h"
+#include "storage/index/inverted/query/exec/cursor_chained_postings.h"
+#include "storage/index/inverted/query/exec/roaring_docid_sink.h"
+#include "storage/index/inverted/query/exec/term_waves.h"
+#include "storage/index/inverted/query/phrase/position_span.h"
+
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+ListedTerms::ListedTerms(index_query::IndexSourcePtr source,
+ std::shared_ptr<roaring::Roaring> nulls)
+ : _source(std::move(source)), _nulls(std::move(nulls)) {}
+
+void ListedTerms::add(size_t clause, std::string term,
+ index_query::ScoringContextPtr<float> similarity) {
+ DCHECK(_clauses.empty() || _clauses.back() < clause);
+ _clauses.push_back(clause);
+ _terms.push_back(std::move(term));
+ _similarities.push_back(std::move(similarity));
+}
+
+bool ListedTerms::holds(size_t clause) const {
+ return std::ranges::binary_search(_clauses, clause);
+}
+
+void ListedTerms::open(bool conjunctive, bool scoring) {
+ if (!conjunctive) {
+ return;
+ }
+ for (const std::string& term : _terms) {
+ bool held = false;
+ THROW_IF_ERROR(_source->may_hold(term, &held));
+ if (!held) {
+ _cursors.resize(_terms.size());
+ return;
+ }
+ }
+ // A scored conjunction reads the positions of the rows it lists.
+ THROW_IF_ERROR(
+ _source->open_terms(_terms, /*positions=*/scoring,
/*scoring=*/false, &_cursors));
+}
+
+bool ListedTerms::has_absent_term() const {
+ return std::ranges::any_of(_cursors, [](const auto& cursor) { return
cursor == nullptr; });
+}
+
+uint64_t ListedTerms::cheapest_doc_freq() const {
+ uint64_t cheapest = UINT64_MAX;
+ for (const auto& cursor : _cursors) {
+ cheapest = std::min<uint64_t>(cheapest, cursor == nullptr ? 0 :
cursor->doc_freq());
+ }
+ return cheapest;
+}
+
+index_query::TruthSet ListedTerms::conjunction(const std::vector<uint32_t>*
candidates) {
+ index_query::TruthSet result;
+ if (has_absent_term()) {
+ return result;
+ }
+ std::vector<uint32_t> docs;
+ THROW_IF_ERROR(index_query::chain_cursors(_cursors, candidates, &docs));
+ result.true_rows.addMany(docs.size(), docs.data());
+ if (_nulls != nullptr) {
+ result.null_rows = *_nulls;
+ }
+ return result;
+}
+
+index_query::TruthSet ListedTerms::disjunction() {
+ index_query::TruthSet result;
+ bool any_present = false;
+ index_query::RoaringDocIdSink sink(result.true_rows);
+ THROW_IF_ERROR(_source->collect_terms(_terms, sink, &any_present));
+ if (any_present && _nulls != nullptr) {
Review Comment:
Fixed in 750d2f248e6. A leaf on an array field whose segment has NULL rows
now compiles behind `NullMaskedQuery`. Its scorer skips those rows and reports
them UNKNOWN, and its listed rows subtract them, so TRUE and UNKNOWN are
disjoint before any Boolean composition. This holds in standard and lucene
mode, scored, unscored and TopK. Leaves on other fields are not wrapped.
Test: `FunctionSearchNullTest.ArrayNullRowWithValuesIsUnknown` gives the
NULL row an `alpha` posting, the shape of the array-null-payload fixture, on a
fake source. It checks `arr:alpha`, `arr:alpha OR other:omega`, `arr:alpha AND
other:x`, `NOT arr:alpha`, the lucene-mode OR and a TopK. The row is now
UNKNOWN in each three-valued query and FALSE in the lucene-mode one.
##########
be/src/exprs/function/search/function_search.cpp:
##########
@@ -0,0 +1,634 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exprs/function/search/function_search.h"
+
+#include <CLucene/config/repl_wchar.h>
+#include <CLucene/debug/error.h>
+#include <fmt/format.h>
+#include <gen_cpp/Exprs_types.h>
+#include <glog/logging.h>
+
+#include <algorithm>
+#include <iterator>
+#include <memory>
+#include <optional>
+#include <roaring/roaring.hh>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "common/config.h"
+#include "common/exception.h"
+#include "common/status.h"
+#include "core/block/columns_with_type_and_name.h"
+#include "exprs/function/search/function_search_leaf.h"
+#include "exprs/function/search/search_lowering.h"
+#include "exprs/function/search/variant_inverted_index_search.h"
+#include "exprs/function/simple_function_factory.h"
+#include "exprs/vexpr_context.h"
+#include "runtime/runtime_profile.h"
+#include "storage/index/index_file_reader.h"
+#include "storage/index/index_query_context.h"
+#include "storage/index/inverted/analysis/analyzer/analyzer.h"
+#include "storage/index/inverted/inverted_index_iterator.h"
+#include "storage/index/inverted/inverted_index_reader.h"
+#include "storage/index/inverted/query/engine/all_query/all_query.h"
+#include "storage/index/inverted/query/engine/bit_set_query/bit_set_query.h"
+#include
"storage/index/inverted/query/engine/boolean_query/boolean_query_builder.h"
+#include "storage/index/inverted/query/engine/boolean_query/operator.h"
+#include "storage/index/inverted/query/engine/collect/doc_set_collector.h"
+#include "storage/index/inverted/query/engine/collect/top_k_collector.h"
+#include "storage/index/inverted/query/exec/term_pattern.h"
+#include "storage/index/inverted/query/logical/node.h"
+#include "storage/index/inverted/reader/collection_statistics.h"
+#include "storage/olap_common.h"
+#include "storage/segment/variant/nested_group_provider.h"
+#include "util/thrift_util.h"
+
+namespace doris {
+
+namespace query_v2 = segment_v2::inverted_index::query_v2;
+
+// Serializes the whole TSearchParam, so every field of the DSL takes part in
the signature, and
+// adds the limit on the terms PREFIX, WILDCARD and REGEXP clauses expand to.
+std::string FunctionSearch::dsl_cache_signature(const TSearchParam&
search_param,
+ const IndexQueryContext*
context) {
+ ThriftSerializer ser(false, 1024);
+ TSearchParam copy = search_param;
+ std::string sig;
+ auto st = ser.serialize(©, &sig);
+ if (UNLIKELY(!st.ok())) {
+ LOG(WARNING) << "build_dsl_signature: Thrift serialization failed: "
<< st.to_string()
+ << ", caching disabled for this query";
+ return "";
+ }
+ sig += "|max_expansions=";
+ sig += std::to_string(
+ index_query::max_expansions(context != nullptr ?
context->runtime_state : nullptr));
+ return sig;
+}
+
+// Extract segment path prefix from the first available inverted index
iterator.
+// All fields in the same segment share the same path prefix.
+static std::string extract_segment_prefix(
+ const std::unordered_map<std::string, IndexIterator*>& iterators) {
+ for (const auto& [field_name, iter] : iterators) {
+ auto* inv_iter = dynamic_cast<InvertedIndexIterator*>(iter);
+ if (!inv_iter) continue;
+ // Try fulltext reader first, then string type
+ for (auto type :
+ {InvertedIndexReaderType::FULLTEXT,
InvertedIndexReaderType::STRING_TYPE}) {
+ IndexReaderType reader_type = type;
+ auto reader = inv_iter->get_reader(reader_type);
+ if (!reader) continue;
+ auto inv_reader =
std::dynamic_pointer_cast<InvertedIndexReader>(reader);
+ if (!inv_reader) continue;
+ auto file_reader = inv_reader->get_index_file_reader();
+ if (!file_reader) continue;
+ return file_reader->get_index_path_prefix();
+ }
+ }
+ VLOG_DEBUG << "extract_segment_prefix: no suitable inverted index reader
found across "
+ << iterators.size() << " iterators, caching disabled for this
query";
+ return "";
+}
+
+namespace {
+
+bool is_nested_group_search_supported() {
+ auto provider = segment_v2::create_nested_group_read_provider();
+ return provider != nullptr &&
provider->should_enable_nested_group_read_path();
+}
+
+namespace logical = index_query::logical;
+
+// Lowering's view of the resolver: what index a field binds to and how that
+// index analyzes values.
+class SearchFieldCatalog final : public logical::FieldCatalog {
+public:
+ SearchFieldCatalog(FieldReaderResolver& resolver,
std::shared_ptr<IndexQueryContext> context)
+ : _resolver(resolver), _context(std::move(context)) {}
+
+ Status resolve(const std::string& field, InvertedIndexQueryType query_type,
+ logical::FieldProps* out) override {
+ FieldReaderBinding binding;
+ RETURN_IF_ERROR(_resolver.resolve(field, query_type, &binding));
+ if (!binding.is_bound()) {
+ LOG(INFO) << "search: No inverted index for field '" << field
+ << "' in this segment, query_type=" <<
static_cast<int>(query_type)
+ << ", returning UNKNOWN bitmap";
+ *out = logical::FieldProps {};
+ return Status::OK();
+ }
+ const bool scalar = binding.inverted_reader != nullptr &&
+ binding.inverted_reader->type() ==
InvertedIndexReaderType::BKD;
+ const bool analyzed = !scalar &&
inverted_index::InvertedIndexAnalyzer::should_analyzer(
+ binding.index_properties);
+ *out = logical::FieldProps {.bound = true,
+ .direct_index = scalar,
+ .analyzed = analyzed,
+ .binding = binding.binding_key};
+ return Status::OK();
+ }
+
+ Status analyze(const logical::FieldProps& props, const std::string& value,
+ std::vector<logical::Token>* out) override {
+ const FieldReaderBinding* binding =
_resolver.find_binding(props.binding);
+ if (binding == nullptr) {
+ return Status::InternalError("search: no binding '{}' to analyze
with", props.binding);
+ }
+ int64_t unused_timer = 0;
+ SCOPED_RAW_TIMER(_context != nullptr && _context->stats != nullptr
+ ?
&_context->stats->inverted_index_analyzer_timer
+ : &unused_timer);
+ InvertedIndexAnalyzerCtxSPtr analyzer_ctx;
+ RETURN_IF_ERROR(_resolver.analyzer_context_for(props.binding,
&analyzer_ctx));
+ return inverted_index::InvertedIndexAnalyzer::analyze(value,
analyzer_ctx.get(),
+
binding->index_properties, out);
+ }
+
+ Status normalize(const logical::FieldProps& props, const std::string&
value,
+ std::string* out) override {
+ const FieldReaderBinding* binding =
_resolver.find_binding(props.binding);
+ if (binding == nullptr) {
+ return Status::InternalError("search: no binding '{}' to normalize
with",
+ props.binding);
+ }
+ try {
+ *out = inverted_index::InvertedIndexAnalyzer::normalize(value,
+
binding->index_properties);
+ } catch (const CLuceneError& e) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>(
+ "search: normalizing '{}' failed: {}", value, e.what());
+ } catch (const Exception& e) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>(
+ "search: normalizing '{}' failed: {}", value, e.what());
+ }
+ return Status::OK();
+ }
+
+private:
+ FieldReaderResolver& _resolver;
+ std::shared_ptr<IndexQueryContext> _context;
+};
+
+query_v2::Occur to_query_occur(logical::Occur occur) {
+ switch (occur) {
+ case logical::Occur::kShould:
+ return query_v2::Occur::SHOULD;
+ case logical::Occur::kMustNot:
+ return query_v2::Occur::MUST_NOT;
+ case logical::Occur::kMust:
+ default:
+ return query_v2::Occur::MUST;
+ }
+}
+
+Status compile_node(const logical::Node& node, const SearchLeafContext& ctx,
+ FieldReaderResolver& resolver, query_v2::QueryPtr* out,
+ std::string* binding_key);
+
+using Clauses = std::vector<std::pair<logical::Occur, logical::NodePtr>>;
+
+bool has_required_clause(const Clauses& clauses) {
+ return std::ranges::any_of(
+ clauses, [](const auto& clause) { return clause.first ==
logical::Occur::kMust; });
+}
+
+// Without scoring, optional clauses beside a required one change neither the
rows nor their
+// UNKNOWN state, so they are left out.
+Clauses matching_clauses(const logical::Bool& boolean, bool scoring) {
+ if (scoring || boolean.op != logical::BoolOp::kOccur ||
boolean.min_should_match != 0 ||
+ !has_required_clause(boolean.clauses)) {
+ return boolean.clauses;
+ }
+ Clauses kept;
+ std::ranges::copy_if(boolean.clauses, std::back_inserter(kept), [](const
auto& clause) {
+ return clause.first != logical::Occur::kShould;
+ });
+ return kept;
+}
+
+// AND, OR and NOT ignore the per-clause occur; OCCUR keeps it and the
threshold. A Boolean
+// left with one clause after dropping clauses that only score is that clause.
+Status compile_bool(const logical::Bool& boolean, const SearchLeafContext& ctx,
+ FieldReaderResolver& resolver, query_v2::QueryPtr* out) {
+ const Clauses clauses = matching_clauses(boolean, ctx.scoring);
+ if (clauses.size() == 1 && boolean.clauses.size() > 1) {
+ return compile_node(*clauses.front().second, ctx, resolver, out,
nullptr);
+ }
+ std::vector<query_v2::QueryPtr> queries(clauses.size());
+ std::vector<std::string> keys(clauses.size());
+ for (size_t i = 0; i < clauses.size(); ++i) {
+ RETURN_IF_ERROR(compile_node(*clauses[i].second, ctx, resolver,
&queries[i], &keys[i]));
+ }
+ if (boolean.op == logical::BoolOp::kOccur) {
+ auto builder = query_v2::create_occur_boolean_query_builder();
+ builder->set_minimum_number_should_match(boolean.min_should_match);
+ for (size_t i = 0; i < clauses.size(); ++i) {
+ builder->add(queries[i], to_query_occur(clauses[i].first),
std::move(keys[i]));
+ }
+ *out = builder->build();
+ return Status::OK();
+ }
+ query_v2::OperatorType op = query_v2::OperatorType::OP_AND;
+ if (boolean.op == logical::BoolOp::kOr) {
+ op = query_v2::OperatorType::OP_OR;
+ } else if (boolean.op == logical::BoolOp::kNot) {
+ op = query_v2::OperatorType::OP_NOT;
+ }
+ auto builder = query_v2::create_operator_boolean_query_builder(op);
+ for (size_t i = 0; i < clauses.size(); ++i) {
+ builder->add(queries[i], std::move(keys[i]));
+ }
+ *out = builder->build();
+ return Status::OK();
+}
+
+// A threshold counts the set's terms above the field's compiler, so every
format
+// answers it the same way: each term is its own leaf on the field.
+Status compile_term_threshold(const logical::TermSet& set, const
SearchLeafContext& ctx,
+ SearchLeafCompiler& compiler,
query_v2::QueryPtr* out) {
+ auto builder = query_v2::create_occur_boolean_query_builder();
+ builder->set_minimum_number_should_match(set.min_should_match);
+ const query_v2::Occur occur = set.require_all ? query_v2::Occur::MUST :
query_v2::Occur::SHOULD;
+ for (const auto& term : set.terms) {
+ const auto leaf = logical::make_node(logical::TermSet {
+ .field = set.field, .terms = {term}, .require_all = false,
.min_should_match = 0});
+ query_v2::QueryPtr term_query;
+ RETURN_IF_ERROR(compiler.compile(*leaf, ctx, &term_query));
+ builder->add(term_query, occur, set.field.binding);
+ }
+ *out = builder->build();
+ return Status::OK();
+}
+
+// A leaf goes to the compiler of the index it was bound to; its binding key
and
+// the leaf mapper follow it. A term threshold stays one leaf for the mapper.
+Status compile_node(const logical::Node& node, const SearchLeafContext& ctx,
+ FieldReaderResolver& resolver, query_v2::QueryPtr* out,
+ std::string* binding_key) {
+ *out = nullptr;
+ if (binding_key != nullptr) {
+ binding_key->clear();
+ }
+ if (node.as<logical::All>() != nullptr) {
+ *out = std::make_shared<query_v2::AllQuery>();
+ return Status::OK();
+ }
+ if (const auto* boolean = node.as<logical::Bool>()) {
+ return compile_bool(*boolean, ctx, resolver, out);
+ }
+ const logical::FieldRef* field = node.field();
+ DCHECK(field != nullptr);
+ if (binding_key != nullptr) {
+ *binding_key = field->binding;
+ }
+ if (node.as<logical::Unknown>() != nullptr) {
+ *out = make_unknown_leaf_query(ctx.num_rows);
+ return resolver.map_leaf_query(field->name, out);
+ }
+ const FieldReaderBinding* binding = resolver.find_binding(field->binding);
+ if (binding == nullptr || binding->leaf_compiler == nullptr) {
+ return Status::InternalError("search: field '{}' has no compiler for
binding '{}'",
+ field->name, field->binding);
+ }
+ const auto* set = node.as<logical::TermSet>();
+ if (set != nullptr && set->min_should_match > 0) {
+ RETURN_IF_ERROR(compile_term_threshold(*set, ctx,
*binding->leaf_compiler, out));
+ } else {
+ RETURN_IF_ERROR(binding->leaf_compiler->compile(node, ctx, out));
+ }
+ return resolver.map_leaf_query(field->name, out);
+}
+
+} // namespace
+
+Status FunctionSearch::execute_impl(FunctionContext* /*context*/, Block&
/*block*/,
+ const ColumnNumbers& /*arguments*/,
uint32_t /*result*/,
+ size_t /*input_rows_count*/) const {
+ return Status::RuntimeError("only inverted index queries are supported");
+}
+
+// Enhanced implementation: Handle new parameter structure (DSL +
SlotReferences)
+Status FunctionSearch::evaluate_inverted_index(
+ const ColumnsWithTypeAndName& arguments,
+ const std::vector<IndexFieldNameAndTypePair>& data_type_with_names,
+ std::vector<IndexIterator*> iterators, uint32_t num_rows,
+ const InvertedIndexAnalyzerCtx* /*analyzer_ctx*/,
+ InvertedIndexResultBitmap& bitmap_result) const {
+ return Status::OK();
+}
+
+Status FunctionSearch::evaluate_inverted_index_with_search_param(
+ const TSearchParam& search_param,
+ const std::unordered_map<std::string, IndexFieldNameAndTypePair>&
data_type_with_names,
+ std::unordered_map<std::string, IndexIterator*> iterators, uint32_t
num_rows,
+ InvertedIndexResultBitmap& bitmap_result, bool enable_cache) const {
+ return evaluate_inverted_index_with_search_param(search_param,
data_type_with_names,
+ std::move(iterators),
num_rows, bitmap_result,
+ enable_cache, nullptr,
nullptr);
+}
+
+// A lazy leaf reads its index while the result is listed; what it throws is
returned like the
+// errors the evaluation returns.
+Status FunctionSearch::evaluate_inverted_index_with_search_param(
+ const TSearchParam& search_param,
+ const std::unordered_map<std::string, IndexFieldNameAndTypePair>&
data_type_with_names,
+ std::unordered_map<std::string, IndexIterator*> iterators, uint32_t
num_rows,
+ InvertedIndexResultBitmap& bitmap_result, bool enable_cache,
+ const IndexExecContext* index_exec_ctx, const TabletColumn*
nested_column,
+ const std::shared_ptr<IndexQueryContext>& index_query_context) const {
+ // VSearchExpr enters here directly, outside IFunction::execute() and its
exception boundary,
+ // so this is where a failure inside the search, such as an analyzer whose
first token
+ // stream throws, has to become a Status.
+ try {
+ RETURN_IF_ERROR_OR_CATCH_EXCEPTION(_evaluate_search_param(
+ search_param, data_type_with_names, std::move(iterators),
num_rows, bitmap_result,
+ enable_cache, index_exec_ctx, nested_column,
index_query_context));
+ } catch (const CLuceneError& e) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>("search
failed: {}",
+
e.what());
+ }
+ return Status::OK();
+}
+
+// The evaluation keeps its result cache, the tree's compile and the
collection in one path.
+//
NOLINTNEXTLINE(readability-function-cognitive-complexity,readability-function-size)
+Status FunctionSearch::_evaluate_search_param(
+ const TSearchParam& search_param,
+ const std::unordered_map<std::string, IndexFieldNameAndTypePair>&
data_type_with_names,
+ std::unordered_map<std::string, IndexIterator*> iterators, uint32_t
num_rows,
+ InvertedIndexResultBitmap& bitmap_result, bool enable_cache,
+ const IndexExecContext* index_exec_ctx, const TabletColumn*
nested_column,
+ const std::shared_ptr<IndexQueryContext>& index_query_context) const {
+ const bool is_nested_query = search_param.root.clause_type == "NESTED";
+ if (is_nested_query && !is_nested_group_search_supported()) {
+ return Status::NotSupported(
+ "NESTED query requires NestedGroup support, which is
unavailable in this build");
+ }
+
+ if (!is_nested_query && (iterators.empty() ||
data_type_with_names.empty())) {
+ LOG(INFO) << "No indexed columns or iterators available, returning
empty result, dsl:"
+ << search_param.original_dsl;
+ bitmap_result =
InvertedIndexResultBitmap(std::make_shared<roaring::Roaring>(),
+
std::make_shared<roaring::Roaring>());
+ return Status::OK();
+ }
+
+ // Track overall query time (equivalent to inverted_index_query_timer in
MATCH path).
+ // Must be declared before the DSL cache lookup so that cache-hit fast
paths are
+ // also covered by the timer.
+ int64_t query_timer_dummy = 0;
+ OlapReaderStatistics* outer_stats = index_query_context ?
index_query_context->stats : nullptr;
+ SCOPED_RAW_TIMER(outer_stats ? &outer_stats->inverted_index_query_timer :
&query_timer_dummy);
+
+ // DSL result cache only stores bitmap/null bitmap. It does not store BM25
scores,
+ // so score() queries must execute scorers to populate
CollectionSimilarity.
+ const bool enable_scoring =
+ index_query_context != nullptr &&
index_query_context->collection_similarity != nullptr;
+ auto* dsl_cache =
+ enable_cache && !enable_scoring ?
InvertedIndexQueryCache::instance() : nullptr;
+ std::string seg_prefix;
+ std::string dsl_sig;
+ InvertedIndexQueryCache::CacheKey dsl_cache_key;
+ bool cache_usable = false;
+ if (dsl_cache) {
+ seg_prefix = extract_segment_prefix(iterators);
+ dsl_sig = dsl_cache_signature(search_param, index_query_context.get());
+ if (!seg_prefix.empty() && !dsl_sig.empty()) {
+ dsl_cache_key = InvertedIndexQueryCache::CacheKey {
+ seg_prefix, "__search_dsl__",
InvertedIndexQueryType::SEARCH_DSL_QUERY,
+ dsl_sig};
+ cache_usable = true;
+ InvertedIndexQueryCacheHandle dsl_cache_handle;
+ bool dsl_hit = false;
+ {
+ int64_t lookup_dummy = 0;
+ SCOPED_RAW_TIMER(outer_stats ?
&outer_stats->inverted_index_lookup_timer
+ : &lookup_dummy);
+ dsl_hit = dsl_cache->lookup(dsl_cache_key, &dsl_cache_handle);
+ }
+ if (dsl_hit) {
+ auto cached_bitmap = dsl_cache_handle.get_bitmap();
+ if (cached_bitmap) {
+ if (outer_stats) {
+ outer_stats->inverted_index_query_cache_hit++;
+ }
+ // Also retrieve cached null bitmap for three-valued SQL
logic
+ // (needed by compound operators NOT, OR, AND in
VCompoundPred)
+ auto null_cache_key = InvertedIndexQueryCache::CacheKey {
+ seg_prefix, "__search_dsl__",
InvertedIndexQueryType::SEARCH_DSL_QUERY,
+ dsl_sig + "__null"};
+ InvertedIndexQueryCacheHandle null_cache_handle;
+ std::shared_ptr<roaring::Roaring> null_bitmap;
+ if (dsl_cache->lookup(null_cache_key, &null_cache_handle))
{
+ null_bitmap = null_cache_handle.get_bitmap();
+ }
+ if (!null_bitmap) {
+ null_bitmap = std::make_shared<roaring::Roaring>();
+ }
+ bitmap_result =
+ InvertedIndexResultBitmap(cached_bitmap,
std::move(null_bitmap));
+ return Status::OK();
+ }
+ }
+ if (outer_stats) {
+ outer_stats->inverted_index_query_cache_miss++;
+ }
+ }
+ }
+
+ std::shared_ptr<IndexQueryContext> context;
+ if (index_query_context) {
+ context = index_query_context;
+ } else {
+ context = std::make_shared<IndexQueryContext>();
+ context->collection_statistics =
std::make_shared<CollectionStatistics>();
+ context->collection_similarity =
std::make_shared<CollectionSimilarity>();
+ }
+
+ const auto* effective_data_type_with_names = &data_type_with_names;
+
+ // Pass field_bindings to resolver for variant subcolumn detection
+ FieldReaderResolver resolver(*effective_data_type_with_names, iterators,
context,
+ search_param.field_bindings);
+
+ if (is_nested_query) {
+ std::shared_ptr<roaring::Roaring> row_bitmap;
+ VariantNestedSearchEvaluator nested_evaluator(*this);
+ RETURN_IF_ERROR(nested_evaluator.evaluate(search_param,
search_param.root, context,
+ resolver, num_rows,
index_exec_ctx, nested_column,
+ row_bitmap));
+ bitmap_result = InvertedIndexResultBitmap(std::move(row_bitmap),
+
std::make_shared<roaring::Roaring>());
+ bitmap_result.mask_out_null();
+ return Status::OK();
+ }
+
+ // Extract default_operator from TSearchParam (default: "or")
+ std::string default_operator = "or";
+ if (search_param.__isset.default_operator &&
!search_param.default_operator.empty()) {
+ default_operator = search_param.default_operator;
+ }
+ // Extract minimum_should_match from TSearchParam (-1 means not set)
+ int32_t minimum_should_match = -1;
+ if (search_param.__isset.minimum_should_match) {
+ minimum_should_match = search_param.minimum_should_match;
+ }
+
+ auto* stats = context->stats;
+ int64_t dummy_timer = 0;
+ SCOPED_RAW_TIMER(stats ? &stats->inverted_index_searcher_search_timer :
&dummy_timer);
+
+ query_v2::QueryPtr root_query;
+ std::string root_binding_key;
+ {
+ int64_t init_dummy = 0;
+ SCOPED_RAW_TIMER(stats ?
&stats->inverted_index_searcher_search_init_timer : &init_dummy);
+ RETURN_IF_ERROR(build_query_recursive(search_param.root, context,
resolver, &root_query,
+ &root_binding_key,
default_operator,
+ minimum_should_match, num_rows,
enable_scoring));
+ }
+ if (root_query == nullptr) {
+ LOG(INFO) << "search: Query tree resolved to empty query, dsl:"
+ << search_param.original_dsl;
+ bitmap_result =
InvertedIndexResultBitmap(std::make_shared<roaring::Roaring>(),
+
std::make_shared<roaring::Roaring>());
+ return Status::OK();
+ }
+
+ VariantSearchNullBitmapAdapter null_resolver(resolver);
Review Comment:
Fixed in 750d2f248e6. `VariantSearchNullBitmapAdapter::bitmap_for` now
throws the read Status instead of logging it.
`evaluate_inverted_index_with_search_param` turns it into the query's Status,
as the native SNII leaf did with `RETURN_IF_ERROR`. The array-field NULL read
added for the ARRAY fix returns its Status the same way.
Test: `FunctionSearchNullTest.FailedNullBitmapReadFailsTheSearch` fails the
NULL-section read of a scalar field under `NOT body:foo`. The search now
returns that error, scored and unscored; before the change it returned OK.
##########
be/src/storage/index/inverted/reader/text_index_reader.cpp:
##########
@@ -0,0 +1,676 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/reader/text_index_reader.h"
+
+#include <algorithm>
+#include <map>
+#include <memory>
+#include <roaring/roaring.hh>
+#include <set>
+#include <string>
+#include <string_view>
+#include <type_traits>
+
+#include "common/config.h"
+#include "common/exception.h"
+#include "common/logging.h"
+#include "common/status.h"
+#include "core/data_type/primitive_type.h"
+#include "runtime/runtime_profile.h"
+#include "runtime/runtime_state.h"
+#include "storage/index/index_file_reader.h"
+#include "storage/index/index_reader_helper.h"
+#include "storage/index/inverted/analysis/analyzer/analyzer.h"
+#include "storage/index/inverted/analysis/inverted_index_parser.h"
+#include "storage/index/inverted/common/inverted_index_cache.h"
+#include "storage/index/inverted/common/inverted_index_query_type.h"
+#include "storage/index/inverted/common/single_flight.h"
+#include "storage/index/inverted/common/string_helper.h"
+#include "storage/index/inverted/inverted_index_iterator.h"
+#include "storage/index/inverted/query/engine/all_query/all_query.h"
+#include "storage/index/inverted/query/engine/bit_set_query/bit_set_query.h"
+#include
"storage/index/inverted/query/engine/boolean_query/boolean_query_builder.h"
+#include "storage/index/inverted/query/engine/boolean_query/listed_terms.h"
+#include "storage/index/inverted/query/engine/boolean_query/operator.h"
+#include "storage/index/inverted/query/engine/collect/doc_set_collector.h"
+#include "storage/index/inverted/query/engine/complete_null_bitmap.h"
+#include "storage/index/inverted/query/engine/expand_query/expand_query.h"
+#include
"storage/index/inverted/query/engine/phrase_prefix_query/phrase_prefix_query.h"
+#include
"storage/index/inverted/query/engine/phrase_query/multi_phrase_query.h"
+#include "storage/index/inverted/query/engine/phrase_query/phrase_query.h"
+#include "storage/index/inverted/query/engine/term_query/term_query.h"
+#include "storage/index/inverted/query/exec/docid_set_ops.h"
+#include "storage/index/inverted/query/exec/roaring_docid_sink.h"
+#include "storage/index/inverted/query/exec/term_pattern.h"
+#include "storage/index/inverted/query/logical/match_lowering.h"
+#include "storage/index/inverted/query/scoring/bm25_similarity.h"
+#include "storage/olap_common.h"
+#include "util/defer_op.h"
+
+namespace doris::segment_v2 {
+
+namespace logical = index_query::logical;
+namespace query_v2 = inverted_index::query_v2;
+
+namespace {
+
+namespace logical = index_query::logical;
+namespace query_v2 = inverted_index::query_v2;
+
+query_v2::QueryPtr term_query(const IndexQueryContextPtr& context, const
std::wstring& field,
+ const std::string& term) {
+ return std::make_shared<query_v2::TermQuery>(context, field, term);
+}
+
+// One term is queried as itself; several form the boolean the set asks for.
SEARCH counts a
+// threshold before a set reaches here.
+query_v2::QueryPtr term_set_query(const IndexQueryContextPtr& context, const
std::wstring& field,
+ const std::string& binding_key, const
logical::TermSet& set) {
+ DORIS_CHECK(set.min_should_match == 0);
+ if (set.terms.size() == 1) {
+ return term_query(context, field, set.terms.front());
+ }
+ auto builder = query_v2::create_operator_boolean_query_builder(
+ set.require_all ? query_v2::OperatorType::OP_AND :
query_v2::OperatorType::OP_OR);
+ for (const auto& term : set.terms) {
+ builder->add(term_query(context, field, term), binding_key);
+ }
+ return builder->build();
+}
+
+Status phrase_query(const IndexQueryContextPtr& context, const std::wstring&
field,
+ const logical::Phrase& phrase, const roaring::Roaring*
candidates,
+ query_v2::QueryPtr* out) {
+ const bool single_terms = std::ranges::all_of(
+ phrase.slots, [](const TermInfo& slot) { return
slot.is_single_term(); });
+ if (phrase.prefix) {
+ if (!single_terms) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
+ "a phrase prefix with several terms at one position is not
supported");
+ }
+ *out = std::make_shared<query_v2::PhrasePrefixQuery>(context, field,
phrase.slots,
+ candidates,
phrase.suffix);
+ return Status::OK();
+ }
+ const index_query::PhraseQueryOptions options {
+ .slop = static_cast<uint32_t>(phrase.slop),
+ .ordered = phrase.ordered,
+ .candidates = candidates,
+ .candidate_rows_consumed =
+ candidates == nullptr ? nullptr :
&context->candidate_rows_consumed};
+ if (single_terms) {
+ *out = std::make_shared<query_v2::PhraseQuery>(context, field,
phrase.slots, options);
+ } else {
+ *out = std::make_shared<query_v2::MultiPhraseQuery>(context, field,
phrase.slots, options);
+ }
+ return Status::OK();
+}
+
+index_query::TermPatternKind pattern_kind(logical::ExpandKind kind) {
+ switch (kind) {
+ case logical::ExpandKind::kPrefix:
+ return index_query::TermPatternKind::kPrefix;
+ case logical::ExpandKind::kRegexp:
+ return index_query::TermPatternKind::kRegexp;
+ case logical::ExpandKind::kContains:
+ return index_query::TermPatternKind::kContains;
+ case logical::ExpandKind::kWildcard:
+ default:
+ return index_query::TermPatternKind::kWildcard;
+ }
+}
+
+} // namespace
+
+Status TextIndexReader::open_source(const IndexQueryContextPtr& context, const
std::wstring& field,
+ std::unique_ptr<OpenedIndex>* opened,
+ index_query::IndexSourcePtr* source) {
+ RETURN_IF_ERROR(_open_index(context, opened));
+ *source = _bind_source(context, field, **opened);
+ return Status::OK();
+}
+
+Status run_leaf(const IndexQueryContextPtr& context, const std::wstring& field,
+ const logical::Node& leaf, const roaring::Roaring* candidates,
bool scoring,
+ index_query::IndexSourcePtr source, uint32_t doc_count,
+ const std::shared_ptr<roaring::Roaring>& result) {
+ SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_search_timer);
+ if (const auto* expand = leaf.as<logical::Expand>(); expand != nullptr &&
!scoring) {
+
SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_search_exec_timer);
+ if (doc_count == 0) {
+ return Status::OK();
+ }
+ const auto kind = pattern_kind(expand->kind);
+ index_query::TermPattern pattern;
+ THROW_IF_ERROR(index_query::TermPattern::create(kind, expand->pattern,
&pattern));
+ THROW_IF_ERROR(query_v2::collect_expanded_rows(
+ *source, pattern,
+ index_query::expansion_limit(kind,
index_query::max_expansions(*context)), nullptr,
+ result.get()));
+ return Status::OK();
+ }
+ std::span<const std::string> terms;
+ if (const auto* term = leaf.as<logical::Term>(); term != nullptr) {
+ terms = std::span(&term->term, 1);
+ } else if (const auto* set = leaf.as<logical::TermSet>();
+ set != nullptr && (!set->require_all || set->terms.size() == 1)
&&
+ set->min_should_match == 0) {
+ terms = set->terms;
+ }
+ if (!terms.empty() && (!scoring || terms.size() > 1)) {
+
SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_search_exec_timer);
+ if (!scoring) {
+ index_query::RoaringDocIdSink sink(*result);
+ return source->collect_terms(terms, sink);
+ }
+ query_v2::ListedTerms listed(std::move(source), nullptr);
+ for (size_t i = 0; i < terms.size(); ++i) {
+ auto similarity = std::make_shared<BM25Similarity>();
+ similarity->for_one_term(context, field,
+
inverted_index::StringHelper::to_wstring(terms[i]));
+ listed.add(i, terms[i], std::move(similarity));
+ }
+ const auto scorer = listed.scored_disjunction();
+ if (context->collection_similarity != nullptr) {
+ *result |= query_v2::collect_scored_rows(scorer, 0,
*context->collection_similarity);
+ } else {
+ query_v2::collect_true_rows(scorer, result.get());
+ }
+ return Status::OK();
+ }
+ query_v2::WeightPtr weight;
+ {
+
SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_search_init_timer);
+ query_v2::QueryPtr query;
+ RETURN_IF_ERROR(plan_query(leaf, context, field, "", candidates,
&query));
+ weight = query->weight(scoring);
+ }
+
SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_search_exec_timer);
+ query_v2::QueryExecutionContext exec_ctx;
+ exec_ctx.segment_num_rows = doc_count;
+ exec_ctx.field_sources.emplace(field, std::move(source));
+ query_v2::collect_multi_segment_doc_set(weight, exec_ctx, "", result,
+ context->collection_similarity,
scoring);
+ return Status::OK();
+}
+
+Status plan_query(const logical::Node& leaf, const IndexQueryContextPtr&
context,
+ const std::wstring& field, const std::string& binding_key,
+ const roaring::Roaring* candidates, query_v2::QueryPtr* out)
{
+ if (const auto* term = leaf.as<logical::Term>()) {
+ *out = term_query(context, field, term->term);
+ } else if (const auto* set = leaf.as<logical::TermSet>()) {
+ *out = term_set_query(context, field, binding_key, *set);
+ } else if (const auto* phrase = leaf.as<logical::Phrase>()) {
+ return phrase_query(context, field, *phrase, candidates, out);
+ } else if (const auto* expand = leaf.as<logical::Expand>()) {
+ *out = std::make_shared<query_v2::ExpandQuery>(context, field,
pattern_kind(expand->kind),
+ expand->pattern);
+ } else if (leaf.as<logical::Exists>() != nullptr) {
+ *out = std::make_shared<query_v2::AllQuery>(field, /*nullable=*/true);
+ } else if (leaf.as<logical::Empty>() != nullptr) {
+ *out = std::make_shared<query_v2::BitSetQuery>(roaring::Roaring());
Review Comment:
Fixed in 750d2f248e6. `logical::Empty` now compiles to `EmptyQuery`, an
empty leaf bound to its field whose scorer keeps the field's NULL rows.
Test: `FunctionSearchNullTest.AnalyzerEmptyLeafKeepsTheFieldNullRows` runs
`NOT` over a value the analyzer turns into no token, for TERM, PHRASE, MATCH,
ANY and ALL, scored and unscored. The NULL row is now UNKNOWN in each; before
the change it was returned.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
