airborne12 commented on code in PR #68758:
URL: https://github.com/apache/doris/pull/68758#discussion_r4265800702
##########
be/src/storage/index/inverted/query/engine/term_query/term_weight.h:
##########
@@ -62,29 +63,31 @@ class TermWeight : public Weight {
std::move(result));
}
+ const std::wstring& field() const { return _field; }
+ const std::string& term() const { return _term; }
+ const index_query::ScoringContextPtr<float>& similarity() const { return
_similarity; }
+ bool scores() const { return _enable_scoring; }
+
private:
TermOrEmptyScorer specialized_scorer(const QueryExecutionContext& ctx,
const std::string& binding_key) {
- auto reader = lookup_reader(_field, ctx, binding_key);
+ auto source = lookup_source(_field, ctx, binding_key);
auto logical_field = logical_field_or_fallback(ctx, binding_key,
_field);
- if (!reader) {
+ if (!source) {
return std::make_shared<EmptyScorer>();
}
- SegmentPostingsPtr segment_postings;
- segment_postings = create_term_posting(reader.get(), _field, _term,
_enable_scoring,
- _similarity, _context->io_ctx);
+ SegmentPostingsPtr segment_postings =
+ open_postings(*source, _term, /*positions=*/false,
_enable_scoring, _similarity);
if (segment_postings) {
return std::make_shared<TermScorer>(segment_postings, _similarity,
logical_field);
}
return std::make_shared<EmptyScorer>();
Review Comment:
By design, SEARCH is going to stop handling NULL and ignore it: it will not
track NULL or UNKNOWN rows for nullable fields. We are therefore leaving its
current NULL handling as is in this PR instead of fixing it.
Without NULL tracking, an absent term is an empty result with no field NULL
bitmap to carry, and `NOT` treats the NULL row like any other row the clause
does not match.
##########
be/src/storage/index/inverted/query/engine/boolean_query/listed_terms.cpp:
##########
@@ -0,0 +1,304 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/query/engine/boolean_query/listed_terms.h"
+
+#include <parallel_hashmap/phmap.h>
+
+#include <algorithm>
+#include <bit>
+#include <optional>
+#include <span>
+#include <utility>
+
+#include "common/check.h"
+#include "common/exception.h"
+#include "storage/index/inverted/query/engine/postings/listed_walk.h"
+#include "storage/index/inverted/query/engine/scored_rows_scorer.h"
+#include "storage/index/inverted/query/exec/block_doc_set.h"
+#include "storage/index/inverted/query/exec/cursor_chained_postings.h"
+#include "storage/index/inverted/query/exec/roaring_docid_sink.h"
+#include "storage/index/inverted/query/exec/term_waves.h"
+#include "storage/index/inverted/query/phrase/position_span.h"
+
+namespace doris::segment_v2::inverted_index::query_v2 {
+
+ListedTerms::ListedTerms(index_query::IndexSourcePtr source,
+ std::shared_ptr<roaring::Roaring> nulls)
+ : _source(std::move(source)), _nulls(std::move(nulls)) {}
+
+void ListedTerms::add(size_t clause, std::string term,
+ index_query::ScoringContextPtr<float> similarity) {
+ DCHECK(_clauses.empty() || _clauses.back() < clause);
+ _clauses.push_back(clause);
+ _terms.push_back(std::move(term));
+ _similarities.push_back(std::move(similarity));
+}
+
+bool ListedTerms::holds(size_t clause) const {
+ return std::ranges::binary_search(_clauses, clause);
+}
+
+void ListedTerms::open(bool conjunctive, bool scoring) {
+ if (!conjunctive) {
+ return;
+ }
+ for (const std::string& term : _terms) {
+ bool held = false;
+ THROW_IF_ERROR(_source->may_hold(term, &held));
+ if (!held) {
+ _cursors.resize(_terms.size());
+ return;
+ }
+ }
+ // A scored conjunction reads the positions of the rows it lists.
+ THROW_IF_ERROR(
+ _source->open_terms(_terms, /*positions=*/scoring,
/*scoring=*/false, &_cursors));
+}
+
+bool ListedTerms::has_absent_term() const {
+ return std::ranges::any_of(_cursors, [](const auto& cursor) { return
cursor == nullptr; });
+}
+
+uint64_t ListedTerms::cheapest_doc_freq() const {
+ uint64_t cheapest = UINT64_MAX;
+ for (const auto& cursor : _cursors) {
+ cheapest = std::min<uint64_t>(cheapest, cursor == nullptr ? 0 :
cursor->doc_freq());
+ }
+ return cheapest;
+}
+
+index_query::TruthSet ListedTerms::conjunction(const std::vector<uint32_t>*
candidates) {
+ index_query::TruthSet result;
+ if (has_absent_term()) {
+ return result;
+ }
+ std::vector<uint32_t> docs;
+ THROW_IF_ERROR(index_query::chain_cursors(_cursors, candidates, &docs));
+ result.true_rows.addMany(docs.size(), docs.data());
+ if (_nulls != nullptr) {
+ result.null_rows = *_nulls;
+ }
+ return result;
+}
+
+index_query::TruthSet ListedTerms::disjunction() {
+ index_query::TruthSet result;
+ bool any_present = false;
+ index_query::RoaringDocIdSink sink(result.true_rows);
+ THROW_IF_ERROR(_source->collect_terms(_terms, sink, &any_present));
+ if (any_present && _nulls != nullptr) {
Review Comment:
By design, SEARCH is going to stop handling NULL and ignore it: it will not
track NULL or UNKNOWN rows for nullable fields. We are therefore leaving its
current NULL handling as is in this PR instead of fixing it.
Without NULL tracking, SEARCH does not mask rows with the field NULL bitmap;
it answers from what the index holds, so this TRUE/UNKNOWN overlap no longer
arises. The row in this example matches because the nested `alpha` written
under the outer NULL is in the index. Whether the writer should index such
values is a separate question; this PR does not change how arrays are written.
##########
be/src/exprs/function/search/function_search.cpp:
##########
@@ -0,0 +1,634 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exprs/function/search/function_search.h"
+
+#include <CLucene/config/repl_wchar.h>
+#include <CLucene/debug/error.h>
+#include <fmt/format.h>
+#include <gen_cpp/Exprs_types.h>
+#include <glog/logging.h>
+
+#include <algorithm>
+#include <iterator>
+#include <memory>
+#include <optional>
+#include <roaring/roaring.hh>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "common/config.h"
+#include "common/exception.h"
+#include "common/status.h"
+#include "core/block/columns_with_type_and_name.h"
+#include "exprs/function/search/function_search_leaf.h"
+#include "exprs/function/search/search_lowering.h"
+#include "exprs/function/search/variant_inverted_index_search.h"
+#include "exprs/function/simple_function_factory.h"
+#include "exprs/vexpr_context.h"
+#include "runtime/runtime_profile.h"
+#include "storage/index/index_file_reader.h"
+#include "storage/index/index_query_context.h"
+#include "storage/index/inverted/analysis/analyzer/analyzer.h"
+#include "storage/index/inverted/inverted_index_iterator.h"
+#include "storage/index/inverted/inverted_index_reader.h"
+#include "storage/index/inverted/query/engine/all_query/all_query.h"
+#include "storage/index/inverted/query/engine/bit_set_query/bit_set_query.h"
+#include
"storage/index/inverted/query/engine/boolean_query/boolean_query_builder.h"
+#include "storage/index/inverted/query/engine/boolean_query/operator.h"
+#include "storage/index/inverted/query/engine/collect/doc_set_collector.h"
+#include "storage/index/inverted/query/engine/collect/top_k_collector.h"
+#include "storage/index/inverted/query/exec/term_pattern.h"
+#include "storage/index/inverted/query/logical/node.h"
+#include "storage/index/inverted/reader/collection_statistics.h"
+#include "storage/olap_common.h"
+#include "storage/segment/variant/nested_group_provider.h"
+#include "util/thrift_util.h"
+
+namespace doris {
+
+namespace query_v2 = segment_v2::inverted_index::query_v2;
+
+// Serializes the whole TSearchParam, so every field of the DSL takes part in
the signature, and
+// adds the limit on the terms PREFIX, WILDCARD and REGEXP clauses expand to.
+std::string FunctionSearch::dsl_cache_signature(const TSearchParam&
search_param,
+ const IndexQueryContext*
context) {
+ ThriftSerializer ser(false, 1024);
+ TSearchParam copy = search_param;
+ std::string sig;
+ auto st = ser.serialize(©, &sig);
+ if (UNLIKELY(!st.ok())) {
+ LOG(WARNING) << "build_dsl_signature: Thrift serialization failed: "
<< st.to_string()
+ << ", caching disabled for this query";
+ return "";
+ }
+ sig += "|max_expansions=";
+ sig += std::to_string(
+ index_query::max_expansions(context != nullptr ?
context->runtime_state : nullptr));
+ return sig;
+}
+
+// Extract segment path prefix from the first available inverted index
iterator.
+// All fields in the same segment share the same path prefix.
+static std::string extract_segment_prefix(
+ const std::unordered_map<std::string, IndexIterator*>& iterators) {
+ for (const auto& [field_name, iter] : iterators) {
+ auto* inv_iter = dynamic_cast<InvertedIndexIterator*>(iter);
+ if (!inv_iter) continue;
+ // Try fulltext reader first, then string type
+ for (auto type :
+ {InvertedIndexReaderType::FULLTEXT,
InvertedIndexReaderType::STRING_TYPE}) {
+ IndexReaderType reader_type = type;
+ auto reader = inv_iter->get_reader(reader_type);
+ if (!reader) continue;
+ auto inv_reader =
std::dynamic_pointer_cast<InvertedIndexReader>(reader);
+ if (!inv_reader) continue;
+ auto file_reader = inv_reader->get_index_file_reader();
+ if (!file_reader) continue;
+ return file_reader->get_index_path_prefix();
+ }
+ }
+ VLOG_DEBUG << "extract_segment_prefix: no suitable inverted index reader
found across "
+ << iterators.size() << " iterators, caching disabled for this
query";
+ return "";
+}
+
+namespace {
+
+bool is_nested_group_search_supported() {
+ auto provider = segment_v2::create_nested_group_read_provider();
+ return provider != nullptr &&
provider->should_enable_nested_group_read_path();
+}
+
+namespace logical = index_query::logical;
+
+// Lowering's view of the resolver: what index a field binds to and how that
+// index analyzes values.
+class SearchFieldCatalog final : public logical::FieldCatalog {
+public:
+ SearchFieldCatalog(FieldReaderResolver& resolver,
std::shared_ptr<IndexQueryContext> context)
+ : _resolver(resolver), _context(std::move(context)) {}
+
+ Status resolve(const std::string& field, InvertedIndexQueryType query_type,
+ logical::FieldProps* out) override {
+ FieldReaderBinding binding;
+ RETURN_IF_ERROR(_resolver.resolve(field, query_type, &binding));
+ if (!binding.is_bound()) {
+ LOG(INFO) << "search: No inverted index for field '" << field
+ << "' in this segment, query_type=" <<
static_cast<int>(query_type)
+ << ", returning UNKNOWN bitmap";
+ *out = logical::FieldProps {};
+ return Status::OK();
+ }
+ const bool scalar = binding.inverted_reader != nullptr &&
+ binding.inverted_reader->type() ==
InvertedIndexReaderType::BKD;
+ const bool analyzed = !scalar &&
inverted_index::InvertedIndexAnalyzer::should_analyzer(
+ binding.index_properties);
+ *out = logical::FieldProps {.bound = true,
+ .direct_index = scalar,
+ .analyzed = analyzed,
+ .binding = binding.binding_key};
+ return Status::OK();
+ }
+
+ Status analyze(const logical::FieldProps& props, const std::string& value,
+ std::vector<logical::Token>* out) override {
+ const FieldReaderBinding* binding =
_resolver.find_binding(props.binding);
+ if (binding == nullptr) {
+ return Status::InternalError("search: no binding '{}' to analyze
with", props.binding);
+ }
+ int64_t unused_timer = 0;
+ SCOPED_RAW_TIMER(_context != nullptr && _context->stats != nullptr
+ ?
&_context->stats->inverted_index_analyzer_timer
+ : &unused_timer);
+ InvertedIndexAnalyzerCtxSPtr analyzer_ctx;
+ RETURN_IF_ERROR(_resolver.analyzer_context_for(props.binding,
&analyzer_ctx));
+ return inverted_index::InvertedIndexAnalyzer::analyze(value,
analyzer_ctx.get(),
+
binding->index_properties, out);
+ }
+
+ Status normalize(const logical::FieldProps& props, const std::string&
value,
+ std::string* out) override {
+ const FieldReaderBinding* binding =
_resolver.find_binding(props.binding);
+ if (binding == nullptr) {
+ return Status::InternalError("search: no binding '{}' to normalize
with",
+ props.binding);
+ }
+ try {
+ *out = inverted_index::InvertedIndexAnalyzer::normalize(value,
+
binding->index_properties);
+ } catch (const CLuceneError& e) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>(
+ "search: normalizing '{}' failed: {}", value, e.what());
+ } catch (const Exception& e) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>(
+ "search: normalizing '{}' failed: {}", value, e.what());
+ }
+ return Status::OK();
+ }
+
+private:
+ FieldReaderResolver& _resolver;
+ std::shared_ptr<IndexQueryContext> _context;
+};
+
+query_v2::Occur to_query_occur(logical::Occur occur) {
+ switch (occur) {
+ case logical::Occur::kShould:
+ return query_v2::Occur::SHOULD;
+ case logical::Occur::kMustNot:
+ return query_v2::Occur::MUST_NOT;
+ case logical::Occur::kMust:
+ default:
+ return query_v2::Occur::MUST;
+ }
+}
+
+Status compile_node(const logical::Node& node, const SearchLeafContext& ctx,
+ FieldReaderResolver& resolver, query_v2::QueryPtr* out,
+ std::string* binding_key);
+
+using Clauses = std::vector<std::pair<logical::Occur, logical::NodePtr>>;
+
+bool has_required_clause(const Clauses& clauses) {
+ return std::ranges::any_of(
+ clauses, [](const auto& clause) { return clause.first ==
logical::Occur::kMust; });
+}
+
+// Without scoring, optional clauses beside a required one change neither the
rows nor their
+// UNKNOWN state, so they are left out.
+Clauses matching_clauses(const logical::Bool& boolean, bool scoring) {
+ if (scoring || boolean.op != logical::BoolOp::kOccur ||
boolean.min_should_match != 0 ||
+ !has_required_clause(boolean.clauses)) {
+ return boolean.clauses;
+ }
+ Clauses kept;
+ std::ranges::copy_if(boolean.clauses, std::back_inserter(kept), [](const
auto& clause) {
+ return clause.first != logical::Occur::kShould;
+ });
+ return kept;
+}
+
+// AND, OR and NOT ignore the per-clause occur; OCCUR keeps it and the
threshold. A Boolean
+// left with one clause after dropping clauses that only score is that clause.
+Status compile_bool(const logical::Bool& boolean, const SearchLeafContext& ctx,
+ FieldReaderResolver& resolver, query_v2::QueryPtr* out) {
+ const Clauses clauses = matching_clauses(boolean, ctx.scoring);
+ if (clauses.size() == 1 && boolean.clauses.size() > 1) {
+ return compile_node(*clauses.front().second, ctx, resolver, out,
nullptr);
+ }
+ std::vector<query_v2::QueryPtr> queries(clauses.size());
+ std::vector<std::string> keys(clauses.size());
+ for (size_t i = 0; i < clauses.size(); ++i) {
+ RETURN_IF_ERROR(compile_node(*clauses[i].second, ctx, resolver,
&queries[i], &keys[i]));
+ }
+ if (boolean.op == logical::BoolOp::kOccur) {
+ auto builder = query_v2::create_occur_boolean_query_builder();
+ builder->set_minimum_number_should_match(boolean.min_should_match);
+ for (size_t i = 0; i < clauses.size(); ++i) {
+ builder->add(queries[i], to_query_occur(clauses[i].first),
std::move(keys[i]));
+ }
+ *out = builder->build();
+ return Status::OK();
+ }
+ query_v2::OperatorType op = query_v2::OperatorType::OP_AND;
+ if (boolean.op == logical::BoolOp::kOr) {
+ op = query_v2::OperatorType::OP_OR;
+ } else if (boolean.op == logical::BoolOp::kNot) {
+ op = query_v2::OperatorType::OP_NOT;
+ }
+ auto builder = query_v2::create_operator_boolean_query_builder(op);
+ for (size_t i = 0; i < clauses.size(); ++i) {
+ builder->add(queries[i], std::move(keys[i]));
+ }
+ *out = builder->build();
+ return Status::OK();
+}
+
+// A threshold counts the set's terms above the field's compiler, so every
format
+// answers it the same way: each term is its own leaf on the field.
+Status compile_term_threshold(const logical::TermSet& set, const
SearchLeafContext& ctx,
+ SearchLeafCompiler& compiler,
query_v2::QueryPtr* out) {
+ auto builder = query_v2::create_occur_boolean_query_builder();
+ builder->set_minimum_number_should_match(set.min_should_match);
+ const query_v2::Occur occur = set.require_all ? query_v2::Occur::MUST :
query_v2::Occur::SHOULD;
+ for (const auto& term : set.terms) {
+ const auto leaf = logical::make_node(logical::TermSet {
+ .field = set.field, .terms = {term}, .require_all = false,
.min_should_match = 0});
+ query_v2::QueryPtr term_query;
+ RETURN_IF_ERROR(compiler.compile(*leaf, ctx, &term_query));
+ builder->add(term_query, occur, set.field.binding);
+ }
+ *out = builder->build();
+ return Status::OK();
+}
+
+// A leaf goes to the compiler of the index it was bound to; its binding key
and
+// the leaf mapper follow it. A term threshold stays one leaf for the mapper.
+Status compile_node(const logical::Node& node, const SearchLeafContext& ctx,
+ FieldReaderResolver& resolver, query_v2::QueryPtr* out,
+ std::string* binding_key) {
+ *out = nullptr;
+ if (binding_key != nullptr) {
+ binding_key->clear();
+ }
+ if (node.as<logical::All>() != nullptr) {
+ *out = std::make_shared<query_v2::AllQuery>();
+ return Status::OK();
+ }
+ if (const auto* boolean = node.as<logical::Bool>()) {
+ return compile_bool(*boolean, ctx, resolver, out);
+ }
+ const logical::FieldRef* field = node.field();
+ DCHECK(field != nullptr);
+ if (binding_key != nullptr) {
+ *binding_key = field->binding;
+ }
+ if (node.as<logical::Unknown>() != nullptr) {
+ *out = make_unknown_leaf_query(ctx.num_rows);
+ return resolver.map_leaf_query(field->name, out);
+ }
+ const FieldReaderBinding* binding = resolver.find_binding(field->binding);
+ if (binding == nullptr || binding->leaf_compiler == nullptr) {
+ return Status::InternalError("search: field '{}' has no compiler for
binding '{}'",
+ field->name, field->binding);
+ }
+ const auto* set = node.as<logical::TermSet>();
+ if (set != nullptr && set->min_should_match > 0) {
+ RETURN_IF_ERROR(compile_term_threshold(*set, ctx,
*binding->leaf_compiler, out));
+ } else {
+ RETURN_IF_ERROR(binding->leaf_compiler->compile(node, ctx, out));
+ }
+ return resolver.map_leaf_query(field->name, out);
+}
+
+} // namespace
+
+Status FunctionSearch::execute_impl(FunctionContext* /*context*/, Block&
/*block*/,
+ const ColumnNumbers& /*arguments*/,
uint32_t /*result*/,
+ size_t /*input_rows_count*/) const {
+ return Status::RuntimeError("only inverted index queries are supported");
+}
+
+// Enhanced implementation: Handle new parameter structure (DSL +
SlotReferences)
+Status FunctionSearch::evaluate_inverted_index(
+ const ColumnsWithTypeAndName& arguments,
+ const std::vector<IndexFieldNameAndTypePair>& data_type_with_names,
+ std::vector<IndexIterator*> iterators, uint32_t num_rows,
+ const InvertedIndexAnalyzerCtx* /*analyzer_ctx*/,
+ InvertedIndexResultBitmap& bitmap_result) const {
+ return Status::OK();
+}
+
+Status FunctionSearch::evaluate_inverted_index_with_search_param(
+ const TSearchParam& search_param,
+ const std::unordered_map<std::string, IndexFieldNameAndTypePair>&
data_type_with_names,
+ std::unordered_map<std::string, IndexIterator*> iterators, uint32_t
num_rows,
+ InvertedIndexResultBitmap& bitmap_result, bool enable_cache) const {
+ return evaluate_inverted_index_with_search_param(search_param,
data_type_with_names,
+ std::move(iterators),
num_rows, bitmap_result,
+ enable_cache, nullptr,
nullptr);
+}
+
+// A lazy leaf reads its index while the result is listed; what it throws is
returned like the
+// errors the evaluation returns.
+Status FunctionSearch::evaluate_inverted_index_with_search_param(
+ const TSearchParam& search_param,
+ const std::unordered_map<std::string, IndexFieldNameAndTypePair>&
data_type_with_names,
+ std::unordered_map<std::string, IndexIterator*> iterators, uint32_t
num_rows,
+ InvertedIndexResultBitmap& bitmap_result, bool enable_cache,
+ const IndexExecContext* index_exec_ctx, const TabletColumn*
nested_column,
+ const std::shared_ptr<IndexQueryContext>& index_query_context) const {
+ // VSearchExpr enters here directly, outside IFunction::execute() and its
exception boundary,
+ // so this is where a failure inside the search, such as an analyzer whose
first token
+ // stream throws, has to become a Status.
+ try {
+ RETURN_IF_ERROR_OR_CATCH_EXCEPTION(_evaluate_search_param(
+ search_param, data_type_with_names, std::move(iterators),
num_rows, bitmap_result,
+ enable_cache, index_exec_ctx, nested_column,
index_query_context));
+ } catch (const CLuceneError& e) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>("search
failed: {}",
+
e.what());
+ }
+ return Status::OK();
+}
+
+// The evaluation keeps its result cache, the tree's compile and the
collection in one path.
+//
NOLINTNEXTLINE(readability-function-cognitive-complexity,readability-function-size)
+Status FunctionSearch::_evaluate_search_param(
+ const TSearchParam& search_param,
+ const std::unordered_map<std::string, IndexFieldNameAndTypePair>&
data_type_with_names,
+ std::unordered_map<std::string, IndexIterator*> iterators, uint32_t
num_rows,
+ InvertedIndexResultBitmap& bitmap_result, bool enable_cache,
+ const IndexExecContext* index_exec_ctx, const TabletColumn*
nested_column,
+ const std::shared_ptr<IndexQueryContext>& index_query_context) const {
+ const bool is_nested_query = search_param.root.clause_type == "NESTED";
+ if (is_nested_query && !is_nested_group_search_supported()) {
+ return Status::NotSupported(
+ "NESTED query requires NestedGroup support, which is
unavailable in this build");
+ }
+
+ if (!is_nested_query && (iterators.empty() ||
data_type_with_names.empty())) {
+ LOG(INFO) << "No indexed columns or iterators available, returning
empty result, dsl:"
+ << search_param.original_dsl;
+ bitmap_result =
InvertedIndexResultBitmap(std::make_shared<roaring::Roaring>(),
+
std::make_shared<roaring::Roaring>());
+ return Status::OK();
+ }
+
+ // Track overall query time (equivalent to inverted_index_query_timer in
MATCH path).
+ // Must be declared before the DSL cache lookup so that cache-hit fast
paths are
+ // also covered by the timer.
+ int64_t query_timer_dummy = 0;
+ OlapReaderStatistics* outer_stats = index_query_context ?
index_query_context->stats : nullptr;
+ SCOPED_RAW_TIMER(outer_stats ? &outer_stats->inverted_index_query_timer :
&query_timer_dummy);
+
+ // DSL result cache only stores bitmap/null bitmap. It does not store BM25
scores,
+ // so score() queries must execute scorers to populate
CollectionSimilarity.
+ const bool enable_scoring =
+ index_query_context != nullptr &&
index_query_context->collection_similarity != nullptr;
+ auto* dsl_cache =
+ enable_cache && !enable_scoring ?
InvertedIndexQueryCache::instance() : nullptr;
+ std::string seg_prefix;
+ std::string dsl_sig;
+ InvertedIndexQueryCache::CacheKey dsl_cache_key;
+ bool cache_usable = false;
+ if (dsl_cache) {
+ seg_prefix = extract_segment_prefix(iterators);
+ dsl_sig = dsl_cache_signature(search_param, index_query_context.get());
+ if (!seg_prefix.empty() && !dsl_sig.empty()) {
+ dsl_cache_key = InvertedIndexQueryCache::CacheKey {
+ seg_prefix, "__search_dsl__",
InvertedIndexQueryType::SEARCH_DSL_QUERY,
+ dsl_sig};
+ cache_usable = true;
+ InvertedIndexQueryCacheHandle dsl_cache_handle;
+ bool dsl_hit = false;
+ {
+ int64_t lookup_dummy = 0;
+ SCOPED_RAW_TIMER(outer_stats ?
&outer_stats->inverted_index_lookup_timer
+ : &lookup_dummy);
+ dsl_hit = dsl_cache->lookup(dsl_cache_key, &dsl_cache_handle);
+ }
+ if (dsl_hit) {
+ auto cached_bitmap = dsl_cache_handle.get_bitmap();
+ if (cached_bitmap) {
+ if (outer_stats) {
+ outer_stats->inverted_index_query_cache_hit++;
+ }
+ // Also retrieve cached null bitmap for three-valued SQL
logic
+ // (needed by compound operators NOT, OR, AND in
VCompoundPred)
+ auto null_cache_key = InvertedIndexQueryCache::CacheKey {
+ seg_prefix, "__search_dsl__",
InvertedIndexQueryType::SEARCH_DSL_QUERY,
+ dsl_sig + "__null"};
+ InvertedIndexQueryCacheHandle null_cache_handle;
+ std::shared_ptr<roaring::Roaring> null_bitmap;
+ if (dsl_cache->lookup(null_cache_key, &null_cache_handle))
{
+ null_bitmap = null_cache_handle.get_bitmap();
+ }
+ if (!null_bitmap) {
+ null_bitmap = std::make_shared<roaring::Roaring>();
+ }
+ bitmap_result =
+ InvertedIndexResultBitmap(cached_bitmap,
std::move(null_bitmap));
+ return Status::OK();
+ }
+ }
+ if (outer_stats) {
+ outer_stats->inverted_index_query_cache_miss++;
+ }
+ }
+ }
+
+ std::shared_ptr<IndexQueryContext> context;
+ if (index_query_context) {
+ context = index_query_context;
+ } else {
+ context = std::make_shared<IndexQueryContext>();
+ context->collection_statistics =
std::make_shared<CollectionStatistics>();
+ context->collection_similarity =
std::make_shared<CollectionSimilarity>();
+ }
+
+ const auto* effective_data_type_with_names = &data_type_with_names;
+
+ // Pass field_bindings to resolver for variant subcolumn detection
+ FieldReaderResolver resolver(*effective_data_type_with_names, iterators,
context,
+ search_param.field_bindings);
+
+ if (is_nested_query) {
+ std::shared_ptr<roaring::Roaring> row_bitmap;
+ VariantNestedSearchEvaluator nested_evaluator(*this);
+ RETURN_IF_ERROR(nested_evaluator.evaluate(search_param,
search_param.root, context,
+ resolver, num_rows,
index_exec_ctx, nested_column,
+ row_bitmap));
+ bitmap_result = InvertedIndexResultBitmap(std::move(row_bitmap),
+
std::make_shared<roaring::Roaring>());
+ bitmap_result.mask_out_null();
+ return Status::OK();
+ }
+
+ // Extract default_operator from TSearchParam (default: "or")
+ std::string default_operator = "or";
+ if (search_param.__isset.default_operator &&
!search_param.default_operator.empty()) {
+ default_operator = search_param.default_operator;
+ }
+ // Extract minimum_should_match from TSearchParam (-1 means not set)
+ int32_t minimum_should_match = -1;
+ if (search_param.__isset.minimum_should_match) {
+ minimum_should_match = search_param.minimum_should_match;
+ }
+
+ auto* stats = context->stats;
+ int64_t dummy_timer = 0;
+ SCOPED_RAW_TIMER(stats ? &stats->inverted_index_searcher_search_timer :
&dummy_timer);
+
+ query_v2::QueryPtr root_query;
+ std::string root_binding_key;
+ {
+ int64_t init_dummy = 0;
+ SCOPED_RAW_TIMER(stats ?
&stats->inverted_index_searcher_search_init_timer : &init_dummy);
+ RETURN_IF_ERROR(build_query_recursive(search_param.root, context,
resolver, &root_query,
+ &root_binding_key,
default_operator,
+ minimum_should_match, num_rows,
enable_scoring));
+ }
+ if (root_query == nullptr) {
+ LOG(INFO) << "search: Query tree resolved to empty query, dsl:"
+ << search_param.original_dsl;
+ bitmap_result =
InvertedIndexResultBitmap(std::make_shared<roaring::Roaring>(),
+
std::make_shared<roaring::Roaring>());
+ return Status::OK();
+ }
+
+ VariantSearchNullBitmapAdapter null_resolver(resolver);
Review Comment:
By design, SEARCH is going to stop handling NULL and ignore it: it will not
track NULL or UNKNOWN rows for nullable fields. We are therefore leaving its
current NULL handling as is in this PR instead of fixing it.
The NULL-section read here serves only those NULL rows. When SEARCH ignores
NULL it no longer reads that section, so this error path goes away with it
instead of being propagated now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
