github-actions[bot] commented on code in PR #69761:
URL: https://github.com/apache/doris/pull/69761#discussion_r4612201711
##########
be/src/exprs/function/match.cpp:
##########
@@ -53,6 +58,106 @@ const InvertedIndexAnalyzerCtx*
get_match_analyzer_ctx(FunctionContext* context)
return analyzer_ctx;
}
+// Collects the tokens of every scalar leaf of one Variant document, the
scalar mirror of what a
+// VARIANT values index stores. The shared leaf visitor ignores object keys
and unsupported
+// primitives; each canonical scalar text is tokenized independently.
+Status append_variant_leaf_tokens(const InvertedIndexAnalyzerCtx* analyzer_ctx,
+ const VariantRef& value,
+ std::vector<segment_v2::TermInfo>* tokens) {
+ DORIS_CHECK(tokens != nullptr);
+ if (analyzer_ctx == nullptr) {
+ return Status::OK();
+ }
+ const bool requires_analysis =
+ analyzer_ctx->requires_analysis() && analyzer_ctx->analyzer !=
nullptr;
+ return for_each_variant_leaf_text(value, [&](std::string_view text) {
+ if (!requires_analysis) {
+ tokens->emplace_back(std::string(text));
+ return Status::OK();
+ }
+ std::vector<segment_v2::TermInfo> value_tokens;
+
RETURN_IF_ERROR(segment_v2::inverted_index::InvertedIndexAnalyzer::analyze(
+ text, analyzer_ctx, {}, &value_tokens));
+ tokens->insert(tokens->end(),
std::make_move_iterator(value_tokens.begin()),
+ std::make_move_iterator(value_tokens.end()));
+ return Status::OK();
+ });
+}
+
+Status execute_variant_values_match(const FunctionMatchBase& function,
FunctionContext* context,
+ const ColumnVariantV2& variant,
+ const InvertedIndexAnalyzerCtx*
analyzer_ctx,
+ const std::string& match_query_str,
+ const std::string& column_name, Block&
block, uint32_t result,
+ size_t input_rows_count) {
+ const InvertedIndexQueryType query_type =
function.get_query_type_from_fn_name();
+ if (query_type != InvertedIndexQueryType::MATCH_ANY_QUERY &&
+ query_type != InvertedIndexQueryType::MATCH_ALL_QUERY) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
+ "VARIANT root supports only MATCH, MATCH_ANY, and MATCH_ALL");
+ }
+ RETURN_IF_ERROR(function.check(context, function.get_name()));
+
+ // Only root VARIANT inputs reach this branch; FE casts every other
VARIANT input to STRING.
+ if (analyzer_ctx == nullptr ||
+ (analyzer_ctx->requires_analysis() && analyzer_ctx->analyzer ==
nullptr)) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
+ "VARIANT MATCH requires a bound analyzer");
+ }
+
+ auto result_values = ColumnUInt8::create();
+ ColumnUInt8::Container& result_data = result_values->get_data();
+ result_data.resize_fill(input_rows_count);
+ const auto query_tokens =
+ function.analyse_query_str_token(analyzer_ctx, match_query_str,
column_name);
+ if (query_tokens.empty()) {
+ block.replace_by_position(result, std::move(result_values));
+ return Status::OK();
+ }
+
+ Status evaluation_status = Status::OK();
+ try {
+ visit_variant_v2_values(
+ variant, 0, input_rows_count, {}, [](size_t) {},
+ [&](size_t row, const VariantRef& value) {
+ if (!evaluation_status.ok()) {
+ return;
+ }
+ std::vector<segment_v2::TermInfo> data_tokens;
Review Comment:
[P2] Account for scalar values-MATCH token buffers
On a values-indexed table with `enable_match_without_inverted_index=true`
and `enable_inverted_index_query=false`, `SELECT v MATCH_ANY 'needle' FROM t`
reaches this scalar path. A valid row with a million short scalar leaves first
builds all owning `TermInfo` objects, then allocates another `string_view`
array and sorts it. These standard containers and token strings have no Doris
memory reservation/accounting (the jemalloc hook only forwards allocations),
adding tens of MiB per evaluated row outside the query limit, multiplied by
concurrent tasks. Stream tokens into query-sized ANY/ALL state, or
track/reserve both container capacities and owned token bytes. This is separate
from fixed writer-staging thread 4598841093: these buffers belong to scalar
projection/missing-index evaluation.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]