This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 21db7165cea branch-4.1: [fix](be) Disable DSL cache for score queries
#65436 (#65590)
21db7165cea is described below
commit 21db7165cea8623804dd233ceca1900c2e1fccf3
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Jul 15 11:47:58 2026 +0800
branch-4.1: [fix](be) Disable DSL cache for score queries #65436 (#65590)
Cherry-picked from #65436
Co-authored-by: liangj777
<[email protected]>
---
be/src/exprs/function/function_search.cpp | 9 +-
.../data/search/test_search_score_cache.out | 42 ++++
.../suites/search/test_search_cache.groovy | 30 +++
.../suites/search/test_search_score_cache.groovy | 254 +++++++++++++++++++++
4 files changed, 333 insertions(+), 2 deletions(-)
diff --git a/be/src/exprs/function/function_search.cpp
b/be/src/exprs/function/function_search.cpp
index 9c33752598c..e87aff95ee0 100644
--- a/be/src/exprs/function/function_search.cpp
+++ b/be/src/exprs/function/function_search.cpp
@@ -274,8 +274,13 @@ Status
FunctionSearch::evaluate_inverted_index_with_search_param(
OlapReaderStatistics* outer_stats = index_query_context ?
index_query_context->stats : nullptr;
SCOPED_RAW_TIMER(outer_stats ? &outer_stats->inverted_index_query_timer :
&query_timer_dummy);
- // DSL result cache: reuse InvertedIndexQueryCache with SEARCH_DSL_QUERY
type
- auto* dsl_cache = enable_cache ? InvertedIndexQueryCache::instance() :
nullptr;
+ const bool need_similarity_score =
+ index_query_context && index_query_context->collection_similarity;
+
+ // DSL result cache only stores bitmap/null bitmap. It does not store BM25
scores,
+ // so score() queries must execute scorers to populate
CollectionSimilarity.
+ auto* dsl_cache = (enable_cache && !need_similarity_score) ?
InvertedIndexQueryCache::instance()
+ : nullptr;
std::string seg_prefix;
std::string dsl_sig;
InvertedIndexQueryCache::CacheKey dsl_cache_key;
diff --git a/regression-test/data/search/test_search_score_cache.out
b/regression-test/data/search/test_search_score_cache.out
new file mode 100644
index 00000000000..58f33cb879d
--- /dev/null
+++ b/regression-test/data/search/test_search_score_cache.out
@@ -0,0 +1,42 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !search_dsl_score --
+8
+1
+2
+4
+6
+
+-- !match_any_score --
+8
+1
+2
+4
+6
+
+-- !match_all_score --
+2
+1
+
+-- !match_phrase_score --
+2
+1
+
+-- !match_any_with_filter_score --
+8
+1
+2
+4
+6
+
+-- !variant_match_phrase_score --
+2
+4
+
+-- !union_all_score --
+3
+7
+8
+1
+2
+2
+
diff --git a/regression-test/suites/search/test_search_cache.groovy
b/regression-test/suites/search/test_search_cache.groovy
index 39af2184bb7..4ebb5c173a2 100644
--- a/regression-test/suites/search/test_search_cache.groovy
+++ b/regression-test/suites/search/test_search_cache.groovy
@@ -134,5 +134,35 @@ suite("test_search_cache", "p0") {
"""
assertEquals(complex_cached, complex_uncached)
+ // Test 7: SEARCH DSL + score() must not reuse bitmap-only DSL cache.
+ // A warmed DSL cache or repeated score query would return 0.0 scores if
cache hit
+ // skipped scorer collection.
+ def score_cached_1 = sql """
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id, score() AS s FROM ${tableName}
+ WHERE search('title:apple')
+ ORDER BY s DESC
+ LIMIT 10
+ """
+
+ def score_cached_2 = sql """
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id, score() AS s FROM ${tableName}
+ WHERE search('title:apple')
+ ORDER BY s DESC
+ LIMIT 10
+ """
+
+ assertEquals(score_cached_1.size(), score_cached_2.size())
+ assertTrue(score_cached_1.size() > 0)
+ assertEquals(score_cached_1.collect { it[0] as int }.sort(),
+ score_cached_2.collect { it[0] as int }.sort())
+ for (def row : score_cached_1) {
+ assertTrue(Double.parseDouble(row[1].toString()) > 0.0)
+ }
+ for (def row : score_cached_2) {
+ assertTrue(Double.parseDouble(row[1].toString()) > 0.0)
+ }
+
sql "DROP TABLE IF EXISTS ${tableName}"
}
diff --git a/regression-test/suites/search/test_search_score_cache.groovy
b/regression-test/suites/search/test_search_score_cache.groovy
new file mode 100644
index 00000000000..caa3259195d
--- /dev/null
+++ b/regression-test/suites/search/test_search_score_cache.groovy
@@ -0,0 +1,254 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_search_score_cache", "p0") {
+ def tableName = "search_score_cache_test"
+
+ def assertPositiveScores = { result ->
+ assertTrue(result.size() > 0)
+ for (def row : result) {
+ assertTrue(Double.parseDouble(row[1].toString()) > 0.0)
+ }
+ }
+
+ def assertStablePositiveScoreQuery = { tag, warmSql, scoreSql ->
+ def warmResult1 = sql warmSql
+ def warmResult2 = sql warmSql
+ assertEquals(warmResult1, warmResult2)
+
+ def scoreResult1 = sql scoreSql
+ def scoreResult2 = sql scoreSql
+ assertEquals(scoreResult1.collect { it[0] as int },
scoreResult2.collect { it[0] as int })
+ assertPositiveScores(scoreResult1)
+ assertPositiveScores(scoreResult2)
+
+ quickTest(tag, """
+ SELECT id
+ FROM (${scoreSql}) t
+ ORDER BY s DESC
+ """)
+ }
+
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ CREATE TABLE ${tableName} (
+ id INT,
+ status INT,
+ title VARCHAR(200),
+ content VARCHAR(500),
+ v VARIANT,
+ INDEX idx_title(title) USING INVERTED PROPERTIES(
+ "parser" = "english",
+ "support_phrase" = "true"
+ ),
+ INDEX idx_content(content) USING INVERTED PROPERTIES(
+ "parser" = "english",
+ "support_phrase" = "true"
+ ),
+ INDEX idx_v(v) USING INVERTED PROPERTIES(
+ "parser" = "english",
+ "support_phrase" = "true"
+ )
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+
+ sql """INSERT INTO ${tableName} VALUES
+ (1, 1, 'apple apple apple banana cherry', 'red fruit sweet apple',
'{"host":"apple banana server"}'),
+ (2, 1, 'apple apple banana date', 'fresh apple banana salad',
'{"host":"apple server cluster"}'),
+ (3, 0, 'banana banana banana grape mango', 'yellow fruit tropical',
'{"host":"banana server"}'),
+ (4, 1, 'apple grape kiwi', 'green fruit fresh apple', '{"host":"green
green apple server"}'),
+ (5, 0, 'mango pineapple coconut', 'tropical fruit exotic',
'{"host":"mango server"}'),
+ (6, 1, 'apple cherry plum apricot fig', 'mixed fruit apple salad',
'{"host":"apple cherry node"}'),
+ (7, 0, 'banana banana coconut papaya', 'smoothie blend tropical',
'{"host":"banana coconut node"}'),
+ (8, 1, 'grape cherry apple apple apple apple', 'wine fruit tart
apple', '{"host":"grape apple node"}')
+ """
+ sql "sync"
+
+ sql """ set enable_common_expr_pushdown = true """
+ sql """ set enable_inverted_index_query_cache = true """
+
+ // SEARCH DSL + score() must execute scorers even when the DSL bitmap
cache is warm.
+ assertStablePositiveScoreQuery(
+ "search_dsl_score",
+ """
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id
+ FROM ${tableName}
+ WHERE search('title:apple')
+ ORDER BY id
+ """,
+ """
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id, score() AS s
+ FROM ${tableName}
+ WHERE search('title:apple')
+ ORDER BY s DESC
+ LIMIT 10
+ """
+ )
+
+ // MATCH_ANY + score() must bypass bitmap-only inverted index query cache.
+ assertStablePositiveScoreQuery(
+ "match_any_score",
+ """
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id
+ FROM ${tableName}
+ WHERE title MATCH_ANY 'apple'
+ ORDER BY id
+ """,
+ """
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id, score() AS s
+ FROM ${tableName}
+ WHERE title MATCH_ANY 'apple'
+ ORDER BY s DESC
+ LIMIT 10
+ """
+ )
+
+ // MATCH_ALL + score() should still produce collected BM25 scores after
cache warmup.
+ assertStablePositiveScoreQuery(
+ "match_all_score",
+ """
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id
+ FROM ${tableName}
+ WHERE title MATCH_ALL 'apple banana'
+ ORDER BY id
+ """,
+ """
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id, score() AS s
+ FROM ${tableName}
+ WHERE title MATCH_ALL 'apple banana'
+ ORDER BY s DESC
+ LIMIT 10
+ """
+ )
+
+ // MATCH_PHRASE + score() should not materialize default 0 scores from
cached bitmaps.
+ assertStablePositiveScoreQuery(
+ "match_phrase_score",
+ """
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id
+ FROM ${tableName}
+ WHERE title MATCH_PHRASE 'apple banana'
+ ORDER BY id
+ """,
+ """
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id, score() AS s
+ FROM ${tableName}
+ WHERE title MATCH_PHRASE 'apple banana'
+ ORDER BY s DESC
+ LIMIT 10
+ """
+ )
+
+
+ // score() with ordinary filters should still execute scorers after the
indexed bitmap cache is warm.
+ assertStablePositiveScoreQuery(
+ "match_any_with_filter_score",
+ """
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id
+ FROM ${tableName}
+ WHERE title MATCH_ANY 'apple' AND status = 1
+ ORDER BY id
+ """,
+ """
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id, score() AS s
+ FROM ${tableName}
+ WHERE title MATCH_ANY 'apple' AND status = 1
+ ORDER BY s DESC
+ LIMIT 10
+ """
+ )
+
+ // score() on VARIANT subfields must not fall back to default 0 after
cache warmup.
+ assertStablePositiveScoreQuery(
+ "variant_match_phrase_score",
+ """
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id
+ FROM ${tableName}
+ WHERE CAST(v["host"] AS STRING) MATCH_PHRASE 'apple server'
+ ORDER BY id
+ """,
+ """
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id, score() AS s
+ FROM ${tableName}
+ WHERE CAST(v["host"] AS STRING) MATCH_PHRASE 'apple server'
+ ORDER BY s DESC
+ LIMIT 10
+ """
+ )
+
+ // UNION ALL branches each have their own score() pushdown and must
collect non-zero scores.
+ def unionWarmSql = """
+ (
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id
+ FROM ${tableName}
+ WHERE title MATCH_ANY 'apple'
+ ORDER BY id
+ LIMIT 3
+ )
+ UNION ALL
+ (
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id
+ FROM ${tableName}
+ WHERE title MATCH_ANY 'banana'
+ ORDER BY id
+ LIMIT 3
+ )
+ ORDER BY id
+ """
+ def unionScoreSql = """
+ SELECT * FROM (
+ (
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id, score() AS s
+ FROM ${tableName}
+ WHERE title MATCH_ANY 'apple'
+ ORDER BY s DESC
+ LIMIT 3
+ )
+ UNION ALL
+ (
+ SELECT
/*+SET_VAR(enable_common_expr_pushdown=true,enable_inverted_index_query_cache=true)
*/
+ id, score() AS s
+ FROM ${tableName}
+ WHERE title MATCH_ANY 'banana'
+ ORDER BY s DESC
+ LIMIT 3
+ )
+ ) t
+ ORDER BY s DESC
+ """
+ assertStablePositiveScoreQuery("union_all_score", unionWarmSql,
unionScoreSql)
+
+ sql "DROP TABLE IF EXISTS ${tableName}"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
