This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 03b7020ea0e [opt](point query) reduce short circuit lookup cache usage
(#67228) (#69020)
03b7020ea0e is described below
commit 03b7020ea0ed27c631e7cd0476119d218d9d7a57
Author: HonestManXin <[email protected]>
AuthorDate: Thu Oct 15 09:17:40 2026 +0800
[opt](point query) reduce short circuit lookup cache usage (#67228) (#69020)
https://github.com/apache/doris/pull/67228
Co-authored-by: TRAE CLI <[email protected]>
---
be/src/common/config.cpp | 16 ++++++++++++++++
be/src/common/config.h | 4 ++++
be/src/service/point_query_executor.cpp | 16 +++++-----------
be/test/common/config_validator_test.cpp | 19 +++++++++++++++++++
4 files changed, 44 insertions(+), 11 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index ed484c622dd..bbf4f446902 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1551,6 +1551,19 @@ DEFINE_mInt64(auto_inc_low_water_level_mark_size_ratio,
"3");
DEFINE_mInt64(auto_inc_fetch_thread_num, "3");
// default max to 2048 connections
DEFINE_mInt64(lookup_connection_cache_capacity, "2048");
+// Number of reusable result blocks retained by each point lookup cache entry,
in [1, 32].
+DEFINE_mInt32(lookup_connection_cache_block_pool_size, "1");
+DEFINE_Validator(lookup_connection_cache_block_pool_size,
+ [](int32_t value) -> bool { return value >= 1 && value <= 32;
});
+static std::atomic<size_t> atomic_lookup_connection_cache_block_pool_size {1};
+DEFINE_ON_UPDATE(lookup_connection_cache_block_pool_size, [](int32_t, int32_t
value) {
+
atomic_lookup_connection_cache_block_pool_size.store(static_cast<size_t>(value),
+
std::memory_order_relaxed);
+});
+
+size_t get_lookup_connection_cache_block_pool_size() {
+ return
atomic_lookup_connection_cache_block_pool_size.load(std::memory_order_relaxed);
+}
// level of compression when using LZ4_HC, whose defalut value is
LZ4HC_CLEVEL_DEFAULT
DEFINE_mInt64(LZ4_HC_compression_level, "9");
@@ -2322,6 +2335,9 @@ bool init(const char* conf_file, bool fill_conf_map, bool
must_exist, bool set_t
}
published_inverted_index_candidate_pushdown_ratio.store(
inverted_index_candidate_pushdown_ratio);
+ atomic_lookup_connection_cache_block_pool_size.store(
+ static_cast<size_t>(lookup_connection_cache_block_pool_size),
+ std::memory_order_relaxed);
if (config::is_cloud_mode()) {
auto st = config::set_config("enable_file_cache", "true", true, true);
diff --git a/be/src/common/config.h b/be/src/common/config.h
index d9c2ea4c750..5ea29f1b634 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -17,6 +17,7 @@
#pragma once
+#include <cstddef>
#include <cstdint>
#include <functional>
#include <map>
@@ -1540,6 +1541,9 @@ DECLARE_mInt64(auto_inc_low_water_level_mark_size_ratio);
DECLARE_mInt64(auto_inc_fetch_thread_num);
// Max connection cache num for point lookup queries
DECLARE_mInt64(lookup_connection_cache_capacity);
+// Number of reusable result blocks retained by each point lookup cache entry,
in [1, 32].
+DECLARE_mInt32(lookup_connection_cache_block_pool_size);
+size_t get_lookup_connection_cache_block_pool_size();
// level of compression when using LZ4_HC, whose defalut value is
LZ4HC_CLEVEL_DEFAULT
DECLARE_mInt64(LZ4_HC_compression_level);
diff --git a/be/src/service/point_query_executor.cpp
b/be/src/service/point_query_executor.cpp
index 3e9088525c7..642c1a5528b 100644
--- a/be/src/service/point_query_executor.cpp
+++ b/be/src/service/point_query_executor.cpp
@@ -113,8 +113,6 @@ static void get_missing_and_include_cids(const
TabletSchema& schema,
}
}
-constexpr static int s_preallocted_blocks_num = 32;
-
static void extract_slot_ref(const VExprSPtr& expr, TupleDescriptor*
tuple_desc,
std::vector<SlotDescriptor*>& slots) {
const auto& children = expr->children();
@@ -204,8 +202,9 @@ void Reusable::return_block(std::unique_ptr<Block>& block) {
}
block->clear_column_data();
_block_pool.push_back(std::move(block));
- if (_block_pool.size() > s_preallocted_blocks_num) {
- _block_pool.resize(s_preallocted_blocks_num);
+ const auto block_pool_size =
config::get_lookup_connection_cache_block_pool_size();
+ if (_block_pool.size() > block_pool_size) {
+ _block_pool.resize(block_pool_size);
}
}
@@ -342,15 +341,10 @@ Status PointQueryExecutor::init(const
PTabletKeyLookupRequest* request,
reinterpret_cast<const
uint8_t*>(request->query_options().data()), &len, false,
&t_query_options));
}
+ RETURN_IF_ERROR(reusable_ptr->init(t_desc_tbl, t_output_exprs.exprs,
t_query_options,
+ *_tablet->tablet_schema(), 1));
if (uuid != 0) {
- // could be reused by requests after, pre allocte more blocks
- RETURN_IF_ERROR(reusable_ptr->init(t_desc_tbl,
t_output_exprs.exprs, t_query_options,
- *_tablet->tablet_schema(),
- s_preallocted_blocks_num));
LookupConnectionCache::instance()->add(uuid, reusable_ptr);
- } else {
- RETURN_IF_ERROR(reusable_ptr->init(t_desc_tbl,
t_output_exprs.exprs, t_query_options,
- *_tablet->tablet_schema(), 1));
}
}
_init_remote_scan_cache_write_limiter();
diff --git a/be/test/common/config_validator_test.cpp
b/be/test/common/config_validator_test.cpp
index 9f6e849ed3a..7d6dca7219c 100644
--- a/be/test/common/config_validator_test.cpp
+++ b/be/test/common/config_validator_test.cpp
@@ -50,4 +50,23 @@ TEST(ConfigValidatorTest, Validator) {
EXPECT_EQ(cfg_validator_2, 8);
}
+TEST(ConfigValidatorTest, LookupConnectionCacheBlockPoolSize) {
+ const auto old_value =
config::get_lookup_connection_cache_block_pool_size();
+
+ EXPECT_TRUE(config::set_config("lookup_connection_cache_block_pool_size",
"1").ok());
+ EXPECT_EQ(config::get_lookup_connection_cache_block_pool_size(), 1);
+ EXPECT_FALSE(config::set_config("lookup_connection_cache_block_pool_size",
"0").ok());
+ EXPECT_FALSE(config::set_config("lookup_connection_cache_block_pool_size",
"-1").ok());
+ EXPECT_EQ(config::get_lookup_connection_cache_block_pool_size(), 1);
+
+ EXPECT_TRUE(config::set_config("lookup_connection_cache_block_pool_size",
"32").ok());
+ EXPECT_EQ(config::get_lookup_connection_cache_block_pool_size(), 32);
+ EXPECT_FALSE(config::set_config("lookup_connection_cache_block_pool_size",
"33").ok());
+ EXPECT_EQ(config::get_lookup_connection_cache_block_pool_size(), 32);
+
+ EXPECT_TRUE(
+ config::set_config("lookup_connection_cache_block_pool_size",
std::to_string(old_value))
+ .ok());
+}
+
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
