This is an automated email from the ASF dual-hosted git repository.
zouxinyi pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new 225a1baf52 [improvement](memory) simplify memory config related to
tcmalloc (#13784)
225a1baf52 is described below
commit 225a1baf522a9976816195d95d89755fabf328a9
Author: Yongqiang YANG <98214048+dataroar...@users.noreply.github.com>
AuthorDate: Thu Nov 3 23:33:20 2022 +0800
[improvement](memory) simplify memory config related to tcmalloc (#13784)
There are several configs related to tcmalloc, users do know how to config
them. Actually users just want two modes, performance or compact, in
performance mode, users want doris run query and load quickly while in compact
mode, users want doris run with less memory usage.
If we want to config tcmalloc individually, we can use env variables which
are supported by tcmalloc.
---
be/src/common/config.h | 22 +++-------------------
be/src/common/daemon.cpp | 13 +++++++++++--
be/src/service/doris_main.cpp | 33 ++++++++++++++++++---------------
3 files changed, 32 insertions(+), 36 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 132d7b43fd..bdae169ad7 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -42,25 +42,9 @@ CONF_Int32(brpc_num_threads, "-1");
// If no ip match this rule, will choose one randomly.
CONF_String(priority_networks, "");
-////
-//// tcmalloc gc parameter
-////
-// min memory for TCmalloc, when used memory is smaller than this, do not
returned to OS
-CONF_mInt64(tc_use_memory_min, "10737418240");
-// free memory rate.[0-100]
-CONF_mInt64(tc_free_memory_rate, "20");
-// tcmallc aggressive_memory_decommit
-CONF_mBool(tc_enable_aggressive_memory_decommit, "true");
-
-// Bound on the total amount of bytes allocated to thread caches.
-// This bound is not strict, so it is possible for the cache to go over this
bound
-// in certain circumstances. This value defaults to 1GB
-// If you suspect your application is not scaling to many threads due to lock
contention in TCMalloc,
-// you can try increasing this value. This may improve performance, at a cost
of extra memory
-// use by TCMalloc.
-// reference: https://gperftools.github.io/gperftools/tcmalloc.html:
TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES
-// https://github.com/gperftools/gperftools/issues/1111
-CONF_Int64(tc_max_total_thread_cache_bytes, "1073741824");
+// memory mode
+// performance or compact
+CONF_String(memory_mode, "performance");
// process memory limit specified as number of bytes
// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index bb39bf13ef..ed5fe06ffa 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -69,6 +69,15 @@ bool k_doris_exit = false;
void Daemon::tcmalloc_gc_thread() {
// TODO All cache GC wish to be supported
+
+ size_t tc_use_memory_min = MemInfo::mem_limit();
+ if (config::memory_mode == std::string("performance")) {
+ tc_use_memory_min = std::max(tc_use_memory_min / 10 * 9,
+ tc_use_memory_min - size_t(10) * 1024 *
1024 * 1024);
+ } else {
+ tc_use_memory_min = tc_use_memory_min >> 1;
+ }
+
while
(!_stop_background_threads_latch.wait_for(MonoDelta::FromSeconds(10))) {
size_t used_size = 0;
size_t free_size = 0;
@@ -78,8 +87,8 @@ void Daemon::tcmalloc_gc_thread() {
MallocExtension::instance()->GetNumericProperty("tcmalloc.pageheap_free_bytes",
&free_size);
size_t alloc_size = used_size + free_size;
- if (alloc_size > config::tc_use_memory_min) {
- size_t max_free_size = alloc_size * config::tc_free_memory_rate /
100;
+ if (alloc_size > tc_use_memory_min) {
+ size_t max_free_size = alloc_size * 20 / 100;
if (free_size > max_free_size) {
MallocExtension::instance()->ReleaseToSystem(free_size -
max_free_size);
}
diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp
index bd5fe3aa93..4556cf939b 100644
--- a/be/src/service/doris_main.cpp
+++ b/be/src/service/doris_main.cpp
@@ -314,26 +314,29 @@ int main(int argc, char** argv) {
return -1;
}
-#if !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) &&
!defined(THREAD_SANITIZER)
- // Aggressive decommit is required so that unused pages in the TCMalloc
page heap are
- // not backed by physical pages and do not contribute towards memory
consumption.
- if (doris::config::tc_enable_aggressive_memory_decommit) {
-
MallocExtension::instance()->SetNumericProperty("tcmalloc.aggressive_memory_decommit",
1);
- }
- // Change the total TCMalloc thread cache size if necessary.
- if (!MallocExtension::instance()->SetNumericProperty(
- "tcmalloc.max_total_thread_cache_bytes",
- doris::config::tc_max_total_thread_cache_bytes)) {
- fprintf(stderr, "Failed to change TCMalloc total thread cache
size.\n");
- return -1;
- }
-#endif
-
if (!doris::Env::init()) {
LOG(FATAL) << "init env failed.";
exit(-1);
}
+#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) &&
!defined(LEAK_SANITIZER) && \
+ !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC)
+ // Change the total TCMalloc thread cache size if necessary.
+ size_t total_thread_cache_bytes;
+ if
(!MallocExtension::instance()->GetNumericProperty("tcmalloc.max_total_thread_cache_bytes",
+
&total_thread_cache_bytes)) {
+ fprintf(stderr, "Failed to get TCMalloc total thread cache size.\n");
+ }
+ const size_t kDefaultTotalThreadCacheBytes = 1024 * 1024 * 1024;
+ if (total_thread_cache_bytes < kDefaultTotalThreadCacheBytes) {
+ if (!MallocExtension::instance()->SetNumericProperty(
+ "tcmalloc.max_total_thread_cache_bytes",
kDefaultTotalThreadCacheBytes)) {
+ fprintf(stderr, "Failed to change TCMalloc total thread cache
size.\n");
+ return -1;
+ }
+ }
+#endif
+
std::vector<doris::StorePath> paths;
auto olap_res =
doris::parse_conf_store_paths(doris::config::storage_root_path, &paths);
if (olap_res != doris::OLAP_SUCCESS) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
