This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new a805e0b366c [cherry-pick](branch-2.0) Pick
"[Enhancement](merge-on-write) Support dynamic delete bitmap cache (#32991)"
(#37602)
a805e0b366c is described below
commit a805e0b366c34c1e4606ab7477c46cbc44761228
Author: abmdocrt <[email protected]>
AuthorDate: Sat Jul 13 22:09:25 2024 +0800
[cherry-pick](branch-2.0) Pick "[Enhancement](merge-on-write) Support
dynamic delete bitmap cache (#32991)" (#37602)
The default delete bitmap cache is set to 100MB, which can be
insufficient and cause performance issues when the amount of user data
is large. To mitigate the problem of an inadequate cache, we will take
the larger of 5% of the total memory and 100MB as the delete bitmap
cache size.
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
---
be/src/common/config.cpp | 6 ++++++
be/src/common/config.h | 2 ++
be/src/olap/tablet_meta.cpp | 15 ++++++++++++++-
3 files changed, 22 insertions(+), 1 deletion(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index ee9201bf6b5..54f90675610 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -777,6 +777,12 @@ DEFINE_Int64(delete_bitmap_agg_cache_capacity,
"104857600");
// s3 config
DEFINE_mInt32(max_remote_storage_count, "10");
+// The default delete bitmap cache is set to 100MB,
+// which can be insufficient and cause performance issues when the amount of
user data is large.
+// To mitigate the problem of an inadequate cache,
+// we will take the larger of 0.5% of the total memory and 100MB as the delete
bitmap cache size.
+DEFINE_String(delete_bitmap_dynamic_agg_cache_limit, "0.5%");
+DEFINE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec, "1800");
// reference
https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#broker-version-compatibility
// If the dependent kafka broker version older than 0.10.0.0,
diff --git a/be/src/common/config.h b/be/src/common/config.h
index dc750fc511f..050987a8398 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -828,6 +828,8 @@ DECLARE_mInt32(jdbc_connection_pool_cache_clear_time_sec);
// Global bitmap cache capacity for aggregation cache, size in bytes
DECLARE_Int64(delete_bitmap_agg_cache_capacity);
+DECLARE_String(delete_bitmap_dynamic_agg_cache_limit);
+DECLARE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec);
// s3 config
DECLARE_mInt32(max_remote_storage_count);
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 22cfdce462f..adbfd4d6f38 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -39,6 +39,8 @@
#include "olap/tablet_meta_manager.h"
#include "olap/utils.h"
#include "util/debug_points.h"
+#include "util/mem_info.h"
+#include "util/parse_util.h"
#include "util/string_util.h"
#include "util/time.h"
#include "util/uid_util.h"
@@ -926,7 +928,18 @@ bool operator!=(const TabletMeta& a, const TabletMeta& b) {
}
DeleteBitmap::DeleteBitmap(int64_t tablet_id) : _tablet_id(tablet_id) {
- _agg_cache.reset(new AggCache(config::delete_bitmap_agg_cache_capacity));
+ // The default delete bitmap cache is set to 100MB,
+ // which can be insufficient and cause performance issues when the amount
of user data is large.
+ // To mitigate the problem of an inadequate cache,
+ // we will take the larger of 0.5% of the total memory and 100MB as the
delete bitmap cache size.
+ bool is_percent = false;
+ int64_t delete_bitmap_agg_cache_cache_limit =
+
ParseUtil::parse_mem_spec(config::delete_bitmap_dynamic_agg_cache_limit,
+ MemInfo::mem_limit(),
MemInfo::physical_mem(), &is_percent);
+ _agg_cache.reset(new AggCache(delete_bitmap_agg_cache_cache_limit >
+
config::delete_bitmap_agg_cache_capacity
+ ? delete_bitmap_agg_cache_cache_limit
+ :
config::delete_bitmap_agg_cache_capacity));
}
DeleteBitmap::DeleteBitmap(const DeleteBitmap& o) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
