This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new baed3db8f8c [refactor](cloud) improve FDB transaction approximate size
calculation accuracy (#60351)
baed3db8f8c is described below
commit baed3db8f8c1d9d33f4493483c9dbddcc9546509
Author: walter <[email protected]>
AuthorDate: Fri Jan 30 12:14:23 2026 +0800
[refactor](cloud) improve FDB transaction approximate size calculation
accuracy (#60351)
---
cloud/src/meta-store/txn_kv.cpp | 72 +++++++++++++++++++++++++++++++++--------
1 file changed, 58 insertions(+), 14 deletions(-)
diff --git a/cloud/src/meta-store/txn_kv.cpp b/cloud/src/meta-store/txn_kv.cpp
index d55bcec618c..70740ac3747 100644
--- a/cloud/src/meta-store/txn_kv.cpp
+++ b/cloud/src/meta-store/txn_kv.cpp
@@ -266,6 +266,53 @@ namespace doris::cloud::fdb {
//
https://apple.github.io/foundationdb/known-limitations.html#design-limitations
constexpr size_t FDB_VALUE_BYTES_LIMIT = 100'000; // 100 KB
+// FDB internal structure sizes for approximate size calculation
+// See fdbclient/ReadYourWrites.actor.cpp for details
+constexpr size_t FDB_SIZEOF_KEYRANGEREF = 32; // sizeof(KeyRangeRef)
+constexpr size_t FDB_SIZEOF_MUTATIONREF = 56; // sizeof(MutationRef)
+
+// Helper functions for calculating approximate size according to FDB's
implementation
+namespace {
+
+// Calculate approximate size for read operation on single key
+// Formula: 2*key + 1 + sizeof(KeyRangeRef)
+// See fdbclient/ReadYourWrites.actor.cpp:329
+inline constexpr size_t read_get_approximate_size(size_t key_size) {
+ return key_size * 2 + 1 + FDB_SIZEOF_KEYRANGEREF;
+}
+
+// Calculate approximate size for read operation on range
+// Formula: begin + end + sizeof(KeyRangeRef)
+// See fdbclient/ReadYourWrites.actor.cpp:245-280
+inline constexpr size_t read_get_range_approximate_size(size_t begin_size,
size_t end_size) {
+ return begin_size + end_size + FDB_SIZEOF_KEYRANGEREF;
+}
+
+// Calculate approximate size for write operation (set/atomic_op)
+// Formula: key + val + sizeof(MutationRef) + sizeof(KeyRangeRef) + 2*key + 1
+// See fdbclient/ReadYourWrites.actor.cpp:2267-2269
+inline constexpr size_t write_set_approximate_size(size_t key_size, size_t
val_size) {
+ size_t write_conflict = FDB_SIZEOF_KEYRANGEREF + key_size * 2 + 1;
+ return key_size + val_size + FDB_SIZEOF_MUTATIONREF + write_conflict;
+}
+
+// Calculate approximate size for clear operation on single key
+// Formula: 2*key + 2*sizeof(KeyRangeRef)
+// See fdbclient/ReadYourWrites.actor.cpp:2361-2362
+inline constexpr size_t write_clear_approximate_size(size_t key_size) {
+ return key_size * 2 + FDB_SIZEOF_KEYRANGEREF * 2;
+}
+
+// Calculate approximate size for clear_range operation
+// Formula: begin+end + sizeof(MutationRef) + sizeof(KeyRangeRef) + begin+end
+// See fdbclient/ReadYourWrites.actor.cpp:2304-2306
+inline constexpr size_t write_clear_range_approximate_size(size_t begin_size,
size_t end_size) {
+ size_t write_conflict = FDB_SIZEOF_KEYRANGEREF + begin_size + end_size;
+ return begin_size + end_size + FDB_SIZEOF_MUTATIONREF + write_conflict;
+}
+
+} // anonymous namespace
+
// Ref
https://apple.github.io/foundationdb/api-error-codes.html#developer-guide-error-codes.
constexpr fdb_error_t FDB_ERROR_CODE_TIMED_OUT = 1004;
constexpr fdb_error_t FDB_ERROR_CODE_TXN_TOO_OLD = 1007;
@@ -488,7 +535,7 @@ void Transaction::put(std::string_view key,
std::string_view val) {
++num_put_keys_;
put_bytes_ += key.size() + val.size();
- approximate_bytes_ += key.size() * 3 + val.size(); // See
fdbclient/ReadYourWrites.actor.cpp
+ approximate_bytes_ += write_set_approximate_size(key.size(), val.size());
if (val.size() > FDB_VALUE_BYTES_LIMIT) {
LOG_WARNING("txn put with large value")
@@ -534,8 +581,7 @@ TxnErrorCode Transaction::get(std::string_view key,
std::string* val, bool snaps
StopWatch sw;
if (!snapshot) {
- // See fdbclient/ReadYourWrites.actor.cpp for details
- approximate_bytes_ += key.size() * 2;
+ approximate_bytes_ += read_get_approximate_size(key.size());
}
auto* fut = fdb_transaction_get(txn_, (uint8_t*)key.data(), key.size(),
snapshot);
@@ -581,8 +627,7 @@ TxnErrorCode Transaction::get(std::string_view begin,
std::string_view end,
StopWatch sw;
if (!opts.snapshot) {
- // See fdbclient/ReadYourWrites.actor.cpp for details
- approximate_bytes_ += begin.size() * 2 + end.size() * 2;
+ approximate_bytes_ += read_get_range_approximate_size(begin.size(),
end.size());
}
DORIS_CLOUD_DEFER {
g_bvar_txn_kv_range_get << sw.elapsed_us();
@@ -643,7 +688,7 @@ void Transaction::atomic_set_ver_key(std::string_view
key_prefix, std::string_vi
g_bvar_txn_kv_atomic_set_ver_key << sw.elapsed_us();
++num_put_keys_;
put_bytes_ += key.size() + val.size();
- approximate_bytes_ += key.size() * 3 + val.size();
+ approximate_bytes_ += write_set_approximate_size(key.size(), val.size());
if (val.size() > FDB_VALUE_BYTES_LIMIT) {
LOG_WARNING("atomic_set_ver_key with large value")
@@ -672,7 +717,7 @@ bool Transaction::atomic_set_ver_key(std::string_view key,
uint32_t offset, std:
g_bvar_txn_kv_atomic_set_ver_key << sw.elapsed_us();
++num_put_keys_;
put_bytes_ += key_buf.size() + val.size();
- approximate_bytes_ += key_buf.size() * 3 + val.size();
+ approximate_bytes_ += write_set_approximate_size(key_buf.size(),
val.size());
if (val.size() > FDB_VALUE_BYTES_LIMIT) {
LOG_WARNING("atomic_set_ver_key with large value")
@@ -700,7 +745,7 @@ void Transaction::atomic_set_ver_value(std::string_view
key, std::string_view va
g_bvar_txn_kv_atomic_set_ver_value << sw.elapsed_us();
++num_put_keys_;
put_bytes_ += key.size() + val.size();
- approximate_bytes_ += key.size() * 3 + val.size();
+ approximate_bytes_ += write_set_approximate_size(key.size(), val.size());
if (val.size() > FDB_VALUE_BYTES_LIMIT) {
LOG_WARNING("atomic_set_ver_value with large value")
@@ -720,7 +765,7 @@ void Transaction::atomic_add(std::string_view key, int64_t
to_add) {
g_bvar_txn_kv_atomic_add << sw.elapsed_us();
++num_put_keys_;
put_bytes_ += key.size() + 8;
- approximate_bytes_ += key.size() * 3 + 8;
+ approximate_bytes_ += write_set_approximate_size(key.size(), 8);
}
bool Transaction::decode_atomic_int(std::string_view data, int64_t* val) {
@@ -742,7 +787,7 @@ void Transaction::remove(std::string_view key) {
g_bvar_txn_kv_remove << sw.elapsed_us();
++num_del_keys_;
delete_bytes_ += key.size();
- approximate_bytes_ += key.size() * 4; // See
fdbclient/ReadYourWrites.actor.cpp for details.
+ approximate_bytes_ += write_clear_approximate_size(key.size());
}
void Transaction::remove(std::string_view begin, std::string_view end) {
@@ -752,8 +797,7 @@ void Transaction::remove(std::string_view begin,
std::string_view end) {
g_bvar_txn_kv_range_remove << sw.elapsed_us();
num_del_keys_ += 2;
delete_bytes_ += begin.size() + end.size();
- approximate_bytes_ +=
- (begin.size() + end.size()) * 2; // See
fdbclient/ReadYourWrites.actor.cpp for details.
+ approximate_bytes_ += write_clear_range_approximate_size(begin.size(),
end.size());
}
TxnErrorCode Transaction::commit() {
@@ -1185,7 +1229,7 @@ TxnErrorCode
Transaction::batch_get(std::vector<std::optional<std::string>>* res
futures.emplace_back(
fdb_transaction_get(txn_, (uint8_t*)k.data(), k.size(),
opts.snapshot));
if (!opts.snapshot) {
- approximate_bytes_ += k.size() * 2;
+ approximate_bytes_ += read_get_approximate_size(k.size());
}
}
@@ -1265,7 +1309,7 @@ TxnErrorCode Transaction::batch_scan(
futures.emplace_back(fut);
if (!opts.snapshot) {
- approximate_bytes_ += start.size() + end.size();
+ approximate_bytes_ +=
read_get_range_approximate_size(start.size(), end.size());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
