This is an automated email from the ASF dual-hosted git repository.
wyxxxcat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 94ad13936af [improvement](cloud) Skip delete bitmap lookup for non-MoW
rowsets (#68256)
94ad13936af is described below
commit 94ad13936afd43426e5882930a829ebb5e32618e
Author: Yixuan Wang <[email protected]>
AuthorDate: Wed Oct 28 15:04:42 2026 +0800
[improvement](cloud) Skip delete bitmap lookup for non-MoW rowsets (#68256)
### What problem does this PR solve?
Issue Number: N/A
Related PR: N/A
Problem Summary: Rowset recycling queried versioned delete bitmap
metadata for non-MoW rowsets, producing an unnecessary empty range read
per rowset.
Check whether delete bitmap cleanup is needed before reading its storage
metadata in single-rowset, batched-rowset, and tablet recycling paths.
For a 5,000-rowset non-MoW workload, this avoids 5,000 bitmap metadata
range reads.
Update the recycler benchmark to seed tablet and partition metadata,
warm the MoW cache, and include range-read requests in get counts. Add a
unit test that checks the non-MoW skip and preserves the MoW path.
### Release note
None
### Check List (For Author)
- Test: Unit test added and recycler benchmark updated; passing status
not verified.
- Behavior changed: Yes. Non-MoW rowsets skip delete bitmap metadata
lookup.
- Does this need documentation: No.
---
cloud/src/recycler/recycler.cpp | 91 ++++++++++++++++----------
cloud/src/recycler/recycler.h | 2 +-
cloud/test/recycler_benchmark_test.cpp | 115 ++++++++++++++++++++++++---------
cloud/test/recycler_test.cpp | 56 ++++++++++++++++
4 files changed, 199 insertions(+), 65 deletions(-)
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index d5f30852600..ce9cd8f03bb 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -3939,19 +3939,24 @@ int InstanceRecycler::delete_rowset_data(const
RowsetMetaCloudPB& rs_meta_pb) {
}
}
- // Process delete bitmap - check where it's stored.
- DeleteBitmapStorageType delete_bitmap_storage_type =
DeleteBitmapStorageType::NOT_FOUND;
- if (decrement_delete_bitmap_packed_file_ref_counts(tablet_id, rowset_id,
-
&delete_bitmap_storage_type) != 0) {
- LOG_WARNING("failed to decrement delete bitmap packed file ref count")
- .tag("instance_id", instance_id_)
- .tag("tablet_id", tablet_id)
- .tag("rowset_id", rowset_id);
+ int should_delete_dbm =
may_have_versioned_delete_bitmap(rs_meta_pb.partition_id(), tablet_id);
+ if (should_delete_dbm == 1) {
+ DeleteBitmapStorageType delete_bitmap_storage_type =
DeleteBitmapStorageType::NOT_FOUND;
+ std::vector<std::string> rowset_dbm_keys;
+ if (decrement_delete_bitmap_packed_file_ref_counts(
+ tablet_id, rowset_id, &delete_bitmap_storage_type,
nullptr) != 0) {
+ LOG_WARNING("failed to decrement delete bitmap packed file ref
count")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", tablet_id)
+ .tag("rowset_id", rowset_id);
+ return -1;
+ }
+ if (delete_bitmap_storage_type ==
DeleteBitmapStorageType::STANDALONE_FILE) {
+ file_paths.push_back(delete_bitmap_path(tablet_id, rowset_id));
+ }
+ } else if (should_delete_dbm != 0) {
return -1;
}
- if (delete_bitmap_storage_type ==
DeleteBitmapStorageType::STANDALONE_FILE) {
- file_paths.push_back(delete_bitmap_path(tablet_id, rowset_id));
- }
// TODO(AlexYue): seems could do do batch
return accessor->delete_files(file_paths);
}
@@ -4198,6 +4203,7 @@ int
InstanceRecycler::decrement_packed_file_ref_counts(const doris::RowsetMetaCl
int InstanceRecycler::decrement_delete_bitmap_packed_file_ref_counts(
int64_t tablet_id, const std::string& rowset_id,
DeleteBitmapStorageType* out_storage_type,
std::vector<std::string>* keys) {
+
TEST_SYNC_POINT("InstanceRecycler::decrement_delete_bitmap_packed_file_ref_counts");
if (out_storage_type) {
*out_storage_type = DeleteBitmapStorageType::NOT_FOUND;
}
@@ -4605,24 +4611,31 @@ int InstanceRecycler::delete_rowset_data(
continue;
}
- DeleteBitmapStorageType delete_bitmap_storage_type =
DeleteBitmapStorageType::NOT_FOUND;
- std::vector<std::string> rowset_dbm_keys;
- if (decrement_delete_bitmap_packed_file_ref_counts(
- tablet_id, rowset_id, &delete_bitmap_storage_type,
- delete_bitmap_key_groups ? &rowset_dbm_keys : nullptr) !=
0) {
- LOG_WARNING("failed to decrement delete bitmap packed file ref
count")
- .tag("instance_id", instance_id_)
- .tag("tablet_id", tablet_id)
- .tag("rowset_id", rowset_id);
+ int should_delete_dbm =
may_have_versioned_delete_bitmap(rs.partition_id(), tablet_id);
+ if (should_delete_dbm == 1) {
+ DeleteBitmapStorageType delete_bitmap_storage_type =
DeleteBitmapStorageType::NOT_FOUND;
+ std::vector<std::string> rowset_dbm_keys;
+ if (decrement_delete_bitmap_packed_file_ref_counts(
+ tablet_id, rowset_id, &delete_bitmap_storage_type,
+ delete_bitmap_key_groups ? &rowset_dbm_keys : nullptr)
!= 0) {
+ LOG_WARNING("failed to decrement delete bitmap packed file ref
count")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", tablet_id)
+ .tag("rowset_id", rowset_id)
+ .tag("task_type", metrics_context.operation_type);
+ ret = -1;
+ continue;
+ }
+ if (!rowset_dbm_keys.empty() && delete_bitmap_key_groups) {
+
delete_bitmap_key_groups->emplace_back(std::move(rowset_dbm_keys));
+ }
+ if (delete_bitmap_storage_type ==
DeleteBitmapStorageType::STANDALONE_FILE) {
+ file_paths.push_back(delete_bitmap_path(tablet_id, rowset_id));
+ }
+ } else if (should_delete_dbm != 0) {
ret = -1;
continue;
}
- if (!rowset_dbm_keys.empty() && delete_bitmap_key_groups) {
- delete_bitmap_key_groups->emplace_back(std::move(rowset_dbm_keys));
- }
- if (delete_bitmap_storage_type ==
DeleteBitmapStorageType::STANDALONE_FILE) {
- file_paths.push_back(delete_bitmap_path(tablet_id, rowset_id));
- }
// Process inverted indexes
std::vector<std::pair<int64_t, std::string>> index_ids;
@@ -4827,8 +4840,7 @@ bool InstanceRecycler::is_tablet_recycled(int64_t
tablet_id) {
return recycled_tablets_.contains(tablet_id);
}
-int InstanceRecycler::should_delete_versioned_delete_bitmap_kvs(int64_t
partition_id,
- int64_t
tablet_id) {
+int InstanceRecycler::may_have_versioned_delete_bitmap(int64_t partition_id,
int64_t tablet_id) {
bool is_mow = true;
if (partition_id != -1) {
std::lock_guard lock(partition_mow_cache_mutex);
@@ -4898,7 +4910,7 @@ int
InstanceRecycler::should_delete_versioned_delete_bitmap_kvs(int64_t partitio
int InstanceRecycler::delete_versioned_delete_bitmap_kvs(int64_t partition_id,
int64_t tablet_id,
const std::string&
rowset_id) {
- int ret = should_delete_versioned_delete_bitmap_kvs(partition_id,
tablet_id);
+ int ret = may_have_versioned_delete_bitmap(partition_id, tablet_id);
if (ret <= 0) {
return ret;
}
@@ -5250,12 +5262,21 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id,
RecyclerMetricsContext&
.tag("rowset_id", rs_meta.rowset_id_v2());
return -1;
}
- if (decrement_delete_bitmap_packed_file_ref_counts(tablet_id,
rs_meta.rowset_id_v2(),
- nullptr) != 0) {
- LOG_WARNING("failed to decrement delete bitmap packed file ref
count")
- .tag("instance_id", instance_id_)
- .tag("tablet_id", tablet_id)
- .tag("rowset_id", rs_meta.rowset_id_v2());
+ int should_delete_dbm =
may_have_versioned_delete_bitmap(rs_meta.partition_id(), tablet_id);
+ if (should_delete_dbm == 1) {
+ DeleteBitmapStorageType delete_bitmap_storage_type =
DeleteBitmapStorageType::NOT_FOUND;
+ std::vector<std::string> rowset_dbm_keys;
+ if (decrement_delete_bitmap_packed_file_ref_counts(tablet_id,
rs_meta.rowset_id_v2(),
+
&delete_bitmap_storage_type,
+ nullptr) != 0) {
+ LOG_WARNING("failed to decrement delete bitmap packed file ref
count")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", tablet_id)
+ .tag("rowset_id", rs_meta.rowset_id_v2())
+ .tag("task_type", metrics_context.operation_type);
+ return -1;
+ }
+ } else if (should_delete_dbm != 0) {
return -1;
}
recycle_rowsets_number += 1;
diff --git a/cloud/src/recycler/recycler.h b/cloud/src/recycler/recycler.h
index 427cc419e71..46950970a76 100644
--- a/cloud/src/recycler/recycler.h
+++ b/cloud/src/recycler/recycler.h
@@ -517,7 +517,7 @@ private:
// Return 1 if the versioned delete bitmap should be deleted,
// Return 0 if it can be skipped,
// negative on error.
- int should_delete_versioned_delete_bitmap_kvs(int64_t partition_id,
int64_t tablet_id);
+ int may_have_versioned_delete_bitmap(int64_t partition_id, int64_t
tablet_id);
int delete_versioned_delete_bitmap_kvs(int64_t partition_id, int64_t
tablet_id,
const std::string& rowset_id);
diff --git a/cloud/test/recycler_benchmark_test.cpp
b/cloud/test/recycler_benchmark_test.cpp
index fe35c315b89..a6323efb41b 100644
--- a/cloud/test/recycler_benchmark_test.cpp
+++ b/cloud/test/recycler_benchmark_test.cpp
@@ -22,7 +22,6 @@
#include <gtest/gtest.h>
#include <algorithm>
-#include <atomic>
#include <chrono>
#include <cstddef>
#include <cstdint>
@@ -119,6 +118,7 @@ constexpr int kMaxPackedRecyclePasses = 10;
static_assert(kRowsetsPerBranch % kRowsetsPerPackedFile == 0);
static_assert(kSeedCommitBatch % kRowsetsPerPackedFile == 0);
constexpr int64_t kBenchmarkIndexId = 20000;
+constexpr int64_t kBenchmarkTableId = 30000;
constexpr int32_t kBenchmarkSchemaVersion = 1;
constexpr int64_t kBenchmarkDbId = 10000;
@@ -310,11 +310,33 @@ void put_benchmark_delete_bitmap(Transaction* txn, const
std::string& instance_i
}
}
-// Seed `count` recycle rowset KVs for one branch. Every rowset gets a distinct
-// tablet_id so the per-tablet recycle batch limit never truncates the
workload.
+void put_benchmark_tablet(Transaction* txn, const std::string& instance_id,
int64_t partition_id,
+ int64_t tablet_id, bool is_mow) {
+ TabletIndexPB tablet_index;
+ tablet_index.set_db_id(kBenchmarkDbId);
+ tablet_index.set_table_id(kBenchmarkTableId);
+ tablet_index.set_index_id(kBenchmarkIndexId);
+ tablet_index.set_partition_id(partition_id);
+ tablet_index.set_tablet_id(tablet_id);
+ txn->put(meta_tablet_idx_key({instance_id, tablet_id}),
tablet_index.SerializeAsString());
+
+ doris::TabletMetaCloudPB tablet_meta;
+ tablet_meta.set_table_id(kBenchmarkTableId);
+ tablet_meta.set_partition_id(partition_id);
+ tablet_meta.set_tablet_id(tablet_id);
+ tablet_meta.set_index_id(kBenchmarkIndexId);
+ tablet_meta.set_enable_unique_key_merge_on_write(is_mow);
+ txn->put(meta_tablet_key(
+ {instance_id, kBenchmarkTableId, kBenchmarkIndexId,
partition_id, tablet_id}),
+ tablet_meta.SerializeAsString());
+}
+
+// Seed `count` recycle rowset KVs for one branch. Compacted rowsets share
tablets
+// within one partition, as they do in a real tablet's version history.
int seed_recycle_rowsets(TxnKv* txn_kv, const std::string& instance_id,
RecycleRowsetBranch branch,
int64_t count, int64_t tablet_id_base, bool
write_schema_kv = true,
DeleteBitmapVersion bitmap_version =
DeleteBitmapVersion::kNone) {
+ constexpr int64_t kCompactedRowsetsPerTablet = 100;
if (write_schema_kv && put_benchmark_schema(txn_kv, instance_id) != 0) {
return -1;
}
@@ -330,9 +352,18 @@ int seed_recycle_rowsets(TxnKv* txn_kv, const std::string&
instance_id, RecycleR
return -1;
}
}
- int64_t tablet_id = tablet_id_base + i;
+ int64_t tablet_id = tablet_id_base + (branch ==
RecycleRowsetBranch::kCompactedWithData
+ ? i /
kCompactedRowsetsPerTablet
+ : i);
std::string rowset_id = fmt::format("{:018d}", i);
auto meta = make_rowset_meta(branch, tablet_id, rowset_id);
+ if (branch == RecycleRowsetBranch::kCompactedWithData) {
+ meta.set_partition_id(tablet_id_base);
+ if (i % kCompactedRowsetsPerTablet == 0) {
+ put_benchmark_tablet(txn.get(), instance_id, tablet_id_base,
tablet_id,
+ bitmap_version !=
DeleteBitmapVersion::kNone);
+ }
+ }
auto pb = make_recycle_rowset(branch, meta);
std::string key;
@@ -393,6 +424,9 @@ void put_packed_recycle_rowsets(Transaction* txn, const
std::string& instance_id
const auto tablet_id = tablet_id_base + rowset_number;
const auto rowset_id = fmt::format("{:018d}", rowset_number);
auto meta = make_rowset_meta(RecycleRowsetBranch::kCompactedWithData,
tablet_id, rowset_id);
+ meta.set_partition_id(tablet_id_base);
+ put_benchmark_tablet(txn, instance_id, tablet_id_base, tablet_id,
+ bitmap_version != DeleteBitmapVersion::kNone);
const std::pair<std::string, int64_t> files[] = {
{segment_path(tablet_id, rowset_id, 0), meta.data_disk_size()},
{inverted_index_path_v1(tablet_id, rowset_id, 0, 1, ""),
meta.index_disk_size()}};
@@ -597,16 +631,16 @@ protected:
}
report += fmt::format("recycler benchmark: operation={}\n",
operation_type);
report += fmt::format(
- " {:<{}} {:>12} {:>13} {:>16} {:>12} {:>12} {:>12}"
- " {:>12}\n",
- "branch", branch_width, "total_ms", "recycled_num",
"recycled_bytes",
- "get_keys", "put_keys", "del_keys", "total_keys");
+ " {:<{}} {:>12} {:>13} {:>16} {:>12} {:>12} {:>12}
{:>12}\n", "branch",
+ branch_width, "total_ms", "recycled_num",
"recycled_bytes", "get", "put_keys",
+ "del_keys", "total_keys");
double total_elapsed_ms = 0;
RecycleMetrics total_recycle_metrics;
TxnKvCounts total_txn_kv_counts;
for (const auto& [branch, result] : branches) {
report += fmt::format(
- " {:<{}} {:>12.2f} {:>13} {:>16} {:>12} {:>12}
{:>12} {:>12}\n",
+ " {:<{}} {:>12.2f} {:>13} {:>16} {:>12} {:>12}
{:>12}"
+ " {:>12}\n",
branch, branch_width, result.elapsed_ms,
result.metrics.num,
result.metrics.bytes, result.txn_kv_counts.get,
result.txn_kv_counts.put,
result.txn_kv_counts.del,
result.txn_kv_counts.total());
@@ -616,7 +650,8 @@ protected:
total_txn_kv_counts += result.txn_kv_counts;
}
report += fmt::format(
- " {:<{}} {:>12.2f} {:>13} {:>16} {:>12} {:>12}
{:>12} {:>12}\n",
+ " {:<{}} {:>12.2f} {:>13} {:>16} {:>12} {:>12}
{:>12}"
+ " {:>12}\n",
"total", branch_width, total_elapsed_ms,
total_recycle_metrics.num,
total_recycle_metrics.bytes, total_txn_kv_counts.get,
total_txn_kv_counts.put,
total_txn_kv_counts.del, total_txn_kv_counts.total());
@@ -653,7 +688,8 @@ protected:
}
TxnKvCounts read_txn_kv_counts() const {
- return {.get = g_bvar_txn_kv_get_count_normalized.get_value(),
+ return {.get = g_bvar_txn_kv_get_count_normalized.get_value() +
+ g_bvar_txn_kv_range_get.count(),
.put = g_bvar_txn_kv_put.count() +
g_bvar_txn_kv_atomic_set_ver_key.count() +
g_bvar_txn_kv_atomic_set_ver_value.count() +
g_bvar_txn_kv_atomic_add.count(),
@@ -719,6 +755,14 @@ protected:
ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
}
+ void warm_partition_mow_cache(int64_t partition_id, bool is_mow) {
+ // Keep the timed run focused on the steady-state per-rowset path. The
first lookup
+ // validates the seeded tablet index/meta and populates the partition
cache.
+ ASSERT_EQ(recycler_->may_have_versioned_delete_bitmap(partition_id,
+ /*tablet_id*/
partition_id),
+ is_mow ? 1 : 0);
+ }
+
int64_t count_recycle_rowsets() {
std::string begin = recycle_rowset_key({kBenchmarkInstanceId, 0, ""});
const auto end =
@@ -847,13 +891,15 @@ TEST_F(RecyclerBenchmarkTest, RecycleRowsets) {
RecycleRowsetBranch::kCompactedWithData, kRowsetsPerBranch,
tablet_id_base),
0);
+ ASSERT_NO_FATAL_FAILURE(warm_partition_mow_cache(tablet_id_base,
false));
ASSERT_EQ(remove_benchmark_schema(txn_kv_.get(),
kBenchmarkInstanceId), 0);
ASSERT_NO_FATAL_FAILURE(measure("recycle_rowsets",
"compacted_without_schema", "delete"));
ASSERT_EQ(count_recycle_rowsets(), 0);
});
// Hold rowset count, schema and file sizes fixed; vary only data packing
and
- // bitmap version. V1 bitmap KVs survive rowset recycling, so isolate each
case.
+ // bitmap version. Data-only cases model non-MoW tablets; bitmap cases
model MoW.
+ // V1 bitmap KVs survive rowset recycling, so isolate each case.
for (const auto& [bitmap_version, bitmap_name] :
{std::pair {DeleteBitmapVersion::kNone, "data_only"},
std::pair {DeleteBitmapVersion::kV1, "delete_bitmap_v1"},
@@ -867,6 +913,8 @@ TEST_F(RecyclerBenchmarkTest, RecycleRowsets) {
RecycleRowsetBranch::kCompactedWithData,
kRowsetsPerBranch, tablet_id_base,
true, bitmap_version),
0);
+ ASSERT_NO_FATAL_FAILURE(warm_partition_mow_cache(
+ tablet_id_base, bitmap_version !=
DeleteBitmapVersion::kNone));
ASSERT_EQ(count_recycle_rowsets(), kRowsetsPerBranch);
ASSERT_NO_FATAL_FAILURE(measure("recycle_rowsets",
fmt::format("compacted_with_data/{}", bitmap_name),
@@ -878,6 +926,9 @@ TEST_F(RecyclerBenchmarkTest, RecycleRowsets) {
run_benchmark(fmt::format("compacted_with_packed_data/{}",
bitmap_name), [&] {
ASSERT_NO_FATAL_FAILURE(seed_packed_recycle_rowsets(tablet_id_base
+ kRowsetsPerBranch,
bitmap_version));
+ ASSERT_NO_FATAL_FAILURE(
+ warm_partition_mow_cache(tablet_id_base +
kRowsetsPerBranch,
+ bitmap_version !=
DeleteBitmapVersion::kNone));
int64_t remaining = count_recycle_rowsets();
ASSERT_EQ(remaining, kRowsetsPerBranch);
@@ -911,7 +962,6 @@ TEST_F(RecyclerBenchmarkTest, RecycleRowsets) {
check_recycled_bitmap_range(tablet_id_base +
kRowsetsPerBranch));
});
}
-
run_benchmark("compacted_empty", [&] {
ASSERT_EQ(seed_recycle_rowsets(txn_kv_.get(), kBenchmarkInstanceId,
RecycleRowsetBranch::kCompactedEmpty,
kRowsetsPerBranch,
@@ -981,6 +1031,9 @@ TEST_F(RecyclerBenchmarkTest, RecycleRowsets) {
mixed_tablet_offset +
tablet_base_for(RecycleRowsetBranch::kCompactedWithData)),
0);
+ ASSERT_NO_FATAL_FAILURE(warm_partition_mow_cache(
+ mixed_tablet_offset +
tablet_base_for(RecycleRowsetBranch::kCompactedWithData),
+ false));
ASSERT_EQ(
seed_recycle_rowsets(txn_kv_.get(), kBenchmarkInstanceId,
RecycleRowsetBranch::kCompactedEmpty,
kRowsetsPerBranch,
@@ -993,26 +1046,30 @@ TEST_F(RecyclerBenchmarkTest, RecycleRowsets) {
ASSERT_EQ(count_recycle_rowsets(), 0);
});
- // Recorded with 5,000 rowsets per branch. Recalibrate if the workload
changes.
+ // Bulk branches use 5,000 rowsets. Recalibrate if the workload changes.
static_assert(kRowsetsPerBranch == 5000);
const std::map<std::string, TxnKvCounts> baseline_txn_kv_counts = {
- {"compacted_empty", {.get = 5000, .put = 0, .del = 5000}},
- // First schema cache fill reads one KV in addition to the 5,000
rowset KVs.
- {"compacted_with_data/data_only", {.get = 5001, .put = 0, .del =
5000}},
- {"compacted_with_data/delete_bitmap_v1", {.get = 5000, .put = 0,
.del = 5000}},
- {"compacted_with_data/delete_bitmap_v2", {.get = 10000, .put = 0,
.del = 10000}},
- {"compacted_with_packed_data/data_only", {.get = 12500, .put =
5000, .del = 7500}},
+ {"compacted_empty", {.get = 5014, .put = 0, .del = 5000}},
+ // get = returned KVs + range-get requests (empty results count as
one request).
+ // data_only: 5000 rowset KVs + 13 scan reads + 1 schema KV + 1
schema read = 5015.
+ // v1: 5000 rowset KVs + 13 scan reads + 5000 empty bitmap reads =
10013.
+ // v2: 5000 rowset KVs + 13 scan reads + 5000 bitmap KVs + 5000
bitmap reads = 15013.
+ {"compacted_with_data/data_only", {.get = 5015, .put = 0, .del =
5000}},
+ {"compacted_with_data/delete_bitmap_v1", {.get = 10013, .put = 0,
.del = 5000}},
+ {"compacted_with_data/delete_bitmap_v2", {.get = 15013, .put = 0,
.del = 10000}},
+ {"compacted_with_packed_data/data_only", {.get = 12526, .put =
5000, .del = 7500}},
{"compacted_with_packed_data/delete_bitmap_v1",
- {.get = 12500, .put = 5000, .del = 7500}},
+ {.get = 17526, .put = 5000, .del = 7500}},
{"compacted_with_packed_data/delete_bitmap_v2",
- {.get = 17500, .put = 5000, .del = 12500}},
- {"compacted_without_schema", {.get = 5000, .put = 0, .del = 5000}},
- {"legacy_empty_resource", {.get = 5000, .put = 0, .del = 5000}},
- {"legacy_with_resource", {.get = 10000, .put = 0, .del = 10000}},
- {"mixed", {.get = 100000, .put = 20000, .del = 60000}},
- {"prepare_abort", {.get = 45000, .put = 15000, .del = 15000}},
- {"prepare_direct", {.get = 10000, .put = 0, .del = 10000}},
- {"prepare_mark", {.get = 20000, .put = 5000, .del = 10000}},
+ {.get = 22526, .put = 5000, .del = 12500}},
+ // Missing schema is read once per rowset because the cache does
not retain misses.
+ {"compacted_without_schema", {.get = 10013, .put = 0, .del =
5000}},
+ {"legacy_empty_resource", {.get = 5008, .put = 0, .del = 5000}},
+ {"legacy_with_resource", {.get = 10009, .put = 0, .del = 10000}},
+ {"mixed", {.get = 100111, .put = 20000, .del = 60000}},
+ {"prepare_abort", {.get = 45030, .put = 15000, .del = 15000}},
+ {"prepare_direct", {.get = 10014, .put = 0, .del = 10000}},
+ {"prepare_mark", {.get = 20028, .put = 5000, .del = 10000}},
};
const auto& results = benchmark_results_["recycle_rowsets"];
for (const auto& [branch, baseline_counts] : baseline_txn_kv_counts) {
diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp
index b84c4ff09c3..1b65192150f 100644
--- a/cloud/test/recycler_test.cpp
+++ b/cloud/test/recycler_test.cpp
@@ -7519,6 +7519,62 @@ TEST(RecyclerTest,
delete_rowset_data_without_delete_bitmap_meta) {
EXPECT_EQ(deleted_paths[0], segment_path(rowset.tablet_id(),
rowset.rowset_id_v2(), 0));
}
+TEST(RecyclerTest, delete_rowset_data_skips_delete_bitmap_for_non_mow_table) {
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ constexpr std::string_view resource_id =
"delete_rowset_data_skip_non_mow_dbm";
+ InstanceRecycler recycler(txn_kv,
create_recycler_test_instance(std::string(resource_id)),
+ thread_group,
std::make_shared<TxnLazyCommitter>(txn_kv));
+ ASSERT_EQ(recycler.init(), 0);
+
+ constexpr int64_t table_id = 29901;
+ constexpr int64_t index_id = 29902;
+ constexpr int64_t non_mow_partition_id = 29903;
+ constexpr int64_t non_mow_tablet_id = 29904;
+ constexpr int64_t mow_partition_id = 29905;
+ constexpr int64_t mow_tablet_id = 29906;
+ ASSERT_EQ(create_tablet(txn_kv.get(), table_id, index_id,
non_mow_partition_id,
+ non_mow_tablet_id, false),
+ 0);
+ ASSERT_EQ(
+ create_tablet(txn_kv.get(), table_id, index_id, mow_partition_id,
mow_tablet_id, true),
+ 0);
+
+ doris::TabletSchemaCloudPB schema;
+ schema.set_schema_version(1);
+ schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1);
+
+ auto non_mow_rowset =
+ create_rowset(std::string(resource_id), non_mow_tablet_id,
index_id, 1, schema);
+ non_mow_rowset.set_partition_id(non_mow_partition_id);
+ auto mow_rowset = create_rowset(std::string(resource_id), mow_tablet_id,
index_id, 1, schema);
+ mow_rowset.set_partition_id(mow_partition_id);
+
+ std::atomic<int> decrement_calls {0};
+ auto* sp = SyncPoint::get_instance();
+ SyncPoint::CallbackGuard guard;
+ sp->set_call_back(
+ "InstanceRecycler::decrement_delete_bitmap_packed_file_ref_counts",
+ [&](auto&&) { ++decrement_calls; }, &guard);
+ sp->enable_processing();
+
+ RecyclerMetricsContext metrics_context;
+ std::map<std::string, doris::RowsetMetaCloudPB> rowsets;
+ rowsets.emplace(non_mow_rowset.rowset_id_v2(), non_mow_rowset);
+ ASSERT_EQ(recycler.delete_rowset_data(rowsets,
RowsetRecyclingState::FORMAL_ROWSET,
+ metrics_context),
+ 0);
+ EXPECT_EQ(decrement_calls.load(), 0);
+
+ rowsets.clear();
+ rowsets.emplace(mow_rowset.rowset_id_v2(), mow_rowset);
+ ASSERT_EQ(recycler.delete_rowset_data(rowsets,
RowsetRecyclingState::FORMAL_ROWSET,
+ metrics_context),
+ 0);
+ EXPECT_EQ(decrement_calls.load(), 1);
+}
+
TEST(RecyclerTest,
delete_versioned_delete_bitmap_kvs_caches_partition_mow_state) {
auto txn_kv = std::make_shared<MemTxnKv>();
ASSERT_EQ(txn_kv->init(), 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
