This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new eb470282090 branch-3.0: [fix](mow) remove agg cache version for unused
rowsets #49511 (#49578)
eb470282090 is described below
commit eb470282090c78699ab884e8d6f0619f65b4f695
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Mar 30 10:34:03 2025 +0800
branch-3.0: [fix](mow) remove agg cache version for unused rowsets #49511
(#49578)
Cherry-picked from #49511
Co-authored-by: meiyi <[email protected]>
---
be/src/olap/storage_engine.cpp | 6 +++++
be/src/olap/tablet_meta.cpp | 51 ++++++++++++++++++++++++++++++++++--------
be/src/olap/tablet_meta.h | 2 ++
3 files changed, 50 insertions(+), 9 deletions(-)
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 3ba65813492..84a654e366d 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -949,6 +949,11 @@ void StorageEngine::_clean_unused_rowset_metas() {
for (auto& rowset_meta : invalid_rowset_metas) {
static_cast<void>(RowsetMetaManager::remove(
data_dir->get_meta(), rowset_meta->tablet_uid(),
rowset_meta->rowset_id()));
+ TabletSharedPtr tablet =
_tablet_manager->get_tablet(rowset_meta->tablet_id());
+ if (tablet &&
tablet->tablet_meta()->enable_unique_key_merge_on_write()) {
+
tablet->tablet_meta()->delete_bitmap().remove_rowset_cache_version(
+ rowset_meta->rowset_id());
+ }
}
LOG(INFO) << "remove " << invalid_rowset_metas.size()
<< " invalid rowset meta from dir: " << data_dir->path();
@@ -1223,6 +1228,7 @@ void StorageEngine::start_delete_unused_rowset() {
tablet && tablet->enable_unique_key_merge_on_write()) {
tablet->tablet_meta()->delete_bitmap().remove({rs->rowset_id(), 0,
0},
{rs->rowset_id(),
UINT32_MAX, 0});
+
tablet->tablet_meta()->delete_bitmap().remove_rowset_cache_version(rs->rowset_id());
}
Status status = rs->remove();
unused_rowsets_counter << -1;
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 3b58006f8d0..c843e3abad5 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -1004,17 +1004,38 @@ void TabletMeta::clear_rowsets() {
}
void TabletMeta::_check_mow_rowset_cache_version_size(size_t
rowset_cache_version_size) {
- if (_enable_unique_key_merge_on_write &&
+ if (_enable_unique_key_merge_on_write && config::enable_mow_verbose_log &&
rowset_cache_version_size > _rs_metas.size() + _stale_rs_metas.size())
{
- std::string err_msg = fmt::format(
- ". tablet: {}, rowset_cache_version size: {}, "
- "_rs_metas size: {}, _stale_rs_metas size: {}",
- _tablet_id, rowset_cache_version_size, _rs_metas.size(),
_stale_rs_metas.size());
- if (config::enable_mow_get_agg_correctness_check_core) {
- CHECK(false) << err_msg;
- } else {
- DCHECK(false) << err_msg;
+ std::stringstream ss;
+ auto rowset_ids = _delete_bitmap->get_rowset_cache_version();
+ for (const auto& rowset_id : rowset_ids) {
+ bool found = false;
+ for (auto& rs_meta : _rs_metas) {
+ if (rs_meta->rowset_id() == rowset_id) {
+ found = true;
+ break;
+ }
+ }
+ if (found) {
+ continue;
+ }
+ for (auto& rs_meta : _stale_rs_metas) {
+ if (rs_meta->rowset_id() == rowset_id) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ ss << rowset_id.to_string() << ", ";
+ }
}
+ // size(rowset_cache_version) <= size(_rs_metas) +
size(_stale_rs_metas) + size(_unused_rs)
+ std::string msg = fmt::format(
+ "tablet: {}, rowset_cache_version size: {}, "
+ "_rs_metas size: {}, _stale_rs_metas size: {}, delta: {}.
rowset only in cache: {}",
+ _tablet_id, rowset_cache_version_size, _rs_metas.size(),
_stale_rs_metas.size(),
+ rowset_cache_version_size - _rs_metas.size() -
_stale_rs_metas.size(), ss.str());
+ LOG(INFO) << msg;
}
}
@@ -1330,12 +1351,24 @@ bool
DeleteBitmap::has_calculated_for_multi_segments(const RowsetId& rowset_id)
size_t DeleteBitmap::remove_rowset_cache_version(const RowsetId& rowset_id) {
std::lock_guard l(_rowset_cache_version_lock);
_rowset_cache_version.erase(rowset_id);
+ VLOG_DEBUG << "remove agg cache version for tablet=" << _tablet_id
+ << ", rowset=" << rowset_id.to_string();
return _rowset_cache_version.size();
}
void DeleteBitmap::clear_rowset_cache_version() {
std::lock_guard l(_rowset_cache_version_lock);
_rowset_cache_version.clear();
+ VLOG_DEBUG << "clear agg cache version for tablet=" << _tablet_id;
+}
+
+std::set<RowsetId> DeleteBitmap::get_rowset_cache_version() {
+ std::set<RowsetId> set;
+ std::shared_lock l(_rowset_cache_version_lock);
+ for (auto& [k, _] : _rowset_cache_version) {
+ set.insert(k);
+ }
+ return set;
}
DeleteBitmap::Version DeleteBitmap::_get_rowset_cache_version(const BitmapKey&
bmk) const {
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 4970551ff24..a0a2ab3d321 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -563,6 +563,8 @@ public:
void clear_rowset_cache_version();
+ std::set<RowsetId> get_rowset_cache_version();
+
class AggCachePolicy : public LRUCachePolicy {
public:
AggCachePolicy(size_t capacity)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
