This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new c7b707f2575 [fix](cloud-mow)Full compaction no need to sync delete
bitmap (#43613)
c7b707f2575 is described below
commit c7b707f2575928f9e1416dc759a7b760891fc4b8
Author: huanghaibin <284824...@qq.com>
AuthorDate: Mon Nov 11 22:33:46 2024 +0800
[fix](cloud-mow)Full compaction no need to sync delete bitmap (#43613)
sync delete bitmap may cost too much time,which may lead to full
compaction fail, but full compaction no need to sync it.
pick pr:https://github.com/apache/doris/pull/43337
Co-authored-by: huanghaibin <huanghai...@selectdb.com>
---
be/src/cloud/cloud_compaction_action.cpp | 5 +++--
be/src/cloud/cloud_meta_mgr.cpp | 5 +++--
be/src/cloud/cloud_meta_mgr.h | 3 ++-
be/src/cloud/cloud_tablet_mgr.cpp | 11 ++++++-----
be/src/cloud/cloud_tablet_mgr.h | 3 ++-
5 files changed, 16 insertions(+), 11 deletions(-)
diff --git a/be/src/cloud/cloud_compaction_action.cpp
b/be/src/cloud/cloud_compaction_action.cpp
index 13161c32c8e..481f7b589fe 100644
--- a/be/src/cloud/cloud_compaction_action.cpp
+++ b/be/src/cloud/cloud_compaction_action.cpp
@@ -149,8 +149,9 @@ Status
CloudCompactionAction::_handle_run_compaction(HttpRequest* req, std::stri
compaction_type != PARAM_COMPACTION_FULL) {
return Status::NotSupported("The compaction type '{}' is not
supported", compaction_type);
}
-
- CloudTabletSPtr tablet =
DORIS_TRY(_engine.tablet_mgr().get_tablet(tablet_id));
+ bool sync_delete_bitmap = compaction_type != PARAM_COMPACTION_FULL;
+ CloudTabletSPtr tablet =
+ DORIS_TRY(_engine.tablet_mgr().get_tablet(tablet_id, false,
sync_delete_bitmap));
if (tablet == nullptr) {
return Status::NotFound("Tablet not found. tablet_id={}", tablet_id);
}
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index 6f820a8b2a5..22b96b0d4d7 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -382,7 +382,8 @@ Status CloudMetaMgr::get_tablet_meta(int64_t tablet_id,
TabletMetaSharedPtr* tab
return Status::OK();
}
-Status CloudMetaMgr::sync_tablet_rowsets(CloudTablet* tablet, bool
warmup_delta_data) {
+Status CloudMetaMgr::sync_tablet_rowsets(CloudTablet* tablet, bool
warmup_delta_data,
+ bool sync_delete_bitmap) {
using namespace std::chrono;
TEST_SYNC_POINT_RETURN_WITH_VALUE("CloudMetaMgr::sync_tablet_rowsets",
Status::OK(), tablet);
@@ -463,7 +464,7 @@ Status CloudMetaMgr::sync_tablet_rowsets(CloudTablet*
tablet, bool warmup_delta_
// If is mow, the tablet has no delete bitmap in base rowsets.
// So dont need to sync it.
- if (tablet->enable_unique_key_merge_on_write() &&
+ if (sync_delete_bitmap && tablet->enable_unique_key_merge_on_write() &&
tablet->tablet_state() == TABLET_RUNNING) {
DeleteBitmap delete_bitmap(tablet_id);
int64_t old_max_version = req.start_version() - 1;
diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h
index d407b8b4ff1..2b955d40e85 100644
--- a/be/src/cloud/cloud_meta_mgr.h
+++ b/be/src/cloud/cloud_meta_mgr.h
@@ -57,7 +57,8 @@ public:
Status get_tablet_meta(int64_t tablet_id, std::shared_ptr<TabletMeta>*
tablet_meta);
- Status sync_tablet_rowsets(CloudTablet* tablet, bool warmup_delta_data =
false);
+ Status sync_tablet_rowsets(CloudTablet* tablet, bool warmup_delta_data =
false,
+ bool sync_delete_bitmap = true);
Status prepare_rowset(const RowsetMeta& rs_meta,
std::shared_ptr<RowsetMeta>* existed_rs_meta =
nullptr);
diff --git a/be/src/cloud/cloud_tablet_mgr.cpp
b/be/src/cloud/cloud_tablet_mgr.cpp
index 7ecb72e62fd..e7a7d254f3f 100644
--- a/be/src/cloud/cloud_tablet_mgr.cpp
+++ b/be/src/cloud/cloud_tablet_mgr.cpp
@@ -149,8 +149,8 @@ void set_tablet_access_time_ms(CloudTablet* tablet) {
tablet->last_access_time_ms = now;
}
-Result<std::shared_ptr<CloudTablet>> CloudTabletMgr::get_tablet(int64_t
tablet_id,
- bool
warmup_data) {
+Result<std::shared_ptr<CloudTablet>> CloudTabletMgr::get_tablet(int64_t
tablet_id, bool warmup_data,
+ bool
sync_delete_bitmap) {
// LRU value type. `Value`'s lifetime MUST NOT be longer than
`CloudTabletMgr`
class Value : public LRUCacheValueBase {
public:
@@ -168,8 +168,8 @@ Result<std::shared_ptr<CloudTablet>>
CloudTabletMgr::get_tablet(int64_t tablet_i
CacheKey key(tablet_id_str);
auto* handle = _cache->lookup(key);
if (handle == nullptr) {
- auto load_tablet = [this, &key,
- warmup_data](int64_t tablet_id) ->
std::shared_ptr<CloudTablet> {
+ auto load_tablet = [this, &key, warmup_data,
+ sync_delete_bitmap](int64_t tablet_id) ->
std::shared_ptr<CloudTablet> {
TabletMetaSharedPtr tablet_meta;
auto st = _engine.meta_mgr().get_tablet_meta(tablet_id,
&tablet_meta);
if (!st.ok()) {
@@ -180,7 +180,8 @@ Result<std::shared_ptr<CloudTablet>>
CloudTabletMgr::get_tablet(int64_t tablet_i
auto tablet = std::make_shared<CloudTablet>(_engine,
std::move(tablet_meta));
auto value = std::make_unique<Value>(tablet, *_tablet_map);
// MUST sync stats to let compaction scheduler work correctly
- st = _engine.meta_mgr().sync_tablet_rowsets(tablet.get(),
warmup_data);
+ st = _engine.meta_mgr().sync_tablet_rowsets(tablet.get(),
warmup_data,
+ sync_delete_bitmap);
if (!st.ok()) {
LOG(WARNING) << "failed to sync tablet " << tablet_id << ": "
<< st;
return nullptr;
diff --git a/be/src/cloud/cloud_tablet_mgr.h b/be/src/cloud/cloud_tablet_mgr.h
index 903f372cbde..cbbd119a36b 100644
--- a/be/src/cloud/cloud_tablet_mgr.h
+++ b/be/src/cloud/cloud_tablet_mgr.h
@@ -43,7 +43,8 @@ public:
// If the tablet is in cache, return this tablet directly; otherwise will
get tablet meta first,
// sync rowsets after, and download segment data in background if
`warmup_data` is true.
- Result<std::shared_ptr<CloudTablet>> get_tablet(int64_t tablet_id, bool
warmup_data = false);
+ Result<std::shared_ptr<CloudTablet>> get_tablet(int64_t tablet_id, bool
warmup_data = false,
+ bool sync_delete_bitmap =
true);
void erase_tablet(int64_t tablet_id);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
