This is an automated email from the ASF dual-hosted git repository.
liaoxin01 pushed a commit to branch branch-4.1.2-recycler
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1.2-recycler by this
push:
new e82d62029c3 [fix](cloud) Add skip-delete accessor for blacklisted
vaults (#68951)
e82d62029c3 is described below
commit e82d62029c3d942f006f05569c7f7f18bd9b3bb3
Author: Yixuan Wang <[email protected]>
AuthorDate: Mon Oct 12 18:24:22 2026 +0800
[fix](cloud) Add skip-delete accessor for blacklisted vaults (#68951)
Create a SkipDeleteAccessor for vaults in
recycler_storage_vault_black_list. It records delete requests in logs
and bvar without deleting objects, while recycler callers continue using
the existing accessor flow.
---
cloud/src/common/config.h | 7 ++-
cloud/src/recycler/recycler.cpp | 9 +++
cloud/src/recycler/skip_delete_accessor.cpp | 91 +++++++++++++++++++++++++++++
cloud/src/recycler/skip_delete_accessor.h | 54 +++++++++++++++++
cloud/test/recycler_test.cpp | 18 ++++++
5 files changed, 177 insertions(+), 2 deletions(-)
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 5991938c938..251cd68603c 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -371,9 +371,12 @@ CONF_mString(delete_bitmap_lock_v2_white_list, "");
// FOR DEBUGGING
CONF_mBool(use_delete_bitmap_lock_random_version, "false");
-// Which vaults should be recycled. If empty, recycle all vaults.
-// Comma seprated list: recycler_storage_vault_white_list="aaa,bbb,ccc"
+// Which vaults should be recycled. If empty, recycle all vaults except
blacklisted vaults.
+// Comma separated list: recycler_storage_vault_white_list="aaa,bbb,ccc"
CONF_Strings(recycler_storage_vault_white_list, "");
+// Vaults whose objects must not be deleted by the recycler.
+// Comma separated list of vault names:
recycler_storage_vault_black_list="aaa,bbb,ccc"
+CONF_Strings(recycler_storage_vault_black_list, "");
// for test only
CONF_mBool(enable_update_delete_bitmap_kv_check, "false");
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index 4325c16eb52..a05ae89556f 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -60,6 +60,7 @@
#include "recycler/hdfs_accessor.h"
#endif
#include "recycler/s3_accessor.h"
+#include "recycler/skip_delete_accessor.h"
#include "recycler/storage_vault_accessor.h"
#ifdef UNIT_TEST
#include "../test/mock_accessor.h"
@@ -654,6 +655,14 @@ int InstanceRecycler::init_storage_vault_accessors() {
LOG(WARNING) << "malformed storage vault, unable to deserialize
key=" << hex(k);
return -1;
}
+ if (std::find(config::recycler_storage_vault_black_list.begin(),
+ config::recycler_storage_vault_black_list.end(),
+ vault.name()) !=
config::recycler_storage_vault_black_list.end()) {
+ accessor_map_.insert_or_assign(
+ vault.id(),
+ std::make_shared<SkipDeleteAccessor>(instance_id_,
vault.id(), vault.name()));
+ continue;
+ }
std::string recycler_storage_vault_white_list = accumulate(
config::recycler_storage_vault_white_list.begin(),
config::recycler_storage_vault_white_list.end(), std::string(),
diff --git a/cloud/src/recycler/skip_delete_accessor.cpp
b/cloud/src/recycler/skip_delete_accessor.cpp
new file mode 100644
index 00000000000..02052d091c6
--- /dev/null
+++ b/cloud/src/recycler/skip_delete_accessor.cpp
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "recycler/skip_delete_accessor.h"
+
+#include <utility>
+
+#include "common/bvars.h"
+#include "common/logging.h"
+
+namespace doris::cloud {
+
+SkipDeleteAccessor::SkipDeleteAccessor(std::string instance_id, std::string
resource_id,
+ std::string vault_name)
+ : StorageVaultAccessor(AccessorType::S3),
+ instance_id_(std::move(instance_id)),
+ resource_id_(std::move(resource_id)),
+ vault_name_(std::move(vault_name)) {
+ uri_ = "skip-delete://" + resource_id_;
+}
+
+void SkipDeleteAccessor::record_skip(const char* operation, const std::string&
path) const {
+ LOG_INFO("skip deleting from recycler blacklisted storage vault")
+ .tag("instance_id", instance_id_)
+ .tag("resource_id", resource_id_)
+ .tag("vault_name", vault_name_)
+ .tag("operation", operation)
+ .tag("path", path);
+}
+
+int SkipDeleteAccessor::delete_prefix(const std::string& path_prefix, int64_t)
{
+ record_skip("delete_prefix", path_prefix);
+ return 0;
+}
+
+int SkipDeleteAccessor::delete_directory(const std::string& dir_path) {
+ record_skip("delete_directory", dir_path);
+ return 0;
+}
+
+int SkipDeleteAccessor::delete_all(int64_t) {
+ record_skip("delete_all", "*");
+ return 0;
+}
+
+int SkipDeleteAccessor::delete_files(const std::vector<std::string>& paths) {
+ record_skip("delete_files", paths.empty() ? "" : paths.front());
+ return 0;
+}
+
+int SkipDeleteAccessor::delete_file(const std::string& path) {
+ record_skip("delete_file", path);
+ return 0;
+}
+
+int SkipDeleteAccessor::abort_multipart_upload(const std::string& path, const
std::string&) {
+ record_skip("abort_multipart_upload", path);
+ return 0;
+}
+
+int SkipDeleteAccessor::list_directory(const std::string&,
std::unique_ptr<ListIterator>*) {
+ return -1;
+}
+
+int SkipDeleteAccessor::list_all(std::unique_ptr<ListIterator>*) {
+ return -1;
+}
+
+int SkipDeleteAccessor::put_file(const std::string&, const std::string&) {
+ return -1;
+}
+
+int SkipDeleteAccessor::exists(const std::string&) {
+ return -1;
+}
+
+} // namespace doris::cloud
diff --git a/cloud/src/recycler/skip_delete_accessor.h
b/cloud/src/recycler/skip_delete_accessor.h
new file mode 100644
index 00000000000..5752f82b166
--- /dev/null
+++ b/cloud/src/recycler/skip_delete_accessor.h
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "recycler/storage_vault_accessor.h"
+
+namespace doris::cloud {
+
+// Records recycler delete requests without touching a blacklisted storage
vault.
+class SkipDeleteAccessor final : public StorageVaultAccessor {
+public:
+ SkipDeleteAccessor(std::string instance_id, std::string resource_id,
std::string vault_name);
+
+ int delete_prefix(const std::string& path_prefix, int64_t expiration_time
= 0) override;
+ int delete_directory(const std::string& dir_path) override;
+ int delete_all(int64_t expiration_time = 0) override;
+ int delete_files(const std::vector<std::string>& paths) override;
+ int delete_file(const std::string& path) override;
+ int abort_multipart_upload(const std::string& path, const std::string&
upload_id) override;
+
+ int list_directory(const std::string& dir_path,
std::unique_ptr<ListIterator>* res) override;
+ int list_all(std::unique_ptr<ListIterator>* res) override;
+ int put_file(const std::string& path, const std::string& content) override;
+ int exists(const std::string& path) override;
+
+private:
+ void record_skip(const char* operation, const std::string& path) const;
+
+ std::string instance_id_;
+ std::string resource_id_;
+ std::string vault_name_;
+};
+
+} // namespace doris::cloud
diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp
index 856347af27d..06f492375e3 100644
--- a/cloud/test/recycler_test.cpp
+++ b/cloud/test/recycler_test.cpp
@@ -6968,7 +6968,11 @@ TEST(RecyclerTest, init_all_vault_accessors_failed_test)
{
TEST(RecyclerTest, recycler_storage_vault_white_list_test) {
auto* sp = SyncPoint::get_instance();
+ auto old_white_list = config::recycler_storage_vault_white_list;
+ auto old_black_list = config::recycler_storage_vault_black_list;
DORIS_CLOUD_DEFER {
+ config::recycler_storage_vault_white_list = old_white_list;
+ config::recycler_storage_vault_black_list = old_black_list;
sp->clear_all_call_backs();
sp->clear_trace();
sp->disable_processing();
@@ -7072,6 +7076,7 @@ TEST(RecyclerTest,
recycler_storage_vault_white_list_test) {
EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
EXPECT_EQ(accessor->exists("data/0/test.csv"), 0);
+ config::recycler_storage_vault_black_list = {};
{
config::recycler_storage_vault_white_list = {};
@@ -7098,6 +7103,19 @@ TEST(RecyclerTest,
recycler_storage_vault_white_list_test) {
EXPECT_EQ(recycler.accessor_map_.at("s3_1")->exists("data/0/test.csv"), 0);
EXPECT_EQ(recycler.accessor_map_.at("hdfs_1")->exists("data/0/test.csv"), 0);
}
+
+ {
+ config::recycler_storage_vault_white_list = {"s3_1", "s3_2"};
+ config::recycler_storage_vault_black_list = {"s3_2"};
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
+ EXPECT_EQ(recycler.init(), 0);
+ EXPECT_EQ(recycler.accessor_map_.size(), 2);
+ auto skip_accessor = recycler.accessor_map_.at("s3_2");
+ EXPECT_EQ(skip_accessor->type(), AccessorType::S3);
+ EXPECT_EQ(skip_accessor->delete_file("data/0/test.csv"), 0);
+ EXPECT_EQ(accessor->exists("data/0/test.csv"), 0);
+ }
}
TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v1) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
