This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 66681cfe5cf branch-3.0: [feat](cloud) Support alter operation for
obj_info and s3 vault obj_info #51162 (#51685)
66681cfe5cf is described below
commit 66681cfe5cf8244cd62a0b49f71314934cd0af3e
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Jun 19 16:36:43 2025 +0800
branch-3.0: [feat](cloud) Support alter operation for obj_info and s3 vault
obj_info #51162 (#51685)
Cherry-picked from #51162
Co-authored-by: Lei Zhang <[email protected]>
---
cloud/src/meta-service/meta_service_http.cpp | 8 +-
cloud/src/meta-service/meta_service_resource.cpp | 131 +++++++--
cloud/test/meta_service_test.cpp | 301 +++++++++++++++++++++
.../org/apache/doris/catalog/S3StorageVault.java | 6 +-
.../org/apache/doris/catalog/StorageVault.java | 6 +-
.../apache/doris/cloud/rpc/MetaServiceClient.java | 4 +
.../apache/doris/cloud/rpc/MetaServiceProxy.java | 4 +
gensrc/proto/cloud.proto | 1 +
.../test_alter_s3_vault_with_role.groovy | 126 +++++++++
.../node_mgr/test_ms_alter_obj_info.groovy | 206 ++++++++++++++
.../node_mgr/test_ms_alter_s3_vault.groovy | 237 ++++++++++++++++
11 files changed, 999 insertions(+), 31 deletions(-)
diff --git a/cloud/src/meta-service/meta_service_http.cpp
b/cloud/src/meta-service/meta_service_http.cpp
index 53d9dbbef83..d12ff59ffe0 100644
--- a/cloud/src/meta-service/meta_service_http.cpp
+++ b/cloud/src/meta-service/meta_service_http.cpp
@@ -229,7 +229,8 @@ static HttpResponse
process_get_obj_store_info(MetaServiceImpl* service, brpc::C
static HttpResponse process_alter_obj_store_info(MetaServiceImpl* service,
brpc::Controller* ctrl) {
static std::unordered_map<std::string_view,
AlterObjStoreInfoRequest::Operation> operations {
{"add_obj_info", AlterObjStoreInfoRequest::ADD_OBJ_INFO},
- {"legacy_update_ak_sk",
AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK}};
+ {"legacy_update_ak_sk",
AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK},
+ {"alter_obj_info", AlterObjStoreInfoRequest::ALTER_OBJ_INFO}};
auto& path = ctrl->http_request().unresolved_path();
auto it = operations.find(remove_version_prefix(path));
@@ -251,6 +252,7 @@ static HttpResponse
process_alter_storage_vault(MetaServiceImpl* service, brpc::
static std::unordered_map<std::string_view,
AlterObjStoreInfoRequest::Operation> operations {
{"drop_s3_vault", AlterObjStoreInfoRequest::DROP_S3_VAULT},
{"add_s3_vault", AlterObjStoreInfoRequest::ADD_S3_VAULT},
+ {"alter_s3_vault", AlterObjStoreInfoRequest::ALTER_S3_VAULT},
{"drop_hdfs_vault", AlterObjStoreInfoRequest::DROP_HDFS_INFO},
{"add_hdfs_vault", AlterObjStoreInfoRequest::ADD_HDFS_INFO}};
@@ -740,6 +742,10 @@ void
MetaServiceImpl::http(::google::protobuf::RpcController* controller,
{"alter_s3_vault", process_alter_storage_vault},
{"drop_s3_vault", process_alter_storage_vault},
{"drop_hdfs_vault", process_alter_storage_vault},
+ {"alter_obj_info", process_alter_obj_store_info},
+ {"v1/alter_obj_info", process_alter_obj_store_info},
+ {"v1/alter_s3_vault", process_alter_storage_vault},
+
// for tools
{"decode_key", process_decode_key},
{"encode_key", process_encode_key},
diff --git a/cloud/src/meta-service/meta_service_resource.cpp
b/cloud/src/meta-service/meta_service_resource.cpp
index eb88694ff57..bbd94b577b1 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -677,14 +677,6 @@ static int alter_s3_storage_vault(InstanceInfoPB&
instance, std::unique_ptr<Tran
return -1;
}
- if (obj_info.has_ak() ^ obj_info.has_sk()) {
- code = MetaServiceCode::INVALID_ARGUMENT;
- std::stringstream ss;
- ss << "Accesskey and secretkey must be alter together";
- msg = ss.str();
- return -1;
- }
-
const auto& name = vault.name();
// Here we try to get mutable iter since we might need to alter the vault
name
auto name_itr =
std::find_if(instance.mutable_storage_vault_names()->begin(),
@@ -723,6 +715,8 @@ static int alter_s3_storage_vault(InstanceInfoPB& instance,
std::unique_ptr<Tran
return -1;
}
+ auto origin_vault_info = new_vault.DebugString();
+
if (vault.has_alter_name()) {
if (!is_valid_storage_vault_name(vault.alter_name())) {
code = MetaServiceCode::INVALID_ARGUMENT;
@@ -742,13 +736,26 @@ static int alter_s3_storage_vault(InstanceInfoPB&
instance, std::unique_ptr<Tran
new_vault.set_name(vault.alter_name());
*name_itr = vault.alter_name();
}
- auto origin_vault_info = new_vault.DebugString();
- // For ak or sk is not altered.
- EncryptionInfoPB encryption_info = new_vault.obj_info().encryption_info();
- AkSkPair new_ak_sk_pair {new_vault.obj_info().ak(),
new_vault.obj_info().sk()};
+ if (obj_info.has_role_arn() && (obj_info.has_ak() || obj_info.has_sk())) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "invaild argument, both set ak/sk and role_arn is not allowed";
+ LOG(WARNING) << msg;
+ return -1;
+ }
+
+ if (obj_info.has_ak() ^ obj_info.has_sk()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "Accesskey and secretkey must be alter together";
+ msg = ss.str();
+ return -1;
+ }
if (obj_info.has_ak()) {
+ EncryptionInfoPB encryption_info =
new_vault.obj_info().encryption_info();
+ AkSkPair new_ak_sk_pair {new_vault.obj_info().ak(),
new_vault.obj_info().sk()};
+
// ak and sk must be altered together, there is check before.
auto ret = encrypt_ak_sk_helper(obj_info.ak(), obj_info.sk(),
&encryption_info,
&new_ak_sk_pair, code, msg);
@@ -758,15 +765,36 @@ static int alter_s3_storage_vault(InstanceInfoPB&
instance, std::unique_ptr<Tran
LOG(WARNING) << msg;
return -1;
}
+ new_vault.mutable_obj_info()->clear_role_arn();
+ new_vault.mutable_obj_info()->clear_external_id();
+ new_vault.mutable_obj_info()->clear_cred_provider_type();
+
+ new_vault.mutable_obj_info()->set_ak(new_ak_sk_pair.first);
+ new_vault.mutable_obj_info()->set_sk(new_ak_sk_pair.second);
+
new_vault.mutable_obj_info()->mutable_encryption_info()->CopyFrom(encryption_info);
+ }
+
+ if (obj_info.has_role_arn()) {
+ new_vault.mutable_obj_info()->clear_ak();
+ new_vault.mutable_obj_info()->clear_sk();
+ new_vault.mutable_obj_info()->clear_encryption_info();
+
+ new_vault.mutable_obj_info()->set_role_arn(obj_info.role_arn());
+
new_vault.mutable_obj_info()->set_cred_provider_type(CredProviderTypePB::INSTANCE_PROFILE);
+ if (obj_info.has_external_id()) {
+
new_vault.mutable_obj_info()->set_external_id(obj_info.external_id());
+ }
}
- new_vault.mutable_obj_info()->set_ak(new_ak_sk_pair.first);
- new_vault.mutable_obj_info()->set_sk(new_ak_sk_pair.second);
-
new_vault.mutable_obj_info()->mutable_encryption_info()->CopyFrom(encryption_info);
if (obj_info.has_use_path_style()) {
new_vault.mutable_obj_info()->set_use_path_style(obj_info.use_path_style());
}
+ auto now_time = std::chrono::system_clock::now();
+ uint64_t time =
+
std::chrono::duration_cast<std::chrono::seconds>(now_time.time_since_epoch()).count();
+ new_vault.mutable_obj_info()->set_mtime(time);
+
auto new_vault_info = new_vault.DebugString();
val = new_vault.SerializeAsString();
if (val.empty()) {
@@ -825,6 +853,7 @@ static int extract_object_storage_info(const
AlterObjStoreInfoRequest* request,
if (!obj.has_ak() || !obj.has_sk()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "s3 obj info err " + proto_to_json(*request);
+ LOG(INFO) << msg;
return -1;
}
@@ -839,13 +868,12 @@ static int extract_object_storage_info(const
AlterObjStoreInfoRequest* request,
ak = cipher_ak_sk_pair.first;
sk = cipher_ak_sk_pair.second;
} else {
- if (!obj.has_cred_provider_type() ||
- obj.cred_provider_type() != CredProviderTypePB::INSTANCE_PROFILE ||
- !obj.has_provider() || obj.provider() != ObjectStoreInfoPB::S3) {
+ if (obj.has_ak() || obj.has_sk()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 conf info err with role_arn, please check it";
+ msg = "invaild argument, both set ak/sk and role_arn is not
allowed";
return -1;
}
+
role_arn = obj.has_role_arn() ? obj.role_arn() : "";
external_id = obj.has_external_id() ? obj.external_id() : "";
}
@@ -1044,6 +1072,16 @@ void
MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
return;
}
+ if (!role_arn.empty()) {
+ if (!obj.has_cred_provider_type() ||
+ obj.cred_provider_type() !=
CredProviderTypePB::INSTANCE_PROFILE ||
+ !obj.has_provider() || obj.provider() !=
ObjectStoreInfoPB::S3) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "s3 conf info err with role_arn, please check it";
+ return;
+ }
+ }
+
auto& objs = instance.obj_info();
for (auto& it : objs) {
if (bucket == it.bucket() && prefix == it.prefix() && endpoint ==
it.endpoint() &&
@@ -1210,6 +1248,7 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
switch (request->op()) {
case AlterObjStoreInfoRequest::ADD_OBJ_INFO:
case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK:
+ case AlterObjStoreInfoRequest::ALTER_OBJ_INFO:
case AlterObjStoreInfoRequest::UPDATE_AK_SK: {
auto tmp_desc = ObjectStorageDesc {ak, sk,
bucket, prefix,
@@ -1287,7 +1326,8 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
}
switch (request->op()) {
- case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK: {
+ case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK:
+ case AlterObjStoreInfoRequest::ALTER_OBJ_INFO: {
// get id
std::string id = request->obj().has_id() ? request->obj().id() : "0";
int idx = std::stoi(id);
@@ -1301,20 +1341,55 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
const_cast<std::decay_t<decltype(instance.obj_info())>&>(instance.obj_info());
for (auto& it : obj_info) {
if (std::stoi(it.id()) == idx) {
- if (it.ak() == ak && it.sk() == sk) {
- // not change, just return ok
- code = MetaServiceCode::OK;
- msg = "";
- return;
+ if (role_arn.empty()) {
+ if (it.ak() == ak && it.sk() == sk) {
+ // not change, just return ok
+ code = MetaServiceCode::OK;
+ msg = "ak/sk not changed";
+ return;
+ }
+ it.clear_role_arn();
+ it.clear_external_id();
+ it.clear_cred_provider_type();
+
+ it.set_ak(ak);
+ it.set_sk(sk);
+ it.mutable_encryption_info()->CopyFrom(encryption_info);
+ } else {
+ if (!ak.empty() || !sk.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "invaild argument, both set ak/sk and role_arn
is not allowed";
+ LOG(INFO) << msg;
+ return;
+ }
+
+ if (it.provider() != ObjectStoreInfoPB::S3) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "role_arn is only supported for s3 provider";
+ LOG(INFO) << msg << " provider=" << it.provider();
+ return;
+ }
+
+ if (it.role_arn() == role_arn && it.external_id() ==
external_id) {
+ // not change, just return ok
+ code = MetaServiceCode::OK;
+ msg = "ak/sk not changed";
+ return;
+ }
+ it.clear_ak();
+ it.clear_sk();
+ it.clear_encryption_info();
+
+ it.set_role_arn(role_arn);
+ it.set_external_id(external_id);
+
it.set_cred_provider_type(CredProviderTypePB::INSTANCE_PROFILE);
}
+
auto now_time = std::chrono::system_clock::now();
uint64_t time =
std::chrono::duration_cast<std::chrono::seconds>(
now_time.time_since_epoch())
.count();
it.set_mtime(time);
- it.set_ak(ak);
- it.set_sk(sk);
- it.mutable_encryption_info()->CopyFrom(encryption_info);
}
}
} break;
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index b5e609be3a8..fbbfbff19fe 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -9445,4 +9445,305 @@ TEST(MetaServiceTest, StaleCommitRowset) {
ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT) <<
res.status().code();
}
+TEST(MetaServiceTest, AlterObjInfoTest) {
+ auto meta_service = get_meta_service();
+
+ auto sp = SyncPoint::get_instance();
+ sp->enable_processing();
+
+ sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) {
+ auto* ret = try_any_cast<int*>(args[0]);
+ *ret = 0;
+ auto* key = try_any_cast<std::string*>(args[1]);
+ *key = "selectdbselectdbselectdbselectdb";
+ auto* key_id = try_any_cast<int64_t*>(args[2]);
+ *key_id = 1;
+ });
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string key;
+ std::string val;
+ InstanceKeyInfo key_info {"test_instance"};
+ instance_key(key_info, &key);
+
+ ObjectStoreInfoPB obj_info;
+ obj_info.set_id("1");
+ obj_info.set_ak("access_key_132131");
+ obj_info.set_sk("secret_key_434124");
+
obj_info.set_provider(ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3);
+ InstanceInfoPB instance;
+ instance.add_obj_info()->CopyFrom(obj_info);
+ val = instance.SerializeAsString();
+ txn->put(key, val);
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+ auto get_test_instance = [&](InstanceInfoPB& i) {
+ std::string key;
+ std::string val;
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn),
TxnErrorCode::TXN_OK);
+ InstanceKeyInfo key_info {"test_instance"};
+ instance_key(key_info, &key);
+ ASSERT_EQ(txn->get(key, &val), TxnErrorCode::TXN_OK);
+ i.ParseFromString(val);
+ };
+
+ std::string cipher_sk =
"JUkuTDctR+ckJtnPkLScWaQZRcOtWBhsLLpnCRxQLxr734qB8cs6gNLH6grE1FxO";
+ std::string plain_sk = "Hx60p12123af234541nsVsffdfsdfghsdfhsdf34t";
+
+ // update failed
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_OBJ_INFO);
+ req.mutable_obj()->set_id("2");
+ req.mutable_obj()->set_ak("new_ak");
+ req.mutable_obj()->set_sk(plain_sk);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_obj_store_info(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT);
+ InstanceInfoPB instance;
+ get_test_instance(instance);
+ ASSERT_EQ(instance.obj_info(0).id(), "1");
+ ASSERT_EQ(instance.obj_info(0).ak(), "access_key_132131");
+ ASSERT_EQ(instance.obj_info(0).sk(), "secret_key_434124");
+ }
+
+ // update ak/sk successful
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_OBJ_INFO);
+ req.mutable_obj()->set_id("1");
+ req.mutable_obj()->set_ak("new_access_key_132131");
+ req.mutable_obj()->set_sk(plain_sk);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_obj_store_info(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+ InstanceInfoPB instance;
+ get_test_instance(instance);
+ LOG(INFO) << "instance:" << instance.ShortDebugString();
+ ASSERT_EQ(instance.obj_info(0).id(), "1");
+ ASSERT_EQ(instance.obj_info(0).ak(), "new_access_key_132131");
+ ASSERT_EQ(instance.obj_info(0).sk(), cipher_sk);
+ }
+
+ // update from ak/sk to role_arn
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_OBJ_INFO);
+ req.mutable_obj()->set_id("1");
+
req.mutable_obj()->set_role_arn("arn:aws:iam::1453123012:role/test-role");
+ req.mutable_obj()->set_external_id("external_id_13123");
+
req.mutable_obj()->set_cred_provider_type(CredProviderTypePB::INSTANCE_PROFILE);
+
req.mutable_obj()->set_provider(ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_obj_store_info(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+ InstanceInfoPB instance;
+ get_test_instance(instance);
+ LOG(INFO) << "instance:" << instance.ShortDebugString();
+ ASSERT_EQ(instance.obj_info(0).id(), "1");
+ ASSERT_EQ(instance.obj_info(0).role_arn(),
"arn:aws:iam::1453123012:role/test-role");
+ ASSERT_EQ(instance.obj_info(0).external_id(), "external_id_13123");
+ ASSERT_EQ(instance.obj_info(0).provider(),
+ ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3);
+ ASSERT_EQ(instance.obj_info(0).cred_provider_type(),
CredProviderTypePB::INSTANCE_PROFILE);
+ ASSERT_TRUE(instance.obj_info(0).ak().empty());
+ ASSERT_TRUE(instance.obj_info(0).sk().empty());
+ ASSERT_FALSE(instance.obj_info(0).has_encryption_info());
+ }
+
+ // update from role_arn to ak/sk
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_OBJ_INFO);
+ req.mutable_obj()->set_id("1");
+ req.mutable_obj()->set_ak("new_access_key_132131");
+ req.mutable_obj()->set_sk(plain_sk);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_obj_store_info(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+ InstanceInfoPB instance;
+ get_test_instance(instance);
+ LOG(INFO) << "instance:" << instance.ShortDebugString();
+ ASSERT_EQ(instance.obj_info(0).id(), "1");
+ ASSERT_EQ(instance.obj_info(0).ak(), "new_access_key_132131");
+ ASSERT_EQ(instance.obj_info(0).sk(), cipher_sk);
+ ASSERT_EQ(instance.obj_info(0).provider(),
+ ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3);
+ ASSERT_FALSE(instance.obj_info(0).has_cred_provider_type());
+ ASSERT_FALSE(instance.obj_info(0).has_role_arn());
+ ASSERT_FALSE(instance.obj_info(0).has_external_id());
+ }
+
+ SyncPoint::get_instance()->disable_processing();
+ SyncPoint::get_instance()->clear_all_call_backs();
+}
+
+TEST(MetaServiceTest, AlterS3StorageVaultWithRoleArnTest) {
+ auto meta_service = get_meta_service();
+
+ auto sp = SyncPoint::get_instance();
+ sp->enable_processing();
+ sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) {
+ auto* ret = try_any_cast<int*>(args[0]);
+ *ret = 0;
+ auto* key = try_any_cast<std::string*>(args[1]);
+ *key = "selectdbselectdbselectdbselectdb";
+ auto* key_id = try_any_cast<int64_t*>(args[2]);
+ *key_id = 1;
+ });
+ std::pair<std::string, std::string> pair;
+ sp->set_call_back("extract_object_storage_info:get_aksk_pair", [&](auto&&
args) {
+ auto* ret = try_any_cast<std::pair<std::string,
std::string>*>(args[0]);
+ pair = *ret;
+ });
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string key;
+ std::string val;
+ InstanceKeyInfo key_info {"test_instance"};
+ instance_key(key_info, &key);
+
+ ObjectStoreInfoPB obj_info;
+ obj_info.set_id("1");
+ obj_info.set_ak("123456ab");
+ obj_info.set_sk("@ak$");
+
obj_info.set_provider(ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3);
+ StorageVaultPB vault;
+ constexpr char vault_name[] = "test_alter_s3_vault_111";
+ vault.mutable_obj_info()->MergeFrom(obj_info);
+ vault.set_name(vault_name);
+ vault.set_id("2");
+ InstanceInfoPB instance;
+ instance.add_storage_vault_names(vault.name());
+ instance.add_resource_ids(vault.id());
+ instance.set_instance_id("GetObjStoreInfoTestInstance");
+ val = instance.SerializeAsString();
+ txn->put(key, val);
+ txn->put(storage_vault_key({instance.instance_id(), "2"}),
vault.SerializeAsString());
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ txn = nullptr;
+
+ auto get_test_instance = [&](InstanceInfoPB& i) {
+ std::string key;
+ std::string val;
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn),
TxnErrorCode::TXN_OK);
+ InstanceKeyInfo key_info {"test_instance"};
+ instance_key(key_info, &key);
+ ASSERT_EQ(txn->get(key, &val), TxnErrorCode::TXN_OK);
+ i.ParseFromString(val);
+ };
+
+ // update from ak/sk to role_arn
+ {
+ AlterObjStoreInfoRequest req;
+ constexpr char new_vault_name[] = "new_test_alter_s3_vault_111";
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT);
+ StorageVaultPB vault;
+ vault.set_alter_name(new_vault_name);
+ ObjectStoreInfoPB obj;
+ obj.set_role_arn("arn:aws:iam::12311321:role/test-alter-role");
+ obj.set_external_id("external_id_123123");
+ vault.mutable_obj_info()->MergeFrom(obj);
+ vault.set_name(vault_name);
+ req.mutable_vault()->CopyFrom(vault);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) <<
res.status().msg();
+
+ InstanceInfoPB instance;
+ get_test_instance(instance);
+ LOG(INFO) << "instance:" << instance.ShortDebugString();
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn),
TxnErrorCode::TXN_OK);
+ std::string val;
+ ASSERT_EQ(txn->get(storage_vault_key({instance.instance_id(), "2"}),
&val),
+ TxnErrorCode::TXN_OK);
+ StorageVaultPB get_obj;
+ get_obj.ParseFromString(val);
+ ASSERT_EQ(get_obj.id(), "2");
+ ASSERT_EQ(get_obj.obj_info().role_arn(),
"arn:aws:iam::12311321:role/test-alter-role");
+ ASSERT_EQ(get_obj.obj_info().external_id(), "external_id_123123");
+ ASSERT_EQ(get_obj.obj_info().provider(),
+ ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3);
+ ASSERT_EQ(get_obj.obj_info().cred_provider_type(),
CredProviderTypePB::INSTANCE_PROFILE);
+ ASSERT_TRUE(get_obj.obj_info().ak().empty());
+ ASSERT_TRUE(get_obj.obj_info().sk().empty());
+ ASSERT_FALSE(get_obj.obj_info().has_encryption_info());
+ ASSERT_EQ(get_obj.name(), new_vault_name) <<
get_obj.obj_info().ShortDebugString();
+ }
+
+ std::string cipher_sk =
"JUkuTDctR+ckJtnPkLScWaQZRcOtWBhsLLpnCRxQLxr734qB8cs6gNLH6grE1FxO";
+ std::string plain_sk = "Hx60p12123af234541nsVsffdfsdfghsdfhsdf34t";
+
+ // update from role_arn to ak_sk
+ {
+ AlterObjStoreInfoRequest req;
+ constexpr char new_vault_name[] = "new_test_alter_s3_vault_111";
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT);
+ StorageVaultPB vault;
+ ObjectStoreInfoPB obj;
+ obj.set_ak("123456ab");
+ obj.set_sk(plain_sk);
+ vault.mutable_obj_info()->MergeFrom(obj);
+ vault.set_name(new_vault_name);
+ req.mutable_vault()->CopyFrom(vault);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) <<
res.status().msg();
+
+ InstanceInfoPB instance;
+ get_test_instance(instance);
+ LOG(INFO) << "instance:" << instance.ShortDebugString();
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn),
TxnErrorCode::TXN_OK);
+ std::string val;
+ ASSERT_EQ(txn->get(storage_vault_key({instance.instance_id(), "2"}),
&val),
+ TxnErrorCode::TXN_OK);
+ StorageVaultPB get_obj;
+ get_obj.ParseFromString(val);
+ ASSERT_EQ(get_obj.id(), "2");
+ ASSERT_EQ(get_obj.obj_info().provider(),
+ ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3);
+ ASSERT_EQ(get_obj.obj_info().ak(), "123456ab");
+ ASSERT_EQ(get_obj.obj_info().sk(), cipher_sk);
+ ASSERT_TRUE(get_obj.obj_info().role_arn().empty());
+ ASSERT_TRUE(get_obj.obj_info().external_id().empty());
+ ASSERT_TRUE(get_obj.obj_info().has_encryption_info());
+ ASSERT_FALSE(get_obj.obj_info().has_cred_provider_type());
+ ASSERT_EQ(get_obj.name(), new_vault_name) <<
get_obj.obj_info().ShortDebugString();
+ }
+
+ SyncPoint::get_instance()->disable_processing();
+ SyncPoint::get_instance()->clear_all_call_backs();
+}
} // namespace doris::cloud
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
index f8196c7ea80..58097d3c1bc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
@@ -70,6 +70,8 @@ public class S3StorageVault extends StorageVault {
public static final String REGION = S3Properties.REGION;
public static final String ENDPOINT = S3Properties.ENDPOINT;
public static final String BUCKET = S3Properties.BUCKET;
+ public static final String ROLE_ARN = S3Properties.ROLE_ARN;
+ public static final String EXTERNAL_ID = S3Properties.EXTERNAL_ID;
}
public static final HashSet<String> ALLOW_ALTER_PROPERTIES = new
HashSet<>(Arrays.asList(
@@ -77,7 +79,9 @@ public class S3StorageVault extends StorageVault {
StorageVault.PropertyKey.TYPE,
PropertyKey.ACCESS_KEY,
PropertyKey.SECRET_KEY,
- PropertyKey.USE_PATH_STYLE
+ PropertyKey.USE_PATH_STYLE,
+ PropertyKey.ROLE_ARN,
+ PropertyKey.EXTERNAL_ID
));
@SerializedName(value = "properties")
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
index c9fe0c8bcc6..906b4585313 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
@@ -222,7 +222,11 @@ public abstract class StorageVault {
Cloud.ObjectStoreInfoPB.Builder builder =
Cloud.ObjectStoreInfoPB.newBuilder();
builder.mergeFrom(vault.getObjInfo());
builder.clearId();
- builder.setSk("xxxxxxx");
+
+ if (vault.getObjInfo().hasAk()) {
+ builder.setSk("xxxxxxx");
+ }
+
if (!vault.getObjInfo().hasUsePathStyle()) {
// There is no `use_path_style` field in old version, think
`use_path_style` false
builder.setUsePathStyle(false);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java
index d027777bf19..048d8ab93df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java
@@ -364,6 +364,10 @@ public class MetaServiceClient {
.alterCluster(request);
}
+ /**
+ * This method is deprecated, there is no code to call it.
+ */
+ @Deprecated
public Cloud.AlterObjStoreInfoResponse
alterObjStoreInfo(Cloud.AlterObjStoreInfoRequest request) {
if (!request.hasCloudUniqueId()) {
Cloud.AlterObjStoreInfoRequest.Builder builder =
Cloud.AlterObjStoreInfoRequest.newBuilder();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java
index 1e5aa7ed111..95753821c35 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java
@@ -385,6 +385,10 @@ public class MetaServiceProxy {
return w.executeRequest((client) ->
client.removeDeleteBitmapUpdateLock(request));
}
+ /**
+ * This method is deprecated, there is no code to call it.
+ */
+ @Deprecated
public Cloud.AlterObjStoreInfoResponse
alterObjStoreInfo(Cloud.AlterObjStoreInfoRequest request)
throws RpcException {
return w.executeRequest((client) -> client.alterObjStoreInfo(request));
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index 7b50747856c..4e5c27f6a43 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -878,6 +878,7 @@ message AlterObjStoreInfoRequest {
UPDATE_AK_SK = 1;
ADD_OBJ_INFO = 2;
LEGACY_UPDATE_AK_SK = 3;
+ ALTER_OBJ_INFO = 4;
ADD_HDFS_INFO = 100;
DROP_HDFS_INFO = 101;
diff --git
a/regression-test/suites/aws_iam_role_p0/test_alter_s3_vault_with_role.groovy
b/regression-test/suites/aws_iam_role_p0/test_alter_s3_vault_with_role.groovy
new file mode 100644
index 00000000000..990afd876f5
--- /dev/null
+++
b/regression-test/suites/aws_iam_role_p0/test_alter_s3_vault_with_role.groovy
@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import com.google.common.base.Strings;
+
+suite("test_alter_s3_vault_with_role") {
+ if (!isCloudMode()) {
+ logger.info("skip ${name} case, because not cloud mode")
+ return
+ }
+
+ if (!enableStoragevault()) {
+ logger.info("skip ${name} case, because storage vault not enabled")
+ return
+ }
+
+ def randomStr = UUID.randomUUID().toString().replace("-", "")
+ def s3VaultName = "s3_" + randomStr
+
+ def endpoint = context.config.awsEndpoint
+ def region = context.config.awsRegion
+ def bucket = context.config.awsBucket
+ def roleArn = context.config.awsRoleArn
+ def externalId = context.config.awsExternalId
+ def prefix = context.config.awsPrefix
+ def awsAccessKey = context.config.awsAccessKey
+ def awsSecretKey = context.config.awsSecretKey
+
+ sql """
+ CREATE STORAGE VAULT IF NOT EXISTS ${s3VaultName}
+ PROPERTIES (
+ "type"="S3",
+ "s3.endpoint"="${endpoint}",
+ "s3.region" = "${region}",
+ "s3.role_arn" = "${roleArn}",
+ "s3.external_id" = "${externalId}",
+ "s3.root.path" = "${prefix}/aws_iam_role_p0/${s3VaultName}",
+ "s3.bucket" = "${bucket}",
+ "s3.external_endpoint" = "",
+ "provider" = "S3",
+ "use_path_style" = "false"
+ );
+ """
+
+ sql """
+ CREATE TABLE ${s3VaultName} (
+ C_CUSTKEY INTEGER NOT NULL,
+ C_NAME INTEGER NOT NULL
+ )
+ DUPLICATE KEY(C_CUSTKEY, C_NAME)
+ DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "storage_vault_name" = ${s3VaultName}
+ )
+ """
+ sql """ insert into ${s3VaultName} values(1, 1); """
+ sql """ sync;"""
+ def result = sql """ select * from ${s3VaultName}; """
+ assertEquals(result.size(), 1);
+
+ sql """
+ ALTER STORAGE VAULT ${s3VaultName}
+ PROPERTIES (
+ "type"="S3",
+ "s3.access_key" = "${awsAccessKey}",
+ "s3.secret_key" = "${awsSecretKey}"
+ );
+ """
+
+ def vaultInfos = sql """SHOW STORAGE VAULTS;"""
+
+ for (int i = 0; i < vaultInfos.size(); i++) {
+ logger.info("vault info: ${vaultInfos[i]}")
+ if (vaultInfos[i][0].equals(s3VaultName)) {
+ def newProperties = vaultInfos[i][2]
+ logger.info("newProperties: ${newProperties}")
+ assertTrue(newProperties.contains(awsAccessKey))
+ assertFalse(newProperties.contains("role_arn"))
+ }
+ }
+
+ sql """ insert into ${s3VaultName} values(2, 2); """
+ sql """ sync;"""
+ result = sql """ select * from ${s3VaultName}; """
+ assertEquals(result.size(), 2);
+
+ sql """
+ ALTER STORAGE VAULT ${s3VaultName}
+ PROPERTIES (
+ "type"="S3",
+ "s3.role_arn" = "${roleArn}",
+ "s3.external_id" = "${externalId}"
+ );
+ """
+
+ vaultInfos = sql """SHOW STORAGE VAULTS;"""
+ for (int i = 0; i < vaultInfos.size(); i++) {
+ logger.info("vault info: ${vaultInfos[i]}")
+ if (vaultInfos[i][0].equals(s3VaultName)) {
+ def newProperties = vaultInfos[i][2]
+ logger.info("newProperties: ${newProperties}")
+ assertFalse(newProperties.contains(awsAccessKey))
+ assertTrue(newProperties.contains(roleArn))
+ }
+ }
+
+ sql """ insert into ${s3VaultName} values(3, 3); """
+ sql """ sync;"""
+ result = sql """ select * from ${s3VaultName}; """
+ assertEquals(result.size(), 3);
+}
\ No newline at end of file
diff --git
a/regression-test/suites/cloud_p0/node_mgr/test_ms_alter_obj_info.groovy
b/regression-test/suites/cloud_p0/node_mgr/test_ms_alter_obj_info.groovy
new file mode 100644
index 00000000000..b864db1838e
--- /dev/null
+++ b/regression-test/suites/cloud_p0/node_mgr/test_ms_alter_obj_info.groovy
@@ -0,0 +1,206 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import groovy.json.JsonOutput
+import org.apache.doris.regression.suite.ClusterOptions
+
+suite('test_ms_alter_obj_info', 'p0, docker') {
+ if (!isCloudMode()) {
+ return;
+ }
+
+ def options = new ClusterOptions()
+ options.feConfigs += [
+ 'cloud_cluster_check_interval_second=1',
+ 'sys_log_verbose_modules=org',
+ 'heartbeat_interval_second=1'
+ ]
+ options.setFeNum(1)
+ options.setBeNum(1)
+ options.cloudMode = true
+
+ def create_instance_api = { msHttpPort, request_body, check_func ->
+ httpTest {
+ endpoint msHttpPort
+ uri "/MetaService/http/create_instance?token=$token"
+ body request_body
+ check check_func
+ }
+ }
+
+
+ def get_instance_api = { msHttpPort, instance_id, check_func ->
+ httpTest {
+ op "get"
+ endpoint msHttpPort
+ uri
"/MetaService/http/get_instance?token=${token}&instance_id=${instance_id}"
+ check check_func
+ }
+ }
+
+ def alter_obj_info_api = { msHttpPort, request_body, check_func ->
+ httpTest {
+ endpoint msHttpPort
+ uri "/MetaService/http/alter_obj_info?token=$token"
+ body request_body
+ check check_func
+ }
+ }
+
+ docker(options) {
+ def ms = cluster.getAllMetaservices().get(0)
+ def msHttpPort = ms.host + ":" + ms.httpPort
+ logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host,
ms.httpPort, msHttpPort)
+
+ // Inventory function test
+ def token = "greedisgood9999"
+ def instance_id = "instance_id_test_in_docker"
+ def name = "user_1"
+ def user_id = "10000"
+
+ def cloudUniqueId = "1:${instance_id}:xxxxx"
+ // create instance
+ /*
+ curl -X GET
'127.0.0.1:5000/MetaService/http/create_instance?token=greedisgood9999' -d '{
+ "instance_id": "instance_id_deadbeef",
+ "name": "user_1",
+ "user_id": "10000",
+ "obj_info": {
+ "ak": "test-ak1",
+ "sk": "test-sk1",
+ "bucket": "test-bucket",
+ "prefix": "test-prefix",
+ "endpoint": "test-endpoint",
+ "region": "test-region",
+ "provider" : S3",
+ "external_endpoint" : "endpoint"
+ }
+ }'
+ */
+ def jsonOutput = new JsonOutput()
+ def s3 = [
+ ak: "test-ak1",
+ sk : "test-sk1",
+ bucket : "test-bucket",
+ prefix: "test-prefix",
+ endpoint: "test-endpoint",
+ region: "test-region",
+ provider : "S3",
+ external_endpoint: "test-external-endpoint"
+ ]
+ def map = [instance_id: "${instance_id}", name: "${name}", user_id:
"${user_id}", obj_info: s3]
+ def instance_body = jsonOutput.toJson(map)
+
+ create_instance_api.call(msHttpPort, instance_body) {
+ respCode, body ->
+ log.info("http cli result: ${body} ${respCode}".toString())
+ def json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+ def json
+ get_instance_api.call(msHttpPort, instance_id) {
+ respCode, body ->
+ log.info("get instance resp: ${body} ${respCode}".toString())
+ json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+ // alter s3 info to instance
+ /*
+ curl
'127.0.0.1:5000/MetaService/http/add_obj_info?token=greedisgood9999' -d '{
+ "cloud_unique_id": "cloud_unique_id_compute_node0",
+ "obj": {
+ "id": `1`,
+ "ak": "test-ak2",
+ "sk": "test-sk2"
+ }
+ }'
+ */
+
+ def alter_obj_info_api_body = [cloud_unique_id:"${cloudUniqueId}",
+ obj:[id:"1", ak:"new-ak2", sk:"new-sk2"]]
+ jsonOutput = new JsonOutput()
+ def alterObjInfoBody = jsonOutput.toJson(alter_obj_info_api_body)
+ logger.info("alter obj info body: ${alterObjInfoBody}")
+
+ alter_obj_info_api.call(msHttpPort, alterObjInfoBody) {
+ respCode, body ->
+ log.info("http cli result: ${body} ${respCode}".toString())
+ json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+ get_instance_api.call(msHttpPort, instance_id) {
+ respCode, body ->
+ log.info("get instance resp: ${body} ${respCode}".toString())
+ json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+ assertTrue(json.result.obj_info[0]["ak"].equalsIgnoreCase("new-ak2"))
+ assertTrue(json.result.obj_info[0]["sk"].equalsIgnoreCase("new-sk2"))
+
+
+ alter_obj_info_api_body = [cloud_unique_id:"${cloudUniqueId}",
+ obj:[id:"1", role_arn:"new-role-arn",
external_id:"new-external-id"]]
+ jsonOutput = new JsonOutput()
+ alterObjInfoBody = jsonOutput.toJson(alter_obj_info_api_body)
+ logger.info("alter obj info body: ${alterObjInfoBody}")
+
+ alter_obj_info_api.call(msHttpPort, alterObjInfoBody) {
+ respCode, body ->
+ log.info("http cli result: ${body} ${respCode}".toString())
+ json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+ get_instance_api.call(msHttpPort, instance_id) {
+ respCode, body ->
+ log.info("get instance resp: ${body} ${respCode}".toString())
+ json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+
assertTrue(json.result.obj_info[0]["role_arn"].equalsIgnoreCase("new-role-arn"))
+
assertTrue(json.result.obj_info[0]["external_id"].equalsIgnoreCase("new-external-id"))
+
assertTrue(json.result.obj_info[0]["cred_provider_type"].equalsIgnoreCase("INSTANCE_PROFILE"))
+
+ alter_obj_info_api_body = [cloud_unique_id:"${cloudUniqueId}",
+ obj:[id:"1", ak:"new-ak3", sk:"new-sk3"]]
+ jsonOutput = new JsonOutput()
+ alterObjInfoBody = jsonOutput.toJson(alter_obj_info_api_body)
+ logger.info("alter obj info body: ${alterObjInfoBody}")
+
+ alter_obj_info_api.call(msHttpPort, alterObjInfoBody) {
+ respCode, body ->
+ log.info("http cli result: ${body} ${respCode}".toString())
+ json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+ get_instance_api.call(msHttpPort, instance_id) {
+ respCode, body ->
+ log.info("get instance resp: ${body} ${respCode}".toString())
+ json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+ assertTrue(json.result.obj_info[0]["ak"].equalsIgnoreCase("new-ak3"))
+ assertTrue(json.result.obj_info[0]["sk"].equalsIgnoreCase("new-sk3"))
+ }
+
+}
\ No newline at end of file
diff --git
a/regression-test/suites/cloud_p0/node_mgr/test_ms_alter_s3_vault.groovy
b/regression-test/suites/cloud_p0/node_mgr/test_ms_alter_s3_vault.groovy
new file mode 100644
index 00000000000..c837ecf90c6
--- /dev/null
+++ b/regression-test/suites/cloud_p0/node_mgr/test_ms_alter_s3_vault.groovy
@@ -0,0 +1,237 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import groovy.json.JsonOutput
+import org.apache.doris.regression.suite.ClusterOptions
+
+suite('test_ms_alter_s3_vault', 'p0, docker') {
+ if (!isCloudMode()) {
+ return;
+ }
+
+ def options = new ClusterOptions()
+ options.feConfigs += [
+ 'cloud_cluster_check_interval_second=1',
+ 'sys_log_verbose_modules=org',
+ 'heartbeat_interval_second=1'
+ ]
+ options.setFeNum(1)
+ options.setBeNum(1)
+ options.cloudMode = true
+
+ def create_instance_api = { msHttpPort, request_body, check_func ->
+ httpTest {
+ endpoint msHttpPort
+ uri "/MetaService/http/create_instance?token=$token"
+ body request_body
+ check check_func
+ }
+ }
+
+
+ def get_instance_api = { msHttpPort, instance_id, check_func ->
+ httpTest {
+ op "get"
+ endpoint msHttpPort
+ uri
"/MetaService/http/get_instance?token=${token}&instance_id=${instance_id}"
+ check check_func
+ }
+ }
+
+ def show_storage_vaults_api = { msHttpPort, request_body, check_func ->
+ httpTest {
+ endpoint msHttpPort
+ uri "/MetaService/http/show_storage_vaults?token=${token}"
+ body request_body
+ check check_func
+ }
+ }
+
+ def alter_s3_vault_api = { msHttpPort, request_body, check_func ->
+ httpTest {
+ endpoint msHttpPort
+ uri "/MetaService/http/alter_s3_vault?token=$token"
+ body request_body
+ check check_func
+ }
+ }
+
+ docker(options) {
+ def ms = cluster.getAllMetaservices().get(0)
+ def msHttpPort = ms.host + ":" + ms.httpPort
+ logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host,
ms.httpPort, msHttpPort)
+
+ // Inventory function test
+ def token = "greedisgood9999"
+ def instance_id = "instance_id_test_in_docker"
+ def name = "user_1"
+ def user_id = "10000"
+
+ def cloudUniqueId = "1:${instance_id}:xxxxx"
+ // create instance
+ /*
+ curl -X GET
'127.0.0.1:5000/MetaService/http/create_instance?token=greedisgood9999' -d '{
+ "instance_id": "instance_id_deadbeef",
+ "name": "user_1",
+ "user_id": "10000",
+ "vault": {
+ "obj_info": {
+ "ak": "test-ak1",
+ "sk": "test-sk1",
+ "bucket": "test-bucket",
+ "prefix": "test-prefix",
+ "endpoint": "test-endpoint",
+ "region": "test-region",
+ "provider" : S3",
+ "external_endpoint" : "endpoint"
+ }
+ }
+ }'
+ */
+ def jsonOutput = new JsonOutput()
+ def s3 = [
+ ak: "test-ak1",
+ sk : "test-sk1",
+ bucket : "test-bucket",
+ prefix: "test-prefix",
+ endpoint: "test-endpoint",
+ region: "test-region",
+ provider : "S3",
+ external_endpoint: "test-external-endpoint"
+ ]
+ def map = [instance_id: "${instance_id}", name: "${name}", user_id:
"${user_id}", vault: [obj_info: s3]]
+ def instance_body = jsonOutput.toJson(map)
+ logger.info("instance_body: ${instance_body}")
+
+ create_instance_api.call(msHttpPort, instance_body) {
+ respCode, body ->
+ log.info("http cli result: ${body} ${respCode}".toString())
+ def json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+ def json
+ get_instance_api.call(msHttpPort, instance_id) {
+ respCode, body ->
+ log.info("get instance resp: ${body} ${respCode}".toString())
+ json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+ // show stoarge vaults
+ /*
+
+ curl
http://127.0.0.1:5000/MetaService/http/show_storage_vaults?token=greedisgood9999
-d '{
+ "cloud_unique_id":"cloud_unique_id_compute_node0"
+ }'
+ */
+
+ def show_storage_vaults_api_body = [cloud_unique_id:"${cloudUniqueId}"]
+ show_storage_vaults_api.call(msHttpPort,
jsonOutput.toJson(show_storage_vaults_api_body)) {
+ respCode, body ->
+ log.info("show_storage_vaults_api resp: ${body}
${respCode}".toString())
+ json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+ def mtime = json.result.storage_vault[0]["obj_info"]["mtime"]
+ def ctime = json.result.storage_vault[0]["obj_info"]["ctime"]
+
+ // alter s3 vaults
+ /*
+ curl
http://127.0.0.1:5000/MetaService/http/v1/alter_s3_vault?token=greedisgood9999
-d '{
+ "cloud_unique_id":"cloud_unique_id_compute_node0",
+ "vault": {
+ "name": "built_in_storage_vault",
+ "obj_info": {
+ "ak": "test-ak2",
+ "sk": "test-sk2"
+ }
+ }
+ }'
+
+ curl
http://127.0.0.1:5000/MetaService/http/v1/alter_s3_vault?token=greedisgood9999
-d '{
+ "cloud_unique_id":"cloud_unique_id_compute_node0",
+ "vault": {
+ "name": "built_in_storage_vault",
+ "obj_info": {
+ "role_arn": "test-role-arn",
+ "external_id": "test-external-id"
+ }
+ }
+ }'
+ */
+ sleep(2000)
+ def alter_s3_vault_body = [cloud_unique_id:"${cloudUniqueId}",
+ vault:[
+ name:"built_in_storage_vault",
+ obj_info:[ak:"test-ak2", sk:"test-sk2"]
+ ]]
+
+ alter_s3_vault_api.call(msHttpPort,
jsonOutput.toJson(alter_s3_vault_body)) {
+ respCode, body ->
+ log.info("alter_s3_vault_api resp: ${body}
${respCode}".toString())
+ json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+ show_storage_vaults_api.call(msHttpPort,
jsonOutput.toJson(show_storage_vaults_api_body)) {
+ respCode, body ->
+ log.info("show_storage_vaults_api resp: ${body}
${respCode}".toString())
+ json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+
assertTrue(json.result.storage_vault[0]["obj_info"]["ak"].equalsIgnoreCase("test-ak2"))
+
assertTrue(json.result.storage_vault[0]["obj_info"]["sk"].equalsIgnoreCase("test-sk2"))
+
+ def mtime2 = json.result.storage_vault[0]["obj_info"]["mtime"]
+ def ctime2 = json.result.storage_vault[0]["obj_info"]["ctime"]
+ assertTrue(mtime2 > mtime)
+ assertTrue(ctime2 == ctime)
+
+
+ sleep(2000)
+ alter_s3_vault_body = [cloud_unique_id:"${cloudUniqueId}",
+ vault:[
+ name:"built_in_storage_vault",
+ obj_info:[role_arn:"test-role-arn",
external_id:"test-external-id"]
+ ]]
+
+ alter_s3_vault_api.call(msHttpPort,
jsonOutput.toJson(alter_s3_vault_body)) {
+ respCode, body ->
+ log.info("alter_s3_vault_api resp: ${body}
${respCode}".toString())
+ json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+ show_storage_vaults_api.call(msHttpPort,
jsonOutput.toJson(show_storage_vaults_api_body)) {
+ respCode, body ->
+ log.info("show_storage_vaults_api resp: ${body}
${respCode}".toString())
+ json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+
+
assertTrue(json.result.storage_vault[0]["obj_info"]["role_arn"].equalsIgnoreCase("test-role-arn"))
+
assertTrue(json.result.storage_vault[0]["obj_info"]["external_id"].equalsIgnoreCase("test-external-id"))
+
assertTrue(json.result.storage_vault[0]["obj_info"]["cred_provider_type"].equalsIgnoreCase("INSTANCE_PROFILE"))
+
+ def mtime3 = json.result.storage_vault[0]["obj_info"]["mtime"]
+ def ctime3 = json.result.storage_vault[0]["obj_info"]["ctime"]
+ assertTrue(mtime3 > mtime)
+ assertTrue(ctime3 == ctime)
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
