This is an automated email from the ASF dual-hosted git repository.
sollhui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7b0d4ffa079 [feat](s3) support S3 storage vault credentials provider
without role ARN (#64766)
7b0d4ffa079 is described below
commit 7b0d4ffa079616bf00057b13bd4187088b234e27
Author: hui lai <[email protected]>
AuthorDate: Wed Jun 24 14:13:08 2026 +0800
[feat](s3) support S3 storage vault credentials provider without role ARN
(#64766)
### What problem does this PR solve?
S3 storage vault creation only treated role ARN as the
credential-provider path. When users configured
`s3.credentials_provider_type` without `s3.role_arn`, FE did not persist
the provider type into `ObjectStoreInfoPB`, and Cloud meta-service still
required AK/SK for the vault. The recycler also only read credential
provider type inside the role ARN branch.
This change allows S3 storage vaults to use an explicit credentials
provider type without role ARN. FE now writes `cred_provider_type` when
`s3.credentials_provider_type` or `AWS_CREDENTIALS_PROVIDER_TYPE` is
set, Cloud meta-service accepts credential-provider-based S3 vaults
without AK/SK, and the recycler reads the provider type independently
from role ARN.
---
cloud/src/meta-service/meta_service_resource.cpp | 29 +++++++++-----
cloud/src/recycler/s3_accessor.cpp | 9 +++--
cloud/test/meta_service_test.cpp | 46 ++++++++++++++++++++++
.../datasource/property/storage/S3Properties.java | 20 ++++++++--
.../property/storage/S3PropertiesTest.java | 19 +++++++++
5 files changed, 107 insertions(+), 16 deletions(-)
diff --git a/cloud/src/meta-service/meta_service_resource.cpp
b/cloud/src/meta-service/meta_service_resource.cpp
index ed36b54322c..2e19ebbb8a3 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -760,6 +760,14 @@ static int add_hdfs_storage_vault(InstanceInfoPB&
instance, Transaction* txn,
return 0;
}
+static bool has_non_empty_role_arn(const ObjectStoreInfoPB& obj) {
+ return obj.has_role_arn() && !obj.role_arn().empty();
+}
+
+static bool use_credential_provider(const ObjectStoreInfoPB& obj) {
+ return obj.has_cred_provider_type() || has_non_empty_role_arn(obj);
+}
+
static void create_object_info_with_encrypt(const InstanceInfoPB& instance,
ObjectStoreInfoPB* obj,
bool sse_enabled, MetaServiceCode&
code,
std::string& msg) {
@@ -1250,7 +1258,9 @@ static int extract_object_storage_info(const
AlterObjStoreInfoRequest* request,
auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region,
use_path_style, role_arn,
external_id] = obj_desc;
- if (!obj.has_role_arn()) {
+ bool use_credential_provider_for_add_vault =
+ request->op() == AlterObjStoreInfoRequest::ADD_S3_VAULT &&
obj.has_cred_provider_type();
+ if (!obj.has_role_arn() && !use_credential_provider_for_add_vault) {
if (!obj.has_ak() || !obj.has_sk()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "s3 obj info err " + proto_to_json(*request);
@@ -1306,13 +1316,15 @@ static ObjectStoreInfoPB
object_info_pb_factory(ObjectStorageDesc& obj_desc,
last_item.set_user_id(obj.user_id());
}
- if (!obj.has_role_arn()) {
+ if (!use_credential_provider(obj)) {
last_item.set_ak(std::move(cipher_ak_sk_pair.first));
last_item.set_sk(std::move(cipher_ak_sk_pair.second));
last_item.mutable_encryption_info()->CopyFrom(encryption_info);
} else {
- last_item.set_role_arn(role_arn);
- last_item.set_external_id(external_id);
+ if (has_non_empty_role_arn(obj)) {
+ last_item.set_role_arn(role_arn);
+ last_item.set_external_id(external_id);
+ }
last_item.set_cred_provider_type(get_cred_provider_type(obj));
}
last_item.set_bucket(bucket);
@@ -1476,18 +1488,17 @@ void
MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
return;
}
// ATTN: prefix may be empty
- if (((ak.empty() || sk.empty()) && role_arn.empty()) || bucket.empty()
||
+ if (((ak.empty() || sk.empty()) && !use_credential_provider(obj)) ||
bucket.empty() ||
endpoint.empty() || region.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "s3 conf info err, please check it";
return;
}
- if (!role_arn.empty()) {
- if (!obj.has_cred_provider_type() || !obj.has_provider() ||
- obj.provider() != ObjectStoreInfoPB::S3) {
+ if (use_credential_provider(obj)) {
+ if (!obj.has_provider() || obj.provider() !=
ObjectStoreInfoPB::S3) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 conf info err with role_arn, please check it";
+ msg = "s3 conf info err with credentials_provider_type, please
check it";
return;
}
}
diff --git a/cloud/src/recycler/s3_accessor.cpp
b/cloud/src/recycler/s3_accessor.cpp
index d48c51c8469..51cc9023904 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -237,13 +237,14 @@ std::optional<S3Conf> S3Conf::from_obj_store_info(const
ObjectStoreInfoPB& obj_i
}
}
+ if (obj_info.has_cred_provider_type()) {
+ s3_conf.cred_provider_type =
cred_provider_type_from_pb(obj_info.cred_provider_type());
+ }
+
if (obj_info.has_role_arn() && !obj_info.role_arn().empty()) {
s3_conf.role_arn = obj_info.role_arn();
s3_conf.external_id = obj_info.external_id();
- if (obj_info.has_cred_provider_type()) {
- s3_conf.cred_provider_type =
-
cred_provider_type_from_pb(obj_info.cred_provider_type());
- } else {
+ if (!obj_info.has_cred_provider_type()) {
s3_conf.cred_provider_type = CredProviderType::InstanceProfile;
}
}
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index 8b17a385659..98011dd9fe0 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -10498,6 +10498,52 @@ TEST(MetaServiceTest, CreateS3VaultWithIamRole) {
}
}
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ADD_S3_VAULT);
+ StorageVaultPB vault;
+ vault.mutable_obj_info()->set_endpoint("s3.us-east-1.amazonaws.com");
+ vault.mutable_obj_info()->set_region("us-east-1");
+
vault.mutable_obj_info()->set_bucket("test_credential_provider_bucket");
+
vault.mutable_obj_info()->set_prefix("test_credential_provider_prefix");
+ vault.mutable_obj_info()->set_provider(
+ ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3);
+
vault.mutable_obj_info()->set_cred_provider_type(CredProviderTypePB::CONTAINER);
+
+ vault.set_name("s3_vault_with_credential_provider");
+ req.mutable_vault()->CopyFrom(vault);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+
+ {
+ InstanceInfoPB instance;
+ get_test_instance(instance);
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn),
TxnErrorCode::TXN_OK);
+ std::string val;
+ ASSERT_EQ(txn->get(storage_vault_key({instance.instance_id(),
"5"}), &val),
+ TxnErrorCode::TXN_OK);
+ StorageVaultPB get_obj;
+ get_obj.ParseFromString(val);
+ ASSERT_TRUE(get_obj.obj_info().ak().empty()) <<
get_obj.obj_info().ak();
+ ASSERT_TRUE(get_obj.obj_info().sk().empty()) <<
get_obj.obj_info().sk();
+ ASSERT_FALSE(get_obj.obj_info().has_role_arn());
+ ASSERT_FALSE(get_obj.obj_info().has_external_id());
+ ASSERT_EQ(get_obj.obj_info().cred_provider_type(),
CredProviderTypePB::CONTAINER)
+ << get_obj.obj_info().cred_provider_type();
+ ASSERT_EQ(get_obj.obj_info().bucket(),
"test_credential_provider_bucket")
+ << get_obj.obj_info().bucket();
+ ASSERT_EQ(get_obj.obj_info().prefix(),
"test_credential_provider_prefix")
+ << get_obj.obj_info().prefix();
+ ASSERT_EQ(get_obj.id(), "5") << get_obj.id();
+ }
+ }
+
LOG(INFO) << "instance:" << instance.ShortDebugString();
SyncPoint::get_instance()->disable_processing();
SyncPoint::get_instance()->clear_all_call_backs();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index d0166e195f9..01c5cc6033c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -657,6 +657,11 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
properties.putIfAbsent(Env.CONNECTION_TIMEOUT_MS,
Env.DEFAULT_CONNECTION_TIMEOUT_MS);
}
+ private static boolean hasCredentialsProviderType(Map<String, String>
properties) {
+ return properties.containsKey(CREDENTIALS_PROVIDER_TYPE)
+ || properties.containsKey(Env.CREDENTIALS_PROVIDER_TYPE);
+ }
+
public static Cloud.ObjectStoreInfoPB.Builder
getObjStoreInfoPB(Map<String, String> properties) {
Cloud.ObjectStoreInfoPB.Builder builder =
Cloud.ObjectStoreInfoPB.newBuilder();
if (properties.containsKey(S3Properties.ENDPOINT)) {
@@ -696,12 +701,21 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
builder.setUsePathStyle(value.equalsIgnoreCase("true"));
}
+ if (hasCredentialsProviderType(properties)) {
+ builder.setCredProviderType(getCredProviderTypePB(properties));
+ }
+
if (properties.containsKey(S3Properties.ROLE_ARN)) {
- builder.setRoleArn(properties.get(S3Properties.ROLE_ARN));
- if (properties.containsKey(S3Properties.EXTERNAL_ID)) {
+ String roleArn = properties.get(S3Properties.ROLE_ARN);
+ if (!Strings.isNullOrEmpty(roleArn)) {
+ builder.setRoleArn(roleArn);
+ }
+ if (!Strings.isNullOrEmpty(roleArn) &&
properties.containsKey(S3Properties.EXTERNAL_ID)) {
builder.setExternalId(properties.get(S3Properties.EXTERNAL_ID));
}
- builder.setCredProviderType(getCredProviderTypePB(properties));
+ if (!Strings.isNullOrEmpty(roleArn) &&
!builder.hasCredProviderType()) {
+ builder.setCredProviderType(getCredProviderTypePB(properties));
+ }
}
return builder;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
index 97bb862df33..f9cfde32b06 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
@@ -292,6 +292,25 @@ public class S3PropertiesTest {
Assertions.assertTrue(S3StorageVault.ALLOW_ALTER_PROPERTIES.contains(S3Properties.CREDENTIALS_PROVIDER_TYPE));
}
+ @Test
+ public void testS3CredentialsProviderTypeWithoutIamRoleForCloud() {
+ origProps.put("s3.endpoint", "s3.us-west-2.amazonaws.com");
+ origProps.put("s3.region", "us-west-2");
+ origProps.put("s3.bucket", "bucket");
+ origProps.put("s3.root.path", "root");
+ origProps.put("s3.credentials_provider_type", "container");
+
+ Assertions.assertEquals(CredProviderTypePB.CONTAINER,
+
S3Properties.getObjStoreInfoPB(origProps).getCredProviderType());
+
Assertions.assertFalse(S3Properties.getObjStoreInfoPB(origProps).hasRoleArn());
+
+ origProps.remove("s3.credentials_provider_type");
+ origProps.put("AWS_CREDENTIALS_PROVIDER_TYPE", "env");
+ Assertions.assertEquals(CredProviderTypePB.ENV,
+
S3Properties.getObjStoreInfoPB(origProps).getCredProviderType());
+
Assertions.assertFalse(S3Properties.getObjStoreInfoPB(origProps).hasRoleArn());
+ }
+
@Test
public void testGetAwsCredentialsProviderWithIamRoleAndExternalId() {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
