This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new d344f83a33e branch-4.1: [feat](s3) support S3 storage vault
credentials provider without role ARN (#65013)
d344f83a33e is described below
commit d344f83a33e8b1a0afc744bef6b7e8b8882cc257
Author: hui lai <[email protected]>
AuthorDate: Wed Jul 1 10:10:35 2026 +0800
branch-4.1: [feat](s3) support S3 storage vault credentials provider
without role ARN (#65013)
pick https://github.com/apache/doris/pull/64766
---
cloud/src/meta-service/meta_service_resource.cpp | 29 +++++++++-----
cloud/src/recycler/s3_accessor.cpp | 9 +++--
cloud/test/meta_service_test.cpp | 46 ++++++++++++++++++++++
.../datasource/property/storage/S3Properties.java | 20 ++++++++--
.../property/storage/S3PropertiesTest.java | 19 +++++++++
5 files changed, 107 insertions(+), 16 deletions(-)
diff --git a/cloud/src/meta-service/meta_service_resource.cpp
b/cloud/src/meta-service/meta_service_resource.cpp
index ed36b54322c..2e19ebbb8a3 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -760,6 +760,14 @@ static int add_hdfs_storage_vault(InstanceInfoPB&
instance, Transaction* txn,
return 0;
}
+static bool has_non_empty_role_arn(const ObjectStoreInfoPB& obj) {
+ return obj.has_role_arn() && !obj.role_arn().empty();
+}
+
+static bool use_credential_provider(const ObjectStoreInfoPB& obj) {
+ return obj.has_cred_provider_type() || has_non_empty_role_arn(obj);
+}
+
static void create_object_info_with_encrypt(const InstanceInfoPB& instance,
ObjectStoreInfoPB* obj,
bool sse_enabled, MetaServiceCode&
code,
std::string& msg) {
@@ -1250,7 +1258,9 @@ static int extract_object_storage_info(const
AlterObjStoreInfoRequest* request,
auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region,
use_path_style, role_arn,
external_id] = obj_desc;
- if (!obj.has_role_arn()) {
+ bool use_credential_provider_for_add_vault =
+ request->op() == AlterObjStoreInfoRequest::ADD_S3_VAULT &&
obj.has_cred_provider_type();
+ if (!obj.has_role_arn() && !use_credential_provider_for_add_vault) {
if (!obj.has_ak() || !obj.has_sk()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "s3 obj info err " + proto_to_json(*request);
@@ -1306,13 +1316,15 @@ static ObjectStoreInfoPB
object_info_pb_factory(ObjectStorageDesc& obj_desc,
last_item.set_user_id(obj.user_id());
}
- if (!obj.has_role_arn()) {
+ if (!use_credential_provider(obj)) {
last_item.set_ak(std::move(cipher_ak_sk_pair.first));
last_item.set_sk(std::move(cipher_ak_sk_pair.second));
last_item.mutable_encryption_info()->CopyFrom(encryption_info);
} else {
- last_item.set_role_arn(role_arn);
- last_item.set_external_id(external_id);
+ if (has_non_empty_role_arn(obj)) {
+ last_item.set_role_arn(role_arn);
+ last_item.set_external_id(external_id);
+ }
last_item.set_cred_provider_type(get_cred_provider_type(obj));
}
last_item.set_bucket(bucket);
@@ -1476,18 +1488,17 @@ void
MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
return;
}
// ATTN: prefix may be empty
- if (((ak.empty() || sk.empty()) && role_arn.empty()) || bucket.empty()
||
+ if (((ak.empty() || sk.empty()) && !use_credential_provider(obj)) ||
bucket.empty() ||
endpoint.empty() || region.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "s3 conf info err, please check it";
return;
}
- if (!role_arn.empty()) {
- if (!obj.has_cred_provider_type() || !obj.has_provider() ||
- obj.provider() != ObjectStoreInfoPB::S3) {
+ if (use_credential_provider(obj)) {
+ if (!obj.has_provider() || obj.provider() !=
ObjectStoreInfoPB::S3) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 conf info err with role_arn, please check it";
+ msg = "s3 conf info err with credentials_provider_type, please
check it";
return;
}
}
diff --git a/cloud/src/recycler/s3_accessor.cpp
b/cloud/src/recycler/s3_accessor.cpp
index d48c51c8469..51cc9023904 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -237,13 +237,14 @@ std::optional<S3Conf> S3Conf::from_obj_store_info(const
ObjectStoreInfoPB& obj_i
}
}
+ if (obj_info.has_cred_provider_type()) {
+ s3_conf.cred_provider_type =
cred_provider_type_from_pb(obj_info.cred_provider_type());
+ }
+
if (obj_info.has_role_arn() && !obj_info.role_arn().empty()) {
s3_conf.role_arn = obj_info.role_arn();
s3_conf.external_id = obj_info.external_id();
- if (obj_info.has_cred_provider_type()) {
- s3_conf.cred_provider_type =
-
cred_provider_type_from_pb(obj_info.cred_provider_type());
- } else {
+ if (!obj_info.has_cred_provider_type()) {
s3_conf.cred_provider_type = CredProviderType::InstanceProfile;
}
}
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index 3dd78bd247a..9b04e98a2e0 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -10508,6 +10508,52 @@ TEST(MetaServiceTest, CreateS3VaultWithIamRole) {
}
}
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ADD_S3_VAULT);
+ StorageVaultPB vault;
+ vault.mutable_obj_info()->set_endpoint("s3.us-east-1.amazonaws.com");
+ vault.mutable_obj_info()->set_region("us-east-1");
+
vault.mutable_obj_info()->set_bucket("test_credential_provider_bucket");
+
vault.mutable_obj_info()->set_prefix("test_credential_provider_prefix");
+ vault.mutable_obj_info()->set_provider(
+ ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3);
+
vault.mutable_obj_info()->set_cred_provider_type(CredProviderTypePB::CONTAINER);
+
+ vault.set_name("s3_vault_with_credential_provider");
+ req.mutable_vault()->CopyFrom(vault);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+
+ {
+ InstanceInfoPB instance;
+ get_test_instance(instance);
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn),
TxnErrorCode::TXN_OK);
+ std::string val;
+ ASSERT_EQ(txn->get(storage_vault_key({instance.instance_id(),
"5"}), &val),
+ TxnErrorCode::TXN_OK);
+ StorageVaultPB get_obj;
+ get_obj.ParseFromString(val);
+ ASSERT_TRUE(get_obj.obj_info().ak().empty()) <<
get_obj.obj_info().ak();
+ ASSERT_TRUE(get_obj.obj_info().sk().empty()) <<
get_obj.obj_info().sk();
+ ASSERT_FALSE(get_obj.obj_info().has_role_arn());
+ ASSERT_FALSE(get_obj.obj_info().has_external_id());
+ ASSERT_EQ(get_obj.obj_info().cred_provider_type(),
CredProviderTypePB::CONTAINER)
+ << get_obj.obj_info().cred_provider_type();
+ ASSERT_EQ(get_obj.obj_info().bucket(),
"test_credential_provider_bucket")
+ << get_obj.obj_info().bucket();
+ ASSERT_EQ(get_obj.obj_info().prefix(),
"test_credential_provider_prefix")
+ << get_obj.obj_info().prefix();
+ ASSERT_EQ(get_obj.id(), "5") << get_obj.id();
+ }
+ }
+
LOG(INFO) << "instance:" << instance.ShortDebugString();
SyncPoint::get_instance()->disable_processing();
SyncPoint::get_instance()->clear_all_call_backs();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index d0166e195f9..01c5cc6033c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -657,6 +657,11 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
properties.putIfAbsent(Env.CONNECTION_TIMEOUT_MS,
Env.DEFAULT_CONNECTION_TIMEOUT_MS);
}
+ private static boolean hasCredentialsProviderType(Map<String, String>
properties) {
+ return properties.containsKey(CREDENTIALS_PROVIDER_TYPE)
+ || properties.containsKey(Env.CREDENTIALS_PROVIDER_TYPE);
+ }
+
public static Cloud.ObjectStoreInfoPB.Builder
getObjStoreInfoPB(Map<String, String> properties) {
Cloud.ObjectStoreInfoPB.Builder builder =
Cloud.ObjectStoreInfoPB.newBuilder();
if (properties.containsKey(S3Properties.ENDPOINT)) {
@@ -696,12 +701,21 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
builder.setUsePathStyle(value.equalsIgnoreCase("true"));
}
+ if (hasCredentialsProviderType(properties)) {
+ builder.setCredProviderType(getCredProviderTypePB(properties));
+ }
+
if (properties.containsKey(S3Properties.ROLE_ARN)) {
- builder.setRoleArn(properties.get(S3Properties.ROLE_ARN));
- if (properties.containsKey(S3Properties.EXTERNAL_ID)) {
+ String roleArn = properties.get(S3Properties.ROLE_ARN);
+ if (!Strings.isNullOrEmpty(roleArn)) {
+ builder.setRoleArn(roleArn);
+ }
+ if (!Strings.isNullOrEmpty(roleArn) &&
properties.containsKey(S3Properties.EXTERNAL_ID)) {
builder.setExternalId(properties.get(S3Properties.EXTERNAL_ID));
}
- builder.setCredProviderType(getCredProviderTypePB(properties));
+ if (!Strings.isNullOrEmpty(roleArn) &&
!builder.hasCredProviderType()) {
+ builder.setCredProviderType(getCredProviderTypePB(properties));
+ }
}
return builder;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
index 7871dfb6bfb..c772c25bfc7 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
@@ -308,6 +308,25 @@ public class S3PropertiesTest {
Assertions.assertNull(backendProperties.get("AWS_EXTERNAL_ID"));
}
+ @Test
+ public void testS3CredentialsProviderTypeWithoutIamRoleForCloud() {
+ origProps.put("s3.endpoint", "s3.us-west-2.amazonaws.com");
+ origProps.put("s3.region", "us-west-2");
+ origProps.put("s3.bucket", "bucket");
+ origProps.put("s3.root.path", "root");
+ origProps.put("s3.credentials_provider_type", "container");
+
+ Assertions.assertEquals(CredProviderTypePB.CONTAINER,
+
S3Properties.getObjStoreInfoPB(origProps).getCredProviderType());
+
Assertions.assertFalse(S3Properties.getObjStoreInfoPB(origProps).hasRoleArn());
+
+ origProps.remove("s3.credentials_provider_type");
+ origProps.put("AWS_CREDENTIALS_PROVIDER_TYPE", "env");
+ Assertions.assertEquals(CredProviderTypePB.ENV,
+
S3Properties.getObjStoreInfoPB(origProps).getCredProviderType());
+
Assertions.assertFalse(S3Properties.getObjStoreInfoPB(origProps).hasRoleArn());
+ }
+
@Test
public void testGetAwsCredentialsProviderWithIamRoleAndExternalId(@Mocked
StsClientBuilder mockBuilder,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
