This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new d43b359106a [fix](s3Client) Add `ca_cert_file_paths` conf for
stsClient` and recycler (#50837) (#50943)
d43b359106a is described below
commit d43b359106a69849fcccb33c76e8c0dec332676d
Author: Lei Zhang <[email protected]>
AuthorDate: Sat May 17 11:19:38 2025 +0800
[fix](s3Client) Add `ca_cert_file_paths` conf for stsClient` and recycler
(#50837) (#50943)
* https://github.com/apache/doris/pull/32285 In previous, the pr add a
`ca_cert_file_paths` config for be s3Client, but lack of recycler and
stsClient
---
be/src/util/s3_util.cpp | 36 ++++++++++-------------
be/src/util/s3_util.h | 1 -
be/test/io/fs/s3_obj_stroage_client_mock_test.cpp | 7 +++++
cloud/src/common/config.h | 6 ++++
cloud/src/recycler/s3_accessor.cpp | 20 ++++++++++++-
cloud/src/recycler/s3_accessor.h | 1 +
cloud/test/util_test.cpp | 7 +++++
common/cpp/aws_common.cpp | 8 +++++
common/cpp/aws_common.h | 7 ++++-
9 files changed, 70 insertions(+), 23 deletions(-)
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index 39887625942..a9f82537184 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -140,7 +140,7 @@ S3ClientFactory::S3ClientFactory() {
return std::make_shared<DorisAWSLogger>(logLevel);
};
Aws::InitAPI(_aws_options);
- _ca_cert_file_path = get_valid_ca_cert_path();
+ _ca_cert_file_path =
get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";"));
_rate_limiters = {
std::make_unique<S3RateLimiterHolder>(
config::s3_get_token_per_second,
config::s3_get_bucket_tokens,
@@ -152,17 +152,6 @@ S3ClientFactory::S3ClientFactory() {
metric_func_factory(put_rate_limit_ns,
put_rate_limit_exceed_req_num))};
}
-std::string S3ClientFactory::get_valid_ca_cert_path() {
- auto vec_ca_file_path = doris::split(config::ca_cert_file_paths, ";");
- auto it = vec_ca_file_path.begin();
- for (; it != vec_ca_file_path.end(); ++it) {
- if (std::filesystem::exists(*it)) {
- return *it;
- }
- }
- return "";
-}
-
S3ClientFactory::~S3ClientFactory() {
Aws::ShutdownAPI(_aws_options);
}
@@ -243,6 +232,14 @@ std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
S3ClientFactory::get_aws_cred
Aws::Client::ClientConfiguration clientConfiguration =
S3ClientFactory::getClientConfiguration();
+ if (_ca_cert_file_path.empty()) {
+ _ca_cert_file_path =
+
get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";"));
+ }
+ if (!_ca_cert_file_path.empty()) {
+ clientConfiguration.caFile = _ca_cert_file_path;
+ }
+
auto stsClient = std::make_shared<Aws::STS::STSClient>(
std::make_shared<Aws::Auth::InstanceProfileCredentialsProvider>(),
clientConfiguration);
@@ -264,16 +261,15 @@ std::shared_ptr<io::ObjStorageClient>
S3ClientFactory::_create_s3_client(
aws_config.endpointOverride = s3_conf.endpoint;
}
aws_config.region = s3_conf.region;
- std::string ca_cert = get_valid_ca_cert_path();
- if ("" != _ca_cert_file_path) {
+
+ if (_ca_cert_file_path.empty()) {
+ _ca_cert_file_path =
get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";"));
+ }
+
+ if (!_ca_cert_file_path.empty()) {
aws_config.caFile = _ca_cert_file_path;
- } else {
- // config::ca_cert_file_paths is valmutable,get newest value if file
path invaild
- _ca_cert_file_path = get_valid_ca_cert_path();
- if ("" != _ca_cert_file_path) {
- aws_config.caFile = _ca_cert_file_path;
- }
}
+
if (s3_conf.max_connections > 0) {
aws_config.maxConnections = s3_conf.max_connections;
} else {
diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h
index c45c6afa6ef..8b96fb0776b 100644
--- a/be/src/util/s3_util.h
+++ b/be/src/util/s3_util.h
@@ -158,7 +158,6 @@ private:
const S3ClientConf& s3_conf);
S3ClientFactory();
- static std::string get_valid_ca_cert_path();
Aws::SDKOptions _aws_options;
std::mutex _lock;
diff --git a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp
b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp
index 2fb61c92201..b7e635c1f1d 100644
--- a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp
+++ b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp
@@ -24,6 +24,7 @@
#include "gmock/gmock.h"
#include "io/fs/s3_obj_storage_client.h"
#include "util/s3_util.h"
+#include "util/string_util.h"
using namespace Aws::S3::Model;
@@ -118,4 +119,10 @@ TEST_F(S3ObjStorageClientMockTest,
list_objects_with_pagination) {
EXPECT_EQ(files.size(), 5);
files.clear();
}
+
+TEST_F(S3ObjStorageClientMockTest, test_ca_cert) {
+ auto path =
doris::get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";"));
+ LOG(INFO) << "config:" << config::ca_cert_file_paths << " path:" << path;
+ ASSERT_FALSE(path.empty());
+}
} // namespace doris::io
\ No newline at end of file
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 70d8ca8849f..1e40c5cfb25 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -282,4 +282,10 @@ CONF_Strings(recycler_storage_vault_white_list, "");
// Trace = 6
CONF_Int32(aws_log_level, "2");
+// ca_cert_file is in this path by default, Normally no modification is
required
+// ca cert default path is different from different OS
+CONF_mString(ca_cert_file_paths,
+
"/etc/pki/tls/certs/ca-bundle.crt;/etc/ssl/certs/ca-certificates.crt;"
+ "/etc/ssl/ca-bundle.pem");
+
} // namespace doris::cloud::config
diff --git a/cloud/src/recycler/s3_accessor.cpp
b/cloud/src/recycler/s3_accessor.cpp
index 63844665e3a..ada9a4a1e8c 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -260,8 +260,18 @@ std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
S3Accessor::get_aws_credentia
return
std::make_shared<Aws::Auth::InstanceProfileCredentialsProvider>();
}
+ Aws::Client::ClientConfiguration clientConfiguration;
+ if (_ca_cert_file_path.empty()) {
+ _ca_cert_file_path =
+
get_valid_ca_cert_path(doris::cloud::split(config::ca_cert_file_paths, ';'));
+ }
+ if (!_ca_cert_file_path.empty()) {
+ clientConfiguration.caFile = _ca_cert_file_path;
+ }
+
auto stsClient = std::make_shared<Aws::STS::STSClient>(
-
std::make_shared<Aws::Auth::InstanceProfileCredentialsProvider>());
+
std::make_shared<Aws::Auth::InstanceProfileCredentialsProvider>(),
+ clientConfiguration);
return std::make_shared<Aws::Auth::STSAssumeRoleCredentialsProvider>(
s3_conf.role_arn, Aws::String(), s3_conf.external_id,
@@ -334,6 +344,14 @@ int S3Accessor::init() {
}
aws_config.retryStrategy = std::make_shared<S3CustomRetryStrategy>(
config::max_s3_client_retry /*scaleFactor = 25*/);
+
+ if (_ca_cert_file_path.empty()) {
+ _ca_cert_file_path =
+
get_valid_ca_cert_path(doris::cloud::split(config::ca_cert_file_paths, ';'));
+ }
+ if (!_ca_cert_file_path.empty()) {
+ aws_config.caFile = _ca_cert_file_path;
+ }
auto s3_client = std::make_shared<Aws::S3::S3Client>(
get_aws_credentials_provider(conf_), std::move(aws_config),
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
diff --git a/cloud/src/recycler/s3_accessor.h b/cloud/src/recycler/s3_accessor.h
index faa8392373c..544c142d09b 100644
--- a/cloud/src/recycler/s3_accessor.h
+++ b/cloud/src/recycler/s3_accessor.h
@@ -139,6 +139,7 @@ protected:
S3Conf conf_;
std::shared_ptr<ObjStorageClient> obj_client_;
+ std::string _ca_cert_file_path;
};
class GcsAccessor final : public S3Accessor {
diff --git a/cloud/test/util_test.cpp b/cloud/test/util_test.cpp
index e505b2b99a5..e0cd54acc8b 100644
--- a/cloud/test/util_test.cpp
+++ b/cloud/test/util_test.cpp
@@ -29,6 +29,7 @@
#include "common/logging.h"
#include "common/simple_thread_pool.h"
#include "common/string_util.h"
+#include "cpp/aws_common.h"
#include "cpp/sync_point.h"
#include "gtest/gtest.h"
#include "recycler/recycler.h"
@@ -324,4 +325,10 @@ TEST(UtilTest, test_sync_executor) {
EXPECT_EQ(1, res.size());
EXPECT_EQ(finished, true);
std::for_each(res.begin(), res.end(), [](auto&& n) { EXPECT_EQ(0, n); });
+}
+
+TEST(UtilTest, test_split) {
+ auto path =
doris::get_valid_ca_cert_path(doris::cloud::split(config::ca_cert_file_paths,
';'));
+ LOG(INFO) << "config:" << config::ca_cert_file_paths << " path:" << path;
+ ASSERT_FALSE(path.empty());
}
\ No newline at end of file
diff --git a/common/cpp/aws_common.cpp b/common/cpp/aws_common.cpp
index 15a34f7c11a..5c615b843ae 100644
--- a/common/cpp/aws_common.cpp
+++ b/common/cpp/aws_common.cpp
@@ -37,4 +37,12 @@ CredProviderType
cred_provider_type_from_pb(cloud::CredProviderTypePB cred_provi
}
}
+std::string get_valid_ca_cert_path(const std::vector<std::string>&
ca_cert_file_paths) {
+ for (const auto& path : ca_cert_file_paths) {
+ if (std::filesystem::exists(path)) {
+ return path;
+ }
+ }
+ return "";
+}
}
\ No newline at end of file
diff --git a/common/cpp/aws_common.h b/common/cpp/aws_common.h
index 895ba7a6736..183a2ba80c5 100644
--- a/common/cpp/aws_common.h
+++ b/common/cpp/aws_common.h
@@ -19,9 +19,14 @@
#include <gen_cpp/cloud.pb.h>
+#include <filesystem>
+
namespace doris {
//AWS Credentials Provider Type
enum class CredProviderType { Default = 0, Simple = 1, InstanceProfile = 2
};
CredProviderType cred_provider_type_from_pb(cloud::CredProviderTypePB
cred_provider_type);
-}
\ No newline at end of file
+
+ std::string get_valid_ca_cert_path(const std::vector<std::string>&
ca_cert_file_paths);
+
+ } // namespace doris
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
