[doris] branch master updated: [enhance](ColdHeatSeparation) carry use path style info along with cold heat separation to support using minio (#22249)

[dataroaring]

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 06e4061b94 [enhance](ColdHeatSeparation) carry use path style info 
along with cold heat separation to support using minio (#22249)
06e4061b94 is described below

commit 06e4061b94281ee53d2a01ba105219ab5074698f
Author: AlexYue <yj976240...@gmail.com>
AuthorDate: Sun Jul 30 21:03:33 2023 +0800

    [enhance](ColdHeatSeparation) carry use path style info along with cold 
heat separation to support using minio (#22249)
---
 be/src/agent/task_worker_pool.cpp                                   | 3 +++
 .../apache/doris/datasource/property/constants/S3Properties.java    | 6 ++++++
 gensrc/thrift/AgentService.thrift                                   | 1 +
 3 files changed, 10 insertions(+)

diff --git a/be/src/agent/task_worker_pool.cpp 
b/be/src/agent/task_worker_pool.cpp
index f40ff6ef4c..6fa6c88197 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -1116,6 +1116,9 @@ void 
TaskWorkerPool::_push_storage_policy_worker_thread_callback() {
                 s3_conf.connect_timeout_ms = 
resource.s3_storage_param.conn_timeout_ms;
                 s3_conf.max_connections = resource.s3_storage_param.max_conn;
                 s3_conf.request_timeout_ms = 
resource.s3_storage_param.request_timeout_ms;
+                // When using cold heat separation in minio, user might use ip 
address directly,
+                // which needs enable use_virtual_addressing to true
+                s3_conf.use_virtual_addressing = 
resource.s3_storage_param.use_path_style;
                 std::shared_ptr<io::S3FileSystem> fs;
                 if (existed_resource.fs == nullptr) {
                     st = io::S3FileSystem::create(s3_conf, 
std::to_string(resource.id), &fs);
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
 
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
index 13fb6ee336..3d3a003466 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
@@ -22,6 +22,7 @@ import org.apache.doris.common.DdlException;
 import org.apache.doris.datasource.credentials.CloudCredential;
 import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint;
 import org.apache.doris.datasource.credentials.DataLakeAWSCredentialsProvider;
+import org.apache.doris.datasource.property.PropertyConverter;
 import org.apache.doris.thrift.TS3StorageParam;
 
 import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
@@ -241,6 +242,9 @@ public class S3Properties extends BaseProperties {
         if (properties.containsKey(S3Properties.Env.BUCKET)) {
             properties.putIfAbsent(S3Properties.BUCKET, 
properties.get(S3Properties.Env.BUCKET));
         }
+        if (properties.containsKey(PropertyConverter.USE_PATH_STYLE)) {
+            properties.putIfAbsent(PropertyConverter.USE_PATH_STYLE, 
properties.get(PropertyConverter.USE_PATH_STYLE));
+        }
     }
 
     public static TS3StorageParam getS3TStorageParam(Map<String, String> 
properties) {
@@ -261,6 +265,8 @@ public class S3Properties extends BaseProperties {
         String connTimeoutMs = 
properties.get(S3Properties.CONNECTION_TIMEOUT_MS);
         s3Info.setMaxConn(Integer.parseInt(connTimeoutMs == null
                 ? S3Properties.Env.DEFAULT_CONNECTION_TIMEOUT_MS : 
connTimeoutMs));
+        String usePathStyle = 
properties.getOrDefault(PropertyConverter.USE_PATH_STYLE, "false");
+        s3Info.setUsePathStyle(Boolean.parseBoolean(usePathStyle));
         return s3Info;
     }
 }
diff --git a/gensrc/thrift/AgentService.thrift 
b/gensrc/thrift/AgentService.thrift
index 13bee01da8..c1d25d5e66 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -70,6 +70,7 @@ struct TS3StorageParam {
     7: optional i32 conn_timeout_ms = 1000
     8: optional string root_path
     9: optional string bucket
+    10: optional bool use_path_style = false
 }
 
 struct TStoragePolicy {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org