This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to tag 3.0.3-rc04
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5098a4af7a907a23e05497123247b4da405278e6
Author: Gavin Chou <ga...@selectdb.com>
AuthorDate: Sun Dec 8 04:26:23 2024 +0800
[fix](storage vault) Fix missing use_path_style when create storage vault
(#45155)
---
cloud/src/meta-service/meta_service_resource.cpp | 27 +++++++++++++---------
.../org/apache/doris/catalog/StorageVault.java | 4 ++++
.../property/constants/S3Properties.java | 3 +++
3 files changed, 23 insertions(+), 11 deletions(-)
diff --git a/cloud/src/meta-service/meta_service_resource.cpp
b/cloud/src/meta-service/meta_service_resource.cpp
index db232eb5314..4fa8cc5a132 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -765,6 +765,7 @@ struct ObjectStorageDesc {
std::string& endpoint;
std::string& external_endpoint;
std::string& region;
+ bool& use_path_style;
};
static int extract_object_storage_info(const AlterObjStoreInfoRequest* request,
@@ -777,7 +778,7 @@ static int extract_object_storage_info(const
AlterObjStoreInfoRequest* request,
msg = "s3 obj info err " + proto_to_json(*request);
return -1;
}
- auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region] =
obj_desc;
+ auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region,
use_path_style] = obj_desc;
const auto& obj = request->has_obj() ? request->obj() :
request->vault().obj_info();
// Prepare data
if (!obj.has_ak() || !obj.has_sk()) {
@@ -803,6 +804,7 @@ static int extract_object_storage_info(const
AlterObjStoreInfoRequest* request,
endpoint = obj.has_endpoint() ? obj.endpoint() : "";
external_endpoint = obj.has_external_endpoint() ? obj.external_endpoint()
: "";
region = obj.has_region() ? obj.region() : "";
+ use_path_style = obj.use_path_style();
// obj size > 1k, refuse
if (obj.ByteSizeLong() > 1024) {
code = MetaServiceCode::INVALID_ARGUMENT;
@@ -812,13 +814,13 @@ static int extract_object_storage_info(const
AlterObjStoreInfoRequest* request,
return 0;
}
-static ObjectStoreInfoPB object_info_pb_factory(ObjectStorageDesc& obj_info,
+static ObjectStoreInfoPB object_info_pb_factory(ObjectStorageDesc& obj_desc,
const ObjectStoreInfoPB& obj,
InstanceInfoPB& instance,
EncryptionInfoPB&
encryption_info,
AkSkPair& cipher_ak_sk_pair) {
ObjectStoreInfoPB last_item;
- auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region] =
obj_info;
+ auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region,
use_path_style] = obj_desc;
auto now_time = std::chrono::system_clock::now();
uint64_t time =
std::chrono::duration_cast<std::chrono::seconds>(now_time.time_since_epoch()).count();
@@ -840,6 +842,7 @@ static ObjectStoreInfoPB
object_info_pb_factory(ObjectStorageDesc& obj_info,
last_item.set_region(region);
last_item.set_provider(obj.provider());
last_item.set_sse_enabled(instance.sse_enabled());
+ last_item.set_use_path_style(use_path_style);
return last_item;
}
@@ -848,14 +851,15 @@ void
MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
AlterObjStoreInfoResponse* response,
::google::protobuf::Closure* done) {
std::string ak, sk, bucket, prefix, endpoint, external_endpoint, region;
+ bool use_path_style;
EncryptionInfoPB encryption_info;
AkSkPair cipher_ak_sk_pair;
RPC_PREPROCESS(alter_storage_vault);
switch (request->op()) {
case AlterObjStoreInfoRequest::ADD_S3_VAULT:
case AlterObjStoreInfoRequest::DROP_S3_VAULT: {
- auto tmp_desc =
- ObjectStorageDesc {ak, sk, bucket, prefix, endpoint,
external_endpoint, region};
+ auto tmp_desc = ObjectStorageDesc {
+ ak, sk, bucket, prefix, endpoint, external_endpoint, region,
use_path_style};
if (0 != extract_object_storage_info(request, code, msg, tmp_desc,
encryption_info,
cipher_ak_sk_pair)) {
return;
@@ -994,8 +998,8 @@ void
MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
}
}
// calc id
- auto tmp_tuple =
- ObjectStorageDesc {ak, sk, bucket, prefix, endpoint,
external_endpoint, region};
+ auto tmp_tuple = ObjectStorageDesc {
+ ak, sk, bucket, prefix, endpoint, external_endpoint, region,
use_path_style};
ObjectStoreInfoPB last_item = object_info_pb_factory(tmp_tuple, obj,
instance,
encryption_info,
cipher_ak_sk_pair);
if (instance.storage_vault_names().end() !=
@@ -1138,6 +1142,7 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
AlterObjStoreInfoResponse* response,
::google::protobuf::Closure* done) {
std::string ak, sk, bucket, prefix, endpoint, external_endpoint, region;
+ bool use_path_style;
EncryptionInfoPB encryption_info;
AkSkPair cipher_ak_sk_pair;
RPC_PREPROCESS(alter_obj_store_info);
@@ -1145,8 +1150,8 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
case AlterObjStoreInfoRequest::ADD_OBJ_INFO:
case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK:
case AlterObjStoreInfoRequest::UPDATE_AK_SK: {
- auto tmp_desc =
- ObjectStorageDesc {ak, sk, bucket, prefix, endpoint,
external_endpoint, region};
+ auto tmp_desc = ObjectStorageDesc {
+ ak, sk, bucket, prefix, endpoint, external_endpoint, region,
use_path_style};
if (0 != extract_object_storage_info(request, code, msg, tmp_desc,
encryption_info,
cipher_ak_sk_pair)) {
return;
@@ -1285,8 +1290,8 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
}
}
// calc id
- auto tmp_tuple =
- ObjectStorageDesc {ak, sk, bucket, prefix, endpoint,
external_endpoint, region};
+ auto tmp_tuple = ObjectStorageDesc {
+ ak, sk, bucket, prefix, endpoint, external_endpoint, region,
use_path_style};
ObjectStoreInfoPB last_item = object_info_pb_factory(tmp_tuple, obj,
instance,
encryption_info,
cipher_ak_sk_pair);
instance.add_obj_info()->CopyFrom(last_item);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
index df9310526e4..9d45ce7bdd8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
@@ -23,6 +23,7 @@ import org.apache.doris.cloud.proto.Cloud;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.qe.ShowResultSetMetaData;
import com.google.common.base.Strings;
@@ -144,6 +145,9 @@ public abstract class StorageVault {
vault.modifyProperties(stmt.getProperties());
break;
case S3:
+ if
(!stmt.getProperties().containsKey(PropertyConverter.USE_PATH_STYLE)) {
+ stmt.getProperties().put(PropertyConverter.USE_PATH_STYLE,
"true");
+ }
CreateResourceStmt resourceStmt =
new CreateResourceStmt(false, ifNotExists, name,
stmt.getProperties());
resourceStmt.analyzeResourceType();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
index 7e94d81518e..18286b2fbc3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
@@ -343,6 +343,9 @@ public class S3Properties extends BaseProperties {
|| value.equalsIgnoreCase("false"),
"Invalid use_path_style value: %s only 'true' or 'false'
is acceptable", value);
builder.setUsePathStyle(value.equalsIgnoreCase("true"));
+ } else {
+ // use path style
+ builder.setUsePathStyle(true);
}
return builder;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
