This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-refactor_property
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-refactor_property by
this push:
new 3d6f67696a0 [Test](refactor-storage)Add BACKUP/RESTORE/REPOSITORY Test
(#49372)
3d6f67696a0 is described below
commit 3d6f67696a08df0f62d2a7422064c14c5d1d99a1
Author: Calvin Kirs <[email protected]>
AuthorDate: Mon Mar 24 14:39:34 2025 +0800
[Test](refactor-storage)Add BACKUP/RESTORE/REPOSITORY Test (#49372)
### What problem does this PR solve?
This PR continues the optimization of the S3Filesystem by removing
deprecated parameters and adapting the Backup&Restore logic to use the
new parameter format. Additionally, it introduces relevant test cases to
ensure compatibility and correctness.
#### Main Changes
Remove Legacy Parameters from S3Filesystem
Clean up outdated S3 parameters to simplify the codebase and eliminate
potential compatibility issues.
#### Adapt BackupRestore to Use New Parameters
Modify Backup & Restore to correctly parse and use the new parameter
format in S3Filesystem, ensuring stable backup and restore
functionality.
#### Add Test Cases
Introduce unit tests and integration tests for S3 to verify
compatibility with the new parameters and validate the BackupRestore
logic.
---
.../java/org/apache/doris/analysis/LoadStmt.java | 2 +
.../doris/datasource/property/PropertyUtils.java | 18 ++-
.../storage/AbstractObjectStorageProperties.java | 24 +--
.../datasource/property/storage/COSProperties.java | 17 +-
.../datasource/property/storage/OBSProperties.java | 14 +-
.../datasource/property/storage/OSSProperties.java | 16 +-
.../property/storage/ObjectStorageProperties.java | 8 +
.../datasource/property/storage/S3Properties.java | 22 ++-
.../java/org/apache/doris/fs/obj/S3ObjStorage.java | 50 ++----
.../org/apache/doris/fs/remote/S3FileSystem.java | 4 +-
.../property/storage/COSPropertiesTest.java | 39 ++++-
.../property/storage/OBSPropertyTest.java | 20 +++
.../property/storage/OSSPropertiesTest.java | 23 +++
.../property/storage/S3PropertiesTest.java | 34 +++-
.../org/apache/doris/fs/obj/S3FileSystemTest.java | 7 +-
.../org/apache/doris/fs/obj/S3ObjStorageTest.java | 18 ++-
.../backup_restore_cos.groovy | 178 +++++++++++++++++++++
17 files changed, 423 insertions(+), 71 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
index 9ceec9415e3..2863f7c8a18 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
@@ -504,6 +504,8 @@ public class LoadStmt extends DdlStmt implements
NotFallbackInParser {
}
} else if (brokerDesc != null) {
etlJobType = EtlJobType.BROKER;
+ //todo Storage-related parameter validation should not be placed
here.
+ // This section should focus solely on business logic.
checkS3Param();
} else if (isMysqlLoad) {
etlJobType = EtlJobType.LOCAL_FILE;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyUtils.java
index 86c25655db7..f02f01efad3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyUtils.java
@@ -27,15 +27,19 @@ public class PropertyUtils {
// Get all fields of a class with annotation @ConnectorProperty
public static List<Field> getConnectorProperties(Class<?> clazz) {
List<Field> fields = Lists.newArrayList();
- for (Field field : clazz.getDeclaredFields()) {
- field.setAccessible(true);
- if (field.isAnnotationPresent(ConnectorProperty.class)) {
- // Get annotation of the field
- ConnectorProperty connectorProperty =
field.getAnnotation(ConnectorProperty.class);
- if (connectorProperty.supported()) {
- fields.add(field);
+ Class<?> currentClass = clazz;
+
+ while (currentClass != null) {
+ for (Field field : currentClass.getDeclaredFields()) {
+ field.setAccessible(true);
+ if (field.isAnnotationPresent(ConnectorProperty.class)) {
+ ConnectorProperty connectorProperty =
field.getAnnotation(ConnectorProperty.class);
+ if (connectorProperty.supported()) {
+ fields.add(field);
+ }
}
}
+ currentClass = currentClass.getSuperclass();
}
return fields;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractObjectStorageProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractObjectStorageProperties.java
index b9d84d2a7c9..7d177d3bd3d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractObjectStorageProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractObjectStorageProperties.java
@@ -41,24 +41,25 @@ public abstract class AbstractObjectStorageProperties
extends StorageProperties
* This value is optional and can be configured by the user.
*/
@Getter
- @ConnectorProperty(names = {"maxConnections"}, required = false,
description = "Maximum number of connections.")
- protected int maxConnections = 100;
+ @ConnectorProperty(names = {"connection.maximum"}, required = false,
description = "Maximum number of connections.")
+ protected String maxConnections = "100";
/**
* The timeout (in milliseconds) for requests made to the object storage
system.
* This value is optional and can be configured by the user.
*/
@Getter
- @ConnectorProperty(names = {"requestTimeoutS"}, required = false,
description = "Request timeout in seconds.")
- protected int requestTimeoutS = 10000;
+ @ConnectorProperty(names = {"connection.request.timeout"}, required =
false,
+ description = "Request timeout in seconds.")
+ protected String requestTimeoutS = "10000";
/**
* The timeout (in milliseconds) for establishing a connection to the
object storage system.
* This value is optional and can be configured by the user.
*/
@Getter
- @ConnectorProperty(names = {"connectionTimeoutS"}, required = false,
description = "Connection timeout in seconds.")
- protected int connectionTimeoutS = 10000;
+ @ConnectorProperty(names = {"connection.timeout"}, required = false,
description = "Connection timeout in seconds.")
+ protected String connectionTimeoutS = "10000";
/**
* Flag indicating whether to use path-style URLs for the object storage
system.
@@ -66,9 +67,14 @@ public abstract class AbstractObjectStorageProperties
extends StorageProperties
*/
@Setter
@Getter
- @ConnectorProperty(names = {"usePathStyle", "s3.path-style-access"},
required = false,
+ @ConnectorProperty(names = {"use_path_style", "s3.path-style-access"},
required = false,
description = "Whether to use path style URL for the storage.")
- protected boolean usePathStyle = false;
+ protected String usePathStyle = "false";
+ @ConnectorProperty(names = {"force_parsing_by_standard_uri"}, required =
false,
+ description = "Whether to use path style URL for the storage.")
+ @Setter
+ @Getter
+ protected String forceParsingByStandardUrl = "false";
/**
* Constructor to initialize the object storage properties with the
provided type and original properties map.
@@ -124,7 +130,7 @@ public abstract class AbstractObjectStorageProperties
extends StorageProperties
String accessKey,
String secretKey) {
return generateAWSS3Properties(endpoint, region, accessKey, secretKey,
String.valueOf(getMaxConnections()),
String.valueOf(getRequestTimeoutS()),
- String.valueOf(getConnectionTimeoutS()),
String.valueOf(isUsePathStyle()));
+ String.valueOf(getConnectionTimeoutS()), usePathStyle);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
index 1791ba67a51..5416819dd36 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
@@ -71,7 +71,7 @@ public class COSProperties extends
AbstractObjectStorageProperties {
config.putAll(generateAWSS3Properties(cosEndpoint, getRegion(),
cosAccessKey, cosSecretKey));
}
- private String getRegion() {
+ public String getRegion() {
if (Strings.isNullOrEmpty(this.cosRegion)) {
if (cosEndpoint.contains("myqcloud.com")) {
Pattern cosPattern =
Pattern.compile("cos\\.([a-z0-9-]+)\\.myqcloud\\.com");
@@ -83,4 +83,19 @@ public class COSProperties extends
AbstractObjectStorageProperties {
}
return this.cosRegion;
}
+
+ @Override
+ public String getEndpoint() {
+ return cosEndpoint;
+ }
+
+ @Override
+ public String getAccessKey() {
+ return cosAccessKey;
+ }
+
+ @Override
+ public String getSecretKey() {
+ return cosSecretKey;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
index 1cdb4a83139..d561eb5fdca 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
@@ -65,7 +65,7 @@ public class OBSProperties extends
AbstractObjectStorageProperties {
config.putAll(generateAWSS3Properties(obsEndpoint, getRegion(),
obsAccessKey, obsSecretKey));
}
- private String getRegion() {
+ public String getRegion() {
if (Strings.isNullOrEmpty(this.region) &&
obsEndpoint.contains("myhuaweicloud.com")) {
Pattern obsPattern =
Pattern.compile("obs\\.([a-z0-9-]+)\\.myhuaweicloud\\.com");
Matcher matcher = obsPattern.matcher(obsEndpoint);
@@ -75,4 +75,16 @@ public class OBSProperties extends
AbstractObjectStorageProperties {
}
return this.region;
}
+
+ public String getEndpoint() {
+ return obsEndpoint;
+ }
+
+ public String getAccessKey() {
+ return obsAccessKey;
+ }
+
+ public String getSecretKey() {
+ return obsSecretKey;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
index 712a1ab61e5..3a5ede7a539 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
@@ -51,6 +51,7 @@ public class OSSProperties extends
AbstractObjectStorageProperties {
public Configuration getHadoopConfiguration() {
Configuration conf = new Configuration(false);
conf.set("fs.oss.endpoint", endpoint);
+ conf.set("fs.oss.region", getRegion());
conf.set("fs.oss.accessKeyId", accessKey);
conf.set("fs.oss.accessKeySecret", secretKey);
conf.set("fs.oss.impl",
"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem");
@@ -62,7 +63,7 @@ public class OSSProperties extends
AbstractObjectStorageProperties {
config.putAll(generateAWSS3Properties(endpoint, getRegion(),
accessKey, secretKey));
}
- private String getRegion() {
+ public String getRegion() {
// Return the region if it is already set
if (!Strings.isNullOrEmpty(this.region)) {
return region;
@@ -91,5 +92,18 @@ public class OSSProperties extends
AbstractObjectStorageProperties {
return this.region;
}
+ @Override
+ public String getEndpoint() {
+ return endpoint;
+ }
+ @Override
+ public String getAccessKey() {
+ return accessKey;
+ }
+
+ @Override
+ public String getSecretKey() {
+ return secretKey;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/ObjectStorageProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/ObjectStorageProperties.java
index 2465fbf6192..22694293e25 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/ObjectStorageProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/ObjectStorageProperties.java
@@ -48,4 +48,12 @@ public interface ObjectStorageProperties {
* to the object storage system.
*/
void toNativeS3Configuration(Map<String, String> config);
+
+ String getEndpoint();
+
+ String getRegion();
+
+ String getAccessKey();
+
+ String getSecretKey();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index ad4fcd2eb81..6963a0b024a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -147,7 +147,7 @@ public class S3Properties extends
AbstractObjectStorageProperties {
catalogProps.put("s3.access-key-id", s3AccessKey);
catalogProps.put("s3.secret-access-key", s3SecretKey);
catalogProps.put("client.region", s3Region);
- catalogProps.put("s3.path-style-access",
Boolean.toString(usePathStyle));
+ catalogProps.put("s3.path-style-access", usePathStyle);
}
@Override
@@ -177,4 +177,24 @@ public class S3Properties extends
AbstractObjectStorageProperties {
toNativeS3Configuration(config);
return config;
}
+
+ @Override
+ public String getEndpoint() {
+ return s3Endpoint;
+ }
+
+ @Override
+ public String getRegion() {
+ return s3Region;
+ }
+
+ @Override
+ public String getAccessKey() {
+ return s3AccessKey;
+ }
+
+ @Override
+ public String getSecretKey() {
+ return s3SecretKey;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java
index edcb54bf8fa..a6411aea219 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java
@@ -23,8 +23,7 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.credentials.CloudCredential;
import org.apache.doris.common.util.S3URI;
import org.apache.doris.common.util.S3Util;
-import org.apache.doris.datasource.property.PropertyConverter;
-import org.apache.doris.datasource.property.constants.S3Properties;
+import
org.apache.doris.datasource.property.storage.AbstractObjectStorageProperties;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Triple;
@@ -68,11 +67,13 @@ public class S3ObjStorage implements ObjStorage<S3Client> {
protected Map<String, String> properties;
+ protected AbstractObjectStorageProperties s3Properties;
+
private boolean isUsePathStyle = false;
private boolean forceParsingByStandardUri = false;
- public S3ObjStorage(Map<String, String> properties) {
+ public S3ObjStorage(AbstractObjectStorageProperties properties) {
this.properties = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
setProperties(properties);
}
@@ -81,49 +82,28 @@ public class S3ObjStorage implements ObjStorage<S3Client> {
return properties;
}
- protected void setProperties(Map<String, String> properties) {
- this.properties.putAll(properties);
- try {
- S3Properties.requiredS3Properties(this.properties);
- } catch (DdlException e) {
- throw new IllegalArgumentException(e);
- }
- // Virtual hosted-style is recommended in the s3 protocol.
- // The path-style has been abandoned, but for some unexplainable
reasons,
- // the s3 client will determine whether the endpiont starts with `s3`
- // when generating a virtual hosted-sytle request.
- // If not, it will not be converted (
https://github.com/aws/aws-sdk-java-v2/pull/763),
- // but the endpoints of many cloud service providers for object
storage do not start with s3,
- // so they cannot be converted to virtual hosted-sytle.
- // Some of them, such as aliyun's oss, only support virtual
hosted-style,
- // and some of them(ceph) may only support
- // path-style, so we need to do some additional conversion.
- isUsePathStyle =
this.properties.getOrDefault(PropertyConverter.USE_PATH_STYLE, "false")
- .equalsIgnoreCase("true");
- forceParsingByStandardUri =
this.properties.getOrDefault(PropertyConverter.FORCE_PARSING_BY_STANDARD_URI,
- "false").equalsIgnoreCase("true");
-
- String endpoint = this.properties.get(S3Properties.ENDPOINT);
- String region = this.properties.get(S3Properties.REGION);
-
- this.properties.put(S3Properties.REGION,
PropertyConverter.checkRegion(endpoint, region, S3Properties.REGION));
+ protected void setProperties(AbstractObjectStorageProperties properties) {
+ this.s3Properties = properties;
+ isUsePathStyle = Boolean.parseBoolean(properties.getUsePathStyle());
+ forceParsingByStandardUri =
Boolean.parseBoolean(s3Properties.getForceParsingByStandardUrl());
}
@Override
public S3Client getClient() throws UserException {
if (client == null) {
- String endpointStr = properties.get(S3Properties.ENDPOINT);
+ String endpointStr = s3Properties.getEndpoint();
if (!endpointStr.contains("://")) {
endpointStr = "http://"; + endpointStr;
}
URI endpoint = URI.create(endpointStr);
CloudCredential credential = new CloudCredential();
- credential.setAccessKey(properties.get(S3Properties.ACCESS_KEY));
- credential.setSecretKey(properties.get(S3Properties.SECRET_KEY));
- if (properties.containsKey(S3Properties.SESSION_TOKEN)) {
+ credential.setAccessKey(s3Properties.getAccessKey());
+ credential.setSecretKey(s3Properties.getSecretKey());
+
+ /* if (properties.containsKey(S3Properties.SESSION_TOKEN)) {
credential.setSessionToken(properties.get(S3Properties.SESSION_TOKEN));
- }
- client = S3Util.buildS3Client(endpoint,
properties.get(S3Properties.REGION), credential, isUsePathStyle);
+ }*/
+ client = S3Util.buildS3Client(endpoint, s3Properties.getRegion(),
credential, isUsePathStyle);
}
return client;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java
index 5910ed80c9e..555d1e42508 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java
@@ -49,7 +49,7 @@ public class S3FileSystem extends ObjFileSystem {
public S3FileSystem(AbstractObjectStorageProperties s3Properties) {
super(StorageBackend.StorageType.S3.name(),
StorageBackend.StorageType.S3,
- new S3ObjStorage(s3Properties.getOrigProps()));
+ new S3ObjStorage(s3Properties));
this.s3Properties = s3Properties;
this.storageProperties = s3Properties;
initFsProperties();
@@ -63,7 +63,7 @@ public class S3FileSystem extends ObjFileSystem {
}
private void initFsProperties() {
- this.properties.putAll(((S3ObjStorage) objStorage).getProperties());
+ this.properties.putAll(storageProperties.getOrigProps());
}
@Override
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
index 18a0f710a86..b2148f8e53a 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
@@ -50,18 +50,30 @@ public class COSPropertiesTest {
origProps.put("cos.access_key", "myCOSAccessKey");
origProps.put("cos.secret_key", "myCOSSecretKey");
origProps.put("cos.region", "us-west-1");
- origProps.put("cos.max_connections", "100");
- origProps.put("cos.request_timeout", "3000");
- origProps.put("cos.connection_timeout", "1000");
- origProps.put("cos.use_path_style", "true");
+ origProps.put("connection.maximum", "88");
+ origProps.put("connection.request.timeout", "100");
+ origProps.put("connection.timeout", "1000");
+ origProps.put("use_path_style", "true");
origProps.put(StorageProperties.FS_COS_SUPPORT, "true");
+ origProps.put("test_non_storage_param", "6000");
+
COSProperties cosProperties = (COSProperties)
StorageProperties.create(origProps).get(1);
Configuration config = cosProperties.getHadoopConfiguration();
+ Map<String, String> cosConfig = cosProperties.getOrigProps();
+
Assertions.assertTrue(!cosConfig.containsKey("test_non_storage_param"));
+
+ origProps.forEach((k, v) -> {
+ if (!k.equals("test_non_storage_param") &&
!k.equals(StorageProperties.FS_COS_SUPPORT)) {
+ Assertions.assertEquals(v, cosConfig.get(k));
+ }
+ });
// Validate the configuration
Assertions.assertEquals("https://cos.example.com";,
config.get("fs.cos.endpoint"));
Assertions.assertEquals("myCOSAccessKey",
config.get("fs.cosn.userinfo.secretId"));
Assertions.assertEquals("myCOSSecretKey",
config.get("fs.cosn.userinfo.secretKey"));
+ Assertions.assertEquals("myCOSAccessKey",
config.get("fs.cosn.userinfo.secretId"));
+ Assertions.assertEquals("myCOSSecretKey",
config.get("fs.cosn.userinfo.secretKey"));
origProps = new HashMap<>();
origProps.put("cos.endpoint", "https://cos.example.com";);
origProps.put(StorageProperties.FS_COS_SUPPORT, "true");
@@ -78,18 +90,37 @@ public class COSPropertiesTest {
origProps.put("cos.endpoint", "cos.ap-beijing.myqcloud.com");
origProps.put("cos.access_key", "myCOSAccessKey");
origProps.put("cos.secret_key", "myCOSSecretKey");
+ origProps.put("test_non_storage_param", "6000");
+ origProps.put("connection.maximum", "88");
+ origProps.put("connection.request.timeout", "100");
+ origProps.put("connection.timeout", "1000");
origProps.put(StorageProperties.FS_COS_SUPPORT, "true");
//origProps.put("cos.region", "ap-beijing");
COSProperties cosProperties = (COSProperties)
StorageProperties.create(origProps).get(1);
Map<String, String> s3Props = new HashMap<>();
cosProperties.toNativeS3Configuration(s3Props);
+ Map<String, String> cosConfig = cosProperties.getOrigProps();
+
Assertions.assertTrue(!cosConfig.containsKey("test_non_storage_param"));
+ origProps.forEach((k, v) -> {
+ if (!k.equals("test_non_storage_param") &&
!k.equals(StorageProperties.FS_COS_SUPPORT)) {
+ Assertions.assertEquals(v, cosConfig.get(k));
+ }
+ });
// Validate the S3 properties
Assertions.assertEquals("cos.ap-beijing.myqcloud.com",
s3Props.get("AWS_ENDPOINT"));
Assertions.assertEquals("ap-beijing", s3Props.get("AWS_REGION"));
Assertions.assertEquals("myCOSAccessKey",
s3Props.get("AWS_ACCESS_KEY"));
Assertions.assertEquals("myCOSSecretKey",
s3Props.get("AWS_SECRET_KEY"));
+ Assertions.assertEquals("88", s3Props.get("AWS_MAX_CONNECTIONS"));
+ Assertions.assertEquals("100", s3Props.get("AWS_REQUEST_TIMEOUT_MS"));
+ Assertions.assertEquals("1000",
s3Props.get("AWS_CONNECTION_TIMEOUT_MS"));
+ Assertions.assertEquals("false", s3Props.get("use_path_style"));
+ origProps.put("use_path_style", "true");
+ cosProperties = (COSProperties)
StorageProperties.create(origProps).get(1);
+ cosProperties.toNativeS3Configuration(s3Props);
+ Assertions.assertEquals("true", s3Props.get("use_path_style"));
// Add any additional assertions for other properties if needed
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
index d4191d4be26..b7f3b521044 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
@@ -67,16 +67,36 @@ public class OBSPropertyTest {
origProps.put("obs.access_key", "myOBSAccessKey");
origProps.put("obs.secret_key", "myOBSSecretKey");
origProps.put("obs.endpoint", "obs.cn-north-4.myhuaweicloud.com");
+ origProps.put("connection.maximum", "88");
+ origProps.put("connection.request.timeout", "100");
+ origProps.put("connection.timeout", "1000");
+ origProps.put("use_path_style", "true");
origProps.put(StorageProperties.FS_OBS_SUPPORT, "true");
OBSProperties obsProperties = (OBSProperties)
StorageProperties.create(origProps).get(1);
Map<String, String> s3Props = new HashMap<>();
+ Map<String, String> obsConfig = obsProperties.getOrigProps();
+
Assertions.assertTrue(!obsConfig.containsKey("test_non_storage_param"));
+ origProps.forEach((k, v) -> {
+ if (!k.equals("test_non_storage_param") &&
!k.equals(StorageProperties.FS_OBS_SUPPORT)) {
+ Assertions.assertEquals(v, obsConfig.get(k));
+ }
+ });
obsProperties.toNativeS3Configuration(s3Props);
Assertions.assertEquals("obs.cn-north-4.myhuaweicloud.com",
s3Props.get("AWS_ENDPOINT"));
Assertions.assertEquals("cn-north-4", s3Props.get("AWS_REGION"));
Assertions.assertEquals("myOBSAccessKey",
s3Props.get("AWS_ACCESS_KEY"));
Assertions.assertEquals("myOBSSecretKey",
s3Props.get("AWS_SECRET_KEY"));
+ Assertions.assertEquals("88", s3Props.get("AWS_MAX_CONNECTIONS"));
+ Assertions.assertEquals("100", s3Props.get("AWS_REQUEST_TIMEOUT_MS"));
+ Assertions.assertEquals("1000",
s3Props.get("AWS_CONNECTION_TIMEOUT_MS"));
+ Assertions.assertEquals("true", s3Props.get("use_path_style"));
+ origProps.remove("use_path_style");
+ obsProperties = (OBSProperties)
StorageProperties.create(origProps).get(1);
+ s3Props = new HashMap<>();
+ obsProperties.toNativeS3Configuration(s3Props);
+ Assertions.assertEquals("false", s3Props.get("use_path_style"));
}
private static String obsAccessKey = "";
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
index 8002f8781fa..e582da4307e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
@@ -63,15 +63,38 @@ public class OSSPropertiesTest {
origProps.put("oss.secret_key", "myOSSSecretKey");
origProps.put("oss.endpoint", "oss-cn-beijing-internal.aliyuncs.com");
origProps.put(StorageProperties.FS_OSS_SUPPORT, "true");
+ origProps.put("connection.maximum", "88");
+ origProps.put("connection.request.timeout", "100");
+ origProps.put("connection.timeout", "1000");
+ origProps.put("use_path_style", "true");
+ origProps.put("test_non_storage_param", "6000");
OSSProperties ossProperties = (OSSProperties)
StorageProperties.create(origProps).get(1);
Map<String, String> s3Props = new HashMap<>();
+ Map<String, String> ossConfig = ossProperties.getOrigProps();
+
Assertions.assertTrue(!ossConfig.containsKey("test_non_storage_param"));
+
+ origProps.forEach((k, v) -> {
+ if (!k.equals("test_non_storage_param") &&
!k.equals(StorageProperties.FS_OSS_SUPPORT)) {
+ Assertions.assertEquals(v, ossConfig.get(k));
+ }
+ });
+
ossProperties.toNativeS3Configuration(s3Props);
Assertions.assertEquals("oss-cn-beijing-internal.aliyuncs.com",
s3Props.get("AWS_ENDPOINT"));
Assertions.assertEquals("cn-beijing-internal",
s3Props.get("AWS_REGION"));
Assertions.assertEquals("myOSSAccessKey",
s3Props.get("AWS_ACCESS_KEY"));
Assertions.assertEquals("myOSSSecretKey",
s3Props.get("AWS_SECRET_KEY"));
+ Assertions.assertEquals("88", s3Props.get("AWS_MAX_CONNECTIONS"));
+ Assertions.assertEquals("100", s3Props.get("AWS_REQUEST_TIMEOUT_MS"));
+ Assertions.assertEquals("1000",
s3Props.get("AWS_CONNECTION_TIMEOUT_MS"));
+ Assertions.assertEquals("true", s3Props.get("use_path_style"));
+ origProps.remove("use_path_style");
+ ossProperties = (OSSProperties)
StorageProperties.create(origProps).get(1);
+ s3Props = new HashMap<>();
+ ossProperties.toNativeS3Configuration(s3Props);
+ Assertions.assertEquals("false", s3Props.get("use_path_style"));
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
index 0e2085b90cc..dd0536f9c82 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
@@ -74,17 +74,42 @@ public class S3PropertiesTest {
origProps.put("s3.secret_key", "myS3SecretKey");
origProps.put("s3.region", "us-west-1");
origProps.put(StorageProperties.FS_S3_SUPPORT, "true");
+ origProps.put("use_path_style", "true");
+ origProps.put("s3.connection.maximum", "88");
+ origProps.put("s3.connection.timeout", "6000");
+ origProps.put("test_non_storage_param", "6000");
+
S3Properties s3Properties = (S3Properties)
StorageProperties.create(origProps).get(1);
Map<String, String> s3Props = new HashMap<>();
s3Properties.toNativeS3Configuration(s3Props);
+ Map<String, String> s3Config = s3Properties.getOrigProps();
+ Assertions.assertTrue(!s3Config.containsKey("test_non_storage_param"));
+
+ origProps.forEach((k, v) -> {
+ if (!k.equals("test_non_storage_param") &&
!k.equals(StorageProperties.FS_S3_SUPPORT)) {
+ Assertions.assertEquals(v, s3Config.get(k));
+ }
+ });
+
// Validate the S3 properties
Assertions.assertEquals("https://cos.example.com";,
s3Props.get("AWS_ENDPOINT"));
Assertions.assertEquals("us-west-1", s3Props.get("AWS_REGION"));
Assertions.assertEquals("myS3AccessKey",
s3Props.get("AWS_ACCESS_KEY"));
Assertions.assertEquals("myS3SecretKey",
s3Props.get("AWS_SECRET_KEY"));
- // Add any additional assertions for other properties if needed
+ Assertions.assertEquals("88", s3Props.get("AWS_MAX_CONNECTIONS"));
+ Assertions.assertEquals("6000",
s3Props.get("AWS_CONNECTION_TIMEOUT_MS"));
+ Assertions.assertEquals("true", s3Props.get("use_path_style"));
+ origProps.remove("use_path_style");
+ origProps.remove("s3.connection.maximum");
+ origProps.remove("s3.connection.timeout");
+ s3Properties = (S3Properties)
StorageProperties.create(origProps).get(1);
+ s3Props = new HashMap<>();
+ s3Properties.toNativeS3Configuration(s3Props);
+ Assertions.assertEquals("false", s3Props.get("use_path_style"));
+ Assertions.assertEquals("50", s3Props.get("AWS_MAX_CONNECTIONS"));
+ Assertions.assertEquals("1000",
s3Props.get("AWS_CONNECTION_TIMEOUT_MS"));
}
/**
@@ -115,8 +140,15 @@ public class S3PropertiesTest {
origProps.put("s3.secret_key", secretKey);
origProps.put("s3.region", "ap-northeast-1");
origProps.put(StorageProperties.FS_S3_SUPPORT, "true");
+ origProps.put("use_path_style", "true");
+ origProps.put("s3.connection.maximum", "88");
+ origProps.put("s3.connection.timeout", "6000");
+ origProps.put("test_non_storage_param", "6000");
S3Properties s3Properties = (S3Properties)
StorageProperties.create(origProps).get(1);
Configuration configuration = s3Properties.getHadoopConfiguration();
+ Assertions.assertEquals("88",
configuration.get("fs.s3a.connection.maximum"));
+ Assertions.assertEquals("6000",
configuration.get("fs.s3a.connection.timeout"));
+ Assertions.assertEquals("6000",
configuration.get("fs.s3a.request.timeout"));
FileSystem fs = FileSystem.get(new URI(hdfsPath), configuration);
FileStatus[] fileStatuses = fs.listStatus(new Path(hdfsPath));
for (FileStatus status : fileStatuses) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java
b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java
index 442883573ce..7691d5580b7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java
@@ -22,6 +22,8 @@ import org.apache.doris.backup.Status;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.S3URI;
import org.apache.doris.datasource.property.PropertyConverter;
+import org.apache.doris.datasource.property.storage.S3Properties;
+import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.fs.remote.RemoteFile;
import org.apache.doris.fs.remote.S3FileSystem;
@@ -102,10 +104,11 @@ public class S3FileSystemTest {
return mockedClient;
}
};
- S3ObjStorage mockedStorage = new S3ObjStorage(properties);
+ S3Properties s3Properties = (S3Properties)
StorageProperties.createStorageProperties(properties);
+ S3ObjStorage mockedStorage = new S3ObjStorage(s3Properties);
Assertions.assertTrue(mockedStorage.getClient() instanceof
MockedS3Client);
// inject storage to file system.
- fileSystem = new S3FileSystem(mockedStorage);
+ fileSystem = new S3FileSystem(s3Properties);
new MockUp<S3FileSystem>(S3FileSystem.class) {
@Mock
public Status globList(String remotePath, List<RemoteFile>
result, boolean fileNameOnly) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3ObjStorageTest.java
b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3ObjStorageTest.java
index e565120600a..3688459f9e4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3ObjStorageTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3ObjStorageTest.java
@@ -19,6 +19,8 @@ package org.apache.doris.fs.obj;
import org.apache.doris.backup.Status;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.S3Properties;
+import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.commons.lang3.StringUtils;
import org.junit.jupiter.api.Assertions;
@@ -56,7 +58,7 @@ class S3ObjStorageTest {
properties.put("s3.access_key", ak);
properties.put("s3.secret_key", sk);
properties.put("s3.region", region);
- S3ObjStorage storage = new S3ObjStorage(properties);
+ S3ObjStorage storage = new S3ObjStorage(null);
String baseUrl = "s3://" + bucket + "/" + prefix + "/";
String content = "mocked";
@@ -109,7 +111,8 @@ class S3ObjStorageTest {
properties.put("s3.endpoint", "s3.e.c");
properties.put("s3.access_key", "abc");
properties.put("s3.secret_key", "123");
- S3ObjStorage storage = new S3ObjStorage(properties);
+ S3Properties s3Properties = (S3Properties)
StorageProperties.createStorageProperties(properties);
+ S3ObjStorage storage = new S3ObjStorage(s3Properties);
Field client = storage.getClass().getDeclaredField("client");
client.setAccessible(true);
MockedS3Client mockedClient = new MockedS3Client();
@@ -145,10 +148,11 @@ class S3ObjStorageTest {
RemoteObject remoteObject = objectList.get(i);
Assertions.assertEquals("key" + i, remoteObject.getRelativePath());
}
+ properties.put("use_path_style", "false");
+ properties.put("s3.endpoint", "oss.a.c");
+ S3Properties newS3Properties = (S3Properties)
StorageProperties.createStorageProperties(properties);
- storage.properties.put("use_path_style", "false");
- storage.properties.put("s3.endpoint", "oss.a.c");
- storage.setProperties(storage.properties);
+ storage.setProperties(newS3Properties);
RemoteObjects remoteObjectsVBucket =
storage.listObjects("oss://bucket/keys", null);
List<RemoteObject> list = remoteObjectsVBucket.getObjectList();
for (int i = 0; i < list.size(); i++) {
@@ -156,8 +160,8 @@ class S3ObjStorageTest {
Assertions.assertTrue(remoteObject.getRelativePath().startsWith("key" + i));
}
- storage.properties.put("use_path_style", "true");
- storage.setProperties(storage.properties);
+ properties.put("use_path_style", "true");
+ storage.setProperties((S3Properties)
StorageProperties.createStorageProperties(properties));
remoteObjectsVBucket = storage.listObjects("oss://bucket/keys", null);
list = remoteObjectsVBucket.getObjectList();
for (int i = 0; i < list.size(); i++) {
diff --git
a/regression-test/suites/refactor_storage_param_p0/backup_restore_cos.groovy
b/regression-test/suites/refactor_storage_param_p0/backup_restore_cos.groovy
new file mode 100644
index 00000000000..884eafe2966
--- /dev/null
+++ b/regression-test/suites/refactor_storage_param_p0/backup_restore_cos.groovy
@@ -0,0 +1,178 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import org.awaitility.Awaitility;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static groovy.test.GroovyAssert.shouldFail
+suite("refactor_storage_backup_restore_cos") {
+ String enabled=
context.config.otherConfigs.get("enableBackUpRestoreCOSTest");
+ if (enabled == null && enabled.equalsIgnoreCase("false")){
+ return
+ }
+ def s3table = "test_backup_restore_cos";
+ sql """
+ drop table if exists ${s3table};
+ """
+ sql """
+ CREATE TABLE ${s3table}(
+ user_id BIGINT NOT NULL COMMENT "user id",
+ name VARCHAR(20) COMMENT "name",
+ age INT COMMENT "age"
+ )
+ DUPLICATE KEY(user_id)
+ DISTRIBUTED BY HASH(user_id) BUCKETS 10
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+ sql """
+ insert into ${s3table} values (1, 'a', 10);
+ """
+
+ def insertResult = sql """
+ SELECT count(1) FROM ${s3table}
+ """
+
+ println "insertResult: ${insertResult}"
+
+ assert insertResult.get(0).get(0) == 1
+ def databaseQueryResult = sql """
+ select database();
+ """
+ println databaseQueryResult
+ def currentDBName = databaseQueryResult.get(0).get(0)
+ println currentDBName
+ // cos
+
+ String objectAccessKey = context.config.otherConfigs.get("cosAK");
+ String objectSecretKey = context.config.otherConfigs.get("cosSK");
+ String objectStorageEndpoint
=context.config.otherConfigs.get("cosEndpoint");
+ String objectStorageRegion = context.config.otherConfigs.get("cosRegion");
+ String objectStorageFilePathPrefix
=context.config.otherConfigs.get("cosFilePathPrefix");
+
+ def objectStorageRepoName = "cos_repo_test_1";
+ try {
+ sql """
+ drop repository ${objectStorageRepoName};
+ """
+ } catch (Exception e) {
+ //ignore exception, repo may not exist
+ }
+
+
+ shouldFail {
+ sql """
+ CREATE REPOSITORY ${objectStorageRepoName}
+ WITH S3
+ ON LOCATION "${objectStorageFilePathPrefix}"
+ PROPERTIES (
+ "s3.endpoint" = "${objectStorageEndpoint}",
+ "s3.region" = "${objectStorageRegion}",
+ "s3.secret_key" = "${objectSecretKey}"
+ );
+ """
+ }
+ // Invalid export path https:// please use valid 's3://' path.
+ shouldFail {
+ objectStorageHttpsFilePathPrefix =
objectStorageFilePathPrefix.replaceAll("^s3://", "https://";);
+ sql """
+ CREATE REPOSITORY ${objectStorageRepoName}_https_prefix
+ WITH S3
+ ON LOCATION "https://${objectStorageHttpsFilePathPrefix}";
+ PROPERTIES (
+ "s3.endpoint" = "${objectStorageEndpoint}",
+ "s3.region" = "${objectStorageRegion}",
+ "s3.access_key" = "${objectAccessKey}",
+ "s3.secret_key" = "${objectSecretKey}"
+ );
+ """
+ }
+ shouldFail {
+ sql """
+ CREATE REPOSITORY ${objectStorageRepoName}
+ WITH S3
+ ON LOCATION "${objectStorageFilePathPrefix}"
+ PROPERTIES (
+ "s3.endpoint" = "${objectStorageEndpoint}",
+ "s3.region" = "${objectStorageRegion}",
+ "s3.secret_key" = "${objectSecretKey}"
+ );
+ """
+ }
+ sql """
+ CREATE REPOSITORY ${objectStorageRepoName}
+ WITH S3
+ ON LOCATION "${objectStorageFilePathPrefix}"
+ PROPERTIES (
+ "s3.endpoint" = "${objectStorageEndpoint}",
+ "s3.region" = "${objectStorageRegion}",
+ "s3.access_key" = "${objectAccessKey}",
+ "s3.secret_key" = "${objectSecretKey}"
+ );
+ """
+
+ def objectStorageBackupLabel = "oss_label_1" + System.currentTimeMillis()
+ sql """
+ BACKUP SNAPSHOT ${currentDBName}.${objectStorageBackupLabel}
+ TO ${objectStorageRepoName}
+ ON (${s3table})
+ PROPERTIES ("type" = "full");
+
+ """
+ Awaitility.await().atMost(60, SECONDS).pollInterval(5, SECONDS).until(
+ {
+ def backupResult = sql """
+ show backup from ${currentDBName} where SnapshotName =
'${objectStorageBackupLabel}';
+ """
+ println "backupResult: ${backupResult}"
+ return backupResult.get(0).get(3) == "FINISHED"
+ })
+
+ def queryCosSnapshotResult = sql """
+ SHOW SNAPSHOT ON ${objectStorageRepoName} WHERE SNAPSHOT =
'${objectStorageBackupLabel}';
+ """
+ println queryCosSnapshotResult
+ def objectSecretSnapshotTime = queryCosSnapshotResult.get(0).get(1)
+ sql """
+ drop table if exists ${s3table};
+ """
+ //restore
+ sql """
+ RESTORE SNAPSHOT ${currentDBName}.${objectStorageBackupLabel}
+ FROM ${objectStorageRepoName}
+ ON (`${s3table}`)
+ PROPERTIES
+ (
+ "backup_timestamp"="${objectSecretSnapshotTime}",
+ "replication_num" = "1"
+ );
+ """
+ Awaitility.await().atMost(60, SECONDS).pollInterval(5, SECONDS).until(
+ {
+ try {
+ def restoreResult = sql """
+ SELECT count(1) FROM ${s3table}
+ """
+ println "restoreResult: ${restoreResult}"
+ return restoreResult.get(0).get(0) == 1
+ } catch (Exception e) {
+ //tbl not found
+ return false
+ }
+ })
+
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
