This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 4be107a3a74 [Fix](PaimonCatalog) fix the problem that paimon catalog
can not access to OSS-HDFS #42585 (#43309)
4be107a3a74 is described below
commit 4be107a3a7472132719348c5aa7f07a107395fbe
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Wed Nov 6 15:08:26 2024 +0800
[Fix](PaimonCatalog) fix the problem that paimon catalog can not access to
OSS-HDFS #42585 (#43309)
cherry pick from #42585
Co-authored-by: Tiewei Fang <[email protected]>
---
.../org/apache/doris/common/util/LocationPath.java | 16 ++++++++++++++--
.../datasource/paimon/PaimonFileExternalCatalog.java | 19 +++++++++++++------
.../apache/doris/common/util/LocationPathTest.java | 4 +++-
3 files changed, 30 insertions(+), 9 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
index 267e20a1f95..4604e4deabb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
@@ -131,7 +131,19 @@ public class LocationPath {
tmpLocation = convertPath ? convertToS3(tmpLocation) :
tmpLocation;
break;
case FeConstants.FS_PREFIX_OSS:
- if (isHdfsOnOssEndpoint(tmpLocation)) {
+ String endpoint = "";
+ if (props.containsKey(OssProperties.ENDPOINT)) {
+ endpoint = props.get(OssProperties.ENDPOINT);
+ if (endpoint.startsWith(OssProperties.OSS_PREFIX)) {
+ // may use oss.oss-cn-beijing.aliyuncs.com
+ endpoint = endpoint.replace(OssProperties.OSS_PREFIX,
"");
+ }
+ } else if (props.containsKey(S3Properties.ENDPOINT)) {
+ endpoint = props.get(S3Properties.ENDPOINT);
+ } else if (props.containsKey(S3Properties.Env.ENDPOINT)) {
+ endpoint = props.get(S3Properties.Env.ENDPOINT);
+ }
+ if (isHdfsOnOssEndpoint(endpoint)) {
this.scheme = Scheme.OSS_HDFS;
} else {
if (useS3EndPoint(props)) {
@@ -398,7 +410,7 @@ public class LocationPath {
}
}
- private FileSystemType getFileSystemType() {
+ public FileSystemType getFileSystemType() {
FileSystemType fsType;
switch (scheme) {
case S3:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
index 9b956a551d5..e74f3deeaf5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
@@ -17,9 +17,11 @@
package org.apache.doris.datasource.paimon;
+import org.apache.doris.common.util.LocationPath;
import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.datasource.property.constants.CosProperties;
import org.apache.doris.datasource.property.constants.ObsProperties;
+import org.apache.doris.datasource.property.constants.OssProperties;
import org.apache.doris.datasource.property.constants.PaimonProperties;
import org.apache.logging.log4j.LogManager;
@@ -53,12 +55,17 @@ public class PaimonFileExternalCatalog extends
PaimonExternalCatalog {
options.put(PaimonProperties.PAIMON_S3_SECRET_KEY,
properties.get(PaimonProperties.PAIMON_S3_SECRET_KEY));
} else if
(properties.containsKey(PaimonProperties.PAIMON_OSS_ENDPOINT)) {
- options.put(PaimonProperties.PAIMON_OSS_ENDPOINT,
- properties.get(PaimonProperties.PAIMON_OSS_ENDPOINT));
- options.put(PaimonProperties.PAIMON_OSS_ACCESS_KEY,
- properties.get(PaimonProperties.PAIMON_OSS_ACCESS_KEY));
- options.put(PaimonProperties.PAIMON_OSS_SECRET_KEY,
- properties.get(PaimonProperties.PAIMON_OSS_SECRET_KEY));
+ boolean hdfsEnabled = Boolean.parseBoolean(properties.getOrDefault(
+ OssProperties.OSS_HDFS_ENABLED, "false"));
+ if
(!LocationPath.isHdfsOnOssEndpoint(properties.get(PaimonProperties.PAIMON_OSS_ENDPOINT))
+ && !hdfsEnabled) {
+ options.put(PaimonProperties.PAIMON_OSS_ENDPOINT,
+ properties.get(PaimonProperties.PAIMON_OSS_ENDPOINT));
+ options.put(PaimonProperties.PAIMON_OSS_ACCESS_KEY,
+
properties.get(PaimonProperties.PAIMON_OSS_ACCESS_KEY));
+ options.put(PaimonProperties.PAIMON_OSS_SECRET_KEY,
+
properties.get(PaimonProperties.PAIMON_OSS_SECRET_KEY));
+ }
} else if (properties.containsKey(CosProperties.ENDPOINT)) {
options.put(PaimonProperties.PAIMON_S3_ENDPOINT,
properties.get(CosProperties.ENDPOINT));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
index 23f052d6131..9d1edadd919 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
@@ -19,6 +19,7 @@ package org.apache.doris.common.util;
import org.apache.doris.catalog.HdfsResource;
import org.apache.doris.common.util.LocationPath.Scheme;
+import org.apache.doris.datasource.property.constants.OssProperties;
import org.apache.doris.fs.FileSystemType;
import org.junit.jupiter.api.Assertions;
@@ -119,13 +120,14 @@ public class LocationPathTest {
Assertions.assertTrue(beLocation.startsWith("s3://"));
Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
null).first, FileSystemType.S3);
+ rangeProps.put(OssProperties.ENDPOINT, "oss-dls.aliyuncs.com");
locationPath = new
LocationPath("oss://test.oss-dls.aliyuncs.com/path", rangeProps);
// FE
Assertions.assertTrue(locationPath.get().startsWith("oss://test.oss-dls.aliyuncs"));
// BE
beLocation = locationPath.toStorageLocation().toString();
Assertions.assertTrue(beLocation.startsWith("oss://test.oss-dls.aliyuncs"));
- Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
null).first, FileSystemType.DFS);
+ Assertions.assertEquals(locationPath.getFileSystemType(),
FileSystemType.DFS);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
