This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 45e2fae64b3 branch-4.0: [paimon](obs) support pfs use in paimon
catalog #60995 (#61217)
45e2fae64b3 is described below
commit 45e2fae64b3cbb8fb982c0bdf4a2b28e3ffef898
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Mar 12 10:09:14 2026 +0800
branch-4.0: [paimon](obs) support pfs use in paimon catalog #60995 (#61217)
Cherry-picked from #60995
Co-authored-by: zhangstar333 <[email protected]>
---
fe/be-java-extensions/hadoop-deps/pom.xml | 12 ++++++++++++
.../datasource/property/storage/OBSProperties.java | 20 +++++++++++++++++++-
2 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/fe/be-java-extensions/hadoop-deps/pom.xml
b/fe/be-java-extensions/hadoop-deps/pom.xml
index 102a92d356e..54591563091 100644
--- a/fe/be-java-extensions/hadoop-deps/pom.xml
+++ b/fe/be-java-extensions/hadoop-deps/pom.xml
@@ -71,6 +71,10 @@ under the License.
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-protobuf_3_25</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.huaweicloud</groupId>
+ <artifactId>hadoop-huaweicloud</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
@@ -95,6 +99,14 @@ under the License.
</dependency>
</dependencies>
+ <repositories>
+ <!-- for huawei obs sdk -->
+ <repository>
+ <id>huawei-obs-sdk</id>
+
<url>https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/</url>
+ </repository>
+ </repositories>
+
<build>
<plugins>
<plugin>
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
index 25afb2bbf03..237fc67fc88 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
@@ -168,10 +168,28 @@ public class OBSProperties extends
AbstractS3CompatibleProperties {
return null;
}
+ private static final boolean OBS_FILE_SYSTEM_AVAILABLE =
+ isClassAvailable("org.apache.hadoop.fs.obs.OBSFileSystem");
+
+ private static boolean isClassAvailable(String className) {
+ try {
+ Class.forName(className, false,
OBSProperties.class.getClassLoader());
+ return true;
+ } catch (ClassNotFoundException e) {
+ return false;
+ }
+ }
+
@Override
public void initializeHadoopStorageConfig() {
super.initializeHadoopStorageConfig();
- hadoopStorageConfig.set("fs.obs.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem");
+ // obs is not compatible with s3a well; prefer native OBSFileSystem if
available on the classpath
+ if (OBS_FILE_SYSTEM_AVAILABLE) {
+ hadoopStorageConfig.set("fs.obs.impl",
"org.apache.hadoop.fs.obs.OBSFileSystem");
+ hadoopStorageConfig.set("fs.AbstractFileSystem.obs.impl",
"org.apache.hadoop.fs.obs.OBS");
+ } else {
+ hadoopStorageConfig.set("fs.obs.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem");
+ }
hadoopStorageConfig.set("fs.obs.access.key", accessKey);
hadoopStorageConfig.set("fs.obs.secret.key", secretKey);
hadoopStorageConfig.set("fs.obs.endpoint", endpoint);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
