This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new bee2f157a69 branch-3.1: [test](docker) adapt paimon on hms and
s3/oss/obs/cos #54556 (#54605)
bee2f157a69 is described below
commit bee2f157a6944d7413aefdbaa4fd91dbd7516dad
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Aug 13 14:29:52 2025 +0800
branch-3.1: [test](docker) adapt paimon on hms and s3/oss/obs/cos #54556
(#54605)
Cherry-picked from #54556
Co-authored-by: zgxme <[email protected]>
---
.../docker-compose/hive/hadoop-hive-3x.env.tpl | 19 ++++++++++++
.../docker-compose/hive/hive-3x_settings.env | 16 ++++++++++
.../create_paimon_tables.hql | 35 ++++++++++++++++++++++
.../docker-compose/hive/scripts/hive-metastore.sh | 27 +++++++++++++++++
.../kerberos/entrypoint-hive-master.sh | 15 ++++++++++
.../docker-compose/kerberos/hadoop-hive.env.tpl | 22 +++++++++++++-
.../docker-compose/kerberos/kerberos1_settings.env | 18 ++++++++++-
.../docker-compose/kerberos/kerberos2_settings.env | 18 ++++++++++-
docker/thirdparties/run-thirdparties-docker.sh | 2 ++
9 files changed, 169 insertions(+), 3 deletions(-)
diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive-3x.env.tpl
b/docker/thirdparties/docker-compose/hive/hadoop-hive-3x.env.tpl
index 4d92bab5351..c8afe5046d0 100644
--- a/docker/thirdparties/docker-compose/hive/hadoop-hive-3x.env.tpl
+++ b/docker/thirdparties/docker-compose/hive/hadoop-hive-3x.env.tpl
@@ -20,3 +20,22 @@
HIVE_SITE_CONF_hive_metastore_event_db_notification_api_auth=false
HIVE_SITE_CONF_hive_metastore_dml_events=true
HIVE_SITE_CONF_hive_metastore_transactional_event_listeners=org.apache.hive.hcatalog.listener.DbNotificationListener
HIVE_SITE_CONF_hive_stats_column_autogather=false
+HIVE_SITE_CONF_fs_s3a_impl=org.apache.hadoop.fs.s3a.S3AFileSystem
+HIVE_SITE_CONF_fs_s3a_access_key=${AWSAk}
+HIVE_SITE_CONF_fs_s3a_secret_key=${AWSSk}
+HIVE_SITE_CONF_fs_s3a_endpoint=${AWSEndpoint}
+HIVE_SITE_CONF_fs_AbstractFileSystem_obs_impl=org.apache.hadoop.fs.obs.OBS
+HIVE_SITE_CONF_fs_obs_access_key=${OBSAk}
+HIVE_SITE_CONF_fs_obs_secret_key=${OBSSk}
+HIVE_SITE_CONF_fs_obs_endpoint=${OBSEndpoint}
+HIVE_SITE_CONF_fs_cosn_credentials_provider=org.apache.hadoop.fs.auth.SimpleCredentialProvider
+HIVE_SITE_CONF_fs_cosn_userinfo_secretId=${COSAk}
+HIVE_SITE_CONF_fs_cosn_userinfo_secretKey=${COSSk}
+HIVE_SITE_CONF_fs_cosn_bucket_region=${COSRegion}
+HIVE_SITE_CONF_fs_cosn_impl=org.apache.hadoop.fs.CosFileSystem
+HIVE_SITE_CONF_fs_AbstractFileSystem_cosn_impl=org.apache.hadoop.fs.CosN
+HIVE_SITE_CONF_fs_oss_impl=org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem
+HIVE_SITE_CONF_fs_oss_accessKeyId=${OSSAk}
+HIVE_SITE_CONF_fs_oss_accessKeySecret=${OSSSk}
+HIVE_SITE_CONF_fs_oss_endpoint=${OSSEndpoint}
+enablePaimonHms=${enablePaimonHms}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
index 7322f71f307..21f9d1908c3 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
+++ b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
@@ -24,3 +24,19 @@ export FS_PORT=8320 # should be same as hive3HmsPort in
regression-conf.groovy
export HMS_PORT=9383 # should be same as hive3HmsPort in regression-conf.groovy
export HS_PORT=13000 # should be same as hive3ServerPort in
regression-conf.groovy
export PG_PORT=5732 # should be same as hive3PgPort in regression-conf.groovy
+
+# prepare for paimon hms test,control load paimon hms data or not
+export enablePaimonHms="false"
+# hms on s3/oss/obs/cos
+export AWSAk="*****************"
+export AWSSk="*****************"
+export AWSEndpoint="s3.ap-northeast-1.amazonaws.com"
+export OSSAk="*****************"
+export OSSSk="*****************"
+export OSSEndpoint="oss-cn-beijing.aliyuncs.com"
+export OBSAk="*****************"
+export OBSSk="*****************"
+export OBSEndpoint="obs.cn-north-4.myhuaweicloud.com"
+export COSAk="*****************"
+export COSSk="*****************"
+export COSRegion="ap-beijing"
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/hive/scripts/create_external_paimon_scripts/create_paimon_tables.hql
b/docker/thirdparties/docker-compose/hive/scripts/create_external_paimon_scripts/create_paimon_tables.hql
new file mode 100644
index 00000000000..17526476199
--- /dev/null
+++
b/docker/thirdparties/docker-compose/hive/scripts/create_external_paimon_scripts/create_paimon_tables.hql
@@ -0,0 +1,35 @@
+CREATE DATABASE IF NOT EXISTS hdfs_db;
+USE hdfs_db;
+CREATE TABLE external_test_table(
+ a INT COMMENT 'The a field',
+ b STRING COMMENT 'The b field'
+)
+STORED BY 'org.apache.paimon.hive.PaimonStorageHandler';
+INSERT INTO external_test_table VALUES(11111111, "hdfs_db_test");
+
+SET
hive.metastore.warehouse.dir=oss://doris-regression-bj/regression/paimon_warehouse;
+CREATE DATABASE ali_db;
+USE ali_db;
+CREATE EXTERNAL TABLE external_test_table
+STORED BY 'org.apache.paimon.hive.PaimonStorageHandler'
+LOCATION
'oss://doris-regression-bj/regression/paimon_warehouse/ali_db.db/hive_test_table';
+
+
+SET hive.metastore.warehouse.dir=obs://doris-build/regression/paimon_warehouse;
+CREATE DATABASE hw_db;
+USE hw_db;
+CREATE EXTERNAL TABLE external_test_table
+STORED BY 'org.apache.paimon.hive.PaimonStorageHandler'
+LOCATION
'obs://doris-build/regression/paimon_warehouse/hw_db.db/hive_test_table';
+
+
+SET
hive.metastore.warehouse.dir=cosn://sdb-qa-datalake-test-1308700295/paimon_warehouse;
+CREATE DATABASE tx_db;
+USE tx_db;
+CREATE EXTERNAL TABLE external_test_table
+STORED BY 'org.apache.paimon.hive.PaimonStorageHandler'
+LOCATION
'cosn://sdb-qa-datalake-test-1308700295/paimon_warehouse/tx_db.db/hive_test_table';
+
+
+
+
diff --git a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
index 59c326b2703..0571735c32a 100755
--- a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
+++ b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
@@ -30,6 +30,22 @@ done
ls "${AUX_LIB}/"
cp -r "${AUX_LIB}"/ /opt/hive
+# download auxiliary jars
+cd /opt/hive/auxlib
+curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/jdom-1.1.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-core-3.4.0.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-ecs-4.2.0.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-ram-3.0.0.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-sts-3.0.0.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-sdk-oss-3.4.1.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-aliyun-3.2.1.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/aws-java-sdk-bundle-1.11.375.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-huaweicloud-3.1.1-hw-54.5.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-cos-3.1.0-8.3.22.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/cos_api-bundle-5.6.244.4.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-aws-3.2.1.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/paimon-hive-connector-3.1-1.3-SNAPSHOT.jar
+
nohup /opt/hive/bin/hive --service metastore &
# wait lockfile
@@ -118,6 +134,17 @@ hadoop_put_pids+=($!)
hadoop fs -copyFromLocal -f /mnt/scripts/paimon1 /user/doris/ &
hadoop_put_pids+=($!)
+# create paimon external table
+if [[ ${enablePaimonHms} == "true" ]]; then
+ START_TIME=$(date +%s)
+ hive -f
/mnt/scripts/create_external_paimon_scripts/create_paimon_tables.hql || (echo
"Failed to executing create_paimon_table.hql" && exit 1)
+ END_TIME=$(date +%s)
+ EXECUTION_TIME=$((END_TIME - START_TIME))
+ echo "Script: create_paimon_table.hql executed in $EXECUTION_TIME seconds"
+else
+ echo "enablePaimonHms is false, skip create paimon table"
+fi
+
## put tvf_data
if [[ -z "$(ls /mnt/scripts/tvf_data)" ]]; then
echo "tvf_data does not exist"
diff --git
a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
index f12b76734f7..246e0edab79 100755
--- a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
+++ b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
@@ -40,6 +40,21 @@ else
echo "Invalid index parameter. Exiting."
exit 1
fi
+cd /usr/hdp/3.1.0.0-78/hive/auxlib
+curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/jdom-1.1.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-core-3.4.0.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-ecs-4.2.0.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-ram-3.0.0.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-sts-3.0.0.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-sdk-oss-3.4.1.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-aliyun-3.2.1.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/aws-java-sdk-bundle-1.11.375.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-huaweicloud-3.1.1-hw-54.5.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-cos-3.1.0-8.3.22.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/cos_api-bundle-5.6.244.4.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-aws-3.2.1.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/paimon-hive-connector-3.1-1.3-SNAPSHOT.jar
+
/usr/local/hadoop-run.sh &
# check healthy hear
diff --git a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
index 41c95057a92..f06c1ef7d7a 100644
--- a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
+++ b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
@@ -62,4 +62,24 @@
YARN_CONF_yarn_resourcemanager_resource___tracker_address=${HOST}:${YARN_RM_TRAC
YARN_CONF_yarn_resourcemanager_admin_address=${HOST}:${YARN_RM_ADMIN_PORT}
YARN_CONF_yarn_resourcemanager_webapp_address=${HOST}:${YARN_RM_WEBAPP_PORT}
YARN_CONF_yarn_nodemanager_localizer_address=${HOST}:${YARN_NM_LOCAL_PORT}
-YARN_CONF_yarn_nodemanager_webapp_address=${HOST}:${YARN_NM_WEBAPP_PORT}
\ No newline at end of file
+YARN_CONF_yarn_nodemanager_webapp_address=${HOST}:${YARN_NM_WEBAPP_PORT}
+
+HIVE_SITE_CONF_fs_s3a_impl=org.apache.hadoop.fs.s3a.S3AFileSystem
+HIVE_SITE_CONF_fs_s3a_access_key=${AWSAk}
+HIVE_SITE_CONF_fs_s3a_secret_key=${AWSSk}
+HIVE_SITE_CONF_fs_s3a_endpoint=${AWSEndpoint}
+HIVE_SITE_CONF_fs_AbstractFileSystem_obs_impl=org.apache.hadoop.fs.obs.OBS
+HIVE_SITE_CONF_fs_obs_access_key=${OBSAk}
+HIVE_SITE_CONF_fs_obs_secret_key=${OBSSk}
+HIVE_SITE_CONF_fs_obs_endpoint=${OBSEndpoint}
+HIVE_SITE_CONF_fs_cosn_credentials_provider=org.apache.hadoop.fs.auth.SimpleCredentialProvider
+HIVE_SITE_CONF_fs_cosn_userinfo_secretId=${COSAk}
+HIVE_SITE_CONF_fs_cosn_userinfo_secretKey=${COSSk}
+HIVE_SITE_CONF_fs_cosn_bucket_region=${COSRegion}
+HIVE_SITE_CONF_fs_cosn_impl=org.apache.hadoop.fs.CosFileSystem
+HIVE_SITE_CONF_fs_AbstractFileSystem_cosn_impl=org.apache.hadoop.fs.CosN
+HIVE_SITE_CONF_fs_oss_impl=org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem
+HIVE_SITE_CONF_fs_oss_accessKeyId=${OSSAk}
+HIVE_SITE_CONF_fs_oss_accessKeySecret=${OSSSk}
+HIVE_SITE_CONF_fs_oss_endpoint=${OSSEndpoint}
+enablePaimonHms=${enablePaimonHms}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env
b/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env
index aaf4c639fa8..62de14dfafa 100644
--- a/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env
+++ b/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env
@@ -44,4 +44,20 @@ export KDC_PORT2=5589
export KADMIND_PORT1=5749
export KADMIND_PORT2=5750
export KPASSWD_PORT1=5464
-export KPASSWD_PORT2=5465
\ No newline at end of file
+export KPASSWD_PORT2=5465
+
+# prepare for paimon hms test,control load paimon hms data or not
+export enablePaimonHms="false"
+# hms on s3/oss/obs/cos
+export AWSAk="*****************"
+export AWSSk="*****************"
+export AWSEndpoint="s3.ap-northeast-1.amazonaws.com"
+export OSSAk="*****************"
+export OSSSk="*****************"
+export OSSEndpoint="oss-cn-beijing.aliyuncs.com"
+export OBSAk="*****************"
+export OBSSk="*****************"
+export OBSEndpoint="obs.cn-north-4.myhuaweicloud.com"
+export COSAk="*****************"
+export COSSk="*****************"
+export COSRegion="ap-beijing"
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env
b/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env
index fca68d60162..776fd50eb2d 100644
--- a/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env
+++ b/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env
@@ -40,4 +40,20 @@ export YARN_NM_WEBAPP_PORT=8642
export MAPREDUCE_SHUFFLE_PORT=13662
export KDC_PORT1=6688
export KADMIND_PORT1=6749
-export KPASSWD_PORT1=6464
\ No newline at end of file
+export KPASSWD_PORT1=6464
+
+# prepare for paimon hms test,control load paimon hms data or not
+export enablePaimonHms="false"
+# hms on s3/oss/obs/cos
+export AWSAk="*****************"
+export AWSSk="*****************"
+export AWSEndpoint="s3.ap-northeast-1.amazonaws.com"
+export OSSAk="*****************"
+export OSSSk="*****************"
+export OSSEndpoint="oss-cn-beijing.aliyuncs.com"
+export OBSAk="*****************"
+export OBSSk="*****************"
+export OBSEndpoint="obs.cn-north-4.myhuaweicloud.com"
+export COSAk="*****************"
+export COSSk="*****************"
+export COSRegion="ap-beijing"
\ No newline at end of file
diff --git a/docker/thirdparties/run-thirdparties-docker.sh
b/docker/thirdparties/run-thirdparties-docker.sh
index 0c7241ea141..74c3c302655 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -623,6 +623,8 @@ start_kerberos() {
export IP_HOST=${IP_HOST}
export CONTAINER_UID=${CONTAINER_UID}
envsubst <"${ROOT}"/docker-compose/kerberos/kerberos.yaml.tpl
>"${ROOT}"/docker-compose/kerberos/kerberos.yaml
+ sed -i "s/s3Endpoint/${s3Endpoint}/g"
"${ROOT}"/docker-compose/kerberos/entrypoint-hive-master.sh
+ sed -i "s/s3BucketName/${s3BucketName}/g"
"${ROOT}"/docker-compose/kerberos/entrypoint-hive-master.sh
for i in {1..2}; do
. "${ROOT}"/docker-compose/kerberos/kerberos${i}_settings.env
envsubst <"${ROOT}"/docker-compose/kerberos/hadoop-hive.env.tpl
>"${ROOT}"/docker-compose/kerberos/hadoop-hive-${i}.env
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
