This is an automated email from the ASF dual-hosted git repository.
zhangstar333 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1f3daebd49a [chore](ci) support custom conf for ms and recycler
(#31558)
1f3daebd49a is described below
commit 1f3daebd49aaec9b98637f1759e59d68dadb40d0
Author: Dongyang Li <[email protected]>
AuthorDate: Fri Mar 1 14:16:32 2024 +0800
[chore](ci) support custom conf for ms and recycler (#31558)
* [chore](ci) support custom conf for ms and recycler
Co-authored-by: stephen <[email protected]>
---
regression-test/pipeline/cloud_p0/conf/ms_custom.conf | 2 ++
regression-test/pipeline/cloud_p0/conf/recycler_custom.conf | 2 ++
regression-test/pipeline/cloud_p0/deploy.sh | 9 +++++----
regression-test/pipeline/cloud_p0/run.sh | 6 +++---
regression-test/pipeline/cloud_p1/conf/ms_custom.conf | 1 +
regression-test/pipeline/cloud_p1/conf/recycler_custom.conf | 2 ++
regression-test/pipeline/common/doris-utils.sh | 13 +++++++++++--
regression-test/pipeline/performance/deploy.sh | 2 +-
regression-test/pipeline/performance/run-clickbench.sh | 2 +-
regression-test/pipeline/performance/run-load.sh | 2 +-
regression-test/pipeline/performance/run-tpcds.sh | 2 +-
regression-test/pipeline/performance/run-tpch.sh | 2 +-
12 files changed, 31 insertions(+), 14 deletions(-)
diff --git a/regression-test/pipeline/cloud_p0/conf/ms_custom.conf
b/regression-test/pipeline/cloud_p0/conf/ms_custom.conf
new file mode 100644
index 00000000000..1c88dc87938
--- /dev/null
+++ b/regression-test/pipeline/cloud_p0/conf/ms_custom.conf
@@ -0,0 +1,2 @@
+# below lines will be appended to the default doris_cloud.conf when deploying
meta service
+meta_schema_value_version = 1
diff --git a/regression-test/pipeline/cloud_p0/conf/recycler_custom.conf
b/regression-test/pipeline/cloud_p0/conf/recycler_custom.conf
new file mode 100644
index 00000000000..62deff2b870
--- /dev/null
+++ b/regression-test/pipeline/cloud_p0/conf/recycler_custom.conf
@@ -0,0 +1,2 @@
+# below lines will be appended to the default doris_cloud.conf when deploying
recycler
+brpc_listen_port = 6000
diff --git a/regression-test/pipeline/cloud_p0/deploy.sh
b/regression-test/pipeline/cloud_p0/deploy.sh
index dc3ed41c8d9..788d5f24457 100644
--- a/regression-test/pipeline/cloud_p0/deploy.sh
+++ b/regression-test/pipeline/cloud_p0/deploy.sh
@@ -61,10 +61,11 @@ exit_flag=0
cp -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/cloud_p0/conf/be_custom.conf
"${DORIS_HOME}"/be/conf/
fdb_cluster="$(cat /etc/foundationdb/fdb.cluster)"
sed -i "s/^fdb_cluster = .*/fdb_cluster = ${fdb_cluster}/"
"${DORIS_HOME}"/ms/conf/doris_cloud.conf
- # this is a temporary config, need to replace in
cloud_p0/conf/doris_cloud.conf
- echo "meta_schema_value_version = 1"
>>"${DORIS_HOME}"/ms/conf/doris_cloud.conf
sed -i "s/^fdb_cluster = .*/fdb_cluster = ${fdb_cluster}/"
"${DORIS_HOME}"/recycler/conf/doris_cloud.conf
- sed -i "s/^brpc_listen_port = .*/fbrpc_listen_port = 6000/"
"${DORIS_HOME}"/recycler/conf/doris_cloud.conf
+ cat
"${teamcity_build_checkoutDir}"/regression-test/pipeline/cloud_p0/conf/ms_custom.conf
>>"${DORIS_HOME}"/ms/conf/doris_cloud.conf
+ echo >>"${DORIS_HOME}"/ms/conf/doris_cloud.conf
+ cat
"${teamcity_build_checkoutDir}"/regression-test/pipeline/cloud_p0/conf/recycler_custom.conf
>>"${DORIS_HOME}"/recycler/conf/doris_cloud.conf
+ echo >>"${DORIS_HOME}"/recycler/conf/doris_cloud.conf
print_doris_conf
echo "#### 4. start Doris"
@@ -93,7 +94,7 @@ if [[ ${exit_flag} != "0" ]]; then
stop_doris
print_doris_fe_log
print_doris_be_log
- if file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_doris_logs.tar.gz"); then
+ if file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_logs.tar.gz"); then
upload_doris_log_to_oss "${file_name}"
fi
fi
diff --git a/regression-test/pipeline/cloud_p0/run.sh
b/regression-test/pipeline/cloud_p0/run.sh
index 4d2094fee19..4cfe066d1d0 100644
--- a/regression-test/pipeline/cloud_p0/run.sh
+++ b/regression-test/pipeline/cloud_p0/run.sh
@@ -81,7 +81,7 @@ run() {
}
export -f run
# 设置超时时间（以分为单位）
-timeout_minutes=$((${repeat_times_from_trigger:-1} * 90))m
+timeout_minutes=$((${repeat_times_from_trigger:-1} *
${BUILD_TIMEOUT_MINUTES:-180}))m
timeout "${timeout_minutes}" bash -cx run
exit_flag="$?"
@@ -91,10 +91,10 @@ if [[ ${exit_flag} != "0" ]]; then
stop_doris
print_doris_fe_log
print_doris_be_log
- if file_name=$(archive_doris_coredump
"${pr_num_from_trigger}_${commit_id_from_trigger}_doris_coredump.tar.gz"); then
+ if file_name=$(archive_doris_coredump
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_coredump.tar.gz"); then
upload_doris_log_to_oss "${file_name}"
fi
- if file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_doris_logs.tar.gz"); then
+ if file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_logs.tar.gz"); then
upload_doris_log_to_oss "${file_name}"
fi
fi
diff --git a/regression-test/pipeline/cloud_p1/conf/ms_custom.conf
b/regression-test/pipeline/cloud_p1/conf/ms_custom.conf
new file mode 100644
index 00000000000..d5056833bac
--- /dev/null
+++ b/regression-test/pipeline/cloud_p1/conf/ms_custom.conf
@@ -0,0 +1 @@
+# below lines will be appended to the default doris_cloud.conf when deploying
meta service
diff --git a/regression-test/pipeline/cloud_p1/conf/recycler_custom.conf
b/regression-test/pipeline/cloud_p1/conf/recycler_custom.conf
new file mode 100644
index 00000000000..62deff2b870
--- /dev/null
+++ b/regression-test/pipeline/cloud_p1/conf/recycler_custom.conf
@@ -0,0 +1,2 @@
+# below lines will be appended to the default doris_cloud.conf when deploying
recycler
+brpc_listen_port = 6000
diff --git a/regression-test/pipeline/common/doris-utils.sh
b/regression-test/pipeline/common/doris-utils.sh
index dc1de6e270c..ab2f3a86e54 100644
--- a/regression-test/pipeline/common/doris-utils.sh
+++ b/regression-test/pipeline/common/doris-utils.sh
@@ -453,7 +453,9 @@ function set_doris_session_variables_from_file() {
archive_doris_logs() {
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
local archive_name="$1"
- if [[ -z ${archive_name} ]]; then echo "ERROR: archive file name required"
&& return 1; fi
+ if [[ -z ${archive_name} || ${archive_name} != *".tar.gz" ]]; then
+ echo "USAGE: ${FUNCNAME[0]} xxxx.tar.gz" && return 1
+ fi
local archive_dir="${archive_name%.tar.gz}"
rm -rf "${DORIS_HOME:?}/${archive_dir}"
mkdir -p "${DORIS_HOME}/${archive_dir}"
@@ -468,12 +470,18 @@ archive_doris_logs() {
if sed -i "s/${cos_ak:-}//g;s/${cos_sk:-}//g"
regression-test/log/* &>/dev/null; then :; fi
cp --parents -rf "regression-test/log" "${archive_dir}"/
fi
+ if [[ -d "${DORIS_HOME}"/../regression-test/conf ]]; then
+ # try to hide ak and sk
+ if sed -i "s/${cos_ak:-}//g;s/${cos_sk:-}//g"
../regression-test/conf/* &>/dev/null; then :; fi
+ mkdir -p "${archive_dir}"/regression-test/conf
+ cp -rf ../regression-test/conf/*
"${archive_dir}"/regression-test/conf/
+ fi
if [[ -f "${DORIS_HOME}"/session_variables ]]; then
cp --parents -rf "session_variables" "${archive_dir}"/
fi
if [[ -d "${DORIS_HOME}"/ms ]]; then
mkdir -p "${archive_dir}"/foundationdb/log
- cp --parents -rf /var/log/foundationdb/*
"${archive_dir}"/foundationdb/log/
+ cp -rf /var/log/foundationdb/* "${archive_dir}"/foundationdb/log/
cp --parents -rf "ms/conf" "${archive_dir}"/
cp --parents -rf "ms/log" "${archive_dir}"/
fi
@@ -551,6 +559,7 @@ archive_doris_coredump() {
--directory "${DORIS_HOME}" \
-cf "${DORIS_HOME}/${archive_name}" \
"${archive_dir}"; then
+ rm -rf "${DORIS_HOME:?}/${archive_dir}"
echo "${DORIS_HOME}/${archive_name}"
else
return 1
diff --git a/regression-test/pipeline/performance/deploy.sh
b/regression-test/pipeline/performance/deploy.sh
index 7d8e44544b3..dfcf1ecbabd 100644
--- a/regression-test/pipeline/performance/deploy.sh
+++ b/regression-test/pipeline/performance/deploy.sh
@@ -90,7 +90,7 @@ if [[ ${exit_flag} != "0" ]]; then
stop_doris
print_doris_fe_log
print_doris_be_log
- if file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_doris_logs.tar.gz"); then
+ if file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_logs.tar.gz"); then
upload_doris_log_to_oss "${file_name}"
fi
fi
diff --git a/regression-test/pipeline/performance/run-clickbench.sh
b/regression-test/pipeline/performance/run-clickbench.sh
index cd19b1d3aa0..6ed29a464f9 100644
--- a/regression-test/pipeline/performance/run-clickbench.sh
+++ b/regression-test/pipeline/performance/run-clickbench.sh
@@ -289,7 +289,7 @@ if [[ ${exit_flag} != "0" ]]; then
stop_doris
print_doris_fe_log
print_doris_be_log
- if file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_doris_logs.tar.gz"); then
+ if file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_logs.tar.gz"); then
upload_doris_log_to_oss "${file_name}"
fi
fi
diff --git a/regression-test/pipeline/performance/run-load.sh
b/regression-test/pipeline/performance/run-load.sh
index fbd60920cbe..57fa5b49f2d 100644
--- a/regression-test/pipeline/performance/run-load.sh
+++ b/regression-test/pipeline/performance/run-load.sh
@@ -694,7 +694,7 @@ if [[ ${exit_flag} != "0" ]]; then
stop_doris
print_doris_fe_log
print_doris_be_log
- if file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_doris_logs.tar.gz"); then
+ if file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_logs.tar.gz"); then
upload_doris_log_to_oss "${file_name}"
fi
fi
diff --git a/regression-test/pipeline/performance/run-tpcds.sh
b/regression-test/pipeline/performance/run-tpcds.sh
index 72244764aad..5c24c7ab9c5 100644
--- a/regression-test/pipeline/performance/run-tpcds.sh
+++ b/regression-test/pipeline/performance/run-tpcds.sh
@@ -153,7 +153,7 @@ if [[ ${exit_flag} != "0" ]]; then
stop_doris
print_doris_fe_log
print_doris_be_log
- if file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_doris_logs.tar.gz"); then
+ if file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_logs.tar.gz"); then
upload_doris_log_to_oss "${file_name}"
fi
fi
diff --git a/regression-test/pipeline/performance/run-tpch.sh
b/regression-test/pipeline/performance/run-tpch.sh
index a9d1a9519f3..86ea13b6a0c 100644
--- a/regression-test/pipeline/performance/run-tpch.sh
+++ b/regression-test/pipeline/performance/run-tpch.sh
@@ -157,7 +157,7 @@ if [[ ${exit_flag} != "0" ]]; then
stop_doris
print_doris_fe_log
print_doris_be_log
- if file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_doris_logs.tar.gz"); then
+ if file_name=$(archive_doris_logs
"${pr_num_from_trigger}_${commit_id_from_trigger}_$(date
+%Y%m%d%H%M%S)_doris_logs.tar.gz"); then
upload_doris_log_to_oss "${file_name}"
fi
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
