This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 4d5328930b KYLIN-5954 Optimize Bootstrap
4d5328930b is described below
commit 4d5328930bc2dcfd14fb3c3f250d6fd3da3c5342
Author: zhong.zhu <[email protected]>
AuthorDate: Thu Aug 29 17:10:42 2024 +0800
KYLIN-5954 Optimize Bootstrap
---------
Co-authored-by: jiale.he <[email protected]>
Co-authored-by: junqing.cai <[email protected]>
---
build/bin/get-properties.sh | 189 ++++++++++++++++++---
build/bin/kylin.sh | 108 ++++++++++--
build/sbin/bootstrap.sh | 176 +++++++++++++------
build/sbin/check-1600-spark-dir.sh | 8 +-
build/sbin/check-2000-zookeeper-role.sh | 6 +-
build/sbin/header.sh | 10 ++
build/sbin/init-kerberos.sh | 30 ++--
build/sbin/load-zookeeper-config.sh | 8 +-
build/sbin/rotate-logs.sh | 34 ++--
.../resources/metadata-jdbc-postgresql.properties | 3 +
10 files changed, 437 insertions(+), 135 deletions(-)
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index eea27b4ed7..031db36f3e 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -17,42 +17,181 @@
# limitations under the License.
#
-if [ $# != 1 ]
-then
- if [[ $# < 2 || $2 != 'DEC' ]]
- then
- echo 'invalid input'
- exit 1
- fi
+if [[ -z $KYLIN_HOME ]]; then
+ export KYLIN_HOME=$(cd -P -- "$(dirname -- "$0")"/../ && pwd -P)
fi
-if [ -z $KYLIN_HOME ];then
- export KYLIN_HOME=$(cd -P -- "$(dirname -- "$0")"/../ && pwd -P)
-fi
+# tmp config file
+_KYLIN_CACHED_CONFIG_FILE=${_KYLIN_CACHED_CONFIG_FILE:-"${KYLIN_HOME}/conf/._kylin_properties_"}
+# read the configuration from the local cache (true | false)
+GET_PROPERTIES_FROM_LOCAL_CACHE=${_KYLIN_GET_PROPERTIES_FROM_LOCAL_CACHE}
+
+function verboseLog() {
+ echo $(date '+%F %H:%M:%S') "$@" >>"${KYLIN_HOME}"/logs/shell.stderr
+}
+
+function quit() {
+ verboseLog "$@"
+ exit 1
+}
-if [ -z $MAPR_HOME ];then
+function help() {
+ echo "Usage: get-properties <COMMAND>"
+ echo
+ echo "Commands:"
+ echo " -c get properties from cache(local disk)"
+ echo " -r get properties in real time, this options support
backwards config"
+ echo " -e [file_path]only export properties to cache(local disk)"
+ echo " -h print help"
+ echo " interactive Enter to get kylin properties"
+}
+
+function runTool() {
+
+ if [[ -z ${MAPR_HOME} ]]; then
export MAPR_HOME="/opt/mapr"
-fi
+ fi
-if [ -z ${kylin_hadoop_conf_dir} ]; then
+ if [[ -z ${kylin_hadoop_conf_dir} ]]; then
export kylin_hadoop_conf_dir=$KYLIN_HOME/hadoop_conf
-fi
+ fi
-export KYLIN_KERBEROS_OPTS=""
-if [ -f ${KYLIN_HOME}/conf/krb5.conf ];then
+ local KYLIN_KERBEROS_OPTS=""
+ if [[ -f ${KYLIN_HOME}/conf/krb5.conf ]]; then
KYLIN_KERBEROS_OPTS="-Djava.security.krb5.conf=${KYLIN_HOME}/conf/krb5.conf"
-fi
+ fi
-export SPARK_HOME=$KYLIN_HOME/spark
+ local SPARK_HOME=${KYLIN_HOME}/spark
-if [[ -f ${KYLIN_HOME}/conf/kylin-tools-log4j.xml ]]; then
+ local kylin_tools_log4j=""
+ if [[ -f ${KYLIN_HOME}/conf/kylin-tools-log4j.xml ]]; then
kylin_tools_log4j="file:${KYLIN_HOME}/conf/kylin-tools-log4j.xml"
- else
+ else
kylin_tools_log4j="file:${KYLIN_HOME}/tool/conf/kylin-tools-log4j.xml"
-fi
+ fi
+
+ mkdir -p "${KYLIN_HOME}"/logs
+ local result=$(java ${KYLIN_KERBEROS_OPTS}
-Dlog4j.configurationFile=${kylin_tools_log4j}
-Dkylin.hadoop.conf.dir=${kylin_hadoop_conf_dir} -Dhdp.version=current -cp
"${kylin_hadoop_conf_dir}:${KYLIN_HOME}/lib/ext/*:${KYLIN_HOME}/server/jars/*:${SPARK_HOME}/jars/*"
"$@" 2>>${KYLIN_HOME}/logs/shell.stderr)
+ echo "$result"
+}
+
+# get properties from cache(local disk)
+function getPropertiesFromLocal() {
+ if [[ $# != 1 ]]; then
+ echo 'Invalid input from local'
+ exit 1
+ fi
+
+ if [[ "${GET_PROPERTIES_FROM_LOCAL_CACHE}" == "true" && -e
${_KYLIN_CACHED_CONFIG_FILE} ]]; then
+ local inputP=$1
+ if [[ ${inputP} = kap.* ]]; then
+ quit "local properties doesn't support kap.*"
+ elif [[ ${inputP} = *. ]]; then
+ grep -F "${inputP}" "${_KYLIN_CACHED_CONFIG_FILE}" | sed -e
"s/${inputP}//g"
+ else
+ grep -F "${inputP}=" "${_KYLIN_CACHED_CONFIG_FILE}" | cut -d "=" -f 2-
+ verboseLog "getProperties ${inputP} from local success"
+ fi
+ else
+ if [[ "${GET_PROPERTIES_FROM_LOCAL_CACHE}" != true ]]; then
+ verboseLog "Please turn
_KYLIN_GET_PROPERTIES_FROM_LOCAL_CACHE:${GET_PROPERTIES_FROM_LOCAL_CACHE} to
true."
+ else
+ verboseLog "Cannot find local cache file:${_KYLIN_CACHED_CONFIG_FILE}!"
+ fi
+ return 1
+ fi
+}
+
+# get properties in real time
+function getPropertiesFromKylin() {
+ if [ $# != 1 ]; then
+ if [[ $# -lt 2 || $2 != 'DEC' ]]; then
+ echo 'Invalid input'
+ exit 1
+ fi
+ fi
+ runTool org.apache.kylin.tool.KylinConfigCLI "$@"
+}
+
+# 1. export properties to tmp file ${_KYLIN_CACHED_CONFIG_FILE}.tmp
+# 2. rename .tmp file
+function exportPropertiesToLocal() {
+ if [[ "${GET_PROPERTIES_FROM_LOCAL_CACHE}" != "true" ]]; then
+ quit "cannot export properties to local when
_KYLIN_GET_PROPERTIES_FROM_LOCAL_CACHE=false"
+ fi
+
+ local export_file=${1:-${_KYLIN_CACHED_CONFIG_FILE}}
+ local tmp_config_file="${export_file}.tmp"
+
+ if [[ -e ${tmp_config_file} ]]; then
+ verboseLog "tmp file already exist, try to remove it.
path:${tmp_config_file}"
+ rm -f "${tmp_config_file}"
+ fi
+
+ # export config file
+ local result=$(runTool org.apache.kylin.tool.KylinConfigExporterCLI
"${tmp_config_file}")
+ verboseLog "${result:-"export success"}"
+
+ if [[ $? -ne 0 || ! -e ${tmp_config_file} ]]; then
+ quit "export properties failed"
+ else
+ # rename
+ mv "${tmp_config_file}" "${export_file}"
+ if [[ $? -ne 0 || ! -e ${export_file} ]]; then
+ quit "mv properties failed"
+ fi
+ fi
+}
+
+function getProperties() {
+ if [[ "${GET_PROPERTIES_FROM_LOCAL_CACHE}" == "true" ]]; then
+ getPropertiesFromLocal "$@"
+ if [[ $? -ne 0 ]]; then
+ verboseLog "get from cache failed, try to get in real time again" "$@"
+ GET_PROPERTIES_FROM_LOCAL_CACHE=""
+ getPropertiesFromKylin "$@"
+ fi
+ else
+ getPropertiesFromKylin "$@"
+ fi
+}
+
+function main() {
+ if [[ $# == 0 ]]; then
+ help
+ exit 0
+ fi
-mkdir -p ${KYLIN_HOME}/logs
-export KYLIN_EXTERNAL_CONFIG_INITIALIZED=false
-result=`java ${KYLIN_KERBEROS_OPTS}
-Dlog4j.configurationFile=${kylin_tools_log4j}
-Dkylin.hadoop.conf.dir=${kylin_hadoop_conf_dir} -Dhdp.version=current -cp
"${kylin_hadoop_conf_dir}:${KYLIN_HOME}/lib/ext/*:${KYLIN_HOME}/server/jars/*:${SPARK_HOME}/jars/*"
org.apache.kylin.tool.KylinConfigCLI $@ 2>>${KYLIN_HOME}/logs/shell.stderr`
+ while getopts "c:r:eh" opt; do
+ case ${opt} in
+ c)
+ shift
+ GET_PROPERTIES_FROM_LOCAL_CACHE="true"
+ getPropertiesFromLocal "$@" || echo ""
+ exit $?
+ ;;
+ r)
+ shift
+ GET_PROPERTIES_FROM_LOCAL_CACHE=""
+ getProperties "$@"
+ exit $?
+ ;;
+ e)
+ shift
+ exportPropertiesToLocal "$@"
+ exit $?
+ ;;
+ h)
+ help
+ exit $?
+ ;;
+ *)
+ verboseLog "Invalid option: -$OPTARG"
+ exit 1
+ ;;
+ esac
+ done
+ getProperties "$@"
+}
-echo "$result"
+main "$@"
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 654096dc9e..8fe5e8b011 100755
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -17,14 +17,100 @@
# limitations under the License.
#
-source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/../sbin/header.sh "$@"
-mkdir -p ${KYLIN_HOME}/logs
-ERR_LOG=${KYLIN_HOME}/logs/shell.stderr
-OUT_LOG=${KYLIN_HOME}/logs/shell.stdout
-echo "----------------------- log start -----------------------" >>${ERR_LOG}
-echo "----------------------- log start -----------------------" >>${OUT_LOG}
-bash -x ${KYLIN_HOME}/sbin/bootstrap.sh "$@" 2>>${ERR_LOG} | tee -a ${OUT_LOG}
-ret=${PIPESTATUS[0]}
-echo "----------------------- log end -------------------------" >>${ERR_LOG}
-echo "----------------------- log end -------------------------" >>${OUT_LOG}
-exit ${ret}
\ No newline at end of file
+function help() {
+ echo "Usage: kylin.sh <COMMAND>"
+ echo
+ echo "Commands:"
+ echo " -D -D[ skipCheck ]"
+ echo " skip some check when bootstrap, eg. bash
kylin.sh -DskipCheck start"
+ echo " -C -C[ true | false ], default true, use the
local properties or not"
+ echo " start start kylin"
+ echo " restart restart kylin"
+ echo " stop stop kylin"
+ echo " io.kyligence.* run tool"
+ echo " interactive Enter for bootstrap"
+}
+
+function parseSkipCheckArgs() {
+ case $1 in
+ "skipCheck")
+ skipValue=1
+ ;;
+ *)
+ echo "Invalid option -D: -$1" >&2
+ exit 1
+ ;;
+ esac
+ echo ${skipValue}
+}
+
+function checkArguments() {
+ local local_properties="false"
+ # enabled when check env by pass
+ if [[ -f ${KYLIN_HOME}/bin/check-env-bypass ]]; then
+ local_properties="true"
+ fi
+
+ case $1 in
+ "start" | "restart")
+ export
_KYLIN_GET_PROPERTIES_FROM_LOCAL_CACHE=${_KYLIN_GET_PROPERTIES_FROM_LOCAL_CACHE:-"${local_properties}"}
+ exportPropertiesToFile
+ "${KYLIN_HOME}"/sbin/rotate-logs.sh "$@"
+ export KYLIN_SKIP_ROTATE_LOG=1
+ ;;
+ "stop")
+ export
_KYLIN_GET_PROPERTIES_FROM_LOCAL_CACHE=${_KYLIN_GET_PROPERTIES_FROM_LOCAL_CACHE:-"${local_properties}"}
+ ;;
+ "spawn")
+ export
_KYLIN_GET_PROPERTIES_FROM_LOCAL_CACHE=${_KYLIN_GET_PROPERTIES_FROM_LOCAL_CACHE:-"${local_properties}"}
+ "${KYLIN_HOME}"/sbin/rotate-logs.sh "$@"
+ export KYLIN_SKIP_ROTATE_LOG=1
+ ;;
+ *) ;;
+ esac
+}
+
+function main() {
+ # parsed arguments
+ while getopts "vD:C:h" opt; do
+ case ${opt} in
+ v)
+ export verbose=true
+ ;;
+ D)
+ skipArgs=$(parseSkipCheckArgs "$OPTARG") || exit 1
+ export KYLIN_SKIP_CHECK=${skipArgs}
+ ;;
+ C)
+ export _KYLIN_GET_PROPERTIES_FROM_LOCAL_CACHE="$OPTARG"
+ ;;
+ h)
+ help
+ exit 0
+ ;;
+ *)
+ echo "Invalid option: -$OPTARG" && exit 1
+ ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ # init
+ source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/../sbin/header.sh "$@"
+ mkdir -p "${KYLIN_HOME}"/logs
+ ERR_LOG=${KYLIN_HOME}/logs/shell.stderr
+ OUT_LOG=${KYLIN_HOME}/logs/shell.stdout
+
+ # check action arguments
+ checkArguments "$@"
+
+ echo "----------------------- log start -----------------------"
>>${ERR_LOG}
+ echo "----------------------- log start -----------------------"
>>${OUT_LOG}
+ bash -x "${KYLIN_HOME}"/sbin/bootstrap.sh "$@" 2>>"${ERR_LOG}" | tee -a
${OUT_LOG}
+ ret=${PIPESTATUS[0]}
+ echo "----------------------- log end -------------------------"
>>${ERR_LOG}
+ echo "----------------------- log end -------------------------"
>>${OUT_LOG}
+ exit ${ret}
+}
+
+main "$@"
diff --git a/build/sbin/bootstrap.sh b/build/sbin/bootstrap.sh
index 6c8480051b..60cd8fb8e4 100755
--- a/build/sbin/bootstrap.sh
+++ b/build/sbin/bootstrap.sh
@@ -18,17 +18,17 @@
#
source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh $@
-version=`cat ${KYLIN_HOME}/VERSION | awk '{print $3}'`
-${KYLIN_HOME}/sbin/rotate-logs.sh $@
+source "${KYLIN_HOME}"/sbin/init-customer-env.sh
+version=$(cat "${KYLIN_HOME}"/VERSION | awk '{print $3}')
+"${KYLIN_HOME}"/sbin/rotate-logs.sh "$@"
+
+export KYLIN_SKIP_CHECK=${KYLIN_SKIP_CHECK:-0}
+KYLIN_SKIP_CHECK_MODE=1
if [ "$1" == "-v" ]; then
shift
fi
-KYLIN_ENV_CHANNEL=`$KYLIN_HOME/bin/get-properties.sh kylin.env.channel`
-SPARK_SCHEDULER_MODE=`$KYLIN_HOME/bin/get-properties.sh
kylin.query.engine.spark-scheduler-mode`
-MAX_CONCURRENT_JOBS=`$KYLIN_HOME/bin/get-properties.sh
kylin.job.max-concurrent-jobs`
-
if [[ $(hadoop version 2>/dev/null) == *"mapr"* ]]; then
MAPR_AUTHENTICATION="-Djava.security.auth.login.config=${MAPR_HOME}/conf/mapr.login.conf"
fi
@@ -37,7 +37,14 @@ if [ "${SPARK_SCHEDULER_MODE}" == "" ] || [[
"${SPARK_SCHEDULER_MODE}" != "FAIR"
SPARK_SCHEDULER_MODE="FAIR"
fi
-function prepareEnv {
+function prepareEnv() {
+ # avoid re-entering
+ if [[ -n $SKIP_PRE_ENV ]]; then
+ return
+ fi
+
+ SKIP_PRE_ENV=1
+
export KYLIN_CONFIG_FILE="${KYLIN_HOME}/conf/kylin.properties"
export SPARK_HOME=${KYLIN_HOME}/spark
@@ -68,45 +75,73 @@ function retrieveDependency() {
}
function checkRestPort() {
- used=`netstat -tpln | grep "\<$port\>" | awk '{print $7}' | sed "s/\// /g"`
+ SERVER_PORT=${SERVER_PORT:-$("$KYLIN_HOME"/bin/get-properties.sh
server.port)}
+ local used=$(netstat -tpln | grep "\<$SERVER_PORT\>" | awk '{print $7}' |
sed "s/\// /g")
if [ ! -z "$used" ]; then
- echo "<$used> already listen on $port"
- exit -1
+ echo "<$used> already listen on $SERVER_PORT"
+ exit 1
+ fi
+}
+
+function skipCheckOrNot() {
+ if [[ $KYLIN_SKIP_CHECK -ge $1 ]]; then
+ echo "true"
+ else
+ echo ""
fi
}
-function checkZookeeperRole {
- source ${KYLIN_HOME}/sbin/check-2000-zookeeper-role.sh
+function checkZookeeperRole() {
+ # this is necessary in FI
+ source "${KYLIN_HOME}"/sbin/load-zookeeper-config.sh
+ if [[ $(skipCheckOrNot $KYLIN_SKIP_CHECK_MODE) ]]; then
+ return 0
+ fi
+ verboseLog "checking zookeeper role"
+ source "${KYLIN_HOME}"/sbin/check-2000-zookeeper-role.sh
}
function checkSparkDir() {
- source ${KYLIN_HOME}/sbin/check-1600-spark-dir.sh
-}
+ if [[ $(skipCheckOrNot $KYLIN_SKIP_CHECK_MODE) ]]; then
+ return 0
+ fi
-function checkHiveDirAcl() {
- source ${KYLIN_HOME}/sbin/check-2100-hive-acl.sh
+ if [[ ${KYLIN_ENV_CHANNEL} == "on-premises" || -z ${KYLIN_ENV_CHANNEL} ]];
then
+ verboseLog "checking spark dir"
+ source "${KYLIN_HOME}"/sbin/check-1600-spark-dir.sh
+ fi
}
function checkIfStopUserSameAsStartUser() {
- startUser=`ps -p $1 -o user=`
- currentUser=`whoami`
+ if [[ `skipCheckOrNot $KYLIN_SKIP_CHECK_MODE` ]]; then
+ return 0
+ fi
+ verboseLog "checking stop user"
+
+ startUser=$(ps -p "$1" -o user=)
+ currentUser=$(whoami)
- if [ ${startUser} != ${currentUser} ]; then
- echo `setColor 33 "Warning: You started Kylin as user [${startUser}],
please stop the instance as the same user."`
+ if [[ ${startUser} != "${currentUser}" ]]; then
+ echo $(setColor 33 "Warning: You started Kylin as user [${startUser}],
please stop the instance as the same user.")
fi
}
function quit {
echo "$@"
if [[ -n "${QUIT_MESSAGE_LOG}" ]]; then
- echo `setColor 31 "$@"` >> ${QUIT_MESSAGE_LOG}
+ echo $(setColor 31 "$@") >> "${QUIT_MESSAGE_LOG}"
fi
exit 1
}
function prepareFairScheduler() {
- cat > ${KYLIN_HOME}/conf/fairscheduler.xml <<EOL
+ local spark_scheduler_mode=$("$KYLIN_HOME"/bin/get-properties.sh
kylin.query.engine.spark-scheduler-mode)
+ if [ "${spark_scheduler_mode}" == "" ] || [[ "${spark_scheduler_mode}" !=
"FAIR" && "${spark_scheduler_mode}" != "SJF" ]]; then
+ spark_scheduler_mode="FAIR"
+ fi
+
+ cat > "${KYLIN_HOME}"/conf/fairscheduler.xml <<EOL
<?xml version="1.0"?>
<!--
@@ -169,12 +204,13 @@ function runToolInternal() {
else
kylin_tools_log4j="file:${KYLIN_HOME}/tool/conf/kylin-tools-log4j.xml"
fi
- java -Xms${JAVA_VM_TOOL_XMS} -Xmx${JAVA_VM_TOOL_XMX}
${KYLIN_KERBEROS_OPTS} ${MAPR_AUTHENTICATION} -Dfile.encoding=UTF-8
-Dlog4j.configurationFile=${kylin_tools_log4j}
-Dkylin.hadoop.conf.dir=${kylin_hadoop_conf_dir} -Dhdp.version=current -cp
"${kylin_hadoop_conf_dir}:${KYLIN_HOME}/conf/:${KYLIN_HOME}/lib/ext/*:${KYLIN_HOME}/server/jars/*:${SPARK_HOME}/jars/*"
"$@"
+ java -Xms${JAVA_VM_TOOL_XMS} -Xmx${JAVA_VM_TOOL_XMX}
${KYLIN_KERBEROS_OPTS} -Dfile.encoding=UTF-8
-Dlog4j.configurationFile=${kylin_tools_log4j}
-Dkylin.hadoop.conf.dir=${kylin_hadoop_conf_dir} -Dhdp.version=current -cp
"${kylin_hadoop_conf_dir}:${KYLIN_HOME}/conf/:${KYLIN_HOME}/lib/ext/*:${KYLIN_HOME}/server/jars/*:${SPARK_HOME}/jars/*"
"$@"
}
function killChildProcess {
if [ -f "${KYLIN_HOME}/child_process" ]
then
+ MAX_CONCURRENT_JOBS=$("$KYLIN_HOME"/bin/get-properties.sh
kylin.job.max-concurrent-jobs)
count=0
while childPid='' read -r line || [[ -n "$line" ]]; do
# only kill orphan processes and spark-submit processes
@@ -208,6 +244,17 @@ function killChildProcess {
}
function clearRedundantProcess {
+ if [[ $(skipCheckOrNot $KYLIN_SKIP_CHECK_MODE) ]]; then
+ return 0
+ fi
+
+ verboseLog "checking redundant process"
+
+ #sleep or not
+ if [[ -n $1 ]]; then
+ sleep "$1"
+ fi
+
if [ -f "${KYLIN_HOME}/pid" ]
then
pidKeep=0
@@ -240,6 +287,38 @@ function clearRedundantProcess {
fi
}
+function checkKylinMetaList() {
+ if [[ $(skipCheckOrNot $KYLIN_SKIP_CHECK_MODE) ]]; then
+ return 0
+ fi
+
+ verboseLog "checking kylin meta"
+
+ runToolInternal
org.apache.kylin.tool.upgrade.UpdateSessionTableColumnLengthCLI
+
+ runToolInternal org.apache.kylin.tool.security.AdminUserInitCLI
+
+}
+
+function checkLog4jConf() {
+ if [[ -f ${KYLIN_HOME}/conf/kylin-server-log4j.xml ]]; then
+ KYLIN_SERVER_LOG4J="file:${KYLIN_HOME}/conf/kylin-server-log4j.xml"
+ else
+
KYLIN_SERVER_LOG4J="file:${KYLIN_HOME}/server/conf/kylin-server-log4j.xml"
+ fi
+}
+
+function checkTimeZone() {
+ TIME_ZONE=$("$KYLIN_HOME"/bin/get-properties.sh kylin.web.timezone)
+ if [[ -n ${TIME_ZONE} ]]; then
+ TIME_ZONE="-Duser.timezone=${TIME_ZONE}"
+ fi
+}
+
+function checkEnv() {
+ "${KYLIN_HOME}"/bin/check-env.sh "if-not-yet" || exit 1
+}
+
function clearCrontab() {
if ! isCrontabUseable;then
return 1
@@ -283,7 +362,7 @@ function startKylin(){
fi
fi
- ${KYLIN_HOME}/bin/check-env.sh "if-not-yet" || exit 1
+ checkEnv
START_TIME=$(date "+%Y-%m-%d %H:%M:%S")
@@ -293,54 +372,39 @@ function startKylin(){
prepareEnv
- cd ${KYLIN_HOME}/server
- source ${KYLIN_HOME}/sbin/load-zookeeper-config.sh
- fetchFIZkInfo
-
prepareFairScheduler
- port=`$KYLIN_HOME/bin/get-properties.sh server.port`
+ #this is necessary in FI
+ source "${KYLIN_HOME}"/sbin/load-zookeeper-config.sh
+
if [[ -f ${KYLIN_HOME}/bin/check-env-bypass ]]; then
checkRestPort
checkZookeeperRole
fi
- if [[ ${KYLIN_ENV_CHANNEL} == "on-premises" || -z ${KYLIN_ENV_CHANNEL} ]];
then
- checkSparkDir
- fi
+ checkSparkDir
- checkHiveDirAcl
+ checkKylinMetaList
- runToolInternal org.apache.kylin.tool.security.AdminUserInitCLI
- if [[ $? == 1 ]]; then
- quit "Create Admin user failed, for more details please refer to
\"\$KYLIN_HOME/logs/shell.stderr\"."
- fi
+ checkLog4jConf
- if [[ -f ${KYLIN_HOME}/conf/kylin-server-log4j.xml ]]; then
- kylin_server_log4j="file:${KYLIN_HOME}/conf/kylin-server-log4j.xml"
- else
-
kylin_server_log4j="file:${KYLIN_HOME}/server/conf/kylin-server-log4j.xml"
- fi
+ checkTimeZone
- TIME_ZONE=`${KYLIN_HOME}/bin/get-properties.sh kylin.web.timezone`
- if [[ -n ${TIME_ZONE} ]]; then
- TIME_ZONE="-Duser.timezone=${TIME_ZONE}"
- fi
+ SERVER_MODE=$("${KYLIN_HOME}"/bin/get-properties.sh kylin.server.mode)
- SERVER_MODE=`${KYLIN_HOME}/bin/get-properties.sh kylin.server.mode`
+ cd "${KYLIN_HOME}"/server
+ nohup java ${KYLIN_KERBEROS_OPTS} ${KYLIN_EXTRA_START_OPTS} ${TIME_ZONE}
-Dfile.encoding=UTF-8 -Dlogging.path=${KYLIN_HOME}/logs
-Dspring.profiles.active=prod -Dlogging.config=${KYLIN_SERVER_LOG4J}
-Dkylin.hadoop.conf.dir=${kylin_hadoop_conf_dir} -Dhdp.version=current
-Dloader.path="${kylin_hadoop_conf_dir},${KYLIN_HOME}/conf,${KYLIN_HOME}/lib/ext,${KYLIN_HOME}/server/jars,${SPARK_HOME}/jars"
-XX:OnOutOfMemoryError="sh ${KYLIN_HOME}/bin/guardian.sh kill" -jar
newten.jar --kylin.serv [...]
- nohup java ${KYLIN_KERBEROS_OPTS} ${KYLIN_EXTRA_START_OPTS} ${TIME_ZONE}
-Dfile.encoding=UTF-8 -Dlogging.path=${KYLIN_HOME}/logs
-Dspring.profiles.active=prod -Dlogging.config=${kylin_server_log4j}
-Dkylin.hadoop.conf.dir=${kylin_hadoop_conf_dir} -Dhdp.version=current
-Dloader.path="${kylin_hadoop_conf_dir},${KYLIN_HOME}/conf,${KYLIN_HOME}/lib/ext,${KYLIN_HOME}/server/jars,${SPARK_HOME}/jars"
-XX:OnOutOfMemoryError="sh ${KYLIN_HOME}/bin/guardian.sh kill" -jar
newten.jar --kylin.serv [...]
- sleep 3
- clearRedundantProcess
+ clearRedundantProcess 3
- PID=`cat ${KYLIN_HOME}/pid`
+ PID=$(cat "${KYLIN_HOME}"/pid)
CUR_DATE=$(date "+%Y-%m-%d %H:%M:%S")
echo $CUR_DATE" new Kylin process pid is "$PID >>
${KYLIN_HOME}/logs/kylin.log
- sh ${KYLIN_HOME}/bin/guardian.sh start
+ sh "${KYLIN_HOME}"/bin/guardian.sh start
- echo "Kylin is starting. It may take a while. For status, please visit
http://`hostname`:$port/kylin/index.html.";
- echo "You may also check status via: PID:`cat ${KYLIN_HOME}/pid`, or Log:
${KYLIN_HOME}/logs/kylin.log."
+ echo "Kylin is starting. It may take a while. For status, please visit
http://$(hostname):$SERVER_PORT/kylin/index.html."
+ echo "You may also check status via: PID:$(cat "${KYLIN_HOME}"/pid), or
Log: ${KYLIN_HOME}/logs/kylin.log."
recordKylinStartOrStop "start success" "${START_TIME}"
}
@@ -384,8 +448,8 @@ function stopKylin(){
function recordKylinStartOrStop() {
currentIp=$(ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep
-Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1' | head -n 1)
- serverPort=`$KYLIN_HOME/bin/get-properties.sh server.port`
- echo `date '+%Y-%m-%d %H:%M:%S '`"INFO : [Operation: $1] user:`whoami`,
start time:$2, ip and port:${currentIp}:${serverPort}" >>
${KYLIN_HOME}/logs/security.log
+ SERVER_PORT=${SERVER_PORT:-$("$KYLIN_HOME"/bin/get-properties.sh
server.port)}
+ echo $(date '+%Y-%m-%d %H:%M:%S ')"INFO : [Operation: $1] user:$(whoami),
start time:$2, ip and port:${currentIp}:${SERVER_PORT}" >>
"${KYLIN_HOME}"/logs/security.log
}
if [[ "$1" == org.apache.kylin.* ]]; then
diff --git a/build/sbin/check-1600-spark-dir.sh
b/build/sbin/check-1600-spark-dir.sh
index 45ed7007f4..b0041dfc11 100755
--- a/build/sbin/check-1600-spark-dir.sh
+++ b/build/sbin/check-1600-spark-dir.sh
@@ -20,8 +20,8 @@
#title=Checking Spark Dir
source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
-source ${KYLIN_HOME}/sbin/prepare-hadoop-conf-dir.sh
-source ${KYLIN_HOME}/sbin/init-kerberos.sh
+source "${KYLIN_HOME}"/sbin/prepare-hadoop-conf-dir.sh
+source "${KYLIN_HOME}"/sbin/init-kerberos.sh
## init Kerberos if needed
initKerberosIfNeeded
@@ -66,8 +66,8 @@ if [[ ${spark_log_dir} != ${spark_eventlog_dir} ]]; then
checkDirExistAndPermission ${spark_eventlog_dir} "spark history event log"
fi
-sparder_log_dir=`$KYLIN_HOME/bin/get-properties.sh
kap.storage.columnar.spark-conf.spark.history.fs.logDirectory`
-sparder_eventlog_dir=`$KYLIN_HOME/bin/get-properties.sh
kap.storage.columnar.spark-conf.spark.eventLog.dir`
+sparder_log_dir=`$KYLIN_HOME/bin/get-properties.sh
kylin.storage.columnar.spark-conf.spark.history.fs.logDirectory`
+sparder_eventlog_dir=`$KYLIN_HOME/bin/get-properties.sh
kylin.storage.columnar.spark-conf.spark.eventLog.dir`
checkDirExistAndPermission ${sparder_log_dir} "sparder history log"
diff --git a/build/sbin/check-2000-zookeeper-role.sh
b/build/sbin/check-2000-zookeeper-role.sh
index 96f69ad457..3121c99553 100755
--- a/build/sbin/check-2000-zookeeper-role.sh
+++ b/build/sbin/check-2000-zookeeper-role.sh
@@ -21,13 +21,9 @@
source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
-source ${KYLIN_HOME}/sbin/init-kerberos.sh
-## init Kerberos if needed
-initKerberosIfNeeded
-
echo "Checking Zookeeper role..."
-zk_connect_string=`${KYLIN_HOME}/bin/get-properties.sh
kylin.env.zookeeper-connect-string`
+zk_connect_string=$("${KYLIN_HOME}"/bin/get-properties.sh
kylin.env.zookeeper-connect-string)
if [[ -z $zk_connect_string ]]; then
quit "Failed: Zookeeper connect string is empty, please set
'kylin.env.zookeeper-connect-string' in {KYLIN_HOME}/conf/kylin.properties"
diff --git a/build/sbin/header.sh b/build/sbin/header.sh
index 1d679135a5..04ed913494 100755
--- a/build/sbin/header.sh
+++ b/build/sbin/header.sh
@@ -32,6 +32,16 @@ function isValidJavaVersion() {
echo "true"
}
+function verboseLog() {
+ (>&2 echo $(date '+%F %H:%M:%S') "$@")
+}
+
+function exportPropertiesToFile() {
+ if [[ "${_KYLIN_GET_PROPERTIES_FROM_LOCAL_CACHE}" == "true" ]]; then
+ "${KYLIN_HOME}"/bin/get-properties.sh -e || quit "export properties
error"
+ fi
+}
+
# avoid re-entering
if [[ "$dir" == "" ]]
then
diff --git a/build/sbin/init-kerberos.sh b/build/sbin/init-kerberos.sh
index 4667368161..6a00f55e98 100644
--- a/build/sbin/init-kerberos.sh
+++ b/build/sbin/init-kerberos.sh
@@ -16,8 +16,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
-KAP_KERBEROS_ENABLED=`$KYLIN_HOME/bin/get-properties.sh kylin.kerberos.enabled`
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh "$@"
+KAP_KERBEROS_ENABLED=$("$KYLIN_HOME"/bin/get-properties.sh
kylin.kerberos.enabled)
function is_kap_kerberos_enabled(){
if [[ "${KAP_KERBEROS_ENABLED}" == "true" ]];then
@@ -30,11 +30,11 @@ function is_kap_kerberos_enabled(){
}
function exportKRB5() {
- KAP_KERBEROS_CACHE=`$KYLIN_HOME/bin/get-properties.sh kylin.kerberos.cache`
+ KAP_KERBEROS_CACHE=$("$KYLIN_HOME"/bin/get-properties.sh
kylin.kerberos.cache)
export KRB5CCNAME=${KYLIN_HOME}"/conf/"${KAP_KERBEROS_CACHE}
- KAP_KERBEROS_KRB5=`$KYLIN_HOME/bin/get-properties.sh
kylin.kerberos.krb5-conf`
- if [ ! -n "$KAP_KERBEROS_KRB5" ]; then
+ KAP_KERBEROS_KRB5=$("$KYLIN_HOME"/bin/get-properties.sh
kylin.kerberos.krb5-conf)
+ if [ -z "$KAP_KERBEROS_KRB5" ]; then
quit "kylin.kerberos.krb5-conf cannot be set to empty in
kylin.properties"
fi
@@ -50,24 +50,24 @@ function prepareKerberosOpts() {
}
function initKerberos() {
- KAP_KERBEROS_PRINCIPAL=`$KYLIN_HOME/bin/get-properties.sh
kylin.kerberos.principal`
- KAP_KERBEROS_KEYTAB=`$KYLIN_HOME/bin/get-properties.sh
kylin.kerberos.keytab`
+ KAP_KERBEROS_PRINCIPAL=$("$KYLIN_HOME"/bin/get-properties.sh
kylin.kerberos.principal)
+ KAP_KERBEROS_KEYTAB=$("$KYLIN_HOME"/bin/get-properties.sh
kylin.kerberos.keytab)
KAP_KERBEROS_KEYTAB_PATH=${KYLIN_HOME}"/conf/"${KAP_KERBEROS_KEYTAB}
- if [ ! -e ${KRB5_CONFIG} ]; then
+ if [[ ! -e ${KRB5_CONFIG} ]]; then
quit "${KRB5_CONFIG} file doesn't exist"
fi
echo "Kerberos is enabled, init..."
- kinit -kt $KAP_KERBEROS_KEYTAB_PATH $KAP_KERBEROS_PRINCIPAL
+ kinit -kt "$KAP_KERBEROS_KEYTAB_PATH" "$KAP_KERBEROS_PRINCIPAL"
}
function prepareJaasConf() {
- if [ -f ${KYLIN_HOME}/conf/jaas.conf ]; then
+ if [[ -f ${KYLIN_HOME}/conf/jaas.conf ]]; then
return
fi
- cat > ${KYLIN_HOME}/conf/jaas.conf <<EOL
+ cat > "${KYLIN_HOME}"/conf/jaas.conf <<EOL
Client{
com.sun.security.auth.module.Krb5LoginModule required
useKeyTab=false
@@ -79,15 +79,15 @@ EOL
}
function prepareZKPrincipal() {
- params=`env | grep "HADOOP_OPTS"`
+ params=$(env | grep "HADOOP_OPTS")
splitParams=(${params//'-D'/ })
for param in ${splitParams[@]}
do
if [[ "$param" == zookeeper* ]];then
infos=(${param//'zookeeper.server.principal='/ })
envZKPrincipal=${infos[0]}
- zkPrincipal=`$KYLIN_HOME/bin/get-properties.sh
kylin.kerberos.zookeeper-server-principal`
- if [ $zkPrincipal != $envZKPrincipal ]
+ zkPrincipal=$("$KYLIN_HOME"/bin/get-properties.sh
kylin.kerberos.zookeeper-server-principal)
+ if [[ $zkPrincipal != "$envZKPrincipal" ]]
then
sed -i '/kap.kerberos.zookeeper.server.principal/d'
${KYLIN_CONFIG_FILE}
sed -i '/kylin.kerberos.zookeeper-server-principal/d'
${KYLIN_CONFIG_FILE}
@@ -99,7 +99,7 @@ function prepareZKPrincipal() {
function prepareFIKerberosInfoIfNeeded() {
prepareJaasConf
- KERBEROS_PALTFORM=`$KYLIN_HOME/bin/get-properties.sh
kylin.kerberos.platform`
+ KERBEROS_PALTFORM=$("$KYLIN_HOME"/bin/get-properties.sh
kylin.kerberos.platform)
if [[ "${KERBEROS_PALTFORM}" == "FI" || "${KERBEROS_PALTFORM}" == "TDH" ]]
then
prepareZKPrincipal
diff --git a/build/sbin/load-zookeeper-config.sh
b/build/sbin/load-zookeeper-config.sh
index ec1f3ba825..72f70983f1 100644
--- a/build/sbin/load-zookeeper-config.sh
+++ b/build/sbin/load-zookeeper-config.sh
@@ -20,15 +20,17 @@
function fetchFIZkInfo(){
if [ -n "$FI_ENV_PLATFORM" ]
then
- CLIENT_HIVE_URI=`env | grep "CLIENT_HIVE_URI"`
+ CLIENT_HIVE_URI=$(env | grep "CLIENT_HIVE_URI")
array=(${CLIENT_HIVE_URI//// })
verbose "FI_ZK_CONNECT_INFO is:${array[1]}"
FI_ZK_CONNECT_INFO=${array[1]}
- zkConnectString=`$KYLIN_HOME/bin/get-properties.sh
kylin.env.zookeeper-connect-string`
+ zkConnectString=$("$KYLIN_HOME"/bin/get-properties.sh
kylin.env.zookeeper-connect-string)
if [ -z "$zkConnectString" ]
then
sed -i
'$a\kylin.env.zookeeper-connect-string='$FI_ZK_CONNECT_INFO''
${KYLIN_CONFIG_FILE}
fi
fi
-}
\ No newline at end of file
+}
+
+fetchFIZkInfo
diff --git a/build/sbin/rotate-logs.sh b/build/sbin/rotate-logs.sh
index 18584a8836..fa7f0efd39 100755
--- a/build/sbin/rotate-logs.sh
+++ b/build/sbin/rotate-logs.sh
@@ -21,7 +21,7 @@ source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh $@
function checkFileOccupied() {
target_file=$1
- pids="`fuser $target_file 2>&1`"
+ pids="$(fuser "$target_file" 2>&1)"
if [[ "${pids}" == "" ]]; then
echo false
else
@@ -31,8 +31,8 @@ function checkFileOccupied() {
function checkSizeExceedLimit() {
target_file=$1
- file_threshold=`${KYLIN_HOME}/bin/get-properties.sh
kylin.env.max-keep-log-file-threshold-mb`
- file_size=`du -b "$target_file" | cut -f 1`
+ file_threshold=$("${KYLIN_HOME}"/bin/get-properties.sh
kylin.env.max-keep-log-file-threshold-mb)
+ file_size=$(du -b "$target_file" | cut -f 1)
let file_threshold=file_threshold*1024*1024
if [[ ${file_size} -gt ${file_threshold} ]]; then
echo true
@@ -44,14 +44,14 @@ function checkSizeExceedLimit() {
function logRotate() {
target_file=$1
# keep 10 history log files
- keep_limit=`${KYLIN_HOME}/bin/get-properties.sh
kylin.env.max-keep-log-file-number`
+ keep_limit=$("${KYLIN_HOME}"/bin/get-properties.sh
kylin.env.max-keep-log-file-number)
- is_occupied=`checkFileOccupied ${target_file}`
+ is_occupied=$(checkFileOccupied "${target_file}")
if [[ "${is_occupied}" == "true" ]]; then
return
fi
- is_too_large=`checkSizeExceedLimit ${target_file}`
+ is_too_large=$(checkSizeExceedLimit "${target_file}")
if [[ "${is_too_large}" == "false" ]]; then
return
fi
@@ -59,21 +59,21 @@ function logRotate() {
if [[ -f $target_file ]]; then
if [[ -f ${target_file}.${keep_limit} ]]; then
# clean oldest log file first
- rm -f ${target_file}.${keep_limit}
+ rm -f "${target_file}"."${keep_limit}"
fi
let p_cnt=keep_limit-1
# renames logs .1 trough .${keep_limit}
while [[ $keep_limit -gt 1 ]]; do
- if [ -f ${target_file}.${p_cnt} ] ; then
- mv -f ${target_file}.${p_cnt} ${target_file}.${keep_limit}
+ if [ -f "${target_file}"."${p_cnt}" ] ; then
+ mv -f "${target_file}"."${p_cnt}"
"${target_file}"."${keep_limit}"
fi
let keep_limit=keep_limit-1
let p_cnt=p_cnt-1
done
# rename current log to .1
- mv -f $target_file $target_file.1
+ mv -f "$target_file" "$target_file".1
fi
}
@@ -81,9 +81,11 @@ ERR_LOG=${KYLIN_HOME}/logs/shell.stderr
OUT_LOG=${KYLIN_HOME}/logs/shell.stdout
KYLIN_OUT=${KYLIN_HOME}/logs/kylin.out
-if [ "$1" == "start" ] || [ "$1" == "spawn" ]
-then
- logRotate $ERR_LOG
- logRotate $OUT_LOG
- logRotate $KYLIN_OUT
-fi
\ No newline at end of file
+if [ "$1" == "start" ] || [ "$1" == "spawn" ]; then
+ # avoid re-entering
+ if [[ -z ${KYLIN_SKIP_ROTATE_LOG} ]]; then
+ logRotate "$ERR_LOG"
+ logRotate "$OUT_LOG"
+ logRotate "$KYLIN_OUT"
+ fi
+fi
diff --git
a/src/core-common/src/main/resources/metadata-jdbc-postgresql.properties
b/src/core-common/src/main/resources/metadata-jdbc-postgresql.properties
index 3726ed7840..59dc3bcb12 100644
--- a/src/core-common/src/main/resources/metadata-jdbc-postgresql.properties
+++ b/src/core-common/src/main/resources/metadata-jdbc-postgresql.properties
@@ -637,6 +637,9 @@ create.queryhistoryrealization.store.table=CREATE TABLE IF
NOT EXISTS %s ( \
duration BIGINT, \
query_time BIGINT, \
project_name VARCHAR(255), \
+ query_first_day_of_month BIGINT, \
+ query_first_day_of_week BIGINT, \
+ query_day BIGINT, \
primary key(id , project_name) \
);
