This is an automated email from the ASF dual-hosted git repository. dbecker pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 23a14a249c783a9dfb38586052513c6455ce3dad Author: zhangyifan27 <[email protected]> AuthorDate: Thu Feb 29 15:36:27 2024 +0800 IMPALA-12852: Make Kudu service start and stop independent This patch decouples run-kudu.sh and kill-kudu.sh from run-mini-dfs.sh and kill-mini-dfs.sh. These scripts can be useful for setting up test environments that require no or only Kudu service. Testing: - Ran the modified and new scripts and checked they worked as expected. Change-Id: I9624aaa61353bb4520e879570e5688d5e3493201 Reviewed-on: http://gerrit.cloudera.org:8080/21090 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- testdata/bin/kill-kudu.sh | 21 +++++++++++++++ testdata/bin/run-all.sh | 8 +++--- testdata/bin/run-kudu.sh | 42 +++++++++++++++++++++++++++++ testdata/cluster/admin | 68 ++++++++++++++++++++++++++++++++++++++--------- 4 files changed, 123 insertions(+), 16 deletions(-) diff --git a/testdata/bin/kill-kudu.sh b/testdata/bin/kill-kudu.sh new file mode 100755 index 000000000..916e22629 --- /dev/null +++ b/testdata/bin/kill-kudu.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +$IMPALA_HOME/testdata/cluster/admin stop_kudu_cluster +sleep 2 diff --git a/testdata/bin/run-all.sh b/testdata/bin/run-all.sh index dda171c23..333d31803 100755 --- a/testdata/bin/run-all.sh +++ b/testdata/bin/run-all.sh @@ -23,11 +23,11 @@ set -euo pipefail . $IMPALA_HOME/bin/report_build_error.sh setup_report_build_error -# If -format is passed, format the mini-dfs cluster. +# If -format is passed, format the mini-dfs cluster and kudu cluster. if [[ $# -eq 1 && "$1" == "-format" ]]; then echo "Formatting cluster" - HDFS_FORMAT_CLUSTER="-format" + FORMAT_CLUSTER="-format" elif [[ $# -ne 0 ]]; then echo "Usage: run-all.sh [-format]" echo "[-format] : Format the mini-dfs cluster before starting" @@ -52,8 +52,10 @@ fi popd echo "Starting cluster services..." -$IMPALA_HOME/testdata/bin/run-mini-dfs.sh ${HDFS_FORMAT_CLUSTER-} 2>&1 | \ +$IMPALA_HOME/testdata/bin/run-mini-dfs.sh ${FORMAT_CLUSTER-} 2>&1 | \ tee ${IMPALA_CLUSTER_LOGS_DIR}/run-mini-dfs.log +$IMPALA_HOME/testdata/bin/run-kudu.sh ${FORMAT_CLUSTER-} 2>&1 | \ + tee ${IMPALA_CLUSTER_LOGS_DIR}/run-kudu.log # Starts up a mini-cluster which includes: # - HDFS with 3 DNs diff --git a/testdata/bin/run-kudu.sh b/testdata/bin/run-kudu.sh new file mode 100755 index 000000000..b2aa0d8ba --- /dev/null +++ b/testdata/bin/run-kudu.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error + +if [[ $# -eq 1 && "$1" == -format ]]; then + SHOULD_FORMAT=true +elif [[ $# -ne 0 ]]; then + echo "Usage: $0 [-format]" + echo "[-format] : Format the kudu cluster before starting" + exit 1 +else + SHOULD_FORMAT=false +fi + +# Kill and clean data for a clean start. +$IMPALA_HOME/testdata/bin/kill-kudu.sh + +if $SHOULD_FORMAT; then + $IMPALA_HOME/testdata/cluster/admin delete_kudu_data +fi + +set +e +$IMPALA_HOME/testdata/cluster/admin start_kudu_cluster diff --git a/testdata/cluster/admin b/testdata/cluster/admin index 179bb11ab..60bafdaa4 100755 --- a/testdata/cluster/admin +++ b/testdata/cluster/admin @@ -64,7 +64,6 @@ else # the other services could work after the proper configuration changes. SUPPORTED_SERVICES=() fi -SUPPORTED_SERVICES+=(kudu) # All DataNodes and NodeManagers need a unique but fixed address. The IP is fixed at # 127.0.0.1, so the only difference is the port. The address must be fixed because it is @@ -186,6 +185,13 @@ function cluster_exists { fi } +function kudu_cluster_exists { + # Just use the first master as an indicator. + if [[ ! -e "$IMPALA_CLUSTER_NODES_DIR/${NODE_PREFIX}1/var/lib/kudu/master" ]]; then + return 1 + fi +} + function create_cluster { mkdir -p "$IMPALA_CLUSTER_NODES_DIR" @@ -366,6 +372,16 @@ function start_cluster { return $? } +function start_kudu_cluster { + if ! kudu_cluster_exists; then + echo "The Kudu cluster must be created first" + return 1 + fi + start "kudu" + sleep 10 + check_kudu_cluster_status +} + function start { if [[ $# -ne 1 ]]; then echo start must be called with a single argument -- the service to start. 1>&2 @@ -444,23 +460,42 @@ function check_cluster_status { return 1 fi + for SERVICE in ${SUPPORTED_SERVICES[@]-}; do + check_status $SERVICE + done +} + +function check_kudu_cluster_status { + if ! kudu_cluster_exists; then + echo "The Kudu cluster does not exist" + return 1 + fi + + check_status "kudu" +} + +function check_status { + if [[ $# -ne 1 ]]; then + echo stop must be called with a single argument -- the service to check. 1>&2 + exit 1 + fi + local SERVICE=$1 + ROLE_COUNT=0 NOT_RUNNING=() for NODE_DIR in "$IMPALA_CLUSTER_NODES_DIR/$NODE_PREFIX"*; do - for SERVICE in ${SUPPORTED_SERVICES[@]-}; do - for SCRIPT in $(find "$NODE_DIR" -path "*/etc/init.d/$SERVICE*" $FIND_EXECUTABLE_FILTER \ - -type f); do - ROLE_COUNT=$((ROLE_COUNT + 1)) - if ! "$SCRIPT" status &>/dev/null; then - NOT_RUNNING+=("\n$(basename $SCRIPT) is not running on $(basename $NODE_DIR)") - fi - done + for SCRIPT in $(find "$NODE_DIR" -path "*/etc/init.d/$SERVICE*" $FIND_EXECUTABLE_FILTER \ + -type f); do + ROLE_COUNT=$((ROLE_COUNT + 1)) + if ! "$SCRIPT" status &>/dev/null; then + NOT_RUNNING+=("\n$(basename $SCRIPT) is not running on $(basename $NODE_DIR)") + fi done done case ${#NOT_RUNNING[@]} in - 0) echo "The cluster is running"; return;; - $ROLE_COUNT) echo "The cluster is not running"; return 1;; + 0) echo "The ${SERVICE} cluster is running"; return;; + $ROLE_COUNT) echo "The ${SERVICE} cluster is not running"; return 1;; *) echo -e "${NOT_RUNNING[@]}"; return 1;; esac } @@ -476,6 +511,12 @@ function stop_cluster { pkill -u $USER -f $KILL_CLUSTER_MARKER || true } +function stop_kudu_cluster { + if kudu_cluster_exists; then + stop "kudu" + fi +} + function stop { if [[ $# -ne 1 ]]; then echo stop must be called with a single argument -- the service to stop. 1>&2 @@ -502,7 +543,6 @@ function delete_data { rm -rf "$IMPALA_CLUSTER_NODES_DIR/$NODE_PREFIX"*/data/dfs/{nn,dn}/* rm -f "$IMPALA_CLUSTER_NODES_DIR/$NODE_PREFIX"*/data/kms.keystore rm -rf "$IMPALA_CLUSTER_NODES_DIR/$NODE_PREFIX"*/data/ozone - delete_kudu_data } function delete_kudu_data { @@ -510,6 +550,7 @@ function delete_kudu_data { } function delete_cluster { + # Delete all cluster data and directory structure. pkill -u $USER -f $KILL_CLUSTER_MARKER || true rm -rf "$IMPALA_CLUSTER_NODES_DIR" } @@ -529,7 +570,8 @@ function get_hadoop_client_conf_dir { COMMAND=$1 shift case $COMMAND in - check_cluster_status | cluster_exists | is_kerberized) + check_cluster_status | check_kudu_cluster_status | cluster_exists | \ + kudu_cluster_exists | is_kerberized) # Use an "if" to avoid triggering the ERR trap. if ! $COMMAND "$@"; then exit 1
