maksaska commented on code in PR #285: URL: https://github.com/apache/ignite-extensions/pull/285#discussion_r1945240717
########## modules/cdc-ext/examples/cdc-start-up/cdc-start-up.sh: ########## @@ -0,0 +1,486 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# All-in-one CDC start-up manager. Use this script to run examples for CDC with Apache Ignite. +# + +set -Eeuo pipefail +trap 'cleanup $LINENO' SIGINT SIGTERM ERR EXIT + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) + +IGNITE_BIN_DIR="${SCRIPT_DIR}/../../bin" +IGNITE_HOME="${SCRIPT_DIR}/../../" +IGNITE_LIBS="${SCRIPT_DIR}/../../libs" +IGNITE_CDC_EXAMPLES_DIR="${SCRIPT_DIR}/../config/cdc-start-up" + +CURRENT_PID=$$ + +# +# Help message +# +usage() { + cat <<EOF +This is a simple start-up script designed to ease the user experience with starting CDC for ignite clusters. + +Available options: + +-h, --help Prints help summary +-i, --ignite igniteProperties Starts a single node with provided properties. ` + `An ignite instance will be started with basic CDC configuration ` + `\$IGNITE_HOME/examples/config/cdc-start-up/cdc-base-configuration.xml + + Available options for igniteProperties include: + * cluster-1 + * cluster-2 + + Properties files are preconfigured for data replication between cluster-1 and cluster-2. + +-c, --cdc-consumer consumerMode ignitePropertiesPath ` + `Starts CDC consumer with specified transfer mode to parse WAL archives ` + `from source cluster. + + Available options for --cdc-consumer include: + * --ignite-to-ignite Creates a single thick client, ` + `used to transfer data from source-cluster to destination-cluster. + * --ignite-to-ignite-thin Creates a single thin client, ` + `used to transfer data from source-cluster to destination-cluster. + * --ignite-to-kafka Creates a cdc consumer, used to transfer data from source-cluster to specified Kafka topic. + +-k, --kafka-to-ignite clientMode ignitePropertiesPath ` + `Starts Kafka topic consumer for data replication to destination cluster. + + Available options for --kafka-to-ignite include: + * thick Creates a single server client (Thick client), ` + `used to transfer data from Kafka to destination-cluster. + * thin Creates a single thin client, used to transfer data from Kafka to destination-cluster. + +--check-cdc --key intNum1 --value intNum2 --version intNum3 [--cluster clusterNum] ` + `Starts CDC check with proposed (key, value) entry. ` + `The command puts the entry in the chosen cluster, and shows the comparison ` + `of caches between clusters as the entry reaches the other cluster. + + Options: + * --key intNum1 Specifies key of the entry. + * --value intNum2 Specifies value of the entry. + * --version intNum3 Specifies version of the entry. The value is used to resolve conflicted entries. + * --cluster clusterNum Optional parameter for the cluster number (1 or 2) that initially stores the entry. +EOF + exit +} + +# +# General message output function +# Arguments: +# 1 - message to print +# +msg() { + echo >&2 -e "${ORANGE}[PID=${CURRENT_PID-}]:${NOFORMAT} ${1-}" +} + +# +# Exits with error +# +die() { + local msg=$1 + local code=${2-1} + msg "$msg" + exit "$code" +} + +# +# Colors setup function +# +setupColors() { + if [[ -t 2 ]] && [[ -z "${NO_COLOR-}" ]] && [[ "${TERM-}" != "dumb" ]]; then + NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' + BLUE='\033[0;34m' PURPLE='\033[0;35m' CYAN='\033[0;36m' YELLOW='\033[1;33m' + else + NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE='' PURPLE='' CYAN='' YELLOW='' + fi +} + +# +# Script setup. Sets colors for messaging. +# +setup() { + setupColors + + msg "${PURPLE}CDC start-up manager [PID=${CURRENT_PID-}]${NOFORMAT}" +} + +# +# Clean-up function for exit and interruption +# +cleanup() { + trap - SIGINT SIGTERM ERR EXIT + + msg "${PURPLE}CDC start-up manager [PID=${CURRENT_PID-}] is closed ${NOFORMAT}" +} + +# +# General information message output function +# Arguments: +# 1 - message to print +# +infoMsg() { + msg "${GREEN}${1-}${NOFORMAT}" +} + +# +# Simple util function to check argument presence for specified parent argument +# Arguments: +# 1 - parent command argument +# 2 - argument name to check +# 3 - argument to check +# +checkMissing() { + local parent_arg_name=$1 + local arg_name=$2 + + local arg_to_check=$3 + + [[ -z $arg_to_check ]] && die "Missing script argument [""${arg_name-}""] for ""${parent_arg_name-}""!" + + return 0 +} + +# +# Checks --ignite arguments +# Globals: +# ignite_properties_path - '.properties' holder path. The file is used to configure server node +# Arguments: +# "$@" - script command arguments +# +checkServerParams() { + checkMissing "${script_param-}" "ignitePropertiesPath" "${2-}" Review Comment: Typo. Fixed it ########## modules/cdc-ext/examples/cdc-start-up/cdc-start-up.sh: ########## @@ -0,0 +1,486 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# All-in-one CDC start-up manager. Use this script to run examples for CDC with Apache Ignite. +# + +set -Eeuo pipefail +trap 'cleanup $LINENO' SIGINT SIGTERM ERR EXIT + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) + +IGNITE_BIN_DIR="${SCRIPT_DIR}/../../bin" +IGNITE_HOME="${SCRIPT_DIR}/../../" +IGNITE_LIBS="${SCRIPT_DIR}/../../libs" +IGNITE_CDC_EXAMPLES_DIR="${SCRIPT_DIR}/../config/cdc-start-up" + +CURRENT_PID=$$ + +# +# Help message +# +usage() { + cat <<EOF +This is a simple start-up script designed to ease the user experience with starting CDC for ignite clusters. + +Available options: + +-h, --help Prints help summary +-i, --ignite igniteProperties Starts a single node with provided properties. ` + `An ignite instance will be started with basic CDC configuration ` + `\$IGNITE_HOME/examples/config/cdc-start-up/cdc-base-configuration.xml + + Available options for igniteProperties include: + * cluster-1 + * cluster-2 + + Properties files are preconfigured for data replication between cluster-1 and cluster-2. + +-c, --cdc-consumer consumerMode ignitePropertiesPath ` + `Starts CDC consumer with specified transfer mode to parse WAL archives ` + `from source cluster. + + Available options for --cdc-consumer include: + * --ignite-to-ignite Creates a single thick client, ` + `used to transfer data from source-cluster to destination-cluster. + * --ignite-to-ignite-thin Creates a single thin client, ` + `used to transfer data from source-cluster to destination-cluster. + * --ignite-to-kafka Creates a cdc consumer, used to transfer data from source-cluster to specified Kafka topic. + +-k, --kafka-to-ignite clientMode ignitePropertiesPath ` + `Starts Kafka topic consumer for data replication to destination cluster. + + Available options for --kafka-to-ignite include: + * thick Creates a single server client (Thick client), ` + `used to transfer data from Kafka to destination-cluster. + * thin Creates a single thin client, used to transfer data from Kafka to destination-cluster. + +--check-cdc --key intNum1 --value intNum2 --version intNum3 [--cluster clusterNum] ` + `Starts CDC check with proposed (key, value) entry. ` + `The command puts the entry in the chosen cluster, and shows the comparison ` + `of caches between clusters as the entry reaches the other cluster. + + Options: + * --key intNum1 Specifies key of the entry. + * --value intNum2 Specifies value of the entry. + * --version intNum3 Specifies version of the entry. The value is used to resolve conflicted entries. + * --cluster clusterNum Optional parameter for the cluster number (1 or 2) that initially stores the entry. +EOF + exit +} + +# +# General message output function +# Arguments: +# 1 - message to print +# +msg() { + echo >&2 -e "${ORANGE}[PID=${CURRENT_PID-}]:${NOFORMAT} ${1-}" +} + +# +# Exits with error +# +die() { + local msg=$1 + local code=${2-1} + msg "$msg" + exit "$code" +} + +# +# Colors setup function +# +setupColors() { + if [[ -t 2 ]] && [[ -z "${NO_COLOR-}" ]] && [[ "${TERM-}" != "dumb" ]]; then + NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' + BLUE='\033[0;34m' PURPLE='\033[0;35m' CYAN='\033[0;36m' YELLOW='\033[1;33m' + else + NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE='' PURPLE='' CYAN='' YELLOW='' + fi +} + +# +# Script setup. Sets colors for messaging. +# +setup() { + setupColors + + msg "${PURPLE}CDC start-up manager [PID=${CURRENT_PID-}]${NOFORMAT}" +} + +# +# Clean-up function for exit and interruption +# +cleanup() { + trap - SIGINT SIGTERM ERR EXIT + + msg "${PURPLE}CDC start-up manager [PID=${CURRENT_PID-}] is closed ${NOFORMAT}" +} + +# +# General information message output function +# Arguments: +# 1 - message to print +# +infoMsg() { + msg "${GREEN}${1-}${NOFORMAT}" +} + +# +# Simple util function to check argument presence for specified parent argument +# Arguments: +# 1 - parent command argument +# 2 - argument name to check +# 3 - argument to check +# +checkMissing() { + local parent_arg_name=$1 + local arg_name=$2 + + local arg_to_check=$3 + + [[ -z $arg_to_check ]] && die "Missing script argument [""${arg_name-}""] for ""${parent_arg_name-}""!" + + return 0 +} + +# +# Checks --ignite arguments +# Globals: +# ignite_properties_path - '.properties' holder path. The file is used to configure server node +# Arguments: +# "$@" - script command arguments +# +checkServerParams() { + checkMissing "${script_param-}" "ignitePropertiesPath" "${2-}" + + ignite_properties_path="${IGNITE_CDC_EXAMPLES_DIR}"/${2-} +} + +# +# Checks --cdc-consumer arguments +# Globals: +# consumer_mode - Transfer type for CDC +# cdc_streamer_xml_file_name - '.xml' filename of the specified transfer type +# ignite_properties_path - '.properties' holder path. The file is used to configure CDC client +# Arguments: +# "$@" - script command arguments +# +checkConsumerParams() { + consumer_mode=${2-} + + checkMissing "${script_param-}" "consumerMode" "${consumer_mode-}" + + case $consumer_mode in + --ignite-to-ignite) export cdc_streamer_xml_file_name="cdc-streamer-I2I.xml" ;; + --ignite-to-ignite-thin) export cdc_streamer_xml_file_name="cdc-streamer-I2I-thin.xml" ;; + --ignite-to-kafka) export cdc_streamer_xml_file_name="cdc-streamer-I2K.xml" ;; + *) die "Unknown consumer mode for CDC: ${consumer_mode-}" ;; + esac + + checkMissing "${consumer_mode-}" "ignitePropertiesPath" "${3-}" + + ignite_properties_path="${IGNITE_CDC_EXAMPLES_DIR}"/${3-} + + return 0 +} + +# +# Checks --kafka-to-ignite arguments +# Globals: +# client_mode - Transfer type for CDC +# cdc_streamer_xml_file_name - '.xml' filename of the specified transfer type +# ignite_properties_path - '.properties' holder path. The file is used to configure CDC client +# Arguments: +# "$@" - script command arguments +# +checkKafkaConsumerParams() { + client_mode=${2-} + + checkMissing "${script_param-}" "clientMode" "${client_mode-}" + + case $client_mode in + thick) ;; + thin) ;; + *) die "Unknown client mode for CDC: ${client_mode-}" ;; + esac + + checkMissing "${client_mode-}" "ignitePropertiesPath" "${3-}" + + ignite_properties_path="${IGNITE_CDC_EXAMPLES_DIR}"/${3-} + + return 0 +} + +# +# Checks --check-cdc arguments +# Globals: +# key - Entity key +# value - Entity value +# cluster - Cluster to put entity in +# version - Entity version. Used to resolve conflict entries +# Arguments: +# "$@" - script command arguments +# +checkEntriesParams() { + cluster=1 + + while :; do + case "${2-}" in + --key) key="${3-}"; shift ;; + --value) value="${3-}"; shift ;; + --cluster) cluster="${3-}"; shift ;; + --version) version="${3-}"; shift ;; + -?*) die "Unknown option: ${script_param-}" ;; + *) break ;; + esac + shift + done + + checkMissing "${script_param-}" "key" "${key-}" + checkMissing "${script_param-}" "value" "${value-}" + checkMissing "${script_param-}" "version" "${version-}" + + return 0 +} + +# +# Checks if all required optional libraries enabled for CDC check +# +checkLibraries() { + local lib1="ignite-rest-http"; + local lib2="ignite-json"; + + if [ ! -d "$IGNITE_LIBS/$lib1" ] && [ ! -d "$IGNITE_LIBS/$lib2" ]; then + die "${RED}Failure! Check that $lib1 and $lib2 optional libraries are enabled."; + elif [ ! -d "$IGNITE_LIBS/$lib1" ]; then + die "${RED}Failure! Check that $lib1 optional library is enabled."; + elif [ ! -d "$IGNITE_LIBS/$lib2" ]; then + die "${RED}Failure! Check that $lib2 optional library is enabled."; + fi +} + +# +# Starts single Ignite instance +# cdc-base-configuration needs dummy streamer to start, which is why cdc_streamer_xml_file_name is exported +# +startIgnite() { + infoMsg "Starting Ignite for ${ignite_properties_path-}" + + export cdc_streamer_xml_file_name="cdc-streamer-I2I.xml" + export ignite_properties_path + + "${IGNITE_BIN_DIR}"/ignite.sh -v "${IGNITE_CDC_EXAMPLES_DIR}"/cdc-base-configuration.xml +} + +# +# Starts single CDC client instance +# +startCdcConsumer() { + infoMsg "Starting CDC client for ${ignite_properties_path-} with ${consumer_mode-}" Review Comment: Fixed it -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: notifications-unsubscr...@ignite.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
