timoninmaxim commented on code in PR #285: URL: https://github.com/apache/ignite-extensions/pull/285#discussion_r1944936654
########## modules/cdc-ext/examples/cdc-start-up/cdc-start-up.sh: ########## @@ -0,0 +1,481 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# All-in-one CDC start-up manager. Use this script to run examples for CDC with Apache Ignite. +# + +set -Eeuo pipefail +trap 'cleanup $LINENO' SIGINT SIGTERM ERR EXIT + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) + +IGNITE_BIN_DIR="${SCRIPT_DIR}/../../bin" +IGNITE_HOME="${SCRIPT_DIR}/../../" +IGNITE_LIBS="${SCRIPT_DIR}/../../libs" +IGNITE_CDC_EXAMPLES_DIR="${SCRIPT_DIR}/../config/cdc-start-up" + +CURRENT_PID=$$ + +# +# Help message +# +usage() { + cat <<EOF +This is a simple start-up script designed to ease the user experience with starting CDC for ignite clusters. + +Available options: + +-h, --help Prints help summary +-i, --ignite igniteProperties Starts a single node with provided properties. ` + `An ignite instance will be started with basic CDC configuration. + + * igniteProperties ignite properties folder under \$IGNITE_HOME/examples/config/cdc-start-up + +-c, --cdc-consumer consumerMode ignitePropertiesPath ` + `Starts CDC consumer with specified transfer mode to parse WAL archives ` + `from source cluster. + + Available options for --cdc-consumer include: + * --ignite-to-ignite Creates a single server client (Thick client), ` + `used to transfer data from source-cluster to destination-cluster. + * --ignite-to-ignite-thin Creates a single thin client, ` + `used to transfer data from source-cluster to destination-cluster. + * --ignite-to-kafka Creates a cdc client, used to transfer data from source-cluster to specified Kafka topic. Review Comment: What is cdc client? It must be one of thin or thick? ########## modules/cdc-ext/examples/config/cdc-start-up/cdc-base-configuration.xml: ########## @@ -0,0 +1,83 @@ +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xmlns:util="http://www.springframework.org/schema/util"; + xmlns:context="http://www.springframework.org/schema/context"; + xsi:schemaLocation=" + http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans.xsd + http://www.springframework.org/schema/util + http://www.springframework.org/schema/util/spring-util.xsd + http://www.springframework.org/schema/context + http://www.springframework.org/schema/context/spring-context-3.0.xsd";> + + <import resource="${cdc_streamer_xml_file_name}"/> + + <context:property-placeholder location="file:${ignite_properties_path}/ignite-cdc.properties"/> + + <!--IgniteConfiguration for source cluster connection--> + <bean class="org.apache.ignite.configuration.IgniteConfiguration"> + <property name="igniteInstanceName" value="${server_instance_name}"/> + <property name="consistentId" value="${server_instance_name}"/> + <property name="localHost" value="${server_host}"/> + <property name="discoverySpi" ref="ignTcpDiscoverySpi"/> + <property name="dataStorageConfiguration" ref="ignDataStorageConfiguration"/> + <property name="pluginProviders" ref="ignCacheVersionConflictResolverPluginProvider"/> + <property name="cacheConfiguration" ref="ignCacheConfiguration"/> + <property name="clientConnectorConfiguration" ref="ignClientConnectorConfiguration"/> + </bean> + + <!--CdcConfiguration--> + <bean id="cdc.cfg" class="org.apache.ignite.cdc.CdcConfiguration"> + <property name="consumer" ref="cdc.streamer"/> + </bean> + + <!--IgniteConfiguration property declaration--> + + <!--TcpDiscoverySpi--> + <bean id="ignTcpDiscoverySpi" class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> + <property name="ipFinder"> + <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"> Review Comment: Fix indentions in configs also ########## modules/cdc-ext/examples/cdc-start-up/cdc-start-up.sh: ########## @@ -0,0 +1,481 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# All-in-one CDC start-up manager. Use this script to run examples for CDC with Apache Ignite. +# + +set -Eeuo pipefail +trap 'cleanup $LINENO' SIGINT SIGTERM ERR EXIT + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) + +IGNITE_BIN_DIR="${SCRIPT_DIR}/../../bin" +IGNITE_HOME="${SCRIPT_DIR}/../../" +IGNITE_LIBS="${SCRIPT_DIR}/../../libs" +IGNITE_CDC_EXAMPLES_DIR="${SCRIPT_DIR}/../config/cdc-start-up" + +CURRENT_PID=$$ + +# +# Help message +# +usage() { + cat <<EOF +This is a simple start-up script designed to ease the user experience with starting CDC for ignite clusters. + +Available options: + +-h, --help Prints help summary +-i, --ignite igniteProperties Starts a single node with provided properties. ` + `An ignite instance will be started with basic CDC configuration. + + * igniteProperties ignite properties folder under \$IGNITE_HOME/examples/config/cdc-start-up + +-c, --cdc-consumer consumerMode ignitePropertiesPath ` + `Starts CDC consumer with specified transfer mode to parse WAL archives ` + `from source cluster. + + Available options for --cdc-consumer include: + * --ignite-to-ignite Creates a single server client (Thick client), ` + `used to transfer data from source-cluster to destination-cluster. + * --ignite-to-ignite-thin Creates a single thin client, ` + `used to transfer data from source-cluster to destination-cluster. + * --ignite-to-kafka Creates a cdc client, used to transfer data from source-cluster to specified Kafka topic. + +-k, --kafka-to-ignite clientMode ignitePropertiesPath ` + `Starts Kafka topic consumer for data replication to destination cluster. + + Available options for --kafka-to-ignite include: + * thick Creates a single server client (Thick client), ` + `used to transfer data from Kafka to destination-cluster. + * thin Creates a single thin client, used to transfer data from Kafka to destination-cluster. + +--check-cdc --key intNum1 --value intNum2 --version intNum3 [--cluster clusterNum] ` + `Starts CDC check with proposed (key, value) entry. ` + `The command puts the entry in the chosen cluster, and shows the comparison ` + `of caches between clusters as the entry reaches the other cluster. + + Options: + * --key intNum1 Specifies key of the entry. + * --value intNum2 Specifies value of the entry. + * --version intNum3 Specifies version of the entry. The value is used to resolve conflicted entries. + * --cluster clusterNum Optional parameter for the cluster number (1 or 2) that initially stores the entry. +EOF + exit +} + +# +# General message output function +# Arguments: +# 1 - message to print +# +msg() { + echo >&2 -e "${ORANGE}[PID=${CURRENT_PID-}]:${NOFORMAT} ${1-}" +} + +# +# Exits with error +# +die() { + local msg=$1 + local code=${2-1} + msg "$msg" + exit "$code" +} + +# +# Colors setup function +# +setupColors() { + if [[ -t 2 ]] && [[ -z "${NO_COLOR-}" ]] && [[ "${TERM-}" != "dumb" ]]; then + NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' + BLUE='\033[0;34m' PURPLE='\033[0;35m' CYAN='\033[0;36m' YELLOW='\033[1;33m' + else + NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE='' PURPLE='' CYAN='' YELLOW='' + fi +} + +# +# Script setup. Sets colors for messaging. +# +setup() { + setupColors + + msg "${PURPLE}CDC start-up manager [PID=${CURRENT_PID-}]${NOFORMAT}" +} + +# +# Clean-up function for exit and interruption +# +cleanup() { + trap - SIGINT SIGTERM ERR EXIT + + msg "${PURPLE}CDC start-up manager [PID=${CURRENT_PID-}] is closed ${NOFORMAT}" +} + +# +# General information message output function +# Arguments: +# 1 - message to print +# +infoMsg() { + msg "${GREEN}${1-}${NOFORMAT}" +} + +# +# Simple util function to check argument presence for specified parent argument +# Arguments: +# 1 - parent command argument +# 2 - argument name to check +# 3 - argument to check +# +checkMissing() { + local parent_arg_name=$1 + local arg_name=$2 + + local arg_to_check=$3 + + [[ -z $arg_to_check ]] && die "Missing script argument [""${arg_name-}""] for ""${parent_arg_name-}""!" + + return 0 +} + +# +# Checks --ignite arguments +# Globals: +# ignite_properties_path - '.properties' holder path. The file is used to configure server node +# Arguments: +# "$@" - script command arguments +# +checkServerParams() { + checkMissing "${script_param-}" "ignitePropertiesPath" "${2-}" + + ignite_properties_path="${IGNITE_CDC_EXAMPLES_DIR}"/${2-} +} + +# +# Checks --cdc-client arguments Review Comment: No such parameter ########## modules/cdc-ext/README.txt: ########## @@ -0,0 +1,29 @@ +Apache Ignite Change Data Capture Module +------------------------ + +Apache Ignite CDC is a data processing pattern used to asynchronously receive entries that have been changed on the local node so that action can be taken using the changed entry. + +This module provides the clients with a simple CDC implementations strategies for inter-cluster communication. Active-Passive and Active-Active replication regimes can be established with different CDC clients. Each such client should be started up for each node participating in CDC. + +==== Installation + +. Build `cdc-ext` module with maven: ++ +```console + $~/src/ignite-extensions/> mvn clean install -f modules/cdc-ext -Pcheckstyle,extension-release,skip-docs -DskipTests + $~/src/ignite-extensions/> ls modules/cdc-ext/target | grep zip +ignite-cdc-ext.zip +``` + +. The resulting binary will be located under 'target' directory. Unpack `ignite-cdc-ext.zip` archive to `$IGNITE_HOME` folder to enable CDC. + +For Linux/Macos you can use in the ignite root with cdc binary: ++ +```console +$ unzip ignite-cdc-ext-bin.zip +$ cp -r ignite-cdc-ext/* . +``` + +Now, you have additional binary `$IGNITE_HOME/bin/kafka-to-ignite.sh` and `$IGNITE_HOME/libs/optional/ignite-cdc-ext` module. Review Comment: And examples ########## modules/cdc-ext/examples/cdc-start-up/cdc-start-up.sh: ########## @@ -0,0 +1,432 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# All-in-one CDC start-up manager. Use this script to run examples for CDC with Apache Ignite. +# + +set -Eeuo pipefail +trap 'cleanup $LINENO' SIGINT SIGTERM ERR EXIT + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) + +IGNITE_BIN_DIR="${SCRIPT_DIR}/../../bin" +IGNITE_HOME="${SCRIPT_DIR}/../../" +IGNITE_LIBS="${SCRIPT_DIR}/../../libs" +IGNITE_CDC_EXAMPLES_DIR="${SCRIPT_DIR}/../config/cdc-start-up" + +CURRENT_PID=$$ + +# +# Help message +# +usage() { + cat <<EOF +This is a simple start-up script designed to ease the user experience with starting CDC for ignite clusters. + +Available options: + +-h, --help Prints help summary +-i, --ignite igniteProperties Starts a single node with provided properties. ` + `An ignite instance will be started with basic persistence configuration with CDC. + + * igniteProperties ignite properties folder under \$IGNITE_HOME/examples/config/cdc-start-up + +-c, --cdc-client clientMode ignitePropertiesPath ` + `Starts CDC client with specified transfer mode. + + Available options for --cdc-client include: + * --ignite-to-ignite Creates a single server client (Thick client), ` Review Comment: just "thick client", server client sounds weird :) ########## modules/cdc-ext/examples/cdc-start-up/cdc-start-up.sh: ########## @@ -0,0 +1,481 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# All-in-one CDC start-up manager. Use this script to run examples for CDC with Apache Ignite. +# + +set -Eeuo pipefail +trap 'cleanup $LINENO' SIGINT SIGTERM ERR EXIT + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) + +IGNITE_BIN_DIR="${SCRIPT_DIR}/../../bin" +IGNITE_HOME="${SCRIPT_DIR}/../../" +IGNITE_LIBS="${SCRIPT_DIR}/../../libs" +IGNITE_CDC_EXAMPLES_DIR="${SCRIPT_DIR}/../config/cdc-start-up" + +CURRENT_PID=$$ + +# +# Help message +# +usage() { + cat <<EOF +This is a simple start-up script designed to ease the user experience with starting CDC for ignite clusters. + +Available options: + +-h, --help Prints help summary +-i, --ignite igniteProperties Starts a single node with provided properties. ` + `An ignite instance will be started with basic CDC configuration. Review Comment: add path to ignite configuration. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: notifications-unsubscr...@ignite.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
