This is an automated email from the ASF dual-hosted git repository. lhotari pushed a commit to branch branch-4.0 in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit e52b4e0fa62b8afd2b6b9525f636d0b72f7986c0 Author: Lari Hotari <[email protected]> AuthorDate: Tue Aug 26 07:44:05 2025 +0300 [improve][io] Upgrade AWS SDK v1 & v2, Kinesis KPL and KPC versions (#24661) (cherry picked from commit dc2ab26675bdbe11eabdbf86ff01f928bb820ebe) --- docker/kinesis-producer-alpine/Dockerfile | 2 +- docker/kinesis-producer-alpine/README.md | 2 +- docker/kinesis-producer-alpine/build-alpine.sh | 112 +++++++++------- .../kinesis_producer_alpine.patch | 23 ++-- docker/pulsar-all/pom.xml | 2 +- pom.xml | 11 +- pulsar-io/aws/pom.xml | 1 - pulsar-io/kinesis-kpl-shaded/pom.xml | 149 +++++++++++++++++++++ pulsar-io/kinesis/pom.xml | 24 ++-- .../org/apache/pulsar/io/kinesis/KinesisSink.java | 15 ++- .../pulsar/io/kinesis/KinesisSinkConfig.java | 4 +- .../pulsar/io/kinesis/KinesisSinkConfigTest.java | 2 +- pulsar-io/pom.xml | 2 + tests/integration/pom.xml | 2 +- 14 files changed, 271 insertions(+), 80 deletions(-) diff --git a/docker/kinesis-producer-alpine/Dockerfile b/docker/kinesis-producer-alpine/Dockerfile index ac04e69aea0..a33dc82f158 100644 --- a/docker/kinesis-producer-alpine/Dockerfile +++ b/docker/kinesis-producer-alpine/Dockerfile @@ -23,7 +23,7 @@ ARG ALPINE_VERSION=3.21 # Build stage FROM alpine:$ALPINE_VERSION AS kinesis-producer-build -ENV KINESIS_PRODUCER_LIB_VERSION=0.15.12 +ENV KINESIS_PRODUCER_LIB_VERSION=1.0.4 # Install build dependencies RUN apk update && apk add --no-cache \ diff --git a/docker/kinesis-producer-alpine/README.md b/docker/kinesis-producer-alpine/README.md index dc21b05dec3..e3fa0f88602 100644 --- a/docker/kinesis-producer-alpine/README.md +++ b/docker/kinesis-producer-alpine/README.md @@ -32,7 +32,7 @@ This image only needs to be re-created when we want to upgrade to a newer versio 2. Rebuild the image and push it to Docker Hub: ``` IMAGE=apachepulsar/pulsar-io-kinesis-sink-kinesis_producer -KINESIS_PRODUCER_VERSION=0.15.12 +KINESIS_PRODUCER_VERSION=1.0.4 docker buildx build --platform=linux/amd64,linux/arm64 \ -t "$IMAGE:$KINESIS_PRODUCER_VERSION" -t "$IMAGE:${KINESIS_PRODUCER_VERSION}-$(date -I)" \ . --push diff --git a/docker/kinesis-producer-alpine/build-alpine.sh b/docker/kinesis-producer-alpine/build-alpine.sh index 6ea41c55582..52ae9c1a32e 100644 --- a/docker/kinesis-producer-alpine/build-alpine.sh +++ b/docker/kinesis-producer-alpine/build-alpine.sh @@ -20,13 +20,25 @@ set -e set -x +SILENT="n" -INSTALL_DIR=/build/third_party -AWS_SDK_CPP_VERSION="1.11.420" -PROTOBUF_VERSION="3.11.4" -BOOST_VERSION="1.76.0" -BOOST_VERSION_UNDERSCORED="${BOOST_VERSION//\./_}" +silence() { + if [ -n "$SILENT" ]; then + "$@" >/dev/null + else + "$@" + fi +} + +BOOST_VERSION="1.88.0" +BOOST_VERSION_UNDERSCORED="${BOOST_VERSION//\./_}" # convert from 1.76.0 to 1_76_0 +PROTOBUF_VERSION="21.12" +AWS_SDK_CPP_VERSION="1.11.615" + +LIB_BOOST="https://archives.boost.io/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORED}.tar.gz"; +LIB_PROTOBUF="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz"; +INSTALL_DIR=/build/third_party # Create install directory mkdir -p $INSTALL_DIR @@ -39,61 +51,67 @@ export LD_LIBRARY_PATH="$INSTALL_DIR/lib:$LD_LIBRARY_PATH" cd $INSTALL_DIR -# Build protobuf -if [ ! -d "protobuf-${PROTOBUF_VERSION}" ]; then - curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz - tar xf protobuf-all-${PROTOBUF_VERSION}.tar.gz - rm protobuf-all-${PROTOBUF_VERSION}.tar.gz - - cd protobuf-${PROTOBUF_VERSION} - ./configure --prefix=${INSTALL_DIR} \ - --disable-shared \ - CFLAGS="-fPIC" \ - CXXFLAGS="-fPIC ${CXXFLAGS}" \ - --with-pic - make -j4 - make install - cd .. -fi - -# Build Boost +SED="sed -i" +CMAKE=$(which cmake3 &>/dev/null && echo "cmake3 " || echo "cmake") +function _curl { + curl -L "$@" +} + +function conf { + silence ./configure \ + --prefix="$INSTALL_DIR" \ + LD_LIBRARY_PATH="$LD_LIBRARY_PATH" \ + LDFLAGS="$LDFLAGS" \ + CXXFLAGS="$CXXFLAGS" \ + C_INCLUDE_PATH="$C_INCLUDE_PATH" \ + "$@" +} + +# Boost C++ Libraries if [ ! -d "boost_${BOOST_VERSION_UNDERSCORED}" ]; then - curl -LO https://archives.boost.io/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORED}.tar.gz - tar xf boost_${BOOST_VERSION_UNDERSCORED}.tar.gz - rm boost_${BOOST_VERSION_UNDERSCORED}.tar.gz + _curl "$LIB_BOOST" >boost.tgz + tar xf boost.tgz + rm boost.tgz cd boost_${BOOST_VERSION_UNDERSCORED} - BOOST_LIBS="regex,thread,log,system,random,filesystem,chrono,atomic,date_time,program_options,test" + LIBS="atomic,chrono,log,system,test,random,regex,thread,filesystem" + OPTS="--build-type=minimal --layout=system --prefix=$INSTALL_DIR link=static threading=multi release install" - ./bootstrap.sh --with-libraries=$BOOST_LIBS --with-toolset=gcc + silence ./bootstrap.sh --with-libraries="$LIBS" --with-toolset=gcc + silence ./b2 toolset=gcc $OPTS - ./b2 \ - -j4 \ - variant=release \ - link=static \ - threading=multi \ - runtime-link=static \ - --prefix=${INSTALL_DIR} \ - cxxflags="-fPIC ${CXXFLAGS}" \ - install + cd .. +fi + +# Google Protocol Buffers +if [ ! -d "protobuf-${PROTOBUF_VERSION}" ]; then + _curl "$LIB_PROTOBUF" >protobuf.tgz + tar xf protobuf.tgz + rm protobuf.tgz + + cd protobuf-${PROTOBUF_VERSION} + silence conf --enable-shared=no + silence make -j 4 + silence make install cd .. fi -# Download and build AWS SDK +# AWS C++ SDK if [ ! -d "aws-sdk-cpp" ]; then - git clone --depth 1 --branch ${AWS_SDK_CPP_VERSION} https://github.com/awslabs/aws-sdk-cpp.git aws-sdk-cpp + git clone https://github.com/awslabs/aws-sdk-cpp.git aws-sdk-cpp pushd aws-sdk-cpp - git config submodule.fetchJobs 8 - git submodule update --init --depth 1 --recursive + git checkout ${AWS_SDK_CPP_VERSION} + git submodule update --init --recursive popd rm -rf aws-sdk-cpp-build mkdir aws-sdk-cpp-build + cd aws-sdk-cpp-build - cmake \ + silence $CMAKE \ -DBUILD_ONLY="kinesis;monitoring;sts" \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DSTATIC_LINKING=1 \ @@ -105,15 +123,19 @@ if [ ! -d "aws-sdk-cpp" ]; then -DCMAKE_FIND_FRAMEWORK=LAST \ -DENABLE_TESTING="OFF" \ ../aws-sdk-cpp - make -j4 - make install + silence make -j4 + silence make install + cd .. + fi +cd .. + # Build the native kinesis producer cd /build/amazon-kinesis-producer ln -fs ../third_party -cmake -DCMAKE_PREFIX_PATH="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=RelWithDebInfo . +$CMAKE -DCMAKE_PREFIX_PATH="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=RelWithDebInfo . make -j4 FINAL_DIR=/opt/amazon-kinesis-producer diff --git a/docker/kinesis-producer-alpine/kinesis_producer_alpine.patch b/docker/kinesis-producer-alpine/kinesis_producer_alpine.patch index d1ddd6a8550..d31e61a3f8e 100644 --- a/docker/kinesis-producer-alpine/kinesis_producer_alpine.patch +++ b/docker/kinesis-producer-alpine/kinesis_producer_alpine.patch @@ -1,10 +1,9 @@ -From 96ba2eb7145363586529e6c770dcc0920bf04ac2 Mon Sep 17 00:00:00 2001 +From 76518befa2b3b79931044267722feecfe1465d19 Mon Sep 17 00:00:00 2001 From: Lari Hotari <[email protected]> -Date: Wed, 18 Dec 2024 17:16:02 +0200 -Subject: [PATCH] Adapt build for Alpine, fix issue with NULL_BACKTRACE support +Date: Fri, 22 Aug 2025 21:25:49 +0300 +Subject: [PATCH] [PATCH] Adapt build for Alpine, fix issue with NULL_BACKTRACE + support -- also use dynamic linking to some libraries (zlib, openssl, libz, libcurl, libcrypto) - to reduce binary size --- CMakeLists.txt | 20 +++++++++++--------- aws/utils/backtrace/bsd_backtrace.cc | 2 +- @@ -14,12 +13,12 @@ Subject: [PATCH] Adapt build for Alpine, fix issue with NULL_BACKTRACE support 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt -index 2dd7084..2ba47e6 100644 +index 8803f5c..6152b3d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -213,22 +213,24 @@ set(STATIC_LIBS - boost_chrono) +@@ -213,8 +213,10 @@ set(STATIC_LIBS + find_package(Protobuf REQUIRED) find_package(Threads) -find_package(ZLIB) +find_package(ZLIB REQUIRED) @@ -27,6 +26,12 @@ index 2dd7084..2ba47e6 100644 +find_package(OpenSSL REQUIRED) +find_package(CURL REQUIRED) + # Specify the output directory for generated protobuf files + set(PROTO_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/aws/kinesis/protobuf/) +@@ -239,19 +241,19 @@ foreach(proto ${PROTO_FILES}) + ) + endforeach() + -add_library(LibCrypto STATIC IMPORTED) -set_property(TARGET LibCrypto PROPERTY IMPORTED_LOCATION ${THIRD_PARTY_LIB_DIR}/libcrypto.a) +add_library(LibCrypto SHARED IMPORTED) @@ -123,5 +128,5 @@ index b58ab0e..f483c77 100644 #include <cstring> #include <cstdint> -- -2.47.1 +2.51.0 diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 4f6907881e0..bb8d4363352 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -82,7 +82,7 @@ <!-- include the docker image only when docker profile is active --> <properties> <!-- reference to the image built from docker/kinesis-producer-alpine/Dockerfile --> - <PULSAR_IO_KINESIS_KPL_IMAGE>apachepulsar/pulsar-io-kinesis-sink-kinesis_producer:0.15.12</PULSAR_IO_KINESIS_KPL_IMAGE> + <PULSAR_IO_KINESIS_KPL_IMAGE>apachepulsar/pulsar-io-kinesis-sink-kinesis_producer:1.0.4</PULSAR_IO_KINESIS_KPL_IMAGE> </properties> <dependencies> <dependency> diff --git a/pom.xml b/pom.xml index f8b31c61451..aa099c9ef09 100644 --- a/pom.xml +++ b/pom.xml @@ -227,7 +227,8 @@ flexible messaging model and an intuitive client API.</description> <aerospike-client.version>4.5.0</aerospike-client.version> <kafka-client.version>3.9.0</kafka-client.version> <rabbitmq-client.version>5.18.0</rabbitmq-client.version> - <aws-sdk.version>1.12.638</aws-sdk.version> + <aws-sdk.version>1.12.788</aws-sdk.version> + <aws-sdk2.version>2.32.28</aws-sdk2.version> <avro.version>1.11.4</avro.version> <joda.version>2.10.10</joda.version> <jclouds.version>2.6.0</jclouds.version> @@ -1362,6 +1363,14 @@ flexible messaging model and an intuitive client API.</description> <scope>import</scope> </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>bom</artifactId> + <version>${aws-sdk2.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + <dependency> <groupId>org.apache.distributedlog</groupId> <artifactId>distributedlog-core</artifactId> diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index bca87245832..5ac6c61bc27 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -51,7 +51,6 @@ <dependency> <groupId>software.amazon.awssdk</groupId> <artifactId>sts</artifactId> - <version>2.10.56</version> </dependency> <!-- /aws dependencies --> diff --git a/pulsar-io/kinesis-kpl-shaded/pom.xml b/pulsar-io/kinesis-kpl-shaded/pom.xml new file mode 100644 index 00000000000..2574d6df23d --- /dev/null +++ b/pulsar-io/kinesis-kpl-shaded/pom.xml @@ -0,0 +1,149 @@ +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.pulsar</groupId> + <artifactId>pulsar</artifactId> + <version>4.0.7-SNAPSHOT</version> + <relativePath>../../pom.xml</relativePath> + </parent> + + <artifactId>pulsar-io-kinesis-kpl-shaded</artifactId> + <name>Pulsar IO :: Kinesis KPL Shaded</name> + + <!-- + There's a need to shade the protobuf-java dependency in the AWS Kinesis Producer Library (KPL) + since it conflicts at runtime with Pulsar Functions GRPC protobuf-java version. + --> + + <properties> + <amazon-kinesis-producer.version>1.0.4</amazon-kinesis-producer.version> + <protobuf-java.version>4.29.0</protobuf-java.version> + </properties> + + <dependencyManagement> + <dependencies> + <!-- enforce protobuf-java version that is compatible with KPL 1.0.x --> + <!-- see https://github.com/awslabs/amazon-kinesis-producer/blob/v1.0.4/java/amazon-kinesis-producer/pom.xml#L20 --> + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-bom</artifactId> + <version>${protobuf-java.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + <dependencies> + <dependency> + <groupId>software.amazon.kinesis</groupId> + <artifactId>amazon-kinesis-producer</artifactId> + <version>${amazon-kinesis-producer.version}</version> + </dependency> + </dependencies> + <build> + <finalName>${project.artifactId}-${project.version}</finalName> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <id>attach-shade-jar</id> + <phase>package</phase> + <goals> + <goal>attach-artifact</goal> + </goals> + <configuration> + <artifacts> + <artifact> + <file>${project.build.directory}/${project.build.finalName}.jar</file> + <type>jar</type> + <classifier>shaded</classifier> + </artifact> + </artifacts> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <createDependencyReducedPom>true</createDependencyReducedPom> + <promoteTransitiveDependencies>true</promoteTransitiveDependencies> + <minimizeJar>false</minimizeJar> + <artifactSet> + <includes> + <include>software.amazon.kinesis:amazon-kinesis-producer</include> + <include>com.google.protobuf:protobuf-java</include> + </includes> + <excludes> + <!-- This is required for execute shade multiple times without clean --> + <exclude>org.apache.pulsar:pulsar-io-kinesis-kpl-shaded</exclude> + </excludes> + </artifactSet> + <relocations> + <relocation> + <pattern>com.google.protobuf</pattern> + <shadedPattern>org.apache.pulsar.io.kinesis.shaded.com.google.protobuf</shadedPattern> + </relocation> + </relocations> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/maven/${project.groupId}/${project.artifactId}/pom.xml</exclude> + </excludes> + </filter> + </filters> + <transformers> + <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <manifestEntries> + <Multi-Release>true</Multi-Release> + </manifestEntries> + </transformer> + <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> + <transformer implementation="org.apache.maven.plugins.shade.resource.PluginXmlResourceTransformer"/> + <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer"> + <resource>META-INF/maven/${project.groupId}/${project.artifactId}/pom.xml</resource> + <file>${project.basedir}/dependency-reduced-pom.xml</file> + </transformer> + </transformers> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> \ No newline at end of file diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index 35f09540091..3bef295a06f 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -19,7 +19,7 @@ --> <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.pulsar</groupId> @@ -31,8 +31,7 @@ <name>Pulsar IO :: Kinesis</name> <properties> - <amazon-kinesis-client.version>2.2.8</amazon-kinesis-client.version> - <amazon-kinesis-producer.version>0.15.12</amazon-kinesis-producer.version> + <amazon-kinesis-client.version>3.1.2</amazon-kinesis-client.version> <json-flattener.version>0.13.0</json-flattener.version> <flatbuffers-java.version>1.9.0</flatbuffers-java.version> <jaxb-api.version>2.3.0</jaxb-api.version> @@ -50,6 +49,12 @@ </dependencyManagement> <dependencies> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>pulsar-io-kinesis-kpl-shaded</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>${project.groupId}</groupId> <artifactId>pulsar-io-common</artifactId> @@ -123,7 +128,6 @@ <dependency> <groupId>software.amazon.awssdk</groupId> <artifactId>utils</artifactId> - <version>2.22.12</version> </dependency> <dependency> @@ -132,12 +136,7 @@ <version>${amazon-kinesis-client.version}</version> </dependency> - <dependency> - <groupId>com.amazonaws</groupId> - <artifactId>amazon-kinesis-producer</artifactId> - <version>${amazon-kinesis-producer.version}</version> - </dependency> - <!-- /kinesis dependencies --> + <!-- /kinesis dependencies --> <dependency> <groupId>com.google.flatbuffers</groupId> @@ -177,6 +176,11 @@ <plugin> <groupId>org.apache.nifi</groupId> <artifactId>nifi-nar-maven-plugin</artifactId> + <configuration> + <excludes> + <exclude>**/amazon-kinesis-producer-*.jar</exclude> + </excludes> + </configuration> </plugin> </plugins> </build> diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java index 76686fbd899..db23972905a 100644 --- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java +++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java @@ -22,11 +22,6 @@ import static com.google.common.util.concurrent.Futures.addCallback; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static org.apache.commons.lang3.StringUtils.isNotBlank; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.services.kinesis.producer.KinesisProducer; -import com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration; -import com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration.ThreadingModel; -import com.amazonaws.services.kinesis.producer.UserRecordFailedException; -import com.amazonaws.services.kinesis.producer.UserRecordResult; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -56,6 +51,12 @@ import org.apache.pulsar.io.core.annotations.IOType; import org.apache.pulsar.io.kinesis.KinesisSinkConfig.MessageFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.kinesis.producer.KinesisProducer; +import software.amazon.kinesis.producer.KinesisProducerConfiguration; +import software.amazon.kinesis.producer.KinesisProducerConfiguration.ThreadingModel; +import software.amazon.kinesis.producer.UserRecordFailedException; +import software.amazon.kinesis.producer.UserRecordResult; /** * A Kinesis sink which can be configured by {@link KinesisSinkConfig}. @@ -191,10 +192,10 @@ public class KinesisSink extends AbstractAwsConnector implements Sink<GenericObj && kinesisSinkConfig.getSkipCertificateValidation()) { kinesisConfig.setVerifyCertificate(false); } - AWSCredentialsProvider credentialsProvider = createCredentialProvider( + AwsCredentialsProvider credentialsProvider = createCredentialProvider( kinesisSinkConfig.getAwsCredentialPluginName(), kinesisSinkConfig.getAwsCredentialPluginParam()) - .getCredentialProvider(); + .getV2CredentialsProvider(); kinesisConfig.setCredentialsProvider(credentialsProvider); kinesisConfig.setNativeExecutable(StringUtils.trimToEmpty(kinesisSinkConfig.getNativeExecutable())); kinesisConfig.setAggregationEnabled(kinesisSinkConfig.isAggregationEnabled()); diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java index 9d316e54419..d4f34a05596 100644 --- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java +++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java @@ -20,7 +20,6 @@ package org.apache.pulsar.io.kinesis; import static com.google.common.base.Preconditions.checkArgument; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.Serializable; @@ -31,6 +30,7 @@ import lombok.EqualsAndHashCode; import org.apache.pulsar.io.common.IOConfigUtils; import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.core.annotations.FieldDoc; +import software.amazon.kinesis.producer.KinesisProducerConfiguration; @Data @EqualsAndHashCode(callSuper = true) @@ -245,7 +245,7 @@ public class KinesisSinkConfig extends BaseKinesisConfig implements Serializable @FieldDoc( defaultValue = "", - help = "Extra KinesisProducerConfiguration parameters. See https://javadoc.io/static/com.amazonaws/amazon-kinesis-producer/0.15.2/index.html?com/amazonaws/services/kinesis/producer/KinesisProducerConfiguration.html for all the available parameters." + help = "Extra KinesisProducerConfiguration parameters. See https://javadoc.io/static/software.amazon.kinesis/amazon-kinesis-producer/1.0.4/software/amazon/kinesis/producer/KinesisProducerConfiguration.html for all the available parameters." + "Parameters that are explicitly set take preference over extra config.") private Map<String, String> extraKinesisProducerConfig = new HashMap<>(); diff --git a/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTest.java b/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTest.java index 1c6beee96b8..1a81965d38e 100644 --- a/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTest.java +++ b/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTest.java @@ -20,13 +20,13 @@ package org.apache.pulsar.io.kinesis; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; -import com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration; import java.io.IOException; import java.util.HashMap; import java.util.Map; import org.apache.pulsar.io.core.SinkContext; import org.mockito.Mockito; import org.testng.annotations.Test; +import software.amazon.kinesis.producer.KinesisProducerConfiguration; public class KinesisSinkConfigTest { diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index e4341d6a707..123d257596a 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -56,6 +56,7 @@ <module>kafka</module> <module>rabbitmq</module> <module>kinesis</module> + <module>kinesis-kpl-shaded</module> <module>hdfs3</module> <module>jdbc</module> <module>data-generator</module> @@ -93,6 +94,7 @@ <module>http</module> <module>rabbitmq</module> <module>kinesis</module> + <module>kinesis-kpl-shaded</module> <module>hdfs3</module> <module>jdbc</module> <module>data-generator</module> diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index ed822f1f072..0a5f26f9c7a 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -222,7 +222,7 @@ <dependency> <groupId>software.amazon.kinesis</groupId> <artifactId>amazon-kinesis-client</artifactId> - <version>2.2.8</version> + <version>3.1.2</version> <scope>test</scope> </dependency> <dependency>
