This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 2e6a8eb1f20 [improve][build] Support building docker images that use
Wolfi base image which is glibc based (#24692)
2e6a8eb1f20 is described below
commit 2e6a8eb1f200ca687064058a101963713b896fe2
Author: Lari Hotari <[email protected]>
AuthorDate: Tue Sep 2 23:09:31 2025 +0300
[improve][build] Support building docker images that use Wolfi base image
which is glibc based (#24692)
---
.github/changes-filter.yaml | 8 +-
.github/workflows/pulsar-ci.yaml | 81 +++++++++++---
.../Dockerfile.wolfi} | 44 +++-----
docker/pulsar-all/pom.xml | 11 ++
docker/pulsar/Dockerfile.wolfi | 123 +++++++++++++++++++++
.../remove-unnecessary-native-binaries.sh | 8 +-
docker/pulsar/pom.xml | 11 ++
.../docker-images/latest-version-image/Dockerfile | 3 +-
tests/docker-images/latest-version-image/pom.xml | 11 ++
.../integration/profiling/PulsarProfilingTest.java | 16 ++-
.../integration/topologies/PulsarCluster.java | 20 ++++
11 files changed, 287 insertions(+), 49 deletions(-)
diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml
index 12850852082..681dd63718b 100644
--- a/.github/changes-filter.yaml
+++ b/.github/changes-filter.yaml
@@ -22,4 +22,10 @@ need_owasp:
- 'pom.xml'
- '**/pom.xml'
- 'src/owasp-dependency-check-false-positives.xml'
- - 'src/owasp-dependency-check-suppressions.xml'
\ No newline at end of file
+ - 'src/owasp-dependency-check-suppressions.xml'
+pulsar_dockerfile_wolfi:
+ - 'docker/pulsar/Dockerfile.wolfi'
+ - 'tests/docker-images/java-test-image/Dockerfile'
+pulsar_all_dockerfile_wolfi:
+ - 'docker/pulsar-all/Dockerfile.wolfi'
+ - 'tests/docker-images/latest-version-image/Dockerfile'
\ No newline at end of file
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index 679bd08009d..ad234bec6f2 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -70,6 +70,14 @@ on:
- 'fail_on_leak'
- 'off'
default: 'report'
+ docker_image_type:
+ description: 'Type of docker image to build. When set to "alpine", an
Alpine based docker image will be built. When set to "wolfi", a Wolfi based
docker image will be built.'
+ required: true
+ type: choice
+ options:
+ - 'alpine'
+ - 'wolfi'
+ default: 'alpine'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{
github.event_name == 'workflow_dispatch' &&
github.event.inputs.jdk_major_version || '' }}
@@ -95,6 +103,8 @@ jobs:
jdk_major_version: ${{ steps.jdk_major_version.outputs.jdk_major_version
}}
java_non_tests: ${{ steps.changes.outputs.java_non_tests }}
netty_leak_detection: ${{
steps.netty_leak_detection.outputs.netty_leak_detection }}
+ java_test_image_base_matrix: ${{
steps.docker_images_matrix.outputs.java_test_image_base_matrix }}
+ latest_version_image_base_matrix: ${{
steps.docker_images_matrix.outputs.latest_version_image_base_matrix }}
steps:
- name: Cancel scheduled jobs in forks by default
if: ${{ github.repository != 'apache/pulsar' && github.event_name ==
'schedule' }}
@@ -137,19 +147,57 @@ jobs:
echo docs_only=false >> $GITHUB_OUTPUT
fi
+ - name: Choose docker images to build
+ id: docker_images_matrix
+ run: |
+ # build JSON for java_test_image_base_matrix and
latest_version_image_base_matrix
+ # This JSON is used as input or the build matrixes in the docker
image build jobs.
+ # For workflow_dispatch, the docker_image_type input is used to
determine the base image
+ # For scheduled builds, the default base image is alpine
+ # For pull requests, the default base image is alpine. If the
changed files include Dockerfile.wolfi changes
+ # the wolfi base image is built in addition to the alpine image.
+ # create json for the default base image of the build
+ main_type="alpine"
+ if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then
+ main_type="${{ github.event.inputs.docker_image_type }}"
+ fi
+ if [[ "$main_type" == "wolfi" ]]; then
+ main_type_json='{"type": "'$main_type'", "save_artifact": true,
"additional_build_args": "-Pdocker-wolfi"}'
+ else
+ main_type_json='{"type": "'$main_type'", "save_artifact": true}'
+ fi
+ # build JSON for java_test_image_base_matrix
+ echo "java_test_image_base_matrix<<EOF" >> $GITHUB_OUTPUT
+ echo "[" >> $GITHUB_OUTPUT
+ echo "$main_type_json" >> $GITHUB_OUTPUT
+ if [[ ${{ github.event_name == 'pull_request' &&
fromJSON(steps.changes.outputs.pulsar_dockerfile_wolfi_count) || 0 }} -gt 0 ]];
then
+ echo ',{"type": "wolfi", "save_artifact": false,
"additional_build_args": "-Pdocker-wolfi"}' >> $GITHUB_OUTPUT
+ fi
+ echo "]" >> $GITHUB_OUTPUT
+ echo "EOF" >> $GITHUB_OUTPUT
+ # build JSON for latest_version_image_base_matrix
+ echo "latest_version_image_base_matrix<<EOF" >> $GITHUB_OUTPUT
+ echo "[" >> $GITHUB_OUTPUT
+ echo "$main_type_json" >> $GITHUB_OUTPUT
+ if [[ ${{ github.event_name == 'pull_request' &&
fromJSON(steps.changes.outputs.pulsar_all_dockerfile_wolfi_count) || 0 }} -gt 0
]]; then
+ echo ',{"type": "wolfi", "save_artifact": false,
"additional_build_args": "-Pdocker-wolfi"}' >> $GITHUB_OUTPUT
+ fi
+ echo "]" >> $GITHUB_OUTPUT
+ echo "EOF" >> $GITHUB_OUTPUT
+
- name: Check if coverage should be collected
id: check_coverage
run: |
- echo "collect_coverage=${{
- (steps.check_changes.outputs.docs_only != 'true' &&
github.event_name != 'workflow_dispatch'
- && (github.base_ref == 'master' || github.ref_name == 'master'))
+ echo "collect_coverage=${{
+ (steps.check_changes.outputs.docs_only != 'true' &&
github.event_name != 'workflow_dispatch'
+ && (github.base_ref == 'master' || github.ref_name == 'master'))
|| (github.event_name == 'workflow_dispatch' &&
github.event.inputs.collect_coverage == 'true')
}}" >> $GITHUB_OUTPUT
- name: Set Netty leak detection mode
id: netty_leak_detection
run: |
- echo "netty_leak_detection=${{
+ echo "netty_leak_detection=${{
github.event_name == 'workflow_dispatch' &&
github.event.inputs.netty_leak_detection || 'report'
}}" >> $GITHUB_OUTPUT
@@ -457,7 +505,7 @@ jobs:
- name: Restore coverage files from build artifacts and create Jacoco
reports
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh
restore_unittest_coverage_files
- $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh
create_test_coverage_report
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
cd $GITHUB_WORKSPACE/target
zip -qr jacoco_test_coverage_report_unittests.zip
jacoco_test_coverage_report || true
@@ -487,7 +535,7 @@ jobs:
action: wait
pulsar-java-test-image:
- name: Build Pulsar java-test-image docker image
+ name: Build Pulsar java-test-image ${{ matrix.base.type }} docker image
for ${{ matrix.platform }}
runs-on: ubuntu-22.04
timeout-minutes: 60
needs: ['preconditions', 'build-and-license-check']
@@ -495,6 +543,7 @@ jobs:
strategy:
fail-fast: false
matrix:
+ base: ${{
fromJSON(needs.preconditions.outputs.java_test_image_base_matrix) }}
platform:
- linux/amd64
- linux/arm64
@@ -547,16 +596,16 @@ jobs:
with:
platforms: arm64
- - name: Build java-test-image docker image - ${{ matrix.platform }}
+ - name: Build java-test-image ${{ matrix.base.type }} docker image - ${{
matrix.platform }}
run: |
# build docker image
DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl
docker/pulsar,tests/docker-images/java-test-image install
-Pcore-modules,-main,integrationTests,docker \
- -Ddocker.platforms=${{ matrix.platform }} \
+ -Ddocker.platforms=${{ matrix.platform }} ${{
matrix.base.additional_build_args }} \
-Dmaven.test.skip=true -DskipSourceReleaseAssembly=true \
-Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true
-Drat.skip=true
- name: save docker image apachepulsar/java-test-image:latest to Github
artifact cache
- if: ${{ matrix.platform == 'linux/amd64' }}
+ if: ${{ matrix.base.save_artifact && matrix.platform == 'linux/amd64'
}}
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh
docker_save_image_to_github_actions_artifacts
apachepulsar/java-test-image:latest pulsar-java-test-image
@@ -814,7 +863,7 @@ jobs:
- name: Restore coverage files from build artifacts and create Jacoco
reports
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh
restore_inttest_coverage_files
- $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh
create_inttest_coverage_report
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh
create_inttest_coverage_report
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
cd $GITHUB_WORKSPACE/target
zip -qr jacoco_test_coverage_report_inttests.zip
jacoco_test_coverage_report jacoco_inttest_coverage_report || true
@@ -869,11 +918,15 @@ jobs:
gh-actions-artifact-client.js delete pulsar-java-test-image.zst
pulsar-test-latest-version-image:
- name: Build Pulsar docker image
+ name: Build Pulsar ${{ matrix.base.type }} docker image
runs-on: ubuntu-22.04
timeout-minutes: 60
needs: ['preconditions', 'build-and-license-check']
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
+ strategy:
+ fail-fast: false
+ matrix:
+ base: ${{
fromJSON(needs.preconditions.outputs.latest_version_image_base_matrix) }}
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version
}}
@@ -924,12 +977,12 @@ jobs:
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh
restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
- - name: Build latest-version-image docker image
+ - name: Build latest-version-image docker ${{ matrix.base.type }} image
run: |
# build docker image
# include building of Connectors, Offloaders and server distros
DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl
distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image
install \
- -Pmain,docker -Dmaven.test.skip=true \
+ -Pmain,docker -Dmaven.test.skip=true ${{
matrix.base.additional_build_args }} \
-Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true
-Drat.skip=true
# check full build artifacts licenses
@@ -957,6 +1010,7 @@ jobs:
sarif_file: 'trivy-results.sarif'
- name: Clean up disk space
+ if: ${{ matrix.base.save_artifact }}
run: |
# release disk space since saving docker image consumes local disk
space
#
@@ -981,6 +1035,7 @@ jobs:
echo "::endgroup::"
- name: save docker image apachepulsar/pulsar-test-latest-version:latest
to Github artifact cache
+ if: ${{ matrix.base.save_artifact }}
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh
docker_save_image_to_github_actions_artifacts
apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image
diff --git a/docker/pulsar/build-scripts/remove-unnecessary-native-binaries.sh
b/docker/pulsar-all/Dockerfile.wolfi
old mode 100755
new mode 100644
similarity index 50%
copy from docker/pulsar/build-scripts/remove-unnecessary-native-binaries.sh
copy to docker/pulsar-all/Dockerfile.wolfi
index fe97b71179d..ce9f4235331
--- a/docker/pulsar/build-scripts/remove-unnecessary-native-binaries.sh
+++ b/docker/pulsar-all/Dockerfile.wolfi
@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+# syntax=docker/dockerfile:1
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,34 +18,20 @@
# under the License.
#
-set -e
+# Dockerfile for pulsar-all image which uses Wolfi (chainguard/wolfi-base) as
it's base image
+# More information about Wolfi OS: https://github.com/wolfi-dev/
+# wolfi-base CVEs:
https://images.chainguard.dev/directory/image/wolfi-base/vulnerabilities
-# Retain only native libraries for the architecture of this
-# image
-ARCH=$(uname -m)
+# global arguments (only to be used in FROM clauses)
+ARG PULSAR_IMAGE
-# Remove extra binaries for Netty TCNative
-if [ "$ARCH" = "aarch64" ]
-then
- TC_NATIVE_TO_KEEP=linux-aarch_64
-else
- TC_NATIVE_TO_KEEP=linux-$ARCH
-fi
-ls /pulsar/lib/io.netty-netty-tcnative-boringssl-static*Final-*.jar | grep -v
$TC_NATIVE_TO_KEEP | xargs rm
+FROM busybox AS pulsar-extensions
+ARG PULSAR_IO_DIR
+ADD ${PULSAR_IO_DIR} /connectors
+ARG PULSAR_OFFLOADER_TARBALL
+ADD ${PULSAR_OFFLOADER_TARBALL} /
+RUN mv /apache-pulsar-offloaders-*/offloaders /offloaders
-# Prune extra libs from RocksDB JAR
-mkdir /tmp/rocksdb
-cd /tmp/rocksdb
-ROCKSDB_JAR=$(ls /pulsar/lib/org.rocksdb-rocksdbjni-*.jar)
-unzip $ROCKSDB_JAR > /dev/null
-
-if [ "$ARCH" = "x86_64" ]
-then
- ROCKSDB_TO_KEEP=linux64-musl
-else
- ROCKSDB_TO_KEEP=linux-$ARCH-musl
-fi
-
-ls librocksdbjni-* | grep -v librocksdbjni-${ROCKSDB_TO_KEEP}.so | xargs rm
-rm $ROCKSDB_JAR
-zip -r -9 $ROCKSDB_JAR * > /dev/null
+FROM ${PULSAR_IMAGE}
+COPY --from=pulsar-extensions /connectors /pulsar/connectors
+COPY --from=pulsar-extensions /offloaders /pulsar/offloaders
\ No newline at end of file
diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml
index 4f579aa0f04..bd59bbeac9a 100644
--- a/docker/pulsar-all/pom.xml
+++ b/docker/pulsar-all/pom.xml
@@ -30,6 +30,10 @@
<name>Apache Pulsar :: Docker Images :: Pulsar Latest Version (Include All
Components)</name>
<packaging>pom</packaging>
+ <properties>
+ <docker.dockerfile>Dockerfile</docker.dockerfile>
+ </properties>
+
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
@@ -156,6 +160,7 @@
<image>
<name>${docker.organization}/${docker.image}-all:${docker.tag}</name>
<build>
+ <dockerFile>${docker.dockerfile}</dockerFile>
<contextDir>${project.basedir}</contextDir>
<skipTag>${docker.skip.tag}</skipTag>
<tags>
@@ -182,6 +187,12 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>docker-wolfi</id>
+ <properties>
+ <docker.dockerfile>Dockerfile.wolfi</docker.dockerfile>
+ </properties>
+ </profile>
<profile>
<id>docker-push</id>
<properties>
diff --git a/docker/pulsar/Dockerfile.wolfi b/docker/pulsar/Dockerfile.wolfi
new file mode 100644
index 00000000000..061dead5955
--- /dev/null
+++ b/docker/pulsar/Dockerfile.wolfi
@@ -0,0 +1,123 @@
+# syntax=docker/dockerfile:1
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Dockerfile for pulsar image which uses Wolfi (chainguard/wolfi-base) as it's
base image
+# More information about Wolfi OS: https://github.com/wolfi-dev/
+# wolfi-base CVEs:
https://images.chainguard.dev/directory/image/wolfi-base/vulnerabilities
+
+ARG IMAGE_JDK_MAJOR_VERSION=21
+
+# First create a stage with just the Pulsar tarball and scripts
+FROM chainguard/wolfi-base as pulsar
+
+RUN apk add zip
+
+ARG PULSAR_TARBALL
+
+ADD ${PULSAR_TARBALL} /
+RUN mv /apache-pulsar-* /pulsar
+RUN rm -rf /pulsar/bin/*.cmd
+
+COPY build-scripts /build-scripts/
+RUN /build-scripts/remove-unnecessary-native-binaries.sh glibc
+
+COPY scripts/* /pulsar/bin/
+
+# The final image needs to give the root group sufficient permission for
Pulsar components
+# to write to specific directories within /pulsar
+# The ownership is changed to uid 10000 to allow using a different root group.
This is necessary when running the
+# container when gid=0 is prohibited. In that case, the container must be run
with uid 10000 with
+# any group id != 0 (for example 10001).
+# The file permissions are preserved when copying files from this builder
image to the target image.
+RUN for SUBDIRECTORY in conf data download logs instances/deps
packages-storage; do \
+ mkdir -p /pulsar/$SUBDIRECTORY; \
+ chmod -R ug+rwx /pulsar/$SUBDIRECTORY; \
+ chown -R 10000:0 /pulsar/$SUBDIRECTORY; \
+ done
+
+RUN chmod -R g+rx /pulsar/bin
+RUN chmod -R o+rx /pulsar
+
+### Create one stage to include JVM distribution
+FROM amazoncorretto:${IMAGE_JDK_MAJOR_VERSION} AS jvm
+
+# Install binutils which is required by jlink
+RUN yum update -y && yum install -y binutils
+
+# Use JLink to create a slimmer JDK distribution (see:
https://adoptium.net/blog/2021/10/jlink-to-produce-own-runtime/)
+# This still includes all JDK modules, though in the future we could compile a
list of required modules
+# first try with Java 17/21 compatible jlink command and if it fails, fallback
to Java 24+ compatible command
+RUN /usr/lib/jvm/java/bin/jlink --add-modules ALL-MODULE-PATH --compress=2
--no-man-pages --no-header-files --strip-debug --output /opt/jvm ||
/usr/lib/jvm/java/bin/jlink --module-path /usr/lib/jvm/java/jmods --add-modules
ALL-MODULE-PATH --compress=zip-9 --no-man-pages --no-header-files --strip-debug
--output /opt/jvm
+RUN echo networkaddress.cache.ttl=1 >> /opt/jvm/conf/security/java.security
+RUN echo networkaddress.cache.negative.ttl=1 >>
/opt/jvm/conf/security/java.security
+
+## Create final stage
+## and add OpenJDK and Python dependencies (for Pulsar functions)
+FROM chainguard/wolfi-base
+ENV LANG C.UTF-8
+
+ARG PYTHON_VERSION=3.12
+# Install some utilities, some are required by Pulsar scripts
+RUN apk add --no-cache \
+ bash \
+ python-${PYTHON_VERSION} \
+ py${PYTHON_VERSION}-pip \
+ libgcc \
+ libstdc++ \
+ libuuid \
+ ca-certificates \
+ procps \
+ curl \
+ bind-tools \
+ openssl \
+ coreutils
+
+# Upgrade all packages to get latest versions with security fixes
+RUN apk upgrade --no-cache
+
+# Python dependencies
+# The pinned grpcio and protobuf versions should be compatible with the
generated Protobuf and gRPC stubs used
+# in Pulsar Functions Python runtime. You should also update the grpcio
version in src/update_python_protobuf_stubs.sh
+# and regenerate the Python stubs if you change the grpcio version here.
Please see
+# pulsar-functions/instance/src/main/python/README.md for more details.
+ARG PULSAR_CLIENT_PYTHON_VERSION
+RUN pip3 install --break-system-packages --no-cache-dir \
+ --only-binary \
+ grpcio==1.73.1 \
+ protobuf==6.31.1 \
+ pulsar-client[all]==${PULSAR_CLIENT_PYTHON_VERSION} \
+ kazoo \
+ pyyaml
+
+COPY --from=jvm /opt/jvm /opt/jvm
+ENV JAVA_HOME=/opt/jvm
+
+# The default is /pulsat/bin and cannot be written.
+ENV PULSAR_PID_DIR=/pulsar/logs
+
+COPY --from=pulsar /pulsar /pulsar
+
+WORKDIR /pulsar
+ENV PATH=$PATH:$JAVA_HOME/bin:/pulsar/bin
+
+# The UID must be non-zero. Otherwise, it is arbitrary. No logic should rely
on its specific value.
+ARG DEFAULT_USERNAME=pulsar
+RUN adduser ${DEFAULT_USERNAME} -u 10000 -G root -D -H -h /pulsar/data
+USER 10000
diff --git a/docker/pulsar/build-scripts/remove-unnecessary-native-binaries.sh
b/docker/pulsar/build-scripts/remove-unnecessary-native-binaries.sh
index fe97b71179d..d96e0460351 100755
--- a/docker/pulsar/build-scripts/remove-unnecessary-native-binaries.sh
+++ b/docker/pulsar/build-scripts/remove-unnecessary-native-binaries.sh
@@ -41,9 +41,13 @@ unzip $ROCKSDB_JAR > /dev/null
if [ "$ARCH" = "x86_64" ]
then
- ROCKSDB_TO_KEEP=linux64-musl
+ ROCKSDB_TO_KEEP=linux64
else
- ROCKSDB_TO_KEEP=linux-$ARCH-musl
+ ROCKSDB_TO_KEEP=linux-$ARCH
+fi
+
+if [ "$1" != "glibc" ]; then
+ ROCKSDB_TO_KEEP="${ROCKSDB_TO_KEEP}-musl"
fi
ls librocksdbjni-* | grep -v librocksdbjni-${ROCKSDB_TO_KEEP}.so | xargs rm
diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
index 95f5ed21844..72ab40852fb 100644
--- a/docker/pulsar/pom.xml
+++ b/docker/pulsar/pom.xml
@@ -30,6 +30,10 @@
<name>Apache Pulsar :: Docker Images :: Pulsar Latest Version</name>
<packaging>pom</packaging>
+ <properties>
+ <docker.dockerfile>Dockerfile</docker.dockerfile>
+ </properties>
+
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
@@ -85,6 +89,7 @@
<SNAPPY_VERSION>${snappy.version}</SNAPPY_VERSION>
<IMAGE_JDK_MAJOR_VERSION>${IMAGE_JDK_MAJOR_VERSION}</IMAGE_JDK_MAJOR_VERSION>
</args>
+ <dockerFile>${docker.dockerfile}</dockerFile>
<contextDir>${project.basedir}</contextDir>
<skipTag>${docker.skip.tag}</skipTag>
<tags>
@@ -123,6 +128,12 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>docker-wolfi</id>
+ <properties>
+ <docker.dockerfile>Dockerfile.wolfi</docker.dockerfile>
+ </properties>
+ </profile>
<profile>
<id>docker-push</id>
<properties>
diff --git a/tests/docker-images/latest-version-image/Dockerfile
b/tests/docker-images/latest-version-image/Dockerfile
index 981c892d60e..e4bfa77355c 100644
--- a/tests/docker-images/latest-version-image/Dockerfile
+++ b/tests/docker-images/latest-version-image/Dockerfile
@@ -18,9 +18,10 @@
#
ARG PULSAR_ALL_IMAGE
+ARG GOLANG_IMAGE
# build go lang examples first in a separate layer
-FROM golang:1.23-alpine as pulsar-function-go
+FROM $GOLANG_IMAGE as pulsar-function-go
COPY target/pulsar-function-go/
/go/src/github.com/apache/pulsar/pulsar-function-go
RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go && go install ./...
diff --git a/tests/docker-images/latest-version-image/pom.xml
b/tests/docker-images/latest-version-image/pom.xml
index 3e1495e1de4..ba2f94e7474 100644
--- a/tests/docker-images/latest-version-image/pom.xml
+++ b/tests/docker-images/latest-version-image/pom.xml
@@ -30,6 +30,10 @@
<name>Apache Pulsar :: Tests :: Docker Images :: Latest Version
Testing</name>
<packaging>pom</packaging>
+ <properties>
+ <docker.golang.image>golang:1.23-alpine</docker.golang.image>
+ </properties>
+
<profiles>
<profile>
<id>docker</id>
@@ -169,6 +173,7 @@
<contextDir>${project.basedir}</contextDir>
<args>
<PULSAR_ALL_IMAGE>${docker.organization}/${docker.image}-all:${project.version}-${git.commit.id.abbrev}</PULSAR_ALL_IMAGE>
+ <GOLANG_IMAGE>${docker.golang.image}</GOLANG_IMAGE>
<INSTALL_ASYNC_PROFILER>${docker.install.asyncprofiler}</INSTALL_ASYNC_PROFILER>
</args>
<noCache>true</noCache>
@@ -187,5 +192,11 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>docker-wolfi</id>
+ <properties>
+ <docker.golang.image>golang:1.23</docker.golang.image>
+ </properties>
+ </profile>
</profiles>
</project>
diff --git
a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingTest.java
b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingTest.java
index 1ca4352615e..ac312f8cfd5 100644
---
a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingTest.java
+++
b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingTest.java
@@ -100,7 +100,8 @@ public class PulsarProfilingTest extends PulsarTestSuite {
"/pulsar/bin/pulsar-perf", "consume", topicName,
"-u", "pulsar://" + brokerHostname + ":6650",
"-st", "Shared",
- "-m", String.valueOf(numberOfMessages), "-ml", "200M");
+ "-aq",
+ "-m", String.valueOf(numberOfMessages), "-ml", "400M");
}
public CompletableFuture<Long> produce(String topicName) throws
Exception {
@@ -110,7 +111,7 @@ public class PulsarProfilingTest extends PulsarTestSuite {
"-au", "http://"; + brokerHostname + ":8080",
"-r", String.valueOf(Integer.MAX_VALUE), // max-rate
"-s", "8192", // 8kB message size
- "-m", String.valueOf(numberOfMessages), "-ml", "200M");
+ "-m", String.valueOf(numberOfMessages), "-ml", "400M");
}
}
@@ -136,6 +137,14 @@ public class PulsarProfilingTest extends PulsarTestSuite {
super.tearDownCluster();
}
+ @Override
+ protected void beforeStartCluster() throws Exception {
+ super.beforeStartCluster();
+ pulsarCluster.forEachContainer(
+ // This is effective only when -Pdocker-wolfi has been passed
when building java-test-image
+ c -> c.withEnv("GLIBC_TUNABLES",
"glibc.malloc.hugetlb=1:glibc.malloc.mmap_threshold=2097152"));
+ }
+
@Override
protected PulsarClusterSpec.PulsarClusterSpecBuilder
beforeSetupCluster(String clusterName,
PulsarClusterSpec.PulsarClusterSpecBuilder specBuilder) {
@@ -157,7 +166,8 @@ public class PulsarProfilingTest extends PulsarTestSuite {
"managedLedgerMaxSizePerLedgerMbytes", "512",
"managedLedgerDefaultEnsembleSize", "1",
"managedLedgerDefaultWriteQuorum", "1",
- "managedLedgerDefaultAckQuorum", "1"
+ "managedLedgerDefaultAckQuorum", "1",
+ "maxPendingPublishRequestsPerConnection", "100000"
));
// Increase memory for bookkeepers and make compaction run more often
diff --git
a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java
b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java
index 2ee571dafbb..73d9076ad08 100644
---
a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java
+++
b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java
@@ -34,6 +34,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.function.Consumer;
import java.util.function.Function;
import lombok.Cleanup;
import lombok.Getter;
@@ -310,6 +311,25 @@ public class PulsarCluster {
return proxyContainer.getPlainTextServiceUrl();
}
+ public void forEachContainer(Consumer<GenericContainer<?>> consumer) {
+ if (zkContainer != null) {
+ consumer.accept(zkContainer);
+ }
+ if (csContainer != null) {
+ consumer.accept(csContainer);
+ }
+ if (oxiaContainer != null) {
+ consumer.accept(oxiaContainer);
+ }
+ if (proxyContainer != null) {
+ consumer.accept(proxyContainer);
+ }
+ bookieContainers.values().forEach(consumer);
+ brokerContainers.values().forEach(consumer);
+ workerContainers.values().forEach(consumer);
+ externalServices.values().forEach(consumer);
+ }
+
public String getHttpServiceUrl() {
return proxyContainer.getHttpServiceUrl();
}
