This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 3d269e465e59b7f3af9f124598e66781ba8cd829 Author: Joe McDonnell <[email protected]> AuthorDate: Wed Sep 21 21:18:27 2022 -0700 IMPALA-11634: Provide an option to use Java 11 for docker images Currently, Docker images install Java 8 for Impala's use. This adds the IMPALA_DOCKER_USE_JAVA11 environment variable. When set to true, this installs Java 11 rather than Java 8. It defaults to false. The daemon_entrypoint.sh script is modified to detect Java 11 correctly. As a workaround for IMPALA-11260, this appends a list of "--add-opens" statements to JAVA_TOOL_OPTIONS when running with Java 11. Testing: - Ran a set of dockerized tests on Rocky 8.5 with Java 11 Change-Id: Icc1dbd3f6a2279840218dc1da2b60077e211a328 Reviewed-on: http://gerrit.cloudera.org:8080/19031 Reviewed-by: Joe McDonnell <[email protected]> Reviewed-by: Laszlo Gaal <[email protected]> Tested-by: Joe McDonnell <[email protected]> --- bin/impala-config.sh | 6 ++ .../dockerized-impala-bootstrap-and-test.sh | 3 +- docker/CMakeLists.txt | 3 + docker/daemon_entrypoint.sh | 65 +++++++++++++++++++++- docker/install_os_packages.sh | 27 ++++++++- 5 files changed, 99 insertions(+), 5 deletions(-) diff --git a/bin/impala-config.sh b/bin/impala-config.sh index fd705d84d..151f4b4b8 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -236,6 +236,12 @@ export IMPALA_COS_VERSION=3.1.0-5.9.3 export IMPALA_REDHAT7_DOCKER_BASE=${IMPALA_REDHAT7_DOCKER_BASE:-"centos:centos7.9.2009"} export IMPALA_REDHAT8_DOCKER_BASE=${IMPALA_REDHAT8_DOCKER_BASE:-"rockylinux:8.5"} +# When set to true, this modifies the Docker image build logic to install Java 11 +# rather than the default of Java 8. This is strictly about what is shipped in the +# Docker image, and it has no impact on what version of Java is used to compile +# Impala's Java code. +export IMPALA_DOCKER_USE_JAVA11=${IMPALA_DOCKER_USE_JAVA11:-"false"} + # When IMPALA_(CDP_COMPONENT)_URL are overridden, they may contain '$(platform_label)' # which will be substituted for the CDP platform label in bootstrap_toolchain.py unset IMPALA_HADOOP_URL diff --git a/bin/jenkins/dockerized-impala-bootstrap-and-test.sh b/bin/jenkins/dockerized-impala-bootstrap-and-test.sh index 7317ba93c..19b8c0220 100755 --- a/bin/jenkins/dockerized-impala-bootstrap-and-test.sh +++ b/bin/jenkins/dockerized-impala-bootstrap-and-test.sh @@ -35,7 +35,8 @@ source ./bin/bootstrap_system.sh # NOTE: A Jenkins job can also call dockerized-impala-preserve-vars.py directly # to preserve additional variables. ./bin/jenkins/dockerized-impala-preserve-vars.py \ - EE_TEST EE_TEST_FILES JDBC_TEST EXPLORATION_STRATEGY CMAKE_BUILD_TYPE + EE_TEST EE_TEST_FILES JDBC_TEST EXPLORATION_STRATEGY CMAKE_BUILD_TYPE \ + IMPALA_DOCKER_USE_JAVA11 # Execute the tests using su to re-login so that group change made above # setup_docker takes effect. This does a full re-login and does not stay diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt index 4692a4830..c94e97898 100644 --- a/docker/CMakeLists.txt +++ b/docker/CMakeLists.txt @@ -88,6 +88,9 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED") if (${build_type} STREQUAL "debug") set(INSTALL_OS_PACKAGES_ARGS "--install-debug-tools") endif() + if ($ENV{IMPALA_DOCKER_USE_JAVA11} STREQUAL "true") + set(INSTALL_OS_PACKAGES_ARGS "${INSTALL_OS_PACKAGES_ARGS} --use-java11") + endif() # Target for the base Impala image. add_custom_target(impala_base_image_${build_type} # Run docker build inside the build context directory so that all dependencies are diff --git a/docker/daemon_entrypoint.sh b/docker/daemon_entrypoint.sh index 6cc7bd89e..a62cc81f5 100755 --- a/docker/daemon_entrypoint.sh +++ b/docker/daemon_entrypoint.sh @@ -52,11 +52,35 @@ fi # Need libjvm.so and libjsig.so on LD_LIBRARY_PATH # Ubuntu and Redhat use different locations for JAVA_HOME. +# This detection logic handles both Java 8 and Java 11. +# If both are present (which shouldn't happen), Java 11 is preferred. JAVA_HOME=Unknown +USING_JAVA11=false if [[ $DISTRIBUTION == Ubuntu ]]; then - JAVA_HOME=$(compgen -G /usr/lib/jvm/java-8-openjdk-*) + # Since the Java location includes the CPU architecture, use a glob + # to find Java home + if compgen -G "/usr/lib/jvm/java-11-openjdk*" ; then + JAVA_HOME=$(compgen -G "/usr/lib/jvm/java-11-openjdk*") + USING_JAVA11=true + if compgen -G "/usr/lib/jvm/java-8-openjdk*" ; then + echo "WARNING: Java 8 is also present" + fi + elif compgen -G "/usr/lib/jvm/java-8-openjdk*" ; then + echo "Detected Java 8" + JAVA_HOME=$(compgen -G "/usr/lib/jvm/java-8-openjdk*") + fi elif [[ $DISTRIBUTION == Redhat ]]; then - JAVA_HOME=/usr/lib/jvm/jre-1.8.0 + if [[ -d /usr/lib/jvm/jre-11 ]]; then + echo "Detected Java 11" + JAVA_HOME=/usr/lib/jvm/jre-11 + USING_JAVA11=true + if [[ -d /usr/lib/jvm/jre-1.8.0 ]]; then + echo "WARNING: Java 8 is also present" + fi + elif [[ -d /usr/lib/jvm/jre-1.8.0 ]]; then + echo "Detected Java 8" + JAVA_HOME=/usr/lib/jvm/jre-1.8.0 + fi fi if [[ $JAVA_HOME == Unknown ]]; then @@ -93,9 +117,46 @@ done echo "CLASSPATH: $CLASSPATH" echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" +# IMPALA-11260: ehcache underestimates sizes on Java 9+ due to restrictions +# on reflection. As a workaround, this turns off the restrictions to allow +# ehcache to function. The real fix needs to happen on the ehcache side. +if $USING_JAVA11 ; then + JAVA11_OPTIONS=" --add-opens=java.base/java.io=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/java.lang.module=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/java.lang.ref=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/java.lang.reflect=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/java.lang=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/java.net=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/java.nio.charset=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/java.nio.file.attribute=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/java.nio=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/java.security=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/java.util.concurrent=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/java.util.jar=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/java.util.zip=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/java.util=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/jdk.internal.math=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/jdk.internal.module=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/jdk.internal.util.jar=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=java.base/sun.nio.fs=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=jdk.dynalink/jdk.dynalink.beans=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=jdk.dynalink/jdk.dynalink.linker.support=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=jdk.dynalink/jdk.dynalink.linker=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=jdk.dynalink/jdk.dynalink.support=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=jdk.dynalink/jdk.dynalink=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED" + JAVA11_OPTIONS+=" --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED" + JAVA_TOOL_OPTIONS="$JAVA11_OPTIONS $JAVA_TOOL_OPTIONS" +fi + # Default to 2GB heap. Allow overriding by externally-set JAVA_TOOL_OPTIONS. export JAVA_TOOL_OPTIONS="-Xmx2g $JAVA_TOOL_OPTIONS" +echo "JAVA_TOOL_OPTIONS: $JAVA_TOOL_OPTIONS" + # Various Hadoop libraries depend on having a username. If we're running under # an unknown username, create an entry in the password file for this user. if ! whoami ; then diff --git a/docker/install_os_packages.sh b/docker/install_os_packages.sh index db4348d1a..e05fdaa68 100755 --- a/docker/install_os_packages.sh +++ b/docker/install_os_packages.sh @@ -24,10 +24,12 @@ set -euo pipefail INSTALL_DEBUG_TOOLS=false +USE_JAVA11=false function print_usage { echo "install_os_packages.sh - Helper script to install OS dependencies" echo "[--install-debug-tools] : Also install debug tools like curl, iproute, etc" + echo "[--use-java11] : Use Java 11 rather than the default Java 8." } while [ -n "$*" ] @@ -36,6 +38,9 @@ do --install-debug-tools) INSTALL_DEBUG_TOOLS=true ;; + --use-java11) + USE_JAVA11=true + ;; --help|*) print_usage exit 1 @@ -45,6 +50,7 @@ do done echo "INSTALL_DEBUG_TOOLS=${INSTALL_DEBUG_TOOLS}" +echo "USE_JAVA11=${USE_JAVA11}" # This can get more detailed if there are specific steps # for specific versions, but at the moment the distribution @@ -58,6 +64,13 @@ else if [[ $DISTRIB_ID == Ubuntu ]]; then echo "Identified Ubuntu system." DISTRIBUTION=Ubuntu + + # Ubuntu 16.04 does not have Java 11 in its package repository, + # so exit with an error. + if [[ $DISTRIB_RELEASE == 16.04 ]] && $USE_JAVA11 ; then + echo "ERROR: Java 11 is not supported on Ubuntu 16.04" + exit 1 + fi fi fi @@ -72,12 +85,16 @@ fi if [[ $DISTRIBUTION == Ubuntu ]]; then export DEBIAN_FRONTEND=noninteractive apt-get update + if $USE_JAVA11 ; then + apt-get install -y openjdk-11-jre-headless + else + apt-get install -y openjdk-8-jre-headless + fi apt-get install -y \ krb5-user \ libsasl2-2 \ libsasl2-modules \ libsasl2-modules-gssapi-mit \ - openjdk-8-jre-headless \ tzdata if $INSTALL_DEBUG_TOOLS ; then echo "Installing extra debug tools" @@ -92,10 +109,16 @@ if [[ $DISTRIBUTION == Ubuntu ]]; then vim fi elif [[ $DISTRIBUTION == Redhat ]]; then + if $USE_JAVA11 ; then + yum install -y --disableplugin=subscription-manager \ + java-11-openjdk-headless + else + yum install -y --disableplugin=subscription-manager \ + java-1.8.0-openjdk-headless + fi yum install -y --disableplugin=subscription-manager \ cyrus-sasl-gssapi \ cyrus-sasl-plain \ - java-1.8.0-openjdk-headless \ krb5-workstation \ openldap-devel \ tzdata
