This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 234d641d7bba4c2cab8e7c2b5fb72d2861bea23f Author: Joe McDonnell <[email protected]> AuthorDate: Tue Jun 13 22:20:44 2023 -0700 IMPALA-11961/IMPALA-12207: Add Redhat 9 / Ubuntu 22 support This adds support for Redhat 9 / Ubuntu 22. It updates to a newer toolchain that has those builds, and it adds supporting code in bootstrap_system.sh. Redhat 9 and Ubuntu 22 use python = python3, which requires various changes to build scripts and tests. Ubuntu 22 uses Python 3.10, which deprecates certain ssl.PROTOCOL_TLS, so this adapts test_client_ssl.py to that change until it can be fully addressed in IMPALA-12219. Various OpenSSL methods have been deprecated. As a workaround until these can be addressed properly, this specifies -Wno-deprecated-declarations. This can be removed once the code is adapted to the non-deprecated APIs in IMPALA-12226. Impala crashes with tcmalloc errors unless we update to a newer gperftools, so this moves to gperftools 2.10. gperftools changed the default for tcmalloc.aggressive_memory_decommit to off, so this adapts our code to set it for backend tests. The gperftools upgrade does not show any performance regression: +----------+-----------------------+---------+------------+------------+----------------+ | Workload | File Format | Avg (s) | Delta(Avg) | GeoMean(s) | Delta(GeoMean) | +----------+-----------------------+---------+------------+------------+----------------+ | TPCH(42) | parquet / none / none | 3.08 | -0.64% | 2.20 | -0.37% | +----------+-----------------------+---------+------------+------------+----------------+ With newer Python versions, the impala-virtualenv command fails to create a Python 3 virtualenv. This switches to using Python 3's builtin venv command for Python >=3.6. Kudu needed a newer version and LLVM required a couple patches. Testing: - Ran a core job on Ubuntu 22 and Redhat 9. The tests run to completion without crashing. There are test failures that will be addressed in follow-up JIRAs. - Ran dockerised tests on Ubuntu 22. - Ran dockerised tests on Ubuntu 20 and Rocky 8.5. Change-Id: If1fcdb2f8c635ecd6dc7a8a1db81f5f389c78b86 Reviewed-on: http://gerrit.cloudera.org:8080/20073 Reviewed-by: Michael Smith <[email protected]> Tested-by: Joe McDonnell <[email protected]> --- be/CMakeLists.txt | 5 +- be/src/runtime/exec-env.cc | 8 +- be/src/runtime/exec-env.h | 7 ++ be/src/runtime/test-env.cc | 5 + bin/bootstrap_build.sh | 4 +- bin/bootstrap_system.sh | 126 ++++++++++++++------- bin/bootstrap_toolchain.py | 6 +- bin/cmake_aux/create_py3_virtualenv.sh | 53 +++++++++ bin/impala-config.sh | 13 ++- docker/CMakeLists.txt | 8 +- docker/entrypoint.sh | 5 +- .../impala/customcluster/LdapImpalaShellTest.java | 2 +- .../impala_py_lib/jenkins/junitxml_prune_notrun.py | 2 +- shell/CMakeLists.txt | 4 +- tests/custom_cluster/test_client_ssl.py | 13 ++- 15 files changed, 199 insertions(+), 62 deletions(-) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index b19423d90..572d944d3 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -35,6 +35,9 @@ PROJECT(ASSEMBLER) # -fsigned-char: on aarch64 platform, type of char default is unsigned char, here # set it to signed-char to be compatible with x86-64 # -Wno-deprecated: gutil contains deprecated headers +# -Wno-deprecated-declarations: OpenSSL3 deprecated various APIs currently used by +# Impala, so this disables those warnings until they can be addressed. +# See IMPALA-12226. # -Wno-vla: we use C99-style variable-length arrays # -pthread: enable multithreaded malloc # -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG: enable nanosecond precision for boost @@ -46,7 +49,7 @@ SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wno-sign-compare -Wno-unknown-p SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fno-strict-aliasing -fno-omit-frame-pointer") SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fsigned-char") SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=c++17") -SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated -Wno-vla") +SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated -Wno-deprecated-declarations -Wno-vla") SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG") SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_SYSTEM_NO_DEPRECATED") SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_BIND_GLOBAL_PLACEHOLDERS") diff --git a/be/src/runtime/exec-env.cc b/be/src/runtime/exec-env.cc index 087a8ef80..79be4b1f2 100644 --- a/be/src/runtime/exec-env.cc +++ b/be/src/runtime/exec-env.cc @@ -568,8 +568,7 @@ void ExecEnv::SetImpalaServer(ImpalaServer* server) { } } -void ExecEnv::InitBufferPool(int64_t min_buffer_size, int64_t capacity, - int64_t clean_pages_limit) { +void ExecEnv::InitTcMallocAggressiveDecommit() { #if !defined(ADDRESS_SANITIZER) && !defined(THREAD_SANITIZER) // Aggressive decommit is required so that unused pages in the TCMalloc page heap are // not backed by physical pages and do not contribute towards memory consumption. @@ -577,6 +576,11 @@ void ExecEnv::InitBufferPool(int64_t min_buffer_size, int64_t capacity, MallocExtension::instance()->SetNumericProperty( "tcmalloc.aggressive_memory_decommit", 1); #endif +} + +void ExecEnv::InitBufferPool(int64_t min_buffer_size, int64_t capacity, + int64_t clean_pages_limit) { + InitTcMallocAggressiveDecommit(); buffer_pool_.reset( new BufferPool(metrics_.get(), min_buffer_size, capacity, clean_pages_limit)); buffer_reservation_.reset(new ReservationTracker()); diff --git a/be/src/runtime/exec-env.h b/be/src/runtime/exec-env.h index 57e373b9a..521b81615 100644 --- a/be/src/runtime/exec-env.h +++ b/be/src/runtime/exec-env.h @@ -327,6 +327,13 @@ class ExecEnv { /// Initialize ExecEnv based on Hadoop config from frontend. Status InitHadoopConfig(); + /// Set tcmalloc's aggressive_memory_decommit=1. This needs to be called before + /// initializing the buffer pool, because the buffer pool asserts that this + /// property is set and newer versions of tcmalloc do not set it by default. + /// InitBufferPool() calls this automatically, so this is only used directly by + /// TestEnv. + void InitTcMallocAggressiveDecommit(); + /// Initialise 'buffer_pool_' and 'buffer_reservation_' with given capacity. void InitBufferPool(int64_t min_page_len, int64_t capacity, int64_t clean_pages_limit); diff --git a/be/src/runtime/test-env.cc b/be/src/runtime/test-env.cc index 54aebafa4..bd1d70410 100644 --- a/be/src/runtime/test-env.cc +++ b/be/src/runtime/test-env.cc @@ -73,6 +73,11 @@ Status TestEnv::Init() { if (enable_buffer_pool_) { exec_env_->InitBufferPool(buffer_pool_min_buffer_len_, buffer_pool_capacity_, static_cast<int64_t>(0.1 * buffer_pool_capacity_)); + } else { + // The buffer pool requires tcmalloc's aggressive_memory_decommit to be set. + // Since this codepath will never call InitBufferPool(), this needs to manually + // set it. + exec_env_->InitTcMallocAggressiveDecommit(); } if (process_mem_tracker_use_metrics_) { exec_env_->InitMemTracker(process_mem_limit_); diff --git a/bin/bootstrap_build.sh b/bin/bootstrap_build.sh index 71b43705b..d69394e3d 100755 --- a/bin/bootstrap_build.sh +++ b/bin/bootstrap_build.sh @@ -34,8 +34,8 @@ set -euxo pipefail export DEBIAN_FRONTEND=noninteractive sudo -E apt-get update sudo -E apt-get --yes install g++ gcc git libsasl2-dev libssl-dev make python-dev \ - python-setuptools python3-dev python3-setuptools libffi-dev libkrb5-dev \ - krb5-admin-server krb5-kdc krb5-user libxml2-dev libxslt-dev + python-setuptools python3-dev python3-setuptools python3-venv libffi-dev \ + libkrb5-dev krb5-admin-server krb5-kdc krb5-user libxml2-dev libxslt-dev source /etc/lsb-release diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh index d470e9036..965d434ea 100755 --- a/bin/bootstrap_system.sh +++ b/bin/bootstrap_system.sh @@ -70,17 +70,22 @@ set -x # Determine whether we're running on redhat or ubuntu REDHAT= -REDHAT6= REDHAT7= REDHAT8= +REDHAT9= UBUNTU= UBUNTU16= UBUNTU18= UBUNTU20= +UBUNTU22= IN_DOCKER= if [[ -f /etc/redhat-release ]]; then REDHAT=true echo "Identified redhat system." + if grep 'release 9\.' /etc/redhat-release; then + REDHAT9=true + echo "Identified redhat9 system." + fi if grep 'release 8\.' /etc/redhat-release; then REDHAT8=true echo "Identified redhat8 system." @@ -89,10 +94,6 @@ if [[ -f /etc/redhat-release ]]; then REDHAT7=true echo "Identified redhat7 system." fi - if grep 'release 6\.' /etc/redhat-release; then - REDHAT6=true - echo "Identified redhat6 system." - fi # TODO: restrict redhat versions else source /etc/lsb-release @@ -114,8 +115,12 @@ else then UBUNTU20=true echo "Identified Ubuntu 20.04 system." + elif [[ $DISTRIB_RELEASE = 22.04 ]] + then + UBUNTU22=true + echo "Identified Ubuntu 22.04 system." else - echo "This script supports Ubuntu versions 16.04, 18.04 or 20.04" >&2 + echo "This script supports Ubuntu versions 16.04, 18.04, 20.04, or 22.04" >&2 exit 1 fi else @@ -155,19 +160,19 @@ function ubuntu20 { fi } -# Helper function to execute following command only on RedHat -function redhat { - if [[ "$REDHAT" == true ]]; then +function ubuntu22 { + if [[ "$UBUNTU22" == true ]]; then "$@" fi } -# Helper function to execute following command only on RedHat6 -function redhat6 { - if [[ "$REDHAT6" == true ]]; then +# Helper function to execute following command only on RedHat +function redhat { + if [[ "$REDHAT" == true ]]; then "$@" fi } + # Helper function to execute following command only on RedHat7 function redhat7 { if [[ "$REDHAT7" == true ]]; then @@ -180,6 +185,12 @@ function redhat8 { "$@" fi } +# Helper function to execute following command only on RedHat8 +function redhat9 { + if [[ "$REDHAT9" == true ]]; then + "$@" + fi +} # Helper function to execute following command only in docker function indocker { if [[ "$IN_DOCKER" == true ]]; then @@ -219,16 +230,22 @@ fi source "$IMPALA_HOME/bin/impala-config-java.sh" ubuntu apt-get update -ubuntu apt-get --yes install ccache curl gawk g++ gcc apt-utils git libffi-dev \ +ubuntu apt-get --yes install ccache curl file gawk g++ gcc apt-utils git libffi-dev \ libkrb5-dev krb5-admin-server krb5-kdc krb5-user libsasl2-dev \ libsasl2-modules libsasl2-modules-gssapi-mit libssl-dev make ninja-build \ - python-dev python-setuptools python3-dev python3-setuptools postgresql \ + python3-dev python3-setuptools python3-venv postgresql \ ssh wget vim-common psmisc lsof net-tools language-pack-en libxml2-dev \ libxslt-dev openjdk-${UBUNTU_JAVA_VERSION}-jdk \ openjdk-${UBUNTU_JAVA_VERSION}-source openjdk-${UBUNTU_JAVA_VERSION}-dbg +# Regular python packages don't exist on Ubuntu 22. Everything is Python 3. +ubuntu16 apt-get --yes install python python-dev python-setuptools +ubuntu18 apt-get --yes install python python-dev python-setuptools +ubuntu20 apt-get --yes install python python-dev python-setuptools + # Required by Kudu in the minicluster ubuntu20 apt-get --yes install libtinfo5 +ubuntu22 apt-get --yes install libtinfo5 ARCH_NAME=$(uname -p) if [[ $ARCH_NAME == 'aarch64' ]]; then ubuntu apt-get --yes install unzip pkg-config flex maven python3-pip build-essential \ @@ -240,18 +257,24 @@ fi ubuntu sudo update-java-alternatives -s \ java-1.${UBUNTU_JAVA_VERSION}.0-openjdk-${UBUNTU_PACKAGE_ARCH} -redhat sudo yum install -y curl gawk gcc gcc-c++ git krb5-devel krb5-server \ +redhat sudo yum install -y file gawk gcc gcc-c++ git krb5-devel krb5-server \ krb5-workstation libevent-devel libffi-devel make openssl-devel cyrus-sasl \ cyrus-sasl-gssapi cyrus-sasl-devel cyrus-sasl-plain \ postgresql postgresql-server \ - wget vim-common nscd cmake fuse-devel zlib-devel \ - psmisc lsof openssh-server python3-devel python3-setuptools \ + wget vim-common nscd cmake zlib-devel \ + procps psmisc lsof openssh-server python3-devel python3-setuptools \ net-tools langpacks-en glibc-langpack-en libxml2-devel libxslt-devel \ java-${REDHAT_JAVA_VERSION}-openjdk-src java-${REDHAT_JAVA_VERSION}-openjdk-devel +# fuse-devel doesn't exist for Redhat 9 +redhat7 sudo yum install -y fuse-devel curl +redhat8 sudo yum install -y fuse-devel curl +# Redhat9 can have curl-minimal preinstalled, which can conflict with curl. +# Adding --allowerasing allows curl to replace curl-minimal. +redhat9 sudo yum install -y --allowerasing curl + # RedHat / CentOS 8 exposes only specific versions of Python. # Set up unversioned default Python 2.x for older CentOS versions -redhat6 sudo yum install -y python-devel python-setuptools python-argparse redhat7 sudo yum install -y python-devel python-setuptools python-argparse # Install Python 2.x explicitly for CentOS 8 @@ -278,6 +301,38 @@ function setup_python2() { redhat8 setup_python2 redhat8 pip install --user argparse +# Point Python to Python 3 for Redhat 9 and Ubuntu 22 +function setup_python3() { + # If python is already set, then use it. Otherwise, try to point python to python3. + if ! command -v python > /dev/null; then + if command -v python3 ; then + # Newer OSes (e.g. Redhat 9 and equivalents) make it harder to get Python 2, and we + # need to start using Python 3 by default. + # For these new OSes (Ubuntu 22, Redhat 9), there is no alternative entry for + # python, so we need to create one from scratch. + if command -v alternatives > /dev/null; then + if sudo alternatives --list | grep python > /dev/null ; then + sudo alternatives --set python /usr/bin/python3 + else + # The alternative doesn't exist, create it + sudo alternatives --install /usr/bin/python python /usr/bin/python3 20 + fi + elif command -v update-alternatives > /dev/null; then + # This is what Ubuntu 20/22+ does. There is no official python alternative, + # so we need to create one. + sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 20 + else + echo "ERROR: trying to set python to point to python3" + echo "ERROR: alternatives/update-alternatives don't exist, so giving up..." + exit 1 + fi + fi + fi +} + +redhat9 setup_python3 +ubuntu22 setup_python3 + # CentOS repos don't contain ccache, so install from EPEL redhat sudo yum install -y epel-release redhat sudo yum install -y ccache @@ -299,19 +354,16 @@ if [ ! -d /usr/local/apache-maven-3.5.4 ]; then MAVEN_DIRECTORY="/usr/local/apache-maven-3.5.4" redhat8 indocker sudo chmod 0755 ${MAVEN_DIRECTORY} redhat8 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}/{bin,boot} + redhat9 indocker sudo chmod 0755 ${MAVEN_DIRECTORY} + redhat9 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}/{bin,boot} fi if ! { service --status-all | grep -E '^ \[ \+ \] ssh$'; } then ubuntu sudo service ssh start - # TODO: CentOS/RH 7 uses systemd, and this doesn't work. - redhat6 sudo service sshd start - redhat7 notindocker sudo service sshd start - redhat8 notindocker sudo service sshd start - redhat7 indocker sudo /usr/bin/ssh-keygen -A - redhat7 indocker sudo /usr/sbin/sshd - redhat8 indocker sudo /usr/bin/ssh-keygen -A - redhat8 indocker sudo /usr/sbin/sshd + redhat notindocker sudo service sshd start + redhat indocker sudo /usr/bin/ssh-keygen -A + redhat indocker sudo /usr/sbin/sshd # The CentOS 8.1 image includes /var/run/nologin by mistake; this file prevents # SSH logins. See https://github.com/CentOS/sig-cloud-instance-images/issues/60 redhat8 indocker sudo rm -f /var/run/nologin @@ -323,14 +375,9 @@ fi echo ">>> Configuring system" -redhat6 sudo service postgresql initdb -redhat6 sudo service postgresql stop -redhat7 notindocker sudo service postgresql initdb -redhat7 notindocker sudo service postgresql stop -redhat7 indocker sudo -u postgres PGDATA=/var/lib/pgsql/data pg_ctl init -redhat8 notindocker sudo service postgresql initdb -redhat8 notindocker sudo service postgresql stop -redhat8 indocker sudo -u postgres PGDATA=/var/lib/pgsql/data pg_ctl init +redhat notindocker sudo service postgresql initdb +redhat notindocker sudo service postgresql stop +redhat indocker sudo -u postgres PGDATA=/var/lib/pgsql/data pg_ctl init ubuntu sudo service postgresql stop # These configurations expose connectiong to PostgreSQL via md5-hashed @@ -344,14 +391,10 @@ redhat sudo sed -ri 's/local +all +all +(ident|peer)/local all all trust/g' \ redhat sudo sed -i -e 's,\(host.*\)ident,\1md5,' /var/lib/pgsql/data/pg_hba.conf ubuntu sudo service postgresql start -redhat6 sudo service postgresql start -redhat7 notindocker sudo service postgresql start -redhat8 notindocker sudo service postgresql start +redhat notindocker sudo service postgresql start # Important to redirect pg_ctl to a logfile, lest it keep the stdout # file descriptor open, preventing the shell from exiting. -redhat7 indocker sudo -u postgres PGDATA=/var/lib/pgsql/data bash -c \ - "pg_ctl start -w --timeout=120 >> /var/lib/pgsql/pg.log 2>&1" -redhat8 indocker sudo -u postgres PGDATA=/var/lib/pgsql/data bash -c \ +redhat indocker sudo -u postgres PGDATA=/var/lib/pgsql/data bash -c \ "pg_ctl start -w --timeout=120 >> /var/lib/pgsql/pg.log 2>&1" # Set up postgres for HMS @@ -419,11 +462,10 @@ echo -e "\n* - nofile 1048576" | sudo tee -a /etc/security/limits.conf # Default on CentOS limits a user to 1024 or 4096 processes (threads) , which isn't # enough for minicluster with all of its friends. -redhat6 sudo sed -i 's,\*\s*soft\s*nproc\s*[0-9]*$,* soft nproc unlimited,' \ - /etc/security/limits.d/*-nproc.conf redhat7 sudo sed -i 's,\*\s*soft\s*nproc\s*[0-9]*$,* soft nproc unlimited,' \ /etc/security/limits.d/*-nproc.conf redhat8 echo -e "* soft nproc unlimited" | sudo tee -a /etc/security/limits.conf +redhat9 echo -e "* soft nproc unlimited" | sudo tee -a /etc/security/limits.conf echo ">>> Checking out Impala" diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py index 7d9cc70b6..f52c76d20 100755 --- a/bin/bootstrap_toolchain.py +++ b/bin/bootstrap_toolchain.py @@ -80,11 +80,15 @@ OS_MAPPING = [ OsMapping("centos8", "ec2-package-centos-8"), OsMapping("rocky8", "ec2-package-centos-8"), OsMapping("almalinux8", "ec2-package-centos-8"), + OsMapping("rhel9", "ec2-package-rocky-9"), + OsMapping("rocky9", "ec2-package-rocky-9"), + OsMapping("almalinux9", "ec2-package-rocky-9"), OsMapping("sles12", "ec2-package-sles-12"), OsMapping("sles15", "ec2-package-sles-15"), OsMapping('ubuntu16', "ec2-package-ubuntu-16-04"), OsMapping('ubuntu18', "ec2-package-ubuntu-18-04"), - OsMapping('ubuntu20', "ec2-package-ubuntu-20-04") + OsMapping('ubuntu20', "ec2-package-ubuntu-20-04"), + OsMapping('ubuntu22', "ec2-package-ubuntu-22-04") ] diff --git a/bin/cmake_aux/create_py3_virtualenv.sh b/bin/cmake_aux/create_py3_virtualenv.sh new file mode 100755 index 000000000..5e0a8fe98 --- /dev/null +++ b/bin/cmake_aux/create_py3_virtualenv.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Create a python3 virtualenv. When system python is 3.6 or higher, +# we can just use the built-in venv module to create the virtualenv. +# If system python is older or the built-in venv module fails, then +# this falls back to impala-virtualenv, which uses python2 to +# initialize a virtualenv using python3. +# +# This takes a single argument, which is the destination directory +# for the virtualenv: +# create_py3_virtualenv.sh venv_dir + +set -euo pipefail + +# We should only be calling this when system python3 is available +[[ -n ${IMPALA_SYSTEM_PYTHON3} ]] + +VENV_DIR=$1 + +IS_PY36_OR_HIGHER=$(${IMPALA_SYSTEM_PYTHON3} -c \ + "import sys; print('true' if sys.version_info.minor >= 6 else 'false')") + +# If using Python >= 3.6, try to use the builtin venv package. +if $IS_PY36_OR_HIGHER ; then + if ${IMPALA_SYSTEM_PYTHON3} -m venv ${VENV_DIR} ; then + # Success + exit 0 + fi +fi + +if $IS_PY36_OR_HIGHER ; then + echo "WARNING: Tried to create virtualenv with Python3's venv module and failed." + echo "Falling back to old impala-virtualenv path..." +fi +# Fall back to using the old impala-virtualenv method +impala-virtualenv --python ${IMPALA_SYSTEM_PYTHON3} ${VENV_DIR} + diff --git a/bin/impala-config.sh b/bin/impala-config.sh index cb4ca79a4..778f7b9ca 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -81,7 +81,7 @@ export USE_APACHE_HIVE=${USE_APACHE_HIVE-false} # moving to a different build of the toolchain, e.g. when a version is bumped or a # compile option is changed. The build id can be found in the output of the toolchain # build jobs, it is constructed from the build number and toolchain git hash prefix. -export IMPALA_TOOLCHAIN_BUILD_ID=296-f7e1d0d78b +export IMPALA_TOOLCHAIN_BUILD_ID=320-33e177225e # Versions of toolchain dependencies. # ----------------------------------- export IMPALA_AVRO_VERSION=1.7.4-p5 @@ -114,7 +114,7 @@ export IMPALA_GFLAGS_VERSION=2.2.0-p2 unset IMPALA_GFLAGS_URL export IMPALA_GLOG_VERSION=0.3.5-p3 unset IMPALA_GLOG_URL -export IMPALA_GPERFTOOLS_VERSION=2.5-p4 +export IMPALA_GPERFTOOLS_VERSION=2.10 unset IMPALA_GPERFTOOLS_URL export IMPALA_GTEST_VERSION=1.6.0 unset IMPALA_GTEST_URL @@ -124,9 +124,9 @@ export IMPALA_LIBEV_VERSION=4.20-p1 unset IMPALA_LIBEV_URL export IMPALA_LIBUNWIND_VERSION=1.5.0-p1 unset IMPALA_LIBUNWIND_URL -export IMPALA_LLVM_VERSION=5.0.1-p5 +export IMPALA_LLVM_VERSION=5.0.1-p7 unset IMPALA_LLVM_URL -export IMPALA_LLVM_ASAN_VERSION=5.0.1-p5 +export IMPALA_LLVM_ASAN_VERSION=5.0.1-p7 unset IMPALA_LLVM_ASAN_URL # To limit maximum memory available for the mini-cluster and CDH cluster, add the @@ -140,7 +140,7 @@ export IMPALA_LLVM_UBSAN_BASE_VERSION=5.0.1 # Debug builds should use the release+asserts build to get additional coverage. # Don't use the LLVM debug build because the binaries are too large to distribute. -export IMPALA_LLVM_DEBUG_VERSION=5.0.1-asserts-p5 +export IMPALA_LLVM_DEBUG_VERSION=5.0.1-asserts-p7 unset IMPALA_LLVM_DEBUG_URL export IMPALA_LZ4_VERSION=1.9.3 unset IMPALA_LZ4_URL @@ -265,6 +265,7 @@ export IMPALA_OBS_VERSION=3.1.1-hw-42 # free distributions, but Redhat UBI images are known to work. export IMPALA_REDHAT7_DOCKER_BASE=${IMPALA_REDHAT7_DOCKER_BASE:-"centos:centos7.9.2009"} export IMPALA_REDHAT8_DOCKER_BASE=${IMPALA_REDHAT8_DOCKER_BASE:-"rockylinux:8.5"} +export IMPALA_REDHAT9_DOCKER_BASE=${IMPALA_REDHAT9_DOCKER_BASE:-"rockylinux:9.2"} # When set to true, this instructs start-impala-cluster.py to start with Java 11 images # created via 'make docker_java11_images' (or 'docker_debug_java11_images'). The Java @@ -947,7 +948,7 @@ fi # overall build type) and does not apply when using a local Kudu build. export USE_KUDU_DEBUG_BUILD=${USE_KUDU_DEBUG_BUILD-false} -export IMPALA_KUDU_VERSION=${IMPALA_KUDU_VERSION-"345fd44ca3"} +export IMPALA_KUDU_VERSION=${IMPALA_KUDU_VERSION-"14c326461c"} export IMPALA_KUDU_HOME=${IMPALA_TOOLCHAIN_PACKAGES_HOME}/kudu-$IMPALA_KUDU_VERSION export IMPALA_KUDU_JAVA_HOME=\ ${IMPALA_TOOLCHAIN_PACKAGES_HOME}/kudu-${IMPALA_KUDU_VERSION}/java diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt index b4aebf352..fb620fa31 100644 --- a/docker/CMakeLists.txt +++ b/docker/CMakeLists.txt @@ -39,14 +39,16 @@ set(DISTRO_BASE_IMAGE "UNSUPPORTED") if(${OS_DISTRIB_ID} STREQUAL "ubuntu") if(${OS_DISTRIB_VERSION_ID} STREQUAL "16.04" OR ${OS_DISTRIB_VERSION_ID} STREQUAL "18.04" OR - ${OS_DISTRIB_VERSION_ID} STREQUAL "20.04") + ${OS_DISTRIB_VERSION_ID} STREQUAL "20.04" OR + ${OS_DISTRIB_VERSION_ID} STREQUAL "22.04") set(DISTRO_BASE_IMAGE "ubuntu:${OS_DISTRIB_VERSION_ID}") set(QUICKSTART_BASE_IMAGE "ubuntu:${OS_DISTRIB_VERSION_ID}") endif() if (${OS_DISTRIB_VERSION_ID} STREQUAL "16.04" OR ${OS_DISTRIB_VERSION_ID} STREQUAL "18.04") set(PIP "python-pip") - elseif (${OS_DISTRIB_VERSION_ID} STREQUAL "20.04") + elseif (${OS_DISTRIB_VERSION_ID} STREQUAL "20.04" OR + ${OS_DISTRIB_VERSION_ID} STREQUAL "22.04") set(PIP "python3-pip") endif() elseif(${OS_DISTRIB_ID} STREQUAL "rhel" OR @@ -59,6 +61,8 @@ elseif(${OS_DISTRIB_ID} STREQUAL "rhel" OR set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT7_DOCKER_BASE}") elseif(${OS_DISTRIB_VERSION_ID} MATCHES "8.*") set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT8_DOCKER_BASE}") + elseif(${OS_DISTRIB_VERSION_ID} MATCHES "9.*") + set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT9_DOCKER_BASE}") endif() endif() MESSAGE(STATUS "Picked docker base image based on host OS: ${DISTRO_BASE_IMAGE}") diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index a26585a1a..9de5b64fa 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -108,7 +108,10 @@ function build() { fi if command -v apt-get > /dev/null; then apt-get update - apt-get install -y sudo git lsb-release python + # The 'python' package doesn't exist on Ubuntu 22, so this installs python3. + # TODO: It might not be necessary to install python here, as the container + # will invoke bootstrap_system.sh. + apt-get install -y sudo git python3 elif grep 'release 8\.' /etc/redhat-release; then # WARNING: Install the following packages one by one! # Installing them in a common transaction breaks something inside yum/dnf, diff --git a/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java b/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java index c6177a045..3cc6c8e31 100644 --- a/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java +++ b/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java @@ -75,7 +75,7 @@ public class LdapImpalaShellTest { // Runs the following command: // python -c "import ssl; print hasattr(ssl, 'create_default_context')" String[] cmd = { - "python", "-c", "import ssl; print hasattr(ssl, 'create_default_context')"}; + "python", "-c", "import ssl; print(hasattr(ssl, 'create_default_context'))"}; return Boolean.parseBoolean(RunShellCommand.Run(cmd, true, "", "").replace("\n", "")); } diff --git a/lib/python/impala_py_lib/jenkins/junitxml_prune_notrun.py b/lib/python/impala_py_lib/jenkins/junitxml_prune_notrun.py index 02bc04ec5..3abfb8823 100755 --- a/lib/python/impala_py_lib/jenkins/junitxml_prune_notrun.py +++ b/lib/python/impala_py_lib/jenkins/junitxml_prune_notrun.py @@ -58,7 +58,7 @@ def junitxml_prune_notrun(junitxml_filename): # not support an XML declaration on Python 2.6, so use minidom to write the XML. root_node_minidom = minidom.parseString(ET.tostring(root)) junitxml_string = root_node_minidom.toxml(encoding="utf-8") - with open(junitxml_filename, "w") as f: + with open(junitxml_filename, "wb") as f: f.write(junitxml_string) diff --git a/shell/CMakeLists.txt b/shell/CMakeLists.txt index 104125bc4..3c670a9c7 100644 --- a/shell/CMakeLists.txt +++ b/shell/CMakeLists.txt @@ -53,8 +53,10 @@ add_custom_target(shell_python2_install DEPENDS "${PYTHON2_VENV}" shell_pypi_tes COMMAND "${PYTHON2_VENV}/bin/pip" install --cache-dir "${PYTHON2_PIP_CACHE}" "${SHELL_TEST_PKG}" ) +# In cases where system python3 is old, this can use impala-virtualenv, so it +# needs to depend on impala_python. add_custom_command(OUTPUT "${PYTHON3_VENV}" DEPENDS impala_python - COMMAND impala-virtualenv --python "$ENV{IMPALA_SYSTEM_PYTHON3}" "${PYTHON3_VENV}" + COMMAND "${CMAKE_SOURCE_DIR}/bin/cmake_aux/create_py3_virtualenv.sh" "${PYTHON3_VENV}" ) add_custom_target(shell_python3_install DEPENDS "${PYTHON3_VENV}" shell_pypi_test_package diff --git a/tests/custom_cluster/test_client_ssl.py b/tests/custom_cluster/test_client_ssl.py index 99109cd00..1a23a9872 100644 --- a/tests/custom_cluster/test_client_ssl.py +++ b/tests/custom_cluster/test_client_ssl.py @@ -21,6 +21,7 @@ import json import logging import os import pytest +import re import requests import signal import ssl @@ -244,18 +245,26 @@ class TestClientSsl(CustomClusterTestSuite): run_impala_shell_cmd(vector, args, expect_success=False) def _validate_positive_cases(self, vector, ca_cert=""): + python3_10_version_re = re.compile(r"using Python 3\.1[0-9]") shell_options = ["--ssl", "-q", "select 1 + 2"] result = run_impala_shell_cmd(vector, shell_options, wait_until_connected=False) for msg in [self.SSL_ENABLED, self.CONNECTED, self.FETCHED]: assert msg in result.stderr - assert self.DEPRECATION_WARNING not in result.stderr + # Python >3.10 has deprecated ssl.PROTOCOL_TLS and impala-shell currently emits a + # DeprecationWarning for that version. As a temporary workaround, this skips the + # assert about deprecation for Python 3.10 or above. This can be removed when + # IMPALA-12219 is fixed. + # Note: This is the version that impala-shell uses, not the version pytests uses. + if not python3_10_version_re.search(result.stderr): + assert self.DEPRECATION_WARNING not in result.stderr if ca_cert != "": shell_options = shell_options + ["--ca_cert=%s" % ca_cert] result = run_impala_shell_cmd(vector, shell_options, wait_until_connected=False) for msg in [self.SSL_ENABLED, self.CONNECTED, self.FETCHED]: assert msg in result.stderr - assert self.DEPRECATION_WARNING not in result.stderr + if not python3_10_version_re.search(result.stderr): + assert self.DEPRECATION_WARNING not in result.stderr def _verify_ssl_webserver(self): for port in ["25000", "25010", "25020"]:
