This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit aa27f16d00bc5644427b8e1deeed205b9244d068 Author: Joe McDonnell <[email protected]> AuthorDate: Wed Feb 18 17:06:50 2026 -0800 IMPALA-14620 (part 2): Fix code to use 'python3' rather than 'python' This changes bootstrap to stop installing Python 2. It also stops setting 'python' to refer to 'python3'. It adjusts scripts to point to python3 rather than python. For certain unused scripts (e.g. for the Docker-based tests), it leaves them using python with the understanding that they can be updated when revived (and may need other changes to adjust to Python 3). Testing: - Ran core jobs on Ubuntu 20 and Redhat 8 / 9 Change-Id: Ibe36d2b2a1464a5abb4fd1599e57cae2db45fbda Reviewed-on: http://gerrit.cloudera.org:8080/23999 Reviewed-by: Joe McDonnell <[email protected]> Tested-by: Joe McDonnell <[email protected]> --- bin/bootstrap_build.sh | 31 +------- bin/bootstrap_system.sh | 85 ---------------------- bin/create-test-configuration.sh | 2 +- bin/impala-python3-common.sh | 2 +- common/thrift/CMakeLists.txt | 4 +- docker/entrypoint.sh | 31 +------- .../impala/customcluster/LdapImpalaShellTest.java | 4 +- testdata/bin/download-impala-jdbc-driver.sh | 4 +- testdata/bin/jwt-generate.sh | 4 +- testdata/bin/setup-ranger.sh | 8 +- .../cluster/ranger/install.properties.template | 2 +- 11 files changed, 19 insertions(+), 158 deletions(-) diff --git a/bin/bootstrap_build.sh b/bin/bootstrap_build.sh index ea81e4439..e54895ea3 100755 --- a/bin/bootstrap_build.sh +++ b/bin/bootstrap_build.sh @@ -17,8 +17,8 @@ # specific language governing permissions and limitations # under the License. -# This script builds Impala from scratch. It is known to work on Ubuntu versions 16.04, -# 18.04, 20.04, 22.04 and 24.04. To run it you need to have: +# This script builds Impala from scratch. It is known to work on Ubuntu versions 20.04, +# 22.04 and 24.04. To run it you need to have: # # 1. At least 8GB of free disk space # 4. A connection to the internet (parts of the build download dependencies) @@ -33,38 +33,13 @@ set -euxo pipefail # Kerberos setup would pop up dialog boxes without this export DEBIAN_FRONTEND=noninteractive sudo -E apt-get --quiet update -# unversioned python-dev and python-setuptools are not available on newer releases -# that don't support Python 2. Add them only when they exist for the platform, -# otherwise set Python 3 to be the default Python version. +# Always install Python 3 and set it to be the default Python PACKAGES='g++ gcc git libsasl2-dev libssl-dev make ninja-build python3-dev python3-setuptools python3-venv libffi-dev language-pack-en libkrb5-dev krb5-admin-server krb5-kdc krb5-user libxml2-dev libxslt-dev wget' -if sudo apt-get --quiet install -s python-dev python-setuptools > /dev/null 2>&1; then - PACKAGES="${PACKAGES} python-dev python-setuptools" -else - PACKAGES="${PACKAGES} python-is-python3 python-dev-is-python3" -fi sudo -E apt-get --yes --quiet install ${PACKAGES} -source /etc/lsb-release - -# Ubuntu 20's Python 2.7.18-1~20.04.5 version has a bug in its tarfile support. -# If we detect the affected tarfile.py, download a patched version and overwrite it. -if [[ $DISTRIB_ID == Ubuntu && $DISTRIB_RELEASE == 20.04 ]]; then - if [[ -f /usr/lib/python2.7/tarfile.py ]]; then - TARFILE_PY_HASH=$(sha1sum /usr/lib/python2.7/tarfile.py | cut -d' ' -f1) - if [[ "${TARFILE_PY_HASH}" == "6e1a6d9ea2a535cbb17fe266ed9ac76eb5e27b89" ]]; then - TMP_DIR=$(mktemp -d) - pushd $TMP_DIR - wget -nv https://launchpadlibrarian.net/759546541/tarfile.py - sudo cp tarfile.py /usr/lib/python2.7/tarfile.py - popd - rm -rf $TMP_DIR - fi - fi -fi - JDK_VERSION=17 if [[ "$(uname -p)" == 'aarch64' ]]; then PACKAGE_ARCH='arm64' diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh index 9ee10a2e8..47fcf968e 100755 --- a/bin/bootstrap_system.sh +++ b/bin/bootstrap_system.sh @@ -252,27 +252,6 @@ ubuntu apt-get --yes install ccache curl file gawk g++ gcc apt-utils git libffi- libxslt-dev openjdk-${UBUNTU_JAVA_VERSION}-jdk \ openjdk-${UBUNTU_JAVA_VERSION}-source openjdk-${UBUNTU_JAVA_VERSION}-dbg -# Regular python packages don't exist on Ubuntu 22. Everything is Python 3. -ubuntu16 apt-get --yes install python python-dev python-setuptools -ubuntu18 apt-get --yes install python python-dev python-setuptools -ubuntu20 apt-get --yes install python python-dev python-setuptools - -# Ubuntu 20's Python 2.7.18-1~20.04.5 version has a bug in its tarfile support. -# If we detect the affected tarfile.py, download a patched version and overwrite it. -if [[ $UBUNTU20 == true ]]; then - if [[ -f /usr/lib/python2.7/tarfile.py ]]; then - TARFILE_PY_HASH=$(sha1sum /usr/lib/python2.7/tarfile.py | cut -d' ' -f1) - if [[ "${TARFILE_PY_HASH}" == "6e1a6d9ea2a535cbb17fe266ed9ac76eb5e27b89" ]]; then - TMP_DIR=$(mktemp -d) - pushd $TMP_DIR - wget -nv https://launchpadlibrarian.net/759546541/tarfile.py - sudo cp tarfile.py /usr/lib/python2.7/tarfile.py - popd - rm -rf $TMP_DIR - fi - fi -fi - # Required by Kudu in the minicluster. Older Kudu versions depend on libtinfo5, # versions that can be compiled for Ubuntu 24.04 depend on libtinfo6. ubuntu20 apt-get --yes install libtinfo5 libtinfo6 @@ -319,70 +298,6 @@ redhat8 sudo yum install -y fuse-devel curl # Adding --allowerasing allows curl to replace curl-minimal. redhat9 sudo yum install -y --allowerasing curl -# RedHat / CentOS 8 exposes only specific versions of Python. -# Set up unversioned default Python 2.x for older CentOS versions -redhat7 sudo yum install -y python-devel python-setuptools python-argparse - -# Install Python 2.x explicitly for CentOS 8 -function setup_python2() { - if command -v python && [[ $(python --version 2>&1 | cut -d ' ' -f 2) =~ 2\. ]]; then - echo "We have Python 2.x"; - else - if ! command -v python2; then - # Python2 needs to be installed - sudo dnf install -y python2 - fi - # Here Python2 is installed, but is not the default Python. - # 1. Link pip's version to Python's version - sudo alternatives --add-slave python /usr/bin/python2 /usr/bin/pip pip /usr/bin/pip2 - sudo alternatives --add-slave python /usr/libexec/no-python /usr/bin/pip pip \ - /usr/libexec/no-python - # 2. Set Python2 (with pip2) to be the system default. - sudo alternatives --set python /usr/bin/python2 - fi - # Here the Python2 runtime is already installed, add the dev package - sudo dnf -y install python2-devel -} - -# IMPALA-14606: Stop building using Python 2 and always run with -# IMPALA_USE_PYTHON3_TESTS=true. -# redhat8 setup_python2 -# redhat8 pip install --user argparse - -# Point Python to Python 3 for Redhat 9 and Ubuntu 22, or newer -function setup_python3() { - # If python is already set, then use it. Otherwise, try to point python to python3. - if ! command -v python > /dev/null; then - if command -v python3 ; then - # Newer OSes (e.g. Redhat 9 and equivalents) make it harder to get Python 2, and we - # need to start using Python 3 by default. - # For these new OSes (Ubuntu 22+, Redhat 9), there is no alternative entry for - # python, so we need to create one from scratch. - if command -v alternatives > /dev/null; then - if sudo alternatives --list | grep python > /dev/null ; then - sudo alternatives --set python /usr/bin/python3 - else - # The alternative doesn't exist, create it - sudo alternatives --install /usr/bin/python python /usr/bin/python3 20 - fi - elif command -v update-alternatives > /dev/null; then - # This is what Ubuntu 20/22+ does. There is no official python alternative, - # so we need to create one. - sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 20 - else - echo "ERROR: trying to set python to point to python3" - echo "ERROR: alternatives/update-alternatives don't exist, so giving up..." - exit 1 - fi - fi - fi -} - -redhat8 setup_python3 -redhat9 setup_python3 -ubuntu22 setup_python3 -ubuntu24 setup_python3 - # CentOS repos don't contain ccache, so install from EPEL redhat sudo yum install -y epel-release redhat sudo yum install -y ccache diff --git a/bin/create-test-configuration.sh b/bin/create-test-configuration.sh index ad6675aed..39507e590 100755 --- a/bin/create-test-configuration.sh +++ b/bin/create-test-configuration.sh @@ -223,7 +223,7 @@ if [ $CREATE_RANGER_POLICY_DB -eq 1 ]; then createdb -U hiveuser "${RANGER_POLICY_DB}" pushd "${RANGER_HOME}" generate_config "${RANGER_TEST_CONF_DIR}/install.properties.template" install.properties - python ./db_setup.py + python3 ./db_setup.py popd fi diff --git a/bin/impala-python3-common.sh b/bin/impala-python3-common.sh index c6cca8737..e800ad1e4 100644 --- a/bin/impala-python3-common.sh +++ b/bin/impala-python3-common.sh @@ -24,7 +24,7 @@ setup_report_build_error . $IMPALA_HOME/bin/set-pythonpath.sh -export LD_LIBRARY_PATH="$(python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \ +export LD_LIBRARY_PATH="$(python3 "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \ --print-ld-library-path)" PY_DIR="$(dirname "$0")/../infra/python" diff --git a/common/thrift/CMakeLists.txt b/common/thrift/CMakeLists.txt index bb9e5e189..6499e8ac3 100644 --- a/common/thrift/CMakeLists.txt +++ b/common/thrift/CMakeLists.txt @@ -234,11 +234,11 @@ SET_SOURCE_FILES_PROPERTIES(beeswax.thrift PROPERTIES OBJECT_DEPENDS ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift) add_custom_command(OUTPUT ErrorCodes.thrift - COMMAND python generate_error_codes.py + COMMAND python3 generate_error_codes.py DEPENDS generate_error_codes.py) add_custom_command(OUTPUT MetricDefs.thrift - COMMAND python generate_metrics.py + COMMAND python3 generate_metrics.py DEPENDS generate_metrics.py metrics.json) # The thrift-generated java classes defined in TCLIService are also pulled into our build diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index bdaad8bf8..5cf62988f 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -55,32 +55,6 @@ function _pg_ctl() { sudo service postgresql $1 } -# Install Python2 with pip2 and make them the default Python and pip commands -# on RedHat / CentOS 8. -# This has no notion of "default" Python, and can install both Python2 and Python3 -# side by side. Impala currently needs Python2 as the default version. -# The function is adaptive; it performs only the necessary steps; it shares the installer -# logic with bin/bootstrap_system.sh -function install_python2_for_centos8() { - if command -v python && [[ $(python --version 2>&1 | cut -d ' ' -f 2) =~ 2\. ]]; then - echo "We have Python 2.x"; - else - if ! command -v python2; then - # Python2 needs to be installed - dnf install -y python2 - fi - # Here Python2 is installed, but is not the default Python. - # 1. Link pip's version to Python's version - alternatives --add-slave python /usr/bin/python2 /usr/bin/pip pip /usr/bin/pip2 - alternatives --add-slave python /usr/libexec/no-python /usr/bin/pip pip \ - /usr/libexec/no-python - # 2. Set Python2 (with pip2) to be the system default. - alternatives --set python /usr/bin/python2 - fi - # Here the Python2 runtime is already installed, add the dev package - dnf -y install python2-devel -} - # Boostraps the container by creating a user and adding basic tools like Python and git. # Takes a uid as an argument for the user to be created. function build() { @@ -114,13 +88,10 @@ function build() { apt-get install -y sudo git python3 elif grep 'release 8\.' /etc/redhat-release; then # WARNING: Install the following packages one by one! - # Installing them in a common transaction breaks something inside yum/dnf, - # and the subsequent step installing Python2 will fail with a GPG signature error. + # Installing them in a common transaction breaks something inside yum/dnf. dnf -y install sudo dnf -y install which dnf -y install git-core - - install_python2_for_centos8 else yum -y install which sudo git python fi diff --git a/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java b/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java index 1e62919a5..7dfabcde9 100644 --- a/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java +++ b/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java @@ -96,9 +96,9 @@ public class LdapImpalaShellTest { */ protected boolean pythonSupportsSSLContext() throws Exception { // Runs the following command: - // python -c "import ssl; print hasattr(ssl, 'create_default_context')" + // python3 -c "import ssl; print hasattr(ssl, 'create_default_context')" String[] cmd = { - "python", "-c", "import ssl; print(hasattr(ssl, 'create_default_context'))"}; + "python3", "-c", "import ssl; print(hasattr(ssl, 'create_default_context'))"}; return Boolean.parseBoolean( RunShellCommand.Run(cmd, true, "", "").stdout.replace("\n", "")); } diff --git a/testdata/bin/download-impala-jdbc-driver.sh b/testdata/bin/download-impala-jdbc-driver.sh index 05d445295..cbe6eee45 100755 --- a/testdata/bin/download-impala-jdbc-driver.sh +++ b/testdata/bin/download-impala-jdbc-driver.sh @@ -57,8 +57,8 @@ pzf.extractall() __EOT__ # Extract driver jar file from zip file. -python ./unzip.py ${SIMBA_DRIVER_ZIP_FILENAME}.zip -python ./unzip.py ${SIMBA_DRIVER_ZIP_FILENAME}/${INNER_SIMBA_DRIVER_ZIP_FILENAME}.zip +python3 ./unzip.py ${SIMBA_DRIVER_ZIP_FILENAME}.zip +python3 ./unzip.py ${SIMBA_DRIVER_ZIP_FILENAME}/${INNER_SIMBA_DRIVER_ZIP_FILENAME}.zip # Copy driver jar file to Hadoop FS. hadoop fs -put -f /tmp/impala_jdbc_driver/${SIMBA_DRIVER_JAR_FILENAME} \ diff --git a/testdata/bin/jwt-generate.sh b/testdata/bin/jwt-generate.sh index 63a98c25c..f99dafada 100755 --- a/testdata/bin/jwt-generate.sh +++ b/testdata/bin/jwt-generate.sh @@ -44,6 +44,6 @@ export VIRTUAL_ENV_DISABLE_PROMPT source "${VENV_DIR}/bin/activate" # install necessary dependencies for the jwt generation python script -python -m pip install -r "$(dirname "${0}")/jwt_requirements.txt" +python3 -m pip install -r "$(dirname "${0}")/jwt_requirements.txt" -python "$(dirname "${0}")/jwt-util.py" "${DATA_DIR}" +python3 "$(dirname "${0}")/jwt-util.py" "${DATA_DIR}" diff --git a/testdata/bin/setup-ranger.sh b/testdata/bin/setup-ranger.sh index c8645dbed..52161ad02 100755 --- a/testdata/bin/setup-ranger.sh +++ b/testdata/bin/setup-ranger.sh @@ -36,7 +36,7 @@ function setup-ranger { --header="accept:application/json" \ --header="Content-Type:application/json" \ http://localhost:6080/service/xusers/secure/groups | - python -c "import sys, json; print(json.load(sys.stdin)['id'])") + python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") export GROUP_ID_OWNER GROUP_ID_NON_OWNER=$(wget -qO - --auth-no-challenge --user=admin \ @@ -44,7 +44,7 @@ function setup-ranger { --header="accept:application/json" \ --header="Content-Type:application/json" \ http://localhost:6080/service/xusers/secure/groups | - python -c "import sys, json; print(json.load(sys.stdin)['id'])") + python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") export GROUP_ID_NON_OWNER GROUP_ID_NON_OWNER_2=$(wget -qO - --auth-no-challenge --user=admin \ @@ -52,7 +52,7 @@ function setup-ranger { --header="accept:application/json" \ --header="Content-Type:application/json" \ http://localhost:6080/service/xusers/secure/groups | - python -c "import sys, json; print(json.load(sys.stdin)['id'])") + python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") export GROUP_ID_NON_OWNER_2 perl -wpl -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' \ @@ -109,7 +109,7 @@ function setup-ranger { http://localhost:6080/service/public/v2/api/service/test_impala/policy/\ all%20-%20database \ -H 'accept: application/json' | \ - python -c "import sys, json; print(json.load(sys.stdin)['id'])") + python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") export ALL_DATABASE_POLICY_ID perl -wpl -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' \ diff --git a/testdata/cluster/ranger/install.properties.template b/testdata/cluster/ranger/install.properties.template index 5358894c3..684af677c 100755 --- a/testdata/cluster/ranger/install.properties.template +++ b/testdata/cluster/ranger/install.properties.template @@ -14,7 +14,7 @@ # limitations under the License. #------------------------- DB CONFIG - BEGIN ---------------------------------- -PYTHON_COMMAND_INVOKER=python +PYTHON_COMMAND_INVOKER=python3 DB_FLAVOR=POSTGRES SQL_CONNECTOR_JAR=${POSTGRES_JDBC_DRIVER}
