This is an automated email from the ASF dual-hosted git repository. boroknagyz pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 9cc19eb5f67248dff8edf8ea0193ec4f029f8b2a Author: Michael Smith <[email protected]> AuthorDate: Tue Feb 28 10:23:53 2023 -0800 IMPALA-11955: Detect system Pythons Detect system Pythons (2 and 3) during build configuration. Build the impala-shell tarball only using available Python versions, and test available versions. Drops support for DISABLE_PYTHON3_TEST as it's now automatically detected. If python3 is present on the system, it's expected to be usable. Testing: - built in SLES 15 SP4 container with Python 3 Change-Id: Iba36d0feba163e1251c66a6a49121d4dac625afc Reviewed-on: http://gerrit.cloudera.org:8080/19560 Reviewed-by: Michael Smith <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- CMakeLists.txt | 12 +++++++----- bin/impala-config.sh | 6 ++++++ shell/make_shell_tarball.sh | 32 +++++++++++++++++++------------- tests/shell/util.py | 19 ++++++++++--------- 4 files changed, 42 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 611c5a533..cb35b393a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -514,12 +514,14 @@ add_custom_target(impala_python ALL COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh" ) -if(NOT $ENV{DISABLE_PYTHON3_TEST} EQUAL "") - message(STATUS "DISABLE_PYTHON3_TEST is set, disabling Python3 virtualenv and tests") - add_custom_target(impala_shell_pypi ALL DEPENDS shell_python2_install) -else() - add_custom_target(impala_shell_pypi ALL DEPENDS shell_python2_install shell_python3_install) +set(IMPALA_PYTHON_INSTALLS "") +if (NOT $ENV{IMPALA_SYSTEM_PYTHON2} EQUAL "") + list(APPEND IMPALA_PYTHON_INSTALLS shell_python2_install) +endif() +if (NOT $ENV{IMPALA_SYSTEM_PYTHON3} EQUAL "") + list(APPEND IMPALA_PYTHON_INSTALLS shell_python3_install) endif() +add_custom_target(impala_shell_pypi ALL DEPENDS ${IMPALA_PYTHON_INSTALLS}) add_custom_target(notests_independent_targets DEPENDS java cscope tarballs impala_python impala_shell_pypi diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 8e08a2fce..ae9a00a42 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -196,6 +196,10 @@ unset IMPALA_THRIFT_JAVA_URL export IMPALA_THRIFT_PY_VERSION=0.16.0-p3 unset IMPALA_THRIFT_PY_URL +# Find system python versions for testing +export IMPALA_SYSTEM_PYTHON2=$(command -v python2) +export IMPALA_SYSTEM_PYTHON3=$(command -v python3) + if [[ $OSTYPE == "darwin"* ]]; then IMPALA_CYRUS_SASL_VERSION=2.1.26 unset IMPALA_CYRUS_SASL_URL @@ -982,6 +986,8 @@ echo "IMPALA_ICEBERG_VERSION = $IMPALA_ICEBERG_VERSION" echo "IMPALA_COS_VERSION = $IMPALA_COS_VERSION" echo "IMPALA_OSS_VERSION = $IMPALA_OSS_VERSION" echo "IMPALA_OBS_VERSION = $IMPALA_OBS_VERSION" +echo "IMPALA_SYSTEM_PYTHON2 = $IMPALA_SYSTEM_PYTHON2" +echo "IMPALA_SYSTEM_PYTHON3 = $IMPALA_SYSTEM_PYTHON3" # Kerberos things. If the cluster exists and is kerberized, source # the required environment. This is required for any hadoop tool to diff --git a/shell/make_shell_tarball.sh b/shell/make_shell_tarball.sh index 6a2273b44..775a63c48 100755 --- a/shell/make_shell_tarball.sh +++ b/shell/make_shell_tarball.sh @@ -118,25 +118,31 @@ for MODULE in ${SHELL_HOME}/ext-py/*; do continue; fi pushd ${MODULE} > /dev/null 2>&1 - echo "Cleaning up old build artifacts." - rm -rf dist 2>&1 > /dev/null - rm -rf build 2>&1 > /dev/null - echo "Creating an egg for ${MODULE}" - if [[ "$MODULE" == *"/bitarray"* ]]; then - # Need to use setuptools to build egg for bitarray module - python -c "import setuptools; exec(open('setup.py').read())" -q bdist_egg - else - python setup.py -q bdist_egg clean + if [ ! -z "${IMPALA_SYSTEM_PYTHON2:-}" ]; then + echo "Cleaning up old build artifacts." + rm -rf dist 2>&1 > /dev/null + rm -rf build 2>&1 > /dev/null + echo "Creating a Python 2 egg for ${MODULE}" + if [[ "$MODULE" == *"/bitarray"* ]]; then + # Need to use setuptools to build egg for bitarray module + ${IMPALA_SYSTEM_PYTHON2} -c "import setuptools; exec(open('setup.py').read())" \ + -q bdist_egg + else + ${IMPALA_SYSTEM_PYTHON2} setup.py -q bdist_egg clean + fi + cp dist/*.egg ${TARBALL_ROOT}/ext-py2 fi - cp dist/*.egg ${TARBALL_ROOT}/ext-py2 - if [ -z "${DISABLE_PYTHON3_TEST:-}" ]; then + if [ ! -z "${IMPALA_SYSTEM_PYTHON3:-}" ]; then + echo "Cleaning up old build artifacts." rm -rf dist 2>&1 > /dev/null rm -rf build 2>&1 > /dev/null + echo "Creating a Python 3 egg for ${MODULE}" if [[ "$MODULE" == *"/bitarray"* ]]; then # Need to use setuptools to build egg for bitarray module - python3 -c "import setuptools; exec(open('setup.py').read())" -q bdist_egg + ${IMPALA_SYSTEM_PYTHON3} -c "import setuptools; exec(open('setup.py').read())" \ + -q bdist_egg else - python3 setup.py -q bdist_egg clean + ${IMPALA_SYSTEM_PYTHON3} setup.py -q bdist_egg clean fi cp dist/*.egg ${TARBALL_ROOT}/ext-py3 fi diff --git a/tests/shell/util.py b/tests/shell/util.py index f369c646d..2097eabd3 100755 --- a/tests/shell/util.py +++ b/tests/shell/util.py @@ -337,16 +337,17 @@ def get_dev_impala_shell_executable(): def create_impala_shell_executable_dimension(dev_only=False): _, include_pypi = get_dev_impala_shell_executable() + dimensions = [] + if os.getenv("IMPALA_SYSTEM_PYTHON2"): + dimensions.append('dev') + if os.getenv("IMPALA_SYSTEM_PYTHON3"): + dimensions.append('dev3') if include_pypi and not dev_only: - if 'DISABLE_PYTHON3_TEST' in os.environ: - return ImpalaTestDimension('impala_shell', 'dev', 'python2') - else: - return ImpalaTestDimension('impala_shell', 'dev', 'dev3', 'python2', 'python3') - else: - if 'DISABLE_PYTHON3_TEST' in os.environ: - return ImpalaTestDimension('impala_shell', 'dev') - else: - return ImpalaTestDimension('impala_shell', 'dev', 'dev3') + if os.getenv("IMPALA_SYSTEM_PYTHON2"): + dimensions.append('python2') + if os.getenv("IMPALA_SYSTEM_PYTHON3"): + dimensions.append('python3') + return ImpalaTestDimension('impala_shell', *dimensions) def get_impala_shell_executable(vector):
