This is an automated email from the ASF dual-hosted git repository. michaelsmith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 683bef1ca46cb986b33797b7d1dcff63c0ebe65f Author: Michael Smith <[email protected]> AuthorDate: Wed May 24 10:36:23 2023 -0700 IMPALA-11253: Support testing with Java 11 (take 2) Adds new environment variable IMPALA_JDK_VERSION which can be 'system', '8', or '11'. The default is 'system', which uses the same logic as before. If set to 8 or 11, it will ignore the system java and search for java of that specific version (based on specific directories for Ubuntu and Redhat). This is used by bin/bootstrap_system.sh to determine whether to install java 8 or java 11 (other versions can come later). If IMPALA_JDK_VERSION=11, then bin/start-impala-cluster.py adds the opens needed to deal with the ehcache issue. This no longer puts JAVA_HOME in bin/impala-config-local.sh as part of bootstrap_system.sh. Instead, it provides a new environment variable IMPALA_JAVA_HOME_OVERRIDE, which will be preferred over IMPALA_JDK_VERSION. This also updates the versions of Maven plugins related to the build. Source and target releases are still set to Java 8 compatibility. Adds a verifier to the end of run-all-tests that InaccessibleObjectException is not present in impalad logs. Tested with JDBC_TEST=false EE_TEST=false FE_TEST=false BE_TEST=false \ CLUSTER_TEST_FILES=custom_cluster/test_local_catalog.py \ run-all-tests.sh Testing: ran test suite with Java 11 This reverts the revert commit 1b6011c, restoring these changes minus code to update IMPALA_JDK_VERSION based on $JAVA -version as that could break subsequent sourcing of impala-config.sh. Change-Id: Ie16504ad5738b1f228f97044afd3d9017ccc6c53 Reviewed-on: http://gerrit.cloudera.org:8080/19928 Reviewed-by: Joe McDonnell <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- CMakeLists.txt | 2 + LICENSE.txt | 34 --- README-build.md | 4 +- bin/bootstrap_system.sh | 66 +---- .../impala-config-java.sh | 31 +-- bin/impala-config.sh | 74 +++++- bin/run-all-tests.sh | 14 + cmake_modules/FindJNI.cmake | 281 --------------------- fe/pom.xml | 16 +- .../org/apache/impala/util/JMXJsonUtilTest.java | 3 +- java/TableFlattener/pom.xml | 2 +- java/datagenerator/pom.xml | 4 +- java/executor-deps/pom.xml | 2 +- java/ext-data-source/api/pom.xml | 6 +- java/ext-data-source/sample/pom.xml | 4 +- java/ext-data-source/test/pom.xml | 2 +- java/query-event-hook-api/pom.xml | 4 +- java/shaded-deps/hive-exec/pom.xml | 2 +- java/shaded-deps/s3a-aws-sdk/pom.xml | 2 +- java/test-corrupt-hive-udfs/pom.xml | 2 +- java/test-hive-udfs/pom.xml | 2 +- java/yarn-extras/pom.xml | 2 +- testdata/bin/run-ranger-server.sh | 8 +- tests/verifiers/test_banned_log_messages.py | 43 ++++ 24 files changed, 188 insertions(+), 422 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b7e858812..e1a02ddeb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -384,6 +384,8 @@ IMPALA_ADD_THIRDPARTY_LIB(crcutil ${CRCUTIL_INCLUDE_DIR} ${CRCUTIL_STATIC_LIB} ${CRCUTIL_SHARED_LIB}) # find jni headers and libs +set(JAVA_AWT_LIBRARY NotNeeded) +set(JAVA_AWT_INCLUDE_PATH NotNeeded) find_package(JNI REQUIRED) IMPALA_ADD_THIRDPARTY_LIB(java_jvm "${JNI_INCLUDE_DIRS}" "" ${JAVA_JVM_LIBRARY}) diff --git a/LICENSE.txt b/LICENSE.txt index b0f39c838..62598a490 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -701,40 +701,6 @@ cmake_modules/FindGTest.cmake: MIT license -------------------------------------------------------------------------------- -cmake_modules/FindJNI.cmake: 3-clause BSD - - Copyright 2001-2009 Kitware, Inc. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the names of Kitware, Inc., the Insight Software Consortium, - nor the names of their contributors may be used to endorse or promote - products derived from this software without specific prior written - permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - be/src/kudu/util (some portions): 3-clause BSD license Some portions of this module are derived from code from LevelDB diff --git a/README-build.md b/README-build.md index 4afead00a..51f750552 100644 --- a/README-build.md +++ b/README-build.md @@ -31,8 +31,8 @@ can do so through the environment variables and scripts listed below. | CDP_COMPONENTS_HOME | "${IMPALA_HOME}/toolchain/cdp_components-${CDP_BUILD_NUMBER}" | Location of the CDP components within the toolchain. | | CDH_MAJOR_VERSION | "7" | Identifier used to uniqueify paths for potentially incompatible component builds. | | IMPALA_CONFIG_SOURCED | "1" | Set by ${IMPALA_HOME}/bin/impala-config.sh (internal use) | -| JAVA_HOME | "/usr/lib/jvm/${JAVA_VERSION}" | Used to locate Java | -| JAVA_VERSION | "java-7-oracle-amd64" | Can override to set a local Java version. | +| IMPALA_JAVA_HOME_OVERRIDE | | Specify a non-system Java version. Overrides IMPALA_JDK_VERSION behavior. | +| IMPALA_JDK_VERSION | "system" | Set to 8 or 11 to select a system Java version. Default will set JAVA_HOME based on the javac symlink in PATH. | | JAVA | "${JAVA_HOME}/bin/java" | Java binary location. | | CLASSPATH | | See bin/set-classpath.sh for details. | | PYTHONPATH | Will be changed to include: "${IMPALA_HOME}/shell/gen-py" "${IMPALA_HOME}/testdata" "${THRIFT_PY_HOME}/python/lib/python2.7/site-packages" "${HIVE_HOME}/lib/py" | diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh index 2483ead5e..5a9186e5c 100755 --- a/bin/bootstrap_system.sh +++ b/bin/bootstrap_system.sh @@ -214,13 +214,19 @@ if [[ "$UBUNTU" == true ]]; then sleep 1 done fi + +# Set UBUNTU_JAVA_VERSION, UBUNTU_PACKAGE_ARCH, REDHAT_JAVA_VERSION +source "$IMPALA_HOME/bin/impala-config-java.sh" + ubuntu apt-get update ubuntu apt-get --yes install ccache curl gawk g++ gcc apt-utils git libffi-dev \ libkrb5-dev krb5-admin-server krb5-kdc krb5-user libsasl2-dev \ libsasl2-modules libsasl2-modules-gssapi-mit libssl-dev make ninja-build \ python-dev python-setuptools python3-dev python3-setuptools postgresql \ - ssh wget vim-common psmisc lsof openjdk-8-jdk openjdk-8-source openjdk-8-dbg \ - net-tools language-pack-en libxml2-dev libxslt-dev + ssh wget vim-common psmisc lsof net-tools language-pack-en libxml2-dev \ + libxslt-dev openjdk-${UBUNTU_JAVA_VERSION}-jdk \ + openjdk-${UBUNTU_JAVA_VERSION}-source openjdk-${UBUNTU_JAVA_VERSION}-dbg + # Required by Kudu in the minicluster ubuntu20 apt-get --yes install libtinfo5 ARCH_NAME=$(uname -p) @@ -230,46 +236,18 @@ if [[ $ARCH_NAME == 'aarch64' ]]; then libncurses5-dev libreadline-dev fi -if [[ "$UBUNTU" == true ]]; then - # Don't use openjdk-8-jdk 8u181-b13-1ubuntu0.16.04.1 which is known to break the - # surefire tests. If we detect that version, we downgrade to the last known good one. - # See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=911925 for details. - JDK_BAD_VERSION="8u181-b13-1ubuntu0.16.04.1" - if dpkg -l openjdk-8-jdk | grep -q $JDK_BAD_VERSION; then - JDK_TARGET_VERSION="8u181-b13-0ubuntu0.16.04.1" - DEB_DIR=$(mktemp -d) - pushd $DEB_DIR - wget --no-verbose \ - "https://launchpadlibrarian.net/380913637/openjdk-8-jdk_8u181-b13-0ubuntu0.16.04.1_amd64.deb"; \ - "https://launchpadlibrarian.net/380913636/openjdk-8-jdk-headless_8u181-b13-0ubuntu0.16.04.1_amd64.deb"; \ - "https://launchpadlibrarian.net/380913641/openjdk-8-jre_8u181-b13-0ubuntu0.16.04.1_amd64.deb"; \ - "https://launchpadlibrarian.net/380913638/openjdk-8-jre-headless_8u181-b13-0ubuntu0.16.04.1_amd64.deb"; \ - "https://launchpadlibrarian.net/380913642/openjdk-8-source_8u181-b13-0ubuntu0.16.04.1_all.deb"; \ - "https://launchpadlibrarian.net/380913633/openjdk-8-dbg_8u181-b13-0ubuntu0.16.04.1_amd64.deb"; - sudo dpkg -i *.deb - popd - rm -rf $DEB_DIR - fi -fi - -# Ubuntu 18.04 and 20.04 install OpenJDK 11 and configure it as the default Java version. -# Impala is currently tested with OpenJDK 8, so configure that version as the default. -if [[ $ARCH_NAME == 'aarch64' ]]; then - ubuntu20 sudo update-java-alternatives -s java-1.8.0-openjdk-arm64 - ubuntu18 sudo update-java-alternatives -s java-1.8.0-openjdk-arm64 -else - ubuntu18 sudo update-java-alternatives -s java-1.8.0-openjdk-amd64 - ubuntu20 sudo update-java-alternatives -s java-1.8.0-openjdk-amd64 -fi +# Configure the default Java version to be the version we selected. +ubuntu sudo update-java-alternatives -s \ + java-1.${UBUNTU_JAVA_VERSION}.0-openjdk-${UBUNTU_PACKAGE_ARCH} redhat sudo yum install -y curl gawk gcc gcc-c++ git krb5-devel krb5-server \ krb5-workstation libevent-devel libffi-devel make openssl-devel cyrus-sasl \ cyrus-sasl-gssapi cyrus-sasl-devel cyrus-sasl-plain \ postgresql postgresql-server \ wget vim-common nscd cmake fuse-devel zlib-devel \ - psmisc lsof openssh-server redhat-lsb java-1.8.0-openjdk-devel \ - java-1.8.0-openjdk-src python3-devel python3-setuptools net-tools \ - langpacks-en glibc-langpack-en libxml2-devel libxslt-devel + psmisc lsof openssh-server redhat-lsb python3-devel python3-setuptools \ + net-tools langpacks-en glibc-langpack-en libxml2-devel libxslt-devel \ + java-${REDHAT_JAVA_VERSION}-openjdk-src java-${REDHAT_JAVA_VERSION}-openjdk-devel # Enable the Powertools repo for snappy-devel on RedHat 8 redhat8 sudo yum install -y dnf-plugins-core @@ -465,22 +443,6 @@ SET_IMPALA_HOME="export IMPALA_HOME=$(pwd)" echo -e "\n$SET_IMPALA_HOME" >> ~/.bashrc eval "$SET_IMPALA_HOME" -# Ubuntu and RH install JDK's in slightly different paths. -if [[ $UBUNTU == true ]]; then - # Assert that there's only one glob match. - [ 1 == $(compgen -G "/usr/lib/jvm/java-8-openjdk-*" | wc -l) ] - SET_JAVA_HOME="export JAVA_HOME=$(compgen -G '/usr/lib/jvm/java-8-openjdk-*')" -else - # Assert that there's only one glob match. - [ 1 == $(compgen -G "/usr/lib/jvm/java-1.8.0-openjdk-*" | wc -l) ] - SET_JAVA_HOME="export JAVA_HOME=$(compgen -G '/usr/lib/jvm/java-1.8.0-openjdk-*')" -fi - -echo -e "\n$SET_JAVA_HOME" >> "${IMPALA_HOME}/bin/impala-config-local.sh" -eval "$SET_JAVA_HOME" -# Assert that we have a java available -test -f $JAVA_HOME/bin/java - if [[ $ARCH_NAME == 'aarch64' ]]; then echo -e "\nexport SKIP_TOOLCHAIN_BOOTSTRAP=true" >> \ "${IMPALA_HOME}/bin/impala-config-local.sh" diff --git a/testdata/bin/run-ranger-server.sh b/bin/impala-config-java.sh old mode 100755 new mode 100644 similarity index 57% copy from testdata/bin/run-ranger-server.sh copy to bin/impala-config-java.sh index 1290e81a4..0b1fa929b --- a/testdata/bin/run-ranger-server.sh +++ b/bin/impala-config-java.sh @@ -1,5 +1,3 @@ -#!/bin/bash -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -17,19 +15,22 @@ # specific language governing permissions and limitations # under the License. -set -euo pipefail -. $IMPALA_HOME/bin/report_build_error.sh -setup_report_build_error +IMPALA_JDK_VERSION=${IMPALA_JDK_VERSION:-system} -RANGER_LOG_DIR="${IMPALA_CLUSTER_LOGS_DIR}/ranger" -if [[ ! -d "${RANGER_LOG_DIR}" ]]; then - mkdir -p "${RANGER_LOG_DIR}" +# Set OS Java package variables for bootstrap_system and Docker builds +if [[ "${IMPALA_JDK_VERSION}" == "system" || "${IMPALA_JDK_VERSION}" == "8" ]]; then + UBUNTU_JAVA_VERSION=8 + REDHAT_JAVA_VERSION=1.8.0 +elif [[ "${IMPALA_JDK_VERSION}" == "11" ]]; then + UBUNTU_JAVA_VERSION=11 + REDHAT_JAVA_VERSION=11 +else + echo "Unknown value for IMPALA_JDK_VERSION=${IMPALA_JDK_VERSION}" + exit 1 fi -# IMPALA-8815: don't allow additional potentially incompatible jars to get onto -# the ranger classpath. We should only need the test cluster configs on the classpath. -unset CLASSPATH -. $IMPALA_HOME/bin/impala-config.sh - -JAVA_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=30130" \ - "${RANGER_HOME}"/ews/ranger-admin-services.sh restart +if [[ "$(uname -p)" == 'aarch64' ]]; then + UBUNTU_PACKAGE_ARCH='arm64' +else + UBUNTU_PACKAGE_ARCH='amd64' +fi diff --git a/bin/impala-config.sh b/bin/impala-config.sh index de57cc95d..f07cdf4b8 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -380,20 +380,72 @@ fi # It is important to have a coherent view of the JAVA_HOME and JAVA executable. # The JAVA_HOME should be determined first, then the JAVA executable should be -# derived from JAVA_HOME. bin/bootstrap_development.sh adds code to -# bin/impala-config-local.sh to set JAVA_HOME, so it is important to pick up that -# setting before deciding what JAVA_HOME to use. - -# Try to detect the system's JAVA_HOME -# If javac exists, then the system has a Java SDK (JRE does not have javac). -# Follow the symbolic links and use this to determine the system's JAVA_HOME. -SYSTEM_JAVA_HOME="/usr/java/default" -if [ -n "$(which javac)" ]; then - SYSTEM_JAVA_HOME=$(which javac | xargs readlink -f | sed "s:/bin/javac::") +# derived from JAVA_HOME. For development, it is useful to be able to specify +# the JDK version as part of bin/impala-config-local.sh + +# Decision tree: +# if IMPALA_JAVA_HOME_OVERRIDE is set, respect it +# else if IMPALA_JDK_VERSION == system, look for system JDK +# else if IMPALA_JDK_VERSION == 8, look for Java 8 JDK +# else if IMPALA_JDK_VERSION == 11, look for Java 11 JDK + +# Initialize IMPALA_JDK_VERSION and set package variables for Docker builds +. "$IMPALA_HOME/bin/impala-config-java.sh" + +if [[ -n "${IMPALA_JAVA_HOME_OVERRIDE-}" ]]; then + IMPALA_JDK_VERSION=override +fi + +DETECTED_JAVA_HOME=Invalid +if [[ "${IMPALA_JDK_VERSION}" == "system" ]]; then + # Try to detect the system's JAVA_HOME + # If javac exists, then the system has a Java SDK (JRE does not have javac). + # Follow the symbolic links and use this to determine the system's JAVA_HOME. + DETECTED_JAVA_HOME="/usr/java/default" + if [ -n "$(which javac)" ]; then + DETECTED_JAVA_HOME=$(dirname $(dirname $(readlink -f $(which javac)))) + fi +elif [[ "${IMPALA_JDK_VERSION}" != "override" ]]; then + # Now, we are looking for a specific version, and that will depend on the + # distribution. Currently, this is implemented for Redhat and Ubuntu. + DISTRIBUTION=Unknown + if [[ -f /etc/redhat-release ]]; then + echo "Identified Redhat image." + DISTRIBUTION=Redhat + else + source /etc/lsb-release + if [[ $DISTRIB_ID == Ubuntu ]]; then + echo "Identified Ubuntu image." + DISTRIBUTION=Ubuntu + fi + fi + if [[ "${DISTRIBUTION}" == "Unknown" ]]; then + echo "ERROR: auto-detection of JAVA_HOME only supported for Ubuntu and RedHat." + echo "Use IMPALA_JAVA_HOME_OVERRIDE to configure JAVA_HOME." + return 1 + fi + + JVMS_PATH=/usr/lib/jvm + if [[ "${DISTRIBUTION}" == "Ubuntu" ]]; then + JAVA_PACKAGE_NAME="java-${IMPALA_JDK_VERSION}-openjdk-${UBUNTU_PACKAGE_ARCH}" + DETECTED_JAVA_HOME="${JVMS_PATH}/${JAVA_PACKAGE_NAME}" + elif [[ "${DISTRIBUTION}" == "Redhat" ]]; then + if [[ "${IMPALA_JDK_VERSION}" == "8" ]]; then + DETECTED_JAVA_HOME="${JVMS_PATH}/java-1.8.0" + else + DETECTED_JAVA_HOME="${JVMS_PATH}/java-${IMPALA_JDK_VERSION}" + fi + fi + + if [[ ! -d "${DETECTED_JAVA_HOME}" ]]; then + echo "ERROR: Could not detect Java ${IMPALA_JDK_VERSION}."\ + "${DETECTED_JAVA_HOME} is not a directory." + return 1 + fi fi # Prefer the JAVA_HOME set in the environment, but use the system's JAVA_HOME otherwise -export JAVA_HOME="${JAVA_HOME:-${SYSTEM_JAVA_HOME}}" +export JAVA_HOME="${IMPALA_JAVA_HOME_OVERRIDE:-${DETECTED_JAVA_HOME}}" if [ ! -d "$JAVA_HOME" ]; then echo "JAVA_HOME must be set to the location of your JDK!" return 1 diff --git a/bin/run-all-tests.sh b/bin/run-all-tests.sh index e4381293c..22bb4e44e 100755 --- a/bin/run-all-tests.sh +++ b/bin/run-all-tests.sh @@ -52,6 +52,9 @@ fi # Run Cluster Tests : ${CLUSTER_TEST:=true} : ${CLUSTER_TEST_FILES:=} +# Verifiers to run after all tests. Skipped if empty. +: ${TEST_SUITE_VERIFIERS:=verifiers/test_banned_log_messages.py} +: ${TEST_SUITE_VERIFIERS_LOG_DIR:=${IMPALA_LOGS_DIR}/verifiers} # Extra arguments passed to start-impala-cluster for tests. These do not apply to custom # cluster tests. : ${TEST_START_CLUSTER_ARGS:=} @@ -330,6 +333,17 @@ do export IMPALA_MAX_LOG_FILES="${IMPALA_MAX_LOG_FILES_SAVE}" fi + if [ ! -z "${TEST_SUITE_VERIFIERS}" ]; then + mkdir -p "${TEST_SUITE_VERIFIERS_LOG_DIR}" + pushd "${IMPALA_HOME}/tests" + if ! impala-py.test ${TEST_SUITE_VERIFIERS} \ + --junitxml=${TEST_SUITE_VERIFIERS_LOG_DIR}/TEST-impala-verifiers.xml \ + --resultlog=${TEST_SUITE_VERIFIERS_LOG_DIR}/TEST-impala-verifiers.log; then + TEST_RET_CODE=1 + fi + popd + fi + # Run the process failure tests. # Disabled temporarily until we figure out the proper timeouts required to make the test # succeed. diff --git a/cmake_modules/FindJNI.cmake b/cmake_modules/FindJNI.cmake deleted file mode 100644 index 225c79f1f..000000000 --- a/cmake_modules/FindJNI.cmake +++ /dev/null @@ -1,281 +0,0 @@ -# - Find JNI java libraries. -# This module finds if Java is installed and determines where the -# include files and libraries are. It also determines what the name of -# the library is. This code sets the following variables: -# -# JNI_INCLUDE_DIRS = the include dirs to use -# JNI_LIBRARIES = the libraries to use -# JAVA_AWT_LIBRARY = the path to the jawt library -# JAVA_JVM_LIBRARY = the path to the jvm library -# JAVA_INCLUDE_PATH = the include path to jni.h -# JAVA_INCLUDE_PATH2 = the include path to jni_md.h -# JAVA_AWT_INCLUDE_PATH = the include path to jawt.h -# - -#============================================================================= -# Copyright 2001-2009 Kitware, Inc. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the names of Kitware, Inc., the Insight Software Consortium, -# nor the names of their contributors may be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#============================================================================= - -# Expand {libarch} occurences to java_libarch subdirectory(-ies) and set ${_var} -macro(JAVA_APPEND_LIBRARY_DIRECTORIES _var) - # Determine java arch-specific library subdir - if (CMAKE_SYSTEM_NAME MATCHES "Linux") - # Based on openjdk/jdk/make/common/shared/Platform.gmk as of 6b16 - # and kaffe as of 1.1.8 which uses the first part of the - # GNU config.guess platform triplet. - if (CMAKE_SYSTEM_PROCESSOR MATCHES "^i[3-9]86$") - set(_java_libarch "i386") - elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - set(_java_libarch "amd64" "x86_64") - elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^ppc") - set(_java_libarch "ppc" "powerpc" "ppc64") - elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^sparc") - set(_java_libarch "sparc" "sparcv9") - else (CMAKE_SYSTEM_PROCESSOR MATCHES "^i[3-9]86$") - set(_java_libarch "${CMAKE_SYSTEM_PROCESSOR}") - endif(CMAKE_SYSTEM_PROCESSOR MATCHES "^i[3-9]86$") - else (CMAKE_SYSTEM_NAME MATCHES "Linux") - set(_java_libarch "i386" "amd64" "ppc") # previous default - endif (CMAKE_SYSTEM_NAME MATCHES "Linux") - - foreach(_path ${ARGN}) - if (_path MATCHES "{libarch}") - foreach(_libarch ${_java_libarch}) - string(REPLACE "{libarch}" "${_libarch}" _newpath "${_path}") - list(APPEND ${_var} "${_newpath}") - endforeach(_libarch) - else (_path MATCHES "{libarch}") - list(APPEND ${_var} "${_path}") - endif (_path MATCHES "{libarch}") - endforeach(_path) -endmacro(JAVA_APPEND_LIBRARY_DIRECTORIES) - -file(TO_CMAKE_PATH "$ENV{JAVA_HOME}" _JAVA_HOME) - -get_filename_component(java_install_version - "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit;CurrentVersion]" NAME -) - -set(JAVA_AWT_LIBRARY_DIRECTORIES - "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.4;JavaHome]/lib" - "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.3;JavaHome]/lib" - "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\${java_install_version};JavaHome]/lib" -) - - -java_append_library_directories(JAVA_AWT_LIBRARY_DIRECTORIES - ${_JAVA_HOME}/jre/lib/{libarch} - ${_JAVA_HOME}/jre/lib - ${_JAVA_HOME}/lib/{libarch} - ${_JAVA_HOME}/lib - ${_JAVA_HOME} - /usr/lib - /usr/local/lib - /usr/lib/jvm/java/lib - /usr/lib/java/jre/lib/{libarch} - /usr/local/lib/java/jre/lib/{libarch} - /usr/local/share/java/jre/lib/{libarch} - /usr/lib/j2sdk1.4-sun/jre/lib/{libarch} - /usr/lib/j2sdk1.5-sun/jre/lib/{libarch} - /opt/sun-jdk-1.5.0.04/jre/lib/{libarch} - /usr/lib/jvm/java-6-sun/jre/lib/{libarch} - /usr/lib/jvm/java-1.5.0-sun/jre/lib/{libarch} - /usr/lib/jvm/java-6-sun-1.6.0.00/jre/lib/{libarch} # can this one be removed according to #8821 ? Alex - /usr/lib/jvm/java-openjdk/jre/lib/{libarch} - /usr/lib/jvm/java-6-openjdk/jre/lib/{libarch} - /usr/lib/jvm/java-openjdk/jre/lib/{libarch} - # Debian specific paths for default JVM - /usr/lib/jvm/default-java/jre/lib/{libarch} - /usr/lib/jvm/default-java/jre/lib - /usr/lib/jvm/default-java/lib -) - -set(JAVA_JVM_LIBRARY_DIRECTORIES) -foreach (dir ${JAVA_AWT_LIBRARY_DIRECTORIES}) - set(JAVA_JVM_LIBRARY_DIRECTORIES - ${JAVA_JVM_LIBRARY_DIRECTORIES} - "${dir}" - "${dir}/client" - "${dir}/server" - ) -endforeach (dir) - - -set(JAVA_AWT_INCLUDE_DIRECTORIES - "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.4;JavaHome]/include" - "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.3;JavaHome]/include" - "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\${java_install_version};JavaHome]/include" - ${_JAVA_HOME}/include - /usr/include - /usr/local/include - /usr/lib/java/include - /usr/local/lib/java/include - /usr/lib/jvm/java/include - /usr/lib/jvm/java-6-sun/include - /usr/lib/jvm/java-1.5.0-sun/include - /usr/lib/jvm/java-6-sun-1.6.0.00/include # can this one be removed according to #8821 ? Alex - /usr/lib/jvm/java-6-openjdk/include - /usr/local/share/java/include - /usr/lib/j2sdk1.4-sun/include - /usr/lib/j2sdk1.5-sun/include - /opt/sun-jdk-1.5.0.04/include - # Debian specific path for default JVM - /usr/lib/jvm/default-java/include -) - -foreach(JAVA_PROG "${JAVA_RUNTIME}" "${JAVA_COMPILE}" "${JAVA_ARCHIVE}" "${JAVA_HEADER}") - get_filename_component(jpath "${JAVA_PROG}" PATH) - foreach (JAVA_INC_PATH ../include ../java/include ../share/java/include) - if (EXISTS ${jpath}/${JAVA_INC_PATH}) - set(JAVA_AWT_INCLUDE_DIRECTORIES - ${JAVA_AWT_INCLUDE_DIRECTORIES} - "${jpath}/${JAVA_INC_PATH}" - ) - endif (EXISTS ${jpath}/${JAVA_INC_PATH}) - endforeach (JAVA_INC_PATH) - - foreach (JAVA_LIB_PATH - ../lib ../jre/lib ../jre/lib/i386 - ../java/lib ../java/jre/lib ../java/jre/lib/i386 - ../share/java/lib ../share/java/jre/lib ../share/java/jre/lib/i386) - if (EXISTS ${jpath}/${JAVA_LIB_PATH}) - set(JAVA_AWT_LIBRARY_DIRECTORIES - ${JAVA_AWT_LIBRARY_DIRECTORIES} - "${jpath}/${JAVA_LIB_PATH}" - ) - endif (EXISTS ${jpath}/${JAVA_LIB_PATH}) - endforeach (JAVA_LIB_PATH) -endforeach (JAVA_PROG) - -if (APPLE) - if (EXISTS ~/Library/Frameworks/JavaVM.framework) - set(JAVA_HAVE_FRAMEWORK 1) - endif (EXISTS ~/Library/Frameworks/JavaVM.framework) - - if (EXISTS /Library/Frameworks/JavaVM.framework) - SET(JAVA_HAVE_FRAMEWORK 1) - endif (EXISTS /Library/Frameworks/JavaVM.framework) - - if (EXISTS /System/Library/Frameworks/JavaVM.framework) - set(JAVA_HAVE_FRAMEWORK 1) - endif(EXISTS /System/Library/Frameworks/JavaVM.framework) - - if (JAVA_HAVE_FRAMEWORK) - if(NOT JAVA_AWT_LIBRARY) - set(JAVA_AWT_LIBRARY "-framework JavaVM" CACHE FILEPATH "Java Frameworks" FORCE) - endif(NOT JAVA_AWT_LIBRARY) - - if (NOT JAVA_JVM_LIBRARY) - set (JAVA_JVM_LIBRARY "-framework JavaVM" CACHE FILEPATH "Java Frameworks" FORCE) - endif (NOT JAVA_JVM_LIBRARY) - - if (NOT JAVA_AWT_INCLUDE_PATH) - if (EXISTS /System/Library/Frameworks/JavaVM.framework/Headers/jawt.h) - set(JAVA_AWT_INCLUDE_PATH "/System/Library/Frameworks/JavaVM.framework/Headers" CACHE FILEPATH "jawt.h location" FORCE) - endif(EXISTS /System/Library/Frameworks/JavaVM.framework/Headers/jawt.h) - endif (NOT JAVA_AWT_INCLUDE_PATH) - - # - # If using "-framework JavaVM", prefer its headers *before* the others in - # JAVA_AWT_INCLUDE_DIRECTORIES... (*prepend* to the list here) - # - set(JAVA_AWT_INCLUDE_DIRECTORIES - ~/Library/Frameworks/JavaVM.framework/Headers - /Library/Frameworks/JavaVM.framework/Headers - /System/Library/Frameworks/JavaVM.framework/Headers - ${JAVA_AWT_INCLUDE_DIRECTORIES} - ) - endif(JAVA_HAVE_FRAMEWORK) -else (APPLE) - find_library(JAVA_AWT_LIBRARY - NAMES - jawt - PATHS - ${JAVA_AWT_LIBRARY_DIRECTORIES} - ) - - find_library(JAVA_JSIG_LIBRARY - NAMES - jsig - PATHS - ${JAVA_JVM_LIBRARY_DIRECTORIES} - ) - find_library(JAVA_JVM_LIBRARY - NAMES - jvm - JavaVM - PATHS - ${JAVA_JVM_LIBRARY_DIRECTORIES} - ) -endif (APPLE) - -# add in the include path -find_path(JAVA_INCLUDE_PATH - NAMES - jni.h - PATHS - ${JAVA_AWT_INCLUDE_DIRECTORIES} -) - -find_path(JAVA_INCLUDE_PATH2 - NAMES - jni_md.h - PATHS - ${JAVA_INCLUDE_PATH} - ${JAVA_INCLUDE_PATH}/win32 - ${JAVA_INCLUDE_PATH}/linux - ${JAVA_INCLUDE_PATH}/freebsd - ${JAVA_INCLUDE_PATH}/solaris -) - -find_path(JAVA_AWT_INCLUDE_PATH - NAMES - jawt.h - PATHS - ${JAVA_INCLUDE_PATH} -) - -set(JNI_LIBRARIES - ${JAVA_AWT_LIBRARY} - ${JAVA_JSIG_LIBRARY} - ${JAVA_JVM_LIBRARY} -) - -set(JNI_INCLUDE_DIRS - ${JAVA_INCLUDE_PATH} - ${JAVA_INCLUDE_PATH2} - ${JAVA_AWT_INCLUDE_PATH} -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(JNI DEFAULT_MSG JNI_LIBRARIES JNI_INCLUDE_DIRS JAVA_AWT_LIBRARY JAVA_JSIG_LIBRARY JAVA_JVM_LIBRARY) - -mark_as_advanced(JNI_LIBRARIES JNI_INCLUDE_DIRS JAVA_AWT_LIBRARY JAVA_JSIG_LIBRARY JAVA_JVM_LIBRARY) diff --git a/fe/pom.xml b/fe/pom.xml index 7362d1770..6cea36f97 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -621,7 +621,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> + <version>3.11.0</version> <configuration> <source>1.8</source> <target>1.8</target> @@ -632,7 +632,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> - <version>2.6</version> + <version>3.3.0</version> <executions> <execution> <goals> @@ -685,7 +685,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> - <version>3.1.1</version> + <version>3.5.0</version> <executions> <!-- TODO(todd): consider removing this execution or moving it to some kind of 'dist' profile. No need to copy all of these jars @@ -734,7 +734,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> - <version>2.20</version> + <version>3.0.0</version> <configuration> <trimStackTrace>false</trimStackTrace> <reportsDirectory>${surefire.reports.dir}</reportsDirectory> @@ -761,7 +761,7 @@ under the License. <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>build-helper-maven-plugin</artifactId> - <version>1.5</version> + <version>3.3.0</version> <executions> <!-- Tell maven about our generated files --> <execution> @@ -822,7 +822,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-enforcer-plugin</artifactId> - <version>3.0.0-M1</version> + <version>3.1.0</version> <executions> <execution> <id>enforce-banned-dependencies</id> @@ -1059,7 +1059,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> + <version>3.11.0</version> <configuration> <showWarnings>true</showWarnings> <compilerId>javac-with-errorprone</compilerId> @@ -1103,7 +1103,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> + <version>3.11.0</version> <configuration> <excludes> <exclude>**/org/apache/impala/catalog/metastore/*.java</exclude> diff --git a/fe/src/test/java/org/apache/impala/util/JMXJsonUtilTest.java b/fe/src/test/java/org/apache/impala/util/JMXJsonUtilTest.java index 95e9c3de1..79efac1b9 100644 --- a/fe/src/test/java/org/apache/impala/util/JMXJsonUtilTest.java +++ b/fe/src/test/java/org/apache/impala/util/JMXJsonUtilTest.java @@ -50,7 +50,8 @@ public class JMXJsonUtilTest { assertTrue("Invalid JSON: " + jmxJson, rootNode.hasNonNull("beans")); List<String> values = rootNode.get("beans").findValuesAsText("name"); assertTrue("Invalid JSON: " + jmxJson, - values.contains("java.lang:type=MemoryPool,name=Metaspace")); + values.contains("java.lang:type=MemoryPool,name=Metaspace") || + values.contains("java.lang:name=Metaspace,type=MemoryPool")); assertTrue("Invalid JSON: " + jmxJson, values.contains("java.lang:type=Runtime")); } } diff --git a/java/TableFlattener/pom.xml b/java/TableFlattener/pom.xml index 623ff91a7..e4061e909 100644 --- a/java/TableFlattener/pom.xml +++ b/java/TableFlattener/pom.xml @@ -34,7 +34,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> + <version>3.11.0</version> <configuration> <source>1.8</source> <target>1.8</target> diff --git a/java/datagenerator/pom.xml b/java/datagenerator/pom.xml index ce02ee20d..20230f612 100644 --- a/java/datagenerator/pom.xml +++ b/java/datagenerator/pom.xml @@ -81,7 +81,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> + <version>3.11.0</version> <configuration> <source>1.8</source> <target>1.8</target> @@ -91,7 +91,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> - <version>2.18</version> + <version>3.0.0</version> <configuration> <redirectTestOutputToFile>true</redirectTestOutputToFile> </configuration> diff --git a/java/executor-deps/pom.xml b/java/executor-deps/pom.xml index 0847462e1..25f6f6a68 100644 --- a/java/executor-deps/pom.xml +++ b/java/executor-deps/pom.xml @@ -188,7 +188,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> - <version>3.1.1</version> + <version>3.5.0</version> <executions> <execution> <id>write-executor-classpath</id> diff --git a/java/ext-data-source/api/pom.xml b/java/ext-data-source/api/pom.xml index e68ec983a..c4b5bc8c0 100644 --- a/java/ext-data-source/api/pom.xml +++ b/java/ext-data-source/api/pom.xml @@ -44,7 +44,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-source-plugin</artifactId> - <version>2.4</version> + <version>3.2.1</version> <executions> <execution> <id>attach-sources</id> @@ -57,7 +57,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> + <version>3.11.0</version> <configuration> <source>1.8</source> <target>1.8</target> @@ -67,7 +67,7 @@ <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>build-helper-maven-plugin</artifactId> - <version>1.5</version> + <version>3.3.0</version> <executions> <!-- Tell maven about our generated files --> <execution> diff --git a/java/ext-data-source/sample/pom.xml b/java/ext-data-source/sample/pom.xml index ef24e525b..99bcae62a 100644 --- a/java/ext-data-source/sample/pom.xml +++ b/java/ext-data-source/sample/pom.xml @@ -48,7 +48,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-source-plugin</artifactId> - <version>2.4</version> + <version>3.2.1</version> <executions> <execution> <id>attach-sources</id> @@ -61,7 +61,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> + <version>3.11.0</version> <configuration> <source>1.8</source> <target>1.8</target> diff --git a/java/ext-data-source/test/pom.xml b/java/ext-data-source/test/pom.xml index a3fa56768..8ea768835 100644 --- a/java/ext-data-source/test/pom.xml +++ b/java/ext-data-source/test/pom.xml @@ -48,7 +48,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> + <version>3.11.0</version> <configuration> <source>1.8</source> <target>1.8</target> diff --git a/java/query-event-hook-api/pom.xml b/java/query-event-hook-api/pom.xml index 9d047a984..da437e03b 100644 --- a/java/query-event-hook-api/pom.xml +++ b/java/query-event-hook-api/pom.xml @@ -35,7 +35,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-source-plugin</artifactId> - <version>2.4</version> + <version>3.2.1</version> <executions> <execution> <id>attach-sources</id> @@ -48,7 +48,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> + <version>3.11.0</version> <configuration> <source>1.8</source> <target>1.8</target> diff --git a/java/shaded-deps/hive-exec/pom.xml b/java/shaded-deps/hive-exec/pom.xml index c718f40ac..d509fdf8a 100644 --- a/java/shaded-deps/hive-exec/pom.xml +++ b/java/shaded-deps/hive-exec/pom.xml @@ -55,7 +55,7 @@ the same dependencies <plugins> <plugin> <artifactId>maven-shade-plugin</artifactId> - <version>3.2.1</version> + <version>3.4.1</version> <configuration> <artifactSet> <includes> diff --git a/java/shaded-deps/s3a-aws-sdk/pom.xml b/java/shaded-deps/s3a-aws-sdk/pom.xml index 8b996dd09..aa755d1bf 100644 --- a/java/shaded-deps/s3a-aws-sdk/pom.xml +++ b/java/shaded-deps/s3a-aws-sdk/pom.xml @@ -43,7 +43,7 @@ though some of them might not be necessary. The exclusions are sorted alphabetic <plugins> <plugin> <artifactId>maven-shade-plugin</artifactId> - <version>3.2.1</version> + <version>3.4.1</version> <configuration> <artifactSet> <includes> diff --git a/java/test-corrupt-hive-udfs/pom.xml b/java/test-corrupt-hive-udfs/pom.xml index 19c20a647..e45c04366 100644 --- a/java/test-corrupt-hive-udfs/pom.xml +++ b/java/test-corrupt-hive-udfs/pom.xml @@ -42,7 +42,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> + <version>3.11.0</version> <configuration> <source>1.8</source> <target>1.8</target> diff --git a/java/test-hive-udfs/pom.xml b/java/test-hive-udfs/pom.xml index 790c42ec7..be43607c5 100644 --- a/java/test-hive-udfs/pom.xml +++ b/java/test-hive-udfs/pom.xml @@ -69,7 +69,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> + <version>3.11.0</version> <configuration> <source>1.8</source> <target>1.8</target> diff --git a/java/yarn-extras/pom.xml b/java/yarn-extras/pom.xml index a75b718ad..e9caa4aaa 100644 --- a/java/yarn-extras/pom.xml +++ b/java/yarn-extras/pom.xml @@ -89,7 +89,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> + <version>3.11.0</version> <configuration> <source>1.8</source> <target>1.8</target> diff --git a/testdata/bin/run-ranger-server.sh b/testdata/bin/run-ranger-server.sh index 1290e81a4..9a6e29676 100755 --- a/testdata/bin/run-ranger-server.sh +++ b/testdata/bin/run-ranger-server.sh @@ -31,5 +31,11 @@ fi unset CLASSPATH . $IMPALA_HOME/bin/impala-config.sh -JAVA_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=30130" \ +# Required to start Ranger with Java 11 +if [[ ! -d "${RANGER_HOME}"/ews/logs ]]; then + mkdir -p "${RANGER_HOME}"/ews/logs +fi + +JAVA_DBG_SOCKET="-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=30130" +JAVA_OPTS="-XX:+IgnoreUnrecognizedVMOptions -Xdebug ${JAVA_DBG_SOCKET}" \ "${RANGER_HOME}"/ews/ranger-admin-services.sh restart diff --git a/tests/verifiers/test_banned_log_messages.py b/tests/verifiers/test_banned_log_messages.py new file mode 100644 index 000000000..ac5d1e3e0 --- /dev/null +++ b/tests/verifiers/test_banned_log_messages.py @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Test that impalad logs omit specific messages we shouldn't see. + +from __future__ import absolute_import, division, print_function +import os +import subprocess + +from tests.common.impala_test_suite import ImpalaTestSuite +from tests.common.skip import SkipIfDockerizedCluster + + +class TestBannedLogMessages(ImpalaTestSuite): + """Verify that specific log messages are banned from Impala logs. + + This test suite should be run after all the tests have been run. + """ + + @SkipIfDockerizedCluster.daemon_logs_not_exposed + def test_no_inaccessible_objects(self): + """Test that cluster logs do not contain InaccessibleObjectException""" + log_dir = os.environ["IMPALA_LOGS_DIR"] + message = 'InaccessibleObjectException' + for root, _, files in os.walk(log_dir): + for file in files: + log_file_path = os.path.join(root, file) + returncode = subprocess.call(['grep', message, log_file_path]) + assert returncode == 1, "%s contains '%s'" % (log_file_path, message)
