This is an automated email from the ASF dual-hosted git repository. csringhofer pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 5eea4f6f797e5549623a4dc4891ec82af08a41ac Author: Joe McDonnell <[email protected]> AuthorDate: Mon Oct 6 16:36:24 2025 -0700 IMPALA-14559: Ship calcite-planner jar in Impala packages This adds the java/impala-package Maven project to make it easier to ship / test the Calcite planner. impala-package has a dependency on impala-frontend and calcite-planner, so its classpath requires no extra work when constructing the classpath. An additional cleanup is that this no longer puts the impala-frontend-*-tests.jar on the classpath by default. This requires updating the query event hooks test, as it relies on that jar being present. This does not change the default value for the use_calcite_planner query option, so there is no change in behavior. Testing: - Ran a core job - Built docker images and OS packages locally Change-Id: I81dec2a5b59e279229a735c8bb1a23c77111a793 Reviewed-on: http://gerrit.cloudera.org:8080/23497 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- bin/set-classpath.sh | 34 +++-------- docker/setup_build_context.py | 24 ++++---- java/impala-package/pom.xml | 79 ++++++++++++++++++++++++++ java/pom.xml | 1 + package/CMakeLists.txt | 4 +- tests/custom_cluster/test_query_event_hooks.py | 17 ++++++ 6 files changed, 119 insertions(+), 40 deletions(-) diff --git a/bin/set-classpath.sh b/bin/set-classpath.sh index abfe54e43..a3e051fb0 100644 --- a/bin/set-classpath.sh +++ b/bin/set-classpath.sh @@ -33,40 +33,22 @@ fi # frontend tests. This uses jars to avoid issues related to that # inconsistency. CLASSPATH=\ -"$IMPALA_HOME"/fe/src/test/resources:\ -"$IMPALA_HOME"/fe/target/impala-frontend-${IMPALA_VERSION}.jar:\ -"$IMPALA_HOME"/fe/target/dependency:\ -"$IMPALA_HOME"/fe/target/impala-frontend-${IMPALA_VERSION}-tests.jar: +"$IMPALA_HOME"/fe/src/test/resources: -FE_CP_FILE="$IMPALA_HOME/fe/target/build-classpath.txt" +PACKAGE_CP_FILE="$IMPALA_HOME/java/impala-package/target/package-classpath.txt" -if [ ! -s "$FE_CP_FILE" ]; then - >&2 echo FE classpath file $FE_CP_FILE missing. - >&2 echo Build the front-end first. +if [ ! -s "$PACKAGE_CP_FILE" ]; then + >&2 echo impala-package classpath file "$PACKAGE_CP_FILE" missing. + >&2 echo Build the impala-package first. return 1 fi -CLASSPATH=$(cat $FE_CP_FILE):"$CLASSPATH" - -# Currently the Calcite planner is experimental and not included by default. -# If the USE_CALCITE_PLANNER is explicitly set, then the jar dependencies -# are added to the CLASSPATH -USE_CALCITE_PLANNER=${USE_CALCITE_PLANNER:-false} -if [ "true" = "$USE_CALCITE_PLANNER" ]; then - - CALCITE_CP_FILE="$IMPALA_HOME/java/calcite-planner/target/calcite-build-classpath.txt" - if [ ! -s "$CALCITE_CP_FILE" ]; then - >&2 echo Calcite front-end classpath file $CALCITE_CP_FILE missing. - >&2 echo Build the Calcite front-end first. - return 1 - fi - CLASSPATH="$CLASSPATH":$(cat $CALCITE_CP_FILE):\ -"$IMPALA_HOME"/java/calcite-planner/target/calcite-planner-${IMPALA_VERSION}.jar: -fi +CLASSPATH=$(cat "$PACKAGE_CP_FILE"):"$CLASSPATH" if [[ "${1:-notest}" = "test" ]]; then FE_TEST_CP_FILE="$IMPALA_HOME/fe/target/test-classpath.txt" - CLASSPATH=$(cat $FE_TEST_CP_FILE):"$CLASSPATH" + CLASSPATH=$(cat "$FE_TEST_CP_FILE"):\ +"$IMPALA_HOME"/fe/target/impala-frontend-${IMPALA_VERSION}-tests.jar:"$CLASSPATH" fi : ${CUSTOM_CLASSPATH=} diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py index 1761452cc..0dd6714cc 100755 --- a/docker/setup_build_context.py +++ b/docker/setup_build_context.py @@ -193,26 +193,28 @@ if args.utility_context: os.path.join(IMPALA_HOME, "docker/utility_entrypoint.sh"), BIN_DIR) else: # Impala Coordinator dependencies. + impala_package_classpath_file = "java/impala-package/target/package-classpath.txt" num_jars_on_classpath = 0 - dep_classpath = open(os.path.join(IMPALA_HOME, "fe/target/build-classpath.txt")).read() + num_frontend_jars = 0 + num_calcite_jars = 0 + dep_classpath = open(os.path.join(IMPALA_HOME, impala_package_classpath_file)).read() for jar in dep_classpath.split(":"): num_jars_on_classpath += 1 assert os.path.exists(jar), "missing jar from classpath: {0}".format(jar) + if jar.find("calcite-planner") != -1: + assert jar.find("tests") == -1 + num_calcite_jars += 1 + if jar.find("impala-frontend") != -1: + assert jar.find("tests") == -1 + num_frontend_jars += 1 symlink_file_into_dir(jar, LIB_DIR) if num_jars_on_classpath == 0: raise Exception("No jars listed in {0}".format(os.path.join(IMPALA_HOME, - "fe/target/build-classpath.txt"))) + impala_package_classpath_file))) - # Impala Coordinator jars. - num_frontend_jars = 0 - for jar in glob.glob(os.path.join(IMPALA_HOME, "fe/target/impala-frontend-*.jar")): - # Ignore the tests jar - if jar.find("-tests") != -1: - continue - symlink_file_into_dir(jar, LIB_DIR) - num_frontend_jars += 1 - # There must be exactly one impala-frontend jar. + # There must be exactly one jar for each of impala-frontend and calcite-planner + assert num_calcite_jars == 1 assert num_frontend_jars == 1 # Impala Executor dependencies. diff --git a/java/impala-package/pom.xml b/java/impala-package/pom.xml new file mode 100644 index 000000000..e256340e4 --- /dev/null +++ b/java/impala-package/pom.xml @@ -0,0 +1,79 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <groupId>org.apache.impala</groupId> + <artifactId>impala-parent</artifactId> + <version>5.0.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>impala-package</artifactId> + <packaging>pom</packaging> + <name>Apache Impala Package</name> + + <dependencies> + <dependency> + <groupId>org.apache.impala</groupId> + <artifactId>impala-frontend</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.impala</groupId> + <artifactId>calcite-planner</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <excludeTypes>pom</excludeTypes> + <includeScope>runtime</includeScope> + <silent>true</silent> + </configuration> + </execution> + <execution> + <id>write-classpath</id> + <goals> + <goal>build-classpath</goal> + </goals> + <configuration> + <outputFile>${project.build.directory}/package-classpath.txt</outputFile> + <includeScope>runtime</includeScope> + <excludeTypes>pom</excludeTypes> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/java/pom.xml b/java/pom.xml index 1a837e69b..4eed32513 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -579,6 +579,7 @@ under the License. <module>../fe</module> <module>external-frontend</module> <module>calcite-planner</module> + <module>impala-package</module> <module>query-event-hook-api</module> <module>shaded-deps/hive-exec</module> <module>shaded-deps/s3a-aws-sdk</module> diff --git a/package/CMakeLists.txt b/package/CMakeLists.txt index d264718ee..8e4c4273f 100644 --- a/package/CMakeLists.txt +++ b/package/CMakeLists.txt @@ -25,9 +25,7 @@ message(STATUS "DISTRIB_VERSION_ID: ${OS_DISTRIB_VERSION_ID}") install(FILES ${CMAKE_SOURCE_DIR}/LICENSE.txt DESTINATION ${IMPALA_INSTALLDIR} RENAME LICENSE) install(FILES ${CMAKE_SOURCE_DIR}/NOTICE.txt DESTINATION ${IMPALA_INSTALLDIR} RENAME NOTICE) install(DIRECTORY "${CMAKE_SOURCE_DIR}/www/" DESTINATION ${IMPALA_INSTALLDIR}/www) -install(FILES ${CMAKE_SOURCE_DIR}/fe/target/impala-frontend-$ENV{IMPALA_VERSION}.jar - DESTINATION ${IMPALA_INSTALLDIR}/lib/jars) -install(DIRECTORY ${CMAKE_SOURCE_DIR}/fe/target/dependency/ +install(DIRECTORY ${CMAKE_SOURCE_DIR}/java/impala-package/target/dependency/ DESTINATION ${IMPALA_INSTALLDIR}/lib/jars FILES_MATCHING PATTERN "*.jar") diff --git a/tests/custom_cluster/test_query_event_hooks.py b/tests/custom_cluster/test_query_event_hooks.py index 84ca8852a..dc45a1cbf 100644 --- a/tests/custom_cluster/test_query_event_hooks.py +++ b/tests/custom_cluster/test_query_event_hooks.py @@ -18,17 +18,31 @@ # Client tests for Query Event Hooks from __future__ import absolute_import, division, print_function +import os import pytest from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +def add_frontend_test_jar_to_classpath(): + """Put the impala-frontend-*-tests.jar on the classpath""" + os.environ["CUSTOM_CLASSPATH"] = os.path.join(os.getenv("IMPALA_HOME"), "fe", + "target", "impala-frontend-{0}-tests.jar".format(os.getenv("IMPALA_VERSION"))) + + class TestHooks(CustomClusterTestSuite): """ Tests for FE QueryEventHook invocations. """ DUMMY_HOOK = "org.apache.impala.testutil.DummyQueryEventHook" + @classmethod + def setup_class(cls): + super(TestHooks, cls).setup_class() + # This needs the impala-frontend-*-tests.jar on the classpath to make the test query + # hooks available. + add_frontend_test_jar_to_classpath() + @pytest.mark.execute_serially @CustomClusterTestSuite.with_args( impala_log_dir="{query_event_hooks_log}", @@ -65,6 +79,9 @@ class TestHooksStartupFail(CustomClusterTestSuite): if cls.exploration_strategy() != 'exhaustive': pytest.skip('runs only in exhaustive') super(TestHooksStartupFail, cls).setup_class() + # This needs the impala-frontend-*-tests.jar on the classpath to make the test query + # hooks available. + add_frontend_test_jar_to_classpath() FAILING_HOOK = "org.apache.impala.testutil.AlwaysErrorQueryEventHook" NONEXIST_HOOK = "captain.hook"
