This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 39daf930d IMPALA-13921: Make Java 17 the default in tests
39daf930d is described below
commit 39daf930d8badf47752d719bbcc27782f5680bbc
Author: Zoltan Borok-Nagy <[email protected]>
AuthorDate: Wed Jan 28 18:25:29 2026 +0100
IMPALA-13921: Make Java 17 the default in tests
This patch makes Java 17 the default in our builds. Once it has been
proven stable, we can explicitly remove support for Java 8 and
possibly Java 11, and remove all related code also.
Fixes required for this change:
* Fix DESCRIPTION output of bin/jenkins/build-all-flag-combinations.sh
* Use Java 17 in the docker containers
* Hive 3:
- Add javax.annotation-api as a direct dependency
- Disable DataNucleus (which is only needed for the interaction
between HMS and its backend DB)
- Remove Cleaner logic in the ORC EncodedReaderImpl, because its
API changed and it isn't needed for the Impala frontend/catalog.
* Hive 2:
- Patching Hive 2 and building with Java 17 is cumbersome. A cleaner
solution was to add some extra shimming around notification events
and catalog name.
- hive_metastore.thrift still need to be patched to add 'catName'
field to NotificationEvent, so we don't need to modify our C++ code
Generated-by: Github Copilot (Claude Sonnet 4.5)
Generated-by: Gemini Pro
Change-Id: I99cc3b752142e6936a039c02505e3c35e71277aa
Reviewed-on: http://gerrit.cloudera.org:8080/23916
Reviewed-by: Michael Smith <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
bin/impala-config-java.sh | 7 +-
bin/impala-config.sh | 5 +-
bin/jenkins/build-all-flag-combinations.sh | 7 +-
bin/jenkins/dockerized-impala-run-tests.sh | 8 +-
bin/start-impala-cluster.py | 4 +-
docker/CMakeLists.txt | 25 +-
docker/install_os_packages.sh | 4 +-
.../impala/catalog/events/MetastoreEvents.java | 26 +-
.../catalog/events/EventExecutorService.java | 4 +-
.../impala/catalog/events/MetastoreEvents.java | 18 ++
.../catalog/events/MetastoreEventsProcessor.java | 9 +-
.../apache/impala/service/CatalogOpExecutor.java | 3 +-
java/ext-data-source/api/pom.xml | 5 +
testdata/bin/patch_hive.sh | 15 +-
testdata/bin/patch_hive2.sh | 10 -
.../cluster/hive/java17/patch0-JDK-17-changes.diff | 81 ++++++
testdata/cluster/hive2/patch0-CATALOG_NAME.diff | 10 +
testdata/cluster/hive2/patch0-HIVE-18755.diff | 306 ---------------------
18 files changed, 186 insertions(+), 361 deletions(-)
diff --git a/bin/impala-config-java.sh b/bin/impala-config-java.sh
index 002f1dbbb..7f85f6bdc 100644
--- a/bin/impala-config-java.sh
+++ b/bin/impala-config-java.sh
@@ -18,8 +18,11 @@
IMPALA_JDK_VERSION=${IMPALA_JDK_VERSION:-}
# Set OS Java package variables for bootstrap_system and Docker builds.
-# Defaults to installing Java 8.
-if [[ "${IMPALA_JDK_VERSION}" == "" || "${IMPALA_JDK_VERSION}" == "8" ]]; then
+# Defaults to installing Java 17.
+if [[ "${IMPALA_JDK_VERSION}" == "" ]]; then
+ UBUNTU_JAVA_VERSION=17
+ REDHAT_JAVA_VERSION=17
+elif [[ "${IMPALA_JDK_VERSION}" == "8" ]]; then
UBUNTU_JAVA_VERSION=8
REDHAT_JAVA_VERSION=1.8.0
else
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 5c0b5e09a..d480ccc44 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -327,10 +327,7 @@ export
USE_CUSTOM_IMPALA_BASE_IMAGE=${USE_CUSTOM_IMPALA_BASE_IMAGE:-false}
# images (created via e.g. 'make docker_debug_java11_images'). The Java
version used in
# these images is independent of the Java version used to compile Impala.
# Accepts 8, 11, 17.
-export IMPALA_DOCKER_JAVA=${IMPALA_DOCKER_JAVA:-"8"}
-if [ "${IMPALA_DOCKER_USE_JAVA11:-}" = "true" ]; then
- export IMPALA_DOCKER_JAVA=11
-fi
+export IMPALA_DOCKER_JAVA=${IMPALA_DOCKER_JAVA:-"17"}
# There are multiple compatible implementations of zlib. Cloudflare Zlib is an
# implementation with optimizations to use platform-specific CPU features that
are not
diff --git a/bin/jenkins/build-all-flag-combinations.sh
b/bin/jenkins/build-all-flag-combinations.sh
index 842492efd..44534f8f1 100755
--- a/bin/jenkins/build-all-flag-combinations.sh
+++ b/bin/jenkins/build-all-flag-combinations.sh
@@ -80,7 +80,6 @@ for CONFIG in "${CONFIGS[@]}"; do
else
export USE_APACHE_COMPONENTS=false
fi
- DESCRIPTION="Options $CONFIG USE_APACHE_COMPONENTS=$USE_APACHE_COMPONENTS"
CONFIG2=${CONFIG/-use_apache_hive_3/}
if [[ "$CONFIG" != "$CONFIG2" ]]; then
@@ -88,7 +87,6 @@ for CONFIG in "${CONFIGS[@]}"; do
export USE_APACHE_HIVE_3=true
export USE_APACHE_HIVE_2=false
fi
- DESCRIPTION="Options $CONFIG USE_APACHE_HIVE_3=$USE_APACHE_HIVE_3"
CONFIG2=${CONFIG/-use_apache_hive_2/}
if [[ "$CONFIG" != "$CONFIG2" ]]; then
@@ -96,7 +94,10 @@ for CONFIG in "${CONFIGS[@]}"; do
export USE_APACHE_HIVE_2=true
export USE_APACHE_HIVE_3=false
fi
- DESCRIPTION="Options $CONFIG USE_APACHE_HIVE_2=$USE_APACHE_HIVE_2"
+ DESCRIPTION="Options $CONFIG"
+ DESCRIPTION+=" | USE_APACHE_COMPONENTS=$USE_APACHE_COMPONENTS"
+ DESCRIPTION+=" | USE_APACHE_HIVE_3=$USE_APACHE_HIVE_3"
+ DESCRIPTION+=" | USE_APACHE_HIVE_2=$USE_APACHE_HIVE_2"
if [[ $# == 1 && $1 == "--dryrun" ]]; then
echo $DESCRIPTION
diff --git a/bin/jenkins/dockerized-impala-run-tests.sh
b/bin/jenkins/dockerized-impala-run-tests.sh
index 4c06efe50..ce7576880 100755
--- a/bin/jenkins/dockerized-impala-run-tests.sh
+++ b/bin/jenkins/dockerized-impala-run-tests.sh
@@ -81,13 +81,13 @@ start-impala-cluster.py --kill
# parquet-reader and impala-profile-tool are needed for e2e tests but not
built for
# non-test build.
IMAGE_TYPE=docker_debug
-case ${IMPALA_DOCKER_JAVA:-8} in
+case ${IMPALA_DOCKER_JAVA:-17} in
+ 8)
+ IMAGE_TYPE=${IMAGE_TYPE}_java8
+ ;;
11)
IMAGE_TYPE=${IMAGE_TYPE}_java11
;;
- 17)
- IMAGE_TYPE=${IMAGE_TYPE}_java17
- ;;
*)
;;
esac
diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index e7d2be2b9..4dee99b8b 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -1076,8 +1076,8 @@ class DockerMiniClusterOperations(object):
image_tag = daemon + "_debug"
else:
image_tag = daemon
- java_versions = {"8": "", "11": "_java11", "17": "_java17"}
- image_tag += java_versions[os.getenv('IMPALA_DOCKER_JAVA', '8')]
+ java_versions = {"17": "", "8": "_java8", "11": "_java11"}
+ image_tag += java_versions[os.getenv('IMPALA_DOCKER_JAVA', '17')]
host_name = self.__gen_host_name__(daemon, instance)
container_name = self.__gen_container_name__(daemon, instance)
# Mount configuration into container so that we don't need to rebuild
container
diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt
index e9c232eb3..c5d2481ea 100644
--- a/docker/CMakeLists.txt
+++ b/docker/CMakeLists.txt
@@ -115,21 +115,20 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
)
endfunction()
add_base_image(release "" "--install-debug-tools basic")
+ add_base_image(release_java8 "" "--install-debug-tools basic --java 8")
add_base_image(release_java11 "" "--install-debug-tools basic --java 11")
- add_base_image(release_java17 "" "--install-debug-tools basic --java 17")
# Debug images include full debug tools
add_base_image(debug "--debug-build" "--install-debug-tools full")
+ add_base_image(debug_java8 "--debug-build" "--install-debug-tools full
--java 8")
add_base_image(debug_java11 "--debug-build" "--install-debug-tools full
--java 11")
- add_base_image(debug_java17 "--debug-build" "--install-debug-tools full
--java 17")
-
# Target to build all docker images. Dependencies are added for each docker
image
# instantiated below.
add_custom_target(docker_images)
+ add_custom_target(docker_java8_images)
add_custom_target(docker_java11_images)
- add_custom_target(docker_java17_images)
add_custom_target(docker_debug_images)
+ add_custom_target(docker_debug_java8_images)
add_custom_target(docker_debug_java11_images)
- add_custom_target(docker_debug_java17_images)
add_custom_target(quickstart_docker_images)
set(exported_image_names "")
@@ -160,28 +159,28 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
function(add_daemon_docker_images daemon_name)
set(release_image ${daemon_name})
set(release_target ${daemon_name}_image)
+ set(release_java8_image ${daemon_name}_java8)
+ set(release_java8_target ${daemon_name}_java8_image)
set(release_java11_image ${daemon_name}_java11)
set(release_java11_target ${daemon_name}_java11_image)
- set(release_java17_image ${daemon_name}_java17)
- set(release_java17_target ${daemon_name}_java17_image)
set(debug_image ${daemon_name}_debug)
set(debug_target ${daemon_name}_debug_image)
+ set(debug_java8_image ${daemon_name}_debug_java8)
+ set(debug_java8_target ${daemon_name}_debug_java8_image)
set(debug_java11_image ${daemon_name}_debug_java11)
set(debug_java11_target ${daemon_name}_debug_java11_image)
- set(debug_java17_image ${daemon_name}_debug_java17)
- set(debug_java17_target ${daemon_name}_debug_java17_image)
add_daemon_docker_image(${release_target} ${daemon_name} ${release_image}
release)
+ add_daemon_docker_image(${release_java8_target} ${daemon_name}
${release_java8_image} release_java8)
add_daemon_docker_image(${release_java11_target} ${daemon_name}
${release_java11_image} release_java11)
- add_daemon_docker_image(${release_java17_target} ${daemon_name}
${release_java17_image} release_java17)
add_daemon_docker_image(${debug_target} ${daemon_name} ${debug_image}
debug)
+ add_daemon_docker_image(${debug_java8_target} ${daemon_name}
${debug_java8_image} debug_java8)
add_daemon_docker_image(${debug_java11_target} ${daemon_name}
${debug_java11_image} debug_java11)
- add_daemon_docker_image(${debug_java17_target} ${daemon_name}
${debug_java17_image} debug_java17)
ADD_DEPENDENCIES(docker_images ${release_target})
+ ADD_DEPENDENCIES(docker_java8_images ${release_java8_target})
ADD_DEPENDENCIES(docker_java11_images ${release_java11_target})
- ADD_DEPENDENCIES(docker_java17_images ${release_java17_target})
ADD_DEPENDENCIES(docker_debug_images ${debug_target})
+ ADD_DEPENDENCIES(docker_debug_java8_images ${debug_java8_target})
ADD_DEPENDENCIES(docker_debug_java11_images ${debug_java11_target})
- ADD_DEPENDENCIES(docker_debug_java17_images ${debug_java17_target})
# add_daemon_docker_image modifies 'exported_image_names'. Publish it to
the parent scope.
set(exported_image_names "${exported_image_names}" PARENT_SCOPE)
endfunction()
diff --git a/docker/install_os_packages.sh b/docker/install_os_packages.sh
index 328ffb5ea..2d39c15b0 100755
--- a/docker/install_os_packages.sh
+++ b/docker/install_os_packages.sh
@@ -26,7 +26,7 @@ set -euo pipefail
# Default level of extra debugging tools, controlled by the
--install-debug-tools flag.
INSTALL_DEBUG_TOOLS=none
-JAVA_VERSION=8
+JAVA_VERSION=17
DRY_RUN=false
PKG_LIST=""
NON_PKG_NAMES=(apt-get yum apk install update add)
@@ -35,7 +35,7 @@ function print_usage {
echo "install_os_packages.sh - Helper script to install OS dependencies"
echo "[--install-debug-tools <none|basic|full>] : set the level of debug
tools"\
"to install"
- echo "[--java <version>] : Use specified Java version rather than the
default Java 8."
+ echo "[--java <version>] : Use specified Java version rather than the
default Java 17."
echo "[--dry-run] : Print the list of packages to install."
}
diff --git
a/fe/src/compat-apache-hive-2/java/org/apache/impala/catalog/events/MetastoreEvents.java
b/fe/src/compat-apache-hive-2/java/org/apache/impala/catalog/events/MetastoreEvents.java
index 605acbb12..d456c54c8 100644
---
a/fe/src/compat-apache-hive-2/java/org/apache/impala/catalog/events/MetastoreEvents.java
+++
b/fe/src/compat-apache-hive-2/java/org/apache/impala/catalog/events/MetastoreEvents.java
@@ -61,6 +61,7 @@ import org.apache.impala.catalog.Db;
import org.apache.impala.catalog.FeCatalogUtils;
import org.apache.impala.catalog.FileMetadataLoadOpts;
import org.apache.impala.catalog.HdfsTable;
+import org.apache.impala.catalog.Hive2MetastoreShimBase;
import org.apache.impala.catalog.IncompleteTable;
import org.apache.impala.catalog.MetastoreClientInstantiationException;
import org.apache.impala.catalog.MetaStoreClientPool;
@@ -601,9 +602,6 @@ public class MetastoreEvents {
// Logger available for all the sub-classes
protected final Logger LOG = LoggerFactory.getLogger(this.getClass());
- // catalog name from the event
- protected final String catalogName_;
-
// dbName from the event
protected String dbName_;
@@ -648,7 +646,6 @@ public class MetastoreEvents {
this.eventId_ = event_.getEventId();
this.eventType_ = MetastoreEventType.from(event.getEventType());
// certain event types in Hive-3 like COMMIT_TXN may not have dbName set
- this.catalogName_ = event.getCatName();
this.dbName_ = event.getDbName();
this.tblName_ = event.getTableName();
this.metastoreNotificationEvent_ = event;
@@ -718,7 +715,11 @@ public class MetastoreEvents {
this.eventType_ = type;
}
- public String getCatalogName() { return catalogName_; }
+ public String getCatalogName() {
+ // In Hive 2 there is no catalog name support. So we return the Hive 3
default
+ // catalog name
+ return Hive2MetastoreShimBase.getDefaultCatalogName();
+ }
public String getDbName() { return dbName_; }
@@ -1053,6 +1054,21 @@ public class MetastoreEvents {
}
}
+ /**
+ * In Hive 2 there is no catalog name support. So we return the Hive 3
default
+ * catalog name
+ */
+ public static String getCatName(NotificationEvent event) {
+ return Hive2MetastoreShimBase.getDefaultCatalogName();
+ }
+
+ /**
+ * In Hive 2 there is no catalog name support. So this method is a no-op.
+ */
+ public static void setCatName(NotificationEvent event, String unused) {
+ // No-op
+ }
+
public static String getStringProperty(
Map<String, String> params, String key, String defaultVal) {
if (params == null) return defaultVal;
diff --git
a/fe/src/main/java/org/apache/impala/catalog/events/EventExecutorService.java
b/fe/src/main/java/org/apache/impala/catalog/events/EventExecutorService.java
index 734ec6f6a..cfedc8922 100644
---
a/fe/src/main/java/org/apache/impala/catalog/events/EventExecutorService.java
+++
b/fe/src/main/java/org/apache/impala/catalog/events/EventExecutorService.java
@@ -368,7 +368,7 @@ public class EventExecutorService {
pseudoDropTableEvent.setEventType(MetastoreEventType.DROP_TABLE.toString());
pseudoDropTableEvent.setTableName(alterEvent.getBeforeTable().getTableName());
pseudoDropTableEvent.setDbName(alterEvent.getBeforeTable().getDbName());
- pseudoDropTableEvent.setCatName(alterEvent.getCatalogName());
+ MetastoreEvents.setCatName(pseudoDropTableEvent,
alterEvent.getCatalogName());
pseudoDropTableEvent.setEventId(alterEvent.getEventId());
MetastoreTableEvent dropTableEvent = new DropTableEvent(
alterEvent.getCatalogOpExecutor(), alterEvent.getMetrics(),
pseudoDropTableEvent,
@@ -378,7 +378,7 @@ public class EventExecutorService {
pseudoCreateTableEvent.setEventType(MetastoreEventType.CREATE_TABLE.toString());
pseudoCreateTableEvent.setTableName(alterEvent.getAfterTable().getTableName());
pseudoCreateTableEvent.setDbName(alterEvent.getAfterTable().getDbName());
- pseudoCreateTableEvent.setCatName(alterEvent.getCatalogName());
+ MetastoreEvents.setCatName(pseudoCreateTableEvent,
alterEvent.getCatalogName());
pseudoCreateTableEvent.setEventId(alterEvent.getEventId());
MetastoreTableEvent createTableEvent = new CreateTableEvent(
alterEvent.getCatalogOpExecutor(), alterEvent.getMetrics(),
diff --git
a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
index 2a938c251..5b67b3274 100644
--- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
+++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
@@ -1060,6 +1060,24 @@ public class MetastoreEvents {
}
}
+ /**
+ * We need this method for Hive 2 compatibility because NotificationEvent
+ * does not have getCatName() method in Hive 2.
+ * See its counterpart in MetastoreEvents.java under compat-apache-hive-2.
+ */
+ public static String getCatName(NotificationEvent event) {
+ return event.getCatName();
+ }
+
+ /**
+ * We need this method for Hive 2 compatibility because NotificationEvent
+ * does not have setCatName() method in Hive 2.
+ * See its counterpart in MetastoreEvents.java under compat-apache-hive-2.
+ */
+ public static void setCatName(NotificationEvent event, String catName) {
+ event.setCatName(catName);
+ }
+
public static String getStringProperty(
Map<String, String> params, String key, String defaultVal) {
if (params == null) return defaultVal;
diff --git
a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
index 68ec104e3..46ce22cb9 100644
---
a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
+++
b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
@@ -334,7 +334,7 @@ public class MetastoreEventsProcessor implements
ExternalEventsProcessor {
Preconditions.checkNotNull(dbName, "dbName is null in fetching db events");
NotificationFilter filter = notificationEvent ->
eventType.equals(notificationEvent.getEventType())
- && catName.equalsIgnoreCase(notificationEvent.getCatName())
+ &&
catName.equalsIgnoreCase(MetastoreEvents.getCatName(notificationEvent))
&& dbName.equalsIgnoreCase(notificationEvent.getDbName());
MetaDataFilter metaDataFilter = new MetaDataFilter(filter, catName,
dbName);
return getNextMetastoreEventsWithFilterInBatches(catalog, eventId,
metaDataFilter,
@@ -357,7 +357,7 @@ public class MetastoreEventsProcessor implements
ExternalEventsProcessor {
Preconditions.checkNotNull(tblName, "tblName is null in fetching table
events");
NotificationFilter filter = notificationEvent ->
eventType.equals(notificationEvent.getEventType())
- && catName.equalsIgnoreCase(notificationEvent.getCatName())
+ &&
catName.equalsIgnoreCase(MetastoreEvents.getCatName(notificationEvent))
&& dbName.equalsIgnoreCase(notificationEvent.getDbName())
&& tblName.equalsIgnoreCase(notificationEvent.getTableName());
MetaDataFilter metaDataFilter = new MetaDataFilter(filter, catName,
dbName, tblName);
@@ -2132,7 +2132,7 @@ public class MetastoreEventsProcessor implements
ExternalEventsProcessor {
List<String> eventTypes = wantTableList ?
MetaDataFilter.TABLE_LIST_EVENT_TYPES :
MetaDataFilter.DB_EVENT_TYPES;
NotificationFilter filter = e -> dbName.equalsIgnoreCase(e.getDbName())
- && MetastoreShim.isDefaultCatalog(e.getCatName())
+ && MetastoreShim.isDefaultCatalog(MetastoreEvents.getCatName(e))
&& eventTypes.contains(e.getEventType());
// Use 'requiredEventId' as the startEventId since events before it are
decided
// to wait for.
@@ -2242,7 +2242,8 @@ public class MetastoreEventsProcessor implements
ExternalEventsProcessor {
*/
private long getMinRequiredEventIdForDbList(long startEventId)
throws MetastoreNotificationFetchException {
- NotificationFilter filter = e ->
MetastoreShim.isDefaultCatalog(e.getCatName())
+ NotificationFilter filter = e ->
+ MetastoreShim.isDefaultCatalog(MetastoreEvents.getCatName(e))
&& MetaDataFilter.DB_EVENT_TYPES.contains(e.getEventType());
List<NotificationEvent> dbEvents = getNextMetastoreEventsInBatches(
catalog_, startEventId, filter,
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index 979983c2e..79ca38519 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -144,6 +144,7 @@ import org.apache.impala.catalog.Transaction;
import org.apache.impala.catalog.Type;
import org.apache.impala.catalog.View;
import org.apache.impala.catalog.events.DeleteEventLog;
+import org.apache.impala.catalog.events.MetastoreEvents;
import org.apache.impala.catalog.events.MetastoreEvents.AddPartitionEvent;
import org.apache.impala.catalog.events.MetastoreEvents.AlterTableEvent;
import org.apache.impala.catalog.events.MetastoreEvents.CreateDatabaseEvent;
@@ -2463,7 +2464,7 @@ public class CatalogOpExecutor {
List<String> eventTypes = Lists.newArrayList(
DropDatabaseEvent.EVENT_TYPE, DropTableEvent.EVENT_TYPE);
NotificationFilter filter = e -> dbName.equalsIgnoreCase(e.getDbName())
- && MetastoreShim.isDefaultCatalog(e.getCatName())
+ && MetastoreShim.isDefaultCatalog(MetastoreEvents.getCatName(e))
&& eventTypes.contains(e.getEventType());
List<NotificationEvent> events = MetastoreEventsProcessor
.getNextMetastoreEventsInBatches(catalog_, eventId, filter,
diff --git a/java/ext-data-source/api/pom.xml b/java/ext-data-source/api/pom.xml
index 2a9209130..028303222 100644
--- a/java/ext-data-source/api/pom.xml
+++ b/java/ext-data-source/api/pom.xml
@@ -36,6 +36,11 @@
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
</dependency>
+ <dependency>
+ <groupId>javax.annotation</groupId>
+ <artifactId>javax.annotation-api</artifactId>
+ <version>1.3.2</version>
+ </dependency>
</dependencies>
<build>
diff --git a/testdata/bin/patch_hive.sh b/testdata/bin/patch_hive.sh
index 1953b8926..45f52bcbb 100755
--- a/testdata/bin/patch_hive.sh
+++ b/testdata/bin/patch_hive.sh
@@ -38,7 +38,7 @@ HIVE_REBUILD=${HIVE_REBUILD-false}
PATCHED_CACHE_FILE="$HIVE_SRC_DIR/.patched"
if [ ! -f "$PATCHED_CACHE_FILE" ]; then touch "$PATCHED_CACHE_FILE"; fi
# Apache Hive patch dir
-HIVE_PARCH_DIR="${IMPALA_HOME}/testdata/cluster/hive"
+HIVE_PATCH_DIR="${IMPALA_HOME}/testdata/cluster/hive"
# Apply the patch and save the patch name to .patched
function apply_patch {
@@ -53,7 +53,7 @@ function apply_patch {
done < $PATCHED_CACHE_FILE
if [ $status = "1" ] ;then
echo "Apply patch: $p"
- patch -p1 < ${HIVE_PARCH_DIR}/$p
+ patch -p1 < ${HIVE_PATCH_DIR}/$p
echo $p >> $PATCHED_CACHE_FILE
HIVE_REBUILD=true
fi
@@ -66,12 +66,21 @@ cp $HADOOP_HOME/share/hadoop/hdfs/lib/guava-*.jar
$HIVE_HOME/lib/
# 2. Apply patches
pushd "$HIVE_SRC_DIR"
-for file in `ls ${HIVE_PARCH_DIR}/patch*.diff | sort`
+for file in `ls ${HIVE_PATCH_DIR}/patch*.diff | sort`
do
p=$(basename $file)
apply_patch $p
done
+if (( IMPALA_JAVA_TARGET >= 17 )); then
+ echo "Apply patches for Java 17 compatibility"
+ for file in `ls ${HIVE_PATCH_DIR}/java17/patch*.diff | sort`
+ do
+ p=java17/$(basename $file)
+ apply_patch $p
+ done
+fi
+
# 3. Repackage the hive submodules affected by the patch
if [[ "${HIVE_REBUILD}" = "true" ]]; then
echo "Repackage the hive-exec module"
diff --git a/testdata/bin/patch_hive2.sh b/testdata/bin/patch_hive2.sh
index b88284ec9..5b49d524e 100755
--- a/testdata/bin/patch_hive2.sh
+++ b/testdata/bin/patch_hive2.sh
@@ -59,13 +59,3 @@ do
p=$(basename $file)
apply_patch $p
done
-
-# 2. Repackage the hive submodules affected by the patch
-if [[ "${HIVE_REBUILD}" = "true" ]]; then
- echo "Repackage the hive-metastore module"
- ${IMPALA_HOME}/bin/mvn-quiet.sh -pl metastore clean package \
- -Dmaven.test.skip
- cp
$HIVE_SRC_DIR/metastore/target/hive-metastore-${APACHE_HIVE_2_VERSION}.jar \
- $HIVE_HOME/lib/
-fi
-popd
diff --git a/testdata/cluster/hive/java17/patch0-JDK-17-changes.diff
b/testdata/cluster/hive/java17/patch0-JDK-17-changes.diff
new file mode 100644
index 000000000..2e2b0724d
--- /dev/null
+++ b/testdata/cluster/hive/java17/patch0-JDK-17-changes.diff
@@ -0,0 +1,81 @@
+diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+index 348f9df773f..558a7fb0ae8 100644
+---
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
++++
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+@@ -69,7 +69,7 @@
+ import com.google.common.annotations.VisibleForTesting;
+ import com.google.protobuf.CodedInputStream;
+
+-import sun.misc.Cleaner;
++
+
+
+ /**
+@@ -1653,18 +1653,12 @@ private void releaseBuffer(ByteBuffer bb, boolean
isFromDataReader) {
+ }
+ Field localCf = cleanerField;
+ if (!bb.isDirect() || localCf == null) return;
+- try {
+- Cleaner cleaner = (Cleaner) localCf.get(bb);
+- if (cleaner != null) {
+- cleaner.clean();
+- } else {
+- LOG.debug("Unable to clean a buffer using cleaner - no cleaner");
+- }
+- } catch (Exception e) {
+- // leave it for GC to clean up
+- LOG.warn("Unable to clean direct buffers using Cleaner.");
+- cleanerField = null;
+- }
++ // sun.misc.Cleaner has been relocated and changed in OpenJDK 9.
++ // The direct cleaning is not critical for the Impala Frontend, so we just
++ // fall back to GC.
++ LOG.info("Unable to clean a buffer using cleaner. " +
++ "GC should eventually clean this buffer");
++ cleanerField = null;
+ }
+
+
+diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
+index e36f1e64f04..d945c0ef0f9 100644
+--- a/standalone-metastore/pom.xml
++++ b/standalone-metastore/pom.xml
+@@ -93,6 +93,11 @@
+ </properties>
+
+ <dependencies>
++ <dependency>
++ <groupId>javax.annotation</groupId>
++ <artifactId>javax.annotation-api</artifactId>
++ <version>1.3.2</version>
++ </dependency>
+ <dependency>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-core</artifactId>
+@@ -504,6 +509,11 @@
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>${maven.antrun.plugin.version}</version>
+ <dependencies>
++ <dependency>
++ <groupId>javax.annotation</groupId>
++ <artifactId>javax.annotation-api</artifactId>
++ <version>1.3.2</version>
++ </dependency>
+ <dependency>
+ <groupId>ant-contrib</groupId>
+ <artifactId>ant-contrib</artifactId>
+@@ -786,9 +796,9 @@
+ </configuration>
+ <executions>
+ <execution>
+- <phase>process-classes</phase>
++ <phase>none</phase>
+ <goals>
+- <goal>enhance</goal>
++ <goal>none</goal>
+ </goals>
+ </execution>
+ </executions>
+--
+2.34.1
+
diff --git a/testdata/cluster/hive2/patch0-CATALOG_NAME.diff
b/testdata/cluster/hive2/patch0-CATALOG_NAME.diff
new file mode 100644
index 000000000..e75c9b291
--- /dev/null
+++ b/testdata/cluster/hive2/patch0-CATALOG_NAME.diff
@@ -0,0 +1,10 @@
+diff --git a/metastore/if/hive_metastore.thrift
b/metastore/if/hive_metastore.thrift
+index 8a66bbcc70..bf33ae1954 100755
+--- a/metastore/if/hive_metastore.thrift
++++ b/metastore/if/hive_metastore.thrift
+@@ -820,6 +820,7 @@ struct NotificationEvent {
+ 5: optional string tableName,
+ 6: required string message,
+ 7: optional string messageFormat,
++ 8: optional string catName,
+ }
diff --git a/testdata/cluster/hive2/patch0-HIVE-18755.diff
b/testdata/cluster/hive2/patch0-HIVE-18755.diff
deleted file mode 100644
index 7fdfff53e..000000000
--- a/testdata/cluster/hive2/patch0-HIVE-18755.diff
+++ /dev/null
@@ -1,306 +0,0 @@
-From 92a3a88870623dad3b8fa1e02d5e855ae55dfcff Mon Sep 17 00:00:00 2001
-From: ttttttz <[email protected]>
-Date: Mon, 7 Jul 2025 01:56:58 +0800
-Subject: [PATCH] Add the field catName to the NotificationEvent
-
----
- metastore/if/hive_metastore.thrift | 1 +
- .../hive/metastore/api/NotificationEvent.java | 116 +++++++++++++++++-
- 2 files changed, 112 insertions(+), 5 deletions(-)
-
-diff --git a/metastore/if/hive_metastore.thrift
b/metastore/if/hive_metastore.thrift
-index 8a66bbcc70..bf33ae1954 100755
---- a/metastore/if/hive_metastore.thrift
-+++ b/metastore/if/hive_metastore.thrift
-@@ -820,6 +820,7 @@ struct NotificationEvent {
- 5: optional string tableName,
- 6: required string message,
- 7: optional string messageFormat,
-+ 8: optional string catName,
- }
-
- struct NotificationEventResponse {
-diff --git
a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEvent.java
b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEvent.java
-index 8e0fb40bcc..e0e1cd4dc5 100644
----
a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEvent.java
-+++
b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEvent.java
-@@ -35,7 +35,7 @@
-
- @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
- @Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
--public class NotificationEvent implements
org.apache.thrift.TBase<NotificationEvent, NotificationEvent._Fields>,
java.io.Serializable, Cloneable, Comparable<NotificationEvent> {
[email protected]
@org.apache.hadoop.classification.InterfaceStability.Stable public class
NotificationEvent implements org.apache.thrift.TBase<NotificationEvent,
NotificationEvent._Fields>, java.io.Serializable, Cloneable,
Comparable<NotificationEvent> {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new
org.apache.thrift.protocol.TStruct("NotificationEvent");
-
- private static final org.apache.thrift.protocol.TField EVENT_ID_FIELD_DESC
= new org.apache.thrift.protocol.TField("eventId",
org.apache.thrift.protocol.TType.I64, (short)1);
-@@ -45,6 +45,7 @@
- private static final org.apache.thrift.protocol.TField
TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName",
org.apache.thrift.protocol.TType.STRING, (short)5);
- private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC =
new org.apache.thrift.protocol.TField("message",
org.apache.thrift.protocol.TType.STRING, (short)6);
- private static final org.apache.thrift.protocol.TField
MESSAGE_FORMAT_FIELD_DESC = new
org.apache.thrift.protocol.TField("messageFormat",
org.apache.thrift.protocol.TType.STRING, (short)7);
-+ private static final org.apache.thrift.protocol.TField CAT_NAME_FIELD_DESC
= new org.apache.thrift.protocol.TField("catName",
org.apache.thrift.protocol.TType.STRING, (short)8);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes =
new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
-@@ -59,6 +60,7 @@
- private String tableName; // optional
- private String message; // required
- private String messageFormat; // optional
-+ private String catName; // optional
-
- /** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-@@ -68,7 +70,8 @@
- DB_NAME((short)4, "dbName"),
- TABLE_NAME((short)5, "tableName"),
- MESSAGE((short)6, "message"),
-- MESSAGE_FORMAT((short)7, "messageFormat");
-+ MESSAGE_FORMAT((short)7, "messageFormat"),
-+ CAT_NAME((short)8, "catName");
-
- private static final Map<String, _Fields> byName = new HashMap<String,
_Fields>();
-
-@@ -97,6 +100,8 @@ public static _Fields findByThriftId(int fieldId) {
- return MESSAGE;
- case 7: // MESSAGE_FORMAT
- return MESSAGE_FORMAT;
-+ case 8: // CAT_NAME
-+ return CAT_NAME;
- default:
- return null;
- }
-@@ -140,7 +145,7 @@ public String getFieldName() {
- private static final int __EVENTID_ISSET_ID = 0;
- private static final int __EVENTTIME_ISSET_ID = 1;
- private byte __isset_bitfield = 0;
-- private static final _Fields optionals[] =
{_Fields.DB_NAME,_Fields.TABLE_NAME,_Fields.MESSAGE_FORMAT};
-+ private static final _Fields optionals[] =
{_Fields.DB_NAME,_Fields.TABLE_NAME,_Fields.MESSAGE_FORMAT,_Fields.CAT_NAME};
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData>
metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new
EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
-@@ -158,6 +163,8 @@ public String getFieldName() {
- new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.MESSAGE_FORMAT, new
org.apache.thrift.meta_data.FieldMetaData("messageFormat",
org.apache.thrift.TFieldRequirementType.OPTIONAL,
- new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
-+ tmpMap.put(_Fields.CAT_NAME, new
org.apache.thrift.meta_data.FieldMetaData("catName",
org.apache.thrift.TFieldRequirementType.OPTIONAL,
-+ new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
-
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(NotificationEvent.class,
metaDataMap);
- }
-@@ -202,6 +209,9 @@ public NotificationEvent(NotificationEvent other) {
- if (other.isSetMessageFormat()) {
- this.messageFormat = other.messageFormat;
- }
-+ if (other.isSetCatName()) {
-+ this.catName = other.catName;
-+ }
- }
-
- public NotificationEvent deepCopy() {
-@@ -219,6 +229,7 @@ public void clear() {
- this.tableName = null;
- this.message = null;
- this.messageFormat = null;
-+ this.catName = null;
- }
-
- public long getEventId() {
-@@ -380,6 +391,29 @@ public void setMessageFormatIsSet(boolean value) {
- }
- }
-
-+ public String getCatName() {
-+ return this.catName;
-+ }
-+
-+ public void setCatName(String catName) {
-+ this.catName = catName;
-+ }
-+
-+ public void unsetCatName() {
-+ this.catName = null;
-+ }
-+
-+ /** Returns true if field catName is set (has been assigned a value) and
false otherwise */
-+ public boolean isSetCatName() {
-+ return this.catName != null;
-+ }
-+
-+ public void setCatNameIsSet(boolean value) {
-+ if (!value) {
-+ this.catName = null;
-+ }
-+ }
-+
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case EVENT_ID:
-@@ -438,6 +472,14 @@ public void setFieldValue(_Fields field, Object value) {
- }
- break;
-
-+ case CAT_NAME:
-+ if (value == null) {
-+ unsetCatName();
-+ } else {
-+ setCatName((String)value);
-+ }
-+ break;
-+
- }
- }
-
-@@ -464,6 +506,9 @@ public Object getFieldValue(_Fields field) {
- case MESSAGE_FORMAT:
- return getMessageFormat();
-
-+ case CAT_NAME:
-+ return getCatName();
-+
- }
- throw new IllegalStateException();
- }
-@@ -489,6 +534,8 @@ public boolean isSet(_Fields field) {
- return isSetMessage();
- case MESSAGE_FORMAT:
- return isSetMessageFormat();
-+ case CAT_NAME:
-+ return isSetCatName();
- }
- throw new IllegalStateException();
- }
-@@ -569,6 +616,15 @@ public boolean equals(NotificationEvent that) {
- return false;
- }
-
-+ boolean this_present_catName = true && this.isSetCatName();
-+ boolean that_present_catName = true && that.isSetCatName();
-+ if (this_present_catName || that_present_catName) {
-+ if (!(this_present_catName && that_present_catName))
-+ return false;
-+ if (!this.catName.equals(that.catName))
-+ return false;
-+ }
-+
- return true;
- }
-
-@@ -611,6 +667,11 @@ public int hashCode() {
- if (present_messageFormat)
- list.add(messageFormat);
-
-+ boolean present_catName = true && (isSetCatName());
-+ list.add(present_catName);
-+ if (present_catName)
-+ list.add(catName);
-+
- return list.hashCode();
- }
-
-@@ -692,6 +753,16 @@ public int compareTo(NotificationEvent other) {
- return lastComparison;
- }
- }
-+ lastComparison =
Boolean.valueOf(isSetCatName()).compareTo(other.isSetCatName());
-+ if (lastComparison != 0) {
-+ return lastComparison;
-+ }
-+ if (isSetCatName()) {
-+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.catName,
other.catName);
-+ if (lastComparison != 0) {
-+ return lastComparison;
-+ }
-+ }
- return 0;
- }
-
-@@ -765,6 +836,16 @@ public String toString() {
- }
- first = false;
- }
-+ if (isSetCatName()) {
-+ if (!first) sb.append(", ");
-+ sb.append("catName:");
-+ if (this.catName == null) {
-+ sb.append("null");
-+ } else {
-+ sb.append(this.catName);
-+ }
-+ first = false;
-+ }
- sb.append(")");
- return sb.toString();
- }
-@@ -882,6 +963,14 @@ public void read(org.apache.thrift.protocol.TProtocol
iprot, NotificationEvent s
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
- }
- break;
-+ case 8: // CAT_NAME
-+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
-+ struct.catName = iprot.readString();
-+ struct.setCatNameIsSet(true);
-+ } else {
-+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
-+ }
-+ break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
- }
-@@ -932,6 +1021,13 @@ public void write(org.apache.thrift.protocol.TProtocol
oprot, NotificationEvent
- oprot.writeFieldEnd();
- }
- }
-+ if (struct.catName != null) {
-+ if (struct.isSetCatName()) {
-+ oprot.writeFieldBegin(CAT_NAME_FIELD_DESC);
-+ oprot.writeString(struct.catName);
-+ oprot.writeFieldEnd();
-+ }
-+ }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-@@ -963,7 +1059,10 @@ public void write(org.apache.thrift.protocol.TProtocol
prot, NotificationEvent s
- if (struct.isSetMessageFormat()) {
- optionals.set(2);
- }
-- oprot.writeBitSet(optionals, 3);
-+ if (struct.isSetCatName()) {
-+ optionals.set(3);
-+ }
-+ oprot.writeBitSet(optionals, 4);
- if (struct.isSetDbName()) {
- oprot.writeString(struct.dbName);
- }
-@@ -973,6 +1072,9 @@ public void write(org.apache.thrift.protocol.TProtocol
prot, NotificationEvent s
- if (struct.isSetMessageFormat()) {
- oprot.writeString(struct.messageFormat);
- }
-+ if (struct.isSetCatName()) {
-+ oprot.writeString(struct.catName);
-+ }
- }
-
- @Override
-@@ -986,7 +1088,7 @@ public void read(org.apache.thrift.protocol.TProtocol
prot, NotificationEvent st
- struct.setEventTypeIsSet(true);
- struct.message = iprot.readString();
- struct.setMessageIsSet(true);
-- BitSet incoming = iprot.readBitSet(3);
-+ BitSet incoming = iprot.readBitSet(4);
- if (incoming.get(0)) {
- struct.dbName = iprot.readString();
- struct.setDbNameIsSet(true);
-@@ -999,6 +1101,10 @@ public void read(org.apache.thrift.protocol.TProtocol
prot, NotificationEvent st
- struct.messageFormat = iprot.readString();
- struct.setMessageFormatIsSet(true);
- }
-+ if (incoming.get(3)) {
-+ struct.catName = iprot.readString();
-+ struct.setCatNameIsSet(true);
-+ }
- }
- }
-
---
-2.32.1 (Apple Git-133)
-
