This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit a52721e62761957753f482124c1e76f453d22b1d Author: stiga-huang <[email protected]> AuthorDate: Sun Oct 9 10:37:03 2022 +0800 IMPALA-11644: updateLatestEventId should handle cases of empty events IMPALA-11490 adds a catalog metric for the latest event id in HMS. The fetched events could be empty if there are no more events in the past 24 hours, since the retention duration for notification events in HMS is 24 hours by default. This case is not handled so the thread for updating the latestEventId metric will keep throwing a NoSuchElementException until there are new events generated. This patch handles the case to avoid exceptions, also sets the initial value of latestEventId to 0 which is the returned value of getCurrentNotificationEventId() when there are no events in HMS. Tests - Clean up the notification events in HMS by truncating the NOTIFICATION_SEQUENCE and NOTIFICATION_LOG tables in the underlying PostgreSQL. Then launch HMS and Impala. Verified the exception disappears. - Add a breakpoint in updateLatestEventId() to stop before fetching the events and after getting the current event id. Clean up the notification events in HMS. Then resume catalogd. Verified no exceptions are thrown. Change-Id: I0f207fff1ff59376e30afdc3cd074c950a1c3ddb Reviewed-on: http://gerrit.cloudera.org:8080/19112 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../apache/impala/catalog/events/MetastoreEventsProcessor.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java index 264caab4e..7c2d6d368 100644 --- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java +++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java @@ -26,6 +26,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -497,7 +498,7 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor { // The event id and eventTime of the latest event in HMS. Only used in metrics to show // how far we are lagging behind. - private final AtomicLong latestEventId_ = new AtomicLong(-1); + private final AtomicLong latestEventId_ = new AtomicLong(0); private final AtomicLong latestEventTimeMs_ = new AtomicLong(0); // The duration in nanoseconds of the processing of the last event batch. @@ -888,7 +889,10 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor { eventRequest.setMaxEvents(1); NotificationEventResponse response = MetastoreShim .getNextNotification(msClient.getHiveClient(), eventRequest); - NotificationEvent event = response.getEventsIterator().next(); + Iterator<NotificationEvent> eventIter = response.getEventsIterator(); + // Events could be empty if they are just cleaned up. + if (!eventIter.hasNext()) return; + NotificationEvent event = eventIter.next(); Preconditions.checkState(event.getEventId() == currentEventId); LOG.info("Latest event in HMS: id={}, time={}", currentEventId, event.getEventTime());
