This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 98709beb2f9 branch-3.0: [fix](hms)fix hive catalog follower not get
event. #46485 (#46491)
98709beb2f9 is described below
commit 98709beb2f96fd839b7d3c004e30175a81856cbe
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Jan 7 12:13:42 2025 +0800
branch-3.0: [fix](hms)fix hive catalog follower not get event. #46485
(#46491)
Cherry-picked from #46485
Co-authored-by: daidai <changyu...@selectdb.com>
---
.../doris/datasource/hive/HMSExternalCatalog.java | 28 +++++++++++-----------
.../hive/event/MetastoreEventsProcessor.java | 2 +-
2 files changed, 15 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
index 2d6c66af444..2c80901da4a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
@@ -117,20 +117,6 @@ public class HMSExternalCatalog extends ExternalCatalog {
throw new DdlException(
"The parameter " + FILE_META_CACHE_TTL_SECOND + " is
wrong, value is " + fileMetaCacheTtlSecond);
}
- Map<String, String> properties = catalogProperty.getProperties();
- if
(properties.containsKey(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC)) {
- enableHmsEventsIncrementalSync =
-
properties.get(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC).equals("true");
- } else {
- enableHmsEventsIncrementalSync =
Config.enable_hms_events_incremental_sync;
- }
-
- if
(properties.containsKey(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC)) {
- hmsEventsBatchSizePerRpc =
Integer.valueOf(properties.get(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC));
- } else {
- hmsEventsBatchSizePerRpc = Config.hms_events_batch_size_per_rpc;
- }
-
// check the dfs.ha properties
// 'dfs.nameservices'='your-nameservice',
// 'dfs.ha.namenodes.your-nameservice'='nn1,nn2',
@@ -290,6 +276,20 @@ public class HMSExternalCatalog extends ExternalCatalog {
// always allow fallback to simple auth, so to support both
kerberos and simple auth
catalogProperty.addProperty(DFSFileSystem.PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH,
"true");
}
+
+ Map<String, String> properties = catalogProperty.getProperties();
+ if
(properties.containsKey(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC)) {
+ enableHmsEventsIncrementalSync =
+
properties.get(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC).equals("true");
+ } else {
+ enableHmsEventsIncrementalSync =
Config.enable_hms_events_incremental_sync;
+ }
+
+ if
(properties.containsKey(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC)) {
+ hmsEventsBatchSizePerRpc =
Integer.valueOf(properties.get(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC));
+ } else {
+ hmsEventsBatchSizePerRpc = Config.hms_events_batch_size_per_rpc;
+ }
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
index 73054773402..23bf324360b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
@@ -269,7 +269,7 @@ public class MetastoreEventsProcessor extends MasterDaemon {
// For slave FE nodes, only fetch events which id is lower than
masterLastSyncedEventId
int maxEventSize = Math.min((int) (masterLastSyncedEventId -
lastSyncedEventId),
- Config.hms_events_batch_size_per_rpc);
+ hmsExternalCatalog.getHmsEventsBatchSizePerRpc());
try {
return
hmsExternalCatalog.getClient().getNextNotification(lastSyncedEventId,
maxEventSize, null);
} catch (MetastoreNotificationFetchException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
