This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 3a8f12c318f9 [HUDI-9749] Pass correct base path key to metrics of sync
tool (#13764)
3a8f12c318f9 is described below
commit 3a8f12c318f9c003b7ed43d2cc53887c8038cfde
Author: Lin Liu <[email protected]>
AuthorDate: Tue Aug 26 10:43:42 2025 -0700
[HUDI-9749] Pass correct base path key to metrics of sync tool (#13764)
---
.../hudi/common/config/HoodieCommonConfig.java | 2 +
.../hudi/config/metrics/HoodieMetricsConfig.java | 10 ++++-
.../config/metrics/TestHoodieMetricsConfig.java | 48 ++++++++++++++++++++++
.../hudi/gcp/bigquery/TestBigQuerySyncTool.java | 3 +-
.../hudi/sync/datahub/TestDataHubSyncTool.java | 3 +-
.../apache/hudi/sync/common/HoodieSyncConfig.java | 3 +-
.../hudi/sync/common/TestHoodieSyncTool.java | 3 +-
7 files changed, 67 insertions(+), 5 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java
b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java
index 7f9098bd7cf3..e488324af341 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java
@@ -38,6 +38,8 @@ import static
org.apache.hudi.common.util.ConfigUtils.enumNames;
description = "The following set of configurations are common across
Hudi.")
public class HoodieCommonConfig extends HoodieConfig {
+ public static final String META_SYNC_BASE_PATH_KEY =
"hoodie.datasource.meta.sync.base.path";
+
public static final ConfigProperty<String> BASE_PATH = ConfigProperty
.key("hoodie.base.path")
.noDefaultValue()
diff --git
a/hudi-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java
b/hudi-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java
index d869070a39ca..7b9230218881 100644
---
a/hudi-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java
+++
b/hudi-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java
@@ -41,6 +41,8 @@ import java.util.Properties;
import java.util.function.Supplier;
import java.util.stream.Collectors;
+import static
org.apache.hudi.common.config.HoodieCommonConfig.META_SYNC_BASE_PATH_KEY;
+
/**
* Fetch the configurations used by the Metrics system.
*/
@@ -168,7 +170,13 @@ public class HoodieMetricsConfig extends HoodieConfig {
* base properties.
*/
public String getBasePath() {
- return getString(HoodieCommonConfig.BASE_PATH);
+ // First search default base path key.
+ String basePathKey = getString(HoodieCommonConfig.BASE_PATH);
+ // This works when initialized in a meta-sync tool.
+ if (basePathKey == null) {
+ return getString(META_SYNC_BASE_PATH_KEY);
+ }
+ return basePathKey;
}
/**
diff --git
a/hudi-common/src/test/java/org/apache/hudi/config/metrics/TestHoodieMetricsConfig.java
b/hudi-common/src/test/java/org/apache/hudi/config/metrics/TestHoodieMetricsConfig.java
new file mode 100644
index 000000000000..f8cac40195f2
--- /dev/null
+++
b/hudi-common/src/test/java/org/apache/hudi/config/metrics/TestHoodieMetricsConfig.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.config.metrics;
+
+import org.junit.jupiter.api.Test;
+
+import static
org.apache.hudi.common.config.HoodieCommonConfig.META_SYNC_BASE_PATH_KEY;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+class TestHoodieMetricsConfig {
+ @Test
+ void testReturnsBasePathWhenSetViaBuilder() {
+ HoodieMetricsConfig config = HoodieMetricsConfig.newBuilder()
+ .withPath("primary/base/path")
+ .build();
+ assertEquals("primary/base/path", config.getBasePath());
+ }
+
+ @Test
+ void testReturnsMetaSyncPathWhenBasePathIsNotSet() {
+ HoodieMetricsConfig config = HoodieMetricsConfig.newBuilder().build();
+ config.setValue(META_SYNC_BASE_PATH_KEY, "base/path/set/during/sync");
+ assertEquals("base/path/set/during/sync", config.getBasePath());
+ }
+
+ @Test
+ void testReturnsNullWhenNeitherBasePathNorMetaSyncIsSet() {
+ HoodieMetricsConfig config = HoodieMetricsConfig.newBuilder().build();
+ assertNull(config.getBasePath());
+ }
+}
diff --git
a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncTool.java
b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncTool.java
index 9fd7d2fb5b99..f7f442ac008a 100644
---
a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncTool.java
+++
b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncTool.java
@@ -44,6 +44,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Properties;
+import static
org.apache.hudi.common.config.HoodieCommonConfig.META_SYNC_BASE_PATH_KEY;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -188,7 +189,7 @@ public class TestBigQuerySyncTool extends
HoodieCommonTestHarness {
TypedProperties typedProperties = new TypedProperties();
String location = "us-central1";
typedProperties.setProperty("hoodie.gcp.bigquery.sync.dataset_location",
location);
- typedProperties.setProperty("hoodie.datasource.meta.sync.base.path",
metaClient.getBasePath().toString());
+ typedProperties.setProperty(META_SYNC_BASE_PATH_KEY,
metaClient.getBasePath().toString());
try (MockedStatic<BigQueryOptions> mockedStatic =
mockStatic(BigQueryOptions.class)) {
BigQueryOptions.Builder builder = mock(BigQueryOptions.Builder.class);
BigQueryOptions.Builder builderWithLocation =
mock(BigQueryOptions.Builder.class);
diff --git
a/hudi-sync/hudi-datahub-sync/src/test/java/org/apache/hudi/sync/datahub/TestDataHubSyncTool.java
b/hudi-sync/hudi-datahub-sync/src/test/java/org/apache/hudi/sync/datahub/TestDataHubSyncTool.java
index ca5d7564a058..a62a27cd114b 100644
---
a/hudi-sync/hudi-datahub-sync/src/test/java/org/apache/hudi/sync/datahub/TestDataHubSyncTool.java
+++
b/hudi-sync/hudi-datahub-sync/src/test/java/org/apache/hudi/sync/datahub/TestDataHubSyncTool.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;
+import static
org.apache.hudi.common.config.HoodieCommonConfig.META_SYNC_BASE_PATH_KEY;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
@@ -57,7 +58,7 @@ class TestDataHubSyncTool extends HoodieCommonTestHarness {
void validatePropsConstructor() throws Exception {
initMetaClient();
TypedProperties typedProperties = new TypedProperties();
- typedProperties.setProperty("hoodie.datasource.meta.sync.base.path",
metaClient.getBasePath().toString());
+ typedProperties.setProperty(META_SYNC_BASE_PATH_KEY,
metaClient.getBasePath().toString());
assertDoesNotThrow(() -> {
HoodieSyncTool syncTool = new DataHubSyncTool(typedProperties);
syncTool.close();
diff --git
a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncConfig.java
b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncConfig.java
index d5e7f3dfae97..7fd76ef52390 100644
---
a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncConfig.java
+++
b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncConfig.java
@@ -46,6 +46,7 @@ import java.util.Properties;
import java.util.stream.Collectors;
import static org.apache.hudi.common.config.HoodieCommonConfig.BASE_PATH;
+import static
org.apache.hudi.common.config.HoodieCommonConfig.META_SYNC_BASE_PATH_KEY;
import static
org.apache.hudi.common.config.HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS;
import static org.apache.hudi.common.table.HoodieTableConfig.BASE_FILE_FORMAT;
import static org.apache.hudi.common.table.HoodieTableConfig.DATABASE_NAME;
@@ -67,7 +68,7 @@ public class HoodieSyncConfig extends HoodieConfig {
private static final Logger LOG =
LoggerFactory.getLogger(HoodieSyncConfig.class);
public static final ConfigProperty<String> META_SYNC_BASE_PATH =
ConfigProperty
- .key("hoodie.datasource.meta.sync.base.path")
+ .key(META_SYNC_BASE_PATH_KEY)
.defaultValue("")
.markAdvanced()
.withDocumentation("Base path of the hoodie table to sync");
diff --git
a/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/TestHoodieSyncTool.java
b/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/TestHoodieSyncTool.java
index 93876312550d..692c9b65e92f 100644
---
a/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/TestHoodieSyncTool.java
+++
b/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/TestHoodieSyncTool.java
@@ -25,6 +25,7 @@ import
org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hadoop.conf.Configuration;
import org.junit.jupiter.api.Test;
+import static
org.apache.hudi.common.config.HoodieCommonConfig.META_SYNC_BASE_PATH_KEY;
import static org.junit.jupiter.api.Assertions.assertEquals;
class TestHoodieSyncTool extends HoodieCommonTestHarness {
@@ -32,7 +33,7 @@ class TestHoodieSyncTool extends HoodieCommonTestHarness {
void testBuildMetaClient() throws Exception {
initMetaClient();
TypedProperties properties = new TypedProperties();
- properties.put("hoodie.datasource.meta.sync.base.path",
metaClient.getBasePath().toString());
+ properties.put(META_SYNC_BASE_PATH_KEY,
metaClient.getBasePath().toString());
HoodieSyncConfig syncConfig = new HoodieSyncConfig(properties, new
Configuration());
HoodieTableMetaClient actual = HoodieSyncTool.buildMetaClient(syncConfig);
assertEquals(actual.getBasePath(), metaClient.getBasePath());
