This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch release-0.13.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit eb6bceefa5e6b8e7be77e5785dd0d009805cb37c Author: Y Ethan Guo <[email protected]> AuthorDate: Sat Feb 4 13:49:07 2023 -0800 [HUDI-5700] Annotate config classes for docs generation (#7853) This PR annotates the missing config classes so that the configs in these classes are properly processed and populated on our "All Configurations" page. --- .../apache/hudi/common/config/ConfigGroups.java | 1 + .../hudi/gcp/bigquery/BigQuerySyncConfig.java | 8 ++++++++ .../sync/datahub/config/DataHubSyncConfig.java | 24 ++++++++++++++-------- .../java/org/apache/hudi/hive/HiveSyncConfig.java | 8 ++++++++ .../hive/replication/GlobalHiveSyncConfig.java | 8 ++++++++ .../apache/hudi/sync/common/HoodieSyncConfig.java | 8 ++++++++ 6 files changed, 49 insertions(+), 8 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java index 0488401f10a..6fe8e5002a8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java @@ -32,6 +32,7 @@ public class ConfigGroups { SPARK_DATASOURCE("Spark Datasource Configs"), FLINK_SQL("Flink Sql Configs"), WRITE_CLIENT("Write Client Configs"), + META_SYNC("Metastore and Catalog Sync Configs"), METRICS("Metrics Configs"), RECORD_PAYLOAD("Record Payload Config"), KAFKA_CONNECT("Kafka Connect Configs"), diff --git a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncConfig.java b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncConfig.java index 52b3d3b74e5..8c04c0fa136 100644 --- a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncConfig.java +++ b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncConfig.java @@ -19,6 +19,8 @@ package org.apache.hudi.gcp.bigquery; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.sync.common.HoodieSyncConfig; @@ -26,12 +28,18 @@ import org.apache.hudi.sync.common.HoodieSyncConfig; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParametersDelegate; +import javax.annotation.concurrent.Immutable; + import java.io.Serializable; import java.util.Properties; /** * Configs needed to sync data into BigQuery. */ +@Immutable +@ConfigClassProperty(name = "BigQuery Sync Configs", + groupName = ConfigGroups.Names.META_SYNC, + description = "Configurations used by the Hudi to sync metadata to Google BigQuery.") public class BigQuerySyncConfig extends HoodieSyncConfig implements Serializable { public static final ConfigProperty<String> BIGQUERY_SYNC_PROJECT_ID = ConfigProperty diff --git a/hudi-sync/hudi-datahub-sync/src/main/java/org/apache/hudi/sync/datahub/config/DataHubSyncConfig.java b/hudi-sync/hudi-datahub-sync/src/main/java/org/apache/hudi/sync/datahub/config/DataHubSyncConfig.java index e4fa695ecef..d9a3f00e46b 100644 --- a/hudi-sync/hudi-datahub-sync/src/main/java/org/apache/hudi/sync/datahub/config/DataHubSyncConfig.java +++ b/hudi-sync/hudi-datahub-sync/src/main/java/org/apache/hudi/sync/datahub/config/DataHubSyncConfig.java @@ -19,6 +19,8 @@ package org.apache.hudi.sync.datahub.config; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.util.ReflectionUtils; @@ -28,11 +30,17 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.ParametersDelegate; import datahub.client.rest.RestEmitter; +import javax.annotation.concurrent.Immutable; + import java.util.Properties; import static org.apache.hudi.sync.datahub.config.HoodieDataHubDatasetIdentifier.DEFAULT_DATAHUB_ENV; import static org.apache.hudi.sync.datahub.config.HoodieDataHubDatasetIdentifier.DEFAULT_HOODIE_DATAHUB_PLATFORM_NAME; +@Immutable +@ConfigClassProperty(name = "DataHub Sync Configs", + groupName = ConfigGroups.Names.META_SYNC, + description = "Configurations used by the Hudi to sync metadata to DataHub.") public class DataHubSyncConfig extends HoodieSyncConfig { public static final ConfigProperty<String> META_SYNC_DATAHUB_DATASET_IDENTIFIER_CLASS = ConfigProperty @@ -56,15 +64,15 @@ public class DataHubSyncConfig extends HoodieSyncConfig { .withDocumentation("Pluggable class to supply a DataHub REST emitter to connect to the DataHub instance. This overwrites other emitter configs."); public static final ConfigProperty<String> META_SYNC_DATAHUB_DATAPLATFORM_NAME = ConfigProperty - .key("hoodie.meta.sync.datahub.dataplatform.name") - .defaultValue(DEFAULT_HOODIE_DATAHUB_PLATFORM_NAME) - .withDocumentation("String used to represent Hudi when creating its corresponding DataPlatform entity " - + "within Datahub"); + .key("hoodie.meta.sync.datahub.dataplatform.name") + .defaultValue(DEFAULT_HOODIE_DATAHUB_PLATFORM_NAME) + .withDocumentation("String used to represent Hudi when creating its corresponding DataPlatform entity " + + "within Datahub"); public static final ConfigProperty<String> META_SYNC_DATAHUB_DATASET_ENV = ConfigProperty - .key("hoodie.meta.sync.datahub.dataset.env") - .defaultValue(DEFAULT_DATAHUB_ENV.name()) - .withDocumentation("Environment to use when pushing entities to Datahub"); + .key("hoodie.meta.sync.datahub.dataset.env") + .defaultValue(DEFAULT_DATAHUB_ENV.name()) + .withDocumentation("Environment to use when pushing entities to Datahub"); public final HoodieDataHubDatasetIdentifier datasetIdentifier; @@ -102,7 +110,7 @@ public class DataHubSyncConfig extends HoodieSyncConfig { public String emitterSupplierClass; @Parameter(names = {"--data-platform-name"}, description = "String used to represent Hudi when creating its " - + "corresponding DataPlatform entity within Datahub") + + "corresponding DataPlatform entity within Datahub") public String dataPlatformName; @Parameter(names = {"--dataset-env"}, description = "Which Datahub Environment to use when pushing entities") diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java index db0fe820942..9d23b88cc1b 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java @@ -18,6 +18,8 @@ package org.apache.hudi.hive; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.util.ValidationUtils; @@ -28,11 +30,17 @@ import com.beust.jcommander.ParametersDelegate; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import javax.annotation.concurrent.Immutable; + import java.util.Properties; /** * Configs needed to sync data into the Hive Metastore. */ +@Immutable +@ConfigClassProperty(name = "Hive Sync Configs", + groupName = ConfigGroups.Names.META_SYNC, + description = "Configurations used by the Hudi to sync metadata to Hive Metastore.") public class HiveSyncConfig extends HoodieSyncConfig { /* diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/replication/GlobalHiveSyncConfig.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/replication/GlobalHiveSyncConfig.java index dc4355c57c3..0f0a99bbbb1 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/replication/GlobalHiveSyncConfig.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/replication/GlobalHiveSyncConfig.java @@ -18,6 +18,8 @@ package org.apache.hudi.hive.replication; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.hive.HiveSyncConfig; @@ -26,8 +28,14 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.ParametersDelegate; import org.apache.hadoop.conf.Configuration; +import javax.annotation.concurrent.Immutable; + import java.util.Properties; +@Immutable +@ConfigClassProperty(name = "Global Hive Sync Configs", + groupName = ConfigGroups.Names.META_SYNC, + description = "Global replication configurations used by the Hudi to sync metadata to Hive Metastore.") public class GlobalHiveSyncConfig extends HiveSyncConfig { public static final ConfigProperty<String> META_SYNC_GLOBAL_REPLICATE_TIMESTAMP = ConfigProperty diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncConfig.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncConfig.java index c0d1075c779..eb9affe3085 100644 --- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncConfig.java +++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncConfig.java @@ -18,6 +18,8 @@ package org.apache.hudi.sync.common; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.common.config.HoodieMetadataConfig; @@ -35,6 +37,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import javax.annotation.concurrent.Immutable; + import java.util.Comparator; import java.util.List; import java.util.Properties; @@ -51,6 +55,10 @@ import static org.apache.hudi.common.table.HoodieTableConfig.URL_ENCODE_PARTITIO /** * Configs needed to sync data into external meta stores, catalogs, etc. */ +@Immutable +@ConfigClassProperty(name = "Metadata Sync Configs", + groupName = ConfigGroups.Names.META_SYNC, + description = "Configurations used by the Hudi to sync metadata to external metastores and catalogs.") public class HoodieSyncConfig extends HoodieConfig { private static final Logger LOG = LogManager.getLogger(HoodieSyncConfig.class);
