codope commented on code in PR #5207:
URL: https://github.com/apache/hudi/pull/5207#discussion_r844982369
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java:
##########
@@ -2480,41 +2494,42 @@ protected void setDefaults() {
HoodieLayoutConfig.newBuilder().fromProperties(writeConfig.getProps()).build());
writeConfig.setDefaultValue(TIMELINE_LAYOUT_VERSION_NUM,
String.valueOf(TimelineLayoutVersion.CURR_VERSION));
- autoAdjustConfigsForConcurrencyMode();
- }
-
- private void autoAdjustConfigsForConcurrencyMode() {
- boolean isMetadataTableEnabled =
writeConfig.getBoolean(HoodieMetadataConfig.ENABLE);
+ // isLockProviderPropertySet must be fetched before setting defaults of
HoodieLockConfig
final TypedProperties writeConfigProperties = writeConfig.getProps();
final boolean isLockProviderPropertySet =
writeConfigProperties.containsKey(HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME)
||
writeConfigProperties.containsKey(HoodieLockConfig.LOCK_PROVIDER_CLASS_PROP);
-
- if (!isLockConfigSet) {
- HoodieLockConfig.Builder lockConfigBuilder =
HoodieLockConfig.newBuilder().fromProperties(writeConfig.getProps());
- writeConfig.setDefault(lockConfigBuilder.build());
- }
-
- if (isMetadataTableEnabled) {
- // When metadata table is enabled, optimistic concurrency control must
be used for
- // single writer with async table services.
- // Async table services can update the metadata table and a lock
provider is
- // needed to guard against any concurrent table write operations. If
user has
- // not configured any lock provider, let's use the InProcess lock
provider.
- boolean areTableServicesEnabled =
writeConfig.areTableServicesEnabled();
- boolean areAsyncTableServicesEnabled =
writeConfig.areAnyTableServicesAsync();
-
- if (!isLockProviderPropertySet && areTableServicesEnabled &&
areAsyncTableServicesEnabled) {
- // This is targeted at Single writer with async table services
- // If user does not set the lock provider, likely that the
concurrency mode is not set either
- // Override the configs for metadata table
- writeConfig.setValue(WRITE_CONCURRENCY_MODE.key(),
- WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL.value());
- writeConfig.setValue(HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME.key(),
- InProcessLockProvider.class.getName());
- LOG.info(String.format("Automatically set %s=%s and %s=%s since user
has not set the "
- + "lock provider for single writer with async table
services",
- WRITE_CONCURRENCY_MODE.key(),
WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL.value(),
- HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME.key(),
InProcessLockProvider.class.getName()));
+ writeConfig.setDefaultOnCondition(!isLockConfigSet,
+
HoodieLockConfig.newBuilder().fromProperties(writeConfig.getProps()).build());
+
+ autoAdjustConfigsForConcurrencyMode(isLockProviderPropertySet);
+ }
+
+ private void autoAdjustConfigsForConcurrencyMode(boolean
isLockProviderPropertySet) {
+ if (writeConfig.isAutoAdjustLockConfigs()) {
+ // auto adjustment is required only for deltastreamer and spark
streaming where async table services can be executed in the same JVM.
+ boolean isMetadataTableEnabled =
writeConfig.getBoolean(HoodieMetadataConfig.ENABLE);
+
+ if (isMetadataTableEnabled) {
Review Comment:
Is it really necessary to guard by metadata enabled? Even with metadata
disabled, async table service can still run into confict with deltastreamer
continous mode as we saw in HUDI-2774
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
