This is an automated email from the ASF dual-hosted git repository.
jonvex pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 7cbb72786eb [HUDI-7641] Adding metadata enablement metrics and index
type metrics (#11053)
7cbb72786eb is described below
commit 7cbb72786eb2ff84938eb4b9c730b3a8d96d8c5a
Author: Sivabalan Narayanan <[email protected]>
AuthorDate: Mon May 6 07:59:58 2024 -0700
[HUDI-7641] Adding metadata enablement metrics and index type metrics
(#11053)
* Adding metadata enablement metrics
* fixing build failures
* Adding tests
---
.../apache/hudi/client/BaseHoodieWriteClient.java | 5 ++--
.../org/apache/hudi/metrics/HoodieMetrics.java | 16 +++++++++++
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 31 +++++++++++++++++++++-
.../apache/hudi/client/HoodieJavaWriteClient.java | 2 +-
.../apache/hudi/client/SparkRDDWriteClient.java | 26 +++++++++++++++---
5 files changed, 73 insertions(+), 7 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
index 6f41ec80bd6..003836d0cac 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
@@ -164,6 +164,7 @@ public abstract class BaseHoodieWriteClient<T, I, K, O>
extends BaseHoodieClient
super(context, writeConfig, timelineService);
this.index = createIndex(writeConfig);
this.upgradeDowngradeHelper = upgradeDowngradeHelper;
+ this.metrics.emitIndexTypeMetrics(config.getIndexType().ordinal());
}
protected abstract HoodieIndex<?, ?> createIndex(HoodieWriteConfig
writeConfig);
@@ -1230,7 +1231,7 @@ public abstract class BaseHoodieWriteClient<T, I, K, O>
extends BaseHoodieClient
this.txnManager.beginTransaction(ownerInstant, Option.empty());
try {
tryUpgrade(metaClient, instantTime);
- initMetadataTable(instantTime);
+ initMetadataTable(instantTime, metaClient);
} finally {
this.txnManager.endTransaction(ownerInstant);
}
@@ -1241,7 +1242,7 @@ public abstract class BaseHoodieWriteClient<T, I, K, O>
extends BaseHoodieClient
*
* @param instantTime current inflight instant time
*/
- protected void initMetadataTable(Option<String> instantTime) {
+ protected void initMetadataTable(Option<String> instantTime,
HoodieTableMetaClient metaClient) {
// by default do nothing.
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java
index 2a72c2236c6..9f1ef7a44c5 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java
@@ -399,6 +399,22 @@ public class HoodieMetrics {
}
}
+ public void emitMetadataEnablementMetrics(boolean isMetadataEnabled, boolean
isMetadataColStatsEnabled, boolean isMetadataBloomFilterEnabled,
+ boolean isMetadataRliEnabled) {
+ if (config.isMetricsOn()) {
+ metrics.registerGauge(getMetricsName("metadata", "isEnabled"),
isMetadataEnabled ? 1 : 0);
+ metrics.registerGauge(getMetricsName("metadata", "isColSatsEnabled"),
isMetadataColStatsEnabled ? 1 : 0);
+ metrics.registerGauge(getMetricsName("metadata",
"isBloomFilterEnabled"), isMetadataBloomFilterEnabled ? 1 : 0);
+ metrics.registerGauge(getMetricsName("metadata", "isRliEnabled"),
isMetadataRliEnabled ? 1 : 0);
+ }
+ }
+
+ public void emitIndexTypeMetrics(int indexTypeOrdinal) {
+ if (config.isMetricsOn()) {
+ metrics.registerGauge(getMetricsName("index", "type"), indexTypeOrdinal);
+ }
+ }
+
private Counter getCounter(Counter counter, String name) {
if (counter == null) {
return metrics.getRegistry().counter(name);
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
index 8c34931d93e..7b1b918535b 100755
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
@@ -23,6 +23,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
+import org.apache.hudi.index.HoodieIndex;
import com.codahale.metrics.Timer;
import org.junit.jupiter.api.AfterEach;
@@ -73,7 +74,7 @@ public class TestHoodieMetrics {
}
@Test
- public void testTimerCtx() throws InterruptedException {
+ public void testTimerCtxandGauges() throws InterruptedException {
Random rand = new Random();
// Index metrics
Timer.Context timer = hoodieMetrics.getIndexCtx();
@@ -83,6 +84,34 @@ public class TestHoodieMetrics {
long msec =
(Long)metrics.getRegistry().getGauges().get(metricName).getValue();
assertTrue(msec > 0);
+ // test index type
+ metricName = hoodieMetrics.getMetricsName("index", "type");
+ for (HoodieIndex.IndexType indexType: HoodieIndex.IndexType.values()) {
+ hoodieMetrics.emitIndexTypeMetrics(indexType.ordinal());
+ long indexTypeOrdinal =
(Long)metrics.getRegistry().getGauges().get(metricName).getValue();
+ assertEquals(indexTypeOrdinal, indexType.ordinal());
+ }
+
+ // test metadata enablement metrics
+ metricName = hoodieMetrics.getMetricsName("metadata", "isEnabled");
+ String colStatsMetricName = hoodieMetrics.getMetricsName("metadata",
"isColSatsEnabled");
+ String bloomFilterMetricName = hoodieMetrics.getMetricsName("metadata",
"isBloomFilterEnabled");
+ String rliMetricName = hoodieMetrics.getMetricsName("metadata",
"isRliEnabled");
+ Boolean[] boolValues = new Boolean[]{true, false};
+ for (Boolean mdt: boolValues) {
+ for (Boolean colStats : boolValues) {
+ for (Boolean bloomFilter : boolValues) {
+ for (Boolean rli : boolValues) {
+ hoodieMetrics.emitMetadataEnablementMetrics(mdt, colStats,
bloomFilter, rli);
+ assertEquals(mdt ? 1L : 0L,
metrics.getRegistry().getGauges().get(metricName).getValue());
+ assertEquals(colStats ? 1L : 0L,
metrics.getRegistry().getGauges().get(colStatsMetricName).getValue());
+ assertEquals(bloomFilter ? 1L : 0L,
metrics.getRegistry().getGauges().get(bloomFilterMetricName).getValue());
+ assertEquals(rli ? 1L : 0L,
metrics.getRegistry().getGauges().get(rliMetricName).getValue());
+ }
+ }
+ }
+ }
+
// Rollback metrics
timer = hoodieMetrics.getRollbackCtx();
Thread.sleep(5); // Ensure timer duration is > 0
diff --git
a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaWriteClient.java
b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaWriteClient.java
index c07fdf3afcd..596767e8cc6 100644
---
a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaWriteClient.java
+++
b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaWriteClient.java
@@ -209,7 +209,7 @@ public class HoodieJavaWriteClient<T> extends
}
@Override
- protected void initMetadataTable(Option<String> instantTime) {
+ protected void initMetadataTable(Option<String> instantTime,
HoodieTableMetaClient metaClient) {
// Initialize Metadata Table to make sure it's bootstrapped _before_ the
operation,
// if it didn't exist before
// See https://issues.apache.org/jira/browse/HUDI-3343 for more details
diff --git
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java
index 4ea9f9c07d0..4997faebe70 100644
---
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java
+++
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java
@@ -29,6 +29,7 @@ import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.model.WriteOperationType;
+import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.Option;
@@ -39,6 +40,7 @@ import org.apache.hudi.hadoop.fs.HoodieWrapperFileSystem;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.index.SparkHoodieIndexFactory;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
+import org.apache.hudi.metadata.MetadataPartitionType;
import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter;
import org.apache.hudi.metrics.DistributedRegistry;
import org.apache.hudi.table.BulkInsertPartitioner;
@@ -272,11 +274,11 @@ public class SparkRDDWriteClient<T> extends
}
@Override
- protected void initMetadataTable(Option<String> instantTime) {
+ protected void initMetadataTable(Option<String> instantTime,
HoodieTableMetaClient metaClient) {
// Initialize Metadata Table to make sure it's bootstrapped _before_ the
operation,
// if it didn't exist before
// See https://issues.apache.org/jira/browse/HUDI-3343 for more details
- initializeMetadataTable(instantTime);
+ initializeMetadataTable(instantTime, metaClient);
}
/**
@@ -285,10 +287,28 @@ public class SparkRDDWriteClient<T> extends
*
* @param inFlightInstantTimestamp - The in-flight action responsible for
the metadata table initialization
*/
- private void initializeMetadataTable(Option<String>
inFlightInstantTimestamp) {
+ private void initializeMetadataTable(Option<String>
inFlightInstantTimestamp, HoodieTableMetaClient metaClient) {
if (!config.isMetadataTableEnabled()) {
return;
}
+ // if metadata table is enabled, emit enablement metrics
+ HoodieTableConfig tableConfig = metaClient.getTableConfig();
+ if (tableConfig.isMetadataTableAvailable()) {
+ // if metadata table is available, lets emit partitions of interest
+ boolean isMetadataColStatsAvailable = false;
+ boolean isMetadataBloomFilterAvailable = false;
+ boolean isMetadataRliAvailable = false;
+ if
(tableConfig.getMetadataPartitions().contains(MetadataPartitionType.COLUMN_STATS.getPartitionPath()))
{
+ isMetadataColStatsAvailable = true;
+ }
+ if
(tableConfig.getMetadataPartitions().contains(MetadataPartitionType.BLOOM_FILTERS.getPartitionPath()))
{
+ isMetadataBloomFilterAvailable = true;
+ }
+ if
(tableConfig.getMetadataPartitions().contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath()))
{
+ isMetadataRliAvailable = true;
+ }
+ metrics.emitMetadataEnablementMetrics(true, isMetadataColStatsAvailable,
isMetadataBloomFilterAvailable, isMetadataRliAvailable);
+ }
try (HoodieTableMetadataWriter writer =
SparkHoodieBackedTableMetadataWriter.create(
context.getStorageConf(), config, context, inFlightInstantTimestamp)) {
