This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new a4d311ceed Fixed major compaction metrics (#5074)
a4d311ceed is described below
commit a4d311ceed1cee74e96399a1b21d0ece43fdf7c9
Author: Dave Marion <[email protected]>
AuthorDate: Tue Nov 26 08:05:42 2024 -0500
Fixed major compaction metrics (#5074)
Renamed compaction metrics, removed MAJC_QUEUED and
MAJC_RUNNING, added new MAJC_IN_PROGRESS metric.
Closes #5062
---
.../org/apache/accumulo/core/metrics/Metric.java | 28 +++++++++++-----------
.../org/apache/accumulo/compactor/Compactor.java | 18 +++++++++++---
.../coordinator/CompactionCoordinator.java | 8 -------
.../tserver/metrics/TabletServerMetrics.java | 6 ++---
4 files changed, 32 insertions(+), 28 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java
b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java
index 4ef5afbce4..6b71ed2f49 100644
--- a/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/Metric.java
@@ -31,14 +31,20 @@ public enum Metric {
MetricCategory.GENERAL_SERVER),
// Compactor Metrics
+ COMPACTOR_MAJC_IN_PROGRESS("accumulo.compaction.majc.in_progress",
MetricType.GAUGE,
+ "Indicator of whether a compaction is in-progress (value: 1) or not
(value: 0). An"
+ + " in-progress compaction could also be stuck.",
+ MetricCategory.COMPACTION),
COMPACTOR_MAJC_STUCK("accumulo.compaction.majc.stuck",
MetricType.LONG_TASK_TIMER,
"Number and duration of stuck major compactions.",
MetricCategory.COMPACTION),
- COMPACTOR_ENTRIES_READ("accumulo.compactor.entries.read",
MetricType.FUNCTION_COUNTER,
- "Number of entries read by all compactions that have run on this
compactor.",
- MetricCategory.COMPACTOR),
- COMPACTOR_ENTRIES_WRITTEN("accumulo.compactor.entries.written",
MetricType.FUNCTION_COUNTER,
- "Number of entries written by all compactions that have run on this
compactor.",
- MetricCategory.COMPACTOR),
+ COMPACTOR_MINC_STUCK("accumulo.compaction.minc.stuck",
MetricType.LONG_TASK_TIMER,
+ "Number and duration of stuck minor compactions.",
MetricCategory.COMPACTION),
+ COMPACTOR_ENTRIES_READ("accumulo.compaction.entries.read",
MetricType.FUNCTION_COUNTER,
+ "Number of entries read by all compactions that have run on this
compactor (majc) or tserver (minc).",
+ MetricCategory.COMPACTION),
+ COMPACTOR_ENTRIES_WRITTEN("accumulo.compaction.entries.written",
MetricType.FUNCTION_COUNTER,
+ "Number of entries written by all compactions that have run on this
compactor (majc) or tserver (minc).",
+ MetricCategory.COMPACTION),
COMPACTOR_JOB_PRIORITY_QUEUES("accumulo.compaction.queue.count",
MetricType.GAUGE,
"Number of priority queues for compaction jobs.",
MetricCategory.COMPACTION),
COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH("accumulo.compaction.queue.length",
MetricType.GAUGE,
@@ -188,14 +194,8 @@ public enum Metric {
"Number of scan threads that have no associated client session.",
MetricCategory.SCAN),
// Major Compaction Metrics
- MAJC_QUEUED("accumulo.tserver.compactions.majc.queued", MetricType.GAUGE,
- "Number of queued major compactions. The compaction service information
is in a tag: `id={i|e}_{compactionServiceName}_{executor_name}`.",
- MetricCategory.COMPACTION),
- MAJC_RUNNING("accumulo.tserver.compactions.majc.running", MetricType.GAUGE,
- "Number of running major compactions. The compaction service information
is in a tag: `id={i|e}_{compactionServiceName}_{executor_name}`.",
- MetricCategory.COMPACTION),
- MAJC_PAUSED("accumulo.tserver.compactions.majc.paused", MetricType.COUNTER,
- "Number of paused major compactions.", MetricCategory.COMPACTION),
+ MAJC_PAUSED("accumulo.compaction.majc.paused", MetricType.COUNTER,
+ "Number of paused major compactions.", MetricCategory.COMPACTOR),
// Minor Compaction Metrics
MINC_QUEUED("accumulo.tserver.compactions.minc.queued", MetricType.TIMER,
diff --git
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index 463937e9ba..3eabcedbf6 100644
---
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.compactor;
import static
com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_ENTRIES_READ;
import static
org.apache.accumulo.core.metrics.Metric.COMPACTOR_ENTRIES_WRITTEN;
+import static
org.apache.accumulo.core.metrics.Metric.COMPACTOR_MAJC_IN_PROGRESS;
import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_MAJC_STUCK;
import static org.apache.accumulo.core.util.LazySingletons.RANDOM;
@@ -132,6 +133,7 @@ import com.google.common.base.Preconditions;
import com.google.common.net.HostAndPort;
import io.micrometer.core.instrument.FunctionCounter;
+import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.LongTaskTimer;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tag;
@@ -186,16 +188,26 @@ public class Compactor extends AbstractServer implements
MetricsProducer, Compac
return FileCompactor.getTotalEntriesWritten();
}
+ private long compactionInProgress() {
+ return compactionRunning.get() ? 1 : 0;
+ }
+
@Override
public void registerMetrics(MeterRegistry registry) {
super.registerMetrics(registry);
FunctionCounter.builder(COMPACTOR_ENTRIES_READ.getName(), this,
Compactor::getTotalEntriesRead)
-
.description(COMPACTOR_ENTRIES_READ.getDescription()).register(registry);
+ .description(COMPACTOR_ENTRIES_READ.getDescription())
+ .tags(List.of(Tag.of("queue.id",
this.getResourceGroup()))).register(registry);
FunctionCounter
.builder(COMPACTOR_ENTRIES_WRITTEN.getName(), this,
Compactor::getTotalEntriesWritten)
-
.description(COMPACTOR_ENTRIES_WRITTEN.getDescription()).register(registry);
+ .description(COMPACTOR_ENTRIES_WRITTEN.getDescription())
+ .tags(List.of(Tag.of("queue.id",
this.getResourceGroup()))).register(registry);
+ Gauge.builder(COMPACTOR_MAJC_IN_PROGRESS.getName(), this,
Compactor::compactionInProgress)
+ .description(COMPACTOR_MAJC_IN_PROGRESS.getDescription())
+ .tags(List.of(Tag.of("queue.id",
this.getResourceGroup()))).register(registry);
LongTaskTimer timer = LongTaskTimer.builder(COMPACTOR_MAJC_STUCK.getName())
- .description(COMPACTOR_MAJC_STUCK.getDescription()).register(registry);
+ .description(COMPACTOR_MAJC_STUCK.getDescription())
+ .tags(List.of(Tag.of("queue.id",
this.getResourceGroup()))).register(registry);
CompactionWatcher.setTimer(timer);
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
index 4e49b825b0..e06229c21b 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
@@ -26,8 +26,6 @@ import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SELECTED;
-import static org.apache.accumulo.core.metrics.Metric.MAJC_QUEUED;
-import static org.apache.accumulo.core.metrics.Metric.MAJC_RUNNING;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -149,7 +147,6 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
import com.google.common.net.HostAndPort;
-import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
public class CompactionCoordinator
@@ -645,11 +642,6 @@ public class CompactionCoordinator
@Override
public void registerMetrics(MeterRegistry registry) {
- Gauge.builder(MAJC_QUEUED.getName(), jobQueues,
CompactionJobQueues::getQueuedJobCount)
- .description(MAJC_QUEUED.getDescription()).register(registry);
- Gauge.builder(MAJC_RUNNING.getName(), this,
CompactionCoordinator::getNumRunningCompactions)
- .description(MAJC_RUNNING.getDescription()).register(registry);
-
queueMetrics.registerMetrics(registry);
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java
index a70112f5f5..6c3382df0d 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java
@@ -20,7 +20,7 @@ package org.apache.accumulo.tserver.metrics;
import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_ENTRIES_READ;
import static
org.apache.accumulo.core.metrics.Metric.COMPACTOR_ENTRIES_WRITTEN;
-import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_MAJC_STUCK;
+import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_MINC_STUCK;
import static org.apache.accumulo.core.metrics.Metric.TSERVER_ENTRIES;
import static org.apache.accumulo.core.metrics.Metric.TSERVER_HOLD;
import static org.apache.accumulo.core.metrics.Metric.TSERVER_INGEST_BYTES;
@@ -72,8 +72,8 @@ public class TabletServerMetrics implements MetricsProducer {
.builder(COMPACTOR_ENTRIES_WRITTEN.getName(), this,
TabletServerMetrics::getTotalEntriesWritten)
.description(COMPACTOR_ENTRIES_WRITTEN.getDescription()).register(registry);
- LongTaskTimer timer = LongTaskTimer.builder(COMPACTOR_MAJC_STUCK.getName())
- .description(COMPACTOR_MAJC_STUCK.getDescription()).register(registry);
+ LongTaskTimer timer = LongTaskTimer.builder(COMPACTOR_MINC_STUCK.getName())
+ .description(COMPACTOR_MINC_STUCK.getDescription()).register(registry);
CompactionWatcher.setTimer(timer);
Gauge
