This is an automated email from the ASF dual-hosted git repository.
gian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 98e8c8ae705 fix: Emit sys metrics from k8s peons. (#19305)
98e8c8ae705 is described below
commit 98e8c8ae705ae58d3e838458869753e3ed6bf503
Author: Gian Merlino <[email protected]>
AuthorDate: Thu Apr 16 09:45:46 2026 -0700
fix: Emit sys metrics from k8s peons. (#19305)
Currently the SysMonitor and OshiSysMonitor are both skipped for peons,
on the logic that they are redundant to the MiddleManager (see #12802).
However, with the Kubernetes runner, we launch peons directly and we
want them to emit sys metrics.
To make it possible to differentiate these cases, this patch adds a
property "druid.peon.managed" that informs the MetricsModule that the
peon is managed by another process and should not emit metrics.
---
.../druid/indexing/overlord/ForkingTaskRunner.java | 2 ++
.../apache/druid/server/metrics/MetricsModule.java | 28 ++++++++++++++++++----
.../druid/server/metrics/MetricsModuleTest.java | 9 ++++---
3 files changed, 32 insertions(+), 7 deletions(-)
diff --git
a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ForkingTaskRunner.java
b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ForkingTaskRunner.java
index 6485d6d72da..1f7e4218107 100644
---
a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ForkingTaskRunner.java
+++
b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ForkingTaskRunner.java
@@ -62,6 +62,7 @@ import org.apache.druid.java.util.emitter.EmittingLogger;
import org.apache.druid.query.DruidMetrics;
import org.apache.druid.server.DruidNode;
import org.apache.druid.server.log.StartupLoggingConfig;
+import org.apache.druid.server.metrics.MetricsModule;
import org.apache.druid.server.metrics.MonitorsConfig;
import org.apache.druid.server.metrics.WorkerTaskCountStatsProvider;
import org.apache.druid.tasklogs.TaskLogPusher;
@@ -372,6 +373,7 @@ public class ForkingTaskRunner
command.addSystemProperty("druid.indexer.task.baseTaskDir",
storageSlot.getDirectory().getAbsolutePath());
command.addSystemProperty("druid.indexer.task.tmpStorageBytesPerTask",
storageSlot.getNumBytes());
+
command.addSystemProperty(MetricsModule.PROPERTY_PEON_MANAGED, true);
command.add("org.apache.druid.cli.Main");
command.add("internal");
diff --git
a/server/src/main/java/org/apache/druid/server/metrics/MetricsModule.java
b/server/src/main/java/org/apache/druid/server/metrics/MetricsModule.java
index c1425b81652..7935c1b0a3e 100644
--- a/server/src/main/java/org/apache/druid/server/metrics/MetricsModule.java
+++ b/server/src/main/java/org/apache/druid/server/metrics/MetricsModule.java
@@ -57,6 +57,7 @@ import org.apache.druid.query.ExecutorServiceMonitor;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
+import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
@@ -70,6 +71,14 @@ import java.util.stream.Collectors;
public class MetricsModule implements Module
{
public static final String MONITORING_PROPERTY_PREFIX = "druid.monitoring";
+
+ /**
+ * Property set by {@code ForkingTaskRunner} when launching peons as child
processes of a MiddleManager.
+ * When this property is "true", the peon is managed by a MiddleManager, and
system-level monitors
+ * should be skipped because the MiddleManager already emits them.
+ */
+ public static final String PROPERTY_PEON_MANAGED = "druid.peon.managed";
+
private static final Logger log = new Logger(MetricsModule.class);
public static void register(Binder binder, Class<? extends Monitor>
monitorClazz)
@@ -163,9 +172,9 @@ public class MetricsModule implements Module
@Provides
@ManageLifecycle
- public SysMonitor getSysMonitor(@Self Set<NodeRole> nodeRoles)
+ public SysMonitor getSysMonitor(@Self Set<NodeRole> nodeRoles, Properties
properties)
{
- if (nodeRoles.contains(NodeRole.PEON)) {
+ if (nodeRoles.contains(NodeRole.PEON) && isManagedPeon(properties)) {
return new NoopSysMonitor();
} else {
return new SysMonitor();
@@ -176,16 +185,27 @@ public class MetricsModule implements Module
@ManageLifecycle
public OshiSysMonitor getOshiSysMonitor(
@Self Set<NodeRole> nodeRoles,
- OshiSysMonitorConfig oshiSysConfig
+ OshiSysMonitorConfig oshiSysConfig,
+ Properties properties
)
{
- if (nodeRoles.contains(NodeRole.PEON)) {
+ if (nodeRoles.contains(NodeRole.PEON) && isManagedPeon(properties)) {
return new NoopOshiSysMonitor();
} else {
return new OshiSysMonitor(oshiSysConfig);
}
}
+ /**
+ * Returns true if this peon was launched by a MiddleManager (as opposed to
running standalone,
+ * such as under the Kubernetes task runner). MiddleManager-launched peons
share the host with
+ * the MiddleManager, so system-level metrics would be redundant.
+ */
+ private static boolean isManagedPeon(Properties properties)
+ {
+ return Boolean.parseBoolean(properties.getProperty(PROPERTY_PEON_MANAGED));
+ }
+
/**
* Checks if a monitor needs to be loaded on this service based on its node
role.
*/
diff --git
a/server/src/test/java/org/apache/druid/server/metrics/MetricsModuleTest.java
b/server/src/test/java/org/apache/druid/server/metrics/MetricsModuleTest.java
index 5b54f0d5ccf..5e95a141cd9 100644
---
a/server/src/test/java/org/apache/druid/server/metrics/MetricsModuleTest.java
+++
b/server/src/test/java/org/apache/druid/server/metrics/MetricsModuleTest.java
@@ -243,7 +243,9 @@ public class MetricsModuleTest
// Do not run the tests on ARM64. Sigar library has no binaries for ARM64
Assume.assumeFalse("aarch64".equals(CPU_ARCH));
- final Injector injector = createInjector(new Properties(),
ImmutableSet.of(NodeRole.PEON));
+ final Properties properties = new Properties();
+ properties.setProperty(MetricsModule.PROPERTY_PEON_MANAGED, "true");
+ final Injector injector = createInjector(properties,
ImmutableSet.of(NodeRole.PEON));
final SysMonitor sysMonitor = injector.getInstance(SysMonitor.class);
final ServiceEmitter emitter = Mockito.mock(ServiceEmitter.class);
sysMonitor.doMonitor(emitter);
@@ -269,8 +271,9 @@ public class MetricsModuleTest
@Test
public void testGetOshiSysMonitorViaInjector()
{
-
- final Injector injector = createInjector(new Properties(),
ImmutableSet.of(NodeRole.PEON));
+ final Properties properties = new Properties();
+ properties.setProperty(MetricsModule.PROPERTY_PEON_MANAGED, "true");
+ final Injector injector = createInjector(properties,
ImmutableSet.of(NodeRole.PEON));
final OshiSysMonitor sysMonitor =
injector.getInstance(OshiSysMonitor.class);
final ServiceEmitter emitter = Mockito.mock(ServiceEmitter.class);
sysMonitor.doMonitor(emitter);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
