This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 5547a2917f [#8593]improve(server): expose more server metrics (#8670)
5547a2917f is described below
commit 5547a2917f8f4c411c2afd41853901c56cf8ac6e
Author: Junda Yang <[email protected]>
AuthorDate: Wed Sep 24 00:28:37 2025 -0700
[#8593]improve(server): expose more server metrics (#8670)
### What changes were proposed in this pull request?
Expose these jetty server thread pool metrics -
min-threads
max-threads
total-threads
busy-threads
idle-threads
queued-requests
### Why are the changes needed?
This PR exposes more metrics about jetty server. We can observe if the
server is overloaded by checking idle-threads, busy-threads, as well as
queued requests.
Fix: #8593
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
1. Unit test added
2. Tested in our staging / production deployment
---
.../org/apache/gravitino/metrics/MetricNames.java | 7 +-
.../server/web/HttpServerMetricsSource.java | 17 +-
.../server/web/TestHttpServerMetricsSource.java | 275 +++++++++++++++++++++
3 files changed, 296 insertions(+), 3 deletions(-)
diff --git a/core/src/main/java/org/apache/gravitino/metrics/MetricNames.java
b/core/src/main/java/org/apache/gravitino/metrics/MetricNames.java
index b3c8271942..a6c9d16db8 100644
--- a/core/src/main/java/org/apache/gravitino/metrics/MetricNames.java
+++ b/core/src/main/java/org/apache/gravitino/metrics/MetricNames.java
@@ -21,11 +21,16 @@ package org.apache.gravitino.metrics;
public class MetricNames {
public static final String HTTP_PROCESS_DURATION =
"http-request-duration-seconds";
- public static final String SERVER_IDLE_THREAD_NUM =
"http-server.idle-thread.num";
public static final String DATASOURCE_ACTIVE_CONNECTIONS =
"datasource.active-connections";
public static final String DATASOURCE_IDLE_CONNECTIONS =
"datasource.idle-connections";
public static final String DATASOURCE_MAX_CONNECTIONS =
"datasource.max-connections";
public static final String FILESYSTEM_CACHE = "filesystem-cache";
+ public static final String SERVER_BUSY_THREAD_NUM =
"http-server.busy-thread.num";
+ public static final String SERVER_IDLE_THREAD_NUM =
"http-server.idle-thread.num";
+ public static final String SERVER_QUEUED_REQUEST_NUM =
"http-server.queued-request.num";
+ public static final String SERVER_TOTAL_THREAD_NUM =
"http-server.total-thread.num";
+ public static final String SERVER_MIN_THREAD_NUM =
"http-server.min-thread.num";
+ public static final String SERVER_MAX_THREAD_NUM =
"http-server.max-thread.num";
private MetricNames() {}
}
diff --git
a/server-common/src/main/java/org/apache/gravitino/server/web/HttpServerMetricsSource.java
b/server-common/src/main/java/org/apache/gravitino/server/web/HttpServerMetricsSource.java
index 013c5fe7c0..437cb30090 100644
---
a/server-common/src/main/java/org/apache/gravitino/server/web/HttpServerMetricsSource.java
+++
b/server-common/src/main/java/org/apache/gravitino/server/web/HttpServerMetricsSource.java
@@ -25,6 +25,8 @@ import
com.codahale.metrics.jersey2.InstrumentedResourceMethodApplicationListene
import java.util.concurrent.TimeUnit;
import org.apache.gravitino.metrics.MetricNames;
import org.apache.gravitino.metrics.source.MetricsSource;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.eclipse.jetty.util.thread.ThreadPool;
import org.glassfish.jersey.server.ResourceConfig;
public class HttpServerMetricsSource extends MetricsSource {
@@ -38,7 +40,18 @@ public class HttpServerMetricsSource extends MetricsSource {
() ->
new SlidingTimeWindowArrayReservoir(
getTimeSlidingWindowSeconds(), TimeUnit.SECONDS)));
- registerGauge(
- MetricNames.SERVER_IDLE_THREAD_NUM, () ->
server.getThreadPool().getIdleThreads());
+
+ // Register QueuedThreadPool specific metrics with instance checks
+ ThreadPool threadPool = server.getThreadPool();
+ registerGauge(MetricNames.SERVER_IDLE_THREAD_NUM, () ->
threadPool.getIdleThreads());
+ registerGauge(MetricNames.SERVER_TOTAL_THREAD_NUM, () ->
threadPool.getThreads());
+
+ if (threadPool instanceof QueuedThreadPool) {
+ QueuedThreadPool queuedThreadPool = (QueuedThreadPool) threadPool;
+ registerGauge(MetricNames.SERVER_BUSY_THREAD_NUM, () ->
queuedThreadPool.getBusyThreads());
+ registerGauge(MetricNames.SERVER_QUEUED_REQUEST_NUM, () ->
queuedThreadPool.getQueueSize());
+ registerGauge(MetricNames.SERVER_MIN_THREAD_NUM, () ->
queuedThreadPool.getMinThreads());
+ registerGauge(MetricNames.SERVER_MAX_THREAD_NUM, () ->
queuedThreadPool.getMaxThreads());
+ }
}
}
diff --git
a/server-common/src/test/java/org/apache/gravitino/server/web/TestHttpServerMetricsSource.java
b/server-common/src/test/java/org/apache/gravitino/server/web/TestHttpServerMetricsSource.java
new file mode 100644
index 0000000000..7ed99b8219
--- /dev/null
+++
b/server-common/src/test/java/org/apache/gravitino/server/web/TestHttpServerMetricsSource.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.server.web;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.MetricRegistry;
+import
com.codahale.metrics.jersey2.InstrumentedResourceMethodApplicationListener;
+import org.apache.gravitino.metrics.MetricNames;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.eclipse.jetty.util.thread.ThreadPool;
+import org.glassfish.jersey.server.ResourceConfig;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+public class TestHttpServerMetricsSource {
+
+ private JettyServer mockJettyServer;
+ private ThreadPool mockThreadPool;
+ private ResourceConfig mockResourceConfig;
+
+ private HttpServerMetricsSource metricsSource;
+
+ @BeforeEach
+ void setUp() {
+ mockJettyServer = mock(JettyServer.class);
+ mockThreadPool = mock(ThreadPool.class);
+ mockResourceConfig = mock(ResourceConfig.class);
+ }
+
+ @Test
+ void testConstructorWithQueuedThreadPool() {
+ // Arrange - Use a real QueuedThreadPool instance
+ QueuedThreadPool realQueuedThreadPool = new QueuedThreadPool(20, 1, 60000);
+ when(mockJettyServer.getThreadPool()).thenReturn(realQueuedThreadPool);
+
+ // Act
+ metricsSource = new HttpServerMetricsSource("test-server",
mockResourceConfig, mockJettyServer);
+
+ // Assert
+ assertNotNull(metricsSource);
+ assertNotNull(metricsSource.getMetricRegistry());
+
+ // Verify ResourceConfig registration
+
verify(mockResourceConfig).register(any(InstrumentedResourceMethodApplicationListener.class));
+
+ // Verify basic thread pool metrics are registered
+ MetricRegistry registry = metricsSource.getMetricRegistry();
+
assertTrue(registry.getGauges().containsKey(MetricNames.SERVER_IDLE_THREAD_NUM));
+
assertTrue(registry.getGauges().containsKey(MetricNames.SERVER_TOTAL_THREAD_NUM));
+
+ // Verify QueuedThreadPool specific metrics are registered
+
assertTrue(registry.getGauges().containsKey(MetricNames.SERVER_BUSY_THREAD_NUM));
+
assertTrue(registry.getGauges().containsKey(MetricNames.SERVER_QUEUED_REQUEST_NUM));
+
assertTrue(registry.getGauges().containsKey(MetricNames.SERVER_MIN_THREAD_NUM));
+
assertTrue(registry.getGauges().containsKey(MetricNames.SERVER_MAX_THREAD_NUM));
+ }
+
+ @Test
+ void testConstructorWithNonQueuedThreadPool() {
+ // Arrange - Use a mock ThreadPool that is not QueuedThreadPool
+ when(mockJettyServer.getThreadPool()).thenReturn(mockThreadPool);
+ when(mockThreadPool.getIdleThreads()).thenReturn(3);
+ when(mockThreadPool.getThreads()).thenReturn(8);
+
+ // Act
+ metricsSource = new HttpServerMetricsSource("test-server",
mockResourceConfig, mockJettyServer);
+
+ // Assert
+ assertNotNull(metricsSource);
+ assertNotNull(metricsSource.getMetricRegistry());
+
+ // Verify ResourceConfig registration
+
verify(mockResourceConfig).register(any(InstrumentedResourceMethodApplicationListener.class));
+
+ // Verify only basic thread pool metrics are registered
+ MetricRegistry registry = metricsSource.getMetricRegistry();
+
assertTrue(registry.getGauges().containsKey(MetricNames.SERVER_IDLE_THREAD_NUM));
+
assertTrue(registry.getGauges().containsKey(MetricNames.SERVER_TOTAL_THREAD_NUM));
+
+ // Verify QueuedThreadPool specific metrics are NOT registered
+ assertTrue(
+ registry.getGauges().isEmpty()
+ ||
!registry.getGauges().containsKey(MetricNames.SERVER_BUSY_THREAD_NUM));
+ assertTrue(
+ registry.getGauges().isEmpty()
+ ||
!registry.getGauges().containsKey(MetricNames.SERVER_QUEUED_REQUEST_NUM));
+ assertTrue(
+ registry.getGauges().isEmpty()
+ ||
!registry.getGauges().containsKey(MetricNames.SERVER_MIN_THREAD_NUM));
+ assertTrue(
+ registry.getGauges().isEmpty()
+ ||
!registry.getGauges().containsKey(MetricNames.SERVER_MAX_THREAD_NUM));
+ }
+
+ @Test
+ void testGaugeValuesWithQueuedThreadPool() {
+ // Arrange - Use a real QueuedThreadPool instance
+ QueuedThreadPool realQueuedThreadPool = new QueuedThreadPool(20, 1, 60000);
+ when(mockJettyServer.getThreadPool()).thenReturn(realQueuedThreadPool);
+
+ // Act
+ metricsSource = new HttpServerMetricsSource("test-server",
mockResourceConfig, mockJettyServer);
+
+ // Assert - Test gauge values
+ MetricRegistry registry = metricsSource.getMetricRegistry();
+
+ @SuppressWarnings("unchecked")
+ Gauge<Integer> idleThreadsGauge =
+ (Gauge<Integer>)
registry.getGauges().get(MetricNames.SERVER_IDLE_THREAD_NUM);
+ assertTrue(idleThreadsGauge.getValue() >= 0, "Idle threads should be
non-negative");
+
+ @SuppressWarnings("unchecked")
+ Gauge<Integer> totalThreadsGauge =
+ (Gauge<Integer>)
registry.getGauges().get(MetricNames.SERVER_TOTAL_THREAD_NUM);
+ assertTrue(totalThreadsGauge.getValue() >= 0, "Total threads should be
non-negative");
+
+ @SuppressWarnings("unchecked")
+ Gauge<Integer> busyThreadsGauge =
+ (Gauge<Integer>)
registry.getGauges().get(MetricNames.SERVER_BUSY_THREAD_NUM);
+ assertTrue(busyThreadsGauge.getValue() >= 0, "Busy threads should be
non-negative");
+
+ @SuppressWarnings("unchecked")
+ Gauge<Integer> queuedRequestsGauge =
+ (Gauge<Integer>)
registry.getGauges().get(MetricNames.SERVER_QUEUED_REQUEST_NUM);
+ assertTrue(queuedRequestsGauge.getValue() >= 0, "Queued requests should be
non-negative");
+
+ @SuppressWarnings("unchecked")
+ Gauge<Integer> minThreadsGauge =
+ (Gauge<Integer>)
registry.getGauges().get(MetricNames.SERVER_MIN_THREAD_NUM);
+ assertEquals(1, minThreadsGauge.getValue().intValue());
+
+ @SuppressWarnings("unchecked")
+ Gauge<Integer> maxThreadsGauge =
+ (Gauge<Integer>)
registry.getGauges().get(MetricNames.SERVER_MAX_THREAD_NUM);
+ assertEquals(20, maxThreadsGauge.getValue().intValue());
+ }
+
+ @Test
+ void testGaugeValuesWithNonQueuedThreadPool() {
+ // Arrange
+ when(mockJettyServer.getThreadPool()).thenReturn(mockThreadPool);
+ when(mockThreadPool.getIdleThreads()).thenReturn(3);
+ when(mockThreadPool.getThreads()).thenReturn(8);
+
+ // Act
+ metricsSource = new HttpServerMetricsSource("test-server",
mockResourceConfig, mockJettyServer);
+
+ // Assert - Test gauge values
+ MetricRegistry registry = metricsSource.getMetricRegistry();
+
+ @SuppressWarnings("unchecked")
+ Gauge<Integer> idleThreadsGauge =
+ (Gauge<Integer>)
registry.getGauges().get(MetricNames.SERVER_IDLE_THREAD_NUM);
+ assertEquals(3, idleThreadsGauge.getValue().intValue());
+
+ @SuppressWarnings("unchecked")
+ Gauge<Integer> totalThreadsGauge =
+ (Gauge<Integer>)
registry.getGauges().get(MetricNames.SERVER_TOTAL_THREAD_NUM);
+ assertEquals(8, totalThreadsGauge.getValue().intValue());
+ }
+
+ @Test
+ void testResourceConfigRegistration() {
+ // Arrange
+ QueuedThreadPool realQueuedThreadPool = new QueuedThreadPool(10, 5, 60000);
+ when(mockJettyServer.getThreadPool()).thenReturn(realQueuedThreadPool);
+
+ // Act
+ metricsSource = new HttpServerMetricsSource("test-server",
mockResourceConfig, mockJettyServer);
+
+ // Assert
+ ArgumentCaptor<InstrumentedResourceMethodApplicationListener> captor =
+
ArgumentCaptor.forClass(InstrumentedResourceMethodApplicationListener.class);
+ verify(mockResourceConfig).register(captor.capture());
+
+ InstrumentedResourceMethodApplicationListener listener = captor.getValue();
+ assertNotNull(listener);
+ }
+
+ @Test
+ void testMetricsSourceName() {
+ // Arrange
+ when(mockJettyServer.getThreadPool()).thenReturn(mockThreadPool);
+ when(mockThreadPool.getIdleThreads()).thenReturn(0);
+ when(mockThreadPool.getThreads()).thenReturn(0);
+
+ // Act
+ metricsSource = new HttpServerMetricsSource("test-server",
mockResourceConfig, mockJettyServer);
+
+ // Assert
+ assertEquals("test-server", metricsSource.getMetricsSourceName());
+ }
+
+ @Test
+ void testGaugeLambdasAreCalled() {
+ // Arrange
+ when(mockJettyServer.getThreadPool()).thenReturn(mockThreadPool);
+ when(mockThreadPool.getIdleThreads()).thenReturn(5);
+ when(mockThreadPool.getThreads()).thenReturn(10);
+
+ // Act
+ metricsSource = new HttpServerMetricsSource("test-server",
mockResourceConfig, mockJettyServer);
+
+ // Assert - Verify that the gauge lambdas call the thread pool methods
+ MetricRegistry registry = metricsSource.getMetricRegistry();
+
+ // Call getValue() on each gauge to trigger the lambda
+ registry.getGauges().get(MetricNames.SERVER_IDLE_THREAD_NUM).getValue();
+ registry.getGauges().get(MetricNames.SERVER_TOTAL_THREAD_NUM).getValue();
+
+ // Verify that the thread pool methods were called
+ verify(mockThreadPool).getIdleThreads();
+ verify(mockThreadPool).getThreads();
+ }
+
+ @Test
+ void testQueuedThreadPoolCast() {
+ // Arrange - Create a real QueuedThreadPool instance
+ QueuedThreadPool realQueuedThreadPool = new QueuedThreadPool(10, 5, 60000);
+ when(mockJettyServer.getThreadPool()).thenReturn(realQueuedThreadPool);
+
+ // Act
+ metricsSource = new HttpServerMetricsSource("test-server",
mockResourceConfig, mockJettyServer);
+
+ // Assert
+ MetricRegistry registry = metricsSource.getMetricRegistry();
+
+ // Verify all metrics are registered including QueuedThreadPool specific
ones
+
assertTrue(registry.getGauges().containsKey(MetricNames.SERVER_IDLE_THREAD_NUM));
+
assertTrue(registry.getGauges().containsKey(MetricNames.SERVER_TOTAL_THREAD_NUM));
+
assertTrue(registry.getGauges().containsKey(MetricNames.SERVER_BUSY_THREAD_NUM));
+
assertTrue(registry.getGauges().containsKey(MetricNames.SERVER_QUEUED_REQUEST_NUM));
+
assertTrue(registry.getGauges().containsKey(MetricNames.SERVER_MIN_THREAD_NUM));
+
assertTrue(registry.getGauges().containsKey(MetricNames.SERVER_MAX_THREAD_NUM));
+
+ // Verify gauge values are reasonable (not null and non-negative)
+ registry
+ .getGauges()
+ .values()
+ .forEach(
+ gauge -> {
+ Object value = gauge.getValue();
+ assertNotNull(value);
+ if (value instanceof Integer) {
+ assertTrue((Integer) value >= 0, "Gauge value should be
non-negative: " + value);
+ }
+ });
+ }
+}
