This is an automated email from the ASF dual-hosted git repository.
jamesbognar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/juneau.git
The following commit(s) were added to refs/heads/master by this push:
new d39be085de Add graceful shutdown and readiness gating for Jetty/Tomcat
microservices; externalize remaining starter projects; document graceful
shutdown.
d39be085de is described below
commit d39be085deef8ffea48947964a897e972090a0a4
Author: James Bognar <[email protected]>
AuthorDate: Fri Jun 12 11:41:56 2026 -0400
Add graceful shutdown and readiness gating for Jetty/Tomcat microservices;
externalize remaining starter projects; document graceful shutdown.
---
.../rest/server/health/ReadinessGate_Test.java | 116 +++++++++++++++++
.../microservice/jetty/JettyServerComponent.java | 40 ++++++
.../juneau/microservice/jetty/JettySettings.java | 62 +++++++++
.../src/main/resources/jetty.xml | 4 +
.../src/main/resources/juneau.cfg | 11 ++
.../jetty/JettyServerComponent.properties | 1 +
.../jetty/JettyGracefulShutdown_Test.java | 142 +++++++++++++++++++++
.../microservice/tomcat/TomcatServerComponent.java | 64 ++++++++++
.../juneau/microservice/tomcat/TomcatSettings.java | 63 +++++++++
.../tomcat/TomcatServerComponent.properties | 1 +
.../tomcat/TomcatGracefulShutdown_Test.java | 91 +++++++++++++
.../juneau-my-springboot-microservice/.gitignore | 1 +
.../juneau-my-tomcat-microservice/.gitignore | 1 +
.../rest/server/health/HealthAggregator.java | 8 ++
.../juneau/rest/server/health/ReadinessState.java | 125 ++++++++++++++++++
15 files changed, 730 insertions(+)
diff --git
a/juneau-integration-tests/src/test/java/org/apache/juneau/rest/server/health/ReadinessGate_Test.java
b/juneau-integration-tests/src/test/java/org/apache/juneau/rest/server/health/ReadinessGate_Test.java
new file mode 100644
index 0000000000..a79ee15f41
--- /dev/null
+++
b/juneau-integration-tests/src/test/java/org/apache/juneau/rest/server/health/ReadinessGate_Test.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.juneau.rest.server.health;
+
+import static java.util.EnumSet.*;
+
+import java.util.*;
+
+import org.apache.juneau.*;
+import org.apache.juneau.commons.inject.*;
+import org.apache.juneau.rest.mock.classic.*;
+import org.apache.juneau.rest.server.*;
+import org.junit.jupiter.api.*;
+
+/**
+ * Validates the zero-downtime readiness gate (TODO-174a) surfaced by {@link
HealthAggregator}.
+ *
+ * <p>
+ * When the shared {@link ReadinessState} flips {@link
ReadinessState#markOutOfService() out of service} (as the
+ * embedded-server shutdown hooks do at the very start of shutdown), {@code
/readyz} must return {@code 503} so a
+ * load balancer / Kubernetes stops routing new traffic, while {@code /livez}
(so the pod is not killed mid-drain)
+ * and {@code /healthz} stay unaffected. A registered {@code ReadinessState}
bean (the per-context resolution
+ * winner over the process-wide shared instance) drives the gate here so the
test is isolated.
+ *
+ * @since 10.0.0
+ */
+class ReadinessGate_Test extends TestBase {
+
+ static final ReadinessState READINESS = new ReadinessState();
+
+ @Rest
+ public static class A extends HealthServlet {
+ private static final long serialVersionUID = 1L;
+
+ @Bean
+ public ReadinessState readinessState() {
+ return READINESS;
+ }
+
+ @Override
+ protected Map<String,HealthIndicator> indicators() {
+ return Map.of(
+ "liveOnly", new HealthIndicator() {
+ @Override public Health check() {
return Health.up("liveOnly").build(); }
+ @Override public EnumSet<HealthProbe>
probes() { return of(HealthProbe.LIVE); }
+ },
+ "readyOnly", new HealthIndicator() {
+ @Override public Health check() {
return Health.up("readyOnly").build(); }
+ @Override public EnumSet<HealthProbe>
probes() { return of(HealthProbe.READY); }
+ }
+ );
+ }
+ }
+
+ private static final MockRestClient c =
MockRestClient.buildLax(A.class);
+
+ @BeforeEach void resetReadiness() {
+ READINESS.markReady();
+ }
+
+
//-----------------------------------------------------------------------------------------------------------------
+ // A. Regression: default (non-shutdown) probe behavior is unchanged.
+
//-----------------------------------------------------------------------------------------------------------------
+
+ @Test void a01_default_readyzLivezHealthzAllHealthy() throws Exception {
+
c.get("/readyz").accept("application/json").run().assertStatus(200);
+
c.get("/livez").accept("application/json").run().assertStatus(200);
+
c.get("/healthz").accept("application/json").run().assertStatus(200);
+ }
+
+
//-----------------------------------------------------------------------------------------------------------------
+ // B. Shutdown gate: readyz flips to 503; livez and healthz stay
healthy.
+
//-----------------------------------------------------------------------------------------------------------------
+
+ @Test void b01_outOfService_readyzFlips503() throws Exception {
+ READINESS.markOutOfService();
+ c.get("/readyz").accept("application/json").run()
+ .assertStatus(503)
+
.assertContent().asString().isContains("OUT_OF_SERVICE");
+ }
+
+ @Test void b02_outOfService_livezStaysHealthy() throws Exception {
+ READINESS.markOutOfService();
+
c.get("/livez").accept("application/json").run().assertStatus(200);
+ }
+
+ @Test void b03_outOfService_healthzUnaffected() throws Exception {
+ READINESS.markOutOfService();
+
c.get("/healthz").accept("application/json").run().assertStatus(200);
+ }
+
+
//-----------------------------------------------------------------------------------------------------------------
+ // C. Recovery: flipping back ready restores readyz.
+
//-----------------------------------------------------------------------------------------------------------------
+
+ @Test void c01_readyAgain_readyzRecovers() throws Exception {
+ READINESS.markOutOfService();
+
c.get("/readyz").accept("application/json").run().assertStatus(503);
+ READINESS.markReady();
+
c.get("/readyz").accept("application/json").run().assertStatus(200);
+ }
+}
diff --git
a/juneau-microservice/juneau-microservice-jetty/src/main/java/org/apache/juneau/microservice/jetty/JettyServerComponent.java
b/juneau-microservice/juneau-microservice-jetty/src/main/java/org/apache/juneau/microservice/jetty/JettyServerComponent.java
index 0619c8b6fb..8fa7880f9f 100644
---
a/juneau-microservice/juneau-microservice-jetty/src/main/java/org/apache/juneau/microservice/jetty/JettyServerComponent.java
+++
b/juneau-microservice/juneau-microservice-jetty/src/main/java/org/apache/juneau/microservice/jetty/JettyServerComponent.java
@@ -26,6 +26,7 @@ import static
org.apache.juneau.marshall.collections.JsonMap.*;
import java.io.*;
import java.net.*;
+import java.time.*;
import java.util.*;
import java.util.concurrent.atomic.*;
import java.util.logging.*;
@@ -37,6 +38,7 @@ import org.apache.juneau.marshall.cp.*;
import org.apache.juneau.microservice.*;
import org.apache.juneau.rest.server.*;
import org.apache.juneau.rest.server.auth.*;
+import org.apache.juneau.rest.server.health.*;
import org.apache.juneau.rest.server.servlet.*;
import org.eclipse.jetty.ee11.servlet.*;
import org.eclipse.jetty.server.*;
@@ -82,9 +84,13 @@ public class JettyServerComponent implements
MicroserviceListener {
private static final String KEY_SERVLET_CONTEXT_HANDLER =
"ServletContextHandler";
private static final Random RANDOM = new Random();
+ /** Default bounded graceful-shutdown drain timeout applied to the
Jetty server when none is configured. */
+ static final Duration DEFAULT_STOP_TIMEOUT = Duration.ofSeconds(30);
+
private final Messages messages =
Messages.of(JettyServerComponent.class);
private final AtomicReference<Server> server = new AtomicReference<>();
private final AtomicReference<Microservice> microservice = new
AtomicReference<>();
+ private final AtomicReference<Duration> shutdownSettleDelay = new
AtomicReference<>(Duration.ZERO);
/**
* Env-driven sentinel for {@code availablePort}; {@link
Optional#empty()} when unset (in which case
@@ -235,6 +241,21 @@ public class JettyServerComponent implements
MicroserviceListener {
// Publish back to the store for downstream beans /
lookups.
store.addBean(Server.class, server.get());
+ // Graceful-shutdown wiring (zero-downtime k8s
rollouts):
+ // - Apply a bounded stopTimeout so server.stop()
drains in-flight requests before the connector closes.
+ // Precedence: JettySettings > Jetty/stopTimeout
config > jetty.xml value (if any) > 30s default.
+ // - Remember the settle delay so onStop() can let the
LB observe the /readyz 503 before draining.
+ // - Mark readiness ready so a freshly-(re)started
service serves traffic.
+ var stopTimeoutOverride = settings.getStopTimeout() !=
null
+ ? settings.getStopTimeout()
+ :
cf.get("Jetty/stopTimeout").asLong().map(Duration::ofMillis).orElse(null);
+ if (stopTimeoutOverride != null)
+ server.get().setStopTimeout(Math.max(0L,
stopTimeoutOverride.toMillis()));
+ else if (server.get().getStopTimeout() <= 0L)
+
server.get().setStopTimeout(DEFAULT_STOP_TIMEOUT.toMillis());
+
shutdownSettleDelay.set(firstNonNull(settings.getShutdownSettleDelay(),
cf.get("Jetty/shutdownSettleDelay").asLong().map(Duration::ofMillis).orElse(Duration.ZERO)));
+ ReadinessState.resolve(store).markReady();
+
// Track each servlet pathSpec with its declaring
source so we can fail loudly on collisions.
var mountedPaths = new LinkedHashMap<String,String>();
@@ -297,8 +318,17 @@ public class JettyServerComponent implements
MicroserviceListener {
}
@Override /* Overridden from MicroserviceListener */
+ @SuppressWarnings({
+ "resource" // ms.getBeanStore() is owned by the microservice
lifecycle; do not close here.
+ })
public void onStop(Microservice ms) {
final Logger logger = ms.getLogger();
+ // Flip readiness out of service BEFORE the connector stops so
/readyz returns 503 and the load balancer /
+ // Kubernetes stops routing new traffic while in-flight
requests drain. /livez stays healthy so the pod is
+ // not killed mid-drain. The settle delay (default 0) gives
the LB a window to observe the 503.
+ ReadinessState.resolve(ms.getBeanStore()).markOutOfService();
+ ms.out(messages, "DrainingRequests");
+ sleepQuietly(shutdownSettleDelay.get());
var t = new Thread("JettyServerComponentStop") {
@Override /* Overridden from Thread */
public void run() {
@@ -322,6 +352,16 @@ public class JettyServerComponent implements
MicroserviceListener {
}
}
+ private static void sleepQuietly(Duration d) {
+ if (d == null || d.isZero() || d.isNegative())
+ return;
+ try {
+ Thread.sleep(d.toMillis());
+ } catch (@SuppressWarnings("unused") InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
@Override /* Overridden from MicroserviceListener */
public void onConfigChange(Microservice ms, ConfigEvents events) {
// No-op: Intentional empty implementation - this component is
purely lifecycle-driven.
diff --git
a/juneau-microservice/juneau-microservice-jetty/src/main/java/org/apache/juneau/microservice/jetty/JettySettings.java
b/juneau-microservice/juneau-microservice-jetty/src/main/java/org/apache/juneau/microservice/jetty/JettySettings.java
index 7389bc0ff1..7cad486d55 100644
---
a/juneau-microservice/juneau-microservice-jetty/src/main/java/org/apache/juneau/microservice/jetty/JettySettings.java
+++
b/juneau-microservice/juneau-microservice-jetty/src/main/java/org/apache/juneau/microservice/jetty/JettySettings.java
@@ -22,6 +22,7 @@ import static org.apache.juneau.commons.utils.Utils.*;
import java.io.*;
import java.nio.file.*;
+import java.time.*;
/**
* Programmatic settings for the Jetty server contributed by {@link
JettyConfiguration}.
@@ -61,6 +62,8 @@ public class JettySettings {
int[] ports;
String jettyXml;
Boolean jettyXmlResolveVars;
+ Duration stopTimeout;
+ Duration shutdownSettleDelay;
Builder() {}
@@ -127,6 +130,43 @@ public class JettySettings {
jettyXmlResolveVars = resolveVars;
return this;
}
+
+ /**
+ * Specifies the bounded graceful-shutdown drain timeout for
the Jetty server.
+ *
+ * <p>
+ * On stop, the server stops accepting new connections and
waits up to this duration for in-flight requests
+ * to complete before the connector closes (Jetty's
<c>stopTimeout</c>). When set, this value takes
+ * precedence over the <c>Jetty/stopTimeout</c> config-file
entry (in milliseconds). When neither is set,
+ * a default of {@code 30s} is applied.
+ *
+ * @param value The drain timeout. Can be <jk>null</jk> to
defer to config, then to the default.
+ * @return This object.
+ */
+ public Builder stopTimeout(Duration value) {
+ stopTimeout = value;
+ return this;
+ }
+
+ /**
+ * Specifies the settle delay applied between flipping the
readiness probe out of service and stopping the
+ * connector.
+ *
+ * <p>
+ * On stop, the readiness probe ({@code /readyz}) flips to
{@code 503} <i>before</i> the connector closes so a
+ * load balancer / Kubernetes stops routing new traffic. This
brief delay gives the load balancer time to
+ * observe the {@code 503} before in-flight requests are
drained. When set, this value takes precedence over
+ * the <c>Jetty/shutdownSettleDelay</c> config-file entry (in
milliseconds). When neither is set, no settle
+ * delay is applied (defaults to {@code 0}) — the
recommended Kubernetes pattern is a {@code preStop}
+ * hook sleep instead.
+ *
+ * @param value The settle delay. Can be <jk>null</jk> to
defer to config, then to the default.
+ * @return This object.
+ */
+ public Builder shutdownSettleDelay(Duration value) {
+ shutdownSettleDelay = value;
+ return this;
+ }
}
/**
@@ -141,11 +181,15 @@ public class JettySettings {
private final int[] ports;
private final String jettyXml;
private final Boolean jettyXmlResolveVars;
+ private final Duration stopTimeout;
+ private final Duration shutdownSettleDelay;
JettySettings(Builder b) {
ports = b.ports;
jettyXml = b.jettyXml;
jettyXmlResolveVars = b.jettyXmlResolveVars;
+ stopTimeout = b.stopTimeout;
+ shutdownSettleDelay = b.shutdownSettleDelay;
}
/**
@@ -174,4 +218,22 @@ public class JettySettings {
public Boolean getJettyXmlResolveVars() {
return jettyXmlResolveVars;
}
+
+ /**
+ * Returns the bounded graceful-shutdown drain timeout for the Jetty
server.
+ *
+ * @return The drain timeout, or <jk>null</jk> if not set.
+ */
+ public Duration getStopTimeout() {
+ return stopTimeout;
+ }
+
+ /**
+ * Returns the settle delay applied between flipping the readiness
probe out of service and stopping the connector.
+ *
+ * @return The settle delay, or <jk>null</jk> if not set.
+ */
+ public Duration getShutdownSettleDelay() {
+ return shutdownSettleDelay;
+ }
}
diff --git
a/juneau-microservice/juneau-microservice-jetty/src/main/resources/jetty.xml
b/juneau-microservice/juneau-microservice-jetty/src/main/resources/jetty.xml
index efa4ca51df..d5a24b31e0 100644
--- a/juneau-microservice/juneau-microservice-jetty/src/main/resources/jetty.xml
+++ b/juneau-microservice/juneau-microservice-jetty/src/main/resources/jetty.xml
@@ -28,6 +28,10 @@
jetty.xml on the working directory or classpath.
- All defaults are overridable: a consumer-supplied jetty.xml on the working
directory or earlier on
the classpath wins, as does a @Bean JettyServerFactory in the consumer's
@Configuration.
+ - Graceful shutdown: stopTimeout is intentionally NOT set here.
JettyServerComponent applies a sensible
+ default (30s) at startup so server.stop() drains in-flight requests, and
lets the [Jetty]/stopTimeout
+ config or a @Bean JettySettings override it. Setting <Set
name="stopTimeout"> here would take effect only
+ when neither of those overrides is supplied.
-->
<Configure id="JuneauJettyServer" class="org.eclipse.jetty.server.Server">
diff --git
a/juneau-microservice/juneau-microservice-jetty/src/main/resources/juneau.cfg
b/juneau-microservice/juneau-microservice-jetty/src/main/resources/juneau.cfg
index 09e8edd193..87eaec712f 100644
---
a/juneau-microservice/juneau-microservice-jetty/src/main/resources/juneau.cfg
+++
b/juneau-microservice/juneau-microservice-jetty/src/main/resources/juneau.cfg
@@ -37,6 +37,17 @@ resolveVars = true
# The resolved port gets set as the system property "availablePort",
referenced in jetty.xml as "$S{availablePort}".
port = 10000,0,0,0
+# Graceful-shutdown drain timeout in milliseconds (zero-downtime k8s rollouts).
+# On stop, JettyServerComponent flips the readiness probe (/readyz) to 503
BEFORE the connector closes, then
+# server.stop() drains in-flight requests for up to this long. Defaults to
30000 (30s) when unset.
+# stopTimeout = 30000
+
+# Settle delay in milliseconds applied between the /readyz 503 flip and the
connector stop, giving the load
+# balancer a window to observe the 503 before in-flight requests drain.
Defaults to 0 when unset.
+# The recommended Kubernetes pattern is a preStop hook sleep + a
terminationGracePeriodSeconds that exceeds
+# (shutdownSettleDelay + stopTimeout) rather than an in-process settle delay.
+# shutdownSettleDelay = 0
+
#=======================================================================================================================
# REST settings
#=======================================================================================================================
diff --git
a/juneau-microservice/juneau-microservice-jetty/src/main/resources/org/apache/juneau/microservice/jetty/JettyServerComponent.properties
b/juneau-microservice/juneau-microservice-jetty/src/main/resources/org/apache/juneau/microservice/jetty/JettyServerComponent.properties
index ed10350985..d56367530e 100644
---
a/juneau-microservice/juneau-microservice-jetty/src/main/resources/org/apache/juneau/microservice/jetty/JettyServerComponent.properties
+++
b/juneau-microservice/juneau-microservice-jetty/src/main/resources/org/apache/juneau/microservice/jetty/JettyServerComponent.properties
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+DrainingRequests = Readiness flipped out of service; draining in-flight
requests.
StoppingServer = Stopping server.
ServerStopped = Server stopped.
ServerStarted = Server started on port {0}
diff --git
a/juneau-microservice/juneau-microservice-jetty/src/test/java/org/apache/juneau/microservice/jetty/JettyGracefulShutdown_Test.java
b/juneau-microservice/juneau-microservice-jetty/src/test/java/org/apache/juneau/microservice/jetty/JettyGracefulShutdown_Test.java
new file mode 100644
index 0000000000..07379b67e0
--- /dev/null
+++
b/juneau-microservice/juneau-microservice-jetty/src/test/java/org/apache/juneau/microservice/jetty/JettyGracefulShutdown_Test.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.juneau.microservice.jetty;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.time.*;
+
+import org.apache.juneau.*;
+import org.apache.juneau.commons.inject.*;
+import org.apache.juneau.microservice.*;
+import org.apache.juneau.rest.server.health.*;
+import org.eclipse.jetty.ee11.servlet.*;
+import org.eclipse.jetty.server.*;
+import org.junit.jupiter.api.*;
+
+/**
+ * Graceful-shutdown / readiness-gating tests for {@link JettyServerComponent}
(TODO-174a).
+ *
+ * <p>
+ * Verifies the zero-downtime shutdown contract on the Jetty side:
+ * <ul>
+ * <li>A sensible default {@code stopTimeout} ({@link
JettyServerComponent#DEFAULT_STOP_TIMEOUT}) is applied to the
+ * Jetty server when neither {@link
JettySettings#getStopTimeout()} nor the {@code Jetty/stopTimeout} config
+ * entry supplies one, so {@code server.stop()} drains in-flight
requests before the connector closes.
+ * <li>A programmatic {@link JettySettings#stopTimeout(Duration)} override
wins over the default.
+ * <li>On stop, readiness flips out of service <i>before</i> the connector
stops; on start it is marked ready.
+ * </ul>
+ *
+ * @since 10.0.0
+ */
[email protected]
+@SuppressWarnings("resource") // Microservice/Server instances are test
fixtures managed by the test lifecycle; explicit close is not needed for these
assertions.
+class JettyGracefulShutdown_Test extends TestBase {
+
+ private static Microservice create(Class<?>... configurations) throws
Exception {
+ var classes = new Class<?>[configurations.length + 1];
+ System.arraycopy(configurations, 0, classes, 0,
configurations.length);
+ classes[configurations.length] = JettyConfiguration.class;
+ return Microservice.create().configurations(classes).build();
+ }
+
+ // An ephemeral-port Jetty server with stopTimeout pre-set to 0 so the
component default/override is observable.
+ private static Server ephemeralServer() {
+ var server = new Server();
+ var connector = new ServerConnector(server);
+ connector.setPort(0);
+ server.addConnector(connector);
+ var ctx = new ServletContextHandler();
+ ctx.setContextPath("/");
+ server.setAttribute("ServletContextHandler", ctx);
+ server.setHandler(ctx);
+ server.setStopTimeout(0L);
+ return server;
+ }
+
+
//-----------------------------------------------------------------------------------------------------------------
+ // A. Default stopTimeout applied when nothing else supplies one.
+
//-----------------------------------------------------------------------------------------------------------------
+
+ @Configuration
+ static class A_DefaultConfig {
+ @Bean Server jettyServer() { return ephemeralServer(); }
+ @Bean ReadinessState readinessState() { return new
ReadinessState(); }
+ }
+
+ @Test void a01_defaultStopTimeoutApplied() throws Exception {
+ var ms = create(A_DefaultConfig.class);
+ try {
+ ms.start();
+ var server =
ms.getBeanStore().getBean(JettyServerComponent.class).orElseThrow().getServer();
+
assertEquals(JettyServerComponent.DEFAULT_STOP_TIMEOUT.toMillis(),
server.getStopTimeout());
+ } finally {
+ ms.stop();
+ }
+ }
+
+
//-----------------------------------------------------------------------------------------------------------------
+ // B. Programmatic JettySettings.stopTimeout override wins over the
default.
+
//-----------------------------------------------------------------------------------------------------------------
+
+ @Configuration
+ static class B_OverrideConfig {
+ @Bean Server jettyServer() { return ephemeralServer(); }
+ @Bean JettySettings jettySettings() { return
JettySettings.create().stopTimeout(Duration.ofSeconds(5)).build(); }
+ @Bean ReadinessState readinessState() { return new
ReadinessState(); }
+ }
+
+ @Test void b01_settingsStopTimeoutOverridesDefault() throws Exception {
+ var ms = create(B_OverrideConfig.class);
+ try {
+ ms.start();
+ var server =
ms.getBeanStore().getBean(JettyServerComponent.class).orElseThrow().getServer();
+ assertEquals(5000L, server.getStopTimeout());
+ } finally {
+ ms.stop();
+ }
+ }
+
+
//-----------------------------------------------------------------------------------------------------------------
+ // C. Readiness flips out of service on stop, ready on start.
+
//-----------------------------------------------------------------------------------------------------------------
+
+ @Test void c01_readinessFlipsOutOfServiceOnStop() throws Exception {
+ var ms = create(A_DefaultConfig.class);
+ ReadinessState rs;
+ boolean readyAfterStart;
+ try {
+ ms.start();
+ rs =
ms.getBeanStore().getBean(ReadinessState.class).orElseThrow();
+ readyAfterStart = rs.isReady();
+ } finally {
+ ms.stop();
+ }
+ assertTrue(readyAfterStart, "Service should be ready after
start");
+ assertFalse(rs.isReady(), "Service should be out of service
after stop");
+ }
+
+
//-----------------------------------------------------------------------------------------------------------------
+ // D. Settings round-trip.
+
//-----------------------------------------------------------------------------------------------------------------
+
+ @Test void d01_settingsRoundTrip() {
+ var s =
JettySettings.create().stopTimeout(Duration.ofSeconds(7)).shutdownSettleDelay(Duration.ofSeconds(2)).build();
+ assertEquals(Duration.ofSeconds(7), s.getStopTimeout());
+ assertEquals(Duration.ofSeconds(2), s.getShutdownSettleDelay());
+ }
+}
diff --git
a/juneau-microservice/juneau-microservice-tomcat/src/main/java/org/apache/juneau/microservice/tomcat/TomcatServerComponent.java
b/juneau-microservice/juneau-microservice-tomcat/src/main/java/org/apache/juneau/microservice/tomcat/TomcatServerComponent.java
index c1d33e1fca..b9935ff3e3 100644
---
a/juneau-microservice/juneau-microservice-tomcat/src/main/java/org/apache/juneau/microservice/tomcat/TomcatServerComponent.java
+++
b/juneau-microservice/juneau-microservice-tomcat/src/main/java/org/apache/juneau/microservice/tomcat/TomcatServerComponent.java
@@ -28,7 +28,9 @@ import java.io.*;
import java.net.*;
import java.nio.file.*;
import java.nio.file.attribute.*;
+import java.time.*;
import java.util.*;
+import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import java.util.logging.*;
@@ -41,6 +43,7 @@ import org.apache.juneau.marshall.cp.*;
import org.apache.juneau.microservice.*;
import org.apache.juneau.rest.server.*;
import org.apache.juneau.rest.server.auth.*;
+import org.apache.juneau.rest.server.health.*;
import org.apache.juneau.rest.server.servlet.*;
import org.apache.tomcat.util.descriptor.web.*;
@@ -87,11 +90,16 @@ public class TomcatServerComponent implements
MicroserviceListener {
private static final String ROOT_CONTEXT_PATH = "";
private static final String ROOT_DOC_BASE = ".";
+ /** Default bounded graceful-shutdown drain timeout applied when none
is configured. */
+ static final Duration DEFAULT_STOP_TIMEOUT = Duration.ofSeconds(30);
+
private final Messages messages =
Messages.of(TomcatServerComponent.class);
private final AtomicReference<Tomcat> tomcat = new AtomicReference<>();
private final AtomicReference<Microservice> microservice = new
AtomicReference<>();
private final AtomicReference<File> baseDir = new AtomicReference<>();
private final AtomicBoolean ownsBaseDir = new AtomicBoolean(false);
+ private final AtomicReference<Duration> stopTimeout = new
AtomicReference<>(DEFAULT_STOP_TIMEOUT);
+ private final AtomicReference<Duration> shutdownSettleDelay = new
AtomicReference<>(Duration.ZERO);
/**
* Env-driven sentinel for {@code availablePort}; {@link
Optional#empty()} when unset (in which case
@@ -221,6 +229,15 @@ public class TomcatServerComponent implements
MicroserviceListener {
// Publish back to the store for downstream beans /
lookups.
store.addBean(Tomcat.class, tomcat.get());
+ // Graceful-shutdown wiring (zero-downtime k8s
rollouts), kept identical to the Jetty contract:
+ // - Resolve the bounded drain timeout (TomcatSettings
> Tomcat/stopTimeout config > 30s default) so
+ // onStop() can wait for in-flight requests to
complete before the server stops.
+ // - Remember the settle delay so onStop() can let the
LB observe the /readyz 503 before draining.
+ // - Mark readiness ready so a freshly-(re)started
service serves traffic.
+ stopTimeout.set(firstNonNull(settings.getStopTimeout(),
cf.get("Tomcat/stopTimeout").asLong().map(Duration::ofMillis).orElse(DEFAULT_STOP_TIMEOUT)));
+
shutdownSettleDelay.set(firstNonNull(settings.getShutdownSettleDelay(),
cf.get("Tomcat/shutdownSettleDelay").asLong().map(Duration::ofMillis).orElse(Duration.ZERO)));
+ ReadinessState.resolve(store).markReady();
+
// Track each servlet pathSpec with its declaring
source so we can fail loudly on collisions.
var mountedPaths = new LinkedHashMap<String,String>();
@@ -260,8 +277,17 @@ public class TomcatServerComponent implements
MicroserviceListener {
}
@Override /* Overridden from MicroserviceListener */
+ @SuppressWarnings({
+ "resource" // ms.getBeanStore() is owned by the microservice
lifecycle; do not close here.
+ })
public void onStop(Microservice ms) {
final Logger logger = ms.getLogger();
+ // Flip readiness out of service BEFORE the connector stops so
/readyz returns 503 and the load balancer /
+ // Kubernetes stops routing new traffic while in-flight
requests drain. /livez stays healthy so the pod is
+ // not killed mid-drain. The settle delay (default 0) gives
the LB a window to observe the 503.
+ ReadinessState.resolve(ms.getBeanStore()).markOutOfService();
+ ms.out(messages, "DrainingRequests");
+ sleepQuietly(shutdownSettleDelay.get());
var t = new Thread("TomcatServerComponentStop") {
@Override /* Overridden from Thread */
public void run() {
@@ -269,6 +295,7 @@ public class TomcatServerComponent implements
MicroserviceListener {
var t2 = tomcat.get();
if (t2 == null)
return;
+ drainConnector(t2, stopTimeout.get(),
logger);
ms.out(messages, "StoppingServer");
t2.stop();
t2.destroy();
@@ -288,6 +315,43 @@ public class TomcatServerComponent implements
MicroserviceListener {
}
}
+ /**
+ * Pauses the connector (stops accepting new connections) and waits up
to {@code timeout} for in-flight requests
+ * to drain before the caller stops the server. Best-effort: never
throws out of the shutdown sequence.
+ */
+ private static void drainConnector(Tomcat t2, Duration timeout, Logger
logger) {
+ try {
+ var connector = t2.getConnector();
+ if (connector == null)
+ return;
+ connector.pause();
+ var executor = connector.getProtocolHandler() == null ?
null : connector.getProtocolHandler().getExecutor();
+ if (executor instanceof ThreadPoolExecutor tpe) {
+ var deadline = System.nanoTime() + Math.max(0L,
timeout.toNanos());
+ while (tpe.getActiveCount() > 0 &&
System.nanoTime() < deadline) {
+ try {
+ Thread.sleep(50L);
+ } catch (@SuppressWarnings("unused")
InterruptedException e) {
+
Thread.currentThread().interrupt();
+ return;
+ }
+ }
+ }
+ } catch (Exception e) {
+ logger.log(Level.WARNING, lm(e), e);
+ }
+ }
+
+ private static void sleepQuietly(Duration d) {
+ if (d == null || d.isZero() || d.isNegative())
+ return;
+ try {
+ Thread.sleep(d.toMillis());
+ } catch (@SuppressWarnings("unused") InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
@Override /* Overridden from MicroserviceListener */
public void onConfigChange(Microservice ms, ConfigEvents events) {
// No-op: Intentional empty implementation - this component is
purely lifecycle-driven.
diff --git
a/juneau-microservice/juneau-microservice-tomcat/src/main/java/org/apache/juneau/microservice/tomcat/TomcatSettings.java
b/juneau-microservice/juneau-microservice-tomcat/src/main/java/org/apache/juneau/microservice/tomcat/TomcatSettings.java
index 2a386dfa2d..8701927a87 100644
---
a/juneau-microservice/juneau-microservice-tomcat/src/main/java/org/apache/juneau/microservice/tomcat/TomcatSettings.java
+++
b/juneau-microservice/juneau-microservice-tomcat/src/main/java/org/apache/juneau/microservice/tomcat/TomcatSettings.java
@@ -16,6 +16,8 @@
*/
package org.apache.juneau.microservice.tomcat;
+import java.time.*;
+
/**
* Programmatic settings for the embedded Tomcat server contributed by {@link
TomcatConfiguration}.
*
@@ -53,6 +55,8 @@ public class TomcatSettings {
int[] ports;
String baseDir;
+ Duration stopTimeout;
+ Duration shutdownSettleDelay;
Builder() {}
@@ -97,6 +101,43 @@ public class TomcatSettings {
baseDir = value;
return this;
}
+
+ /**
+ * Specifies the bounded graceful-shutdown drain timeout for
the Tomcat server.
+ *
+ * <p>
+ * On stop, the connector is paused (stops accepting new
requests) and the component waits up to this
+ * duration for in-flight requests to complete before the
server stops. When set, this value takes
+ * precedence over the <c>Tomcat/stopTimeout</c> config-file
entry (in milliseconds). When neither is set,
+ * a default of {@code 30s} is applied.
+ *
+ * @param value The drain timeout. Can be <jk>null</jk> to
defer to config, then to the default.
+ * @return This object.
+ */
+ public Builder stopTimeout(Duration value) {
+ stopTimeout = value;
+ return this;
+ }
+
+ /**
+ * Specifies the settle delay applied between flipping the
readiness probe out of service and stopping the
+ * connector.
+ *
+ * <p>
+ * On stop, the readiness probe ({@code /readyz}) flips to
{@code 503} <i>before</i> the connector is paused
+ * so a load balancer / Kubernetes stops routing new traffic.
This brief delay gives the load balancer time
+ * to observe the {@code 503} before in-flight requests are
drained. When set, this value takes precedence
+ * over the <c>Tomcat/shutdownSettleDelay</c> config-file entry
(in milliseconds). When neither is set, no
+ * settle delay is applied (defaults to {@code 0}) — the
recommended Kubernetes pattern is a
+ * {@code preStop} hook sleep instead.
+ *
+ * @param value The settle delay. Can be <jk>null</jk> to
defer to config, then to the default.
+ * @return This object.
+ */
+ public Builder shutdownSettleDelay(Duration value) {
+ shutdownSettleDelay = value;
+ return this;
+ }
}
/**
@@ -110,10 +151,14 @@ public class TomcatSettings {
private final int[] ports;
private final String baseDir;
+ private final Duration stopTimeout;
+ private final Duration shutdownSettleDelay;
TomcatSettings(Builder b) {
ports = b.ports;
baseDir = b.baseDir;
+ stopTimeout = b.stopTimeout;
+ shutdownSettleDelay = b.shutdownSettleDelay;
}
/**
@@ -133,4 +178,22 @@ public class TomcatSettings {
public String getBaseDir() {
return baseDir;
}
+
+ /**
+ * Returns the bounded graceful-shutdown drain timeout for the Tomcat
server.
+ *
+ * @return The drain timeout, or <jk>null</jk> if not set.
+ */
+ public Duration getStopTimeout() {
+ return stopTimeout;
+ }
+
+ /**
+ * Returns the settle delay applied between flipping the readiness
probe out of service and stopping the connector.
+ *
+ * @return The settle delay, or <jk>null</jk> if not set.
+ */
+ public Duration getShutdownSettleDelay() {
+ return shutdownSettleDelay;
+ }
}
diff --git
a/juneau-microservice/juneau-microservice-tomcat/src/main/resources/org/apache/juneau/microservice/tomcat/TomcatServerComponent.properties
b/juneau-microservice/juneau-microservice-tomcat/src/main/resources/org/apache/juneau/microservice/tomcat/TomcatServerComponent.properties
index a857f69a1f..97b3472542 100644
---
a/juneau-microservice/juneau-microservice-tomcat/src/main/resources/org/apache/juneau/microservice/tomcat/TomcatServerComponent.properties
+++
b/juneau-microservice/juneau-microservice-tomcat/src/main/resources/org/apache/juneau/microservice/tomcat/TomcatServerComponent.properties
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+DrainingRequests = Readiness flipped out of service; draining in-flight
requests.
StoppingServer = Stopping server.
ServerStopped = Server stopped.
ServerStarted = Server started on port {0}
diff --git
a/juneau-microservice/juneau-microservice-tomcat/src/test/java/org/apache/juneau/microservice/tomcat/TomcatGracefulShutdown_Test.java
b/juneau-microservice/juneau-microservice-tomcat/src/test/java/org/apache/juneau/microservice/tomcat/TomcatGracefulShutdown_Test.java
new file mode 100644
index 0000000000..49efc50290
--- /dev/null
+++
b/juneau-microservice/juneau-microservice-tomcat/src/test/java/org/apache/juneau/microservice/tomcat/TomcatGracefulShutdown_Test.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.juneau.microservice.tomcat;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.time.*;
+
+import org.apache.juneau.*;
+import org.apache.juneau.commons.inject.*;
+import org.apache.juneau.microservice.*;
+import org.apache.juneau.rest.server.health.*;
+import org.junit.jupiter.api.*;
+
+/**
+ * Graceful-shutdown / readiness-gating tests for {@link
TomcatServerComponent} (TODO-174a).
+ *
+ * <p>
+ * Mirrors {@code JettyGracefulShutdown_Test} so both embedded servers share
one zero-downtime contract:
+ * <ul>
+ * <li>On stop, readiness flips out of service <i>before</i> the connector
is paused / the server stops; on start
+ * it is marked ready.
+ * <li>A sensible default drain timeout ({@link
TomcatServerComponent#DEFAULT_STOP_TIMEOUT}) backs the bounded
+ * in-flight drain, overridable via {@link
TomcatSettings#stopTimeout(Duration)} / {@code Tomcat/stopTimeout}.
+ * </ul>
+ *
+ * @since 10.0.0
+ */
[email protected]
+class TomcatGracefulShutdown_Test extends TestBase {
+
+ private static Microservice create(Class<?>... configurations) throws
Exception {
+ var classes = new Class<?>[configurations.length + 1];
+ System.arraycopy(configurations, 0, classes, 0,
configurations.length);
+ classes[configurations.length] = TomcatConfiguration.class;
+ return Microservice.create().configurations(classes).build();
+ }
+
+
//-----------------------------------------------------------------------------------------------------------------
+ // A. Readiness flips out of service on stop, ready on start.
+
//-----------------------------------------------------------------------------------------------------------------
+
+ @Configuration
+ static class A_Config {
+ @Bean TomcatSettings tomcatSettings() { return
TomcatSettings.create().ports(0).build(); }
+ @Bean ReadinessState readinessState() { return new
ReadinessState(); }
+ }
+
+ @Test void a01_readinessFlipsOutOfServiceOnStop() throws Exception {
+ var ms = create(A_Config.class);
+ ReadinessState rs;
+ boolean readyAfterStart;
+ try {
+ ms.start();
+ rs =
ms.getBeanStore().getBean(ReadinessState.class).orElseThrow();
+ readyAfterStart = rs.isReady();
+ } finally {
+ ms.stop();
+ }
+ assertTrue(readyAfterStart, "Service should be ready after
start");
+ assertFalse(rs.isReady(), "Service should be out of service
after stop");
+ }
+
+
//-----------------------------------------------------------------------------------------------------------------
+ // B. Default drain timeout + settings round-trip (parity with the
Jetty contract).
+
//-----------------------------------------------------------------------------------------------------------------
+
+ @Test void b01_defaultStopTimeoutConstant() {
+ assertEquals(Duration.ofSeconds(30),
TomcatServerComponent.DEFAULT_STOP_TIMEOUT);
+ }
+
+ @Test void b02_settingsRoundTrip() {
+ var s =
TomcatSettings.create().stopTimeout(Duration.ofSeconds(5)).shutdownSettleDelay(Duration.ofSeconds(2)).build();
+ assertEquals(Duration.ofSeconds(5), s.getStopTimeout());
+ assertEquals(Duration.ofSeconds(2), s.getShutdownSettleDelay());
+ }
+}
diff --git a/juneau-microservice/juneau-my-springboot-microservice/.gitignore
b/juneau-microservice/juneau-my-springboot-microservice/.gitignore
new file mode 100644
index 0000000000..ae3c172604
--- /dev/null
+++ b/juneau-microservice/juneau-my-springboot-microservice/.gitignore
@@ -0,0 +1 @@
+/bin/
diff --git a/juneau-microservice/juneau-my-tomcat-microservice/.gitignore
b/juneau-microservice/juneau-my-tomcat-microservice/.gitignore
new file mode 100644
index 0000000000..ae3c172604
--- /dev/null
+++ b/juneau-microservice/juneau-my-tomcat-microservice/.gitignore
@@ -0,0 +1 @@
+/bin/
diff --git
a/juneau-rest/juneau-rest-server/src/main/java/org/apache/juneau/rest/server/health/HealthAggregator.java
b/juneau-rest/juneau-rest-server/src/main/java/org/apache/juneau/rest/server/health/HealthAggregator.java
index c4d94a734e..ccd8ed831f 100644
---
a/juneau-rest/juneau-rest-server/src/main/java/org/apache/juneau/rest/server/health/HealthAggregator.java
+++
b/juneau-rest/juneau-rest-server/src/main/java/org/apache/juneau/rest/server/health/HealthAggregator.java
@@ -78,6 +78,14 @@ public class HealthAggregator {
}
var status = summarize(out.values());
+
+ // Readiness gate: when shutdown has flipped the readiness
state out of service, force the READY probe
+ // down so /readyz returns 503 before the connector closes.
/livez (LIVE) and /healthz (null) are unaffected.
+ if (probe == HealthProbe.READY && !
ReadinessState.resolve(context.getBeanStore()).isReady()) {
+ status = HealthStatus.DOWN;
+ out.put("readiness", new
ComponentHealth(HealthStatus.DOWN, Map.of("state", "OUT_OF_SERVICE")));
+ }
+
res.setStatus(status == HealthStatus.DOWN ? 503 : 200);
return new HealthResponse(status, out);
}
diff --git
a/juneau-rest/juneau-rest-server/src/main/java/org/apache/juneau/rest/server/health/ReadinessState.java
b/juneau-rest/juneau-rest-server/src/main/java/org/apache/juneau/rest/server/health/ReadinessState.java
new file mode 100644
index 0000000000..8a22839818
--- /dev/null
+++
b/juneau-rest/juneau-rest-server/src/main/java/org/apache/juneau/rest/server/health/ReadinessState.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.juneau.rest.server.health;
+
+import org.apache.juneau.commons.inject.*;
+
+/**
+ * Shared, observable readiness flag consumed by {@link HealthAggregator} to
gate the readiness probe.
+ *
+ * <p>
+ * This is the building block for zero-downtime shutdown. When a microservice
begins shutting down it flips
+ * the readiness state {@link #markOutOfService() out of service}
<i>before</i> the listening connector stops.
+ * The {@link HealthAggregator} consults this state for the {@link
HealthProbe#READY READY} probe so
+ * {@code /readyz} returns {@code 503} (out of service) as soon as shutdown
begins — this lets a load
+ * balancer / Kubernetes stop routing new traffic to the pod while in-flight
requests drain. The
+ * {@link HealthProbe#LIVE LIVE} probe ({@code /livez}) is intentionally
<b>not</b> gated, so Kubernetes does
+ * not kill the pod mid-drain.
+ *
+ * <h5 class='section'>Resolution:</h5>
+ * <p>
+ * {@link HealthAggregator} and the embedded-server shutdown hooks resolve the
state via
+ * {@link #resolve(BeanStore)}: a {@code ReadinessState} bean registered in
the relevant {@link BeanStore} wins;
+ * otherwise the process-wide {@link #shared() shared} instance is used. In
the standalone microservice runtime
+ * the auto-mounted health probe servlet and the server lifecycle component
live in <i>different</i> bean stores,
+ * so they bridge through the shared instance — the server flips it on
stop and the probe observes the flip.
+ *
+ * <p>
+ * This class is thread-safe; the flag is {@code volatile}.
+ *
+ * <h5 class='section'>See Also:</h5><ul>
+ * <li class='jc'>{@link HealthAggregator}
+ * </ul>
+ *
+ * @since 10.0.0
+ */
+public final class ReadinessState {
+
+ private static final ReadinessState SHARED = new ReadinessState();
+
+ /**
+ * Returns the process-wide shared readiness state.
+ *
+ * <p>
+ * Used as the fallback when no {@code ReadinessState} bean is
registered in the relevant {@link BeanStore}.
+ * In the standalone microservice runtime this is the bridge between
the embedded-server lifecycle component
+ * (which flips it on shutdown) and the auto-mounted health probe
servlet (which observes the flip), since the
+ * two live in separate bean stores.
+ *
+ * @return The shared instance. Never <jk>null</jk>.
+ */
+ public static ReadinessState shared() {
+ return SHARED;
+ }
+
+ /**
+ * Resolves the readiness state from the given bean store, falling back
to the {@link #shared() shared} instance.
+ *
+ * @param beanStore The bean store to consult. Can be <jk>null</jk>
(resolves to the shared instance).
+ * @return The resolved readiness state. Never <jk>null</jk>.
+ */
+ public static ReadinessState resolve(BeanStore beanStore) {
+ if (beanStore == null)
+ return SHARED;
+ return beanStore.getBean(ReadinessState.class).orElse(SHARED);
+ }
+
+ private volatile boolean ready = true;
+
+ /**
+ * Constructor.
+ *
+ * <p>
+ * The new state starts {@link #isReady() ready}.
+ */
+ public ReadinessState() {}
+
+ /**
+ * Returns whether this service is currently ready to receive traffic.
+ *
+ * @return <jk>true</jk> if ready, <jk>false</jk> if out of service.
+ */
+ public boolean isReady() {
+ return ready;
+ }
+
+ /**
+ * Marks this service as ready to receive traffic.
+ *
+ * <p>
+ * Called when an embedded server starts so a freshly-(re)started
service is ready.
+ *
+ * @return This object.
+ */
+ public ReadinessState markReady() {
+ ready = true;
+ return this;
+ }
+
+ /**
+ * Marks this service as out of service so the readiness probe ({@code
/readyz}) returns {@code 503}.
+ *
+ * <p>
+ * Called at the very beginning of shutdown, before the listening
connector stops.
+ *
+ * @return This object.
+ */
+ public ReadinessState markOutOfService() {
+ ready = false;
+ return this;
+ }
+}
