This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 27bbe6a1e7 Merges admin signalShutdown into admin stop command (#5745)
27bbe6a1e7 is described below
commit 27bbe6a1e71a4068a144d5e8ec50029e50737a2a
Author: Keith Turner <[email protected]>
AuthorDate: Wed Jul 23 12:33:43 2025 -0400
Merges admin signalShutdown into admin stop command (#5745)
Currently the admin stop command takes a list of hostnames which are
assumed to be tservers. This change allows admin stop to also take a
list of host:port pairs. For these host:port pairs they can be any
accumulo server type and the new gracefulShutdown code will be called for
them. The admin signalShutdown command was removed since the stop
command is calling the same code internally now.
When ports are not specified the command behaves exactly as it used to.
However a deprecation warning is logged because the old command would
use local config to guess at ports, so it would never stop tservers on a
host where its local config had different ports. Hopefully the
deprecated behavior of only specifying host can be replaced w/ something
like the following. Need to see what the actual command would look
like.
```bash
acummulo admin stop $(accumulo admin serviceStatus | grep
$tserver_host_to_stop | ???)
```
---
.../org/apache/accumulo/server/util/Admin.java | 80 ++++++++++++++++------
.../org/apache/accumulo/server/util/ZooZap.java | 8 +--
.../test/functional/GracefulShutdownIT.java | 10 +--
3 files changed, 69 insertions(+), 29 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
index 0f7f534751..e2ec6f7fa1 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -105,18 +105,12 @@ public class Admin implements KeywordExecutable {
}
@Parameters(
- commandDescription = "signal the server process to shutdown normally,
finishing anything it might be working on, but not starting any new tasks")
- static class GracefulShutdownCommand extends SubCommandOpts {
- @Parameter(required = true, names = {"-a", "--address"}, description =
"<host:port>")
- String address = null;
- }
-
- @Parameters(commandDescription = "stop the tablet server on the given hosts")
+ commandDescription = "Stop the servers at the given addresses allowing
them to complete current task but not start new task. When no port is
specified uses ports from tserver.port.client property.")
static class StopCommand extends SubCommandOpts {
@Parameter(names = {"-f", "--force"},
- description = "force the given server to stop by removing its lock")
+ description = "force the given server to stop immediately by removing
its lock. Does not wait for any task the server is currently working.")
boolean force = false;
- @Parameter(description = "<host> {<host> ... }")
+ @Parameter(description = "<host[:port]> {<host[:port]> ... }")
List<String> args = new ArrayList<>();
}
@@ -321,9 +315,6 @@ public class Admin implements KeywordExecutable {
FateOpsCommand fateOpsCommand = new FateOpsCommand();
cl.addCommand("fate", fateOpsCommand);
- GracefulShutdownCommand gracefulShutdownCommand = new
GracefulShutdownCommand();
- cl.addCommand("signalShutdown", gracefulShutdownCommand);
-
ListInstancesCommand listInstancesOpts = new ListInstancesCommand();
cl.addCommand("listInstances", listInstancesOpts);
@@ -413,9 +404,7 @@ public class Admin implements KeywordExecutable {
}
} else if (cl.getParsedCommand().equals("stop")) {
- stopTabletServer(context, stopOpts.args, stopOpts.force);
- } else if (cl.getParsedCommand().equals("signalShutdown")) {
- signalGracefulShutdown(context, gracefulShutdownCommand.address);
+ stopServers(context, stopOpts.args, stopOpts.force);
} else if (cl.getParsedCommand().equals("dumpConfig")) {
printConfig(context, dumpConfigCommand);
} else if (cl.getParsedCommand().equals("volumes")) {
@@ -563,18 +552,69 @@ public class Admin implements KeywordExecutable {
client -> client.shutdown(TraceUtil.traceInfo(), context.rpcCreds(),
tabletServersToo));
}
- // Visible for tests
- public static void signalGracefulShutdown(final ClientContext context,
String address) {
+ private static void stopServers(final ServerContext context, List<String>
servers,
+ final boolean force)
+ throws AccumuloException, AccumuloSecurityException,
InterruptedException, KeeperException {
+ List<String> hostOnly = new ArrayList<>();
+ Set<HostAndPort> hostAndPort = new TreeSet<>();
+
+ for (var server : servers) {
+ if (server.contains(":")) {
+ hostAndPort.add(HostAndPort.fromString(server));
+ } else {
+ hostOnly.add(server);
+ }
+ }
+
+ if (!hostOnly.isEmpty()) {
+ // The old impl of this command with the old behavior
+ stopTabletServer(context, hostOnly, force);
+ }
+
+ if (!hostAndPort.isEmpty()) {
+ // New behavior for this command when ports are present, supports more
than just tservers. Is
+ // also async.
+ if (force) {
+ ZooZap.Opts opts = new ZooZap.Opts();
+ var zk = context.getZooReaderWriter();
+ var iid = context.getInstanceID();
+
+ String tserversPath = Constants.ZROOT + "/" + iid +
Constants.ZTSERVERS;
+ ZooZap.removeLocks(zk, tserversPath, hostAndPort::contains, opts);
+ String compactorsBasepath = Constants.ZROOT + "/" + iid +
Constants.ZCOMPACTORS;
+ ZooZap.removeGroupedLocks(zk, compactorsBasepath, rg -> true,
hostAndPort::contains, opts);
+ String sserversPath = Constants.ZROOT + "/" + iid +
Constants.ZSSERVERS;
+ ZooZap.removeGroupedLocks(zk, sserversPath, rg -> true,
hostAndPort::contains, opts);
+
+ String managerLockPath = Constants.ZROOT + "/" + iid +
Constants.ZMANAGER_LOCK;
+ ZooZap.removeSingletonLock(zk, managerLockPath, hostAndPort::contains,
opts);
+ String gcLockPath = Constants.ZROOT + "/" + iid + Constants.ZGC_LOCK;
+ ZooZap.removeSingletonLock(zk, gcLockPath, hostAndPort::contains,
opts);
+ String monitorLockPath = Constants.ZROOT + "/" + iid +
Constants.ZMONITOR_LOCK;
+ ZooZap.removeSingletonLock(zk, monitorLockPath, hostAndPort::contains,
opts);
+ } else {
+ for (var server : hostAndPort) {
+ signalGracefulShutdown(context, server);
+ }
+ }
+ }
+ }
- Objects.requireNonNull(address, "address not set");
- final HostAndPort hp = HostAndPort.fromString(address);
+ // Visible for tests
+ public static void signalGracefulShutdown(final ClientContext context,
HostAndPort hp) {
+ Objects.requireNonNull(hp, "address not set");
ServerProcessService.Client client = null;
try {
client =
ThriftClientTypes.SERVER_PROCESS.getServerProcessConnection(context, log,
hp.getHost(), hp.getPort());
+ if (client == null) {
+ log.warn("Failed to initiate shutdown for {}", hp);
+ return;
+ }
client.gracefulShutdown(context.rpcCreds());
+ log.info("Initiated shutdown for {}", hp);
} catch (TException e) {
- throw new RuntimeException("Error invoking graceful shutdown for server:
" + hp, e);
+ log.warn("Failed to initiate shutdown for {}", hp, e);
} finally {
if (client != null) {
ThriftUtil.returnClient(client, context);
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java
b/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java
index b9484176c6..3d548c37cf 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java
@@ -262,8 +262,8 @@ public class ZooZap implements KeywordExecutable {
}
}
- private static void removeGroupedLocks(ZooReaderWriter zoo, String path,
- Predicate<String> groupPredicate, Predicate<HostAndPort>
hostPortPredicate, Opts opts)
+ static void removeGroupedLocks(ZooReaderWriter zoo, String path,
Predicate<String> groupPredicate,
+ Predicate<HostAndPort> hostPortPredicate, Opts opts)
throws KeeperException, InterruptedException {
if (zoo.exists(path)) {
List<String> groups = zoo.getChildren(path);
@@ -275,7 +275,7 @@ public class ZooZap implements KeywordExecutable {
}
}
- private static void removeLocks(ZooReaderWriter zoo, String path,
+ static void removeLocks(ZooReaderWriter zoo, String path,
Predicate<HostAndPort> hostPortPredicate, Opts opts)
throws KeeperException, InterruptedException {
if (zoo.exists(path)) {
@@ -293,7 +293,7 @@ public class ZooZap implements KeywordExecutable {
}
}
- private static void removeSingletonLock(ZooReaderWriter zoo, String path,
+ static void removeSingletonLock(ZooReaderWriter zoo, String path,
Predicate<HostAndPort> hostPortPredicate, Opts ops)
throws KeeperException, InterruptedException {
var lockData = ServiceLock.getLockData(zoo.getZooKeeper(),
ServiceLock.path(path));
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/GracefulShutdownIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/GracefulShutdownIT.java
index ad7aef847d..cbdb01b2f2 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/GracefulShutdownIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/GracefulShutdownIT.java
@@ -173,7 +173,7 @@ public class GracefulShutdownIT extends
SharedMiniClusterBase {
// Don't call `new Admin().execute(new String[] {"signalShutdown", "-h
", host, "-p ",
// Integer.toString(port)})`
// because this poisons the SingletonManager and puts it into SERVER mode
- Admin.signalGracefulShutdown(ctx, gcAddress.toString());
+ Admin.signalGracefulShutdown(ctx, gcAddress);
Wait.waitFor(() -> {
control.refreshProcesses(ServerType.GARBAGE_COLLECTOR);
return control.getProcesses(ServerType.GARBAGE_COLLECTOR).isEmpty();
@@ -183,7 +183,7 @@ public class GracefulShutdownIT extends
SharedMiniClusterBase {
final List<String> tservers =
client.instanceOperations().getTabletServers();
assertEquals(2, tservers.size());
final HostAndPort tserverAddress =
HostAndPort.fromString(tservers.get(0));
- Admin.signalGracefulShutdown(ctx, tserverAddress.toString());
+ Admin.signalGracefulShutdown(ctx, tserverAddress);
Wait.waitFor(() -> {
control.refreshProcesses(ServerType.TABLET_SERVER);
return control.getProcesses(ServerType.TABLET_SERVER).size() == 1;
@@ -197,7 +197,7 @@ public class GracefulShutdownIT extends
SharedMiniClusterBase {
final List<String> managers =
client.instanceOperations().getManagerLocations();
assertEquals(1, managers.size());
final HostAndPort managerAddress =
HostAndPort.fromString(managers.get(0));
- Admin.signalGracefulShutdown(ctx, managerAddress.toString());
+ Admin.signalGracefulShutdown(ctx, managerAddress);
Wait.waitFor(() -> {
control.refreshProcesses(ServerType.MANAGER);
return control.getProcesses(ServerType.MANAGER).isEmpty();
@@ -227,7 +227,7 @@ public class GracefulShutdownIT extends
SharedMiniClusterBase {
client.tableOperations().compact(tableName, cc);
Wait.waitFor(
() ->
ExternalCompactionTestUtils.getRunningCompactions(ctx).getCompactionsSize() >
0);
- Admin.signalGracefulShutdown(ctx, compactorAddress.toString());
+ Admin.signalGracefulShutdown(ctx, compactorAddress);
Wait.waitFor(() -> {
control.refreshProcesses(ServerType.COMPACTOR);
return control.getProcesses(ServerType.COMPACTOR).isEmpty();
@@ -252,7 +252,7 @@ public class GracefulShutdownIT extends
SharedMiniClusterBase {
assertNotNull(e);
count++;
if (count == 2) {
- Admin.signalGracefulShutdown(ctx, sserver.toString());
+ Admin.signalGracefulShutdown(ctx, sserver);
}
}
assertEquals(10, count);
