nick-boss-tech commented on code in PR #5301: URL: https://github.com/apache/solr/pull/5301#discussion_r4636518546
########## solr/core/src/test/org/apache/solr/cloud/ConnectionManagerStaleExpiryTest.java: ########## @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import java.io.IOException; +import java.net.URI; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.config.Property; +import org.apache.solr.SolrTestCase; +import org.apache.solr.client.solrj.cloud.SocketProxy; +import org.apache.solr.common.cloud.ConnectionManager; +import org.apache.solr.common.cloud.DefaultConnectionStrategy; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.util.SuppressForbidden; +import org.apache.solr.common.util.TimeSource; +import org.apache.solr.util.TimeOut; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooKeeper; +import org.junit.Test; + +/** Exercises expiries that are queued behind other connection events. */ +@SuppressForbidden(reason = "needs a log4j appender to delay a connection callback") +public class ConnectionManagerStaleExpiryTest extends SolrTestCase { + private static final int WAIT_SECONDS = 20; + + @Test + public void testStaleExpiryDoesNotReplaceRecoveredSession() throws Exception { + ZkTestServer server = new ZkTestServer(createTempDir("zk-replacement")); + SocketProxy proxy = new SocketProxy(); + RecordingStrategy strategy = new RecordingStrategy(); + CallbackAppender appender = new CallbackAppender(proxy); + LoggerContext context = (LoggerContext) LogManager.getContext(false); + Configuration config = context.getConfiguration(); + String loggerName = ConnectionManager.class.getName(); + LoggerConfig previousLogger = config.getLoggers().get(loggerName); + LoggerConfig logger = new LoggerConfig(loggerName, Level.DEBUG, true); + boolean serverStarted = false; + boolean proxyStarted = false; + appender.start(); + logger.addAppender(appender, Level.DEBUG, null); + config.addLogger(loggerName, logger); + context.updateLoggers(); + try { + server.run(); + serverStarted = true; + proxy.open(new URI("http://127.0.0.1:" + server.getPort())); + proxyStarted = true; + CountDownLatch reconnected = new CountDownLatch(1); + CountDownLatch allowReconnectReturn = new CountDownLatch(1); + AtomicInteger reconnectCalls = new AtomicInteger(); + AtomicBoolean reconnectHoldTimedOut = new AtomicBoolean(); + try (SolrZkClient client = + new SolrZkClient.Builder() + .withUrl("127.0.0.1:" + proxy.getListenPort()) + .withTimeout(20, TimeUnit.SECONDS) + .withConnTimeOut(10, TimeUnit.SECONDS) + .withConnStrategy(strategy) + .withReconnectListener( + () -> { + if (reconnectCalls.incrementAndGet() == 1) { + reconnected.countDown(); + try { + if (!allowReconnectReturn.await(60, TimeUnit.SECONDS)) { + reconnectHoldTimedOut.set(true); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + reconnectHoldTimedOut.set(true); + } + } + }) + .build()) { + try { + ConnectionManager manager = client.getConnectionManager(); + long sessionA = client.getZooKeeper().getSessionId(); + appender.arm(manager.toString()); + server.expire(sessionA); + assertTrue( + "replacement session was not created", + strategy.replacementCreated.await(WAIT_SECONDS, TimeUnit.SECONDS)); + assertTrue( + "replacement connected callback was not held", + appender.connectedHeld.await(WAIT_SECONDS, TimeUnit.SECONDS)); + ZooKeeper replacement = strategy.replacement.get(); + assertTrue( + "replacement must have a real connected session", + replacement.getState().isConnected()); + long sessionB = replacement.getSessionId(); + assertNotEquals(sessionA, sessionB); + + proxy.halfClose(); + assertTrue( + "replacement disconnect was not delivered", + strategy.replacementDisconnected.await(WAIT_SECONDS, TimeUnit.SECONDS)); + assertTrue( + "replacement disconnect was not applied", + appender.disconnected.await(WAIT_SECONDS, TimeUnit.SECONDS)); + assertFalse(manager.isConnected()); + server.expire(sessionB); + // A direct client verifies the server is reachable while the replacement is expired. + try (SolrZkClient control = + new SolrZkClient.Builder() + .withUrl(server.getZkAddress()) + .withTimeout(20, TimeUnit.SECONDS) + .withConnTimeOut(10, TimeUnit.SECONDS) + .build()) { + assertTrue(control.getZooKeeper().getState().isConnected()); + } + proxy.goOn(); + assertTrue( + "replacement expiry was not delivered", + strategy.replacementExpired.await(WAIT_SECONDS, TimeUnit.SECONDS)); + new TimeOut(WAIT_SECONDS, TimeUnit.SECONDS, TimeSource.NANO_TIME) + .waitFor("replacement was not terminal", () -> !replacement.getState().isAlive()); + + // Keep C from connecting until the stale B callback has actually been rejected. + assertTrue( + "next replacement was not requested", + strategy.thirdCreationRequested.await(WAIT_SECONDS, TimeUnit.SECONDS)); + appender.releaseConnected.countDown(); + assertTrue( + "delayed connected callback was not rejected", + appender.staleConnected.await(WAIT_SECONDS, TimeUnit.SECONDS)); + assertFalse("connected callback hold timed out", appender.holdTimedOut.get()); + strategy.allowThirdCreation.countDown(); + assertTrue( + "connection manager did not recover", + reconnected.await(WAIT_SECONDS, TimeUnit.SECONDS)); + ZooKeeper sessionC = client.getZooKeeper(); + assertNotEquals(sessionB, sessionC.getSessionId()); + assertTrue(sessionC.getState().isConnected()); + allowReconnectReturn.countDown(); + assertTrue( + "queued replacement expiry was not processed", + appender.replacementExpiryProcessed.await(WAIT_SECONDS, TimeUnit.SECONDS)); + assertFalse("expiry callback hold timed out", appender.expiryHoldTimedOut.get()); + assertFalse("reconnect callback hold timed out", reconnectHoldTimedOut.get()); + assertTrue( + "stale expiry disconnected the recovered session", manager.isConnectedAndNotClosed()); + assertSame("stale expiry replaced the recovered keeper", sessionC, client.getZooKeeper()); + assertEquals("stale expiry created another keeper", 3, strategy.attempts.get()); + assertEquals("stale expiry invoked another reconnect callback", 1, reconnectCalls.get()); + } finally { + allowReconnectReturn.countDown(); + appender.allowExpiryReturn.countDown(); + appender.releaseConnected.countDown(); + strategy.allowThirdCreation.countDown(); + proxy.goOn(); + } + } + } finally { + appender.releaseConnected.countDown(); + appender.allowExpiryReturn.countDown(); + strategy.allowThirdCreation.countDown(); + config.removeLogger(loggerName); + if (previousLogger != null) { + config.addLogger(loggerName, previousLogger); + } + context.updateLoggers(); + appender.stop(); + if (proxyStarted) { + proxy.close(); + } + if (serverStarted) { + server.shutdown(); + } + } + } + + /** + * The reconnect for session A is still running when session B expires, so B's expiry waits in the + * queue. A late event from the replaced session A is applied in the meantime. B's expiry must + * still be acted on. + */ + @Test + public void testLateEventFromReplacedSessionDoesNotDropCurrentExpiry() throws Exception { + ZkTestServer server = new ZkTestServer(createTempDir("zk-late-event")); + RecordingStrategy strategy = new RecordingStrategy(); + strategy.allowThirdCreation.countDown(); + CountDownLatch reconnected = new CountDownLatch(1); + CountDownLatch allowReconnectReturn = new CountDownLatch(1); + AtomicInteger reconnectCalls = new AtomicInteger(); + try { + server.run(); + try (SolrZkClient client = + new SolrZkClient.Builder() + .withUrl(server.getZkAddress()) + .withTimeout(20, TimeUnit.SECONDS) + .withConnTimeOut(10, TimeUnit.SECONDS) + .withConnStrategy(strategy) + .withReconnectListener( + () -> { + if (reconnectCalls.incrementAndGet() == 1) { + reconnected.countDown(); + try { + allowReconnectReturn.await(60, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + }) + .build()) { + try { + ConnectionManager manager = client.getConnectionManager(); + server.expire(client.getZooKeeper().getSessionId()); + assertTrue( + "replacement session was not installed", + reconnected.await(WAIT_SECONDS, TimeUnit.SECONDS)); + long sessionB = client.getZooKeeper().getSessionId(); + + // B's expiry is delivered, but cannot be handled while the reconnect above is running. + server.expire(sessionB); + assertTrue( + "replacement expiry was not delivered", + strategy.replacementExpired.await(WAIT_SECONDS, TimeUnit.SECONDS)); + + strategy + .watchers + .get(1) + .process( + new WatchedEvent( + Watcher.Event.EventType.None, Watcher.Event.KeeperState.Disconnected, null)); + new TimeOut(WAIT_SECONDS, TimeUnit.SECONDS, TimeSource.NANO_TIME) + .waitFor("late event was not applied", () -> !manager.isConnected()); Review Comment: 🤖 *AI text below* 🤖 Could we wait for a signal tied to the injected callback rather than `!manager.isConnected()`? B’s real disconnect can already make that predicate true, so the test can release the queued expiry before A’s injected callback has run. If we also add the generation check to normal callbacks, I’d have this assert that A’s callback was ignored and that B’s expiry still triggers recovery. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
