Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/939/
Java: 64bit/jdk-9.0.1 -XX:+UseCompressedOops -XX:+UseSerialGC

5 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.BasicZkTest

Error Message:
SolrCore.getOpenCount()==2

Stack Trace:
java.lang.RuntimeException: SolrCore.getOpenCount()==2
        at __randomizedtesting.SeedInfo.seed([B9A4F792DD50842B]:0)
        at org.apache.solr.util.TestHarness.close(TestHarness.java:379)
        at org.apache.solr.SolrTestCaseJ4.deleteCore(SolrTestCaseJ4.java:791)
        at 
org.apache.solr.cloud.AbstractZkTestCase.azt_afterClass(AbstractZkTestCase.java:147)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:564)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.base/java.lang.Thread.run(Thread.java:844)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.BasicZkTest

Error Message:
SolrCore.getOpenCount()==2

Stack Trace:
java.lang.RuntimeException: SolrCore.getOpenCount()==2
        at __randomizedtesting.SeedInfo.seed([B9A4F792DD50842B]:0)
        at org.apache.solr.util.TestHarness.close(TestHarness.java:379)
        at org.apache.solr.SolrTestCaseJ4.deleteCore(SolrTestCaseJ4.java:791)
        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:291)
        at jdk.internal.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
        at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:564)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.base/java.lang.Thread.run(Thread.java:844)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.BasicZkTest

Error Message:
26 threads leaked from SUITE scope at org.apache.solr.cloud.BasicZkTest:     1) 
Thread[id=4277, name=Connection evictor, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/java.lang.Thread.sleep(Native Method)         at 
app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
         at [email protected]/java.lang.Thread.run(Thread.java:844)    2) 
Thread[id=4274, name=zkCallback-657-thread-1, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)    3) 
Thread[id=4282, name=Thread-847, state=WAITING, group=TGRP-BasicZkTest]         
at [email protected]/java.lang.Object.wait(Native Method)         at 
[email protected]/java.lang.Object.wait(Object.java:516)         at 
app//org.apache.solr.core.CloserThread.run(CoreContainer.java:1717)    4) 
Thread[id=4279, name=ScheduledTrigger-2474-thread-1, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1131)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)    5) 
Thread[id=4290, name=zkCallback-657-thread-4, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)    6) 
Thread[id=4267, name=Connection evictor, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/java.lang.Thread.sleep(Native Method)         at 
app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
         at [email protected]/java.lang.Thread.run(Thread.java:844)    7) 
Thread[id=4292, name=Thread-848, state=WAITING, group=TGRP-BasicZkTest]         
at [email protected]/java.lang.Object.wait(Native Method)         at 
[email protected]/java.lang.Thread.join(Thread.java:1353)         at 
[email protected]/java.lang.Thread.join(Thread.java:1427)         at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
         at 
app//org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
         at 
app//org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)    8) 
Thread[id=4296, name=ProcessThread(sid:0 cport:34055):, state=WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
         at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
         at 
app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
    9) Thread[id=4280, name=ScheduledTrigger-2474-thread-2, state=WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1126)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   10) 
Thread[id=4272, 
name=SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker-SendThread(127.0.0.1:34055),
 state=RUNNABLE, group=TGRP-BasicZkTest]         at 
[email protected]/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         
at 
[email protected]/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265)   
      at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
         at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)   
      at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)   
      at 
app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
         at 
app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)   11) 
Thread[id=4289, name=zkCallback-657-thread-3, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   12) 
Thread[id=4286, name=searcherExecutor-2479-thread-1, state=WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
         at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   13) 
Thread[id=4293, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:34055, 
state=RUNNABLE, group=TGRP-BasicZkTest]         at 
[email protected]/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         
at 
[email protected]/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265)   
      at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
         at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)   
      at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)   
      at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   14) 
Thread[id=4273, 
name=SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker-EventThread, 
state=WAITING, group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
         at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
         at 
app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)   15) 
Thread[id=4294, name=SessionTracker, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at [email protected]/java.lang.Object.wait(Native 
Method)         at 
app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
   16) Thread[id=4291, name=ScheduledTrigger-2474-thread-4, state=WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1126)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   17) 
Thread[id=4268, name=Connection evictor, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/java.lang.Thread.sleep(Native Method)         at 
app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   18) 
Thread[id=4276, 
name=OverseerCollectionConfigSetProcessor-99121072745938947-88.99.242.108:8983_-n_0000000000,
 state=TIMED_WAITING, group=Overseer collection creation process.]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104)
         at 
app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435)
         at 
app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)
         at 
app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   19) 
Thread[id=4288, name=zkCallback-657-thread-2, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   20) 
Thread[id=4275, 
name=OverseerStateUpdate-99121072745938947-88.99.242.108:8983_-n_0000000000, 
state=TIMED_WAITING, group=Overseer state updater.]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104)
         at 
app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435)
         at 
app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:203)  
       at [email protected]/java.lang.Thread.run(Thread.java:844)   21) 
Thread[id=4283, name=watches-658-thread-1, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   22) 
Thread[id=4298, name=zkCallback-657-thread-6, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   23) 
Thread[id=4287, name=coreZkRegister-2469-thread-1, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   24) 
Thread[id=4278, 
name=OverseerAutoScalingTriggerThread-99121072745938947-88.99.242.108:8983_-n_0000000000,
 state=WAITING, group=Overseer autoscaling triggers]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
         at 
app//org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   25) 
Thread[id=4284, name=ScheduledTrigger-2474-thread-3, state=WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1126)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   26) 
Thread[id=4297, name=zkCallback-657-thread-5, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 26 threads leaked from 
SUITE scope at org.apache.solr.cloud.BasicZkTest: 
   1) Thread[id=4277, name=Connection evictor, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/java.lang.Thread.sleep(Native Method)
        at 
app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
   2) Thread[id=4274, name=zkCallback-657-thread-1, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
   3) Thread[id=4282, name=Thread-847, state=WAITING, group=TGRP-BasicZkTest]
        at [email protected]/java.lang.Object.wait(Native Method)
        at [email protected]/java.lang.Object.wait(Object.java:516)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:1717)
   4) Thread[id=4279, name=ScheduledTrigger-2474-thread-1, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1131)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
   5) Thread[id=4290, name=zkCallback-657-thread-4, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
   6) Thread[id=4267, name=Connection evictor, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/java.lang.Thread.sleep(Native Method)
        at 
app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
   7) Thread[id=4292, name=Thread-848, state=WAITING, group=TGRP-BasicZkTest]
        at [email protected]/java.lang.Object.wait(Native Method)
        at [email protected]/java.lang.Thread.join(Thread.java:1353)
        at [email protected]/java.lang.Thread.join(Thread.java:1427)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
        at 
app//org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
        at app//org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   8) Thread[id=4296, name=ProcessThread(sid:0 cport:34055):, state=WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
   9) Thread[id=4280, name=ScheduledTrigger-2474-thread-2, state=WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1126)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  10) Thread[id=4272, 
name=SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker-SendThread(127.0.0.1:34055),
 state=RUNNABLE, group=TGRP-BasicZkTest]
        at [email protected]/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at 
app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
        at 
app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
  11) Thread[id=4289, name=zkCallback-657-thread-3, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  12) Thread[id=4286, name=searcherExecutor-2479-thread-1, state=WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  13) Thread[id=4293, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:34055, 
state=RUNNABLE, group=TGRP-BasicZkTest]
        at [email protected]/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  14) Thread[id=4273, 
name=SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker-EventThread, 
state=WAITING, group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
  15) Thread[id=4294, name=SessionTracker, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/java.lang.Object.wait(Native Method)
        at 
app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
  16) Thread[id=4291, name=ScheduledTrigger-2474-thread-4, state=WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1126)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  17) Thread[id=4268, name=Connection evictor, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/java.lang.Thread.sleep(Native Method)
        at 
app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  18) Thread[id=4276, 
name=OverseerCollectionConfigSetProcessor-99121072745938947-88.99.242.108:8983_-n_0000000000,
 state=TIMED_WAITING, group=Overseer collection creation process.]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104)
        at 
app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435)
        at 
app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)
        at 
app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  19) Thread[id=4288, name=zkCallback-657-thread-2, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  20) Thread[id=4275, 
name=OverseerStateUpdate-99121072745938947-88.99.242.108:8983_-n_0000000000, 
state=TIMED_WAITING, group=Overseer state updater.]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104)
        at 
app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435)
        at 
app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:203)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  21) Thread[id=4283, name=watches-658-thread-1, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  22) Thread[id=4298, name=zkCallback-657-thread-6, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  23) Thread[id=4287, name=coreZkRegister-2469-thread-1, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  24) Thread[id=4278, 
name=OverseerAutoScalingTriggerThread-99121072745938947-88.99.242.108:8983_-n_0000000000,
 state=WAITING, group=Overseer autoscaling triggers]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
        at 
app//org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  25) Thread[id=4284, name=ScheduledTrigger-2474-thread-3, state=WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1126)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  26) Thread[id=4297, name=zkCallback-657-thread-5, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
        at __randomizedtesting.SeedInfo.seed([B9A4F792DD50842B]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.BasicZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=4274, name=zkCallback-657-thread-1, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)    2) 
Thread[id=4291, name=ScheduledTrigger-2474-thread-4, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1131)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)    3) 
Thread[id=4282, name=Thread-847, state=WAITING, group=TGRP-BasicZkTest]         
at [email protected]/java.lang.Object.wait(Native Method)         at 
[email protected]/java.lang.Object.wait(Object.java:516)         at 
app//org.apache.solr.core.CloserThread.run(CoreContainer.java:1717)    4) 
Thread[id=4279, name=ScheduledTrigger-2474-thread-1, state=WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1126)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)    5) 
Thread[id=4290, name=zkCallback-657-thread-4, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)    6) 
Thread[id=4288, name=zkCallback-657-thread-2, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)    7) 
Thread[id=4283, name=watches-658-thread-1, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)    8) 
Thread[id=4298, name=zkCallback-657-thread-6, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)    9) 
Thread[id=4280, name=ScheduledTrigger-2474-thread-2, state=WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1126)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   10) 
Thread[id=4272, 
name=SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker-SendThread(127.0.0.1:34055),
 state=TIMED_WAITING, group=TGRP-BasicZkTest]         at 
[email protected]/java.lang.Thread.sleep(Native Method)         at 
app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
         at 
app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
         at 
app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)   11) 
Thread[id=4289, name=zkCallback-657-thread-3, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   12) 
Thread[id=4286, name=searcherExecutor-2479-thread-1, state=WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
         at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   13) 
Thread[id=4287, name=coreZkRegister-2469-thread-1, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   14) 
Thread[id=4284, name=ScheduledTrigger-2474-thread-3, state=WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1126)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)   15) 
Thread[id=4297, name=zkCallback-657-thread-5, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
         at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
         at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
         at [email protected]/java.lang.Thread.run(Thread.java:844)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=4274, name=zkCallback-657-thread-1, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
   2) Thread[id=4291, name=ScheduledTrigger-2474-thread-4, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1131)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
   3) Thread[id=4282, name=Thread-847, state=WAITING, group=TGRP-BasicZkTest]
        at [email protected]/java.lang.Object.wait(Native Method)
        at [email protected]/java.lang.Object.wait(Object.java:516)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:1717)
   4) Thread[id=4279, name=ScheduledTrigger-2474-thread-1, state=WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1126)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
   5) Thread[id=4290, name=zkCallback-657-thread-4, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
   6) Thread[id=4288, name=zkCallback-657-thread-2, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
   7) Thread[id=4283, name=watches-658-thread-1, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
   8) Thread[id=4298, name=zkCallback-657-thread-6, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
   9) Thread[id=4280, name=ScheduledTrigger-2474-thread-2, state=WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1126)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  10) Thread[id=4272, 
name=SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker-SendThread(127.0.0.1:34055),
 state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at [email protected]/java.lang.Thread.sleep(Native Method)
        at 
app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at 
app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at 
app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
  11) Thread[id=4289, name=zkCallback-657-thread-3, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  12) Thread[id=4286, name=searcherExecutor-2479-thread-1, state=WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  13) Thread[id=4287, name=coreZkRegister-2469-thread-1, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  14) Thread[id=4284, name=ScheduledTrigger-2474-thread-3, state=WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1126)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
  15) Thread[id=4297, name=zkCallback-657-thread-5, state=TIMED_WAITING, 
group=TGRP-BasicZkTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at 
[email protected]/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at 
[email protected]/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
        at [email protected]/java.lang.Thread.run(Thread.java:844)
        at __randomizedtesting.SeedInfo.seed([B9A4F792DD50842B]:0)


FAILED:  
org.apache.solr.cloud.TestDeleteCollectionOnDownNodes.deleteCollectionWithDownNodes

Error Message:
Timed out waiting for leader elections null Live Nodes: [127.0.0.1:36495_solr, 
127.0.0.1:42315_solr] Last available state: 
DocCollection(halfdeletedcollection2//collections/halfdeletedcollection2/state.json/6)={
   "pullReplicas":"0",   "replicationFactor":"3",   "shards":{     "shard1":{   
    "range":"80000000-bfffffff",       "state":"active",       "replicas":{     
    "core_node3":{           "core":"halfdeletedcollection2_shard1_replica_n1", 
          "base_url":"http://127.0.0.1:36067/solr";,           
"node_name":"127.0.0.1:36067_solr",           "state":"active",           
"type":"NRT",           "leader":"true"},         "core_node5":{           
"core":"halfdeletedcollection2_shard1_replica_n2",           
"base_url":"http://127.0.0.1:36495/solr";,           
"node_name":"127.0.0.1:36495_solr",           "state":"down",           
"type":"NRT"},         "core_node6":{           
"core":"halfdeletedcollection2_shard1_replica_n4",           
"base_url":"http://127.0.0.1:42315/solr";,           
"node_name":"127.0.0.1:42315_solr",           "state":"down",           
"type":"NRT"}}},     "shard2":{       "range":"c0000000-ffffffff",       
"state":"active",       "replicas":{         "core_node8":{           
"core":"halfdeletedcollection2_shard2_replica_n7",           
"base_url":"http://127.0.0.1:34955/solr";,           
"node_name":"127.0.0.1:34955_solr",           "state":"down",           
"type":"NRT"},         "core_node11":{           
"core":"halfdeletedcollection2_shard2_replica_n9",           
"base_url":"http://127.0.0.1:36067/solr";,           
"node_name":"127.0.0.1:36067_solr",           "state":"down",           
"type":"NRT",           "leader":"true"},         "core_node13":{           
"core":"halfdeletedcollection2_shard2_replica_n10",           
"base_url":"http://127.0.0.1:36495/solr";,           
"node_name":"127.0.0.1:36495_solr",           "state":"down",           
"type":"NRT"}}},     "shard3":{       "range":"0-3fffffff",       
"state":"active",       "replicas":{         "core_node15":{           
"core":"halfdeletedcollection2_shard3_replica_n12",           
"base_url":"http://127.0.0.1:42315/solr";,           
"node_name":"127.0.0.1:42315_solr",           "state":"down",           
"type":"NRT"},         "core_node17":{           
"core":"halfdeletedcollection2_shard3_replica_n14",           
"base_url":"http://127.0.0.1:34955/solr";,           
"node_name":"127.0.0.1:34955_solr",           "state":"down",           
"type":"NRT",           "leader":"true"},         "core_node19":{           
"core":"halfdeletedcollection2_shard3_replica_n16",           
"base_url":"http://127.0.0.1:36067/solr";,           
"node_name":"127.0.0.1:36067_solr",           "state":"down",           
"type":"NRT"}}},     "shard4":{       "range":"40000000-7fffffff",       
"state":"active",       "replicas":{         "core_node21":{           
"core":"halfdeletedcollection2_shard4_replica_n18",           
"base_url":"http://127.0.0.1:36495/solr";,           
"node_name":"127.0.0.1:36495_solr",           "state":"down",           
"type":"NRT"},         "core_node23":{           
"core":"halfdeletedcollection2_shard4_replica_n20",           
"base_url":"http://127.0.0.1:42315/solr";,           
"node_name":"127.0.0.1:42315_solr",           "state":"down",           
"type":"NRT"},         "core_node24":{           
"core":"halfdeletedcollection2_shard4_replica_n22",           
"base_url":"http://127.0.0.1:34955/solr";,           
"node_name":"127.0.0.1:34955_solr",           "state":"down",           
"type":"NRT",           "leader":"true"}}}},   "router":{"name":"compositeId"}, 
  "maxShardsPerNode":"3",   "autoAddReplicas":"false",   "nrtReplicas":"3",   
"tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Timed out waiting for leader elections
null
Live Nodes: [127.0.0.1:36495_solr, 127.0.0.1:42315_solr]
Last available state: 
DocCollection(halfdeletedcollection2//collections/halfdeletedcollection2/state.json/6)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{
    "shard1":{
      "range":"80000000-bfffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"halfdeletedcollection2_shard1_replica_n1",
          "base_url":"http://127.0.0.1:36067/solr";,
          "node_name":"127.0.0.1:36067_solr",
          "state":"active",
          "type":"NRT",
          "leader":"true"},
        "core_node5":{
          "core":"halfdeletedcollection2_shard1_replica_n2",
          "base_url":"http://127.0.0.1:36495/solr";,
          "node_name":"127.0.0.1:36495_solr",
          "state":"down",
          "type":"NRT"},
        "core_node6":{
          "core":"halfdeletedcollection2_shard1_replica_n4",
          "base_url":"http://127.0.0.1:42315/solr";,
          "node_name":"127.0.0.1:42315_solr",
          "state":"down",
          "type":"NRT"}}},
    "shard2":{
      "range":"c0000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node8":{
          "core":"halfdeletedcollection2_shard2_replica_n7",
          "base_url":"http://127.0.0.1:34955/solr";,
          "node_name":"127.0.0.1:34955_solr",
          "state":"down",
          "type":"NRT"},
        "core_node11":{
          "core":"halfdeletedcollection2_shard2_replica_n9",
          "base_url":"http://127.0.0.1:36067/solr";,
          "node_name":"127.0.0.1:36067_solr",
          "state":"down",
          "type":"NRT",
          "leader":"true"},
        "core_node13":{
          "core":"halfdeletedcollection2_shard2_replica_n10",
          "base_url":"http://127.0.0.1:36495/solr";,
          "node_name":"127.0.0.1:36495_solr",
          "state":"down",
          "type":"NRT"}}},
    "shard3":{
      "range":"0-3fffffff",
      "state":"active",
      "replicas":{
        "core_node15":{
          "core":"halfdeletedcollection2_shard3_replica_n12",
          "base_url":"http://127.0.0.1:42315/solr";,
          "node_name":"127.0.0.1:42315_solr",
          "state":"down",
          "type":"NRT"},
        "core_node17":{
          "core":"halfdeletedcollection2_shard3_replica_n14",
          "base_url":"http://127.0.0.1:34955/solr";,
          "node_name":"127.0.0.1:34955_solr",
          "state":"down",
          "type":"NRT",
          "leader":"true"},
        "core_node19":{
          "core":"halfdeletedcollection2_shard3_replica_n16",
          "base_url":"http://127.0.0.1:36067/solr";,
          "node_name":"127.0.0.1:36067_solr",
          "state":"down",
          "type":"NRT"}}},
    "shard4":{
      "range":"40000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node21":{
          "core":"halfdeletedcollection2_shard4_replica_n18",
          "base_url":"http://127.0.0.1:36495/solr";,
          "node_name":"127.0.0.1:36495_solr",
          "state":"down",
          "type":"NRT"},
        "core_node23":{
          "core":"halfdeletedcollection2_shard4_replica_n20",
          "base_url":"http://127.0.0.1:42315/solr";,
          "node_name":"127.0.0.1:42315_solr",
          "state":"down",
          "type":"NRT"},
        "core_node24":{
          "core":"halfdeletedcollection2_shard4_replica_n22",
          "base_url":"http://127.0.0.1:34955/solr";,
          "node_name":"127.0.0.1:34955_solr",
          "state":"down",
          "type":"NRT",
          "leader":"true"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"3",
  "autoAddReplicas":"false",
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
        at 
__randomizedtesting.SeedInfo.seed([B9A4F792DD50842B:2791936AFB73C8A3]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
        at 
org.apache.solr.cloud.TestDeleteCollectionOnDownNodes.deleteCollectionWithDownNodes(TestDeleteCollectionOnDownNodes.java:47)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:564)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.base/java.lang.Thread.run(Thread.java:844)




Build Log:
[...truncated 12011 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicZkTest
   [junit4]   2> Creating dataDir: 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.BasicZkTest_B9A4F792DD50842B-001/init-core-data-001
   [junit4]   2> 517694 WARN  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] o.a.s.SolrTestCaseJ4 
startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 517695 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] o.a.s.SolrTestCaseJ4 
Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 517695 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] o.a.s.SolrTestCaseJ4 
Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0, 
clientAuth=0.0/0.0)
   [junit4]   2> 517695 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] o.a.s.SolrTestCaseJ4 
SecureRandom sanity checks: test.solr.allowed.securerandom=null & 
java.security.egd=file:/dev/./urandom
   [junit4]   2> 517695 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] o.a.s.c.ZkTestServer 
STARTING ZK TEST SERVER
   [junit4]   2> 517696 INFO  (Thread-846) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 517696 INFO  (Thread-846) [    ] o.a.s.c.ZkTestServer Starting 
server
   [junit4]   2> 517696 ERROR (Thread-846) [    ] o.a.z.s.ZooKeeperServer 
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action 
on ERROR or SHUTDOWN server state changes
   [junit4]   2> 517796 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] o.a.s.c.ZkTestServer 
start zk server on port:34055
   [junit4]   2> 517803 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
 to /configs/conf1/solrconfig.xml
   [junit4]   2> 517804 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml
 to /configs/conf1/schema.xml
   [junit4]   2> 517805 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
 to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 517806 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
 to /configs/conf1/stopwords.txt
   [junit4]   2> 517806 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt
 to /configs/conf1/protwords.txt
   [junit4]   2> 517806 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml
 to /configs/conf1/currency.xml
   [junit4]   2> 517807 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
 to /configs/conf1/enumsConfig.xml
   [junit4]   2> 517807 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
 to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 517808 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
 to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 517808 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
 to /configs/conf1/old_synonyms.txt
   [junit4]   2> 517809 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] 
o.a.s.c.AbstractZkTestCase put 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
 to /configs/conf1/synonyms.txt
   [junit4]   2> 517809 INFO  
(SUITE-BasicZkTest-seed#[B9A4F792DD50842B]-worker) [    ] o.a.s.SolrTestCaseJ4 
####initCore
   [junit4]   2> 517810 

[...truncated too long message...]

x:halfdeletedcollection2_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing 
metric reporters for 
registry=solr.core.halfdeletedcollection2.shard1.replica_n2, tag=404283744
   [junit4]   2> 853702 INFO  (coreCloseExecutor-2781-thread-2) 
[n:127.0.0.1:36495_solr c:halfdeletedcollection2 s:shard1 r:core_node5 
x:halfdeletedcollection2_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing 
reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6de6d986: rootName 
= solr_36495, domain = solr.core.halfdeletedcollection2.shard1.replica_n2, 
service url = null, agent id = null] for registry 
solr.core.halfdeletedcollection2.shard1.replica_n2 / 
com.codahale.metrics.MetricRegistry@4d8c20f0
   [junit4]   2> 853702 INFO  (coreCloseExecutor-2780-thread-1) 
[n:127.0.0.1:42315_solr c:halfdeletedcollection2 s:shard3 r:core_node15 
x:halfdeletedcollection2_shard3_replica_n12] o.a.s.m.SolrMetricManager Closing 
metric reporters for 
registry=solr.collection.halfdeletedcollection2.shard3.leader, tag=1003965107
   [junit4]   2> 853705 INFO  (coreCloseExecutor-2780-thread-2) 
[n:127.0.0.1:42315_solr c:halfdeletedcollection2 s:shard4 r:core_node23 
x:halfdeletedcollection2_shard4_replica_n20] o.a.s.m.SolrMetricManager Closing 
metric reporters for 
registry=solr.core.halfdeletedcollection2.shard4.replica_n20, tag=1811812505
   [junit4]   2> 853706 INFO  (coreCloseExecutor-2780-thread-2) 
[n:127.0.0.1:42315_solr c:halfdeletedcollection2 s:shard4 r:core_node23 
x:halfdeletedcollection2_shard4_replica_n20] o.a.s.m.r.SolrJmxReporter Closing 
reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3712577e: rootName 
= solr_42315, domain = solr.core.halfdeletedcollection2.shard4.replica_n20, 
service url = null, agent id = null] for registry 
solr.core.halfdeletedcollection2.shard4.replica_n20 / 
com.codahale.metrics.MetricRegistry@6310eeca
   [junit4]   2> 853712 INFO  (coreCloseExecutor-2781-thread-2) 
[n:127.0.0.1:36495_solr c:halfdeletedcollection2 s:shard1 r:core_node5 
x:halfdeletedcollection2_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing 
metric reporters for 
registry=solr.collection.halfdeletedcollection2.shard1.leader, tag=404283744
   [junit4]   2> 853712 INFO  (coreCloseExecutor-2781-thread-3) 
[n:127.0.0.1:36495_solr c:halfdeletedcollection2 s:shard4 r:core_node21 
x:halfdeletedcollection2_shard4_replica_n18] o.a.s.m.SolrMetricManager Closing 
metric reporters for 
registry=solr.core.halfdeletedcollection2.shard4.replica_n18, tag=653649006
   [junit4]   2> 853712 INFO  (coreCloseExecutor-2781-thread-3) 
[n:127.0.0.1:36495_solr c:halfdeletedcollection2 s:shard4 r:core_node21 
x:halfdeletedcollection2_shard4_replica_n18] o.a.s.m.r.SolrJmxReporter Closing 
reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@433738a8: rootName 
= solr_36495, domain = solr.core.halfdeletedcollection2.shard4.replica_n18, 
service url = null, agent id = null] for registry 
solr.core.halfdeletedcollection2.shard4.replica_n18 / 
com.codahale.metrics.MetricRegistry@c559930
   [junit4]   2> 853760 INFO  (coreCloseExecutor-2781-thread-3) 
[n:127.0.0.1:36495_solr c:halfdeletedcollection2 s:shard4 r:core_node21 
x:halfdeletedcollection2_shard4_replica_n18] o.a.s.m.SolrMetricManager Closing 
metric reporters for 
registry=solr.collection.halfdeletedcollection2.shard4.leader, tag=653649006
   [junit4]   2> 853760 INFO  (coreCloseExecutor-2781-thread-1) 
[n:127.0.0.1:36495_solr c:halfdeletedcollection2 s:shard2 r:core_node13 
x:halfdeletedcollection2_shard2_replica_n10] o.a.s.m.SolrMetricManager Closing 
metric reporters for 
registry=solr.collection.halfdeletedcollection2.shard2.leader, tag=1237859024
   [junit4]   2> 853762 INFO  (jetty-closer-889-thread-1) [    ] 
o.a.s.c.Overseer Overseer 
(id=99121086521409545-127.0.0.1:36495_solr-n_0000000000) closing
   [junit4]   2> 853762 INFO  
(OverseerStateUpdate-99121086521409545-127.0.0.1:36495_solr-n_0000000000) 
[n:127.0.0.1:36495_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 
127.0.0.1:36495_solr
   [junit4]   2> 853762 INFO  (coreCloseExecutor-2780-thread-2) 
[n:127.0.0.1:42315_solr c:halfdeletedcollection2 s:shard4 r:core_node23 
x:halfdeletedcollection2_shard4_replica_n20] o.a.s.m.SolrMetricManager Closing 
metric reporters for 
registry=solr.collection.halfdeletedcollection2.shard4.leader, tag=1811812505
   [junit4]   2> 853762 WARN  
(OverseerAutoScalingTriggerThread-99121086521409545-127.0.0.1:36495_solr-n_0000000000)
 [n:127.0.0.1:36495_solr    ] o.a.s.c.a.OverseerTriggerThread 
OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 853766 INFO  (coreCloseExecutor-2780-thread-3) 
[n:127.0.0.1:42315_solr c:halfdeletedcollection2 s:shard1 r:core_node6 
x:halfdeletedcollection2_shard1_replica_n4] o.a.s.m.SolrMetricManager Closing 
metric reporters for 
registry=solr.core.halfdeletedcollection2.shard1.replica_n4, tag=1120931106
   [junit4]   2> 853766 INFO  (coreCloseExecutor-2780-thread-3) 
[n:127.0.0.1:42315_solr c:halfdeletedcollection2 s:shard1 r:core_node6 
x:halfdeletedcollection2_shard1_replica_n4] o.a.s.m.r.SolrJmxReporter Closing 
reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6888edf8: rootName 
= solr_42315, domain = solr.core.halfdeletedcollection2.shard1.replica_n4, 
service url = null, agent id = null] for registry 
solr.core.halfdeletedcollection2.shard1.replica_n4 / 
com.codahale.metrics.MetricRegistry@29903ad2
   [junit4]   2> 853767 INFO  (jetty-closer-889-thread-1) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@3063e4f7{/solr,null,UNAVAILABLE}
   [junit4]   2> 853767 INFO  
(zkCallback-904-thread-4-processing-n:127.0.0.1:42315_solr) 
[n:127.0.0.1:42315_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:42315_solr
   [junit4]   2> 853771 INFO  (coreCloseExecutor-2780-thread-3) 
[n:127.0.0.1:42315_solr c:halfdeletedcollection2 s:shard1 r:core_node6 
x:halfdeletedcollection2_shard1_replica_n4] o.a.s.m.SolrMetricManager Closing 
metric reporters for 
registry=solr.collection.halfdeletedcollection2.shard1.leader, tag=1120931106
   [junit4]   2> 853772 INFO  (jetty-closer-889-thread-2) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@528502ff{/solr,null,UNAVAILABLE}
   [junit4]   2> 853773 ERROR 
(SUITE-TestDeleteCollectionOnDownNodes-seed#[B9A4F792DD50842B]-worker) [    ] 
o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper 
server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 853773 INFO  
(SUITE-TestDeleteCollectionOnDownNodes-seed#[B9A4F792DD50842B]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:38859 38859
   [junit4]   2> 859554 INFO  (Thread-1559) [    ] o.a.s.c.ZkTestServer 
connecting to 127.0.0.1:38859 38859
   [junit4]   2> 859554 WARN  (Thread-1559) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        5       /solr/aliases.json
   [junit4]   2>        5       /solr/clusterprops.json
   [junit4]   2>        4       /solr/security.json
   [junit4]   2>        4       /solr/configs/conf
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        29      
/solr/collections/halfdeletedcollection2/state.json
   [junit4]   2>        5       /solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        5       /solr/live_nodes
   [junit4]   2>        5       /solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestDeleteCollectionOnDownNodes_B9A4F792DD50842B-001
   [junit4]   2> NOTE: test params are: codec=Lucene70, 
sim=RandomSimilarity(queryNorm=true): {}, locale=ak-GH, 
timezone=America/Fortaleza
   [junit4]   2> NOTE: Linux 4.10.0-40-generic amd64/Oracle Corporation 9.0.1 
(64-bit)/cpus=8,threads=1,free=197633072,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [RankQueryTest, 
JavabinLoaderTest, TestMaxTokenLenTokenizer, BlockJoinFacetSimpleTest, 
TestSimpleTrackingShardHandler, DistributedTermsComponentTest, 
TestQuerySenderNoQuery, TestStressRecovery, BlockCacheTest, ZkSolrClientTest, 
ChangedSchemaMergeTest, HttpPartitionTest, OpenCloseCoreStressTest, 
TestRuleBasedAuthorizationPlugin, TestFastWriter, MetricUtilsTest, 
HdfsChaosMonkeySafeLeaderTest, BitVectorTest, InfoHandlerTest, AssignTest, 
TestCollectionAPI, SchemaVersionSpecificBehaviorTest, QueryEqualityTest, 
TestBlobHandler, SpellCheckCollatorWithCollapseTest, 
TestCollationFieldDocValues, TestUniqueKeyFieldResource, HdfsDirectoryTest, 
TestOmitPositions, PreAnalyzedFieldTest, TestSizeLimitedDistributedMap, 
TestTolerantUpdateProcessorCloud, SampleTest, DistributedVersionInfoTest, 
PreAnalyzedFieldManagedSchemaCloudTest, TestApiFramework, TestSSLRandomization, 
RegexBoostProcessorTest, ConvertedLegacyTest, TestCollapseQParserPlugin, 
DeleteShardTest, TestDistribIDF, TestAtomicUpdateErrorCases, 
CollectionsAPIDistributedZkTest, SynonymTokenizerTest, 
SpellingQueryConverterTest, TestBackupRepositoryFactory, JSONWriterTest, 
CursorMarkTest, TestUseDocValuesAsStored2, AtomicUpdateProcessorFactoryTest, 
TestSubQueryTransformerCrossCore, TestCloudRecovery, TestInfoStreamLogging, 
DocumentAnalysisRequestHandlerTest, ChaosMonkeySafeLeaderWithPullReplicasTest, 
TestJsonRequest, OpenExchangeRatesOrgProviderTest, TestRemoteStreaming, 
ClassificationUpdateProcessorIntegrationTest, ExitableDirectoryReaderTest, 
SolrIndexConfigTest, TermVectorComponentDistributedTest, TestStressReorder, 
BlobRepositoryCloudTest, TestBM25SimilarityFactory, TestJmxIntegration, 
TestAuthenticationFramework, TestRawResponseWriter, TestLRUStatsCache, 
CircularListTest, SolrCoreMetricManagerTest, 
TestManagedSynonymGraphFilterFactory, TestSolrCLIRunExample, MBeansHandlerTest, 
TestPseudoReturnFields, TestDeleteCollectionOnDownNodes]
   [junit4] Completed [636/751 (2!)] on J1 in 131.65s, 1 test, 1 failure <<< 
FAILURES!

[...truncated 40615 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to