Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1401/

8 tests failed.
FAILED:  org.apache.solr.cloud.TestLocalFSCloudBackupRestore.test

Error Message:
expected:<COMPLETED> but was:<RUNNING>

Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
        at 
__randomizedtesting.SeedInfo.seed([317B471B1312A108:B92F78C1BDEECCF0]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:147)
        at 
org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:277)
        at 
org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.TriLevelCompositeIdRoutingTest

Error Message:
1 thread leaked from SUITE scope at 
org.apache.solr.cloud.TriLevelCompositeIdRoutingTest:     1) Thread[id=6819, 
name=zkCallback-818-thread-1, state=TIMED_WAITING, 
group=TGRP-TriLevelCompositeIdRoutingTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE 
scope at org.apache.solr.cloud.TriLevelCompositeIdRoutingTest: 
   1) Thread[id=6819, name=zkCallback-818-thread-1, state=TIMED_WAITING, 
group=TGRP-TriLevelCompositeIdRoutingTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([317B471B1312A108]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.TriLevelCompositeIdRoutingTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=6819, name=zkCallback-818-thread-1, state=TIMED_WAITING, 
group=TGRP-TriLevelCompositeIdRoutingTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=6819, name=zkCallback-818-thread-1, state=TIMED_WAITING, 
group=TGRP-TriLevelCompositeIdRoutingTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([317B471B1312A108]:0)


FAILED:  
org.apache.solr.client.solrj.impl.CloudSolrClientTest.testOverwriteOption

Error Message:
Could not load collection from ZK: overwrite

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: 
overwrite
        at 
__randomizedtesting.SeedInfo.seed([F4F708FE41B56716:18CCE9152FE518C0]:0)
        at 
org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
        at 
org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:690)
        at 
org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:154)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClientTest.testOverwriteOption(CloudSolrClientTest.java:155)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:110)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: 
KeeperErrorCode = Session expired for /collections/overwrite/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at 
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at 
org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1182)
        at 
org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1168)
        ... 44 more


FAILED:  org.apache.lucene.index.TestDuelingCodecsAtNight.testBigEquals

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([687DBA3F681653F7]:0)


FAILED:  
junit.framework.TestSuite.org.apache.lucene.index.TestDuelingCodecsAtNight

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([687DBA3F681653F7]:0)


FAILED:  org.apache.solr.security.TestPKIAuthenticationPlugin.test

Error Message:


Stack Trace:
java.lang.NullPointerException
        at 
__randomizedtesting.SeedInfo.seed([317B471B1312A108:B92F78C1BDEECCF0]:0)
        at 
org.apache.solr.security.TestPKIAuthenticationPlugin.test(TestPKIAuthenticationPlugin.java:100)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test

Error Message:
There are still nodes recoverying - waited for 600 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 600 
seconds
        at 
__randomizedtesting.SeedInfo.seed([317B471B1312A108:B92F78C1BDEECCF0]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:914)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1471)
        at 
org.apache.solr.cloud.TlogReplayBufferedWhileIndexingTest.test(TlogReplayBufferedWhileIndexingTest.java:99)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 2201 lines...]
   [junit4] Suite: org.apache.lucene.index.TestDuelingCodecsAtNight
   [junit4]   2> ??? 12, 2017 2:19:53 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: 
org.apache.lucene.index.TestDuelingCodecsAtNight
   [junit4]   2>    1) Thread[id=5590, 
name=SUITE-TestDuelingCodecsAtNight-seed#[687DBA3F681653F7], state=RUNNABLE, 
group=TGRP-TestDuelingCodecsAtNight]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native 
Method)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    2) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    3) Thread[id=5591, 
name=TEST-TestDuelingCodecsAtNight.testBigEquals-seed#[687DBA3F681653F7], 
state=RUNNABLE, group=TGRP-TestDuelingCodecsAtNight]
   [junit4]   2>         at 
org.apache.lucene.store.MockIndexInputWrapper.readByte(MockIndexInputWrapper.java:140)
   [junit4]   2>         at 
org.apache.lucene.codecs.simpletext.SimpleTextUtil.readLine(SimpleTextUtil.java:59)
   [junit4]   2>         at 
org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.nextDoc(SimpleTextFieldsReader.java:409)
   [junit4]   2>         at 
org.apache.lucene.search.DocIdSetIterator.slowAdvance(DocIdSetIterator.java:165)
   [junit4]   2>         at 
org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.advance(SimpleTextFieldsReader.java:448)
   [junit4]   2>         at 
org.apache.lucene.index.CheckIndex.testTermVectors(CheckIndex.java:2475)
   [junit4]   2>         at 
org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:753)
   [junit4]   2>         at 
org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:300)
   [junit4]   2>         at 
org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:865)
   [junit4]   2>         at 
org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
   [junit4]   2>         at 
org.apache.lucene.util.IOUtils.close(IOUtils.java:76)
   [junit4]   2>         at 
org.apache.lucene.index.TestDuelingCodecs.tearDown(TestDuelingCodecs.java:98)
   [junit4]   2>         at 
sun.reflect.GeneratedMethodAccessor13.invoke(Unknown Source)
   [junit4]   2>         at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    4) Thread[id=9, name=JUnit4-serializer-daemon, 
state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at 
com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestDuelingCodecsAtNight -Dtests.method=testBigEquals 
-Dtests.seed=687DBA3F681653F7 -Dtests.multiplier=2 -Dtests.nightly=true 
-Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
 -Dtests.locale=sr-RS -Dtests.timezone=Africa/Dakar -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   7201s J1 | TestDuelingCodecsAtNight.testBigEquals <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because 
suite timeout was reached.
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([687DBA3F681653F7]:0)
   [junit4]   2> ??? 12, 2017 2:19:53 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked 
thread(s).
   [junit4]   2> ??? 12, 2017 2:20:13 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at 
org.apache.lucene.index.TestDuelingCodecsAtNight: 
   [junit4]   2>    1) Thread[id=5591, 
name=TEST-TestDuelingCodecsAtNight.testBigEquals-seed#[687DBA3F681653F7], 
state=RUNNABLE, group=TGRP-TestDuelingCodecsAtNight]
   [junit4]   2>         at 
org.apache.lucene.store.MockIndexInputWrapper.readByte(MockIndexInputWrapper.java:140)
   [junit4]   2>         at 
org.apache.lucene.codecs.simpletext.SimpleTextUtil.readLine(SimpleTextUtil.java:59)
   [junit4]   2>         at 
org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.nextDoc(SimpleTextFieldsReader.java:409)
   [junit4]   2>         at 
org.apache.lucene.search.DocIdSetIterator.slowAdvance(DocIdSetIterator.java:165)
   [junit4]   2>         at 
org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.advance(SimpleTextFieldsReader.java:448)
   [junit4]   2>         at 
org.apache.lucene.index.CheckIndex.testTermVectors(CheckIndex.java:2475)
   [junit4]   2>         at 
org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:753)
   [junit4]   2>         at 
org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:300)
   [junit4]   2>         at 
org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:865)
   [junit4]   2>         at 
org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
   [junit4]   2>         at 
org.apache.lucene.util.IOUtils.close(IOUtils.java:76)
   [junit4]   2>         at 
org.apache.lucene.index.TestDuelingCodecs.tearDown(TestDuelingCodecs.java:98)
   [junit4]   2>         at 
sun.reflect.GeneratedMethodAccessor13.invoke(Unknown Source)
   [junit4]   2>         at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> ??? 12, 2017 2:20:13 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=5591, 
name=TEST-TestDuelingCodecsAtNight.testBigEquals-seed#[687DBA3F681653F7], 
state=RUNNABLE, group=TGRP-TestDuelingCodecsAtNight]
   [junit4]   2> ??? 12, 2017 2:20:16 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be 
terminated:
   [junit4]   2>    1) Thread[id=5591, 
name=TEST-TestDuelingCodecsAtNight.testBigEquals-seed#[687DBA3F681653F7], 
state=RUNNABLE, group=TGRP-TestDuelingCodecsAtNight]
   [junit4]   2>         at 
org.apache.lucene.store.MockIndexInputWrapper.getFilePointer(MockIndexInputWrapper.java:122)
   [junit4]   2>         at 
org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.nextDoc(SimpleTextFieldsReader.java:408)
   [junit4]   2>         at 
org.apache.lucene.search.DocIdSetIterator.slowAdvance(DocIdSetIterator.java:165)
   [junit4]   2>         at 
org.apache.lucene.codecs.simpletext.SimpleTextFieldsReader$SimpleTextPostingsEnum.advance(SimpleTextFieldsReader.java:448)
   [junit4]   2>         at 
org.apache.lucene.index.CheckIndex.testTermVectors(CheckIndex.java:2475)
   [junit4]   2>         at 
org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:753)
   [junit4]   2>         at 
org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:300)
   [junit4]   2>         at 
org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:865)
   [junit4]   2>         at 
org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
   [junit4]   2>         at 
org.apache.lucene.util.IOUtils.close(IOUtils.java:76)
   [junit4]   2>         at 
org.apache.lucene.index.TestDuelingCodecs.tearDown(TestDuelingCodecs.java:98)
   [junit4]   2>         at 
sun.reflect.GeneratedMethodAccessor13.invoke(Unknown Source)
   [junit4]   2>         at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, 
docValues:{}, maxPointsInLeafNode=419, maxMBSortInHeap=7.163448164681653, 
sim=RandomSimilarity(queryNorm=true): {titleTokenized=DFR GLZ(0.3), body=DFR 
I(n)L3(800.0)}, locale=sr-RS, timezone=Africa/Dakar
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 
1.8.0_144 (64-bit)/cpus=4,threads=2,free=247517408,total=530055168
   [junit4]   2> NOTE: All tests run in this JVM: [TestBytesRefAttImpl, 
TestIndexWriterOutOfFileDescriptors, TestNumericDocValuesUpdates, 
TestLucene50StoredFieldsFormat, Test2BDocs, TestPrefixInBooleanQuery, 
TestSpanExplanations, TestConcurrentMergeScheduler, TestOperations, 
TestFilterIterator, TestRAMDirectory, TestRollback, TestBKD, TestMaxPosition, 
TestPhrasePrefixQuery, TestRecyclingIntBlockAllocator, 
TestTimeLimitingCollector, TestPolygon, TestTragicIndexWriterDeadlock, 
TestTermVectorsReader, TestPhraseQuery, Test2BPostingsBytes, 
TestNativeFSLockFactory, TestGrowableByteArrayDataOutput, 
TestSameScoresWithThreads, TestIndexWriterUnicode, TestReusableStringReader, 
TestDocValuesQueries, TestTrackingDirectoryWrapper, TestDeterminism, TestNorms, 
TestSpansEnum, TestCheckIndex, TestLockFactory, TestFieldValueQuery, 
TestSearcherManager, TestLongBitSet, TestOmitPositions, 
TestRecyclingByteBlockAllocator, TestTransactions, 
TestScoreCachingWrappingScorer, FiniteStringsIteratorTest, 
TestFieldInvertState, TestSpanCollection, TestIndexWriterOnJRECrash, 
TestFileSwitchDirectory, TestGraphTokenizers, TestMinimize, 
TestSortedNumericSortField, TestFilterDirectoryReader, 
TestCloseableThreadLocal, TestForTooMuchCloning, TestPointValues, 
TestClassicSimilarity, TestGraphTokenStreamFiniteStrings, TestSynonymQuery, 
TestMultiPhraseEnum, TestCodecUtil, TestToken, TestAddIndexes, 
TestParallelReaderEmptyIndex, TestTieredMergePolicy, TestVirtualMethod, 
TestSegmentInfos, TestOfflineSorter, TestSimpleFSLockFactory, 
TestParallelLeafReader, TestDocIdSetBuilder, TestAxiomaticSimilarity, 
TestIndexFileDeleter, TestByteSlices, TestComplexExplanations, 
TestSpanNotQuery, TestDocumentsWriterDeleteQueue, TestDemo, TestCharArraySet, 
TestIndexWriterMerging, TestFSTs, TestMultiMMap, TestBooleanOr, TestBasics, 
TestIndexWriterDelete, TestIndexWriterMergePolicy, TestDuelingCodecs, 
TestBoolean2, TestAtomicUpdate, TestStressAdvance, TestDirectoryReaderReopen, 
TestSearchWithThreads, TestSearchAfter, TestHighCompressionMode, 
TestDocumentsWriterStallControl, TestArrayUtil, TestLevenshteinAutomata, 
TestBufferedIndexInput, TestTermsEnum, TestCodecHoldsOpenFiles, 
TestReaderClosed, TestNGramPhraseQuery, TestTermdocPerf, TestIndexCommit, 
TestTerm, TestPositiveScoresOnlyCollector, TestNewestSegment, 
TestByteArrayDataInput, Test2BNumericDocValues, TestCharArrayMap, 
TestStopFilter, TestBlockPostingsFormat, TestBlockPostingsFormat3, TestForUtil, 
TestLucene50CompoundFormat, TestLucene50TermVectorsFormat, 
TestLucene60FieldInfoFormat, TestLucene60PointsFormat, 
TestLucene70DocValuesFormat, TestLucene70SegmentInfoFormat, TestFieldType, 
TestGeoEncodingUtils, TestGeoUtils, TestPolygon2D, 
Test2BSortedDocValuesFixedSorted, TestAllFilesDetectTruncation, 
TestAllFilesHaveChecksumFooter, TestCustomTermFreq, TestDocIDMerger, 
TestDocInverterPerFieldErrorInfo, TestDuelingCodecsAtNight]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestDuelingCodecsAtNight -Dtests.seed=687DBA3F681653F7 
-Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
 -Dtests.locale=sr-RS -Dtests.timezone=Africa/Dakar -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestDuelingCodecsAtNight (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 
7200000 msec).
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([687DBA3F681653F7]:0)
   [junit4] Completed [409/455 (1!)] on J1 in 7224.88s, 1 test, 2 errors <<< 
FAILURES!

[...truncated 10644 lines...]
   [junit4] Suite: org.apache.solr.security.TestPKIAuthenticationPlugin
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.security.TestPKIAuthenticationPlugin_317B471B1312A108-001/init-core-data-001
   [junit4]   2> 181512 WARN  
(SUITE-TestPKIAuthenticationPlugin-seed#[317B471B1312A108]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=9 numCloses=9
   [junit4]   2> 181512 INFO  
(SUITE-TestPKIAuthenticationPlugin-seed#[317B471B1312A108]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 181514 INFO  
(SUITE-TestPKIAuthenticationPlugin-seed#[317B471B1312A108]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 181514 INFO  
(SUITE-TestPKIAuthenticationPlugin-seed#[317B471B1312A108]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 181565 INFO  
(TEST-TestPKIAuthenticationPlugin.test-seed#[317B471B1312A108]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 195496 ERROR 
(TEST-TestPKIAuthenticationPlugin.test-seed#[317B471B1312A108]) [    ] 
o.a.s.s.PKIAuthenticationPlugin Invalid key request timestamp: 1507839220544 , 
received timestamp: 1507839233359 , TTL: 10000
   [junit4]   2> 195497 INFO  
(TEST-TestPKIAuthenticationPlugin.test-seed#[317B471B1312A108]) [    ] 
o.a.s.SolrTestCaseJ4 ###Ending test
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestPKIAuthenticationPlugin -Dtests.method=test 
-Dtests.seed=317B471B1312A108 -Dtests.multiplier=2 -Dtests.nightly=true 
-Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
 -Dtests.locale=ar-LY -Dtests.timezone=Europe/Simferopol -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   14.0s J0 | TestPKIAuthenticationPlugin.test <<<
   [junit4]    > Throwable #1: java.lang.NullPointerException
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([317B471B1312A108:B92F78C1BDEECCF0]:0)
   [junit4]    >        at 
org.apache.solr.security.TestPKIAuthenticationPlugin.test(TestPKIAuthenticationPlugin.java:100)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.security.TestPKIAuthenticationPlugin_317B471B1312A108-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, 
docValues:{}, maxPointsInLeafNode=1661, maxMBSortInHeap=7.1266937688702345, 
sim=RandomSimilarity(queryNorm=false): {}, locale=ar-LY, 
timezone=Europe/Simferopol
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 
1.8.0_144 (64-bit)/cpus=4,threads=1,free=350441992,total=430964736
   [junit4]   2> NOTE: All tests run in this JVM: [TestInitQParser, TestDocSet, 
TestBulkSchemaAPI, TestPKIAuthenticationPlugin]
   [junit4] Completed [10/745 (1!)] on J0 in 14.35s, 1 test, 1 error <<< 
FAILURES!

[...truncated 551 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestLocalFSCloudBackupRestore
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/init-core-data-001
   [junit4]   2> 3299920 WARN  
(SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=13 numCloses=13
   [junit4]   2> 3299920 INFO  
(SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 3299921 INFO  
(SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 3299921 INFO  
(SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 3299956 INFO  
(SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001
   [junit4]   2> 3299957 INFO  
(SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 3300141 INFO  (Thread-3150) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3300141 INFO  (Thread-3150) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 3300141 INFO  
(SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:39230
   [junit4]   2> 3300142 ERROR (Thread-3150) [    ] o.a.z.s.ZooKeeperServer 
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action 
on ERROR or SHUTDOWN server state changes
   [junit4]   2> 3300256 INFO  (jetty-launcher-1118-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 3300372 INFO  (jetty-launcher-1118-thread-2) [    ] 
o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 3300373 INFO  (jetty-launcher-1118-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@16e77be4{/solr,null,AVAILABLE}
   [junit4]   2> 3300374 INFO  (jetty-launcher-1118-thread-2) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@1f4721df{HTTP/1.1,[http/1.1]}{127.0.0.1:47294}
   [junit4]   2> 3300374 INFO  (jetty-launcher-1118-thread-2) [    ] 
o.e.j.s.Server Started @3319957ms
   [junit4]   2> 3300374 INFO  (jetty-launcher-1118-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=47294}
   [junit4]   2> 3300374 ERROR (jetty-launcher-1118-thread-2) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 3300374 INFO  (jetty-launcher-1118-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
8.0.0
   [junit4]   2> 3300374 INFO  (jetty-launcher-1118-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3300374 INFO  (jetty-launcher-1118-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config 
dir: null
   [junit4]   2> 3300374 INFO  (jetty-launcher-1118-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-10-12T21:05:37.238Z
   [junit4]   2> 3300376 INFO  (jetty-launcher-1118-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@486f5f7e{/solr,null,AVAILABLE}
   [junit4]   2> 3300376 INFO  (jetty-launcher-1118-thread-1) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@4ddefc04{HTTP/1.1,[http/1.1]}{127.0.0.1:57269}
   [junit4]   2> 3300376 INFO  (jetty-launcher-1118-thread-1) [    ] 
o.e.j.s.Server Started @3319960ms
   [junit4]   2> 3300465 INFO  (jetty-launcher-1118-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=57269}
   [junit4]   2> 3300465 ERROR (jetty-launcher-1118-thread-1) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 3300465 INFO  (jetty-launcher-1118-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
8.0.0
   [junit4]   2> 3300465 INFO  (jetty-launcher-1118-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3300465 INFO  (jetty-launcher-1118-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config 
dir: null
   [junit4]   2> 3300465 INFO  (jetty-launcher-1118-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-10-12T21:05:37.329Z
   [junit4]   2> 3300466 INFO  (jetty-launcher-1118-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3300492 INFO  (jetty-launcher-1118-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3300529 INFO  (jetty-launcher-1118-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39230/solr
   [junit4]   2> 3300547 INFO  (jetty-launcher-1118-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39230/solr
   [junit4]   2> 3300746 INFO  (jetty-launcher-1118-thread-1) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 3300747 INFO  (jetty-launcher-1118-thread-1) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:57269_solr
   [junit4]   2> 3300753 INFO  (jetty-launcher-1118-thread-1) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.Overseer Overseer 
(id=98817955398025222-127.0.0.1:57269_solr-n_0000000000) starting
   [junit4]   2> 3300882 INFO  (jetty-launcher-1118-thread-1) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:57269_solr
   [junit4]   2> 3300896 INFO  
(zkCallback-1130-thread-1-processing-n:127.0.0.1:57269_solr) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 3301253 INFO  (jetty-launcher-1118-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 3301254 INFO  (jetty-launcher-1118-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 3301381 INFO  (jetty-launcher-1118-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:47294_solr
   [junit4]   2> 3301457 INFO  
(zkCallback-1130-thread-1-processing-n:127.0.0.1:57269_solr) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 3301471 INFO  
(zkCallback-1129-thread-1-processing-n:127.0.0.1:47294_solr) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 3302098 INFO  (jetty-launcher-1118-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_47294.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3302180 INFO  (jetty-launcher-1118-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_47294.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3302180 INFO  (jetty-launcher-1118-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_47294.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3302237 INFO  (jetty-launcher-1118-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/.
   [junit4]   2> 3302962 INFO  (jetty-launcher-1118-thread-1) 
[n:127.0.0.1:57269_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_57269.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3303027 INFO  (jetty-launcher-1118-thread-1) 
[n:127.0.0.1:57269_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_57269.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3303028 INFO  (jetty-launcher-1118-thread-1) 
[n:127.0.0.1:57269_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_57269.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3303049 INFO  (jetty-launcher-1118-thread-1) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/.
   [junit4]   2> 3303233 INFO  
(SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3303234 INFO  
(SUITE-TestLocalFSCloudBackupRestore-seed#[317B471B1312A108]-worker) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39230/solr ready
   [junit4]   2> 3304021 INFO  
(TEST-TestLocalFSCloudBackupRestore.test-seed#[317B471B1312A108]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 3304025 INFO  (qtp1531139025-8209) [n:127.0.0.1:47294_solr    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
pullReplicas=1&replicationFactor=2&property.customKey=customValue&collection.configName=conf1&maxShardsPerNode=8&name=backuprestore&nrtReplicas=2&action=CREATE&numShards=2&tlogReplicas=1&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 3304027 INFO  
(OverseerThreadFactory-2370-thread-1-processing-n:127.0.0.1:57269_solr) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
backuprestore
   [junit4]   2> 3304045 WARN  
(OverseerThreadFactory-2370-thread-1-processing-n:127.0.0.1:57269_solr) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.CreateCollectionCmd Specified number of 
replicas of 4 on collection backuprestore is higher than the number of Solr 
instances currently live or live and part of your createNodeSet(2). It's 
unusual to run two replica of the same slice on the same Solr-instance.
   [junit4]   2> 3304183 INFO  
(OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"backuprestore_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:47294/solr";,
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 3304201 INFO  
(OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"backuprestore_shard1_replica_n3",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:57269/solr";,
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 3304204 INFO  
(OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"backuprestore_shard1_replica_t5",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:47294/solr";,
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 3304354 INFO  
(OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"backuprestore_shard1_replica_p7",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:57269/solr";,
   [junit4]   2>   "type":"PULL"} 
   [junit4]   2> 3304359 INFO  
(OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"backuprestore_shard2_replica_n9",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:47294/solr";,
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 3304361 INFO  
(OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"backuprestore_shard2_replica_n11",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:57269/solr";,
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 3304364 INFO  
(OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"backuprestore_shard2_replica_t13",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:47294/solr";,
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 3304366 INFO  
(OverseerStateUpdate-98817955398025222-127.0.0.1:57269_solr-n_0000000000) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"backuprestore_shard2_replica_p14",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:57269/solr";,
   [junit4]   2>   "type":"PULL"} 
   [junit4]   2> 3304539 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node2&name=backuprestore_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin
   [junit4]   2> 3304539 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr    
] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 
2147483647 transient cores
   [junit4]   2> 3304885 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node8&name=backuprestore_shard1_replica_p7&action=CREATE&numShards=2&shard=shard1&wt=javabin
   [junit4]   2> 3304885 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node15&name=backuprestore_shard2_replica_t13&action=CREATE&numShards=2&shard=shard2&wt=javabin
   [junit4]   2> 3304885 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 
transient cores
   [junit4]   2> 3304886 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node4&name=backuprestore_shard1_replica_n3&action=CREATE&numShards=2&shard=shard1&wt=javabin
   [junit4]   2> 3304886 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node6&name=backuprestore_shard1_replica_t5&action=CREATE&numShards=2&shard=shard1&wt=javabin
   [junit4]   2> 3304886 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node10&name=backuprestore_shard2_replica_n9&action=CREATE&numShards=2&shard=shard2&wt=javabin
   [junit4]   2> 3304889 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node16&name=backuprestore_shard2_replica_p14&action=CREATE&numShards=2&shard=shard2&wt=javabin
   [junit4]   2> 3305034 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node12&name=backuprestore_shard2_replica_n11&action=CREATE&numShards=2&shard=shard2&wt=javabin
   [junit4]   2> 3305161 INFO  
(zkCallback-1129-thread-1-processing-n:127.0.0.1:47294_solr) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/backuprestore/state.json] for collection [backuprestore] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3305162 INFO  
(zkCallback-1130-thread-2-processing-n:127.0.0.1:57269_solr) 
[n:127.0.0.1:57269_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/backuprestore/state.json] for collection [backuprestore] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3306028 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard1 r:core_node6 x:backuprestore_shard1_replica_t5] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306042 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n9] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306058 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306077 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard2 r:core_node15 x:backuprestore_shard2_replica_t13] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306084 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard2 r:core_node16 x:backuprestore_shard2_replica_p14] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306132 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] 
o.a.s.s.IndexSchema [backuprestore_shard1_replica_n3] Schema name=minimal
   [junit4]   2> 3306271 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306271 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] 
o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_n3' using 
configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306272 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_57269.solr.core.backuprestore.shard1.replica_n3' (registry 
'solr.core.backuprestore.shard1.replica_n3') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306272 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306272 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] 
o.a.s.c.SolrCore [[backuprestore_shard1_replica_n3] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/backuprestore_shard1_replica_n3],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/./backuprestore_shard1_replica_n3/data/]
   [junit4]   2> 3306289 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_n11] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306292 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_p7] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306314 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard2 r:core_node16 x:backuprestore_shard2_replica_p14] 
o.a.s.s.IndexSchema [backuprestore_shard2_replica_p14] Schema name=minimal
   [junit4]   2> 3306316 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard2 r:core_node16 x:backuprestore_shard2_replica_p14] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306316 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard2 r:core_node16 x:backuprestore_shard2_replica_p14] 
o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_p14' 
using configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306332 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard2 r:core_node15 x:backuprestore_shard2_replica_t13] 
o.a.s.s.IndexSchema [backuprestore_shard2_replica_t13] Schema name=minimal
   [junit4]   2> 3306353 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard1 r:core_node6 x:backuprestore_shard1_replica_t5] 
o.a.s.s.IndexSchema [backuprestore_shard1_replica_t5] Schema name=minimal
   [junit4]   2> 3306355 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard1 r:core_node6 x:backuprestore_shard1_replica_t5] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306355 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard1 r:core_node6 x:backuprestore_shard1_replica_t5] 
o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_t5' using 
configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306356 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard1 r:core_node6 x:backuprestore_shard1_replica_t5] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_47294.solr.core.backuprestore.shard1.replica_t5' (registry 
'solr.core.backuprestore.shard1.replica_t5') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306356 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard1 r:core_node6 x:backuprestore_shard1_replica_t5] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306356 INFO  (qtp1531139025-8213) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard1 r:core_node6 x:backuprestore_shard1_replica_t5] 
o.a.s.c.SolrCore [[backuprestore_shard1_replica_t5] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/backuprestore_shard1_replica_t5],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/./backuprestore_shard1_replica_t5/data/]
   [junit4]   2> 3306158 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n9] 
o.a.s.s.IndexSchema [backuprestore_shard2_replica_n9] Schema name=minimal
   [junit4]   2> 3306139 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica_n1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3306385 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard2 r:core_node16 x:backuprestore_shard2_replica_p14] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_57269.solr.core.backuprestore.shard2.replica_p14' (registry 
'solr.core.backuprestore.shard2.replica_p14') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306385 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard2 r:core_node16 x:backuprestore_shard2_replica_p14] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306385 INFO  (qtp229150333-8221) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard2 r:core_node16 x:backuprestore_shard2_replica_p14] 
o.a.s.c.SolrCore [[backuprestore_shard2_replica_p14] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/backuprestore_shard2_replica_p14],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/./backuprestore_shard2_replica_p14/data/]
   [junit4]   2> 3306387 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n9] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306387 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n9] 
o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_n9' using 
configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306387 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n9] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_47294.solr.core.backuprestore.shard2.replica_n9' (registry 
'solr.core.backuprestore.shard2.replica_n9') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306387 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n9] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306387 INFO  (qtp1531139025-8206) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n9] 
o.a.s.c.SolrCore [[backuprestore_shard2_replica_n9] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/backuprestore_shard2_replica_n9],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/./backuprestore_shard2_replica_n9/data/]
   [junit4]   2> 3306412 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard2 r:core_node15 x:backuprestore_shard2_replica_t13] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306412 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard2 r:core_node15 x:backuprestore_shard2_replica_t13] 
o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_t13' 
using configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306415 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_n11] 
o.a.s.s.IndexSchema [backuprestore_shard2_replica_n11] Schema name=minimal
   [junit4]   2> 3306435 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_p7] 
o.a.s.s.IndexSchema [backuprestore_shard1_replica_p7] Schema name=minimal
   [junit4]   2> 3306454 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica_n1] 
o.a.s.s.IndexSchema [backuprestore_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 3306456 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica_n1] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306456 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica_n1] 
o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_n1' using 
configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306497 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard2 r:core_node15 x:backuprestore_shard2_replica_t13] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_47294.solr.core.backuprestore.shard2.replica_t13' (registry 
'solr.core.backuprestore.shard2.replica_t13') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306497 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard2 r:core_node15 x:backuprestore_shard2_replica_t13] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306497 INFO  (qtp1531139025-8211) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard2 r:core_node15 x:backuprestore_shard2_replica_t13] 
o.a.s.c.SolrCore [[backuprestore_shard2_replica_t13] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/backuprestore_shard2_replica_t13],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/./backuprestore_shard2_replica_t13/data/]
   [junit4]   2> 3306499 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_p7] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306500 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_p7] 
o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_p7' using 
configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306500 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_p7] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_57269.solr.core.backuprestore.shard1.replica_p7' (registry 
'solr.core.backuprestore.shard1.replica_p7') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306500 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_p7] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306500 INFO  (qtp229150333-8218) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_p7] 
o.a.s.c.SolrCore [[backuprestore_shard1_replica_p7] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/backuprestore_shard1_replica_p7],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/./backuprestore_shard1_replica_p7/data/]
   [junit4]   2> 3306502 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_n11] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3306502 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_n11] 
o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_n11' 
using configuration from collection backuprestore, trusted=true
   [junit4]   2> 3306518 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_n11] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_57269.solr.core.backuprestore.shard2.replica_n11' (registry 
'solr.core.backuprestore.shard2.replica_n11') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306518 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_n11] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306518 INFO  (qtp229150333-8220) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_n11] 
o.a.s.c.SolrCore [[backuprestore_shard2_replica_n11] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/backuprestore_shard2_replica_n11],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node1/./backuprestore_shard2_replica_n11/data/]
   [junit4]   2> 3306725 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica_n1] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_47294.solr.core.backuprestore.shard1.replica_n1' (registry 
'solr.core.backuprestore.shard1.replica_n1') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@47e6ecf2
   [junit4]   2> 3306725 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica_n1] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3306725 INFO  (qtp1531139025-8207) [n:127.0.0.1:47294_solr 
c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica_n1] 
o.a.s.c.SolrCore [[backuprestore_shard1_replica_n1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/backuprestore_shard1_replica_n1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_317B471B1312A108-001/tempDir-001/node2/./backuprestore_shard1_replica_n1/data/]
   [junit4]   2> 3307948 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 3307948 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3307949 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3307949 INFO  (qtp229150333-8217) [n:127.0.0.1:57269_solr 
c:backuprestore s:shard1 r:core_node4 x:backuprestore_shard1_replica_n3] 
o.a.s.u.CommitTracker Soft AutoCommit

[...truncated too long message...]

 INFO  (coreCloseExecutor-884-thread-1) [n:127.0.0.1:46367_solr c:foo s:shard1 
r:core_node4 x:foo_shard1_replica_n3] o.a.s.m.r.SolrJmxReporter Closing 
reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2bd59d31: rootName 
= solr_46367, domain = solr.core.foo.shard1.replica_n3, service url = null, 
agent id = null] for registry solr.core.foo.shard1.replica_n3 / 
com.codahale.metrics.MetricRegistry@282db3d0
   [junit4]   2> 695386 INFO  (coreCloseExecutor-885-thread-4) 
[n:127.0.0.1:57824_solr c:multicollection1 s:shard2 r:core_node4 
x:multicollection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.core.multicollection1.shard2.replica_n2, 
tag=1468663982
   [junit4]   2> 695386 INFO  (coreCloseExecutor-885-thread-4) 
[n:127.0.0.1:57824_solr c:multicollection1 s:shard2 r:core_node4 
x:multicollection1_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing 
reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@31e90732: rootName 
= solr_57824, domain = solr.core.multicollection1.shard2.replica_n2, service 
url = null, agent id = null] for registry 
solr.core.multicollection1.shard2.replica_n2 / 
com.codahale.metrics.MetricRegistry@5c9c280d
   [junit4]   2> 695386 INFO  (coreCloseExecutor-883-thread-4) 
[n:127.0.0.1:51897_solr c:multicollection1 s:shard1 r:core_node3 
x:multicollection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.collection.multicollection1.shard1.leader, 
tag=81560481
   [junit4]   2> 695636 INFO  (coreCloseExecutor-885-thread-5) 
[n:127.0.0.1:57824_solr c:localShardsTestColl s:shard1 r:core_node7 
x:localShardsTestColl_shard1_replica_n4] o.a.s.m.SolrMetricManager Closing 
metric reporters for registry=solr.core.localShardsTestColl.shard1.replica_n4, 
tag=1722637580
   [junit4]   2> 695636 INFO  (coreCloseExecutor-885-thread-5) 
[n:127.0.0.1:57824_solr c:localShardsTestColl s:shard1 r:core_node7 
x:localShardsTestColl_shard1_replica_n4] o.a.s.m.r.SolrJmxReporter Closing 
reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@66117c90: rootName 
= solr_57824, domain = solr.core.localShardsTestColl.shard1.replica_n4, service 
url = null, agent id = null] for registry 
solr.core.localShardsTestColl.shard1.replica_n4 / 
com.codahale.metrics.MetricRegistry@3ba7c26c
   [junit4]   2> 695818 INFO  (coreCloseExecutor-884-thread-1) 
[n:127.0.0.1:46367_solr c:foo s:shard1 r:core_node4 x:foo_shard1_replica_n3] 
o.a.s.m.SolrMetricManager Closing metric reporters for 
registry=solr.collection.foo.shard1.leader, tag=1727311193
   [junit4]   2> 695845 INFO  (coreCloseExecutor-884-thread-3) 
[n:127.0.0.1:46367_solr c:multicollection2 s:shard2 r:core_node4 
x:multicollection2_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.collection.multicollection2.shard2.leader, 
tag=82390428
   [junit4]   2> 695850 INFO  (coreCloseExecutor-885-thread-6) 
[n:127.0.0.1:57824_solr c:localShardsTestColl s:shard3 r:core_node18 
x:localShardsTestColl_shard3_replica_n16] o.a.s.m.SolrMetricManager Closing 
metric reporters for registry=solr.core.localShardsTestColl.shard3.replica_n16, 
tag=150773247
   [junit4]   2> 695850 INFO  (coreCloseExecutor-885-thread-6) 
[n:127.0.0.1:57824_solr c:localShardsTestColl s:shard3 r:core_node18 
x:localShardsTestColl_shard3_replica_n16] o.a.s.m.r.SolrJmxReporter Closing 
reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1a4d9292: rootName 
= solr_57824, domain = solr.core.localShardsTestColl.shard3.replica_n16, 
service url = null, agent id = null] for registry 
solr.core.localShardsTestColl.shard3.replica_n16 / 
com.codahale.metrics.MetricRegistry@aa96c9a
   [junit4]   2> 695871 INFO  (coreCloseExecutor-885-thread-5) 
[n:127.0.0.1:57824_solr c:localShardsTestColl s:shard1 r:core_node7 
x:localShardsTestColl_shard1_replica_n4] o.a.s.m.SolrMetricManager Closing 
metric reporters for 
registry=solr.collection.localShardsTestColl.shard1.leader, tag=1722637580
   [junit4]   2> 695878 INFO  (coreCloseExecutor-885-thread-4) 
[n:127.0.0.1:57824_solr c:multicollection1 s:shard2 r:core_node4 
x:multicollection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.collection.multicollection1.shard2.leader, 
tag=1468663982
   [junit4]   2> 695954 INFO  (coreCloseExecutor-885-thread-6) 
[n:127.0.0.1:57824_solr c:localShardsTestColl s:shard3 r:core_node18 
x:localShardsTestColl_shard3_replica_n16] o.a.s.m.SolrMetricManager Closing 
metric reporters for 
registry=solr.collection.localShardsTestColl.shard3.leader, tag=150773247
   [junit4]   2> 696970 WARN  
(zkCallback-119-thread-2-processing-n:127.0.0.1:51897_solr) 
[n:127.0.0.1:51897_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 696971 INFO  (jetty-closer-104-thread-1) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@2ee099e6{/solr,null,UNAVAILABLE}
   [junit4]   2> 696987 INFO  
(zkCallback-120-thread-5-processing-n:127.0.0.1:57824_solr) 
[n:127.0.0.1:57824_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (1)
   [junit4]   2> 696987 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from 
client sessionid 0x15f12b48d610007, likely client has closed socket
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 697042 INFO  (jetty-closer-104-thread-3) [    ] 
o.a.s.c.Overseer Overseer 
(id=98818283500797961-127.0.0.1:57824_solr-n_0000000000) closing
   [junit4]   2> 697042 INFO  
(OverseerStateUpdate-98818283500797961-127.0.0.1:57824_solr-n_0000000000) 
[n:127.0.0.1:57824_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 
127.0.0.1:57824_solr
   [junit4]   2> 697106 WARN  
(OverseerAutoScalingTriggerThread-98818283500797961-127.0.0.1:57824_solr-n_0000000000)
 [n:127.0.0.1:57824_solr    ] o.a.s.c.a.OverseerTriggerThread 
OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 698486 WARN  
(zkCallback-118-thread-3-processing-n:127.0.0.1:46367_solr) 
[n:127.0.0.1:46367_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 698486 INFO  (jetty-closer-104-thread-2) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@cd6badc{/solr,null,UNAVAILABLE}
   [junit4]   2> 698634 WARN  
(zkCallback-120-thread-5-processing-n:127.0.0.1:57824_solr) 
[n:127.0.0.1:57824_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 698634 INFO  (jetty-closer-104-thread-3) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@3fba707c{/solr,null,UNAVAILABLE}
   [junit4]   2> 698650 ERROR 
(SUITE-CloudSolrClientTest-seed#[F4F708FE41B56716]-worker) [    ] 
o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper 
server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 698651 INFO  
(SUITE-CloudSolrClientTest-seed#[F4F708FE41B56716]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:33819 33819
   [junit4]   2> 698802 INFO  (Thread-410) [    ] o.a.s.c.ZkTestServer 
connecting to 127.0.0.1:33819 33819
   [junit4]   2> 698803 WARN  (Thread-410) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        8       /solr/aliases.json
   [junit4]   2>        4       /solr/configs/conf
   [junit4]   2>        3       /solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        30      /solr/collections/foo/state.json
   [junit4]   2>        30      /solr/collections/localShardsTestColl/state.json
   [junit4]   2>        17      /solr/collections/collection1/state.json
   [junit4]   2>        16      /solr/collections/2nd_collection/state.json
   [junit4]   2>        14      /solr/collections/nemesis/state.json
   [junit4]   2>        14      /solr/collections/multicollection2/state.json
   [junit4]   2>        14      /solr/collections/multicollection1/state.json
   [junit4]   2>        8       /solr/clusterprops.json
   [junit4]   2>        8       /solr/clusterstate.json
   [junit4]   2>        6       /solr/collections/overwrite/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        8       /solr/live_nodes
   [junit4]   2>        8       /solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-solrj/test/J2/temp/solr.client.solrj.impl.CloudSolrClientTest_F4F708FE41B56716-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), 
sim=RandomSimilarity(queryNorm=false): {}, locale=ar-QA, timezone=Europe/Warsaw
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 
1.8.0_144 (64-bit)/cpus=4,threads=1,free=172377944,total=410517504
   [junit4]   2> NOTE: All tests run in this JVM: [SchemaTest, 
TestJavaBinCodec, LessThanEvaluatorTest, LargeVolumeJettyTest, 
EmpiricalDistributionEvaluatorTest, FieldValueEvaluatorTest, TestCoreAdmin, 
TestV1toV2ApiMapper, NotEvaluatorTest, SquareRootEvaluatorTest, 
TestToleratedUpdateError, ArcSineEvaluatorTest, GreaterThanEvaluatorTest, 
SolrExampleBinaryTest, TestHash, SolrExampleJettyTest, 
ModifiableSolrParamsTest, SolrParamTest, JettyWebappTest, 
CloudSolrClientCacheTest, StreamExpressionParserTest, TestPolicy, 
AscEvaluatorTest, CloudSolrClientMultiConstructorTest, CloudSolrClientTest]
   [junit4] Completed [101/145 (1!)] on J2 in 189.35s, 15 tests, 1 error <<< 
FAILURES!

[...truncated 46084 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to