Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/205/

6 tests failed.
FAILED:  
org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testUpdateLogSynchronisation

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete 
@source_collection:shard1
        at 
__randomizedtesting.SeedInfo.seed([BEA4C44DC413E54:F58514E71E611D45]:0)
        at 
org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:795)
        at 
org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testUpdateLogSynchronisation(CdcrReplicationDistributedZkTest.java:376)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR

Error Message:
There are still nodes recoverying - waited for 330 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 
seconds
        at 
__randomizedtesting.SeedInfo.seed([BEA4C44DC413E54:D5602239E7E35CA7]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:184)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:141)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:136)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:856)
        at 
org.apache.solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR(LeaderInitiatedRecoveryOnShardRestartTest.java:158)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
junit.framework.TestSuite.org.apache.solr.handler.PingRequestHandlerTest

Error Message:
1 thread leaked from SUITE scope at 
org.apache.solr.handler.PingRequestHandlerTest:     1) Thread[id=17009, 
name=OverseerHdfsCoreFailoverThread-96970927691005965-127.0.0.1:37855_solr-n_0000000002,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE 
scope at org.apache.solr.handler.PingRequestHandlerTest: 
   1) Thread[id=17009, 
name=OverseerHdfsCoreFailoverThread-96970927691005965-127.0.0.1:37855_solr-n_0000000002,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
        at __randomizedtesting.SeedInfo.seed([BEA4C44DC413E54]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler

Error Message:
ObjectTracker found 13 object(s) that were not released!!! 
[NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, 
NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, 
NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, 
NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, 
NRTCachingDirectory] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
  at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
  at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
  at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
  at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
  at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
  at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
  at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
  at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
  at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
  at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
  at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
  at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
  at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 13 object(s) that were not 
released!!! [NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, 
NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, 
NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, 
NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, 
NRTCachingDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
        at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
        at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
        at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
        at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
        at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
        at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
        at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
        at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
        at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
        at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
        at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
        at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
        at 
org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
        at java.lang.Thread.run(Thread.java:745)


        at __randomizedtesting.SeedInfo.seed([BEA4C44DC413E54]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:260)
        at sun.reflect.GeneratedMethodAccessor24.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.testCollectionsAPI

Error Message:
Expected to see collection awhollynewcollection_0 null Last available state: 
DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/3)={
   "replicationFactor":"1",   "shards":{     "shard1":{       
"range":"80000000-bfffffff",       "state":"active",       "replicas":{}},     
"shard2":{       "range":"c0000000-ffffffff",       "state":"active",       
"replicas":{}},     "shard3":{       "range":"0-3fffffff",       
"state":"active",       "replicas":{}},     "shard4":{       
"range":"40000000-7fffffff",       "state":"active",       "replicas":{}}},   
"router":{"name":"compositeId"},   "maxShardsPerNode":"2",   
"autoAddReplicas":"false"}

Stack Trace:
java.lang.AssertionError: Expected to see collection awhollynewcollection_0
null
Last available state: 
DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/3)={
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-bfffffff",
      "state":"active",
      "replicas":{}},
    "shard2":{
      "range":"c0000000-ffffffff",
      "state":"active",
      "replicas":{}},
    "shard3":{
      "range":"0-3fffffff",
      "state":"active",
      "replicas":{}},
    "shard4":{
      "range":"40000000-7fffffff",
      "state":"active",
      "replicas":{}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"2",
  "autoAddReplicas":"false"}
        at 
__randomizedtesting.SeedInfo.seed([BEA4C44DC413E54:439F38F0DA7211C1]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:236)
        at 
org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:496)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.testSpecificConfigsets

Error Message:
KeeperErrorCode = NoNode for /collections/withconfigset2

Stack Trace:
org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode 
for /collections/withconfigset2
        at 
__randomizedtesting.SeedInfo.seed([BEA4C44DC413E54:2694031E2B658458]:0)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:356)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:353)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at 
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:353)
        at 
org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testSpecificConfigsets(CollectionsAPIDistributedZkTest.java:425)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11597 lines...]
   [junit4] Suite: 
org.apache.solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/init-core-data-001
   [junit4]   2> 519210 INFO  
(SUITE-LeaderInitiatedRecoveryOnShardRestartTest-seed#[BEA4C44DC413E54]-worker) 
[    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 519210 INFO  
(SUITE-LeaderInitiatedRecoveryOnShardRestartTest-seed#[BEA4C44DC413E54]-worker) 
[    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: 
/
   [junit4]   2> 519213 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 519217 INFO  (Thread-2403) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 519217 INFO  (Thread-2403) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 519317 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.ZkTestServer start zk server on port:49442
   [junit4]   2> 519330 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
 to /configs/conf1/solrconfig.xml
   [junit4]   2> 519333 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml
 to /configs/conf1/schema.xml
   [junit4]   2> 519334 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
 to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 519336 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
 to /configs/conf1/stopwords.txt
   [junit4]   2> 519337 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt
 to /configs/conf1/protwords.txt
   [junit4]   2> 519339 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml
 to /configs/conf1/currency.xml
   [junit4]   2> 519340 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
 to /configs/conf1/enumsConfig.xml
   [junit4]   2> 519342 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
 to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 519343 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
 to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 519345 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
 to /configs/conf1/old_synonyms.txt
   [junit4]   2> 519354 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
 to /configs/conf1/synonyms.txt
   [junit4]   2> 519465 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/control-001/cores/collection1
   [junit4]   2> 519466 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 519468 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@67dd179b{/,null,AVAILABLE}
   [junit4]   2> 519473 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.e.j.s.ServerConnector Started 
ServerConnector@391d9d0c{HTTP/1.1,[http/1.1]}{127.0.0.1:39164}
   [junit4]   2> 519474 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.e.j.s.Server Started @523930ms
   [junit4]   2> 519474 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/tempDir-001/control/data,
 hostContext=/, hostPort=39164, 
coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/control-001/cores}
   [junit4]   2> 519474 ERROR 
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging 
may be missing or incomplete.
   [junit4]   2> 519474 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? 
version 6.4.0
   [junit4]   2> 519474 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on 
port null
   [junit4]   2> 519474 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 519474 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-11-20T15:41:42.622Z
   [junit4]   2> 519483 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 519483 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/control-001/solr.xml
   [junit4]   2> 519521 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with 
params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 519544 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:49442/solr
   [junit4]   2> 519661 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [n:127.0.0.1:39164_    ] o.a.s.c.OverseerElectionContext I am going to be the 
leader 127.0.0.1:39164_
   [junit4]   2> 519662 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [n:127.0.0.1:39164_    ] o.a.s.c.Overseer Overseer 
(id=96970768539058180-127.0.0.1:39164_-n_0000000000) starting
   [junit4]   2> 519721 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [n:127.0.0.1:39164_    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:39164_
   [junit4]   2> 519729 INFO  
(zkCallback-551-thread-1-processing-n:127.0.0.1:39164_) [n:127.0.0.1:39164_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 519913 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [n:127.0.0.1:39164_    ] o.a.s.c.CorePropertiesLocator Found 1 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/control-001/cores
   [junit4]   2> 519913 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [n:127.0.0.1:39164_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 519940 INFO  
(OverseerStateUpdate-96970768539058180-127.0.0.1:39164_-n_0000000000) 
[n:127.0.0.1:39164_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard 
shard=shard1
   [junit4]   2> 519987 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection   x:collection1] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 6.4.0
   [junit4]   2> 520028 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection   x:collection1] o.a.s.s.IndexSchema 
[collection1] Schema name=test
   [junit4]   2> 520139 WARN  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection   x:collection1] o.a.s.s.IndexSchema 
[collection1] default search field in schema is text. WARNING: Deprecated, 
please use 'df' on request instead.
   [junit4]   2> 520141 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection   x:collection1] o.a.s.s.IndexSchema 
Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 520154 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection   x:collection1] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1' using configuration from collection 
control_collection
   [junit4]   2> 520155 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/control-001/cores/collection1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/control-001/cores/collection1/data/]
   [junit4]   2> 520155 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX 
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e3ddb77
   [junit4]   2> 520158 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: 
minMergeSize=1677721, mergeFactor=22, maxMergeSize=2147483648, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.0]
   [junit4]   2> 520169 WARN  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = 
requestHandler,name = /dump,class = DumpRequestHandler,args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 520178 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 520178 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 520178 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 520178 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 520179 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: 
minMergeSize=0, mergeFactor=10, maxMergeSize=448779299, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.1]
   [junit4]   2> 520179 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.s.SolrIndexSearcher Opening [Searcher@7007b46d[collection1] main]
   [junit4]   2> 520181 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 520182 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 520182 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 520183 INFO  
(searcherExecutor-1316-thread-1-processing-n:127.0.0.1:39164_ x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:39164_ 
c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore 
[collection1] Registered new searcher Searcher@7007b46d[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 520183 INFO  
(coreLoadExecutor-1315-thread-1-processing-n:127.0.0.1:39164_) 
[n:127.0.0.1:39164_ c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1551532297636806656
   [junit4]   2> 520190 INFO  
(coreZkRegister-1308-thread-1-processing-n:127.0.0.1:39164_ x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:39164_ 
c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 520190 INFO  
(coreZkRegister-1308-thread-1-processing-n:127.0.0.1:39164_ x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:39164_ 
c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 520190 INFO  
(coreZkRegister-1308-thread-1-processing-n:127.0.0.1:39164_ x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:39164_ 
c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy 
Sync replicas to http://127.0.0.1:39164/collection1/
   [junit4]   2> 520191 INFO  
(coreZkRegister-1308-thread-1-processing-n:127.0.0.1:39164_ x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:39164_ 
c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy 
Sync Success - now sync replicas to me
   [junit4]   2> 520191 INFO  
(coreZkRegister-1308-thread-1-processing-n:127.0.0.1:39164_ x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:39164_ 
c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy 
http://127.0.0.1:39164/collection1/ has no replicas
   [junit4]   2> 520195 INFO  
(coreZkRegister-1308-thread-1-processing-n:127.0.0.1:39164_ x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:39164_ 
c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:39164/collection1/ shard1
   [junit4]   2> 520346 INFO  
(coreZkRegister-1308-thread-1-processing-n:127.0.0.1:39164_ x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:39164_ 
c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController 
I am the leader, no recovery necessary
   [junit4]   2> 520445 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 520446 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause 
connection loss:false
   [junit4]   2> 520548 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/shard-1-001/cores/collection1
   [junit4]   2> 520549 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/shard-1-001
   [junit4]   2> 520549 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 520551 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@780adc20{/,null,AVAILABLE}
   [junit4]   2> 520551 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.e.j.s.ServerConnector Started 
ServerConnector@3ac332dd{HTTP/1.1,[http/1.1]}{127.0.0.1:46202}
   [junit4]   2> 520551 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.e.j.s.Server Started @525008ms
   [junit4]   2> 520551 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/tempDir-001/jetty1,
 solrconfig=solrconfig.xml, hostContext=/, hostPort=46202, 
coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/shard-1-001/cores}
   [junit4]   2> 520552 ERROR 
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging 
may be missing or incomplete.
   [junit4]   2> 520552 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? 
version 6.4.0
   [junit4]   2> 520552 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on 
port null
   [junit4]   2> 520552 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 520552 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-11-20T15:41:43.700Z
   [junit4]   2> 520560 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 520560 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/shard-1-001/solr.xml
   [junit4]   2> 520566 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with 
params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 520567 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:49442/solr
   [junit4]   2> 520577 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [n:127.0.0.1:46202_    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 520583 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [n:127.0.0.1:46202_    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:46202_
   [junit4]   2> 520584 INFO  (zkCallback-555-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 520584 INFO  
(zkCallback-551-thread-3-processing-n:127.0.0.1:39164_) [n:127.0.0.1:39164_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 520593 INFO  
(zkCallback-560-thread-1-processing-n:127.0.0.1:46202_) [n:127.0.0.1:46202_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 520619 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [n:127.0.0.1:46202_    ] o.a.s.c.CorePropertiesLocator Found 1 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/shard-1-001/cores
   [junit4]   2> 520620 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [n:127.0.0.1:46202_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 520622 INFO  
(OverseerStateUpdate-96970768539058180-127.0.0.1:39164_-n_0000000000) 
[n:127.0.0.1:39164_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard 
shard=shard1
   [junit4]   2> 521635 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using 
Lucene MatchVersion: 6.4.0
   [junit4]   2> 521649 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] Schema name=test
   [junit4]   2> 521747 WARN  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] default search field in schema is text. WARNING: Deprecated, 
please use 'df' on request instead.
   [junit4]   2> 521749 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded 
schema test/1.0 with uniqueid field id
   [junit4]   2> 521762 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1   x:collection1] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 521762 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1 s:shard1 r:core_node1 x:collection1] 
o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/shard-1-001/cores/collection1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/shard-1-001/cores/collection1/data/]
   [junit4]   2> 521763 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1 s:shard1 r:core_node1 x:collection1] 
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX 
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e3ddb77
   [junit4]   2> 521767 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1 s:shard1 r:core_node1 x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: 
minMergeSize=1677721, mergeFactor=22, maxMergeSize=2147483648, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.0]
   [junit4]   2> 521779 WARN  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1 s:shard1 r:core_node1 x:collection1] 
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = 
requestHandler,name = /dump,class = DumpRequestHandler,args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 521792 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1 s:shard1 r:core_node1 x:collection1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 521792 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1 s:shard1 r:core_node1 x:collection1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 521792 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1 s:shard1 r:core_node1 x:collection1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 521792 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1 s:shard1 r:core_node1 x:collection1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 521794 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1 s:shard1 r:core_node1 x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: 
minMergeSize=0, mergeFactor=10, maxMergeSize=448779299, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.1]
   [junit4]   2> 521794 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1 s:shard1 r:core_node1 x:collection1] 
o.a.s.s.SolrIndexSearcher Opening [Searcher@1e333f31[collection1] main]
   [junit4]   2> 521796 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1 s:shard1 r:core_node1 x:collection1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 521796 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1 s:shard1 r:core_node1 x:collection1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 521796 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1 s:shard1 r:core_node1 x:collection1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 521797 INFO  
(coreLoadExecutor-1326-thread-1-processing-n:127.0.0.1:46202_) 
[n:127.0.0.1:46202_ c:collection1 s:shard1 r:core_node1 x:collection1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1551532299329208320
   [junit4]   2> 521799 INFO  
(searcherExecutor-1327-thread-1-processing-n:127.0.0.1:46202_ x:collection1 
s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:46202_ c:collection1 s:shard1 
r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new 
searcher Searcher@1e333f31[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 521814 INFO  
(coreZkRegister-1321-thread-1-processing-n:127.0.0.1:46202_ x:collection1 
s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:46202_ c:collection1 s:shard1 
r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas 
found to continue.
   [junit4]   2> 521814 INFO  
(coreZkRegister-1321-thread-1-processing-n:127.0.0.1:46202_ x:collection1 
s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:46202_ c:collection1 s:shard1 
r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new 
leader - try and sync
   [junit4]   2> 521814 INFO  
(coreZkRegister-1321-thread-1-processing-n:127.0.0.1:46202_ x:collection1 
s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:46202_ c:collection1 s:shard1 
r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to 
http://127.0.0.1:46202/collection1/
   [junit4]   2> 521815 INFO  
(coreZkRegister-1321-thread-1-processing-n:127.0.0.1:46202_ x:collection1 
s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:46202_ c:collection1 s:shard1 
r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync 
replicas to me
   [junit4]   2> 521815 INFO  
(coreZkRegister-1321-thread-1-processing-n:127.0.0.1:46202_ x:collection1 
s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:46202_ c:collection1 s:shard1 
r:core_node1 x:collection1] o.a.s.c.SyncStrategy 
http://127.0.0.1:46202/collection1/ has no replicas
   [junit4]   2> 521821 INFO  
(coreZkRegister-1321-thread-1-processing-n:127.0.0.1:46202_ x:collection1 
s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:46202_ c:collection1 s:shard1 
r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new 
leader: http://127.0.0.1:46202/collection1/ shard1
   [junit4]   2> 521972 INFO  
(coreZkRegister-1321-thread-1-processing-n:127.0.0.1:46202_ x:collection1 
s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:46202_ c:collection1 s:shard1 
r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery 
necessary
   [junit4]   2> 522223 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/shard-2-001/cores/collection1
   [junit4]   2> 522224 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/shard-2-001
   [junit4]   2> 522225 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 522226 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@381a56db{/,null,AVAILABLE}
   [junit4]   2> 522227 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.e.j.s.ServerConnector Started 
ServerConnector@5fcdbae8{HTTP/1.1,[http/1.1]}{127.0.0.1:54399}
   [junit4]   2> 522227 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.e.j.s.Server Started @526683ms
   [junit4]   2> 522227 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/tempDir-001/jetty2,
 solrconfig=solrconfig.xml, hostContext=/, hostPort=54399, 
coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/shard-2-001/cores}
   [junit4]   2> 522228 ERROR 
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging 
may be missing or incomplete.
   [junit4]   2> 522234 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? 
version 6.4.0
   [junit4]   2> 522234 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on 
port null
   [junit4]   2> 522234 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 522234 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-11-20T15:41:45.382Z
   [junit4]   2> 522237 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 522237 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/shard-2-001/solr.xml
   [junit4]   2> 522244 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with 
params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 522244 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:49442/solr
   [junit4]   2> 522261 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [n:127.0.0.1:54399_    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 522266 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [n:127.0.0.1:54399_    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:54399_
   [junit4]   2> 522268 INFO  
(zkCallback-566-thread-1-processing-n:127.0.0.1:54399_) [n:127.0.0.1:54399_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 522269 INFO  
(zkCallback-560-thread-1-processing-n:127.0.0.1:46202_) [n:127.0.0.1:46202_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 522269 INFO  (zkCallback-555-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 522277 INFO  
(zkCallback-551-thread-1-processing-n:127.0.0.1:39164_) [n:127.0.0.1:39164_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 522304 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [n:127.0.0.1:54399_    ] o.a.s.c.CorePropertiesLocator Found 1 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/shard-2-001/cores
   [junit4]   2> 522304 INFO  
(TEST-LeaderInitiatedRecoveryOnShardRestartTest.testRestartWithAllInLIR-seed#[BEA4C44DC413E54])
 [n:127.0.0.1:54399_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 522306 INFO  
(OverseerStateUpdate-96970768539058180-127.0.0.1:39164_-n_0000000000) 
[n:127.0.0.1:39164_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard 
shard=shard1
   [junit4]   2> 523331 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using 
Lucene MatchVersion: 6.4.0
   [junit4]   2> 523345 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] Schema name=test
   [junit4]   2> 523445 WARN  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] default search field in schema is text. WARNING: Deprecated, 
please use 'df' on request instead.
   [junit4]   2> 523449 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded 
schema test/1.0 with uniqueid field id
   [junit4]   2> 523482 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1   x:collection1] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 523483 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/shard-2-001/cores/collection1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnShardRestartTest_BEA4C44DC413E54-001/shard-2-001/cores/collection1/data/]
   [junit4]   2> 523483 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX 
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e3ddb77
   [junit4]   2> 523510 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: 
minMergeSize=1677721, mergeFactor=22, maxMergeSize=2147483648, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.0]
   [junit4]   2> 523531 WARN  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = 
requestHandler,name = /dump,class = DumpRequestHandler,args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 523568 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 523568 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 523569 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 523569 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 523570 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: 
minMergeSize=0, mergeFactor=10, maxMergeSize=448779299, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.1]
   [junit4]   2> 523571 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.s.SolrIndexSearcher Opening [Searcher@405849a1[collection1] main]
   [junit4]   2> 523572 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 523572 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 523573 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 523573 INFO  
(coreLoadExecutor-1337-thread-1-processing-n:127.0.0.1:54399_) 
[n:127.0.0.1:54399_ c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1551532301191479296
   [junit4]   2> 523586 INFO  
(searcherExecutor-1338-thread-1-processing-n:127.0.0.1:54399_ x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54399_ c:collection1 s:shard1 
r:core_node2 x:collection1] o.a.s.c.SolrCore [collection1] Registered new 
searcher Searcher@405849a1[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 523591 INFO  
(coreZkRegister-1332-thread-1-processing-n:127.0.0.1:54399_ x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54399_ c:collection1 s:shard1 
r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to 
recover:collection1
   [junit4]   2> 523598 INFO  
(updateExecutor-563-thread-1-processing-n:127.0.0.1:54399_ x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54399_ c:collection1 s:shard1 
r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 523609 INFO  
(recoveryExecutor-564-thread-1-processing-n:127.0.0.1:54399_ x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54399_ c:collection1 s:shard1 
r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. 
recoveringAfterStartup=true
   [junit4]   2> 523609 INFO  
(recoveryExecutor-564-thread-1-processing-n:127.0.0.1:54399_ x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54399_ c:collection1 s:shard1 
r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 523609 INFO  
(recoveryExecutor-564-thread-1-processing-n:127.0.0.1:54399_ x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54399_ c:collection1 s:shard1 
r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. 
core=[collection1]
   [junit4]   2> 523610 INFO  
(recoveryExecutor-564-thread-1-processing-n:127.0.0.1:54399_ x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54399_ c:collection1 s:shard1 
r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. 
FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 523610 INFO  
(recoveryExecutor-564-thread-1-processing-n:127.0.0.1:54399_ x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54399_ c:collection1 s:shard1 
r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core 
[collection1] as recovering, leader is [http://127.0.0.1:46202/collection1/] 
and I am [http://127.0.0.1:54399/collection1/]
   [junit4]   2> 523612 INFO  
(recoveryExecutor-564-thread-1-processing-n:127.0.0.1:54399_ x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54399_ c:collection1 s:shard1 
r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery 
command to [http://127.0.0

[...truncated too long message...]

4]   2>         at 
org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2>        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2>        at 
org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.teardownClass(HdfsCollectionsAPIDistributedZkTest.java:52)
   [junit4]   2>        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native 
Method)
   [junit4]   2>        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>        at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2>        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2>        ... 54 more
   [junit4]   2> 5010972 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[BEA4C44DC413E54]-worker) [    
] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39243 39243
   [junit4]   2> 5010972 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[BEA4C44DC413E54]-worker) [    
] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_BEA4C44DC413E54-001
   [junit4]   2> Nov 20, 2016 4:56:34 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 33 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, 
docValues:{}, maxPointsInLeafNode=409, maxMBSortInHeap=5.335338837054335, 
sim=RandomSimilarity(queryNorm=false,coord=no): {}, locale=hr-HR, timezone=WET
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 
1.8.0_102 (64-bit)/cpus=4,threads=1,free=343514336,total=518520832
   [junit4]   2> NOTE: All tests run in this JVM: [TestMiniSolrCloudCluster, 
HttpSolrCallGetCoreTest, ZkStateReaderTest, TestSubQueryTransformerDistrib, 
TestQueryTypes, TestSystemIdResolver, SpatialFilterTest, TestTrie, 
TestLuceneMatchVersion, ClusterStateUpdateTest, SoftAutoCommitTest, 
TestNonDefinedSimilarityFactory, TestCursorMarkWithoutUniqueKey, 
TestFieldSortValues, TestSimpleTrackingShardHandler, TestNRTOpen, 
TestCollectionAPI, HighlighterTest, BJQParserTest, TestQueryWrapperFilter, 
HLLSerializationTest, TestFreeTextSuggestions, TestFieldCacheReopen, 
DistributedExpandComponentTest, UpdateRequestProcessorFactoryTest, 
CoreMergeIndexesAdminHandlerTest, HighlighterMaxOffsetTest, 
TestCopyFieldCollectionResource, TestRTimerTree, SystemInfoHandlerTest, 
HdfsWriteToMultipleCollectionsTest, TestGeoJSONResponseWriter, 
DocumentBuilderTest, TestFieldCacheSortRandom, SolrInfoMBeanTest, 
DocValuesTest, TermVectorComponentTest, TestUninvertingReader, 
TestMinMaxOnMultiValuedField, RestartWhileUpdatingTest, 
TestBM25SimilarityFactory, SimpleMLTQParserTest, 
LeaderInitiatedRecoveryOnShardRestartTest, 
TestSolrCloudWithSecureImpersonation, TriLevelCompositeIdRoutingTest, 
ShardRoutingTest, HdfsLockFactoryTest, BitVectorTest, TestHashQParserPlugin, 
SolrCmdDistributorTest, PreAnalyzedFieldManagedSchemaCloudTest, 
TestConfigSetImmutable, TestOnReconnectListenerSupport, 
TestHdfsCloudBackupRestore, VersionInfoTest, TestLRUCache, 
TestLMDirichletSimilarityFactory, TestSearcherReuse, SuggesterFSTTest, 
TestElisionMultitermQuery, SortByFunctionTest, TestExactSharedStatsCache, 
TemplateUpdateProcessorTest, TestRTGBase, CollectionsAPIAsyncDistributedZkTest, 
BadCopyFieldTest, TestQuerySenderListener, TestManagedSchemaThreadSafety, 
ZkControllerTest, TestRebalanceLeaders, RemoteQueryErrorTest, TestRestManager, 
TestCollapseQParserPlugin, TestLeaderInitiatedRecoveryThread, TestBlobHandler, 
TestFieldTypeCollectionResource, ZkSolrClientTest, 
DistributedFacetPivotWhiteBoxTest, LoggingHandlerTest, JsonLoaderTest, 
DeleteNodeTest, DistribJoinFromCollectionTest, 
ClassificationUpdateProcessorFactoryTest, TestWordDelimiterFilterFactory, 
TestFiltering, CSVRequestHandlerTest, TestMacros, TestCloudManagedSchema, 
ChaosMonkeySafeLeaderTest, TestAnalyzeInfixSuggestions, TestStressVersions, 
MultiTermTest, RAMDirectoryFactoryTest, TestSuggestSpellingConverter, 
QueryResultKeyTest, AliasIntegrationTest, IndexBasedSpellCheckerTest, 
OpenCloseCoreStressTest, TermsComponentTest, TestSolrIndexConfig, 
TestSolrConfigHandlerConcurrent, TestXmlQParserPlugin, TestDistributedGrouping, 
ShufflingReplicaListTransformerTest, DeleteStatusTest, 
TestCloudPseudoReturnFields, TestSolr4Spatial2, BasicDistributedZk2Test, 
UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, 
LeaderElectionIntegrationTest, RecoveryZkTest, TestReplicationHandler, 
TestReload, TestCoreContainer, SimpleFacetsTest, SpellCheckComponentTest, 
TestGroupingSearch, ConvertedLegacyTest, TestFunctionQuery, 
BasicFunctionalityTest, TestLazyCores, TestExtendedDismaxParser, 
SuggesterTSTTest, WordBreakSolrSpellCheckerTest, 
SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, 
TestAtomicUpdateErrorCases, DirectUpdateHandlerOptimizeTest, 
DefaultValueUpdateProcessorTest, DistanceFunctionTest, CacheHeaderTest, 
TestSurroundQueryParser, TestQueryUtils, DirectSolrSpellCheckerTest, 
PrimitiveFieldTypeTest, FileBasedSpellCheckerTest, TestIndexingPerformance, 
RequiredFieldsTest, IndexSchemaRuntimeFieldTest, TestJmxIntegration, 
MBeansHandlerTest, TestAnalyzedSuggestions, PingRequestHandlerTest, 
HighlighterConfigTest, SOLR749Test, UpdateParamsTest, CopyFieldTest, 
SolrIndexConfigTest, TestStressRecovery, TestSolrDeletionPolicy2, SampleTest, 
TestBinaryField, TestSearchPerf, NumericFieldsTest, TestConfig, 
TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, 
DirectSolrConnectionTest, TestPhraseSuggestions, SpellPossibilityIteratorTest, 
TestCharFilters, TestCodecSupport, SynonymTokenizerTest, TestXIncludeConfig, 
EchoParamsTest, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, 
TestLMJelinekMercerSimilarityFactory, TestIBSimilarityFactory, TestFastWriter, 
TestJmxMonitoredMap, ResourceLoaderTest, TestFastOutputStream, 
OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, DateMathParserTest, 
ChaosMonkeyNothingIsSafeTest, PreAnalyzedFieldTest, PrimUtilsTest, 
SpellingQueryConverterTest, DOMUtilTest, ZkNodePropsTest, SliceStateTest, 
UUIDFieldTest, FileUtilsTest, CircularListTest, DistributedMLTComponentTest, 
CursorPagingTest, DistributedIntervalFacetingTest, CdcrReplicationHandlerTest, 
CdcrVersionReplicationTest, CleanupOldIndexTest, CollectionReloadTest, 
ConcurrentDeleteAndCreateCollectionTest, CustomCollectionTest, 
DistributedVersionInfoTest, HttpPartitionTest, 
LeaderInitiatedRecoveryOnCommitTest, OutOfBoxZkACLAndCredentialsProvidersTest, 
OverseerModifyCollectionTest, OverseerRolesTest, ReplicationFactorTest, 
SaslZkACLProviderTest, ShardSplitTest, TestTolerantUpdateProcessorCloud, 
TestTolerantUpdateProcessorRandomCloud, HdfsBasicDistributedZk2Test, 
HdfsChaosMonkeyNothingIsSafeTest, HdfsChaosMonkeySafeLeaderTest, 
HdfsCollectionsAPIDistributedZkTest]
   [junit4] Completed [517/654 (5!)] on J1 in 168.08s, 20 tests, 1 failure, 1 
error <<< FAILURES!

[...truncated 61104 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to