Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/704/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.solr.update.HdfsTransactionLog  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at 
org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)  
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203)  at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)  at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)  at 
org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:104)
  at sun.reflect.GeneratedConstructorAccessor513.newInstance(Unknown Source)  
at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  at java.lang.reflect.Constructor.newInstance(Constructor.java:423)  at 
org.apache.solr.core.SolrCore.createInstance(SolrCore.java:741)  at 
org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:803)  at 
org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1053)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:918)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:811)  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:917)  at 
org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:552)  at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
 at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not 
released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.solr.update.HdfsTransactionLog
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at 
org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
        at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203)
        at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
        at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
        at 
org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:104)
        at sun.reflect.GeneratedConstructorAccessor513.newInstance(Unknown 
Source)
        at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:741)
        at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:803)
        at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1053)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:918)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:811)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:917)
        at 
org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:552)
        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


        at __randomizedtesting.SeedInfo.seed([9EA8A67A67E5D1DB]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
        at sun.reflect.GeneratedMethodAccessor119.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.update.TestInPlaceUpdatesDistrib.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when 
talking to server at: http://127.0.0.1:51373/collection1

Stack Trace:
java.util.concurrent.ExecutionException: 
org.apache.solr.client.solrj.SolrServerException: IOException occured when 
talking to server at: http://127.0.0.1:51373/collection1
        at 
__randomizedtesting.SeedInfo.seed([9EA8A67A67E5D1DB:16FC99A0C919BC23]:0)
        at java.util.concurrent.FutureTask.report(FutureTask.java:122)
        at java.util.concurrent.FutureTask.get(FutureTask.java:192)
        at 
org.apache.solr.update.TestInPlaceUpdatesDistrib.outOfOrderUpdatesIndividualReplicaTest(TestInPlaceUpdatesDistrib.java:600)
        at 
org.apache.solr.update.TestInPlaceUpdatesDistrib.test(TestInPlaceUpdatesDistrib.java:143)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException 
occured when talking to server at: http://127.0.0.1:51373/collection1
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:624)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:160)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:177)
        at 
org.apache.solr.update.TestInPlaceUpdatesDistrib$AsyncUpdateWithRandomCommit.call(TestInPlaceUpdatesDistrib.java:928)
        at 
org.apache.solr.update.TestInPlaceUpdatesDistrib$AsyncUpdateWithRandomCommit.call(TestInPlaceUpdatesDistrib.java:915)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        ... 1 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:51373 failed to 
respond
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
        at 
org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
        at 
org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
        at 
org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
        at 
org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
        at 
org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:515)
        ... 11 more




Build Log:
[...truncated 12780 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
   [junit4]   2> Creating dataDir: 
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_9EA8A67A67E5D1DB-001/init-core-data-001
   [junit4]   2> 2687184 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields
   [junit4]   2> 2687186 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2687186 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in 
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_9EA8A67A67E5D1DB-001/tempDir-001
   [junit4]   2> 2687187 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2687187 INFO  (Thread-5508) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2687187 INFO  (Thread-5508) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 2687287 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:53364
   [junit4]   2> 2687306 INFO  (jetty-launcher-5108-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2687306 INFO  (jetty-launcher-5108-thread-2) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2687312 INFO  (jetty-launcher-5108-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@60249651{/solr,null,AVAILABLE}
   [junit4]   2> 2687312 INFO  (jetty-launcher-5108-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@60a47a6a{/solr,null,AVAILABLE}
   [junit4]   2> 2687321 INFO  (jetty-launcher-5108-thread-1) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@58191da{HTTP/1.1,[http/1.1]}{127.0.0.1:61590}
   [junit4]   2> 2687322 INFO  (jetty-launcher-5108-thread-2) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@3932600f{HTTP/1.1,[http/1.1]}{127.0.0.1:48719}
   [junit4]   2> 2687322 INFO  (jetty-launcher-5108-thread-1) [    ] 
o.e.j.s.Server Started @2693055ms
   [junit4]   2> 2687322 INFO  (jetty-launcher-5108-thread-2) [    ] 
o.e.j.s.Server Started @2693055ms
   [junit4]   2> 2687322 INFO  (jetty-launcher-5108-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=61590}
   [junit4]   2> 2687322 INFO  (jetty-launcher-5108-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=48719}
   [junit4]   2> 2687322 ERROR (jetty-launcher-5108-thread-1) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 2687323 ERROR (jetty-launcher-5108-thread-2) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 2687323 INFO  (jetty-launcher-5108-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
6.5.0
   [junit4]   2> 2687323 INFO  (jetty-launcher-5108-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2687323 INFO  (jetty-launcher-5108-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
6.5.0
   [junit4]   2> 2687323 INFO  (jetty-launcher-5108-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2687323 INFO  (jetty-launcher-5108-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2687323 INFO  (jetty-launcher-5108-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2687323 INFO  (jetty-launcher-5108-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-03-05T08:03:09.658Z
   [junit4]   2> 2687323 INFO  (jetty-launcher-5108-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-03-05T08:03:09.658Z
   [junit4]   2> 2687326 INFO  (jetty-launcher-5108-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2687327 INFO  (jetty-launcher-5108-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2687338 INFO  (jetty-launcher-5108-thread-2) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2687339 INFO  (jetty-launcher-5108-thread-1) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2687339 INFO  (jetty-launcher-5108-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53364/solr
   [junit4]   2> 2687341 INFO  (jetty-launcher-5108-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53364/solr
   [junit4]   2> 2687389 INFO  (jetty-launcher-5108-thread-1) 
[n:127.0.0.1:61590_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2687389 INFO  (jetty-launcher-5108-thread-2) 
[n:127.0.0.1:48719_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2687390 INFO  (jetty-launcher-5108-thread-2) 
[n:127.0.0.1:48719_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:48719_solr
   [junit4]   2> 2687391 INFO  (jetty-launcher-5108-thread-1) 
[n:127.0.0.1:61590_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:61590_solr
   [junit4]   2> 2687393 INFO  (jetty-launcher-5108-thread-2) 
[n:127.0.0.1:48719_solr    ] o.a.s.c.Overseer Overseer 
(id=97563508049379333-127.0.0.1:48719_solr-n_0000000000) starting
   [junit4]   2> 2687394 INFO  
(zkCallback-5119-thread-1-processing-n:127.0.0.1:48719_solr) 
[n:127.0.0.1:48719_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 2687395 INFO  
(zkCallback-5120-thread-1-processing-n:127.0.0.1:61590_solr) 
[n:127.0.0.1:61590_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 2687410 INFO  (jetty-launcher-5108-thread-2) 
[n:127.0.0.1:48719_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:48719_solr
   [junit4]   2> 2687413 INFO  
(zkCallback-5120-thread-1-processing-n:127.0.0.1:61590_solr) 
[n:127.0.0.1:61590_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 2687414 INFO  
(zkCallback-5119-thread-1-processing-n:127.0.0.1:48719_solr) 
[n:127.0.0.1:48719_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 2687522 INFO  (jetty-launcher-5108-thread-1) 
[n:127.0.0.1:61590_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_9EA8A67A67E5D1DB-001/tempDir-001/node1/.
   [junit4]   2> 2687683 INFO  (jetty-launcher-5108-thread-2) 
[n:127.0.0.1:48719_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_9EA8A67A67E5D1DB-001/tempDir-001/node2/.
   [junit4]   2> 2688584 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2688586 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53364/solr ready
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 2688634 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] 
o.a.h.m.i.MetricsConfig Cannot locate configuration: tried 
hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 2688647 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2688650 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 2688665 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] o.m.log 
Extract 
jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs
 to ./temp/Jetty_solaris.vm_41750_hdfs____wy98nb/webapp
   [junit4]   2> 2689184 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:41750
   [junit4]   2> 2689310 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2689311 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 2689324 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] o.m.log 
Extract 
jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_51443_datanode____7rvc3i/webapp
   [junit4]   2> 2689815 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:51443
   [junit4]   2> 2689901 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2689902 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 2689925 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] o.m.log 
Extract 
jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_45210_datanode____cfcm8n/webapp
   [junit4]   2> 2689988 INFO  (IPC Server handler 2 on 38007) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-9606c45f-45c4-449a-9836-59bd0678d9de node 
DatanodeRegistration(127.0.0.1:53500, 
datanodeUuid=c843c60a-44d5-4c55-bc0a-1bf4416c0467, infoPort=56527, 
infoSecurePort=0, ipcPort=47351, 
storageInfo=lv=-56;cid=testClusterID;nsid=1675070124;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 1 msecs
   [junit4]   2> 2689988 INFO  (IPC Server handler 2 on 38007) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-c2633e7c-a8ef-44d8-b48f-3bbed45e6e05 node 
DatanodeRegistration(127.0.0.1:53500, 
datanodeUuid=c843c60a-44d5-4c55-bc0a-1bf4416c0467, infoPort=56527, 
infoSecurePort=0, ipcPort=47351, 
storageInfo=lv=-56;cid=testClusterID;nsid=1675070124;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 2690463 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[9EA8A67A67E5D1DB]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45210
   [junit4]   2> 2690594 INFO  (IPC Server handler 6 on 38007) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-92b57cdc-0249-4a79-81db-a3fd2d8330ee node 
DatanodeRegistration(127.0.0.1:41838, 
datanodeUuid=6bac2fc5-d425-4d6d-bb75-b511fba6e3ce, infoPort=34639, 
infoSecurePort=0, ipcPort=41209, 
storageInfo=lv=-56;cid=testClusterID;nsid=1675070124;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 1 msecs
   [junit4]   2> 2690594 INFO  (IPC Server handler 6 on 38007) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-68df8a12-a83e-4131-85ca-49f36d0c59b0 node 
DatanodeRegistration(127.0.0.1:41838, 
datanodeUuid=6bac2fc5-d425-4d6d-bb75-b511fba6e3ce, infoPort=34639, 
infoSecurePort=0, ipcPort=41209, 
storageInfo=lv=-56;cid=testClusterID;nsid=1675070124;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 2690723 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[9EA8A67A67E5D1DB]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 2690730 INFO  (qtp670289028-26288) [n:127.0.0.1:61590_solr    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 2690736 INFO  
(OverseerThreadFactory-10729-thread-1-processing-n:127.0.0.1:48719_solr) 
[n:127.0.0.1:48719_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
recoverytest
   [junit4]   2> 2690847 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2690849 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2690958 INFO  
(zkCallback-5120-thread-1-processing-n:127.0.0.1:61590_solr) 
[n:127.0.0.1:61590_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2690958 INFO  
(zkCallback-5119-thread-1-processing-n:127.0.0.1:48719_solr) 
[n:127.0.0.1:48719_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2691865 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2691865 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2691876 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema 
[recoverytest_shard1_replica1] Schema name=minimal
   [junit4]   2> 2691876 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema 
[recoverytest_shard1_replica2] Schema name=minimal
   [junit4]   2> 2691911 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema 
Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2691911 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema 
Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2691912 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer 
Creating SolrCore 'recoverytest_shard1_replica2' using configuration from 
collection recoverytest
   [junit4]   2> 2691912 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer 
Creating SolrCore 'recoverytest_shard1_replica1' using configuration from 
collection recoverytest
   [junit4]   2> 2691920 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:38007/data
   [junit4]   2> 2691920 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:38007/data
   [junit4]   2> 2691920 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 2691920 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 2691921 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SolrCore 
[[recoverytest_shard1_replica2] ] Opening new SolrCore at 
[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_9EA8A67A67E5D1DB-001/tempDir-001/node2/recoverytest_shard1_replica2],
 dataDir=[hdfs://solaris-vm:38007/data/recoverytest/core_node2/data/]
   [junit4]   2> 2691921 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.SolrCore 
[[recoverytest_shard1_replica1] ] Opening new SolrCore at 
[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_9EA8A67A67E5D1DB-001/tempDir-001/node1/recoverytest_shard1_replica1],
 dataDir=[hdfs://solaris-vm:38007/data/recoverytest/core_node1/data/]
   [junit4]   2> 2691921 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX 
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@14a70695
   [junit4]   2> 2691921 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX 
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@14a70695
   [junit4]   2> 2691921 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://solaris-vm:38007/data/recoverytest/core_node2/data/snapshot_metadata
   [junit4]   2> 2691921 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://solaris-vm:38007/data/recoverytest/core_node1/data/snapshot_metadata
   [junit4]   2> 2691933 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 2691934 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 2691934 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 2691933 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 2691934 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 2691934 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 2692073 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 2692079 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 2692082 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://solaris-vm:38007/data/recoverytest/core_node2/data
   [junit4]   2> 2692085 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://solaris-vm:38007/data/recoverytest/core_node1/data
   [junit4]   2> 2692113 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://solaris-vm:38007/data/recoverytest/core_node2/data/index
   [junit4]   2> 2692114 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://solaris-vm:38007/data/recoverytest/core_node1/data/index
   [junit4]   2> 2692121 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 2692121 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 2692121 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 2692123 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 2692123 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 2692123 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 2692501 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 2692504 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 2692590 INFO  (IPC Server handler 6 on 38007) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53500 is 
added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-92b57cdc-0249-4a79-81db-a3fd2d8330ee:NORMAL:127.0.0.1:41838|RBW],
 
ReplicaUC[[DISK]DS-c2633e7c-a8ef-44d8-b48f-3bbed45e6e05:NORMAL:127.0.0.1:53500|FINALIZED]]}
 size 0
   [junit4]   2> 2692592 INFO  (IPC Server handler 7 on 38007) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41838 is 
added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-c2633e7c-a8ef-44d8-b48f-3bbed45e6e05:NORMAL:127.0.0.1:53500|FINALIZED],
 
ReplicaUC[[DISK]DS-68df8a12-a83e-4131-85ca-49f36d0c59b0:NORMAL:127.0.0.1:41838|FINALIZED]]}
 size 0
   [junit4]   2> 2692598 INFO  (IPC Server handler 5 on 38007) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41838 is 
added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-92b57cdc-0249-4a79-81db-a3fd2d8330ee:NORMAL:127.0.0.1:41838|RBW],
 
ReplicaUC[[DISK]DS-9606c45f-45c4-449a-9836-59bd0678d9de:NORMAL:127.0.0.1:53500|RBW]]}
 size 0
   [junit4]   2> 2692600 INFO  (IPC Server handler 9 on 38007) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53500 is 
added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-92b57cdc-0249-4a79-81db-a3fd2d8330ee:NORMAL:127.0.0.1:41838|RBW],
 
ReplicaUC[[DISK]DS-9606c45f-45c4-449a-9836-59bd0678d9de:NORMAL:127.0.0.1:53500|RBW]]}
 size 0
   [junit4]   2> 2692728 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 2692728 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2692728 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 2692728 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog 
Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 2692728 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2692728 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog 
Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 2692746 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 2692746 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 2692748 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 2692748 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 2692788 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@6e01e872[recoverytest_shard1_replica2] main]
   [junit4]   2> 2692793 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 2692794 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2692794 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2692797 INFO  
(searcherExecutor-10734-thread-1-processing-n:127.0.0.1:48719_solr 
x:recoverytest_shard1_replica2 s:shard1 c:recoverytest) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SolrCore 
[recoverytest_shard1_replica2] Registered new searcher 
Searcher@6e01e872[recoverytest_shard1_replica2] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2692798 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@36f90259[recoverytest_shard1_replica1] main]
   [junit4]   2> 2692800 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 2692801 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2692801 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2692811 INFO  
(searcherExecutor-10735-thread-1-processing-n:127.0.0.1:61590_solr 
x:recoverytest_shard1_replica1 s:shard1 c:recoverytest) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.SolrCore 
[recoverytest_shard1_replica1] Registered new searcher 
Searcher@36f90259[recoverytest_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2692813 WARN  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for 
hdfs://solaris-vm:47774/solr3 due to exception: java.net.ConnectException: Call 
From solaris-vm/127.0.0.1 to solaris-vm:47774 failed on connection exception: 
java.net.ConnectException: Connection refused; For more details see:  
http://wiki.apache.org/hadoop/ConnectionRefused
   [junit4]   2> 2692814 WARN  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for 
hdfs://solaris-vm:47774/solr3 due to exception: java.net.ConnectException: Call 
From solaris-vm/127.0.0.1 to solaris-vm:47774 failed on connection exception: 
java.net.ConnectException: Connection refused; For more details see:  
http://wiki.apache.org/hadoop/ConnectionRefused
   [junit4]   2> 2692815 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1561016134690406400
   [junit4]   2> 2692822 WARN  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for 
hdfs://solaris-vm:47774/solr3 due to exception: java.net.ConnectException: Call 
From solaris-vm/127.0.0.1 to solaris-vm:47774 failed on connection exception: 
java.net.ConnectException: Connection refused; For more details see:  
http://wiki.apache.org/hadoop/ConnectionRefused
   [junit4]   2> 2692829 INFO  
(zkCallback-5119-thread-1-processing-n:127.0.0.1:48719_solr) 
[n:127.0.0.1:48719_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2692831 INFO  
(zkCallback-5120-thread-1-processing-n:127.0.0.1:61590_solr) 
[n:127.0.0.1:61590_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2692832 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for 
shard shard1: total=2 found=1 timeoutin=9995ms
   [junit4]   2> 2692834 WARN  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for 
hdfs://solaris-vm:47774/solr3 due to exception: java.net.ConnectException: Call 
From solaris-vm/127.0.0.1 to solaris-vm:47774 failed on connection exception: 
java.net.ConnectException: Connection refused; For more details see:  
http://wiki.apache.org/hadoop/ConnectionRefused
   [junit4]   2> 2692834 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1561016134710329344
   [junit4]   2> 2693333 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2693334 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2693334 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy 
Sync replicas to http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/
   [junit4]   2> 2693334 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.PeerSync 
PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:48719/solr 
START replicas=[http://127.0.0.1:61590/solr/recoverytest_shard1_replica1/] 
nUpdates=100
   [junit4]   2> 2693334 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.PeerSync 
PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:48719/solr 
DONE.  We have no versions.  sync failed.
   [junit4]   2> 2693337 INFO  (qtp670289028-26287) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.c.S.Request [recoverytest_shard1_replica1]  webapp=/solr path=/get 
params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2}
 status=0 QTime=0
   [junit4]   2> 2693339 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy 
Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 2693339 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we 
can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 2693339 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 2693345 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/ shard1
   [junit4]   2> 2693450 INFO  
(zkCallback-5119-thread-1-processing-n:127.0.0.1:48719_solr) 
[n:127.0.0.1:48719_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2693451 INFO  
(zkCallback-5120-thread-1-processing-n:127.0.0.1:61590_solr) 
[n:127.0.0.1:61590_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2693496 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.ZkController I 
am the leader, no recovery necessary
   [junit4]   2> 2693501 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2}
 status=0 QTime=2653
   [junit4]   2> 2693604 INFO  
(zkCallback-5120-thread-1-processing-n:127.0.0.1:61590_solr) 
[n:127.0.0.1:61590_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2693604 INFO  
(zkCallback-5119-thread-2-processing-n:127.0.0.1:48719_solr) 
[n:127.0.0.1:48719_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2693844 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2}
 status=0 QTime=3001
   [junit4]   2> 2693850 INFO  (qtp670289028-26288) [n:127.0.0.1:61590_solr    
] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 2693950 INFO  
(zkCallback-5120-thread-1-processing-n:127.0.0.1:61590_solr) 
[n:127.0.0.1:61590_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2693950 INFO  
(zkCallback-5119-thread-1-processing-n:127.0.0.1:48719_solr) 
[n:127.0.0.1:48719_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2694742 INFO  
(OverseerCollectionConfigSetProcessor-97563508049379333-127.0.0.1:48719_solr-n_0000000000)
 [n:127.0.0.1:48719_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 2694851 INFO  (qtp670289028-26288) [n:127.0.0.1:61590_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2}
 status=0 QTime=4122
   [junit4]   2> 2694852 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[9EA8A67A67E5D1DB]) [    ] 
o.a.s.c.RecoveryZkTest Indexing 700 documents
   [junit4]   2> 2694900 INFO  (qtp670289028-26291) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-0
 (1561016136830550016)]} 0 25
   [junit4]   2> 2694901 INFO  (qtp1293086874-26300) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 
(1561016136830550016)]} 0 45
   [junit4]   2> 2694909 INFO  (qtp670289028-26296) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-0
 (1561016136829501440)]} 0 33
   [junit4]   2> 2694909 INFO  (qtp670289028-26293) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-1
 (1561016136879833088)]} 0 1
   [junit4]   2> 2694910 INFO  (qtp1293086874-26289) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 
(1561016136829501440)]} 0 54
   [junit4]   2> 2694910 INFO  (qtp1293086874-26302) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 
(1561016136879833088)]} 0 7
   [junit4]   2> 2694916 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-2
 (1561016136890318848)]} 0 1
   [junit4]   2> 2694916 INFO  (qtp670289028-26287) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-1
 (1561016136889270272)]} 0 1
   [junit4]   2> 2694916 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 
(1561016136890318848)]} 0 3
   [junit4]   2> 2694916 INFO  (qtp1293086874-26301) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 
(1561016136889270272)]} 0 3
   [junit4]   2> 2694923 INFO  (qtp670289028-26288) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-3
 (1561016136895561728)]} 0 3
   [junit4]   2> 2694923 INFO  (qtp1293086874-26298) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 
(1561016136895561728)]} 0 5
   [junit4]   2> 2694924 INFO  (qtp670289028-26295) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-2
 (1561016136895561729)]} 0 3
   [junit4]   2> 2694924 INFO  (qtp1293086874-26299) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 
(1561016136895561729)]} 0 6
   [junit4]   2> 2694928 INFO  (qtp670289028-26291) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-4
 (1561016136902901760)]} 0 1
   [junit4]   2> 2694929 INFO  (qtp1293086874-26300) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 
(1561016136902901760)]} 0 3
   [junit4]   2> 2694929 INFO  (qtp670289028-26296) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-3
 (1561016136903950336)]} 0 1
   [junit4]   2> 2694930 INFO  (qtp1293086874-26289) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 
(1561016136903950336)]} 0 3
   [junit4]   2> 2694933 INFO  (qtp670289028-26293) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-5
 (1561016136908144640)]} 0 1
   [junit4]   2> 2694934 INFO  (qtp1293086874-26302) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 
(1561016136908144640)]} 0 3
   [junit4]   2> 2694936 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-4
 (1561016136910241792)]} 0 1
   [junit4]   2> 2694936 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 
(1561016136910241792)]} 0 4
   [junit4]   2> 2694939 INFO  (qtp670289028-26287) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-6
 (1561016136915484672)]} 0 1
   [junit4]   2> 2694940 INFO  (qtp1293086874-26301) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 
(1561016136915484672)]} 0 3
   [junit4]   2> 2694941 INFO  (qtp670289028-26288) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-5
 (1561016136916533248)]} 0 1
   [junit4]   2> 2694942 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 
(1561016136916533248)]} 0 3
   [junit4]   2> 2694945 INFO  (qtp670289028-26295) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-6
 (-1561016136920727552)]} 0 1
   [junit4]   2> 2694945 INFO  (qtp1293086874-26299) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-6 
(-1561016136920727552)]} 0 3
   [junit4]   2> 2694947 INFO  (qtp670289028-26291) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-6
 (1561016136922824704)]} 0 1
   [junit4]   2> 2694947 INFO  (qtp1293086874-26300) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 
(1561016136922824704)]} 0 3
   [junit4]   2> 2694950 INFO  (qtp670289028-26296) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-7
 (1561016136925970432)]} 0 1
   [junit4]   2> 2694950 INFO  (qtp1293086874-26289) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 
(1561016136925970432)]} 0 3
   [junit4]   2> 2694952 INFO  (qtp670289028-26293) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-6
 (-1561016136928067584)]} 0 0
   [junit4]   2> 2694952 INFO  (qtp1293086874-26302) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-6 
(-1561016136928067584)]} 0 3
   [junit4]   2> 2694955 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-8
 (1561016136931213312)]} 0 1
   [junit4]   2> 2694955 INFO  (qtp1293086874-26298) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 
(1561016136931213312)]} 0 2
   [junit4]   2> 2694957 INFO  (qtp670289028-26287) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-7
 (1561016136933310464)]} 0 1
   [junit4]   2> 2694957 INFO  (qtp1293086874-26302) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 
(1561016136933310464)]} 0 2
   [junit4]   2> 2694960 INFO  (qtp670289028-26288) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-9
 (1561016136936456192)]} 0 1
   [junit4]   2> 2694961 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 
(1561016136936456192)]} 0 3
   [junit4]   2> 2694962 INFO  (qtp670289028-26295) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-8
 (1561016136938553344)]} 0 1
   [junit4]   2> 2694962 INFO  (qtp1293086874-26299) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 
(1561016136938553344)]} 0 3
   [junit4]   2> 2694966 INFO  (qtp670289028-26291) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-10
 (1561016136942747648)]} 0 1
   [junit4]   2> 2694966 INFO  (qtp1293086874-26300) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 
(1561016136942747648)]} 0 3
   [junit4]   2> 2694967 INFO  (qtp670289028-26296) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-9
 (1561016136943796224)]} 0 1
   [junit4]   2> 2694968 INFO  (qtp1293086874-26289) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 
(1561016136943796224)]} 0 3
   [junit4]   2> 2694972 INFO  (qtp670289028-26293) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-8
 (-1561016136949039104)]} 0 1
   [junit4]   2> 2694972 INFO  (qtp1293086874-26301) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-8 
(-1561016136949039104)]} 0 4
   [junit4]   2> 2694973 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-10
 (1561016136950087680)]} 0 1
   [junit4]   2> 2694974 INFO  (qtp1293086874-26298) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 
(1561016136950087680)]} 0 3
   [junit4]   2> 2694978 INFO  (qtp670289028-26293) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-11
 (1561016136955330560)]} 0 1
   [junit4]   2> 2694978 INFO  (qtp1293086874-26302) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 
(1561016136955330560)]} 0 3
   [junit4]   2> 2694978 INFO  (qtp670289028-26288) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-8
 (-1561016136956379136)]} 0 1
   [junit4]   2> 2694979 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-8 
(-1561016136956379136)]} 0 3
   [junit4]   2> 2694984 INFO  (qtp670289028-26293) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-12
 (1561016136960573440)]} 0 2
   [junit4]   2> 2694984 INFO  (qtp1293086874-26299) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-12 
(1561016136960573440)]} 0 4
   [junit4]   2> 2694984 INFO  (qtp670289028-26291) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-11
 (1561016136961622016)]} 0 1
   [junit4]   2> 2694985 INFO  (qtp1293086874-26300) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 
(1561016136961622016)]} 0 4
   [junit4]   2> 2694989 INFO  (qtp670289028-26296) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-13
 (1561016136966864896)]} 0 1
   [junit4]   2> 2694990 INFO  (qtp1293086874-26289) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-13 
(1561016136966864896)]} 0 3
   [junit4]   2> 2694991 INFO  (qtp670289028-26287) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-12
 (1561016136968962048)]} 0 1
   [junit4]   2> 2694991 INFO  (qtp1293086874-26301) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-12 
(1561016136968962048)]} 0 3
   [junit4]   2> 2694996 INFO  (qtp670289028-26296) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-14
 (1561016136973156352)]} 0 1
   [junit4]   2> 2694996 INFO  (qtp1293086874-26298) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-14 
(1561016136973156352)]} 0 4
   [junit4]   2> 2694998 INFO  (qtp670289028-26295) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-13
 (1561016136976302080)]} 0 1
   [junit4]   2> 2694998 INFO  (qtp1293086874-26302) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-13 
(1561016136976302080)]} 0 3
   [junit4]   2> 2695001 INFO  (qtp670289028-26288) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-15
 (1561016136980496384)]} 0 1
   [junit4]   2> 2695002 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-15 
(1561016136980496384)]} 0 3
   [junit4]   2> 2695003 INFO  (qtp670289028-26293) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-14
 (1561016136981544960)]} 0 1
   [junit4]   2> 2695003 INFO  (qtp1293086874-26299) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-14 
(1561016136981544960)]} 0 3
   [junit4]   2> 2695007 INFO  (qtp670289028-26291) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-9
 (-1561016136985739264)]} 0 1
   [junit4]   2> 2695007 INFO  (qtp1293086874-26300) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-9 
(-1561016136985739264)]} 0 3
   [junit4]   2> 2695008 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-15
 (1561016136986787840)]} 0 1
   [junit4]   2> 2695008 INFO  (qtp1293086874-26289) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-15 
(1561016136986787840)]} 0 3
   [junit4]   2> 2695014 INFO  (qtp670289028-26287) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-16
 (1561016136990982144)]} 0 3
   [junit4]   2> 2695014 INFO  (qtp1293086874-26301) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-16 
(1561016136990982144)]} 0 5
   [junit4]   2> 2695019 INFO  (qtp670289028-26296) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-9
 (-1561016136997273600)]} 0 3
   [junit4]   2> 2695020 INFO  (qtp1293086874-26298) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-9 
(-1561016136997273600)]} 0 5
   [junit4]   2> 2695021 INFO  (qtp670289028-26295) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-10
 (-1561016137000419328)]} 0 0
   [junit4]   2> 2695022 INFO  (qtp1293086874-26301) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-10 
(-1561016137000419328)]} 0 4
   [junit4]   2> 2695025 INFO  (qtp670289028-26288) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-16
 (1561016137005662208)]} 0 1
   [junit4]   2> 2695026 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-16 
(1561016137005662208)]} 0 3
   [junit4]   2> 2695027 INFO  (qtp670289028-26293) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-17
 (1561016137006710784)]} 0 1
   [junit4]   2> 2695028 INFO  (qtp1293086874-26299) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-17 
(1561016137006710784)]} 0 3
   [junit4]   2> 2695035 INFO  (qtp670289028-26288) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-10
 (-1561016137013002240)]} 0 4
   [junit4]   2> 2695037 INFO  (qtp1293086874-26300) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-10 
(-1561016137013002240)]} 0 7
   [junit4]   2> 2695037 INFO  (qtp670289028-26292) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-18
 (1561016137017196544)]} 0 1
   [junit4]   2> 2695038 INFO  (qtp1293086874-26289) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-18 
(1561016137017196544)]} 0 3
   [junit4]   2> 2695041 INFO  (qtp670289028-26287) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-17
 (1561016137022439424)]} 0 1
   [junit4]   2> 2695042 INFO  (qtp1293086874-26302) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-17 
(1561016137022439424)]} 0 3
   [junit4]   2> 2695043 INFO  (qtp670289028-26287) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-11
 (-1561016137023488000)]} 0 1
   [junit4]   2> 2695043 INFO  (qtp1293086874-26298) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-11 
(-1561016137023488000)]} 0 3
   [junit4]   2> 2695050 INFO  (qtp670289028-26287) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-19
 (1561016137030828032)]} 0 1
   [junit4]   2> 2695050 INFO  (qtp1293086874-26301) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-19 
(1561016137030828032)]} 0 3
   [junit4]   2> 2695051 INFO  (qtp670289028-26295) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-18
 (1561016137030828033)]} 0 1
   [junit4]   2> 2695051 INFO  (qtp1293086874-26297) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-18 
(1561016137030828033)]} 0 3
   [junit4]   2> 2695056 INFO  (qtp670289028-26291) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-12
 (-1561016137036070912)]} 0 2
   [junit4]   2> 2695056 INFO  (qtp670289028-26293) [n:127.0.0.1:61590_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-11
 (-1561016137037119488)]} 0 1
   [junit4]   2> 2695056 INFO  (qtp1293086874-26299) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-12 
(-1561016137036070912)]} 0 4
   [junit4]   2> 2695057 INFO  (qtp1293086874-26300) [n:127.0.0.1:48719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-11 
(-1561016137037119488)]} 0 3
   [junit4]   2> 2695059 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[9EA8A67A67E5D1DB]) [    ] 
o.e.j.s.AbstractConnector Stopped 
ServerConnector@58191da{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2695061 ERROR 
(updateExecutor-5113-thread-2-processing-http:////127.0.0.1:61590//solr//recoverytest_shard1_replica1
 x:recoverytest_shard1_replica2 r:core_node2 n:127.0.0.1:48719_solr s:shard1 
c:recoverytest) [n:127.0.0.1:48719_solr c:recoverytest s:shard1 r:core_node2 
x:recoverytest_shard1_replica2] o.a.s.u.StreamingSolrClients erro

[...truncated too long message...]

e_updatable_int=DocValuesFormat(name=Direct), 
inplace_updatable_float_with_default=DocValuesFormat(name=Direct)}, 
maxPointsInLeafNode=1982, maxMBSortInHeap=5.318216272717953, 
sim=RandomSimilarity(queryNorm=true,coord=crazy): {}, locale=sr-Latn-BA, 
timezone=America/El_Salvador
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 
(64-bit)/cpus=3,threads=1,free=336801496,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestBlobHandler, 
BadComponentTest, TestCollapseQParserPlugin, TestIBSimilarityFactory, 
TestGraphMLResponseWriter, TestLazyCores, DistributedFacetExistsSmallTest, 
SortSpecParsingTest, TestSortByMinMaxFunction, TestComponentsName, 
HardAutoCommitTest, DirectSolrSpellCheckerTest, TestStressRecovery, 
TestImpersonationWithHadoopAuth, JsonValidatorTest, TestRestManager, 
TestSystemCollAutoCreate, TestLRUCache, TestCollationFieldDocValues, 
TestTestInjection, ZkStateWriterTest, TestSolrConfigHandlerCloud, 
TestBulkSchemaConcurrent, TestXmlQParser, TestPseudoReturnFields, 
TestSimpleQParserPlugin, HdfsChaosMonkeySafeLeaderTest, 
TestBlendedInfixSuggestions, ZkCLITest, XmlUpdateRequestHandlerTest, 
CachingDirectoryFactoryTest, UpdateLogTest, TimeZoneUtilsTest, TestPushWriter, 
TestRandomCollapseQParserPlugin, BigEndianAscendingWordDeserializerTest, 
TestElisionMultitermQuery, TestLFUCache, TestUniqueKeyFieldResource, 
SolrRequestParserTest, RequiredFieldsTest, LeaderElectionTest, 
TestReloadAndDeleteDocs, SolrCloudExampleTest, TestPivotHelperCode, 
MultiThreadedOCPTest, BitVectorTest, TestChildDocTransformer, 
TestSubQueryTransformerCrossCore, SystemInfoHandlerTest, ZkControllerTest, 
TestSolrFieldCacheMBean, DOMUtilTest, DirectUpdateHandlerTest, 
TestBinaryResponseWriter, AddSchemaFieldsUpdateProcessorFactoryTest, 
TestPerFieldSimilarity, TestDefaultSearchFieldResource, 
TestMinMaxOnMultiValuedField, TestInitParams, BlobRepositoryCloudTest, 
MergeStrategyTest, AnalysisAfterCoreReloadTest, DirectSolrConnectionTest, 
BlockCacheTest, AlternateDirectoryTest, PeerSyncReplicationTest, 
TestDeleteCollectionOnDownNodes, SmileWriterTest, TestDFISimilarityFactory, 
DistributedQueueTest, TestAuthenticationFramework, TestStressVersions, 
SimpleCollectionCreateDeleteTest, CdcrRequestHandlerTest, 
TestOverriddenPrefixQueryForCustomFieldType, TestReplicationHandler, 
SaslZkACLProviderTest, QueryEqualityTest, ShardRoutingTest, 
RemoteQueryErrorTest, TestStressReorder, TestJsonRequest, 
TestNamedUpdateProcessors, DocumentAnalysisRequestHandlerTest, 
CurrencyFieldXmlFileTest, MultiTermTest, BlockJoinFacetDistribTest, 
StandardRequestHandlerTest, TestFieldResource, TestPerFieldSimilarityClassic, 
LeaderFailoverAfterPartitionTest, TestSweetSpotSimilarityFactory, 
StatelessScriptUpdateProcessorFactoryTest, TestHighlightDedupGrouping, 
FastVectorHighlighterTest, TestCustomStream, TestRemoteStreaming, 
TestWordDelimiterFilterFactory, ShufflingReplicaListTransformerTest, 
BlockJoinFacetSimpleTest, MetricUtilsTest, TestReRankQParserPlugin, 
TestTolerantSearch, JSONWriterTest, TestDistributedStatsComponentCardinality, 
TestUseDocValuesAsStored2, TestMiniSolrCloudClusterSSL, TestSchemaResource, 
DirectoryFactoryTest, TestCollationField, SchemaVersionSpecificBehaviorTest, 
AnalyticsQueryTest, QueryElevationComponentTest, 
CloudExitableDirectoryReaderTest, ParsingFieldUpdateProcessorsTest, 
DocValuesTest, TestConfigSetsAPI, DocumentBuilderTest, 
TestSha256AuthenticationProvider, TestRTGBase, TestRebalanceLeaders, 
DistributedQueryElevationComponentTest, MoreLikeThisHandlerTest, TestSolrXml, 
TestConfigSets, DocExpirationUpdateProcessorFactoryTest, SolrPluginUtilsTest, 
TestDownShardTolerantSearch, SolrCoreTest, PluginInfoTest, 
ConjunctionSolrSpellCheckerTest, BooleanFieldTest, TestDistribIDF, 
SimpleMLTQParserTest, TestSurroundQueryParser, HttpSolrCallGetCoreTest, 
WrapperMergePolicyFactoryTest, HdfsTlogReplayBufferedWhileIndexingTest, 
CdcrReplicationDistributedZkTest, DateRangeFieldTest, 
DistributedSpellCheckComponentTest, ConcurrentDeleteAndCreateCollectionTest, 
DeleteLastCustomShardedReplicaTest, RuleEngineTest, TestXIncludeConfig, 
TestStressUserVersions, TestExceedMaxTermLength, TestNumericTerms32, 
TestBinaryField, DateFieldTest, FileUtilsTest, PeerSyncTest, 
DocValuesMultiTest, TestEmbeddedSolrServerConstructors, DeleteStatusTest, 
CoreAdminCreateDiscoverTest, DistributedFacetPivotLongTailTest, 
OverseerModifyCollectionTest, TestMacroExpander, TestSolrJ, 
SolrJmxReporterTest, TestLocalFSCloudBackupRestore, HdfsDirectoryFactoryTest, 
PolyFieldTest, TestCSVLoader, TestBackupRepositoryFactory, 
TestCursorMarkWithoutUniqueKey, TestRequestForwarding, 
VMParamsZkACLAndCredentialsProvidersTest, BasicDistributedZkTest, 
ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, 
OpenCloseCoreStressTest, OverseerTest, ClusterStateUpdateTest, TestZkChroot, 
ShardRoutingCustomTest, AutoCommitTest, TestFaceting, TestHashPartitioner, 
TestRealTimeGet, TestRangeQuery, TestCoreContainer, SimpleFacetsTest, 
TestSolr4Spatial, TestGroupingSearch, SolrCmdDistributorTest, 
BadIndexSchemaTest, TestSort, TestFiltering, TestFunctionQuery, 
HighlighterTest, SoftAutoCommitTest, CurrencyFieldOpenExchangeTest, 
SolrIndexSplitterTest, SimplePostToolTest, SignatureUpdateProcessorFactoryTest, 
TestExtendedDismaxParser, SpellCheckCollatorTest, SuggesterFSTTest, 
CoreAdminHandlerTest, TestFoldingMultitermQuery, SuggesterTest, 
TestStressLucene, SpatialFilterTest, NoCacheHeaderTest, 
WordBreakSolrSpellCheckerTest, SolrCoreCheckLockOnStartupTest, TestUpdate, 
TestAtomicUpdateErrorCases, DefaultValueUpdateProcessorTest, 
TestSolrDeletionPolicy1, SolrInfoMBeanTest, XsltUpdateRequestHandlerTest, 
DebugComponentTest, CacheHeaderTest, IndexBasedSpellCheckerTest, 
LukeRequestHandlerTest, DisMaxRequestHandlerTest, TestWriterPerf, 
PrimitiveFieldTypeTest, TestOmitPositions, RequestHandlersTest, 
TestIndexingPerformance, TestSolrQueryParser, LoggingHandlerTest, 
TestJmxIntegration, TestCSVResponseWriter, TestAnalyzedSuggestions, 
TestPartialUpdateDeduplication, PingRequestHandlerTest, 
TestBinaryResponseWriter, SOLR749Test, UpdateParamsTest, 
TestQuerySenderNoQuery, ResponseLogComponentTest, TestDocSet, TestConfig, 
TestFuzzyAnalyzedSuggestions, TestPostingsSolrHighlighter, 
SpellPossibilityIteratorTest, TestCharFilters, TestCodecSupport, 
TestJmxMonitoredMap, ResourceLoaderTest, ScriptEngineTest, 
URLClassifyProcessorTest, ChaosMonkeyNothingIsSafeTest, ActionThrottleTest, 
BaseCdcrDistributedZkTest, CdcrBootstrapTest, 
CollectionsAPIAsyncDistributedZkTest, ConnectionManagerTest, 
CustomCollectionTest, DeleteReplicaTest, DocValuesNotIndexedTest, 
HttpPartitionTest, MigrateRouteKeyTest, ReplicationFactorTest, 
RestartWhileUpdatingTest, RollingRestartTest, SharedFSAutoReplicaFailoverTest, 
SolrCLIZkUtilsTest, SolrXmlInZkTest, TestCloudInspectUtil, TestCloudPivotFacet, 
TestCloudPseudoReturnFields, TestCloudRecovery, TestClusterProperties, 
TestConfigSetsAPIZkFailure, TestCryptoKeys, TestDistribDocBasedVersion, 
TestHdfsCloudBackupRestore, TestLeaderElectionZkExpiry, TestLockTree, 
TestMiniSolrCloudCluster, TestOnReconnectListenerSupport, TestRandomFlRTGCloud, 
TestRandomRequestDistribution, TestReplicaProperties, 
TestRequestStatusCollectionAPI, TestSSLRandomization, TestSegmentSorting, 
TestShortCircuitedRequests, TestSizeLimitedDistributedMap, 
TestSolrCloudWithSecureImpersonation, TestStressInPlaceUpdates, 
HdfsRecoveryZkTest, CoreAdminRequestStatusTest, 
CoreMergeIndexesAdminHandlerTest, MetricsHandlerTest, 
SegmentsInfoRequestHandlerTest, StatsReloadRaceTest, TestApiFramework, 
TestConfigsApi, TestCoreAdminApis, DistributedDebugComponentTest, 
DistributedFacetPivotSmallTest, DistributedQueryComponentCustomSortTest, 
DistributedQueryComponentOptimizationTest, InfixSuggestersTest, 
SuggestComponentContextFilterQueryTest, SuggestComponentTest, 
JavabinLoaderTest, HighlighterMaxOffsetTest, TestUnifiedSolrHighlighter, 
TestSlowCompositeReaderWrapper, JvmMetricsTest, SolrCoreMetricManagerTest, 
SolrMetricManagerTest, SolrMetricReporterTest, SolrMetricsIntegrationTest, 
SolrGangliaReporterTest, SolrGraphiteReporterTest, SubstringBytesRefFilterTest, 
JSONWriterTest, SmileWriterTest, TestExportWriter, TestGeoJSONResponseWriter, 
TestRawResponseWriter, TestRawTransformer, TestSolrQueryResponse, 
TestSubQueryTransformerDistrib, TestDynamicFieldResource, 
TestFieldTypeCollectionResource, TestFieldTypeResource, 
TestSchemaSimilarityResource, TestSerializedLuceneMatchVersion, 
TestManagedStopFilterFactory, EnumFieldTest, 
PreAnalyzedFieldManagedSchemaCloudTest, TestCloudManagedSchema, 
TestPointFields, CursorMarkTest, TestAddFieldRealTimeGet, 
TestComplexPhraseQParserPlugin, TestHashQParserPlugin, TestMissingGroups, 
TestQueryWrapperFilter, TestSearcherReuse, TestXmlQParserPlugin, TestOrdValues, 
TestScoreJoinQPNoScore, TestExactSharedStatsCache, TestExactStatsCache, 
PKIAuthenticationIntegrationTest, TestAuthorizationFramework, 
TestFreeTextSuggestions, BufferStoreTest, HdfsLockFactoryTest, TestFieldCache, 
TestFieldCacheSanityChecker, TestFieldCacheSort, TestFieldCacheSortRandom, 
TestFieldCacheWithThreads, TestUninvertingReader, AnalysisErrorHandlingTest, 
DataDrivenBlockJoinTest, PeerSyncWithIndexFingerprintCachingTest, 
SolrIndexMetricsTest, TestDocBasedVersionConstraints, TestInPlaceUpdatesDistrib]
   [junit4] Completed [697/697 (2!)] on J0 in 64.57s, 1 test, 1 error <<< 
FAILURES!

[...truncated 11729 lines...]
-validate-source-patterns:
[source-patterns] tabs instead spaces: solr/webapp/web/partials/collections.html

BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:775: The 
following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:130: Found 1 
violations in source files (tabs instead spaces).

Total time: 80 minutes 22 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to