Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/97/ Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC
9 tests failed. FAILED: org.apache.solr.cloud.TestPullReplica.testCreateDelete { seed=[15C11CC252BA21C5:ED1ED8EA44AD9F5]} Error Message: Could not find collection : pull_replica_test_create_delete Stack Trace: org.apache.solr.common.SolrException: Could not find collection : pull_replica_test_create_delete at __randomizedtesting.SeedInfo.seed([15C11CC252BA21C5:ED1ED8EA44AD9F5]:0) at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:109) at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247) at org.apache.solr.cloud.TestPullReplica.testCreateDelete(TestPullReplica.java:161) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.TestPullReplica.testCreateDelete { seed=[15C11CC252BA21C5:BA875172908812D9]} Error Message: Too many open files Stack Trace: java.net.SocketException: Too many open files at __randomizedtesting.SeedInfo.seed([15C11CC252BA21C5:BA875172908812D9]:0) at java.net.Socket.createImpl(Socket.java:460) at java.net.Socket.getImpl(Socket.java:520) at java.net.Socket.setSoTimeout(Socket.java:1141) at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:119) at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353) at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55) at org.apache.solr.cloud.TestPullReplica.testCreateDelete(TestPullReplica.java:155) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.TestPullReplica.testKillPullReplica Error Message: Expected collection to be created with 1 shard and 2 replicas null Live Nodes: [127.0.0.1:60984_solr, 127.0.0.1:60983_solr] Last available state: null Stack Trace: java.lang.AssertionError: Expected collection to be created with 1 shard and 2 replicas null Live Nodes: [127.0.0.1:60984_solr, 127.0.0.1:60983_solr] Last available state: null at __randomizedtesting.SeedInfo.seed([15C11CC252BA21C5:99300057F20BC0FD]:0) at org.junit.Assert.fail(Assert.java:93) at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269) at org.apache.solr.cloud.TestPullReplica.testKillPullReplica(TestPullReplica.java:467) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.TestTlogReplica.testCreateDelete { seed=[15C11CC252BA21C5:BA875172908812D9]} Error Message: Too many open files Stack Trace: java.net.SocketException: Too many open files at __randomizedtesting.SeedInfo.seed([15C11CC252BA21C5:BA875172908812D9]:0) at java.net.Socket.createImpl(Socket.java:460) at java.net.Socket.getImpl(Socket.java:520) at java.net.Socket.setSoTimeout(Socket.java:1141) at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:119) at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353) at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55) at org.apache.solr.cloud.TestTlogReplica.testCreateDelete(TestTlogReplica.java:181) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.TestTlogReplica.testDeleteById Error Message: Expected collection to be created with 1 shards and 2 replicas null Live Nodes: [127.0.0.1:61437_solr, 127.0.0.1:61436_solr] Last available state: null Stack Trace: java.lang.AssertionError: Expected collection to be created with 1 shards and 2 replicas null Live Nodes: [127.0.0.1:61437_solr, 127.0.0.1:61436_solr] Last available state: null at __randomizedtesting.SeedInfo.seed([15C11CC252BA21C5:4C4C6F510CE84925]:0) at org.junit.Assert.fail(Assert.java:93) at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269) at org.apache.solr.cloud.TestTlogReplica.createAndWaitForCollection(TestTlogReplica.java:716) at org.apache.solr.cloud.TestTlogReplica.testDeleteById(TestTlogReplica.java:603) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates Error Message: Stack Trace: java.lang.NullPointerException at __randomizedtesting.SeedInfo.seed([15C11CC252BA21C5:9300E42F0DEBF725]:0) at org.apache.solr.cloud.TestTlogReplica.getSolrCore(TestTlogReplica.java:846) at org.apache.solr.cloud.TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates(TestTlogReplica.java:652) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.TestTlogReplica.testBasicLeaderElection Error Message: Stack Trace: java.lang.NullPointerException at __randomizedtesting.SeedInfo.seed([15C11CC252BA21C5:8349DA8ABA20663F]:0) at org.apache.solr.cloud.TestTlogReplica.getSolrRunner(TestTlogReplica.java:881) at org.apache.solr.cloud.TestTlogReplica.testBasicLeaderElection(TestTlogReplica.java:635) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.TestTlogReplica.testRecovery Error Message: Stack Trace: java.lang.NullPointerException at __randomizedtesting.SeedInfo.seed([15C11CC252BA21C5:D431656E7FEAEB62]:0) at org.apache.solr.cloud.TestTlogReplica.getSolrRunner(TestTlogReplica.java:881) at org.apache.solr.cloud.TestTlogReplica.testRecovery(TestTlogReplica.java:524) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.TestTlogReplica.testOnlyLeaderIndexes Error Message: Stack Trace: java.lang.NullPointerException at __randomizedtesting.SeedInfo.seed([15C11CC252BA21C5:9C0614F271F5F56]:0) at org.apache.solr.cloud.TestTlogReplica.getSolrCore(TestTlogReplica.java:846) at org.apache.solr.cloud.TestTlogReplica.testOnlyLeaderIndexes(TestTlogReplica.java:467) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Build Log: [...truncated 12548 lines...] [junit4] Suite: org.apache.solr.cloud.TestPullReplica [junit4] 2> 2730722 INFO (SUITE-TestPullReplica-seed#[15C11CC252BA21C5]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/init-core-data-001 [junit4] 2> 2730722 WARN (SUITE-TestPullReplica-seed#[15C11CC252BA21C5]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=18 numCloses=18 [junit4] 2> 2730723 INFO (SUITE-TestPullReplica-seed#[15C11CC252BA21C5]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true [junit4] 2> 2730723 INFO (SUITE-TestPullReplica-seed#[15C11CC252BA21C5]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth [junit4] 2> 2730724 INFO (SUITE-TestPullReplica-seed#[15C11CC252BA21C5]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001 [junit4] 2> 2730724 INFO (SUITE-TestPullReplica-seed#[15C11CC252BA21C5]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 2730724 INFO (Thread-4432) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 2730724 INFO (Thread-4432) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 2730729 ERROR (Thread-4432) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 2730827 INFO (SUITE-TestPullReplica-seed#[15C11CC252BA21C5]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:60981 [junit4] 2> 2730863 INFO (jetty-launcher-2996-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028 [junit4] 2> 2730863 INFO (jetty-launcher-2996-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028 [junit4] 2> 2730865 INFO (jetty-launcher-2996-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@75747671{/solr,null,AVAILABLE} [junit4] 2> 2730866 INFO (jetty-launcher-2996-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2736f530{/solr,null,AVAILABLE} [junit4] 2> 2730866 INFO (jetty-launcher-2996-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3cfa5815{HTTP/1.1,[http/1.1]}{127.0.0.1:60983} [junit4] 2> 2730866 INFO (jetty-launcher-2996-thread-1) [ ] o.e.j.s.Server Started @2735123ms [junit4] 2> 2730866 INFO (jetty-launcher-2996-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60983} [junit4] 2> 2730866 INFO (jetty-launcher-2996-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6bd953dc{HTTP/1.1,[http/1.1]}{127.0.0.1:60984} [junit4] 2> 2730866 INFO (jetty-launcher-2996-thread-2) [ ] o.e.j.s.Server Started @2735123ms [junit4] 2> 2730866 ERROR (jetty-launcher-2996-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 2730866 INFO (jetty-launcher-2996-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60984} [junit4] 2> 2730866 INFO (jetty-launcher-2996-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.1.0 [junit4] 2> 2730866 ERROR (jetty-launcher-2996-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 2730866 INFO (jetty-launcher-2996-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 2730866 INFO (jetty-launcher-2996-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null [junit4] 2> 2730866 INFO (jetty-launcher-2996-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.1.0 [junit4] 2> 2730866 INFO (jetty-launcher-2996-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-06T17:50:29.446Z [junit4] 2> 2730866 INFO (jetty-launcher-2996-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 2730867 INFO (jetty-launcher-2996-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null [junit4] 2> 2730867 INFO (jetty-launcher-2996-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-06T17:50:29.447Z [junit4] 2> 2730869 INFO (jetty-launcher-2996-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading... [junit4] 2> 2730869 INFO (jetty-launcher-2996-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading... [junit4] 2> 2730876 INFO (jetty-launcher-2996-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60981/solr [junit4] 2> 2730876 INFO (jetty-launcher-2996-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60981/solr [junit4] 2> 2731001 INFO (jetty-launcher-2996-thread-1) [n:127.0.0.1:60983_solr ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 2731001 INFO (jetty-launcher-2996-thread-2) [n:127.0.0.1:60984_solr ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 2731001 INFO (jetty-launcher-2996-thread-1) [n:127.0.0.1:60983_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60983_solr [junit4] 2> 2731002 INFO (jetty-launcher-2996-thread-2) [n:127.0.0.1:60984_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60984_solr [junit4] 2> 2731003 INFO (jetty-launcher-2996-thread-1) [n:127.0.0.1:60983_solr ] o.a.s.c.Overseer Overseer (id=98437813325398021-127.0.0.1:60983_solr-n_0000000000) starting [junit4] 2> 2731004 INFO (zkCallback-3007-thread-1-processing-n:127.0.0.1:60983_solr) [n:127.0.0.1:60983_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 2731004 INFO (zkCallback-3008-thread-1-processing-n:127.0.0.1:60984_solr) [n:127.0.0.1:60984_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 2731015 INFO (jetty-launcher-2996-thread-1) [n:127.0.0.1:60983_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60983_solr [junit4] 2> 2731016 INFO (zkCallback-3007-thread-1-processing-n:127.0.0.1:60983_solr) [n:127.0.0.1:60983_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 2731017 INFO (zkCallback-3008-thread-1-processing-n:127.0.0.1:60984_solr) [n:127.0.0.1:60984_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 2731101 INFO (jetty-launcher-2996-thread-1) [n:127.0.0.1:60983_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60983.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71f2ba0 [junit4] 2> 2731111 INFO (jetty-launcher-2996-thread-2) [n:127.0.0.1:60984_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60984.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71f2ba0 [junit4] 2> 2731112 INFO (jetty-launcher-2996-thread-1) [n:127.0.0.1:60983_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60983.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71f2ba0 [junit4] 2> 2731112 INFO (jetty-launcher-2996-thread-1) [n:127.0.0.1:60983_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60983.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71f2ba0 [junit4] 2> 2731113 INFO (jetty-launcher-2996-thread-2) [n:127.0.0.1:60984_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60984.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71f2ba0 [junit4] 2> 2731113 INFO (jetty-launcher-2996-thread-2) [n:127.0.0.1:60984_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60984.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71f2ba0 [junit4] 2> 2731113 INFO (jetty-launcher-2996-thread-1) [n:127.0.0.1:60983_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node1/. [junit4] 2> 2731114 INFO (jetty-launcher-2996-thread-2) [n:127.0.0.1:60984_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node2/. [junit4] 2> 2731146 INFO (SUITE-TestPullReplica-seed#[15C11CC252BA21C5]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2) [junit4] 2> 2731148 INFO (SUITE-TestPullReplica-seed#[15C11CC252BA21C5]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60981/solr ready [junit4] 2> 2731152 INFO (SUITE-TestPullReplica-seed#[15C11CC252BA21C5]-worker) [ ] o.a.s.c.TestPullReplica Using legacyCloud?: false [junit4] 2> 2731153 INFO (qtp1229430273-22263) [n:127.0.0.1:60984_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true [junit4] 2> 2731154 INFO (qtp1229430273-22263) [n:127.0.0.1:60984_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=1 [junit4] 2> 2731251 INFO (TEST-TestPullReplica.testCreateDelete-seed#[15C11CC252BA21C5]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCreateDelete { seed=[15C11CC252BA21C5:ED1ED8EA44AD9F5]} [junit4] 2> 2731267 INFO (OverseerThreadFactory-8912-thread-1-processing-n:127.0.0.1:60983_solr) [n:127.0.0.1:60983_solr ] o.a.s.c.CreateCollectionCmd Create collection pull_replica_test_create_delete [junit4] 2> 2731268 WARN (OverseerThreadFactory-8912-thread-1-processing-n:127.0.0.1:60983_solr) [n:127.0.0.1:60983_solr ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 4 on collection pull_replica_test_create_delete is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance. [junit4] 2> 2731386 INFO (OverseerStateUpdate-98437813325398021-127.0.0.1:60983_solr-n_0000000000) [n:127.0.0.1:60983_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"pull_replica_test_create_delete_shard1_replica_n1", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:60984/solr", [junit4] 2> "type":"NRT"} [junit4] 2> 2731389 INFO (OverseerStateUpdate-98437813325398021-127.0.0.1:60983_solr-n_0000000000) [n:127.0.0.1:60983_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"pull_replica_test_create_delete_shard1_replica_p2", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:60983/solr", [junit4] 2> "type":"PULL"} [junit4] 2> 2731392 INFO (OverseerStateUpdate-98437813325398021-127.0.0.1:60983_solr-n_0000000000) [n:127.0.0.1:60983_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"pull_replica_test_create_delete_shard1_replica_p4", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:60984/solr", [junit4] 2> "type":"PULL"} [junit4] 2> 2731398 INFO (OverseerStateUpdate-98437813325398021-127.0.0.1:60983_solr-n_0000000000) [n:127.0.0.1:60983_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"pull_replica_test_create_delete_shard1_replica_p6", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:60983/solr", [junit4] 2> "type":"PULL"} [junit4] 2> 2731401 INFO (OverseerStateUpdate-98437813325398021-127.0.0.1:60983_solr-n_0000000000) [n:127.0.0.1:60983_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard2", [junit4] 2> "core":"pull_replica_test_create_delete_shard2_replica_n8", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:60984/solr", [junit4] 2> "type":"NRT"} [junit4] 2> 2731404 INFO (OverseerStateUpdate-98437813325398021-127.0.0.1:60983_solr-n_0000000000) [n:127.0.0.1:60983_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard2", [junit4] 2> "core":"pull_replica_test_create_delete_shard2_replica_p10", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:60983/solr", [junit4] 2> "type":"PULL"} [junit4] 2> 2731408 INFO (OverseerStateUpdate-98437813325398021-127.0.0.1:60983_solr-n_0000000000) [n:127.0.0.1:60983_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard2", [junit4] 2> "core":"pull_replica_test_create_delete_shard2_replica_p12", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:60984/solr", [junit4] 2> "type":"PULL"} [junit4] 2> 2731411 INFO (OverseerStateUpdate-98437813325398021-127.0.0.1:60983_solr-n_0000000000) [n:127.0.0.1:60983_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard2", [junit4] 2> "core":"pull_replica_test_create_delete_shard2_replica_p14", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:60983/solr", [junit4] 2> "type":"PULL"} [junit4] 2> 2731634 INFO (qtp775430357-22260) [n:127.0.0.1:60983_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p2&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL [junit4] 2> 2731635 INFO (qtp775430357-22260) [n:127.0.0.1:60983_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores [junit4] 2> 2731635 INFO (qtp1229430273-22264) [n:127.0.0.1:60984_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_n1&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=NRT [junit4] 2> 2731636 INFO (qtp775430357-22262) [n:127.0.0.1:60983_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p6&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL [junit4] 2> 2731636 INFO (qtp1229430273-22264) [n:127.0.0.1:60984_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores [junit4] 2> 2731637 INFO (qtp1229430273-22266) [n:127.0.0.1:60984_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_n8&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=NRT [junit4] 2> 2731637 INFO (qtp775430357-22265) [n:127.0.0.1:60983_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node13&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p10&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL [junit4] 2> 2731638 INFO (qtp1229430273-22270) [n:127.0.0.1:60984_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p4&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL [junit4] 2> 2731639 INFO (qtp775430357-22267) [n:127.0.0.1:60983_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node16&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p14&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL [junit4] 2> 2731639 INFO (qtp1229430273-22271) [n:127.0.0.1:60984_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node15&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p12&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL [junit4] 2> 2731782 INFO (zkCallback-3008-thread-1-processing-n:127.0.0.1:60984_solr) [n:127.0.0.1:60984_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2]) [junit4] 2> 2731782 INFO (zkCallback-3007-thread-1-processing-n:127.0.0.1:60983_solr) [n:127.0.0.1:60983_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2]) [junit4] 2> 2731783 INFO (zkCallback-3008-thread-2-processing-n:127.0.0.1:60984_solr) [n:127.0.0.1:60984_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2]) [junit4] 2> 2731783 INFO (zkCallback-3007-thread-2-processing-n:127.0.0.1:60983_solr) [n:127.0.0.1:60983_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2]) [junit4] 2> 2732687 INFO (qtp1229430273-22270) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2732687 INFO (qtp1229430273-22271) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2732694 INFO (qtp1229430273-22270) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_p4] Schema name=minimal [junit4] 2> 2732694 INFO (qtp1229430273-22271) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_p12] Schema name=minimal [junit4] 2> 2732695 INFO (qtp775430357-22260) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2732697 INFO (qtp1229430273-22271) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2732697 INFO (qtp1229430273-22270) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2732698 INFO (qtp1229430273-22270) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_p4' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2732697 INFO (qtp1229430273-22271) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_p12' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2732698 INFO (qtp1229430273-22270) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60984.solr.core.pull_replica_test_create_delete.shard1.replica_p4' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_p4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71f2ba0 [junit4] 2> 2732698 INFO (qtp1229430273-22271) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60984.solr.core.pull_replica_test_create_delete.shard2.replica_p12' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_p12') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71f2ba0 [junit4] 2> 2732698 INFO (qtp1229430273-22270) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2732698 INFO (qtp1229430273-22271) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2732698 INFO (qtp1229430273-22270) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_p4] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node2/pull_replica_test_create_delete_shard1_replica_p4], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node2/./pull_replica_test_create_delete_shard1_replica_p4/data/] [junit4] 2> 2732698 INFO (qtp1229430273-22271) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_p12] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node2/pull_replica_test_create_delete_shard2_replica_p12], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node2/./pull_replica_test_create_delete_shard2_replica_p12/data/] [junit4] 2> 2732715 INFO (qtp775430357-22267) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2732716 INFO (qtp775430357-22262) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2732716 INFO (qtp775430357-22260) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_p2] Schema name=minimal [junit4] 2> 2732718 INFO (qtp775430357-22260) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2732718 INFO (qtp775430357-22260) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_p2' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2732719 INFO (qtp775430357-22260) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60983.solr.core.pull_replica_test_create_delete.shard1.replica_p2' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_p2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71f2ba0 [junit4] 2> 2732719 INFO (qtp775430357-22260) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2732719 INFO (qtp775430357-22260) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_p2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node1/pull_replica_test_create_delete_shard1_replica_p2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node1/./pull_replica_test_create_delete_shard1_replica_p2/data/] [junit4] 2> 2732720 INFO (qtp1229430273-22266) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2732721 INFO (qtp775430357-22267) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_p14] Schema name=minimal [junit4] 2> 2732722 INFO (qtp775430357-22262) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_p6] Schema name=minimal [junit4] 2> 2732723 INFO (qtp775430357-22267) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2732723 INFO (qtp775430357-22267) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_p14' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2732723 INFO (qtp775430357-22262) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2732723 INFO (qtp775430357-22262) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_p6' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2732723 INFO (qtp775430357-22267) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60983.solr.core.pull_replica_test_create_delete.shard2.replica_p14' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_p14') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71f2ba0 [junit4] 2> 2732723 INFO (qtp775430357-22267) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2732724 INFO (qtp775430357-22267) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_p14] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node1/pull_replica_test_create_delete_shard2_replica_p14], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node1/./pull_replica_test_create_delete_shard2_replica_p14/data/] [junit4] 2> 2732724 INFO (qtp775430357-22262) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60983.solr.core.pull_replica_test_create_delete.shard1.replica_p6' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_p6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71f2ba0 [junit4] 2> 2732724 INFO (qtp775430357-22262) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2732724 INFO (qtp775430357-22262) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_p6] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node1/pull_replica_test_create_delete_shard1_replica_p6], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node1/./pull_replica_test_create_delete_shard1_replica_p6/data/] [junit4] 2> 2732725 INFO (qtp1229430273-22264) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2732726 INFO (qtp775430357-22265) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2732726 INFO (qtp1229430273-22266) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_n8] Schema name=minimal [junit4] 2> 2732728 INFO (qtp1229430273-22266) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2732729 INFO (qtp1229430273-22266) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_n8' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2732729 INFO (qtp1229430273-22266) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60984.solr.core.pull_replica_test_create_delete.shard2.replica_n8' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_n8') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71f2ba0 [junit4] 2> 2732729 INFO (qtp1229430273-22266) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2732729 INFO (qtp1229430273-22266) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_n8] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node2/pull_replica_test_create_delete_shard2_replica_n8], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node2/./pull_replica_test_create_delete_shard2_replica_n8/data/] [junit4] 2> 2732731 INFO (qtp1229430273-22264) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_n1] Schema name=minimal [junit4] 2> 2732731 INFO (qtp775430357-22265) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_p10] Schema name=minimal [junit4] 2> 2732743 INFO (qtp775430357-22265) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2732743 INFO (qtp775430357-22265) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_p10' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2732743 INFO (qtp775430357-22265) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60983.solr.core.pull_replica_test_create_delete.shard2.replica_p10' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_p10') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71f2ba0 [junit4] 2> 2732744 INFO (qtp775430357-22265) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2732744 INFO (qtp775430357-22265) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_p10] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node1/pull_replica_test_create_delete_shard2_replica_p10], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node1/./pull_replica_test_create_delete_shard2_replica_p10/data/] [junit4] 2> 2732744 INFO (qtp1229430273-22264) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2732744 INFO (qtp1229430273-22264) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_n1' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2732745 INFO (qtp1229430273-22264) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60984.solr.core.pull_replica_test_create_delete.shard1.replica_n1' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71f2ba0 [junit4] 2> 2732745 INFO (qtp1229430273-22264) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2732745 INFO (qtp1229430273-22264) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node2/pull_replica_test_create_delete_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_15C11CC252BA21C5-001/tempDir-001/node2/./pull_replica_test_create_delete_shard1_replica_n1/data/] [junit4] 2> 2732815 INFO (qtp1229430273-22266) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard2_replica_n8] CLOSING SolrCore org.apache.solr.core.SolrCore@2095f1ca [junit4] 2> 2732815 INFO (qtp1229430273-22270) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_p4] CLOSING SolrCore org.apache.solr.core.SolrCore@3a2c1cc6 [junit4] 2> 2732817 INFO (qtp775430357-22262) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_p6] CLOSING SolrCore org.apache.solr.core.SolrCore@1a794f53 [junit4] 2> 2732817 INFO (qtp1229430273-22266) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_create_delete.shard2.replica_n8, tag=546697674 [junit4] 2> 2732818 INFO (qtp1229430273-22264) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@557d82fc [junit4] 2> 2732818 INFO (qtp1229430273-22271) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard2_replica_p12] CLOSING SolrCore org.apache.solr.core.SolrCore@2fe37ff9 [junit4] 2> 2732818 INFO (qtp1229430273-22266) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.pull_replica_test_create_delete.shard2.leader, tag=546697674 [junit4] 2> 2732818 INFO (qtp775430357-22265) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard2_replica_p10] CLOSING SolrCore org.apache.solr.core.SolrCore@7603760a [junit4] 2> 2732820 INFO (qtp1229430273-22271) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_create_delete.shard2.replica_p12, tag=803438585 [junit4] 2> 2732821 INFO (qtp775430357-22262) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_create_delete.shard1.replica_p6, tag=444157779 [junit4] 2> 2732821 INFO (qtp1229430273-22271) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.pull_replica_test_create_delete.shard2.leader, tag=803438585 [junit4] 2> 2732821 INFO (qtp1229430273-22264) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_create_delete.shard1.replica_n1, tag=1434288892 [junit4] 2> 2732822 INFO (qtp775430357-22260) [n:127.0.0.1:60983_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_p2] CLOSING SolrCore org.apache.solr.core.SolrCore@1f24a3a2 [junit4] 2> 2732822 ERROR (qtp1229430273-22266) [n:127.0.0.1:60984_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'pull_replica_test_create_delete_shard2_replica_n8': Unable to create core [pull_replica_test_create_delete_shard2_replica_n8] Caused by: invalid API spec: apispec/core.SchemaEdit.json [junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:950) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389) [junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174) [junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726) [junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507) [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:378) [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:322) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699) [junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) [junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) [junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) [junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) [junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) [junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) [junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395) [junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) [junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534) [junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) [junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) [junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273) [junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95) [junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) [junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> Caused by: org.apache.solr.common.SolrException: Unable to create core [pull_replica_test_create_delete_shard2_replica_n8] [junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:997) [junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:917) [junit4] 2> ... 34 more [junit4] 2> Caused by: org.apache.solr.common.SolrException: invalid A [...truncated too long message...] r-5941-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3bcbda2d{/solr,null,UNAVAILABLE} [junit4] 2> 3053070 ERROR (SUITE-TestTlogReplica-seed#[15C11CC252BA21C5]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 3053071 INFO (SUITE-TestTlogReplica-seed#[15C11CC252BA21C5]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:61434 61434 [junit4] 2> 3053090 INFO (Thread-4431) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:61434 61434 [junit4] 2> 3053382 WARN (Thread-4431) [ ] o.a.s.c.ZkTestServer Watch limit violations: [junit4] 2> Maximum concurrent create/delete watches above limit: [junit4] 2> [junit4] 2> 13 /solr/configs/conf [junit4] 2> 5 /solr/aliases.json [junit4] 2> 4 /solr/security.json [junit4] 2> 3 /solr/clusterprops.json [junit4] 2> 2 /solr/collections/tlog_replica_test_create_delete/state.json [junit4] 2> [junit4] 2> Maximum concurrent data watches above limit: [junit4] 2> [junit4] 2> 26 /solr/collections/tlog_replica_test_remove_leader/state.json [junit4] 2> 25 /solr/collections/tlog_replica_test_kill_tlog_replica/state.json [junit4] 2> 25 /solr/collections/tlog_replica_test_add_remove_tlog_replica/state.json [junit4] 2> 21 /solr/collections/tlog_replica_test_kill_leader/state.json [junit4] 2> 14 /solr/collections/tlog_replica_test_create_delete/state.json [junit4] 2> 13 /solr/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json [junit4] 2> 13 /solr/collections/tlog_replica_test_recovery/state.json [junit4] 2> 13 /solr/collections/tlog_replica_test_basic_leader_election/state.json [junit4] 2> 13 /solr/collections/tlog_replica_test_only_leader_indexes/state.json [junit4] 2> 6 /solr/collections/tlog_replica_test_add_docs/state.json [junit4] 2> 6 /solr/collections/tlog_replica_test_real_time_get/state.json [junit4] 2> 5 /solr/clusterprops.json [junit4] 2> 5 /solr/clusterstate.json [junit4] 2> 3 /solr/overseer_elect/election/98437821987225605-127.0.0.1:61436_solr-n_0000000000 [junit4] 2> [junit4] 2> Maximum concurrent children watches above limit: [junit4] 2> [junit4] 2> 5 /solr/collections [junit4] 2> 3 /solr/live_nodes [junit4] 2> [junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_15C11CC252BA21C5-001 [junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {foo=PostingsFormat(name=Direct), id=FST50}, docValues:{_version_=DocValuesFormat(name=Lucene70), id=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1780, maxMBSortInHeap=7.263061468826395, sim=RandomSimilarity(queryNorm=false): {}, locale=en-IN, timezone=Asia/Pyongyang [junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=210820728,total=524288000 [junit4] 2> NOTE: All tests run in this JVM: [DocValuesMissingTest, TestSizeLimitedDistributedMap, AddSchemaFieldsUpdateProcessorFactoryTest, TestDynamicFieldCollectionResource, TestDocSet, TestJettySolrRunner, TestRandomRequestDistribution, LoggingHandlerTest, FieldMutatingUpdateProcessorTest, ClassificationUpdateProcessorIntegrationTest, TestCloudJSONFacetJoinDomain, TestPrepRecovery, TermsComponentTest, TestPostingsSolrHighlighter, TestPullReplicaErrorHandling, UtilsToolTest, SuggesterFSTTest, CdcrBootstrapTest, QueryEqualityTest, AnalysisErrorHandlingTest, TestSchemalessBufferedUpdates, TestCrossCoreJoin, TestConfigSetProperties, PluginInfoTest, TestBadConfig, TestComponentsName, CollectionsAPISolrJTest, TestCloudSearcherWarming, CollectionsAPIDistributedZkTest, SuggestComponentContextFilterQueryTest, TestQueryTypes, BasicDistributedZk2Test, SpellCheckCollatorTest, LukeRequestHandlerTest, TestCustomSort, TestSolrCLIRunExample, PolyFieldTest, TestShardHandlerFactory, TestConfig, DistributedFacetPivotLongTailTest, TestLMJelinekMercerSimilarityFactory, TestSolrConfigHandlerCloud, DistribJoinFromCollectionTest, TestConfigsApi, LeaderElectionIntegrationTest, TestRemoteStreaming, TestFastWriter, TestSha256AuthenticationProvider, TestSolrDeletionPolicy1, BlockDirectoryTest, TestSolrCoreParser, CreateCollectionCleanupTest, HdfsTlogReplayBufferedWhileIndexingTest, TestXmlQParserPlugin, HLLUtilTest, StatsComponentTest, OpenExchangeRatesOrgProviderTest, DistributedDebugComponentTest, DeleteNodeTest, TestSolrCloudSnapshots, BadIndexSchemaTest, BooleanFieldTest, TestNRTOpen, TestSmileRequest, HdfsDirectoryTest, StatelessScriptUpdateProcessorFactoryTest, TestInPlaceUpdatesDistrib, CdcrReplicationHandlerTest, TestFieldCacheReopen, SolrSlf4jReporterTest, VMParamsZkACLAndCredentialsProvidersTest, TestCollectionAPI, TestRandomFaceting, ShardSplitTest, TestLocalFSCloudBackupRestore, TestPartialUpdateDeduplication, SignatureUpdateProcessorFactoryTest, TestSolrJ, CleanupOldIndexTest, CoreAdminRequestStatusTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestComplexPhraseLeadingWildcard, TestObjectReleaseTracker, DistributedFacetExistsSmallTest, TolerantUpdateProcessorTest, TestCustomDocTransformer, TestAnalyzeInfixSuggestions, TestCharFilters, AnalyticsQueryTest, TestSolrCloudWithKerberosAlt, DistributedMLTComponentTest, TestExceedMaxTermLength, TestAnalyzedSuggestions, OverseerTaskQueueTest, DistributedIntervalFacetingTest, HdfsNNFailoverTest, HdfsChaosMonkeySafeLeaderTest, ZkNodePropsTest, TestSQLHandler, SharedFSAutoReplicaFailoverUtilsTest, ConfigSetsAPITest, SchemaVersionSpecificBehaviorTest, TestNonDefinedSimilarityFactory, TestDocTermOrds, TestHashQParserPlugin, HdfsBasicDistributedZkTest, TestManagedSynonymGraphFilterFactory, TestRealTimeGet, TestCloudSchemaless, LeaderInitiatedRecoveryOnShardRestartTest, ExitableDirectoryReaderTest, TestSolrQueryResponse, HdfsWriteToMultipleCollectionsTest, TermVectorComponentTest, RestartWhileUpdatingTest, TestStandardQParsers, DeleteShardTest, LeaderFailureAfterFreshStartTest, FileUtilsTest, TestInfoStreamLogging, TestFieldCacheWithThreads, TestTolerantUpdateProcessorCloud, TestImpersonationWithHadoopAuth, TestFastOutputStream, TestSubQueryTransformer, TestTrieFacet, TestLegacyField, MultiThreadedOCPTest, FullSolrCloudDistribCmdsTest, FullHLLTest, SampleTest, DistribCursorPagingTest, TestSolrIndexConfig, SolrCLIZkUtilsTest, TestDynamicLoading, TestReload, SmileWriterTest, RankQueryTest, TestConfigOverlay, IndexSchemaRuntimeFieldTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestConfigSetsAPI, TestManagedResourceStorage, BlockJoinFacetRandomTest, TestFileDictionaryLookup, TestStressLiveNodes, ChaosMonkeySafeLeaderWithPullReplicasTest, TestNumericRangeQuery64, AtomicUpdatesTest, TestMultiValuedNumericRangeQuery, TestMissingGroups, CoreSorterTest, TestSuggestSpellingConverter, TestElisionMultitermQuery, OverseerRolesTest, PingRequestHandlerTest, TestManagedSchemaAPI, ActionThrottleTest, SolrCoreCheckLockOnStartupTest, SOLR749Test, TestDownShardTolerantSearch, TestDistributedGrouping, UpdateLogTest, MoreLikeThisHandlerTest, TestMergePolicyConfig, DocumentAnalysisRequestHandlerTest, TestAddFieldRealTimeGet, WrapperMergePolicyFactoryTest, TestZkChroot, SolrPluginUtilsTest, MoveReplicaHDFSUlogDirTest, TestQueryUtils, PreAnalyzedFieldTest, TestOrdValues, OpenCloseCoreStressTest, CopyFieldTest, TestLeaderElectionZkExpiry, TestTrie, V2StandaloneTest, ChaosMonkeySafeLeaderTest, SyncSliceTest, ClusterStateUpdateTest, ZkSolrClientTest, TestRecovery, TestHashPartitioner, TermVectorComponentDistributedTest, TestRangeQuery, TestSolr4Spatial, TestGroupingSearch, PeerSyncTest, HighlighterTest, ShowFileRequestHandlerTest, SolrIndexSplitterTest, TestCoreDiscovery, SolrRequestParserTest, SuggesterTSTTest, TestStressLucene, SuggesterWFSTTest, TestCSVLoader, NoCacheHeaderTest, TestUpdate, TestWordDelimiterFilterFactory, DefaultValueUpdateProcessorTest, DocValuesMultiTest, DebugComponentTest, CacheHeaderTest, TestWriterPerf, XmlUpdateRequestHandlerTest, DocumentBuilderTest, TestIndexingPerformance, FieldAnalysisRequestHandlerTest, RegexBoostProcessorTest, UpdateRequestProcessorFactoryTest, UniqFieldsUpdateProcessorFactoryTest, JsonLoaderTest, TestPHPSerializedResponseWriter, UpdateParamsTest, SolrIndexConfigTest, BadComponentTest, TestSolrDeletionPolicy2, TestSearchPerf, NumericFieldsTest, MinimalSchemaTest, OutputWriterTest, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, SynonymTokenizerTest, EchoParamsTest, TestPerFieldSimilarity, TestIBSimilarityFactory, TimeZoneUtilsTest, TestFastLRUCache, ChaosMonkeyNothingIsSafeTest, AsyncCallRequestStatusResponseTest, CdcrRequestHandlerTest, CdcrVersionReplicationTest, CollectionTooManyReplicasTest, ConcurrentDeleteAndCreateCollectionTest, ConnectionManagerTest, DeleteInactiveReplicaTest, DeleteReplicaTest, DistributedVersionInfoTest, ForceLeaderTest, LeaderInitiatedRecoveryOnCommitTest, MissingSegmentRecoveryTest, MoveReplicaTest, OverseerCollectionConfigSetProcessorTest, OverseerStatusTest, ReplaceNodeTest, ReplicationFactorTest, RollingRestartTest, SSLMigrationTest, SharedFSAutoReplicaFailoverTest, SimpleCollectionCreateDeleteTest, TestAuthenticationFramework, TestCloudPseudoReturnFields, TestCloudRecovery, TestExclusionRuleCollectionAccess, TestLeaderInitiatedRecoveryThread, TestOnReconnectListenerSupport, TestRandomFlRTGCloud, TestReplicaProperties, TestRequestForwarding, TestRequestStatusCollectionAPI, TestSSLRandomization, TestSegmentSorting, TestShortCircuitedRequests, TestSolrCloudWithSecureImpersonation, TestStressCloudBlindAtomicUpdates, TestTlogReplica] [junit4] Completed [561/727 (2!)] on J1 in 190.80s, 13 tests, 1 failure, 5 errors <<< FAILURES! [...truncated 43909 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org