Hi. We keep getting "SolrServerException: Max requests queued per destination 3000 exceeded for HttpDestination" for several different collections and instances in our SolrCloud dev environment. The instances don't reply to requests and need to be restarted to function again.
We're not getting those errors in the prod environment even though those are set up similarly. Any ideas what could cause this? We're on Solr 8.11.1 with connTimeout=60000, socketTimeout=60000, zkClientTimeout=30000, solr.jetty.http.idleTimeout=60000. Full stacktrace below. Thanks for any help! Cheers, Henrik java.util.concurrent.RejectedExecutionException: Max requests queued per destination 3000 exceeded for HttpDestination[http://dev-foo3672.bar.no:12621 ]@74097d87,queue=3000,pool=MultiplexConnectionPool@310bfd62 [c=0/4/4,a=0,i=0,q=3000] at org.eclipse.jetty.client.HttpDestination.send(HttpDestination.java:289) ~[jetty-client-9.4.44.v20210927.jar:9.4.44.v20210927] at org.eclipse.jetty.client.HttpDestination.send(HttpDestination.java:262) ~[jetty-client-9.4.44.v20210927.jar:9.4.44.v20210927] at org.eclipse.jetty.client.HttpClient.send(HttpClient.java:600) ~[jetty-client-9.4.44.v20210927.jar:9.4.44.v20210927] at org.eclipse.jetty.client.HttpRequest.sendAsync(HttpRequest.java:778) ~[jetty-client-9.4.44.v20210927.jar:9.4.44.v20210927] at org.eclipse.jetty.client.HttpRequest.send(HttpRequest.java:765) ~[jetty-client-9.4.44.v20210927.jar:9.4.44.v20210927] at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:409) ~[solr-solrj-8.11.1.jar:8.11.1 0b002b11819df70783e83ef36b42ed1223c14b50 - janhoy - 2021-12-14 13:50:57] at org.apache.solr.client.solrj.SolrServerException: Max requests queued per destination 3000 exceeded for HttpDestination[http://dev-foo3672.bar.no:12621 ]@74097d87,queue=3000,pool=MultiplexConnectionPool@310bfd62 [c=0/4/4,a=0,i=0,q=3000] at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:439) ~[solr-solrj-8.11.1.jar:8.11.1 0b002b11819df70783e83ef36b42ed1223c14b50 - janhoy - 2021-12-14 13:50:57] at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:776) ~[solr-solrj-8.11.1.jar:8.11.1 0b002b11819df70783e83ef36b42ed1223c14b50 - janhoy - 2021-12-14 13:50:57] at org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient.request(ConcurrentUpdateHttp2SolrClient.java:370) ~[solr-solrj-8.11.1.jar:8.11.1 0b002b11819df70783e83ef36b42ed1223c14b50 - janhoy - 2021-12-14 13:50:57] at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290) ~[solr-solrj-8.11.1.jar:8.11.1 0b002b11819df70783e83ef36b42ed1223c14b50 - janhoy - 2021-12-14 13:50:57] at org.apache.solr.update.SolrCmdDistributor.doRequest(SolrCmdDistributor.java:345) ~[solr-core-8.11.1.jar:8.11.1 0b002b11819df70783e83ef36b42ed1223c14b50 - janhoy - 2021-12-14 13:50:55] at org.apache.solr.update.SolrCmdDistributor.lambda$submit$0(SolrCmdDistributor.java:334) ~[solr-core-8.11.1.jar:8.11.1 0b002b11819df70783e83ef36b42ed1223c14b50 - janhoy - 2021-12-14 13:50:55] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) ~[metrics-core-4.1.5.jar:4.1.5] at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218) ~[solr-solrj-8.11.1.jar:8.11.1 0b002b11819df70783e83ef36b42ed1223c14b50 - janhoy - 2021-12-14 13:50:57] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?] at java.lang.Thread.run(Thread.java:833) [?:?] Caused by: java.util.concurrent.RejectedExecutionException: Max requests queued per destination 3000 exceeded for HttpDestination[http://dev-foo3672.bar.no:12621 ]@74097d87,queue=3000,pool=MultiplexConnectionPool@310bfd62 [c=0/4/4,a=0,i=0,q=3000] at org.eclipse.jetty.client.HttpDestination.send(HttpDestination.java:289) ~[jetty-client-9.4.44.v20210927.jar:9.4.44.v20210927] at org.eclipse.jetty.client.HttpDestination.send(HttpDestination.java:262) ~[jetty-client-9.4.44.v20210927.jar:9.4.44.v20210927] at org.eclipse.jetty.client.HttpClient.send(HttpClient.java:600) ~[jetty-client-9.4.44.v20210927.jar:9.4.44.v20210927] at org.eclipse.jetty.client.HttpRequest.sendAsync(HttpRequest.java:778) ~[jetty-client-9.4.44.v20210927.jar:9.4.44.v20210927] at org.eclipse.jetty.client.HttpRequest.send(HttpRequest.java:765) ~[jetty-client-9.4.44.v20210927.jar:9.4.44.v20210927] at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:409) ~[solr-solrj-8.11.1.jar:8.11.1 0b002b11819df70783e83ef36b42ed1223c14b50 - janhoy - 2021-12-14 13:50:57]
