It might seem like a target node took more than 20 sec for commit. Since remaining elems are 0, it's worth checking the next side node logs for any errors and hiccups.
On Wed, Apr 17, 2024 at 11:24 AM Stoyanov, Nikolay <nikolay.stoya...@docuware.com.invalid> wrote: > Hello, > I am using solr 9.1.1 in cloud mode with two replicas on different VMs. > > > > Lately we are experiencing issue with task queue processing. > > I have read about the other similar issues and tried the mentioned > solutions but nothing managed to solve the issue. > > Could you please assist me with what could be causing the issue and how to > solve it. > > > > Below you can find details from the exception. > > java.io.IOException: Task queue processing has stalled for 20191 ms with 0 > remaining elements to process. > at > org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient.blockUntilFinished(ConcurrentUpdateHttp2SolrClient.java:508) > at > org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:87) > at > org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:292) > at > org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:91) > at > org.apache.solr.update.processor.DistributedZkUpdateProcessor.doDistribFinish(DistributedZkUpdateProcessor.java:1263) > at > org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1253) > at > org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:78) > at > org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:94) > at > org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:224) > at org.apache.solr.core.SolrCore.execute(SolrCore.java:2865) > at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:887) > at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:606) > at > org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:250) > at > org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:218) > at > org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257) > at > org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227) > at > org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:213) > at > org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:195) > at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:201) > at > org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1626) > at > org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:552) > at > org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) > at > org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:600) > at > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235) > at > org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) > at > org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1440) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) > at > org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:505) > at > org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) > at > org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1355) > at > org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) > at > org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:191) > at > org.eclipse.jetty.server.handler.InetAccessHandler.handle(InetAccessHandler.java:177) > at > org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146) > at > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) > at > org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) > at > org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:772) > at > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) > at org.eclipse.jetty.server.Server.handle(Server.java:516) > at > org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:487) > at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:732) > at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:479) > at > org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) > at org.eclipse.jetty.io > .AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) > at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) > at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) > at > org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409) > at > org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) > at > org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) > at java.base/java.lang.Thread.run(Thread.java:840) > > Thank you in advance! > > Best Regards, > > > Nikolay Stoyanov > Cloud SysOps - Nemetschek OOD > > www.nemetschek.bg<http://www.nemetschek.bg> | e-mail: > nstoya...@nemetschek.bg | office: +359-2-4210-900 | mobile: > +359-887-262-838 > > ________________________________ > > DISCLAIMER: Confidential. Dissemination, distribution or copying of this > e-mail is prohibited. If you have received this e-mail in error, please > notify the sender by replying to this message and delete it immediately. > Nemetschek OOD shall not be liable for the message if altered, changed or > falsified. > > P Please consider the impact on the environment before printing this e-mail > > > -- Sincerely yours Mikhail Khludnev
