Tell a little more about your consumers.Are they getting blocked up? How often are you creating destinations/temp destinations? I don't see a network of brokers in this config... you're not using any, right?
On Fri, Feb 28, 2014 at 11:30 AM, jmachina <[email protected]> wrote: > Experiencing a recurring issue with our activeMQ broker, where it will stop > servicing new connections (any new connection from the client side hangs on > establishing a connection). Thread dump attached, and a small selection of > the dump pasted below. Our configuration is also listed below (and > attached). > > > <beans > xmlns="http://www.springframework.org/schema/beans"; > xmlns:amq="http://activemq.apache.org/schema/core"; > xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; > xsi:schemaLocation="http://www.springframework.org/schema/beans > http://www.springframework.org/schema/beans/spring-beans.xsd > http://activemq.apache.org/schema/core > http://activemq.apache.org/schema/core/activemq-core.xsd";> > > <bean > class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer"> > <property name="locations"> > <value>file:${activemq.conf}/credentials.properties</value> > </property> > </bean> > > <broker xmlns="http://activemq.apache.org/schema/core"; brokerName="main" > dataDirectory="${activemq.data}" offlineDurableSubscriberTimeout="86400000" > offlineDurableSubscriberTaskSchedule="21600000"> > > <destinationPolicy> > <policyMap> > <policyEntries> > <policyEntry topic=">" producerFlowControl="false" > advisoryForSlowConsumers="true" > advisoryForFastProducers="true" > advisoryWhenFull="true" > memoryLimit="4096mb"> > > <dispatchPolicy> > <simpleDispatchPolicy/> > </dispatchPolicy> > > <pendingMessageLimitStrategy> > > <constantPendingMessageLimitStrategy limit="10000"/> > </pendingMessageLimitStrategy> > > <messageEvictionStrategy> > <oldestMessageEvictionStrategy/> > </messageEvictionStrategy> > </policyEntry> > > <policyEntry queue=">" > producerFlowControl="false" > optimizedDispatch="true" > useCache="false" > memoryLimit="4096mb"> > <pendingMessageLimitStrategy> > > <constantPendingMessageLimitStrategy limit="10000"/> > </pendingMessageLimitStrategy> > > > <messageEvictionStrategy> > <oldestMessageEvictionStrategy/> > </messageEvictionStrategy> > </policyEntry> > > <policyEntry topic=">" tempTopic="true" > producerFlowControl="false" > advisoryForSlowConsumers="true" > advisoryForFastProducers="true" > advisoryWhenFull="true" > memoryLimit="4096mb"> > </policyEntry> > > <policyEntry queue=">" tempQueue="true" > producerFlowControl="false" > optimizedDispatch="true" > useCache="false" > advisoryForSlowConsumers="true" > advisoryForFastProducers="true" > advisoryWhenFull="true" > memoryLimit="4096mb"> > </policyEntry> > </policyEntries> > </policyMap> > </destinationPolicy> > > <managementContext> > <managementContext createConnector="true" connectorPort="1099" > rmiServerPort="1199"/> > </managementContext> > > <persistenceAdapter> > <kahaDB directory="${activemq.data}/kahadb-main"/> > </persistenceAdapter> > > <systemUsage> > <systemUsage> > <memoryUsage> > <memoryUsage limit="8 gb"/> > </memoryUsage> > <storeUsage> > <storeUsage limit="20 gb"/> > </storeUsage> > <tempUsage> > <tempUsage limit="50 gb"/> > </tempUsage> > </systemUsage> > </systemUsage> > > <transportConnectors> > <transportConnector name="main-openwire" > uri="tcp://0.0.0.0:61616"/> > </transportConnectors> > > <shutdownHooks> > <bean xmlns="http://www.springframework.org/schema/beans"; > class="org.apache.activemq.hooks.SpringContextHook" /> > </shutdownHooks> > > </broker> > <import resource="jetty-main.xml"/> > </beans> > > > Selection of thread dumps: > "ActiveMQ Transport: tcp:///10.0.1.229:51552@61616" - Thread t@1545 > java.lang.Thread.State: RUNNABLE > at java.net.SocketOutputStream.socketWrite0(Native Method) > at java.net.SocketOutputStream.socketWrite(Unknown Source) > at java.net.SocketOutputStream.write(Unknown Source) > at > org.apache.activemq.transport.tcp.TcpBufferedOutputStream.flush(TcpBufferedOutputStream.java:115) > at java.io.DataOutputStream.flush(Unknown Source) > at > org.apache.activemq.transport.tcp.TcpTransport.oneway(TcpTransport.java:176) > at > org.apache.activemq.transport.AbstractInactivityMonitor.doOnewaySend(AbstractInactivityMonitor.java:304) > at > org.apache.activemq.transport.AbstractInactivityMonitor.oneway(AbstractInactivityMonitor.java:286) > at > org.apache.activemq.transport.TransportFilter.oneway(TransportFilter.java:85) > at > org.apache.activemq.transport.WireFormatNegotiator.oneway(WireFormatNegotiator.java:104) > at > org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68) > at > org.apache.activemq.broker.TransportConnection.dispatch(TransportConnection.java:1339) > at > org.apache.activemq.broker.TransportConnection.processDispatch(TransportConnection.java:858) > at > org.apache.activemq.broker.TransportConnection.dispatchSync(TransportConnection.java:818) > at > org.apache.activemq.broker.region.TopicSubscription.dispatch(TopicSubscription.java:611) > at > org.apache.activemq.broker.region.TopicSubscription.add(TopicSubscription.java:112) > at > org.apache.activemq.broker.region.policy.SimpleDispatchPolicy.dispatch(SimpleDispatchPolicy.java:48) > at org.apache.activemq.broker.region.Topic.dispatch(Topic.java:705) > at > org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:504) > - locked <6f8e78f0> (a org.apache.activemq.broker.region.Topic) > at org.apache.activemq.broker.region.Topic.send(Topic.java:440) > at > org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:394) > at > org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:442) > at > org.apache.activemq.broker.jmx.ManagedRegionBroker.send(ManagedRegionBroker.java:283) > at > org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:585) > at > org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:516) > at > org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:511) > at > org.apache.activemq.advisory.AdvisoryBroker.addDestinationInfo(AdvisoryBroker.java:206) > at > org.apache.activemq.broker.BrokerFilter.addDestinationInfo(BrokerFilter.java:257) > at > org.apache.activemq.broker.BrokerFilter.addDestinationInfo(BrokerFilter.java:257) > at > org.apache.activemq.broker.MutableBrokerFilter.addDestinationInfo(MutableBrokerFilter.java:262) > at > org.apache.activemq.broker.TransportConnection.processAddDestination(TransportConnection.java:495) > at > org.apache.activemq.command.DestinationInfo.visit(DestinationInfo.java:122) > at > org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:292) > at > org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:149) > at > org.apache.activemq.transport.MutexTransport.onCommand(MutexTransport.java:50) > at > org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113) > at > org.apache.activemq.transport.AbstractInactivityMonitor.onCommand(AbstractInactivityMonitor.java:270) > at > org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83) > at > org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:214) > at > org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:196) > at java.lang.Thread.run(Unknown Source) > > Locked ownable synchronizers: > - locked <6c833519> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > > "ActiveMQ Transport: tcp:///10.0.1.229:51551@61616" - Thread t@1544 > java.lang.Thread.State: BLOCKED > at > org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:459) > - waiting to lock <6f8e78f0> (a > org.apache.activemq.broker.region.Topic) > owned by "ActiveMQ Transport: tcp:///10.0.1.229:51552@61616" t@1545 > at org.apache.activemq.broker.region.Topic.send(Topic.java:440) > at > org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:394) > at > org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:442) > at > org.apache.activemq.broker.jmx.ManagedRegionBroker.send(ManagedRegionBroker.java:283) > at > org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:585) > at > org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:516) > at > org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:511) > at > org.apache.activemq.advisory.AdvisoryBroker.addDestinationInfo(AdvisoryBroker.java:206) > at > org.apache.activemq.broker.BrokerFilter.addDestinationInfo(BrokerFilter.java:257) > at > org.apache.activemq.broker.BrokerFilter.addDestinationInfo(BrokerFilter.java:257) > at > org.apache.activemq.broker.MutableBrokerFilter.addDestinationInfo(MutableBrokerFilter.java:262) > at > org.apache.activemq.broker.TransportConnection.processAddDestination(TransportConnection.java:495) > at > org.apache.activemq.command.DestinationInfo.visit(DestinationInfo.java:122) > at > org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:292) > at > org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:149) > at > org.apache.activemq.transport.MutexTransport.onCommand(MutexTransport.java:50) > at > org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113) > at > org.apache.activemq.transport.AbstractInactivityMonitor.onCommand(AbstractInactivityMonitor.java:270) > at > org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83) > at > org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:214) > at > org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:196) > at java.lang.Thread.run(Unknown Source) > > Locked ownable synchronizers: > - None > > "ActiveMQ BrokerService[main] Task-374" - Thread t@1906 > java.lang.Thread.State: BLOCKED > at > org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:459) > - waiting to lock <6f8e78f0> (a > org.apache.activemq.broker.region.Topic) > owned by "ActiveMQ Transport: tcp:///10.0.1.229:51552@61616" t@1545 > at org.apache.activemq.broker.region.Topic.send(Topic.java:440) > at > org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:394) > at > org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:442) > at > org.apache.activemq.broker.jmx.ManagedRegionBroker.send(ManagedRegionBroker.java:283) > at > org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:585) > at > org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:516) > at > org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:511) > at > org.apache.activemq.advisory.AdvisoryBroker.removeDestination(AdvisoryBroker.java:221) > at > org.apache.activemq.broker.BrokerFilter.removeDestination(BrokerFilter.java:172) > at > org.apache.activemq.broker.BrokerFilter.removeDestination(BrokerFilter.java:172) > at > org.apache.activemq.broker.MutableBrokerFilter.removeDestination(MutableBrokerFilter.java:177) > at > org.apache.activemq.broker.TransportConnection.processRemoveConnection(TransportConnection.java:779) > - locked <6f39a3be> (a > org.apache.activemq.broker.jmx.ManagedTransportConnection) > at > org.apache.activemq.broker.TransportConnection.doStop(TransportConnection.java:1107) > at > org.apache.activemq.broker.TransportConnection$4.run(TransportConnection.java:1037) > at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) > at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) > at java.lang.Thread.run(Unknown Source) > > Locked ownable synchronizers: > - locked <312d47fe> (a > java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) > > - locked <1556e994> (a java.util.concurrent.ThreadPoolExecutor$Worker) > > "ActiveMQ BrokerService[main] Task-396" - Thread t@1959 > java.lang.Thread.State: TIMED_WAITING > at sun.misc.Unsafe.park(Native Method) > - parking to wait for <6eb8ca36> (a > java.util.concurrent.SynchronousQueue$TransferStack) > at java.util.concurrent.locks.LockSupport.parkNanos(Unknown Source) > at > java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(Unknown > Source) > at > java.util.concurrent.SynchronousQueue$TransferStack.transfer(Unknown > Source) > at java.util.concurrent.SynchronousQueue.poll(Unknown Source) > at java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source) > at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) > at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) > at java.lang.Thread.run(Unknown Source) > > Locked ownable synchronizers: > - None > > "ActiveMQ Transport: tcp:///10.0.1.202:59370@61616" - Thread t@1498 > java.lang.Thread.State: RUNNABLE > at java.net.SocketInputStream.socketRead0(Native Method) > at java.net.SocketInputStream.read(Unknown Source) > at java.net.SocketInputStream.read(Unknown Source) > at > org.apache.activemq.transport.tcp.TcpBufferedInputStream.fill(TcpBufferedInputStream.java:50) > at > org.apache.activemq.transport.tcp.TcpTransport$2.fill(TcpTransport.java:604) > at > org.apache.activemq.transport.tcp.TcpBufferedInputStream.read(TcpBufferedInputStream.java:58) > at > org.apache.activemq.transport.tcp.TcpTransport$2.read(TcpTransport.java:589) > at java.io.DataInputStream.readInt(Unknown Source) > at > org.apache.activemq.openwire.OpenWireFormat.unmarshal(OpenWireFormat.java:275) > at > org.apache.activemq.transport.tcp.TcpTransport.readCommand(TcpTransport.java:221) > at > org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:213) > at > org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:196) > at java.lang.Thread.run(Unknown Source) > > Locked ownable synchronizers: > - None > > > > threaddump-1393608740161.tdump > <http://activemq.2283324.n4.nabble.com/file/n4678423/threaddump-1393608740161.tdump> > activemq.xml > <http://activemq.2283324.n4.nabble.com/file/n4678423/activemq.xml> > > > > -- > View this message in context: > http://activemq.2283324.n4.nabble.com/Deadlock-on-broker-thread-dumps-included-tp4678423.html > Sent from the ActiveMQ - User mailing list archive at Nabble.com. -- Christian Posta http://www.christianposta.com/blog twitter: @christianposta
