Hi
We are running 0.94.6-cdh4.4.0 with phoenix-2.2.3 and recently started
seeing connections being forcefully closed, ending with a
ClosedByInterruptException [1]. This problem occur on both write and
scans.
I did some searching and found people that had similar problems. Seems
that one answer was overloaded data nodes, but this is not the case
for us.
Any ideas?
Cheers,
-Kristoffer
[1] stacktrace
2015-09-03 08:16:21,263 [pool-10-thread-2938] WARN
o.a.h.h.c.HConnectionManager$HConnectionImplementation:1601 - Failed
all from
region=TTABLE,\x11\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00,1396010829607.e68b18fae7ea7823de8e3de6cd0db35c.,
hostname=hdfs-01, port=60020
java.util.concurrent.ExecutionException: java.io.IOException: Call to
hdfs-01/10.3.24.20:60020 failed on local exception:
java.nio.channels.ClosedByInterruptException
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
~[na:1.8.0_25]
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
~[na:1.8.0_25]
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatchCallback(HConnectionManager.java:1571)
[hbase-0.94.6-cdh4.4.0.jar:na]
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatch(HConnectionManager.java:1423)
[hbase-0.94.6-cdh4.4.0.jar:na]
at org.apache.hadoop.hbase.client.HTable.batch(HTable.java:754)
[hbase-0.94.6-cdh4.4.0.jar:na]
at
org.apache.phoenix.execute.MutationState.commit(MutationState.java:374)
[phoenix-2.2.3-incubating.jar:2.2.3-incubating]
at
org.apache.phoenix.jdbc.PhoenixConnection.commit(PhoenixConnection.java:293)
[phoenix-2.2.3-incubating.jar:2.2.3-incubating]
at org.util.table.Table.insert(Table.java:92) [igor-2.0.52.jar:na]
at
org.service.tracking.HourlyCounterTable$Proxy$_$$_WeldClientProxy.insert(Unknown
Source) [igor-2.0.52.jar:na]
at
org.service.tracking.RawTrackingCounterEndpoint.insertBatch(RawTrackingCounterEndpoint.java:60)
[igor-2.0.52.jar:na]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
~[na:1.8.0_25]
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[na:1.8.0_25]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[na:1.8.0_25]
at java.lang.reflect.Method.invoke(Method.java:483) ~[na:1.8.0_25]
at
org.jboss.weld.bean.proxy.AbstractBeanInstance.invoke(AbstractBeanInstance.java:40)
[weld-core-2.1.2.Final.jar:2014-01-09 09:23]
at
org.jboss.weld.bean.proxy.ProxyMethodHandler.invoke(ProxyMethodHandler.java:100)
[weld-core-2.1.2.Final.jar:2014-01-09 09:23]
at
org.service.tracking.RawTrackingCounterEndpoint$Proxy$_$$_WeldClientProxy.insertBatch(Unknown
Source) [igor-2.0.52.jar:na]
at sun.reflect.GeneratedMethodAccessor50.invoke(Unknown Source) ~[na:na]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[na:1.8.0_25]
at java.lang.reflect.Method.invoke(Method.java:483) ~[na:1.8.0_25]
at
org.jboss.resteasy.core.MethodInjectorImpl.invoke(MethodInjectorImpl.java:137)
[resteasy-jaxrs-3.0.10.Final.jar:na]
at
org.jboss.resteasy.core.ResourceMethodInvoker.invokeOnTarget(ResourceMethodInvoker.java:296)
[resteasy-jaxrs-3.0.10.Final.jar:na]
at
org.jboss.resteasy.core.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:250)
[resteasy-jaxrs-3.0.10.Final.jar:na]
at
org.jboss.resteasy.core.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:237)
[resteasy-jaxrs-3.0.10.Final.jar:na]
at
org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:356)
[resteasy-jaxrs-3.0.10.Final.jar:na]
at
org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:179)
[resteasy-jaxrs-3.0.10.Final.jar:na]
at
org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.service(ServletContainerDispatcher.java:220)
[resteasy-jaxrs-3.0.10.Final.jar:na]
at
org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:56)
[resteasy-jaxrs-3.0.10.Final.jar:na]
at
org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:51)
[resteasy-jaxrs-3.0.10.Final.jar:na]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
[javax.servlet-api-3.1.0.jar:3.1.0]
at
org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:769)
[jetty-servlet-9.2.2.v20140723.jar:9.2.2.v20140723]
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
[jetty-servlet-9.2.2.v20140723.jar:9.2.2.v20140723]
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1125)
[jetty-server-9.2.2.v20140723.jar:9.2.2.v20140723]
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
[jetty-servlet-9.2.2.v20140723.jar:9.2.2.v20140723]
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1059)
[jetty-server-9.2.2.v20140723.jar:9.2.2.v20140723]
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[jetty-server-9.2.2.v20140723.jar:9.2.2.v20140723]
at
org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52)
[jetty-server-9.2.2.v20140723.jar:9.2.2.v20140723]
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
[jetty-server-9.2.2.v20140723.jar:9.2.2.v20140723]
at org.eclipse.jetty.server.Server.handle(Server.java:485)
[jetty-server-9.2.2.v20140723.jar:9.2.2.v20140723]
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:290)
[jetty-server-9.2.2.v20140723.jar:9.2.2.v20140723]
at
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:248)
[jetty-server-9.2.2.v20140723.jar:9.2.2.v20140723]
at
org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
[jetty-io-9.2.2.v20140723.jar:9.2.2.v20140723]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[na:1.8.0_25]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[na:1.8.0_25]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_25]
Caused by: java.io.IOException: Call to hdfs-01/10.4.24.30:60020
failed on local exception:
java.nio.channels.ClosedByInterruptException
at
org.apache.hadoop.hbase.ipc.HBaseClient.wrapException(HBaseClient.java:1039)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:1008)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at
org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at com.sun.proxy.$Proxy72.multi(Unknown Source) ~[na:na]
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3$1.call(HConnectionManager.java:1400)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3$1.call(HConnectionManager.java:1398)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at
org.apache.hadoop.hbase.client.ServerCallable.withoutRetries(ServerCallable.java:210)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3.call(HConnectionManager.java:1407)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3.call(HConnectionManager.java:1395)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[na:1.8.0_25]
... 3 common frames omitted
Caused by: java.nio.channels.ClosedByInterruptException: null
at
java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:202)
~[na:1.8.0_25]
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:477)
~[na:1.8.0_25]
at
org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62)
~[hadoop-common-2.0.0-cdh4.4.0.jar:na]
at
org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:143)
~[hadoop-common-2.0.0-cdh4.4.0.jar:na]
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:153)
~[hadoop-common-2.0.0-cdh4.4.0.jar:na]
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:114)
~[hadoop-common-2.0.0-cdh4.4.0.jar:na]
at
java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
~[na:1.8.0_25]
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
~[na:1.8.0_25]
at java.io.DataOutputStream.flush(DataOutputStream.java:123)
~[na:1.8.0_25]
at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.sendParam(HBaseClient.java:626)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:981)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at
org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at com.sun.proxy.$Proxy72.next(Unknown Source) ~[na:na]
at
org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:126)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at
org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:42)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at
org.apache.hadoop.hbase.client.ServerCallable.withRetries(ServerCallable.java:163)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at
org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:274)
~[hbase-0.94.6-cdh4.4.0.jar:na]
at
org.apache.phoenix.iterate.ScanningResultIterator.next(ScanningResultIterator.java:47)
~[phoenix-2.2.3-incubating.jar:2.2.3-incubating]
at
org.apache.phoenix.iterate.TableResultIterator.next(TableResultIterator.java:76)
~[phoenix-2.2.3-incubating.jar:2.2.3-incubating]
at
org.apache.phoenix.iterate.OrderedResultIterator.getResultIterator(OrderedResultIterator.java:215)
~[phoenix-2.2.3-incubating.jar:2.2.3-incubating]
at
org.apache.phoenix.iterate.OrderedResultIterator.peek(OrderedResultIterator.java:238)
~[phoenix-2.2.3-incubating.jar:2.2.3-incubating]
at
org.apache.phoenix.iterate.MergeSortResultIterator.minIterator(MergeSortResultIterator.java:68)
~[phoenix-2.2.3-incubating.jar:2.2.3-incubating]
at
org.apache.phoenix.iterate.MergeSortResultIterator.next(MergeSortResultIterator.java:90)
~[phoenix-2.2.3-incubating.jar:2.2.3-incubating]
at
org.apache.phoenix.iterate.GroupedAggregatingResultIterator.next(GroupedAggregatingResultIterator.java:68)
~[phoenix-2.2.3-incubating.jar:2.2.3-incubating]
at
org.apache.phoenix.iterate.OrderedResultIterator.getResultIterator(OrderedResultIterator.java:215)
~[phoenix-2.2.3-incubating.jar:2.2.3-incubating]
at
org.apache.phoenix.iterate.OrderedResultIterator.next(OrderedResultIterator.java:169)
~[phoenix-2.2.3-incubating.jar:2.2.3-incubating]
at
org.apache.phoenix.iterate.OrderedAggregatingResultIterator.next(OrderedAggregatingResultIterator.java:50)
~[phoenix-2.2.3-incubating.jar:2.2.3-incubating]
at
org.apache.phoenix.jdbc.PhoenixResultSet.next(PhoenixResultSet.java:731)
~[phoenix-2.2.3-incubating.jar:2.2.3-incubating]
at org.jooq.impl.CursorImpl$CursorResultSet.next(CursorImpl.java:390)
~[jooq-3.5.0.jar:na]
at
org.jooq.impl.CursorImpl$CursorIterator.fetchOne(CursorImpl.java:1403)
~[jooq-3.5.0.jar:na]
at org.jooq.impl.CursorImpl$CursorIterator.hasNext(CursorImpl.java:1380)
~[jooq-3.5.0.jar:na]
at org.jooq.impl.CursorImpl.fetch(CursorImpl.java:203)
~[jooq-3.5.0.jar:na]
at org.jooq.impl.CursorImpl.fetch(CursorImpl.java:178)
~[jooq-3.5.0.jar:na]
