Hi All, I am running Hive Server2 on a big Hadoop cluster space and things looks good, I have a custom code which takes care of running some queries on top of Hive and processing the resultant accordingly.
The Queries are dependent on certain table which are created on Runtime(refreshed every hour), since yesterday I have included some more create statements in the code which have the dependency on the next set of queries, to be more specific the new set of create table statements are quite costly are spawn around 50+ MR in comparison to existing use case where only a few 10 MR would be spawned. Now since when I have included these heavy DML queries (create, drop) I am landing into Transport Exception and my HiveServer goes down(Stack Trace mentioned below). java.sql.SQLException: org.apache.thrift.transport.TTransportException at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:238) at org.apache.hive.jdbc.HiveStatement.executeUpdate(HiveStatement.java:390) at org.apache.hive.jdbc.HivePreparedStatement.executeUpdate(HivePreparedStatement.java:117) at com.sd.dwh.sc.tungsten.misc.CommonFunction.executeUpdate(CommonFunction.java:816) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:95) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.thrift.transport.TTransportException at org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) at org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:355) at org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:432) at org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:414) at org.apache.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) at org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:378) at org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:297) at org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:204) at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:69) at org.apache.hive.service.cli.thrift.TCLIService$Client.recv_ExecuteStatement(TCLIService.java:229) at org.apache.hive.service.cli.thrift.TCLIService$Client.ExecuteStatement(TCLIService.java:216) at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:232) ... 8 more java.sql.SQLException: Error while cleaning up the server resources at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:569) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:141) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.thrift.transport.TTransportException at org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) at org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:355) at org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:432) at org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:414) at org.apache.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) at org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:378) at org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:297) at org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:204) at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:69) at org.apache.hive.service.cli.thrift.TCLIService$Client.recv_CloseSession(TCLIService.java:183) at org.apache.hive.service.cli.thrift.TCLIService$Client.CloseSession(TCLIService.java:170) at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:567) ... 5 more java.sql.SQLException: org.apache.thrift.transport.TTransportException at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:238) at org.apache.hive.jdbc.HiveStatement.executeUpdate(HiveStatement.java:390) at org.apache.hive.jdbc.HivePreparedStatement.executeUpdate(HivePreparedStatement.java:117) at com.sd.dwh.sc.tungsten.misc.CommonFunction.executeUpdate(CommonFunction.java:816) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:95) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.thrift.transport.TTransportException at org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) at org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:355) at org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:432) at org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:414) at org.apache.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) at org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:378) at org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:297) at org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:204) at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:69) at org.apache.hive.service.cli.thrift.TCLIService$Client.recv_ExecuteStatement(TCLIService.java:229) at org.apache.hive.service.cli.thrift.TCLIService$Client.ExecuteStatement(TCLIService.java:216) at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:232) ... 8 more java.sql.SQLException: Error while cleaning up the server resources at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:569) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:141) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.thrift.transport.TTransportException: java.net.SocketException: Broken pipe at org.apache.thrift.transport.TIOStreamTransport.flush(TIOStreamTransport.java:161) at org.apache.thrift.transport.TSaslTransport.flush(TSaslTransport.java:480) at org.apache.thrift.transport.TSaslClientTransport.flush(TSaslClientTransport.java:37) at org.apache.thrift.TServiceClient.sendBase(TServiceClient.java:65) at org.apache.hive.service.cli.thrift.TCLIService$Client.send_CloseSession(TCLIService.java:177) at org.apache.hive.service.cli.thrift.TCLIService$Client.CloseSession(TCLIService.java:169) at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:567) ... 5 more Caused by: java.net.SocketException: Broken pipe at java.net.SocketOutputStream.socketWrite0(Native Method) at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113) at java.net.SocketOutputStream.write(SocketOutputStream.java:159) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at org.apache.thrift.transport.TIOStreamTransport.flush(TIOStreamTransport.java:159) ... 11 more java.sql.SQLException: org.apache.thrift.transport.TTransportException at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:238) at org.apache.hive.jdbc.HiveStatement.executeUpdate(HiveStatement.java:390) at org.apache.hive.jdbc.HivePreparedStatement.executeUpdate(HivePreparedStatement.java:117) at com.sd.dwh.sc.tungsten.misc.CommonFunction.executeUpdate(CommonFunction.java:816) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:95) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.thrift.transport.TTransportException at org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) at org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:355) at org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:432) at org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:414) at org.apache.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) at org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:378) at org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:297) at org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:204) at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:69) at org.apache.hive.service.cli.thrift.TCLIService$Client.recv_ExecuteStatement(TCLIService.java:229) at org.apache.hive.service.cli.thrift.TCLIService$Client.ExecuteStatement(TCLIService.java:216) at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:232) ... 8 more java.sql.SQLException: Error while cleaning up the server resources at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:569) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:141) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.thrift.transport.TTransportException: java.net.SocketException: Broken pipe at org.apache.thrift.transport.TIOStreamTransport.flush(TIOStreamTransport.java:161) at org.apache.thrift.transport.TSaslTransport.flush(TSaslTransport.java:480) at org.apache.thrift.transport.TSaslClientTransport.flush(TSaslClientTransport.java:37) at org.apache.thrift.TServiceClient.sendBase(TServiceClient.java:65) at org.apache.hive.service.cli.thrift.TCLIService$Client.send_CloseSession(TCLIService.java:177) at org.apache.hive.service.cli.thrift.TCLIService$Client.CloseSession(TCLIService.java:169) at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:567) ... 5 more Caused by: java.net.SocketException: Broken pipe at java.net.SocketOutputStream.socketWrite0(Native Method) at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113) at java.net.SocketOutputStream.write(SocketOutputStream.java:159) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at org.apache.thrift.transport.TIOStreamTransport.flush(TIOStreamTransport.java:159) ... 11 more java.sql.SQLException: org.apache.thrift.transport.TTransportException at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:238) at org.apache.hive.jdbc.HiveStatement.executeQuery(HiveStatement.java:376) at org.apache.hive.jdbc.HivePreparedStatement.executeQuery(HivePreparedStatement.java:107) at com.sd.dwh.sc.tungsten.misc.CommonFunction.runQuery(CommonFunction.java:771) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:113) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.thrift.transport.TTransportException at org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) at org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:355) at org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:432) at org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:414) at org.apache.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) at org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:378) at org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:297) at org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:204) at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:69) at org.apache.hive.service.cli.thrift.TCLIService$Client.recv_ExecuteStatement(TCLIService.java:229) at org.apache.hive.service.cli.thrift.TCLIService$Client.ExecuteStatement(TCLIService.java:216) at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:232) ... 8 more java.sql.SQLException: Error while cleaning up the server resources at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:569) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:141) at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.thrift.transport.TTransportException: java.net.SocketException: Broken pipe at org.apache.thrift.transport.TIOStreamTransport.flush(TIOStreamTransport.java:161) at org.apache.thrift.transport.TSaslTransport.flush(TSaslTransport.java:480) at org.apache.thrift.transport.TSaslClientTransport.flush(TSaslClientTransport.java:37) at org.apache.thrift.TServiceClient.sendBase(TServiceClient.java:65) at org.apache.hive.service.cli.thrift.TCLIService$Client.send_CloseSession(TCLIService.java:177) at org.apache.hive.service.cli.thrift.TCLIService$Client.CloseSession(TCLIService.java:169) at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:567) ... 5 more Caused by: java.net.SocketException: Broken pipe at java.net.SocketOutputStream.socketWrite0(Native Method) at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113) at java.net.SocketOutputStream.write(SocketOutputStream.java:159) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at org.apache.thrift.transport.TIOStreamTransport.flush(TIOStreamTransport.java:159) ... 11 more Any suggestion would be appreciated! Thanks!
