[
https://issues.apache.org/jira/browse/HIVE-23271?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ankur Tagra reopened HIVE-23271:
--------------------------------
> Can't start Hive Interactive Server in HDP 3.1.4 Cluster
> --------------------------------------------------------
>
> Key: HIVE-23271
> URL: https://issues.apache.org/jira/browse/HIVE-23271
> Project: Hive
> Issue Type: Bug
> Components: Configuration
> Affects Versions: 3.1.0
> Environment: All nodes have CentOS 7
> Cluster HDP 3.1.4
> Reporter: Gerardo Adrián Aguirre Vivar
> Assignee: Gerardo Adrián Aguirre Vivar
> Priority: Major
>
> Hive interactive server is not working. The installation guide has been
> followed
> ([https://docs.cloudera.com/HDPDocuments/HDP3/HDP-3.1.4/performance-tuning/content/hive_prepare_to_tune_performance.html])
> but when the server try to start, an errors appears.
>
> LOGS:
>
> 2020-04-22T16:43:48,271 INFO [main]: impl.YarnClientImpl
> (YarnClientImpl.java:submitApplication(306)) - Submitted application
> application_1587555843754_0015
> 2020-04-22T16:43:48,275 INFO [main]: client.TezClient
> (TezClient.java:start(404)) - The url to track the Tez Session:
> http://<NODE>:8088/proxy/application_1587555843754_0015/
> 2020-04-22T16:43:53,435 INFO [main]: client.TezClient
> (TezClient.java:getAppMasterStatus(881)) - *{color:#0747a6}Failed to retrieve
> AM Status via proxy{color}*
> com.google.protobuf.ServiceException: java.io.EOFException: End of File
> Exception between local host is:
> "<NODE_HIVE_INTERACTIVE_SERVER>/10.22.39.12"; destination host is:
> "<WORKER2>":33889; : java.io.EOFException; For more details see:
> http://wiki.apache.org/hadoop/EOFException
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:242)
> ~[hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:116)
> ~[hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> at com.sun.proxy.$Proxy75.getAMStatus(Unknown Source) ~[?:?]
> at org.apache.tez.client.TezClient.getAppMasterStatus(TezClient.java:874)
> [tez-api-0.9.1.3.1.4.0-315.jar:0.9.1.3.1.4.0-315]
> at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:1011)
> [tez-api-0.9.1.3.1.4.0-315.jar:0.9.1.3.1.4.0-315]
> at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:982)
> [tez-api-0.9.1.3.1.4.0-315.jar:0.9.1.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionState.startSessionAndContainers(TezSessionState.java:536)
> [hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionState.openInternal(TezSessionState.java:451)
> [hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolSession.openInternal(TezSessionPoolSession.java:124)
> [hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:373)
> [hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:236)
> [hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startInitialSession(TezSessionPool.java:354)
> [hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startUnderInitLock(TezSessionPool.java:166)
> [hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.start(TezSessionPool.java:123)
> [hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.startPool(TezSessionPoolManager.java:112)
> [hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:855)
> [hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hive.service.server.HiveServer2.startOrReconnectTezSessions(HiveServer2.java:828)
> [hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:752)
> [hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:1078)
> [hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hive.service.server.HiveServer2.access$1700(HiveServer2.java:136)
> [hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:1346)
> [hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:1190)
> [hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_232]
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> ~[?:1.8.0_232]
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> ~[?:1.8.0_232]
> at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_232]
> at org.apache.hadoop.util.RunJar.run(RunJar.java:318)
> [hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> at org.apache.hadoop.util.RunJar.main(RunJar.java:232)
> [hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> Caused by: java.io.EOFException: End of File Exception between local host is:
> "bigdata-e0.mova.indra.es/10.22.39.12"; destination host is:
> "bigdata-w2.mova.indra.es":33889; : java.io.EOFException; For more details
> see: http://wiki.apache.org/hadoop/EOFException
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> ~[?:1.8.0_232]
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> ~[?:1.8.0_232]
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> ~[?:1.8.0_232]
> at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
> ~[?:1.8.0_232]
> at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:831)
> ~[hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:789)
> ~[hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1502)
> ~[hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> at org.apache.hadoop.ipc.Client.call(Client.java:1444)
> ~[hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> at org.apache.hadoop.ipc.Client.call(Client.java:1354)
> ~[hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:228)
> ~[hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> ... 27 more
> Caused by: java.io.EOFException
> at java.io.DataInputStream.readInt(DataInputStream.java:392) ~[?:1.8.0_232]
> at org.apache.hadoop.ipc.Client$IpcStreams.readResponse(Client.java:1803)
> ~[hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> at
> org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1168)
> ~[hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1064)
> ~[hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> 2020-04-22T16:43:53,945 INFO [main]: client.TezClient
> (TezClient.java:getAppMasterStatus(862)) - App did not succeed. Diagnostics:
> Application application_1587555843754_0015 failed 1 times (global limit =2;
> local limit is =1) due to AM Container for
> appattempt_1587555843754_0015_000001 exited with exitCode: 1
> Failing this attempt.Diagnostics: [2020-04-22 16:43:53.803]Exception from
> container-launch.
> Container id: container_e81_1587555843754_0015_01_000001
> Exit code: 1
> Exception message: Launch container failed
> Shell output: main : command provided 1
> main : run as user is hive
> main : requested yarn user is hive
> Getting exit code file...
> Creating script paths...
> Writing pid file...
> Writing to tmp file
> /hadoop/yarn/local/nmPrivate/application_1587555843754_0015/container_e81_1587555843754_0015_01_000001/container_e81_1587555843754_0015_01_000001.pid.tmp
> Writing to cgroup task files...
> Creating local dirs...
> Launching container...
> Getting exit code file...
> Creating script paths...
> [2020-04-22 16:43:53.807]Container exited with a non-zero exit code 1. Error
> file: prelaunch.err.
> Last 4096 bytes of prelaunch.err :
> Last 4096 bytes of stderr :
> [2020-04-22 16:43:53.808]Container exited with a non-zero exit code 1. Error
> file: prelaunch.err.
> Last 4096 bytes of prelaunch.err :
> Last 4096 bytes of stderr :
> For more detailed output, check the application tracking page:
> http://<NODE>:8088/cluster/app/application_1587555843754_0015 Then click on
> links to logs of each attempt.
> . Failing the application.
> 2020-04-22T16:43:53,946 ERROR [main]: server.HiveServer2
> (HiveServer2.java:start(754)) - Error starting Tez sessions:
> org.apache.hive.service.ServiceException: Unable to setup tez session pool
> at
> org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:858)
> ~[hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hive.service.server.HiveServer2.startOrReconnectTezSessions(HiveServer2.java:828)
> ~[hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:752)
> [hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:1078)
> [hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hive.service.server.HiveServer2.access$1700(HiveServer2.java:136)
> [hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:1346)
> [hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:1190)
> [hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_232]
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> ~[?:1.8.0_232]
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> ~[?:1.8.0_232]
> at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_232]
> at org.apache.hadoop.util.RunJar.run(RunJar.java:318)
> [hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> at org.apache.hadoop.util.RunJar.main(RunJar.java:232)
> [hadoop-common-3.1.1.3.1.4.0-315.jar:?]
> Caused by: org.apache.tez.dag.api.SessionNotRunning: TezSession has already
> shutdown. Application application_1587555843754_0015 failed 1 times (global
> limit =2; local limit is =1) due to AM Container for
> appattempt_1587555843754_0015_000001 exited with exitCode: 1
> Failing this attempt.Diagnostics: [2020-04-22 16:43:53.803]Exception from
> container-launch.
> Container id: container_e81_1587555843754_0015_01_000001
> Exit code: 1
> Exception message: Launch container failed
> Shell output: main : command provided 1
> main : run as user is hive
> main : requested yarn user is hive
> Getting exit code file...
> Creating script paths...
> Writing pid file...
> Writing to tmp file
> /hadoop/yarn/local/nmPrivate/application_1587555843754_0015/container_e81_1587555843754_0015_01_000001/container_e81_1587555843754_0015_01_000001.pid.tmp
> Writing to cgroup task files...
> Creating local dirs...
> Launching container...
> Getting exit code file...
> Creating script paths...
> [2020-04-22 16:43:53.807]Container exited with a non-zero exit code 1. Error
> file: prelaunch.err.
> Last 4096 bytes of prelaunch.err :
> Last 4096 bytes of stderr :
> [2020-04-22 16:43:53.808]Container exited with a non-zero exit code 1. Error
> file: prelaunch.err.
> Last 4096 bytes of prelaunch.err :
> Last 4096 bytes of stderr :
> For more detailed output, check the application tracking page:
> http://bigdata-m0.mova.indra.es:8088/cluster/app/application_1587555843754_0015
> Then click on links to logs of each attempt.
> . Failing the application.
> at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:1013)
> ~[tez-api-0.9.1.3.1.4.0-315.jar:0.9.1.3.1.4.0-315]
> at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:982)
> ~[tez-api-0.9.1.3.1.4.0-315.jar:0.9.1.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionState.startSessionAndContainers(TezSessionState.java:536)
> ~[hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionState.openInternal(TezSessionState.java:451)
> ~[hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolSession.openInternal(TezSessionPoolSession.java:124)
> ~[hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:373)
> ~[hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:236)
> ~[hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startInitialSession(TezSessionPool.java:354)
> ~[hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startUnderInitLock(TezSessionPool.java:166)
> ~[hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.start(TezSessionPool.java:123)
> ~[hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.startPool(TezSessionPoolManager.java:112)
> ~[hive-exec-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> at
> org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:855)
> ~[hive-service-3.1.0.3.1.4.0-315.jar:3.1.0.3.1.4.0-315]
> ... 12 more
> 2020-04-22T16:43:53,947 INFO [main]: server.HiveServer2
> (HiveServer2.java:stop(918)) - Shutting down HiveServer2
> 2020-04-22T16:43:53,947 INFO [main]: thrift.ThriftCLIService
> (ThriftBinaryCLIService.java:stopServer(186)) - Thrift server has stopped
> 2020-04-22T16:43:53,947 INFO [main]: service.AbstractService
> (AbstractService.java:stop(130)) - Service:ThriftBinaryCLIService is stopped.
> 2020-04-22T16:43:53,947 INFO [main]: service.AbstractService
> (AbstractService.java:stop(130)) - Service:OperationManager is stopped.
> 2020-04-22T16:43:53,948 INFO [main]: service.AbstractService
> (AbstractService.java:stop(130)) - Service:SessionManager is stopped.
> 2020-04-22T16:43:53,966 INFO [main]: service.AbstractService
> (AbstractService.java:stop(130)) - Service:CLIService is stopped
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
