Deadlock in DFSClient#getBlockLocations with the security enabled
-----------------------------------------------------------------
Key: HADOOP-6609
URL: https://issues.apache.org/jira/browse/HADOOP-6609
Project: Hadoop Common
Issue Type: Bug
Reporter: Hairong Kuang
Here is the stack trace:
"IPC Client (47) connection to XX" daemon
prio=10 tid=0x00002aaae0369c00 nid=0x655b waiting for monitor entry
[0x000000004181f000..0x000000004181fb80]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.apache.hadoop.io.UTF8.readChars(UTF8.java:210)
- waiting to lock <0x00002aaab3eaee50> (a org.apache.hadoop.io.DataOutputBuffer)
at org.apache.hadoop.io.UTF8.readString(UTF8.java:203)
at org.apache.hadoop.io.ObjectWritable.readObject(ObjectWritable.java:179)
at org.apache.hadoop.io.ObjectWritable.readFields(ObjectWritable.java:66)
at org.apache.hadoop.ipc.Client$Connection.receiveResponse(Client.java:638)
at org.apache.hadoop.ipc.Client$Connection.run(Client.java:573)
"IPC Client (47) connection to /0.0.0.0:50030 from job_201002262308_0007"
daemon prio=10 tid=0x00002aaae0272800 nid=0x6556 waiting for monitor entry
[0x000000004131a000..0x000000004131ad00]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.apache.hadoop.io.UTF8.readChars(UTF8.java:210)
- waiting to lock <0x00002aaab3eaee50> (a org.apache.hadoop.io.DataOutputBuffer)
at org.apache.hadoop.io.UTF8.readString(UTF8.java:203)
at org.apache.hadoop.io.ObjectWritable.readObject(ObjectWritable.java:179)
at org.apache.hadoop.io.ObjectWritable.readFields(ObjectWritable.java:66)
at org.apache.hadoop.ipc.Client$Connection.receiveResponse(Client.java:638)
at org.apache.hadoop.ipc.Client$Connection.run(Client.java:573)
"main" prio=10 tid=0x0000000046c17800 nid=0x6544 in Object.wait()
[0x0000000040207000..0x0000000040209ec0]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00002aaacee6bc38> (a org.apache.hadoop.ipc.Client$Call)
at java.lang.Object.wait(Object.java:485)
at org.apache.hadoop.ipc.Client.call(Client.java:854) - locked
<0x00002aaacee6bc38> (a org.apache.hadoop.ipc.Client$Call)
at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:223)
at $Proxy2.getBlockLocations(Unknown Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
at $Proxy2.getBlockLocations(Unknown Source)
at org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:333)
at org.apache.hadoop.hdfs.DFSClient.access$2(DFSClient.java:330)
at
org.apache.hadoop.hdfs.DFSClient$DFSInputStream.getBlockAt(DFSClient.java:1606)
- locked <0x00002aaacecb8258> (a
org.apache.hadoop.hdfs.DFSClient$DFSInputStream)
at
org.apache.hadoop.hdfs.DFSClient$DFSInputStream.blockSeekTo(DFSClient.java:1704)
- locked <0x00002aaacecb8258> (a
org.apache.hadoop.hdfs.DFSClient$DFSInputStream)
at org.apache.hadoop.hdfs.DFSClient$DFSInputStream.read(DFSClient.java:1856)
- locked <0x00002aaacecb8258> (a
org.apache.hadoop.hdfs.DFSClient$DFSInputStream)
at java.io.DataInputStream.readFully(DataInputStream.java:178)
at org.apache.hadoop.io.DataOutputBuffer$Buffer.write(DataOutputBuffer.java:63)
at org.apache.hadoop.io.DataOutputBuffer.write(DataOutputBuffer.java:101)
at org.apache.hadoop.io.UTF8.readChars(UTF8.java:211)
- locked <0x00002aaab3eaee50> (a org.apache.hadoop.io.DataOutputBuffer)
at org.apache.hadoop.io.UTF8.readString(UTF8.java:203)
at org.apache.hadoop.mapred.FileSplit.readFields(FileSplit.java:90)
at
org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:67)
at
org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:1)
at org.apache.hadoop.mapred.MapTask.getSplitDetails(MapTask.java:341)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:357)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:317)
at org.apache.hadoop.mapred.Child$4.run(Child.java:211)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:700)
at org.apache.hadoop.mapred.Child.main(Child.java:205)
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
