[ https://issues.apache.org/jira/browse/HDFS-16072?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Ahmed Hussein resolved HDFS-16072. ---------------------------------- Fix Version/s: 2.10.2 Resolution: Fixed > TestBlockRecovery fails consistently on Branch-2.10 > --------------------------------------------------- > > Key: HDFS-16072 > URL: https://issues.apache.org/jira/browse/HDFS-16072 > Project: Hadoop HDFS > Issue Type: Sub-task > Components: datanode, test > Affects Versions: 2.10.1 > Reporter: Ahmed Hussein > Assignee: Ahmed Hussein > Priority: Major > Labels: pull-request-available > Fix For: 2.10.2 > > Time Spent: 40m > Remaining Estimate: 0h > > {{TestBlockRecovery}} fails on branch-2.10 consistently. > I found that the failures were reported in the Qbt-Reports since March 2021 > to say the least. > {code:bash} > [ERROR] Tests run: 20, Failures: 0, Errors: 4, Skipped: 0, Time elapsed: > 21.422 s <<< FAILURE! - in > org.apache.hadoop.hdfs.server.datanode.TestBlockRecovery > [ERROR] > testRaceBetweenReplicaRecoveryAndFinalizeBlock(org.apache.hadoop.hdfs.server.datanode.TestBlockRecovery) > Time elapsed: 2.814 s <<< ERROR! > java.io.IOException: com.google.protobuf.ServiceException: > java.lang.IllegalThreadStateException > at > org.apache.hadoop.ipc.ProtobufHelper.getRemoteException(ProtobufHelper.java:47) > at > org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getDatanodeReport(ClientNamenodeProtocolTranslatorPB.java:656) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:607) > at > org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422) > at > org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165) > at > org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157) > at > org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) > at > org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359) > at com.sun.proxy.$Proxy28.getDatanodeReport(Unknown Source) > at org.apache.hadoop.hdfs.DFSClient.datanodeReport(DFSClient.java:2132) > at > org.apache.hadoop.hdfs.MiniDFSCluster.waitActive(MiniDFSCluster.java:2699) > at > org.apache.hadoop.hdfs.MiniDFSCluster.waitActive(MiniDFSCluster.java:2743) > at > org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes(MiniDFSCluster.java:1723) > at > org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:905) > at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:517) > at > org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:476) > at > org.apache.hadoop.hdfs.server.datanode.TestBlockRecovery.testRaceBetweenReplicaRecoveryAndFinalizeBlock(TestBlockRecovery.java:694) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:607) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at > org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:288) > at > org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:282) > at java.util.concurrent.FutureTask.run(FutureTask.java:262) > at java.lang.Thread.run(Thread.java:748) > Caused by: com.google.protobuf.ServiceException: > java.lang.IllegalThreadStateException > at > org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:244) > at > org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) > at com.sun.proxy.$Proxy27.getDatanodeReport(Unknown Source) > at > org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getDatanodeReport(ClientNamenodeProtocolTranslatorPB.java:653) > ... 30 more > Caused by: java.lang.IllegalThreadStateException > at java.lang.ThreadGroup.addUnstarted(ThreadGroup.java:864) > at java.lang.Thread.init(Thread.java:405) > at java.lang.Thread.init(Thread.java:349) > at java.lang.Thread.<init>(Thread.java:678) > at > java.util.concurrent.Executors$DefaultThreadFactory.newThread(Executors.java:574) > at > com.google.common.util.concurrent.ThreadFactoryBuilder$1.newThread(ThreadFactoryBuilder.java:159) > at > java.util.concurrent.ThreadPoolExecutor$Worker.<init>(ThreadPoolExecutor.java:617) > at > java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:931) > at > java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1381) > at > java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112) > at > org.apache.hadoop.ipc.Client$Connection.sendRpcRequest(Client.java:1147) > at org.apache.hadoop.ipc.Client.call(Client.java:1456) > at org.apache.hadoop.ipc.Client.call(Client.java:1403) > at > org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) > ... 33 more > [ERROR] > testRecoveryTimeout(org.apache.hadoop.hdfs.server.datanode.TestBlockRecovery) > Time elapsed: 1.529 s <<< ERROR! > java.io.IOException: com.google.protobuf.ServiceException: > java.lang.IllegalThreadStateException > at > org.apache.hadoop.ipc.ProtobufHelper.getRemoteException(ProtobufHelper.java:47) > at > org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getDatanodeReport(ClientNamenodeProtocolTranslatorPB.java:656) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:607) > at > org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422) > at > org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165) > at > org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157) > at > org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) > at > org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359) > at com.sun.proxy.$Proxy28.getDatanodeReport(Unknown Source) > at org.apache.hadoop.hdfs.DFSClient.datanodeReport(DFSClient.java:2132) > at > org.apache.hadoop.hdfs.MiniDFSCluster.waitActive(MiniDFSCluster.java:2699) > at > org.apache.hadoop.hdfs.MiniDFSCluster.waitActive(MiniDFSCluster.java:2743) > at > org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes(MiniDFSCluster.java:1723) > at > org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:905) > at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:517) > at > org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:476) > at > org.apache.hadoop.hdfs.server.datanode.TestBlockRecovery.testRecoveryWithDatanodeDelayed(TestBlockRecovery.java:1055) > at > org.apache.hadoop.hdfs.server.datanode.TestBlockRecovery.testRecoveryTimeout(TestBlockRecovery.java:1045) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:607) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at > org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:288) > at > org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:282) > at java.util.concurrent.FutureTask.run(FutureTask.java:262) > at java.lang.Thread.run(Thread.java:748) > Caused by: com.google.protobuf.ServiceException: > java.lang.IllegalThreadStateException > at > org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:244) > at > org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) > at com.sun.proxy.$Proxy27.getDatanodeReport(Unknown Source) > at > org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getDatanodeReport(ClientNamenodeProtocolTranslatorPB.java:653) > ... 31 more > Caused by: java.lang.IllegalThreadStateException > at java.lang.ThreadGroup.addUnstarted(ThreadGroup.java:864) > at java.lang.Thread.init(Thread.java:405) > at java.lang.Thread.init(Thread.java:349) > at java.lang.Thread.<init>(Thread.java:678) > at > java.util.concurrent.Executors$DefaultThreadFactory.newThread(Executors.java:574) > at > com.google.common.util.concurrent.ThreadFactoryBuilder$1.newThread(ThreadFactoryBuilder.java:159) > at > java.util.concurrent.ThreadPoolExecutor$Worker.<init>(ThreadPoolExecutor.java:617) > at > java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:931) > at > java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1381) > at > java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112) > at > org.apache.hadoop.ipc.Client$Connection.sendRpcRequest(Client.java:1147) > at org.apache.hadoop.ipc.Client.call(Client.java:1456) > at org.apache.hadoop.ipc.Client.call(Client.java:1403) > at > org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) > ... 34 more > [ERROR] > testRecoverySlowerThanHeartbeat(org.apache.hadoop.hdfs.server.datanode.TestBlockRecovery) > Time elapsed: 1.352 s <<< ERROR! > java.io.IOException: com.google.protobuf.ServiceException: > java.lang.IllegalThreadStateException > at > org.apache.hadoop.ipc.ProtobufHelper.getRemoteException(ProtobufHelper.java:47) > at > org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getDatanodeReport(ClientNamenodeProtocolTranslatorPB.java:656) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:607) > at > org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422) > at > org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165) > at > org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157) > at > org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) > at > org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359) > at com.sun.proxy.$Proxy28.getDatanodeReport(Unknown Source) > at org.apache.hadoop.hdfs.DFSClient.datanodeReport(DFSClient.java:2132) > at > org.apache.hadoop.hdfs.MiniDFSCluster.waitActive(MiniDFSCluster.java:2699) > at > org.apache.hadoop.hdfs.MiniDFSCluster.waitActive(MiniDFSCluster.java:2743) > at > org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes(MiniDFSCluster.java:1723) > at > org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:905) > at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:517) > at > org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:476) > at > org.apache.hadoop.hdfs.server.datanode.TestBlockRecovery.testRecoveryWithDatanodeDelayed(TestBlockRecovery.java:1055) > at > org.apache.hadoop.hdfs.server.datanode.TestBlockRecovery.testRecoverySlowerThanHeartbeat(TestBlockRecovery.java:1006) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:607) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at > org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:288) > at > org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:282) > at java.util.concurrent.FutureTask.run(FutureTask.java:262) > at java.lang.Thread.run(Thread.java:748) > Caused by: com.google.protobuf.ServiceException: > java.lang.IllegalThreadStateException > at > org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:244) > at > org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) > at com.sun.proxy.$Proxy27.getDatanodeReport(Unknown Source) > at > org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getDatanodeReport(ClientNamenodeProtocolTranslatorPB.java:653) > ... 31 more > Caused by: java.lang.IllegalThreadStateException > at java.lang.ThreadGroup.addUnstarted(ThreadGroup.java:864) > at java.lang.Thread.init(Thread.java:405) > at java.lang.Thread.init(Thread.java:349) > at java.lang.Thread.<init>(Thread.java:678) > at > java.util.concurrent.Executors$DefaultThreadFactory.newThread(Executors.java:574) > at > com.google.common.util.concurrent.ThreadFactoryBuilder$1.newThread(ThreadFactoryBuilder.java:159) > at > java.util.concurrent.ThreadPoolExecutor$Worker.<init>(ThreadPoolExecutor.java:617) > at > java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:931) > at > java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1381) > at > java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112) > at > org.apache.hadoop.ipc.Client$Connection.sendRpcRequest(Client.java:1147) > at org.apache.hadoop.ipc.Client.call(Client.java:1456) > at org.apache.hadoop.ipc.Client.call(Client.java:1403) > at > org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) > ... 34 more > [ERROR] > testRecoveryWillIgnoreMinReplication(org.apache.hadoop.hdfs.server.datanode.TestBlockRecovery) > Time elapsed: 3.062 s <<< ERROR! > java.io.IOException: com.google.protobuf.ServiceException: > java.lang.IllegalThreadStateException > at > org.apache.hadoop.ipc.ProtobufHelper.getRemoteException(ProtobufHelper.java:47) > at > org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getDatanodeReport(ClientNamenodeProtocolTranslatorPB.java:656) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:607) > at > org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422) > at > org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165) > at > org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157) > at > org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) > at > org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359) > at com.sun.proxy.$Proxy28.getDatanodeReport(Unknown Source) > at org.apache.hadoop.hdfs.DFSClient.datanodeReport(DFSClient.java:2132) > at > org.apache.hadoop.hdfs.MiniDFSCluster.waitActive(MiniDFSCluster.java:2699) > at > org.apache.hadoop.hdfs.MiniDFSCluster.waitActive(MiniDFSCluster.java:2743) > at > org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes(MiniDFSCluster.java:1723) > at > org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:905) > at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:517) > at > org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:476) > at > org.apache.hadoop.hdfs.server.datanode.TestBlockRecovery.testRecoveryWillIgnoreMinReplication(TestBlockRecovery.java:1121) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:607) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at > org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:288) > at > org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:282) > at java.util.concurrent.FutureTask.run(FutureTask.java:262) > at java.lang.Thread.run(Thread.java:748) > Caused by: com.google.protobuf.ServiceException: > java.lang.IllegalThreadStateException > at > org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:244) > at > org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) > at com.sun.proxy.$Proxy27.getDatanodeReport(Unknown Source) > at > org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getDatanodeReport(ClientNamenodeProtocolTranslatorPB.java:653) > ... 30 more > Caused by: java.lang.IllegalThreadStateException > at java.lang.ThreadGroup.addUnstarted(ThreadGroup.java:864) > at java.lang.Thread.init(Thread.java:405) > at java.lang.Thread.init(Thread.java:349) > at java.lang.Thread.<init>(Thread.java:678) > at > java.util.concurrent.Executors$DefaultThreadFactory.newThread(Executors.java:574) > at > com.google.common.util.concurrent.ThreadFactoryBuilder$1.newThread(ThreadFactoryBuilder.java:159) > at > java.util.concurrent.ThreadPoolExecutor$Worker.<init>(ThreadPoolExecutor.java:617) > at > java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:931) > at > java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1381) > at > java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112) > at > org.apache.hadoop.ipc.Client$Connection.sendRpcRequest(Client.java:1147) > at org.apache.hadoop.ipc.Client.call(Client.java:1456) > at org.apache.hadoop.ipc.Client.call(Client.java:1403) > at > org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) > ... 33 more > {code} -- This message was sent by Atlassian Jira (v8.3.4#803005) --------------------------------------------------------------------- To unsubscribe, e-mail: hdfs-dev-unsubscr...@hadoop.apache.org For additional commands, e-mail: hdfs-dev-h...@hadoop.apache.org