[
https://issues.apache.org/jira/browse/IGNITE-24932?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Vyacheslav Koptilin updated IGNITE-24932:
-----------------------------------------
Description:
Need to find out the root cause of the hang and fix it. The main goal is that
the test suite should run all tests and not hang.
[https://ci.ignite.apache.org/buildConfiguration/ApacheIgnite3xGradle_Test_IntegrationTests_ModuleRunner?branch=pull%2F5377&buildTypeTab=overview]
UPDATE:
It seems to me the problem relates to the following stack traces:
{noformat}
"%isckvt_tanc_3345%rebalance-scheduler-0" #58534 prio=5
os_prio=0 cpu=1.32ms elapsed=1804.91s tid=0x00007f434de8f0e0 nid=0x28657f
waiting on condition [0x00007f43c5695000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@17.0.6/Native
Method)
- parking to wait for <merged>(a
java.util.concurrent.CompletableFuture$Signaller)
at
java.util.concurrent.locks.LockSupport.park(java.base@17.0.6/LockSupport.java:211)
at
java.util.concurrent.CompletableFuture$Signaller.block(java.base@17.0.6/CompletableFuture.java:1864)
at
java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@17.0.6/ForkJoinPool.java:3463)
at
java.util.concurrent.ForkJoinPool.managedBlock(java.base@17.0.6/ForkJoinPool.java:3434)
at
java.util.concurrent.CompletableFuture.waitingGet(java.base@17.0.6/CompletableFuture.java:1898)
at
java.util.concurrent.CompletableFuture.get(java.base@17.0.6/CompletableFuture.java:2072)
at
org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceRaftGroupEventsListener.doStableKeySwitch(ZoneRebalanceRaftGroupEventsListener.java:364)
at
org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceRaftGroupEventsListener.lambda$onNewPeersConfigurationApplied$1(ZoneRebalanceRaftGroupEventsListener.java:232)
at
org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceRaftGroupEventsListener$$Lambda$3045/0x00000008018a3f50.run(Unknown
Source)
at
java.util.concurrent.Executors$RunnableAdapter.call(java.base@17.0.6/Executors.java:539)
at
java.util.concurrent.FutureTask.run(java.base@17.0.6/FutureTask.java:264)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(java.base@17.0.6/ScheduledThreadPoolExecutor.java:304)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.6/ThreadPoolExecutor.java:1136)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.6/ThreadPoolExecutor.java:635)
at java.lang.Thread.run(java.base@17.0.6/Thread.java:833)
{noformat}
and
{noformat}
"Test worker" #1 prio=5 os_prio=0 cpu=68530.73ms
elapsed=2355.80s tid=0x00007f4454030a30 nid=0x27173a waiting on condition
[0x00007f445b8e5000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(java.base@17.0.6/Native Method)
at
org.apache.ignite.internal.util.IgniteSpinReadWriteLock.writeLock(IgniteSpinReadWriteLock.java:257)
at
org.apache.ignite.internal.util.IgniteSpinBusyLock.block(IgniteSpinBusyLock.java:68)
at
org.apache.ignite.internal.partition.replicator.PartitionReplicaLifecycleManager.beforeNodeStop(PartitionReplicaLifecycleManager.java:758)
at
org.apache.ignite.internal.app.LifecycleManager.initiateAllComponentsStop(LifecycleManager.java:172)
- locked <0x0000000716ac8278> (a
org.apache.ignite.internal.app.LifecycleManager)
at
org.apache.ignite.internal.app.LifecycleManager.stopNode(LifecycleManager.java:152)
at
org.apache.ignite.internal.app.IgniteImpl.stopAsync(IgniteImpl.java:1634)
at
org.apache.ignite.internal.app.IgniteServerImpl.doShutdownAsync(IgniteServerImpl.java:352)
at
org.apache.ignite.internal.app.IgniteServerImpl$$Lambda$3149/0x00000008018f8928.get(Unknown
Source)
at
org.apache.ignite.internal.app.IgniteServerImpl.lambda$chainRestartOrShutdownAction$6(IgniteServerImpl.java:291)
at
org.apache.ignite.internal.app.IgniteServerImpl$$Lambda$3151/0x00000008018f8d80.apply(Unknown
Source)
at
java.util.concurrent.CompletableFuture.uniComposeStage(java.base@17.0.6/CompletableFuture.java:1187)
at
java.util.concurrent.CompletableFuture.thenCompose(java.base@17.0.6/CompletableFuture.java:2309)
at
org.apache.ignite.internal.app.IgniteServerImpl.chainRestartOrShutdownAction(IgniteServerImpl.java:291)
at
org.apache.ignite.internal.app.IgniteServerImpl.shutdownAsync(IgniteServerImpl.java:328)
- locked <0x000000071850d330> (a java.lang.Object)
at
org.apache.ignite.internal.app.IgniteServerImpl.shutdown(IgniteServerImpl.java:368)
at
org.apache.ignite.internal.Cluster$$Lambda$3303/0x0000000801919250.accept(Unknown
Source)
at
java.util.stream.ForEachOps$ForEachOp$OfRef.accept(java.base@17.0.6/ForEachOps.java:183)
at
java.util.stream.ReferencePipeline$2$1.accept(java.base@17.0.6/ReferencePipeline.java:179)
at
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(java.base@17.0.6/ArrayList.java:1625)
at
java.util.stream.AbstractPipeline.copyInto(java.base@17.0.6/AbstractPipeline.java:509)
at
java.util.stream.ForEachOps$ForEachTask.compute(java.base@17.0.6/ForEachOps.java:290)
at
java.util.concurrent.CountedCompleter.exec(java.base@17.0.6/CountedCompleter.java:754)
at
java.util.concurrent.ForkJoinTask.doExec(java.base@17.0.6/ForkJoinTask.java:373)
at
java.util.concurrent.ForkJoinTask.invoke(java.base@17.0.6/ForkJoinTask.java:686)
at
java.util.stream.ForEachOps$ForEachOp.evaluateParallel(java.base@17.0.6/ForEachOps.java:159)
at
java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateParallel(java.base@17.0.6/ForEachOps.java:173)
at
java.util.stream.AbstractPipeline.evaluate(java.base@17.0.6/AbstractPipeline.java:233)
at
java.util.stream.ReferencePipeline.forEach(java.base@17.0.6/ReferencePipeline.java:596)
at
org.apache.ignite.internal.Cluster.shutdown(Cluster.java:562)
at
org.apache.ignite.internal.ClusterPerTestIntegrationTest.stopCluster(ClusterPerTestIntegrationTest.java:138)
at
jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@17.0.6/Native
Method)
at
jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@17.0.6/NativeMethodAccessorImpl.java:77)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@17.0.6/DelegatingMethodAccessorImpl.java:43)
at
java.lang.reflect.Method.invoke(java.base@17.0.6/Method.java:568)
at
org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:775)
at
org.junit.platform.commons.support.ReflectionSupport.invokeMethod(ReflectionSupport.java:479)
at
org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
at
org.junit.jupiter.engine.extension.SameThreadTimeoutInvocation.proceed(SameThreadTimeoutInvocation.java:49)
at
org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:161)
at
org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:133)
at
org.junit.jupiter.engine.extension.TimeoutExtension.interceptAfterEachMethod(TimeoutExtension.java:115)
at
org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor$$Lambda$3308/0x0000000801918800.apply(Unknown
Source)
at
org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(InterceptingExecutableInvoker.java:112)
at
org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$ReflectiveInterceptorCall$$Lambda$306/0x0000000800e9f270.apply(Unknown
Source)
at
org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.lambda$invoke$0(InterceptingExecutableInvoker.java:94)
at
org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$$Lambda$500/0x0000000800ecf9a8.apply(Unknown
Source)
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
at
org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.invoke(InterceptingExecutableInvoker.java:93)
at
org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.invoke(InterceptingExecutableInvoker.java:87)
at
org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:526)
at
org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeAfterEachMethodAdapter$24(ClassBasedTestDescriptor.java:516)
at
org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor$$Lambda$457/0x0000000800ec2cf0.invokeAfterEachMethod(Unknown
Source)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeAfterEachMethods$10(TestMethodTestDescriptor.java:242)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor$$Lambda$3306/0x0000000801919ad8.invoke(Unknown
Source)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeAllAfterMethodsOrCallbacks$13(TestMethodTestDescriptor.java:275)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor$$Lambda$3307/0x0000000801919d00.execute(Unknown
Source)
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeAllAfterMethodsOrCallbacks$14(TestMethodTestDescriptor.java:275)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor$$Lambda$3305/0x00000008019198a0.accept(Unknown
Source)
at
org.junit.platform.commons.util.CollectionUtils.forEachInReverseOrder(CollectionUtils.java:243)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeAllAfterMethodsOrCallbacks(TestMethodTestDescriptor.java:274)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeAfterEachMethods(TestMethodTestDescriptor.java:240)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:141)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:69)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:156)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$418/0x0000000800ebe7e0.execute(Unknown
Source)
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:146)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$417/0x0000000800ebe5b8.invoke(Unknown
Source)
at
org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:144)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$416/0x0000000800ebe190.execute(Unknown
Source)
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:143)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:100)
at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService$$Lambda$422/0x0000000800ebf2f8.accept(Unknown
Source)
at
java.util.ArrayList.forEach(java.base@17.0.6/ArrayList.java:1511)
at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:160)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$418/0x0000000800ebe7e0.execute(Unknown
Source)
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:146)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$417/0x0000000800ebe5b8.invoke(Unknown
Source)
at
org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:144)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$416/0x0000000800ebe190.execute(Unknown
Source)
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:143)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:100)
at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService$$Lambda$422/0x0000000800ebf2f8.accept(Unknown
Source)
at
java.util.ArrayList.forEach(java.base@17.0.6/ArrayList.java:1511)
at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:160)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$418/0x0000000800ebe7e0.execute(Unknown
Source)
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:146)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$417/0x0000000800ebe5b8.invoke(Unknown
Source)
at
org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:144)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$416/0x0000000800ebe190.execute(Unknown
Source)
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:143)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:100)
at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:35)
at
org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
at
org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:54)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:201)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:170)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:94)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:59)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator$$Lambda$353/0x0000000800ea8b10.accept(Unknown
Source)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:142)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:58)
at
org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:103)
at
org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:85)
at
org.junit.platform.launcher.core.DelegatingLauncher.execute(DelegatingLauncher.java:47)
at
org.junit.platform.launcher.core.InterceptingLauncher.lambda$execute$1(InterceptingLauncher.java:39)
at
org.junit.platform.launcher.core.InterceptingLauncher$$Lambda$187/0x0000000800e18290.proceed(Unknown
Source)
at
org.junit.platform.launcher.core.ClasspathAlignmentCheckingLauncherInterceptor.intercept(ClasspathAlignmentCheckingLauncherInterceptor.java:25)
at
org.junit.platform.launcher.core.InterceptingLauncher.execute(InterceptingLauncher.java:38)
at
org.junit.platform.launcher.core.DelegatingLauncher.execute(DelegatingLauncher.java:47)
at
org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor$CollectAllTestClassesExecutor.processAllTestClasses(JUnitPlatformTestClassProcessor.java:119)
at
org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor$CollectAllTestClassesExecutor.access$000(JUnitPlatformTestClassProcessor.java:94)
at
org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor.stop(JUnitPlatformTestClassProcessor.java:89)
at
org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.stop(SuiteTestClassProcessor.java:62)
at
jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@17.0.6/Native
Method)
at
jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@17.0.6/NativeMethodAccessorImpl.java:77)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@17.0.6/DelegatingMethodAccessorImpl.java:43)
at
java.lang.reflect.Method.invoke(java.base@17.0.6/Method.java:568)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at
org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33)
at
org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94)
at jdk.proxy2.$Proxy22.stop(jdk.proxy2/Unknown Source)
at
org.gradle.api.internal.tasks.testing.worker.TestWorker$3.run(TestWorker.java:193)
at
org.gradle.api.internal.tasks.testing.worker.TestWorker.executeAndMaintainThreadName(TestWorker.java:129)
at
org.gradle.api.internal.tasks.testing.worker.TestWorker.execute(TestWorker.java:100)
at
org.gradle.api.internal.tasks.testing.worker.TestWorker.execute(TestWorker.java:60)
at
org.gradle.process.internal.worker.child.ActionExecutionWorker.execute(ActionExecutionWorker.java:56)
at
org.gradle.process.internal.worker.child.SystemApplicationClassLoaderWorker.call(SystemApplicationClassLoaderWorker.java:113)
at
org.gradle.process.internal.worker.child.SystemApplicationClassLoaderWorker.call(SystemApplicationClassLoaderWorker.java:65)
at
worker.org.gradle.process.internal.worker.GradleWorkerMain.run(GradleWorkerMain.java:69)
at
worker.org.gradle.process.internal.worker.GradleWorkerMain.main(GradleWorkerMain.java:74)
{noformat}
So, stopping the node is blocked by `doStableSwitch`, which holds the
`busyLock` and failed to wait for an event from the metastorage.
was:
Need to find out the root cause and fix/disable tests. The main goal is that
the test suite should run all tests and not hang.
https://ci.ignite.apache.org/buildConfiguration/ApacheIgnite3xGradle_Test_IntegrationTests_ModuleRunner?branch=pull%2F5377&buildTypeTab=overview
> Stopping the ignite node may result in a hang
> ---------------------------------------------
>
> Key: IGNITE-24932
> URL: https://issues.apache.org/jira/browse/IGNITE-24932
> Project: Ignite
> Issue Type: Bug
> Reporter: Vyacheslav Koptilin
> Assignee: Vyacheslav Koptilin
> Priority: Major
> Labels: ignite-3
> Time Spent: 10m
> Remaining Estimate: 0h
>
> Need to find out the root cause of the hang and fix it. The main goal is that
> the test suite should run all tests and not hang.
> [https://ci.ignite.apache.org/buildConfiguration/ApacheIgnite3xGradle_Test_IntegrationTests_ModuleRunner?branch=pull%2F5377&buildTypeTab=overview]
>
> UPDATE:
> It seems to me the problem relates to the following stack traces:
> {noformat}
> "%isckvt_tanc_3345%rebalance-scheduler-0" #58534 prio=5
> os_prio=0 cpu=1.32ms elapsed=1804.91s tid=0x00007f434de8f0e0 nid=0x28657f
> waiting on condition [0x00007f43c5695000]
> java.lang.Thread.State: WAITING (parking)
> at jdk.internal.misc.Unsafe.park(java.base@17.0.6/Native
> Method)
> - parking to wait for <merged>(a
> java.util.concurrent.CompletableFuture$Signaller)
> at
> java.util.concurrent.locks.LockSupport.park(java.base@17.0.6/LockSupport.java:211)
> at
> java.util.concurrent.CompletableFuture$Signaller.block(java.base@17.0.6/CompletableFuture.java:1864)
> at
> java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@17.0.6/ForkJoinPool.java:3463)
> at
> java.util.concurrent.ForkJoinPool.managedBlock(java.base@17.0.6/ForkJoinPool.java:3434)
> at
> java.util.concurrent.CompletableFuture.waitingGet(java.base@17.0.6/CompletableFuture.java:1898)
> at
> java.util.concurrent.CompletableFuture.get(java.base@17.0.6/CompletableFuture.java:2072)
> at
> org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceRaftGroupEventsListener.doStableKeySwitch(ZoneRebalanceRaftGroupEventsListener.java:364)
> at
> org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceRaftGroupEventsListener.lambda$onNewPeersConfigurationApplied$1(ZoneRebalanceRaftGroupEventsListener.java:232)
> at
> org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceRaftGroupEventsListener$$Lambda$3045/0x00000008018a3f50.run(Unknown
> Source)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(java.base@17.0.6/Executors.java:539)
> at
> java.util.concurrent.FutureTask.run(java.base@17.0.6/FutureTask.java:264)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(java.base@17.0.6/ScheduledThreadPoolExecutor.java:304)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.6/ThreadPoolExecutor.java:1136)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.6/ThreadPoolExecutor.java:635)
> at java.lang.Thread.run(java.base@17.0.6/Thread.java:833)
> {noformat}
> and
> {noformat}
> "Test worker" #1 prio=5 os_prio=0 cpu=68530.73ms
> elapsed=2355.80s tid=0x00007f4454030a30 nid=0x27173a waiting on condition
> [0x00007f445b8e5000]
> java.lang.Thread.State: TIMED_WAITING (sleeping)
> at java.lang.Thread.sleep(java.base@17.0.6/Native Method)
> at
> org.apache.ignite.internal.util.IgniteSpinReadWriteLock.writeLock(IgniteSpinReadWriteLock.java:257)
> at
> org.apache.ignite.internal.util.IgniteSpinBusyLock.block(IgniteSpinBusyLock.java:68)
> at
> org.apache.ignite.internal.partition.replicator.PartitionReplicaLifecycleManager.beforeNodeStop(PartitionReplicaLifecycleManager.java:758)
> at
> org.apache.ignite.internal.app.LifecycleManager.initiateAllComponentsStop(LifecycleManager.java:172)
> - locked <0x0000000716ac8278> (a
> org.apache.ignite.internal.app.LifecycleManager)
> at
> org.apache.ignite.internal.app.LifecycleManager.stopNode(LifecycleManager.java:152)
> at
> org.apache.ignite.internal.app.IgniteImpl.stopAsync(IgniteImpl.java:1634)
> at
> org.apache.ignite.internal.app.IgniteServerImpl.doShutdownAsync(IgniteServerImpl.java:352)
> at
> org.apache.ignite.internal.app.IgniteServerImpl$$Lambda$3149/0x00000008018f8928.get(Unknown
> Source)
> at
> org.apache.ignite.internal.app.IgniteServerImpl.lambda$chainRestartOrShutdownAction$6(IgniteServerImpl.java:291)
> at
> org.apache.ignite.internal.app.IgniteServerImpl$$Lambda$3151/0x00000008018f8d80.apply(Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.uniComposeStage(java.base@17.0.6/CompletableFuture.java:1187)
> at
> java.util.concurrent.CompletableFuture.thenCompose(java.base@17.0.6/CompletableFuture.java:2309)
> at
> org.apache.ignite.internal.app.IgniteServerImpl.chainRestartOrShutdownAction(IgniteServerImpl.java:291)
> at
> org.apache.ignite.internal.app.IgniteServerImpl.shutdownAsync(IgniteServerImpl.java:328)
> - locked <0x000000071850d330> (a java.lang.Object)
> at
> org.apache.ignite.internal.app.IgniteServerImpl.shutdown(IgniteServerImpl.java:368)
> at
> org.apache.ignite.internal.Cluster$$Lambda$3303/0x0000000801919250.accept(Unknown
> Source)
> at
> java.util.stream.ForEachOps$ForEachOp$OfRef.accept(java.base@17.0.6/ForEachOps.java:183)
> at
> java.util.stream.ReferencePipeline$2$1.accept(java.base@17.0.6/ReferencePipeline.java:179)
> at
> java.util.ArrayList$ArrayListSpliterator.forEachRemaining(java.base@17.0.6/ArrayList.java:1625)
> at
> java.util.stream.AbstractPipeline.copyInto(java.base@17.0.6/AbstractPipeline.java:509)
> at
> java.util.stream.ForEachOps$ForEachTask.compute(java.base@17.0.6/ForEachOps.java:290)
> at
> java.util.concurrent.CountedCompleter.exec(java.base@17.0.6/CountedCompleter.java:754)
> at
> java.util.concurrent.ForkJoinTask.doExec(java.base@17.0.6/ForkJoinTask.java:373)
> at
> java.util.concurrent.ForkJoinTask.invoke(java.base@17.0.6/ForkJoinTask.java:686)
> at
> java.util.stream.ForEachOps$ForEachOp.evaluateParallel(java.base@17.0.6/ForEachOps.java:159)
> at
> java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateParallel(java.base@17.0.6/ForEachOps.java:173)
> at
> java.util.stream.AbstractPipeline.evaluate(java.base@17.0.6/AbstractPipeline.java:233)
> at
> java.util.stream.ReferencePipeline.forEach(java.base@17.0.6/ReferencePipeline.java:596)
> at
> org.apache.ignite.internal.Cluster.shutdown(Cluster.java:562)
> at
> org.apache.ignite.internal.ClusterPerTestIntegrationTest.stopCluster(ClusterPerTestIntegrationTest.java:138)
> at
> jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@17.0.6/Native
> Method)
> at
> jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@17.0.6/NativeMethodAccessorImpl.java:77)
> at
> jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@17.0.6/DelegatingMethodAccessorImpl.java:43)
> at
> java.lang.reflect.Method.invoke(java.base@17.0.6/Method.java:568)
> at
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:775)
> at
> org.junit.platform.commons.support.ReflectionSupport.invokeMethod(ReflectionSupport.java:479)
> at
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> at
> org.junit.jupiter.engine.extension.SameThreadTimeoutInvocation.proceed(SameThreadTimeoutInvocation.java:49)
> at
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:161)
> at
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:133)
> at
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptAfterEachMethod(TimeoutExtension.java:115)
> at
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor$$Lambda$3308/0x0000000801918800.apply(Unknown
> Source)
> at
> org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(InterceptingExecutableInvoker.java:112)
> at
> org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$ReflectiveInterceptorCall$$Lambda$306/0x0000000800e9f270.apply(Unknown
> Source)
> at
> org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.lambda$invoke$0(InterceptingExecutableInvoker.java:94)
> at
> org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$$Lambda$500/0x0000000800ecf9a8.apply(Unknown
> Source)
> at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> at
> org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.invoke(InterceptingExecutableInvoker.java:93)
> at
> org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.invoke(InterceptingExecutableInvoker.java:87)
> at
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:526)
> at
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeAfterEachMethodAdapter$24(ClassBasedTestDescriptor.java:516)
> at
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor$$Lambda$457/0x0000000800ec2cf0.invokeAfterEachMethod(Unknown
> Source)
> at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeAfterEachMethods$10(TestMethodTestDescriptor.java:242)
> at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor$$Lambda$3306/0x0000000801919ad8.invoke(Unknown
> Source)
> at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeAllAfterMethodsOrCallbacks$13(TestMethodTestDescriptor.java:275)
> at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor$$Lambda$3307/0x0000000801919d00.execute(Unknown
> Source)
> at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeAllAfterMethodsOrCallbacks$14(TestMethodTestDescriptor.java:275)
> at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor$$Lambda$3305/0x00000008019198a0.accept(Unknown
> Source)
> at
> org.junit.platform.commons.util.CollectionUtils.forEachInReverseOrder(CollectionUtils.java:243)
> at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeAllAfterMethodsOrCallbacks(TestMethodTestDescriptor.java:274)
> at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeAfterEachMethods(TestMethodTestDescriptor.java:240)
> at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:141)
> at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:69)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:156)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$418/0x0000000800ebe7e0.execute(Unknown
> Source)
> at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:146)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$417/0x0000000800ebe5b8.invoke(Unknown
> Source)
> at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:144)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$416/0x0000000800ebe190.execute(Unknown
> Source)
> at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:143)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:100)
> at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService$$Lambda$422/0x0000000800ebf2f8.accept(Unknown
> Source)
> at
> java.util.ArrayList.forEach(java.base@17.0.6/ArrayList.java:1511)
> at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:160)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$418/0x0000000800ebe7e0.execute(Unknown
> Source)
> at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:146)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$417/0x0000000800ebe5b8.invoke(Unknown
> Source)
> at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:144)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$416/0x0000000800ebe190.execute(Unknown
> Source)
> at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:143)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:100)
> at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService$$Lambda$422/0x0000000800ebf2f8.accept(Unknown
> Source)
> at
> java.util.ArrayList.forEach(java.base@17.0.6/ArrayList.java:1511)
> at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:160)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$418/0x0000000800ebe7e0.execute(Unknown
> Source)
> at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:146)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$417/0x0000000800ebe5b8.invoke(Unknown
> Source)
> at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:144)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$$Lambda$416/0x0000000800ebe190.execute(Unknown
> Source)
> at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:143)
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:100)
> at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:35)
> at
> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
> at
> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:54)
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:201)
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:170)
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:94)
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:59)
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator$$Lambda$353/0x0000000800ea8b10.accept(Unknown
> Source)
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:142)
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:58)
> at
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:103)
> at
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:85)
> at
> org.junit.platform.launcher.core.DelegatingLauncher.execute(DelegatingLauncher.java:47)
> at
> org.junit.platform.launcher.core.InterceptingLauncher.lambda$execute$1(InterceptingLauncher.java:39)
> at
> org.junit.platform.launcher.core.InterceptingLauncher$$Lambda$187/0x0000000800e18290.proceed(Unknown
> Source)
> at
> org.junit.platform.launcher.core.ClasspathAlignmentCheckingLauncherInterceptor.intercept(ClasspathAlignmentCheckingLauncherInterceptor.java:25)
> at
> org.junit.platform.launcher.core.InterceptingLauncher.execute(InterceptingLauncher.java:38)
> at
> org.junit.platform.launcher.core.DelegatingLauncher.execute(DelegatingLauncher.java:47)
> at
> org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor$CollectAllTestClassesExecutor.processAllTestClasses(JUnitPlatformTestClassProcessor.java:119)
> at
> org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor$CollectAllTestClassesExecutor.access$000(JUnitPlatformTestClassProcessor.java:94)
> at
> org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor.stop(JUnitPlatformTestClassProcessor.java:89)
> at
> org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.stop(SuiteTestClassProcessor.java:62)
> at
> jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@17.0.6/Native
> Method)
> at
> jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@17.0.6/NativeMethodAccessorImpl.java:77)
> at
> jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@17.0.6/DelegatingMethodAccessorImpl.java:43)
> at
> java.lang.reflect.Method.invoke(java.base@17.0.6/Method.java:568)
> at
> org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
> at
> org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
> at
> org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33)
> at
> org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94)
> at jdk.proxy2.$Proxy22.stop(jdk.proxy2/Unknown Source)
> at
> org.gradle.api.internal.tasks.testing.worker.TestWorker$3.run(TestWorker.java:193)
> at
> org.gradle.api.internal.tasks.testing.worker.TestWorker.executeAndMaintainThreadName(TestWorker.java:129)
> at
> org.gradle.api.internal.tasks.testing.worker.TestWorker.execute(TestWorker.java:100)
> at
> org.gradle.api.internal.tasks.testing.worker.TestWorker.execute(TestWorker.java:60)
> at
> org.gradle.process.internal.worker.child.ActionExecutionWorker.execute(ActionExecutionWorker.java:56)
> at
> org.gradle.process.internal.worker.child.SystemApplicationClassLoaderWorker.call(SystemApplicationClassLoaderWorker.java:113)
> at
> org.gradle.process.internal.worker.child.SystemApplicationClassLoaderWorker.call(SystemApplicationClassLoaderWorker.java:65)
> at
> worker.org.gradle.process.internal.worker.GradleWorkerMain.run(GradleWorkerMain.java:69)
> at
> worker.org.gradle.process.internal.worker.GradleWorkerMain.main(GradleWorkerMain.java:74)
> {noformat}
> So, stopping the node is blocked by `doStableSwitch`, which holds the
> `busyLock` and failed to wait for an event from the metastorage.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
