Hi, can you provide the jobmanager log of that run? it seems that the operation timed out. The JobManager log will help us to give some insights into the root cause.
On Tue, May 18, 2021 at 1:42 PM V N, Suchithra (Nokia - IN/Bangalore) < [email protected]> wrote: > Hi, > > > > Stop command is failing with below error with apache flink 1.12.3 version. > Could you pls help. > > > > log":"[Flink-RestClusterClient-IO-thread-2] > org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel > Force-closing a channel whose registration task was not accepted by an > event loop: [id: 0x4fb1c35c]"} > > java.util.concurrent.RejectedExecutionException: event executor terminated > > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AbstractUnsafe.register(AbstractChannel.java:471) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:87) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:81) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.channel.MultithreadEventLoopGroup.register(MultithreadEventLoopGroup.java:86) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.bootstrap.AbstractBootstrap.initAndRegister(AbstractBootstrap.java:323) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.doResolveAndConnect(Bootstrap.java:155) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:139) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:123) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:421) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:344) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:258) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.client.program.rest.RestClusterClient.lambda$null$23(RestClusterClient.java:777) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1072) > [?:?] > > at > java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) > [?:?] > > at > java.util.concurrent.CompletableFuture.postFire(CompletableFuture.java:610) > [?:?] > > at > java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:649) > [?:?] > > at > java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:478) > [?:?] > > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) > [?:?] > > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) > [?:?] > > at java.lang.Thread.run(Thread.java:829) [?:?] > > {"type":"log","host":"${env:CLOG_HOST}","level":"ERROR","systemid":"${env:CLOG_SYSTEMID}","system":"${env:CLOG_SYSTEM}","time":"2021-05-18T10:32:04.934Z","timezone":"UTC","log":"[Flink-RestClusterClient-IO-thread-2] > org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.rejectedExecution > Failed to submit a listener notification task. Event loop shut down?"} > > java.util.concurrent.RejectedExecutionException: event executor terminated > > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:183) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:95) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:30) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:425) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:344) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:258) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.client.program.rest.RestClusterClient.lambda$null$23(RestClusterClient.java:777) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1072) > [?:?] > > at > java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) > [?:?] > > at > java.util.concurrent.CompletableFuture.postFire(CompletableFuture.java:610) > [?:?] > > at > java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:649) > [?:?] > > at > java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:478) > [?:?] > > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) > [?:?] > > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) > [?:?] > > at java.lang.Thread.run(Thread.java:829) [?:?] > > > > "log":"[main] org.apache.flink.client.cli.CliFrontend Error while running > the command."} > > org.apache.flink.util.FlinkException: Could not stop with a savepoint job > "00000000000000000000000000000000". > > at > org.apache.flink.client.cli.CliFrontend.lambda$stop$5(CliFrontend.java:581) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.client.cli.CliFrontend.runClusterAction(CliFrontend.java:1002) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.client.cli.CliFrontend.stop(CliFrontend.java:569) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.client.cli.CliFrontend.parseAndRun(CliFrontend.java:1069) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.client.cli.CliFrontend.lambda$main$10(CliFrontend.java:1132) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > at java.security.AccessController.doPrivileged(Native Method) > ~[?:?] > > at javax.security.auth.Subject.doAs(Subject.java:423) [?:?] > > at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1692) > [flink-shaded-hadoop-2-uber-2.6.5-7.0.jar:2.6.5-7.0] > > at > org.apache.flink.runtime.security.contexts.HadoopSecurityContext.runSecured(HadoopSecurityContext.java:41) > [flink-dist_2.11-1.12.3.jar:1.12.3] > > at > org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:1132) > [flink-dist_2.11-1.12.3.jar:1.12.3] > > Caused by: java.util.concurrent.TimeoutException > > at > java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1886) > ~[?:?] > > at > java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2021) > ~[?:?] > > at > org.apache.flink.client.cli.CliFrontend.lambda$stop$5(CliFrontend.java:579) > ~[flink-dist_2.11-1.12.3.jar:1.12.3] > > ... 9 more > > > > ------------------------------------------------------------ > > The program finished with the following exception: > > > > org.apache.flink.util.FlinkException: Could not stop with a savepoint job > "00000000000000000000000000000000". > > at > org.apache.flink.client.cli.CliFrontend.lambda$stop$5(CliFrontend.java:581) > > at > org.apache.flink.client.cli.CliFrontend.runClusterAction(CliFrontend.java:1002) > > at > org.apache.flink.client.cli.CliFrontend.stop(CliFrontend.java:569) > > at > org.apache.flink.client.cli.CliFrontend.parseAndRun(CliFrontend.java:1069) > > at > org.apache.flink.client.cli.CliFrontend.lambda$main$10(CliFrontend.java:1132) > > at java.base/java.security.AccessController.doPrivileged(Native > Method) > > at java.base/javax.security.auth.Subject.doAs(Subject.java:423) > > at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1692) > > at > org.apache.flink.runtime.security.contexts.HadoopSecurityContext.runSecured(HadoopSecurityContext.java:41) > > at > org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:1132) > > Caused by: java.util.concurrent.TimeoutException > > at > java.base/java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1886) > > at > java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2021) > > at > org.apache.flink.client.cli.CliFrontend.lambda$stop$5(CliFrontend.java:579) > > ... 9 more > > > > Regards, > > Suchithra > > >
