[ https://issues.apache.org/jira/browse/FLINK-19027?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17205904#comment-17205904 ]
Dian Fu commented on FLINK-19027: --------------------------------- Another instance: https://dev.azure.com/apache-flink/apache-flink/_build/results?buildId=7158&view=logs&s=ae4f8708-9994-57d3-c2d7-b892156e7812&j=5c8e7682-d68f-54d1-16a2-a09310218a49 {code} 21:38:30,457 [ Map (1/1)] WARN org.apache.flink.runtime.taskmanager.Task [] - Map (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_0) switched from RUNNING to FAILED. java.lang.Exception: Failing map @ 4 (0 attempt); last value 391 at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.failMapper(UnalignedCheckpointITCase.java:616) ~[test-classes/:?] at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.checkFail(UnalignedCheckpointITCase.java:611) ~[test-classes/:?] at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.map(UnalignedCheckpointITCase.java:605) ~[test-classes/:?] at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.map(UnalignedCheckpointITCase.java:580) ~[test-classes/:?] at org.apache.flink.streaming.api.operators.StreamMap.processElement(StreamMap.java:41) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.OneInputStreamTask$StreamTaskNetworkOutput.emitRecord(OneInputStreamTask.java:164) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.StreamTaskNetworkInput.processElement(StreamTaskNetworkInput.java:179) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.StreamTaskNetworkInput.emitNext(StreamTaskNetworkInput.java:152) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.StreamOneInputProcessor.processInput(StreamOneInputProcessor.java:67) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.processInput(StreamTask.java:368) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:185) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:594) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:558) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:722) [flink-runtime_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.runtime.taskmanager.Task.run(Task.java:547) [flink-runtime_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_242] 21:38:30,458 [ Map (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Freeing task resources for Map (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_0). 21:38:30,458 [flink-akka.actor.default-dispatcher-4] INFO org.apache.flink.runtime.taskexecutor.TaskExecutor [] - Un-registering task and sending final execution state FAILED to JobManager for task Map (1/1) cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_0. 21:38:30,459 [flink-akka.actor.default-dispatcher-4] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Map (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_0) switched from RUNNING to FAILED on org.apache.flink.runtime.jobmaster.slotpool.SingleLogicalSlot@2535c61e. java.lang.Exception: Failing map @ 4 (0 attempt); last value 391 at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.failMapper(UnalignedCheckpointITCase.java:616) ~[test-classes/:?] at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.checkFail(UnalignedCheckpointITCase.java:611) ~[test-classes/:?] at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.map(UnalignedCheckpointITCase.java:605) ~[test-classes/:?] at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.map(UnalignedCheckpointITCase.java:580) ~[test-classes/:?] at org.apache.flink.streaming.api.operators.StreamMap.processElement(StreamMap.java:41) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.OneInputStreamTask$StreamTaskNetworkOutput.emitRecord(OneInputStreamTask.java:164) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.StreamTaskNetworkInput.processElement(StreamTaskNetworkInput.java:179) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.StreamTaskNetworkInput.emitNext(StreamTaskNetworkInput.java:152) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.StreamOneInputProcessor.processInput(StreamOneInputProcessor.java:67) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.processInput(StreamTask.java:368) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:185) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:594) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:558) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:722) ~[flink-runtime_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.runtime.taskmanager.Task.run(Task.java:547) ~[flink-runtime_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_242] 21:38:30,460 [flink-akka.actor.default-dispatcher-4] INFO org.apache.flink.runtime.executiongraph.failover.flip1.RestartPipelinedRegionFailoverStrategy [] - Calculating tasks to restart to recover the failed task 0a448493b4782967b150582570326227_0. 21:38:30,460 [flink-akka.actor.default-dispatcher-4] INFO org.apache.flink.runtime.executiongraph.failover.flip1.RestartPipelinedRegionFailoverStrategy [] - 3 tasks should be restarted to recover the failed task 0a448493b4782967b150582570326227_0. 21:38:30,460 [flink-akka.actor.default-dispatcher-4] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Job Flink Streaming Job (cb2b6ee8f94a6516932dfa923bfede3f) switched from state RUNNING to RESTARTING. 21:38:30,460 [flink-akka.actor.default-dispatcher-4] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_0) switched from RUNNING to CANCELING. 21:38:30,461 [flink-akka.actor.default-dispatcher-4] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_0) switched from RUNNING to CANCELING. 21:38:30,461 [flink-akka.actor.default-dispatcher-2] INFO org.apache.flink.runtime.taskmanager.Task [] - Attempting to cancel task Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_0). 21:38:30,461 [flink-akka.actor.default-dispatcher-2] INFO org.apache.flink.runtime.taskmanager.Task [] - Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_0) switched from RUNNING to CANCELING. 21:38:30,461 [flink-akka.actor.default-dispatcher-2] INFO org.apache.flink.runtime.taskmanager.Task [] - Triggering cancellation of task code Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_0). 21:38:30,462 [Channel state writer Source: source (1/1)] INFO org.apache.flink.runtime.checkpoint.channel.ChannelStateWriteRequestExecutorImpl [] - Source: source (1/1) discarding 0 drained requests 21:38:30,462 [flink-akka.actor.default-dispatcher-2] INFO org.apache.flink.runtime.taskmanager.Task [] - Attempting to cancel task Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_0). 21:38:30,462 [flink-akka.actor.default-dispatcher-2] INFO org.apache.flink.runtime.taskmanager.Task [] - Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_0) switched from RUNNING to CANCELING. 21:38:30,462 [flink-akka.actor.default-dispatcher-2] INFO org.apache.flink.runtime.taskmanager.Task [] - Triggering cancellation of task code Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_0). 21:38:30,462 [SourceCoordinator-Source: source] INFO org.apache.flink.runtime.source.coordinator.SourceCoordinator [] - Handling subtask 0 failure of source Source: source. 21:38:30,462 [Source: source (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_0) switched from CANCELING to CANCELED. 21:38:30,462 [Source: source (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Freeing task resources for Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_0). 21:38:30,463 [Channel state writer Sink: Unnamed (1/1)] INFO org.apache.flink.runtime.checkpoint.channel.ChannelStateWriteRequestExecutorImpl [] - Sink: Unnamed (1/1) discarding 0 drained requests 21:38:30,463 [flink-akka.actor.default-dispatcher-4] INFO org.apache.flink.runtime.taskexecutor.TaskExecutor [] - Un-registering task and sending final execution state CANCELED to JobManager for task Source: source (1/1) cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_0. 21:38:30,463 [ Sink: Unnamed (1/1)] INFO org.apache.flink.test.checkpointing.UnalignedCheckpointITCase [] - Last received records [[198]] @ 0 subtask (0 attempt) 21:38:30,463 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_0) switched from CANCELING to CANCELED. 21:38:30,463 [ Sink: Unnamed (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_0) switched from CANCELING to CANCELED. 21:38:30,463 [ Sink: Unnamed (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Freeing task resources for Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_0). 21:38:30,463 [flink-akka.actor.default-dispatcher-5] INFO org.apache.flink.runtime.taskexecutor.TaskExecutor [] - Un-registering task and sending final execution state CANCELED to JobManager for task Sink: Unnamed (1/1) cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_0. 21:38:30,481 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_0) switched from CANCELING to CANCELED. 21:38:30,569 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Job Flink Streaming Job (cb2b6ee8f94a6516932dfa923bfede3f) switched from state RESTARTING to RUNNING. 21:38:30,570 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.checkpoint.CheckpointCoordinator [] - Restoring job cb2b6ee8f94a6516932dfa923bfede3f from Checkpoint 4 @ 1601588310432 for cb2b6ee8f94a6516932dfa923bfede3f located at file:/tmp/junit4240320968876241473/junit1530824567618577449/cb2b6ee8f94a6516932dfa923bfede3f/chk-4. 21:38:30,571 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.checkpoint.CheckpointCoordinator [] - No master state to restore 21:38:30,571 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_1) switched from CREATED to SCHEDULED. 21:38:30,571 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Map (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_1) switched from CREATED to SCHEDULED. 21:38:30,571 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_1) switched from CREATED to SCHEDULED. 21:38:30,573 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_1) switched from SCHEDULED to DEPLOYING. 21:38:30,573 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Deploying Source: source (1/1) (attempt #1) with attempt id cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_1 to 64dbfbce-1d29-40cb-b8c7-075b84d49f9c @ localhost (dataPort=-1) with allocation id 6e56c476945fffb3db10c0debda0f1df 21:38:30,573 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Map (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_1) switched from SCHEDULED to DEPLOYING. 21:38:30,573 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Deploying Map (1/1) (attempt #1) with attempt id cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_1 to 64dbfbce-1d29-40cb-b8c7-075b84d49f9c @ localhost (dataPort=-1) with allocation id 6e56c476945fffb3db10c0debda0f1df 21:38:30,573 [flink-akka.actor.default-dispatcher-5] INFO org.apache.flink.runtime.taskexecutor.slot.TaskSlotTableImpl [] - Activate slot 6e56c476945fffb3db10c0debda0f1df. 21:38:30,574 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_1) switched from SCHEDULED to DEPLOYING. 21:38:30,574 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Deploying Sink: Unnamed (1/1) (attempt #1) with attempt id cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_1 to 64dbfbce-1d29-40cb-b8c7-075b84d49f9c @ localhost (dataPort=-1) with allocation id 6e56c476945fffb3db10c0debda0f1df 21:38:30,574 [flink-akka.actor.default-dispatcher-5] INFO org.apache.flink.runtime.taskexecutor.TaskExecutor [] - Received task Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_1), deploy into slot with allocation id 6e56c476945fffb3db10c0debda0f1df. 21:38:30,574 [flink-akka.actor.default-dispatcher-5] INFO org.apache.flink.runtime.taskexecutor.slot.TaskSlotTableImpl [] - Activate slot 6e56c476945fffb3db10c0debda0f1df. 21:38:30,574 [Source: source (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_1) switched from CREATED to DEPLOYING. 21:38:30,575 [Source: source (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Loading JAR files for task Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_1) [DEPLOYING]. 21:38:30,575 [flink-akka.actor.default-dispatcher-5] INFO org.apache.flink.runtime.taskexecutor.TaskExecutor [] - Received task Map (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_1), deploy into slot with allocation id 6e56c476945fffb3db10c0debda0f1df. 21:38:30,575 [flink-akka.actor.default-dispatcher-5] INFO org.apache.flink.runtime.taskexecutor.slot.TaskSlotTableImpl [] - Activate slot 6e56c476945fffb3db10c0debda0f1df. 21:38:30,575 [Source: source (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Registering task at network: Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_1) [DEPLOYING]. 21:38:30,575 [ Map (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Map (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_1) switched from CREATED to DEPLOYING. 21:38:30,576 [ Map (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Loading JAR files for task Map (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_1) [DEPLOYING]. 21:38:30,576 [ Map (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Registering task at network: Map (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_1) [DEPLOYING]. 21:38:30,576 [flink-akka.actor.default-dispatcher-5] INFO org.apache.flink.runtime.taskexecutor.TaskExecutor [] - Received task Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_1), deploy into slot with allocation id 6e56c476945fffb3db10c0debda0f1df. 21:38:30,576 [ Sink: Unnamed (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_1) switched from CREATED to DEPLOYING. 21:38:30,576 [ Sink: Unnamed (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Loading JAR files for task Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_1) [DEPLOYING]. 21:38:30,577 [Source: source (1/1)] INFO org.apache.flink.streaming.runtime.tasks.StreamTask [] - Using job/cluster config to configure application-defined state backend: File State Backend (checkpoints: 'file:/tmp/junit4240320968876241473/junit1530824567618577449', savepoints: 'null', asynchronous: TRUE, fileStateThreshold: 20480) 21:38:30,577 [Source: source (1/1)] INFO org.apache.flink.streaming.runtime.tasks.StreamTask [] - Using application-defined state backend: File State Backend (checkpoints: 'file:/tmp/junit4240320968876241473/junit1530824567618577449', savepoints: 'null', asynchronous: TRUE, fileStateThreshold: 20480) 21:38:30,577 [ Sink: Unnamed (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Registering task at network: Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_1) [DEPLOYING]. 21:38:30,577 [ Map (1/1)] INFO org.apache.flink.streaming.runtime.tasks.StreamTask [] - Using job/cluster config to configure application-defined state backend: File State Backend (checkpoints: 'file:/tmp/junit4240320968876241473/junit1530824567618577449', savepoints: 'null', asynchronous: TRUE, fileStateThreshold: 20480) 21:38:30,577 [ Map (1/1)] INFO org.apache.flink.streaming.runtime.tasks.StreamTask [] - Using application-defined state backend: File State Backend (checkpoints: 'file:/tmp/junit4240320968876241473/junit1530824567618577449', savepoints: 'null', asynchronous: TRUE, fileStateThreshold: 20480) 21:38:30,577 [ Sink: Unnamed (1/1)] INFO org.apache.flink.streaming.runtime.tasks.StreamTask [] - Using job/cluster config to configure application-defined state backend: File State Backend (checkpoints: 'file:/tmp/junit4240320968876241473/junit1530824567618577449', savepoints: 'null', asynchronous: TRUE, fileStateThreshold: 20480) 21:38:30,577 [ Sink: Unnamed (1/1)] INFO org.apache.flink.streaming.runtime.tasks.StreamTask [] - Using application-defined state backend: File State Backend (checkpoints: 'file:/tmp/junit4240320968876241473/junit1530824567618577449', savepoints: 'null', asynchronous: TRUE, fileStateThreshold: 20480) 21:38:30,577 [Source: source (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_1) switched from DEPLOYING to RUNNING. 21:38:30,578 [flink-akka.actor.default-dispatcher-3] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Source: source (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_bc764cd8ddf7a0cff126f51c16239658_0_1) switched from DEPLOYING to RUNNING. 21:38:30,590 [ Sink: Unnamed (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_1) switched from DEPLOYING to RUNNING. 21:38:30,590 [flink-akka.actor.default-dispatcher-4] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Sink: Unnamed (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_ea632d67b7d595e5b851708ae9ad79d6_0_1) switched from DEPLOYING to RUNNING. 21:38:30,597 [ Map (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Map (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_1) switched from DEPLOYING to RUNNING. 21:38:30,598 [flink-akka.actor.default-dispatcher-4] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Map (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_1) switched from DEPLOYING to RUNNING. 21:38:30,602 [ Sink: Unnamed (1/1)] INFO org.apache.flink.test.checkpointing.UnalignedCheckpointITCase [] - Initialized last snapshotted records [[186]] @ 0 subtask (1 attempt) 21:38:30,656 [ Checkpoint Timer] INFO org.apache.flink.runtime.checkpoint.CheckpointCoordinator [] - Triggering checkpoint 5 (type=CHECKPOINT) @ 1601588310655 for job cb2b6ee8f94a6516932dfa923bfede3f. 21:38:30,657 [Source: source (1/1)] INFO org.apache.flink.test.checkpointing.UnalignedCheckpointITCase [] - Snapshotted next input 782 @ 0 subtask (? attempt) 21:38:30,833 [ Sink: Unnamed (1/1)] INFO org.apache.flink.test.checkpointing.UnalignedCheckpointITCase [] - Last snapshotted records [[390]] @ 0 subtask (1 attempt) 21:38:30,836 [jobmanager-future-thread-29] INFO org.apache.flink.runtime.checkpoint.CheckpointCoordinator [] - Completed checkpoint 5 for job cb2b6ee8f94a6516932dfa923bfede3f (10230 bytes in 181 ms). 21:38:30,837 [SourceCoordinator-Source: source] INFO org.apache.flink.runtime.source.coordinator.SourceCoordinator [] - Marking checkpoint 5 as completed for source Source: source. 21:38:30,837 [ Checkpoint Timer] INFO org.apache.flink.runtime.checkpoint.CheckpointCoordinator [] - Triggering checkpoint 6 (type=CHECKPOINT) @ 1601588310837 for job cb2b6ee8f94a6516932dfa923bfede3f. 21:38:30,838 [Source: source (1/1)] INFO org.apache.flink.test.checkpointing.UnalignedCheckpointITCase [] - Snapshotted next input 978 @ 0 subtask (? attempt) 21:38:30,839 [ Sink: Unnamed (1/1)] INFO org.apache.flink.test.checkpointing.UnalignedCheckpointITCase [] - Last snapshotted records [[395]] @ 0 subtask (1 attempt) 21:38:30,839 [ Map (1/1)] INFO org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl [] - Could not complete snapshot 6 for operator Map (1/1). Failure reason: Checkpoint was declined. org.apache.flink.runtime.checkpoint.CheckpointException: Could not complete snapshot 6 for operator Map (1/1). Failure reason: Checkpoint was declined. at org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:216) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:157) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.AbstractStreamOperator.snapshotState(AbstractStreamOperator.java:313) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.checkpointStreamOperator(SubtaskCheckpointCoordinatorImpl.java:603) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.buildOperatorSnapshotFutures(SubtaskCheckpointCoordinatorImpl.java:529) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.takeSnapshotSync(SubtaskCheckpointCoordinatorImpl.java:496) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.checkpointState(SubtaskCheckpointCoordinatorImpl.java:266) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$performCheckpoint$8(StreamTask.java:941) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:47) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.performCheckpoint(StreamTask.java:931) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.triggerCheckpointOnBarrier(StreamTask.java:899) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointBarrierHandler.notifyCheckpoint(CheckpointBarrierHandler.java:97) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointBarrierUnaligner.processBarrier(CheckpointBarrierUnaligner.java:107) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.AlternatingCheckpointBarrierHandler.processBarrier(AlternatingCheckpointBarrierHandler.java:71) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointedInputGate.handleEvent(CheckpointedInputGate.java:129) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointedInputGate.pollNext(CheckpointedInputGate.java:121) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointedInputGate.processPriorityEvents(CheckpointedInputGate.java:87) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:47) [flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.mailbox.Mail.run(Mail.java:78) [flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.processMail(MailboxProcessor.java:301) [flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:183) [flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:594) [flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:558) [flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:722) [flink-runtime_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.runtime.taskmanager.Task.run(Task.java:547) [flink-runtime_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_242] Caused by: org.apache.flink.util.SerializedThrowable: Failing snapshotState @ 4 (1 attempt); last value 780 at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.failMapper(UnalignedCheckpointITCase.java:616) ~[test-classes/:?] at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.checkFail(UnalignedCheckpointITCase.java:611) ~[test-classes/:?] at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.snapshotState(UnalignedCheckpointITCase.java:630) ~[test-classes/:?] at org.apache.flink.streaming.util.functions.StreamingFunctionUtils.trySnapshotFunctionState(StreamingFunctionUtils.java:120) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.util.functions.StreamingFunctionUtils.snapshotFunctionState(StreamingFunctionUtils.java:101) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator.snapshotState(AbstractUdfStreamOperator.java:90) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:187) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] ... 25 more 21:38:30,839 [Channel state writer Map (1/1)] INFO org.apache.flink.runtime.checkpoint.channel.ChannelStateWriteRequestExecutorImpl [] - Map (1/1) discarding 0 drained requests 21:38:30,840 [ Map (1/1)] INFO org.apache.flink.runtime.checkpoint.channel.ChannelStateWriteRequestExecutorImpl [] - Map (1/1) discarding 1 drained requests 21:38:30,840 [ Map (1/1)] WARN org.apache.flink.runtime.taskmanager.Task [] - Map (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_1) switched from RUNNING to FAILED. java.io.IOException: Could not perform checkpoint 6 for operator Map (1/1). at org.apache.flink.streaming.runtime.tasks.StreamTask.triggerCheckpointOnBarrier(StreamTask.java:912) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointBarrierHandler.notifyCheckpoint(CheckpointBarrierHandler.java:97) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointBarrierUnaligner.processBarrier(CheckpointBarrierUnaligner.java:107) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.AlternatingCheckpointBarrierHandler.processBarrier(AlternatingCheckpointBarrierHandler.java:71) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointedInputGate.handleEvent(CheckpointedInputGate.java:129) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointedInputGate.pollNext(CheckpointedInputGate.java:121) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointedInputGate.processPriorityEvents(CheckpointedInputGate.java:87) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:47) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.mailbox.Mail.run(Mail.java:78) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.processMail(MailboxProcessor.java:301) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:183) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:594) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:558) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:722) [flink-runtime_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.runtime.taskmanager.Task.run(Task.java:547) [flink-runtime_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_242] Caused by: org.apache.flink.runtime.checkpoint.CheckpointException: Could not complete snapshot 6 for operator Map (1/1). Failure reason: Checkpoint was declined. at org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:216) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:157) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.AbstractStreamOperator.snapshotState(AbstractStreamOperator.java:313) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.checkpointStreamOperator(SubtaskCheckpointCoordinatorImpl.java:603) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.buildOperatorSnapshotFutures(SubtaskCheckpointCoordinatorImpl.java:529) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.takeSnapshotSync(SubtaskCheckpointCoordinatorImpl.java:496) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.checkpointState(SubtaskCheckpointCoordinatorImpl.java:266) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$performCheckpoint$8(StreamTask.java:941) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:47) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.performCheckpoint(StreamTask.java:931) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.triggerCheckpointOnBarrier(StreamTask.java:899) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] ... 15 more Caused by: org.apache.flink.util.SerializedThrowable: Failing snapshotState @ 4 (1 attempt); last value 780 at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.failMapper(UnalignedCheckpointITCase.java:616) ~[test-classes/:?] at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.checkFail(UnalignedCheckpointITCase.java:611) ~[test-classes/:?] at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.snapshotState(UnalignedCheckpointITCase.java:630) ~[test-classes/:?] at org.apache.flink.streaming.util.functions.StreamingFunctionUtils.trySnapshotFunctionState(StreamingFunctionUtils.java:120) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.util.functions.StreamingFunctionUtils.snapshotFunctionState(StreamingFunctionUtils.java:101) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator.snapshotState(AbstractUdfStreamOperator.java:90) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:187) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:157) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.AbstractStreamOperator.snapshotState(AbstractStreamOperator.java:313) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.checkpointStreamOperator(SubtaskCheckpointCoordinatorImpl.java:603) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.buildOperatorSnapshotFutures(SubtaskCheckpointCoordinatorImpl.java:529) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.takeSnapshotSync(SubtaskCheckpointCoordinatorImpl.java:496) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.checkpointState(SubtaskCheckpointCoordinatorImpl.java:266) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$performCheckpoint$8(StreamTask.java:941) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:47) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.performCheckpoint(StreamTask.java:931) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.triggerCheckpointOnBarrier(StreamTask.java:899) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] ... 15 more 21:38:30,840 [ Map (1/1)] INFO org.apache.flink.runtime.taskmanager.Task [] - Freeing task resources for Map (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_1). 21:38:30,841 [flink-akka.actor.default-dispatcher-4] INFO org.apache.flink.runtime.taskexecutor.TaskExecutor [] - Un-registering task and sending final execution state FAILED to JobManager for task Map (1/1) cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_1. 21:38:30,872 [flink-akka.actor.default-dispatcher-5] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Map (1/1) (cb2b6ee8f94a6516932dfa923bfede3f_0a448493b4782967b150582570326227_0_1) switched from RUNNING to FAILED on org.apache.flink.runtime.jobmaster.slotpool.SingleLogicalSlot@6e989a5d. java.io.IOException: Could not perform checkpoint 6 for operator Map (1/1). at org.apache.flink.streaming.runtime.tasks.StreamTask.triggerCheckpointOnBarrier(StreamTask.java:912) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointBarrierHandler.notifyCheckpoint(CheckpointBarrierHandler.java:97) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointBarrierUnaligner.processBarrier(CheckpointBarrierUnaligner.java:107) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.AlternatingCheckpointBarrierHandler.processBarrier(AlternatingCheckpointBarrierHandler.java:71) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointedInputGate.handleEvent(CheckpointedInputGate.java:129) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointedInputGate.pollNext(CheckpointedInputGate.java:121) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.io.CheckpointedInputGate.processPriorityEvents(CheckpointedInputGate.java:87) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:47) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.mailbox.Mail.run(Mail.java:78) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.processMail(MailboxProcessor.java:301) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:183) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:594) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:558) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:722) ~[flink-runtime_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.runtime.taskmanager.Task.run(Task.java:547) ~[flink-runtime_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_242] Caused by: org.apache.flink.runtime.checkpoint.CheckpointException: Could not complete snapshot 6 for operator Map (1/1). Failure reason: Checkpoint was declined. at org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:216) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:157) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.AbstractStreamOperator.snapshotState(AbstractStreamOperator.java:313) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.checkpointStreamOperator(SubtaskCheckpointCoordinatorImpl.java:603) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.buildOperatorSnapshotFutures(SubtaskCheckpointCoordinatorImpl.java:529) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.takeSnapshotSync(SubtaskCheckpointCoordinatorImpl.java:496) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.checkpointState(SubtaskCheckpointCoordinatorImpl.java:266) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$performCheckpoint$8(StreamTask.java:941) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:47) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.performCheckpoint(StreamTask.java:931) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.triggerCheckpointOnBarrier(StreamTask.java:899) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] ... 15 more Caused by: org.apache.flink.util.SerializedThrowable: Failing snapshotState @ 4 (1 attempt); last value 780 at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.failMapper(UnalignedCheckpointITCase.java:616) ~[test-classes/:?] at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.checkFail(UnalignedCheckpointITCase.java:611) ~[test-classes/:?] at org.apache.flink.test.checkpointing.UnalignedCheckpointITCase$FailingMapper.snapshotState(UnalignedCheckpointITCase.java:630) ~[test-classes/:?] at org.apache.flink.streaming.util.functions.StreamingFunctionUtils.trySnapshotFunctionState(StreamingFunctionUtils.java:120) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.util.functions.StreamingFunctionUtils.snapshotFunctionState(StreamingFunctionUtils.java:101) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator.snapshotState(AbstractUdfStreamOperator.java:90) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:187) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:157) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.api.operators.AbstractStreamOperator.snapshotState(AbstractStreamOperator.java:313) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.checkpointStreamOperator(SubtaskCheckpointCoordinatorImpl.java:603) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.buildOperatorSnapshotFutures(SubtaskCheckpointCoordinatorImpl.java:529) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.takeSnapshotSync(SubtaskCheckpointCoordinatorImpl.java:496) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.checkpointState(SubtaskCheckpointCoordinatorImpl.java:266) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$performCheckpoint$8(StreamTask.java:941) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:47) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.performCheckpoint(StreamTask.java:931) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] at org.apache.flink.streaming.runtime.tasks.StreamTask.triggerCheckpointOnBarrier(StreamTask.java:899) ~[flink-streaming-java_2.11-1.12-SNAPSHOT.jar:1.12-SNAPSHOT] ... 15 more {code} > UnalignedCheckpointITCase.shouldPerformUnalignedCheckpointOnParallelRemoteChannel > failed because of test timeout > ---------------------------------------------------------------------------------------------------------------- > > Key: FLINK-19027 > URL: https://issues.apache.org/jira/browse/FLINK-19027 > Project: Flink > Issue Type: Bug > Components: Runtime / Checkpointing > Affects Versions: 1.11.2 > Reporter: Dian Fu > Priority: Major > Labels: test-stability > Fix For: 1.12.0, 1.11.3 > > > [https://dev.azure.com/apache-flink/apache-flink/_build/results?buildId=5789&view=logs&j=119bbba7-f5e3-5e08-e72d-09f1529665de&t=ec103906-d047-5b8a-680e-05fc000dfca9] > {code} > 2020-08-22T21:13:05.5315459Z [ERROR] > shouldPerformUnalignedCheckpointOnParallelRemoteChannel(org.apache.flink.test.checkpointing.UnalignedCheckpointITCase) > Time elapsed: 300.075 s <<< ERROR! > 2020-08-22T21:13:05.5316451Z org.junit.runners.model.TestTimedOutException: > test timed out after 300 seconds > 2020-08-22T21:13:05.5317432Z at sun.misc.Unsafe.park(Native Method) > 2020-08-22T21:13:05.5317799Z at > java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) > 2020-08-22T21:13:05.5318247Z at > java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1707) > 2020-08-22T21:13:05.5318885Z at > java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3323) > 2020-08-22T21:13:05.5327035Z at > java.util.concurrent.CompletableFuture.waitingGet(CompletableFuture.java:1742) > 2020-08-22T21:13:05.5328114Z at > java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1908) > 2020-08-22T21:13:05.5328869Z at > org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1719) > 2020-08-22T21:13:05.5329482Z at > org.apache.flink.streaming.api.environment.LocalStreamEnvironment.execute(LocalStreamEnvironment.java:74) > 2020-08-22T21:13:05.5330138Z at > org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1699) > 2020-08-22T21:13:05.5330771Z at > org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1681) > 2020-08-22T21:13:05.5331351Z at > org.apache.flink.test.checkpointing.UnalignedCheckpointITCase.execute(UnalignedCheckpointITCase.java:158) > 2020-08-22T21:13:05.5332015Z at > org.apache.flink.test.checkpointing.UnalignedCheckpointITCase.shouldPerformUnalignedCheckpointOnParallelRemoteChannel(UnalignedCheckpointITCase.java:140) > {code} -- This message was sent by Atlassian Jira (v8.3.4#803005)