Hi Parag, it seems that you are submitting a job with the same job id multiple times. An easy fix would be generating a new job id each time you are submitting the job.
To debug this: check out the Flink jobmanager logs, there are log messages for every job submission. On Thu, Jan 27, 2022 at 9:16 AM Parag Somani <somanipa...@gmail.com> wrote: > Hello All, > > While deploying on our one of environment, we encountered crashloopback of > job manager pod. > Env: K8s > Flink: 1.14.2 > > Could you suggest, how can we troubleshoot this and possible handling of > this? > > > exception snipper as follows: > > 2022-01-27 06:58:07.326 ERROR 44 --- [lt-dispatcher-4] > c.b.a.his.service.FlinkExecutorService : Failed to execute job > > org.apache.flink.util.FlinkException: Failed to execute job 'events rates > calculation'. > at > org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.executeAsync(StreamExecutionEnvironment.java:2056) > ~[flink-streaming-java_2.12-1.14.2.jar:1.14.2] > at > org.apache.flink.client.program.StreamContextEnvironment.executeAsync(StreamContextEnvironment.java:137) > ~[flink-clients_2.12-1.14.2.jar:1.14.2] > at > org.apache.flink.client.program.StreamContextEnvironment.execute(StreamContextEnvironment.java:76) > ~[flink-clients_2.12-1.14.2.jar:1.14.2] > at > org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1917) > ~[flink-streaming-java_2.12-1.14.2.jar:1.14.2] > at > com.bmc.ade.his.service.FlinkExecutorService.init(FlinkExecutorService.java:37) > ~[health-service-1.0.00.jar:1.0.00] > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native > Method) ~[na:na] > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[na:na] > at > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[na:na] > at java.base/java.lang.reflect.Method.invoke(Method.java:566) > ~[na:na] > at > org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleElement.invoke(InitDestroyAnnotationBeanPostProcessor.java:389) > ~[spring-beans-5.3.10.jar:5.3.10] > at > org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleMetadata.invokeInitMethods(InitDestroyAnnotationBeanPostProcessor.java:333) > ~[spring-beans-5.3.10.jar:5.3.10] > at > org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor.postProcessBeforeInitialization(InitDestroyAnnotationBeanPostProcessor.java:157) > ~[spring-beans-5.3.10.jar:5.3.10] > at > org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.applyBeanPostProcessorsBeforeInitialization(AbstractAutowireCapableBeanFactory.java:440) > ~[spring-beans-5.3.10.jar:5.3.10] > at > org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1796) > ~[spring-beans-5.3.10.jar:5.3.10] > at > org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:620) > ~[spring-beans-5.3.10.jar:5.3.10] > at > org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:542) > ~[spring-beans-5.3.10.jar:5.3.10] > at > org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:335) > ~[spring-beans-5.3.10.jar:5.3.10] > at > org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:234) > ~[spring-beans-5.3.10.jar:5.3.10] > at > org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:333) > ~[spring-beans-5.3.10.jar:5.3.10] > at > org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:208) > ~[spring-beans-5.3.10.jar:5.3.10] > at > org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:944) > ~[spring-beans-5.3.10.jar:5.3.10] > at > org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:918) > ~[spring-context-5.3.10.jar:5.3.10] > at > org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:583) > ~[spring-context-5.3.10.jar:5.3.10] > at > org.springframework.boot.SpringApplication.refresh(SpringApplication.java:754) > ~[spring-boot-2.5.5.jar:2.5.5] > at > org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:434) > ~[spring-boot-2.5.5.jar:2.5.5] > at > org.springframework.boot.SpringApplication.run(SpringApplication.java:338) > ~[spring-boot-2.5.5.jar:2.5.5] > at > org.springframework.boot.SpringApplication.run(SpringApplication.java:1343) > ~[spring-boot-2.5.5.jar:2.5.5] > at > org.springframework.boot.SpringApplication.run(SpringApplication.java:1332) > ~[spring-boot-2.5.5.jar:2.5.5] > at > com.bmc.ade.his.HealthAndImpactServiceApplication.main(HealthAndImpactServiceApplication.java:19) > ~[health-service-1.0.00.jar:1.0.00] > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native > Method) ~[na:na] > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[na:na] > at > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[na:na] > at java.base/java.lang.reflect.Method.invoke(Method.java:566) > ~[na:na] > at > org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:355) > ~[flink-clients_2.12-1.14.2.jar:1.14.2] > at > org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:222) > ~[flink-clients_2.12-1.14.2.jar:1.14.2] > at > org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:114) > ~[flink-clients_2.12-1.14.2.jar:1.14.2] > at > org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:253) > ~[flink-clients_2.12-1.14.2.jar:1.14.2] > at > org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.lambda$runApplicationAsync$1(ApplicationDispatcherBootstrap.java:216) > ~[flink-clients_2.12-1.14.2.jar:1.14.2] > at > java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) > ~[na:na] > at > java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[na:na] > at > org.apache.flink.runtime.concurrent.akka.ActorSystemScheduledExecutorAdapter$ScheduledFutureTask.run(ActorSystemScheduledExecutorAdapter.java:171) > ~[na:na] > at > org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68) > ~[na:na] > at > org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.lambda$withContextClassLoader$0(ClassLoadingUtils.java:41) > ~[na:na] > at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:49) > ~[na:na] > at > akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:48) > ~[na:na] > at > java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) > ~[na:na] > at > java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) > ~[na:na] > at > java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) > ~[na:na] > at > java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) > ~[na:na] > at > java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183) > ~[na:na] > Caused by: > org.apache.flink.runtime.client.DuplicateJobSubmissionException: Job has > already been submitted. > at > org.apache.flink.runtime.client.DuplicateJobSubmissionException.ofGloballyTerminated(DuplicateJobSubmissionException.java:33) > ~[flink-runtime-1.14.2.jar:1.14.2] > at > org.apache.flink.runtime.dispatcher.Dispatcher.submitJob(Dispatcher.java:307) > ~[flink-runtime-1.14.2.jar:1.14.2] > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native > Method) ~[na:na] > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[na:na] > at > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[na:na] > at java.base/java.lang.reflect.Method.invoke(Method.java:566) > ~[na:na] > at > org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRpcInvocation$1(AkkaRpcActor.java:316) > ~[na:na] > at > org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:83) > ~[na:na] > at > org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcInvocation(AkkaRpcActor.java:314) > ~[na:na] > at > org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:217) > ~[na:na] > at > org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78) > ~[na:na] > at > org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:163) > ~[na:na] > at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24) > ~[na:na] > at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20) > ~[na:na] > at scala.PartialFunction.applyOrElse(PartialFunction.scala:123) > ~[scala-library-2.12.10.jar:na] > at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122) > ~[scala-library-2.12.10.jar:na] > at > akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20) ~[na:na] > at > scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171) > ~[scala-library-2.12.10.jar:na] > at > scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) > ~[scala-library-2.12.10.jar:na] > at > scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) > ~[scala-library-2.12.10.jar:na] > at akka.actor.Actor.aroundReceive(Actor.scala:537) ~[na:na] > at akka.actor.Actor.aroundReceive$(Actor.scala:535) ~[na:na] > at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220) > ~[na:na] > at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580) > ~[na:na] > at akka.actor.ActorCell.invoke(ActorCell.scala:548) ~[na:na] > at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270) ~[na:na] > at akka.dispatch.Mailbox.run(Mailbox.scala:231) ~[na:na] > at akka.dispatch.Mailbox.exec(Mailbox.scala:243) ~[na:na] > ... 5 common frames omitted > > Currently I have deleted its state in ZK and restart solved the issue. > > Please suggest. > > Thank you in advance...! > > -- > Regards, > Parag Surajmal Somani. >
