Hey,
I upgraded today my Flink application to Flink 1.14.3.
I run it in a HA-Standalone-K8 deployment with 2 JobManagers, so one active and one on standby.
As its only a prototyp I make the UI, port 8081 of the JobManager pods, avaiable via NodePort.
Already with older Flink version I sometimes got some problems within the UI telling "Internal server error.".
With the new 1.14.3 its really often.
It seem, that depending on to which pod the traffic is load balanced, by the NodePort, the UI works or has the problems.
Right now, I dont know which has trouble, but due to the other issues about that, I assume its also the stand-by one.
Here is the exception from the JobManager log, fetched from the UI, from the time the errors happend (so from the erroneous JobManager).
2022-01-20 12:01:11,829 ERROR org.apache.flink.runtime.rest.handler.job.JobsOverviewHandler [] - Unhandled exception.
org.apache.flink.runtime.rpc.akka.exceptions.AkkaRpcException: Failed to serialize the result for RPC call : requestMultipleJobDetails.
at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.serializeRemoteResultAndVerifySize(AkkaRpcActor.java:417) ~[?:?]
at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$sendAsyncResponse$2(AkkaRpcActor.java:373) ~[?:?]
at java.util.concurrent.CompletableFuture.uniHandle(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture$UniHandle.tryFire(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]
at org.apache.flink.util.concurrent.FutureUtils$ResultConjunctFuture.handleCompletedFuture(FutureUtils.java:858) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at org.apache.flink.util.concurrent.FutureUtils$ResultConjunctFuture.lambda$new$0(FutureUtils.java:876) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]
at org.apache.flink.runtime.rpc.akka.AkkaInvocationHandler.lambda$invokeRpc$1(AkkaInvocationHandler.java:258) ~[?:?]
at java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]
at org.apache.flink.util.concurrent.FutureUtils.doForward(FutureUtils.java:1389) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.lambda$null$1(ClassLoadingUtils.java:93) ~[?:?]
at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68) ~[?:?]
at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.lambda$guardCompletionWithContextClassLoader$2(ClassLoadingUtils.java:92) ~[?:?]
at java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]
at org.apache.flink.runtime.concurrent.akka.AkkaFutureUtils$1.onComplete(AkkaFutureUtils.java:47) ~[?:?]
at akka.dispatch.OnComplete.internal(Future.scala:300) ~[?:?]
at akka.dispatch.OnComplete.internal(Future.scala:297) ~[?:?]
at akka.dispatch.japi$CallbackBridge.apply(Future.scala:224) ~[?:?]
at akka.dispatch.japi$CallbackBridge.apply(Future.scala:221) ~[?:?]
at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at org.apache.flink.runtime.concurrent.akka.AkkaFutureUtils$DirectExecutionContext.execute(AkkaFutureUtils.java:65) ~[?:?]
at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:68) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1(Promise.scala:284) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1$adapted(Promise.scala:284) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:621) ~[?:?]
at akka.pattern.PipeToSupport$PipeableFuture$$anonfun$pipeTo$1.applyOrElse(PipeToSupport.scala:24) ~[?:?]
at akka.pattern.PipeToSupport$PipeableFuture$$anonfun$pipeTo$1.applyOrElse(PipeToSupport.scala:23) ~[?:?]
at scala.concurrent.Future.$anonfun$andThen$1(Future.scala:532) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:29) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:29) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:63) ~[?:?]
at akka.dispatch.BatchingExecutor$BlockableBatch.$anonfun$run$1(BatchingExecutor.scala:100) ~[?:?]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:81) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at akka.dispatch.BatchingExecutor$BlockableBatch.run(BatchingExecutor.scala:100) ~[?:?]
at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:49) ~[?:?]
at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:48) ~[?:?]
at java.util.concurrent.ForkJoinTask.doExec(Unknown Source) [?:?]
at java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(Unknown Source) [?:?]
at java.util.concurrent.ForkJoinPool.scan(Unknown Source) [?:?]
at java.util.concurrent.ForkJoinPool.runWorker(Unknown Source) [?:?]
at java.util.concurrent.ForkJoinWorkerThread.run(Unknown Source) [?:?]
Caused by: java.io.NotSerializableException: java.util.HashMap$Values
at java.io.ObjectOutputStream.writeObject0(Unknown Source) ~[?:?]
at java.io.ObjectOutputStream.defaultWriteFields(Unknown Source) ~[?:?]
at java.io.ObjectOutputStream.writeSerialData(Unknown Source) ~[?:?]
at java.io.ObjectOutputStream.writeOrdinaryObject(Unknown Source) ~[?:?]
at java.io.ObjectOutputStream.writeObject0(Unknown Source) ~[?:?]
at java.io.ObjectOutputStream.writeObject(Unknown Source) ~[?:?]
at org.apache.flink.util.InstantiationUtil.serializeObject(InstantiationUtil.java:632) ~[flink-dist_2.12-1.14.3.jar:1.14.3]
at org.apache.flink.runtime.rpc.akka.AkkaRpcSerializedValue.valueOf(AkkaRpcSerializedValue.java:66) ~[?:?]
at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.serializeRemoteResultAndVerifySize(AkkaRpcActor.java:400) ~[?:?]
... 54 more
