Hello Experts Seeing below exceptions thrown by the spark driver every few hours. Using spark 3.3.0
com.fasterxml.jackson.databind.JsonMappingException.wrapWithPath(JsonMappingException.java:392 Caused by: com.fasterxml.jackson.databind.JsonMappingException: timeout (through reference chain: io.fabric8.kubernetes.api.model.PodList["items"]->java.util.ArrayList[0]- More detailed stack trace Uncaught exception in thread kubernetes-executor-pod-polling-sync io.fabric8.kubernetes.client.KubernetesClientException: An error has occurred. at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:103) ~[kubernetes-client-5.12.2.jar:?] at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:97) ~[kubernetes-client-5.12.2.jar:?] at io.fabric8.kubernetes.client.utils.Serialization.unmarshal(Serialization.java:319) ~[kubernetes-client-5.12.2.jar:?] at io.fabric8.kubernetes.client.utils.Serialization.unmarshal(Serialization.java:271) ~[kubernetes-client-5.12.2.jar:?] at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:557) ~[kubernetes-client-5.12.2.jar:?] at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:518) ~[kubernetes-client-5.12.2.jar:?] at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:502) ~[kubernetes-client-5.12.2.jar:?] at io.fabric8.kubernetes.client.dsl.base.BaseOperation.listRequestHelper(BaseOperation.java:133) ~[kubernetes-client-5.12.2.jar:?] at io.fabric8.kubernetes.client.dsl.base.BaseOperation.list(BaseOperation.java:415) ~[kubernetes-client-5.12.2.jar:?] at io.fabric8.kubernetes.client.dsl.base.BaseOperation.list(BaseOperation.java:404) ~[kubernetes-client-5.12.2.jar:?] at io.fabric8.kubernetes.client.dsl.base.BaseOperation.list(BaseOperation.java:83) ~[kubernetes-client-5.12.2.jar:?] at org.apache.spark.scheduler.cluster.k8s.ExecutorPodsPollingSnapshotSource$PollRunnable.$anonfun$run$1(ExecutorPodsPollingSnapshotSource.scala:88) ~[spark-kubernetes_2.12-3.3.0.jar:3.3.0] at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1484) ~[spark-core_2.12-3.3.0.jar:3.3.0] at org.apache.spark.scheduler.cluster.k8s.ExecutorPodsPollingSnapshotSource$PollRunnable.run(ExecutorPodsPollingSnapshotSource.scala:71) ~[spark-kubernetes_2.12-3.3.0.jar:3.3.0] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_342] at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) ~[?:1.8.0_342] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_342] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) ~[?:1.8.0_342] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_342] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_342] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_342] Caused by: com.fasterxml.jackson.databind.JsonMappingException: timeout (through reference chain: io.fabric8.kubernetes.api.model.PodList["items"]->java.util.ArrayList[0]->io.fabric8.kubernetes.api.model.Pod["metadata"]->io.fabric8.kubernetes.api.model.ObjectMeta["managedFields"]->java.util.ArrayList[0]) at com.fasterxml.jackson.databind.JsonMappingException.wrapWithPath(JsonMappingException.java:392) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.JsonMappingException.wrapWithPath(JsonMappingException.java:363) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer._deserializeFromArray(CollectionDeserializer.java:371) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:244) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:28) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:129) ~[jackson-databind-2.13.3.jar:2.13.3] at io.fabric8.kubernetes.client.utils.serialization.SettableBeanPropertyDelegate.deserializeAndSet(SettableBeanPropertyDelegate.java:131) ~[kubernetes-client-5.12.2.jar:?] at com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:313) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:176) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:129) ~[jackson-databind-2.13.3.jar:2.13.3] at io.fabric8.kubernetes.client.utils.serialization.SettableBeanPropertyDelegate.deserializeAndSet(SettableBeanPropertyDelegate.java:131) ~[kubernetes-client-5.12.2.jar:?] at com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:313) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:176) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer._deserializeFromArray(CollectionDeserializer.java:355) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:244) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:28) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:129) ~[jackson-databind-2.13.3.jar:2.13.3] at io.fabric8.kubernetes.client.utils.serialization.SettableBeanPropertyDelegate.deserializeAndSet(SettableBeanPropertyDelegate.java:131) ~[kubernetes-client-5.12.2.jar:?] at com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:313) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:176) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.DefaultDeserializationContext.readRootValue(DefaultDeserializationContext.java:323) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4674) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3674) ~[jackson-databind-2.13.3.jar:2.13.3] at io.fabric8.kubernetes.client.utils.Serialization.unmarshal(Serialization.java:317) ~[kubernetes-client-5.12.2.jar:?] ... 18 more Caused by: java.net.SocketTimeoutException: timeout at okio.Okio$4.newTimeoutException(Okio.java:232) ~[okio-1.14.0.jar:?] at okio.AsyncTimeout.exit(AsyncTimeout.java:275) ~[okio-1.14.0.jar:?] at okio.AsyncTimeout$2.read(AsyncTimeout.java:243) ~[okio-1.14.0.jar:?] at okio.RealBufferedSource.read(RealBufferedSource.java:47) ~[okio-1.14.0.jar:?] at okhttp3.internal.http1.Http1Codec$AbstractSource.read(Http1Codec.java:363) ~[okhttp-3.12.12.jar:?] at okhttp3.internal.http1.Http1Codec$ChunkedSource.read(Http1Codec.java:453) ~[okhttp-3.12.12.jar:?] at okio.RealBufferedSource$1.read(RealBufferedSource.java:440) ~[okio-1.14.0.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) ~[?:1.8.0_342] at java.io.BufferedInputStream.read1(BufferedInputStream.java:286) ~[?:1.8.0_342] at java.io.BufferedInputStream.read(BufferedInputStream.java:345) ~[?:1.8.0_342] at com.fasterxml.jackson.core.json.UTF8StreamJsonParser._loadMore(UTF8StreamJsonParser.java:257) ~[jackson-core-2.13.3.jar:2.13.3] at com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseEscapedName(UTF8StreamJsonParser.java:2054) ~[jackson-core-2.13.3.jar:2.13.3] at com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseLongName(UTF8StreamJsonParser.java:1945) ~[jackson-core-2.13.3.jar:2.13.3] at com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseMediumName2(UTF8StreamJsonParser.java:1883) ~[jackson-core-2.13.3.jar:2.13.3] at com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseMediumName(UTF8StreamJsonParser.java:1842) ~[jackson-core-2.13.3.jar:2.13.3] at com.fasterxml.jackson.core.json.UTF8StreamJsonParser._parseName(UTF8StreamJsonParser.java:1777) ~[jackson-core-2.13.3.jar:2.13.3] at com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextFieldName(UTF8StreamJsonParser.java:1075) ~[jackson-core-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer$Vanilla.mapObject(UntypedObjectDeserializer.java:995) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer$Vanilla.deserialize(UntypedObjectDeserializer.java:718) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer$Vanilla.mapObject(UntypedObjectDeserializer.java:958) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer$Vanilla.deserialize(UntypedObjectDeserializer.java:718) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer$Vanilla.mapObject(UntypedObjectDeserializer.java:958) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer$Vanilla.deserialize(UntypedObjectDeserializer.java:718) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer$Vanilla.deserialize(UntypedObjectDeserializer.java:701) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.SettableAnyProperty.deserialize(SettableAnyProperty.java:153) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.SettableAnyProperty.deserializeAndSet(SettableAnyProperty.java:134) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownVanilla(BeanDeserializerBase.java:1672) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:319) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:176) ~[jackson-databind-2.13.3.jar:2.13.3] at com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:129) ~[jackson-databind-2.13.3.jar:2.13.3] at io.fabric8.kubernetes.client.utils.serialization.SettableBeanPropertyDelegate.deserializeAndSet(SettableBeanPropertyDelegate.java:131) ~[kubernetes-client-5.12.2.jar:?]