Hello Experts
Seeing below exceptions thrown by the spark driver every few hours. Using
spark 3.3.0
com.fasterxml.jackson.databind.JsonMappingException.wrapWithPath(JsonMappingException.java:392
Caused by: com.fasterxml.jackson.databind.JsonMappingException:
timeout (through reference chain:
io.fabric8.kubernetes.api.model.PodList["items"]->java.util.ArrayList[0]-
More detailed stack trace
Uncaught exception in thread kubernetes-executor-pod-polling-sync
io.fabric8.kubernetes.client.KubernetesClientException: An error has occurred.
at
io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:103)
~[kubernetes-client-5.12.2.jar:?]
at
io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:97)
~[kubernetes-client-5.12.2.jar:?]
at
io.fabric8.kubernetes.client.utils.Serialization.unmarshal(Serialization.java:319)
~[kubernetes-client-5.12.2.jar:?]
at
io.fabric8.kubernetes.client.utils.Serialization.unmarshal(Serialization.java:271)
~[kubernetes-client-5.12.2.jar:?]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:557)
~[kubernetes-client-5.12.2.jar:?]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:518)
~[kubernetes-client-5.12.2.jar:?]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:502)
~[kubernetes-client-5.12.2.jar:?]
at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.listRequestHelper(BaseOperation.java:133)
~[kubernetes-client-5.12.2.jar:?]
at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.list(BaseOperation.java:415)
~[kubernetes-client-5.12.2.jar:?]
at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.list(BaseOperation.java:404)
~[kubernetes-client-5.12.2.jar:?]
at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.list(BaseOperation.java:83)
~[kubernetes-client-5.12.2.jar:?]
at
org.apache.spark.scheduler.cluster.k8s.ExecutorPodsPollingSnapshotSource$PollRunnable.$anonfun$run$1(ExecutorPodsPollingSnapshotSource.scala:88)
~[spark-kubernetes_2.12-3.3.0.jar:3.3.0]
at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1484)
~[spark-core_2.12-3.3.0.jar:3.3.0]
at
org.apache.spark.scheduler.cluster.k8s.ExecutorPodsPollingSnapshotSource$PollRunnable.run(ExecutorPodsPollingSnapshotSource.scala:71)
~[spark-kubernetes_2.12-3.3.0.jar:3.3.0]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[?:1.8.0_342]
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
~[?:1.8.0_342]
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
~[?:1.8.0_342]
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
~[?:1.8.0_342]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
~[?:1.8.0_342]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
~[?:1.8.0_342]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_342]
Caused by: com.fasterxml.jackson.databind.JsonMappingException:
timeout (through reference chain:
io.fabric8.kubernetes.api.model.PodList["items"]->java.util.ArrayList[0]->io.fabric8.kubernetes.api.model.Pod["metadata"]->io.fabric8.kubernetes.api.model.ObjectMeta["managedFields"]->java.util.ArrayList[0])
at
com.fasterxml.jackson.databind.JsonMappingException.wrapWithPath(JsonMappingException.java:392)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.JsonMappingException.wrapWithPath(JsonMappingException.java:363)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.std.CollectionDeserializer._deserializeFromArray(CollectionDeserializer.java:371)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:244)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:28)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:129)
~[jackson-databind-2.13.3.jar:2.13.3]
at
io.fabric8.kubernetes.client.utils.serialization.SettableBeanPropertyDelegate.deserializeAndSet(SettableBeanPropertyDelegate.java:131)
~[kubernetes-client-5.12.2.jar:?]
at
com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:313)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:176)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:129)
~[jackson-databind-2.13.3.jar:2.13.3]
at
io.fabric8.kubernetes.client.utils.serialization.SettableBeanPropertyDelegate.deserializeAndSet(SettableBeanPropertyDelegate.java:131)
~[kubernetes-client-5.12.2.jar:?]
at
com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:313)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:176)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.std.CollectionDeserializer._deserializeFromArray(CollectionDeserializer.java:355)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:244)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:28)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:129)
~[jackson-databind-2.13.3.jar:2.13.3]
at
io.fabric8.kubernetes.client.utils.serialization.SettableBeanPropertyDelegate.deserializeAndSet(SettableBeanPropertyDelegate.java:131)
~[kubernetes-client-5.12.2.jar:?]
at
com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:313)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:176)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.DefaultDeserializationContext.readRootValue(DefaultDeserializationContext.java:323)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4674)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3674)
~[jackson-databind-2.13.3.jar:2.13.3]
at
io.fabric8.kubernetes.client.utils.Serialization.unmarshal(Serialization.java:317)
~[kubernetes-client-5.12.2.jar:?]
... 18 more
Caused by: java.net.SocketTimeoutException: timeout
at okio.Okio$4.newTimeoutException(Okio.java:232) ~[okio-1.14.0.jar:?]
at okio.AsyncTimeout.exit(AsyncTimeout.java:275) ~[okio-1.14.0.jar:?]
at okio.AsyncTimeout$2.read(AsyncTimeout.java:243) ~[okio-1.14.0.jar:?]
at okio.RealBufferedSource.read(RealBufferedSource.java:47)
~[okio-1.14.0.jar:?]
at
okhttp3.internal.http1.Http1Codec$AbstractSource.read(Http1Codec.java:363)
~[okhttp-3.12.12.jar:?]
at
okhttp3.internal.http1.Http1Codec$ChunkedSource.read(Http1Codec.java:453)
~[okhttp-3.12.12.jar:?]
at okio.RealBufferedSource$1.read(RealBufferedSource.java:440)
~[okio-1.14.0.jar:?]
at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
~[?:1.8.0_342]
at java.io.BufferedInputStream.read1(BufferedInputStream.java:286)
~[?:1.8.0_342]
at java.io.BufferedInputStream.read(BufferedInputStream.java:345)
~[?:1.8.0_342]
at
com.fasterxml.jackson.core.json.UTF8StreamJsonParser._loadMore(UTF8StreamJsonParser.java:257)
~[jackson-core-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseEscapedName(UTF8StreamJsonParser.java:2054)
~[jackson-core-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseLongName(UTF8StreamJsonParser.java:1945)
~[jackson-core-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseMediumName2(UTF8StreamJsonParser.java:1883)
~[jackson-core-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseMediumName(UTF8StreamJsonParser.java:1842)
~[jackson-core-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.core.json.UTF8StreamJsonParser._parseName(UTF8StreamJsonParser.java:1777)
~[jackson-core-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextFieldName(UTF8StreamJsonParser.java:1075)
~[jackson-core-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer$Vanilla.mapObject(UntypedObjectDeserializer.java:995)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer$Vanilla.deserialize(UntypedObjectDeserializer.java:718)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer$Vanilla.mapObject(UntypedObjectDeserializer.java:958)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer$Vanilla.deserialize(UntypedObjectDeserializer.java:718)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer$Vanilla.mapObject(UntypedObjectDeserializer.java:958)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer$Vanilla.deserialize(UntypedObjectDeserializer.java:718)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer$Vanilla.deserialize(UntypedObjectDeserializer.java:701)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.SettableAnyProperty.deserialize(SettableAnyProperty.java:153)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.SettableAnyProperty.deserializeAndSet(SettableAnyProperty.java:134)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownVanilla(BeanDeserializerBase.java:1672)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:319)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:176)
~[jackson-databind-2.13.3.jar:2.13.3]
at
com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:129)
~[jackson-databind-2.13.3.jar:2.13.3]
at
io.fabric8.kubernetes.client.utils.serialization.SettableBeanPropertyDelegate.deserializeAndSet(SettableBeanPropertyDelegate.java:131)
~[kubernetes-client-5.12.2.jar:?]
