Ethan Guo created HUDI-3638:
-------------------------------
Summary: Task not serializable due to ZookeeperBasedLockProvider
in clean action
Key: HUDI-3638
URL: https://issues.apache.org/jira/browse/HUDI-3638
Project: Apache Hudi
Issue Type: Task
Reporter: Ethan Guo
{code:java}
22/03/14 10:58:05 WARN CleanActionExecutor: Failed to perform previous clean
operation, instant: [==>20220314105144254__clean__REQUESTED]
org.apache.spark.SparkException: Task not serializable
at
org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:416)
at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:406)
at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:162)
at org.apache.spark.SparkContext.clean(SparkContext.scala:2477)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitions$1(RDD.scala:860)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:414)
at org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:859)
at
org.apache.spark.api.java.JavaRDDLike.mapPartitionsToPair(JavaRDDLike.scala:188)
at
org.apache.spark.api.java.JavaRDDLike.mapPartitionsToPair$(JavaRDDLike.scala:183)
at
org.apache.spark.api.java.AbstractJavaRDDLike.mapPartitionsToPair(JavaRDDLike.scala:45)
at
org.apache.hudi.client.common.HoodieSparkEngineContext.mapPartitionsToPairAndReduceByKey(HoodieSparkEngineContext.java:116)
at
org.apache.hudi.table.action.clean.CleanActionExecutor.clean(CleanActionExecutor.java:141)
at
org.apache.hudi.table.action.clean.CleanActionExecutor.runClean(CleanActionExecutor.java:197)
at
org.apache.hudi.table.action.clean.CleanActionExecutor.runPendingClean(CleanActionExecutor.java:176)
at
org.apache.hudi.table.action.clean.CleanActionExecutor.lambda$execute$6(CleanActionExecutor.java:238)
at java.util.ArrayList.forEach(ArrayList.java:1259)
at
org.apache.hudi.table.action.clean.CleanActionExecutor.execute(CleanActionExecutor.java:232)
at
org.apache.hudi.table.HoodieSparkCopyOnWriteTable.clean(HoodieSparkCopyOnWriteTable.java:339)
at
org.apache.hudi.client.BaseHoodieWriteClient.clean(BaseHoodieWriteClient.java:780)
at
org.apache.hudi.client.BaseHoodieWriteClient.clean(BaseHoodieWriteClient.java:737)
at
org.apache.hudi.async.AsyncCleanerService.lambda$startService$0(AsyncCleanerService.java:55)
at
java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.io.NotSerializableException:
org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider
Serialization stack:
- object not serializable (class:
org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider, value:
org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider@2e4da831)
- field (class: org.apache.hudi.client.transaction.lock.LockManager, name:
lockProvider, type: interface org.apache.hudi.common.lock.LockProvider)
- object (class org.apache.hudi.client.transaction.lock.LockManager,
org.apache.hudi.client.transaction.lock.LockManager@60d74cfd)
- field (class: org.apache.hudi.client.transaction.TransactionManager,
name: lockManager, type: class
org.apache.hudi.client.transaction.lock.LockManager)
- object (class org.apache.hudi.client.transaction.TransactionManager,
org.apache.hudi.client.transaction.TransactionManager@54e830d4)
- field (class: org.apache.hudi.table.action.clean.CleanActionExecutor,
name: txnManager, type: class
org.apache.hudi.client.transaction.TransactionManager)
- object (class org.apache.hudi.table.action.clean.CleanActionExecutor,
org.apache.hudi.table.action.clean.CleanActionExecutor@71c368a0)
- element of array (index: 0)
- array (class [Ljava.lang.Object;, size 1)
- field (class: java.lang.invoke.SerializedLambda, name: capturedArgs,
type: class [Ljava.lang.Object;)
- object (class java.lang.invoke.SerializedLambda,
SerializedLambda[capturingClass=class
org.apache.hudi.table.action.clean.CleanActionExecutor,
functionalInterfaceMethod=org/apache/hudi/common/function/SerializablePairFlatMapFunction.call:(Ljava/lang/Object;)Ljava/util/stream/Stream;,
implementation=invokeSpecial
org/apache/hudi/table/action/clean/CleanActionExecutor.lambda$clean$59ddfded$1:(Ljava/util/Iterator;)Ljava/util/stream/Stream;,
instantiatedMethodType=(Ljava/util/Iterator;)Ljava/util/stream/Stream;,
numCaptured=1])
- writeReplace data (class: java.lang.invoke.SerializedLambda)
- object (class
org.apache.hudi.table.action.clean.CleanActionExecutor$$Lambda$3706/863841076,
org.apache.hudi.table.action.clean.CleanActionExecutor$$Lambda$3706/863841076@5e1f7ca3)
- element of array (index: 0)
- array (class [Ljava.lang.Object;, size 1)
- field (class: java.lang.invoke.SerializedLambda, name: capturedArgs,
type: class [Ljava.lang.Object;)
- object (class java.lang.invoke.SerializedLambda,
SerializedLambda[capturingClass=class
org.apache.hudi.client.common.HoodieSparkEngineContext,
functionalInterfaceMethod=org/apache/spark/api/java/function/PairFlatMapFunction.call:(Ljava/lang/Object;)Ljava/util/Iterator;,
implementation=invokeStatic
org/apache/hudi/client/common/HoodieSparkEngineContext.lambda$mapPartitionsToPairAndReduceByKey$4e9dbab9$1:(Lorg/apache/hudi/common/function/SerializablePairFlatMapFunction;Ljava/util/Iterator;)Ljava/util/Iterator;,
instantiatedMethodType=(Ljava/util/Iterator;)Ljava/util/Iterator;,
numCaptured=1])
- writeReplace data (class: java.lang.invoke.SerializedLambda)
- object (class
org.apache.hudi.client.common.HoodieSparkEngineContext$$Lambda$3709/1146198397,
org.apache.hudi.client.common.HoodieSparkEngineContext$$Lambda$3709/1146198397@7aff92c3)
- element of array (index: 0)
- array (class [Ljava.lang.Object;, size 1)
- field (class: java.lang.invoke.SerializedLambda, name: capturedArgs,
type: class [Ljava.lang.Object;)
- object (class java.lang.invoke.SerializedLambda,
SerializedLambda[capturingClass=interface
org.apache.spark.api.java.JavaRDDLike,
functionalInterfaceMethod=scala/Function1.apply:(Ljava/lang/Object;)Ljava/lang/Object;,
implementation=invokeStatic
org/apache/spark/api/java/JavaRDDLike.$anonfun$mapPartitionsToPair$1:(Lorg/apache/spark/api/java/function/PairFlatMapFunction;Lscala/collection/Iterator;)Lscala/collection/Iterator;,
instantiatedMethodType=(Lscala/collection/Iterator;)Lscala/collection/Iterator;,
numCaptured=1])
- writeReplace data (class: java.lang.invoke.SerializedLambda)
- object (class
org.apache.spark.api.java.JavaRDDLike$$Lambda$3710/1144734313,
org.apache.spark.api.java.JavaRDDLike$$Lambda$3710/1144734313@4a33e3f8)
at
org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:41)
at
org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:47)
at
org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:101)
at
org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:413)
... 26 more {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
