Darshan Mehta created SPARK-20225:
-------------------------------------
Summary: Spark Job hangs while writing parquet files to HDFS
Key: SPARK-20225
URL: https://issues.apache.org/jira/browse/SPARK-20225
Project: Spark
Issue Type: Bug
Components: Input/Output
Affects Versions: 2.1.0
Reporter: Darshan Mehta
While saving a dataframe to parquet using {{baseDataset.write.parquet("some
location")}} 3 stages failed, however, it did not notify the (parent) tasks
which got stuck on 80%.
Here's the sample (pseudo) code:
{code:title=Test.scala|borderStyle=solid}
val values = Map(...)
val range = Seq(..)
val collection = //get collection
collection.foreach(range=>{
values.foreach(e=> {
val baseDataset = //get dataset
baseDataset.write.parquet("some location")
})
})
{code}
This is used as a paragraph in zeppelin and it gets stuck at 80% after
completing 5-6 iterations (I have tried debugging it by writing {{println}}
statements).
Here's what I can see in Thread dump :
{code}
"pool-2-thread-10" #174 prio=5 os_prio=0 tid=0x00007f9f1807e800 nid=0x3945
waiting on condition [0x00007f9eddec1000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000552b4bef0> (a
scala.concurrent.impl.Promise$CompletionLatch)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at
scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:202)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:153)
at
org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:619)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1951)
at
org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply$mcV$sp(FileFormatWriter.scala:127)
at
org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:121)
at
org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:121)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:57)
at
org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:121)
at
org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:101)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
- locked <0x0000000552b92c08> (a
org.apache.spark.sql.execution.command.ExecutedCommandExec)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:74)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:135)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at
org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:132)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:113)
at
org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:87)
- locked <0x0000000552b92c80> (a
org.apache.spark.sql.execution.QueryExecution)
at
org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:87)
at
org.apache.spark.sql.execution.datasources.DataSource.write(DataSource.scala:492)
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:215)
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:198)
at
org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:494)
at
$line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1$$anonfun$apply$1.apply(<console>:82)
at
$line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1$$anonfun$apply$1.apply(<console>:63)
at scala.collection.immutable.Map$Map4.foreach(Map.scala:188)
at
$line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:63)
at
$line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:62)
at scala.collection.immutable.Stream.foreach(Stream.scala:594)
at
$line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:62)
at
$line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:89)
at $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:91)
at $line167049345059.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:93)
at $line167049345059.$read$$iw$$iw$$iw$$iw.<init>(<console>:95)
at $line167049345059.$read$$iw$$iw$$iw.<init>(<console>:97)
at $line167049345059.$read$$iw$$iw.<init>(<console>:99)
at $line167049345059.$read$$iw.<init>(<console>:101)
at $line167049345059.$read.<init>(<console>:103)
at $line167049345059.$read$.<init>(<console>:107)
at $line167049345059.$read$.<clinit>(<console>)
at $line167049345059.$eval$.$print$lzycompute(<console>:7)
- locked <0x000000054940c200> (a $line167049345059.$eval$)
at $line167049345059.$eval$.$print(<console>:6)
at $line167049345059.$eval.$print(<console>)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
at
scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
at
scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
at
scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
at
scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
at
scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
at
scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:38)
at
org.apache.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:961)
at
org.apache.zeppelin.spark.SparkInterpreter.interpretInput(SparkInterpreter.java:1187)
at
org.apache.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:1133)
- locked <0x0000000531629650> (a
org.apache.zeppelin.spark.SparkInterpreter)
at
org.apache.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:1126)
at
org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:94)
at
org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:489)
at org.apache.zeppelin.scheduler.Job.run(Job.java:175)
at
org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Locked ownable synchronizers:
- <0x0000000549eb7590> (a
java.util.concurrent.ThreadPoolExecutor$Worker)
{code}
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
