when i run spark application longtime.
15/05/11 12:50:47 INFO spark.SparkContext: Created broadcast 8 from broadcast
at DAGScheduler.scala:839
15/05/11 12:50:47 INFO storage.BlockManagerInfo: Removed input-0-1431345102800
on dn2.vi:45688 in memory (size: 27.6 KB, free: 524.2 MB)
15/05/11 12:50:47 ERROR LocalActorRefProvider(akka://sparkDriver): guardian
failed, shutting down system
java.lang.AssertionError: assertion failed
at scala.Predef$.assert(Predef.scala:165)
at
org.apache.spark.streaming.scheduler.ReceivedBlockTracker.cleanupOldBatches(ReceivedBlockTracker.scala:153)
at
org.apache.spark.streaming.scheduler.ReceiverTracker.cleanupOldBlocksAndBatches(ReceiverTracker.scala:127)
at
org.apache.spark.streaming.scheduler.JobGenerator.clearMetadata(JobGenerator.scala:264)
at
org.apache.spark.streaming.scheduler.JobGenerator.org$apache$spark$streaming$scheduler$JobGenerator$$processEvent(JobGenerator.scala:175)
at
org.apache.spark.streaming.scheduler.JobGenerator$$anonfun$start$1$$anon$1$$anonfun$receive$1.applyOrElse(JobGenerator.scala:85)
at akka.actor.Actor$class.aroundReceive(Actor.scala:465)
at
org.apache.spark.streaming.scheduler.JobGenerator$$anonfun$start$1$$anon$1.aroundReceive(JobGenerator.scala:83)
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516)
at akka.actor.ActorCell.invoke(ActorCell.scala:487)
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238)
at akka.dispatch.Mailbox.run(Mailbox.scala:220)
at
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:393)
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at
scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at
scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
15/05/11 12:50:47 ERROR actor.OneForOneStrategy: assertion failed
java.lang.AssertionError: assertion failed
at scala.Predef$.assert(Predef.scala:165)
at
org.apache.spark.streaming.scheduler.ReceivedBlockTracker.cleanupOldBatches(ReceivedBlockTracker.scala:153)
at
org.apache.spark.streaming.scheduler.ReceiverTracker.cleanupOldBlocksAndBatches(ReceiverTracker.scala:127)
at
org.apache.spark.streaming.scheduler.JobGenerator.clearMetadata(JobGenerator.scala:264)
at
org.apache.spark.streaming.scheduler.JobGenerator.org$apache$spark$streaming$scheduler$JobGenerator$$processEvent(JobGenerator.scala:175)
at
org.apache.spark.streaming.scheduler.JobGenerator$$anonfun$start$1$$anon$1$$anonfun$receive$1.applyOrElse(JobGenerator.scala:85)
at akka.actor.Actor$class.aroundReceive(Actor.scala:465)
at
org.apache.spark.streaming.scheduler.JobGenerator$$anonfun$start$1$$anon$1.aroundReceive(JobGenerator.scala:83)
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516)
at akka.actor.ActorCell.invoke(ActorCell.scala:487)
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238)
at akka.dispatch.Mailbox.run(Mailbox.scala:220)
at
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:393)
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at
scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at
scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
15/05/11 12:50:47 WARN util.AkkaUtils: Error sending message [message =
GetLocationsMultipleBlockIds([Lorg.apache.spark.storage.BlockId;@7e6a12b8)] in
1 attempts
akka.pattern.AskTimeoutException:
Recipient[Actor[akka://sparkDriver/user/BlockManagerMaster#-1781979320]] had
already been terminated.
at akka.pattern.AskableActorRef$.ask$extension(AskSupport.scala:132)
at org.apache.spark.util.AkkaUtils$.askWithReply(AkkaUtils.scala:194)
at
org.apache.spark.storage.BlockManagerMaster.askDriverWithReply(BlockManagerMaster.scala:221)
at
org.apache.spark.storage.BlockManagerMaster.getLocations(BlockManagerMaster.scala:75)
at
org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$getLocationBlockIds(BlockManager.scala:421)
at
org.apache.spark.storage.BlockManager$.blockIdsToBlockManagers(BlockManager.scala:1256)
at
org.apache.spark.storage.BlockManager$.blockIdsToHosts(BlockManager.scala:1279)
at
org.apache.spark.rdd.BlockRDD.locations_$lzycompute(BlockRDD.scala:34)
at org.apache.spark.rdd.BlockRDD.locations_(BlockRDD.scala:34)
at
org.apache.spark.rdd.BlockRDD.getPreferredLocations(BlockRDD.scala:57)
at
org.apache.spark.rdd.RDD$$anonfun$preferredLocations$2.apply(RDD.scala:231)
at
org.apache.spark.rdd.RDD$$anonfun$preferredLocations$2.apply(RDD.scala:231)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.preferredLocations(RDD.scala:230)
at
org.apache.spark.rdd.UnionPartition.preferredLocations(UnionRDD.scala:47)
at
org.apache.spark.rdd.UnionRDD.getPreferredLocations(UnionRDD.scala:91)
at
org.apache.spark.rdd.RDD$$anonfun$preferredLocations$2.apply(RDD.scala:231)
at
org.apache.spark.rdd.RDD$$anonfun$preferredLocations$2.apply(RDD.scala:231)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.preferredLocations(RDD.scala:230)
at
org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$getPreferredLocsInternal(DAGScheduler.scala:1324)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$org$apache$spark$scheduler$DAGScheduler$$getPreferredLocsInternal$2$$anonfun$apply$2.apply$mcVI$sp(DAGScheduler.scala:1334)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$org$apache$spark$scheduler$DAGScheduler$$getPreferredLocsInternal$2$$anonfun$apply$2.apply(DAGScheduler.scala:1333)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$org$apache$spark$scheduler$DAGScheduler$$getPreferredLocsInternal$2$$anonfun$apply$2.apply(DAGScheduler.scala:1333)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$org$apache$spark$scheduler$DAGScheduler$$getPreferredLocsInternal$2.apply(DAGScheduler.scala:1333)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$org$apache$spark$scheduler$DAGScheduler$$getPreferredLocsInternal$2.apply(DAGScheduler.scala:1331)
at scala.collection.immutable.List.foreach(List.scala:318)
at
org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$getPreferredLocsInternal(DAGScheduler.scala:1331)
at
org.apache.spark.scheduler.DAGScheduler.getPreferredLocs(DAGScheduler.scala:1296)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$14.apply(DAGScheduler.scala:863)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$14.apply(DAGScheduler.scala:860)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
at scala.collection.AbstractTraversable.map(Traversable.scala:105)
at
org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$submitMissingTasks(DAGScheduler.scala:860)
at
org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$submitStage(DAGScheduler.scala:778)
at
org.apache.spark.scheduler.DAGScheduler.handleJobSubmitted(DAGScheduler.scala:762)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1362)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1354)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
15/05/11 12:50:47 INFO remote.RemoteActorRefProvider$RemotingTerminator:
Shutting down remote daemon.
15/05/11 12:50:47 INFO remote.RemoteActorRefProvider$RemotingTerminator: Remote
daemon shut down; proceeding with flushing remote transports.
15/05/11 12:50:47 INFO remote.RemoteActorRefProvider$RemotingTerminator:
Remoting shut down.
15/05/11 12:50:47 ERROR cluster.YarnClientSchedulerBackend: Yarn application
has already exited with state FINISHED!
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/streaming/json,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/streaming,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/metrics/json,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/stages/stage/kill,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/static,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/executors/threadDump/json,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/executors/threadDump,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/executors/json,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/executors,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/environment/json,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/environment,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/storage/rdd/json,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/storage/rdd,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/storage/json,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/storage,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/stages/pool/json,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/stages/pool,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/stages/stage/json,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/stages/stage,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/stages/json,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/stages,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/jobs/job/json,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/jobs/job,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/jobs/json,null}
15/05/11 12:50:47 INFO handler.ContextHandler: stopped
o.s.j.s.ServletContextHandler{/jobs,null}
15/05/11 12:50:47 INFO ui.SparkUI: Stopped Spark web UI at http://master.vi:4041
15/05/11 12:50:47 INFO scheduler.DAGScheduler: Stopping DAGScheduler
15/05/11 12:50:47 INFO scheduler.DAGScheduler: Job 3 failed: start at
SocketStreamingTest.scala:110, took 387.065955 s
Exception in thread "Thread-40" org.apache.spark.SparkException: Job cancelled
because SparkContext was shut down
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$cleanUpAfterSchedulerStop$1.apply(DAGScheduler.scala:699)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$cleanUpAfterSchedulerStop$1.apply(DAGScheduler.scala:698)
at scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
at
org.apache.spark.scheduler.DAGScheduler.cleanUpAfterSchedulerStop(DAGScheduler.scala:698)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onStop(DAGScheduler.scala:1411)
at org.apache.spark.util.EventLoop.stop(EventLoop.scala:81)
at org.apache.spark.scheduler.DAGScheduler.stop(DAGScheduler.scala:1346)
at org.apache.spark.SparkContext.stop(SparkContext.scala:1380)
at
org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend$$anon$1.run(YarnClientSchedulerBackend.scala:143)
15/05/11 12:50:47 INFO scheduler.DAGScheduler: Job 53 failed: foreachRDD at
SocketStreamingTest.scala:107, took 0.553739 s
15/05/11 12:50:47 INFO cluster.YarnClientSchedulerBackend: Shutting down all
executors
15/05/11 12:50:47 INFO cluster.YarnClientSchedulerBackend: Stopped
15/05/11 12:50:47 WARN util.AkkaUtils: Error sending message [message =
StopMapOutputTracker] in 1 attempts
akka.pattern.AskTimeoutException:
Recipient[Actor[akka://sparkDriver/user/MapOutputTracker#-890977520]] had
already been terminated.
at akka.pattern.AskableActorRef$.ask$extension(AskSupport.scala:132)
at org.apache.spark.util.AkkaUtils$.askWithReply(AkkaUtils.scala:194)
at
org.apache.spark.MapOutputTracker.askTracker(MapOutputTracker.scala:113)
at
org.apache.spark.MapOutputTracker.sendTracker(MapOutputTracker.scala:123)
at
org.apache.spark.MapOutputTrackerMaster.stop(MapOutputTracker.scala:329)
at org.apache.spark.SparkEnv.stop(SparkEnv.scala:85)
at org.apache.spark.SparkContext.stop(SparkContext.scala:1385)
at
org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend$$anon$1.run(YarnClientSchedulerBackend.scala:143)
15/05/11 12:50:50 WARN util.AkkaUtils: Error sending message [message =
StopMapOutputTracker] in 2 attempts
akka.pattern.AskTimeoutException:
Recipient[Actor[akka://sparkDriver/user/MapOutputTracker#-890977520]] had
already been terminated.
at akka.pattern.AskableActorRef$.ask$extension(AskSupport.scala:132)
at org.apache.spark.util.AkkaUtils$.askWithReply(AkkaUtils.scala:194)
at
org.apache.spark.MapOutputTracker.askTracker(MapOutputTracker.scala:113)
at
org.apache.spark.MapOutputTracker.sendTracker(MapOutputTracker.scala:123)
at
org.apache.spark.MapOutputTrackerMaster.stop(MapOutputTracker.scala:329)
at org.apache.spark.SparkEnv.stop(SparkEnv.scala:85)
at org.apache.spark.SparkContext.stop(SparkContext.scala:1385)
at
org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend$$anon$1.run(YarnClientSchedulerBackend.scala:143)
15/05/11 12:50:53 WARN util.AkkaUtils: Error sending message [message =
StopMapOutputTracker] in 3 attempts
akka.pattern.AskTimeoutException:
Recipient[Actor[akka://sparkDriver/user/MapOutputTracker#-890977520]] had
already been terminated.
at akka.pattern.AskableActorRef$.ask$extension(AskSupport.scala:132)
at org.apache.spark.util.AkkaUtils$.askWithReply(AkkaUtils.scala:194)
at
org.apache.spark.MapOutputTracker.askTracker(MapOutputTracker.scala:113)
at
org.apache.spark.MapOutputTracker.sendTracker(MapOutputTracker.scala:123)
at
org.apache.spark.MapOutputTrackerMaster.stop(MapOutputTracker.scala:329)
at org.apache.spark.SparkEnv.stop(SparkEnv.scala:85)
at org.apache.spark.SparkContext.stop(SparkContext.scala:1385)
at
org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend$$anon$1.run(YarnClientSchedulerBackend.scala:143)
15/05/11 12:50:56 ERROR spark.MapOutputTrackerMaster: Error communicating with
MapOutputTracker
org.apache.spark.SparkException: Error sending message [message =
StopMapOutputTracker]
at org.apache.spark.util.AkkaUtils$.askWithReply(AkkaUtils.scala:209)
at
org.apache.spark.MapOutputTracker.askTracker(MapOutputTracker.scala:113)
at
org.apache.spark.MapOutputTracker.sendTracker(MapOutputTracker.scala:123)
at
org.apache.spark.MapOutputTrackerMaster.stop(MapOutputTracker.scala:329)
at org.apache.spark.SparkEnv.stop(SparkEnv.scala:85)
at org.apache.spark.SparkContext.stop(SparkContext.scala:1385)
at
org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend$$anon$1.run(YarnClientSchedulerBackend.scala:143)
Caused by: akka.pattern.AskTimeoutException:
Recipient[Actor[akka://sparkDriver/user/MapOutputTracker#-890977520]] had
already been terminated.
at akka.pattern.AskableActorRef$.ask$extension(AskSupport.scala:132)
at org.apache.spark.util.AkkaUtils$.askWithReply(AkkaUtils.scala:194)
... 6 more
Exception in thread "Yarn application state monitor"
org.apache.spark.SparkException: Error communicating with MapOutputTracker
at
org.apache.spark.MapOutputTracker.askTracker(MapOutputTracker.scala:117)
at
org.apache.spark.MapOutputTracker.sendTracker(MapOutputTracker.scala:123)
at
org.apache.spark.MapOutputTrackerMaster.stop(MapOutputTracker.scala:329)
at org.apache.spark.SparkEnv.stop(SparkEnv.scala:85)
at org.apache.spark.SparkContext.stop(SparkContext.scala:1385)
at
org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend$$anon$1.run(YarnClientSchedulerBackend.scala:143)
Caused by: org.apache.spark.SparkException: Error sending message [message =
StopMapOutputTracker]
at org.apache.spark.util.AkkaUtils$.askWithReply(AkkaUtils.scala:209)
at
org.apache.spark.MapOutputTracker.askTracker(MapOutputTracker.scala:113)
... 5 more
Caused by: akka.pattern.AskTimeoutException:
Recipient[Actor[akka://sparkDriver/user/MapOutputTracker#-890977520]] had
already been terminated.
at akka.pattern.AskableActorRef$.ask$extension(AskSupport.scala:132)
at org.apache.spark.util.AkkaUtils$.askWithReply(AkkaUtils.scala:194)
... 6 more
