Ah nevermind, the fix is to get rid of "return" from my method. There's
probably a bug somewhere related to the repl taking bad input more cleanly,
but this isn't the end of the world once you figure out what the issue is.
Thanks for the time,
Andrew
On Mon, Jun 2, 2014 at 11:35 PM, Andrew Ash <and...@andrewash.com> wrote:
> // observed in Spark 1.0
>
> Scala devs,
>
> I was observing an unusual NPE in my code recently, and came up with the
> below minimal test case:
>
> class Super extends Serializable {
> lazy val superVal: String = null
> }
>
> class Sub extends Super {
> lazy val subVal: String = {
> try {
> "literal"
> } catch {
> case _:Throwable => return superVal
> }
> }
> }
>
> Save this to a file, open the Spark shell, and load with ":l
> /tmp/test.scala"
>
> I got the below really unusual exception. It goes away though when
> removing the try/catch inside subVal and just returning either a straight
> literal or superVal.
>
>
> Is this a bug in Spark, or Scala, or my code, or what? I think it might
> be related to the Spark Repl doing magic but I'm unsure what.
>
> Cheers!
> Andrew
>
>
>
>
> scala> :l /tmp/test.scala
> Loading /tmp/test.scala...
> defined class Super
>
> while compiling: <console>
> during phase: mixin
> library version: version 2.10.4
> compiler version: version 2.10.4
> reconstructed args:
>
> last tree to typer: Apply(constructor $read)
> symbol: constructor $read in class $read (flags: <method>
> <triedcooking>)
> symbol definition: def <init>(): $line9.$read
> tpe: $line9.$read
> symbol owners: constructor $read -> class $read -> package $line9
> context owners: class iwC$Sub -> package $line9
>
> == Enclosing template or block ==
>
> Template( // val <local Sub>: <notype>, tree.tpe=$line9.iwC$Sub
> "$line5.$read$$iwC$$iwC$$iwC$$iwC$Super" // parents
> ValDef(
> private
> "_"
> <tpt>
> <empty>
> )
> // 6 statements
> ValDef( // lazy private[this] var subVal: String
> private <mutable> <local> lazy <triedcooking>
> "subVal "
> <tpt> // tree.tpe=String
> <empty>
> )
> DefDef( // lazy val subVal(): String
> <method> <stable> <accessor> lazy
> "subVal"
> []
> List(Nil)
> <tpt> // tree.tpe=String
> Block( // tree.tpe=String
> ValDef( // val nonLocalReturnKey1: Object
> <synthetic> <triedcooking>
> "nonLocalReturnKey1"
> <tpt> // tree.tpe=Object
> Apply( // def <init>(): Object in class Object, tree.tpe=Object
> new Object."<init>" // def <init>(): Object in class Object,
> tree.tpe=()Object
> Nil
> )
> )
> Try( // tree.tpe=String
> Block( // tree.tpe=String
> Assign( // tree.tpe=Unit
> $read$$iwC$$iwC$$iwC$$iwC$Sub.this."subVal " // lazy
> private[this] var subVal: String, tree.tpe=String
> Block( // tree.tpe=String
> {}
> Apply( // final private[this] def
> liftedTree1$1(nonLocalReturnKey1$1: Object): String, tree.tpe=String
> $read$$iwC$$iwC$$iwC$$iwC$Sub.this."liftedTree1$1" //
> final private[this] def liftedTree1$1(nonLocalReturnKey1$1: Object):
> String, tree.tpe=(nonLocalReturnKey1$1: Object)String
> "nonLocalReturnKey1" // val nonLocalReturnKey1: Object,
> tree.tpe=Object
> )
> )
> )
> $read$$iwC$$iwC$$iwC$$iwC$Sub.this."subVal " // lazy
> private[this] var subVal: String, tree.tpe=String
> )
> CaseDef( // tree.tpe=String
> Bind( // val ex: runtime.NonLocalReturnControl,
> tree.tpe=runtime.NonLocalReturnControl
> "ex"
> Typed( // tree.tpe=runtime.NonLocalReturnControl
> "_" // tree.tpe=runtime.NonLocalReturnControl
> <tpt> // tree.tpe=runtime.NonLocalReturnControl
> )
> )
> If( // tree.tpe=String
> Apply( // final def eq(x$1: Object): Boolean in class Object,
> tree.tpe=Boolean
> ex.key()."eq" // final def eq(x$1: Object): Boolean in class
> Object, tree.tpe=(x$1: Object)Boolean
> "nonLocalReturnKey1" // val nonLocalReturnKey1: Object,
> tree.tpe=Object
> )
> Apply( // final def $asInstanceOf[T0 >: ? <: ?](): T0 in class
> Object, tree.tpe=String
> TypeApply( // final def $asInstanceOf[T0 >: ? <: ?](): T0 in
> class Object, tree.tpe=()String
> ex.value()."$asInstanceOf" // final def $asInstanceOf[T0
> >: ? <: ?](): T0 in class Object, tree.tpe=[T0 >: ? <: ?]()T0
> <tpt> // tree.tpe=String
> )
> Nil
> )
> Throw("ex")tree.tpe=Nothing
> )
> )
> )
> )
> )
> ValDef( // protected val $outer: $line9.iwC
> protected <synthetic> <paramaccessor> <triedcooking>
> "$outer "
> <tpt> // tree.tpe=$line9.iwC
> <empty>
> )
> DefDef( // val $outer(): $line9.iwC
> <method> <synthetic> <stable> <expandedname>
> "$line9$$read$$iwC$$iwC$$iwC$$iwC$Sub$$$outer"
> []
> List(Nil)
> <tpt> // tree.tpe=$line9.iwC
> $read$$iwC$$iwC$$iwC$$iwC$Sub.this."$outer " // protected val $outer:
> $line9.iwC, tree.tpe=$line9.iwC
> )
> DefDef( // final private[this] def liftedTree1$1(nonLocalReturnKey1$1:
> Object): String
> <method> private final <local> <lifted> <triedcooking>
> "liftedTree1"
> []
> // 1 parameter list
> ValDef( // nonLocalReturnKey1$1: Object
> <param> <synthetic>
> "nonLocalReturnKey1$1"
> <tpt> // tree.tpe=Object
> <empty>
> )
> <tpt> // tree.tpe=String
> Try( // tree.tpe=String
> "literal"
> CaseDef( // tree.tpe=Nothing
> Typed( // tree.tpe=Throwable
> "_" // tree.tpe=Throwable
> <tpt> // tree.tpe=Throwable
> )
> Throw( // tree.tpe=Nothing
> Apply( // def <init>(key: Object,value: Object):
> scala.runtime.NonLocalReturnControl in class NonLocalReturnControl,
> tree.tpe=scala.runtime.NonLocalReturnControl
> new runtime.NonLocalReturnControl."<init>" // def <init>(key:
> Object,value: Object): scala.runtime.NonLocalReturnControl in class
> NonLocalReturnControl, tree.tpe=(key: Object, value:
> Object)scala.runtime.NonLocalReturnControl
> // 2 arguments
> "nonLocalReturnKey1$1" // nonLocalReturnKey1$1: Object,
> tree.tpe=Object
> Apply( // lazy val superVal(): String, tree.tpe=String
> $read$$iwC$$iwC$$iwC$$iwC$Sub.this."superVal" // lazy val
> superVal(): String, tree.tpe=()String
> Nil
> )
> )
> )
> )
> )
> )
> DefDef( // def <init>(arg$outer: $line9.iwC): $line9.iwC$Sub
> <method>
> "<init>"
> []
> // 1 parameter list
> ValDef( // $outer: $line9.iwC
> <param> <triedcooking>
> "$outer"
> <tpt> // tree.tpe=$line9.iwC
> <empty>
> )
> <tpt> // tree.tpe=$line9.iwC$Sub
> Block( // tree.tpe=Unit
> // 2 statements
> If( // tree.tpe=Unit
> Apply( // final def eq(x$1: Object): Boolean in class Object,
> tree.tpe=Boolean
> "$outer"."eq" // final def eq(x$1: Object): Boolean in class
> Object, tree.tpe=(x$1: Object)Boolean
> null
> )
> Throw( // tree.tpe=Nothing
> Apply( // def <init>(): NullPointerException in class
> NullPointerException, tree.tpe=NullPointerException
> new NullPointerException."<init>" // def <init>():
> NullPointerException in class NullPointerException,
> tree.tpe=()NullPointerException
> Nil
> )
> )
> Assign( // tree.tpe=Unit
> $read$$iwC$$iwC$$iwC$$iwC$Sub.this."$outer " // protected val
> $outer: $line9.iwC, tree.tpe=$line9.iwC
> "$outer" // $outer: $line9.iwC, tree.tpe=$line9.iwC
> )
> )
> Apply( // def <init>(arg$outer: $line5.iwC): $line5.iwC$Super,
> tree.tpe=$line5.iwC$Super
> $read$$iwC$$iwC$$iwC$$iwC$Sub.super."<init>" // def
> <init>(arg$outer: $line5.iwC): $line5.iwC$Super, tree.tpe=(arg$outer:
> $line5.iwC)$line5.iwC$Super
> Apply( // val $iw(): $line5.iwC, tree.tpe=$line5.iwC
>
> $outer.$line9$$read$$iwC$$iwC$$iwC$$iwC$$$outer().$VAL1().$iw().$iw().$iw()."$iw"
> // val $iw(): $line5.iwC, tree.tpe=()$line5.iwC
> Nil
> )
> )
> ()
> )
> )
> )
>
> == Expanded type of tree ==
>
> TypeRef(TypeSymbol(class $read extends Serializable))
>
> unhandled exception while transforming <console>
> error: uncaught exception during compilation:
> java.lang.NullPointerException
> java.lang.NullPointerException
> at scala.reflect.internal.Trees$class.Select(Trees.scala:1066)
> at scala.reflect.internal.SymbolTable.Select(SymbolTable.scala:13)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$scala$tools$nsc$transform$Mixin$MixinTransformer$$dd$1$2.apply(Mixin.scala:908)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$scala$tools$nsc$transform$Mixin$MixinTransformer$$dd$1$2.apply(Mixin.scala:904)
> at scala.reflect.internal.Trees$class.deriveDefDef(Trees.scala:1598)
> at scala.reflect.internal.SymbolTable.deriveDefDef(SymbolTable.scala:13)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer.scala$tools$nsc$transform$Mixin$MixinTransformer$$dd$1(Mixin.scala:904)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$addCheckedGetters$1$1.apply(Mixin.scala:945)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$addCheckedGetters$1$1.apply(Mixin.scala:945)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
> at scala.collection.immutable.List.foreach(List.scala:318)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
> at scala.collection.AbstractTraversable.map(Traversable.scala:105)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer.addCheckedGetters$1(Mixin.scala:945)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer.addNewDefs(Mixin.scala:1013)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer.scala$tools$nsc$transform$Mixin$MixinTransformer$$postTransform(Mixin.scala:1147)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$transform$1.apply(Mixin.scala:1261)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$transform$1.apply(Mixin.scala:1261)
> at scala.reflect.internal.SymbolTable.atPhase(SymbolTable.scala:207)
> at scala.reflect.internal.SymbolTable.afterPhase(SymbolTable.scala:216)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:1261)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:471)
> at
> scala.reflect.api.Trees$Transformer.transformTemplate(Trees.scala:2904)
> at
> scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1280)
> at
> scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1279)
> at scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)
> at scala.reflect.internal.Trees$class.itransform(Trees.scala:1278)
> at scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)
> at scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)
> at scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:1258)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:471)
> at
> scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2927)
> at
> scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2925)
> at scala.collection.immutable.List.loop$1(List.scala:170)
> at scala.collection.immutable.List.mapConserve(List.scala:186)
> at scala.reflect.api.Trees$Transformer.transformStats(Trees.scala:2925)
> at
> scala.reflect.internal.Trees$$anonfun$itransform$7.apply(Trees.scala:1298)
> at
> scala.reflect.internal.Trees$$anonfun$itransform$7.apply(Trees.scala:1298)
> at scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)
> at scala.reflect.internal.Trees$class.itransform(Trees.scala:1297)
> at scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)
> at scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)
> at scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:1258)
> at
> scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:471)
> at scala.tools.nsc.ast.Trees$Transformer.transformUnit(Trees.scala:227)
> at scala.tools.nsc.transform.Transform$Phase.apply(Transform.scala:30)
> at scala.tools.nsc.Global$GlobalPhase.applyPhase(Global.scala:464)
> at
> scala.tools.nsc.Global$GlobalPhase$$anonfun$run$1.apply(Global.scala:431)
> at
> scala.tools.nsc.Global$GlobalPhase$$anonfun$run$1.apply(Global.scala:431)
> at scala.collection.Iterator$class.foreach(Iterator.scala:727)
> at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
> at scala.tools.nsc.Global$GlobalPhase.run(Global.scala:431)
> at scala.tools.nsc.Global$Run.compileUnitsInternal(Global.scala:1583)
> at scala.tools.nsc.Global$Run.compileUnits(Global.scala:1557)
> at scala.tools.nsc.Global$Run.compileSources(Global.scala:1553)
> at
> org.apache.spark.repl.SparkIMain.compileSourcesKeepingRun(SparkIMain.scala:468)
> at
> org.apache.spark.repl.SparkIMain$ReadEvalPrint.compileAndSaveRun(SparkIMain.scala:859)
> at
> org.apache.spark.repl.SparkIMain$ReadEvalPrint.compile(SparkIMain.scala:815)
> at
> org.apache.spark.repl.SparkIMain$Request.compile$lzycompute(SparkIMain.scala:1009)
> at org.apache.spark.repl.SparkIMain$Request.compile(SparkIMain.scala:1004)
> at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:644)
> at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:609)
> at
> org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:796)
> at
> org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
> at
> org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
> at
> org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
> at
> org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
> at
> org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
> at
> org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
> at
> org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
> at
> org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
> at
> org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
> at
> org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
> at
> org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
> at
> org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
> at
> org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
> at
> org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
> at
> org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
> at
> org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
> at
> org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
> at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:753)
> at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:601)
> at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:608)
> at org.apache.spark.repl.SparkILoop.loop(SparkILoop.scala:611)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1$$anonfun$apply$mcV$sp$2.apply(SparkILoop.scala:621)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1$$anonfun$apply$mcV$sp$2.apply(SparkILoop.scala:618)
> at
> scala.reflect.io.Streamable$Chars$class.applyReader(Streamable.scala:104)
> at scala.reflect.io.File.applyReader(File.scala:82)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SparkILoop.scala:618)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1.apply(SparkILoop.scala:618)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1.apply(SparkILoop.scala:618)
> at org.apache.spark.repl.SparkILoop.savingReplayStack(SparkILoop.scala:150)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1.apply$mcV$sp(SparkILoop.scala:617)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1.apply(SparkILoop.scala:617)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1.apply(SparkILoop.scala:617)
> at org.apache.spark.repl.SparkILoop.savingReader(SparkILoop.scala:155)
> at
> org.apache.spark.repl.SparkILoop.interpretAllFrom(SparkILoop.scala:616)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$loadCommand$1.apply(SparkILoop.scala:681)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$loadCommand$1.apply(SparkILoop.scala:680)
> at org.apache.spark.repl.SparkILoop.withFile(SparkILoop.scala:674)
> at org.apache.spark.repl.SparkILoop.loadCommand(SparkILoop.scala:680)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$standardCommands$7.apply(SparkILoop.scala:294)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$standardCommands$7.apply(SparkILoop.scala:294)
> at
> scala.tools.nsc.interpreter.LoopCommands$LineCmd.apply(LoopCommands.scala:81)
> at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:748)
> at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:601)
> at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:608)
> at org.apache.spark.repl.SparkILoop.loop(SparkILoop.scala:611)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply$mcZ$sp(SparkILoop.scala:936)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:884)
> at
> org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:884)
> at
> scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
> at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:884)
> at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:983)
> at org.apache.spark.repl.Main$.main(Main.scala:31)
> at org.apache.spark.repl.Main.main(Main.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:256)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:54)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
>
>
> Abandoning crashed session.
>
> scala>
>
