Hi All
Does anyone know a fix for below exception. The XML parsing function
works fine for unit test as you see in below code but fails while
using in RDD.
new_xml: org.apache.spark.rdd.RDD[List[(String, String)]] =
MapPartitionsRDD[119] at map at <console>:57
17/07/10 08:29:54 ERROR Executor: Exception in task 0.0 in stage 31.0 (TID 50)
java.lang.NullPointerException
at $line103.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.xmlParse(<console>:52)
at
$line109.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:57)
at
$line109.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:57)
at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
at scala.collection.Iterator$class.foreach(Iterator.scala:893)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
at
org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$27.apply(RDD.scala:875)
at
org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$27.apply(RDD.scala:875)
at
org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1897)
at
org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1897)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:70)
at org.apache.spark.scheduler.Task.run(Task.scala:85)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
17/07/10 08:29:54 ERROR Executor: Exception in task 1.0 in stage 31.0 (TID 51)
java.lang.NullPointerException
cat /home/spark/XML_Project/XML_Prog.scala
println(">>>>>>>START UnitTest for xmlParse")
import com.databricks.spark.xml.XmlReader
def xmlParse (xml:String) = {
var xRDD = sc.parallelize(Seq(xml))
var df = new XmlReader().xmlRdd(spark.sqlContext,xRDD)
var out_rdd = df.withColumn("comment",
explode(df("Comments.Comment"))).select($"comment.Description",$"comment.Title").rdd
out_rdd.collect.map(x=>(x(0).toString,x(1).toString)).toList
}
val
xml1="<books><Comments><Comment><Title>Title1.1</Title><Description>Descript1.1</Description></Comment><Comment><Title>Title1.2</Title><Description>Descript1.2</Description></Comment><Comment><Title>Title1.3</Title><Description>Descript1.3</Description></Comment></Comments></books>"
val xml_parse = xmlParse(xml1)
println("<<<<<<<<END UnitTest for xmlParse")
val rdd = sc.textFile("file:///home/spark/XML_Project/data.txt")
val xml_pRDDs = rdd.map(x=>(x.split(',')(0).toInt, x.split(',')(3)))
val new_xml = xml_pRDDs.map({case (key,value)=>(xmlParse(value.toString))})
new_xml.foreach(println)
cat /home/spark/XML_Project/data.txt
1,Amol,Kolhapur,<books><Comments><Comment><Title>Title1.1</Title><Description>Descript1.1</Description></Comment><Comment><Title>Title1.2</Title><Description>Descript1.2</Description></Comment><Comment><Title>Title1.3</Title><Description>Descript1.3</Description></Comment></Comments></books>
2,Ameet,Bangalore,<books><Comments><Comment><Title>Title2.1</Title><Description>Descript2.1</Description></Comment><Comment><Title>Title2.2</Title><Description>Descript2.2</Description></Comment></Comments></books>
3,Rajesh,Jaipur,<books><Comments><Comment><Title>Title3.1</Title><Description>Descript3.1</Description></Comment><Comment><Title>Title3.2</Title><Description>Descript3.2</Description></Comment><Comment><Title>Title3.3</Title><Description>Descript3.3</Description></Comment><Comment><Title>Title3.4</Title><Description>Descript3.4</Description></Comment></Comments></books>
Regards,
Amol
---------------------------------------------------------------------
To unsubscribe e-mail: user-unsubscr...@spark.apache.org
