Hi
It worked for me like this. Just define the case class outside of any class
to write to parquet format successfully. I am using Spark version 1.1.1.
case class person(id: Int, name: String, fathername: String, officeid: Int)
object Program {
def main (args: Array[String]) {
val conf: SparkConf = new
SparkConf().setAppName("Test").setMaster("local")
val sc: SparkContext = new SparkContext(conf)
val sqlContext = new org.apache.spark.sql.SQLContext(sc)
import sqlContext.createSchemaRDD
val baseperson: RDD[Array[String]] =
sc.textFile("/home/ahsan/Desktop/1.csv").flatMap(line =>
line.split("\n")).map(_.split(","))
val x: RDD[person] = baseperson.map(p =>
person(p(0).asInstanceOf[Int], p(1).asInstanceOf[String],
p(2).asInstanceOf[String], p(3).asInstanceOf[Int]))
x.saveAsParquetFile("/home/ahsan/Desktop/pqt")
}
}
--
View this message in context:
http://apache-spark-user-list.1001560.n3.nabble.com/parquet-file-not-loading-spark-v-1-1-0-tp20618p20628.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.
---------------------------------------------------------------------
To unsubscribe, e-mail: user-unsubscr...@spark.apache.org
For additional commands, e-mail: user-h...@spark.apache.org
