I could trace where the problem is. If I run without any threads, it works
fine. When I allocate threads, I run into Not serializable problem. But, I
need to have threads in my code.
Any help please!!!
This is my code:
object SparkKart
{
def parseVector(line: String): Vector[Double] = {
DenseVector(line.split(' ').map(_.toDouble))
}
def main(args: Array[String]) {
val sparkConf = new SparkConf().setAppName("SparkKart")
val sc = new SparkContext(sparkConf)
val lines = sc.textFile(args(0))
val data = lines.map(parseVector
_).persist(StorageLevel.MEMORY_ONLY_SER)
val kPoints = data.takeSample(withReplacement = false, 4, 42).toArray
val thread1= new Thread(new Runnable {
def run() {
val dist1 =data.map(x => squaredDistance(x,kPoints(0)))
dist1.saveAsTextFile("hdfs:/kart")
}
})
thread1.start
}
}
--
View this message in context:
http://apache-spark-user-list.1001560.n3.nabble.com/Task-not-serializable-exception-tp21776p21778.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.
---------------------------------------------------------------------
To unsubscribe, e-mail: user-unsubscr...@spark.apache.org
For additional commands, e-mail: user-h...@spark.apache.org
