Hi ,
how to convert this script to java 8 with lampdas ?
My problem is the function getactivations() returns a
scala.collection.Iterator<Node> to
mapPartitions() that need a java.util.Iterator<String> ...
Thks !
===================================================
// Step 1 - Stub code to copy into Spark Shell
// load XML files containing device activation records.
// Find the most common device models activated
import scala.xml._
// Given a partition containing multi-line XML, parse the contents.
// Return an iterator of activation XML nodes contained in the partition
def getactivations(fileiterator: Iterator[String]): Iterator[Node] = {
val nodes = XML.loadString(fileiterator.mkString) \\ "activation"
nodes.toIterator
}
// Get the model name from a device activation record
def getmodel(activation: Node): String = {
(activation \ "model").text
}
// Step 2 - activation XML files
var filename="hdfs://localhost/user/cloudera/data/activations/*"
// parse each partition as a file into an activation XML record
var activations = sc.textFile(filename)
// Step 3 - Parse each partition as a file into an activation XML record
var activationTrees = activations.mapPartitions(getactivations)
// Step 4 - Map each activation record to a device model name
var models = activationTrees.map(getmodel)
// Step 5 - Show the partitioning
println(models.toDebugString)
// Step 6 - XCount activations by model
var modelcounts = models.
map(model => (model,1)).
reduceByKey((v1,v2) => v1+v2)
--
View this message in context:
http://apache-spark-user-list.1001560.n3.nabble.com/mapPartitions-in-Java-8-tp22533.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
