Github user rjurney commented on the pull request:
https://github.com/apache/spark/pull/455#issuecomment-49697592
My data has doubles in it, could that be the issue?
Using Python version 2.7.6rc1 (v2.7.6rc1:4913d0e9be30+, Oct 27 2013
20:52:11)
SparkContext available as sc.
In [1]: avroRdd = sc.newAPIHadoopFile('/tmp/part-m-00000.avro',
...: "org.apache.avro.mapreduce.AvroKeyInputFormat",
...: "org.apache.avro.mapred.AvroKey",
...: "org.apache.hadoop.io.NullWritable",
...:
keyConverter="org.apache.spark.examples.pythonconverters.AvroGenericConverter")
14/07/21 21:38:48 INFO MemoryStore: ensureFreeSpace(56082) called with
curMem=0, maxMem=309225062
14/07/21 21:38:48 INFO MemoryStore: Block broadcast_0 stored as values in
memory (estimated size 54.8 KB, free 294.8 MB)
14/07/21 21:38:48 INFO Converter: Loaded converter:
org.apache.spark.examples.pythonconverters.AvroGenericConverter
14/07/21 21:38:48 INFO FileInputFormat: Total input paths to process : 1
14/07/21 21:38:48 INFO SparkContext: Starting job: first at
SerDeUtil.scala:69
14/07/21 21:38:48 INFO DAGScheduler: Got job 0 (first at
SerDeUtil.scala:69) with 1 output partitions (allowLocal=true)
14/07/21 21:38:48 INFO DAGScheduler: Final stage: Stage 0(first at
SerDeUtil.scala:69)
14/07/21 21:38:48 INFO DAGScheduler: Parents of final stage: List()
14/07/21 21:38:48 INFO DAGScheduler: Missing parents: List()
14/07/21 21:38:48 INFO DAGScheduler: Computing the requested partition
locally
14/07/21 21:38:48 INFO NewHadoopRDD: Input split:
file:/tmp/part-m-00000.avro:0+1996482
14/07/21 21:38:48 WARN AvroKeyInputFormat: Reader schema was not set. Use
AvroJob.setInputKeySchema() if desired.
14/07/21 21:38:48 INFO AvroKeyInputFormat: Using a reader schema equal to
the writer schema.
14/07/21 21:38:48 INFO DAGScheduler: Failed to run first at
SerDeUtil.scala:69
---------------------------------------------------------------------------
Py4JJavaError Traceback (most recent call last)
<ipython-input-1-6b7a8c841ffd> in <module>()
3 "org.apache.avro.mapred.AvroKey",
4 "org.apache.hadoop.io.NullWritable",
----> 5
keyConverter="org.apache.spark.examples.pythonconverters.AvroGenericConverter")
/Users/rjurney/Software/spark-trunk/python/pyspark/context.pyc in
newAPIHadoopFile(self, path, inputFormatClass, keyClass, valueClass,
keyConverter, valueConverter, conf)
402 jconf = self._dictToJavaMap(conf)
403 jrdd = self._jvm.PythonRDD.newAPIHadoopFile(self._jsc,
path, inputFormatClass, keyClass,
--> 404 valueClass,
keyConverter, valueConverter, jconf)
405 return RDD(jrdd, self, PickleSerializer())
406
/Users/rjurney/Software/spark-trunk/python/lib/py4j-0.8.1-src.zip/py4j/java_gateway.py
in __call__(self, *args)
535 answer = self.gateway_client.send_command(command)
536 return_value = get_return_value(answer, self.gateway_client,
--> 537 self.target_id, self.name)
538
539 for temp_arg in temp_args:
/Users/rjurney/Software/spark-trunk/python/lib/py4j-0.8.1-src.zip/py4j/protocol.py
in get_return_value(answer, gateway_client, target_id, name)
298 raise Py4JJavaError(
299 'An error occurred while calling {0}{1}{2}.\n'.
--> 300 format(target_id, '.', name), value)
301 else:
302 raise Py4JError(
Py4JJavaError: An error occurred while calling
z:org.apache.spark.api.python.PythonRDD.newAPIHadoopFile.
: org.apache.spark.SparkDriverExecutionException: Execution error
at
org.apache.spark.scheduler.DAGScheduler.runLocallyWithinThread(DAGScheduler.scala:585)
at
org.apache.spark.scheduler.DAGScheduler$$anon$1.run(DAGScheduler.scala:563)
Caused by: java.lang.NullPointerException
at
org.apache.spark.examples.pythonconverters.AvroGenericConverter.org$apache$spark$examples$pythonconverters$AvroGenericConverter$$unpack$1(AvroGenericConverter.scala:31)
at
org.apache.spark.examples.pythonconverters.AvroGenericConverter$$anonfun$unpackRecord$1$1.apply(AvroGenericConverter.scala:17)
at
org.apache.spark.examples.pythonconverters.AvroGenericConverter$$anonfun$unpackRecord$1$1.apply(AvroGenericConverter.scala:17)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
at scala.collection.AbstractTraversable.map(Traversable.scala:105)
at
org.apache.spark.examples.pythonconverters.AvroGenericConverter.unpackRecord$1(AvroGenericConverter.scala:17)
at
org.apache.spark.examples.pythonconverters.AvroGenericConverter.convert(AvroGenericConverter.scala:34)
at
org.apache.spark.examples.pythonconverters.AvroGenericConverter.convert(AvroGenericConverter.scala:12)
at
org.apache.spark.api.python.PythonHadoopUtil$$anonfun$convertRDD$1.apply(PythonHadoopUtil.scala:126)
at
org.apache.spark.api.python.PythonHadoopUtil$$anonfun$convertRDD$1.apply(PythonHadoopUtil.scala:126)
at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
at scala.collection.Iterator$$anon$10.next(Iterator.scala:312)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at
scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48)
at
scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103)
at
scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47)
at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273)
at scala.collection.AbstractIterator.to(Iterator.scala:1157)
at
scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265)
at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157)
at
scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252)
at scala.collection.AbstractIterator.toArray(Iterator.scala:1157)
at org.apache.spark.rdd.RDD$$anonfun$28.apply(RDD.scala:1068)
at org.apache.spark.rdd.RDD$$anonfun$28.apply(RDD.scala:1068)
at
org.apache.spark.SparkContext$$anonfun$runJob$4.apply(SparkContext.scala:1096)
at
org.apache.spark.SparkContext$$anonfun$runJob$4.apply(SparkContext.scala:1096)
at
org.apache.spark.scheduler.DAGScheduler.runLocallyWithinThread(DAGScheduler.scala:578)
... 1 more
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
