Hi Stan, I have filed a issue on JIRA for this exception, and I have tested it on spark 1.3.0 (local mode) without exception.
https://issues.apache.org/jira/browse/SPARK-8368 On Sun, Jun 14, 2015 at 11:25 AM, StanZhai <m...@zhaishidan.cn> wrote: > I have encountered the similar error too at spark 1.4.0. > > The same code can be run on spark 1.3.1. > > My code is(it can be run on spark-shell): > =============================== > // hc is a instance of HiveContext > val df = hc.sql("select * from test limit 10") > val sb = new mutable.StringBuilder > def mapHandle = (row: Row) => { > val rowData = ArrayBuffer[String]() > for (i <- 0 until row.size) { > val d = row.get(i) > > d match { > case data: ArrayBuffer[Any] => > sb.clear() > sb.append('[') > for (j <- 0 until data.length) { > val elm = data(j) > if (elm != null) { > sb.append('"') > sb.append(elm.toString) > sb.append('"') > } else { > sb.append("null") > } > sb.append(',') > } > if (sb.length > 1) { > sb.deleteCharAt(sb.length - 1) > } > sb.append(']') > rowData += sb.toString() > case _ => > rowData += (if (d != null) d.toString else null) > } > } > rowData > } > df.map(mapHandle).foreach(println) > > > My submit script is: spark-submit --class cn.zhaishidan.trans.Main --master > local[8] test-spark.jar > ===============the error================ > java.lang.ClassNotFoundException: > > cn.zhaishidan.trans.service.SparkHiveService$$anonfun$mapHandle$1$1$$anonfun$apply$1 > at java.net.URLClassLoader$1.run(URLClassLoader.java:366) > at java.net.URLClassLoader$1.run(URLClassLoader.java:355) > at java.security.AccessController.doPrivileged(Native Method) > at java.net.URLClassLoader.findClass(URLClassLoader.java:354) > at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308) > at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > at java.lang.Class.forName0(Native Method) > at java.lang.Class.forName(Class.java:270) > at > > org.apache.spark.util.InnerClosureFinder$$anon$4.visitMethodInsn(ClosureCleaner.scala:455) > at > > com.esotericsoftware.reflectasm.shaded.org.objectweb.asm.ClassReader.accept(Unknown > Source) > at > > com.esotericsoftware.reflectasm.shaded.org.objectweb.asm.ClassReader.accept(Unknown > Source) > at > > org.apache.spark.util.ClosureCleaner$.getInnerClosureClasses(ClosureCleaner.scala:101) > at > > org.apache.spark.util.ClosureCleaner$.org$apache$spark$util$ClosureCleaner$$clean(ClosureCleaner.scala:197) > at > org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:132) > at org.apache.spark.SparkContext.clean(SparkContext.scala:1891) > at org.apache.spark.rdd.RDD$$anonfun$map$1.apply(RDD.scala:294) > at org.apache.spark.rdd.RDD$$anonfun$map$1.apply(RDD.scala:293) > at > > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:148) > at > > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:109) > at org.apache.spark.rdd.RDD.withScope(RDD.scala:286) > at org.apache.spark.rdd.RDD.map(RDD.scala:293) > at org.apache.spark.sql.DataFrame.map(DataFrame.scala:1210) > at > > cn.zhaishidan.trans.service.SparkHiveService.formatDF(SparkHiveService.scala:66) > at > > cn.zhaishidan.trans.service.SparkHiveService.query(SparkHiveService.scala:80) > at > > cn.zhaishidan.trans.api.DatabaseApi$$anonfun$query$1.apply(DatabaseApi.scala:39) > at > > cn.zhaishidan.trans.api.DatabaseApi$$anonfun$query$1.apply(DatabaseApi.scala:30) > at > cn.zhaishidan.trans.web.JettyUtils$$anon$1.getOrPost(JettyUtils.scala:56) > at > cn.zhaishidan.trans.web.JettyUtils$$anon$1.doGet(JettyUtils.scala:73) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:735) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:848) > at > org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684) > at > org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:501) > at > > org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086) > at > org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:428) > at > > org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020) > at > > org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) > at > org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) > at > > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) > at org.eclipse.jetty.server.Server.handle(Server.java:370) > at > > org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494) > at > > org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:971) > at > > org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1033) > at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644) > at > org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) > at > > org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) > at > > org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:667) > at > > org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52) > at > > org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) > at > > org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) > at java.lang.Thread.run(Thread.java:745) > > > > > -- > View this message in context: > http://apache-spark-developers-list.1001551.n3.nabble.com/A-confusing-ClassNotFoundException-error-tp12720p12749.html > Sent from the Apache Spark Developers List mailing list archive at > Nabble.com. > > --------------------------------------------------------------------- > To unsubscribe, e-mail: dev-unsubscr...@spark.apache.org > For additional commands, e-mail: dev-h...@spark.apache.org > >
