Spark and Flink

Wed, 13 May 2015 06:10:34 -0700 

hi,

i use spark and flink in the same maven project,

now i get a exception on working with spark, flink work well

the problem are transitiv dependencies.

maybe somebody know a solution, or versions, which work together.

best regards
paul

ps: a cloudera maven repo flink would be desirable

my pom:

<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="
http://www.w3.org/2001/XMLSchema-instance";
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
  <modelVersion>4.0.0</modelVersion>

  <groupId>mgm.tp.bigdata</groupId>
  <artifactId>tempGeoKmeans</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  <packaging>jar</packaging>

  <name>tempGeoKmeans</name>
  <url>http://maven.apache.org</url>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  </properties>

    <repositories>
        <repository>
              <id>cloudera</id>
              <url>
https://repository.cloudera.com/artifactory/cloudera-repos/</url>
        </repository>
      </repositories>

  <dependencies>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-core_2.10</artifactId>
        <version>1.1.0-cdh5.2.5</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-mapreduce-client-common</artifactId>
        <version>2.5.0-cdh5.2.5</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
        <version>2.5.0-cdh5.2.5</version>
    </dependency>
    <dependency>
        <groupId>org.apache.mahout</groupId>
        <artifactId>mahout-core</artifactId>
        <version>0.9-cdh5.2.5</version>
    </dependency>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>3.8.1</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-core</artifactId>
        <version>0.8.1</version>
    </dependency>
    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-java</artifactId>
        <version>0.8.1</version>
    </dependency>
    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-clients</artifactId>
        <version>0.8.1</version>
    </dependency>
  </dependencies>
</project>

my exception:

5/05/13 15:00:48 WARN component.AbstractLifeCycle: FAILED
org.eclipse.jetty.servlet.DefaultServlet-461261579:
java.lang.NoSuchMethodError:
org.eclipse.jetty.server.ResourceCache.<init>(Lorg/eclipse/jetty/http/MimeTypes;)V
java.lang.NoSuchMethodError:
org.eclipse.jetty.server.ResourceCache.<init>(Lorg/eclipse/jetty/http/MimeTypes;)V
    at
org.eclipse.jetty.servlet.NIOResourceCache.<init>(NIOResourceCache.java:41)
    at
org.eclipse.jetty.servlet.DefaultServlet.init(DefaultServlet.java:223)
    at javax.servlet.GenericServlet.init(GenericServlet.java:244)
    at
org.eclipse.jetty.servlet.ServletHolder.initServlet(ServletHolder.java:442)
    at
org.eclipse.jetty.servlet.ServletHolder.doStart(ServletHolder.java:270)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:721)
    at
org.eclipse.jetty.servlet.ServletContextHandler.startContext(ServletContextHandler.java:279)
    at
org.eclipse.jetty.server.handler.ContextHandler.doStart(ContextHandler.java:717)
    at
org.eclipse.jetty.servlet.ServletContextHandler.doStart(ServletContextHandler.java:155)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.server.handler.HandlerCollection.doStart(HandlerCollection.java:229)
    at
org.eclipse.jetty.server.handler.ContextHandlerCollection.doStart(ContextHandlerCollection.java:172)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.server.handler.HandlerWrapper.doStart(HandlerWrapper.java:95)
    at org.eclipse.jetty.server.Server.doStart(Server.java:282)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:199)
    at org.apache.spark.ui.JettyUtils$$anonfun$4.apply(JettyUtils.scala:209)
    at org.apache.spark.ui.JettyUtils$$anonfun$4.apply(JettyUtils.scala:209)
    at
org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1454)
    at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
    at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1450)
    at
org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:209)
    at org.apache.spark.ui.WebUI.bind(WebUI.scala:102)
    at org.apache.spark.SparkContext.<init>(SparkContext.scala:224)
    at
org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:53)
    at mgm.tp.bigdata.tempGeoKmeans.Spark.SparkMain.main(SparkMain.java:37)
15/05/13 15:00:48 WARN component.AbstractLifeCycle: FAILED
o.e.j.s.ServletContextHandler{/static,null}: java.lang.NoSuchMethodError:
org.eclipse.jetty.server.ResourceCache.<init>(Lorg/eclipse/jetty/http/MimeTypes;)V
java.lang.NoSuchMethodError:
org.eclipse.jetty.server.ResourceCache.<init>(Lorg/eclipse/jetty/http/MimeTypes;)V
    at
org.eclipse.jetty.servlet.NIOResourceCache.<init>(NIOResourceCache.java:41)
    at
org.eclipse.jetty.servlet.DefaultServlet.init(DefaultServlet.java:223)
    at javax.servlet.GenericServlet.init(GenericServlet.java:244)
    at
org.eclipse.jetty.servlet.ServletHolder.initServlet(ServletHolder.java:442)
    at
org.eclipse.jetty.servlet.ServletHolder.doStart(ServletHolder.java:270)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:721)
    at
org.eclipse.jetty.servlet.ServletContextHandler.startContext(ServletContextHandler.java:279)
    at
org.eclipse.jetty.server.handler.ContextHandler.doStart(ContextHandler.java:717)
    at
org.eclipse.jetty.servlet.ServletContextHandler.doStart(ServletContextHandler.java:155)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.server.handler.HandlerCollection.doStart(HandlerCollection.java:229)
    at
org.eclipse.jetty.server.handler.ContextHandlerCollection.doStart(ContextHandlerCollection.java:172)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.server.handler.HandlerWrapper.doStart(HandlerWrapper.java:95)
    at org.eclipse.jetty.server.Server.doStart(Server.java:282)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:199)
    at org.apache.spark.ui.JettyUtils$$anonfun$4.apply(JettyUtils.scala:209)
    at org.apache.spark.ui.JettyUtils$$anonfun$4.apply(JettyUtils.scala:209)
    at
org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1454)
    at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
    at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1450)
    at
org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:209)
    at org.apache.spark.ui.WebUI.bind(WebUI.scala:102)
    at org.apache.spark.SparkContext.<init>(SparkContext.scala:224)
    at
org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:53)
    at mgm.tp.bigdata.tempGeoKmeans.Spark.SparkMain.main(SparkMain.java:37)
15/05/13 15:00:48 WARN component.AbstractLifeCycle: FAILED
org.eclipse.jetty.server.handler.ContextHandlerCollection@48f860cd:
java.lang.NoSuchMethodError:
org.eclipse.jetty.server.ResourceCache.<init>(Lorg/eclipse/jetty/http/MimeTypes;)V
java.lang.NoSuchMethodError:
org.eclipse.jetty.server.ResourceCache.<init>(Lorg/eclipse/jetty/http/MimeTypes;)V
    at
org.eclipse.jetty.servlet.NIOResourceCache.<init>(NIOResourceCache.java:41)
    at
org.eclipse.jetty.servlet.DefaultServlet.init(DefaultServlet.java:223)
    at javax.servlet.GenericServlet.init(GenericServlet.java:244)
    at
org.eclipse.jetty.servlet.ServletHolder.initServlet(ServletHolder.java:442)
    at
org.eclipse.jetty.servlet.ServletHolder.doStart(ServletHolder.java:270)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:721)
    at
org.eclipse.jetty.servlet.ServletContextHandler.startContext(ServletContextHandler.java:279)
    at
org.eclipse.jetty.server.handler.ContextHandler.doStart(ContextHandler.java:717)
    at
org.eclipse.jetty.servlet.ServletContextHandler.doStart(ServletContextHandler.java:155)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.server.handler.HandlerCollection.doStart(HandlerCollection.java:229)
    at
org.eclipse.jetty.server.handler.ContextHandlerCollection.doStart(ContextHandlerCollection.java:172)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.server.handler.HandlerWrapper.doStart(HandlerWrapper.java:95)
    at org.eclipse.jetty.server.Server.doStart(Server.java:282)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:199)
    at org.apache.spark.ui.JettyUtils$$anonfun$4.apply(JettyUtils.scala:209)
    at org.apache.spark.ui.JettyUtils$$anonfun$4.apply(JettyUtils.scala:209)
    at
org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1454)
    at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
    at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1450)
    at
org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:209)
    at org.apache.spark.ui.WebUI.bind(WebUI.scala:102)
    at org.apache.spark.SparkContext.<init>(SparkContext.scala:224)
    at
org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:53)
    at mgm.tp.bigdata.tempGeoKmeans.Spark.SparkMain.main(SparkMain.java:37)
15/05/13 15:00:48 WARN component.AbstractLifeCycle: FAILED
org.eclipse.jetty.server.Server@2a9b5828: java.lang.NoSuchMethodError:
org.eclipse.jetty.server.ResourceCache.<init>(Lorg/eclipse/jetty/http/MimeTypes;)V
java.lang.NoSuchMethodError:
org.eclipse.jetty.server.ResourceCache.<init>(Lorg/eclipse/jetty/http/MimeTypes;)V
    at
org.eclipse.jetty.servlet.NIOResourceCache.<init>(NIOResourceCache.java:41)
    at
org.eclipse.jetty.servlet.DefaultServlet.init(DefaultServlet.java:223)
    at javax.servlet.GenericServlet.init(GenericServlet.java:244)
    at
org.eclipse.jetty.servlet.ServletHolder.initServlet(ServletHolder.java:442)
    at
org.eclipse.jetty.servlet.ServletHolder.doStart(ServletHolder.java:270)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:721)
    at
org.eclipse.jetty.servlet.ServletContextHandler.startContext(ServletContextHandler.java:279)
    at
org.eclipse.jetty.server.handler.ContextHandler.doStart(ContextHandler.java:717)
    at
org.eclipse.jetty.servlet.ServletContextHandler.doStart(ServletContextHandler.java:155)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.server.handler.HandlerCollection.doStart(HandlerCollection.java:229)
    at
org.eclipse.jetty.server.handler.ContextHandlerCollection.doStart(ContextHandlerCollection.java:172)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.server.handler.HandlerWrapper.doStart(HandlerWrapper.java:95)
    at org.eclipse.jetty.server.Server.doStart(Server.java:282)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:199)
    at org.apache.spark.ui.JettyUtils$$anonfun$4.apply(JettyUtils.scala:209)
    at org.apache.spark.ui.JettyUtils$$anonfun$4.apply(JettyUtils.scala:209)
    at
org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1454)
    at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
    at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1450)
    at
org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:209)
    at org.apache.spark.ui.WebUI.bind(WebUI.scala:102)
    at org.apache.spark.SparkContext.<init>(SparkContext.scala:224)
    at
org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:53)
    at mgm.tp.bigdata.tempGeoKmeans.Spark.SparkMain.main(SparkMain.java:37)
Exception in thread "main" java.lang.NoSuchMethodError:
org.eclipse.jetty.server.ResourceCache.<init>(Lorg/eclipse/jetty/http/MimeTypes;)V
    at
org.eclipse.jetty.servlet.NIOResourceCache.<init>(NIOResourceCache.java:41)
    at
org.eclipse.jetty.servlet.DefaultServlet.init(DefaultServlet.java:223)
    at javax.servlet.GenericServlet.init(GenericServlet.java:244)
    at
org.eclipse.jetty.servlet.ServletHolder.initServlet(ServletHolder.java:442)
    at
org.eclipse.jetty.servlet.ServletHolder.doStart(ServletHolder.java:270)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:721)
    at
org.eclipse.jetty.servlet.ServletContextHandler.startContext(ServletContextHandler.java:279)
    at
org.eclipse.jetty.server.handler.ContextHandler.doStart(ContextHandler.java:717)
    at
org.eclipse.jetty.servlet.ServletContextHandler.doStart(ServletContextHandler.java:155)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.server.handler.HandlerCollection.doStart(HandlerCollection.java:229)
    at
org.eclipse.jetty.server.handler.ContextHandlerCollection.doStart(ContextHandlerCollection.java:172)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.eclipse.jetty.server.handler.HandlerWrapper.doStart(HandlerWrapper.java:95)
    at org.eclipse.jetty.server.Server.doStart(Server.java:282)
    at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
    at
org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:199)
    at org.apache.spark.ui.JettyUtils$$anonfun$4.apply(JettyUtils.scala:209)
    at org.apache.spark.ui.JettyUtils$$anonfun$4.apply(JettyUtils.scala:209)
    at
org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1454)
    at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
    at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1450)
    at
org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:209)
    at org.apache.spark.ui.WebUI.bind(WebUI.scala:102)
    at org.apache.spark.SparkContext.<init>(SparkContext.scala:224)
    at
org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:53)
    at mgm.tp.bigdata.tempGeoKmeans.Spark.SparkMain.main(SparkMain.java:37)

in my class SparkMain line 37 are: JavaSparkContext sc = new
JavaSparkContext(conf);

Reply via email to