Hi, I’m trying to run the Spark test suite on an EC2 instance, but I can’t get Yarn tests to pass. The hostname I get on that machine is not resolvable, but adding a line in /etc/hosts makes the other tests pass, except for Yarn tests.
Any help is greatly appreciated! thanks, iulian ubuntu@ip-172-30-0-248:~/spark$ cat /etc/hosts 127.0.0.1 localhost 172.30.0.248 ip-172-30-0-248 # The following lines are desirable for IPv6 capable hosts ::1 ip6-localhost ip6-loopback fe00::0 ip6-localnet ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters ff02::3 ip6-allhosts and the exception: [info] - run Spark in yarn-client mode *** FAILED *** (2 seconds, 249 milliseconds) [info] java.net.UnknownHostException: Invalid host name: local host is: (unknown); destination host is: "ip-172-30-0-248":57041; java.net.UnknownHostException; For more details see: http://wiki.apache.org/hadoop/UnknownHost [info] at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) [info] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57) [info] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) [info] at java.lang.reflect.Constructor.newInstance(Constructor.java:526) [info] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:783) [info] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:742) [info] at org.apache.hadoop.ipc.Client$Connection.<init>(Client.java:400) [info] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1448) [info] at org.apache.hadoop.ipc.Client.call(Client.java:1377) [info] at org.apache.hadoop.ipc.Client.call(Client.java:1359) [info] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206) [info] at com.sun.proxy.$Proxy69.getClusterMetrics(Unknown Source) [info] at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationClientProtocolPBClientImpl.getClusterMetrics(ApplicationClientProtocolPBClientImpl.java:152) [info] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [info] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) [info] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [info] at java.lang.reflect.Method.invoke(Method.java:606) [info] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:186) [info] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) [info] at com.sun.proxy.$Proxy70.getClusterMetrics(Unknown Source) [info] at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.getYarnClusterMetrics(YarnClientImpl.java:294) [info] at org.apache.spark.deploy.yarn.Client$$anonfun$submitApplication$1.apply(Client.scala:91) [info] at org.apache.spark.deploy.yarn.Client$$anonfun$submitApplication$1.apply(Client.scala:91) [info] at org.apache.spark.Logging$class.logInfo(Logging.scala:59) [info] at org.apache.spark.deploy.yarn.Client.logInfo(Client.scala:49) [info] at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:90) [info] at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:57) [info] at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:141) [info] at org.apache.spark.SparkContext.<init>(SparkContext.scala:343) [info] at org.apache.spark.deploy.yarn.YarnClusterDriver$.main(YarnClusterSuite.scala:175) [info] at org.apache.spark.deploy.yarn.YarnClusterSuite$$anonfun$1.apply$mcV$sp(YarnClusterSuite.scala:118) [info] at org.apache.spark.deploy.yarn.YarnClusterSuite$$anonfun$1.apply(YarnClusterSuite.scala:116) [info] at org.apache.spark.deploy.yarn.YarnClusterSuite$$anonfun$1.apply(YarnClusterSuite.scala:116) [info] at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22) [info] at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) [info] at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) [info] at org.scalatest.Transformer.apply(Transformer.scala:22) [info] at org.scalatest.Transformer.apply(Transformer.scala:20) [info] at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:166) [info] at org.scalatest.Suite$class.withFixture(Suite.scala:1122) [info] at org.scalatest.FunSuite.withFixture(FunSuite.scala:1555) [info] at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:163) [info] at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) [info] at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) [info] at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306) [info] at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:175) [info] at org.scalatest.FunSuite.runTest(FunSuite.scala:1555) [info] at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) [info] at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) [info] at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413) [info] at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401) [info] at scala.collection.immutable.List.foreach(List.scala:318) [info] at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) [info] at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396) [info] at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483) [info] at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:208) [info] at org.scalatest.FunSuite.runTests(FunSuite.scala:1555) [info] at org.scalatest.Suite$class.run(Suite.scala:1424) [info] at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1555) [info] at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) [info] at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) [info] at org.scalatest.SuperEngine.runImpl(Engine.scala:545) [info] at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:212) [info] at org.apache.spark.deploy.yarn.YarnClusterSuite.org$scalatest$BeforeAndAfterAll$$super$run(YarnClusterSuite.scala:36) [info] at org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:257) [info] at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:256) [info] at org.apache.spark.deploy.yarn.YarnClusterSuite.run(YarnClusterSuite.scala:36) [info] at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:462) [info] at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:671) [info] at sbt.ForkMain$Run$2.call(ForkMain.java:294) [info] at sbt.ForkMain$Run$2.call(ForkMain.java:284) [info] at java.util.concurrent.FutureTask.run(FutureTask.java:262) [info] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [info] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [info] at java.lang.Thread.run(Thread.java:745) [info] Cause: java.net.UnknownHostException: [info] at org.apache.hadoop.ipc.Client$Connection.<init>(Client.java:400) [info] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1448) [info] at org.apache.hadoop.ipc.Client.call(Client.java:1377) [info] at org.apache.hadoop.ipc.Client.call(Client.java:1359) [info] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206) [info] at com.sun.proxy.$Proxy69.getClusterMetrics(Unknown Source) [info] at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationClientProtocolPBClientImpl.getClusterMetrics(ApplicationClientProtocolPBClientImpl.java:152) [info] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [info] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) [info] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [info] at java.lang.reflect.Method.invoke(Method.java:606) [info] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:186) [info] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) [info] at com.sun.proxy.$Proxy70.getClusterMetrics(Unknown Source) [info] at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.getYarnClusterMetrics(YarnClientImpl.java:294) [info] at org.apache.spark.deploy.yarn.Client$$anonfun$submitApplication$1.apply(Client.scala:91) [info] at org.apache.spark.deploy.yarn.Client$$anonfun$submitApplication$1.apply(Client.scala:91) [info] at org.apache.spark.Logging$class.logInfo(Logging.scala:59) [info] at org.apache.spark.deploy.yarn.Client.logInfo(Client.scala:49) [info] at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:90) [info] at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:57) [info] at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:141) [info] at org.apache.spark.SparkContext.<init>(SparkContext.scala:343) [info] at org.apache.spark.deploy.yarn.YarnClusterDriver$.main(YarnClusterSuite.scala:175) [info] at org.apache.spark.deploy.yarn.YarnClusterSuite$$anonfun$1.apply$mcV$sp(YarnClusterSuite.scala:118) [info] at org.apache.spark.deploy.yarn.YarnClusterSuite$$anonfun$1.apply(YarnClusterSuite.scala:116) [info] at org.apache.spark.deploy.yarn.YarnClusterSuite$$anonfun$1.apply(YarnClusterSuite.scala:116) [info] at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22) [info] at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) [info] at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) [info] at org.scalatest.Transformer.apply(Transformer.scala:22) [info] at org.scalatest.Transformer.apply(Transformer.scala:20) [info] at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:166) [info] at org.scalatest.Suite$class.withFixture(Suite.scala:1122) -- -- Iulian Dragos ------ Reactive Apps on the JVM www.typesafe.com
