Github user kayousterhout commented on a diff in the pull request:
https://github.com/apache/spark/pull/159#discussion_r10640410
--- Diff:
core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala ---
@@ -298,6 +298,94 @@ class TaskSetManagerSuite extends FunSuite with
LocalSparkContext with Logging {
}
}
+ test("executors should be blacklisted after task failure, in spite of
locality preferences") {
+ val rescheduleDelay = 300L
+ val conf = new SparkConf().
+ set("spark.task.executorBlacklistTimeout", rescheduleDelay.toString).
+ // dont wait to jump locality levels in this test
+ set("spark.locality.wait", "0")
+
+ sc = new SparkContext("local", "test", conf)
+ // two executors on same host, one on different.
+ val sched = new FakeTaskScheduler(sc, ("exec1", "host1"),
+ ("exec1.1", "host1"), ("exec2", "host2"))
+ // affinity to exec1 on host1 - which we will fail.
+ val taskSet = FakeTask.createTaskSet(1, Seq(TaskLocation("host1",
"exec1")))
+ // we need actual time measurement
--- End diff --
remove this comment (since you're using a fake clock -- not the actual
clock)
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
