peter-toth commented on code in PR #49094:
URL: https://github.com/apache/spark/pull/49094#discussion_r1879696412
##########
core/src/main/scala/org/apache/spark/status/AppStatusListener.scala:
##########
@@ -496,22 +496,25 @@ private[spark] class AppStatusListener(
val now = System.nanoTime()
// Check if there are any pending stages that match this job; mark those
as skipped.
- val it = liveStages.entrySet.iterator()
- while (it.hasNext()) {
- val e = it.next()
- if (job.stageIds.contains(e.getKey()._1)) {
- val stage = e.getValue()
+ liveStages.forEach { (key, stage) =>
+ val stageId = key._1
+ if (job.stageIds.contains(stageId)) {
if (v1.StageStatus.PENDING.equals(stage.status)) {
stage.status = v1.StageStatus.SKIPPED
job.skippedStages += stage.info.stageId
job.skippedTasks += stage.info.numTasks
+ liveJobs.values.filter(_.stageIds.contains(stageId)).foreach {
otherJob =>
Review Comment:
Can you please add some comments here why we need to check other jobs?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
