akshaychitneni commented on code in PR #2633:
URL:
https://github.com/apache/datafusion-ballista/pull/2633#discussion_r4395391976
##########
ballista/scheduler/src/state/mod.rs:
##########
@@ -433,20 +454,31 @@ impl<T: 'static + AsLogicalPlan, U: 'static +
AsExecutionPlan> SchedulerState<T,
Ok(())
}
- /// Immediately reclaim intermediate shuffle data, then spawn a delayed
- /// future to clean up the remaining job data (final-stage output + job
- /// state) on both Scheduler and Executors.
- pub(crate) fn clean_up_successful_job(
+ /// Reclaim a finished job's shuffle data and schedule state cleanup. A job
+ /// backing a cache (recorded by [`on_job_terminal`](Self::on_job_terminal)
+ /// before this runs) keeps its final-stage data; unpinning is future work.
+ pub(crate) async fn clean_up_successful_job(
&self,
job_id: JobId,
intermediate_stage_ids: Vec<u32>,
) {
self.executor_manager
.clean_up_intermediate_job_data(job_id.clone(),
intermediate_stage_ids);
- self.executor_manager.clean_up_job_data_delayed(
- job_id.clone(),
- self.config.finished_job_data_clean_up_interval_seconds,
- );
+ let pinned = match self.cache_state.pinned_stages().await {
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
