comphead commented on code in PR #11218:
URL: https://github.com/apache/datafusion/pull/11218#discussion_r1678505039
##########
datafusion/physical-plan/src/joins/sort_merge_join.rs:
##########
@@ -602,15 +622,38 @@ impl BufferedBatch {
+ mem::size_of::<Range<usize>>()
+ mem::size_of::<usize>();
+ let num_rows = batch.num_rows();
BufferedBatch {
batch,
range,
join_arrays,
null_joined: vec![],
size_estimation,
join_filter_failed_idxs: HashSet::new(),
+ num_rows,
+ spill_file: None,
}
}
+
+ fn spill_to_disk(
+ &mut self,
+ path: RefCountedTempFile,
+ buffered_schema: SchemaRef,
+ batch_size: usize,
+ ) -> Result<()> {
+ let batch = std::mem::replace(
+ &mut self.batch,
+ RecordBatch::new_empty(Arc::clone(&buffered_schema)),
+ );
+ let _ = spill_record_batch_by_size(
+ batch,
+ path.path().into(),
+ buffered_schema,
+ batch_size,
+ );
+ self.spill_file = Some(path);
Review Comment:
The batch can be spilled only once to the disk. I'm not sure if there is a
necessity of spilling multiple times. But even if so the spill file will be
pointing to the most recent valid spill file
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
