Dandandan commented on code in PR #18972:
URL: https://github.com/apache/datafusion/pull/18972#discussion_r2597222745
##########
datafusion/physical-plan/src/joins/hash_join/stream.rs:
##########
@@ -280,21 +289,35 @@ pub(super) fn lookup_join_hashmap(
hashes_buffer: &[u64],
limit: usize,
offset: JoinHashMapOffset,
+ probe_indices_buffer: &mut Vec<u32>,
+ build_indices_buffer: &mut Vec<u64>,
) -> Result<(UInt64Array, UInt32Array, Option<JoinHashMapOffset>)> {
- let (probe_indices, build_indices, next_offset) =
- build_hashmap.get_matched_indices_with_limit_offset(hashes_buffer,
limit, offset);
-
- let build_indices: UInt64Array = build_indices.into();
- let probe_indices: UInt32Array = probe_indices.into();
-
+ let next_offset = build_hashmap.get_matched_indices_with_limit_offset(
+ hashes_buffer,
+ limit,
+ offset,
+ probe_indices_buffer,
+ build_indices_buffer,
+ );
+
+ let build_indices_unfiltered: UInt64Array =
+ std::mem::take(build_indices_buffer).into();
+ let probe_indices_unfiltered: UInt32Array =
+ std::mem::take(probe_indices_buffer).into();
+
+ // TODO: optimize equal_rows_arr to avoid allocation of new buffers
Review Comment:
Added some link, updated the issue description
##########
datafusion/physical-plan/src/joins/hash_join/stream.rs:
##########
@@ -280,21 +289,35 @@ pub(super) fn lookup_join_hashmap(
hashes_buffer: &[u64],
limit: usize,
offset: JoinHashMapOffset,
+ probe_indices_buffer: &mut Vec<u32>,
+ build_indices_buffer: &mut Vec<u64>,
) -> Result<(UInt64Array, UInt32Array, Option<JoinHashMapOffset>)> {
- let (probe_indices, build_indices, next_offset) =
- build_hashmap.get_matched_indices_with_limit_offset(hashes_buffer,
limit, offset);
-
- let build_indices: UInt64Array = build_indices.into();
- let probe_indices: UInt32Array = probe_indices.into();
-
+ let next_offset = build_hashmap.get_matched_indices_with_limit_offset(
+ hashes_buffer,
+ limit,
+ offset,
+ probe_indices_buffer,
+ build_indices_buffer,
+ );
+
+ let build_indices_unfiltered: UInt64Array =
+ std::mem::take(build_indices_buffer).into();
+ let probe_indices_unfiltered: UInt32Array =
+ std::mem::take(probe_indices_buffer).into();
+
+ // TODO: optimize equal_rows_arr to avoid allocation of new buffers
Review Comment:
Added a link to https://github.com/apache/datafusion/issues/12131, updated
the issue description
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
