comphead commented on code in PR #2258:
URL: https://github.com/apache/datafusion-comet/pull/2258#discussion_r2373379906
##########
native/core/src/execution/planner.rs:
##########
@@ -2344,16 +2351,58 @@ impl PhysicalPlanner {
))
}
PartitioningStruct::RangePartition(range_partition) => {
+ // Generate the lexical ordering for comparisons
let exprs: Result<Vec<PhysicalSortExpr>, ExecutionError> =
range_partition
.sort_orders
.iter()
.map(|expr| self.create_sort_expr(expr,
Arc::clone(&input_schema)))
.collect();
let lex_ordering = LexOrdering::new(exprs?).unwrap();
+
+ // Generate the row converter for comparing incoming batches
to boundary rows
+ let sort_fields: Vec<SortField> = lex_ordering
+ .iter()
+ .map(|sort_expr| {
+ sort_expr
+ .expr
+ .data_type(input_schema.as_ref())
+ .map(|dt| SortField::new_with_options(dt,
sort_expr.options))
+ })
+ .collect::<Result<Vec<_>, _>>()?;
+
+ // Deserialize the literals to columnar collections of
ScalarValues
+ let mut scalar_values: Vec<Vec<ScalarValue>> = vec![vec![];
lex_ordering.len()];
+ for boundary_row in &range_partition.boundary_rows {
+ // For each serialized expr in a boundary row, convert to
a Literal
+ // expression, then extract the ScalarValue from the
Literal and push it
+ // into the collection of ScalarValues
+ for col_idx in 0..lex_ordering.len() {
+ let expr = self.create_expr(
+ &boundary_row.partition_bounds[col_idx],
+ Arc::clone(&input_schema),
+ )?;
+ let literal_expr =
+
expr.as_any().downcast_ref::<Literal>().expect("Literal");
+
scalar_values[col_idx].push(literal_expr.value().clone());
+ }
+ }
+
+ // Convert the collection of ScalarValues to collection of
Arrow Arrays
+ let arrays: Vec<ArrayRef> = scalar_values
+ .iter()
+ .map(|scalar_vec|
ScalarValue::iter_to_array(scalar_vec.iter().cloned()))
+ .collect::<Result<Vec<_>, _>>()?;
+
+ // Create a RowConverter and use to create OwnedRows from the
Arrays
+ let converter = RowConverter::new(sort_fields)?;
+ let rows = converter.convert_columns(&arrays)?;
+ let owned_rows: Vec<OwnedRow> = rows.iter().map(|row|
row.owned()).collect();
Review Comment:
maybe we comment here what is `owned_rows` here?
Is it actual rows before shuffle?
For simplicity attaching a diagram with RR flow
```
[ Before Shuffle ]
Executor 1: (5,A) (15,B) (30,C) Executor 2: (40,D) (70,E) (90,F)
| |
v v
[ Shuffle Write: Buckets on Disk ]
Executor 1: [P0:(5,15,30)] [P1: ] [P2: ] Executor 2: [P0: ] [P1:(40)]
[P2:(70,90)]
| |
v v
[ Shuffle Read: Reducers Fetch Buckets ]
Reducer P0 <---- E1.Bucket0 + E2.Bucket0 ----> (5,15,30)
Reducer P1 <---- E1.Bucket1 + E2.Bucket1 ----> (40)
Reducer P2 <---- E1.Bucket2 + E2.Bucket2 ----> (70,90)
[ After Shuffle = Range Partitions ]
P0: (5,15,30) P1: (40) P2: (70,90)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
