jayzhan211 commented on PR #25497:
URL: https://github.com/apache/datafusion/pull/25497#issuecomment-5814129630
The new tests only coalesce Int64, but `concat` works differently for
Dictionary (it merges the dictionary values) and Utf8View (it appends data
buffers). Please add one case for each that coalesces past 64 rows and checks
the evaluated order. It's fine to handle this in a follow-up.
```rs
#[test]
fn ordered_array_agg_coalesces_dictionary_and_view_payloads() -> Result<()> {
use arrow::array::{DictionaryArray, Int64Array, StringViewArray};
use arrow::datatypes::Int8Type;
let dictionary_type =
DataType::Dictionary(Box::new(DataType::Int8),
Box::new(DataType::Utf8));
for payload_type in [dictionary_type, DataType::Utf8View] {
let mut acc = ordered_accumulator(
payload_type.clone(),
DataType::Int64,
SortOptions::new(false, false),
false,
false,
)?;
// Longer than 12 bytes so view payloads are stored in data buffers.
let value = |i: usize| format!("payload-value-{i:03}");
for i in (0..100).rev() {
let values: ArrayRef = match payload_type {
DataType::Utf8View => {
Arc::new(StringViewArray::from_iter_values([value(i)]))
}
_ => Arc::new(
[value(i).as_str()]
.into_iter()
.collect::<DictionaryArray<Int8Type>>(),
),
};
let ordering =
Arc::new(Int64Array::from(vec![i64::try_from(i).unwrap()]))
as ArrayRef;
acc.update_batch(&[values, ordering])?;
}
assert_eq!(
acc.batches
.iter()
.map(|batch| batch.len())
.collect::<Vec<_>>(),
vec![64, 36],
);
let ScalarValue::List(result) = acc.evaluate()? else {
return internal_err!("expect list");
};
let values = cast(result.values(), &DataType::Utf8)?;
let values = values.as_string::<i32>();
for i in 0..100 {
assert_eq!(values.value(i), value(i));
}
}
Ok(())
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
