Dandandan commented on code in PR #1192:
URL: https://github.com/apache/datafusion-comet/pull/1192#discussion_r1896016551
##########
native/core/benches/shuffle_writer.rs:
##########
@@ -35,23 +35,42 @@ fn criterion_benchmark(c: &mut Criterion) {
group.bench_function("shuffle_writer: encode (no compression))", |b| {
let batch = create_batch(8192, true);
let mut buffer = vec![];
- let mut cursor = Cursor::new(&mut buffer);
let ipc_time = Time::default();
- b.iter(|| write_ipc_compressed(&batch, &mut cursor,
&CompressionCodec::None, &ipc_time));
+ b.iter(|| {
+ buffer.clear();
+ let mut cursor = Cursor::new(&mut buffer);
+ write_ipc_compressed(&batch, &mut cursor, &CompressionCodec::None,
&ipc_time)
+ });
+ });
+ group.bench_function("shuffle_writer: encode and compress (lz4)", |b| {
+ let batch = create_batch(8192, true);
+ let mut buffer = vec![];
+ let ipc_time = Time::default();
+ b.iter(|| {
+ buffer.clear();
+ let mut cursor = Cursor::new(&mut buffer);
+ write_ipc_compressed(&batch, &mut cursor,
&CompressionCodec::Lz4Frame, &ipc_time)
+ });
});
group.bench_function("shuffle_writer: encode and compress (zstd level 1)",
|b| {
let batch = create_batch(8192, true);
let mut buffer = vec![];
- let mut cursor = Cursor::new(&mut buffer);
let ipc_time = Time::default();
- b.iter(|| write_ipc_compressed(&batch, &mut cursor,
&CompressionCodec::Zstd(1), &ipc_time));
+ b.iter(|| {
+ buffer.clear();
+ let mut cursor = Cursor::new(&mut buffer);
+ write_ipc_compressed(&batch, &mut cursor,
&CompressionCodec::Zstd(1), &ipc_time)
Review Comment:
I think zstd should have faster negative levels as well (-4 or -5), would be
interesting to see how it compares. Not sure if it is available in the rust
bindings.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
