Dandandan commented on code in PR #1192:
URL: https://github.com/apache/datafusion-comet/pull/1192#discussion_r1896016551
##########
native/core/benches/shuffle_writer.rs:
##########
@@ -35,23 +35,42 @@ fn criterion_benchmark(c: &mut Criterion) {
group.bench_function("shuffle_writer: encode (no compression))", |b| {
let batch = create_batch(8192, true);
let mut buffer = vec![];
- let mut cursor = Cursor::new(&mut buffer);
let ipc_time = Time::default();
- b.iter(|| write_ipc_compressed(&batch, &mut cursor,
&CompressionCodec::None, &ipc_time));
+ b.iter(|| {
+ buffer.clear();
+ let mut cursor = Cursor::new(&mut buffer);
+ write_ipc_compressed(&batch, &mut cursor, &CompressionCodec::None,
&ipc_time)
+ });
+ });
+ group.bench_function("shuffle_writer: encode and compress (lz4)", |b| {
+ let batch = create_batch(8192, true);
+ let mut buffer = vec![];
+ let ipc_time = Time::default();
+ b.iter(|| {
+ buffer.clear();
+ let mut cursor = Cursor::new(&mut buffer);
+ write_ipc_compressed(&batch, &mut cursor,
&CompressionCodec::Lz4Frame, &ipc_time)
+ });
});
group.bench_function("shuffle_writer: encode and compress (zstd level 1)",
|b| {
let batch = create_batch(8192, true);
let mut buffer = vec![];
- let mut cursor = Cursor::new(&mut buffer);
let ipc_time = Time::default();
- b.iter(|| write_ipc_compressed(&batch, &mut cursor,
&CompressionCodec::Zstd(1), &ipc_time));
+ b.iter(|| {
+ buffer.clear();
+ let mut cursor = Cursor::new(&mut buffer);
+ write_ipc_compressed(&batch, &mut cursor,
&CompressionCodec::Zstd(1), &ipc_time)
Review Comment:
I think zstd should have faster negative levels as well (-4 or -5 might come
clos), would be interesting to see how it compares. Not sure if it is available
in the rust bindings.
##########
native/core/benches/shuffle_writer.rs:
##########
@@ -35,23 +35,42 @@ fn criterion_benchmark(c: &mut Criterion) {
group.bench_function("shuffle_writer: encode (no compression))", |b| {
let batch = create_batch(8192, true);
let mut buffer = vec![];
- let mut cursor = Cursor::new(&mut buffer);
let ipc_time = Time::default();
- b.iter(|| write_ipc_compressed(&batch, &mut cursor,
&CompressionCodec::None, &ipc_time));
+ b.iter(|| {
+ buffer.clear();
+ let mut cursor = Cursor::new(&mut buffer);
+ write_ipc_compressed(&batch, &mut cursor, &CompressionCodec::None,
&ipc_time)
+ });
+ });
+ group.bench_function("shuffle_writer: encode and compress (lz4)", |b| {
+ let batch = create_batch(8192, true);
+ let mut buffer = vec![];
+ let ipc_time = Time::default();
+ b.iter(|| {
+ buffer.clear();
+ let mut cursor = Cursor::new(&mut buffer);
+ write_ipc_compressed(&batch, &mut cursor,
&CompressionCodec::Lz4Frame, &ipc_time)
+ });
});
group.bench_function("shuffle_writer: encode and compress (zstd level 1)",
|b| {
let batch = create_batch(8192, true);
let mut buffer = vec![];
- let mut cursor = Cursor::new(&mut buffer);
let ipc_time = Time::default();
- b.iter(|| write_ipc_compressed(&batch, &mut cursor,
&CompressionCodec::Zstd(1), &ipc_time));
+ b.iter(|| {
+ buffer.clear();
+ let mut cursor = Cursor::new(&mut buffer);
+ write_ipc_compressed(&batch, &mut cursor,
&CompressionCodec::Zstd(1), &ipc_time)
Review Comment:
I think zstd should have faster negative levels as well (-4 or -5 might come
close), would be interesting to see how it compares. Not sure if it is
available in the rust bindings.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
