comphead commented on code in PR #7674: URL: https://github.com/apache/datafusion-comet/pull/7674#discussion_r4619979105
########## docs/source/contributor-guide/expression-audits/agg_funcs.md: ########## @@ -118,4 +119,8 @@ - Spark 4.2.0 (audited 2026-10-12): the `TreeMap` is chosen for every pivot column type without proper equals (`!TypeUtils.typeWithProperEquals`), which adds binary and strings with a non-binary-equality collation, so binary pivot values now match by content. A `findPivotIndex` wrapper returns `-1` for a null key on the `TreeMap` path instead of throwing. - `CometPivotFirst` (in `spark/src/main/scala/org/apache/comet/serde/aggregates.scala`) forwards the aggregate to the native `SparkPivotFirst` UDAF (`native/spark-expr/src/agg_funcs/pivot_first.rs`) when the value type is in the supported set and the pivot column is a Boolean, Byte, Short, Int, Long, Float, Double, Decimal, UTF8_BINARY String, Date, Timestamp or TimestampNTZ. Those are the types Spark keys by value equality on every version, which the native lookup reproduces, including the float rules. Other pivot columns fall back, as does a list where `indexSize` is smaller than the list (duplicates, including `0.0` next to `-0.0`). State layout matches Spark's `aggBufferAttributes` (one scalar column per pivot slot) so the shuffle schema between Partial and Final stays consistent. The result reassembles the slots into a `ListArray` matching `PivotFirst.dataType = ArrayType(valueDataType)`. +## sum + +- Performance (tuned 2026-11-17): the grouped decimal accumulator loads the result's precision bounds once and compares against them on every update, merge, and evaluate, instead of calling `Decimal128Type::is_valid_decimal_precision`, which release builds keep out of line. 9% to 12% faster on most shapes even in bench builds, which inline that call. Benchmark: `benches/decimal_groups.rs`. Review Comment: I don't have an x86-64 machine to rerun `decimal_groups` on, so I dropped the number and kept the call that release builds no longer make, with the PR link. ########## native/spark-expr/benches/decimal_groups.rs: ########## @@ -0,0 +1,235 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Grouped decimal SUM and AVG accumulators, driven directly through +//! `GroupsAccumulator::update_batch` so the numbers cover the accumulators and not grouping or +//! the execution framework. + +use arrow::array::{ArrayRef, Decimal128Array}; +use arrow::datatypes::{DataType, Field, Schema}; +use criterion::{criterion_group, criterion_main, Criterion}; +use datafusion::functions_aggregate::sum::sum_udaf; +use datafusion::logical_expr::function::AccumulatorArgs; +use datafusion::logical_expr::{AggregateUDFImpl, EmitTo, GroupsAccumulator}; +use datafusion_comet_spark_expr::{create_query_context_map, AvgDecimal, EvalMode, SumDecimal}; +use std::hint::black_box; +use std::sync::Arc; + +const ROWS: usize = 8192; +const BATCHES: usize = 10; + +/// TPC-H Q1 grouping: 4 groups in random order, weighted 25%, 0.6%, 50% and 25%. +fn q1_groups() -> Vec<usize> { + let mut x: u64 = 0x9E37_79B9_7F4A_7C15; + (0..ROWS) + .map(|_| { + x = x + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + match (x >> 33) % 1000 { + 0..=249 => 0, + 250..=255 => 1, + 256..=755 => 2, + _ => 3, + } + }) + .collect() +} + +fn decimals(precision: u8, scale: i8, value: impl Fn(usize) -> Option<i128>) -> ArrayRef { + Arc::new( + (0..ROWS) + .map(value) + .collect::<Decimal128Array>() + .with_precision_and_scale(precision, scale) + .unwrap(), + ) +} + +/// One input shape: values, group ids, and the (precision, scale) of the input and of the sum. +struct Shape<'a> { + name: &'a str, + values: &'a ArrayRef, + groups: &'a [usize], + num_groups: usize, + types: ((u8, i8), (u8, i8)), +} + +impl<'a> Shape<'a> { + fn new( + name: &'a str, + values: &'a ArrayRef, + groups: &'a [usize], + num_groups: usize, + types: ((u8, i8), (u8, i8)), + ) -> Self { + Self { + name, + values, + groups, + num_groups, + types, + } + } +} + +fn bench( + c: &mut Criterion, + name: &str, + make: &dyn Fn() -> Box<dyn GroupsAccumulator>, + shape: &Shape, +) { + let values = [Arc::clone(shape.values)]; + c.bench_function(name, |b| { + b.iter(|| { + let mut acc = make(); + for _ in 0..BATCHES { + acc.update_batch(&values, shape.groups, None, shape.num_groups) + .unwrap(); + } + black_box(acc.evaluate(EmitTo::All).unwrap()) + }) + }); +} + +fn criterion_benchmark(c: &mut Criterion) { + let q1 = q1_groups(); + let cyclic: Vec<usize> = (0..ROWS).map(|i| i % 1024).collect(); + // A bijection on 0..65536, so each batch touches 8192 scattered groups. + let scattered: Vec<usize> = (0..ROWS) + .map(|i| (i as u64 * 2654435761 % 65536) as usize) + .collect(); + + // Q1's `l_quantity`: DECIMAL(15,2) values 1.00..50.00 summed as DECIMAL(25,2). + let quantity = decimals(15, 2, |i| Some(((i % 50) as i128 + 1) * 100)); + let quantity_nulls = decimals(15, 2, |i| { + (i % 10 != 0).then(|| ((i % 50) as i128 + 1) * 100) + }); + // A DECIMAL(38,6) input keeps the sum at the capped precision 38. Values near 1e5 stay far + // from it. Values of 1e27 to 5e28 come within a few batches of it without overflowing. Review Comment: You're right. I checked the shape: no group mixes signs, the largest running sum after the 10 batches is 2.8e30 (2.8% of 1e32), and one batch's absolute values sum to 2.09e32. I took your comment. ########## native/spark-expr/src/agg_funcs/avg_decimal.rs: ########## @@ -398,7 +398,7 @@ impl AvgDecimalGroupsAccumulator { target_precision, target_scale, sum_data_type: sum_data_type.clone(), - sum_precision, + sum_bounds: Decimal128Bounds::new(sum_precision), Review Comment: Added as `grouped_latches_at_the_sum_precision_below_38`. Every existing grouped AVG test uses precision 38 for the sum and the result, so none of them could catch this without changing what it tests. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
