https://gcc.gnu.org/g:3adb3325954177939d59031055fc08a81ae21be3
commit r17-2478-g3adb3325954177939d59031055fc08a81ae21be3 Author: Richard Biener <[email protected]> Date: Wed Jul 15 11:33:50 2026 +0200 Improve BB vectorization of reductions When there's not a uniform chain of operations gathered from the reduction operation chain we currently simply fail and to make success more likely we strip off the last operation to make the number of lanes at least even. This isn't ideal and somewhat random as can be seen in PR126028 which is the motivating case and has a three lane reduction. So the following removes the early stripping down to an even number of lanes and uses SLP discovery of the whole group to direct re-analysis of the larger of the matching or non-matching part. For gcc.dg/vect/pr106081.c we now BB vectorize parts which just confuses the loop vectorization dump scanning, so disable it. PR tree-optimization/126028 * tree-vect-slp.cc (vect_slp_check_for_roots): Do not force the BB reduction root to have an even number of lanes. (vect_build_slp_instance): For failed discovery of a BB reduction attempt to re-try discovery on the matching or non-matching part. * gcc.dg/vect/bb-slp-reduc-2.c: New testcase. * gcc.dg/vect/bb-slp-reduc-3.c: Likewise. * gcc.dg/vect/pr106081.c: Disable BB vectorization. Diff: --- gcc/testsuite/gcc.dg/vect/bb-slp-reduc-2.c | 12 ++++++ gcc/testsuite/gcc.dg/vect/bb-slp-reduc-3.c | 12 ++++++ gcc/testsuite/gcc.dg/vect/pr106081.c | 2 +- gcc/tree-vect-slp.cc | 68 +++++++++++++++++++++++++----- 4 files changed, 83 insertions(+), 11 deletions(-) diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-reduc-2.c b/gcc/testsuite/gcc.dg/vect/bb-slp-reduc-2.c new file mode 100644 index 000000000000..d4cfadfaa3b0 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/bb-slp-reduc-2.c @@ -0,0 +1,12 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target vect_int } */ + +int foo (int *a, int *b, int c) +{ + return (c ^ 1) + ((a[0] | b[0]) + (a[1] | b[1]) + (a[2] | b[2]) + (a[3] | b[3])); +} + +/* Make sure that we pick matching lanes when attempting to BB vectorize + a reduction rather than arbitrarily cutting back to the number of + vector lanes. */ +/* { dg-final { scan-tree-dump "optimized: basic block part vectorized" "slp2" { target { vect_hw_misalign && { x86_64-*-* i?86-*-* aarch64-*-* } } } } } */ diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-reduc-3.c b/gcc/testsuite/gcc.dg/vect/bb-slp-reduc-3.c new file mode 100644 index 000000000000..acb9bf59f76a --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/bb-slp-reduc-3.c @@ -0,0 +1,12 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target vect_int } */ + +int foo (int *a, int *b, int c) +{ + return (c ^ 1) + ((a[0] | b[0]) + (a[1] | b[1]) + (a[2] | b[2]) + (a[3] | b[3]) + (a[4] | b[4])); +} + +/* Make sure that we pick matching lanes when attempting to BB vectorize + a reduction rather than arbitrarily cutting back to the number of + vector lanes. */ +/* { dg-final { scan-tree-dump "optimized: basic block part vectorized" "slp2" { target { vect_hw_misalign && { x86_64-*-* i?86-*-* aarch64-*-* } } } } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr106081.c b/gcc/testsuite/gcc.dg/vect/pr106081.c index 8f97af2d642b..23b3127be0aa 100644 --- a/gcc/testsuite/gcc.dg/vect/pr106081.c +++ b/gcc/testsuite/gcc.dg/vect/pr106081.c @@ -1,5 +1,5 @@ /* { dg-do compile } */ -/* { dg-additional-options "-ffast-math -fdump-tree-optimized" } */ +/* { dg-additional-options "-ffast-math -fno-tree-slp-vectorize -fdump-tree-optimized" } */ /* { dg-additional-options "-mavx2" { target x86_64-*-* i?86-*-* } } */ /* { dg-require-effective-target vect_double } */ /* { dg-require-effective-target vect_unpack } */ diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc index 7abf0faa765e..850cb1efacc5 100644 --- a/gcc/tree-vect-slp.cc +++ b/gcc/tree-vect-slp.cc @@ -4321,6 +4321,63 @@ vect_build_slp_instance (vec_info *vinfo, vect_analyze_slp_instance now. */ gcc_assert (kind != slp_inst_kind_store || group_size == 1); + /* For BB vectorization we get failures only in case of the need of + unrolling, as otherwise we'll simply get operands built from scalars. + Iff there is any mismatches in the toplevel stmts those will prevail, + otherwise we get the non-power-of-two tail of the lanes failed. + For BB reductions we mainly want to catch the first case so we pick + a more useful subset of lanes to reduce. */ + if (kind == slp_inst_kind_bb_reduc && matches[0]) + { + unsigned n_matching = 0; + for (unsigned i = 0; i < group_size; ++i) + if (matches[i]) + n_matching++; + vec<stmt_vec_info> scalar_stmts2 = vNULL; + /* Try matched parts and put the rest to remain. */ + if (n_matching >= 2 && n_matching >= group_size / 2) + { + /* As we know the matches[] stmts match up, recursing for + non-power-of-two sizes will just force-fail the tail + for us at hopefully optimal vector size and succesfully + finish discovery. */ + scalar_stmts2.create (n_matching); + for (unsigned i = 0; i < group_size; ++i) + if (matches[i]) + scalar_stmts2.quick_push (scalar_stmts[i]); + else + remain.safe_push + (gimple_get_lhs (vect_orig_stmt (scalar_stmts[i])->stmt)); + } + /* Try the non-matching part. */ + else if (group_size - n_matching >= 2) + { + /* We do not know whether the !matches[] part matches, so avoid + cutting to a multiple of the vector size too early. We should + make progress by means of remain only growing and most of the + time prefering the matching[] part. */ + scalar_stmts2.create (scalar_stmts.length () - n_matching); + for (unsigned i = 0; i < group_size; ++i) + if (!matches[i]) + scalar_stmts2.quick_push (scalar_stmts[i]); + else + remain.safe_push + (gimple_get_lhs (vect_orig_stmt (scalar_stmts[i])->stmt)); + } + if (scalar_stmts2.exists ()) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, "Splitting %d " + "non-matching lanes to scalar remains\n", + scalar_stmts.length () - scalar_stmts2.length ()); + scalar_stmts.release (); + return vect_build_slp_instance (vinfo, kind, scalar_stmts2, + root_stmt_infos, remain, + max_tree_size, limit, bst_map, + force_single_lane); + } + } + /* Free the allocated memory. */ scalar_stmts.release (); @@ -9980,7 +10037,6 @@ vect_slp_check_for_roots (bb_vec_info bb_vinfo) /* ??? For now do not allow mixing ops or externs/constants. */ bool invalid = false; unsigned remain_cnt = 0; - unsigned last_idx = 0; for (unsigned i = 0; i < chain.length (); ++i) { if (chain[i].code != code) @@ -9995,13 +10051,7 @@ vect_slp_check_for_roots (bb_vec_info bb_vinfo) (chain[i].op)->stmt) != chain[i].op)) remain_cnt++; - else - last_idx = i; } - /* Make sure to have an even number of lanes as we later do - all-or-nothing discovery, not trying to split further. */ - if ((chain.length () - remain_cnt) & 1) - remain_cnt++; if (!invalid && chain.length () - remain_cnt > 1) { vec<stmt_vec_info> stmts; @@ -10014,9 +10064,7 @@ vect_slp_check_for_roots (bb_vec_info bb_vinfo) stmt_vec_info stmt_info; if (chain[i].dt == vect_internal_def && ((stmt_info = bb_vinfo->lookup_def (chain[i].op)), - gimple_get_lhs (stmt_info->stmt) == chain[i].op) - && (i != last_idx - || (stmts.length () & 1))) + gimple_get_lhs (stmt_info->stmt) == chain[i].op)) stmts.quick_push (stmt_info); else remain.quick_push (chain[i].op);
