On Mon, 20 Jul 2026, liuhongt wrote:
> Update in V3:
> > > + if (SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_group
> >
> > ... I wonder about this, a slp_inst_kind_reduc_group is multiple
> > independent reductions (or one) in the same vector, so for that
> > we have SLP_TREE_LANES independent fold-left reductions of
> > VF lanes each. Esp. when VF == 1 this means the fold-left
> > reductions are "free" (this correlates well with the
> > heuristic to prefer smaller vectors)
> >
> > And for slp_inst_kind_reduc_chain we have a single fold-left
> > reduction with VF * SLP_TREE_LANES lanes.
> >
> > I guess for the overall sum this difference doesn't matter, but
> > you do not cover slp_inst_kind_reduc_chain at all here? So
> > add a || SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_chain?
> Added || SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_chain
>
>
> Bootstrapped and regtested on x86_64-pc-linux-gnu{-m32,}.
> Ok for trunk?
OK.
Thanks,
Richard.
> -------------------------
> Fold-left FP reductions are lowered to scalar operations on x86. In
> mixed-width loops the per-statement costs can therefore make a wider
> vector mode look better even though it lengthens the serial reduction
> chain.
>
> Count the fold-left reduction lanes from reduction groups and chains during
> finish_cost and use that as an extra loop-candidate preference. Do not let
> emulated sub-SSE modes win on this preference alone. Dump the lane count when
> present.
>
> gcc/ChangeLog:
>
> * config/i386/i386.cc (ix86_vector_costs): Add
> better_fold_left_reduc_than_p and m_num_fold_left_reduc_lanes.
> (ix86_vector_costs::ix86_vector_costs): Initialize it.
> (ix86_vector_costs::finish_cost): Count and dump fold-left
> reduction lanes.
> (ix86_vector_costs::better_fold_left_reduc_than_p): New function.
> (ix86_vector_costs::better_main_loop_than_p): Use it.
> (ix86_vector_costs::better_epilogue_loop_than_p): Likewise.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.target/i386/fold-left-reduc-cost.c: New test.
> * gcc.target/i386/fold-left-reduc-chain-cost.c: New test.
> * gcc.target/i386/vect-epilogues-6.c: Update expected vector size.
> * gcc.target/i386/vect-epilogues-7.c: Likewise.
> * gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c: Likewise.
> ---
> gcc/config/i386/i386.cc | 42 ++++++++++++++++++-
> .../costmodel/x86_64/costmodel-vect-epil-1.c | 7 ++--
> .../i386/fold-left-reduc-chain-cost.c | 19 +++++++++
> .../gcc.target/i386/fold-left-reduc-cost.c | 18 ++++++++
> .../gcc.target/i386/vect-epilogues-6.c | 3 +-
> .../gcc.target/i386/vect-epilogues-7.c | 4 +-
> 6 files changed, 84 insertions(+), 9 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c
> create mode 100644 gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c
>
> diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
> index e66958db7ac..8e22a43fbd1 100644
> --- a/gcc/config/i386/i386.cc
> +++ b/gcc/config/i386/i386.cc
> @@ -26172,6 +26172,7 @@ public:
>
> private:
>
> + bool better_fold_left_reduc_than_p (const vector_costs *) const;
> /* Estimate register pressure of the vectorized code. */
> void ix86_vect_estimate_reg_pressure ();
> /* Number of GENERAL_REGS/SSE_REGS used in the vectorizer, it's used for
> @@ -26188,6 +26189,8 @@ private:
> unsigned m_num_reduc[X86_REDUC_LAST];
> /* Don't do unroll if m_prefer_unroll is false, default is true. */
> bool m_prefer_unroll;
> + /* Scalar lanes in fold-left reductions. */
> + unsigned int m_num_fold_left_reduc_lanes;
> };
>
> ix86_vector_costs::ix86_vector_costs (vec_info* vinfo, bool
> costing_for_scalar)
> @@ -26197,7 +26200,8 @@ ix86_vector_costs::ix86_vector_costs (vec_info*
> vinfo, bool costing_for_scalar)
> m_num_avx256_vec_perm (),
> m_num_avx512_vec_perm (),
> m_num_reduc (),
> - m_prefer_unroll (true)
> + m_prefer_unroll (true),
> + m_num_fold_left_reduc_lanes (0)
> {}
>
> /* Implement targetm.vectorize.create_costs. */
> @@ -26903,6 +26907,20 @@ ix86_vector_costs::finish_cost (const vector_costs
> *scalar_costs)
> loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo);
> if (loop_vinfo && !m_costing_for_scalar)
> {
> + unsigned int vf = vect_vf_for_cost (loop_vinfo);
> + for (auto inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
> + if ((SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_group
> + || SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_chain)
> + && (vect_reduc_type (loop_vinfo, SLP_INSTANCE_TREE (inst))
> + == FOLD_LEFT_REDUCTION))
> + m_num_fold_left_reduc_lanes
> + += vf * SLP_TREE_LANES (SLP_INSTANCE_TREE (inst));
> +
> + if (m_num_fold_left_reduc_lanes && dump_enabled_p ())
> + dump_printf_loc (MSG_NOTE, vect_location,
> + "in-order FP reduction lanes: %u\n",
> + m_num_fold_left_reduc_lanes);
> +
> /* We are currently not asking the vectorizer to compare costs
> between different vector mode sizes. When using predication
> that will end up always choosing the preferred mode size even
> @@ -27067,6 +27085,21 @@ ix86_vector_costs::finish_cost (const vector_costs
> *scalar_costs)
> vector_costs::finish_cost (scalar_costs);
> }
>
> +/* Return true if THIS has a shorter fold-left reduction chain than OTHER.
> */
> +
> +bool
> +ix86_vector_costs::better_fold_left_reduc_than_p
> + (const vector_costs *other) const
> +{
> + auto other_costs = static_cast<const ix86_vector_costs *> (other);
> + if (m_num_fold_left_reduc_lanes >=
> other_costs->m_num_fold_left_reduc_lanes)
> + return false;
> +
> + /* Do not let emulated sub-SSE modes win on this alone. */
> + loop_vec_info loop_vinfo = as_a<loop_vec_info> (m_vinfo);
> + return known_ge (GET_MODE_SIZE (loop_vinfo->vector_mode), 16);
> +}
> +
> /* Return true if THIS should be preferred over OTHER as main vector loop.
> */
>
> bool
> @@ -27075,6 +27108,9 @@ ix86_vector_costs::better_main_loop_than_p (const
> vector_costs *other) const
> loop_vec_info this_loop_vinfo = as_a<loop_vec_info> (this->vinfo ());
> loop_vec_info other_loop_vinfo = as_a<loop_vec_info> (other->vinfo ());
>
> + if (better_fold_left_reduc_than_p (other))
> + return true;
> +
> /* If the other loop is masked it does not need an epilog. Prefer that
> if the current loop cannot be vectorized fully with a vector
> epilogs with at most one scalar iteration left. */
> @@ -27097,6 +27133,10 @@ ix86_vector_costs::better_epilogue_loop_than_p
> (const vector_costs *other,
> loop_vec_info main_loop) const
> {
> loop_vec_info this_loop_info = as_a <loop_vec_info> (this->vinfo ());
> +
> + if (better_fold_left_reduc_than_p (other))
> + return true;
> +
> /* The x86 target allows for multiple vector epilogues, if THIS is
> the suggested epilog mode of OTHER then keep the latter unless
> THIS has a VF of one which means no further epilog needed. */
> diff --git
> a/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c
> b/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c
> index abb9f6681c2..370e7ed4d11 100644
> --- a/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c
> +++ b/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c
> @@ -52,7 +52,6 @@ void test (const unsigned char * __restrict__ pi,
> pp_avg_rgb[3] = pp_avg_rgb_3;
> }
>
> -/* Even though there's an SLP opportunity in-order reductions should never
> use
> - masked epilogs. */
> -/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte
> vectors" "vect" } } */
> -/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using 32
> byte vectors" "vect" } } */
> +/* In-order reductions should not use masked epilogs here. */
> +/* { dg-final { scan-tree-dump "optimized: loop vectorized using 16 byte
> vectors" "vect" } } */
> +/* { dg-final { scan-tree-dump-not "optimized: epilogue loop vectorized
> using masked" "vect" } } */
> diff --git a/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c
> b/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c
> new file mode 100644
> index 00000000000..634647287d8
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c
> @@ -0,0 +1,19 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O2 -march=x86-64-v3 -fdump-tree-vect-details" } */
> +
> +/* The two dependent updates form a fold-left reduction chain. */
> +
> +float
> +foo (float *__restrict__ a, int n)
> +{
> + float sum = 0;
> + for (int i = 0; i != n; i++)
> + {
> + sum += a[2 * i];
> + sum += a[2 * i + 1];
> + }
> + return sum;
> +}
> +
> +/* { dg-final { scan-tree-dump "Starting SLP discovery of reduction chain"
> "vect" } } */
> +/* { dg-final { scan-tree-dump "in-order FP reduction lanes" "vect" } } */
> diff --git a/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c
> b/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c
> new file mode 100644
> index 00000000000..6babd761d6f
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c
> @@ -0,0 +1,18 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O2 -march=x86-64-v3 -fdump-tree-vect-details" } */
> +
> +/* The byte loads make V32QI available, but the fold-left reduction should
> + make the smaller SSE loop win. */
> +
> +float
> +foo (char *a, char *b, int n)
> +{
> + float sum = 0;
> + for (int i = 0; i != n; i++)
> + sum += a[i] * b[i];
> + return sum;
> +}
> +
> +/* { dg-final { scan-tree-dump "in-order FP reduction lanes" "vect" } } */
> +/* { dg-final { scan-tree-dump "loop vectorized using 16 byte vectors"
> "vect" } } */
> +/* { dg-final { scan-tree-dump-not "loop vectorized using 32 byte vectors"
> "vect" } } */
> diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c
> b/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c
> index 8cd8740c6ec..9f0445390f4 100644
> --- a/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c
> +++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c
> @@ -17,5 +17,4 @@ foo (double *a, char *mask, int n)
> return sum;
> }
>
> -/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte
> vectors" "vect" } } */
> -/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using 32
> byte vectors" "vect" } } */
> +/* { dg-final { scan-tree-dump "optimized: loop vectorized using 32 byte
> vectors" "vect" } } */
> diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c
> b/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c
> index 63c29895f9b..be56819fc54 100644
> --- a/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c
> +++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c
> @@ -17,5 +17,5 @@ foo (double *a, char *mask, int n)
> return sum;
> }
>
> -/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte
> vectors" "vect" } } */
> -/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using
> masked 64 byte vectors" "vect" } } */
> +/* { dg-final { scan-tree-dump "optimized: loop vectorized using 32 byte
> vectors" "vect" } } */
> +/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using
> masked 32 byte vectors" "vect" } } */
>
--
Richard Biener <[email protected]>
SUSE Software Solutions Germany GmbH,
Frankenstrasse 146, 90461 Nuernberg, Germany;
GF: Jochen Jaser, Andrew McDonald, Abhinav Puri; (HRB 36809, AG Nuernberg)
