On Thu, 22 Aug 2024, Jennifer Schmitz wrote:
> On 19 Aug 2024, at 21:02, Richard Sandiford <richard.sandif...@arm.com> wrote:
> >
> > External email: Use caution opening links or attachments
> >
> >
> > Jennifer Schmitz <jschm...@nvidia.com> writes:
> >> Thanks for the comments. I updated the patch accordingly and bootstrapped
> >> and tested again.
> >> Best, Jennifer
> >>
> >> From 9ef423f23afaeaa650d511c51bbc1a167e40b349 Mon Sep 17 00:00:00 2001
> >> From: Jennifer Schmitz <jschm...@nvidia.com>
> >> Date: Wed, 7 Aug 2024 08:56:45 -0700
> >> Subject: [PATCH] PR tree-optimization/101390: Vectorize modulo operator
> >>
> >> This patch adds a new vectorization pattern that detects the modulo
> >> operation where the second operand is a variable.
> >> It replaces the statement by division, multiplication, and subtraction.
> >>
> >> The patch was bootstrapped and regtested on aarch64-linux-gnu, no
> >> regression.
> >> Ok for mainline?
> >>
> >> Signed-off-by: Jennifer Schmitz <jschm...@nvidia.com>
> >>
> >> gcc/
> >>
> >> PR tree-optimization/101390
> >> * tree-vect-pattern.cc (vect_recog_mod_var_pattern): Add new pattern.
> >>
> >> gcc/testsuite/
> >> PR tree-optimization/101390
> >> * gcc.dg/vect/vect-mod-var.c: New test.
> >> * gcc.target/aarch64/sve/mod_1.c: Likewise.
> >> * lib/target-supports.exp: New selector expression.
> >
> > LGTM, thanks. Please give others a couple of days to comment though.
> >
> Pushed to trunk with 9bbad3685131ec95d970f81bf75f9556d4d92742.
The gcc.dg/vect/vect-mod-var.c seems to FAIL execution for me on
x86_64-linux:
FAIL: gcc.dg/vect/vect-mod-var.c -flto -ffat-lto-objects execution test
FAIL: gcc.dg/vect/vect-mod-var.c execution test
Richard.
> Best, Jennifer
> > Richard
> >
> >> ---
> >> gcc/testsuite/gcc.dg/vect/vect-mod-var.c | 37 +++++++++++
> >> gcc/testsuite/gcc.target/aarch64/sve/mod_1.c | 28 +++++++++
> >> gcc/testsuite/lib/target-supports.exp | 5 ++
> >> gcc/tree-vect-patterns.cc | 66 ++++++++++++++++++++
> >> 4 files changed, 136 insertions(+)
> >> create mode 100644 gcc/testsuite/gcc.dg/vect/vect-mod-var.c
> >> create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mod_1.c
> >>
> >> diff --git a/gcc/testsuite/gcc.dg/vect/vect-mod-var.c
> >> b/gcc/testsuite/gcc.dg/vect/vect-mod-var.c
> >> new file mode 100644
> >> index 00000000000..eeed318c62b
> >> --- /dev/null
> >> +++ b/gcc/testsuite/gcc.dg/vect/vect-mod-var.c
> >> @@ -0,0 +1,37 @@
> >> +#include "tree-vect.h"
> >> +
> >> +#define N 64
> >> +
> >> +__attribute__ ((noinline)) int
> >> +f (int *restrict a, int *restrict b, int *restrict c)
> >> +{
> >> + for (int i = 0; i < N; ++i)
> >> + c[i] = a[i] % b[i];
> >> +}
> >> +
> >> +#define BASE1 -126
> >> +#define BASE2 116
> >> +
> >> +int
> >> +main (void)
> >> +{
> >> + check_vect ();
> >> +
> >> + int a[N], b[N], c[N];
> >> +
> >> + for (int i = 0; i < N; ++i)
> >> + {
> >> + a[i] = BASE1 + i * 5;
> >> + b[i] = BASE2 - i * 4;
> >> + __asm__ volatile ("");
> >> + }
> >> +
> >> + f (a, b, c);
> >> +
> >> +#pragma GCC novector
> >> + for (int i = 0; i < N; ++i)
> >> + if (c[i] != a[i] % b[i])
> >> + __builtin_abort ();
> >> +}
> >> +
> >> +/* { dg-final { scan-tree-dump "vect_recog_mod_var_pattern: detected"
> >> "vect" { target vect_int_div } } } */
> >> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mod_1.c
> >> b/gcc/testsuite/gcc.target/aarch64/sve/mod_1.c
> >> new file mode 100644
> >> index 00000000000..eb37f1e3636
> >> --- /dev/null
> >> +++ b/gcc/testsuite/gcc.target/aarch64/sve/mod_1.c
> >> @@ -0,0 +1,28 @@
> >> +/* { dg-do assemble { target aarch64_asm_sve_ok } } */
> >> +/* { dg-options "-Ofast -ftree-vectorize -fno-vect-cost-model
> >> --save-temps" } */
> >> +
> >> +#include <stdint.h>
> >> +
> >> +#define DEF_LOOP(TYPE) \
> >> +void __attribute__ ((noipa)) \
> >> +mod_##TYPE (TYPE *restrict dst, TYPE *restrict src1, \
> >> + TYPE *restrict src2, int count) \
> >> +{ \
> >> + for (int i = 0; i < count; ++i) \
> >> + dst[i] = src1[i] % src2[i]; \
> >> +}
> >> +
> >> +#define TEST_ALL(T) \
> >> + T (int32_t) \
> >> + T (uint32_t) \
> >> + T (int64_t) \
> >> + T (uint64_t)
> >> +
> >> +TEST_ALL (DEF_LOOP)
> >> +
> >> +/* { dg-final { scan-assembler-times {\tsdiv\tz[0-9]+\.s, p[0-7]/m,
> >> z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
> >> +/* { dg-final { scan-assembler-times {\tudiv\tz[0-9]+\.s, p[0-7]/m,
> >> z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
> >> +/* { dg-final { scan-assembler-times {\tmsb\tz[0-9]+\.s, p[0-7]/m,
> >> z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
> >> +/* { dg-final { scan-assembler-times {\tsdiv\tz[0-9]+\.d, p[0-7]/m,
> >> z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
> >> +/* { dg-final { scan-assembler-times {\tudiv\tz[0-9]+\.d, p[0-7]/m,
> >> z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
> >> +/* { dg-final { scan-assembler-times {\tmsb\tz[0-9]+\.d, p[0-7]/m,
> >> z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
> >> diff --git a/gcc/testsuite/lib/target-supports.exp
> >> b/gcc/testsuite/lib/target-supports.exp
> >> index 26820b146d4..b8072eaf781 100644
> >> --- a/gcc/testsuite/lib/target-supports.exp
> >> +++ b/gcc/testsuite/lib/target-supports.exp
> >> @@ -4258,6 +4258,11 @@ proc check_effective_target_vect_int { } {
> >> }}]
> >> }
> >>
> >> +# Return 1 if the target supports vector integer division, 0 otherwise.
> >> +proc check_effective_target_vect_int_div { } {
> >> + return [check_effective_target_aarch64_sve]
> >> +}
> >> +
> >> # Return 1 if the target supports vectorization of early breaks,
> >> # 0 otherwise.
> >> #
> >> diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
> >> index f52de2b6972..18b322c63b8 100644
> >> --- a/gcc/tree-vect-patterns.cc
> >> +++ b/gcc/tree-vect-patterns.cc
> >> @@ -5264,6 +5264,71 @@ vect_recog_divmod_pattern (vec_info *vinfo,
> >> return pattern_stmt;
> >> }
> >>
> >> +/* Detects pattern with a modulo operation (S1) where both arguments
> >> + are variables of integral type.
> >> + The statement is replaced by division, multiplication, and subtraction.
> >> + The last statement (S4) is returned.
> >> +
> >> + Example:
> >> + S1 c_t = a_t % b_t;
> >> +
> >> + is replaced by
> >> + S2 x_t = a_t / b_t;
> >> + S3 y_t = x_t * b_t;
> >> + S4 z_t = a_t - y_t; */
> >> +
> >> +static gimple *
> >> +vect_recog_mod_var_pattern (vec_info *vinfo,
> >> + stmt_vec_info stmt_vinfo, tree *type_out)
> >> +{
> >> + gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
> >> + tree oprnd0, oprnd1, vectype, itype;
> >> + gimple *pattern_stmt, *def_stmt;
> >> + enum tree_code rhs_code;
> >> +
> >> + if (!is_gimple_assign (last_stmt))
> >> + return NULL;
> >> +
> >> + rhs_code = gimple_assign_rhs_code (last_stmt);
> >> + if (rhs_code != TRUNC_MOD_EXPR)
> >> + return NULL;
> >> +
> >> + oprnd0 = gimple_assign_rhs1 (last_stmt);
> >> + oprnd1 = gimple_assign_rhs2 (last_stmt);
> >> + itype = TREE_TYPE (oprnd0);
> >> + if (TREE_CODE (oprnd0) != SSA_NAME
> >> + || TREE_CODE (oprnd1) != SSA_NAME
> >> + || TREE_CODE (itype) != INTEGER_TYPE)
> >> + return NULL;
> >> +
> >> + vectype = get_vectype_for_scalar_type (vinfo, itype);
> >> +
> >> + if (!vectype
> >> + || target_has_vecop_for_code (TRUNC_MOD_EXPR, vectype)
> >> + || !target_has_vecop_for_code (TRUNC_DIV_EXPR, vectype)
> >> + || !target_has_vecop_for_code (MULT_EXPR, vectype)
> >> + || !target_has_vecop_for_code (MINUS_EXPR, vectype))
> >> + return NULL;
> >> +
> >> + tree q, tmp, r;
> >> + q = vect_recog_temp_ssa_var (itype, NULL);
> >> + def_stmt = gimple_build_assign (q, TRUNC_DIV_EXPR, oprnd0, oprnd1);
> >> + append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vectype);
> >> +
> >> + tmp = vect_recog_temp_ssa_var (itype, NULL);
> >> + def_stmt = gimple_build_assign (tmp, MULT_EXPR, q, oprnd1);
> >> + append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vectype);
> >> +
> >> + r = vect_recog_temp_ssa_var (itype, NULL);
> >> + pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, tmp);
> >> +
> >> + /* Pattern detected. */
> >> + *type_out = vectype;
> >> + vect_pattern_detected ("vect_recog_mod_var_pattern", last_stmt);
> >> +
> >> + return pattern_stmt;
> >> +}
> >> +
> >> /* Function vect_recog_mixed_size_cond_pattern
> >>
> >> Try to find the following pattern:
> >> @@ -7343,6 +7408,7 @@ static vect_recog_func vect_vect_recog_func_ptrs[] =
> >> {
> >> { vect_recog_rotate_pattern, "rotate" },
> >> { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
> >> { vect_recog_divmod_pattern, "divmod" },
> >> + { vect_recog_mod_var_pattern, "modvar" },
> >> { vect_recog_mult_pattern, "mult" },
> >> { vect_recog_sat_add_pattern, "sat_add" },
> >> { vect_recog_sat_sub_pattern, "sat_sub" },
>
>
--
Richard Biener <rguent...@suse.de>
SUSE Software Solutions Germany GmbH,
Frankenstrasse 146, 90461 Nuernberg, Germany;
GF: Ivo Totev, Andrew McDonald, Werner Knoblich; (HRB 36809, AG Nuernberg)
