On Fri, Nov 12, 2021 at 7:06 PM Richard Sandiford via Gcc-patches
<gcc-patches@gcc.gnu.org> wrote:
>
> This patch extends the previous SLP gather load support so
> that it can handle masked loads too.
>
> Regstrapped on aarch64-linux-gnu and x86_64-linux-gnu. OK to install?
OK.
Thanks,
Richard.
> Richard
>
>
> gcc/
> * tree-vect-slp.c (arg1_arg4_map): New variable.
> (vect_get_operand_map): Handle IFN_MASK_GATHER_LOAD.
> (vect_build_slp_tree_1): Likewise.
> (vect_build_slp_tree_2): Likewise.
> * tree-vect-stmts.c (vectorizable_load): Expect the mask to be
> the last SLP child node rather than the first.
>
> gcc/testsuite/
> * gcc.dg/vect/vect-gather-3.c: New test.
> * gcc.dg/vect/vect-gather-4.c: Likewise.
> * gcc.target/aarch64/sve/mask_gather_load_8.c: Likewise.
> ---
> gcc/testsuite/gcc.dg/vect/vect-gather-3.c | 64 ++++++++++++++++++
> gcc/testsuite/gcc.dg/vect/vect-gather-4.c | 48 ++++++++++++++
> .../aarch64/sve/mask_gather_load_8.c | 65 +++++++++++++++++++
> gcc/tree-vect-slp.c | 15 ++++-
> gcc/tree-vect-stmts.c | 21 ++++--
> 5 files changed, 203 insertions(+), 10 deletions(-)
> create mode 100644 gcc/testsuite/gcc.dg/vect/vect-gather-3.c
> create mode 100644 gcc/testsuite/gcc.dg/vect/vect-gather-4.c
> create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mask_gather_load_8.c
>
> diff --git a/gcc/testsuite/gcc.dg/vect/vect-gather-3.c
> b/gcc/testsuite/gcc.dg/vect/vect-gather-3.c
> new file mode 100644
> index 00000000000..738bd3f3106
> --- /dev/null
> +++ b/gcc/testsuite/gcc.dg/vect/vect-gather-3.c
> @@ -0,0 +1,64 @@
> +#include "tree-vect.h"
> +
> +#define N 16
> +
> +void __attribute__((noipa))
> +f (int *restrict y, int *restrict x, int *restrict indices)
> +{
> + for (int i = 0; i < N; ++i)
> + {
> + y[i * 2] = (indices[i * 2] < N * 2
> + ? x[indices[i * 2]] + 1
> + : 1);
> + y[i * 2 + 1] = (indices[i * 2 + 1] < N * 2
> + ? x[indices[i * 2 + 1]] + 2
> + : 2);
> + }
> +}
> +
> +int y[N * 2];
> +int x[N * 2] = {
> + 72704, 52152, 51301, 96681,
> + 57937, 60490, 34504, 60944,
> + 42225, 28333, 88336, 74300,
> + 29250, 20484, 38852, 91536,
> + 86917, 63941, 31590, 21998,
> + 22419, 26974, 28668, 13968,
> + 3451, 20247, 44089, 85521,
> + 22871, 87362, 50555, 85939
> +};
> +int indices[N * 2] = {
> + 15, 0x10000, 0xcafe0, 19,
> + 7, 22, 19, 1,
> + 0x20000, 0x70000, 15, 30,
> + 5, 12, 11, 11,
> + 10, 25, 5, 20,
> + 22, 24, 32, 28,
> + 30, 19, 6, 0xabcdef,
> + 7, 12, 8, 21
> +};
> +int expected[N * 2] = {
> + 91537, 2, 1, 22000,
> + 60945, 28670, 21999, 52154,
> + 1, 2, 91537, 50557,
> + 60491, 29252, 74301, 74302,
> + 88337, 20249, 60491, 22421,
> + 28669, 3453, 1, 22873,
> + 50556, 22000, 34505, 2,
> + 60945, 29252, 42226, 26976
> +};
> +
> +int
> +main (void)
> +{
> + check_vect ();
> +
> + f (y, x, indices);
> + for (int i = 0; i < 32; ++i)
> + if (y[i] != expected[i])
> + __builtin_abort ();
> +
> + return 0;
> +}
> +
> +/* { dg-final { scan-tree-dump "Loop contains only SLP stmts" vect { target
> { vect_gather_load_ifn && vect_masked_load } } } } */
> diff --git a/gcc/testsuite/gcc.dg/vect/vect-gather-4.c
> b/gcc/testsuite/gcc.dg/vect/vect-gather-4.c
> new file mode 100644
> index 00000000000..ee2e4e4999a
> --- /dev/null
> +++ b/gcc/testsuite/gcc.dg/vect/vect-gather-4.c
> @@ -0,0 +1,48 @@
> +/* { dg-do compile } */
> +
> +#define N 16
> +
> +void
> +f1 (int *restrict y, int *restrict x1, int *restrict x2,
> + int *restrict indices)
> +{
> + for (int i = 0; i < N; ++i)
> + {
> + y[i * 2] = (indices[i * 2] < N * 2
> + ? x1[indices[i * 2]] + 1
> + : 1);
> + y[i * 2 + 1] = (indices[i * 2 + 1] < N * 2
> + ? x2[indices[i * 2 + 1]] + 2
> + : 2);
> + }
> +}
> +
> +void
> +f2 (int *restrict y, int *restrict x, int *restrict indices)
> +{
> + for (int i = 0; i < N; ++i)
> + {
> + y[i * 2] = (indices[i * 2] < N * 2
> + ? x[indices[i * 2]] + 1
> + : 1);
> + y[i * 2 + 1] = (indices[i * 2 + 1] < N * 2
> + ? x[indices[i * 2 + 1] * 2] + 2
> + : 2);
> + }
> +}
> +
> +void
> +f3 (int *restrict y, int *restrict x, int *restrict indices)
> +{
> + for (int i = 0; i < N; ++i)
> + {
> + y[i * 2] = (indices[i * 2] < N * 2
> + ? x[indices[i * 2]] + 1
> + : 1);
> + y[i * 2 + 1] = (indices[i * 2 + 1] < N * 2
> + ? x[(unsigned int) indices[i * 2 + 1]] + 2
> + : 2);
> + }
> +}
> +
> +/* { dg-final { scan-tree-dump-not "Loop contains only SLP stmts" vect {
> target vect_gather_load_ifn } } } */
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mask_gather_load_8.c
> b/gcc/testsuite/gcc.target/aarch64/sve/mask_gather_load_8.c
> new file mode 100644
> index 00000000000..95767f30a80
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/mask_gather_load_8.c
> @@ -0,0 +1,65 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O3 -fno-vect-cost-model" } */
> +
> +#include <stdint.h>
> +
> +void
> +f1 (int32_t *restrict y, int32_t *restrict x, int32_t *restrict index)
> +{
> + for (int i = 0; i < 100; ++i)
> + {
> + y[i * 2] = (index[i * 2] < 128
> + ? x[index[i * 2]] + 1
> + : 1);
> + y[i * 2 + 1] = (index[i * 2 + 1] < 128
> + ? x[index[i * 2 + 1]] + 2
> + : 2);
> + }
> +}
> +
> +void
> +f2 (int32_t *restrict y, int32_t *restrict x, uint32_t *restrict index)
> +{
> + for (int i = 0; i < 100; ++i)
> + {
> + y[i * 2] = (index[i * 2] < 128
> + ? x[index[i * 2]] + 1
> + : 1);
> + y[i * 2 + 1] = (index[i * 2 + 1] < 128
> + ? x[index[i * 2 + 1]] + 2
> + : 2);
> + }
> +}
> +
> +void
> +f3 (int32_t *restrict y, int32_t *restrict x, uint64_t *restrict index)
> +{
> + for (int i = 0; i < 100; ++i)
> + {
> + y[i * 2] = (index[i * 2] < 128
> + ? x[index[i * 2]] + 1
> + : 1);
> + y[i * 2 + 1] = (index[i * 2 + 1] < 128
> + ? x[index[i * 2 + 1]] + 2
> + : 2);
> + }
> +}
> +
> +void
> +f4 (int64_t *restrict y, int64_t *restrict x, uint64_t *restrict index)
> +{
> + for (int i = 0; i < 100; ++i)
> + {
> + y[i * 2] = (index[i * 2] < 128
> + ? x[index[i * 2]] + 1
> + : 1);
> + y[i * 2 + 1] = (index[i * 2 + 1] < 128
> + ? x[index[i * 2 + 1]] + 2
> + : 2);
> + }
> +}
> +
> +/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z,
> \[x[0-9]+, z[0-9]+\.s, sxtw #?2\]} 1 } } */
> +/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z,
> \[x[0-9]+, z[0-9]+\.s, uxtw #?2\]} 1 } } */
> +/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d, p[0-7]/z,
> \[x[0-9]+, z[0-9]+\.d, lsl #?2\]} 1 } } */
> +/* { dg-final { scan-assembler-times {\tld1d\tz[0-9]+\.d, p[0-7]/z,
> \[x[0-9]+, z[0-9]+\.d, lsl #?3\]} 1 } } */
> diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
> index 0f09fc1fda8..35ec2e2ad5e 100644
> --- a/gcc/tree-vect-slp.c
> +++ b/gcc/tree-vect-slp.c
> @@ -461,6 +461,7 @@ static const int cond_expr_maps[3][5] = {
> };
> static const int arg1_map[] = { 1, 1 };
> static const int arg2_map[] = { 1, 2 };
> +static const int arg1_arg4_map[] = { 2, 1, 4 };
>
> /* For most SLP statements, there is a one-to-one mapping between
> gimple arguments and child nodes. If that is not true for STMT,
> @@ -494,6 +495,9 @@ vect_get_operand_map (const gimple *stmt, unsigned char
> swap = 0)
> case IFN_GATHER_LOAD:
> return arg1_map;
>
> + case IFN_MASK_GATHER_LOAD:
> + return arg1_arg4_map;
> +
> default:
> break;
> }
> @@ -1000,7 +1004,9 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char
> *swap,
> else
> rhs_code = CALL_EXPR;
>
> - if (cfn == CFN_MASK_LOAD || cfn == CFN_GATHER_LOAD)
> + if (cfn == CFN_MASK_LOAD
> + || cfn == CFN_GATHER_LOAD
> + || cfn == CFN_MASK_GATHER_LOAD)
> load_p = true;
> else if ((internal_fn_p (cfn)
> && !vectorizable_internal_fn_p (as_internal_fn (cfn)))
> @@ -1229,7 +1235,9 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char
> *swap,
> } /* Grouped access. */
> else
> {
> - if (load_p && rhs_code != CFN_GATHER_LOAD)
> + if (load_p
> + && rhs_code != CFN_GATHER_LOAD
> + && rhs_code != CFN_MASK_GATHER_LOAD)
> {
> /* Not grouped load. */
> if (dump_enabled_p ())
> @@ -1711,7 +1719,8 @@ vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
> {
> if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
> gcc_assert (gimple_call_internal_p (stmt, IFN_MASK_LOAD)
> - || gimple_call_internal_p (stmt, IFN_GATHER_LOAD));
> + || gimple_call_internal_p (stmt, IFN_GATHER_LOAD)
> + || gimple_call_internal_p (stmt, IFN_MASK_GATHER_LOAD));
> else
> {
> *max_nunits = this_max_nunits;
> diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
> index 06da5a9bc13..8642acbc0b4 100644
> --- a/gcc/tree-vect-stmts.c
> +++ b/gcc/tree-vect-stmts.c
> @@ -8595,6 +8595,7 @@ vectorizable_load (vec_info *vinfo,
> return false;
>
> tree mask = NULL_TREE, mask_vectype = NULL_TREE;
> + int mask_index = -1;
> if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
> {
> scalar_dest = gimple_assign_lhs (assign);
> @@ -8626,12 +8627,12 @@ vectorizable_load (vec_info *vinfo,
> if (!scalar_dest)
> return false;
>
> - int mask_index = internal_fn_mask_index (ifn);
> + mask_index = internal_fn_mask_index (ifn);
> + /* ??? For SLP the mask operand is always last. */
> + if (mask_index >= 0 && slp_node)
> + mask_index = SLP_TREE_CHILDREN (slp_node).length () - 1;
> if (mask_index >= 0
> - && !vect_check_scalar_mask (vinfo, stmt_info, slp_node,
> - /* ??? For SLP we only have operands for
> - the mask operand. */
> - slp_node ? 0 : mask_index,
> + && !vect_check_scalar_mask (vinfo, stmt_info, slp_node, mask_index,
> &mask, NULL, &mask_dt, &mask_vectype))
> return false;
> }
> @@ -9393,8 +9394,14 @@ vectorizable_load (vec_info *vinfo,
> vec<tree> vec_offsets = vNULL;
> auto_vec<tree> vec_masks;
> if (mask)
> - vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
> - mask, &vec_masks, mask_vectype, NULL_TREE);
> + {
> + if (slp_node)
> + vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[mask_index],
> + &vec_masks);
> + else
> + vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies, mask,
> + &vec_masks, mask_vectype);
> + }
> tree vec_mask = NULL_TREE;
> poly_uint64 group_elt = 0;
> for (j = 0; j < ncopies; j++)
> --
> 2.25.1
>
