> I think this is better, but I'd make this partial_store_elidable_p only and
> for
> loads replace with 'else'? Alternatively rename it to
> partial_load_store_all_lanes_masked_p
> or so? Btw, I see we're oddly rejecting any mask != -1 even when len
> == 0? Likewise
> we don't seem to treat mask == 0 the same as len == 0?
Yes, I also noticed the inconsistencies. At first I wanted to change as little
as possible but maybe another approach is also still ok:
What we're still missing to treat things more uniformly is an else operand for
len_load. Then we can use internal_fn_else_index for all partial loads and use
its value.
So I went ahead and did that in a separate preparation patch.
The attached v3 of the elision patch goes a bit further in than v1/v2 in it
tries to classify "all active", "all inactive", and "mixed" and also checks all
partial loads (like gathers, lanes). It depends on the preparation patch,
though, because it calls internal_fn_else_index unconditionally.
They have been bootstrapped and regtested individually as well as together on
x86 and power10. Regtested on riscv64, aarch64, and s390 (via qemu).
Regards
Robin
[PATCH v3] fold: Elide MASK_LEN_LOAD/STORE with zero length [PR122635].
This patch adds zero-length handling to gimple_fold_partial_store and
gimple_fold_partial_load and unifies them into
gimple_fold_partial_load_store.
It introduces a new function partial_load_store_mask_state that
returns
MASK_ALL_INACTIVE,
MASK_ALL_ACTIVE, or
MASK_UNKNOWN.
This result is used to either replace a load with its else value and
elide a store (when all inactive), turn the load/store into a regular
mem ref (all_active), or do nothing.
PR tree-optimization/122635
gcc/ChangeLog:
* gimple-fold.cc (enum mask_load_store_state): New enum.
(gimple_fold_partial_load_store_mem_ref): Only fold
"all active" loads/stores.
(partial_load_store_mask_state): New function to compute mask
state.
(gimple_fold_partial_load): Remove.
(gimple_fold_partial_load_store): New function.
(gimple_fold_partial_store): Remove.
(gimple_fold_call): Use new function.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/pfalse-store.c: Expect more elided
stores.
* gcc.target/riscv/rvv/autovec/pr122635-1.c: New test.
* gcc.target/riscv/rvv/autovec/pr122635-2.c: New test.
* gcc.target/powerpc/p9-vec-length-epil-8.c: Expect two lxvl
less.
---
gcc/gimple-fold.cc | 225 ++++++++++++------
.../gcc.target/aarch64/sve/pfalse-store.c | 5 +-
.../gcc.target/powerpc/p9-vec-length-epil-8.c | 2 +-
.../gcc.target/riscv/rvv/autovec/pr122635-1.c | 20 ++
.../gcc.target/riscv/rvv/autovec/pr122635-2.c | 18 ++
5 files changed, 198 insertions(+), 72 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-1.c
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-2.c
diff --git a/gcc/gimple-fold.cc b/gcc/gimple-fold.cc
index 3fc76313622..a7b5d8352e7 100644
--- a/gcc/gimple-fold.cc
+++ b/gcc/gimple-fold.cc
@@ -5757,50 +5757,112 @@ arith_overflowed_p (enum tree_code code, const_tree
type,
return wi::min_precision (wres, sign) > TYPE_PRECISION (type);
}
-/* If IFN_{MASK,LEN,MASK_LEN}_LOAD/STORE call CALL is unconditional,
- return a MEM_REF for the memory it references, otherwise return null.
- VECTYPE is the type of the memory vector. MASK_P indicates it's for
- MASK if true, otherwise it's for LEN. */
+/* Mask state for partial load/store operations (mask and length). */
+enum mask_load_store_state {
+ MASK_ALL_INACTIVE, /* All lanes/elements are inactive (can be elided). */
+ MASK_ALL_ACTIVE, /* All lanes/elements are active (unconditional). */
+ MASK_UNKNOWN
+};
-static tree
-gimple_fold_partial_load_store_mem_ref (gcall *call, tree vectype, bool mask_p)
+/* Check the mask/length state of IFN_{MASK,LEN,MASK_LEN}_LOAD/STORE call CALL.
+ Returns whether all elements are active, all inactive, or mixed.
+ VECTYPE is the vector type of the operation. */
+
+static enum mask_load_store_state
+partial_load_store_mask_state (gcall *call, tree vectype)
{
- tree ptr = gimple_call_arg (call, 0);
- tree alias_align = gimple_call_arg (call, 1);
- if (!tree_fits_uhwi_p (alias_align))
- return NULL_TREE;
+ internal_fn ifn = gimple_call_internal_fn (call);
+ int mask_index = internal_fn_mask_index (ifn);
+ int len_index = internal_fn_len_index (ifn);
+
+ /* Extract length and mask arguments up front. */
+ tree len = len_index != -1 ? gimple_call_arg (call, len_index) : NULL_TREE;
+ tree bias = len ? gimple_call_arg (call, len_index + 1) : NULL_TREE;
+ tree mask = mask_index != -1 ? gimple_call_arg (call, mask_index) :
NULL_TREE;
+
+ poly_int64 nelts = GET_MODE_NUNITS (TYPE_MODE (vectype));
- if (mask_p)
+ poly_widest_int wlen = -1;
+ bool full_length_p = !len; /* No length means full length. */
+
+ /* Compute effective length. */
+ if (len && poly_int_tree_p (len))
{
- tree mask = gimple_call_arg (call, 2);
- if (!integer_all_onesp (mask))
- return NULL_TREE;
+ gcc_assert (TREE_CODE (bias) == INTEGER_CST);
+ wlen = wi::to_poly_widest (len) + wi::to_widest (bias);
+
+ if (known_eq (wlen, 0))
+ return MASK_ALL_INACTIVE;
+
+ if (known_eq (wlen, nelts))
+ full_length_p = true;
+ else
+ full_length_p = false;
}
- else
+
+ /* Check mask for early return cases. */
+ if (mask)
{
- internal_fn ifn = gimple_call_internal_fn (call);
- int len_index = internal_fn_len_index (ifn);
- tree basic_len = gimple_call_arg (call, len_index);
- if (!poly_int_tree_p (basic_len))
- return NULL_TREE;
- tree bias = gimple_call_arg (call, len_index + 1);
- gcc_assert (TREE_CODE (bias) == INTEGER_CST);
- /* For LEN_LOAD/LEN_STORE/MASK_LEN_LOAD/MASK_LEN_STORE,
- we don't fold when (bias + len) != VF. */
- if (maybe_ne (wi::to_poly_widest (basic_len) + wi::to_widest (bias),
- GET_MODE_NUNITS (TYPE_MODE (vectype))))
- return NULL_TREE;
+ if (integer_zerop (mask))
+ return MASK_ALL_INACTIVE;
+
+ if (full_length_p && integer_all_onesp (mask))
+ return MASK_ALL_ACTIVE;
+ }
+ else if (full_length_p)
+ /* No mask and full length means all active. */
+ return MASK_ALL_ACTIVE;
+
+ /* For VLA vectors, we can't do much more. */
+ if (!nelts.is_constant ())
+ return MASK_UNKNOWN;
+
+ /* Same for VLS vectors with non-constant mask. */
+ if (mask && TREE_CODE (mask) != VECTOR_CST)
+ return MASK_UNKNOWN;
- /* For MASK_LEN_{LOAD,STORE}, we should also check whether
- the mask is all ones mask. */
- if (ifn == IFN_MASK_LEN_LOAD || ifn == IFN_MASK_LEN_STORE)
+ /* Check VLS vector elements. */
+ gcc_assert (wlen.is_constant ());
+
+ HOST_WIDE_INT active_len = wlen.to_constant ().to_shwi ();
+ if (active_len == -1)
+ active_len = nelts.to_constant ();
+
+ /* Check if all elements in the active range match the mask. */
+ for (HOST_WIDE_INT i = 0; i < active_len; i++)
+ {
+ bool elt_active = !mask || !integer_zerop (vector_cst_elt (mask, i));
+ if (!elt_active)
{
- tree mask = gimple_call_arg (call, internal_fn_mask_index (ifn));
- if (!integer_all_onesp (mask))
- return NULL_TREE;
+ /* Found an inactive element. Check if all are inactive. */
+ for (HOST_WIDE_INT j = 0; j < active_len; j++)
+ if (!mask || !integer_zerop (vector_cst_elt (mask, j)))
+ return MASK_UNKNOWN; /* Mixed state. */
+ return MASK_ALL_INACTIVE;
}
}
+ /* All elements in active range are active. */
+ return full_length_p ? MASK_ALL_ACTIVE : MASK_UNKNOWN;
+}
+
+
+/* If IFN_{MASK,LEN,MASK_LEN}_LOAD/STORE call CALL is unconditional
+ (all lanes active), return a MEM_REF for the memory it references.
+ Otherwise return NULL_TREE. VECTYPE is the type of the memory vector. */
+
+static tree
+gimple_fold_partial_load_store_mem_ref (gcall *call, tree vectype)
+{
+ /* Only fold if all lanes are active (unconditional). */
+ if (partial_load_store_mask_state (call, vectype) != MASK_ALL_ACTIVE)
+ return NULL_TREE;
+
+ tree ptr = gimple_call_arg (call, 0);
+ tree alias_align = gimple_call_arg (call, 1);
+ if (!tree_fits_uhwi_p (alias_align))
+ return NULL_TREE;
+
unsigned HOST_WIDE_INT align = tree_to_uhwi (alias_align);
if (TYPE_ALIGN (vectype) != align)
vectype = build_aligned_type (vectype, align);
@@ -5808,41 +5870,68 @@ gimple_fold_partial_load_store_mem_ref (gcall *call,
tree vectype, bool mask_p)
return fold_build2 (MEM_REF, vectype, ptr, offset);
}
-/* Try to fold IFN_{MASK,LEN}_LOAD call CALL. Return true on success.
- MASK_P indicates it's for MASK if true, otherwise it's for LEN. */
+/* Try to fold IFN_{MASK,LEN}_LOAD/STORE call CALL. Return true on success.
*/
static bool
-gimple_fold_partial_load (gimple_stmt_iterator *gsi, gcall *call, bool mask_p)
+gimple_fold_partial_load_store (gimple_stmt_iterator *gsi, gcall *call)
{
+ internal_fn ifn = gimple_call_internal_fn (call);
tree lhs = gimple_call_lhs (call);
- if (!lhs)
- return false;
+ bool is_load = (lhs != NULL_TREE);
+ tree vectype;
- if (tree rhs
- = gimple_fold_partial_load_store_mem_ref (call, TREE_TYPE (lhs), mask_p))
+ if (is_load)
+ vectype = TREE_TYPE (lhs);
+ else
{
- gassign *new_stmt = gimple_build_assign (lhs, rhs);
- gimple_set_location (new_stmt, gimple_location (call));
- gimple_move_vops (new_stmt, call);
- gsi_replace (gsi, new_stmt, false);
- return true;
+ tree rhs = gimple_call_arg (call, internal_fn_stored_value_index (ifn));
+ vectype = TREE_TYPE (rhs);
}
- return false;
-}
-/* Try to fold IFN_{MASK,LEN}_STORE call CALL. Return true on success.
- MASK_P indicates it's for MASK if true, otherwise it's for LEN. */
+ enum mask_load_store_state state
+ = partial_load_store_mask_state (call, vectype);
-static bool
-gimple_fold_partial_store (gimple_stmt_iterator *gsi, gcall *call,
- bool mask_p)
-{
- internal_fn ifn = gimple_call_internal_fn (call);
- tree rhs = gimple_call_arg (call, internal_fn_stored_value_index (ifn));
- if (tree lhs
- = gimple_fold_partial_load_store_mem_ref (call, TREE_TYPE (rhs), mask_p))
+ /* Handle all-inactive case. */
+ if (state == MASK_ALL_INACTIVE)
{
- gassign *new_stmt = gimple_build_assign (lhs, rhs);
+ if (is_load)
+ {
+ /* Replace load with else value. */
+ int else_index = internal_fn_else_index (ifn);
+ tree else_value = gimple_call_arg (call, else_index);
+ gassign *new_stmt = gimple_build_assign (lhs, else_value);
+ gimple_set_location (new_stmt, gimple_location (call));
+ gsi_replace (gsi, new_stmt, false);
+ return true;
+ }
+ else
+ {
+ /* Remove inactive store. */
+ unlink_stmt_vdef (call);
+ release_defs (call);
+ gsi_replace (gsi, gimple_build_nop (), true);
+ return true;
+ }
+ }
+
+ /* We cannot simplify a gather/scatter or load/store lanes further. */
+ if (internal_gather_scatter_fn_p (ifn)
+ || TREE_CODE (vectype) == ARRAY_TYPE)
+ return false;
+
+ /* Handle all-active case - fold to regular memory operation. */
+ if (tree mem_ref = gimple_fold_partial_load_store_mem_ref (call, vectype))
+ {
+ gassign *new_stmt;
+ if (is_load)
+ new_stmt = gimple_build_assign (lhs, mem_ref);
+ else
+ {
+ tree rhs
+ = gimple_call_arg (call, internal_fn_stored_value_index (ifn));
+ new_stmt = gimple_build_assign (mem_ref, rhs);
+ }
+
gimple_set_location (new_stmt, gimple_location (call));
gimple_move_vops (new_stmt, call);
gsi_replace (gsi, new_stmt, false);
@@ -6075,19 +6164,21 @@ gimple_fold_call (gimple_stmt_iterator *gsi, bool
inplace)
cplx_result = true;
uaddc_usubc = true;
break;
- case IFN_MASK_LOAD:
- changed |= gimple_fold_partial_load (gsi, stmt, true);
- break;
- case IFN_MASK_STORE:
- changed |= gimple_fold_partial_store (gsi, stmt, true);
- break;
case IFN_LEN_LOAD:
+ case IFN_MASK_LOAD:
case IFN_MASK_LEN_LOAD:
- changed |= gimple_fold_partial_load (gsi, stmt, false);
- break;
+ case IFN_MASK_GATHER_LOAD:
+ case IFN_MASK_LEN_GATHER_LOAD:
+ case IFN_MASK_LOAD_LANES:
+ case IFN_MASK_LEN_LOAD_LANES:
case IFN_LEN_STORE:
+ case IFN_MASK_STORE:
case IFN_MASK_LEN_STORE:
- changed |= gimple_fold_partial_store (gsi, stmt, false);
+ case IFN_MASK_SCATTER_STORE:
+ case IFN_MASK_LEN_SCATTER_STORE:
+ case IFN_MASK_STORE_LANES:
+ case IFN_MASK_LEN_STORE_LANES:
+ changed |= gimple_fold_partial_load_store (gsi, stmt);
break;
default:
break;
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store.c
b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store.c
index 1539f58c824..39db13bbcd6 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store.c
@@ -46,8 +46,5 @@ ALL_DATA (st2, x2_t)
ALL_DATA (st3, x3_t)
ALL_DATA (st4, x4_t)
-/* FIXME: Currently, st1/2/3/4 are not folded with a pfalse
- predicate, which is the reason for the 48 missing cases below. Once
- folding is implemented for these intrinsics, the sum should be 60. */
-/* { dg-final { scan-assembler-times {\t.cfi_startproc\n\tret\n} 12 } } */
+/* { dg-final { scan-assembler-times {\t.cfi_startproc\n\tret\n} 60 } } */
/* { dg-final { scan-assembler-times {\t.cfi_startproc\n} 60 } } */
diff --git a/gcc/testsuite/gcc.target/powerpc/p9-vec-length-epil-8.c
b/gcc/testsuite/gcc.target/powerpc/p9-vec-length-epil-8.c
index 34a2c8eb11b..5dff0d0ceb9 100644
--- a/gcc/testsuite/gcc.target/powerpc/p9-vec-length-epil-8.c
+++ b/gcc/testsuite/gcc.target/powerpc/p9-vec-length-epil-8.c
@@ -13,5 +13,5 @@
#include "p9-vec-length-8.h"
-/* { dg-final { scan-assembler-times {\mlxvl\M} 16 } } */
+/* { dg-final { scan-assembler-times {\mlxvl\M} 14 } } */
/* { dg-final { scan-assembler-times {\mstxvl\M} 7 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-1.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-1.c
new file mode 100644
index 00000000000..0beb3d70866
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-1.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=rv64gcv_zvl256b -mabi=lp64d -mrvv-vector-bits=zvl
-mno-autovec-segment" } */
+
+typedef struct {
+ int a[6];
+ float b[3];
+} c;
+
+int d(c *e) {
+ int f =0;
+ for (; f < 3; f++) {
+ e->a[2 * f] = e->b[f];
+ e->a[2 * f + 1] = -e->a[2 * f];
+ e->a[2 * f] = f + 3 * e->a[2 * f];
+ e->a[2 * f + 1] = f + 3 * e->a[2 * f + 1];
+ }
+ return 0;
+}
+
+/* { dg-final { scan-assembler-not "vsetivli.*zero,0" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-2.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-2.c
new file mode 100644
index 00000000000..0de69b52cb0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/pr122635-2.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=rv64gcv_zvl256b -mabi=lp64d -mrvv-vector-bits=zvl
-mno-autovec-segment" } */
+
+typedef struct {
+ int A[6];
+ float b[];
+} a;
+
+int b(a *a) {
+ int b = 0;
+ for (; b < 3; b++) {
+ a->A[2 * b] = a->b[b] - b + a->A[2 * b];
+ a->A[2 * b + 1] = b * a->A[2 * b + 1];
+ }
+ return 0;
+}
+
+/* { dg-final { scan-assembler-not "vsetivli.*zero,0" } } */
--
2.51.1
