Tamar Christina <[email protected]> writes:
> Fold integer VEC_UNPACK_LO_EXPR and VEC_UNPACK_HI_EXPR for suitable
> variable-length VECTOR_CSTs.
>
> The patch adds support for other _LO, _HI expressions but it's expected
> that those be almost never use but the code unifies the VLA and VLS
> paths.
>
> The fixed-length case already folds. This extends the same idea to VLA
> constants when the input has twice as many elements as the output and the
> selected half can be represented as a valid VECTOR_CST. This helps fold
> known predicate constants before RTL expansion, avoiding unnecessary
> predicate unpack instructions.
>
> The idea here is that the VEC_CST should be a repeating pattern with a
> fill above the minimum VL.
>
> i.e. {a, b, a, b, ..., c, c, c} has the requirement that the minimum
> vector length consume the entire repeating pattern such that the _hi
> folds contain only {c,c,..}.
>
> This show up often with known iteration loops and folding these in gimple
> allows us to fold away unrolled masks which end up being empty.
>
> The selftests cover the foldable low/high cases and cases that should not
> fold.
>
>
> Bootstrapped Regtested on aarch64-none-linux-gnu,
> arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
> -m32, -m64 and no issues.
>
> OK for master?
>
> Thanks,
> Tamar
>
> gcc/ChangeLog:
>
> * fold-const.cc (const_unop): Fold VEC_UNPACK_LO_EXPR and
> VEC_UNPACK_HI_EXPR for suitable variable-length VECTOR_CSTs.
> (selftest::test_vec_unpack_folding): New function.
> (selftest::fold_const_cc_tests): Call it.
>
> ---
>
> diff --git a/gcc/fold-const.cc b/gcc/fold-const.cc
> index
> b9262b34f76683fa75e4c5f7ce938753e9c695a2..f919dbab4390127b604302fad06c7a672e7f9876
> 100644
> --- a/gcc/fold-const.cc
> +++ b/gcc/fold-const.cc
> @@ -1948,22 +1948,16 @@ const_unop (enum tree_code code, tree type, tree arg0)
> case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
> case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
> {
> - unsigned HOST_WIDE_INT out_nelts, in_nelts, i;
> + poly_uint64 out_nelts = TYPE_VECTOR_SUBPARTS (type);
> + unsigned HOST_WIDE_INT const_out_nelts;
> + unsigned int encoded_nelts, i, offset = 0;
> enum tree_code subcode;
>
> if (TREE_CODE (arg0) != VECTOR_CST)
> return NULL_TREE;
>
> - if (!VECTOR_CST_NELTS (arg0).is_constant (&in_nelts))
> + if (!known_eq (VECTOR_CST_NELTS (arg0), out_nelts * 2))
> return NULL_TREE;
> - out_nelts = in_nelts / 2;
> - gcc_assert (known_eq (out_nelts, TYPE_VECTOR_SUBPARTS (type)));
I'm not sure why the assert becomes an early exit. Shouldn't the
condition hold for VLA too?
> -
> - unsigned int offset = 0;
> - if ((!BYTES_BIG_ENDIAN) ^ (code == VEC_UNPACK_LO_EXPR
> - || code == VEC_UNPACK_FLOAT_LO_EXPR
> - || code == VEC_UNPACK_FIX_TRUNC_LO_EXPR))
> - offset = out_nelts;
>
> if (code == VEC_UNPACK_LO_EXPR || code == VEC_UNPACK_HI_EXPR)
> subcode = NOP_EXPR;
> @@ -1973,8 +1967,57 @@ const_unop (enum tree_code code, tree type, tree arg0)
> else
> subcode = FIX_TRUNC_EXPR;
>
> - tree_vector_builder elts (type, out_nelts, 1);
> - for (i = 0; i < out_nelts; i++)
> + bool low_p = (code == VEC_UNPACK_LO_EXPR
> + || code == VEC_UNPACK_FLOAT_LO_EXPR
> + || code == VEC_UNPACK_FIX_TRUNC_LO_EXPR);
> + bool use_second_encoded_half_p = (!BYTES_BIG_ENDIAN) ^ low_p;
> +
> + unsigned int nelts_per_pattern = 1;
> + tree fill = NULL_TREE;
> + if (out_nelts.is_constant (&const_out_nelts))
> + {
> + encoded_nelts = const_out_nelts;
> + if (use_second_encoded_half_p)
> + offset = encoded_nelts;
> + }
> + else
> + {
> + if (VECTOR_CST_STEPPED_P (arg0))
> + return NULL_TREE;
> +
> + unsigned int min_out_nelts = constant_lower_bound (out_nelts);
> + encoded_nelts = min_out_nelts;
> + fill = fold_convert_const (subcode, TREE_TYPE (type),
> + vector_cst_elt (arg0, encoded_nelts));
> + if (fill == NULL_TREE || !CONSTANT_CLASS_P (fill))
> + return NULL_TREE;
> +
> + /* Check that the selected high half is a single repeated
> + value. Since the input is not stepped, checking one full
> + pattern is enough to prove the scalable tail. */
> + for (i = 1; i < VECTOR_CST_NPATTERNS (arg0); ++i)
> + {
> + tree elt
> + = fold_convert_const (subcode, TREE_TYPE (type),
> + vector_cst_elt (arg0,
> + encoded_nelts + i));
> + if (elt == NULL_TREE
> + || !CONSTANT_CLASS_P (elt)
> + || !operand_equal_p (elt, fill))
> + return NULL_TREE;
> + }
> +
> + if (use_second_encoded_half_p)
> + {
> + encoded_nelts = 1;
> + offset = min_out_nelts;
> + }
> + else
> + nelts_per_pattern = 2;
> + }
I don't think the fold needs to be this specific. If:
- the input constant isn't stepped, and
- out_nelts is a multiple of the input's npatterns ("n" below)
then we know that the input contains n "foreground" elements followed by
repeats of n "background" elements (which may or may not be the same as
the foreground elements). That is:
{ a1,...,an,b1,....bn,b1,....,bn,b1,...,bn,... }
If we're unpacking the first half, this is also the shape of the output
(although the elements are wider). If we're unpacking the second half,
the shape is:
{ b1,...,bn,b1,....bn,b1,....,bn,b1,...,bn,... }
And this should always be at least as efficient as the status quo for
VLS as well as VLA.
So I'd suggest the attached (untested!) patch on top of this one.
Thanks,
Richard
diff --git a/gcc/fold-const.cc b/gcc/fold-const.cc
index f919dbab439..414a712b936 100644
--- a/gcc/fold-const.cc
+++ b/gcc/fold-const.cc
@@ -1949,15 +1949,13 @@ const_unop (enum tree_code code, tree type, tree arg0)
case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
{
poly_uint64 out_nelts = TYPE_VECTOR_SUBPARTS (type);
- unsigned HOST_WIDE_INT const_out_nelts;
- unsigned int encoded_nelts, i, offset = 0;
+ unsigned HOST_WIDE_INT npatterns, nelts_per_pattern, i, offset = 0;
enum tree_code subcode;
if (TREE_CODE (arg0) != VECTOR_CST)
return NULL_TREE;
- if (!known_eq (VECTOR_CST_NELTS (arg0), out_nelts * 2))
- return NULL_TREE;
+ gcc_assert (known_eq (VECTOR_CST_NELTS (arg0), out_nelts * 2));
if (code == VEC_UNPACK_LO_EXPR || code == VEC_UNPACK_HI_EXPR)
subcode = NOP_EXPR;
@@ -1970,54 +1968,37 @@ const_unop (enum tree_code code, tree type, tree arg0)
bool low_p = (code == VEC_UNPACK_LO_EXPR
|| code == VEC_UNPACK_FLOAT_LO_EXPR
|| code == VEC_UNPACK_FIX_TRUNC_LO_EXPR);
- bool use_second_encoded_half_p = (!BYTES_BIG_ENDIAN) ^ low_p;
- unsigned int nelts_per_pattern = 1;
- tree fill = NULL_TREE;
- if (out_nelts.is_constant (&const_out_nelts))
- {
- encoded_nelts = const_out_nelts;
- if (use_second_encoded_half_p)
- offset = encoded_nelts;
- }
+ npatterns = VECTOR_CST_NPATTERNS (arg0);
+ nelts_per_pattern = VECTOR_CST_NELTS_PER_PATTERN (arg0);
+ if (nelts_per_pattern <= 2 && multiple_p (out_nelts, npatterns))
+ /* The input is:
+
+ { a1,...,an,b1,...,bn,b1,...,bn,b1,...,bn,... }
+
+ where N == NPATTERNS and where ai == bi for NELTS_PER_PATTERN == 1
+ but not necessarily for NELTS_PER_PATTERN == 2.
+
+ When unpacking the first half, the shape of the output is the same
+ as the input. That shape would also work for the second half,
+ but setting the output nelts_per_patterns to 1 is always enough
+ in that case (since the output always starts on b1). */
+ ;
else
{
- if (VECTOR_CST_STEPPED_P (arg0))
- return NULL_TREE;
-
- unsigned int min_out_nelts = constant_lower_bound (out_nelts);
- encoded_nelts = min_out_nelts;
- fill = fold_convert_const (subcode, TREE_TYPE (type),
- vector_cst_elt (arg0, encoded_nelts));
- if (fill == NULL_TREE || !CONSTANT_CLASS_P (fill))
+ if (!out_nelts.is_constant (&npatterns))
return NULL_TREE;
+ nelts_per_pattern = 1;
+ }
- /* Check that the selected high half is a single repeated
- value. Since the input is not stepped, checking one full
- pattern is enough to prove the scalable tail. */
- for (i = 1; i < VECTOR_CST_NPATTERNS (arg0); ++i)
- {
- tree elt
- = fold_convert_const (subcode, TREE_TYPE (type),
- vector_cst_elt (arg0,
- encoded_nelts + i));
- if (elt == NULL_TREE
- || !CONSTANT_CLASS_P (elt)
- || !operand_equal_p (elt, fill))
- return NULL_TREE;
- }
-
- if (use_second_encoded_half_p)
- {
- encoded_nelts = 1;
- offset = min_out_nelts;
- }
- else
- nelts_per_pattern = 2;
+ if ((!BYTES_BIG_ENDIAN) ^ low_p)
+ {
+ offset = npatterns;
+ nelts_per_pattern = 1;
}
- tree_vector_builder elts (type, encoded_nelts, nelts_per_pattern);
- for (i = 0; i < encoded_nelts; i++)
+ tree_vector_builder elts (type, npatterns, nelts_per_pattern);
+ for (i = 0; i < npatterns * nelts_per_pattern; i++)
{
tree elt = fold_convert_const (subcode, TREE_TYPE (type),
VECTOR_CST_ELT (arg0, i + offset));
@@ -2026,10 +2007,6 @@ const_unop (enum tree_code code, tree type, tree arg0)
elts.quick_push (elt);
}
- if (nelts_per_pattern == 2)
- for (i = 0; i < encoded_nelts; ++i)
- elts.quick_push (fill);
-
return elts.build ();
}
