From: Kyrylo Tkachov <[email protected]>
vect_recog_mulhs_pattern turns (a * b) >> N into IFN_MULH and casts the
result back to the type of the shift, which the over-widening machinery
then drops. Building that cast needs a vector type for the wide type, and
the pattern gives up when there is none.
For a 64-bit high-part multiply the wide type is 128 bits, and no target
has a vector of 128-bit integers, so on AArch64 the pattern never fires
for scalable SVE and the loop stays scalar. It does fire under
-msve-vector-bits=128, where the prevailing vector size is 128 bits and a
one-element TImode vector exists, which is what makes the failure specific
to variable-length vectors.
The pattern has already established that the users of the result need at
most min_output_precision bits and that NEW_TYPE holds at least that many,
so when the wide vector type is unavailable we can simply leave the result
at NEW_TYPE precision and skip the cast.
for (int i = 0; i < n; i++)
d[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
on -march=armv8.2-a+sve2 now gives
ld1d z28.d, p7/z, [x1, x6, lsl 3]
ld1d z27.d, p7/z, [x2, x6, lsl 3]
umulh z28.d, z28.d, z27.d
st1d z28.d, p7, [x0, x6, lsl 3]
instead of a scalar loop.
Bootstrapped and regression-tested on aarch64-unknown-linux-gnu and
x86_64-linux.
Ok for trunk?
Thanks,
Kyrill
gcc/ChangeLog:
* tree-vect-patterns.cc (vect_recog_mulhs_pattern): Keep the
pattern result at NEW_TYPE precision when LHS_TYPE has no vector
type.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/mul_highpart_scalable_1.c: New test.
* gcc.target/aarch64/sve/mul_highpart_scalable_run.c: New test.
Signed-off-by: Kyrylo Tkachov <[email protected]>
---
.../aarch64/sve/mul_highpart_scalable_1.c | 23 ++++++++++
.../aarch64/sve/mul_highpart_scalable_run.c | 42 +++++++++++++++++++
gcc/tree-vect-patterns.cc | 17 ++++++--
3 files changed, 78 insertions(+), 4 deletions(-)
create mode 100644
gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c
create mode 100644
gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_run.c
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c
b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c
new file mode 100644
index 00000000000..f61d235b7a8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c
@@ -0,0 +1,23 @@
+/* A 64-bit high-part multiply has a 128-bit product type, and there is no
+ scalable vector of 128-bit integers. Check that the vectoriser recognises
+ the high-part multiply anyway. */
+/* { dg-do assemble { target aarch64_asm_sve2_ok } } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2
-mautovec-preference=sve-only -msve-vector-bits=scalable --save-temps" } */
+
+#include <stdint.h>
+
+#define DEF_LOOP(TYPE, WIDE) \
+ void __attribute__ ((noipa)) \
+ mulh_##TYPE (TYPE *restrict dst, TYPE *restrict a, \
+ TYPE *restrict b, int count) \
+ { \
+ for (int i = 0; i < count; ++i) \
+ dst[i] = (TYPE) (((WIDE) a[i] * b[i]) >> 64); \
+ }
+
+DEF_LOOP (int64_t, __int128)
+DEF_LOOP (uint64_t, unsigned __int128)
+
+/* { dg-final { scan-assembler-times {\tsmulh\tz[0-9]+\.d, z[0-9]+\.d,
z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumulh\tz[0-9]+\.d, z[0-9]+\.d,
z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler {\twhilelo\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_run.c
b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_run.c
new file mode 100644
index 00000000000..d856302d7a1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_run.c
@@ -0,0 +1,42 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2
-mautovec-preference=sve-only -msve-vector-bits=scalable" } */
+
+#include "mul_highpart_scalable_1.c"
+
+#define N 77
+
+static int64_t sa[N], sb[N], sd[N];
+static uint64_t ua[N], ub[N], ud[N];
+
+int
+main (void)
+{
+ uint64_t s = 0x243f6a8885a308d3ULL;
+ for (int i = 0; i < N; ++i)
+ {
+ s ^= s << 13; s ^= s >> 7; s ^= s << 17;
+ sa[i] = (int64_t) s;
+ ua[i] = s;
+ s ^= s << 13; s ^= s >> 7; s ^= s << 17;
+ sb[i] = (int64_t) s;
+ ub[i] = s;
+ }
+ sa[0] = INT64_MIN; sb[0] = INT64_MIN;
+ sa[1] = INT64_MIN; sb[1] = -1;
+ sa[2] = -1; sb[2] = -1;
+ ua[0] = 0; ub[0] = ~0ULL;
+ ua[1] = ~0ULL; ub[1] = ~0ULL;
+ ua[2] = 1ULL << 63; ub[2] = 1ULL << 63;
+
+ mulh_int64_t (sd, sa, sb, N);
+ mulh_uint64_t (ud, ua, ub, N);
+
+ for (int i = 0; i < N; ++i)
+ {
+ if (sd[i] != (int64_t) (((__int128) sa[i] * sb[i]) >> 64))
+ __builtin_abort ();
+ if (ud[i] != (uint64_t) (((unsigned __int128) ua[i] * ub[i]) >> 64))
+ __builtin_abort ();
+ }
+ return 0;
+}
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index c57e215be57..9934ab8306a 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -3412,11 +3412,20 @@ vect_recog_mulhs_pattern (vec_info *vinfo,
(ifn, new_vectype, OPTIMIZE_FOR_SPEED))
return NULL;
- /* The IR requires a valid vector type for the cast result, even though
- it's likely to be discarded. */
+ /* The IFN result is cast back to LHS_TYPE, a cast that the over-widening
+ machinery then drops. Some targets have no vector form of LHS_TYPE at
+ all: a 64-bit high-part multiply has a 128-bit LHS_TYPE, and there is no
+ vector of 128-bit integers. Keep the pattern result at NEW_TYPE
+ precision in that case. This is safe because the check at the top of the
+ function proved that the users of the result need at most
+ TARGET_PRECISION bits, and NEW_TYPE holds at least that many. */
+ tree out_type = lhs_type;
*type_out = get_vectype_for_scalar_type (vinfo, lhs_type);
if (!*type_out)
- return NULL;
+ {
+ out_type = new_type;
+ *type_out = new_vectype;
+ }
/* Generate the IFN_MULHRS call. */
tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
@@ -3432,7 +3441,7 @@ vect_recog_mulhs_pattern (vec_info *vinfo,
dump_printf_loc (MSG_NOTE, vect_location,
"created pattern stmt: %G", (gimple *) mulhrs_stmt);
- return vect_convert_output (vinfo, last_stmt_info, lhs_type,
+ return vect_convert_output (vinfo, last_stmt_info, out_type,
mulhrs_stmt, new_vectype);
}
--
2.50.1 (Apple Git-155)
