From: Kyrylo Tkachov <[email protected]>
Advanced SIMD has no 64x64 high-part multiply, so the vectorizer
currently cannot use its modes for loops such as:
for (int i = 0; i < n; i++)
d[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
SVE has UMULH and SMULH for D elements and the Z registers overlap the V
registers, so the Advanced SIMD optab can be implemented on top of the SVE
instruction. Extend the high-part multiply patterns from SVE_I to
SVE_I_SIMD_DI, exactly as mul<mode>3 already does for vector DImode
multiplication, and print the operands with %Z so that the V2DI form uses
the Z register names.
For the loop above on a 128-bit SVE implementation we now generate:
ldr q31, [x1, x4]
ldr q30, [x2, x4]
umulh z30.d, z31.d, z30.d
str q30, [x0, x4]
rather than:
ldr x5, [x1, x4]
ldr x6, [x2, x4]
umulh x5, x5, x6
str x5, [x0, x4]
I don't think people often run the testuite with -march=armv8.2-a+sve
-mautovec-preference=asimd-only and I don't know how clean it is, so
I've added the runtime test generically to torture and included it in
gcc.target/aarch64/ with the aarch64 flags added on top.
Bootstrapped and regression-tested on aarch64-unknown-linux-gnu.
Ok for trunk?
Thanks,
Kyrill
gcc/ChangeLog:
* config/aarch64/aarch64-sve.md (<su>mul<mode>3_highpart): Extend
from SVE_I to SVE_I_SIMD_DI.
(@aarch64_pred_<optab><mode>): Likewise for the MUL_HIGHPART form,
and print the data operands with %Z.
* config/aarch64/aarch64-sve2.md (@aarch64_sve2_<optab><mode>):
Likewise.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/mul_highpart_v2di_1.c: New test.
* gcc.target/aarch64/sve/mul_highpart_v2di_2.c: New test.
* gcc.target/aarch64/sve/mul_highpart_v2di_3.c: New test.
* gcc.target/aarch64/sve/mul_highpart_v2di_run.c: New test.
* gcc.dg/torture/mul-highpart-1.c: New test.
Signed-off-by: Kyrylo Tkachov <[email protected]>
---
gcc/config/aarch64/aarch64-sve.md | 29 +++++----
gcc/config/aarch64/aarch64-sve2.md | 13 ++--
gcc/testsuite/gcc.dg/torture/mul-highpart-1.c | 62 +++++++++++++++++++
.../aarch64/sve/mul_highpart_v2di_1.c | 37 +++++++++++
.../aarch64/sve/mul_highpart_v2di_2.c | 35 +++++++++++
.../aarch64/sve/mul_highpart_v2di_3.c | 26 ++++++++
.../aarch64/sve/mul_highpart_v2di_run.c | 7 +++
7 files changed, 190 insertions(+), 19 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/torture/mul-highpart-1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c
diff --git a/gcc/config/aarch64/aarch64-sve.md
b/gcc/config/aarch64/aarch64-sve.md
index f665417dba4..5f9a19c42e9 100644
--- a/gcc/config/aarch64/aarch64-sve.md
+++ b/gcc/config/aarch64/aarch64-sve.md
@@ -4881,13 +4881,16 @@
;; -------------------------------------------------------------------------
;; Unpredicated highpart multiplication.
+;; Advanced SIMD has no vector DImode high-part multiply, but SVE does.
+;; Make use of the overlap between Z and V registers to implement the V2DI
+;; optab for TARGET_SVE, in the same way as the mul<mode>3 expander above.
(define_expand "<su>mul<mode>3_highpart"
- [(set (match_operand:SVE_I 0 "register_operand")
- (unspec:SVE_I
+ [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand")
+ (unspec:SVE_I_SIMD_DI
[(match_dup 3)
- (unspec:SVE_I
- [(match_operand:SVE_I 1 "register_operand")
- (match_operand:SVE_I 2 "register_operand")]
+ (unspec:SVE_I_SIMD_DI
+ [(match_operand:SVE_I_SIMD_DI 1 "register_operand")
+ (match_operand:SVE_I_SIMD_DI 2 "register_operand")]
MUL_HIGHPART)]
UNSPEC_PRED_X))]
"TARGET_SVE"
@@ -4898,22 +4901,22 @@
;; Predicated highpart multiplication.
(define_insn_and_split "@aarch64_pred_<optab><mode>"
- [(set (match_operand:SVE_I 0 "register_operand")
- (unspec:SVE_I
+ [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand")
+ (unspec:SVE_I_SIMD_DI
[(match_operand:<VPRED> 1 "register_operand")
- (unspec:SVE_I
- [(match_operand:SVE_I 2 "register_operand")
- (match_operand:SVE_I 3 "register_operand")]
+ (unspec:SVE_I_SIMD_DI
+ [(match_operand:SVE_I_SIMD_DI 2 "register_operand")
+ (match_operand:SVE_I_SIMD_DI 3 "register_operand")]
MUL_HIGHPART)]
UNSPEC_PRED_X))]
"TARGET_SVE"
{@ [ cons: =0 , 1 , %2 , 3 ; attrs: movprfx ]
- [ w , Upl , 0 , w ; * ] <su>mulh\t%0.<Vetype>, %1/m,
%0.<Vetype>, %3.<Vetype>
- [ ?&w , Upl , w , w ; yes ] movprfx\t%0,
%2\;<su>mulh\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
+ [ w , Upl , 0 , w ; * ] <su>mulh\t%Z0.<Vetype>,
%1/m, %Z0.<Vetype>, %Z3.<Vetype>
+ [ ?&w , Upl , w , w ; yes ] movprfx\t%Z0,
%Z2\;<su>mulh\t%Z0.<Vetype>, %1/m, %Z0.<Vetype>, %Z3.<Vetype>
}
"TARGET_SVE2"
[(set (match_dup 0)
- (unspec:SVE_I
+ (unspec:SVE_I_SIMD_DI
[(match_dup 2)
(match_dup 3)]
MUL_HIGHPART))]
diff --git a/gcc/config/aarch64/aarch64-sve2.md
b/gcc/config/aarch64/aarch64-sve2.md
index a8bee6b6dc7..1cc0cefbee7 100644
--- a/gcc/config/aarch64/aarch64-sve2.md
+++ b/gcc/config/aarch64/aarch64-sve2.md
@@ -983,15 +983,16 @@
;; ---- [INT] Unpredicated high-part multiplication
;; -------------------------------------------------------------------------
-;; SVE2 unpredicated SMULH/UMULH.
+;; SVE2 unpredicated SMULH/UMULH. V2DI is included so that the Advanced
+;; SIMD high-part multiply optabs can be implemented on top of SVE.
(define_insn "@aarch64_sve2_<optab><mode>"
- [(set (match_operand:SVE_I 0 "register_operand" "=w")
- (unspec:SVE_I
- [(match_operand:SVE_I 1 "register_operand" "w")
- (match_operand:SVE_I 2 "register_operand" "w")]
+ [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand" "=w")
+ (unspec:SVE_I_SIMD_DI
+ [(match_operand:SVE_I_SIMD_DI 1 "register_operand" "w")
+ (match_operand:SVE_I_SIMD_DI 2 "register_operand" "w")]
MUL_HIGHPART))]
"TARGET_SVE2"
- "<su>mulh\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
+ "<su>mulh\t%Z0.<Vetype>, %Z1.<Vetype>, %Z2.<Vetype>"
[(set_attr "sve_type" "sve_int_mul")]
)
diff --git a/gcc/testsuite/gcc.dg/torture/mul-highpart-1.c
b/gcc/testsuite/gcc.dg/torture/mul-highpart-1.c
new file mode 100644
index 00000000000..0c6b7812863
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/mul-highpart-1.c
@@ -0,0 +1,62 @@
+/* Check that a 64-bit high-part multiply gives the same answer whether or not
+ it is vectorized. */
+/* { dg-do run } */
+/* { dg-require-effective-target int128 } */
+/* { dg-additional-options "-ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define N 77
+
+static int64_t sa[N], sb[N], sd[N];
+static uint64_t ua[N], ub[N], ud[N];
+
+void __attribute__ ((noipa))
+mulh_s64 (int64_t *restrict dst, int64_t *restrict a, int64_t *restrict b,
+ int count)
+{
+ for (int i = 0; i < count; ++i)
+ dst[i] = (int64_t) (((__int128) a[i] * b[i]) >> 64);
+}
+
+void __attribute__ ((noipa))
+mulh_u64 (uint64_t *restrict dst, uint64_t *restrict a, uint64_t *restrict b,
+ int count)
+{
+ for (int i = 0; i < count; ++i)
+ dst[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
+}
+
+int
+main (void)
+{
+ uint64_t s = 0x243f6a8885a308d3ULL;
+ for (int i = 0; i < N; ++i)
+ {
+ s ^= s << 13; s ^= s >> 7; s ^= s << 17;
+ sa[i] = (int64_t) s;
+ ua[i] = s;
+ s ^= s << 13; s ^= s >> 7; s ^= s << 17;
+ sb[i] = (int64_t) s;
+ ub[i] = s;
+ }
+ /* Boundary values. */
+ sa[0] = INT64_MIN; sb[0] = INT64_MIN;
+ sa[1] = INT64_MIN; sb[1] = -1;
+ sa[2] = -1; sb[2] = -1;
+ ua[0] = 0; ub[0] = ~(uint64_t) 0;
+ ua[1] = ~(uint64_t) 0; ub[1] = ~(uint64_t) 0;
+ ua[2] = (uint64_t) 1 << 63; ub[2] = (uint64_t) 1 << 63;
+
+ mulh_s64 (sd, sa, sb, N);
+ mulh_u64 (ud, ua, ub, N);
+
+ for (int i = 0; i < N; ++i)
+ {
+ if (sd[i] != (int64_t) (((__int128) sa[i] * sb[i]) >> 64))
+ __builtin_abort ();
+ if (ud[i] != (uint64_t) (((unsigned __int128) ua[i] * ub[i]) >> 64))
+ __builtin_abort ();
+ }
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c
b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c
new file mode 100644
index 00000000000..2544e4c6b79
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c
@@ -0,0 +1,37 @@
+/* Advanced SIMD has no 64-bit high-part multiply, but SVE does, and the two
+ register files overlap. Check that the SVE instruction is used for a
+ 128-bit vector high-part multiply when the vectoriser is restricted to
+ Advanced SIMD modes. */
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve
-mautovec-preference=asimd-only" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <stdint.h>
+
+/*
+** mulh_s64:
+** ...
+** smulh z[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d
+** ...
+*/
+void __attribute__ ((noipa))
+mulh_s64 (int64_t *restrict dst, int64_t *restrict a, int64_t *restrict b,
+ int count)
+{
+ for (int i = 0; i < count; ++i)
+ dst[i] = (int64_t) (((__int128) a[i] * b[i]) >> 64);
+}
+
+/*
+** mulh_u64:
+** ...
+** umulh z[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d
+** ...
+*/
+void __attribute__ ((noipa))
+mulh_u64 (uint64_t *restrict dst, uint64_t *restrict a, uint64_t *restrict b,
+ int count)
+{
+ for (int i = 0; i < count; ++i)
+ dst[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c
b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c
new file mode 100644
index 00000000000..3f3989fa901
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c
@@ -0,0 +1,35 @@
+/* Same as mul_highpart_v2di_1.c, but for SVE2, where the high-part multiply
+ has an unpredicated form. */
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2
-mautovec-preference=asimd-only" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <stdint.h>
+
+/*
+** mulh_s64:
+** ...
+** smulh z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d
+** ...
+*/
+void __attribute__ ((noipa))
+mulh_s64 (int64_t *restrict dst, int64_t *restrict a, int64_t *restrict b,
+ int count)
+{
+ for (int i = 0; i < count; ++i)
+ dst[i] = (int64_t) (((__int128) a[i] * b[i]) >> 64);
+}
+
+/*
+** mulh_u64:
+** ...
+** umulh z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d
+** ...
+*/
+void __attribute__ ((noipa))
+mulh_u64 (uint64_t *restrict dst, uint64_t *restrict a, uint64_t *restrict b,
+ int count)
+{
+ for (int i = 0; i < count; ++i)
+ dst[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c
b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c
new file mode 100644
index 00000000000..b459bbc27cc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c
@@ -0,0 +1,26 @@
+/* The Shoup modular multiply used by lattice cryptography. The high-part
+ multiply is the only operation Advanced SIMD lacks, so without a 128-bit
+ vector form of it the whole loop stays scalar. */
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2
-mautovec-preference=asimd-only" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <stdint.h>
+
+/*
+** mul_mod:
+** ...
+** umulh z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d
+** ...
+*/
+void __attribute__ ((noipa))
+mul_mod (uint64_t *restrict dst, const uint64_t *restrict src,
+ uint64_t operand, uint64_t quotient, uint64_t modulus, int count)
+{
+ for (int i = 0; i < count; ++i)
+ {
+ uint64_t hi = (uint64_t) (((unsigned __int128) src[i] * quotient) >> 64);
+ uint64_t t = operand * src[i] - hi * modulus;
+ dst[i] = t >= modulus ? t - modulus : t;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c
b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c
new file mode 100644
index 00000000000..a77f2ef2207
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c
@@ -0,0 +1,7 @@
+/* Run the generic high-part multiply correctness test with the vectoriser
+ restricted to Advanced SIMD modes, so that the V2DI patterns are the ones
+ being exercised. */
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve
-mautovec-preference=asimd-only" } */
+
+#include "../../../gcc.dg/torture/mul-highpart-1.c"
--
2.50.1 (Apple Git-155)
