in r16-6508-g772b847d99d5e3 I made some typos
1. I forgot to negate the else value
2. one of the patterns was missing the mask.
This fixes it. The patterns moving the COND inwards are still useful because
they allow FMA forwarding as most micro-architectures don't forward FMA when
there's a random instruction like fneg in between.
Sorry for the mistakes. I added more tests to cover these now.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
Ok for master? and backport to GCC 16?
Thanks,
Tamar
gcc/ChangeLog:
* match.pd: Fix FMS patterns.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/cond_fma.c: New test.
* gcc.target/aarch64/sve/cond_fma_neg_addend.c: New test.
* gcc.target/aarch64/sve/cond_fms.c: New test.
* gcc.target/aarch64/sve/cond_fnma.c: New test.
* gcc.target/aarch64/sve/cond_fnms.c: New test.
---
diff --git a/gcc/match.pd b/gcc/match.pd
index
536d5125a0b6ea5e0cb54d46f682b0840b025617..7df879733ce3816de5eb061804266857e81ca3af
100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -10382,14 +10382,14 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
(IFN_COND_FNMA @3 @0 @1 @2 @4))
(simplify
(fmas @3 @0 @1 (negate @2) @4)
- (IFN_COND_FMS @0 @1 @2 @4))
+ (IFN_COND_FMS @3 @0 @1 @2 @4))
(simplify
(fmas:c @3 (negate @0) @1 (negate @2) @4)
(IFN_COND_FNMS @3 @0 @1 @2 @4))
(simplify
(negate (fmas@3 @4 @0 @1 @2 @5))
(if (!HONOR_SIGN_DEPENDENT_ROUNDING (type) && single_use (@3))
- (IFN_COND_FNMS @4 @0 @1 @2 @5))))
+ (IFN_COND_FNMS @4 @0 @1 @2 (negate @5)))))
(simplify
(IFN_COND_FMS:c @3 (negate @0) @1 @2 @4)
@@ -10401,9 +10401,9 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
(IFN_COND_FMS:c @3 (negate @0) @1 (negate @2) @4)
(IFN_COND_FNMA @3 @0 @1 @2 @4))
(simplify
- (negate (IFN_FMS@3 @4 @0 @1 @2 @5))
+ (negate (IFN_COND_FMS@3 @4 @0 @1 @2 @5))
(if (!HONOR_SIGN_DEPENDENT_ROUNDING (type) && single_use (@3))
- (IFN_COND_FNMA @4 @0 @1 @2 @5)))
+ (IFN_COND_FNMA @4 @0 @1 @2 (negate @5))))
(simplify
(IFN_COND_FNMA:c @3 (negate @0) @1 @2 @4)
@@ -10417,7 +10417,7 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
(simplify
(negate (IFN_COND_FNMA@3 @4 @0 @1 @2 @5))
(if (!HONOR_SIGN_DEPENDENT_ROUNDING (type) && single_use (@3))
- (IFN_COND_FMS @4 @0 @1 @2 @5)))
+ (IFN_COND_FMS @4 @0 @1 @2 (negate @5))))
(simplify
(IFN_COND_FNMS:c @3 (negate @0) @1 @2 @4)
@@ -10431,7 +10431,7 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
(simplify
(negate (IFN_COND_FNMS@3 @4 @0 @1 @2 @5))
(if (!HONOR_SIGN_DEPENDENT_ROUNDING (type) && single_use (@3))
- (IFN_COND_FMA @4 @0 @1 @2 @5))))
+ (IFN_COND_FMA @4 @0 @1 @2 (negate @5)))))
/* CLZ simplifications. */
(for clz (CLZ)
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_fma.c
b/gcc/testsuite/gcc.target/aarch64/sve/cond_fma.c
new file mode 100644
index
0000000000000000000000000000000000000000..e4bc381a193dde6122d594e973c5b05ac32fb8ca
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_fma.c
@@ -0,0 +1,37 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize" } */
+
+float a[256], b[256], c[256], d[256], r[256];
+
+__attribute__((noipa)) void
+f (int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ float t = d[i] > 0.f ? __builtin_fmaf (a[i], b[i], c[i]) : d[i];
+ r[i] = -t;
+ }
+}
+
+int
+main (void)
+{
+ for (int i = 0; i < 256; i++)
+ {
+ a[i] = 2.f;
+ b[i] = 3.f;
+ c[i] = 5.f;
+ d[i] = i & 1 ? 4.f : -7.f;
+ r[i] = 0.f;
+ }
+
+ f (256);
+ for (int i = 0; i < 256; i++)
+ {
+ float expected = d[i] > 0.f ? -11.f : 7.f;
+ if (r[i] != expected)
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_fma_neg_addend.c
b/gcc/testsuite/gcc.target/aarch64/sve/cond_fma_neg_addend.c
new file mode 100644
index
0000000000000000000000000000000000000000..e2c9dd1a68066bf3706282290387ef5e9216318b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_fma_neg_addend.c
@@ -0,0 +1,34 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize" } */
+
+float a[256], b[256], c[256], d[256], r[256];
+
+__attribute__((noipa)) void
+f (int n)
+{
+ for (int i = 0; i < n; i++)
+ r[i] = d[i] > 0.f ? __builtin_fmaf (a[i], b[i], -c[i]) : d[i];
+}
+
+int
+main (void)
+{
+ for (int i = 0; i < 256; i++)
+ {
+ a[i] = 2.f;
+ b[i] = 3.f;
+ c[i] = 5.f;
+ d[i] = i & 1 ? 4.f : -7.f;
+ r[i] = 0.f;
+ }
+
+ f (256);
+ for (int i = 0; i < 256; i++)
+ {
+ float expected = d[i] > 0.f ? 1.f : -7.f;
+ if (r[i] != expected)
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_fms.c
b/gcc/testsuite/gcc.target/aarch64/sve/cond_fms.c
new file mode 100644
index
0000000000000000000000000000000000000000..e383d8eba02534233ecd4c5dcfbe01ac48ea09e9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_fms.c
@@ -0,0 +1,37 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize" } */
+
+float a[256], b[256], c[256], d[256], r[256];
+
+__attribute__((noipa)) void
+f (int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ float t = d[i] > 0.f ? __builtin_fmaf (a[i], b[i], -c[i]) : d[i];
+ r[i] = -t;
+ }
+}
+
+int
+main (void)
+{
+ for (int i = 0; i < 256; i++)
+ {
+ a[i] = 2.f;
+ b[i] = 3.f;
+ c[i] = 5.f;
+ d[i] = i & 1 ? 4.f : -7.f;
+ r[i] = 0.f;
+ }
+
+ f (256);
+ for (int i = 0; i < 256; i++)
+ {
+ float expected = d[i] > 0.f ? -1.f : 7.f;
+ if (r[i] != expected)
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_fnma.c
b/gcc/testsuite/gcc.target/aarch64/sve/cond_fnma.c
new file mode 100644
index
0000000000000000000000000000000000000000..dcee73c0c2e3e72275dbafc3197a7d90a9418137
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_fnma.c
@@ -0,0 +1,37 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize" } */
+
+float a[256], b[256], c[256], d[256], r[256];
+
+__attribute__((noipa)) void
+f (int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ float t = d[i] > 0.f ? __builtin_fmaf (-a[i], b[i], c[i]) : d[i];
+ r[i] = -t;
+ }
+}
+
+int
+main (void)
+{
+ for (int i = 0; i < 256; i++)
+ {
+ a[i] = 2.f;
+ b[i] = 3.f;
+ c[i] = 5.f;
+ d[i] = i & 1 ? 4.f : -7.f;
+ r[i] = 0.f;
+ }
+
+ f (256);
+ for (int i = 0; i < 256; i++)
+ {
+ float expected = d[i] > 0.f ? 1.f : 7.f;
+ if (r[i] != expected)
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_fnms.c
b/gcc/testsuite/gcc.target/aarch64/sve/cond_fnms.c
new file mode 100644
index
0000000000000000000000000000000000000000..c2951bc5aae80032ee8889621d811de4da92167b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_fnms.c
@@ -0,0 +1,37 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize" } */
+
+float a[256], b[256], c[256], d[256], r[256];
+
+__attribute__((noipa)) void
+f (int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ float t = d[i] > 0.f ? __builtin_fmaf (-a[i], b[i], -c[i]) : d[i];
+ r[i] = -t;
+ }
+}
+
+int
+main (void)
+{
+ for (int i = 0; i < 256; i++)
+ {
+ a[i] = 2.f;
+ b[i] = 3.f;
+ c[i] = 5.f;
+ d[i] = i & 1 ? 4.f : -7.f;
+ r[i] = 0.f;
+ }
+
+ f (256);
+ for (int i = 0; i < 256; i++)
+ {
+ float expected = d[i] > 0.f ? 11.f : 7.f;
+ if (r[i] != expected)
+ __builtin_abort ();
+ }
+
+ return 0;
+}
--
diff --git a/gcc/match.pd b/gcc/match.pd
index 536d5125a0b6ea5e0cb54d46f682b0840b025617..7df879733ce3816de5eb061804266857e81ca3af 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -10382,14 +10382,14 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
(IFN_COND_FNMA @3 @0 @1 @2 @4))
(simplify
(fmas @3 @0 @1 (negate @2) @4)
- (IFN_COND_FMS @0 @1 @2 @4))
+ (IFN_COND_FMS @3 @0 @1 @2 @4))
(simplify
(fmas:c @3 (negate @0) @1 (negate @2) @4)
(IFN_COND_FNMS @3 @0 @1 @2 @4))
(simplify
(negate (fmas@3 @4 @0 @1 @2 @5))
(if (!HONOR_SIGN_DEPENDENT_ROUNDING (type) && single_use (@3))
- (IFN_COND_FNMS @4 @0 @1 @2 @5))))
+ (IFN_COND_FNMS @4 @0 @1 @2 (negate @5)))))
(simplify
(IFN_COND_FMS:c @3 (negate @0) @1 @2 @4)
@@ -10401,9 +10401,9 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
(IFN_COND_FMS:c @3 (negate @0) @1 (negate @2) @4)
(IFN_COND_FNMA @3 @0 @1 @2 @4))
(simplify
- (negate (IFN_FMS@3 @4 @0 @1 @2 @5))
+ (negate (IFN_COND_FMS@3 @4 @0 @1 @2 @5))
(if (!HONOR_SIGN_DEPENDENT_ROUNDING (type) && single_use (@3))
- (IFN_COND_FNMA @4 @0 @1 @2 @5)))
+ (IFN_COND_FNMA @4 @0 @1 @2 (negate @5))))
(simplify
(IFN_COND_FNMA:c @3 (negate @0) @1 @2 @4)
@@ -10417,7 +10417,7 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
(simplify
(negate (IFN_COND_FNMA@3 @4 @0 @1 @2 @5))
(if (!HONOR_SIGN_DEPENDENT_ROUNDING (type) && single_use (@3))
- (IFN_COND_FMS @4 @0 @1 @2 @5)))
+ (IFN_COND_FMS @4 @0 @1 @2 (negate @5))))
(simplify
(IFN_COND_FNMS:c @3 (negate @0) @1 @2 @4)
@@ -10431,7 +10431,7 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
(simplify
(negate (IFN_COND_FNMS@3 @4 @0 @1 @2 @5))
(if (!HONOR_SIGN_DEPENDENT_ROUNDING (type) && single_use (@3))
- (IFN_COND_FMA @4 @0 @1 @2 @5))))
+ (IFN_COND_FMA @4 @0 @1 @2 (negate @5)))))
/* CLZ simplifications. */
(for clz (CLZ)
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_fma.c b/gcc/testsuite/gcc.target/aarch64/sve/cond_fma.c
new file mode 100644
index 0000000000000000000000000000000000000000..e4bc381a193dde6122d594e973c5b05ac32fb8ca
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_fma.c
@@ -0,0 +1,37 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize" } */
+
+float a[256], b[256], c[256], d[256], r[256];
+
+__attribute__((noipa)) void
+f (int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ float t = d[i] > 0.f ? __builtin_fmaf (a[i], b[i], c[i]) : d[i];
+ r[i] = -t;
+ }
+}
+
+int
+main (void)
+{
+ for (int i = 0; i < 256; i++)
+ {
+ a[i] = 2.f;
+ b[i] = 3.f;
+ c[i] = 5.f;
+ d[i] = i & 1 ? 4.f : -7.f;
+ r[i] = 0.f;
+ }
+
+ f (256);
+ for (int i = 0; i < 256; i++)
+ {
+ float expected = d[i] > 0.f ? -11.f : 7.f;
+ if (r[i] != expected)
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_fma_neg_addend.c b/gcc/testsuite/gcc.target/aarch64/sve/cond_fma_neg_addend.c
new file mode 100644
index 0000000000000000000000000000000000000000..e2c9dd1a68066bf3706282290387ef5e9216318b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_fma_neg_addend.c
@@ -0,0 +1,34 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize" } */
+
+float a[256], b[256], c[256], d[256], r[256];
+
+__attribute__((noipa)) void
+f (int n)
+{
+ for (int i = 0; i < n; i++)
+ r[i] = d[i] > 0.f ? __builtin_fmaf (a[i], b[i], -c[i]) : d[i];
+}
+
+int
+main (void)
+{
+ for (int i = 0; i < 256; i++)
+ {
+ a[i] = 2.f;
+ b[i] = 3.f;
+ c[i] = 5.f;
+ d[i] = i & 1 ? 4.f : -7.f;
+ r[i] = 0.f;
+ }
+
+ f (256);
+ for (int i = 0; i < 256; i++)
+ {
+ float expected = d[i] > 0.f ? 1.f : -7.f;
+ if (r[i] != expected)
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_fms.c b/gcc/testsuite/gcc.target/aarch64/sve/cond_fms.c
new file mode 100644
index 0000000000000000000000000000000000000000..e383d8eba02534233ecd4c5dcfbe01ac48ea09e9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_fms.c
@@ -0,0 +1,37 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize" } */
+
+float a[256], b[256], c[256], d[256], r[256];
+
+__attribute__((noipa)) void
+f (int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ float t = d[i] > 0.f ? __builtin_fmaf (a[i], b[i], -c[i]) : d[i];
+ r[i] = -t;
+ }
+}
+
+int
+main (void)
+{
+ for (int i = 0; i < 256; i++)
+ {
+ a[i] = 2.f;
+ b[i] = 3.f;
+ c[i] = 5.f;
+ d[i] = i & 1 ? 4.f : -7.f;
+ r[i] = 0.f;
+ }
+
+ f (256);
+ for (int i = 0; i < 256; i++)
+ {
+ float expected = d[i] > 0.f ? -1.f : 7.f;
+ if (r[i] != expected)
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_fnma.c b/gcc/testsuite/gcc.target/aarch64/sve/cond_fnma.c
new file mode 100644
index 0000000000000000000000000000000000000000..dcee73c0c2e3e72275dbafc3197a7d90a9418137
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_fnma.c
@@ -0,0 +1,37 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize" } */
+
+float a[256], b[256], c[256], d[256], r[256];
+
+__attribute__((noipa)) void
+f (int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ float t = d[i] > 0.f ? __builtin_fmaf (-a[i], b[i], c[i]) : d[i];
+ r[i] = -t;
+ }
+}
+
+int
+main (void)
+{
+ for (int i = 0; i < 256; i++)
+ {
+ a[i] = 2.f;
+ b[i] = 3.f;
+ c[i] = 5.f;
+ d[i] = i & 1 ? 4.f : -7.f;
+ r[i] = 0.f;
+ }
+
+ f (256);
+ for (int i = 0; i < 256; i++)
+ {
+ float expected = d[i] > 0.f ? 1.f : 7.f;
+ if (r[i] != expected)
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_fnms.c b/gcc/testsuite/gcc.target/aarch64/sve/cond_fnms.c
new file mode 100644
index 0000000000000000000000000000000000000000..c2951bc5aae80032ee8889621d811de4da92167b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_fnms.c
@@ -0,0 +1,37 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize" } */
+
+float a[256], b[256], c[256], d[256], r[256];
+
+__attribute__((noipa)) void
+f (int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ float t = d[i] > 0.f ? __builtin_fmaf (-a[i], b[i], -c[i]) : d[i];
+ r[i] = -t;
+ }
+}
+
+int
+main (void)
+{
+ for (int i = 0; i < 256; i++)
+ {
+ a[i] = 2.f;
+ b[i] = 3.f;
+ c[i] = 5.f;
+ d[i] = i & 1 ? 4.f : -7.f;
+ r[i] = 0.f;
+ }
+
+ f (256);
+ for (int i = 0; i < 256; i++)
+ {
+ float expected = d[i] > 0.f ? 11.f : 7.f;
+ if (r[i] != expected)
+ __builtin_abort ();
+ }
+
+ return 0;
+}
