[COMMITTED/12] Fix PR 105532: match.pd patterns calling tree_nonzero_bits with vector types

[Andrew Pinski via Gcc-patches]

Even though this PR was reported with an ubsan issue, the problem is
tree_nonzero_bits is being called with an expression which is a vector type.
This fixes three patterns I noticed which does that.
And adds a testcase for one of the patterns.

Committed after a bootstrapped and tested on x86_64-linux-gnu with no 
regressions

gcc/ChangeLog:

        PR tree-optimization/105532
        * match.pd (~(X >> Y) -> ~X >> Y): Check if it is an integral
        type before calling tree_nonzero_bits.
        (popcount(X) + popcount(Y)): Likewise.
        (popcount(X&C1)): Likewise.

gcc/testsuite/ChangeLog:

        * gcc.c-torture/compile/vector-shift-1.c: New test.

(cherry picked from commit 193fccaa5c3525e979a989835c47c76d2c49d10c)
---
 gcc/match.pd                                  | 25 +++++++++++--------
 .../gcc.c-torture/compile/vector-shift-1.c    |  8 ++++++
 2 files changed, 22 insertions(+), 11 deletions(-)
 create mode 100644 gcc/testsuite/gcc.c-torture/compile/vector-shift-1.c

diff --git a/gcc/match.pd b/gcc/match.pd
index ef352af1572..fc2833bbdca 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -1268,7 +1268,8 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
    /* For logical right shifts, this is possible only if @0 doesn't
       have MSB set and the logical right shift is changed into
       arithmetic shift.  */
-   (if (!wi::neg_p (tree_nonzero_bits (@0)))
+   (if (INTEGRAL_TYPE_P (type)
+        && !wi::neg_p (tree_nonzero_bits (@0)))
     (with { tree stype = signed_type_for (TREE_TYPE (@0)); }
      (convert (rshift (bit_not! (convert:stype @0)) @1))))))
 #endif
@@ -7169,7 +7170,8 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
 /* popcount(X) + popcount(Y) is popcount(X|Y) when X&Y must be zero.  */
 (simplify
   (plus (POPCOUNT:s @0) (POPCOUNT:s @1))
-  (if (wi::bit_and (tree_nonzero_bits (@0), tree_nonzero_bits (@1)) == 0)
+  (if (INTEGRAL_TYPE_P (type)
+       && wi::bit_and (tree_nonzero_bits (@0), tree_nonzero_bits (@1)) == 0)
     (POPCOUNT (bit_ior @0 @1))))
 
 /* popcount(X) == 0 is X == 0, and related (in)equalities.  */
@@ -7201,15 +7203,16 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
 (for pfun (POPCOUNT PARITY)
   (simplify
     (pfun @0)
-    (with { wide_int nz = tree_nonzero_bits (@0); }
-      (switch
-       (if (nz == 1)
-         (convert @0))
-       (if (wi::popcount (nz) == 1)
-         (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
-           (convert (rshift:utype (convert:utype @0)
-                                  { build_int_cst (integer_type_node,
-                                                   wi::ctz (nz)); }))))))))
+    (if (INTEGRAL_TYPE_P (type))
+     (with { wide_int nz = tree_nonzero_bits (@0); }
+       (switch
+        (if (nz == 1)
+          (convert @0))
+        (if (wi::popcount (nz) == 1)
+          (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
+            (convert (rshift:utype (convert:utype @0)
+                                   { build_int_cst (integer_type_node,
+                                                    wi::ctz (nz)); })))))))))
 
 #if GIMPLE
 /* 64- and 32-bits branchless implementations of popcount are detected:
diff --git a/gcc/testsuite/gcc.c-torture/compile/vector-shift-1.c 
b/gcc/testsuite/gcc.c-torture/compile/vector-shift-1.c
new file mode 100644
index 00000000000..142ea56d5bb
--- /dev/null
+++ b/gcc/testsuite/gcc.c-torture/compile/vector-shift-1.c
@@ -0,0 +1,8 @@
+typedef unsigned char __attribute__((__vector_size__ (1))) U;
+
+U
+foo (U u)
+{
+  u = u == u;
+  return (~(u >> 255));
+}
-- 
2.17.1