Hi,
This patch adds a support of pattern recognition in basic block SLP.
Bootstrapped and tested on powerpc64-suse-linux.
Ira
ChangeLog:
* tree-vectorizer.h (vect_pattern_recog): Add new argument.
* tree-vect-loop.c (vect_analyze_loop_2): Update call to
vect_pattern_recog.
* tree-vect-patterns.c (widened_name_p): Pass basic block
info to vect_is_simple_use.
(vect_recog_dot_prod_pattern): Fail for basic blocks.
(vect_recog_widen_sum_pattern): Likewise.
(vect_handle_widen_op_by_const): Support basic blocks.
(vect_operation_fits_smaller_type,
vect_recog_over_widening_pattern): Likewise.
(vect_recog_vector_vector_shift_pattern): Support basic blocks.
Update call to vect_is_simple_use.
(vect_recog_mixed_size_cond_pattern): Support basic blocks.
Add printing.
(check_bool_pattern): Add an argument, update call to
vect_is_simple_use and the recursive calls.
(vect_recog_bool_pattern): Update relevant function calls.
Add printing.
(vect_mark_pattern_stmts): Update calls to new_stmt_vec_info.
(vect_pattern_recog_1): Check for reduction only in loops.
(vect_pattern_recog): Add new argument. Support basic blocks.
* tree-vect-stmts.c (vectorizable_conversion): Pass basic block
info to vect_is_simple_use_1.
* tree-vect-slp.c (vect_get_and_check_slp_defs): Support basic
blocks.
(vect_slp_analyze_bb_1): Call vect_pattern_recog.
testsuite/ChangeLog:
* gcc.dg/vect/bb-slp-pattern-1.c: New test.
* gcc.dg/vect/bb-slp-pattern-2.c: New test.
(See attached file: bb-patterns.txt)Index: testsuite/gcc.dg/vect/bb-slp-pattern-1.c
===================================================================
--- testsuite/gcc.dg/vect/bb-slp-pattern-1.c (revision 0)
+++ testsuite/gcc.dg/vect/bb-slp-pattern-1.c (revision 0)
@@ -0,0 +1,54 @@
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 8
+
+unsigned short X[N];
+unsigned short Y[N];
+unsigned int result[N];
+
+/* unsigned short->unsigned int widening-mult. */
+__attribute__ ((noinline, noclone)) void
+foo (void)
+{
+ result[0] = (unsigned int) (X[0] * Y[0]);
+ result[1] = (unsigned int) (X[1] * Y[1]);
+ result[2] = (unsigned int) (X[2] * Y[2]);
+ result[3] = (unsigned int) (X[3] * Y[3]);
+ result[4] = (unsigned int) (X[4] * Y[4]);
+ result[5] = (unsigned int) (X[5] * Y[5]);
+ result[6] = (unsigned int) (X[6] * Y[6]);
+ result[7] = (unsigned int) (X[7] * Y[7]);
+}
+
+int main (void)
+{
+ int i, tmp;
+
+ check_vect ();
+
+ for (i = 0; i < N; i++)
+ {
+ X[i] = i;
+ Y[i] = 64-i;
+ }
+
+ foo ();
+
+ for (i = 0; i < N; i++)
+ {
+ __asm__ volatile ("");
+ tmp = X[i] * Y[i];
+ if (result[i] != tmp)
+ abort ();
+ }
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 1 "slp" {
target { vect_widen_mult_hi_to_si || vect_unpack } } } } */
+/* { dg-final { scan-tree-dump-times "vect_recog_widen_mult_pattern: detected"
8 "slp" { target vect_widen_mult_hi_to_si_pattern } } } */
+/* { dg-final { scan-tree-dump-times "pattern recognized" 8 "slp" { target
vect_widen_mult_hi_to_si_pattern } } } */
+/* { dg-final { cleanup-tree-dump "slp" } } */
Index: testsuite/gcc.dg/vect/bb-slp-pattern-2.c
===================================================================
--- testsuite/gcc.dg/vect/bb-slp-pattern-2.c (revision 0)
+++ testsuite/gcc.dg/vect/bb-slp-pattern-2.c (revision 0)
@@ -0,0 +1,52 @@
+/* { dg-require-effective-target vect_condition } */
+
+#include "tree-vect.h"
+
+#define N 128
+
+__attribute__((noinline, noclone)) void
+foo (short * __restrict__ a, int * __restrict__ b, int stride)
+{
+ int i;
+
+ for (i = 0; i < N/stride; i++, a += stride, b += stride)
+ {
+ a[0] = b[0] ? 1 : 7;
+ a[1] = b[1] ? 2 : 0;
+ a[2] = b[2] ? 3 : 0;
+ a[3] = b[3] ? 4 : 0;
+ a[4] = b[4] ? 5 : 0;
+ a[5] = b[5] ? 6 : 0;
+ a[6] = b[6] ? 7 : 0;
+ a[7] = b[7] ? 8 : 0;
+ }
+}
+
+short a[N];
+int b[N];
+int main ()
+{
+ int i;
+
+ check_vect ();
+
+ for (i = 0; i < N; i++)
+ {
+ a[i] = i;
+ b[i] = -i;
+ }
+
+ foo (a, b, 8);
+
+ for (i = 1; i < N; i++)
+ if (a[i] != i%8 + 1)
+ abort ();
+
+ if (a[0] != 7)
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "basic block vectorized using SLP" 1
"slp" { target { vect_element_align && vect_pack_trunc } } } } */
+/* { dg-final { cleanup-tree-dump "slp" } } */
Index: tree-vectorizer.h
===================================================================
--- tree-vectorizer.h (revision 183902)
+++ tree-vectorizer.h (working copy)
@@ -933,7 +933,7 @@ extern void vect_slp_transform_bb (basic_block);
in the future. */
typedef gimple (* vect_recog_func_ptr) (VEC (gimple, heap) **, tree *, tree *);
#define NUM_PATTERNS 10
-void vect_pattern_recog (loop_vec_info);
+void vect_pattern_recog (loop_vec_info, bb_vec_info);
/* In tree-vectorizer.c. */
unsigned vectorize_loops (void);
Index: tree-vect-loop.c
===================================================================
--- tree-vect-loop.c (revision 183902)
+++ tree-vect-loop.c (working copy)
@@ -1514,7 +1514,7 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo)
vect_analyze_scalar_cycles (loop_vinfo);
- vect_pattern_recog (loop_vinfo);
+ vect_pattern_recog (loop_vinfo, NULL);
/* Data-flow analysis to detect stmts that do not need to be vectorized. */
Index: tree-vect-patterns.c
===================================================================
--- tree-vect-patterns.c (revision 183902)
+++ tree-vect-patterns.c (working copy)
@@ -105,12 +105,13 @@ widened_name_p (tree name, gimple use_stmt, tree *
tree oprnd0;
enum vect_def_type dt;
tree def;
+ bb_vec_info bb_vinfo;
stmt_vinfo = vinfo_for_stmt (use_stmt);
loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
-
- if (!vect_is_simple_use (name, use_stmt, loop_vinfo, NULL, def_stmt, &def,
- &dt))
+ bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
+ if (!vect_is_simple_use (name, use_stmt, loop_vinfo, bb_vinfo, def_stmt,
+ &def, &dt))
return false;
if (dt != vect_internal_def
@@ -135,7 +136,7 @@ widened_name_p (tree name, gimple use_stmt, tree *
return false;
if (!vect_is_simple_use (oprnd0, *def_stmt, loop_vinfo,
- NULL, &dummy_gimple, &dummy, &dt))
+ bb_vinfo, &dummy_gimple, &dummy, &dt))
return false;
return true;
@@ -211,9 +212,14 @@ vect_recog_dot_prod_pattern (VEC (gimple, heap) **
gimple pattern_stmt;
tree prod_type;
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
- struct loop *loop = LOOP_VINFO_LOOP (loop_info);
+ struct loop *loop;
tree var;
+ if (!loop_info)
+ return NULL;
+
+ loop = LOOP_VINFO_LOOP (loop_info);
+
if (!is_gimple_assign (last_stmt))
return NULL;
@@ -383,9 +389,17 @@ vect_handle_widen_op_by_const (gimple stmt, enum t
{
tree new_type, new_oprnd, tmp;
gimple new_stmt;
- loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (vinfo_for_stmt (stmt));
- struct loop *loop = LOOP_VINFO_LOOP (loop_info);
+ loop_vec_info loop_vinfo;
+ struct loop *loop = NULL;
+ bb_vec_info bb_vinfo;
+ stmt_vec_info stmt_vinfo;
+ stmt_vinfo = vinfo_for_stmt (stmt);
+ loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
+ if (loop_vinfo)
+ loop = LOOP_VINFO_LOOP (loop_vinfo);
+
if (code != MULT_EXPR && code != LSHIFT_EXPR)
return false;
@@ -402,7 +416,9 @@ vect_handle_widen_op_by_const (gimple stmt, enum t
if (TYPE_PRECISION (type) < (TYPE_PRECISION (*half_type) * 4)
|| !gimple_bb (def_stmt)
- || !flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
+ || (loop && !flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
+ || (!loop && gimple_bb (def_stmt) != BB_VINFO_BB (bb_vinfo)
+ && gimple_code (def_stmt) != GIMPLE_PHI)
|| !vinfo_for_stmt (def_stmt))
return false;
@@ -798,9 +814,14 @@ vect_recog_widen_sum_pattern (VEC (gimple, heap) *
tree type, half_type;
gimple pattern_stmt;
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
- struct loop *loop = LOOP_VINFO_LOOP (loop_info);
+ struct loop *loop;
tree var;
+ if (!loop_info)
+ return NULL;
+
+ loop = LOOP_VINFO_LOOP (loop_info);
+
if (!is_gimple_assign (last_stmt))
return NULL;
@@ -899,8 +920,12 @@ vect_operation_fits_smaller_type (gimple stmt, tre
gimple def_stmt, new_stmt;
bool first = false;
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (vinfo_for_stmt (stmt));
- struct loop *loop = LOOP_VINFO_LOOP (loop_info);
+ bb_vec_info bb_info = STMT_VINFO_BB_VINFO (vinfo_for_stmt (stmt));
+ struct loop *loop = NULL;
+ if (loop_info)
+ loop = LOOP_VINFO_LOOP (loop_info);
+
*op0 = NULL_TREE;
*op1 = NULL_TREE;
*new_def_stmt = NULL;
@@ -933,7 +958,9 @@ vect_operation_fits_smaller_type (gimple stmt, tre
first = true;
if (!widened_name_p (oprnd, stmt, &half_type, &def_stmt, false)
|| !gimple_bb (def_stmt)
- || !flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
+ || (loop && !flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
+ || (!loop && gimple_bb (def_stmt) != BB_VINFO_BB (bb_info)
+ && gimple_code (def_stmt) != GIMPLE_PHI)
|| !vinfo_for_stmt (def_stmt))
return false;
}
@@ -1107,9 +1134,18 @@ vect_recog_over_widening_pattern (VEC (gimple, hea
int nuses = 0;
tree var = NULL_TREE, new_type = NULL_TREE, tmp, new_oprnd;
bool first;
- struct loop *loop = (gimple_bb (stmt))->loop_father;
tree type = NULL;
+ loop_vec_info loop_vinfo;
+ struct loop *loop = NULL;
+ bb_vec_info bb_vinfo;
+ stmt_vec_info stmt_vinfo;
+ stmt_vinfo = vinfo_for_stmt (stmt);
+ loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
+ if (loop_vinfo)
+ loop = LOOP_VINFO_LOOP (loop_vinfo);
+
first = true;
while (1)
{
@@ -1141,7 +1177,8 @@ vect_recog_over_widening_pattern (VEC (gimple, hea
if (nuses != 1 || !is_gimple_assign (use_stmt)
|| !gimple_bb (use_stmt)
- || !flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
+ || (loop && !flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
+ || (!loop && gimple_bb (use_stmt) != BB_VINFO_BB (bb_vinfo)))
return NULL;
/* Create pattern statement for STMT. */
@@ -1518,6 +1555,7 @@ vect_recog_vector_vector_shift_pattern (VEC (gimpl
enum tree_code rhs_code;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
enum vect_def_type dt;
tree def;
@@ -1551,7 +1589,7 @@ vect_recog_vector_vector_shift_pattern (VEC (gimpl
!= TYPE_PRECISION (TREE_TYPE (oprnd0)))
return NULL;
- if (!vect_is_simple_use (oprnd1, last_stmt, loop_vinfo, NULL, &def_stmt,
+ if (!vect_is_simple_use (oprnd1, last_stmt, loop_vinfo, bb_vinfo, &def_stmt,
&def, &dt))
return NULL;
@@ -1840,6 +1878,7 @@ vect_recog_mixed_size_cond_pattern (VEC (gimple, h
enum machine_mode cmpmode;
gimple pattern_stmt, def_stmt;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
if (!is_gimple_assign (last_stmt)
|| gimple_assign_rhs_code (last_stmt) != COND_EXPR
@@ -1907,12 +1946,15 @@ vect_recog_mixed_size_cond_pattern (VEC (gimple, h
gimple_assign_lhs (def_stmt), NULL_TREE);
new_pattern_def_seq (stmt_vinfo, def_stmt);
- def_stmt_info = new_stmt_vec_info (def_stmt, loop_vinfo, NULL);
+ def_stmt_info = new_stmt_vec_info (def_stmt, loop_vinfo, bb_vinfo);
set_vinfo_for_stmt (def_stmt, def_stmt_info);
STMT_VINFO_VECTYPE (def_stmt_info) = vecitype;
*type_in = vecitype;
*type_out = vectype;
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "vect_recog_mixed_size_cond_pattern: detected: ");
+
return pattern_stmt;
}
@@ -1921,14 +1963,15 @@ vect_recog_mixed_size_cond_pattern (VEC (gimple, h
true if bool VAR can be optimized that way. */
static bool
-check_bool_pattern (tree var, loop_vec_info loop_vinfo)
+check_bool_pattern (tree var, loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
{
gimple def_stmt;
enum vect_def_type dt;
tree def, rhs1;
enum tree_code rhs_code;
- if (!vect_is_simple_use (var, NULL, loop_vinfo, NULL, &def_stmt, &def, &dt))
+ if (!vect_is_simple_use (var, NULL, loop_vinfo, bb_vinfo, &def_stmt, &def,
+ &dt))
return false;
if (dt != vect_internal_def)
@@ -1945,24 +1988,25 @@ static bool
switch (rhs_code)
{
case SSA_NAME:
- return check_bool_pattern (rhs1, loop_vinfo);
+ return check_bool_pattern (rhs1, loop_vinfo, bb_vinfo);
CASE_CONVERT:
if ((TYPE_PRECISION (TREE_TYPE (rhs1)) != 1
|| !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
&& TREE_CODE (TREE_TYPE (rhs1)) != BOOLEAN_TYPE)
return false;
- return check_bool_pattern (rhs1, loop_vinfo);
+ return check_bool_pattern (rhs1, loop_vinfo, bb_vinfo);
case BIT_NOT_EXPR:
- return check_bool_pattern (rhs1, loop_vinfo);
+ return check_bool_pattern (rhs1, loop_vinfo, bb_vinfo);
case BIT_AND_EXPR:
case BIT_IOR_EXPR:
case BIT_XOR_EXPR:
- if (!check_bool_pattern (rhs1, loop_vinfo))
+ if (!check_bool_pattern (rhs1, loop_vinfo, bb_vinfo))
return false;
- return check_bool_pattern (gimple_assign_rhs2 (def_stmt), loop_vinfo);
+ return check_bool_pattern (gimple_assign_rhs2 (def_stmt), loop_vinfo,
+ bb_vinfo);
default:
if (TREE_CODE_CLASS (rhs_code) == tcc_comparison)
@@ -2260,6 +2304,7 @@ vect_recog_bool_pattern (VEC (gimple, heap) **stmt
tree var, lhs, rhs, vectype;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
gimple pattern_stmt;
if (!is_gimple_assign (last_stmt))
@@ -2283,7 +2328,7 @@ vect_recog_bool_pattern (VEC (gimple, heap) **stmt
if (vectype == NULL_TREE)
return NULL;
- if (!check_bool_pattern (var, loop_vinfo))
+ if (!check_bool_pattern (var, loop_vinfo, bb_vinfo))
return NULL;
rhs = adjust_bool_pattern (var, TREE_TYPE (lhs), NULL_TREE, stmts);
@@ -2297,6 +2342,9 @@ vect_recog_bool_pattern (VEC (gimple, heap) **stmt
*type_out = vectype;
*type_in = vectype;
VEC_safe_push (gimple, heap, *stmts, last_stmt);
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "vect_recog_bool_pattern: detected: ");
+
return pattern_stmt;
}
else if (rhs_code == SSA_NAME
@@ -2307,7 +2355,7 @@ vect_recog_bool_pattern (VEC (gimple, heap) **stmt
gcc_assert (vectype != NULL_TREE);
if (!VECTOR_MODE_P (TYPE_MODE (vectype)))
return NULL;
- if (!check_bool_pattern (var, loop_vinfo))
+ if (!check_bool_pattern (var, loop_vinfo, bb_vinfo))
return NULL;
rhs = adjust_bool_pattern (var, TREE_TYPE (vectype), NULL_TREE, stmts);
@@ -2322,7 +2370,8 @@ vect_recog_bool_pattern (VEC (gimple, heap) **stmt
}
pattern_stmt
= gimple_build_assign_with_ops (SSA_NAME, lhs, rhs, NULL_TREE);
- pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo, NULL);
+ pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo,
+ bb_vinfo);
set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info);
STMT_VINFO_DATA_REF (pattern_stmt_info)
= STMT_VINFO_DATA_REF (stmt_vinfo);
@@ -2338,6 +2387,8 @@ vect_recog_bool_pattern (VEC (gimple, heap) **stmt
*type_out = vectype;
*type_in = vectype;
VEC_safe_push (gimple, heap, *stmts, last_stmt);
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "vect_recog_bool_pattern: detected: ");
return pattern_stmt;
}
else
@@ -2354,12 +2405,14 @@ vect_mark_pattern_stmts (gimple orig_stmt, gimple
stmt_vec_info pattern_stmt_info, def_stmt_info;
stmt_vec_info orig_stmt_info = vinfo_for_stmt (orig_stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (orig_stmt_info);
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (orig_stmt_info);
gimple def_stmt;
pattern_stmt_info = vinfo_for_stmt (pattern_stmt);
if (pattern_stmt_info == NULL)
{
- pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo, NULL);
+ pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo,
+ bb_vinfo);
set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info);
}
gimple_set_bb (pattern_stmt, gimple_bb (orig_stmt));
@@ -2382,7 +2435,8 @@ vect_mark_pattern_stmts (gimple orig_stmt, gimple
def_stmt_info = vinfo_for_stmt (def_stmt);
if (def_stmt_info == NULL)
{
- def_stmt_info = new_stmt_vec_info (def_stmt, loop_vinfo, NULL);
+ def_stmt_info = new_stmt_vec_info (def_stmt, loop_vinfo,
+ bb_vinfo);
set_vinfo_for_stmt (def_stmt, def_stmt_info);
}
gimple_set_bb (def_stmt, gimple_bb (orig_stmt));
@@ -2493,9 +2547,10 @@ vect_pattern_recog_1 (vect_recog_func_ptr vect_rec
/* Patterns cannot be vectorized using SLP, because they change the order of
computation. */
- FOR_EACH_VEC_ELT (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i, next)
- if (next == stmt)
- VEC_ordered_remove (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i);
+ if (loop_vinfo)
+ FOR_EACH_VEC_ELT (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i, next)
+ if (next == stmt)
+ VEC_ordered_remove (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i);
/* It is possible that additional pattern stmts are created and inserted in
STMTS_TO_REPLACE. We create a stmt_info for each of them, and mark the
@@ -2595,19 +2650,34 @@ vect_pattern_recog_1 (vect_recog_func_ptr vect_rec
be recorded in S3. */
void
-vect_pattern_recog (loop_vec_info loop_vinfo)
+vect_pattern_recog (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
{
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
- unsigned int nbbs = loop->num_nodes;
+ struct loop *loop;
+ basic_block *bbs, bb;
+ unsigned int nbbs;
gimple_stmt_iterator si;
unsigned int i, j;
vect_recog_func_ptr vect_recog_func;
VEC (gimple, heap) *stmts_to_replace = VEC_alloc (gimple, heap, 1);
+ gimple stmt;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_pattern_recog ===");
+ if (loop_vinfo)
+ {
+ loop = LOOP_VINFO_LOOP (loop_vinfo);
+ bbs = LOOP_VINFO_BBS (loop_vinfo);
+ nbbs = loop->num_nodes;
+ }
+ else
+ {
+ bb = BB_VINFO_BB (bb_vinfo);
+ nbbs = 1;
+ bbs = XNEW (basic_block);
+ bbs[0] = bb;
+ }
+
/* Scan through the loop stmts, applying the pattern recognition
functions starting at each stmt visited: */
for (i = 0; i < nbbs; i++)
@@ -2615,6 +2685,11 @@ void
basic_block bb = bbs[i];
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
{
+ if (bb_vinfo && (stmt = gsi_stmt (si))
+ && vinfo_for_stmt (stmt)
+ && !STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt)))
+ continue;
+
/* Scan over all generic vect_recog_xxx_pattern functions. */
for (j = 0; j < NUM_PATTERNS; j++)
{
@@ -2626,4 +2701,6 @@ void
}
VEC_free (gimple, heap, stmts_to_replace);
+ if (bb_vinfo)
+ free (bbs);
}
Index: tree-vect-stmts.c
===================================================================
--- tree-vect-stmts.c (revision 183902)
+++ tree-vect-stmts.c (working copy)
@@ -2224,10 +2224,10 @@ vectorizable_conversion (gimple stmt, gimple_stmt_
/* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
OP1. */
if (CONSTANT_CLASS_P (op0))
- ok = vect_is_simple_use_1 (op1, stmt, loop_vinfo, NULL,
+ ok = vect_is_simple_use_1 (op1, stmt, loop_vinfo, bb_vinfo,
&def_stmt, &def, &dt[1], &vectype_in);
else
- ok = vect_is_simple_use (op1, stmt, loop_vinfo, NULL, &def_stmt,
+ ok = vect_is_simple_use (op1, stmt, loop_vinfo, bb_vinfo, &def_stmt,
&def, &dt[1]);
if (!ok)
Index: tree-vect-slp.c
===================================================================
--- tree-vect-slp.c (revision 183902)
+++ tree-vect-slp.c (working copy)
@@ -249,12 +249,14 @@ vect_get_and_check_slp_defs (loop_vec_info loop_vi
/* Check if DEF_STMT is a part of a pattern in LOOP and get the def stmt
from the pattern. Check that all the stmts of the node are in the
pattern. */
- if (loop && def_stmt && gimple_bb (def_stmt)
- && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
+ if (def_stmt && gimple_bb (def_stmt)
+ && ((loop && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
+ || (!loop && gimple_bb (def_stmt) == BB_VINFO_BB (bb_vinfo)
+ && gimple_code (def_stmt) != GIMPLE_PHI))
&& vinfo_for_stmt (def_stmt)
&& STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (def_stmt))
- && !STMT_VINFO_RELEVANT (vinfo_for_stmt (def_stmt))
- && !STMT_VINFO_LIVE_P (vinfo_for_stmt (def_stmt)))
+ && !STMT_VINFO_RELEVANT (vinfo_for_stmt (def_stmt))
+ && !STMT_VINFO_LIVE_P (vinfo_for_stmt (def_stmt)))
{
pattern = true;
if (!first && !oprnd_info->first_pattern)
@@ -2010,7 +2012,9 @@ vect_slp_analyze_bb_1 (basic_block bb)
return NULL;
}
- if (!vect_analyze_data_ref_dependences (NULL, bb_vinfo, &max_vf)
+ vect_pattern_recog (NULL, bb_vinfo);
+
+ if (!vect_analyze_data_ref_dependences (NULL, bb_vinfo, &max_vf)
|| min_vf > max_vf)
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
