This fixes the vectorizer testsuite fallout from folding all stmts
which can swap tree operands where the SLP vectorizer currently
cannot deal with that. It fixes it by making the SLP vectorizer
deal with swapped operands in more cases, mostly SLP tree leafs
and operations with one dt_external operand.
Bootstrapped on x86_64-unknown-linux-gnu, testing in progress.
Might also fix some PR but a cursory look didn't find one.
Thanks,
Richard.
2014-10-20 Richard Biener <rguent...@suse.de>
* tree-vect-slp.c (vect_get_and_check_slp_defs): Try swapping
operands to get a def operand kind match. Signal mismatches
to the parent so we can try swapping its operands.
(vect_build_slp_tree): Try swapping operands if they have
a mismatched operand kind.
Index: gcc/tree-vect-slp.c
===================================================================
--- gcc/tree-vect-slp.c (revision 216448)
+++ gcc/tree-vect-slp.c (working copy)
@@ -205,9 +205,11 @@ vect_get_place_in_interleaving_chain (gi
/* Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that
they are of a valid type and that they match the defs of the first stmt of
- the SLP group (stored in OPRNDS_INFO). */
+ the SLP group (stored in OPRNDS_INFO). If there was a fatal error
+ return -1, if the error could be corrected by swapping operands of the
+ operation return 1, if everything is ok return 0. */
-static bool
+static int
vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
gimple stmt, bool first,
vec<slp_oprnd_info> *oprnds_info)
@@ -220,8 +222,9 @@ vect_get_and_check_slp_defs (loop_vec_in
struct loop *loop = NULL;
bool pattern = false;
slp_oprnd_info oprnd_info;
- int op_idx = 1;
- tree compare_rhs = NULL_TREE;
+ int first_op_idx = 1;
+ bool commutative = false;
+ bool first_op_cond = false;
if (loop_vinfo)
loop = LOOP_VINFO_LOOP (loop_vinfo);
@@ -229,35 +232,41 @@ vect_get_and_check_slp_defs (loop_vec_in
if (is_gimple_call (stmt))
{
number_of_oprnds = gimple_call_num_args (stmt);
- op_idx = 3;
+ first_op_idx = 3;
}
else if (is_gimple_assign (stmt))
{
+ enum tree_code code = gimple_assign_rhs_code (stmt);
number_of_oprnds = gimple_num_ops (stmt) - 1;
if (gimple_assign_rhs_code (stmt) == COND_EXPR)
- number_of_oprnds++;
+ {
+ first_op_cond = true;
+ commutative = true;
+ number_of_oprnds++;
+ }
+ else
+ commutative = commutative_tree_code (code);
}
else
- return false;
+ return -1;
+ bool swapped = false;
for (i = 0; i < number_of_oprnds; i++)
{
- if (compare_rhs)
+again:
+ if (first_op_cond)
{
- oprnd = compare_rhs;
- compare_rhs = NULL_TREE;
+ if (i == 0 || i == 1)
+ oprnd = TREE_OPERAND (gimple_op (stmt, first_op_idx),
+ swapped ? !i : i);
+ else
+ oprnd = gimple_op (stmt, first_op_idx + i - 1);
}
else
- oprnd = gimple_op (stmt, op_idx++);
+ oprnd = gimple_op (stmt, first_op_idx + (swapped ? !i : i));
oprnd_info = (*oprnds_info)[i];
- if (COMPARISON_CLASS_P (oprnd))
- {
- compare_rhs = TREE_OPERAND (oprnd, 1);
- oprnd = TREE_OPERAND (oprnd, 0);
- }
-
if (!vect_is_simple_use (oprnd, NULL, loop_vinfo, bb_vinfo, &def_stmt,
&def, &dt)
|| (!def_stmt && dt != vect_constant_def))
@@ -270,7 +279,7 @@ vect_get_and_check_slp_defs (loop_vec_in
dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
}
- return false;
+ return -1;
}
/* Check if DEF_STMT is a part of a pattern in LOOP and get the def stmt
@@ -288,6 +297,14 @@ vect_get_and_check_slp_defs (loop_vec_in
pattern = true;
if (!first && !oprnd_info->first_pattern)
{
+ if (i == 0
+ && !swapped
+ && commutative)
+ {
+ swapped = true;
+ goto again;
+ }
+
if (dump_enabled_p ())
{
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -297,7 +314,7 @@ vect_get_and_check_slp_defs (loop_vec_in
dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
}
- return false;
+ return 1;
}
def_stmt = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt));
@@ -308,7 +325,7 @@ vect_get_and_check_slp_defs (loop_vec_in
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
"Unsupported pattern.\n");
- return false;
+ return -1;
}
switch (gimple_code (def_stmt))
@@ -325,7 +342,7 @@ vect_get_and_check_slp_defs (loop_vec_in
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
"unsupported defining stmt:\n");
- return false;
+ return -1;
}
}
@@ -352,11 +369,20 @@ vect_get_and_check_slp_defs (loop_vec_in
|| !types_compatible_p (oprnd_info->first_op_type,
TREE_TYPE (oprnd))))
{
+ /* Try swapping operands if we got a mismatch. */
+ if (i == 0
+ && !swapped
+ && commutative)
+ {
+ swapped = true;
+ goto again;
+ }
+
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
"Build SLP failed: different types\n");
- return false;
+ return 1;
}
}
@@ -382,11 +408,26 @@ vect_get_and_check_slp_defs (loop_vec_in
dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
}
- return false;
+ return -1;
}
}
- return true;
+ /* Swap operands. */
+ if (swapped)
+ {
+ if (first_op_cond)
+ {
+ tree cond = gimple_assign_rhs1 (stmt);
+ swap_ssa_operands (stmt, &TREE_OPERAND (cond, 0),
+ &TREE_OPERAND (cond, 1));
+ TREE_SET_CODE (cond, swap_tree_comparison (TREE_CODE (cond)));
+ }
+ else
+ swap_ssa_operands (stmt, gimple_assign_rhs1_ptr (stmt),
+ gimple_assign_rhs2_ptr (stmt));
+ }
+
+ return 0;
}
@@ -892,13 +933,26 @@ vect_build_slp_tree (loop_vec_info loop_
slp_oprnd_info oprnd_info;
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (*node), i, stmt)
{
- if (!vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo,
- stmt, (i == 0), &oprnds_info))
+ switch (vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo,
+ stmt, (i == 0), &oprnds_info))
{
+ case 0:
+ break;
+ case -1:
+ matches[0] = false;
vect_free_oprnd_info (oprnds_info);
return false;
+ case 1:
+ matches[i] = false;
+ break;
}
}
+ for (i = 0; i < group_size; ++i)
+ if (!matches[i])
+ {
+ vect_free_oprnd_info (oprnds_info);
+ return false;
+ }
stmt = SLP_TREE_SCALAR_STMTS (*node)[0];
@@ -958,13 +1012,17 @@ vect_build_slp_tree (loop_vec_info loop_
*max_nunits = old_max_nunits;
loads->truncate (old_nloads);
/* Swap mismatched definition stmts. */
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "Re-trying with swapped operands of stmts ");
for (unsigned j = 0; j < group_size; ++j)
if (!matches[j])
{
gimple tem = oprnds_info[0]->def_stmts[j];
oprnds_info[0]->def_stmts[j] = oprnds_info[1]->def_stmts[j];
oprnds_info[1]->def_stmts[j] = tem;
+ dump_printf (MSG_NOTE, "%d ", j);
}
+ dump_printf (MSG_NOTE, "\n");
/* And try again ... */
if (vect_build_slp_tree (loop_vinfo, bb_vinfo, &child,
group_size, max_nunits, loads,
