This splits the core loop matching lanes from plumbing around it so
that the core loop fully populates matches[] and returns false only
on fatal errors. The plumbing then handles soft-failing and unrelated
things like vector type computations.
Bootstrapped and tested on x86_64-unknown-linux-gnu.
* tree-vect-slp.cc (vect_build_slp_tree_12): New function,
split out from ...
(vect_build_slp_tree_1): ... here.
---
gcc/tree-vect-slp.cc | 145 ++++++++++++++++++++++++-------------------
1 file changed, 81 insertions(+), 64 deletions(-)
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index bdd9026c14f..eb42b384962 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -1133,11 +1133,11 @@ vect_record_max_nunits (vec_info *vinfo, stmt_vec_info
stmt_info,
}
/* Verify if the scalar stmts STMTS are isomorphic, require data
- permutation or are of unsupported types of operation. Return
- true if they are, otherwise return false and indicate in *MATCHES
- which stmts are not isomorphic to the first one. If MATCHES[0]
- is false then this indicates the comparison could not be
- carried out or the stmts will never be vectorized by SLP.
+ permutation or are of unsupported types of operation.
+ Return false if at least one stmt is unvectorizable or the comparison
+ could not be carried out.
+ Return true if they all are and indicate in *MATCHES which stmts are
+ not isomorphic to the first one.
Note COND_EXPR is possibly isomorphic to another one after swapping its
operands. Set SWAP[i] to 1 if stmt I is COND_EXPR and isomorphic to
@@ -1147,12 +1147,10 @@ vect_record_max_nunits (vec_info *vinfo, stmt_vec_info
stmt_info,
to (B1 <= A1 ? X1 : Y1); or be inverted to (A1 < B1) ? Y1 : X1. */
static bool
-vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
- vec<stmt_vec_info> stmts,
- poly_uint64 *max_nunits, bool *matches,
- bool *two_operators, tree *node_vectype)
+vect_build_slp_tree_12 (vec_info *vinfo, unsigned char *swap,
+ vec<stmt_vec_info> stmts, bool *matches,
+ bool *two_operators, tree vectype)
{
- unsigned int group_size = stmts.length ();
unsigned int i;
stmt_vec_info first_stmt_info = stmts[0];
code_helper first_stmt_code = ERROR_MARK;
@@ -1165,43 +1163,7 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char
*swap,
bool first_stmt_ldst_p = false, first_stmt_ldst_masklen_p = false;
bool first_stmt_phi_p = false;
int first_reduc_idx = -1;
- bool maybe_soft_fail = false;
- tree soft_fail_nunits_vectype = NULL_TREE;
-
- tree vectype, nunits_vectype;
- if (!vect_get_vector_types_for_stmt (vinfo, first_stmt_info, &vectype,
- &nunits_vectype, group_size))
- {
- /* Fatal mismatch. */
- matches[0] = false;
- return false;
- }
- if (is_a <bb_vec_info> (vinfo)
- && known_le (TYPE_VECTOR_SUBPARTS (vectype), 1U))
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "Build SLP failed: not using single lane "
- "vector type %T\n", vectype);
- matches[0] = false;
- return false;
- }
- /* Record nunits required but continue analysis, producing matches[]
- as if nunits was not an issue. This allows splitting of groups
- to happen. */
- if (nunits_vectype
- && !vect_record_max_nunits (vinfo, first_stmt_info, group_size,
- nunits_vectype, max_nunits))
- {
- gcc_assert (is_a <bb_vec_info> (vinfo));
- maybe_soft_fail = true;
- soft_fail_nunits_vectype = nunits_vectype;
- }
-
- gcc_assert (vectype || !gimple_get_lhs (first_stmt_info->stmt));
- *node_vectype = vectype;
- /* For every stmt in NODE find its def stmt/s. */
stmt_vec_info stmt_info;
FOR_EACH_VEC_ELT (stmts, i, stmt_info)
{
@@ -1238,8 +1200,6 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char
*swap,
work for this though but it's the easiest we can do here. */
if (is_a <bb_vec_info> (vinfo) && i != 0)
continue;
- /* Fatal mismatch. */
- matches[0] = false;
return false;
}
@@ -1253,8 +1213,6 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char
*swap,
"GIMPLE_CALL %G", stmt);
if (is_a <bb_vec_info> (vinfo) && i != 0)
continue;
- /* Fatal mismatch. */
- matches[0] = false;
return false;
}
@@ -1300,8 +1258,6 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char
*swap,
(gimple *) call_stmt);
if (is_a <bb_vec_info> (vinfo) && i != 0)
continue;
- /* Fatal mismatch. */
- matches[0] = false;
return false;
}
}
@@ -1344,8 +1300,6 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char
*swap,
"op not supported by target.\n");
if (is_a <bb_vec_info> (vinfo) && i != 0)
continue;
- /* Fatal mismatch. */
- matches[0] = false;
return false;
}
need_same_oprnds = true;
@@ -1375,8 +1329,6 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char
*swap,
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
"Build SLP failed: "
"BIT_FIELD_REF not supported\n");
- /* Fatal mismatch. */
- matches[0] = false;
return false;
}
}
@@ -1601,8 +1553,6 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char
*swap,
if (i != 0)
continue;
- /* Fatal mismatch. */
- matches[0] = false;
return false;
}
}
@@ -1626,8 +1576,6 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char
*swap,
stmt);
if (is_a <bb_vec_info> (vinfo) && i != 0)
continue;
- /* Fatal mismatch. */
- matches[0] = false;
return false;
}
@@ -1688,10 +1636,6 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char
*swap,
matches[i] = true;
}
- for (i = 0; i < group_size; ++i)
- if (!matches[i])
- return false;
-
/* If we allowed a two-operation SLP node verify the target can cope
with the permute we are going to use. */
if (alt_stmt_code != ERROR_MARK
@@ -1702,12 +1646,85 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char
*swap,
*two_operators = true;
}
+ return true;
+}
+
+/* Verify if the scalar stmts STMTS are isomorphic, require data
+ permutation or are of unsupported types of operation. Return
+ true if they are, otherwise return false and indicate in *MATCHES
+ which stmts are not isomorphic to the first one. If MATCHES[0]
+ is false then this indicates the comparison could not be
+ carried out or the stmts will never be vectorized by SLP.
+
+ Note COND_EXPR is possibly isomorphic to another one after swapping its
+ operands. Set SWAP[i] to 1 if stmt I is COND_EXPR and isomorphic to
+ the first stmt by swapping the two operands of comparison; set SWAP[i]
+ to 2 if stmt I is isormorphic to the first stmt by inverting the code
+ of comparison. Take A1 >= B1 ? X1 : Y1 as an example, it can be swapped
+ to (B1 <= A1 ? X1 : Y1); or be inverted to (A1 < B1) ? Y1 : X1. */
+
+static bool
+vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
+ vec<stmt_vec_info> stmts,
+ poly_uint64 *max_nunits, bool *matches,
+ bool *two_operators, tree *node_vectype)
+{
+ stmt_vec_info first_stmt_info = stmts[0];
+ unsigned int group_size = stmts.length ();
+ tree vectype, nunits_vectype;
+ if (!vect_get_vector_types_for_stmt (vinfo, first_stmt_info, &vectype,
+ &nunits_vectype, group_size))
+ {
+ /* Fatal mismatch. */
+ matches[0] = false;
+ return false;
+ }
+ if (is_a <bb_vec_info> (vinfo)
+ && known_le (TYPE_VECTOR_SUBPARTS (vectype), 1U))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "Build SLP failed: not using single lane "
+ "vector type %T\n", vectype);
+ matches[0] = false;
+ return false;
+ }
+ bool maybe_soft_fail = false;
+ tree soft_fail_nunits_vectype = NULL_TREE;
+ /* Record nunits required but continue analysis, producing matches[]
+ as if nunits was not an issue. This allows splitting of groups
+ to happen. */
+ if (nunits_vectype
+ && !vect_record_max_nunits (vinfo, first_stmt_info, group_size,
+ nunits_vectype, max_nunits))
+ {
+ gcc_assert (is_a <bb_vec_info> (vinfo));
+ maybe_soft_fail = true;
+ soft_fail_nunits_vectype = nunits_vectype;
+ }
+
+ gcc_assert (vectype || !gimple_get_lhs (first_stmt_info->stmt));
+ *node_vectype = vectype;
+
+ if (!vect_build_slp_tree_12 (vinfo, swap, stmts, matches, two_operators,
+ vectype))
+ {
+ /* Fatal mismatch. */
+ matches[0] = false;
+ return false;
+ }
+
+ for (unsigned i = 0; i < group_size; ++i)
+ if (!matches[i])
+ return false;
+
if (maybe_soft_fail)
{
unsigned HOST_WIDE_INT const_nunits;
if (!TYPE_VECTOR_SUBPARTS
(soft_fail_nunits_vectype).is_constant (&const_nunits)
|| const_nunits > group_size)
+ /* Fatal mismatch. */
matches[0] = false;
else
{
--
2.51.0
