The point of this patch is to put pattern statements in the same
vec_basic_block as the statements they replace, with the pattern
statements for S coming between S and S's original predecessor.
This removes the need to handle them specially in various places.
2018-07-30 Richard Sandiford <richard.sandif...@arm.com>
gcc/
* tree-vectorizer.h (vec_basic_block): Expand comment.
(_stmt_vec_info::pattern_def_seq): Delete.
(STMT_VINFO_PATTERN_DEF_SEQ): Likewise.
(is_main_pattern_stmt_p): New function.
* tree-vect-loop.c (vect_determine_vf_for_stmt_1): Rename to...
(vect_determine_vf_for_stmt): ...this, deleting the original
function with this name. Remove vectype_maybe_set_p argument
and test is_pattern_stmt_p instead. Retain the "examining..."
message from the previous vect_determine_vf_for_stmt.
(vect_compute_single_scalar_iteration_cost, vect_update_vf_for_slp)
(vect_analyze_loop_2): Don't treat pattern statements specially.
(vect_transform_loop): Likewise. Use vect_orig_stmt to find the
insertion point.
* tree-vect-slp.c (vect_detect_hybrid_slp): Expect pattern statements
to be in the statement list, without needing to follow
STMT_VINFO_RELATED_STMT. Remove PATTERN_DEF_SEQ handling.
* tree-vect-stmts.c (vect_analyze_stmt): Don't handle pattern
statements specially.
(vect_remove_dead_scalar_stmts): Ignore pattern statements.
* tree-vect-patterns.c (vect_set_pattern_stmt): Insert the pattern
statement into the vec_basic_block immediately before the statement
it replaces.
(append_pattern_def_seq): Likewise. If the original statement is
itself a pattern statement, associate the new one with the original
statement.
(vect_split_statement): Use append_pattern_def_seq to insert the
first pattern statement.
(vect_recog_vector_vector_shift_pattern): Remove mention of
STMT_VINFO_PATTERN_DEF_SEQ.
(adjust_bool_stmts): Get the last pattern statement from the
stmt_vec_info chain.
(vect_mark_pattern_stmts): Rename to...
(vect_replace_stmt_with_pattern): ...this. Remove the
PATTERN_DEF_SEQ handling and process only the pattern statement given.
Use append_pattern_def_seq when replacing a pattern statement with
another pattern statement, and use vec_basic_block::remove instead
of gsi_remove to remove the old one.
(vect_pattern_recog_1): Update accordingly. Remove PATTERN_DEF_SEQ
handling. On failure, remove any half-formed pattern sequence from
the vec_basic_block. Install the vector type in pattern statements
that don't yet have one.
(vect_pattern_recog): Iterate over statements that are added
by previous recognizers, but skipping those that have already
been replaced, or the main pattern statement in such a replacement.
Index: gcc/tree-vectorizer.h
===================================================================
*** gcc/tree-vectorizer.h 2018-07-30 12:32:46.658356275 +0100
--- gcc/tree-vectorizer.h 2018-07-30 12:32:49.898327734 +0100
*************** #define SLP_TREE_TWO_OPERATORS(S) (S)-
*** 172,178 ****
#define SLP_TREE_DEF_TYPE(S) (S)->def_type
/* Information about the phis and statements in a block that we're trying
! to vectorize, in their original order. */
class vec_basic_block
{
public:
--- 172,184 ----
#define SLP_TREE_DEF_TYPE(S) (S)->def_type
/* Information about the phis and statements in a block that we're trying
! to vectorize. This includes the phis and statements that were in the
! original scalar code, in their original order. It also includes any
! pattern statements that the vectorizer has created to replace some
! of the scalar ones. Such pattern statements come immediately before
! the statement that they replace; that is, all pattern statements P for
! which vect_orig_stmt (P) == S form a sequence that comes immediately
! before S. */
class vec_basic_block
{
public:
*************** struct _stmt_vec_info {
*** 870,880 ****
pattern). */
stmt_vec_info related_stmt;
- /* Used to keep a sequence of def stmts of a pattern stmt if such exists.
- The sequence is attached to the original statement rather than the
- pattern statement. */
- gimple_seq pattern_def_seq;
-
/* List of datarefs that are known to have the same alignment as the dataref
of this stmt. */
vec<dr_p> same_align_refs;
--- 876,881 ----
*************** #define STMT_VINFO_DR_INFO(S) \
*** 1048,1054 ****
#define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
#define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
- #define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq
#define STMT_VINFO_SAME_ALIGN_REFS(S) (S)->same_align_refs
#define STMT_VINFO_SIMD_CLONE_INFO(S) (S)->simd_clone_info
#define STMT_VINFO_DEF_TYPE(S) (S)->def_type
--- 1049,1054 ----
*************** is_pattern_stmt_p (stmt_vec_info stmt_in
*** 1176,1181 ****
--- 1176,1192 ----
return stmt_info->pattern_stmt_p;
}
+ /* Return TRUE if a statement represented by STMT_INFO is the final
+ statement in a pattern. */
+
+ static inline bool
+ is_main_pattern_stmt_p (stmt_vec_info stmt_info)
+ {
+ stmt_vec_info orig_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
+ return (is_pattern_stmt_p (stmt_info)
+ && STMT_VINFO_RELATED_STMT (orig_stmt_info) == stmt_info);
+ }
+
/* If STMT_INFO is a pattern statement, return the statement that it
replaces, otherwise return STMT_INFO itself. */
Index: gcc/tree-vect-loop.c
===================================================================
*** gcc/tree-vect-loop.c 2018-07-30 12:32:46.654356310 +0100
--- gcc/tree-vect-loop.c 2018-07-30 12:32:49.894327770 +0100
*************** Software Foundation; either version 3, o
*** 155,172 ****
static void vect_estimate_min_profitable_iters (loop_vec_info, int *, int *);
! /* Subroutine of vect_determine_vf_for_stmt that handles only one
! statement. VECTYPE_MAYBE_SET_P is true if STMT_VINFO_VECTYPE
! may already be set for general statements (not just data refs). */
static bool
! vect_determine_vf_for_stmt_1 (stmt_vec_info stmt_info,
! bool vectype_maybe_set_p,
! poly_uint64 *vf,
! vec<stmt_vec_info > *mask_producers)
{
gimple *stmt = stmt_info->stmt;
if ((!STMT_VINFO_RELEVANT_P (stmt_info)
&& !STMT_VINFO_LIVE_P (stmt_info))
|| gimple_clobber_p (stmt))
--- 155,178 ----
static void vect_estimate_min_profitable_iters (loop_vec_info, int *, int *);
! /* Subroutine of vect_determine_vectorization_factor. Set the vector
! type of STMT_INFO and update the vectorization factor VF accordingly.
! If the statement produces a mask result whose vector type can only be
! calculated later, add it to MASK_PRODUCERS. Return true on success
! or false if something prevented vectorization. */
static bool
! vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf,
! vec<stmt_vec_info > *mask_producers)
{
gimple *stmt = stmt_info->stmt;
+ if (dump_enabled_p ())
+ {
+ dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: ");
+ dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
+ }
+
if ((!STMT_VINFO_RELEVANT_P (stmt_info)
&& !STMT_VINFO_LIVE_P (stmt_info))
|| gimple_clobber_p (stmt))
*************** vect_determine_vf_for_stmt_1 (stmt_vec_i
*** 188,194 ****
that contain a data ref, or for "pattern-stmts" (stmts generated
by the vectorizer to represent/replace a certain idiom). */
gcc_assert ((STMT_VINFO_DATA_REF (stmt_info)
! || vectype_maybe_set_p)
&& STMT_VINFO_VECTYPE (stmt_info) == stmt_vectype);
else if (stmt_vectype == boolean_type_node)
mask_producers->safe_push (stmt_info);
--- 194,200 ----
that contain a data ref, or for "pattern-stmts" (stmts generated
by the vectorizer to represent/replace a certain idiom). */
gcc_assert ((STMT_VINFO_DATA_REF (stmt_info)
! || is_pattern_stmt_p (stmt_info))
&& STMT_VINFO_VECTYPE (stmt_info) == stmt_vectype);
else if (stmt_vectype == boolean_type_node)
mask_producers->safe_push (stmt_info);
*************** vect_determine_vf_for_stmt_1 (stmt_vec_i
*** 202,263 ****
return true;
}
- /* Subroutine of vect_determine_vectorization_factor. Set the vector
- types of STMT_INFO and all attached pattern statements and update
- the vectorization factor VF accordingly. If some of the statements
- produce a mask result whose vector type can only be calculated later,
- add them to MASK_PRODUCERS. Return true on success or false if
- something prevented vectorization. */
-
- static bool
- vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf,
- vec<stmt_vec_info > *mask_producers)
- {
- vec_info *vinfo = stmt_info->vinfo;
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
- }
- if (!vect_determine_vf_for_stmt_1 (stmt_info, false, vf, mask_producers))
- return false;
-
- if (STMT_VINFO_IN_PATTERN_P (stmt_info)
- && STMT_VINFO_RELATED_STMT (stmt_info))
- {
- gimple *pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
- stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
-
- /* If a pattern statement has def stmts, analyze them too. */
- for (gimple_stmt_iterator si = gsi_start (pattern_def_seq);
- !gsi_end_p (si); gsi_next (&si))
- {
- stmt_vec_info def_stmt_info = vinfo->lookup_stmt (gsi_stmt (si));
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "==> examining pattern def stmt: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM,
- def_stmt_info->stmt, 0);
- }
- if (!vect_determine_vf_for_stmt_1 (def_stmt_info, true,
- vf, mask_producers))
- return false;
- }
-
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "==> examining pattern statement: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
- }
- if (!vect_determine_vf_for_stmt_1 (stmt_info, true, vf, mask_producers))
- return false;
- }
-
- return true;
- }
-
/* Function vect_determine_vectorization_factor
Determine the vectorization factor (VF). VF is the number of data elements
--- 208,213 ----
*************** vect_compute_single_scalar_iteration_cos
*** 1078,1086 ****
/* Skip stmts that are not vectorized inside the loop. */
if (stmt_info
&& !STMT_VINFO_RELEVANT_P (stmt_info)
! && (!STMT_VINFO_LIVE_P (stmt_info)
! || !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE
(stmt_info)))
! && !STMT_VINFO_IN_PATTERN_P (stmt_info))
continue;
vect_cost_for_stmt kind;
--- 1028,1035 ----
/* Skip stmts that are not vectorized inside the loop. */
if (stmt_info
&& !STMT_VINFO_RELEVANT_P (stmt_info)
! && (!VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info))
! || !STMT_VINFO_LIVE_P (stmt_info)))
continue;
vect_cost_for_stmt kind;
*************** vect_update_vf_for_slp (loop_vec_info lo
*** 1394,1407 ****
vec_basic_block *vec_bb;
FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! {
! stmt_vec_info final_info = vect_stmt_to_vectorize (stmt_info);
! if ((STMT_VINFO_RELEVANT_P (final_info)
! || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (final_info)))
! && !PURE_SLP_STMT (final_info))
! /* STMT needs both SLP and loop-based vectorization. */
! only_slp_in_loop = false;
! }
if (only_slp_in_loop)
{
--- 1343,1353 ----
vec_basic_block *vec_bb;
FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! if ((STMT_VINFO_RELEVANT_P (stmt_info)
! || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
! && !PURE_SLP_STMT (stmt_info))
! /* STMT needs both SLP and loop-based vectorization. */
! only_slp_in_loop = false;
if (only_slp_in_loop)
{
*************** vect_analyze_loop_2 (loop_vec_info loop_
*** 2164,2181 ****
vec_basic_block *vec_bb;
FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! {
! STMT_SLP_TYPE (stmt_info) = loop_vect;
! if (STMT_VINFO_IN_PATTERN_P (stmt_info))
! {
! gimple *pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
! STMT_SLP_TYPE (STMT_VINFO_RELATED_STMT (stmt_info)) = loop_vect;
! for (gimple_stmt_iterator pi = gsi_start (pattern_def_seq);
! !gsi_end_p (pi); gsi_next (&pi))
! STMT_SLP_TYPE (loop_vinfo->lookup_stmt (gsi_stmt (pi)))
! = loop_vect;
! }
! }
/* Free optimized alias test DDRS. */
LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).truncate (0);
--- 2110,2116 ----
vec_basic_block *vec_bb;
FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! STMT_SLP_TYPE (stmt_info) = loop_vect;
/* Free optimized alias test DDRS. */
LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).truncate (0);
*************** vect_transform_loop (loop_vec_info loop_
*** 8371,8392 ****
loop_vinfo->remove_stmt (stmt_info);
else
{
! gimple_stmt_iterator si = gsi_for_stmt (stmt_info->stmt);
! if (STMT_VINFO_IN_PATTERN_P (stmt_info))
! {
! gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
! for (gimple_stmt_iterator subsi = gsi_start (def_seq);
! !gsi_end_p (subsi); gsi_next (&subsi))
! {
! stmt_vec_info pat_stmt_info
! = loop_vinfo->lookup_stmt (gsi_stmt (subsi));
! vect_transform_loop_stmt (loop_vinfo, pat_stmt_info,
! &si);
! }
! stmt_vec_info pat_stmt_info
! = STMT_VINFO_RELATED_STMT (stmt_info);
! vect_transform_loop_stmt (loop_vinfo, pat_stmt_info, &si);
! }
vect_transform_loop_stmt (loop_vinfo, stmt_info, &si);
}
}
--- 8306,8313 ----
loop_vinfo->remove_stmt (stmt_info);
else
{
! stmt_vec_info place = vect_orig_stmt (stmt_info);
! gimple_stmt_iterator si = gsi_for_stmt (place->stmt);
vect_transform_loop_stmt (loop_vinfo, stmt_info, &si);
}
}
Index: gcc/tree-vect-slp.c
===================================================================
*** gcc/tree-vect-slp.c 2018-07-30 12:32:46.654356310 +0100
--- gcc/tree-vect-slp.c 2018-07-30 12:32:49.894327770 +0100
*************** vect_detect_hybrid_slp (loop_vec_info lo
*** 2411,2428 ****
vec_basic_block *vec_bb;
FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! if (STMT_VINFO_IN_PATTERN_P (stmt_info))
{
walk_stmt_info wi;
memset (&wi, 0, sizeof (wi));
wi.info = loop_vinfo;
! gimple_stmt_iterator gsi2
! = gsi_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info)->stmt);
! walk_gimple_stmt (&gsi2, vect_detect_hybrid_slp_2,
vect_detect_hybrid_slp_1, &wi);
- walk_gimple_seq (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
- vect_detect_hybrid_slp_2,
- vect_detect_hybrid_slp_1, &wi);
}
/* Then walk the SLP instance trees marking stmts with uses in
--- 2411,2424 ----
vec_basic_block *vec_bb;
FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! if (is_pattern_stmt_p (stmt_info))
{
walk_stmt_info wi;
memset (&wi, 0, sizeof (wi));
wi.info = loop_vinfo;
! gimple_stmt_iterator gsi = gsi_for_stmt (stmt_info->stmt);
! walk_gimple_stmt (&gsi, vect_detect_hybrid_slp_2,
vect_detect_hybrid_slp_1, &wi);
}
/* Then walk the SLP instance trees marking stmts with uses in
Index: gcc/tree-vect-stmts.c
===================================================================
*** gcc/tree-vect-stmts.c 2018-07-30 12:32:46.658356275 +0100
--- gcc/tree-vect-stmts.c 2018-07-30 12:32:49.898327734 +0100
*************** vect_analyze_stmt (stmt_vec_info stmt_in
*** 9384,9394 ****
slp_tree node, slp_instance node_instance,
stmt_vector_for_cost *cost_vec)
{
- vec_info *vinfo = stmt_info->vinfo;
bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
bool ok;
- gimple_seq pattern_def_seq;
if (dump_enabled_p ())
{
--- 9384,9392 ----
*************** vect_analyze_stmt (stmt_vec_info stmt_in
*** 9405,9498 ****
return false;
}
- if (STMT_VINFO_IN_PATTERN_P (stmt_info)
- && node == NULL
- && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info)))
- {
- gimple_stmt_iterator si;
-
- for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si))
- {
- stmt_vec_info pattern_def_stmt_info
- = vinfo->lookup_stmt (gsi_stmt (si));
- if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info)
- || STMT_VINFO_LIVE_P (pattern_def_stmt_info))
- {
- /* Analyze def stmt of STMT if it's a pattern stmt. */
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "==> examining pattern def statement: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM,
- pattern_def_stmt_info->stmt, 0);
- }
-
- if (!vect_analyze_stmt (pattern_def_stmt_info,
- need_to_vectorize, node, node_instance,
- cost_vec))
- return false;
- }
- }
- }
-
/* Skip stmts that do not need to be vectorized. In loops this is expected
to include:
- the COND_EXPR which is the loop exit condition
- any LABEL_EXPRs in the loop
- computations that are used only for array indexing or loop control.
In basic blocks we only analyze statements that are a part of some SLP
! instance, therefore, all the statements are relevant.
!
! Pattern statement needs to be analyzed instead of the original statement
! if the original statement is not relevant. Otherwise, we analyze both
! statements. In basic blocks we are called from some SLP instance
! traversal, don't analyze pattern stmts instead, the pattern stmts
! already will be part of SLP instance. */
- stmt_vec_info pattern_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
if (!STMT_VINFO_RELEVANT_P (stmt_info)
&& !STMT_VINFO_LIVE_P (stmt_info))
{
- if (STMT_VINFO_IN_PATTERN_P (stmt_info)
- && pattern_stmt_info
- && (STMT_VINFO_RELEVANT_P (pattern_stmt_info)
- || STMT_VINFO_LIVE_P (pattern_stmt_info)))
- {
- /* Analyze PATTERN_STMT instead of the original stmt. */
- stmt_info = pattern_stmt_info;
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "==> examining pattern statement: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
- }
- }
- else
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
-
- return true;
- }
- }
- else if (STMT_VINFO_IN_PATTERN_P (stmt_info)
- && node == NULL
- && pattern_stmt_info
- && (STMT_VINFO_RELEVANT_P (pattern_stmt_info)
- || STMT_VINFO_LIVE_P (pattern_stmt_info)))
- {
- /* Analyze PATTERN_STMT too. */
if (dump_enabled_p ())
! {
! dump_printf_loc (MSG_NOTE, vect_location,
! "==> examining pattern statement: ");
! dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_stmt_info->stmt, 0);
! }
!
! if (!vect_analyze_stmt (pattern_stmt_info, need_to_vectorize, node,
! node_instance, cost_vec))
! return false;
! }
switch (STMT_VINFO_DEF_TYPE (stmt_info))
{
--- 9403,9423 ----
return false;
}
/* Skip stmts that do not need to be vectorized. In loops this is expected
to include:
- the COND_EXPR which is the loop exit condition
- any LABEL_EXPRs in the loop
- computations that are used only for array indexing or loop control.
In basic blocks we only analyze statements that are a part of some SLP
! instance, therefore, all the statements are relevant. */
if (!STMT_VINFO_RELEVANT_P (stmt_info)
&& !STMT_VINFO_LIVE_P (stmt_info))
{
if (dump_enabled_p ())
! dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
! return true;
! }
switch (STMT_VINFO_DEF_TYPE (stmt_info))
{
*************** vect_remove_dead_scalar_stmts (vec_info
*** 10915,10921 ****
stmt_info = prev_stmt_info)
{
prev_stmt_info = stmt_info->prev;
! vect_maybe_remove_scalar_stmt (stmt_info);
}
}
}
--- 10840,10847 ----
stmt_info = prev_stmt_info)
{
prev_stmt_info = stmt_info->prev;
! if (!is_pattern_stmt_p (stmt_info))
! vect_maybe_remove_scalar_stmt (stmt_info);
}
}
}
Index: gcc/tree-vect-patterns.c
===================================================================
*** gcc/tree-vect-patterns.c 2018-07-30 12:32:42.786390386 +0100
--- gcc/tree-vect-patterns.c 2018-07-30 12:32:49.894327770 +0100
*************** vect_init_pattern_stmt (gimple *pattern_
*** 125,151 ****
vect_set_pattern_stmt (gimple *pattern_stmt, stmt_vec_info orig_stmt_info,
tree vectype)
{
! STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true;
! STMT_VINFO_RELATED_STMT (orig_stmt_info)
= vect_init_pattern_stmt (pattern_stmt, orig_stmt_info, vectype);
}
! /* Add NEW_STMT to STMT_INFO's pattern definition statements. If VECTYPE
! is nonnull, record that NEW_STMT's vector type is VECTYPE, which might
! be different from the vector type of the final pattern statement. */
static inline void
append_pattern_def_seq (stmt_vec_info stmt_info, gimple *new_stmt,
tree vectype = NULL_TREE)
{
! vec_info *vinfo = stmt_info->vinfo;
! if (vectype)
! {
! stmt_vec_info new_stmt_info = vinfo->add_stmt (new_stmt);
! STMT_VINFO_VECTYPE (new_stmt_info) = vectype;
! }
! gimple_seq_add_stmt_without_update (&STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
! new_stmt);
}
/* The caller wants to perform new operations on vect_external variable
--- 125,150 ----
vect_set_pattern_stmt (gimple *pattern_stmt, stmt_vec_info orig_stmt_info,
tree vectype)
{
! stmt_vec_info pattern_stmt_info
= vect_init_pattern_stmt (pattern_stmt, orig_stmt_info, vectype);
+ orig_stmt_info->block->add_before (pattern_stmt_info, orig_stmt_info);
+ STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true;
+ STMT_VINFO_RELATED_STMT (orig_stmt_info) = pattern_stmt_info;
}
! /* Add NEW_STMT to the pattern statements that replace STMT_INFO.
! If VECTYPE is nonnull, record that NEW_STMT's vector type is VECTYPE,
! which might be different from the vector type of the final pattern
! statement. */
static inline void
append_pattern_def_seq (stmt_vec_info stmt_info, gimple *new_stmt,
tree vectype = NULL_TREE)
{
! stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
! stmt_vec_info new_stmt_info
! = vect_init_pattern_stmt (new_stmt, orig_stmt_info, vectype);
! stmt_info->block->add_before (new_stmt_info, stmt_info);
}
/* The caller wants to perform new operations on vect_external variable
*************** vect_split_statement (stmt_vec_info stmt
*** 633,643 ****
{
if (is_pattern_stmt_p (stmt2_info))
{
- /* STMT2_INFO is part of a pattern. Get the statement to which
- the pattern is attached. */
- stmt_vec_info orig_stmt2_info = STMT_VINFO_RELATED_STMT (stmt2_info);
- vect_init_pattern_stmt (stmt1, orig_stmt2_info, vectype);
-
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location,
--- 632,637 ----
*************** vect_split_statement (stmt_vec_info stmt
*** 645,650 ****
--- 639,647 ----
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt2_info->stmt, 0);
}
+ /* Insert STMT1_INFO before STMT2_INFO. */
+ append_pattern_def_seq (stmt2_info, stmt1, vectype);
+
/* Since STMT2_INFO is a pattern statement, we can change it
in-situ without worrying about changing the code for the
containing block. */
*************** vect_split_statement (stmt_vec_info stmt
*** 658,675 ****
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt2_info->stmt, 0);
}
- gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt2_info);
- if (STMT_VINFO_RELATED_STMT (orig_stmt2_info) == stmt2_info)
- /* STMT2_INFO is the actual pattern statement. Add STMT1
- to the end of the definition sequence. */
- gimple_seq_add_stmt_without_update (def_seq, stmt1);
- else
- {
- /* STMT2_INFO belongs to the definition sequence. Insert STMT1
- before it. */
- gimple_stmt_iterator gsi = gsi_for_stmt (stmt2_info->stmt, def_seq);
- gsi_insert_before_without_update (&gsi, stmt1, GSI_SAME_STMT);
- }
return true;
}
else
--- 655,660 ----
*************** vect_split_statement (stmt_vec_info stmt
*** 689,698 ****
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt2_info->stmt, 0);
}
! /* Add STMT1 as a singleton pattern definition sequence. */
! gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt2_info);
! vect_init_pattern_stmt (stmt1, stmt2_info, vectype);
! gimple_seq_add_stmt_without_update (def_seq, stmt1);
/* Build the second of the two pattern statements. */
tree new_lhs = vect_recog_temp_ssa_var (lhs_type, NULL);
--- 674,681 ----
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt2_info->stmt, 0);
}
! /* Insert STMT1_INFO before STMT2_INFO. */
! append_pattern_def_seq (stmt2_info, stmt1, vectype);
/* Build the second of the two pattern statements. */
tree new_lhs = vect_recog_temp_ssa_var (lhs_type, NULL);
*************** vect_recog_rotate_pattern (stmt_vec_info
*** 2164,2170 ****
i.e. the shift/rotate stmt. The original stmt (S3) is replaced
with a shift/rotate which has same type on both operands, in the
second case just b_T op c_T, in the first case with added cast
! from a_t to c_T in STMT_VINFO_PATTERN_DEF_SEQ.
Output:
--- 2147,2153 ----
i.e. the shift/rotate stmt. The original stmt (S3) is replaced
with a shift/rotate which has same type on both operands, in the
second case just b_T op c_T, in the first case with added cast
! from a_t to c_T beforehand.
Output:
*************** adjust_bool_stmts (hash_set <gimple *> &
*** 3518,3526 ****
adjust_bool_pattern (gimple_assign_lhs (bool_stmts[i]),
out_type, stmt_info, defs);
! /* Pop the last pattern seq stmt and install it as pattern root for STMT.
*/
! gimple *pattern_stmt
! = gimple_seq_last_stmt (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
return gimple_assign_lhs (pattern_stmt);
}
--- 3501,3508 ----
adjust_bool_pattern (gimple_assign_lhs (bool_stmts[i]),
out_type, stmt_info, defs);
! /* Return the result of the last statement we emitted. */
! gimple *pattern_stmt = stmt_info->prev->stmt;
return gimple_assign_lhs (pattern_stmt);
}
*************** static vect_recog_func vect_vect_recog_f
*** 4676,4689 ****
const unsigned int NUM_PATTERNS = ARRAY_SIZE (vect_vect_recog_func_ptrs);
! /* Mark statements that are involved in a pattern. */
static inline void
! vect_mark_pattern_stmts (stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
! tree pattern_vectype)
{
- gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
-
gimple *orig_pattern_stmt = NULL;
if (is_pattern_stmt_p (orig_stmt_info))
{
--- 4658,4671 ----
const unsigned int NUM_PATTERNS = ARRAY_SIZE (vect_vect_recog_func_ptrs);
! /* Replace ORIG_STMT_INFO with PATTERN_STMT, using PATTERN_VECTYPE as
! the vector type for PATTERN_STMT. */
static inline void
! vect_replace_stmt_with_pattern (stmt_vec_info orig_stmt_info,
! gimple *pattern_stmt,
! tree pattern_vectype)
{
gimple *orig_pattern_stmt = NULL;
if (is_pattern_stmt_p (orig_stmt_info))
{
*************** vect_mark_pattern_stmts (stmt_vec_info o
*** 4710,4741 ****
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_stmt, 0);
}
- /* Switch to the statement that ORIG replaces. */
- orig_stmt_info = STMT_VINFO_RELATED_STMT (orig_stmt_info);
-
/* We shouldn't be replacing the main pattern statement. */
! gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info)->stmt
! != orig_pattern_stmt);
! }
! if (def_seq)
! for (gimple_stmt_iterator si = gsi_start (def_seq);
! !gsi_end_p (si); gsi_next (&si))
! vect_init_pattern_stmt (gsi_stmt (si), orig_stmt_info, pattern_vectype);
!
! if (orig_pattern_stmt)
! {
! vect_init_pattern_stmt (pattern_stmt, orig_stmt_info, pattern_vectype);
!
! /* Insert all the new pattern statements before the original one. */
! gimple_seq *orig_def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
! gimple_stmt_iterator gsi = gsi_for_stmt (orig_pattern_stmt,
! orig_def_seq);
! gsi_insert_seq_before_without_update (&gsi, def_seq, GSI_SAME_STMT);
! gsi_insert_before_without_update (&gsi, pattern_stmt, GSI_SAME_STMT);
/* Remove the pattern statement that this new pattern replaces. */
! gsi_remove (&gsi, false);
}
else
vect_set_pattern_stmt (pattern_stmt, orig_stmt_info, pattern_vectype);
--- 4692,4705 ----
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_stmt, 0);
}
/* We shouldn't be replacing the main pattern statement. */
! gcc_assert (!is_main_pattern_stmt_p (orig_stmt_info));
! /* Insert the new pattern statement before the original one. */
! append_pattern_def_seq (orig_stmt_info, pattern_stmt, pattern_vectype);
/* Remove the pattern statement that this new pattern replaces. */
! orig_stmt_info->block->remove (orig_stmt_info);
}
else
vect_set_pattern_stmt (pattern_stmt, orig_stmt_info, pattern_vectype);
*************** vect_mark_pattern_stmts (stmt_vec_info o
*** 4762,4791 ****
static void
vect_pattern_recog_1 (vect_recog_func *recog_func, stmt_vec_info stmt_info)
{
- vec_info *vinfo = stmt_info->vinfo;
gimple *pattern_stmt;
loop_vec_info loop_vinfo;
tree pattern_vectype;
! /* If this statement has already been replaced with pattern statements,
! leave the original statement alone, since the first match wins.
! Instead try to match against the definition statements that feed
! the main pattern statement. */
! if (STMT_VINFO_IN_PATTERN_P (stmt_info))
! {
! gimple_stmt_iterator gsi;
! for (gsi = gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
! !gsi_end_p (gsi); gsi_next (&gsi))
! vect_pattern_recog_1 (recog_func, vinfo->lookup_stmt (gsi_stmt (gsi)));
! return;
! }
!
! gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
pattern_stmt = recog_func->fn (stmt_info, &pattern_vectype);
if (!pattern_stmt)
{
! /* Clear any half-formed pattern definition sequence. */
! STMT_VINFO_PATTERN_DEF_SEQ (stmt_info) = NULL;
return;
}
--- 4726,4742 ----
static void
vect_pattern_recog_1 (vect_recog_func *recog_func, stmt_vec_info stmt_info)
{
gimple *pattern_stmt;
loop_vec_info loop_vinfo;
tree pattern_vectype;
! stmt_vec_info prev_stmt_info = stmt_info->prev;
pattern_stmt = recog_func->fn (stmt_info, &pattern_vectype);
if (!pattern_stmt)
{
! /* Delete any half-formed pattern sequence. */
! while (stmt_info->prev != prev_stmt_info)
! stmt_info->block->remove (prev_stmt_info);
return;
}
*************** vect_pattern_recog_1 (vect_recog_func *r
*** 4800,4807 ****
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_stmt, 0);
}
/* Mark the stmts that are involved in the pattern. */
! vect_mark_pattern_stmts (stmt_info, pattern_stmt, pattern_vectype);
/* Patterns cannot be vectorized using SLP, because they change the order of
computation. */
--- 4751,4765 ----
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_stmt, 0);
}
+ /* Install the vector type in pattern definition statements that
+ don't yet have one. */
+ for (stmt_vec_info pat_stmt_info = stmt_info->prev;
+ pat_stmt_info != prev_stmt_info; pat_stmt_info = pat_stmt_info->prev)
+ if (!STMT_VINFO_VECTYPE (pat_stmt_info))
+ STMT_VINFO_VECTYPE (pat_stmt_info) = pattern_vectype;
+
/* Mark the stmts that are involved in the pattern. */
! vect_replace_stmt_with_pattern (stmt_info, pattern_stmt, pattern_vectype);
/* Patterns cannot be vectorized using SLP, because they change the order of
computation. */
*************** vect_pattern_recog (vec_info *vinfo)
*** 4903,4910 ****
vec_basic_block *vec_bb;
FOR_EACH_VEC_ELT (vinfo->blocks, i, vec_bb)
FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! if (STMT_VINFO_VECTORIZABLE (stmt_info))
! /* Scan over all generic vect_recog_xxx_pattern functions. */
! for (unsigned int j = 0; j < NUM_PATTERNS; j++)
! vect_pattern_recog_1 (&vect_vect_recog_func_ptrs[j], stmt_info);
}
--- 4861,4887 ----
vec_basic_block *vec_bb;
FOR_EACH_VEC_ELT (vinfo->blocks, i, vec_bb)
FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! {
! stmt_vec_info begin_prev = stmt_info->prev;
! if (STMT_VINFO_VECTORIZABLE (stmt_info))
! /* Scan over all generic vect_recog_xxx_pattern functions. */
! for (unsigned int j = 0; j < NUM_PATTERNS; j++)
! {
! stmt_vec_info curr_prev;
! /* Scan over STMT_INFO and any pattern definition statements
! that were introduced by previous recognizers. */
! for (stmt_vec_info curr_info = stmt_info;
! curr_info != begin_prev; curr_info = curr_prev)
! {
! curr_prev = curr_info->prev;
! /* The first match wins, so skip statements that have
! already been replaced, and the final statement with
! which they were replaced. */
! if (!STMT_VINFO_IN_PATTERN_P (curr_info)
! && !is_main_pattern_stmt_p (curr_info))
! vect_pattern_recog_1 (&vect_vect_recog_func_ptrs[j],
! curr_info);
! }
! }
! }
}
