This refactors BB vectorization in preparation to make it work
on sub-BB granularity.
Bootstrapped and tested on x86_64-unknown-linux-gnu, applied to trunk.
Richard.
2015-11-03 Richard Biener <rguent...@suse.de>
* tree-vect-data-refs.c (vect_analyze_data_refs): Do not collect
data references here.
* tree-vect-loop.c: Include cgraph.h.
(vect_analyze_loop_2): Collect data references here.
* tree-vect-slp.c (find_bb_location): Inline ...
(vect_slp_bb): ... here. Renamed from vect_slp_analyze_bb.
Factor in vect_slp_transform_bb.
(vect_slp_transform_bb): Removed.
(vect_slp_analyze_bb_1): Collect data references here.
* tree-vectorizer.c (pass_slp_vectorize::execute): Call
vect_slp_bb.
* tree-vectorizer.h (vect_slp_bb): Declare.
(vect_slp_analyze_bb): Remove.
(vect_slp_transform_bb): Remove.
(find_bb_location): Remove.
(vect_analyze_data_refs): Remove stmt count reference parameter.
Index: trunk/gcc/tree-vect-data-refs.c
===================================================================
*** trunk.orig/gcc/tree-vect-data-refs.c 2015-11-03 12:57:31.538588865
+0100
--- trunk/gcc/tree-vect-data-refs.c 2015-11-03 15:05:51.537766823 +0100
*************** vect_check_gather_scatter (gimple *stmt,
*** 3286,3297 ****
*/
bool
! vect_analyze_data_refs (vec_info *vinfo, int *min_vf, unsigned *n_stmts)
{
struct loop *loop = NULL;
- basic_block bb = NULL;
unsigned int i;
- vec<data_reference_p> datarefs;
struct data_reference *dr;
tree scalar_type;
--- 3286,3295 ----
*/
bool
! vect_analyze_data_refs (vec_info *vinfo, int *min_vf)
{
struct loop *loop = NULL;
unsigned int i;
struct data_reference *dr;
tree scalar_type;
*************** vect_analyze_data_refs (vec_info *vinfo,
*** 3300,3405 ****
"=== vect_analyze_data_refs ===\n");
if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
! {
! basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
!
! loop = LOOP_VINFO_LOOP (loop_vinfo);
! datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
! if (!find_loop_nest (loop, &LOOP_VINFO_LOOP_NEST (loop_vinfo)))
! {
! if (dump_enabled_p ())
! dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
! "not vectorized: loop contains function calls"
! " or data references that cannot be analyzed\n");
! return false;
! }
!
! for (i = 0; i < loop->num_nodes; i++)
! {
! gimple_stmt_iterator gsi;
!
! for (gsi = gsi_start_bb (bbs[i]); !gsi_end_p (gsi); gsi_next (&gsi))
! {
! gimple *stmt = gsi_stmt (gsi);
! if (is_gimple_debug (stmt))
! continue;
! ++*n_stmts;
! if (!find_data_references_in_stmt (loop, stmt, &datarefs))
! {
! if (is_gimple_call (stmt) && loop->safelen)
! {
! tree fndecl = gimple_call_fndecl (stmt), op;
! if (fndecl != NULL_TREE)
! {
! struct cgraph_node *node = cgraph_node::get (fndecl);
! if (node != NULL && node->simd_clones != NULL)
! {
! unsigned int j, n = gimple_call_num_args (stmt);
! for (j = 0; j < n; j++)
! {
! op = gimple_call_arg (stmt, j);
! if (DECL_P (op)
! || (REFERENCE_CLASS_P (op)
! && get_base_address (op)))
! break;
! }
! op = gimple_call_lhs (stmt);
! /* Ignore #pragma omp declare simd functions
! if they don't have data references in the
! call stmt itself. */
! if (j == n
! && !(op
! && (DECL_P (op)
! || (REFERENCE_CLASS_P (op)
! && get_base_address (op)))))
! continue;
! }
! }
! }
! LOOP_VINFO_DATAREFS (loop_vinfo) = datarefs;
! if (dump_enabled_p ())
! dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
! "not vectorized: loop contains function "
! "calls or data references that cannot "
! "be analyzed\n");
! return false;
! }
! }
! }
!
! LOOP_VINFO_DATAREFS (loop_vinfo) = datarefs;
! }
! else
! {
! bb_vec_info bb_vinfo = as_a <bb_vec_info> (vinfo);
! gimple_stmt_iterator gsi;
!
! bb = BB_VINFO_BB (bb_vinfo);
! for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
! {
! gimple *stmt = gsi_stmt (gsi);
! if (is_gimple_debug (stmt))
! continue;
! ++*n_stmts;
! if (!find_data_references_in_stmt (NULL, stmt,
! &BB_VINFO_DATAREFS (bb_vinfo)))
! {
! /* Mark the rest of the basic-block as unvectorizable. */
! for (; !gsi_end_p (gsi); gsi_next (&gsi))
! {
! stmt = gsi_stmt (gsi);
! STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt)) = false;
! }
! break;
! }
! }
!
! datarefs = BB_VINFO_DATAREFS (bb_vinfo);
! }
/* Go through the data-refs, check that the analysis succeeded. Update
pointer from stmt_vec_info struct to DR and vectype. */
FOR_EACH_VEC_ELT (datarefs, i, dr)
{
gimple *stmt;
--- 3298,3309 ----
"=== vect_analyze_data_refs ===\n");
if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
! loop = LOOP_VINFO_LOOP (loop_vinfo);
/* Go through the data-refs, check that the analysis succeeded. Update
pointer from stmt_vec_info struct to DR and vectype. */
+ vec<data_reference_p> datarefs = vinfo->datarefs;
FOR_EACH_VEC_ELT (datarefs, i, dr)
{
gimple *stmt;
Index: trunk/gcc/tree-vect-loop.c
===================================================================
*** trunk.orig/gcc/tree-vect-loop.c 2015-11-02 12:37:11.007248642 +0100
--- trunk/gcc/tree-vect-loop.c 2015-11-03 13:48:50.290127748 +0100
*************** along with GCC; see the file COPYING3.
*** 46,51 ****
--- 46,52 ----
#include "tree-scalar-evolution.h"
#include "tree-vectorizer.h"
#include "gimple-fold.h"
+ #include "cgraph.h"
/* Loop Vectorization Pass.
*************** vect_analyze_loop_2 (loop_vec_info loop_
*** 1584,1596 ****
unsigned int n_stmts = 0;
/* Find all data references in the loop (which correspond to vdefs/vuses)
! and analyze their evolution in the loop. Also adjust the minimal
! vectorization factor according to the loads and stores.
! FORNOW: Handle only simple, array references, which
! alignment can be forced, and aligned pointer-references. */
! ok = vect_analyze_data_refs (loop_vinfo, &min_vf, &n_stmts);
if (!ok)
{
if (dump_enabled_p ())
--- 1585,1658 ----
unsigned int n_stmts = 0;
/* Find all data references in the loop (which correspond to vdefs/vuses)
! and analyze their evolution in the loop. */
! basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
!
! loop_p loop = LOOP_VINFO_LOOP (loop_vinfo);
! if (!find_loop_nest (loop, &LOOP_VINFO_LOOP_NEST (loop_vinfo)))
! {
! if (dump_enabled_p ())
! dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
! "not vectorized: loop contains function calls"
! " or data references that cannot be analyzed\n");
! return false;
! }
!
! for (unsigned i = 0; i < loop->num_nodes; i++)
! for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
! !gsi_end_p (gsi); gsi_next (&gsi))
! {
! gimple *stmt = gsi_stmt (gsi);
! if (is_gimple_debug (stmt))
! continue;
! ++n_stmts;
! if (!find_data_references_in_stmt (loop, stmt,
! &LOOP_VINFO_DATAREFS (loop_vinfo)))
! {
! if (is_gimple_call (stmt) && loop->safelen)
! {
! tree fndecl = gimple_call_fndecl (stmt), op;
! if (fndecl != NULL_TREE)
! {
! cgraph_node *node = cgraph_node::get (fndecl);
! if (node != NULL && node->simd_clones != NULL)
! {
! unsigned int j, n = gimple_call_num_args (stmt);
! for (j = 0; j < n; j++)
! {
! op = gimple_call_arg (stmt, j);
! if (DECL_P (op)
! || (REFERENCE_CLASS_P (op)
! && get_base_address (op)))
! break;
! }
! op = gimple_call_lhs (stmt);
! /* Ignore #pragma omp declare simd functions
! if they don't have data references in the
! call stmt itself. */
! if (j == n
! && !(op
! && (DECL_P (op)
! || (REFERENCE_CLASS_P (op)
! && get_base_address (op)))))
! continue;
! }
! }
! }
! if (dump_enabled_p ())
! dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
! "not vectorized: loop contains function "
! "calls or data references that cannot "
! "be analyzed\n");
! return false;
! }
! }
!
! /* Analyze the data references and also adjust the minimal
! vectorization factor according to the loads and stores. */
! ok = vect_analyze_data_refs (loop_vinfo, &min_vf);
if (!ok)
{
if (dump_enabled_p ())
Index: trunk/gcc/tree-vect-slp.c
===================================================================
*** trunk.orig/gcc/tree-vect-slp.c 2015-11-03 12:57:31.538588865 +0100
--- trunk/gcc/tree-vect-slp.c 2015-11-03 14:04:53.845935110 +0100
*************** along with GCC; see the file COPYING3.
*** 40,67 ****
#include "tree-vectorizer.h"
#include "langhooks.h"
#include "gimple-walk.h"
!
! /* Extract the location of the basic block in the source code.
! Return the basic block location if succeed and NULL if not. */
!
! source_location
! find_bb_location (basic_block bb)
! {
! gimple *stmt = NULL;
! gimple_stmt_iterator si;
!
! if (!bb)
! return UNKNOWN_LOCATION;
!
! for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
! {
! stmt = gsi_stmt (si);
! if (gimple_location (stmt) != UNKNOWN_LOCATION)
! return gimple_location (stmt);
! }
!
! return UNKNOWN_LOCATION;
! }
/* Recursively free the memory allocated for the SLP tree rooted at NODE. */
--- 40,46 ----
#include "tree-vectorizer.h"
#include "langhooks.h"
#include "gimple-walk.h"
! #include "dbgcnt.h"
/* Recursively free the memory allocated for the SLP tree rooted at NODE. */
*************** vect_slp_analyze_bb_1 (basic_block bb)
*** 2361,2367 ****
if (!bb_vinfo)
return NULL;
! if (!vect_analyze_data_refs (bb_vinfo, &min_vf, &n_stmts))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
--- 2340,2370 ----
if (!bb_vinfo)
return NULL;
! /* Gather all data references in the basic-block. */
!
! for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
! !gsi_end_p (gsi); gsi_next (&gsi))
! {
! gimple *stmt = gsi_stmt (gsi);
! if (is_gimple_debug (stmt))
! continue;
! ++n_stmts;
! if (!find_data_references_in_stmt (NULL, stmt,
! &BB_VINFO_DATAREFS (bb_vinfo)))
! {
! /* Mark the rest of the basic-block as unvectorizable. */
! for (; !gsi_end_p (gsi); gsi_next (&gsi))
! {
! stmt = gsi_stmt (gsi);
! STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt)) = false;
! }
! break;
! }
! }
!
! /* Analyze the data references. */
!
! if (!vect_analyze_data_refs (bb_vinfo, &min_vf))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
*************** vect_slp_analyze_bb_1 (basic_block bb)
*** 2499,2506 ****
}
! bb_vec_info
! vect_slp_analyze_bb (basic_block bb)
{
bb_vec_info bb_vinfo;
int insns = 0;
--- 2502,2512 ----
}
! /* Main entry for the BB vectorizer. Analyze and transform BB, returns
! true if anything in the basic-block was vectorized. */
!
! bool
! vect_slp_bb (basic_block bb)
{
bb_vec_info bb_vinfo;
int insns = 0;
*************** vect_slp_analyze_bb (basic_block bb)
*** 2517,2522 ****
--- 2523,2530 ----
&& !gimple_nop_p (stmt)
&& gimple_code (stmt) != GIMPLE_LABEL)
insns++;
+ if (gimple_location (stmt) != UNKNOWN_LOCATION)
+ vect_location = gimple_location (stmt);
}
if (insns > PARAM_VALUE (PARAM_SLP_MAX_INSNS_IN_BB))
*************** vect_slp_analyze_bb (basic_block bb)
*** 2526,2532 ****
"not vectorized: too many instructions in "
"basic block.\n");
! return NULL;
}
/* Autodetect first vector size we try. */
--- 2534,2540 ----
"not vectorized: too many instructions in "
"basic block.\n");
! return false;
}
/* Autodetect first vector size we try. */
*************** vect_slp_analyze_bb (basic_block bb)
*** 2537,2550 ****
{
bb_vinfo = vect_slp_analyze_bb_1 (bb);
if (bb_vinfo)
! return bb_vinfo;
destroy_bb_vec_info (bb_vinfo);
vector_sizes &= ~current_vector_size;
if (vector_sizes == 0
|| current_vector_size == 0)
! return NULL;
/* Try the next biggest vector size. */
current_vector_size = 1 << floor_log2 (vector_sizes);
--- 2545,2577 ----
{
bb_vinfo = vect_slp_analyze_bb_1 (bb);
if (bb_vinfo)
! {
! if (!dbg_cnt (vect_slp))
! {
! destroy_bb_vec_info (bb_vinfo);
! return false;
! }
!
! if (dump_enabled_p ())
! dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB\n");
!
! vect_schedule_slp (bb_vinfo);
!
! if (dump_enabled_p ())
! dump_printf_loc (MSG_NOTE, vect_location,
! "BASIC BLOCK VECTORIZED\n");
!
! destroy_bb_vec_info (bb_vinfo);
!
! return true;
! }
destroy_bb_vec_info (bb_vinfo);
vector_sizes &= ~current_vector_size;
if (vector_sizes == 0
|| current_vector_size == 0)
! return false;
/* Try the next biggest vector size. */
current_vector_size = 1 << floor_log2 (vector_sizes);
*************** vect_schedule_slp (vec_info *vinfo)
*** 3533,3580 ****
return is_store;
}
-
-
- /* Vectorize the basic block. */
-
- void
- vect_slp_transform_bb (basic_block bb)
- {
- bb_vec_info bb_vinfo = vec_info_for_bb (bb);
- gimple_stmt_iterator si;
-
- gcc_assert (bb_vinfo);
-
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB\n");
-
- for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
- {
- gimple *stmt = gsi_stmt (si);
- stmt_vec_info stmt_info;
-
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "------>SLPing statement: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
- dump_printf (MSG_NOTE, "\n");
- }
-
- stmt_info = vinfo_for_stmt (stmt);
- gcc_assert (stmt_info);
-
- /* Schedule all the SLP instances when the first SLP stmt is reached.
*/
- if (STMT_SLP_TYPE (stmt_info))
- {
- vect_schedule_slp (bb_vinfo);
- break;
- }
- }
-
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "BASIC BLOCK VECTORIZED\n");
-
- destroy_bb_vec_info (bb_vinfo);
- }
--- 3560,3562 ----
Index: trunk/gcc/tree-vectorizer.c
===================================================================
*** trunk.orig/gcc/tree-vectorizer.c 2015-11-02 12:37:10.986248408 +0100
--- trunk/gcc/tree-vectorizer.c 2015-11-03 14:03:39.794106576 +0100
*************** pass_slp_vectorize::execute (function *f
*** 696,713 ****
FOR_EACH_BB_FN (bb, fun)
{
! vect_location = find_bb_location (bb);
!
! if (vect_slp_analyze_bb (bb))
! {
! if (!dbg_cnt (vect_slp))
! break;
!
! vect_slp_transform_bb (bb);
! if (dump_enabled_p ())
! dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
! "basic block vectorized\n");
! }
}
free_stmt_vec_info_vec ();
--- 696,704 ----
FOR_EACH_BB_FN (bb, fun)
{
! if (vect_slp_bb (bb))
! dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
! "basic block vectorized\n");
}
free_stmt_vec_info_vec ();
Index: trunk/gcc/tree-vectorizer.h
===================================================================
*** trunk.orig/gcc/tree-vectorizer.h 2015-11-03 12:57:31.538588865 +0100
--- trunk/gcc/tree-vectorizer.h 2015-11-03 14:33:42.315259421 +0100
*************** extern bool vect_analyze_data_ref_access
*** 1010,1016 ****
extern bool vect_prune_runtime_alias_test_list (loop_vec_info);
extern tree vect_check_gather_scatter (gimple *, loop_vec_info, tree *, tree
*,
int *);
! extern bool vect_analyze_data_refs (vec_info *, int *, unsigned *);
extern tree vect_create_data_ref_ptr (gimple *, tree, struct loop *, tree,
tree *, gimple_stmt_iterator *,
gimple **, bool, bool *,
--- 1010,1016 ----
extern bool vect_prune_runtime_alias_test_list (loop_vec_info);
extern tree vect_check_gather_scatter (gimple *, loop_vec_info, tree *, tree
*,
int *);
! extern bool vect_analyze_data_refs (vec_info *, int *);
extern tree vect_create_data_ref_ptr (gimple *, tree, struct loop *, tree,
tree *, gimple_stmt_iterator *,
gimple **, bool, bool *,
*************** extern bool vect_make_slp_decision (loop
*** 1072,1081 ****
extern void vect_detect_hybrid_slp (loop_vec_info);
extern void vect_get_slp_defs (vec<tree> , slp_tree,
vec<vec<tree> > *, int);
!
! extern source_location find_bb_location (basic_block);
! extern bb_vec_info vect_slp_analyze_bb (basic_block);
! extern void vect_slp_transform_bb (basic_block);
/* In tree-vect-patterns.c. */
/* Pattern recognition functions.
--- 1072,1078 ----
extern void vect_detect_hybrid_slp (loop_vec_info);
extern void vect_get_slp_defs (vec<tree> , slp_tree,
vec<vec<tree> > *, int);
! extern bool vect_slp_bb (basic_block);
/* In tree-vect-patterns.c. */
/* Pattern recognition functions.
