This fixes the dependency of vectorization of strided loads on
gather support. For that to work we need to lift the restriction
in data-ref analysis that requries a constant DR_STEP. Fortunately
fallout is small.
As a side-effect the data-reference for strided loads now makes
sense (rather than being gobbled over by gather analysis BB-level DR).
Bootstrapped and tested on x86_64-unknown-linux-gnu, applied to trunk.
Richard.
2012-05-11 Richard Guenther <rguent...@suse.de>
* tree-data-ref.h (stride_of_unit_type_p): Handle non-constant
strides.
* tree-data-ref.c (dr_analyze_innermost): Allow non-constant
strides when analyzing data-references in a loop context.
* tree-vect-data-refs.c (vect_mark_for_runtime_alias_test): Reject
non-constant strides for now.
(vect_enhance_data_refs_alignment): Ignore data references
that are strided loads.
(vect_analyze_data_ref_access): Handle non-constant strides.
(vect_check_strided_load): Verify the data-reference is a load.
(vect_analyze_data_refs): Restructure to make strided load
support not dependent on gather support.
* tree-vect-stmts.c (vectorizable_load): Avoid useless work
when doing strided or gather loads.
* tree-vect-loop-manip.c (vect_vfa_segment_size): Use
integer_zerop to compare stride with zero.
Index: gcc/tree-data-ref.h
===================================================================
*** gcc/tree-data-ref.h (revision 187396)
--- gcc/tree-data-ref.h (working copy)
*************** bool stmt_with_adjacent_zero_store_dr_p
*** 618,626 ****
static inline bool
stride_of_unit_type_p (tree stride, tree type)
{
! return tree_int_cst_equal (fold_unary (ABS_EXPR, TREE_TYPE (stride),
! stride),
! TYPE_SIZE_UNIT (type));
}
/* Determines whether RDG vertices V1 and V2 access to similar memory
--- 618,627 ----
static inline bool
stride_of_unit_type_p (tree stride, tree type)
{
! return (TREE_CODE (stride) == INTEGER_CST
! && tree_int_cst_equal (fold_unary (ABS_EXPR, TREE_TYPE (stride),
! stride),
! TYPE_SIZE_UNIT (type)));
}
/* Determines whether RDG vertices V1 and V2 access to similar memory
Index: gcc/tree-data-ref.c
===================================================================
*** gcc/tree-data-ref.c (revision 187396)
--- gcc/tree-data-ref.c (working copy)
*************** dr_analyze_innermost (struct data_refere
*** 736,742 ****
if (in_loop)
{
if (!simple_iv (loop, loop_containing_stmt (stmt), base, &base_iv,
! false))
{
if (nest)
{
--- 736,742 ----
if (in_loop)
{
if (!simple_iv (loop, loop_containing_stmt (stmt), base, &base_iv,
! nest ? true : false))
{
if (nest)
{
*************** dr_analyze_innermost (struct data_refere
*** 773,779 ****
offset_iv.step = ssize_int (0);
}
else if (!simple_iv (loop, loop_containing_stmt (stmt),
! poffset, &offset_iv, false))
{
if (nest)
{
--- 773,780 ----
offset_iv.step = ssize_int (0);
}
else if (!simple_iv (loop, loop_containing_stmt (stmt),
! poffset, &offset_iv,
! nest ? true : false))
{
if (nest)
{
Index: gcc/tree-vect-data-refs.c
===================================================================
*** gcc/tree-vect-data-refs.c (revision 187396)
--- gcc/tree-vect-data-refs.c (working copy)
*************** vect_mark_for_runtime_alias_test (ddr_p
*** 542,547 ****
--- 542,558 ----
return false;
}
+ /* FORNOW: We don't support creating runtime alias tests for non-constant
+ step. */
+ if (TREE_CODE (DR_STEP (DDR_A (ddr))) != INTEGER_CST
+ || TREE_CODE (DR_STEP (DDR_B (ddr))) != INTEGER_CST)
+ {
+ if (vect_print_dump_info (REPORT_DR_DETAILS))
+ fprintf (vect_dump, "versioning not yet supported for non-constant "
+ "step");
+ return false;
+ }
+
VEC_safe_push (ddr_p, heap, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo), ddr);
return true;
}
*************** vect_enhance_data_refs_alignment (loop_v
*** 1522,1527 ****
--- 1533,1543 ----
if (integer_zerop (DR_STEP (dr)))
continue;
+ /* Strided loads perform only component accesses, alignment is
+ irrelevant for them. */
+ if (STMT_VINFO_STRIDE_LOAD_P (stmt_info))
+ continue;
+
supportable_dr_alignment = vect_supportable_dr_alignment (dr, true);
do_peeling = vector_alignment_reachable_p (dr);
if (do_peeling)
*************** vect_enhance_data_refs_alignment (loop_v
*** 1779,1784 ****
--- 1795,1805 ----
&& GROUP_FIRST_ELEMENT (stmt_info) != stmt)
continue;
+ /* Strided loads perform only component accesses, alignment is
+ irrelevant for them. */
+ if (STMT_VINFO_STRIDE_LOAD_P (stmt_info))
+ continue;
+
save_misalignment = DR_MISALIGNMENT (dr);
vect_update_misalignment_for_peel (dr, dr0, npeel);
supportable_dr_alignment = vect_supportable_dr_alignment (dr, false);
*************** vect_enhance_data_refs_alignment (loop_v
*** 1861,1866 ****
--- 1882,1892 ----
&& GROUP_FIRST_ELEMENT (stmt_info) != stmt))
continue;
+ /* Strided loads perform only component accesses, alignment is
+ irrelevant for them. */
+ if (STMT_VINFO_STRIDE_LOAD_P (stmt_info))
+ continue;
+
supportable_dr_alignment = vect_supportable_dr_alignment (dr, false);
if (!supportable_dr_alignment)
*************** vect_analyze_data_ref_access (struct dat
*** 2329,2335 ****
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = NULL;
- HOST_WIDE_INT dr_step;
if (loop_vinfo)
loop = LOOP_VINFO_LOOP (loop_vinfo);
--- 2355,2360 ----
*************** vect_analyze_data_ref_access (struct dat
*** 2342,2349 ****
}
/* Allow invariant loads in loops. */
! dr_step = TREE_INT_CST_LOW (step);
! if (loop_vinfo && dr_step == 0)
{
GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) = NULL;
return DR_IS_READ (dr);
--- 2367,2373 ----
}
/* Allow invariant loads in loops. */
! if (loop_vinfo && integer_zerop (step))
{
GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) = NULL;
return DR_IS_READ (dr);
*************** vect_analyze_data_ref_access (struct dat
*** 2357,2365 ****
/* For the rest of the analysis we use the outer-loop step. */
step = STMT_VINFO_DR_STEP (stmt_info);
! dr_step = TREE_INT_CST_LOW (step);
!
! if (dr_step == 0)
{
if (vect_print_dump_info (REPORT_ALIGNMENT))
fprintf (vect_dump, "zero step in outer loop.");
--- 2381,2387 ----
/* For the rest of the analysis we use the outer-loop step. */
step = STMT_VINFO_DR_STEP (stmt_info);
! if (integer_zerop (step))
{
if (vect_print_dump_info (REPORT_ALIGNMENT))
fprintf (vect_dump, "zero step in outer loop.");
*************** vect_analyze_data_ref_access (struct dat
*** 2371,2383 ****
}
/* Consecutive? */
! if (!tree_int_cst_compare (step, TYPE_SIZE_UNIT (scalar_type))
! || (dr_step < 0
! && !compare_tree_int (TYPE_SIZE_UNIT (scalar_type), -dr_step)))
{
! /* Mark that it is not interleaving. */
! GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) = NULL;
! return true;
}
if (loop && nested_in_vect_loop_p (loop, stmt))
--- 2393,2409 ----
}
/* Consecutive? */
! if (TREE_CODE (step) == INTEGER_CST)
{
! HOST_WIDE_INT dr_step = TREE_INT_CST_LOW (step);
! if (!tree_int_cst_compare (step, TYPE_SIZE_UNIT (scalar_type))
! || (dr_step < 0
! && !compare_tree_int (TYPE_SIZE_UNIT (scalar_type), -dr_step)))
! {
! /* Mark that it is not interleaving. */
! GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) = NULL;
! return true;
! }
}
if (loop && nested_in_vect_loop_p (loop, stmt))
*************** vect_analyze_data_ref_access (struct dat
*** 2387,2392 ****
--- 2413,2422 ----
return false;
}
+ /* Assume this is a DR handled by non-constant strided load case. */
+ if (TREE_CODE (step) != INTEGER_CST)
+ return STMT_VINFO_STRIDE_LOAD_P (stmt_info);
+
/* Not consecutive access - check if it's a part of interleaving group. */
return vect_analyze_group_access (dr);
}
*************** vect_check_strided_load (gimple stmt, lo
*** 2720,2725 ****
--- 2750,2758 ----
tree base, off;
affine_iv iv;
+ if (!DR_IS_READ (dr))
+ return false;
+
base = DR_REF (dr);
if (TREE_CODE (base) == ARRAY_REF)
*************** vect_analyze_data_refs (loop_vec_info lo
*** 3148,3168 ****
VEC (ddr_p, heap) *ddrs = LOOP_VINFO_DDRS (loop_vinfo);
struct data_dependence_relation *ddr, *newddr;
bool bad = false;
- bool strided_load = false;
tree off;
VEC (loop_p, heap) *nest = LOOP_VINFO_LOOP_NEST (loop_vinfo);
- strided_load = vect_check_strided_load (stmt, loop_vinfo, NULL, NULL);
gather = 0 != vect_check_gather (stmt, loop_vinfo, NULL, &off, NULL);
if (gather
&& get_vectype_for_scalar_type (TREE_TYPE (off)) == NULL_TREE)
gather = false;
! if (!gather && !strided_load)
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
fprintf (vect_dump,
! "not vectorized: not suitable for gather/strided
load ");
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
return false;
--- 3181,3199 ----
VEC (ddr_p, heap) *ddrs = LOOP_VINFO_DDRS (loop_vinfo);
struct data_dependence_relation *ddr, *newddr;
bool bad = false;
tree off;
VEC (loop_p, heap) *nest = LOOP_VINFO_LOOP_NEST (loop_vinfo);
gather = 0 != vect_check_gather (stmt, loop_vinfo, NULL, &off, NULL);
if (gather
&& get_vectype_for_scalar_type (TREE_TYPE (off)) == NULL_TREE)
gather = false;
! if (!gather)
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
fprintf (vect_dump,
! "not vectorized: not suitable for gather load ");
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
return false;
*************** vect_analyze_data_refs (loop_vec_info lo
*** 3215,3230 ****
{
fprintf (vect_dump,
"not vectorized: data dependence conflict"
! " prevents gather/strided load");
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
return false;
}
! if (gather)
! STMT_VINFO_GATHER_P (stmt_info) = true;
! else if (strided_load)
! STMT_VINFO_STRIDE_LOAD_P (stmt_info) = true;
}
}
--- 3246,3277 ----
{
fprintf (vect_dump,
"not vectorized: data dependence conflict"
! " prevents gather load");
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
return false;
}
! STMT_VINFO_GATHER_P (stmt_info) = true;
! }
! else if (loop_vinfo
! && TREE_CODE (DR_STEP (dr)) != INTEGER_CST)
! {
! bool strided_load = false;
! if (!nested_in_vect_loop_p (loop, stmt))
! strided_load
! = vect_check_strided_load (stmt, loop_vinfo, NULL, NULL);
! if (!strided_load)
! {
! if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
! {
! fprintf (vect_dump,
! "not vectorized: not suitable for strided load ");
! print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
! }
! return false;
! }
! STMT_VINFO_STRIDE_LOAD_P (stmt_info) = true;
}
}
Index: gcc/tree-vect-stmts.c
===================================================================
*** gcc/tree-vect-stmts.c (revision 187396)
--- gcc/tree-vect-stmts.c (working copy)
*************** vectorizable_load (gimple stmt, gimple_s
*** 4210,4216 ****
bool load_lanes_p = false;
gimple first_stmt;
bool inv_p;
! bool negative;
bool compute_in_loop = false;
struct loop *at_loop;
int vec_num;
--- 4210,4216 ----
bool load_lanes_p = false;
gimple first_stmt;
bool inv_p;
! bool negative = false;
bool compute_in_loop = false;
struct loop *at_loop;
int vec_num;
*************** vectorizable_load (gimple stmt, gimple_s
*** 4280,4296 ****
if (!STMT_VINFO_DATA_REF (stmt_info))
return false;
- negative = tree_int_cst_compare (nested_in_vect_loop
- ? STMT_VINFO_DR_STEP (stmt_info)
- : DR_STEP (dr),
- size_zero_node) < 0;
- if (negative && ncopies > 1)
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "multiple types with negative step.");
- return false;
- }
-
elem_type = TREE_TYPE (vectype);
mode = TYPE_MODE (vectype);
--- 4280,4285 ----
*************** vectorizable_load (gimple stmt, gimple_s
*** 4321,4344 ****
}
}
- if (negative)
- {
- gcc_assert (!grouped_load && !STMT_VINFO_GATHER_P (stmt_info));
- alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
- if (alignment_support_scheme != dr_aligned
- && alignment_support_scheme != dr_unaligned_supported)
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "negative step but alignment required.");
- return false;
- }
- if (!perm_mask_for_reverse (vectype))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "negative step and reversing not supported.");
- return false;
- }
- }
if (STMT_VINFO_GATHER_P (stmt_info))
{
--- 4310,4315 ----
*************** vectorizable_load (gimple stmt, gimple_s
*** 4358,4364 ****
}
else if (STMT_VINFO_STRIDE_LOAD_P (stmt_info))
{
! vect_check_strided_load (stmt, loop_vinfo, &stride_base, &stride_step);
}
if (!vec_stmt) /* transformation not required. */
--- 4329,4369 ----
}
else if (STMT_VINFO_STRIDE_LOAD_P (stmt_info))
{
! if (!vect_check_strided_load (stmt, loop_vinfo,
! &stride_base, &stride_step))
! return false;
! }
! else
! {
! negative = tree_int_cst_compare (nested_in_vect_loop
! ? STMT_VINFO_DR_STEP (stmt_info)
! : DR_STEP (dr),
! size_zero_node) < 0;
! if (negative && ncopies > 1)
! {
! if (vect_print_dump_info (REPORT_DETAILS))
! fprintf (vect_dump, "multiple types with negative step.");
! return false;
! }
!
! if (negative)
! {
! gcc_assert (!grouped_load);
! alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
! if (alignment_support_scheme != dr_aligned
! && alignment_support_scheme != dr_unaligned_supported)
! {
! if (vect_print_dump_info (REPORT_DETAILS))
! fprintf (vect_dump, "negative step but alignment required.");
! return false;
! }
! if (!perm_mask_for_reverse (vectype))
! {
! if (vect_print_dump_info (REPORT_DETAILS))
! fprintf (vect_dump, "negative step and reversing not
supported.");
! return false;
! }
! }
}
if (!vec_stmt) /* transformation not required. */
Index: gcc/tree-vect-loop-manip.c
===================================================================
*** gcc/tree-vect-loop-manip.c (revision 187396)
--- gcc/tree-vect-loop-manip.c (working copy)
*************** vect_vfa_segment_size (struct data_refer
*** 2334,2340 ****
{
tree segment_length;
! if (!compare_tree_int (DR_STEP (dr), 0))
segment_length = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr)));
else
segment_length = size_binop (MULT_EXPR,
--- 2334,2340 ----
{
tree segment_length;
! if (integer_zerop (DR_STEP (dr)))
segment_length = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr)));
else
segment_length = size_binop (MULT_EXPR,
