Various vectorizable_* functions have been updated to return false
to abort the analysis phase of vectorization immediately instead of
setting LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P to false if a partial
vector is required (i.e. the number of subparts in the vector type
may be greater than the number of active lanes for the node) but
not available (e.g., direct_internal_fn_supported_p returned false).
To avoid duplicating the controlling expression, a new helper,
can_use_partial_vectors_p, is used. When analyzing a loop, this
simply returns the result of LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P.
The vect_get_num_copies function used during statement analysis
is updated to return early with 1 if a vector type is long enough for
the specified SLP tree node. This avoids an ICE in vect_get_num_vectors,
which cannot cope with SVE vector types.
The return value of can_use_partial_vectors_p may sometimes appear
to have been assigned unnecessarily. That is to reduce the size of
other patches in this series.
gcc/ChangeLog:
* tree-vect-stmts.cc (can_use_partial_vectors_p): New helper
function to determine whether partial vectors can (still) be
used to vectorize a loop or SLP region. In the case of SLP,
partial vectors can only be used if they might be required
(based on the count of subparts passed by the caller, relative
to the group size).
(vectorizable_call): Call can_use_partial_vectors_p instead of
relying on LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P to determine
whether partial vectors can be used. If so, but no
conditional operation is available, then return false for BB SLP
instead of setting LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P to false.
Pass vec_info and SLP tree node instead of loop_vec_info or null
when calling vect_record_len or vect_record_mask.
(vectorizable_simd_clone_call): Return false if SLP_TREE_LANES
is less than the minimum number of subparts in SLP_TREE_VECTYPE.
(vectorizable_operation): Call can_use_partial_vectors_p instead of
relying on LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P to determine
whether partial vectors can be used. If so, but no
conditional operation is available, then return false for BB SLP
instead of setting LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P to false.
Pass vec_info and SLP tree node instead of loop_vec_info or null
when calling vect_record_len or vect_record_mask.
(vectorizable_store): During the analysis phase, if doing BB SLP
vectorization, call check_load_store_for_partial_vectors as for
loop vectorization but propagate a return value of false up the
callstack instead of assigning the return value to
LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P.
(vectorizable_load): As above.
(vectorizable_condition): Call can_use_partial_vectors_p instead
of relying on LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P to determine
whether partial vectors can be used. If so, but no
conditional operation is available, then return false for BB SLP
instead of setting LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P to false.
Pass vec_info and SLP tree node instead of loop_vec_info or null
when calling vect_record_len or vect_record_mask.
* tree-vectorizer.h (vect_get_num_copies): Return early with 1
if SLP_TREE_VECTYPE is long enough for SLP_TREE_LANES of the
specified SLP tree node to avoid an ICE in vect_get_num_vectors.
---
gcc/tree-vect-loop.cc | 6 +
gcc/tree-vect-stmts.cc | 425 ++++++++++++++++++++++++++---------------
gcc/tree-vectorizer.h | 22 ++-
3 files changed, 295 insertions(+), 158 deletions(-)
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 448d0c3f27d..ec733d01e77 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -2405,6 +2405,12 @@ start_over:
if (!ok)
return ok;
+ /* Clear the chosen partial vector style if the loop does not use partial
+ vectors. This prevents production of SSA names for lengths or masks that
+ are not subsequently defined. */
+ if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
+ LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_none;
+
/* If we're vectorizing a loop that uses length "controls" and
can iterate more than once, we apply decrementing IV approach
in loop control. */
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 27d13ddf524..ca3b2845713 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -1476,7 +1476,7 @@ check_load_store_for_partial_vectors (vec_info *vinfo,
tree vectype,
vect_memory_access_type memory_access_type = ls->memory_access_type;
/* For SLP vectorization, we cannot use partial vectors if the
- group could be wider than the vector type. */
+ group could be wider than the vector type (e.g. 8 lanes of VNx4SI). */
poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
if (!loop_vinfo && maybe_gt (SLP_TREE_LANES (slp_node), nunits))
return false;
@@ -3453,6 +3453,23 @@ simple_integer_narrowing (tree vectype_out, tree
vectype_in,
return true;
}
+/* Return true if the loop or SLP vectorization region represented by
+ VINFO can use partial vectors. SLP_NODE can be null unless VINFO
+ represents an SLP region. NUNITS gives the number of subparts in the
+ vector type that would be used to vectorize stmts. */
+static bool
+can_use_partial_vectors_p (vec_info *vinfo, slp_tree slp_node,
+ poly_uint64 nunits)
+{
+ gcc_assert (!is_a<bb_vec_info> (vinfo) || slp_node);
+ loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
+
+ if (loop_vinfo)
+ return LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo);
+ else
+ return maybe_lt (SLP_TREE_LANES (slp_node), nunits);
+}
+
/* Function vectorizable_call.
Check if STMT_INFO performs a function call that can be vectorized.
@@ -3665,24 +3682,26 @@ vectorizable_call (vec_info *vinfo,
ifn = vectorizable_internal_function (cfn, callee, vectype_out,
vectype_in);
+ bool use_partial_vectors_p
+ = can_use_partial_vectors_p (vinfo, slp_node, nunits_in);
+
/* Check if the operation traps. */
bool could_trap = gimple_could_trap_p (STMT_VINFO_STMT (stmt_info));
- if (could_trap && cost_vec && loop_vinfo)
+ if (could_trap && cost_vec && use_partial_vectors_p)
{
/* If the operation can trap it must be conditional, otherwise fail. */
internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1
- ? ifn : get_conditional_internal_fn (ifn));
+ ? ifn
+ : get_conditional_internal_fn (ifn));
internal_fn cond_len_fn = get_len_internal_fn (cond_fn);
- if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
+ if ((cond_fn == IFN_LAST
+ || !direct_internal_fn_supported_p (cond_fn, vectype_out,
+ OPTIMIZE_FOR_SPEED))
+ && (cond_len_fn == IFN_LAST
+ || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
+ OPTIMIZE_FOR_SPEED)))
{
- /* We assume that BB SLP fills all lanes, so no inactive lanes can
- cause issues. */
- if ((cond_fn == IFN_LAST
- || !direct_internal_fn_supported_p (cond_fn, vectype_out,
- OPTIMIZE_FOR_SPEED))
- && (cond_len_fn == IFN_LAST
- || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
- OPTIMIZE_FOR_SPEED)))
+ if (loop_vinfo)
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -3690,6 +3709,14 @@ vectorizable_call (vec_info *vinfo,
" conditional operation is available.\n");
LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
}
+ else
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "SLP can't use a mask or length because no"
+ " conditional operation is available.\n");
+ return false;
+ }
}
}
@@ -3755,9 +3782,8 @@ vectorizable_call (vec_info *vinfo,
DUMP_VECT_SCOPE ("vectorizable_call");
vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
- if (loop_vinfo
- && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
- && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
+ if ((reduc_idx >= 0 || could_trap || mask_opno >= 0)
+ && use_partial_vectors_p)
{
if (reduc_idx >= 0
&& (cond_fn == IFN_LAST
@@ -3767,11 +3793,22 @@ vectorizable_call (vec_info *vinfo,
|| !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
OPTIMIZE_FOR_SPEED)))
{
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "can't use a fully-masked loop because no"
- " conditional operation is available.\n");
- LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ if (loop_vinfo)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't use a fully-masked loop because no"
+ " conditional operation is available.\n");
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ }
+ else
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "SLP can't use a mask or length because no"
+ " conditional operation is available.\n");
+ return false;
+ }
}
else
{
@@ -3781,10 +3818,10 @@ vectorizable_call (vec_info *vinfo,
if (cond_len_fn != IFN_LAST
&& direct_internal_fn_supported_p (cond_len_fn, vectype_out,
OPTIMIZE_FOR_SPEED))
- vect_record_len (loop_vinfo, nvectors, vectype_out, 1);
+ vect_record_len (vinfo, nvectors, vectype_out, 1, slp_node);
else
- vect_record_mask (loop_vinfo, nvectors, vectype_out,
- scalar_mask);
+ vect_record_mask (vinfo, nvectors, vectype_out, scalar_mask,
+ slp_node);
}
}
return true;
@@ -3799,8 +3836,15 @@ vectorizable_call (vec_info *vinfo,
scalar_dest = gimple_call_lhs (stmt);
vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
- bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
- bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
+ bool masked_loop_p
+ = use_partial_vectors_p
+ && (vinfo->partial_vector_style == vect_partial_vectors_while_ult
+ || vinfo->partial_vector_style == vect_partial_vectors_avx512);
+
+ bool len_loop_p
+ = use_partial_vectors_p
+ && (vinfo->partial_vector_style == vect_partial_vectors_len);
+
unsigned int vect_nargs = nargs;
if (len_loop_p && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
{
@@ -3864,8 +3908,8 @@ vectorizable_call (vec_info *vinfo,
if (masked_loop_p)
{
unsigned int vec_num = vec_oprnds0.length ();
- vargs[varg++] = vect_get_mask (loop_vinfo, gsi, vec_num,
- vectype_out, i);
+ vargs[varg++] = vect_get_mask (vinfo, gsi, vec_num,
+ vectype_out, i, slp_node);
}
else
{
@@ -3924,10 +3968,9 @@ vectorizable_call (vec_info *vinfo,
if (len_opno >= 0 && len_loop_p)
{
unsigned int vec_num = vec_oprnds0.length ();
- tree len = vect_get_len (loop_vinfo, gsi, vec_num,
- vectype_out, i, 1, true);
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ tree len = vect_get_len (vinfo, gsi, vec_num, vectype_out,
+ i, 1, true, slp_node);
+ signed char biasval = vinfo->partial_load_store_bias;
tree bias = build_int_cst (intQI_type_node, biasval);
vargs[len_opno] = len;
vargs[len_opno + 1] = bias;
@@ -3935,10 +3978,10 @@ vectorizable_call (vec_info *vinfo,
else if (mask_opno >= 0 && masked_loop_p)
{
unsigned int vec_num = vec_oprnds0.length ();
- tree mask = vect_get_mask (loop_vinfo, gsi, vec_num,
- vectype_out, i);
+ tree mask = vect_get_mask (vinfo, gsi, vec_num,
+ vectype_out, i, slp_node);
vargs[mask_opno]
- = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
+ = prepare_vec_mask (vinfo, TREE_TYPE (mask), mask,
vargs[mask_opno], gsi);
}
@@ -4164,6 +4207,12 @@ vectorizable_simd_clone_call (vec_info *vinfo,
stmt_vec_info stmt_info,
if (loop_vinfo && nested_in_vect_loop_p (loop, stmt_info))
return false;
+ /* FORNOW Disallow SIMD clones if BB SLP vectorization needs a partial
vector.
+ */
+ if (!loop_vinfo
+ && maybe_lt (SLP_TREE_LANES (slp_node), TYPE_VECTOR_SUBPARTS (vectype)))
+ return false;
+
/* Process function arguments. */
nargs = gimple_call_num_args (stmt) - masked_call_offset;
@@ -5716,17 +5765,29 @@ vectorizable_conversion (vec_info *vinfo,
}
if (modifier == WIDEN
- && loop_vinfo
- && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
- && (code1 == VEC_WIDEN_MULT_EVEN_EXPR
- || widening_evenodd_fn_p (code1)))
+ && (code1 == VEC_WIDEN_MULT_EVEN_EXPR || widening_evenodd_fn_p (code1))
+ && can_use_partial_vectors_p (vinfo, slp_node, nunits_in))
{
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "can't use a fully-masked loop because"
- " widening operation on even/odd elements"
- " mixes up lanes.\n");
- LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ if (loop_vinfo)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't use a fully-masked loop because"
+ " widening operation on even/odd elements"
+ " mixes up lanes.\n");
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ }
+ else
+ {
+ gcc_checking_assert (
+ maybe_lt (SLP_TREE_LANES (slp_node), nunits_out));
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "SLP can't use a mask or length because"
+ " widening operation on even/odd elements"
+ " mixes up lanes.\n");
+ return false;
+ }
}
if (cost_vec) /* transformation not required. */
@@ -6834,27 +6895,39 @@ vectorizable_operation (vec_info *vinfo,
bool mask_out_inactive = ((!is_invariant && gimple_could_trap_p (stmt))
|| reduc_idx >= 0);
+ bool use_partial_vectors_p
+ = can_use_partial_vectors_p (vinfo, slp_node, nunits_in);
+
if (cost_vec) /* transformation not required. */
{
- if (loop_vinfo
- && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
- && mask_out_inactive)
+ if (mask_out_inactive && use_partial_vectors_p)
{
if (cond_len_fn != IFN_LAST
&& direct_internal_fn_supported_p (cond_len_fn, vectype,
OPTIMIZE_FOR_SPEED))
- vect_record_len (loop_vinfo, vec_num, vectype, 1);
+ vect_record_len (vinfo, vec_num, vectype, 1, slp_node);
else if (cond_fn != IFN_LAST
&& direct_internal_fn_supported_p (cond_fn, vectype,
OPTIMIZE_FOR_SPEED))
- vect_record_mask (loop_vinfo, vec_num, vectype, NULL_TREE);
+ vect_record_mask (vinfo, vec_num, vectype, NULL_TREE, slp_node);
else
{
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "can't use a fully-masked loop because no"
- " conditional operation is available.\n");
- LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ if (loop_vinfo)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't use a fully-masked loop because no"
+ " conditional operation is available.\n");
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ }
+ else
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "SLP can't use a mask or length because no"
+ " conditional operation is available.\n");
+ return false;
+ }
}
}
@@ -6914,8 +6987,13 @@ vectorizable_operation (vec_info *vinfo,
dump_printf_loc (MSG_NOTE, vect_location,
"transform binary/unary operation.\n");
- bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
- bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
+ bool masked_loop_p
+ = use_partial_vectors_p
+ && (vinfo->partial_vector_style == vect_partial_vectors_while_ult
+ || vinfo->partial_vector_style == vect_partial_vectors_avx512);
+
+ bool len_loop_p = use_partial_vectors_p
+ && vinfo->partial_vector_style == vect_partial_vectors_len;
/* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
vectors with unsigned elements, but the result is signed. So, we
@@ -7107,7 +7185,7 @@ vectorizable_operation (vec_info *vinfo,
{
tree mask;
if (masked_loop_p)
- mask = vect_get_mask (loop_vinfo, gsi, vec_num, vectype, i);
+ mask = vect_get_mask (vinfo, gsi, vec_num, vectype, i, slp_node);
else
/* Dummy mask. */
mask = build_minus_one_cst (truth_type_for (vectype));
@@ -7133,10 +7211,9 @@ vectorizable_operation (vec_info *vinfo,
}
if (len_loop_p)
{
- tree len
- = vect_get_len (loop_vinfo, gsi, vec_num, vectype, i, 1, true);
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ tree len = vect_get_len (vinfo, gsi, vec_num, vectype, i, 1, true,
+ slp_node);
+ signed char biasval = vinfo->partial_load_store_bias;
tree bias = build_int_cst (intQI_type_node, biasval);
vops.quick_push (len);
vops.quick_push (bias);
@@ -7156,9 +7233,10 @@ vectorizable_operation (vec_info *vinfo,
tree mask = NULL_TREE;
/* When combining two masks check if either of them is elsewhere
combined with a loop mask, if that's the case we can mark that the
- new combined mask doesn't need to be combined with a loop mask. */
- if (masked_loop_p
- && code == BIT_AND_EXPR
+ new combined mask doesn't need to be combined with a loop mask.
+ Not applicable to BB SLP vectorization because two SLP nodes could
+ have the same OP0 but different group sizes or vector types. */
+ if (loop_vinfo && masked_loop_p && code == BIT_AND_EXPR
&& VECTOR_BOOLEAN_TYPE_P (vectype))
{
if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, vec_num
}))
@@ -8316,15 +8394,28 @@ vectorizable_store (vec_info *vinfo,
return false;
}
+ bool use_partial_vectors_p
+ = can_use_partial_vectors_p (vinfo, slp_node, nunits);
+
bool costing_p = cost_vec;
if (costing_p) /* transformation not required. */
{
- if (loop_vinfo
- && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
- && !check_load_store_for_partial_vectors (loop_vinfo, vectype,
- slp_node, vls_type,
- group_size, &ls, mask_node))
- LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ if (use_partial_vectors_p
+ && !check_load_store_for_partial_vectors (vinfo, vectype, slp_node,
+ vls_type, group_size, &ls,
+ mask_node))
+ {
+ if (loop_vinfo)
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ else
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (
+ MSG_MISSED_OPTIMIZATION, vect_location,
+ "SLP store needs but cannot use partial vectors.\n");
+ return false;
+ }
+ }
if (!vect_maybe_update_slp_op_vectype (vinfo, op_node, vectype)
|| (mask_node
@@ -8699,8 +8790,14 @@ vectorizable_store (vec_info *vinfo,
}
gcc_assert (alignment_support_scheme);
- bool masks_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
- bool lens_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
+
+ bool masks_p
+ = use_partial_vectors_p
+ && (vinfo->partial_vector_style == vect_partial_vectors_while_ult
+ || vinfo->partial_vector_style == vect_partial_vectors_avx512);
+
+ bool lens_p = use_partial_vectors_p
+ && vinfo->partial_vector_style == vect_partial_vectors_len;
/* The vect_transform_stmt and vect_analyze_stmt will go here but there
are some difference here. We cannot enable both the lens and masks
@@ -8855,19 +8952,18 @@ vectorizable_store (vec_info *vinfo,
tree bias = NULL;
if (masks_p)
final_mask
- = vect_get_mask (loop_vinfo, gsi, ncopies, vectype, j);
+ = vect_get_mask (vinfo, gsi, ncopies, vectype, j, slp_node);
if (vec_mask)
- final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
+ final_mask = prepare_vec_mask (vinfo, mask_vectype, final_mask,
vec_mask, gsi);
if (lanes_ifn == IFN_MASK_LEN_STORE_LANES)
{
if (lens_p)
- final_len = vect_get_len (loop_vinfo, gsi, ncopies, vectype, j,
- 1, true);
+ final_len = vect_get_len (vinfo, gsi, ncopies, vectype, j, 1,
+ true, slp_node);
else final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ signed char biasval = vinfo->partial_load_store_bias;
bias = build_int_cst (intQI_type_node, biasval);
if (!final_mask)
{
@@ -8996,10 +9092,10 @@ vectorizable_store (vec_info *vinfo,
{
if (masks_p)
final_mask
- = vect_get_mask (loop_vinfo, gsi, num_stmts, vectype, j);
+ = vect_get_mask (vinfo, gsi, num_stmts, vectype, j, slp_node);
if (vec_mask)
- final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
- final_mask, vec_mask, gsi);
+ final_mask = prepare_vec_mask (vinfo, mask_vectype, final_mask,
+ vec_mask, gsi);
}
unsigned align = get_object_alignment (DR_REF (first_dr_info->dr));
@@ -9063,13 +9159,12 @@ vectorizable_store (vec_info *vinfo,
if (ls.gs.ifn == IFN_MASK_LEN_SCATTER_STORE)
{
if (lens_p)
- final_len = vect_get_len (loop_vinfo, gsi, num_stmts,
- vectype, j, 1, true);
+ final_len = vect_get_len (vinfo, gsi, num_stmts, vectype, j,
+ 1, true, slp_node);
else
final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ signed char biasval = vinfo->partial_load_store_bias;
bias = build_int_cst (intQI_type_node, biasval);
if (!final_mask)
{
@@ -9411,12 +9506,12 @@ vectorizable_store (vec_info *vinfo,
tree final_len = NULL_TREE;
tree bias = NULL_TREE;
if (masks_p)
- final_mask = vect_get_mask (loop_vinfo, gsi, vec_num, vectype, i);
+ final_mask = vect_get_mask (vinfo, gsi, vec_num, vectype, i, slp_node);
if (vec_mask)
vec_mask = vec_masks[i];
if (vec_mask)
- final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
- vec_mask, gsi);
+ final_mask
+ = prepare_vec_mask (vinfo, mask_vectype, final_mask, vec_mask, gsi);
if (i > 0)
/* Bump the vector pointer. */
@@ -9451,8 +9546,8 @@ vectorizable_store (vec_info *vinfo,
new_vmode = new_ovmode.require ();
unsigned factor
= (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
- final_len
- = vect_get_len (loop_vinfo, gsi, vec_num, vectype, i, factor, true);
+ final_len = vect_get_len (vinfo, gsi, vec_num, vectype, i, factor,
+ true, slp_node);
}
else if (final_mask)
{
@@ -9480,7 +9575,7 @@ vectorizable_store (vec_info *vinfo,
}
if (final_len)
{
- signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ signed char biasval = vinfo->partial_load_store_bias;
bias = build_int_cst (intQI_type_node, biasval);
}
@@ -9961,17 +10056,28 @@ vectorizable_load (vec_info *vinfo,
}
}
- bool costing_p = cost_vec;
+ bool use_partial_vectors_p
+ = can_use_partial_vectors_p (vinfo, slp_node, nunits);
+ bool costing_p = cost_vec;
if (costing_p) /* transformation not required. */
{
- if (loop_vinfo
- && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
- && !check_load_store_for_partial_vectors (loop_vinfo, vectype,
- slp_node, VLS_LOAD,
- group_size, &ls, mask_node,
- &ls.elsvals))
- LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ if (use_partial_vectors_p
+ && !check_load_store_for_partial_vectors (vinfo, vectype, slp_node,
+ VLS_LOAD, group_size, &ls,
+ mask_node, &ls.elsvals))
+ {
+ if (loop_vinfo)
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ else
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (
+ MSG_MISSED_OPTIMIZATION, vect_location,
+ "SLP load needs but cannot use partial vectors.\n");
+ return false;
+ }
+ }
/* If the type needs padding we must zero inactive elements.
Check if we can do that with a VEC_COND_EXPR and store the
@@ -10600,8 +10706,13 @@ vectorizable_load (vec_info *vinfo,
ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr));
}
- bool masks_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
- bool lens_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
+ bool masks_p
+ = use_partial_vectors_p
+ && (vinfo->partial_vector_style == vect_partial_vectors_while_ult
+ || vinfo->partial_vector_style == vect_partial_vectors_avx512);
+
+ bool lens_p = use_partial_vectors_p
+ && vinfo->partial_vector_style == vect_partial_vectors_len;
/* The vect_transform_stmt and vect_analyze_stmt will go here but there
are some difference here. We cannot enable both the lens and masks
@@ -10823,20 +10934,19 @@ vectorizable_load (vec_info *vinfo,
tree bias = NULL_TREE;
if (masks_p)
final_mask
- = vect_get_mask (loop_vinfo, gsi, ncopies, vectype, j);
+ = vect_get_mask (vinfo, gsi, ncopies, vectype, j, slp_node);
if (vec_mask)
- final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
+ final_mask = prepare_vec_mask (vinfo, mask_vectype, final_mask,
vec_mask, gsi);
if (lanes_ifn == IFN_MASK_LEN_LOAD_LANES)
{
if (lens_p)
- final_len
- = vect_get_len (loop_vinfo, gsi, ncopies, vectype, j, 1,
true);
+ final_len = vect_get_len (vinfo, gsi, ncopies, vectype, j, 1,
+ true, slp_node);
else
final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ signed char biasval = vinfo->partial_load_store_bias;
bias = build_int_cst (intQI_type_node, biasval);
if (!final_mask)
{
@@ -10973,10 +11083,10 @@ vectorizable_load (vec_info *vinfo,
vec_mask = vec_masks[i];
if (masks_p)
final_mask
- = vect_get_mask (loop_vinfo, gsi, vec_num, vectype, i);
+ = vect_get_mask (vinfo, gsi, vec_num, vectype, i, slp_node);
if (vec_mask)
- final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
- final_mask, vec_mask, gsi);
+ final_mask = prepare_vec_mask (vinfo, mask_vectype, final_mask,
+ vec_mask, gsi);
if (i > 0 && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
@@ -11044,13 +11154,12 @@ vectorizable_load (vec_info *vinfo,
if (ls.gs.ifn == IFN_MASK_LEN_GATHER_LOAD)
{
if (lens_p)
- final_len = vect_get_len (loop_vinfo, gsi, vec_num, vectype,
- i, 1, true);
+ final_len = vect_get_len (vinfo, gsi, vec_num, vectype,
+ i, 1, true, slp_node);
else
final_len = build_int_cst (sizetype,
TYPE_VECTOR_SUBPARTS (vectype));
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ signed char biasval = vinfo->partial_load_store_bias;
bias = build_int_cst (intQI_type_node, biasval);
if (!final_mask)
{
@@ -11447,10 +11556,10 @@ vectorizable_load (vec_info *vinfo,
vec_mask = vec_masks[i];
if (masks_p)
final_mask
- = vect_get_mask (loop_vinfo, gsi, vec_num, vectype, i);
+ = vect_get_mask (vinfo, gsi, vec_num, vectype, i, slp_node);
if (vec_mask)
- final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
- final_mask, vec_mask, gsi);
+ final_mask = prepare_vec_mask (vinfo, mask_vectype, final_mask,
+ vec_mask, gsi);
if (i > 0)
dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
@@ -11495,8 +11604,8 @@ vectorizable_load (vec_info *vinfo,
new_vmode = new_ovmode.require ();
unsigned factor
= (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
- final_len = vect_get_len (loop_vinfo, gsi, vec_num, vectype, i,
- factor, true);
+ final_len = vect_get_len (vinfo, gsi, vec_num, vectype, i,
+ factor, true, slp_node);
}
else if (final_mask)
{
@@ -11525,8 +11634,7 @@ vectorizable_load (vec_info *vinfo,
}
if (final_len)
{
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ signed char biasval = vinfo->partial_load_store_bias;
bias = build_int_cst (intQI_type_node, biasval);
}
@@ -12312,6 +12420,10 @@ vectorizable_condition (vec_info *vinfo,
return false;
}
+ bool use_partial_vectors_p
+ = can_use_partial_vectors_p (vinfo, slp_node,
+ TYPE_VECTOR_SUBPARTS (vectype));
+
if (cost_vec)
{
if (bitop1 != NOP_EXPR)
@@ -12358,25 +12470,37 @@ vectorizable_condition (vec_info *vinfo,
return false;
}
- if (loop_vinfo && for_reduction
- && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
+ if (for_reduction && use_partial_vectors_p)
{
if (reduction_type == EXTRACT_LAST_REDUCTION)
{
if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
vectype, OPTIMIZE_FOR_SPEED))
- vect_record_len (loop_vinfo, vec_num, vectype, 1);
+ vect_record_len (vinfo, vec_num, vectype, 1, slp_node);
else
- vect_record_mask (loop_vinfo, vec_num, vectype, NULL_TREE);
+ vect_record_mask (vinfo, vec_num, vectype, NULL_TREE, slp_node);
}
/* Extra inactive lanes should be safe for vect_nested_cycle. */
else if (!nested_cycle_p)
{
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "conditional reduction prevents the use"
- " of partial vectors.\n");
- LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ if (loop_vinfo)
+ {
+ if (dump_enabled_p ()
+ && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "conditional reduction prevents the use"
+ " of partial vectors.\n");
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ }
+ else
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "SLP can't use a mask or length because"
+ " conditional reduction prevents the use"
+ " of partial vectors.\n");
+ return false;
+ }
}
}
@@ -12397,30 +12521,33 @@ vectorizable_condition (vec_info *vinfo,
/* See whether another part of the vectorized code applies a loop
mask to the condition, or to its inverse. */
- bool fully_masked_p = false;
- bool fully_with_length_p = false;
- if (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
+ bool masks_p = false;
+ bool lens_p = false;
+ if (use_partial_vectors_p
+ && vinfo->partial_vector_style == vect_partial_vectors_len)
{
if (reduction_type == EXTRACT_LAST_REDUCTION)
- fully_with_length_p = true;
+ lens_p = true;
}
- else if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
+ else if (use_partial_vectors_p
+ && (vinfo->partial_vector_style == vect_partial_vectors_while_ult
+ || vinfo->partial_vector_style == vect_partial_vectors_avx512))
{
if (reduction_type == EXTRACT_LAST_REDUCTION)
- fully_masked_p = true;
- else
+ masks_p = true;
+ else if (loop_vinfo)
{
scalar_cond_masked_key cond (cond_expr, 1);
if (loop_vinfo->scalar_cond_masked_set.contains (cond))
- fully_masked_p = true;
- else
+ masks_p = true;
+ else if (loop_vinfo)
{
bool honor_nans = HONOR_NANS (TREE_TYPE (cond.op0));
tree_code orig_code = cond.code;
cond.code = invert_tree_comparison (cond.code, honor_nans);
if (!masked && loop_vinfo->scalar_cond_masked_set.contains (cond))
{
- fully_masked_p = true;
+ masks_p = true;
cond_code = cond.code;
swap_cond_operands = true;
}
@@ -12433,7 +12560,7 @@ vectorizable_condition (vec_info *vinfo,
cond.code = orig_code;
if (loop_vinfo->scalar_cond_masked_set.contains (cond))
{
- fully_masked_p = true;
+ masks_p = true;
cond_code = cond.code;
swap_cond_operands = true;
must_invert_cmp_result = true;
@@ -12546,8 +12673,7 @@ vectorizable_condition (vec_info *vinfo,
in cases where that's necessary. */
tree len = NULL_TREE, bias = NULL_TREE;
- if (fully_masked_p || fully_with_length_p
- || reduction_type == EXTRACT_LAST_REDUCTION)
+ if (masks_p || lens_p || reduction_type == EXTRACT_LAST_REDUCTION)
{
if (!is_gimple_val (vec_compare))
{
@@ -12571,14 +12697,13 @@ vectorizable_condition (vec_info *vinfo,
if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
vectype, OPTIMIZE_FOR_SPEED))
{
- if (fully_with_length_p)
+ if (lens_p)
{
/* ??? Do we really want the adjusted LEN here? Isn't this
based on number of elements? */
- len = vect_get_len (loop_vinfo, gsi, vec_num, vectype, i, 1,
- true);
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ len = vect_get_len (vinfo, gsi, vec_num, vectype, i, 1,
+ true, slp_node);
+ signed char biasval = vinfo->partial_load_store_bias;
bias = build_int_cst (intQI_type_node, biasval);
}
else
@@ -12587,10 +12712,10 @@ vectorizable_condition (vec_info *vinfo,
bias = build_int_cst (intQI_type_node, 0);
}
}
- if (fully_masked_p)
+ if (masks_p)
{
tree loop_mask
- = vect_get_mask (loop_vinfo, gsi, vec_num, vectype, i);
+ = vect_get_mask (vinfo, gsi, vec_num, vectype, i, slp_node);
tree tmp2 = make_ssa_name (vec_cmp_type);
gassign *g
= gimple_build_assign (tmp2, BIT_AND_EXPR, vec_compare,
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 9eb4d9f1537..f0adaccb03f 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -2350,17 +2350,23 @@ vect_get_num_vectors (poly_uint64 nunits, tree vectype)
inline unsigned int
vect_get_num_copies (vec_info *vinfo, slp_tree node)
{
- poly_uint64 vf;
+ tree vectype = SLP_TREE_VECTYPE (node);
+ unsigned int group_size = SLP_TREE_LANES (node);
+ poly_uint64 nunits;
- if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
- vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo))
+ nunits = LOOP_VINFO_VECT_FACTOR (loop_vinfo) * group_size;
else
- vf = 1;
-
- vf *= SLP_TREE_LANES (node);
- tree vectype = SLP_TREE_VECTYPE (node);
+ {
+ /* The group size is not necessarily an integral multiple of the number
+ of subparts in the vector type if doing BB SLP vectorization with
+ predicated tails. */
+ if (known_ge (TYPE_VECTOR_SUBPARTS (vectype), group_size))
+ return 1;
+ nunits = group_size;
+ }
- return vect_get_num_vectors (vf, vectype);
+ return vect_get_num_vectors (nunits, vectype);
}
/* Update maximum unit count *MAX_NUNITS so that it accounts for
--
2.43.0
