On Thu, Sep 14, 2017 at 1:22 PM, Richard Sandiford
<richard.sandif...@linaro.org> wrote:
> This patch adds a vectoriser helper routine to calculate how
> many copies of a vector statement we need. At present this
> is always:
>
> LOOP_VINFO_VECT_FACTOR (loop_vinfo) / TYPE_VECTOR_SUBPARTS (vectype)
>
> but later patches add other cases. Another benefit of using
> a helper routine is that it can assert that the division is
> exact (which it must be).
>
> Tested on aarch64-linux-gnu, x86_64-linux-gnu and powerpc64le-linux-gnu.
> OK to install?
Ok.
Richard.
> Richard
>
>
> 2017-09-14 Richard Sandiford <richard.sandif...@linaro.org>
> Alan Hayward <alan.hayw...@arm.com>
> David Sherwood <david.sherw...@arm.com>
>
> gcc/
> * tree-vectorizer.h (vect_get_num_copies): New function.
> * tree-vect-data-refs.c (vect_get_data_access_cost): Use it.
> * tree-vect-loop.c (vectorizable_reduction): Likewise.
> (vectorizable_induction): Likewise.
> (vectorizable_live_operation): Likewise.
> * tree-vect-stmts.c (vectorizable_mask_load_store): Likewise.
> (vectorizable_bswap): Likewise.
> (vectorizable_call): Likewise.
> (vectorizable_conversion): Likewise.
> (vectorizable_assignment): Likewise.
> (vectorizable_shift): Likewise.
> (vectorizable_operation): Likewise.
> (vectorizable_store): Likewise.
> (vectorizable_load): Likewise.
> (vectorizable_condition): Likewise.
> (vectorizable_comparison): Likewise.
> (vect_analyze_stmt): Pass the slp node to vectorizable_live_operation.
>
> Index: gcc/tree-vectorizer.h
> ===================================================================
> --- gcc/tree-vectorizer.h 2017-09-14 11:25:32.166167193 +0100
> +++ gcc/tree-vectorizer.h 2017-09-14 11:27:50.352072753 +0100
> @@ -1076,6 +1076,20 @@ unlimited_cost_model (loop_p loop)
> return (flag_vect_cost_model == VECT_COST_MODEL_UNLIMITED);
> }
>
> +/* Return the number of copies needed for loop vectorization when
> + a statement operates on vectors of type VECTYPE. This is the
> + vectorization factor divided by the number of elements in
> + VECTYPE and is always known at compile time. */
> +
> +static inline unsigned int
> +vect_get_num_copies (loop_vec_info loop_vinfo, tree vectype)
> +{
> + gcc_checking_assert (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
> + % TYPE_VECTOR_SUBPARTS (vectype) == 0);
> + return (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
> + / TYPE_VECTOR_SUBPARTS (vectype));
> +}
> +
> /* Source location */
> extern source_location vect_location;
>
> Index: gcc/tree-vect-data-refs.c
> ===================================================================
> --- gcc/tree-vect-data-refs.c 2017-09-14 11:25:32.163167193 +0100
> +++ gcc/tree-vect-data-refs.c 2017-09-14 11:27:50.350257085 +0100
> @@ -1181,10 +1181,13 @@ vect_get_data_access_cost (struct data_r
> {
> gimple *stmt = DR_STMT (dr);
> stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
> - int nunits = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info));
> loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
> - int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
> - int ncopies = MAX (1, vf / nunits); /* TODO: Handle SLP properly */
> + int ncopies;
> +
> + if (PURE_SLP_STMT (stmt_info))
> + ncopies = 1;
> + else
> + ncopies = vect_get_num_copies (loop_vinfo, STMT_VINFO_VECTYPE
> (stmt_info));
>
> if (DR_IS_READ (dr))
> vect_get_load_cost (dr, ncopies, true, inside_cost, outside_cost,
> Index: gcc/tree-vect-loop.c
> ===================================================================
> --- gcc/tree-vect-loop.c 2017-09-14 11:27:16.962234838 +0100
> +++ gcc/tree-vect-loop.c 2017-09-14 11:27:50.351164919 +0100
> @@ -5683,8 +5683,7 @@ vectorizable_reduction (gimple *stmt, gi
> if (slp_node)
> ncopies = 1;
> else
> - ncopies = (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
> - / TYPE_VECTOR_SUBPARTS (vectype_in));
> + ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
>
> use_operand_p use_p;
> gimple *use_stmt;
> @@ -5980,8 +5979,7 @@ vectorizable_reduction (gimple *stmt, gi
> if (slp_node)
> ncopies = 1;
> else
> - ncopies = (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
> - / TYPE_VECTOR_SUBPARTS (vectype_in));
> + ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
>
> gcc_assert (ncopies >= 1);
>
> @@ -6550,7 +6548,7 @@ vectorizable_induction (gimple *phi,
> if (slp_node)
> ncopies = 1;
> else
> - ncopies = vf / nunits;
> + ncopies = vect_get_num_copies (loop_vinfo, vectype);
> gcc_assert (ncopies >= 1);
>
> /* FORNOW. These restrictions should be relaxed. */
> @@ -7013,12 +7011,17 @@ vectorizable_live_operation (gimple *stm
> tree lhs, lhs_type, bitsize, vec_bitsize;
> tree vectype = STMT_VINFO_VECTYPE (stmt_info);
> int nunits = TYPE_VECTOR_SUBPARTS (vectype);
> - int ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
> + int ncopies;
> gimple *use_stmt;
> auto_vec<tree> vec_oprnds;
>
> gcc_assert (STMT_VINFO_LIVE_P (stmt_info));
>
> + if (slp_node)
> + ncopies = 1;
> + else
> + ncopies = vect_get_num_copies (loop_vinfo, vectype);
> +
> if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
> return false;
>
> Index: gcc/tree-vect-stmts.c
> ===================================================================
> --- gcc/tree-vect-stmts.c 2017-09-14 11:25:32.166167193 +0100
> +++ gcc/tree-vect-stmts.c 2017-09-14 11:27:50.352072753 +0100
> @@ -2038,7 +2038,7 @@ vectorizable_mask_load_store (gimple *st
> if (slp_node != NULL)
> return false;
>
> - ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
> + ncopies = vect_get_num_copies (loop_vinfo, vectype);
> gcc_assert (ncopies >= 1);
>
> mask = gimple_call_arg (stmt, 2);
> @@ -2472,7 +2472,7 @@ vectorizable_bswap (gimple *stmt, gimple
> if (slp_node)
> ncopies = 1;
> else
> - ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
> + ncopies = vect_get_num_copies (loop_vinfo, vectype);
>
> gcc_assert (ncopies >= 1);
>
> @@ -2805,9 +2805,9 @@ vectorizable_call (gimple *gs, gimple_st
> if (slp_node)
> ncopies = 1;
> else if (modifier == NARROW && ifn == IFN_LAST)
> - ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
> + ncopies = vect_get_num_copies (loop_vinfo, vectype_out);
> else
> - ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
> + ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
>
> /* Sanity check: make sure that at least one copy of the vectorized stmt
> needs to be generated. */
> @@ -4204,9 +4204,9 @@ vectorizable_conversion (gimple *stmt, g
> if (slp_node)
> ncopies = 1;
> else if (modifier == NARROW)
> - ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
> + ncopies = vect_get_num_copies (loop_vinfo, vectype_out);
> else
> - ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
> + ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
>
> /* Sanity check: make sure that at least one copy of the vectorized stmt
> needs to be generated. */
> @@ -4690,7 +4690,7 @@ vectorizable_assignment (gimple *stmt, g
> if (slp_node)
> ncopies = 1;
> else
> - ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
> + ncopies = vect_get_num_copies (loop_vinfo, vectype);
>
> gcc_assert (ncopies >= 1);
>
> @@ -4948,7 +4948,7 @@ vectorizable_shift (gimple *stmt, gimple
> if (slp_node)
> ncopies = 1;
> else
> - ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
> + ncopies = vect_get_num_copies (loop_vinfo, vectype);
>
> gcc_assert (ncopies >= 1);
>
> @@ -5379,7 +5379,7 @@ vectorizable_operation (gimple *stmt, gi
> if (slp_node)
> ncopies = 1;
> else
> - ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
> + ncopies = vect_get_num_copies (loop_vinfo, vectype);
>
> gcc_assert (ncopies >= 1);
>
> @@ -5718,7 +5718,7 @@ vectorizable_store (gimple *stmt, gimple
> if (slp)
> ncopies = 1;
> else
> - ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
> + ncopies = vect_get_num_copies (loop_vinfo, vectype);
>
> gcc_assert (ncopies >= 1);
>
> @@ -6713,7 +6713,7 @@ vectorizable_load (gimple *stmt, gimple_
> if (slp)
> ncopies = 1;
> else
> - ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
> + ncopies = vect_get_num_copies (loop_vinfo, vectype);
>
> gcc_assert (ncopies >= 1);
>
> @@ -7926,13 +7926,12 @@ vectorizable_condition (gimple *stmt, gi
> return false;
>
> tree vectype = STMT_VINFO_VECTYPE (stmt_info);
> - int nunits = TYPE_VECTOR_SUBPARTS (vectype);
> tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
>
> if (slp_node)
> ncopies = 1;
> else
> - ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
> + ncopies = vect_get_num_copies (loop_vinfo, vectype);
>
> gcc_assert (ncopies >= 1);
> if (reduc_index && ncopies > 1)
> @@ -8269,7 +8268,7 @@ vectorizable_comparison (gimple *stmt, g
> if (slp_node)
> ncopies = 1;
> else
> - ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
> + ncopies = vect_get_num_copies (loop_vinfo, vectype);
>
> gcc_assert (ncopies >= 1);
> if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
> @@ -8700,7 +8699,7 @@ vect_analyze_stmt (gimple *stmt, bool *n
> need extra handling, except for vectorizable reductions. */
> if (STMT_VINFO_LIVE_P (stmt_info)
> && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
> - ok = vectorizable_live_operation (stmt, NULL, NULL, -1, NULL);
> + ok = vectorizable_live_operation (stmt, NULL, node, -1, NULL);
>
> if (!ok)
> {
