On Tue, Jul 24, 2018 at 11:59 AM Richard Sandiford
<richard.sandif...@arm.com> wrote:
>
> This patch changes SLP_TREE_VEC_STMTS from a vec<gimple *> to a
> vec<stmt_vec_info>. This involved making the same change to the
> phis vector in vectorizable_reduction, since SLP_TREE_VEC_STMTS is
> spliced into it here:
>
> phis.splice (SLP_TREE_VEC_STMTS (slp_node_instance->reduc_phis));
OK, saw that coming - question from earlier patch still stands.
Richard.
>
> 2018-07-24 Richard Sandiford <richard.sandif...@arm.com>
>
> gcc/
> * tree-vectorizer.h (_slp_tree::vec_stmts): Change from a
> vec<gimple *> to a vec<stmt_vec_info>.
> * tree-vect-loop.c (vect_create_epilog_for_reduction): Change
> the reduction_phis argument from a vec<gimple *> to a
> vec<stmt_vec_info>.
> (vectorizable_reduction): Likewise the phis local variable that
> is passed to vect_create_epilog_for_reduction. Update for new type
> of SLP_TREE_VEC_STMTS.
> (vectorizable_induction): Update for new type of SLP_TREE_VEC_STMTS.
> (vectorizable_live_operation): Likewise.
> * tree-vect-slp.c (vect_get_slp_vect_defs): Likewise.
> (vect_transform_slp_perm_load, vect_schedule_slp_instance): Likewise.
>
> Index: gcc/tree-vectorizer.h
> ===================================================================
> --- gcc/tree-vectorizer.h 2018-07-24 10:22:47.489157307 +0100
> +++ gcc/tree-vectorizer.h 2018-07-24 10:22:50.777128110 +0100
> @@ -143,7 +143,7 @@ struct _slp_tree {
> permutation. */
> vec<unsigned> load_permutation;
> /* Vectorized stmt/s. */
> - vec<gimple *> vec_stmts;
> + vec<stmt_vec_info> vec_stmts;
> /* Number of vector stmts that are created to replace the group of scalar
> stmts. It is calculated during the transformation phase as the number of
> scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF
> Index: gcc/tree-vect-loop.c
> ===================================================================
> --- gcc/tree-vect-loop.c 2018-07-24 10:22:47.489157307 +0100
> +++ gcc/tree-vect-loop.c 2018-07-24 10:22:50.777128110 +0100
> @@ -4412,7 +4412,7 @@ get_initial_defs_for_reduction (slp_tree
> vect_create_epilog_for_reduction (vec<tree> vect_defs, gimple *stmt,
> gimple *reduc_def_stmt,
> int ncopies, internal_fn reduc_fn,
> - vec<gimple *> reduction_phis,
> + vec<stmt_vec_info> reduction_phis,
> bool double_reduc,
> slp_tree slp_node,
> slp_instance slp_node_instance,
> @@ -4429,6 +4429,7 @@ vect_create_epilog_for_reduction (vec<tr
> tree scalar_dest;
> tree scalar_type;
> gimple *new_phi = NULL, *phi;
> + stmt_vec_info phi_info;
> gimple_stmt_iterator exit_gsi;
> tree vec_dest;
> tree new_temp = NULL_TREE, new_dest, new_name, new_scalar_dest;
> @@ -4442,7 +4443,8 @@ vect_create_epilog_for_reduction (vec<tr
> tree orig_name, scalar_result;
> imm_use_iterator imm_iter, phi_imm_iter;
> use_operand_p use_p, phi_use_p;
> - gimple *use_stmt, *reduction_phi = NULL;
> + gimple *use_stmt;
> + stmt_vec_info reduction_phi_info = NULL;
> bool nested_in_vect_loop = false;
> auto_vec<gimple *> new_phis;
> auto_vec<stmt_vec_info> inner_phis;
> @@ -4540,7 +4542,7 @@ vect_create_epilog_for_reduction (vec<tr
> }
>
> /* Set phi nodes arguments. */
> - FOR_EACH_VEC_ELT (reduction_phis, i, phi)
> + FOR_EACH_VEC_ELT (reduction_phis, i, phi_info)
> {
> tree vec_init_def = vec_initial_defs[i];
> tree def = vect_defs[i];
> @@ -4548,7 +4550,7 @@ vect_create_epilog_for_reduction (vec<tr
> {
> if (j != 0)
> {
> - phi = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (phi));
> + phi_info = STMT_VINFO_RELATED_STMT (phi_info);
> if (nested_in_vect_loop)
> vec_init_def
> = vect_get_vec_def_for_stmt_copy (initial_def_dt,
> @@ -4557,6 +4559,7 @@ vect_create_epilog_for_reduction (vec<tr
>
> /* Set the loop-entry arg of the reduction-phi. */
>
> + gphi *phi = as_a <gphi *> (phi_info->stmt);
> if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
> == INTEGER_INDUC_COND_REDUCTION)
> {
> @@ -4569,19 +4572,18 @@ vect_create_epilog_for_reduction (vec<tr
> tree induc_val_vec
> = build_vector_from_val (vec_init_def_type, induc_val);
>
> - add_phi_arg (as_a <gphi *> (phi), induc_val_vec,
> - loop_preheader_edge (loop), UNKNOWN_LOCATION);
> + add_phi_arg (phi, induc_val_vec, loop_preheader_edge (loop),
> + UNKNOWN_LOCATION);
> }
> else
> - add_phi_arg (as_a <gphi *> (phi), vec_init_def,
> - loop_preheader_edge (loop), UNKNOWN_LOCATION);
> + add_phi_arg (phi, vec_init_def, loop_preheader_edge (loop),
> + UNKNOWN_LOCATION);
>
> /* Set the loop-latch arg for the reduction-phi. */
> if (j > 0)
> def = vect_get_vec_def_for_stmt_copy (vect_unknown_def_type,
> def);
>
> - add_phi_arg (as_a <gphi *> (phi), def, loop_latch_edge (loop),
> - UNKNOWN_LOCATION);
> + add_phi_arg (phi, def, loop_latch_edge (loop), UNKNOWN_LOCATION);
>
> if (dump_enabled_p ())
> {
> @@ -5599,7 +5601,7 @@ vect_create_epilog_for_reduction (vec<tr
> if (k % ratio == 0)
> {
> epilog_stmt = new_phis[k / ratio];
> - reduction_phi = reduction_phis[k / ratio];
> + reduction_phi_info = reduction_phis[k / ratio];
> if (double_reduc)
> inner_phi = inner_phis[k / ratio];
> }
> @@ -5672,7 +5674,6 @@ vect_create_epilog_for_reduction (vec<tr
> stmt_vec_info use_stmt_vinfo;
> tree vect_phi_init, preheader_arg, vect_phi_res;
> basic_block bb = gimple_bb (use_stmt);
> - gimple *use;
>
> /* Check that USE_STMT is really double reduction phi
> node. */
> @@ -5722,13 +5723,14 @@ vect_create_epilog_for_reduction (vec<tr
> /* Replace the use, i.e., set the correct vs1 in the
> regular
> reduction phi node. FORNOW, NCOPIES is always 1, so the
> loop is redundant. */
> - use = reduction_phi;
> - for (j = 0; j < ncopies; j++)
> - {
> - edge pr_edge = loop_preheader_edge (loop);
> - SET_PHI_ARG_DEF (use, pr_edge->dest_idx, vect_phi_res);
> - use = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (use));
> - }
> + stmt_vec_info use_info = reduction_phi_info;
> + for (j = 0; j < ncopies; j++)
> + {
> + edge pr_edge = loop_preheader_edge (loop);
> + SET_PHI_ARG_DEF (as_a <gphi *> (use_info->stmt),
> + pr_edge->dest_idx, vect_phi_res);
> + use_info = STMT_VINFO_RELATED_STMT (use_info);
> + }
> }
> }
> }
> @@ -6112,7 +6114,7 @@ vectorizable_reduction (gimple *stmt, gi
> auto_vec<tree> vec_oprnds1;
> auto_vec<tree> vec_oprnds2;
> auto_vec<tree> vect_defs;
> - auto_vec<gimple *> phis;
> + auto_vec<stmt_vec_info> phis;
> int vec_num;
> tree def0, tem;
> tree cr_index_scalar_type = NULL_TREE, cr_index_vector_type = NULL_TREE;
> @@ -6218,7 +6220,7 @@ vectorizable_reduction (gimple *stmt, gi
> stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi);
>
> if (slp_node)
> - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi);
> + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info);
> else
> {
> if (j == 0)
> @@ -7075,9 +7077,9 @@ vectorizable_reduction (gimple *stmt, gi
> if (code == COND_EXPR)
> {
> gcc_assert (!slp_node);
> - vectorizable_condition (stmt, gsi, vec_stmt,
> - PHI_RESULT (phis[0]),
> - reduc_index, NULL, NULL);
> + vectorizable_condition (stmt, gsi, vec_stmt,
> + PHI_RESULT (phis[0]->stmt),
> + reduc_index, NULL, NULL);
> /* Multiple types are not supported for condition. */
> break;
> }
> @@ -7501,7 +7503,8 @@ vectorizable_induction (gimple *phi,
> /* Create the induction-phi that defines the induction-operand. */
> vec_dest = vect_get_new_vect_var (vectype, vect_simple_var,
> "vec_iv_");
> induction_phi = create_phi_node (vec_dest, iv_loop->header);
> - loop_vinfo->add_stmt (induction_phi);
> + stmt_vec_info induction_phi_info
> + = loop_vinfo->add_stmt (induction_phi);
> induc_def = PHI_RESULT (induction_phi);
>
> /* Create the iv update inside the loop */
> @@ -7515,7 +7518,7 @@ vectorizable_induction (gimple *phi,
> add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
> UNKNOWN_LOCATION);
>
> - SLP_TREE_VEC_STMTS (slp_node).quick_push (induction_phi);
> + SLP_TREE_VEC_STMTS (slp_node).quick_push (induction_phi_info);
> }
>
> /* Re-use IVs when we can. */
> @@ -7540,7 +7543,7 @@ vectorizable_induction (gimple *phi,
> vec_step = vect_init_vector (phi, new_vec, vectype, NULL);
> for (; ivn < nvects; ++ivn)
> {
> - gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs];
> + gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs]->stmt;
> tree def;
> if (gimple_code (iv) == GIMPLE_PHI)
> def = gimple_phi_result (iv);
> @@ -7556,8 +7559,8 @@ vectorizable_induction (gimple *phi,
> gimple_stmt_iterator tgsi = gsi_for_stmt (iv);
> gsi_insert_after (&tgsi, new_stmt, GSI_CONTINUE_LINKING);
> }
> - loop_vinfo->add_stmt (new_stmt);
> - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
> + SLP_TREE_VEC_STMTS (slp_node).quick_push
> + (loop_vinfo->add_stmt (new_stmt));
> }
> }
>
> @@ -7943,7 +7946,7 @@ vectorizable_live_operation (gimple *stm
> gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
>
> /* Get the correct slp vectorized stmt. */
> - gimple *vec_stmt = SLP_TREE_VEC_STMTS (slp_node)[vec_entry];
> + gimple *vec_stmt = SLP_TREE_VEC_STMTS (slp_node)[vec_entry]->stmt;
> if (gphi *phi = dyn_cast <gphi *> (vec_stmt))
> vec_lhs = gimple_phi_result (phi);
> else
> Index: gcc/tree-vect-slp.c
> ===================================================================
> --- gcc/tree-vect-slp.c 2018-07-24 10:22:44.293185688 +0100
> +++ gcc/tree-vect-slp.c 2018-07-24 10:22:50.777128110 +0100
> @@ -3557,18 +3557,18 @@ vect_get_constant_vectors (tree op, slp_
> vect_get_slp_vect_defs (slp_tree slp_node, vec<tree> *vec_oprnds)
> {
> tree vec_oprnd;
> - gimple *vec_def_stmt;
> + stmt_vec_info vec_def_stmt_info;
> unsigned int i;
>
> gcc_assert (SLP_TREE_VEC_STMTS (slp_node).exists ());
>
> - FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (slp_node), i, vec_def_stmt)
> + FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (slp_node), i, vec_def_stmt_info)
> {
> - gcc_assert (vec_def_stmt);
> - if (gimple_code (vec_def_stmt) == GIMPLE_PHI)
> - vec_oprnd = gimple_phi_result (vec_def_stmt);
> + gcc_assert (vec_def_stmt_info);
> + if (gphi *vec_def_phi = dyn_cast <gphi *> (vec_def_stmt_info->stmt))
> + vec_oprnd = gimple_phi_result (vec_def_phi);
> else
> - vec_oprnd = gimple_get_lhs (vec_def_stmt);
> + vec_oprnd = gimple_get_lhs (vec_def_stmt_info->stmt);
> vec_oprnds->quick_push (vec_oprnd);
> }
> }
> @@ -3687,6 +3687,7 @@ vect_transform_slp_perm_load (slp_tree n
> {
> gimple *stmt = SLP_TREE_SCALAR_STMTS (node)[0];
> stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
> + vec_info *vinfo = stmt_info->vinfo;
> tree mask_element_type = NULL_TREE, mask_type;
> int vec_index = 0;
> tree vectype = STMT_VINFO_VECTYPE (stmt_info);
> @@ -3827,26 +3828,28 @@ vect_transform_slp_perm_load (slp_tree n
> /* Generate the permute statement if necessary. */
> tree first_vec = dr_chain[first_vec_index];
> tree second_vec = dr_chain[second_vec_index];
> - gimple *perm_stmt;
> + stmt_vec_info perm_stmt_info;
> if (! noop_p)
> {
> tree perm_dest
> = vect_create_destination_var (gimple_assign_lhs
> (stmt),
> vectype);
> perm_dest = make_ssa_name (perm_dest);
> - perm_stmt = gimple_build_assign (perm_dest,
> - VEC_PERM_EXPR,
> - first_vec, second_vec,
> - mask_vec);
> - vect_finish_stmt_generation (stmt, perm_stmt, gsi);
> + gassign *perm_stmt
> + = gimple_build_assign (perm_dest, VEC_PERM_EXPR,
> + first_vec, second_vec,
> + mask_vec);
> + perm_stmt_info
> + = vect_finish_stmt_generation (stmt, perm_stmt, gsi);
> }
> else
> /* If mask was NULL_TREE generate the requested
> identity transform. */
> - perm_stmt = SSA_NAME_DEF_STMT (first_vec);
> + perm_stmt_info = vinfo->lookup_def (first_vec);
>
> /* Store the vector statement in NODE. */
> - SLP_TREE_VEC_STMTS (node)[vect_stmts_counter++] = perm_stmt;
> + SLP_TREE_VEC_STMTS (node)[vect_stmts_counter++]
> + = perm_stmt_info;
> }
>
> index = 0;
> @@ -3948,8 +3951,8 @@ vect_schedule_slp_instance (slp_tree nod
> mask.quick_push (0);
> if (ocode != ERROR_MARK)
> {
> - vec<gimple *> v0;
> - vec<gimple *> v1;
> + vec<stmt_vec_info> v0;
> + vec<stmt_vec_info> v1;
> unsigned j;
> tree tmask = NULL_TREE;
> vect_transform_stmt (stmt, &si, &grouped_store, node, instance);
> @@ -3990,10 +3993,11 @@ vect_schedule_slp_instance (slp_tree nod
> gimple *vstmt;
> vstmt = gimple_build_assign (make_ssa_name (vectype),
> VEC_PERM_EXPR,
> - gimple_assign_lhs (v0[j]),
> - gimple_assign_lhs (v1[j]), tmask);
> - vect_finish_stmt_generation (stmt, vstmt, &si);
> - SLP_TREE_VEC_STMTS (node).quick_push (vstmt);
> + gimple_assign_lhs (v0[j]->stmt),
> + gimple_assign_lhs (v1[j]->stmt),
> + tmask);
> + SLP_TREE_VEC_STMTS (node).quick_push
> + (vect_finish_stmt_generation (stmt, vstmt, &si));
> }
> v0.release ();
> v1.release ();
