We have to remove CLOBBERs before SLP is code generated since for store-lanes we are inserting our own CLOBBERs that we want to survive. So the following refactors vect_transform_loop to remove unwanted stmts first.
This resolves the gcc.target/aarch64/sve/store_lane_spill_1.c FAIL. Bootstrap and regtest running on x86_64-unknown-linux-gnu. PR tree-optimization/117277 * tree-vect-loop.cc (vect_transform_loop): Remove CLOBBERs and prefetches before doing any code generation. --- gcc/tree-vect-loop.cc | 110 ++++++++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 51 deletions(-) diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index 50a1531f4c3..322e50882ba 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -12515,6 +12515,25 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call) if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo)) move_early_exit_stmts (loop_vinfo); + /* Remove existing clobber stmts and prefetches. */ + for (i = 0; i < nbbs; i++) + { + basic_block bb = bbs[i]; + for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si);) + { + stmt = gsi_stmt (si); + if (gimple_clobber_p (stmt) + || gimple_call_builtin_p (stmt, BUILT_IN_PREFETCH)) + { + unlink_stmt_vdef (stmt); + gsi_remove (&si, true); + release_defs (stmt); + } + else + gsi_next (&si); + } + } + /* Schedule the SLP instances first, then handle loop vectorization below. */ if (!loop_vinfo->slp_instances.is_empty ()) @@ -12609,66 +12628,55 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call) !gsi_end_p (si);) { stmt = gsi_stmt (si); - /* During vectorization remove existing clobber stmts and - prefetches. */ - if (gimple_clobber_p (stmt) - || gimple_call_builtin_p (stmt, BUILT_IN_PREFETCH)) - { - unlink_stmt_vdef (stmt); - gsi_remove (&si, true); - release_defs (stmt); - } - else + + /* Ignore vector stmts created in the outer loop. */ + stmt_info = loop_vinfo->lookup_stmt (stmt); + + /* vector stmts created in the outer-loop during vectorization of + stmts in an inner-loop may not have a stmt_info, and do not + need to be vectorized. */ + stmt_vec_info seen_store = NULL; + if (stmt_info) { - /* Ignore vector stmts created in the outer loop. */ - stmt_info = loop_vinfo->lookup_stmt (stmt); - - /* vector stmts created in the outer-loop during vectorization of - stmts in an inner-loop may not have a stmt_info, and do not - need to be vectorized. */ - stmt_vec_info seen_store = NULL; - if (stmt_info) + if (STMT_VINFO_IN_PATTERN_P (stmt_info)) { - if (STMT_VINFO_IN_PATTERN_P (stmt_info)) + gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info); + for (gimple_stmt_iterator subsi = gsi_start (def_seq); + !gsi_end_p (subsi); gsi_next (&subsi)) { - gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info); - for (gimple_stmt_iterator subsi = gsi_start (def_seq); - !gsi_end_p (subsi); gsi_next (&subsi)) - { - stmt_vec_info pat_stmt_info - = loop_vinfo->lookup_stmt (gsi_stmt (subsi)); - vect_transform_loop_stmt (loop_vinfo, pat_stmt_info, - &si, &seen_store); - } stmt_vec_info pat_stmt_info - = STMT_VINFO_RELATED_STMT (stmt_info); - if (vect_transform_loop_stmt (loop_vinfo, pat_stmt_info, - &si, &seen_store)) - maybe_set_vectorized_backedge_value (loop_vinfo, - pat_stmt_info); - } - else - { - if (vect_transform_loop_stmt (loop_vinfo, stmt_info, &si, - &seen_store)) - maybe_set_vectorized_backedge_value (loop_vinfo, - stmt_info); + = loop_vinfo->lookup_stmt (gsi_stmt (subsi)); + vect_transform_loop_stmt (loop_vinfo, pat_stmt_info, + &si, &seen_store); } + stmt_vec_info pat_stmt_info + = STMT_VINFO_RELATED_STMT (stmt_info); + if (vect_transform_loop_stmt (loop_vinfo, pat_stmt_info, + &si, &seen_store)) + maybe_set_vectorized_backedge_value (loop_vinfo, + pat_stmt_info); } - gsi_next (&si); - if (seen_store) + else { - if (STMT_VINFO_GROUPED_ACCESS (seen_store)) - /* Interleaving. If IS_STORE is TRUE, the - vectorization of the interleaving chain was - completed - free all the stores in the chain. */ - vect_remove_stores (loop_vinfo, - DR_GROUP_FIRST_ELEMENT (seen_store)); - else - /* Free the attached stmt_vec_info and remove the stmt. */ - loop_vinfo->remove_stmt (stmt_info); + if (vect_transform_loop_stmt (loop_vinfo, stmt_info, &si, + &seen_store)) + maybe_set_vectorized_backedge_value (loop_vinfo, + stmt_info); } } + gsi_next (&si); + if (seen_store) + { + if (STMT_VINFO_GROUPED_ACCESS (seen_store)) + /* Interleaving. If IS_STORE is TRUE, the + vectorization of the interleaving chain was + completed - free all the stores in the chain. */ + vect_remove_stores (loop_vinfo, + DR_GROUP_FIRST_ELEMENT (seen_store)); + else + /* Free the attached stmt_vec_info and remove the stmt. */ + loop_vinfo->remove_stmt (stmt_info); + } } /* Stub out scalar statements that must not survive vectorization. -- 2.43.0