diff --git a/gcc/testsuite/gcc.dg/vect/pr65947-1.c b/gcc/testsuite/gcc.dg/vect/pr65947-1.c index 7933f5c861201a7720ca77d84671898586516914..1e7a05dc8cc6f00fee265b9b8ad65beaa520f1e8 100644 --- a/gcc/testsuite/gcc.dg/vect/pr65947-1.c +++ b/gcc/testsuite/gcc.dg/vect/pr65947-1.c @@ -9,7 +9,7 @@ extern void abort (void) __attribute__ ((noreturn)); int condition_reduction (int *a, int min_v) { - int last = -1; + int last = 66; /* High start value. */ for (int i = 0; i < N; i++) if (a[i] < min_v) @@ -28,12 +28,13 @@ main (void) 31, 32 }; - int ret = condition_reduction (a, 16); + int ret = condition_reduction (a, 1); - if (ret != 19) + if (ret != 17) abort (); return 0; } /* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" { xfail { ! vect_max_reduc } } } } */ +/* { dg-final { scan-tree-dump-times "condition expression based on integer induction." 4 "vect" { xfail { ! vect_max_reduc } } } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr65947-10.c b/gcc/testsuite/gcc.dg/vect/pr65947-10.c index 9a43a6059fabc1ca527dd6305bf6482f2103fab4..b4c6659b77ccbbafd77c62976e408dda37a2f4c4 100644 --- a/gcc/testsuite/gcc.dg/vect/pr65947-10.c +++ b/gcc/testsuite/gcc.dg/vect/pr65947-10.c @@ -37,4 +37,5 @@ main (void) } /* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" { xfail { ! vect_max_reduc } } } } */ +/* { dg-final { scan-tree-dump-not "condition expression based on integer induction." "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr65947-12.c b/gcc/testsuite/gcc.dg/vect/pr65947-12.c new file mode 100644 index 0000000000000000000000000000000000000000..fb5ffd48c7bb87f6e2f9e055e90e4f121a202d5e --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/pr65947-12.c @@ -0,0 +1,41 @@ +/* { dg-require-effective-target vect_condition } */ + +extern void abort (void) __attribute__ ((noreturn)); + +#define N 32 + +/* Simple condition reduction where the result is a negative of the induction. + Will fail to vectorize to a simple case. */ + +signed int +condition_reduction (signed int *a, signed int min_v) +{ + signed int last = -1; + + for (signed int i = 0; i < N; i++) + if (a[i] < min_v) + last = -i; + + return last; +} + +int +main (void) +{ + signed int a[N] = { + 11, -12, 13, 14, 15, 16, 17, 18, 19, 20, + 1, 2, -3, 4, 5, 6, 7, -8, 9, 10, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, + 31, 32 + }; + + signed int ret = condition_reduction (a, 16); + + if (ret != -19) + abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" { xfail { ! vect_max_reduc } } } } */ +/* { dg-final { scan-tree-dump-not "condition expression based on integer induction." "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr65947-13.c b/gcc/testsuite/gcc.dg/vect/pr65947-13.c new file mode 100644 index 0000000000000000000000000000000000000000..8c6faddd189dabf9b96b4e665c59cf268cf9a3aa --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/pr65947-13.c @@ -0,0 +1,41 @@ +/* { dg-require-effective-target vect_condition } */ + +extern void abort (void) __attribute__ ((noreturn)); + +#define N 32 + +/* Simple condition reduction with a reversed loop. + Will fail to vectorize to a simple case. */ + +int +condition_reduction (int *a, int min_v) +{ + int last = -1; + + for (int i = N-1; i >=0; i--) + if (a[i] < min_v) + last = i; + + return last; +} + +int +main (void) +{ + int a[N] = { + 17, 28, 13, 14, 15, 16, 17, 18, 19, 20, + 1, 2, -3, 4, 5, 6, 7, -8, 9, 10, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, + 31, 32 + }; + + int ret = condition_reduction (a, 16); + + if (ret != 2) + abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" { xfail { ! vect_max_reduc } } } } */ +/* { dg-final { scan-tree-dump-not "condition expression based on integer induction." "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr65947-2.c b/gcc/testsuite/gcc.dg/vect/pr65947-2.c index 9c627d9d7d5c1c42f285a29a1a601b3bc60b8a5a..9e9ff53828609b19e6fe97ae2d640294cb980c14 100644 --- a/gcc/testsuite/gcc.dg/vect/pr65947-2.c +++ b/gcc/testsuite/gcc.dg/vect/pr65947-2.c @@ -38,3 +38,4 @@ main (void) } /* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" { xfail { ! vect_max_reduc } } } } */ +/* { dg-final { scan-tree-dump-not "condition expression based on integer induction." "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr65947-3.c b/gcc/testsuite/gcc.dg/vect/pr65947-3.c index e115de2a28259307e4eaa9cbbe2fa93663c22873..4b6aa9216b0c200c43a227bf8b4954ecbe4f0090 100644 --- a/gcc/testsuite/gcc.dg/vect/pr65947-3.c +++ b/gcc/testsuite/gcc.dg/vect/pr65947-3.c @@ -48,3 +48,4 @@ main (void) } /* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" { xfail { ! vect_max_reduc } } } } */ +/* { dg-final { scan-tree-dump-not "condition expression based on integer induction." "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr65947-4.c b/gcc/testsuite/gcc.dg/vect/pr65947-4.c index 76a0567aa5450e00de2d700123ffc930b4028486..f4e7fdc97c89866b19838593cb84fc1754f87494 100644 --- a/gcc/testsuite/gcc.dg/vect/pr65947-4.c +++ b/gcc/testsuite/gcc.dg/vect/pr65947-4.c @@ -37,4 +37,5 @@ main (void) } /* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" { xfail { ! vect_max_reduc } } } } */ +/* { dg-final { scan-tree-dump-times "condition expression based on integer induction." 4 "vect" { xfail { ! vect_max_reduc } } } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr65947-5.c b/gcc/testsuite/gcc.dg/vect/pr65947-5.c index 360e3b51ee15ecea3b5d590c93d524e143f0ffca..21be8d0b749a4c1f2f29b6aed17533ec26b7aede 100644 --- a/gcc/testsuite/gcc.dg/vect/pr65947-5.c +++ b/gcc/testsuite/gcc.dg/vect/pr65947-5.c @@ -39,3 +39,4 @@ main (void) /* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 1 "vect" { xfail { ! vect_max_reduc } } } } */ /* { dg-final { scan-tree-dump "loop size is greater than data size" "vect" { xfail { ! vect_max_reduc } } } } */ +/* { dg-final { scan-tree-dump-not "condition expression based on integer induction." "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr65947-6.c b/gcc/testsuite/gcc.dg/vect/pr65947-6.c index 4997ef79cae83585758436874ac109ca368a1fc1..e1432403b2dd227a7e13db6e228d819780b89ad6 100644 --- a/gcc/testsuite/gcc.dg/vect/pr65947-6.c +++ b/gcc/testsuite/gcc.dg/vect/pr65947-6.c @@ -37,3 +37,4 @@ main (void) } /* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" { xfail { ! vect_max_reduc } } } } */ +/* { dg-final { scan-tree-dump-not "condition expression based on integer induction." "vect" } } */ diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index 43ada18ac7cfebbff06f43b68639a69873ea5c65..86899209692bcea1431c372a4d9c86a17d326513 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -4052,7 +4052,7 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, tree bitsize; tree adjustment_def = NULL; tree vec_initial_def = NULL; - tree reduction_op, expr, def; + tree reduction_op, expr, def, initial_def = NULL; tree orig_name, scalar_result; imm_use_iterator imm_iter, phi_imm_iter; use_operand_p use_p, phi_use_p; @@ -4113,9 +4113,10 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, /* Get at the scalar def before the loop, that defines the initial value of the reduction variable. */ gimple *def_stmt = SSA_NAME_DEF_STMT (reduction_op); - tree op = PHI_ARG_DEF_FROM_EDGE (def_stmt, loop_preheader_edge (loop)); + initial_def = PHI_ARG_DEF_FROM_EDGE (def_stmt, + loop_preheader_edge (loop)); vec_initial_defs.create (1); - vec_initial_def = get_initial_def_for_reduction (stmt, op, + vec_initial_def = get_initial_def_for_reduction (stmt, initial_def, &adjustment_def); vec_initial_defs.quick_push (vec_initial_def); } @@ -4131,9 +4132,25 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, def = vect_defs[i]; for (j = 0; j < ncopies; j++) { - /* Set the loop-entry arg of the reduction-phi. */ - add_phi_arg (as_a (phi), vec_init_def, - loop_preheader_edge (loop), UNKNOWN_LOCATION); + /* Set the loop-entry arg of the reduction-phi. */ + + if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) + == INTEGER_INDUC_COND_REDUCTION) + { + /* Initialise the reduction phi to zero. This prevents initial + values of non-zero interferring with the reduction op. */ + gcc_assert (ncopies == 1); + gcc_assert (i == 0); + + tree vec_init_def_type = TREE_TYPE (vec_init_def); + tree zero_vec = build_zero_cst (vec_init_def_type); + + add_phi_arg (as_a (phi), zero_vec, + loop_preheader_edge (loop), UNKNOWN_LOCATION); + } + else + add_phi_arg (as_a (phi), vec_init_def, + loop_preheader_edge (loop), UNKNOWN_LOCATION); /* Set the loop-latch arg for the reduction-phi. */ if (j > 0) @@ -4475,10 +4492,28 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, } else tmp = build1 (reduc_code, scalar_type, new_phi_result); + epilog_stmt = gimple_build_assign (new_scalar_dest, tmp); new_temp = make_ssa_name (new_scalar_dest, epilog_stmt); gimple_assign_set_lhs (epilog_stmt, new_temp); gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT); + + if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) + == INTEGER_INDUC_COND_REDUCTION) + { + /* Earlier we set the initial value to be zero. Check the result + and if it is zero then replace with the original initial + value. */ + tree zero = build_zero_cst (scalar_type); + tree zcompare = build2 (EQ_EXPR, boolean_type_node, new_temp, zero); + + tmp = make_ssa_name (new_scalar_dest); + epilog_stmt = gimple_build_assign (tmp, COND_EXPR, zcompare, + initial_def, new_temp); + gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT); + new_temp = tmp; + } + scalar_results.safe_push (new_temp); } else @@ -4955,6 +4990,52 @@ vect_finalize_reduction: } +/* Function is_nonwrapping_integer_induction. + + Check if STMT (which is part of loop LOOP) both increments and + does not cause overflow. */ + +static bool +is_nonwrapping_integer_induction (gimple *stmt, struct loop *loop) +{ + stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt); + tree base = PHI_ARG_DEF_FROM_EDGE (stmt, loop_preheader_edge (loop)); + tree step = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo); + tree lhs_type = TREE_TYPE (gimple_phi_result (stmt)); + widest_int ni, max_loop_value, lhs_max; + bool overflow = false; + + /* Make sure the loop is integer based. */ + if (TREE_CODE (base) != INTEGER_CST + || TREE_CODE (step) != INTEGER_CST) + return false; + + /* Check that the induction increments. */ + if (tree_int_cst_sgn (step) == -1) + return false; + + /* Check that the max size of the loop will not wrap. */ + + if (TYPE_OVERFLOW_UNDEFINED (lhs_type)) + return true; + + if (! max_stmt_executions (loop, &ni)) + return false; + + max_loop_value = wi::mul (wi::to_widest (step), ni, TYPE_SIGN (lhs_type), + &overflow); + if (overflow) + return false; + + max_loop_value = wi::add (wi::to_widest (base), max_loop_value, + TYPE_SIGN (lhs_type), &overflow); + if (overflow) + return false; + + return (wi::min_precision (max_loop_value, TYPE_SIGN (lhs_type)) + <= TYPE_PRECISION (lhs_type)); +} + /* Function vectorizable_reduction. Check if STMT performs a reduction operation that can be vectorized. @@ -5053,6 +5134,7 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, tree def0, def1, tem, op0, op1 = NULL_TREE; bool first_p = true; tree cr_index_scalar_type = NULL_TREE, cr_index_vector_type = NULL_TREE; + bool cond_expr_is_nonwrapping_integer_induction = false; /* In case of reduction chain we switch to the first stmt in the chain, but we don't update STMT_INFO, since only the last stmt is marked as reduction @@ -5196,6 +5278,16 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, reduc_def_stmt = def_stmt; reduc_index = i; } + + if (i == 1 && code == COND_EXPR && dt == vect_induction_def + && is_nonwrapping_integer_induction (def_stmt, loop)) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "condition expression based on integer " + "induction.\n"); + cond_expr_is_nonwrapping_integer_induction = true; + } } is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, &def_stmt, &dt, &tem); @@ -5225,6 +5317,11 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, (loop_vinfo, reduc_def_stmt, !nested_cycle, &dummy, false, &STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)); + + if (cond_expr_is_nonwrapping_integer_induction + && STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == COND_REDUCTION) + STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) = INTEGER_INDUC_COND_REDUCTION; + if (orig_stmt) gcc_assert (tmp == orig_stmt || GROUP_FIRST_ELEMENT (vinfo_for_stmt (tmp)) == orig_stmt); @@ -5352,6 +5449,8 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, { /* This is a reduction pattern: get the vectype from the type of the reduction variable, and get the tree-code from orig_stmt. */ + gcc_assert (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) + == TREE_CODE_REDUCTION); orig_code = gimple_assign_rhs_code (orig_stmt); gcc_assert (vectype_out); vec_mode = TYPE_MODE (vectype_out); @@ -5361,6 +5460,12 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, /* Regular reduction: use the same vectype and tree-code as used for the vector code inside the loop can be used for the epilog code. */ orig_code = code; + + /* For simple condition reductions, replace with the actual expression + we want to base our reduction around. */ + if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) + == INTEGER_INDUC_COND_REDUCTION) + orig_code = MAX_EXPR; } if (nested_cycle) @@ -5381,7 +5486,9 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, epilog_reduc_code = ERROR_MARK; - if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == TREE_CODE_REDUCTION) + if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == TREE_CODE_REDUCTION + || STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) + == INTEGER_INDUC_COND_REDUCTION) { if (reduction_code_for_scalar_code (orig_code, &epilog_reduc_code)) { @@ -5407,6 +5514,19 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, epilog_reduc_code = ERROR_MARK; } } + + /* When epilog_reduc_code is ERROR_MARK then a reduction will be + generated in the epilog using multiple expressions. This does not + work for condition reductions. */ + if (epilog_reduc_code == ERROR_MARK + && STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) + == INTEGER_INDUC_COND_REDUCTION) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "no reduc code for scalar code.\n"); + return false; + } } else { @@ -5441,7 +5561,9 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, } if ((double_reduc - || STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == COND_REDUCTION) + || STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == COND_REDUCTION + || STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) + == INTEGER_INDUC_COND_REDUCTION) && ncopies > 1) { if (dump_enabled_p ()) diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index fed80c81c3f3bf38923ed6cd6ed4961129dadc40..620dc7116755585be15eaa599ade1eb25c37c07e 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -63,7 +63,8 @@ enum vect_def_type { /* Define type of reduction. */ enum vect_reduction_type { TREE_CODE_REDUCTION, - COND_REDUCTION + COND_REDUCTION, + INTEGER_INDUC_COND_REDUCTION }; #define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \