The following moves the incomplete validity check to use
WIDEN_MULT_{EVEN,ODD} to the caller of supportable_widening_operation
where we have access to more (but not enough) information. I have
made the test conservative enough I hope. For the testcase what was
broken is that it uses a SLP reduction where lane-swizzling isn't
valid.
Bootstrapped and tested on x86_64-unknown-linux-gnu, pushed.
PR tree-optimization/123002
* tree-vectorizer.h (supportable_widening_operation): Remove
vinfo and stmt_info parameters, add flag to indicate whether
the context would allow OP_{EVEN,ODD}.
* tree-vect-patterns.cc (vect_recog_abd_pattern): Adjust
and pass false.
(vect_recog_widen_op_pattern): Likewise.
(vect_recog_widen_abd_pattern): Likewise.
* tree-vect-stmts.cc (vectorizable_conversion): Move
even/odd validity check here, from supportable_widening_operation.
Adjust it to be conservative.
(supportable_widening_operation): Get flag whether even/odd
is OK to use and remove then unused parameters and code.
* gcc.dg/vect/vect-pr123002.c: New testcase.
---
gcc/testsuite/gcc.dg/vect/vect-pr123002.c | 39 +++++++++++++
gcc/tree-vect-patterns.cc | 12 ++--
gcc/tree-vect-stmts.cc | 70 ++++++++++++-----------
gcc/tree-vectorizer.h | 3 +-
4 files changed, 82 insertions(+), 42 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-pr123002.c
diff --git a/gcc/testsuite/gcc.dg/vect/vect-pr123002.c
b/gcc/testsuite/gcc.dg/vect/vect-pr123002.c
new file mode 100644
index 00000000000..9d9da280800
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-pr123002.c
@@ -0,0 +1,39 @@
+/* { dg-additional-options "-mavx2" { target avx2 } } */
+
+#include "tree-vect.h"
+
+static unsigned int const enc_table_32[8][3] = {
+ {513735U, 77223048U, 437087610U },
+ {0U, 78508U, 646269101U },
+ {0U, 0U, 11997U, },
+ {0U, 0U, 0U, },
+ {0U, 0U, 0U, },
+ {0U, 0U, 0U, },
+ {0U, 0U, 0U, },
+ {0U, 0U, 0U, }};
+
+int __attribute__((noipa)) foo()
+{
+ unsigned long intermediate[3] = {0};
+
+ for (unsigned long i = 0UL; i < 8; i++) {
+ intermediate[0] += 2 * (unsigned long)(enc_table_32)[i][0];
+ intermediate[1] += 2 * (unsigned long)(enc_table_32)[i][1];
+ intermediate[2] += 2 * (unsigned long)(enc_table_32)[i][2];
+ }
+
+ if (intermediate[0] == 0xfad8e &&
+ intermediate[1] == 0x9370e68 && intermediate[2] == 0x8125ca08) {
+ return 0;
+ } else {
+ return 1;
+ }
+}
+
+int main()
+{
+ check_vect ();
+ if (foo ())
+ __builtin_abort ();
+ return 0;
+}
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index db510e5f9b9..555986b4fec 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -1539,9 +1539,8 @@ vect_recog_abd_pattern (vec_info *vinfo,
int dummy_int;
auto_vec<tree> dummy_vec;
if (mid_vectype
- && supportable_widening_operation (vinfo, IFN_VEC_WIDEN_ABD,
- stmt_vinfo, mid_vectype,
- vectype_in,
+ && supportable_widening_operation (IFN_VEC_WIDEN_ABD,
+ mid_vectype, vectype_in, false,
&dummy_code, &dummy_code,
&dummy_int, &dummy_vec))
{
@@ -1661,8 +1660,7 @@ vect_recog_widen_op_pattern (vec_info *vinfo,
if (!vectype
|| !vecitype
|| !vecctype
- || !supportable_widening_operation (vinfo, wide_code, last_stmt_info,
- vecitype, vectype,
+ || !supportable_widening_operation (wide_code, vecitype, vectype, true,
&dummy_code, &dummy_code,
&dummy_int, &dummy_vec))
return NULL;
@@ -1762,8 +1760,8 @@ vect_recog_widen_abd_pattern (vec_info *vinfo,
stmt_vec_info stmt_vinfo,
code_helper dummy_code;
int dummy_int;
auto_vec<tree> dummy_vec;
- if (!supportable_widening_operation (vinfo, IFN_VEC_WIDEN_ABD, stmt_vinfo,
- vectype_out, vectype_in,
+ if (!supportable_widening_operation (IFN_VEC_WIDEN_ABD, vectype_out,
+ vectype_in, false,
&dummy_code, &dummy_code,
&dummy_int, &dummy_vec))
return NULL;
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 12eb5ea5b5e..dc155dcc7fe 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -5424,6 +5424,7 @@ vectorizable_conversion (vec_info *vinfo,
scalar_mode rhs_mode = SCALAR_TYPE_MODE (rhs_type);
opt_scalar_mode rhs_mode_iter;
auto_vec<std::pair<tree, tree_code>, 2> converts;
+ bool evenodd_ok = false;
/* Supportable by target? */
switch (modifier)
@@ -5471,10 +5472,28 @@ vectorizable_conversion (vec_info *vinfo,
gcc_assert (!(multi_step_cvt && op_type == binary_op));
break;
}
- if (supportable_widening_operation (vinfo, code, stmt_info,
- vectype_out, vectype_in, &code1,
- &code2, &multi_step_cvt,
- &interm_types))
+ /* Elements in a vector can only be reordered if used in a reduction
+ operation only. */
+ if (code == WIDEN_MULT_EXPR
+ && loop_vinfo
+ && !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info)
+ /* For a SLP reduction we cannot swizzle lanes, detecting a
+ reduction chain isn't possible here. */
+ && SLP_TREE_LANES (slp_node) == 1)
+ {
+ /* ??? There is no way to look for SLP uses, so work on
+ the stmt and what the stmt-based cycle detection gives us. */
+ tree lhs = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
+ stmt_vec_info use_stmt_info
+ = lhs ? loop_vinfo->lookup_single_use (lhs) : NULL;
+ if (use_stmt_info
+ && STMT_VINFO_REDUC_DEF (use_stmt_info))
+ evenodd_ok = true;
+ }
+ if (supportable_widening_operation (code, vectype_out, vectype_in,
+ evenodd_ok, &code1,
+ &code2, &multi_step_cvt,
+ &interm_types))
{
/* Binary widening operation can only be supported directly by the
architecture. */
@@ -5508,18 +5527,17 @@ vectorizable_conversion (vec_info *vinfo,
goto unsupported;
codecvt1 = tc1;
}
- else if (!supportable_widening_operation (vinfo, code,
- stmt_info, vectype_out,
- cvt_type, &codecvt1,
+ else if (!supportable_widening_operation (code, vectype_out,
+ cvt_type, evenodd_ok,
+ &codecvt1,
&codecvt2, &multi_step_cvt,
&interm_types))
continue;
else
gcc_assert (multi_step_cvt == 0);
- if (supportable_widening_operation (vinfo, NOP_EXPR, stmt_info,
- cvt_type,
- vectype_in, &code1,
+ if (supportable_widening_operation (NOP_EXPR, cvt_type,
+ vectype_in, evenodd_ok, &code1,
&code2, &multi_step_cvt,
&interm_types))
{
@@ -13829,6 +13847,8 @@ vect_maybe_update_slp_op_vectype (slp_tree op, tree
vectype)
are supported by the target platform either directly (via vector
tree-codes), or via target builtins.
+ When EVENODD_OK then also lane-swizzling operations are considered.
+
Output:
- CODE1 and CODE2 are codes of vector operations to be used when
vectorizing the operation, if available.
@@ -13839,17 +13859,14 @@ vect_maybe_update_slp_op_vectype (slp_tree op, tree
vectype)
widening operation (short in the above example). */
bool
-supportable_widening_operation (vec_info *vinfo,
- code_helper code,
- stmt_vec_info stmt_info,
+supportable_widening_operation (code_helper code,
tree vectype_out, tree vectype_in,
+ bool evenodd_ok,
code_helper *code1,
code_helper *code2,
int *multi_step_cvt,
vec<tree> *interm_types)
{
- loop_vec_info loop_info = dyn_cast <loop_vec_info> (vinfo);
- class loop *vect_loop = NULL;
machine_mode vec_mode;
enum insn_code icode1, icode2;
optab optab1 = unknown_optab, optab2 = unknown_optab;
@@ -13862,8 +13879,6 @@ supportable_widening_operation (vec_info *vinfo,
optab optab3, optab4;
*multi_step_cvt = 0;
- if (loop_info)
- vect_loop = LOOP_VINFO_LOOP (loop_info);
switch (code.safe_as_tree_code ())
{
@@ -13905,24 +13920,13 @@ supportable_widening_operation (vec_info *vinfo,
on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
are properly set up for the caller. If we fail, we'll continue with
a VEC_WIDEN_MULT_LO/HI_EXPR check. */
- if (vect_loop
- && !nested_in_vect_loop_p (vect_loop, stmt_info)
- && supportable_widening_operation (vinfo, VEC_WIDEN_MULT_EVEN_EXPR,
- stmt_info, vectype_out,
- vectype_in, code1,
+ if (evenodd_ok
+ && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR,
+ vectype_out, vectype_in,
+ evenodd_ok, code1,
code2, multi_step_cvt,
interm_types))
- {
- /* Elements in a vector with vect_used_by_reduction property cannot
- be reordered if the use chain with this property does not have the
- same operation. One such an example is s += a * b, where elements
- in a and b cannot be reordered. Here we check if the vector
defined
- by STMT is only directly used in the reduction statement. */
- tree lhs = gimple_assign_lhs (vect_orig_stmt (stmt_info)->stmt);
- stmt_vec_info use_stmt_info = loop_info->lookup_single_use (lhs);
- if (use_stmt_info && STMT_VINFO_REDUC_DEF (use_stmt_info))
- return true;
- }
+ return true;
c1 = VEC_WIDEN_MULT_LO_EXPR;
c2 = VEC_WIDEN_MULT_HI_EXPR;
break;
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index b5fbf58ce7a..d6e6b819fb2 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -2496,8 +2496,7 @@ extern bool vect_is_simple_use (vec_info *, slp_tree,
tree *, stmt_vec_info * = NULL);
extern bool vect_maybe_update_slp_op_vectype (slp_tree, tree);
extern tree perm_mask_for_reverse (tree);
-extern bool supportable_widening_operation (vec_info*, code_helper,
- stmt_vec_info, tree, tree,
+extern bool supportable_widening_operation (code_helper, tree, tree, bool,
code_helper*, code_helper*,
int*, vec<tree> *);
extern bool supportable_narrowing_operation (code_helper, tree, tree,
--
2.51.0
