On Tue, Nov 16, 2021 at 5:24 PM Richard Sandiford
<richard.sandif...@arm.com> wrote:
>
> Richard Biener via Gcc-patches <gcc-patches@gcc.gnu.org> writes:
> > On Wed, Nov 10, 2021 at 1:48 PM Richard Sandiford via Gcc-patches
> > <gcc-patches@gcc.gnu.org> wrote:
> >>
> >> This patch extends the reduction code to handle calls. So far
> >> it's a structural change only; a later patch adds support for
> >> specific function reductions.
> >>
> >> Most of the patch consists of using code_helper and gimple_match_op
> >> to describe the reduction operations. The other main change is that
> >> vectorizable_call now needs to handle fully-predicated reductions.
> >>
> >> Tested on aarch64-linux-gnu and x86_64-linux-gnu. OK to install?
> >>
> >> Richard
> >>
> >>
> >> gcc/
> >> * builtins.h (associated_internal_fn): Declare overload that
> >> takes a (combined_cfn, return type) pair.
> >> * builtins.c (associated_internal_fn): Split new overload out
> >> of original fndecl version. Also provide an overload that takes
> >> a (combined_cfn, return type) pair.
> >> * internal-fn.h (commutative_binary_fn_p): Declare.
> >> (associative_binary_fn_p): Likewise.
> >> * internal-fn.c (commutative_binary_fn_p): New function,
> >> split out from...
> >> (first_commutative_argument): ...here.
> >> (associative_binary_fn_p): New function.
> >> * gimple-match.h (code_helper): Add a constructor that takes
> >> internal functions.
> >> (commutative_binary_op_p): Declare.
> >> (associative_binary_op_p): Likewise.
> >> (canonicalize_code): Likewise.
> >> (directly_supported_p): Likewise.
> >> (get_conditional_internal_fn): Likewise.
> >> (gimple_build): New overload that takes a code_helper.
> >> * gimple-fold.c (gimple_build): Likewise.
> >> * gimple-match-head.c (commutative_binary_op_p): New function.
> >> (associative_binary_op_p): Likewise.
> >> (canonicalize_code): Likewise.
> >> (directly_supported_p): Likewise.
> >> (get_conditional_internal_fn): Likewise.
> >> * tree-vectorizer.h: Include gimple-match.h.
> >> (neutral_op_for_reduction): Take a code_helper instead of a
> >> tree_code.
> >> (needs_fold_left_reduction_p): Likewise.
> >> (reduction_fn_for_scalar_code): Likewise.
> >> (vect_can_vectorize_without_simd_p): Declare a nNew overload that
> >> takes
> >> a code_helper.
> >> * tree-vect-loop.c: Include case-cfn-macros.h.
> >> (fold_left_reduction_fn): Take a code_helper instead of a
> >> tree_code.
> >> (reduction_fn_for_scalar_code): Likewise.
> >> (neutral_op_for_reduction): Likewise.
> >> (needs_fold_left_reduction_p): Likewise.
> >> (use_mask_by_cond_expr_p): Likewise.
> >> (build_vect_cond_expr): Likewise.
> >> (vect_create_partial_epilog): Likewise. Use gimple_build rather
> >> than gimple_build_assign.
> >> (check_reduction_path): Handle calls and operate on code_helpers
> >> rather than tree_codes.
> >> (vect_is_simple_reduction): Likewise.
> >> (vect_model_reduction_cost): Likewise.
> >> (vect_find_reusable_accumulator): Likewise.
> >> (vect_create_epilog_for_reduction): Likewise.
> >> (vect_transform_cycle_phi): Likewise.
> >> (vectorizable_reduction): Likewise. Make more use of
> >> lane_reduc_code_p.
> >> (vect_transform_reduction): Use gimple_extract_op but expect
> >> a tree_code for now.
> >> (vect_can_vectorize_without_simd_p): New overload that takes
> >> a code_helper.
> >> * tree-vect-stmts.c (vectorizable_call): Handle reductions in
> >> fully-masked loops.
> >> * tree-vect-patterns.c (vect_mark_pattern_stmts): Use
> >> gimple_extract_op when updating STMT_VINFO_REDUC_IDX.
> >> ---
> >> gcc/builtins.c | 46 ++++-
> >> gcc/builtins.h | 1 +
> >> gcc/gimple-fold.c | 9 +
> >> gcc/gimple-match-head.c | 70 +++++++
> >> gcc/gimple-match.h | 20 ++
> >> gcc/internal-fn.c | 46 ++++-
> >> gcc/internal-fn.h | 2 +
> >> gcc/tree-vect-loop.c | 420 +++++++++++++++++++--------------------
> >> gcc/tree-vect-patterns.c | 23 ++-
> >> gcc/tree-vect-stmts.c | 66 ++++--
> >> gcc/tree-vectorizer.h | 10 +-
> >> 11 files changed, 455 insertions(+), 258 deletions(-)
> >>
> >> diff --git a/gcc/builtins.c b/gcc/builtins.c
> >> index 384864bfb3a..03829c03a5a 100644
> >> --- a/gcc/builtins.c
> >> +++ b/gcc/builtins.c
> >> @@ -2139,17 +2139,17 @@ mathfn_built_in_type (combined_fn fn)
> >> #undef SEQ_OF_CASE_MATHFN
> >> }
> >>
> >> -/* If BUILT_IN_NORMAL function FNDECL has an associated internal function,
> >> - return its code, otherwise return IFN_LAST. Note that this function
> >> - only tests whether the function is defined in internals.def, not
> >> whether
> >> - it is actually available on the target. */
> >> +/* Check whether there is an internal function associated with function FN
> >> + and return type RETURN_TYPE. Return the function if so, otherwise
> >> return
> >> + IFN_LAST.
> >>
> >> -internal_fn
> >> -associated_internal_fn (tree fndecl)
> >> + Note that this function only tests whether the function is defined in
> >> + internals.def, not whether it is actually available on the target. */
> >> +
> >> +static internal_fn
> >> +associated_internal_fn (built_in_function fn, tree return_type)
> >> {
> >> - gcc_checking_assert (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL);
> >> - tree return_type = TREE_TYPE (TREE_TYPE (fndecl));
> >> - switch (DECL_FUNCTION_CODE (fndecl))
> >> + switch (fn)
> >> {
> >> #define DEF_INTERNAL_FLT_FN(NAME, FLAGS, OPTAB, TYPE) \
> >> CASE_FLT_FN (BUILT_IN_##NAME): return IFN_##NAME;
> >> @@ -2177,6 +2177,34 @@ associated_internal_fn (tree fndecl)
> >> }
> >> }
> >>
> >> +/* If BUILT_IN_NORMAL function FNDECL has an associated internal function,
> >> + return its code, otherwise return IFN_LAST. Note that this function
> >> + only tests whether the function is defined in internals.def, not
> >> whether
> >> + it is actually available on the target. */
> >> +
> >> +internal_fn
> >> +associated_internal_fn (tree fndecl)
> >> +{
> >> + gcc_checking_assert (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL);
> >> + return associated_internal_fn (DECL_FUNCTION_CODE (fndecl),
> >> + TREE_TYPE (TREE_TYPE (fndecl)));
> >> +}
> >> +
> >> +/* Check whether there is an internal function associated with function
> >> CFN
> >> + and return type RETURN_TYPE. Return the function if so, otherwise
> >> return
> >> + IFN_LAST.
> >> +
> >> + Note that this function only tests whether the function is defined in
> >> + internals.def, not whether it is actually available on the target. */
> >> +
> >> +internal_fn
> >> +associated_internal_fn (combined_fn cfn, tree return_type)
> >> +{
> >> + if (internal_fn_p (cfn))
> >> + return as_internal_fn (cfn);
> >> + return associated_internal_fn (as_builtin_fn (cfn), return_type);
> >> +}
> >> +
> >> /* If CALL is a call to a BUILT_IN_NORMAL function that could be replaced
> >> on the current target by a call to an internal function, return the
> >> code of that internal function, otherwise return IFN_LAST. The caller
> >> diff --git a/gcc/builtins.h b/gcc/builtins.h
> >> index 5e4d86e9c37..c99670b12f1 100644
> >> --- a/gcc/builtins.h
> >> +++ b/gcc/builtins.h
> >> @@ -148,6 +148,7 @@ extern char target_percent_s_newline[4];
> >> extern bool target_char_cst_p (tree t, char *p);
> >> extern rtx get_memory_rtx (tree exp, tree len);
> >>
> >> +extern internal_fn associated_internal_fn (combined_fn, tree);
> >> extern internal_fn associated_internal_fn (tree);
> >> extern internal_fn replacement_internal_fn (gcall *);
> >>
> >> diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c
> >> index 9daf2cc590c..a937f130815 100644
> >> --- a/gcc/gimple-fold.c
> >> +++ b/gcc/gimple-fold.c
> >> @@ -8808,6 +8808,15 @@ gimple_build (gimple_seq *seq, location_t loc,
> >> combined_fn fn,
> >> return res;
> >> }
> >
> > Toplevel comment missing. You add this for two operands, please
> > also add it for one and three (even if unused).
>
> OK. On the comment side: I was hoping to piggy-back on the comment
> for the previous overload :-) I'll add a new one though.
>
> >> +tree
> >> +gimple_build (gimple_seq *seq, location_t loc, code_helper code,
> >> + tree type, tree op0, tree op1)
> >> +{
> >> + if (code.is_tree_code ())
> >> + return gimple_build (seq, loc, tree_code (code), type, op0, op1);
> >> + return gimple_build (seq, loc, combined_fn (code), type, op0, op1);
> >> +}
> >> +
> >> /* Build the conversion (TYPE) OP with a result of type TYPE
> >> with location LOC if such conversion is neccesary in GIMPLE,
> >> simplifying it first.
> >> diff --git a/gcc/gimple-match-head.c b/gcc/gimple-match-head.c
> >> index d4d7d767075..4558a3db5fc 100644
> >> --- a/gcc/gimple-match-head.c
> >> +++ b/gcc/gimple-match-head.c
> >> @@ -1304,3 +1304,73 @@ optimize_successive_divisions_p (tree divisor, tree
> >> inner_div)
> >> }
> >> return true;
> >> }
> >> +
> >> +/* If CODE, operating on TYPE, represents a built-in function that has an
> >> + associated internal function, return the associated internal function,
> >> + otherwise return CODE. This function does not check whether the
> >> + internal function is supported, only that it exists. */
> >
> > Hmm, why not name the function associated_internal_fn then, or
> > have it contain internal_fn?
>
> I didn't want to call it associated_internal_fn because the existing
> forms of that function return an internal_fn. Here the idea is to
> avoid having multiple representations of the same operation, so that
> code_helpers can be compared for equality. I guess the fact that that
> currently means mapping built-in functions to internal functions
> (and nothing else) is more of an implementation detail.
>
> So I guess the emphasis in the comment is wrong. How about if I change
> it to:
>
> /* Return a canonical form for CODE when operating on TYPE. The idea
> is to remove redundant ways of representing the same operation so
> that code_helpers can be hashed and compared for equality.
>
> The only current canonicalization is to replace built-in functions
> with internal functions, in cases where internal-fn.def defines
> such an internal function.
>
> Note that the new code_helper cannot necessarily be used in place of
> the original code_helper. For example, the new code_helper might be
> an internal function that the target does not support. */
OK, that works for me.
> > I also wonder why all the functions below are not member functions
> > of code_helper?
>
> TBH I don't really like that style very much. :-) E.g. it's not obvious
> that directly_supported_p should be a member of code_helper when it's
> really querying information in the optab array. If code_helper was
> available more widely then the code would probably be in optab*.c
> instead.
>
> The queries are also about (code_helper, type) pairs rather than about
> code_helpers in isolation.
Hmm, true.
> IMO the current code_helper member functions seem like the right set,
> in that they're providing the abstraction “tree_code or combined_fn”.
> That abstraction can then be used in all sorts of places.
>
> >> +code_helper
> >> +canonicalize_code (code_helper code, tree type)
> >> +{
> >> + if (code.is_fn_code ())
> >> + return associated_internal_fn (combined_fn (code), type);
> >> + return code;
> >> +}
> >> +
> >> +/* Return true if CODE is a binary operation that is commutative when
> >> + operating on type TYPE. */
> >> +
> >> +bool
> >> +commutative_binary_op_p (code_helper code, tree type)
> >> +{
> >> + if (code.is_tree_code ())
> >> + return commutative_tree_code (tree_code (code));
> >> + auto cfn = combined_fn (code);
> >> + return commutative_binary_fn_p (associated_internal_fn (cfn, type));
> >> +}
> >
> > Do we need commutative_ternary_op_p? Can we do a more generic
> > commutative_p instead?
>
> How about using the first_commutative_argument interface from internal-fn.c,
> which returns the first argument in a commutative pair or -1 if none?
So I guess both might be useful dependent on the uses. Since you
were fine with commutative_binary_op_p that's good to add. I was merely
wondering about API completeness when we have commutative_ternary_tree_code
but not commutative_ternary_op_p (code_helper ...). So fine either way
(and also when adding an unused commutative_ternary_op_p)
> >> +
> >> +/* Return true if CODE is a binary operation that is associative when
> >> + operating on type TYPE. */
> >> +
> >> +bool
> >> +associative_binary_op_p (code_helper code, tree type)
> >
> > We only have associative_tree_code, is _binary relevant here?
>
> But we do have commutative_ternary_tree_code, like you say :-)
> I guess it didn't seem worth going back and renaming all the
> uses of commutative_tree_code to commutative_binary_tree_code
> to account for that.
>
> So this was partly future-proofing. It was also partly to emphasise
> that the caller doesn't need to check that the operator is a binary
> operator first (although I'm not sure the name actually achieves
> that, oh well).
>
OK, I see.
> >> +{
> >> + if (code.is_tree_code ())
> >> + return associative_tree_code (tree_code (code));
> >> + auto cfn = combined_fn (code);
> >> + return associative_binary_fn_p (associated_internal_fn (cfn, type));
> >> +}
> >> +
> >> +/* Return true if the target directly supports operation CODE on type
> >> TYPE.
> >> + QUERY_TYPE acts as for optab_for_tree_code. */
> >> +
> >> +bool
> >> +directly_supported_p (code_helper code, tree type, optab_subtype
> >> query_type)
> >> +{
> >> + if (code.is_tree_code ())
> >> + {
> >> + direct_optab optab = optab_for_tree_code (tree_code (code), type,
> >> + query_type);
> >> + return (optab != unknown_optab
> >> + && optab_handler (optab, TYPE_MODE (type)) !=
> >> CODE_FOR_nothing);
> >> + }
> >> + gcc_assert (query_type == optab_default
> >> + || (query_type == optab_vector && VECTOR_TYPE_P (type))
> >> + || (query_type == optab_scalar && !VECTOR_TYPE_P (type)));
> >> + internal_fn ifn = associated_internal_fn (combined_fn (code), type);
> >> + return (direct_internal_fn_p (ifn)
> >> + && direct_internal_fn_supported_p (ifn, type,
> >> OPTIMIZE_FOR_SPEED));
> >> +}
> >> +
> >> +/* A wrapper around the internal-fn.c versions of
> >> get_conditional_internal_fn
> >> + for a code_helper CODE operating on type TYPE. */
> >> +
> >> +internal_fn
> >> +get_conditional_internal_fn (code_helper code, tree type)
> >> +{
> >> + if (code.is_tree_code ())
> >> + return get_conditional_internal_fn (tree_code (code));
> >> + auto cfn = combined_fn (code);
> >> + return get_conditional_internal_fn (associated_internal_fn (cfn, type));
> >> +}
> >> diff --git a/gcc/gimple-match.h b/gcc/gimple-match.h
> >> index 1b9dc3851c2..6d24a8a2378 100644
> >> --- a/gcc/gimple-match.h
> >> +++ b/gcc/gimple-match.h
> >> @@ -31,6 +31,7 @@ public:
> >> code_helper () {}
> >> code_helper (tree_code code) : rep ((int) code) {}
> >> code_helper (combined_fn fn) : rep (-(int) fn) {}
> >> + code_helper (internal_fn fn) : rep (-(int) as_combined_fn (fn)) {}
> >> explicit operator tree_code () const { return (tree_code) rep; }
> >> explicit operator combined_fn () const { return (combined_fn) -rep; }
> >
> > Do we want a
> >
> > explicit operator internal_fn () const { ... }
> >
> > for completeness?
>
> Yeah, guess that would simplify some things. Maybe a built_in_function
> one too.
>
> >
> >> bool is_tree_code () const { return rep > 0; }
> >> @@ -346,4 +347,23 @@ tree maybe_push_res_to_seq (gimple_match_op *,
> >> gimple_seq *,
> >> void maybe_build_generic_op (gimple_match_op *);
> >>
> >>
> >> +bool commutative_binary_op_p (code_helper, tree);
> >> +bool associative_binary_op_p (code_helper, tree);
> >> +code_helper canonicalize_code (code_helper, tree);
> >> +
> >> +#ifdef GCC_OPTABS_TREE_H
> >> +bool directly_supported_p (code_helper, tree, optab_subtype =
> >> optab_default);
> >> +#endif
> >> +
> >> +internal_fn get_conditional_internal_fn (code_helper, tree);
> >> +
> >> +extern tree gimple_build (gimple_seq *, location_t,
> >> + code_helper, tree, tree, tree);
> >> +inline tree
> >> +gimple_build (gimple_seq *seq, code_helper code, tree type, tree op0,
> >> + tree op1)
> >> +{
> >> + return gimple_build (seq, UNKNOWN_LOCATION, code, type, op0, op1);
> >> +}
> >
> > That looks a bit misplaced and should be in gimple-fold.h, no?
>
> Files don't need to include gimple-match.h before gimple-fold.h.
> So I saw this as being a bit like the optab stuff: it's generalising
> interfaces provided elsewhere for the "tree_code or combined_fn" union.
>
> One alternative would be to put the functions in gimple-fold.h but
> protect them with #ifdef GCC_GIMPLE_MATCH_H. Another would be to
> move code_helper somewhere else, such as gimple.h.
Hmm, OK. Just leave it here then.
OK with the changes you suggested.
Thanks,
Richard.
> Thanks,
> Richard
>
> >> +
> >> #endif /* GCC_GIMPLE_MATCH_H */
> >> diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c
> >> index da7d8355214..7b13db6dfe3 100644
> >> --- a/gcc/internal-fn.c
> >> +++ b/gcc/internal-fn.c
> >> @@ -3815,6 +3815,43 @@ direct_internal_fn_supported_p (gcall *stmt,
> >> optimization_type opt_type)
> >> return direct_internal_fn_supported_p (fn, types, opt_type);
> >> }
> >>
> >> +/* Return true if FN is a commutative binary operation. */
> >> +
> >> +bool
> >> +commutative_binary_fn_p (internal_fn fn)
> >> +{
> >> + switch (fn)
> >> + {
> >> + case IFN_AVG_FLOOR:
> >> + case IFN_AVG_CEIL:
> >> + case IFN_MULH:
> >> + case IFN_MULHS:
> >> + case IFN_MULHRS:
> >> + case IFN_FMIN:
> >> + case IFN_FMAX:
> >> + return true;
> >> +
> >> + default:
> >> + return false;
> >> + }
> >> +}
> >> +
> >> +/* Return true if FN is an associative binary operation. */
> >> +
> >> +bool
> >> +associative_binary_fn_p (internal_fn fn)
> >
> > See above - without _binary?
> >
> >> +{
> >> + switch (fn)
> >> + {
> >> + case IFN_FMIN:
> >> + case IFN_FMAX:
> >> + return true;
> >> +
> >> + default:
> >> + return false;
> >> + }
> >> +}
> >> +
> >> /* If FN is commutative in two consecutive arguments, return the
> >> index of the first, otherwise return -1. */
> >>
> >> @@ -3827,13 +3864,6 @@ first_commutative_argument (internal_fn fn)
> >> case IFN_FMS:
> >> case IFN_FNMA:
> >> case IFN_FNMS:
> >> - case IFN_AVG_FLOOR:
> >> - case IFN_AVG_CEIL:
> >> - case IFN_MULH:
> >> - case IFN_MULHS:
> >> - case IFN_MULHRS:
> >> - case IFN_FMIN:
> >> - case IFN_FMAX:
> >> return 0;
> >>
> >> case IFN_COND_ADD:
> >> @@ -3852,7 +3882,7 @@ first_commutative_argument (internal_fn fn)
> >> return 1;
> >>
> >> default:
> >> - return -1;
> >> + return commutative_binary_fn_p (fn) ? 0 : -1;
> >> }
> >> }
> >>
> >> diff --git a/gcc/internal-fn.h b/gcc/internal-fn.h
> >> index 19d0f849a5a..82ef4b0d792 100644
> >> --- a/gcc/internal-fn.h
> >> +++ b/gcc/internal-fn.h
> >> @@ -206,6 +206,8 @@ direct_internal_fn_supported_p (internal_fn fn, tree
> >> type0, tree type1,
> >> opt_type);
> >> }
> >>
> >> +extern bool commutative_binary_fn_p (internal_fn);
> >
> > I'm somewhat missing commutative_ternary_fn_p which would work
> > on FMAs?
> >
> > So that was all API comments, the real changes below look good to me.
> >
> > Thanks,
> > Richard.
> >
> >> +extern bool associative_binary_fn_p (internal_fn);
> >> extern int first_commutative_argument (internal_fn);
> >>
> >> extern bool set_edom_supported_p (void);
> >> diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
> >> index 1cd5dbcb6f7..cae895a88f2 100644
> >> --- a/gcc/tree-vect-loop.c
> >> +++ b/gcc/tree-vect-loop.c
> >> @@ -54,6 +54,7 @@ along with GCC; see the file COPYING3. If not see
> >> #include "tree-vector-builder.h"
> >> #include "vec-perm-indices.h"
> >> #include "tree-eh.h"
> >> +#include "case-cfn-macros.h"
> >>
> >> /* Loop Vectorization Pass.
> >>
> >> @@ -3125,17 +3126,14 @@ vect_analyze_loop (class loop *loop,
> >> vec_info_shared *shared)
> >> it in *REDUC_FN if so. */
> >>
> >> static bool
> >> -fold_left_reduction_fn (tree_code code, internal_fn *reduc_fn)
> >> +fold_left_reduction_fn (code_helper code, internal_fn *reduc_fn)
> >> {
> >> - switch (code)
> >> + if (code == PLUS_EXPR)
> >> {
> >> - case PLUS_EXPR:
> >> *reduc_fn = IFN_FOLD_LEFT_PLUS;
> >> return true;
> >> -
> >> - default:
> >> - return false;
> >> }
> >> + return false;
> >> }
> >>
> >> /* Function reduction_fn_for_scalar_code
> >> @@ -3152,21 +3150,22 @@ fold_left_reduction_fn (tree_code code,
> >> internal_fn *reduc_fn)
> >> Return FALSE if CODE currently cannot be vectorized as reduction. */
> >>
> >> bool
> >> -reduction_fn_for_scalar_code (enum tree_code code, internal_fn *reduc_fn)
> >> +reduction_fn_for_scalar_code (code_helper code, internal_fn *reduc_fn)
> >> {
> >> - switch (code)
> >> - {
> >> + if (code.is_tree_code ())
> >> + switch (tree_code (code))
> >> + {
> >> case MAX_EXPR:
> >> - *reduc_fn = IFN_REDUC_MAX;
> >> - return true;
> >> + *reduc_fn = IFN_REDUC_MAX;
> >> + return true;
> >>
> >> case MIN_EXPR:
> >> - *reduc_fn = IFN_REDUC_MIN;
> >> - return true;
> >> + *reduc_fn = IFN_REDUC_MIN;
> >> + return true;
> >>
> >> case PLUS_EXPR:
> >> - *reduc_fn = IFN_REDUC_PLUS;
> >> - return true;
> >> + *reduc_fn = IFN_REDUC_PLUS;
> >> + return true;
> >>
> >> case BIT_AND_EXPR:
> >> *reduc_fn = IFN_REDUC_AND;
> >> @@ -3182,12 +3181,13 @@ reduction_fn_for_scalar_code (enum tree_code code,
> >> internal_fn *reduc_fn)
> >>
> >> case MULT_EXPR:
> >> case MINUS_EXPR:
> >> - *reduc_fn = IFN_LAST;
> >> - return true;
> >> + *reduc_fn = IFN_LAST;
> >> + return true;
> >>
> >> default:
> >> - return false;
> >> + break;
> >> }
> >> + return false;
> >> }
> >>
> >> /* If there is a neutral value X such that a reduction would not be
> >> affected
> >> @@ -3197,32 +3197,35 @@ reduction_fn_for_scalar_code (enum tree_code code,
> >> internal_fn *reduc_fn)
> >> then INITIAL_VALUE is that value, otherwise it is null. */
> >>
> >> tree
> >> -neutral_op_for_reduction (tree scalar_type, tree_code code, tree
> >> initial_value)
> >> +neutral_op_for_reduction (tree scalar_type, code_helper code,
> >> + tree initial_value)
> >> {
> >> - switch (code)
> >> - {
> >> - case WIDEN_SUM_EXPR:
> >> - case DOT_PROD_EXPR:
> >> - case SAD_EXPR:
> >> - case PLUS_EXPR:
> >> - case MINUS_EXPR:
> >> - case BIT_IOR_EXPR:
> >> - case BIT_XOR_EXPR:
> >> - return build_zero_cst (scalar_type);
> >> + if (code.is_tree_code ())
> >> + switch (tree_code (code))
> >> + {
> >> + case WIDEN_SUM_EXPR:
> >> + case DOT_PROD_EXPR:
> >> + case SAD_EXPR:
> >> + case PLUS_EXPR:
> >> + case MINUS_EXPR:
> >> + case BIT_IOR_EXPR:
> >> + case BIT_XOR_EXPR:
> >> + return build_zero_cst (scalar_type);
> >>
> >> - case MULT_EXPR:
> >> - return build_one_cst (scalar_type);
> >> + case MULT_EXPR:
> >> + return build_one_cst (scalar_type);
> >>
> >> - case BIT_AND_EXPR:
> >> - return build_all_ones_cst (scalar_type);
> >> + case BIT_AND_EXPR:
> >> + return build_all_ones_cst (scalar_type);
> >>
> >> - case MAX_EXPR:
> >> - case MIN_EXPR:
> >> - return initial_value;
> >> + case MAX_EXPR:
> >> + case MIN_EXPR:
> >> + return initial_value;
> >>
> >> - default:
> >> - return NULL_TREE;
> >> - }
> >> + default:
> >> + break;
> >> + }
> >> + return NULL_TREE;
> >> }
> >>
> >> /* Error reporting helper for vect_is_simple_reduction below. GIMPLE
> >> statement
> >> @@ -3239,26 +3242,27 @@ report_vect_op (dump_flags_t msg_type, gimple
> >> *stmt, const char *msg)
> >> overflow must wrap. */
> >>
> >> bool
> >> -needs_fold_left_reduction_p (tree type, tree_code code)
> >> +needs_fold_left_reduction_p (tree type, code_helper code)
> >> {
> >> /* CHECKME: check for !flag_finite_math_only too? */
> >> if (SCALAR_FLOAT_TYPE_P (type))
> >> - switch (code)
> >> - {
> >> - case MIN_EXPR:
> >> - case MAX_EXPR:
> >> - return false;
> >> + {
> >> + if (code.is_tree_code ())
> >> + switch (tree_code (code))
> >> + {
> >> + case MIN_EXPR:
> >> + case MAX_EXPR:
> >> + return false;
> >>
> >> - default:
> >> - return !flag_associative_math;
> >> - }
> >> + default:
> >> + break;
> >> + }
> >> + return !flag_associative_math;
> >> + }
> >>
> >> if (INTEGRAL_TYPE_P (type))
> >> - {
> >> - if (!operation_no_trapping_overflow (type, code))
> >> - return true;
> >> - return false;
> >> - }
> >> + return (!code.is_tree_code ()
> >> + || !operation_no_trapping_overflow (type, tree_code (code)));
> >>
> >> if (SAT_FIXED_POINT_TYPE_P (type))
> >> return true;
> >> @@ -3272,7 +3276,7 @@ needs_fold_left_reduction_p (tree type, tree_code
> >> code)
> >>
> >> static bool
> >> check_reduction_path (dump_user_location_t loc, loop_p loop, gphi *phi,
> >> - tree loop_arg, enum tree_code *code,
> >> + tree loop_arg, code_helper *code,
> >> vec<std::pair<ssa_op_iter, use_operand_p> > &path)
> >> {
> >> auto_bitmap visited;
> >> @@ -3347,45 +3351,57 @@ pop:
> >> for (unsigned i = 1; i < path.length (); ++i)
> >> {
> >> gimple *use_stmt = USE_STMT (path[i].second);
> >> - tree op = USE_FROM_PTR (path[i].second);
> >> - if (! is_gimple_assign (use_stmt)
> >> + gimple_match_op op;
> >> + if (!gimple_extract_op (use_stmt, &op))
> >> + {
> >> + fail = true;
> >> + break;
> >> + }
> >> + unsigned int opi = op.num_ops;
> >> + if (gassign *assign = dyn_cast<gassign *> (use_stmt))
> >> + {
> >> /* The following make sure we can compute the operand index
> >> easily plus it mostly disallows chaining via COND_EXPR
> >> condition
> >> operands. */
> >> - || (gimple_assign_rhs1_ptr (use_stmt) != path[i].second->use
> >> - && (gimple_num_ops (use_stmt) <= 2
> >> - || gimple_assign_rhs2_ptr (use_stmt) !=
> >> path[i].second->use)
> >> - && (gimple_num_ops (use_stmt) <= 3
> >> - || gimple_assign_rhs3_ptr (use_stmt) !=
> >> path[i].second->use)))
> >> + for (opi = 0; opi < op.num_ops; ++opi)
> >> + if (gimple_assign_rhs1_ptr (assign) + opi ==
> >> path[i].second->use)
> >> + break;
> >> + }
> >> + else if (gcall *call = dyn_cast<gcall *> (use_stmt))
> >> + {
> >> + for (opi = 0; opi < op.num_ops; ++opi)
> >> + if (gimple_call_arg_ptr (call, opi) == path[i].second->use)
> >> + break;
> >> + }
> >> + if (opi == op.num_ops)
> >> {
> >> fail = true;
> >> break;
> >> }
> >> - tree_code use_code = gimple_assign_rhs_code (use_stmt);
> >> - if (use_code == MINUS_EXPR)
> >> + op.code = canonicalize_code (op.code, op.type);
> >> + if (op.code == MINUS_EXPR)
> >> {
> >> - use_code = PLUS_EXPR;
> >> + op.code = PLUS_EXPR;
> >> /* Track whether we negate the reduction value each iteration.
> >> */
> >> - if (gimple_assign_rhs2 (use_stmt) == op)
> >> + if (op.ops[1] == op.ops[opi])
> >> neg = ! neg;
> >> }
> >> - if (CONVERT_EXPR_CODE_P (use_code)
> >> - && tree_nop_conversion_p (TREE_TYPE (gimple_assign_lhs
> >> (use_stmt)),
> >> - TREE_TYPE (gimple_assign_rhs1
> >> (use_stmt))))
> >> + if (CONVERT_EXPR_CODE_P (op.code)
> >> + && tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0])))
> >> ;
> >> else if (*code == ERROR_MARK)
> >> {
> >> - *code = use_code;
> >> - sign = TYPE_SIGN (TREE_TYPE (gimple_assign_lhs (use_stmt)));
> >> + *code = op.code;
> >> + sign = TYPE_SIGN (op.type);
> >> }
> >> - else if (use_code != *code)
> >> + else if (op.code != *code)
> >> {
> >> fail = true;
> >> break;
> >> }
> >> - else if ((use_code == MIN_EXPR
> >> - || use_code == MAX_EXPR)
> >> - && sign != TYPE_SIGN (TREE_TYPE (gimple_assign_lhs
> >> (use_stmt))))
> >> + else if ((op.code == MIN_EXPR
> >> + || op.code == MAX_EXPR)
> >> + && sign != TYPE_SIGN (op.type))
> >> {
> >> fail = true;
> >> break;
> >> @@ -3397,7 +3413,7 @@ pop:
> >> imm_use_iterator imm_iter;
> >> gimple *op_use_stmt;
> >> unsigned cnt = 0;
> >> - FOR_EACH_IMM_USE_STMT (op_use_stmt, imm_iter, op)
> >> + FOR_EACH_IMM_USE_STMT (op_use_stmt, imm_iter, op.ops[opi])
> >> if (!is_gimple_debug (op_use_stmt)
> >> && (*code != ERROR_MARK
> >> || flow_bb_inside_loop_p (loop, gimple_bb (op_use_stmt))))
> >> @@ -3427,7 +3443,7 @@ check_reduction_path (dump_user_location_t loc,
> >> loop_p loop, gphi *phi,
> >> tree loop_arg, enum tree_code code)
> >> {
> >> auto_vec<std::pair<ssa_op_iter, use_operand_p> > path;
> >> - enum tree_code code_;
> >> + code_helper code_;
> >> return (check_reduction_path (loc, loop, phi, loop_arg, &code_, path)
> >> && code_ == code);
> >> }
> >> @@ -3596,9 +3612,9 @@ vect_is_simple_reduction (loop_vec_info loop_info,
> >> stmt_vec_info phi_info,
> >> gimple *def1 = SSA_NAME_DEF_STMT (op1);
> >> if (gimple_bb (def1)
> >> && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
> >> - && loop->inner
> >> - && flow_bb_inside_loop_p (loop->inner, gimple_bb (def1))
> >> - && is_gimple_assign (def1)
> >> + && loop->inner
> >> + && flow_bb_inside_loop_p (loop->inner, gimple_bb (def1))
> >> + && (is_gimple_assign (def1) || is_gimple_call (def1))
> >> && is_a <gphi *> (phi_use_stmt)
> >> && flow_bb_inside_loop_p (loop->inner, gimple_bb (phi_use_stmt)))
> >> {
> >> @@ -3615,7 +3631,7 @@ vect_is_simple_reduction (loop_vec_info loop_info,
> >> stmt_vec_info phi_info,
> >>
> >> /* Look for the expression computing latch_def from then loop PHI
> >> result. */
> >> auto_vec<std::pair<ssa_op_iter, use_operand_p> > path;
> >> - enum tree_code code;
> >> + code_helper code;
> >> if (check_reduction_path (vect_location, loop, phi, latch_def, &code,
> >> path))
> >> {
> >> @@ -3633,15 +3649,24 @@ vect_is_simple_reduction (loop_vec_info loop_info,
> >> stmt_vec_info phi_info,
> >> {
> >> gimple *stmt = USE_STMT (path[i].second);
> >> stmt_vec_info stmt_info = loop_info->lookup_stmt (stmt);
> >> - STMT_VINFO_REDUC_IDX (stmt_info)
> >> - = path[i].second->use - gimple_assign_rhs1_ptr (stmt);
> >> - enum tree_code stmt_code = gimple_assign_rhs_code (stmt);
> >> - bool leading_conversion = (CONVERT_EXPR_CODE_P (stmt_code)
> >> + gimple_match_op op;
> >> + if (!gimple_extract_op (stmt, &op))
> >> + gcc_unreachable ();
> >> + if (gassign *assign = dyn_cast<gassign *> (stmt))
> >> + STMT_VINFO_REDUC_IDX (stmt_info)
> >> + = path[i].second->use - gimple_assign_rhs1_ptr (assign);
> >> + else
> >> + {
> >> + gcall *call = as_a<gcall *> (stmt);
> >> + STMT_VINFO_REDUC_IDX (stmt_info)
> >> + = path[i].second->use - gimple_call_arg_ptr (call, 0);
> >> + }
> >> + bool leading_conversion = (CONVERT_EXPR_CODE_P (op.code)
> >> && (i == 1 || i == path.length () -
> >> 1));
> >> - if ((stmt_code != code && !leading_conversion)
> >> + if ((op.code != code && !leading_conversion)
> >> /* We can only handle the final value in epilogue
> >> generation for reduction chains. */
> >> - || (i != 1 && !has_single_use (gimple_assign_lhs (stmt))))
> >> + || (i != 1 && !has_single_use (gimple_get_lhs (stmt))))
> >> is_slp_reduc = false;
> >> /* For reduction chains we support a trailing/leading
> >> conversions. We do not store those in the actual chain. */
> >> @@ -4390,8 +4415,6 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
> >> int ncopies, stmt_vector_for_cost *cost_vec)
> >> {
> >> int prologue_cost = 0, epilogue_cost = 0, inside_cost = 0;
> >> - enum tree_code code;
> >> - optab optab;
> >> tree vectype;
> >> machine_mode mode;
> >> class loop *loop = NULL;
> >> @@ -4407,7 +4430,9 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
> >> mode = TYPE_MODE (vectype);
> >> stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
> >>
> >> - code = gimple_assign_rhs_code (orig_stmt_info->stmt);
> >> + gimple_match_op op;
> >> + if (!gimple_extract_op (orig_stmt_info->stmt, &op))
> >> + gcc_unreachable ();
> >>
> >> if (reduction_type == EXTRACT_LAST_REDUCTION)
> >> /* No extra instructions are needed in the prologue. The loop body
> >> @@ -4501,20 +4526,16 @@ vect_model_reduction_cost (loop_vec_info
> >> loop_vinfo,
> >> else
> >> {
> >> int vec_size_in_bits = tree_to_uhwi (TYPE_SIZE (vectype));
> >> - tree bitsize =
> >> - TYPE_SIZE (TREE_TYPE (gimple_assign_lhs
> >> (orig_stmt_info->stmt)));
> >> + tree bitsize = TYPE_SIZE (op.type);
> >> int element_bitsize = tree_to_uhwi (bitsize);
> >> int nelements = vec_size_in_bits / element_bitsize;
> >>
> >> - if (code == COND_EXPR)
> >> - code = MAX_EXPR;
> >> -
> >> - optab = optab_for_tree_code (code, vectype, optab_default);
> >> + if (op.code == COND_EXPR)
> >> + op.code = MAX_EXPR;
> >>
> >> /* We have a whole vector shift available. */
> >> - if (optab != unknown_optab
> >> - && VECTOR_MODE_P (mode)
> >> - && optab_handler (optab, mode) != CODE_FOR_nothing
> >> + if (VECTOR_MODE_P (mode)
> >> + && directly_supported_p (op.code, vectype)
> >> && have_whole_vector_shift (mode))
> >> {
> >> /* Final reduction via vector shifts and the reduction
> >> operator.
> >> @@ -4855,7 +4876,7 @@ vect_find_reusable_accumulator (loop_vec_info
> >> loop_vinfo,
> >> initialize the accumulator with a neutral value instead. */
> >> if (!operand_equal_p (initial_value, main_adjustment))
> >> return false;
> >> - tree_code code = STMT_VINFO_REDUC_CODE (reduc_info);
> >> + code_helper code = STMT_VINFO_REDUC_CODE (reduc_info);
> >> initial_values[0] = neutral_op_for_reduction (TREE_TYPE
> >> (initial_value),
> >> code, initial_value);
> >> }
> >> @@ -4870,7 +4891,7 @@ vect_find_reusable_accumulator (loop_vec_info
> >> loop_vinfo,
> >> CODE emitting stmts before GSI. Returns a vector def of VECTYPE. */
> >>
> >> static tree
> >> -vect_create_partial_epilog (tree vec_def, tree vectype, enum tree_code
> >> code,
> >> +vect_create_partial_epilog (tree vec_def, tree vectype, code_helper code,
> >> gimple_seq *seq)
> >> {
> >> unsigned nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE
> >> (vec_def)).to_constant ();
> >> @@ -4953,9 +4974,7 @@ vect_create_partial_epilog (tree vec_def, tree
> >> vectype, enum tree_code code,
> >> gimple_seq_add_stmt_without_update (seq, epilog_stmt);
> >> }
> >>
> >> - new_temp = make_ssa_name (vectype1);
> >> - epilog_stmt = gimple_build_assign (new_temp, code, dst1, dst2);
> >> - gimple_seq_add_stmt_without_update (seq, epilog_stmt);
> >> + new_temp = gimple_build (seq, code, vectype1, dst1, dst2);
> >> }
> >>
> >> return new_temp;
> >> @@ -5032,7 +5051,7 @@ vect_create_epilog_for_reduction (loop_vec_info
> >> loop_vinfo,
> >> }
> >> gphi *reduc_def_stmt
> >> = as_a <gphi *> (STMT_VINFO_REDUC_DEF (vect_orig_stmt
> >> (stmt_info))->stmt);
> >> - enum tree_code code = STMT_VINFO_REDUC_CODE (reduc_info);
> >> + code_helper code = STMT_VINFO_REDUC_CODE (reduc_info);
> >> internal_fn reduc_fn = STMT_VINFO_REDUC_FN (reduc_info);
> >> tree vectype;
> >> machine_mode mode;
> >> @@ -5699,14 +5718,9 @@ vect_create_epilog_for_reduction (loop_vec_info
> >> loop_vinfo,
> >> tree vectype1 = get_related_vectype_for_scalar_type (TYPE_MODE
> >> (vectype),
> >> stype, nunits1);
> >> reduce_with_shift = have_whole_vector_shift (mode1);
> >> - if (!VECTOR_MODE_P (mode1))
> >> + if (!VECTOR_MODE_P (mode1)
> >> + || !directly_supported_p (code, vectype1))
> >> reduce_with_shift = false;
> >> - else
> >> - {
> >> - optab optab = optab_for_tree_code (code, vectype1,
> >> optab_default);
> >> - if (optab_handler (optab, mode1) == CODE_FOR_nothing)
> >> - reduce_with_shift = false;
> >> - }
> >>
> >> /* First reduce the vector to the desired vector size we should
> >> do shift reduction on by combining upper and lower halves. */
> >> @@ -5944,7 +5958,7 @@ vect_create_epilog_for_reduction (loop_vec_info
> >> loop_vinfo,
> >> for (k = 0; k < live_out_stmts.size (); k++)
> >> {
> >> stmt_vec_info scalar_stmt_info = vect_orig_stmt (live_out_stmts[k]);
> >> - scalar_dest = gimple_assign_lhs (scalar_stmt_info->stmt);
> >> + scalar_dest = gimple_get_lhs (scalar_stmt_info->stmt);
> >>
> >> phis.create (3);
> >> /* Find the loop-closed-use at the loop exit of the original scalar
> >> @@ -6277,7 +6291,7 @@ is_nonwrapping_integer_induction (stmt_vec_info
> >> stmt_vinfo, class loop *loop)
> >> CODE is the code for the operation. COND_FN is the conditional
> >> internal
> >> function, if it exists. VECTYPE_IN is the type of the vector input.
> >> */
> >> static bool
> >> -use_mask_by_cond_expr_p (enum tree_code code, internal_fn cond_fn,
> >> +use_mask_by_cond_expr_p (code_helper code, internal_fn cond_fn,
> >> tree vectype_in)
> >> {
> >> if (cond_fn != IFN_LAST
> >> @@ -6285,15 +6299,17 @@ use_mask_by_cond_expr_p (enum tree_code code,
> >> internal_fn cond_fn,
> >> OPTIMIZE_FOR_SPEED))
> >> return false;
> >>
> >> - switch (code)
> >> - {
> >> - case DOT_PROD_EXPR:
> >> - case SAD_EXPR:
> >> - return true;
> >> + if (code.is_tree_code ())
> >> + switch (tree_code (code))
> >> + {
> >> + case DOT_PROD_EXPR:
> >> + case SAD_EXPR:
> >> + return true;
> >>
> >> - default:
> >> - return false;
> >> - }
> >> + default:
> >> + break;
> >> + }
> >> + return false;
> >> }
> >>
> >> /* Insert a conditional expression to enable masked vectorization. CODE
> >> is the
> >> @@ -6301,10 +6317,10 @@ use_mask_by_cond_expr_p (enum tree_code code,
> >> internal_fn cond_fn,
> >> mask. GSI is a statement iterator used to place the new conditional
> >> expression. */
> >> static void
> >> -build_vect_cond_expr (enum tree_code code, tree vop[3], tree mask,
> >> +build_vect_cond_expr (code_helper code, tree vop[3], tree mask,
> >> gimple_stmt_iterator *gsi)
> >> {
> >> - switch (code)
> >> + switch (tree_code (code))
> >> {
> >> case DOT_PROD_EXPR:
> >> {
> >> @@ -6390,12 +6406,10 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >> slp_instance slp_node_instance,
> >> stmt_vector_for_cost *cost_vec)
> >> {
> >> - tree scalar_dest;
> >> tree vectype_in = NULL_TREE;
> >> class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
> >> enum vect_def_type cond_reduc_dt = vect_unknown_def_type;
> >> stmt_vec_info cond_stmt_vinfo = NULL;
> >> - tree scalar_type;
> >> int i;
> >> int ncopies;
> >> bool single_defuse_cycle = false;
> >> @@ -6508,18 +6522,18 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >> info_for_reduction to work. */
> >> if (STMT_VINFO_LIVE_P (vdef))
> >> STMT_VINFO_REDUC_DEF (def) = phi_info;
> >> - gassign *assign = dyn_cast <gassign *> (vdef->stmt);
> >> - if (!assign)
> >> + gimple_match_op op;
> >> + if (!gimple_extract_op (vdef->stmt, &op))
> >> {
> >> if (dump_enabled_p ())
> >> dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> >> - "reduction chain includes calls.\n");
> >> + "reduction chain includes unsupported"
> >> + " statement type.\n");
> >> return false;
> >> }
> >> - if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (assign)))
> >> + if (CONVERT_EXPR_CODE_P (op.code))
> >> {
> >> - if (!tree_nop_conversion_p (TREE_TYPE (gimple_assign_lhs
> >> (assign)),
> >> - TREE_TYPE (gimple_assign_rhs1
> >> (assign))))
> >> + if (!tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0])))
> >> {
> >> if (dump_enabled_p ())
> >> dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> >> @@ -6530,7 +6544,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >> else if (!stmt_info)
> >> /* First non-conversion stmt. */
> >> stmt_info = vdef;
> >> - reduc_def = gimple_op (vdef->stmt, 1 + STMT_VINFO_REDUC_IDX (vdef));
> >> + reduc_def = op.ops[STMT_VINFO_REDUC_IDX (vdef)];
> >> reduc_chain_length++;
> >> if (!stmt_info && slp_node)
> >> slp_for_stmt_info = SLP_TREE_CHILDREN (slp_for_stmt_info)[0];
> >> @@ -6588,26 +6602,24 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >>
> >> tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
> >> STMT_VINFO_REDUC_VECTYPE (reduc_info) = vectype_out;
> >> - gassign *stmt = as_a <gassign *> (stmt_info->stmt);
> >> - enum tree_code code = gimple_assign_rhs_code (stmt);
> >> - bool lane_reduc_code_p
> >> - = (code == DOT_PROD_EXPR || code == WIDEN_SUM_EXPR || code ==
> >> SAD_EXPR);
> >> - int op_type = TREE_CODE_LENGTH (code);
> >> + gimple_match_op op;
> >> + if (!gimple_extract_op (stmt_info->stmt, &op))
> >> + gcc_unreachable ();
> >> + bool lane_reduc_code_p = (op.code == DOT_PROD_EXPR
> >> + || op.code == WIDEN_SUM_EXPR
> >> + || op.code == SAD_EXPR);
> >> enum optab_subtype optab_query_kind = optab_vector;
> >> - if (code == DOT_PROD_EXPR
> >> - && TYPE_SIGN (TREE_TYPE (gimple_assign_rhs1 (stmt)))
> >> - != TYPE_SIGN (TREE_TYPE (gimple_assign_rhs2 (stmt))))
> >> + if (op.code == DOT_PROD_EXPR
> >> + && (TYPE_SIGN (TREE_TYPE (op.ops[0]))
> >> + != TYPE_SIGN (TREE_TYPE (op.ops[1]))))
> >> optab_query_kind = optab_vector_mixed_sign;
> >>
> >> -
> >> - scalar_dest = gimple_assign_lhs (stmt);
> >> - scalar_type = TREE_TYPE (scalar_dest);
> >> - if (!POINTER_TYPE_P (scalar_type) && !INTEGRAL_TYPE_P (scalar_type)
> >> - && !SCALAR_FLOAT_TYPE_P (scalar_type))
> >> + if (!POINTER_TYPE_P (op.type) && !INTEGRAL_TYPE_P (op.type)
> >> + && !SCALAR_FLOAT_TYPE_P (op.type))
> >> return false;
> >>
> >> /* Do not try to vectorize bit-precision reductions. */
> >> - if (!type_has_mode_precision_p (scalar_type))
> >> + if (!type_has_mode_precision_p (op.type))
> >> return false;
> >>
> >> /* For lane-reducing ops we're reducing the number of reduction PHIs
> >> @@ -6626,25 +6638,23 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >> The last use is the reduction variable. In case of nested cycle this
> >> assumption is not true: we use reduc_index to record the index of the
> >> reduction variable. */
> >> - slp_tree *slp_op = XALLOCAVEC (slp_tree, op_type);
> >> + slp_tree *slp_op = XALLOCAVEC (slp_tree, op.num_ops);
> >> /* We need to skip an extra operand for COND_EXPRs with embedded
> >> comparison. */
> >> unsigned opno_adjust = 0;
> >> - if (code == COND_EXPR
> >> - && COMPARISON_CLASS_P (gimple_assign_rhs1 (stmt)))
> >> + if (op.code == COND_EXPR && COMPARISON_CLASS_P (op.ops[0]))
> >> opno_adjust = 1;
> >> - for (i = 0; i < op_type; i++)
> >> + for (i = 0; i < (int) op.num_ops; i++)
> >> {
> >> /* The condition of COND_EXPR is checked in
> >> vectorizable_condition(). */
> >> - if (i == 0 && code == COND_EXPR)
> >> + if (i == 0 && op.code == COND_EXPR)
> >> continue;
> >>
> >> stmt_vec_info def_stmt_info;
> >> enum vect_def_type dt;
> >> - tree op;
> >> if (!vect_is_simple_use (loop_vinfo, stmt_info, slp_for_stmt_info,
> >> - i + opno_adjust, &op, &slp_op[i], &dt, &tem,
> >> - &def_stmt_info))
> >> + i + opno_adjust, &op.ops[i], &slp_op[i],
> >> &dt,
> >> + &tem, &def_stmt_info))
> >> {
> >> if (dump_enabled_p ())
> >> dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> >> @@ -6669,13 +6679,13 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >> < GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (tem))))))
> >> vectype_in = tem;
> >>
> >> - if (code == COND_EXPR)
> >> + if (op.code == COND_EXPR)
> >> {
> >> /* Record how the non-reduction-def value of COND_EXPR is
> >> defined. */
> >> if (dt == vect_constant_def)
> >> {
> >> cond_reduc_dt = dt;
> >> - cond_reduc_val = op;
> >> + cond_reduc_val = op.ops[i];
> >> }
> >> if (dt == vect_induction_def
> >> && def_stmt_info
> >> @@ -6845,7 +6855,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >> (and also the same tree-code) when generating the epilog code
> >> and
> >> when generating the code inside the loop. */
> >>
> >> - enum tree_code orig_code = STMT_VINFO_REDUC_CODE (phi_info);
> >> + code_helper orig_code = STMT_VINFO_REDUC_CODE (phi_info);
> >> STMT_VINFO_REDUC_CODE (reduc_info) = orig_code;
> >>
> >> vect_reduction_type reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info);
> >> @@ -6864,7 +6874,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >> && !REDUC_GROUP_FIRST_ELEMENT (stmt_info)
> >> && known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1u))
> >> ;
> >> - else if (needs_fold_left_reduction_p (scalar_type, orig_code))
> >> + else if (needs_fold_left_reduction_p (op.type, orig_code))
> >> {
> >> /* When vectorizing a reduction chain w/o SLP the reduction PHI
> >> is not directy used in stmt. */
> >> @@ -6879,8 +6889,8 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >> STMT_VINFO_REDUC_TYPE (reduc_info)
> >> = reduction_type = FOLD_LEFT_REDUCTION;
> >> }
> >> - else if (!commutative_tree_code (orig_code)
> >> - || !associative_tree_code (orig_code))
> >> + else if (!commutative_binary_op_p (orig_code, op.type)
> >> + || !associative_binary_op_p (orig_code, op.type))
> >> {
> >> if (dump_enabled_p ())
> >> dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> >> @@ -6935,7 +6945,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >> else if (reduction_type == COND_REDUCTION)
> >> {
> >> int scalar_precision
> >> - = GET_MODE_PRECISION (SCALAR_TYPE_MODE (scalar_type));
> >> + = GET_MODE_PRECISION (SCALAR_TYPE_MODE (op.type));
> >> cr_index_scalar_type = make_unsigned_type (scalar_precision);
> >> cr_index_vector_type = get_same_sized_vectype (cr_index_scalar_type,
> >> vectype_out);
> >> @@ -7121,28 +7131,19 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >>
> >> if (single_defuse_cycle || lane_reduc_code_p)
> >> {
> >> - gcc_assert (code != COND_EXPR);
> >> + gcc_assert (op.code != COND_EXPR);
> >>
> >> /* 4. Supportable by target? */
> >> bool ok = true;
> >>
> >> /* 4.1. check support for the operation in the loop */
> >> - optab optab = optab_for_tree_code (code, vectype_in,
> >> optab_query_kind);
> >> - if (!optab)
> >> - {
> >> - if (dump_enabled_p ())
> >> - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> >> - "no optab.\n");
> >> - ok = false;
> >> - }
> >> -
> >> machine_mode vec_mode = TYPE_MODE (vectype_in);
> >> - if (ok && optab_handler (optab, vec_mode) == CODE_FOR_nothing)
> >> + if (!directly_supported_p (op.code, vectype_in, optab_query_kind))
> >> {
> >> if (dump_enabled_p ())
> >> dump_printf (MSG_NOTE, "op not supported by target.\n");
> >> if (maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD)
> >> - || !vect_can_vectorize_without_simd_p (code))
> >> + || !vect_can_vectorize_without_simd_p (op.code))
> >> ok = false;
> >> else
> >> if (dump_enabled_p ())
> >> @@ -7150,7 +7151,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >> }
> >>
> >> if (vect_emulated_vector_p (vectype_in)
> >> - && !vect_can_vectorize_without_simd_p (code))
> >> + && !vect_can_vectorize_without_simd_p (op.code))
> >> {
> >> if (dump_enabled_p ())
> >> dump_printf (MSG_NOTE, "using word mode not possible.\n");
> >> @@ -7183,11 +7184,9 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >>
> >> if (slp_node
> >> && !(!single_defuse_cycle
> >> - && code != DOT_PROD_EXPR
> >> - && code != WIDEN_SUM_EXPR
> >> - && code != SAD_EXPR
> >> + && !lane_reduc_code_p
> >> && reduction_type != FOLD_LEFT_REDUCTION))
> >> - for (i = 0; i < op_type; i++)
> >> + for (i = 0; i < (int) op.num_ops; i++)
> >> if (!vect_maybe_update_slp_op_vectype (slp_op[i], vectype_in))
> >> {
> >> if (dump_enabled_p ())
> >> @@ -7206,10 +7205,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >> /* Cost the reduction op inside the loop if transformed via
> >> vect_transform_reduction. Otherwise this is costed by the
> >> separate vectorizable_* routines. */
> >> - if (single_defuse_cycle
> >> - || code == DOT_PROD_EXPR
> >> - || code == WIDEN_SUM_EXPR
> >> - || code == SAD_EXPR)
> >> + if (single_defuse_cycle || lane_reduc_code_p)
> >> record_stmt_cost (cost_vec, ncopies, vector_stmt, stmt_info, 0,
> >> vect_body);
> >>
> >> if (dump_enabled_p ()
> >> @@ -7220,9 +7216,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >> /* All but single defuse-cycle optimized, lane-reducing and fold-left
> >> reductions go through their own vectorizable_* routines. */
> >> if (!single_defuse_cycle
> >> - && code != DOT_PROD_EXPR
> >> - && code != WIDEN_SUM_EXPR
> >> - && code != SAD_EXPR
> >> + && !lane_reduc_code_p
> >> && reduction_type != FOLD_LEFT_REDUCTION)
> >> {
> >> stmt_vec_info tem
> >> @@ -7238,10 +7232,10 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
> >> else if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P
> >> (loop_vinfo))
> >> {
> >> vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
> >> - internal_fn cond_fn = get_conditional_internal_fn (code);
> >> + internal_fn cond_fn = get_conditional_internal_fn (op.code,
> >> op.type);
> >>
> >> if (reduction_type != FOLD_LEFT_REDUCTION
> >> - && !use_mask_by_cond_expr_p (code, cond_fn, vectype_in)
> >> + && !use_mask_by_cond_expr_p (op.code, cond_fn, vectype_in)
> >> && (cond_fn == IFN_LAST
> >> || !direct_internal_fn_supported_p (cond_fn, vectype_in,
> >> OPTIMIZE_FOR_SPEED)))
> >> @@ -7294,24 +7288,11 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
> >> gcc_assert (STMT_VINFO_DEF_TYPE (reduc_info) ==
> >> vect_double_reduction_def);
> >> }
> >>
> >> - gassign *stmt = as_a <gassign *> (stmt_info->stmt);
> >> - enum tree_code code = gimple_assign_rhs_code (stmt);
> >> - int op_type = TREE_CODE_LENGTH (code);
> >> -
> >> - /* Flatten RHS. */
> >> - tree ops[3];
> >> - switch (get_gimple_rhs_class (code))
> >> - {
> >> - case GIMPLE_TERNARY_RHS:
> >> - ops[2] = gimple_assign_rhs3 (stmt);
> >> - /* Fall thru. */
> >> - case GIMPLE_BINARY_RHS:
> >> - ops[0] = gimple_assign_rhs1 (stmt);
> >> - ops[1] = gimple_assign_rhs2 (stmt);
> >> - break;
> >> - default:
> >> - gcc_unreachable ();
> >> - }
> >> + gimple_match_op op;
> >> + if (!gimple_extract_op (stmt_info->stmt, &op))
> >> + gcc_unreachable ();
> >> + gcc_assert (op.code.is_tree_code ());
> >> + auto code = tree_code (op.code);
> >>
> >> /* All uses but the last are expected to be defined in the loop.
> >> The last use is the reduction variable. In case of nested cycle this
> >> @@ -7359,7 +7340,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
> >> internal_fn reduc_fn = STMT_VINFO_REDUC_FN (reduc_info);
> >> return vectorize_fold_left_reduction
> >> (loop_vinfo, stmt_info, gsi, vec_stmt, slp_node, reduc_def_phi,
> >> code,
> >> - reduc_fn, ops, vectype_in, reduc_index, masks);
> >> + reduc_fn, op.ops, vectype_in, reduc_index, masks);
> >> }
> >>
> >> bool single_defuse_cycle = STMT_VINFO_FORCE_SINGLE_CYCLE (reduc_info);
> >> @@ -7369,22 +7350,22 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
> >> || code == SAD_EXPR);
> >>
> >> /* Create the destination vector */
> >> - tree scalar_dest = gimple_assign_lhs (stmt);
> >> + tree scalar_dest = gimple_assign_lhs (stmt_info->stmt);
> >> tree vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
> >>
> >> vect_get_vec_defs (loop_vinfo, stmt_info, slp_node, ncopies,
> >> single_defuse_cycle && reduc_index == 0
> >> - ? NULL_TREE : ops[0], &vec_oprnds0,
> >> + ? NULL_TREE : op.ops[0], &vec_oprnds0,
> >> single_defuse_cycle && reduc_index == 1
> >> - ? NULL_TREE : ops[1], &vec_oprnds1,
> >> - op_type == ternary_op
> >> + ? NULL_TREE : op.ops[1], &vec_oprnds1,
> >> + op.num_ops == 3
> >> && !(single_defuse_cycle && reduc_index == 2)
> >> - ? ops[2] : NULL_TREE, &vec_oprnds2);
> >> + ? op.ops[2] : NULL_TREE, &vec_oprnds2);
> >> if (single_defuse_cycle)
> >> {
> >> gcc_assert (!slp_node);
> >> vect_get_vec_defs_for_operand (loop_vinfo, stmt_info, 1,
> >> - ops[reduc_index],
> >> + op.ops[reduc_index],
> >> reduc_index == 0 ? &vec_oprnds0
> >> : (reduc_index == 1 ? &vec_oprnds1
> >> : &vec_oprnds2));
> >> @@ -7414,7 +7395,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
> >> }
> >> else
> >> {
> >> - if (op_type == ternary_op)
> >> + if (op.num_ops == 3)
> >> vop[2] = vec_oprnds2[i];
> >>
> >> if (masked_loop_p && mask_by_cond_expr)
> >> @@ -7546,7 +7527,7 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
> >> {
> >> tree initial_value
> >> = (num_phis == 1 ? initial_values[0] : NULL_TREE);
> >> - tree_code code = STMT_VINFO_REDUC_CODE (reduc_info);
> >> + code_helper code = STMT_VINFO_REDUC_CODE (reduc_info);
> >> tree neutral_op
> >> = neutral_op_for_reduction (TREE_TYPE (vectype_out),
> >> code, initial_value);
> >> @@ -7603,7 +7584,7 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
> >> if (!reduc_info->reduc_initial_values.is_empty ())
> >> {
> >> initial_def = reduc_info->reduc_initial_values[0];
> >> - enum tree_code code = STMT_VINFO_REDUC_CODE (reduc_info);
> >> + code_helper code = STMT_VINFO_REDUC_CODE (reduc_info);
> >> tree neutral_op
> >> = neutral_op_for_reduction (TREE_TYPE (initial_def),
> >> code, initial_def);
> >> @@ -7901,6 +7882,15 @@ vect_can_vectorize_without_simd_p (tree_code code)
> >> }
> >> }
> >>
> >> +/* Likewise, but taking a code_helper. */
> >> +
> >> +bool
> >> +vect_can_vectorize_without_simd_p (code_helper code)
> >> +{
> >> + return (code.is_tree_code ()
> >> + && vect_can_vectorize_without_simd_p (tree_code (code)));
> >> +}
> >> +
> >> /* Function vectorizable_induction
> >>
> >> Check if STMT_INFO performs an induction computation that can be
> >> vectorized.
> >> diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
> >> index 854cbcff390..26421ee5511 100644
> >> --- a/gcc/tree-vect-patterns.c
> >> +++ b/gcc/tree-vect-patterns.c
> >> @@ -5594,8 +5594,10 @@ vect_mark_pattern_stmts (vec_info *vinfo,
> >> /* Transfer reduction path info to the pattern. */
> >> if (STMT_VINFO_REDUC_IDX (orig_stmt_info_saved) != -1)
> >> {
> >> - tree lookfor = gimple_op (orig_stmt_info_saved->stmt,
> >> - 1 + STMT_VINFO_REDUC_IDX (orig_stmt_info));
> >> + gimple_match_op op;
> >> + if (!gimple_extract_op (orig_stmt_info_saved->stmt, &op))
> >> + gcc_unreachable ();
> >> + tree lookfor = op.ops[STMT_VINFO_REDUC_IDX (orig_stmt_info)];
> >> /* Search the pattern def sequence and the main pattern stmt. Note
> >> we may have inserted all into a containing pattern def sequence
> >> so the following is a bit awkward. */
> >> @@ -5615,14 +5617,15 @@ vect_mark_pattern_stmts (vec_info *vinfo,
> >> do
> >> {
> >> bool found = false;
> >> - for (unsigned i = 1; i < gimple_num_ops (s); ++i)
> >> - if (gimple_op (s, i) == lookfor)
> >> - {
> >> - STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i - 1;
> >> - lookfor = gimple_get_lhs (s);
> >> - found = true;
> >> - break;
> >> - }
> >> + if (gimple_extract_op (s, &op))
> >> + for (unsigned i = 0; i < op.num_ops; ++i)
> >> + if (op.ops[i] == lookfor)
> >> + {
> >> + STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i;
> >> + lookfor = gimple_get_lhs (s);
> >> + found = true;
> >> + break;
> >> + }
> >> if (s == pattern_stmt)
> >> {
> >> if (!found && dump_enabled_p ())
> >> diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
> >> index 03cc7267cf8..1e197023b98 100644
> >> --- a/gcc/tree-vect-stmts.c
> >> +++ b/gcc/tree-vect-stmts.c
> >> @@ -3202,7 +3202,6 @@ vectorizable_call (vec_info *vinfo,
> >> int ndts = ARRAY_SIZE (dt);
> >> int ncopies, j;
> >> auto_vec<tree, 8> vargs;
> >> - auto_vec<tree, 8> orig_vargs;
> >> enum { NARROW, NONE, WIDEN } modifier;
> >> size_t i, nargs;
> >> tree lhs;
> >> @@ -3426,6 +3425,8 @@ vectorizable_call (vec_info *vinfo,
> >> needs to be generated. */
> >> gcc_assert (ncopies >= 1);
> >>
> >> + int reduc_idx = STMT_VINFO_REDUC_IDX (stmt_info);
> >> + internal_fn cond_fn = get_conditional_internal_fn (ifn);
> >> vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) :
> >> NULL);
> >> if (!vec_stmt) /* transformation not required. */
> >> {
> >> @@ -3446,14 +3447,33 @@ vectorizable_call (vec_info *vinfo,
> >> record_stmt_cost (cost_vec, ncopies / 2,
> >> vec_promote_demote, stmt_info, 0, vect_body);
> >>
> >> - if (loop_vinfo && mask_opno >= 0)
> >> + if (loop_vinfo
> >> + && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
> >> + && (reduc_idx >= 0 || mask_opno >= 0))
> >> {
> >> - unsigned int nvectors = (slp_node
> >> - ? SLP_TREE_NUMBER_OF_VEC_STMTS
> >> (slp_node)
> >> - : ncopies);
> >> - tree scalar_mask = gimple_call_arg (stmt_info->stmt, mask_opno);
> >> - vect_record_loop_mask (loop_vinfo, masks, nvectors,
> >> - vectype_out, scalar_mask);
> >> + if (reduc_idx >= 0
> >> + && (cond_fn == IFN_LAST
> >> + || !direct_internal_fn_supported_p (cond_fn, vectype_out,
> >> + OPTIMIZE_FOR_SPEED)))
> >> + {
> >> + if (dump_enabled_p ())
> >> + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> >> + "can't use a fully-masked loop because no"
> >> + " conditional operation is available.\n");
> >> + LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
> >> + }
> >> + else
> >> + {
> >> + unsigned int nvectors
> >> + = (slp_node
> >> + ? SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node)
> >> + : ncopies);
> >> + tree scalar_mask = NULL_TREE;
> >> + if (mask_opno >= 0)
> >> + scalar_mask = gimple_call_arg (stmt_info->stmt, mask_opno);
> >> + vect_record_loop_mask (loop_vinfo, masks, nvectors,
> >> + vectype_out, scalar_mask);
> >> + }
> >> }
> >> return true;
> >> }
> >> @@ -3468,12 +3488,17 @@ vectorizable_call (vec_info *vinfo,
> >> vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
> >>
> >> bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P
> >> (loop_vinfo);
> >> + unsigned int vect_nargs = nargs;
> >> + if (masked_loop_p && reduc_idx >= 0)
> >> + {
> >> + ifn = cond_fn;
> >> + vect_nargs += 2;
> >> + }
> >>
> >> if (modifier == NONE || ifn != IFN_LAST)
> >> {
> >> tree prev_res = NULL_TREE;
> >> - vargs.safe_grow (nargs, true);
> >> - orig_vargs.safe_grow (nargs, true);
> >> + vargs.safe_grow (vect_nargs, true);
> >> auto_vec<vec<tree> > vec_defs (nargs);
> >> for (j = 0; j < ncopies; ++j)
> >> {
> >> @@ -3488,12 +3513,23 @@ vectorizable_call (vec_info *vinfo,
> >> /* Arguments are ready. Create the new vector stmt. */
> >> FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
> >> {
> >> + int varg = 0;
> >> + if (masked_loop_p && reduc_idx >= 0)
> >> + {
> >> + unsigned int vec_num = vec_oprnds0.length ();
> >> + /* Always true for SLP. */
> >> + gcc_assert (ncopies == 1);
> >> + vargs[varg++] = vect_get_loop_mask (gsi, masks,
> >> vec_num,
> >> + vectype_out, i);
> >> + }
> >> size_t k;
> >> for (k = 0; k < nargs; k++)
> >> {
> >> vec<tree> vec_oprndsk = vec_defs[k];
> >> - vargs[k] = vec_oprndsk[i];
> >> + vargs[varg++] = vec_oprndsk[i];
> >> }
> >> + if (masked_loop_p && reduc_idx >= 0)
> >> + vargs[varg++] = vargs[reduc_idx + 1];
> >> gimple *new_stmt;
> >> if (modifier == NARROW)
> >> {
> >> @@ -3546,6 +3582,10 @@ vectorizable_call (vec_info *vinfo,
> >> continue;
> >> }
> >>
> >> + int varg = 0;
> >> + if (masked_loop_p && reduc_idx >= 0)
> >> + vargs[varg++] = vect_get_loop_mask (gsi, masks, ncopies,
> >> + vectype_out, j);
> >> for (i = 0; i < nargs; i++)
> >> {
> >> op = gimple_call_arg (stmt, i);
> >> @@ -3556,8 +3596,10 @@ vectorizable_call (vec_info *vinfo,
> >> op, &vec_defs[i],
> >> vectypes[i]);
> >> }
> >> - orig_vargs[i] = vargs[i] = vec_defs[i][j];
> >> + vargs[varg++] = vec_defs[i][j];
> >> }
> >> + if (masked_loop_p && reduc_idx >= 0)
> >> + vargs[varg++] = vargs[reduc_idx + 1];
> >>
> >> if (mask_opno >= 0 && masked_loop_p)
> >> {
> >> diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
> >> index f8f30641512..8330cd897b8 100644
> >> --- a/gcc/tree-vectorizer.h
> >> +++ b/gcc/tree-vectorizer.h
> >> @@ -28,6 +28,7 @@ typedef class _stmt_vec_info *stmt_vec_info;
> >> #include "target.h"
> >> #include "internal-fn.h"
> >> #include "tree-ssa-operands.h"
> >> +#include "gimple-match.h"
> >>
> >> /* Used for naming of new temporaries. */
> >> enum vect_var_kind {
> >> @@ -1192,7 +1193,7 @@ public:
> >> enum vect_reduction_type reduc_type;
> >>
> >> /* The original reduction code, to be used in the epilogue. */
> >> - enum tree_code reduc_code;
> >> + code_helper reduc_code;
> >> /* An internal function we should use in the epilogue. */
> >> internal_fn reduc_fn;
> >>
> >> @@ -2151,7 +2152,7 @@ extern tree vect_create_addr_base_for_vector_ref
> >> (vec_info *,
> >> tree);
> >>
> >> /* In tree-vect-loop.c. */
> >> -extern tree neutral_op_for_reduction (tree, tree_code, tree);
> >> +extern tree neutral_op_for_reduction (tree, code_helper, tree);
> >> extern widest_int vect_iv_limit_for_partial_vectors (loop_vec_info
> >> loop_vinfo);
> >> bool vect_rgroup_iv_might_wrap_p (loop_vec_info, rgroup_controls *);
> >> /* Used in tree-vect-loop-manip.c */
> >> @@ -2160,7 +2161,7 @@ extern opt_result
> >> vect_determine_partial_vectors_and_peeling (loop_vec_info,
> >> /* Used in gimple-loop-interchange.c and tree-parloops.c. */
> >> extern bool check_reduction_path (dump_user_location_t, loop_p, gphi *,
> >> tree,
> >> enum tree_code);
> >> -extern bool needs_fold_left_reduction_p (tree, tree_code);
> >> +extern bool needs_fold_left_reduction_p (tree, code_helper);
> >> /* Drive for loop analysis stage. */
> >> extern opt_loop_vec_info vect_analyze_loop (class loop *, vec_info_shared
> >> *);
> >> extern tree vect_build_loop_niters (loop_vec_info, bool * = NULL);
> >> @@ -2178,7 +2179,7 @@ extern tree vect_get_loop_len (loop_vec_info,
> >> vec_loop_lens *, unsigned int,
> >> unsigned int);
> >> extern gimple_seq vect_gen_len (tree, tree, tree, tree);
> >> extern stmt_vec_info info_for_reduction (vec_info *, stmt_vec_info);
> >> -extern bool reduction_fn_for_scalar_code (enum tree_code, internal_fn *);
> >> +extern bool reduction_fn_for_scalar_code (code_helper, internal_fn *);
> >>
> >> /* Drive for loop transformation stage. */
> >> extern class loop *vect_transform_loop (loop_vec_info, gimple *);
> >> @@ -2216,6 +2217,7 @@ extern bool vectorizable_phi (vec_info *,
> >> stmt_vec_info, gimple **, slp_tree,
> >> stmt_vector_for_cost *);
> >> extern bool vect_emulated_vector_p (tree);
> >> extern bool vect_can_vectorize_without_simd_p (tree_code);
> >> +extern bool vect_can_vectorize_without_simd_p (code_helper);
> >> extern int vect_get_known_peeling_cost (loop_vec_info, int, int *,
> >> stmt_vector_for_cost *,
> >> stmt_vector_for_cost *,
> >> --
> >> 2.25.1
> >>
