From: oluade01 <oluwatamilore.adeb...@arm.com>
This updates vect_recog_abd_pattern to recognize the widening
variant of absolute difference (ABDL, ABDL2).
gcc/ChangeLog:
* internal-fn.cc (widening_fn_p, decomposes_to_hilo_fn_p):
Add IFN_VEC_WIDEN_ABD to the switch statement.
* internal-fn.def (VEC_WIDEN_ABD): New internal hilo optab.
* optabs.def (vec_widen_sabd_optab,
vec_widen_sabd_hi_optab, vec_widen_sabd_lo_optab,
vec_widen_sabd_odd_even, vec_widen_sabd_even_optab,
vec_widen_uabd_optab,
vec_widen_uabd_hi_optab, vec_widen_uabd_lo_optab,
vec_widen_uabd_odd_even, vec_widen_uabd_even_optab):
New optabs.
* tree-vect-patterns.cc (vect_recog_abd_pattern): Update to
to build a VEC_WIDEN_ABD call if the input precision is smaller
than the precision of the output.
(vect_recog_widen_abd_pattern): Should an ABD expression be
found preceeding an extension, replace the two with a
VEC_WIDEN_ABD.
---
gcc/internal-fn.def | 5 ++
gcc/optabs.def | 10 +++
gcc/tree-vect-patterns.cc | 174 ++++++++++++++++++++++++++++++++------
3 files changed, 161 insertions(+), 28 deletions(-)
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index
116965f4830cec8f60642ff011a86b6562e2c509..d67274d68b49943a88c531e903fd03b42343ab97
100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -352,6 +352,11 @@ DEF_INTERNAL_WIDENING_OPTAB_FN (VEC_WIDEN_MINUS,
first,
vec_widen_ssub, vec_widen_usub,
binary)
+DEF_INTERNAL_WIDENING_OPTAB_FN (VEC_WIDEN_ABD,
+ ECF_CONST | ECF_NOTHROW,
+ first,
+ vec_widen_sabd, vec_widen_uabd,
+ binary)
DEF_INTERNAL_OPTAB_FN (VEC_FMADDSUB, ECF_CONST, vec_fmaddsub, ternary)
DEF_INTERNAL_OPTAB_FN (VEC_FMSUBADD, ECF_CONST, vec_fmsubadd, ternary)
diff --git a/gcc/optabs.def b/gcc/optabs.def
index
35b835a6ac56d72417dac8ddfd77a8a7e2475e65..68dfa1550f791a2fe833012157601ecfa68f1e09
100644
--- a/gcc/optabs.def
+++ b/gcc/optabs.def
@@ -418,6 +418,11 @@ OPTAB_D (vec_widen_sadd_hi_optab, "vec_widen_sadd_hi_$a")
OPTAB_D (vec_widen_sadd_lo_optab, "vec_widen_sadd_lo_$a")
OPTAB_D (vec_widen_sadd_odd_optab, "vec_widen_sadd_odd_$a")
OPTAB_D (vec_widen_sadd_even_optab, "vec_widen_sadd_even_$a")
+OPTAB_D (vec_widen_sabd_optab, "vec_widen_sabd_$a")
+OPTAB_D (vec_widen_sabd_hi_optab, "vec_widen_sabd_hi_$a")
+OPTAB_D (vec_widen_sabd_lo_optab, "vec_widen_sabd_lo_$a")
+OPTAB_D (vec_widen_sabd_odd_optab, "vec_widen_sabd_odd_$a")
+OPTAB_D (vec_widen_sabd_even_optab, "vec_widen_sabd_even_$a")
OPTAB_D (vec_widen_sshiftl_hi_optab, "vec_widen_sshiftl_hi_$a")
OPTAB_D (vec_widen_sshiftl_lo_optab, "vec_widen_sshiftl_lo_$a")
OPTAB_D (vec_widen_umult_even_optab, "vec_widen_umult_even_$a")
@@ -436,6 +441,11 @@ OPTAB_D (vec_widen_uadd_hi_optab, "vec_widen_uadd_hi_$a")
OPTAB_D (vec_widen_uadd_lo_optab, "vec_widen_uadd_lo_$a")
OPTAB_D (vec_widen_uadd_odd_optab, "vec_widen_uadd_odd_$a")
OPTAB_D (vec_widen_uadd_even_optab, "vec_widen_uadd_even_$a")
+OPTAB_D (vec_widen_uabd_optab, "vec_widen_uabd_$a")
+OPTAB_D (vec_widen_uabd_hi_optab, "vec_widen_uabd_hi_$a")
+OPTAB_D (vec_widen_uabd_lo_optab, "vec_widen_uabd_lo_$a")
+OPTAB_D (vec_widen_uabd_odd_optab, "vec_widen_uabd_odd_$a")
+OPTAB_D (vec_widen_uabd_even_optab, "vec_widen_uabd_even_$a")
OPTAB_D (vec_addsub_optab, "vec_addsub$a3")
OPTAB_D (vec_fmaddsub_optab, "vec_fmaddsub$a4")
OPTAB_D (vec_fmsubadd_optab, "vec_fmsubadd$a4")
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index
e2392113bff4065c909aefc760b4c48978b73a5a..852c4e99edb19d215c354b666b991c87c48620b4
100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -1404,15 +1404,28 @@ vect_recog_sad_pattern (vec_info *vinfo,
gcall *abd_stmt = dyn_cast <gcall *> (abs_stmt_vinfo->stmt);
if (!abd_stmt
|| !gimple_call_internal_p (abd_stmt)
- || gimple_call_internal_fn (abd_stmt) != IFN_ABD)
+ || gimple_call_num_args (abd_stmt) != 2)
return NULL;
tree abd_oprnd0 = gimple_call_arg (abd_stmt, 0);
tree abd_oprnd1 = gimple_call_arg (abd_stmt, 1);
- if (!vect_look_through_possible_promotion (vinfo, abd_oprnd0, &unprom[0])
- || !vect_look_through_possible_promotion (vinfo, abd_oprnd1,
- &unprom[1]))
+ if (gimple_call_internal_fn (abd_stmt) == IFN_ABD)
+ {
+ if (!vect_look_through_possible_promotion (vinfo, abd_oprnd0,
+ &unprom[0])
+ || !vect_look_through_possible_promotion (vinfo, abd_oprnd1,
+ &unprom[1]))
+ return NULL;
+ }
+ else if (gimple_call_internal_fn (abd_stmt) == IFN_VEC_WIDEN_ABD)
+ {
+ unprom[0].op = abd_oprnd0;
+ unprom[0].type = TREE_TYPE (abd_oprnd0);
+ unprom[1].op = abd_oprnd1;
+ unprom[1].type = TREE_TYPE (abd_oprnd1);
+ }
+ else
return NULL;
half_type = unprom[0].type;
@@ -1442,16 +1455,25 @@ vect_recog_sad_pattern (vec_info *vinfo,
/* Function vect_recog_abd_pattern
- Try to find the following ABsolute Difference (ABD) pattern:
+ Try to find the following ABsolute Difference (ABD) or
+ ABsolute Difference Long (ABDL) pattern:
- VTYPE x, y, out;
+ VTYPE x, y;
+ WTYPE out;
type diff;
loop i in range:
S1 diff = x[i] - y[i]
S2 out[i] = ABS_EXPR <diff>;
- where 'type' is a integer and 'VTYPE' is a vector of integers
- the same size as 'type'
+ where 'VTYPE' and 'WTYPE' are vectors of integers.
+ 'WTYPE' may be wider than 'VTYPE'.
+ 'type' is as wide as 'WTYPE'.
+
+ In the case of ABD, should the precision of 'WTYPE' be wider than
+ 'VTYPE' or narrower, conversion operations will succeed the ABD or
+ ABDL expression.
+ In the case of ABDL, the same occurs if the precision of 'WTYPE' is
+ more than twice as wide or narrower.
Input:
@@ -1459,12 +1481,14 @@ vect_recog_sad_pattern (vec_info *vinfo,
Output:
- * TYPE_out: The type of the output of this pattern
+ * TYPE_OUT: The type of the output of this pattern
* Return value: A new stmt that will be used to replace the sequence of
- stmts that constitute the pattern; either SABD or UABD:
+ stmts that constitute the pattern; either SABD, UABD, SABDL or UABDL:
SABD_EXPR<x, y, out>
UABD_EXPR<x, y, out>
+ SABDL_EXPR<x, y, out>
+ UABDL_EXPR<x, y, out>
*/
static gimple *
@@ -1479,8 +1503,9 @@ vect_recog_abd_pattern (vec_info *vinfo,
out[i] = DAD
In which
- - X, Y, DIFF, DAD all have the same type
- - x, y, out are all vectors of the same type
+ - X, Y have the same type
+ - x, y are all vectors of the same type
+ - DIFF, DAD, out may be wider than X, Y
*/
gassign *last_stmt = dyn_cast <gassign *> (STMT_VINFO_STMT (stmt_vinfo));
@@ -1496,37 +1521,74 @@ vect_recog_abd_pattern (vec_info *vinfo,
unprom, &diff_stmt))
return NULL;
- tree abd_type = out_type, vectype;
- tree abd_oprnds[2];
+ internal_fn ifn = IFN_ABD;
+ tree abd_in_type, abd_out_type;
bool extend = false;
+
if (half_type)
{
- vectype = get_vectype_for_scalar_type (vinfo, half_type);
- abd_type = half_type;
- extend = TYPE_PRECISION (abd_type) < TYPE_PRECISION (out_type);
+ abd_in_type = half_type;
+ abd_out_type = abd_in_type;
+
+ if (stmt_vinfo->min_output_precision == TYPE_PRECISION (abd_out_type))
+ extend = TYPE_PRECISION (abd_in_type) < TYPE_PRECISION (out_type);
+ else
+ {
+ ifn = IFN_VEC_WIDEN_ABD;
+ abd_out_type
+ = build_nonstandard_integer_type (TYPE_PRECISION (abd_in_type) * 2,
+ TYPE_UNSIGNED (abd_in_type));
+ extend = TYPE_PRECISION (abd_out_type) < TYPE_PRECISION (out_type);
+ }
}
else
{
unprom[0].op = gimple_assign_rhs1 (diff_stmt);
unprom[1].op = gimple_assign_rhs2 (diff_stmt);
- tree signed_out = signed_type_for (out_type);
- vectype = get_vectype_for_scalar_type (vinfo, signed_out);
+ abd_in_type = signed_type_for (out_type);
+ abd_out_type = abd_in_type;
}
- vect_pattern_detected ("vect_recog_abd_pattern", last_stmt);
+ tree vectype_in = get_vectype_for_scalar_type (vinfo, abd_in_type);
+ tree vectype_out = get_vectype_for_scalar_type (vinfo, abd_out_type);
+ if (!vectype_in || !vectype_out)
+ return NULL;
- if (!vectype
- || !direct_internal_fn_supported_p (IFN_ABD, vectype,
+ if (ifn == IFN_VEC_WIDEN_ABD)
+ {
+ code_helper dummy_code;
+ int dummy_int;
+ auto_vec<tree> dummy_vec;
+ if (!supportable_widening_operation (vinfo, ifn, stmt_vinfo,
+ vectype_out, vectype_in,
+ &dummy_code, &dummy_code,
+ &dummy_int, &dummy_vec))
+ {
+ /* There are architectures that have the ABD instructions
+ but not the ABDL instructions. If we just return NULL here
+ we will miss an occasion where we should have used ABD.
+ So we change back to ABD and try again. */
+ ifn = IFN_ABD;
+ abd_out_type = abd_in_type;
+ extend = true;
+ }
+ }
+
+ if (ifn == IFN_ABD
+ && !direct_internal_fn_supported_p (ifn, vectype_in,
OPTIMIZE_FOR_SPEED))
return NULL;
+ vect_pattern_detected ("vect_recog_abd_pattern", last_stmt);
+
+ tree abd_oprnds[2];
vect_convert_inputs (vinfo, stmt_vinfo, 2, abd_oprnds,
- TREE_TYPE (vectype), unprom, vectype);
+ abd_in_type, unprom, vectype_in);
*type_out = get_vectype_for_scalar_type (vinfo, out_type);
- tree abd_result = vect_recog_temp_ssa_var (abd_type, NULL);
- gcall *abd_stmt = gimple_build_call_internal (IFN_ABD, 2,
+ tree abd_result = vect_recog_temp_ssa_var (abd_out_type, NULL);
+ gcall *abd_stmt = gimple_build_call_internal (ifn, 2,
abd_oprnds[0], abd_oprnds[1]);
gimple_call_set_lhs (abd_stmt, abd_result);
gimple_set_location (abd_stmt, gimple_location (last_stmt));
@@ -1535,15 +1597,15 @@ vect_recog_abd_pattern (vec_info *vinfo,
return abd_stmt;
gimple *stmt = abd_stmt;
- if (!TYPE_UNSIGNED (abd_type))
+ if (!TYPE_UNSIGNED (abd_out_type))
{
- tree unsign = unsigned_type_for (abd_type);
+ tree unsign = unsigned_type_for (abd_out_type);
tree unsign_vectype = get_vectype_for_scalar_type (vinfo, unsign);
stmt = vect_convert_output (vinfo, stmt_vinfo, unsign, stmt,
unsign_vectype);
}
- return vect_convert_output (vinfo, stmt_vinfo, out_type, stmt, vectype);
+ return vect_convert_output (vinfo, stmt_vinfo, out_type, stmt, vectype_out);
}
/* Recognize an operation that performs ORIG_CODE on widened inputs,
@@ -1703,6 +1765,61 @@ vect_recog_widen_minus_pattern (vec_info *vinfo,
stmt_vec_info last_stmt_info,
&subtype);
}
+/* Try to detect abd on widened inputs, converting IFN_ABD
+ to IFN_VEC_WIDEN_ABD. */
+static gimple *
+vect_recog_widen_abd_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
+ tree *type_out)
+{
+ gassign *last_stmt = dyn_cast <gassign *> (STMT_VINFO_STMT (stmt_vinfo));
+ if (!last_stmt || !gimple_assign_cast_p (last_stmt))
+ return NULL;
+
+ tree last_rhs = gimple_assign_rhs1 (last_stmt);
+ stmt_vec_info abs_vinfo = vect_get_internal_def (vinfo, last_rhs);
+ if (!abs_vinfo)
+ return NULL;
+
+ stmt_vec_info abd_pattern_vinfo = STMT_VINFO_RELATED_STMT (abs_vinfo);
+ if (!abd_pattern_vinfo)
+ return NULL;
+
+ gcall *abd_stmt = dyn_cast <gcall *> (STMT_VINFO_STMT (abd_pattern_vinfo));
+ if (!abd_stmt || gimple_call_internal_fn (abd_stmt) != IFN_ABD)
+ return NULL;
+
+ tree abd_oprnd0 = gimple_call_arg (abd_stmt, 0);
+ tree abd_oprnd1 = gimple_call_arg (abd_stmt, 1);
+
+ tree in_type = TREE_TYPE (abd_oprnd0);
+ tree out_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
+ if (TYPE_PRECISION (in_type) * 2 != TYPE_PRECISION (out_type))
+ return NULL;
+
+ tree vectype_in = get_vectype_for_scalar_type (vinfo, in_type);
+ tree vectype_out = get_vectype_for_scalar_type (vinfo, out_type);
+
+ code_helper dummy_code;
+ int dummy_int;
+ auto_vec<tree> dummy_vec;
+ if (!supportable_widening_operation (vinfo, IFN_VEC_WIDEN_ABD, stmt_vinfo,
+ vectype_out, vectype_in,
+ &dummy_code, &dummy_code,
+ &dummy_int, &dummy_vec))
+ return NULL;
+
+ vect_pattern_detected ("vect_recog_widen_abd_pattern", last_stmt);
+
+ *type_out = vectype_out;
+
+ tree abdl_result = vect_recog_temp_ssa_var (out_type, NULL);
+ gcall *abdl_stmt = gimple_build_call_internal (IFN_VEC_WIDEN_ABD, 2,
+ abd_oprnd0, abd_oprnd1);
+ gimple_call_set_lhs (abdl_stmt, abdl_result);
+ gimple_set_location (abdl_stmt, gimple_location (last_stmt));
+ return abdl_stmt;
+}
+
/* Function vect_recog_ctz_ffs_pattern
Try to find the following pattern:
@@ -6670,6 +6787,7 @@ static vect_recog_func vect_vect_recog_func_ptrs[] = {
{ vect_recog_mask_conversion_pattern, "mask_conversion" },
{ vect_recog_widen_plus_pattern, "widen_plus" },
{ vect_recog_widen_minus_pattern, "widen_minus" },
+ { vect_recog_widen_abd_pattern, "widen_abd" },
/* These must come after the double widening ones. */
};
--
2.25.1
