On Sun, 11 Aug 2024, Robin Dapp wrote:
> This patch adds else-operand handling to the internal functions.
LGTM.
> gcc/ChangeLog:
>
> * internal-fn.cc (add_mask_and_len_args): Rename...
> (add_mask_else_and_len_args): ...to this and add else handling.
> (expand_partial_load_optab_fn): Use adjusted function.
> (expand_partial_store_optab_fn): Ditto.
> (expand_scatter_store_optab_fn): Ditto.
> (expand_gather_load_optab_fn): Ditto.
> (internal_fn_len_index): Adjust for masked loads.
> (internal_fn_else_index): Add masked loads.
> ---
> gcc/internal-fn.cc | 69 ++++++++++++++++++++++++++++++++++++++--------
> 1 file changed, 58 insertions(+), 11 deletions(-)
>
> diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
> index 8a2e07f2f96..586978e8f3f 100644
> --- a/gcc/internal-fn.cc
> +++ b/gcc/internal-fn.cc
> @@ -331,17 +331,18 @@ get_multi_vector_move (tree array_type, convert_optab
> optab)
> return convert_optab_handler (optab, imode, vmode);
> }
>
> -/* Add mask and len arguments according to the STMT. */
> +/* Add mask, else, and len arguments according to the STMT. */
>
> static unsigned int
> -add_mask_and_len_args (expand_operand *ops, unsigned int opno, gcall *stmt)
> +add_mask_else_and_len_args (expand_operand *ops, unsigned int opno, gcall
> *stmt)
> {
> internal_fn ifn = gimple_call_internal_fn (stmt);
> int len_index = internal_fn_len_index (ifn);
> /* BIAS is always consecutive next of LEN. */
> int bias_index = len_index + 1;
> int mask_index = internal_fn_mask_index (ifn);
> - /* The order of arguments are always {len,bias,mask}. */
> +
> + /* The order of arguments is always {mask, else, len, bias}. */
> if (mask_index >= 0)
> {
> tree mask = gimple_call_arg (stmt, mask_index);
> @@ -362,6 +363,23 @@ add_mask_and_len_args (expand_operand *ops, unsigned int
> opno, gcall *stmt)
>
> create_input_operand (&ops[opno++], mask_rtx,
> TYPE_MODE (TREE_TYPE (mask)));
> +
> + }
> +
> + int els_index = internal_fn_else_index (ifn);
> + if (els_index >= 0)
> + {
> + tree els = gimple_call_arg (stmt, els_index);
> + tree els_type = TREE_TYPE (els);
> + if (TREE_CODE (els) == SSA_NAME
> + && SSA_NAME_IS_DEFAULT_DEF (els)
> + && VAR_P (SSA_NAME_VAR (els)))
> + create_undefined_input_operand (&ops[opno++], TYPE_MODE (els_type));
> + else
> + {
> + rtx els_rtx = expand_normal (els);
> + create_input_operand (&ops[opno++], els_rtx, TYPE_MODE (els_type));
> + }
> }
> if (len_index >= 0)
> {
> @@ -3014,7 +3032,7 @@ static void
> expand_partial_load_optab_fn (internal_fn ifn, gcall *stmt, convert_optab
> optab)
> {
> int i = 0;
> - class expand_operand ops[5];
> + class expand_operand ops[6];
> tree type, lhs, rhs, maskt;
> rtx mem, target;
> insn_code icode;
> @@ -3044,7 +3062,7 @@ expand_partial_load_optab_fn (internal_fn ifn, gcall
> *stmt, convert_optab optab)
> target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
> create_call_lhs_operand (&ops[i++], target, TYPE_MODE (type));
> create_fixed_operand (&ops[i++], mem);
> - i = add_mask_and_len_args (ops, i, stmt);
> + i = add_mask_else_and_len_args (ops, i, stmt);
> expand_insn (icode, i, ops);
>
> assign_call_lhs (lhs, target, &ops[0]);
> @@ -3090,7 +3108,7 @@ expand_partial_store_optab_fn (internal_fn ifn, gcall
> *stmt, convert_optab optab
> reg = expand_normal (rhs);
> create_fixed_operand (&ops[i++], mem);
> create_input_operand (&ops[i++], reg, TYPE_MODE (type));
> - i = add_mask_and_len_args (ops, i, stmt);
> + i = add_mask_else_and_len_args (ops, i, stmt);
> expand_insn (icode, i, ops);
> }
>
> @@ -3676,7 +3694,7 @@ expand_scatter_store_optab_fn (internal_fn, gcall
> *stmt, direct_optab optab)
> create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
> create_integer_operand (&ops[i++], scale_int);
> create_input_operand (&ops[i++], rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
> - i = add_mask_and_len_args (ops, i, stmt);
> + i = add_mask_else_and_len_args (ops, i, stmt);
>
> insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE
> (rhs)),
> TYPE_MODE (TREE_TYPE (offset)));
> @@ -3705,7 +3723,7 @@ expand_gather_load_optab_fn (internal_fn, gcall *stmt,
> direct_optab optab)
> create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE
> (offset)));
> create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
> create_integer_operand (&ops[i++], scale_int);
> - i = add_mask_and_len_args (ops, i, stmt);
> + i = add_mask_else_and_len_args (ops, i, stmt);
> insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE
> (lhs)),
> TYPE_MODE (TREE_TYPE (offset)));
> expand_insn (icode, i, ops);
> @@ -4590,6 +4608,18 @@ get_len_internal_fn (internal_fn fn)
> case IFN_COND_##NAME:
> \
> return IFN_COND_LEN_##NAME;
> #include "internal-fn.def"
> + default:
> + break;
> + }
> +
> + switch (fn)
> + {
> + case IFN_MASK_LOAD:
> + return IFN_MASK_LEN_LOAD;
> + case IFN_MASK_LOAD_LANES:
> + return IFN_MASK_LEN_LOAD_LANES;
> + case IFN_MASK_GATHER_LOAD:
> + return IFN_MASK_LEN_GATHER_LOAD;
> default:
> return IFN_LAST;
> }
> @@ -4775,8 +4805,12 @@ internal_fn_len_index (internal_fn fn)
> case IFN_LEN_STORE:
> return 2;
>
> - case IFN_MASK_LEN_GATHER_LOAD:
> case IFN_MASK_LEN_SCATTER_STORE:
> + return 5;
> +
> + case IFN_MASK_LEN_GATHER_LOAD:
> + return 6;
> +
> case IFN_COND_LEN_FMA:
> case IFN_COND_LEN_FMS:
> case IFN_COND_LEN_FNMA:
> @@ -4801,13 +4835,15 @@ internal_fn_len_index (internal_fn fn)
> return 4;
>
> case IFN_COND_LEN_NEG:
> - case IFN_MASK_LEN_LOAD:
> case IFN_MASK_LEN_STORE:
> - case IFN_MASK_LEN_LOAD_LANES:
> case IFN_MASK_LEN_STORE_LANES:
> case IFN_VCOND_MASK_LEN:
> return 3;
>
> + case IFN_MASK_LEN_LOAD:
> + case IFN_MASK_LEN_LOAD_LANES:
> + return 4;
> +
> default:
> return -1;
> }
> @@ -4857,6 +4893,12 @@ internal_fn_else_index (internal_fn fn)
> case IFN_COND_LEN_SHR:
> return 3;
>
> + case IFN_MASK_LOAD:
> + case IFN_MASK_LEN_LOAD:
> + case IFN_MASK_LOAD_LANES:
> + case IFN_MASK_LEN_LOAD_LANES:
> + return 3;
> +
> case IFN_COND_FMA:
> case IFN_COND_FMS:
> case IFN_COND_FNMA:
> @@ -4867,6 +4909,10 @@ internal_fn_else_index (internal_fn fn)
> case IFN_COND_LEN_FNMS:
> return 4;
>
> + case IFN_MASK_GATHER_LOAD:
> + case IFN_MASK_LEN_GATHER_LOAD:
> + return 5;
> +
> default:
> return -1;
> }
> @@ -4898,6 +4944,7 @@ internal_fn_mask_index (internal_fn fn)
> case IFN_MASK_LEN_SCATTER_STORE:
> return 4;
>
> + case IFN_VCOND_MASK:
> case IFN_VCOND_MASK_LEN:
> return 0;
>
>
--
Richard Biener <rguent...@suse.de>
SUSE Software Solutions Germany GmbH,
Frankenstrasse 146, 90461 Nuernberg, Germany;
GF: Ivo Totev, Andrew McDonald, Werner Knoblich; (HRB 36809, AG Nuernberg)
