committed, thanks!
On Sun, Jan 29, 2023 at 11:33 PM <juzhe.zh...@rivai.ai> wrote:
>
> From: Ju-Zhe Zhong <juzhe.zh...@rivai.ai>
>
> gcc/ChangeLog:
>
> * config/riscv/riscv-protos.h (get_vector_mode): New function.
> * config/riscv/riscv-v.cc (get_vector_mode): Ditto.
> * config/riscv/riscv-vector-builtins-bases.cc (enum lst_type): New
> enum.
> (class loadstore): Adjust for indexed loads/stores support.
> (BASE): Ditto.
> * config/riscv/riscv-vector-builtins-bases.h: New function declare.
> * config/riscv/riscv-vector-builtins-functions.def (vluxei8): Ditto.
> (vluxei16): Ditto.
> (vluxei32): Ditto.
> (vluxei64): Ditto.
> (vloxei8): Ditto.
> (vloxei16): Ditto.
> (vloxei32): Ditto.
> (vloxei64): Ditto.
> (vsuxei8): Ditto.
> (vsuxei16): Ditto.
> (vsuxei32): Ditto.
> (vsuxei64): Ditto.
> (vsoxei8): Ditto.
> (vsoxei16): Ditto.
> (vsoxei32): Ditto.
> (vsoxei64): Ditto.
> * config/riscv/riscv-vector-builtins-shapes.cc (struct
> indexed_loadstore_def): New class.
> (SHAPE): Ditto.
> * config/riscv/riscv-vector-builtins-shapes.h: Ditto.
> * config/riscv/riscv-vector-builtins.cc (required_extensions_p):
> Adjust for indexed loads/stores support.
> (check_required_extensions): Ditto.
> (rvv_arg_type_info::get_base_vector_type): New function.
> (rvv_arg_type_info::get_tree_type): Ditto.
> (function_builder::add_unique_function): Adjust for indexed
> loads/stores support.
> (function_expander::use_exact_insn): New function.
> * config/riscv/riscv-vector-builtins.h (enum rvv_base_type): Adjust
> for indexed loads/stores support.
> (struct rvv_arg_type_info): Ditto.
> (function_expander::index_mode): New function.
> (function_base::apply_tail_policy_p): Ditto.
> (function_base::apply_mask_policy_p): Ditto.
> * config/riscv/vector-iterators.md (unspec): New unspec.
> * config/riscv/vector.md (unspec): Ditto.
> (@pred_indexed_<order>load<VNX1_QHSD:mode><VNX1_QHSDI:mode>): New
> pattern.
> (@pred_indexed_<order>store<VNX1_QHSD:mode><VNX1_QHSDI:mode>): Ditto.
> (@pred_indexed_<order>load<VNX2_QHSD:mode><VNX2_QHSDI:mode>): Ditto.
> (@pred_indexed_<order>store<VNX2_QHSD:mode><VNX2_QHSDI:mode>): Ditto.
> (@pred_indexed_<order>load<VNX4_QHSD:mode><VNX4_QHSDI:mode>): Ditto.
> (@pred_indexed_<order>store<VNX4_QHSD:mode><VNX4_QHSDI:mode>): Ditto.
> (@pred_indexed_<order>load<VNX8_QHSD:mode><VNX8_QHSDI:mode>): Ditto.
> (@pred_indexed_<order>store<VNX8_QHSD:mode><VNX8_QHSDI:mode>): Ditto.
> (@pred_indexed_<order>load<VNX16_QHS:mode><VNX16_QHSI:mode>): Ditto.
> (@pred_indexed_<order>store<VNX16_QHS:mode><VNX16_QHSI:mode>): Ditto.
> (@pred_indexed_<order>load<VNX32_QH:mode><VNX32_QHI:mode>): Ditto.
> (@pred_indexed_<order>store<VNX32_QH:mode><VNX32_QHI:mode>): Ditto.
> (@pred_indexed_<order>load<VNX64_Q:mode><VNX64_Q:mode>): Ditto.
> (@pred_indexed_<order>store<VNX64_Q:mode><VNX64_Q:mode>): Ditto.
>
> ---
> gcc/config/riscv/riscv-protos.h | 1 +
> gcc/config/riscv/riscv-v.cc | 22 ++
> .../riscv/riscv-vector-builtins-bases.cc | 92 +++++-
> .../riscv/riscv-vector-builtins-bases.h | 16 +
> .../riscv/riscv-vector-builtins-functions.def | 16 +
> .../riscv/riscv-vector-builtins-shapes.cc | 49 +++
> .../riscv/riscv-vector-builtins-shapes.h | 1 +
> gcc/config/riscv/riscv-vector-builtins.cc | 249 +++++++++++++-
> gcc/config/riscv/riscv-vector-builtins.h | 36 ++
> gcc/config/riscv/vector-iterators.md | 87 +++++
> gcc/config/riscv/vector.md | 309 +++++++++++++++++-
> 11 files changed, 845 insertions(+), 33 deletions(-)
>
> diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h
> index 1facad74b3c..ceae4007fd1 100644
> --- a/gcc/config/riscv/riscv-protos.h
> +++ b/gcc/config/riscv/riscv-protos.h
> @@ -172,6 +172,7 @@ enum mask_policy
> enum tail_policy get_prefer_tail_policy ();
> enum mask_policy get_prefer_mask_policy ();
> rtx get_avl_type_rtx (enum avl_type);
> +opt_machine_mode get_vector_mode (scalar_mode, poly_uint64);
> }
>
> /* We classify builtin types into two classes:
> diff --git a/gcc/config/riscv/riscv-v.cc b/gcc/config/riscv/riscv-v.cc
> index 47f2cef459b..992b3437926 100644
> --- a/gcc/config/riscv/riscv-v.cc
> +++ b/gcc/config/riscv/riscv-v.cc
> @@ -349,4 +349,26 @@ get_avl_type_rtx (enum avl_type type)
> return gen_int_mode (type, Pmode);
> }
>
> +/* Return the RVV vector mode that has NUNITS elements of mode INNER_MODE.
> + This function is not only used by builtins, but also will be used by
> + auto-vectorization in the future. */
> +opt_machine_mode
> +get_vector_mode (scalar_mode inner_mode, poly_uint64 nunits)
> +{
> + enum mode_class mclass;
> + if (inner_mode == E_BImode)
> + mclass = MODE_VECTOR_BOOL;
> + else if (FLOAT_MODE_P (inner_mode))
> + mclass = MODE_VECTOR_FLOAT;
> + else
> + mclass = MODE_VECTOR_INT;
> + machine_mode mode;
> + FOR_EACH_MODE_IN_CLASS (mode, mclass)
> + if (inner_mode == GET_MODE_INNER (mode)
> + && known_eq (nunits, GET_MODE_NUNITS (mode))
> + && riscv_v_ext_vector_mode_p (mode))
> + return mode;
> + return opt_machine_mode ();
> +}
> +
> } // namespace riscv_vector
> diff --git a/gcc/config/riscv/riscv-vector-builtins-bases.cc
> b/gcc/config/riscv/riscv-vector-builtins-bases.cc
> index f9a16c68e07..129e89f443e 100644
> --- a/gcc/config/riscv/riscv-vector-builtins-bases.cc
> +++ b/gcc/config/riscv/riscv-vector-builtins-bases.cc
> @@ -48,6 +48,16 @@ using namespace riscv_vector;
>
> namespace riscv_vector {
>
> +/* Enumerates types of loads/stores operations.
> + It's only used in here so we don't define it
> + in riscv-vector-builtins-bases.h. */
> +enum lst_type
> +{
> + LST_UNIT_STRIDE,
> + LST_STRIDED,
> + LST_INDEXED,
> +};
> +
> /* Implements vsetvl<mode> && vsetvlmax<mode>. */
> template<bool VLMAX_P>
> class vsetvl : public function_base
> @@ -84,10 +94,16 @@ public:
> }
> };
>
> -/* Implements vle.v/vse.v/vlm.v/vsm.v/vlse.v/vsse.v codegen. */
> -template <bool STORE_P, bool STRIDED_P = false>
> +/* Implements
> + * vle.v/vse.v/vlm.v/vsm.v/vlse.v/vsse.v/vluxei.v/vloxei.v/vsuxei.v/vsoxei.v
> + * codegen. */
> +template<bool STORE_P, lst_type LST_TYPE, bool ORDERED_P>
> class loadstore : public function_base
> {
> +public:
> + bool apply_tail_policy_p () const override { return !STORE_P; }
> + bool apply_mask_policy_p () const override { return !STORE_P; }
> +
> unsigned int call_properties (const function_instance &) const override
> {
> if (STORE_P)
> @@ -98,27 +114,39 @@ class loadstore : public function_base
>
> bool can_be_overloaded_p (enum predication_type_index pred) const override
> {
> - if (STORE_P)
> + if (STORE_P || LST_TYPE == LST_INDEXED)
> return true;
> return pred != PRED_TYPE_none && pred != PRED_TYPE_mu;
> }
>
> rtx expand (function_expander &e) const override
> {
> - if (STORE_P)
> + if (LST_TYPE == LST_INDEXED)
> + {
> + int unspec = ORDERED_P ? UNSPEC_ORDERED : UNSPEC_UNORDERED;
> + if (STORE_P)
> + return e.use_exact_insn (
> + code_for_pred_indexed_store (unspec, e.vector_mode (),
> + e.index_mode ()));
> + else
> + return e.use_exact_insn (
> + code_for_pred_indexed_load (unspec, e.vector_mode (),
> + e.index_mode ()));
> + }
> + else if (LST_TYPE == LST_STRIDED)
> {
> - if (STRIDED_P)
> + if (STORE_P)
> return e.use_contiguous_store_insn (
> code_for_pred_strided_store (e.vector_mode ()));
> else
> - return e.use_contiguous_store_insn (
> - code_for_pred_store (e.vector_mode ()));
> + return e.use_contiguous_load_insn (
> + code_for_pred_strided_load (e.vector_mode ()));
> }
> else
> {
> - if (STRIDED_P)
> - return e.use_contiguous_load_insn (
> - code_for_pred_strided_load (e.vector_mode ()));
> + if (STORE_P)
> + return e.use_contiguous_store_insn (
> + code_for_pred_store (e.vector_mode ()));
> else
> return e.use_contiguous_load_insn (
> code_for_pred_mov (e.vector_mode ()));
> @@ -128,12 +156,28 @@ class loadstore : public function_base
>
> static CONSTEXPR const vsetvl<false> vsetvl_obj;
> static CONSTEXPR const vsetvl<true> vsetvlmax_obj;
> -static CONSTEXPR const loadstore<false> vle_obj;
> -static CONSTEXPR const loadstore<true> vse_obj;
> -static CONSTEXPR const loadstore<false> vlm_obj;
> -static CONSTEXPR const loadstore<true> vsm_obj;
> -static CONSTEXPR const loadstore<false, true> vlse_obj;
> -static CONSTEXPR const loadstore<true, true> vsse_obj;
> +static CONSTEXPR const loadstore<false, LST_UNIT_STRIDE, false> vle_obj;
> +static CONSTEXPR const loadstore<true, LST_UNIT_STRIDE, false> vse_obj;
> +static CONSTEXPR const loadstore<false, LST_UNIT_STRIDE, false> vlm_obj;
> +static CONSTEXPR const loadstore<true, LST_UNIT_STRIDE, false> vsm_obj;
> +static CONSTEXPR const loadstore<false, LST_STRIDED, false> vlse_obj;
> +static CONSTEXPR const loadstore<true, LST_STRIDED, false> vsse_obj;
> +static CONSTEXPR const loadstore<false, LST_INDEXED, false> vluxei8_obj;
> +static CONSTEXPR const loadstore<false, LST_INDEXED, false> vluxei16_obj;
> +static CONSTEXPR const loadstore<false, LST_INDEXED, false> vluxei32_obj;
> +static CONSTEXPR const loadstore<false, LST_INDEXED, false> vluxei64_obj;
> +static CONSTEXPR const loadstore<false, LST_INDEXED, true> vloxei8_obj;
> +static CONSTEXPR const loadstore<false, LST_INDEXED, true> vloxei16_obj;
> +static CONSTEXPR const loadstore<false, LST_INDEXED, true> vloxei32_obj;
> +static CONSTEXPR const loadstore<false, LST_INDEXED, true> vloxei64_obj;
> +static CONSTEXPR const loadstore<true, LST_INDEXED, false> vsuxei8_obj;
> +static CONSTEXPR const loadstore<true, LST_INDEXED, false> vsuxei16_obj;
> +static CONSTEXPR const loadstore<true, LST_INDEXED, false> vsuxei32_obj;
> +static CONSTEXPR const loadstore<true, LST_INDEXED, false> vsuxei64_obj;
> +static CONSTEXPR const loadstore<true, LST_INDEXED, true> vsoxei8_obj;
> +static CONSTEXPR const loadstore<true, LST_INDEXED, true> vsoxei16_obj;
> +static CONSTEXPR const loadstore<true, LST_INDEXED, true> vsoxei32_obj;
> +static CONSTEXPR const loadstore<true, LST_INDEXED, true> vsoxei64_obj;
>
> /* Declare the function base NAME, pointing it to an instance
> of class <NAME>_obj. */
> @@ -148,5 +192,21 @@ BASE (vlm)
> BASE (vsm)
> BASE (vlse)
> BASE (vsse)
> +BASE (vluxei8)
> +BASE (vluxei16)
> +BASE (vluxei32)
> +BASE (vluxei64)
> +BASE (vloxei8)
> +BASE (vloxei16)
> +BASE (vloxei32)
> +BASE (vloxei64)
> +BASE (vsuxei8)
> +BASE (vsuxei16)
> +BASE (vsuxei32)
> +BASE (vsuxei64)
> +BASE (vsoxei8)
> +BASE (vsoxei16)
> +BASE (vsoxei32)
> +BASE (vsoxei64)
>
> } // end namespace riscv_vector
> diff --git a/gcc/config/riscv/riscv-vector-builtins-bases.h
> b/gcc/config/riscv/riscv-vector-builtins-bases.h
> index 93999e2cbee..238c01dbf1f 100644
> --- a/gcc/config/riscv/riscv-vector-builtins-bases.h
> +++ b/gcc/config/riscv/riscv-vector-builtins-bases.h
> @@ -32,6 +32,22 @@ extern const function_base *const vlm;
> extern const function_base *const vsm;
> extern const function_base *const vlse;
> extern const function_base *const vsse;
> +extern const function_base *const vluxei8;
> +extern const function_base *const vluxei16;
> +extern const function_base *const vluxei32;
> +extern const function_base *const vluxei64;
> +extern const function_base *const vloxei8;
> +extern const function_base *const vloxei16;
> +extern const function_base *const vloxei32;
> +extern const function_base *const vloxei64;
> +extern const function_base *const vsuxei8;
> +extern const function_base *const vsuxei16;
> +extern const function_base *const vsuxei32;
> +extern const function_base *const vsuxei64;
> +extern const function_base *const vsoxei8;
> +extern const function_base *const vsoxei16;
> +extern const function_base *const vsoxei32;
> +extern const function_base *const vsoxei64;
> }
>
> } // end namespace riscv_vector
> diff --git a/gcc/config/riscv/riscv-vector-builtins-functions.def
> b/gcc/config/riscv/riscv-vector-builtins-functions.def
> index 1ddde7b9d76..9719b9b4bf1 100644
> --- a/gcc/config/riscv/riscv-vector-builtins-functions.def
> +++ b/gcc/config/riscv/riscv-vector-builtins-functions.def
> @@ -46,5 +46,21 @@ DEF_RVV_FUNCTION (vlm, loadstore, none_preds,
> b_v_scalar_const_ptr_ops)
> DEF_RVV_FUNCTION (vsm, loadstore, none_preds, b_v_scalar_ptr_ops)
> DEF_RVV_FUNCTION (vlse, loadstore, full_preds,
> all_v_scalar_const_ptr_ptrdiff_ops)
> DEF_RVV_FUNCTION (vsse, loadstore, none_m_preds,
> all_v_scalar_ptr_ptrdiff_ops)
> +DEF_RVV_FUNCTION (vluxei8, indexed_loadstore, full_preds,
> all_v_scalar_const_ptr_uint8_index_ops)
> +DEF_RVV_FUNCTION (vluxei16, indexed_loadstore, full_preds,
> all_v_scalar_const_ptr_uint16_index_ops)
> +DEF_RVV_FUNCTION (vluxei32, indexed_loadstore, full_preds,
> all_v_scalar_const_ptr_uint32_index_ops)
> +DEF_RVV_FUNCTION (vluxei64, indexed_loadstore, full_preds,
> all_v_scalar_const_ptr_uint64_index_ops)
> +DEF_RVV_FUNCTION (vloxei8, indexed_loadstore, full_preds,
> all_v_scalar_const_ptr_uint8_index_ops)
> +DEF_RVV_FUNCTION (vloxei16, indexed_loadstore, full_preds,
> all_v_scalar_const_ptr_uint16_index_ops)
> +DEF_RVV_FUNCTION (vloxei32, indexed_loadstore, full_preds,
> all_v_scalar_const_ptr_uint32_index_ops)
> +DEF_RVV_FUNCTION (vloxei64, indexed_loadstore, full_preds,
> all_v_scalar_const_ptr_uint64_index_ops)
> +DEF_RVV_FUNCTION (vsuxei8, indexed_loadstore, none_m_preds,
> all_v_scalar_ptr_uint8_index_ops)
> +DEF_RVV_FUNCTION (vsuxei16, indexed_loadstore, none_m_preds,
> all_v_scalar_ptr_uint16_index_ops)
> +DEF_RVV_FUNCTION (vsuxei32, indexed_loadstore, none_m_preds,
> all_v_scalar_ptr_uint32_index_ops)
> +DEF_RVV_FUNCTION (vsuxei64, indexed_loadstore, none_m_preds,
> all_v_scalar_ptr_uint64_index_ops)
> +DEF_RVV_FUNCTION (vsoxei8, indexed_loadstore, none_m_preds,
> all_v_scalar_ptr_uint8_index_ops)
> +DEF_RVV_FUNCTION (vsoxei16, indexed_loadstore, none_m_preds,
> all_v_scalar_ptr_uint16_index_ops)
> +DEF_RVV_FUNCTION (vsoxei32, indexed_loadstore, none_m_preds,
> all_v_scalar_ptr_uint32_index_ops)
> +DEF_RVV_FUNCTION (vsoxei64, indexed_loadstore, none_m_preds,
> all_v_scalar_ptr_uint64_index_ops)
>
> #undef DEF_RVV_FUNCTION
> diff --git a/gcc/config/riscv/riscv-vector-builtins-shapes.cc
> b/gcc/config/riscv/riscv-vector-builtins-shapes.cc
> index ef3a31555db..d261dfbceb7 100644
> --- a/gcc/config/riscv/riscv-vector-builtins-shapes.cc
> +++ b/gcc/config/riscv/riscv-vector-builtins-shapes.cc
> @@ -137,8 +137,57 @@ struct loadstore_def : public build_base
> }
> };
>
> +/* indexed_loadstore_def class. */
> +struct indexed_loadstore_def : public function_shape
> +{
> + void build (function_builder &b,
> + const function_group_info &group) const override
> + {
> + for (unsigned int pred_idx = 0; group.preds[pred_idx] != NUM_PRED_TYPES;
> + ++pred_idx)
> + {
> + for (unsigned int vec_type_idx = 0;
> + group.ops_infos.types[vec_type_idx].index != NUM_VECTOR_TYPES;
> + ++vec_type_idx)
> + {
> + tree index_type = group.ops_infos.args[1].get_tree_type (
> + group.ops_infos.types[vec_type_idx].index);
> + if (!index_type)
> + continue;
> + build_one (b, group, pred_idx, vec_type_idx);
> + }
> + }
> + }
> +
> + char *get_name (function_builder &b, const function_instance &instance,
> + bool overloaded_p) const override
> + {
> + /* Return nullptr if it can not be overloaded. */
> + if (overloaded_p && !instance.base->can_be_overloaded_p (instance.pred))
> + return nullptr;
> +
> + b.append_base_name (instance.base_name);
> + /* vop<sew>_v --> vop<sew>_v_<type>. */
> + if (!overloaded_p)
> + {
> + /* vop<sew> --> vop<sew>_v. */
> + b.append_name (operand_suffixes[instance.op_info->op]);
> + /* vop<sew>_v --> vop<sew>_v_<type>. */
> + b.append_name (type_suffixes[instance.type.index].vector);
> + }
> +
> + /* According to rvv-intrinsic-doc, it does not add "_m" suffix
> + for vop_m C++ overloaded API. */
> + if (overloaded_p && instance.pred == PRED_TYPE_m)
> + return b.finish_name ();
> + b.append_name (predication_suffixes[instance.pred]);
> + return b.finish_name ();
> + }
> +};
> +
> SHAPE(vsetvl, vsetvl)
> SHAPE(vsetvl, vsetvlmax)
> SHAPE(loadstore, loadstore)
> +SHAPE(indexed_loadstore, indexed_loadstore)
>
> } // end namespace riscv_vector
> diff --git a/gcc/config/riscv/riscv-vector-builtins-shapes.h
> b/gcc/config/riscv/riscv-vector-builtins-shapes.h
> index 642769208ee..05bc68b7f12 100644
> --- a/gcc/config/riscv/riscv-vector-builtins-shapes.h
> +++ b/gcc/config/riscv/riscv-vector-builtins-shapes.h
> @@ -27,6 +27,7 @@ namespace shapes {
> extern const function_shape *const vsetvl;
> extern const function_shape *const vsetvlmax;
> extern const function_shape *const loadstore;
> +extern const function_shape *const indexed_loadstore;
> }
>
> } // end namespace riscv_vector
> diff --git a/gcc/config/riscv/riscv-vector-builtins.cc
> b/gcc/config/riscv/riscv-vector-builtins.cc
> index 593a5f08e69..df4e2ee1841 100644
> --- a/gcc/config/riscv/riscv-vector-builtins.cc
> +++ b/gcc/config/riscv/riscv-vector-builtins.cc
> @@ -180,6 +180,58 @@ static CONSTEXPR const rvv_arg_type_info
> scalar_ptr_ptrdiff_args[]
> rvv_arg_type_info (RVV_BASE_ptrdiff), rvv_arg_type_info
> (RVV_BASE_vector),
> rvv_arg_type_info_end};
>
> +/* A list of args for vector_type func (const scalar_type *,
> uint8_index_type)
> + * function. */
> +static CONSTEXPR const rvv_arg_type_info scalar_const_ptr_uint8_index_args[]
> + = {rvv_arg_type_info (RVV_BASE_scalar_const_ptr),
> + rvv_arg_type_info (RVV_BASE_uint8_index), rvv_arg_type_info_end};
> +
> +/* A list of args for vector_type func (const scalar_type *,
> uint16_index_type)
> + * function. */
> +static CONSTEXPR const rvv_arg_type_info scalar_const_ptr_uint16_index_args[]
> + = {rvv_arg_type_info (RVV_BASE_scalar_const_ptr),
> + rvv_arg_type_info (RVV_BASE_uint16_index), rvv_arg_type_info_end};
> +
> +/* A list of args for vector_type func (const scalar_type *,
> uint32_index_type)
> + * function. */
> +static CONSTEXPR const rvv_arg_type_info scalar_const_ptr_uint32_index_args[]
> + = {rvv_arg_type_info (RVV_BASE_scalar_const_ptr),
> + rvv_arg_type_info (RVV_BASE_uint32_index), rvv_arg_type_info_end};
> +
> +/* A list of args for vector_type func (const scalar_type *,
> uint64_index_type)
> + * function. */
> +static CONSTEXPR const rvv_arg_type_info scalar_const_ptr_uint64_index_args[]
> + = {rvv_arg_type_info (RVV_BASE_scalar_const_ptr),
> + rvv_arg_type_info (RVV_BASE_uint64_index), rvv_arg_type_info_end};
> +
> +/* A list of args for void func (scalar_type *, uint8_index_type,
> vector_type)
> + * function. */
> +static CONSTEXPR const rvv_arg_type_info scalar_ptr_uint8_index_args[]
> + = {rvv_arg_type_info (RVV_BASE_scalar_ptr),
> + rvv_arg_type_info (RVV_BASE_uint8_index),
> + rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info_end};
> +
> +/* A list of args for void func (scalar_type *, uint16_index_type,
> vector_type)
> + * function. */
> +static CONSTEXPR const rvv_arg_type_info scalar_ptr_uint16_index_args[]
> + = {rvv_arg_type_info (RVV_BASE_scalar_ptr),
> + rvv_arg_type_info (RVV_BASE_uint16_index),
> + rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info_end};
> +
> +/* A list of args for void func (scalar_type *, uint32_index_type,
> vector_type)
> + * function. */
> +static CONSTEXPR const rvv_arg_type_info scalar_ptr_uint32_index_args[]
> + = {rvv_arg_type_info (RVV_BASE_scalar_ptr),
> + rvv_arg_type_info (RVV_BASE_uint32_index),
> + rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info_end};
> +
> +/* A list of args for void func (scalar_type *, uint64_index_type,
> vector_type)
> + * function. */
> +static CONSTEXPR const rvv_arg_type_info scalar_ptr_uint64_index_args[]
> + = {rvv_arg_type_info (RVV_BASE_scalar_ptr),
> + rvv_arg_type_info (RVV_BASE_uint64_index),
> + rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info_end};
> +
> /* A list of none preds that will be registered for intrinsic functions. */
> static CONSTEXPR const predication_type_index none_preds[]
> = {PRED_TYPE_none, NUM_PRED_TYPES};
> @@ -248,6 +300,38 @@ static CONSTEXPR const rvv_op_info
> all_v_scalar_const_ptr_ptrdiff_ops
> rvv_arg_type_info (RVV_BASE_vector), /* Return type */
> scalar_const_ptr_ptrdiff_args /* Args */};
>
> +/* A static operand information for vector_type func (const scalar_type *,
> + * uint8_index_type) function registration. */
> +static CONSTEXPR const rvv_op_info all_v_scalar_const_ptr_uint8_index_ops
> + = {all_ops, /* Types */
> + OP_TYPE_v, /* Suffix */
> + rvv_arg_type_info (RVV_BASE_vector), /* Return type */
> + scalar_const_ptr_uint8_index_args /* Args */};
> +
> +/* A static operand information for vector_type func (const scalar_type *,
> + * uint16_index_type) function registration. */
> +static CONSTEXPR const rvv_op_info all_v_scalar_const_ptr_uint16_index_ops
> + = {all_ops, /* Types */
> + OP_TYPE_v, /* Suffix */
> + rvv_arg_type_info (RVV_BASE_vector), /* Return type */
> + scalar_const_ptr_uint16_index_args /* Args */};
> +
> +/* A static operand information for vector_type func (const scalar_type *,
> + * uint32_index_type) function registration. */
> +static CONSTEXPR const rvv_op_info all_v_scalar_const_ptr_uint32_index_ops
> + = {all_ops, /* Types */
> + OP_TYPE_v, /* Suffix */
> + rvv_arg_type_info (RVV_BASE_vector), /* Return type */
> + scalar_const_ptr_uint32_index_args /* Args */};
> +
> +/* A static operand information for vector_type func (const scalar_type *,
> + * uint64_index_type) function registration. */
> +static CONSTEXPR const rvv_op_info all_v_scalar_const_ptr_uint64_index_ops
> + = {all_ops, /* Types */
> + OP_TYPE_v, /* Suffix */
> + rvv_arg_type_info (RVV_BASE_vector), /* Return type */
> + scalar_const_ptr_uint64_index_args /* Args */};
> +
> /* A static operand information for void func (scalar_type *, ptrdiff_t,
> * vector_type) function registration. */
> static CONSTEXPR const rvv_op_info all_v_scalar_ptr_ptrdiff_ops
> @@ -256,6 +340,38 @@ static CONSTEXPR const rvv_op_info
> all_v_scalar_ptr_ptrdiff_ops
> rvv_arg_type_info (RVV_BASE_void), /* Return type */
> scalar_ptr_ptrdiff_args /* Args */};
>
> +/* A static operand information for void func (scalar_type *,
> uint8_index_type,
> + * vector_type) function registration. */
> +static CONSTEXPR const rvv_op_info all_v_scalar_ptr_uint8_index_ops
> + = {all_ops, /* Types */
> + OP_TYPE_v, /* Suffix */
> + rvv_arg_type_info (RVV_BASE_void), /* Return type */
> + scalar_ptr_uint8_index_args /* Args */};
> +
> +/* A static operand information for void func (scalar_type *,
> uint16_index_type,
> + * vector_type) function registration. */
> +static CONSTEXPR const rvv_op_info all_v_scalar_ptr_uint16_index_ops
> + = {all_ops, /* Types */
> + OP_TYPE_v, /* Suffix */
> + rvv_arg_type_info (RVV_BASE_void), /* Return type */
> + scalar_ptr_uint16_index_args /* Args */};
> +
> +/* A static operand information for void func (scalar_type *,
> uint32_index_type,
> + * vector_type) function registration. */
> +static CONSTEXPR const rvv_op_info all_v_scalar_ptr_uint32_index_ops
> + = {all_ops, /* Types */
> + OP_TYPE_v, /* Suffix */
> + rvv_arg_type_info (RVV_BASE_void), /* Return type */
> + scalar_ptr_uint32_index_args /* Args */};
> +
> +/* A static operand information for void func (scalar_type *,
> uint64_index_type,
> + * vector_type) function registration. */
> +static CONSTEXPR const rvv_op_info all_v_scalar_ptr_uint64_index_ops
> + = {all_ops, /* Types */
> + OP_TYPE_v, /* Suffix */
> + rvv_arg_type_info (RVV_BASE_void), /* Return type */
> + scalar_ptr_uint64_index_args /* Args */};
> +
> /* A list of all RVV intrinsic functions. */
> static function_group_info function_groups[] = {
> #define DEF_RVV_FUNCTION(NAME, SHAPE, PREDS, OPS_INFO)
> \
> @@ -444,11 +560,41 @@ register_vector_type (vector_type_index type)
> builtin_types[type].vector_ptr = build_pointer_type (vectype);
> }
>
> +/* Return true if the type has required_extensions. */
> +static bool
> +required_extensions_p (enum rvv_base_type type)
> +{
> + return type == RVV_BASE_vector || type == RVV_BASE_uint8_index
> + || type == RVV_BASE_uint16_index || type == RVV_BASE_uint32_index
> + || type == RVV_BASE_uint64_index;
> +}
> +
> /* Check whether all the RVV_REQUIRE_* values in REQUIRED_EXTENSIONS are
> enabled. */
> static bool
> -check_required_extensions (uint64_t required_extensions)
> +check_required_extensions (const function_instance &instance)
> {
> + rvv_type_info type_info = instance.type;
> + uint64_t required_extensions = type_info.required_extensions;
> + const rvv_op_info *op_info = instance.op_info;
> + tree type = builtin_types[type_info.index].vector;
> + for (unsigned i = 0; op_info->args[i].base_type != NUM_BASE_TYPES; ++i)
> + {
> + if (!required_extensions_p (op_info->args[i].base_type))
> + continue;
> +
> + enum vector_type_index vector_type
> + = op_info->args[i].get_base_vector_type (type);
> + if (vector_type == NUM_VECTOR_TYPES)
> + continue;
> + required_extensions |= op_info->types[vector_type].required_extensions;
> +
> + /* According to RVV ISA, EEW=64 index of indexed loads/stores require
> + XLEN = 64. */
> + if (op_info->args[i].base_type == RVV_BASE_uint64_index)
> + required_extensions |= RVV_REQUIRE_RV64BIT;
> + }
> +
> uint64_t riscv_isa_flags = 0;
>
> if (TARGET_VECTOR_ELEN_FP_32)
> @@ -502,6 +648,56 @@ get_mask_policy_for_pred (enum predication_type_index
> pred)
> return gen_int_mode (get_prefer_mask_policy (), Pmode);
> }
>
> +vector_type_index
> +rvv_arg_type_info::get_base_vector_type (tree type) const
> +{
> + if (!type)
> + return NUM_VECTOR_TYPES;
> + poly_int64 nunits = GET_MODE_NUNITS (TYPE_MODE (type));
> + machine_mode inner_mode;
> + bool unsigned_p = TYPE_UNSIGNED (type);
> + switch (base_type)
> + {
> + case RVV_BASE_uint8_index:
> + inner_mode = E_QImode;
> + unsigned_p = true;
> + break;
> + case RVV_BASE_uint16_index:
> + inner_mode = E_HImode;
> + unsigned_p = true;
> + break;
> + case RVV_BASE_uint32_index:
> + inner_mode = E_SImode;
> + unsigned_p = true;
> + break;
> + case RVV_BASE_uint64_index:
> + inner_mode = E_DImode;
> + unsigned_p = true;
> + break;
> + default:
> + return NUM_VECTOR_TYPES;
> + }
> +
> + opt_machine_mode mode
> + = get_vector_mode (as_a<scalar_mode> (inner_mode), nunits);
> +
> + if (!mode.exists ())
> + return NUM_VECTOR_TYPES;
> + for (unsigned int i = 0; i < NUM_VECTOR_TYPES + 1; i++)
> + {
> + tree vector_type = builtin_types[i].vector;
> + if (!vector_type)
> + continue;
> +
> + if (TYPE_UNSIGNED (vector_type) != unsigned_p)
> + continue;
> +
> + if (TYPE_MODE (vector_type) == mode.require ())
> + return (enum vector_type_index) i;
> + }
> + return NUM_VECTOR_TYPES;
> +}
> +
> tree
> rvv_arg_type_info::get_tree_type (vector_type_index type_idx) const
> {
> @@ -537,9 +733,20 @@ rvv_arg_type_info::get_tree_type (vector_type_index
> type_idx) const
> return long_unsigned_type_node;
> case RVV_BASE_long:
> return long_integer_type_node;
> + case RVV_BASE_uint8_index:
> + case RVV_BASE_uint16_index:
> + case RVV_BASE_uint32_index:
> + case RVV_BASE_uint64_index:
> + if (get_base_vector_type (builtin_types[type_idx].vector)
> + != NUM_VECTOR_TYPES)
> + return builtin_types[get_base_vector_type (
> + builtin_types[type_idx].vector)].vector;
> + break;
> default:
> gcc_unreachable ();
> }
> + /* Return NULL_TREE if the type we don't want to register. */
> + return NULL_TREE;
> }
>
> function_instance::function_instance (const char *base_name_in,
> @@ -839,7 +1046,7 @@ function_builder::add_unique_function (const
> function_instance &instance,
> vec<tree> &argument_types)
> {
> /* Do not add this function if it is invalid. */
> - if (!check_required_extensions (instance.type.required_extensions))
> + if (!check_required_extensions (instance))
> return;
>
> /* Add the function under its full (unique) name. */
> @@ -918,6 +1125,44 @@ function_expander::add_mem_operand (machine_mode mode,
> unsigned argno)
> add_fixed_operand (mem);
> }
>
> +/* Implement the call using instruction ICODE, with a 1:1 mapping between
> + arguments and input operands. */
> +rtx
> +function_expander::use_exact_insn (insn_code icode)
> +{
> + machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
> + tree mask_type = builtin_types[mask_types[type.index]].vector;
> + machine_mode mask_mode = TYPE_MODE (mask_type);
> +
> + /* Record the offset to get the argument. */
> + int arg_offset = 0;
> +
> + if (use_real_mask_p (pred))
> + add_input_operand (arg_offset++);
> + else
> + add_all_one_mask_operand (mask_mode);
> +
> + /* Store operation doesn't have merge operand. */
> + if (!function_returns_void_p ())
> + {
> + if (use_real_merge_p (pred))
> + add_input_operand (arg_offset++);
> + else
> + add_vundef_operand (mode);
> + }
> +
> + for (int argno = arg_offset; argno < call_expr_nargs (exp); argno++)
> + add_input_operand (argno);
> +
> + if (base->apply_tail_policy_p ())
> + add_input_operand (Pmode, get_tail_policy_for_pred (pred));
> + if (base->apply_mask_policy_p ())
> + add_input_operand (Pmode, get_mask_policy_for_pred (pred));
> +
> + add_input_operand (Pmode, get_avl_type_rtx (avl_type::NONVLMAX));
> + return generate_insn (icode);
> +}
> +
> /* Use contiguous load INSN. */
> rtx
> function_expander::use_contiguous_load_insn (insn_code icode)
> diff --git a/gcc/config/riscv/riscv-vector-builtins.h
> b/gcc/config/riscv/riscv-vector-builtins.h
> index fb3f818e15a..7a4a4d4f74d 100644
> --- a/gcc/config/riscv/riscv-vector-builtins.h
> +++ b/gcc/config/riscv/riscv-vector-builtins.h
> @@ -147,6 +147,10 @@ enum rvv_base_type
> RVV_BASE_ptrdiff,
> RVV_BASE_unsigned_long,
> RVV_BASE_long,
> + RVV_BASE_uint8_index,
> + RVV_BASE_uint16_index,
> + RVV_BASE_uint32_index,
> + RVV_BASE_uint64_index,
> NUM_BASE_TYPES
> };
>
> @@ -176,6 +180,7 @@ struct rvv_arg_type_info
> {}
> enum rvv_base_type base_type;
>
> + vector_type_index get_base_vector_type (tree type) const;
> tree get_tree_type (vector_type_index) const;
> };
>
> @@ -325,7 +330,9 @@ public:
> void add_mem_operand (machine_mode, unsigned);
>
> machine_mode vector_mode (void) const;
> + machine_mode index_mode (void) const;
>
> + rtx use_exact_insn (insn_code);
> rtx use_contiguous_load_insn (insn_code);
> rtx use_contiguous_store_insn (insn_code);
> rtx generate_insn (insn_code);
> @@ -358,6 +365,12 @@ public:
> /* Return true if intrinsics should apply vl operand. */
> virtual bool apply_vl_p () const;
>
> + /* Return true if intrinsics should apply ta operand. */
> + virtual bool apply_tail_policy_p () const;
> +
> + /* Return true if intrinsics should apply ma operand. */
> + virtual bool apply_mask_policy_p () const;
> +
> /* Return true if intrinsic can be overloaded. */
> virtual bool can_be_overloaded_p (enum predication_type_index) const;
>
> @@ -444,6 +457,13 @@ function_expander::vector_mode (void) const
> return TYPE_MODE (builtin_types[type.index].vector);
> }
>
> +/* Return the machine_mode of the corresponding index type. */
> +inline machine_mode
> +function_expander::index_mode (void) const
> +{
> + return TYPE_MODE (op_info->args[1].get_tree_type (type.index));
> +}
> +
> /* Default implementation of function_base::call_properties, with
> conservatively
> correct behavior for floating-point instructions. */
> inline unsigned int
> @@ -463,6 +483,22 @@ function_base::apply_vl_p () const
> return true;
> }
>
> +/* We choose to apply ta operand by default since most of the intrinsics
> + has vl operand. */
> +inline bool
> +function_base::apply_tail_policy_p () const
> +{
> + return true;
> +}
> +
> +/* We choose to apply ma operand by default since most of the intrinsics
> + has ma operand. */
> +inline bool
> +function_base::apply_mask_policy_p () const
> +{
> + return true;
> +}
> +
> /* Since most of intrinsics can be overloaded, we set it true by default. */
> inline bool
> function_base::can_be_overloaded_p (enum predication_type_index) const
> diff --git a/gcc/config/riscv/vector-iterators.md
> b/gcc/config/riscv/vector-iterators.md
> index c1ec23452bc..2ac75b3f94c 100644
> --- a/gcc/config/riscv/vector-iterators.md
> +++ b/gcc/config/riscv/vector-iterators.md
> @@ -18,6 +18,18 @@
> ;; along with GCC; see the file COPYING3. If not see
> ;; <http://www.gnu.org/licenses/>.
>
> +(define_c_enum "unspec" [
> + UNSPEC_VSETVL
> + UNSPEC_VUNDEF
> + UNSPEC_VPREDICATE
> + UNSPEC_VLMAX
> + UNSPEC_STRIDED
> +
> + ;; It's used to specify ordered/unorderd operation.
> + UNSPEC_ORDERED
> + UNSPEC_UNORDERED
> +])
> +
> (define_mode_iterator V [
> VNx1QI VNx2QI VNx4QI VNx8QI VNx16QI VNx32QI (VNx64QI "TARGET_MIN_VLEN >
> 32")
> VNx1HI VNx2HI VNx4HI VNx8HI VNx16HI (VNx32HI "TARGET_MIN_VLEN > 32")
> @@ -35,6 +47,75 @@
> (VNx8DF "TARGET_VECTOR_ELEN_FP_64")
> ])
>
> +(define_mode_iterator VNX1_QHSD [
> + VNx1QI VNx1HI VNx1SI
> + (VNx1DI "TARGET_MIN_VLEN > 32")
> + (VNx1SF "TARGET_VECTOR_ELEN_FP_32")
> + (VNx1DF "TARGET_VECTOR_ELEN_FP_64")
> +])
> +
> +(define_mode_iterator VNX2_QHSD [
> + VNx2QI VNx2HI VNx2SI
> + (VNx2DI "TARGET_MIN_VLEN > 32")
> + (VNx2SF "TARGET_VECTOR_ELEN_FP_32")
> + (VNx2DF "TARGET_VECTOR_ELEN_FP_64")
> +])
> +
> +(define_mode_iterator VNX4_QHSD [
> + VNx4QI VNx4HI VNx4SI
> + (VNx4DI "TARGET_MIN_VLEN > 32")
> + (VNx4SF "TARGET_VECTOR_ELEN_FP_32")
> + (VNx4DF "TARGET_VECTOR_ELEN_FP_64")
> +])
> +
> +(define_mode_iterator VNX8_QHSD [
> + VNx8QI VNx8HI VNx8SI
> + (VNx8DI "TARGET_MIN_VLEN > 32")
> + (VNx8SF "TARGET_VECTOR_ELEN_FP_32")
> + (VNx8DF "TARGET_VECTOR_ELEN_FP_64")
> +])
> +
> +(define_mode_iterator VNX16_QHS [
> + VNx16QI VNx16HI (VNx16SI "TARGET_MIN_VLEN > 32")
> + (VNx16SF "TARGET_VECTOR_ELEN_FP_32 && TARGET_MIN_VLEN > 32")
> +])
> +
> +(define_mode_iterator VNX32_QH [
> + VNx32QI (VNx32HI "TARGET_MIN_VLEN > 32")
> +])
> +
> +(define_mode_iterator VNX64_Q [
> + (VNx64QI "TARGET_MIN_VLEN > 32")
> +])
> +
> +(define_mode_iterator VNX1_QHSDI [
> + VNx1QI VNx1HI VNx1SI
> + (VNx1DI "TARGET_64BIT && TARGET_MIN_VLEN > 32")
> +])
> +
> +(define_mode_iterator VNX2_QHSDI [
> + VNx2QI VNx2HI VNx2SI
> + (VNx2DI "TARGET_64BIT && TARGET_MIN_VLEN > 32")
> +])
> +
> +(define_mode_iterator VNX4_QHSDI [
> + VNx4QI VNx4HI VNx4SI
> + (VNx4DI "TARGET_64BIT && TARGET_MIN_VLEN > 32")
> +])
> +
> +(define_mode_iterator VNX8_QHSDI [
> + VNx8QI VNx8HI VNx8SI
> + (VNx8DI "TARGET_64BIT && TARGET_MIN_VLEN > 32")
> +])
> +
> +(define_mode_iterator VNX16_QHSI [
> + VNx16QI VNx16HI (VNx16SI "TARGET_MIN_VLEN > 32")
> +])
> +
> +(define_mode_iterator VNX32_QHI [
> + VNx32QI (VNx32HI "TARGET_MIN_VLEN > 32")
> +])
> +
> (define_mode_iterator V_WHOLE [
> (VNx4QI "TARGET_MIN_VLEN == 32") VNx8QI VNx16QI VNx32QI (VNx64QI
> "TARGET_MIN_VLEN > 32")
> (VNx2HI "TARGET_MIN_VLEN == 32") VNx4HI VNx8HI VNx16HI (VNx32HI
> "TARGET_MIN_VLEN > 32")
> @@ -90,3 +171,9 @@
> (VNx1SF "32") (VNx2SF "32") (VNx4SF "32") (VNx8SF "32") (VNx16SF "32")
> (VNx1DF "64") (VNx2DF "64") (VNx4DF "64") (VNx8DF "64")
> ])
> +
> +(define_int_iterator ORDER [UNSPEC_ORDERED UNSPEC_UNORDERED])
> +
> +(define_int_attr order [
> + (UNSPEC_ORDERED "o") (UNSPEC_UNORDERED "u")
> +])
> diff --git a/gcc/config/riscv/vector.md b/gcc/config/riscv/vector.md
> index 1453be116a9..403d2f6cda3 100644
> --- a/gcc/config/riscv/vector.md
> +++ b/gcc/config/riscv/vector.md
> @@ -28,14 +28,6 @@
>
> (include "vector-iterators.md")
>
> -(define_c_enum "unspec" [
> - UNSPEC_VSETVL
> - UNSPEC_VUNDEF
> - UNSPEC_VPREDICATE
> - UNSPEC_VLMAX
> - UNSPEC_STRIDED
> -])
> -
> (define_constants [
> (INVALID_ATTRIBUTE 255)
> ])
> @@ -149,7 +141,7 @@
>
> ;; It is valid for instruction that require sew/lmul ratio.
> (define_attr "ratio" ""
> - (cond [(eq_attr "type" "vimov,vfmov")
> + (cond [(eq_attr "type" "vimov,vfmov,vldux,vldox,vstux,vstox")
> (const_int INVALID_ATTRIBUTE)
> (eq_attr "mode" "VNx1QI,VNx1BI")
> (symbol_ref "riscv_vector::get_ratio(E_VNx1QImode)")
> @@ -199,13 +191,13 @@
>
> ;; The index of operand[] to get the merge op.
> (define_attr "merge_op_idx" ""
> - (cond [(eq_attr "type" "vlde,vimov,vfmov,vldm,vlds,vmalu")
> + (cond [(eq_attr "type" "vlde,vimov,vfmov,vldm,vlds,vmalu,vldux,vldox")
> (const_int 2)]
> (const_int INVALID_ATTRIBUTE)))
>
> ;; The index of operand[] to get the avl op.
> (define_attr "vl_op_idx" ""
> - (cond [(eq_attr "type" "vlde,vste,vimov,vfmov,vldm,vstm,vmalu,vsts")
> + (cond [(eq_attr "type"
> "vlde,vste,vimov,vfmov,vldm,vstm,vmalu,vsts,vstux,vstox")
> (const_int 4)
>
> ;; If operands[3] of "vlds" is not vector mode, it is pred_broadcast.
> @@ -213,7 +205,10 @@
> (eq_attr "type" "vlds")
> (if_then_else (match_test "VECTOR_MODE_P (GET_MODE (operands[3]))")
> (const_int 5)
> - (const_int 4))]
> + (const_int 4))
> +
> + (eq_attr "type" "vldux,vldox")
> + (const_int 5)]
> (const_int INVALID_ATTRIBUTE)))
>
> ;; The tail policy op value.
> @@ -226,7 +221,10 @@
> (eq_attr "type" "vlds")
> (if_then_else (match_test "VECTOR_MODE_P (GET_MODE (operands[3]))")
> (symbol_ref "riscv_vector::get_ta(operands[6])")
> - (symbol_ref "riscv_vector::get_ta(operands[5])"))]
> + (symbol_ref "riscv_vector::get_ta(operands[5])"))
> +
> + (eq_attr "type" "vldux,vldox")
> + (symbol_ref "riscv_vector::get_ta(operands[6])")]
> (const_int INVALID_ATTRIBUTE)))
>
> ;; The mask policy op value.
> @@ -239,7 +237,10 @@
> (eq_attr "type" "vlds")
> (if_then_else (match_test "VECTOR_MODE_P (GET_MODE (operands[3]))")
> (symbol_ref "riscv_vector::get_ma(operands[7])")
> - (symbol_ref "riscv_vector::get_ma(operands[6])"))]
> + (symbol_ref "riscv_vector::get_ma(operands[6])"))
> +
> + (eq_attr "type" "vldux,vldox")
> + (symbol_ref "riscv_vector::get_ma(operands[7])")]
> (const_int INVALID_ATTRIBUTE)))
>
> ;; The avl type value.
> @@ -254,7 +255,12 @@
> (eq_attr "type" "vlds")
> (if_then_else (match_test "VECTOR_MODE_P (GET_MODE (operands[3]))")
> (const_int INVALID_ATTRIBUTE)
> - (symbol_ref "INTVAL (operands[7])"))]
> + (symbol_ref "INTVAL (operands[7])"))
> +
> + (eq_attr "type" "vldux,vldox")
> + (symbol_ref "INTVAL (operands[8])")
> + (eq_attr "type" "vstux,vstox")
> + (symbol_ref "INTVAL (operands[5])")]
> (const_int INVALID_ATTRIBUTE)))
>
> ;; -----------------------------------------------------------------
> @@ -832,3 +838,276 @@
> "vsse<sew>.v\t%3,%0,%z2%p1"
> [(set_attr "type" "vsts")
> (set_attr "mode" "<MODE>")])
> +
> +;;
> -------------------------------------------------------------------------------
> +;; ---- Predicated indexed loads/stores
> +;;
> -------------------------------------------------------------------------------
> +;; Includes:
> +;; - 7.6. Vector Indexed Instructions
> +;;
> -------------------------------------------------------------------------------
> +
> +(define_insn "@pred_indexed_<order>load<VNX1_QHSD:mode><VNX1_QHSDI:mode>"
> + [(set (match_operand:VNX1_QHSD 0 "register_operand" "=&vr")
> + (if_then_else:VNX1_QHSD
> + (unspec:<VM>
> + [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1")
> + (match_operand 5 "vector_length_operand" " rK")
> + (match_operand 6 "const_int_operand" " i")
> + (match_operand 7 "const_int_operand" " i")
> + (match_operand 8 "const_int_operand" " i")
> + (reg:SI VL_REGNUM)
> + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
> + (unspec:VNX1_QHSD
> + [(match_operand 3 "pmode_register_operand" " r")
> + (mem:BLK (scratch))
> + (match_operand:VNX1_QHSDI 4 "register_operand" " vr")] ORDER)
> + (match_operand:VNX1_QHSD 2 "vector_merge_operand" "0vu")))]
> + "TARGET_VECTOR"
> + "vl<order>xei<VNX1_QHSDI:sew>.v\t%0,(%3),%4%p1"
> + [(set_attr "type" "vld<order>x")
> + (set_attr "mode" "<VNX1_QHSD:MODE>")])
> +
> +(define_insn "@pred_indexed_<order>store<VNX1_QHSD:mode><VNX1_QHSDI:mode>"
> + [(set (mem:BLK (scratch))
> + (unspec:BLK
> + [(unspec:<VM>
> + [(match_operand:<VM> 0 "vector_mask_operand" "vmWc1")
> + (match_operand 4 "vector_length_operand" " rK")
> + (match_operand 5 "const_int_operand" " i")
> + (reg:SI VL_REGNUM)
> + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
> + (match_operand 1 "pmode_register_operand" " r")
> + (match_operand:VNX1_QHSDI 2 "register_operand" " vr")
> + (match_operand:VNX1_QHSD 3 "register_operand" " vr")] ORDER))]
> + "TARGET_VECTOR"
> + "vs<order>xei<VNX1_QHSDI:sew>.v\t%3,(%1),%2%p0"
> + [(set_attr "type" "vst<order>x")
> + (set_attr "mode" "<VNX1_QHSD:MODE>")])
> +
> +(define_insn "@pred_indexed_<order>load<VNX2_QHSD:mode><VNX2_QHSDI:mode>"
> + [(set (match_operand:VNX2_QHSD 0 "register_operand" "=&vr")
> + (if_then_else:VNX2_QHSD
> + (unspec:<VM>
> + [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1")
> + (match_operand 5 "vector_length_operand" " rK")
> + (match_operand 6 "const_int_operand" " i")
> + (match_operand 7 "const_int_operand" " i")
> + (match_operand 8 "const_int_operand" " i")
> + (reg:SI VL_REGNUM)
> + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
> + (unspec:VNX2_QHSD
> + [(match_operand 3 "pmode_register_operand" " r")
> + (mem:BLK (scratch))
> + (match_operand:VNX2_QHSDI 4 "register_operand" " vr")] ORDER)
> + (match_operand:VNX2_QHSD 2 "vector_merge_operand" "0vu")))]
> + "TARGET_VECTOR"
> + "vl<order>xei<VNX2_QHSDI:sew>.v\t%0,(%3),%4%p1"
> + [(set_attr "type" "vld<order>x")
> + (set_attr "mode" "<VNX2_QHSD:MODE>")])
> +
> +(define_insn "@pred_indexed_<order>store<VNX2_QHSD:mode><VNX2_QHSDI:mode>"
> + [(set (mem:BLK (scratch))
> + (unspec:BLK
> + [(unspec:<VM>
> + [(match_operand:<VM> 0 "vector_mask_operand" "vmWc1")
> + (match_operand 4 "vector_length_operand" " rK")
> + (match_operand 5 "const_int_operand" " i")
> + (reg:SI VL_REGNUM)
> + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
> + (match_operand 1 "pmode_register_operand" " r")
> + (match_operand:VNX2_QHSDI 2 "register_operand" " vr")
> + (match_operand:VNX2_QHSD 3 "register_operand" " vr")] ORDER))]
> + "TARGET_VECTOR"
> + "vs<order>xei<VNX2_QHSDI:sew>.v\t%3,(%1),%2%p0"
> + [(set_attr "type" "vst<order>x")
> + (set_attr "mode" "<VNX2_QHSD:MODE>")])
> +
> +(define_insn "@pred_indexed_<order>load<VNX4_QHSD:mode><VNX4_QHSDI:mode>"
> + [(set (match_operand:VNX4_QHSD 0 "register_operand" "=&vr")
> + (if_then_else:VNX4_QHSD
> + (unspec:<VM>
> + [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1")
> + (match_operand 5 "vector_length_operand" " rK")
> + (match_operand 6 "const_int_operand" " i")
> + (match_operand 7 "const_int_operand" " i")
> + (match_operand 8 "const_int_operand" " i")
> + (reg:SI VL_REGNUM)
> + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
> + (unspec:VNX4_QHSD
> + [(match_operand 3 "pmode_register_operand" " r")
> + (mem:BLK (scratch))
> + (match_operand:VNX4_QHSDI 4 "register_operand" " vr")] ORDER)
> + (match_operand:VNX4_QHSD 2 "vector_merge_operand" "0vu")))]
> + "TARGET_VECTOR"
> + "vl<order>xei<VNX4_QHSDI:sew>.v\t%0,(%3),%4%p1"
> + [(set_attr "type" "vld<order>x")
> + (set_attr "mode" "<VNX4_QHSD:MODE>")])
> +
> +(define_insn "@pred_indexed_<order>store<VNX4_QHSD:mode><VNX4_QHSDI:mode>"
> + [(set (mem:BLK (scratch))
> + (unspec:BLK
> + [(unspec:<VM>
> + [(match_operand:<VM> 0 "vector_mask_operand" "vmWc1")
> + (match_operand 4 "vector_length_operand" " rK")
> + (match_operand 5 "const_int_operand" " i")
> + (reg:SI VL_REGNUM)
> + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
> + (match_operand 1 "pmode_register_operand" " r")
> + (match_operand:VNX4_QHSDI 2 "register_operand" " vr")
> + (match_operand:VNX4_QHSD 3 "register_operand" " vr")] ORDER))]
> + "TARGET_VECTOR"
> + "vs<order>xei<VNX4_QHSDI:sew>.v\t%3,(%1),%2%p0"
> + [(set_attr "type" "vst<order>x")
> + (set_attr "mode" "<VNX4_QHSD:MODE>")])
> +
> +(define_insn "@pred_indexed_<order>load<VNX8_QHSD:mode><VNX8_QHSDI:mode>"
> + [(set (match_operand:VNX8_QHSD 0 "register_operand" "=&vr")
> + (if_then_else:VNX8_QHSD
> + (unspec:<VM>
> + [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1")
> + (match_operand 5 "vector_length_operand" " rK")
> + (match_operand 6 "const_int_operand" " i")
> + (match_operand 7 "const_int_operand" " i")
> + (match_operand 8 "const_int_operand" " i")
> + (reg:SI VL_REGNUM)
> + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
> + (unspec:VNX8_QHSD
> + [(match_operand 3 "pmode_register_operand" " r")
> + (mem:BLK (scratch))
> + (match_operand:VNX8_QHSDI 4 "register_operand" " vr")] ORDER)
> + (match_operand:VNX8_QHSD 2 "vector_merge_operand" "0vu")))]
> + "TARGET_VECTOR"
> + "vl<order>xei<VNX8_QHSDI:sew>.v\t%0,(%3),%4%p1"
> + [(set_attr "type" "vld<order>x")
> + (set_attr "mode" "<VNX8_QHSD:MODE>")])
> +
> +(define_insn "@pred_indexed_<order>store<VNX8_QHSD:mode><VNX8_QHSDI:mode>"
> + [(set (mem:BLK (scratch))
> + (unspec:BLK
> + [(unspec:<VM>
> + [(match_operand:<VM> 0 "vector_mask_operand" "vmWc1")
> + (match_operand 4 "vector_length_operand" " rK")
> + (match_operand 5 "const_int_operand" " i")
> + (reg:SI VL_REGNUM)
> + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
> + (match_operand 1 "pmode_register_operand" " r")
> + (match_operand:VNX8_QHSDI 2 "register_operand" " vr")
> + (match_operand:VNX8_QHSD 3 "register_operand" " vr")] ORDER))]
> + "TARGET_VECTOR"
> + "vs<order>xei<VNX8_QHSDI:sew>.v\t%3,(%1),%2%p0"
> + [(set_attr "type" "vst<order>x")
> + (set_attr "mode" "<VNX8_QHSD:MODE>")])
> +
> +(define_insn "@pred_indexed_<order>load<VNX16_QHS:mode><VNX16_QHSI:mode>"
> + [(set (match_operand:VNX16_QHS 0 "register_operand" "=&vr")
> + (if_then_else:VNX16_QHS
> + (unspec:<VM>
> + [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1")
> + (match_operand 5 "vector_length_operand" " rK")
> + (match_operand 6 "const_int_operand" " i")
> + (match_operand 7 "const_int_operand" " i")
> + (match_operand 8 "const_int_operand" " i")
> + (reg:SI VL_REGNUM)
> + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
> + (unspec:VNX16_QHS
> + [(match_operand 3 "pmode_register_operand" " r")
> + (mem:BLK (scratch))
> + (match_operand:VNX16_QHSI 4 "register_operand" " vr")] ORDER)
> + (match_operand:VNX16_QHS 2 "vector_merge_operand" "0vu")))]
> + "TARGET_VECTOR"
> + "vl<order>xei<VNX16_QHSI:sew>.v\t%0,(%3),%4%p1"
> + [(set_attr "type" "vld<order>x")
> + (set_attr "mode" "<VNX16_QHS:MODE>")])
> +
> +(define_insn "@pred_indexed_<order>store<VNX16_QHS:mode><VNX16_QHSI:mode>"
> + [(set (mem:BLK (scratch))
> + (unspec:BLK
> + [(unspec:<VM>
> + [(match_operand:<VM> 0 "vector_mask_operand" "vmWc1")
> + (match_operand 4 "vector_length_operand" " rK")
> + (match_operand 5 "const_int_operand" " i")
> + (reg:SI VL_REGNUM)
> + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
> + (match_operand 1 "pmode_register_operand" " r")
> + (match_operand:VNX16_QHSI 2 "register_operand" " vr")
> + (match_operand:VNX16_QHS 3 "register_operand" " vr")] ORDER))]
> + "TARGET_VECTOR"
> + "vs<order>xei<VNX16_QHSI:sew>.v\t%3,(%1),%2%p0"
> + [(set_attr "type" "vst<order>x")
> + (set_attr "mode" "<VNX16_QHS:MODE>")])
> +
> +(define_insn "@pred_indexed_<order>load<VNX32_QH:mode><VNX32_QHI:mode>"
> + [(set (match_operand:VNX32_QH 0 "register_operand" "=&vr")
> + (if_then_else:VNX32_QH
> + (unspec:<VM>
> + [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1")
> + (match_operand 5 "vector_length_operand" " rK")
> + (match_operand 6 "const_int_operand" " i")
> + (match_operand 7 "const_int_operand" " i")
> + (match_operand 8 "const_int_operand" " i")
> + (reg:SI VL_REGNUM)
> + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
> + (unspec:VNX32_QH
> + [(match_operand 3 "pmode_register_operand" " r")
> + (mem:BLK (scratch))
> + (match_operand:VNX32_QHI 4 "register_operand" " vr")] ORDER)
> + (match_operand:VNX32_QH 2 "vector_merge_operand" "0vu")))]
> + "TARGET_VECTOR"
> + "vl<order>xei<VNX32_QHI:sew>.v\t%0,(%3),%4%p1"
> + [(set_attr "type" "vld<order>x")
> + (set_attr "mode" "<VNX32_QH:MODE>")])
> +
> +(define_insn "@pred_indexed_<order>store<VNX32_QH:mode><VNX32_QHI:mode>"
> + [(set (mem:BLK (scratch))
> + (unspec:BLK
> + [(unspec:<VM>
> + [(match_operand:<VM> 0 "vector_mask_operand" "vmWc1")
> + (match_operand 4 "vector_length_operand" " rK")
> + (match_operand 5 "const_int_operand" " i")
> + (reg:SI VL_REGNUM)
> + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
> + (match_operand 1 "pmode_register_operand" " r")
> + (match_operand:VNX32_QHI 2 "register_operand" " vr")
> + (match_operand:VNX32_QH 3 "register_operand" " vr")] ORDER))]
> + "TARGET_VECTOR"
> + "vs<order>xei<VNX32_QHI:sew>.v\t%3,(%1),%2%p0"
> + [(set_attr "type" "vst<order>x")
> + (set_attr "mode" "<VNX32_QH:MODE>")])
> +
> +(define_insn "@pred_indexed_<order>load<VNX64_Q:mode><VNX64_Q:mode>"
> + [(set (match_operand:VNX64_Q 0 "register_operand" "=&vr")
> + (if_then_else:VNX64_Q
> + (unspec:<VM>
> + [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1")
> + (match_operand 5 "vector_length_operand" " rK")
> + (match_operand 6 "const_int_operand" " i")
> + (match_operand 7 "const_int_operand" " i")
> + (match_operand 8 "const_int_operand" " i")
> + (reg:SI VL_REGNUM)
> + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
> + (unspec:VNX64_Q
> + [(match_operand 3 "pmode_register_operand" " r")
> + (mem:BLK (scratch))
> + (match_operand:VNX64_Q 4 "register_operand" " vr")] ORDER)
> + (match_operand:VNX64_Q 2 "vector_merge_operand" "0vu")))]
> + "TARGET_VECTOR"
> + "vl<order>xei<VNX64_Q:sew>.v\t%0,(%3),%4%p1"
> + [(set_attr "type" "vld<order>x")
> + (set_attr "mode" "<VNX64_Q:MODE>")])
> +
> +(define_insn "@pred_indexed_<order>store<VNX64_Q:mode><VNX64_Q:mode>"
> + [(set (mem:BLK (scratch))
> + (unspec:BLK
> + [(unspec:<VM>
> + [(match_operand:<VM> 0 "vector_mask_operand" "vmWc1")
> + (match_operand 4 "vector_length_operand" " rK")
> + (match_operand 5 "const_int_operand" " i")
> + (reg:SI VL_REGNUM)
> + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
> + (match_operand 1 "pmode_register_operand" " r")
> + (match_operand:VNX64_Q 2 "register_operand" " vr")
> + (match_operand:VNX64_Q 3 "register_operand" " vr")] ORDER))]
> + "TARGET_VECTOR"
> + "vs<order>xei<VNX64_Q:sew>.v\t%3,(%1),%2%p0"
> + [(set_attr "type" "vst<order>x")
> + (set_attr "mode" "<VNX64_Q:MODE>")])
> --
> 2.36.3
>
