Ajit Agarwal <aagar...@linux.ibm.com> writes:
> [...]
> +static void
> +update_change (set_info *set)
> +{
> + if (!set->has_any_uses ())
> + return;
> +
> + auto *use = *set->all_uses ().begin ();
> + do
> + {
> + auto *next_use = use->next_use ();
> + if (use->is_in_phi ())
> + {
> + update_change (use->phi ());
> + }
> + else
> + {
> + crtl->ssa->remove_use (use);
This isn't right. AFAICT it's simply removing uses from a debug
instruction even though the debug instruction pattern still refers
to the register.
Instead, the debug instruction needs to be updated to refer to the
correct half of the new double-width load result. If the update fails
for any reason, the debug insn should be "reset" (i.e. changed to
INSN_VAR_LOCATION_LOC (use_rtl) = gen_rtx_UNKNOWN_VAR_LOC ();
see insn_combination::substitute_debug_use in late-combine.cc for
an example).
> + }
> + use = next_use;
> + }
> + while (use);
> +}
> [...]
> +// Generate new reg rtx with copy of OLD_DEST for OOmode pair.
> +static rtx
> +new_reg_rtx (rtx old_dest)
> +{
> + rtx new_dest_exp = gen_reg_rtx (OOmode);
> + ORIGINAL_REGNO (new_dest_exp) = ORIGINAL_REGNO (old_dest);
> + REG_USERVAR_P (new_dest_exp) = REG_USERVAR_P (old_dest);
> + REG_POINTER (new_dest_exp) = REG_POINTER (old_dest);
> + REG_ATTRS (new_dest_exp) = REG_ATTRS (old_dest);
> + max_regno = max_reg_num ();
> + return new_dest_exp;
> +}
The new register is a different size and mode from OLD_DEST, so it isn't
appropriate to copy across this much information. The caller should just
use gen_reg_rtx directly.
> +
> +// Set subreg with use of INSN given SRC rtx instruction.
> +static void
> +set_load_subreg (insn_info *i1, rtx src)
> +{
> + rtx set = single_set (i1->rtl());
> + rtx old_dest = SET_DEST (set);
> +
> + for (auto def : i1->defs ())
> + {
> + auto set = dyn_cast<set_info *> (def);
> + for (auto use : set->nondebug_insn_uses ())
> + {
> + insn_info *info = use->insn ();
> + if (!info || !info->rtl ())
> + continue;
> +
> + rtx_insn *rtl_insn = info->rtl ();
> + df_ref ref;
> +
> + FOR_EACH_INSN_USE (ref, rtl_insn)
> + {
> + rtx dest_exp = SET_DEST (PATTERN (i1->rtl ()));
> + if (REG_P (dest_exp)
> + && DF_REF_REGNO (ref) == REGNO (dest_exp))
> + {
> + rtx *loc = DF_REF_LOC (ref);
> + insn_propagation prop (rtl_insn, old_dest, src);
> + if (GET_CODE (*loc) == SUBREG)
> + {
> + if (!prop.apply_to_pattern (loc))
> + {
> + if (dump_file != NULL)
> + {
> + fprintf (dump_file,
> + "Cannot propagate insn \n");
> + print_rtl_single (dump_file, rtl_insn);
We can't simply ignore the failure, since it would leave the instruction
referring to the wrong register. We either need to assert that the failure
can't happen (dangerous) or bail out of the load fusion if propagation fails.
> + }
> + return;
> + }
> + }
> + else
> + *loc = copy_rtx (src);
> + }
> + }
> + }
We shouldn't have two ways of doing this. The above FOR_EACH_INSN_USE
loop should be replaced by something like:
insn_propagation prop (rtl_insn, old_dest, src);
if (!prop.apply_to_pattern (loc))
...
so that the substitution is done in one step, and only done using
insn_propagation.
Like I mentioned before, this change needs to be described as an
insn_change as well, so that rtl-ssa framework will see it. This is
important because later insn movement decisions require the def-use
information to be up-to-date.
> + }
> +}
> +
> +// Set subreg for OO mode store pair to generate registers in pairs
> +// given insn_info I1 and I2.
> +static void
> +set_multiword_subreg_store (insn_info *i1, insn_info *i2)
> +{
> + rtx_insn *insn1 = i1->rtl ();
> + rtx_insn *insn2 = i2->rtl ();
> + rtx body = PATTERN (insn1);
> + rtx src_exp = SET_SRC (body);
> + rtx insn2_body = PATTERN (insn2);
> + rtx insn2_dest_exp = SET_DEST (insn2_body);
> + machine_mode mode = GET_MODE (src_exp);
> + int regoff;
> + rtx src;
> + rtx addr = XEXP (insn2_dest_exp, 0);
> +
> + PUT_MODE_RAW (src_exp, OOmode);
> + if (GET_CODE (addr) == PLUS
> + && XEXP (addr, 1) && CONST_INT_P (XEXP (addr, 1)))
> + regoff = 16;
> + else
> + regoff = 0;
> +
> + src = simplify_gen_subreg (mode,
> + src_exp, GET_MODE (src_exp),
> + regoff);
> +
> + set_store_subreg (i1, src, regoff);
> +
> + int regoff1 = 0;
> + rtx src1;
> +
> + src1 = simplify_gen_subreg (mode,
> + src_exp, GET_MODE (src_exp),
> + regoff1);
> +
> + set_store_subreg (i2, src1, regoff1);
> + set_rescan_store (i1);
> + set_rescan_store (i2);
> + df_insn_rescan (insn1);
> +}
> +
> +// Return true iff insn I1 has already existing subreg.
> +static bool
> +use_has_subreg_p (insn_info *i1)
> +{
> + for (auto def : i1->defs ())
> + {
> + auto set = dyn_cast<set_info *> (def);
> + for (auto use : set->nondebug_insn_uses ())
> + {
> + insn_info *info = use->insn ();
> + if (info && info->rtl ())
> + {
> + rtx_insn *rtl_insn = info->rtl ();
> + df_ref ref;
> + FOR_EACH_INSN_USE (ref, rtl_insn)
> + {
> + rtx dest_exp = SET_DEST (PATTERN (i1->rtl ()));
> + if (REG_P (dest_exp)
> + && DF_REF_REGNO (ref) == REGNO (dest_exp))
> + {
> + rtx *loc = DF_REF_LOC (ref);
> + if (GET_CODE (*loc) == SUBREG)
> + return true;
> + }
> + }
> + }
> + }
> + }
> + return false;
> +}
> +
> +// Set subreg for OO mode pair load for existing subreg rtx to generate
> +// registers in pairs given insn_info I2 and I2.
> +static void
> +set_multiword_subreg_load (insn_info *i1, insn_info *i2)
> +
> +{
> + rtx_insn *insn1 = i1->rtl ();
> + rtx body = PATTERN (insn1);
> + rtx dest_exp = SET_DEST (body);
> + machine_mode mode = GET_MODE (dest_exp);
> + int regoff1;
> + regoff1 = 16;
> +
> + bool subreg_p = false;
> +
> + if (use_has_subreg_p (i1))
> + subreg_p = true;
Like I said in the last review, I don't think we should have a separate
path for uses that involved subregs and uses that didn't. What is currently
the "subreg_p" path should be the only path. We should use it even if
there were no references involving subregs.
We shouldn't in general change the width of an existing pseudo register,
since it would invalidate REG_ATTRS and any debug information associated
with the register. The rs6000 form of load fusion should always create
a fresh pseudo register for the double-width fused result.
> +
> + rtx new_dest_exp = dest_exp;
> +
> + if (subreg_p)
> + new_dest_exp = new_reg_rtx (dest_exp);
> + else
> + PUT_MODE_RAW (new_dest_exp, OOmode);
> +
> + rtx src = simplify_gen_subreg (mode,
> + new_dest_exp,
> + OOmode,
> + regoff1);
> +
> + set_load_subreg (i1, src);
> + if (subreg_p)
> + propagate_insn (i1, new_dest_exp);
> +
> + int regoff = 0;
> + rtx sset = single_set (i2->rtl ());
> + rtx insn2_dest_exp = SET_DEST (sset);
> + machine_mode insn2_mode = GET_MODE (insn2_dest_exp);
> +
> + src = simplify_gen_subreg (insn2_mode,
> + new_dest_exp,
> + OOmode,
> + regoff);
> +
> + set_load_subreg (i2, src);
> +
> + if (subreg_p)
> + propagate_insn (i2, new_dest_exp);
> +
> + if (subreg_p)
> + {
> + auto attempt = crtl->ssa->new_change_attempt ();
> + resource_info resource = { GET_MODE (new_dest_exp), REGNO
> (new_dest_exp) };
> + auto *set = crtl->ssa->allocate<set_info> (i1, resource);
> + if (set)
> + {
> + auto def = find_access (i1->defs (), REGNO (new_dest_exp));
> + if (!def)
> + i1->defs() = insert_access (attempt, set, i1->defs());
> + }
Changes to rtl-ssa insn_infos should always happen through insn_changes,
rather than by direct modification of insns. This allows the framework
to do all the required book-keeping. E.g. the above does not extend
the function_info::m_defs array. (And it shouldn't extend that array
directly -- rtl-ssa should do that internally.)
We already have:
function_info::create_set
which creates a temporary definition of something, for use in an
insn_change. I'm not sure that it handles completely new registers,
because no-one's needed that yet, but it would be a reasonable thing
to add (in function_info::change_insns).
In summary:
- please always use a new pseudo register for the fused load result
- please always use insn_propagation to update uses of the load results
- please create an insn_change for each use, and add the changes to the
existing list of insn_changes made by the main pair-fusion.cc code
- please don't try to change the rtl-ssa ir directly from rs6000 code;
instead, describe the new instruction using insn_changes
- once we know that the fusion is going to go ahead, we should update
all debug uses to refer to the new register half, in a similar way
to late-combine's insn_combination::substitute_optional_uses
(perhaps I should try to provide an abstraction around that).
Thanks,
Richard
> + }
> +
> + set_rescan_load (i1);
> + set_rescan_load (i2);
> + df_insn_rescan (insn1);
> +}
> +
> +// Set subreg for OO mode pair to generate sequential registers given
> +// insn_info pairs I1, I2 and LOAD_P is true iff load insn and false
> +// if store insn.
> +void
> +rs6000_pair_fusion::set_multiword_subreg (insn_info *i1, insn_info *i2,
> + bool load_p)
> +{
> + if (load_p)
> + set_multiword_subreg_load (i1, i2);
> + else
> + set_multiword_subreg_store (i1, i2);
> +}
> +
> +rtx
> +rs6000_pair_fusion::gen_pair (rtx *pats, rtx, bool load_p)
> +{
> + rtx i1 = pats[0];
> + rtx src_exp = SET_SRC (i1);
> + rtx dest_exp = SET_DEST (i1);
> + PUT_MODE_RAW (src_exp, OOmode);
> + PUT_MODE_RAW (dest_exp, OOmode);
> + rtx unspec = gen_rtx_UNSPEC (GET_MODE (dest_exp),
> + gen_rtvec (1, src_exp),
> + UNSPEC_LXVP);
> + rtx set = gen_rtx_SET (dest_exp, unspec);
> + if (dump_file)
> + {
> + if (load_p)
> + fprintf (dump_file, "lxv with lxvp ");
> + else
> + fprintf (dump_file, "stxv with stxvp ");
> + print_rtl_single (dump_file, set);
> + }
> + return set;
> +}
> +
> +const pass_data pass_data_mem_fusion =
> +{
> + RTL_PASS, /* type */
> + "mem_fusion", /* name */
> + OPTGROUP_NONE, /* optinfo_flags */
> + TV_NONE, /* tv_id */
> + 0, /* properties_required */
> + 0, /* properties_provided */
> + 0, /* properties_destroyed */
> + 0, /* todo_flags_start */
> + TODO_df_finish, /* todo_flags_finish */
> +};
> +
> +class pass_mem_fusion : public rtl_opt_pass
> +{
> +public:
> + pass_mem_fusion (gcc::context *ctxt)
> + : rtl_opt_pass (pass_data_mem_fusion, ctxt)
> + {}
> +
> + opt_pass *clone () override { return new pass_mem_fusion (m_ctxt);}
> +
> + /* opt_pass methods: */
> + bool gate (function *)
> + {
> + return (optimize > 0 && TARGET_VSX && TARGET_POWER10);
> + }
> +
> + unsigned int execute (function *) final override
> + {
> + rs6000_pair_fusion pass;
> + pass.run ();
> + return 0;
> + }
> +}; // class pass_mem_fusion
> +
> +rtl_opt_pass *
> +make_pass_mem_fusion (gcc::context *ctxt)
> +{
> + return new pass_mem_fusion (ctxt);
> +}
> diff --git a/gcc/config/rs6000/rs6000-passes.def
> b/gcc/config/rs6000/rs6000-passes.def
> index 46a0d0b8c56..0b48f57014d 100644
> --- a/gcc/config/rs6000/rs6000-passes.def
> +++ b/gcc/config/rs6000/rs6000-passes.def
> @@ -28,7 +28,9 @@ along with GCC; see the file COPYING3. If not see
> The power8 does not have instructions that automaticaly do the byte
> swaps
> for loads and stores. */
> INSERT_PASS_BEFORE (pass_cse, 1, pass_analyze_swaps);
> -
> + /* Pass to replace adjacent memory addresses lxv/stxv instruction with
> + lxvp/stxvp instruction. */
> + INSERT_PASS_BEFORE (pass_early_remat, 1, pass_mem_fusion);
> /* Pass to do the PCREL_OPT optimization that combines the load of an
> external symbol's address along with a single load or store using that
> address as a base register. */
> diff --git a/gcc/config/rs6000/rs6000-protos.h
> b/gcc/config/rs6000/rs6000-protos.h
> index b40557a8557..5295ff66950 100644
> --- a/gcc/config/rs6000/rs6000-protos.h
> +++ b/gcc/config/rs6000/rs6000-protos.h
> @@ -342,6 +342,7 @@ namespace gcc { class context; }
> class rtl_opt_pass;
>
> extern rtl_opt_pass *make_pass_analyze_swaps (gcc::context *);
> +extern rtl_opt_pass *make_pass_mem_fusion (gcc::context *);
> extern rtl_opt_pass *make_pass_pcrel_opt (gcc::context *);
> extern bool rs6000_sum_of_two_registers_p (const_rtx expr);
> extern bool rs6000_quadword_masked_address_p (const_rtx exp);
> diff --git a/gcc/config/rs6000/rs6000.cc b/gcc/config/rs6000/rs6000.cc
> index 5ed64b1e686..449d9e446d5 100644
> --- a/gcc/config/rs6000/rs6000.cc
> +++ b/gcc/config/rs6000/rs6000.cc
> @@ -27442,7 +27442,8 @@ rs6000_split_multireg_move (rtx dst, rtx src)
> reg_mode = word_mode;
> reg_mode_size = GET_MODE_SIZE (reg_mode);
>
> - gcc_assert (reg_mode_size * nregs == GET_MODE_SIZE (mode));
> + gcc_assert (mode == OOmode
> + || reg_mode_size * nregs == GET_MODE_SIZE (mode));
>
> /* TDmode residing in FP registers is special, since the ISA requires that
> the lower-numbered word of a register pair is always the most
> significant
> @@ -27489,6 +27490,11 @@ rs6000_split_multireg_move (rtx dst, rtx src)
> int reg_mode_nregs = hard_regno_nregs (reg, reg_mode);
> if (MEM_P (dst))
> {
> + rtx addr = XEXP (dst, 0);
> + rtx opnd1 = NULL_RTX;
> + if (addr && GET_CODE (addr) == PLUS)
> + opnd1 = XEXP (addr,1);
> +
> unsigned offset = 0;
> unsigned size = GET_MODE_SIZE (reg_mode);
>
> @@ -27502,7 +27508,13 @@ rs6000_split_multireg_move (rtx dst, rtx src)
> {
> unsigned subreg
> = WORDS_BIG_ENDIAN ? i : (nregs - reg_mode_nregs - i);
> - rtx dst2 = adjust_address (dst, reg_mode, offset);
> + rtx dst2 = dst;
> +
> + if ((GET_CODE (addr) != PLUS
> + || (opnd1 && CONST_INT_P(opnd1))))
> + dst2 = adjust_address (dst, reg_mode, offset);
> + else
> + PUT_MODE_RAW (dst, reg_mode);
> rtx src2 = gen_rtx_REG (reg_mode, reg + subreg);
> offset += size;
> emit_insn (gen_rtx_SET (dst2, src2));
> @@ -27513,15 +27525,25 @@ rs6000_split_multireg_move (rtx dst, rtx src)
>
> if (MEM_P (src))
> {
> + rtx addr = XEXP (src, 0);
> + rtx opnd1 = NULL_RTX;
> + if (addr && GET_CODE (addr) == PLUS)
> + opnd1 = XEXP (addr,1);
> +
> unsigned offset = 0;
> unsigned size = GET_MODE_SIZE (reg_mode);
>
> - for (int i = 0; i < nregs; i += reg_mode_nregs)
> + for (int i = nregs-1; i >= 0; i -= reg_mode_nregs)
> {
> unsigned subreg
> = WORDS_BIG_ENDIAN ? i : (nregs - reg_mode_nregs - i);
> rtx dst2 = gen_rtx_REG (reg_mode, reg + subreg);
> - rtx src2 = adjust_address (src, reg_mode, offset);
> + rtx src2 = src;
> +
> + if ((GET_CODE (addr) != PLUS || (opnd1 && CONST_INT_P (opnd1))))
> + src2 = adjust_address (src, reg_mode, offset);
> + else
> + PUT_MODE_RAW (src2, reg_mode);
> offset += size;
> emit_insn (gen_rtx_SET (dst2, src2));
> }
> @@ -27529,7 +27551,8 @@ rs6000_split_multireg_move (rtx dst, rtx src)
> /* If we are writing an accumulator register, we have to
> prime it after we've written it. */
> if (TARGET_MMA
> - && GET_MODE (dst) == XOmode && FP_REGNO_P (REGNO (dst)))
> + && REG_P (dst) && GET_MODE (dst) == XOmode
> + && FP_REGNO_P (REGNO (dst)))
> emit_insn (gen_mma_xxmtacc (dst, dst));
>
> return;
> @@ -27622,9 +27645,12 @@ rs6000_split_multireg_move (rtx dst, rtx src)
> {
> for (i = nregs - 1; i >= 0; i--)
> {
> - rtx dst_i = gen_rtx_REG (reg_mode, REGNO (dst) + i);
> - rtx src_i = gen_rtx_REG (reg_mode, REGNO (src) + i);
> - emit_insn (gen_rtx_SET (dst_i, src_i));
> + if (REG_P (dst) && REG_P (src))
> + {
> + rtx dst_i = gen_rtx_REG (reg_mode, REGNO (dst) + i);
> + rtx src_i = gen_rtx_REG (reg_mode, REGNO (src) + i);
> + emit_insn (gen_rtx_SET (dst_i, src_i));
> + }
> }
> }
> else
> @@ -27639,7 +27665,8 @@ rs6000_split_multireg_move (rtx dst, rtx src)
> /* If we are writing an accumulator register, we have to
> prime it after we've written it. */
> if (TARGET_MMA
> - && GET_MODE (dst) == XOmode && FP_REGNO_P (REGNO (dst)))
> + && REG_P (dst) && GET_MODE (dst) == XOmode
> + && FP_REGNO_P (REGNO (dst)))
> emit_insn (gen_mma_xxmtacc (dst, dst));
> }
> else
> @@ -27696,7 +27723,7 @@ rs6000_split_multireg_move (rtx dst, rtx src)
>
> /* If the base register we are using to address memory is
> also a destination reg, then change that register last. */
> - if (REG_P (breg)
> + if (REG_P (dst) && REG_P (breg)
> && REGNO (breg) >= REGNO (dst)
> && REGNO (breg) < REGNO (dst) + nregs)
> j = REGNO (breg) - REGNO (dst);
> @@ -27794,9 +27821,12 @@ rs6000_split_multireg_move (rtx dst, rtx src)
> /* XO/OO are opaque so cannot use subregs. */
> if (mode == OOmode || mode == XOmode )
> {
> - rtx dst_i = gen_rtx_REG (reg_mode, REGNO (dst) + j);
> - rtx src_i = gen_rtx_REG (reg_mode, REGNO (src) + j);
> - emit_insn (gen_rtx_SET (dst_i, src_i));
> + if (REG_P (dst) && REG_P (src))
> + {
> + rtx dst_i = gen_rtx_REG (reg_mode, REGNO (dst) + j);
> + rtx src_i = gen_rtx_REG (reg_mode, REGNO (src) + j);
> + emit_insn (gen_rtx_SET (dst_i, src_i));
> + }
> }
> else
> emit_insn (gen_rtx_SET (simplify_gen_subreg (reg_mode, dst, mode,
> @@ -27814,7 +27844,9 @@ rs6000_split_multireg_move (rtx dst, rtx src)
> if (restore_basereg != NULL_RTX)
> emit_insn (restore_basereg);
> }
> + return;
> }
> +
>
> /* Return true if the peephole2 can combine a load involving a combination of
> an addis instruction and a load with an offset that can be fused together
> on
> diff --git a/gcc/config/rs6000/rs6000.md b/gcc/config/rs6000/rs6000.md
> index f59be536570..99f070ee24d 100644
> --- a/gcc/config/rs6000/rs6000.md
> +++ b/gcc/config/rs6000/rs6000.md
> @@ -159,6 +159,7 @@
> UNSPEC_XXSPLTIW_CONST
> UNSPEC_FMAX
> UNSPEC_FMIN
> + UNSPEC_LXVP
> ])
>
> ;;
> diff --git a/gcc/config/rs6000/t-rs6000 b/gcc/config/rs6000/t-rs6000
> index 155788de40a..a052163eec4 100644
> --- a/gcc/config/rs6000/t-rs6000
> +++ b/gcc/config/rs6000/t-rs6000
> @@ -34,6 +34,11 @@ rs6000-p8swap.o: $(srcdir)/config/rs6000/rs6000-p8swap.cc
> $(COMPILE) $<
> $(POSTCOMPILE)
>
> +rs6000-mem-fusion.o: $(srcdir)/config/rs6000/rs6000-mem-fusion.cc
> + $(COMPILE) $<
> + $(POSTCOMPILE)
> +
> +
> rs6000-d.o: $(srcdir)/config/rs6000/rs6000-d.cc
> $(COMPILE) $<
> $(POSTCOMPILE)
> diff --git a/gcc/pair-fusion.cc b/gcc/pair-fusion.cc
> index 31d2c21c88f..b1521253474 100644
> --- a/gcc/pair-fusion.cc
> +++ b/gcc/pair-fusion.cc
> @@ -312,9 +312,9 @@ static int
> encode_lfs (lfs_fields fields)
> {
> int size_log2 = exact_log2 (fields.size);
> - gcc_checking_assert (size_log2 >= 2 && size_log2 <= 4);
> - return ((int)fields.load_p << 3)
> - | ((int)fields.fpsimd_p << 2)
> + gcc_checking_assert (size_log2 >= 2 && size_log2 <= 9);
> + return ((int)fields.load_p << 4)
> + | ((int)fields.fpsimd_p << 3)
> | (size_log2 - 2);
> }
>
> @@ -322,8 +322,8 @@ encode_lfs (lfs_fields fields)
> static lfs_fields
> decode_lfs (int lfs)
> {
> - bool load_p = (lfs & (1 << 3));
> - bool fpsimd_p = (lfs & (1 << 2));
> + bool load_p = (lfs & (1 << 4));
> + bool fpsimd_p = (lfs & (1 << 3));
> unsigned size = 1U << ((lfs & 3) + 2);
> return { load_p, fpsimd_p, size };
> }
> @@ -425,6 +425,9 @@ pair_fusion_bb_info::track_access (insn_info *insn, bool
> load_p, rtx mem)
> if (MEM_VOLATILE_P (mem))
> return;
>
> + if (load_p && !m_pass->fuseable_load_p (insn))
> + return;
> +
> // Ignore writeback accesses if the hook says to do so.
> if (!m_pass->should_handle_writeback (writeback_type::EXISTING)
> && GET_RTX_CLASS (GET_CODE (XEXP (mem, 0))) == RTX_AUTOINC)
> @@ -1814,7 +1817,7 @@ pair_fusion_bb_info::fuse_pair (bool load_p,
> }
>
> rtx reg_notes = combine_reg_notes (first, second, load_p);
> -
> + m_pass->set_multiword_subreg (i1, i2, load_p);
> rtx pair_pat = m_pass->gen_pair (pats, writeback_effect, load_p);
> insn_change *pair_change = nullptr;
> auto set_pair_pat = [pair_pat,reg_notes](insn_change *change) {
> @@ -1828,18 +1831,18 @@ pair_fusion_bb_info::fuse_pair (bool load_p,
> changes.safe_push (make_delete (first));
> pair_change = make_change (second);
> changes.safe_push (pair_change);
> -
> pair_change->move_range = move_range;
> pair_change->new_defs = merge_access_arrays (attempt,
> input_defs[0],
> input_defs[1]);
> + m_pass->modify_new_rtx_insn (first, &attempt, &pair_change, changes);
> gcc_assert (pair_change->new_defs.is_valid ());
> -
> pair_change->new_uses
> = merge_access_arrays (attempt,
> drop_memory_access (input_uses[0]),
> drop_memory_access (input_uses[1]));
> gcc_assert (pair_change->new_uses.is_valid ());
> + m_pass->set_uses (i1, i2, &pair_change, &attempt);
> set_pair_pat (pair_change);
> }
> else
> @@ -1872,6 +1875,7 @@ pair_fusion_bb_info::fuse_pair (bool load_p,
> store_def,
> change->new_defs);
> gcc_assert (change->new_defs.is_valid ());
> + m_pass->set_defs (i1, i2, &change, &attempt);
> change->move_range = move_range;
> pair_change = change;
> break;
> @@ -1908,6 +1912,7 @@ pair_fusion_bb_info::fuse_pair (bool load_p,
> change->new_defs = insert_access (attempt, new_set,
> change->new_defs);
> gcc_assert (change->new_defs.is_valid ());
> + m_pass->set_defs (i1, i2, &change, &attempt);
> pair_change = change;
> break;
> }
> @@ -2405,6 +2410,15 @@ pair_fusion_bb_info::try_fuse_pair (bool load_p,
> unsigned access_size,
> reg_ops[i] = XEXP (pats[i], !load_p);
> }
>
> + if (!load_p && !m_pass->fuseable_store_p (i1, i2))
> + {
> + if (dump_file)
> + fprintf (dump_file,
> + "punting on store-mem-pairs due to non fuseable cand
> (%d,%d)\n",
> + insns[0]->uid (), insns[1]->uid ());
> + return false;
> + }
> +
> if (load_p && reg_overlap_mentioned_p (reg_ops[0], reg_ops[1]))
> {
> if (dump_file)
> @@ -2997,6 +3011,8 @@ void pair_fusion::process_block (bb_info *bb)
> if (GET_CODE (pat) != SET)
> continue;
>
> + change_existing_multword_mode (rti);
> +
> if (track_stores && MEM_P (XEXP (pat, 0)))
> bb_state.track_access (insn, false, XEXP (pat, 0));
> else if (track_loads && MEM_P (XEXP (pat, 1)))
> diff --git a/gcc/pair-fusion.h b/gcc/pair-fusion.h
> index 45e4edceecb..b3af49de892 100644
> --- a/gcc/pair-fusion.h
> +++ b/gcc/pair-fusion.h
> @@ -26,8 +26,12 @@ namespace rtl_ssa {
> class insn_info;
> class insn_range_info;
> class bb_info;
> + class insn_change;
> + class insn_range_info;
> }
>
> +class obstack_watermark;
> +
> // Information about a potential base candidate, used in try_fuse_pair.
> // There may be zero, one, or two viable RTL bases for a given pair.
> struct base_cand
> @@ -142,6 +146,19 @@ struct pair_fusion {
> // true iff INSN is a load pair.
> virtual bool pair_mem_insn_p (rtx_insn *insn, bool &load_p) = 0;
>
> + // Given INSN change multiword mode load and store to respective
> + // unspec instruction.
> + virtual void change_existing_multword_mode (rtx_insn *insn) = 0;
> +
> + // Given INSN and watermark ATTEMPT and PAIR_CHANGE sets the
> + // new rtx with INSN. Remove all uses of definition that are
> + // removed given CHANGES.
> + virtual void modify_new_rtx_insn (rtl_ssa::insn_info *first,
> + obstack_watermark *attempt,
> + rtl_ssa::insn_change **pair_change,
> + auto_vec<rtl_ssa::insn_change *> &changes)
> + = 0;
> +
> // Return true if we should track loads.
> virtual bool track_loads_p ()
> {
> @@ -171,6 +188,37 @@ struct pair_fusion {
> virtual rtx gen_promote_writeback_pair (rtx wb_effect, rtx mem,
> rtx regs[2], bool load_p) = 0;
>
> + // Given insn_info pair I1 and I2, sets subreg with multiword registers
> + // to assign register pairs by allocators.
> + // LOAD_P is true iff the pair is a load.
> + virtual void set_multiword_subreg (rtl_ssa::insn_info *i1,
> + rtl_ssa::insn_info *i2,
> + bool load_p) = 0;
> +
> + // Given insn_info pair I1 and I2, sets uses in PAIR_CHANGE.
> + virtual void set_uses (rtl_ssa::insn_info *i1,
> + rtl_ssa::insn_info *i2,
> + rtl_ssa::insn_change **pair_change,
> + obstack_watermark *attempt) = 0;
> +
> + // Given insn_info pair I1 and I2, sets defss in PAIR_CHANGE.
> + virtual void set_defs (rtl_ssa::insn_info *i1,
> + rtl_ssa::insn_info *i2,
> + rtl_ssa::insn_change **pair_change,
> + obstack_watermark *attempt) = 0;
> +
> +
> + // Given insn_info pair I1 and I2, checks if pairs are feasible to perform
> + // store mem pairs.
> + // Return true if feasible to perform store mem pairs otherwise false.
> + virtual bool fuseable_store_p (rtl_ssa::insn_info *i1,
> + rtl_ssa::insn_info *i2) = 0;
> +
> + // Given insn_info pair I1 and I2, checks if pairs are feasible to perform
> + // load mem pairs.
> + // Return true if feasible to perform load mem pairs otherwise false.
> + virtual bool fuseable_load_p (rtl_ssa::insn_info *info) = 0;
> +
> void process_block (rtl_ssa::bb_info *bb);
> rtl_ssa::insn_info *find_trailing_add (rtl_ssa::insn_info *insns[2],
> const rtl_ssa::insn_range_info
> diff --git a/gcc/rtl-ssa/functions.h b/gcc/rtl-ssa/functions.h
> index 8be04f1aa96..27a732b7353 100644
> --- a/gcc/rtl-ssa/functions.h
> +++ b/gcc/rtl-ssa/functions.h
> @@ -222,6 +222,13 @@ public:
> template<typename T, typename... Ts>
> T *change_alloc (obstack_watermark &wm, Ts... args);
>
> + auto_vec<access_info *> &get_m_temp_defs () { return m_temp_defs; }
> +
> + template<typename T, typename... Ts>
> + T *allocate (Ts... args);
> +
> + void remove_use (use_info *);
> +
> private:
> class bb_phi_info;
> class build_info;
> @@ -231,9 +238,6 @@ private:
> // allocate_temp during its lifetime.
> obstack_watermark temp_watermark () { return &m_temp_obstack; }
>
> - template<typename T, typename... Ts>
> - T *allocate (Ts... args);
> -
> template<typename T, typename... Ts>
> T *allocate_temp (Ts... args);
>
> @@ -269,7 +273,6 @@ private:
> static void insert_use_after (use_info *, use_info *);
>
> void add_use (use_info *);
> - void remove_use (use_info *);
>
> insn_info::order_node *need_order_node (insn_info *);
>
> diff --git a/gcc/testsuite/g++.target/powerpc/mem-fusion-1.C
> b/gcc/testsuite/g++.target/powerpc/mem-fusion-1.C
> new file mode 100644
> index 00000000000..d10ff0cdf36
> --- /dev/null
> +++ b/gcc/testsuite/g++.target/powerpc/mem-fusion-1.C
> @@ -0,0 +1,22 @@
> +/* { dg-do compile } */
> +/* { dg-require-effective-target power10_ok } */
> +/* { dg-options "-mdejagnu-cpu=power10 -O2" } */
> +
> +#include <altivec.h>
> +
> +void
> +foo2 ()
> +{
> + __vector_quad *dst1;
> + __vector_quad *dst2;
> + vector unsigned char src;
> + __vector_quad acc;
> + vector unsigned char *ptr;
> + __builtin_mma_xvf32ger(&acc, src, ptr[0]);
> + __builtin_mma_xvf32gerpp(&acc, src, ptr[1]);
> + *dst1 = acc;
> + __builtin_mma_xvf32ger(&acc, src, ptr[2]);
> + __builtin_mma_xvf32gerpp(&acc, src, ptr[3]);
> + *dst2 = acc;
> +}
> +/* { dg-final { scan-assembler {\mlxvp\M} } } */
> diff --git a/gcc/testsuite/g++.target/powerpc/mem-fusion.C
> b/gcc/testsuite/g++.target/powerpc/mem-fusion.C
> new file mode 100644
> index 00000000000..c523572cf3c
> --- /dev/null
> +++ b/gcc/testsuite/g++.target/powerpc/mem-fusion.C
> @@ -0,0 +1,15 @@
> +/* { dg-do compile } */
> +/* { dg-require-effective-target power10_ok } */
> +/* { dg-options "-mdejagnu-cpu=power10 -O2" } */
> +
> +#include <altivec.h>
> +
> +void
> +foo (__vector_quad *dst, vector unsigned char *ptr, vector unsigned char src)
> +{
> + __vector_quad acc;
> + __builtin_mma_xvf32ger(&acc, src, ptr[0]);
> + __builtin_mma_xvf32gerpp(&acc, src, ptr[1]);
> + *dst = acc;
> +}
> +/* { dg-final { scan-assembler {\mlxvp\M} } } */
> diff --git a/gcc/testsuite/gcc.target/powerpc/mma-builtin-1.c
> b/gcc/testsuite/gcc.target/powerpc/mma-builtin-1.c
> index 69ee826e1be..ae29127f954 100644
> --- a/gcc/testsuite/gcc.target/powerpc/mma-builtin-1.c
> +++ b/gcc/testsuite/gcc.target/powerpc/mma-builtin-1.c
> @@ -258,8 +258,8 @@ foo13b (__vector_quad *dst, __vector_quad *src, vec_t
> *vec)
> dst[13] = acc;
> }
>
> -/* { dg-final { scan-assembler-times {\mlxv\M} 40 } } */
> -/* { dg-final { scan-assembler-times {\mlxvp\M} 12 } } */
> +/* { dg-final { scan-assembler-times {\mlxv\M} 0 } } */
> +/* { dg-final { scan-assembler-times {\mlxvp\M} 32 } } */
> /* { dg-final { scan-assembler-times {\mstxvp\M} 40 } } */
> /* { dg-final { scan-assembler-times {\mxxmfacc\M} 20 } } */
> /* { dg-final { scan-assembler-times {\mxxmtacc\M} 6 } } */
