The sched_fusion pass can bring independent memory accesses together
before sched2. Add a priority hook that groups scalar load/store-pair
candidates by register file, access kind, mode and base register, then
orders each group by offset. Prefer decreasing offsets when both
directions are available, matching frame save/restore memory order.
Enable scheduling fusion by default for load/store-pair tunes while
preserving an explicit -f[no-]schedule-fusion option. Keep the positive
test as XFAIL until the tuning patch enables the pairs.
Static sched2 macro-fusion counts for SPEC CPU 2017 Integer built with
-O3 -flto, measured independently of ready-list reordering against a
baseline with xt-c9501fdvt fusion pairs enabled:
Benchmark Baseline With patch Change
500.perlbench_r 15302 17077 +11.6%
502.gcc_r 72806 78135 +7.3%
505.mcf_r 63 85 +34.9%
520.omnetpp_r 20479 22084 +7.8%
523.xalancbmk_r 42442 44651 +5.2%
525.x264_r 6232 7033 +12.9%
531.deepsjeng_r 299 328 +9.7%
541.leela_r 794 851 +7.2%
548.exchange2_r 353 426 +20.7%
557.xz_r 1217 1321 +8.5%
SUM 159987 171991 +7.5%
gcc/ChangeLog:
* config/riscv/riscv-fusion.cc (riscv_sched_fusion_priority): New
function.
* config/riscv/riscv-protos.h (riscv_sched_fusion_priority):
Declare.
* config/riscv/riscv.cc (riscv_override_options_internal): Set the
scheduling-fusion default for the current tune.
(TARGET_SCHED_FUSION_PRIORITY): Define.
gcc/testsuite/ChangeLog:
* gcc.target/riscv/sched-fusion-priority.c: New test.
Signed-off-by: Jin Ma <[email protected]>
---
gcc/config/riscv/riscv-fusion.cc | 68 +++++++++++++++++++
gcc/config/riscv/riscv-protos.h | 1 +
gcc/config/riscv/riscv.cc | 10 +++
.../gcc.target/riscv/sched-fusion-priority.c | 49 +++++++++++++
4 files changed, 128 insertions(+)
create mode 100644 gcc/testsuite/gcc.target/riscv/sched-fusion-priority.c
diff --git a/gcc/config/riscv/riscv-fusion.cc b/gcc/config/riscv/riscv-fusion.cc
index 332f64bd7cc..06cdab5694d 100644
--- a/gcc/config/riscv/riscv-fusion.cc
+++ b/gcc/config/riscv/riscv-fusion.cc
@@ -907,6 +907,74 @@ riscv_fuse_ldst_pair_p (rtx_insn *prev, rtx_insn *curr,
return diff == access_size;
}
+/* Implement TARGET_SCHED_FUSION_PRIORITY. Group load/store pair candidates
+ by register file, access kind, mode and base register, then by offset. */
+
+void
+riscv_sched_fusion_priority (rtx_insn *insn, int max_pri,
+ int *fusion_pri, int *pri)
+{
+ struct riscv_fusion_mem_info mem;
+ unsigned HOST_WIDE_INT fusible_ops;
+ enum riscv_fusion_pairs inc_op, dec_op;
+ HOST_WIDE_INT access_size;
+ unsigned int base_regno;
+ bool isload_p, fp_p, inc_p;
+ int fusion_type, tmp;
+
+ gcc_assert (INSN_P (insn));
+
+ tmp = max_pri - 1;
+ *fusion_pri = tmp;
+ *pri = tmp;
+
+ if (!riscv_fuse_mem_p (insn, &mem)
+ || mem.type == SCHED_FUSION_LD_ZERO_EXTEND
+ || mem.addr.type != ADDRESS_REG
+ || !CONST_INT_P (mem.addr.offset))
+ return;
+
+ base_regno = riscv_regno (mem.addr.reg);
+ if (base_regno >= FIRST_PSEUDO_REGISTER)
+ return;
+
+ isload_p = mem.type != SCHED_FUSION_ST;
+ fp_p = mem.fp_p;
+ access_size = GET_MODE_SIZE (mem.mode).to_constant ();
+ if (access_size != 4 && access_size != 8)
+ return;
+
+ inc_op = fp_p ? RISCV_FUSE_FLDFST_PAIR_INC : RISCV_FUSE_LDST_PAIR_INC;
+ dec_op = fp_p ? RISCV_FUSE_FLDFST_PAIR_DEC : RISCV_FUSE_LDST_PAIR_DEC;
+ fusible_ops = riscv_get_fusible_ops ();
+ if (!(fusible_ops & (inc_op | dec_op)))
+ return;
+
+ /* Prefer decreasing offsets to match frame save/restore order. */
+ inc_p = !(fusible_ops & dec_op);
+
+ /* Give each load/store class and base register a distinct priority below
+ that of unrelated instructions. */
+ fusion_type = (fp_p ? 4 : 0) + (isload_p ? 0 : 2);
+ fusion_type += access_size == 8;
+ fusion_type++;
+ *fusion_pri -= (fusion_type * FIRST_PSEUDO_REGISTER
+ + (int) base_regno);
+
+ tmp /= 2;
+ HOST_WIDE_INT off_val = INTVAL (mem.addr.offset);
+ unsigned HOST_WIDE_INT magnitude = off_val < 0
+ ? -(unsigned HOST_WIDE_INT) off_val
+ : off_val;
+ int offset_pri = magnitude & 0xfffff;
+
+ /* Order offsets in the preferred pair direction. */
+ if (inc_p == (off_val >= 0))
+ *pri = tmp - offset_pri;
+ else
+ *pri = tmp + offset_pri;
+}
+
/* Check the common RTL for ZEXTW, ZEXTWS and ZEXTH fusion. */
static bool
diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h
index cf36efccf75..0921b3a7618 100644
--- a/gcc/config/riscv/riscv-protos.h
+++ b/gcc/config/riscv/riscv-protos.h
@@ -884,6 +884,7 @@ enum riscv_fusion_pairs
extern bool riscv_macro_fusion_p (void);
extern bool riscv_macro_fusion_pair_p (rtx_insn *, rtx_insn *);
+extern void riscv_sched_fusion_priority (rtx_insn *, int, int *, int *);
extern unsigned HOST_WIDE_INT riscv_get_fusible_ops (void);
/* Routines implemented in thead.cc. */
diff --git a/gcc/config/riscv/riscv.cc b/gcc/config/riscv/riscv.cc
index a2a51c019ec..9f0c1d704c2 100644
--- a/gcc/config/riscv/riscv.cc
+++ b/gcc/config/riscv/riscv.cc
@@ -12121,6 +12121,14 @@ riscv_override_options_internal (struct gcc_options
*opts)
? &optimize_size_tune_info
: cpu->tune_param;
+ /* Enable scheduling fusion for load/store-pair tunes. */
+ if (!OPTION_SET_P (flag_schedule_fusion))
+ opts->x_flag_schedule_fusion
+ = (riscv_get_fusible_ops ()
+ & (RISCV_FUSE_LDST_PAIR_INC | RISCV_FUSE_LDST_PAIR_DEC
+ | RISCV_FUSE_FLDFST_PAIR_INC | RISCV_FUSE_FLDFST_PAIR_DEC))
+ != RISCV_FUSE_NOTHING;
+
/* If not optimizing for size, set the default
alignment to what the target wants. */
if (!opts->x_optimize_size)
@@ -16617,6 +16625,8 @@ riscv_memtag_tag_bitsize ()
#define TARGET_SCHED_MACRO_FUSION_P riscv_macro_fusion_p
#undef TARGET_SCHED_MACRO_FUSION_PAIR_P
#define TARGET_SCHED_MACRO_FUSION_PAIR_P riscv_macro_fusion_pair_p
+#undef TARGET_SCHED_FUSION_PRIORITY
+#define TARGET_SCHED_FUSION_PRIORITY riscv_sched_fusion_priority
#undef TARGET_SCHED_INIT
#define TARGET_SCHED_INIT riscv_sched_init
diff --git a/gcc/testsuite/gcc.target/riscv/sched-fusion-priority.c
b/gcc/testsuite/gcc.target/riscv/sched-fusion-priority.c
new file mode 100644
index 00000000000..3d2c100a408
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sched-fusion-priority.c
@@ -0,0 +1,49 @@
+/* { dg-do compile { target { rv64 } } } */
+/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-O3" "-O[sgz]" "-flto" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -mtune=xt-c9501fdvt -O2" } */
+/* { dg-additional-options "-fno-schedule-insns2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+/* Start before sched_fusion and disable sched2 to isolate the priority
+ hook. */
+
+/*
+**sched_fusion_load_pair: { xfail *-*-* }
+** ...
+** ld [a-z][0-9]+,8\(a0\)
+** ld [a-z][0-9]+,0\(a0\)
+** ...
+** ret
+*/
+long __RTL (startwith ("compgotos"))
+sched_fusion_load_pair (void)
+{
+(function "sched_fusion_load_pair"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cnote 2 NOTE_INSN_FUNCTION_BEG)
+ (cinsn 3 (set (reg:DI a1)
+ (mem:DI (reg:DI a0) [0 S8 A64])))
+ (cinsn 4 (set (reg:DI a3)
+ (plus:DI (reg:DI a4) (reg:DI a5))))
+ (cinsn 5 (set (reg:DI a2)
+ (mem:DI
+ (plus:DI (reg:DI a0)
+ (const_int 8)) [0 S8 A64])))
+ (cinsn 6 (use (reg:DI a1)))
+ (cinsn 7 (use (reg:DI a2)))
+ (cinsn 8 (use (reg:DI a3)))
+ (cjump_insn 9 (simple_return))
+ (edge-to exit)
+ ) ;; block 2
+ (cbarrier 10)
+ ) ;; insn-chain
+ (crtl
+ (return_rtx
+ (reg/i:DI a0)
+ ) ;; return_rtx
+ ) ;; crtl
+) ;; function "sched_fusion_load_pair"
+}
--
2.52.0
