https://gcc.gnu.org/g:a846da89ae8112ca0072f7553a1fa395fe9bb2d5
commit a846da89ae8112ca0072f7553a1fa395fe9bb2d5 Author: Michael Meissner <[email protected]> Date: Sat Jul 18 00:36:49 2026 -0400 Add patches 2..12 Diff: --- gcc/config/rs6000/constraints.md | 4 + gcc/config/rs6000/mma.md | 455 ++++++++++++++++++--- gcc/config/rs6000/predicates.md | 37 ++ gcc/config/rs6000/rs6000-builtin.cc | 172 ++++++-- gcc/config/rs6000/rs6000-builtins.def | 64 +++ gcc/config/rs6000/rs6000.cc | 332 ++++++++++++--- gcc/config/rs6000/rs6000.h | 47 ++- gcc/config/rs6000/rs6000.md | 14 +- gcc/doc/extend.texi | 38 ++ gcc/doc/md.texi | 4 + gcc/testsuite/gcc.target/powerpc/dmf-build-dmr.c | 15 + gcc/testsuite/gcc.target/powerpc/dmf-builtin.c | 81 ++++ .../gcc.target/powerpc/dmf-disassemble-dmr.c | 24 ++ gcc/testsuite/gcc.target/powerpc/dmf-extract512.c | 20 + gcc/testsuite/gcc.target/powerpc/dmf-no-mma.c | 19 + .../gcc.target/powerpc/dmr1024-alignment.c | 33 ++ gcc/testsuite/gcc.target/powerpc/dmr1024-compile.c | 46 +++ .../gcc.target/powerpc/dmr1024-invalid-use-1.c | 32 ++ .../gcc.target/powerpc/dmr1024-invalid-use-2.c | 23 ++ .../gcc.target/powerpc/future-dmf-xxacc.c | 18 + .../gcc.target/powerpc/future-nodmf-xxacc.c | 18 + 21 files changed, 1357 insertions(+), 139 deletions(-) diff --git a/gcc/config/rs6000/constraints.md b/gcc/config/rs6000/constraints.md index d0ed47faab84..4b2e703f0497 100644 --- a/gcc/config/rs6000/constraints.md +++ b/gcc/config/rs6000/constraints.md @@ -99,6 +99,10 @@ "@internal Like @code{b}, if @option{-mpowerpc64} is used; otherwise, @code{NO_REGS}.") +(define_register_constraint "wD" "rs6000_constraints[RS6000_CONSTRAINT_wD]" + "@internal Floating point register @code{FPR} if TARGET_MMA is enabled. + 1024 bit Dense math register @code{DMR} if TARGET_DMF is enabled.") + ;; wB needs ISA 2.07 VUPKHSW (define_constraint "wB" "@internal Signed 5-bit constant integer that can be loaded into an diff --git a/gcc/config/rs6000/mma.md b/gcc/config/rs6000/mma.md index 1103f1fc0375..c40070abdcac 100644 --- a/gcc/config/rs6000/mma.md +++ b/gcc/config/rs6000/mma.md @@ -24,7 +24,7 @@ ;; __vector_pair types that the MMA built-in functions reference. We ;; use OPAQUE_MODE to prevent anything from trying to open them up. -(define_constants [(MAX_MMA_OPERANDS 7)]) +(define_constants [(MAX_MMA_OPERANDS 9)]) ;; Constants for creating unspecs @@ -91,6 +91,17 @@ UNSPEC_MMA_XVI8GER4SPP UNSPEC_MMA_XXMFACC UNSPEC_MMA_XXMTACC + UNSPEC_DMF_INSERT512 + UNSPEC_DMF_EXTRACT512 + UNSPEC_DMF_INSERT1024 + UNSPEC_DMF_RELOAD_FROM_MEMORY + UNSPEC_DMF_RELOAD_TO_MEMORY + UNSPEC_DMF_DMXOR + UNSPEC_DMF_DMXVI8GERX4 + UNSPEC_DMF_DMXVI8GERX4PP + UNSPEC_DMF_PMDMXVI8GERX4 + UNSPEC_DMF_PMDMXVI8GERX4PP + UNSPEC_DMF_DMSETDMRZ ]) (define_c_enum "unspecv" @@ -133,12 +144,18 @@ ;; MMA instructions with 1 vector pair and 1 vector arguments (define_int_iterator MMA_PV [UNSPEC_MMA_XVF64GER]) +; DMF instructions with 1 vector pair and 1 vector arguments +(define_int_iterator DMF_PV [UNSPEC_DMF_DMXVI8GERX4]) + ;; MMA instructions with 1 accumulator, 1 vector pair and 1 vector arguments (define_int_iterator MMA_APV [UNSPEC_MMA_XVF64GERPP UNSPEC_MMA_XVF64GERPN UNSPEC_MMA_XVF64GERNP UNSPEC_MMA_XVF64GERNN]) +;; DMF instructions with 1 dmr, 1 vector pair and 1 vector arguments +(define_int_iterator DMF_DPV [UNSPEC_DMF_DMXVI8GERX4PP]) + ;; MMA instructions with 2 vector, 2 4-bit and 1 8-bit arguments (define_int_iterator MMA_VVI4I4I8 [UNSPEC_MMA_PMXVI4GER8]) @@ -188,6 +205,14 @@ (define_int_iterator MMA_AVVI4I4I4 [UNSPEC_MMA_PMXVI8GER4PP UNSPEC_MMA_PMXVI8GER4SPP]) +; DMF instructions with 1 vector pair, 1 vector and 1 8-bit and 2 4-bit +;; arguments +(define_int_iterator DMF_PVI8I4I4 [UNSPEC_DMF_PMDMXVI8GERX4]) + +;; DMF instructions with 1 dmr, 1 vector pair, 1 vector and 1 8-bit and +;; 2 4-bit arguments +(define_int_iterator DMF_DPVI8I4I4 [UNSPEC_DMF_PMDMXVI8GERX4PP]) + (define_int_attr acc [(UNSPEC_MMA_XXMFACC "xxmfacc") (UNSPEC_MMA_XXMTACC "xxmtacc")]) @@ -217,12 +242,14 @@ (UNSPEC_MMA_XVF32GERNP "xvf32gernp") (UNSPEC_MMA_XVF32GERNN "xvf32gernn")]) -(define_int_attr pv [(UNSPEC_MMA_XVF64GER "xvf64ger")]) +(define_int_attr pv [(UNSPEC_MMA_XVF64GER "xvf64ger") + (UNSPEC_DMF_DMXVI8GERX4 "dmxvi8gerx4")]) (define_int_attr apv [(UNSPEC_MMA_XVF64GERPP "xvf64gerpp") (UNSPEC_MMA_XVF64GERPN "xvf64gerpn") (UNSPEC_MMA_XVF64GERNP "xvf64gernp") - (UNSPEC_MMA_XVF64GERNN "xvf64gernn")]) + (UNSPEC_MMA_XVF64GERNN "xvf64gernn") + (UNSPEC_DMF_DMXVI8GERX4PP "dmxvi8gerx4pp")]) (define_int_attr vvi4i4i8 [(UNSPEC_MMA_PMXVI4GER8 "pmxvi4ger8")]) @@ -263,6 +290,9 @@ (define_int_attr avvi4i4i4 [(UNSPEC_MMA_PMXVI8GER4PP "pmxvi8ger4pp") (UNSPEC_MMA_PMXVI8GER4SPP "pmxvi8ger4spp")]) +(define_int_attr pvi8i4i4 [(UNSPEC_DMF_PMDMXVI8GERX4 "pmdmxvi8gerx4")]) + +(define_int_attr dpvi8i4i4 [(UNSPEC_DMF_PMDMXVI8GERX4PP "pmdmxvi8gerx4pp")]) ;; Vector pair support. OOmode can only live in VSRs. (define_expand "movoo" @@ -270,7 +300,7 @@ (match_operand:OO 1 "input_operand"))] "" { - if (TARGET_MMA) + if (TARGET_MMA || TARGET_DMF) { rs6000_emit_move (operands[0], operands[1], OOmode); DONE; @@ -295,7 +325,7 @@ (define_insn_and_split "*movoo" [(set (match_operand:OO 0 "nonimmediate_operand" "=wa,ZwO,wa") (match_operand:OO 1 "input_operand" "ZwO,wa,wa"))] - "TARGET_MMA + "(TARGET_MMA || TARGET_DMF) && (gpc_reg_operand (operands[0], OOmode) || gpc_reg_operand (operands[1], OOmode))" "@ @@ -339,10 +369,10 @@ gcc_assert (false); }) -(define_insn_and_split "*movxo" - [(set (match_operand:XO 0 "nonimmediate_operand" "=d,ZwO,d") - (match_operand:XO 1 "input_operand" "ZwO,d,d"))] - "TARGET_MMA +(define_insn_and_split "*movxo_nodmf" + [(set (match_operand:XO 0 "nonimmediate_operand" "=wD,ZwO,wD") + (match_operand:XO 1 "input_operand" "ZwO,wD,wD"))] + "TARGET_MMA && !TARGET_DMF && (gpc_reg_operand (operands[0], XOmode) || gpc_reg_operand (operands[1], XOmode))" "@ @@ -359,11 +389,36 @@ (set_attr "length" "*,*,16") (set_attr "max_prefixed_insns" "2,2,*")]) +(define_insn_and_split "*movxo_dmf" + [(set (match_operand:XO 0 "nonimmediate_operand" "=wa,ZwO,wa,wD,wD,wa") + (match_operand:XO 1 "input_operand" "ZwO,wa, wa,wa,wD,wD"))] + "TARGET_DMF + && (gpc_reg_operand (operands[0], XOmode) + || gpc_reg_operand (operands[1], XOmode))" + "@ + # + # + # + dmxxinstdmr512 %0,%x1,%W1,0 + dmmr %0,%1 + dmxxextfdmr512 %x0,%W0,%1,0" + "&& reload_completed + && !dmr_register_operand (operands[0], XOmode) + && !dmr_register_operand (operands[1], XOmode)" + [(const_int 0)] +{ + rs6000_split_multireg_move (operands[0], operands[1]); + DONE; +} + [(set_attr "type" "vecload,vecstore,veclogical,dmf,dmf,dmf") + (set_attr "length" "*,*,16,*,*,*") + (set_attr "max_prefixed_insns" "2,2,*,*,*,*")]) + (define_expand "vsx_assemble_pair" [(match_operand:OO 0 "vsx_register_operand") (match_operand:V16QI 1 "mma_assemble_input_operand") (match_operand:V16QI 2 "mma_assemble_input_operand")] - "TARGET_MMA" + "TARGET_MMA || TARGET_DMF" { rtx src = gen_rtx_UNSPEC (OOmode, gen_rtvec (2, operands[1], operands[2]), @@ -380,7 +435,7 @@ (unspec:OO [(match_operand:V16QI 1 "mma_assemble_input_operand" "mwa") (match_operand:V16QI 2 "mma_assemble_input_operand" "mwa")] UNSPEC_VSX_ASSEMBLE))] - "TARGET_MMA" + "TARGET_MMA || TARGET_DMF" "#" "&& reload_completed" [(const_int 0)] @@ -396,7 +451,7 @@ [(match_operand:V16QI 0 "mma_disassemble_output_operand") (match_operand:OO 1 "vsx_register_operand") (match_operand 2 "const_0_to_1_operand")] - "TARGET_MMA" + "TARGET_MMA || TARGET_DMF" { rtx src; int regoff = INTVAL (operands[2]); @@ -412,7 +467,7 @@ (unspec:V16QI [(match_operand:OO 1 "vsx_register_operand" "wa") (match_operand 2 "const_0_to_1_operand")] UNSPEC_MMA_EXTRACT))] - "TARGET_MMA + "(TARGET_MMA || TARGET_DMF) && vsx_register_operand (operands[1], OOmode)" "#" "&& reload_completed" @@ -425,19 +480,205 @@ DONE; }) +(define_insn "dm_insert512" + [(set (match_operand:XO 0 "dmr_register_operand" "=wD") + (unspec:XO [(match_operand:OO 1 "vsx_register_operand" "wa") + (match_operand:OO 2 "vsx_register_operand" "wa") + (match_operand 3 "const_0_to_1_operand")] + UNSPEC_DMF_INSERT512))] + "TARGET_DMF" + "dmxxinstdmr512 %0,%x1,%x2,%3" + [(set_attr "type" "dmf")]) + +;; Move from VSX registers to DMR registers via two insert 512 bit +;; instructions. +(define_insn "dm_insert1024" + [(set (match_operand:TDO 0 "dmr_register_operand" "=wD") + (unspec:TDO [(match_operand:OO 1 "vsx_register_operand" "wa") + (match_operand:OO 2 "vsx_register_operand" "wa") + (match_operand:OO 3 "vsx_register_operand" "wa") + (match_operand:OO 4 "vsx_register_operand" "wa")] + UNSPEC_DMF_INSERT1024))] + "TARGET_DMF" + "dmxxinstdmr512 %0,%x1,%x2,0\n\tdmxxinstdmr512 %0,%x3,%x4,1" + [(set_attr "type" "dmf")]) + +(define_insn "dm_extract512" + [(set (match_operand:XO 0 "vsx_register_operand" "=wa") + (unspec:XO [(match_operand:TDO 1 "dmr_register_operand" "wD") + (match_operand 2 "const_0_to_1_operand" "n")] + UNSPEC_DMF_EXTRACT512))] + "TARGET_DMF" + "dmxxextfdmr512 %x0,%W0,%1,%2" + [(set_attr "type" "dmf")]) + +;; TDO (1024-bit dense-math) move expander. +(define_expand "movtdo" + [(set (match_operand:TDO 0 "nonimmediate_operand") + (match_operand:TDO 1 "input_operand"))] + "TARGET_DMF" +{ + rs6000_emit_move (operands[0], operands[1], TDOmode); + DONE; +}) + +(define_insn_and_split "*movtdo" + [(set (match_operand:TDO 0 "nonimmediate_operand" "=wa,m,wa,wD,wa,wD") + (match_operand:TDO 1 "input_operand" "m,wa,wa,wD,wD,wa"))] + "TARGET_DMF + && (gpc_reg_operand (operands[0], TDOmode) + || gpc_reg_operand (operands[1], TDOmode))" +{ + if (which_alternative == 3) + return "dmmr %0,%1"; + else + return "#"; +} +"reload_completed + && (!dmr_register_operand (operands[0], TDOmode) + || !dmr_register_operand (operands[1], TDOmode))" + [(const_int 0)] +{ + rtx dst = operands[0]; + rtx src = operands[1]; + + /* Memory-involving moves (alt 0/1) and wa<-wa VSX moves (alt 2) + both go through the generic multiregister splitter. */ + if (!REG_P (dst) || !REG_P (src) + || (VSX_REGNO_P (REGNO (dst)) && VSX_REGNO_P (REGNO (src)))) + { + rs6000_split_multireg_move (dst, src); + DONE; + } + + unsigned dst_regno = REGNO (dst); + unsigned src_regno = REGNO (src); + bool dst_is_dmr = DMR_REGNO_P (dst_regno); + bool src_is_dmr = DMR_REGNO_P (src_regno); + bool dst_is_vsx = VSX_REGNO_P (dst_regno); + bool src_is_vsx = VSX_REGNO_P (src_regno); + + /* wD <- wD: already a dmmr move, nothing to split. */ + if (dst_is_dmr && src_is_dmr) + DONE; + + /* wD <- wa */ + if (dst_is_dmr && src_is_vsx) + { + rtx chunk0 = gen_rtx_REG (OOmode, src_regno); + rtx chunk1 = gen_rtx_REG (OOmode, src_regno + 2); + rtx chunk2 = gen_rtx_REG (OOmode, src_regno + 4); + rtx chunk3 = gen_rtx_REG (OOmode, src_regno + 6); + + emit_insn (gen_dm_insert1024 (dst, chunk0, chunk1, chunk2, chunk3)); + DONE; + } + + /* wa <- wD */ + if (dst_is_vsx && src_is_dmr) + { + rtx chunk0 = gen_rtx_REG (XOmode, dst_regno); + rtx chunk1 = gen_rtx_REG (XOmode, dst_regno + 4); + emit_insn (gen_dm_extract512 (chunk0, src, const0_rtx)); + emit_insn (gen_dm_extract512 (chunk1, src, const1_rtx)); + DONE; + } + + gcc_unreachable (); +}) + + +(define_insn_and_split "reload_tdo_load" + [(set (match_operand:TDO 0 "dmr_register_operand" "=wD") + (unspec:TDO [(match_operand:TDO 1 "memory_operand" "m")] + UNSPEC_DMF_RELOAD_FROM_MEMORY)) + (clobber (match_operand:OO 2 "vsx_register_operand" "=wa")) + (clobber (match_operand:OO 3 "vsx_register_operand" "=wa")) + (clobber (match_operand:OO 4 "vsx_register_operand" "=wa")) + (clobber (match_operand:OO 5 "vsx_register_operand" "=wa"))] + "TARGET_DMF" + "#" + "&& reload_completed" + [(const_int 0)] +{ + rtx dest = operands[0]; + rtx src = operands[1]; + rtx pair0 = operands[2]; + rtx pair1 = operands[3]; + rtx pair2 = operands[4]; + rtx pair3 = operands[5]; + + if (BYTES_BIG_ENDIAN) + { + emit_move_insn (pair0, adjust_address (src, OOmode, 0)); + emit_move_insn (pair1, adjust_address (src, OOmode, 32)); + emit_move_insn (pair2, adjust_address (src, OOmode, 64)); + emit_move_insn (pair3, adjust_address (src, OOmode, 96)); + } + else + { + emit_move_insn (pair3, adjust_address (src, OOmode, 0)); + emit_move_insn (pair2, adjust_address (src, OOmode, 32)); + emit_move_insn (pair1, adjust_address (src, OOmode, 64)); + emit_move_insn (pair0, adjust_address (src, OOmode, 96)); + } + emit_insn (gen_dm_insert1024 (dest, pair0, pair1, pair2, pair3)); + DONE; +} + [(set_attr "max_prefixed_insns" "4")]) + + +;; Reload dense math register to memory +(define_insn_and_split "reload_tdo_store" + [(set (match_operand:TDO 0 "memory_operand" "=m") + (unspec:TDO [(match_operand:TDO 1 "dmr_register_operand" "wD")] + UNSPEC_DMF_RELOAD_TO_MEMORY)) + (clobber (match_operand:XO 2 "vsx_register_operand" "=wa"))] + "TARGET_DMF" + "#" + "&& reload_completed" + [(const_int 0)] +{ + rtx dest = operands[0]; + rtx src = operands[1]; + rtx tmp_vsx_512 = operands[2]; + rtx high_mem = adjust_address (dest, XOmode, BYTES_BIG_ENDIAN ? 0 : 64); + rtx low_mem = adjust_address (dest, XOmode, BYTES_BIG_ENDIAN ? 64 : 0); + + emit_insn (gen_dm_extract512 (tmp_vsx_512, src, const0_rtx)); + emit_move_insn (high_mem, tmp_vsx_512); + + emit_insn (gen_dm_extract512 (tmp_vsx_512, src, const1_rtx)); + emit_move_insn (low_mem, tmp_vsx_512); + DONE; +} + [(set_attr "max_prefixed_insns" "4")]) + + (define_expand "mma_assemble_acc" - [(match_operand:XO 0 "fpr_reg_operand") + [(match_operand:XO 0 "accumulator_operand") (match_operand:V16QI 1 "mma_assemble_input_operand") (match_operand:V16QI 2 "mma_assemble_input_operand") (match_operand:V16QI 3 "mma_assemble_input_operand") (match_operand:V16QI 4 "mma_assemble_input_operand")] - "TARGET_MMA" + "TARGET_MMA || TARGET_DMF" { - rtx src = gen_rtx_UNSPEC_VOLATILE (XOmode, - gen_rtvec (4, operands[1], operands[2], - operands[3], operands[4]), - UNSPECV_MMA_ASSEMBLE); - emit_move_insn (operands[0], src); + if (TARGET_DMF) + { + rtx vp0 = gen_reg_rtx (OOmode); + rtx vp1 = gen_reg_rtx (OOmode); + emit_insn (gen_vsx_assemble_pair (vp0, operands[1], operands[2])); + emit_insn (gen_vsx_assemble_pair (vp1, operands[3], operands[4])); + emit_insn (gen_dm_insert512 (operands[0], vp0, vp1, const0_rtx)); + } + else + { + rtx src = gen_rtx_UNSPEC_VOLATILE (XOmode, + gen_rtvec (4, operands[1], operands[2], + operands[3], operands[4]), + UNSPECV_MMA_ASSEMBLE); + emit_move_insn (operands[0], src); + } DONE; }) @@ -445,7 +686,7 @@ ;; as an early clobber so we don't accidentally clobber the input operands. */ (define_insn_and_split "*mma_assemble_acc" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD") (unspec_volatile:XO [(match_operand:V16QI 1 "mma_assemble_input_operand" "mwa") (match_operand:V16QI 2 "mma_assemble_input_operand" "mwa") @@ -453,7 +694,7 @@ (match_operand:V16QI 4 "mma_assemble_input_operand" "mwa")] UNSPECV_MMA_ASSEMBLE))] "TARGET_MMA - && fpr_reg_operand (operands[0], XOmode)" + && accumulator_operand (operands[0], XOmode)" "#" "&& reload_completed" [(const_int 0)] @@ -466,9 +707,33 @@ DONE; }) +(define_expand "dmf_build_dmr" + [(match_operand:TDO 0 "dmr_register_operand") + (match_operand:V16QI 1 "mma_assemble_input_operand") + (match_operand:V16QI 2 "mma_assemble_input_operand") + (match_operand:V16QI 3 "mma_assemble_input_operand") + (match_operand:V16QI 4 "mma_assemble_input_operand") + (match_operand:V16QI 5 "mma_assemble_input_operand") + (match_operand:V16QI 6 "mma_assemble_input_operand") + (match_operand:V16QI 7 "mma_assemble_input_operand") + (match_operand:V16QI 8 "mma_assemble_input_operand")] + "TARGET_DMF" +{ + rtx vp0 = gen_reg_rtx (OOmode); + rtx vp1 = gen_reg_rtx (OOmode); + rtx vp2 = gen_reg_rtx (OOmode); + rtx vp3 = gen_reg_rtx (OOmode); + emit_insn (gen_vsx_assemble_pair (vp0, operands[2], operands[1])); + emit_insn (gen_vsx_assemble_pair (vp1, operands[4], operands[3])); + emit_insn (gen_vsx_assemble_pair (vp2, operands[6], operands[5])); + emit_insn (gen_vsx_assemble_pair (vp3, operands[8], operands[7])); + emit_insn (gen_dm_insert1024 (operands[0], vp0, vp1, vp2, vp3)); + DONE; +}) + (define_expand "mma_disassemble_acc" [(match_operand:V16QI 0 "mma_disassemble_output_operand") - (match_operand:XO 1 "fpr_reg_operand") + (match_operand:XO 1 "accumulator_operand") (match_operand 2 "const_0_to_3_operand")] "TARGET_MMA" { @@ -499,15 +764,36 @@ DONE; }) +;; xxmtacc/xxmfacc prime/deprime an accumulator that lives in 4 adjacent +;; FPRs -- they reformat that shared FPR/accumulator storage in place. On +;; TARGET_DMF, DMRs are a register file entirely separate from the FPRs, +;; so there is no such format to convert and these are a nop. This expand +;; only exists so __builtin_mma_xxmfacc/xxmtacc (which always resolve to +;; this pattern) correctly do nothing on TARGET_DMF instead of failing +;; to match any insn. +(define_expand "mma_<acc>" + [(set (match_operand:XO 0 "accumulator_operand") + (unspec:XO [(match_operand:XO 1 "accumulator_operand")] + MMA_ACC))] + "TARGET_MMA || TARGET_DMF" +{ + if (TARGET_DMF) + { + emit_move_insn (operands[0], operands[1]); + DONE; + } +}) + + ;; MMA instructions that do not use their accumulators as an input, still ;; must not allow their vector operands to overlap the registers used by ;; the accumulator. We enforce this by marking the output as early clobber. -(define_insn "mma_<acc>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d") - (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0")] +(define_insn "*mma_<acc>" + [(set (match_operand:XO 0 "accumulator_operand" "=&wD") + (unspec:XO [(match_operand:XO 1 "accumulator_operand" "0")] MMA_ACC))] - "TARGET_MMA" + "TARGET_MMA && !TARGET_DMF" "<acc> %A0" [(set_attr "type" "mma")]) @@ -515,15 +801,22 @@ ;; UNSPEC_VOLATILE. (define_insn "mma_xxsetaccz" - [(set (match_operand:XO 0 "fpr_reg_operand" "=d") + [(set (match_operand:XO 0 "accumulator_operand" "=wD") (unspec_volatile:XO [(const_int 0)] UNSPECV_MMA_XXSETACCZ))] "TARGET_MMA" "xxsetaccz %A0" [(set_attr "type" "mma")]) +(define_insn "dmf_dmsetdmrz" + [(set (match_operand:TDO 0 "dmr_register_operand" "=wD") + (unspec:TDO [(const_int 0)] UNSPEC_DMF_DMSETDMRZ))] + "TARGET_DMF" + "dmsetdmrz %0" + [(set_attr "type" "dmf")]) + (define_insn "mma_<vv>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD,&wD") (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa") (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")] MMA_VV))] @@ -532,8 +825,8 @@ [(set_attr "type" "mma")]) (define_insn "mma_<avv>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d") - (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD,&wD") + (unspec:XO [(match_operand:XO 1 "accumulator_operand" "0,0") (match_operand:V16QI 2 "vsx_register_operand" "v,?wa") (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")] MMA_AVV))] @@ -542,7 +835,7 @@ [(set_attr "type" "mma")]) (define_insn "mma_<pv>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD,&wD") (unspec:XO [(match_operand:OO 1 "vsx_register_operand" "v,?wa") (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")] MMA_PV))] @@ -551,8 +844,8 @@ [(set_attr "type" "mma")]) (define_insn "mma_<apv>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d") - (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD,&wD") + (unspec:XO [(match_operand:XO 1 "accumulator_operand" "0,0") (match_operand:OO 2 "vsx_register_operand" "v,?wa") (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")] MMA_APV))] @@ -561,7 +854,7 @@ [(set_attr "type" "mma")]) (define_insn "mma_<vvi4i4i8>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD,&wD") (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa") (match_operand:V16QI 2 "vsx_register_operand" "v,?wa") (match_operand:SI 3 "const_0_to_15_operand" "n,n") @@ -574,8 +867,8 @@ (set_attr "prefixed" "yes")]) (define_insn "mma_<avvi4i4i8>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d") - (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD,&wD") + (unspec:XO [(match_operand:XO 1 "accumulator_operand" "0,0") (match_operand:V16QI 2 "vsx_register_operand" "v,?wa") (match_operand:V16QI 3 "vsx_register_operand" "v,?wa") (match_operand:SI 4 "const_0_to_15_operand" "n,n") @@ -588,7 +881,7 @@ (set_attr "prefixed" "yes")]) (define_insn "mma_<vvi4i4i2>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD,&wD") (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa") (match_operand:V16QI 2 "vsx_register_operand" "v,?wa") (match_operand:SI 3 "const_0_to_15_operand" "n,n") @@ -601,8 +894,8 @@ (set_attr "prefixed" "yes")]) (define_insn "mma_<avvi4i4i2>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d") - (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD,&wD") + (unspec:XO [(match_operand:XO 1 "accumulator_operand" "0,0") (match_operand:V16QI 2 "vsx_register_operand" "v,?wa") (match_operand:V16QI 3 "vsx_register_operand" "v,?wa") (match_operand:SI 4 "const_0_to_15_operand" "n,n") @@ -615,7 +908,7 @@ (set_attr "prefixed" "yes")]) (define_insn "mma_<vvi4i4>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD,&wD") (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa") (match_operand:V16QI 2 "vsx_register_operand" "v,?wa") (match_operand:SI 3 "const_0_to_15_operand" "n,n") @@ -627,8 +920,8 @@ (set_attr "prefixed" "yes")]) (define_insn "mma_<avvi4i4>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d") - (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD,&wD") + (unspec:XO [(match_operand:XO 1 "accumulator_operand" "0,0") (match_operand:V16QI 2 "vsx_register_operand" "v,?wa") (match_operand:V16QI 3 "vsx_register_operand" "v,?wa") (match_operand:SI 4 "const_0_to_15_operand" "n,n") @@ -640,7 +933,7 @@ (set_attr "prefixed" "yes")]) (define_insn "mma_<pvi4i2>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD,&wD") (unspec:XO [(match_operand:OO 1 "vsx_register_operand" "v,?wa") (match_operand:V16QI 2 "vsx_register_operand" "v,?wa") (match_operand:SI 3 "const_0_to_15_operand" "n,n") @@ -652,8 +945,8 @@ (set_attr "prefixed" "yes")]) (define_insn "mma_<apvi4i2>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d") - (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD,&wD") + (unspec:XO [(match_operand:XO 1 "accumulator_operand" "0,0") (match_operand:OO 2 "vsx_register_operand" "v,?wa") (match_operand:V16QI 3 "vsx_register_operand" "v,?wa") (match_operand:SI 4 "const_0_to_15_operand" "n,n") @@ -665,7 +958,7 @@ (set_attr "prefixed" "yes")]) (define_insn "mma_<vvi4i4i4>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD,&wD") (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa") (match_operand:V16QI 2 "vsx_register_operand" "v,?wa") (match_operand:SI 3 "const_0_to_15_operand" "n,n") @@ -678,8 +971,8 @@ (set_attr "prefixed" "yes")]) (define_insn "mma_<avvi4i4i4>" - [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d") - (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0") + [(set (match_operand:XO 0 "accumulator_operand" "=&wD,&wD") + (unspec:XO [(match_operand:XO 1 "accumulator_operand" "0,0") (match_operand:V16QI 2 "vsx_register_operand" "v,?wa") (match_operand:V16QI 3 "vsx_register_operand" "v,?wa") (match_operand:SI 4 "const_0_to_15_operand" "n,n") @@ -690,3 +983,67 @@ "<avvi4i4i4> %A0,%x2,%x3,%4,%5,%6" [(set_attr "type" "mma") (set_attr "prefixed" "yes")]) + +(define_insn "dmf_dmxor" + [(set (match_operand:TDO 0 "dmr_register_operand" "=wD") + (unspec:TDO [(match_operand:TDO 1 "dmr_register_operand" "0") + (match_operand:TDO 2 "dmr_register_operand" "wD")] + UNSPEC_DMF_DMXOR))] + "TARGET_DMF" + "dmxor %0,%2" + [(set_attr "type" "dmf")]) + +(define_insn "dmf_<pv>" + [(set (match_operand:TDO 0 "accumulator_operand" "=wD") + (unspec:TDO [(match_operand:OO 1 "vsx_register_operand" "wa") + (match_operand:V16QI 2 "vsx_register_operand" "wa")] + DMF_PV))] + "TARGET_DMF" +{ + return "<pv> %0,%x1,%x2"; +} + [(set_attr "type" "dmf")]) + +(define_insn "dmf_<apv>" + [(set (match_operand:TDO 0 "accumulator_operand" "=wD") + (unspec:TDO [(match_operand:TDO 1 "accumulator_operand" "0") + (match_operand:OO 2 "vsx_register_operand" "wa") + (match_operand:V16QI 3 "vsx_register_operand" "wa")] + DMF_DPV))] + "TARGET_DMF" +{ + return "<apv> %0,%x2,%x3"; +} + [(set_attr "type" "dmf")]) + +(define_insn "dmf_<pvi8i4i4>" + [(set (match_operand:TDO 0 "dmr_register_operand" "=wD") + (unspec:TDO [(match_operand:OO 1 "vsx_register_operand" "wa") + (match_operand:V16QI 2 "vsx_register_operand" "wa") + (match_operand:SI 3 "u8bit_cint_operand" "n") + (match_operand:SI 4 "const_0_to_15_operand" "n") + (match_operand:SI 5 "const_0_to_15_operand" "n")] + DMF_PVI8I4I4))] + "TARGET_DMF" +{ + return "<pvi8i4i4> %0,%x1,%x2,%3,%4,%5"; +} + [(set_attr "type" "dmf") + (set_attr "prefixed" "yes")]) + +(define_insn "dmf_<dpvi8i4i4>" + [(set (match_operand:TDO 0 "dmr_register_operand" "=wD") + (unspec:TDO [(match_operand:TDO 1 "dmr_register_operand" "0") + (match_operand:OO 2 "vsx_register_operand" "wa") + (match_operand:V16QI 3 "vsx_register_operand" "wa") + (match_operand:SI 4 "u8bit_cint_operand" "n") + (match_operand:SI 5 "const_0_to_15_operand" "n") + (match_operand:SI 6 "const_0_to_15_operand" "n")] + DMF_DPVI8I4I4))] + "TARGET_DMF" +{ + return "<dpvi8i4i4> %0,%x2,%x3,%4,%5,%6"; +} + [(set_attr "type" "dmf") + (set_attr "prefixed" "yes")]) + diff --git a/gcc/config/rs6000/predicates.md b/gcc/config/rs6000/predicates.md index 4162c22f8f68..a6ec04f5922d 100644 --- a/gcc/config/rs6000/predicates.md +++ b/gcc/config/rs6000/predicates.md @@ -163,6 +163,40 @@ return VINT_REGNO_P (REGNO (op)); }) +;; Return 1 if op is a dense math register +(define_predicate "dmr_register_operand" + (match_operand 0 "register_operand") +{ + if (!TARGET_DMF) + return 0; + + if (!REG_P (op)) + return 0; + + if (!HARD_REGISTER_P (op)) + return 1; + + return DMR_REGNO_P (REGNO (op)); +}) + +;; Return 1 if op is an accumulator. On power10/11 systems, the accumulators +;; overlap with the FPRs. If TARGET_DMF is true, it will be Dense math register. +(define_predicate "accumulator_operand" + (match_operand 0 "register_operand") +{ + if (SUBREG_P (op)) + op = SUBREG_REG (op); + + if (!REG_P (op)) + return 0; + + if (!HARD_REGISTER_P (op)) + return 1; + + int r = REGNO (op); + return TARGET_DMF ? DMR_REGNO_P (r) : (FP_REGNO_P (r) && (r & 3) == 0); +}) + ;; Return 1 if op is a vector register to do logical operations on (and, or, ;; xor, etc.) (define_predicate "vlogical_operand" @@ -369,6 +403,9 @@ if (TARGET_VSX && VSX_REGNO_P (REGNO (op))) return 1; + if (TARGET_DMF && DMR_REGNO_P (REGNO (op))) + return 1; + return INT_REGNO_P (REGNO (op)) || FP_REGNO_P (REGNO (op)); }) diff --git a/gcc/config/rs6000/rs6000-builtin.cc b/gcc/config/rs6000/rs6000-builtin.cc index 2d35d5861246..c0aa885ac019 100644 --- a/gcc/config/rs6000/rs6000-builtin.cc +++ b/gcc/config/rs6000/rs6000-builtin.cc @@ -517,6 +517,8 @@ const char *rs6000_type_string (tree type_node) return "__vector_pair"; else if (type_node == vector_quad_type_node) return "__vector_quad"; + else if (type_node == dmr1024_type_node) + return "__dmr1024"; return "unknown"; } @@ -818,6 +820,21 @@ rs6000_init_builtins (void) t = build_qualified_type (vector_quad_type_node, TYPE_QUAL_CONST); ptr_vector_quad_type_node = build_pointer_type (t); + /* For TDOmode (1024-bit dense math registers), don't use an alignment + of 1024, use 512. TDOmode loads and stores are always broken up + into two vector pair loads or stores. In addition, we don't have + support for aligning the stack to 1024 bits. */ + dmr1024_type_node = make_node (OPAQUE_TYPE); + SET_TYPE_MODE (dmr1024_type_node, TDOmode); + TYPE_SIZE (dmr1024_type_node) = bitsize_int (GET_MODE_BITSIZE (TDOmode)); + TYPE_PRECISION (dmr1024_type_node) = GET_MODE_BITSIZE (TDOmode); + TYPE_SIZE_UNIT (dmr1024_type_node) = size_int (GET_MODE_SIZE (TDOmode)); + SET_TYPE_ALIGN (dmr1024_type_node, 512); + TYPE_USER_ALIGN (dmr1024_type_node) = 0; + lang_hooks.types.register_builtin_type (dmr1024_type_node, "__dmr1024"); + t = build_qualified_type (dmr1024_type_node, TYPE_QUAL_CONST); + ptr_dmr1024_type_node = build_pointer_type (t); + tdecl = add_builtin_type ("__bool char", bool_char_type_node); TYPE_NAME (bool_char_type_node) = tdecl; @@ -1104,7 +1121,8 @@ rs6000_gimple_fold_mma_builtin (gimple_stmt_iterator *gsi, gimple *stmt = gsi_stmt (*gsi); size_t fncode = (size_t) fn_code; - if (!bif_is_mma (rs6000_builtin_info[fncode])) + if (!bif_is_mma (rs6000_builtin_info[fncode]) + && !bif_is_dm (rs6000_builtin_info[fncode])) return false; /* Each call that can be gimple-expanded has an associated built-in @@ -1112,15 +1130,84 @@ rs6000_gimple_fold_mma_builtin (gimple_stmt_iterator *gsi, already expanded it! Exceptions: lxvp and stxvp. */ if (rs6000_builtin_info[fncode].assoc_bif == RS6000_BIF_NONE && fncode != RS6000_BIF_LXVP - && fncode != RS6000_BIF_STXVP) + && fncode != RS6000_BIF_STXVP + && fncode != RS6000_BIF_DMMR + && fncode != RS6000_BIF_DISASSEMBLE_DMR) return false; bifdata *bd = &rs6000_builtin_info[fncode]; - unsigned nopnds = bd->nargs; gimple_seq new_seq = NULL; gimple *new_call; tree new_decl; + if (fncode == RS6000_BIF_DMF_EXTRACT512 + || fncode == RS6000_BIF_DISASSEMBLE_DMR) + { + unsigned num_extract512; + push_gimplify_context (true); + tree dst_ptr = gimple_call_arg (stmt, 0); + tree src_ptr = gimple_call_arg (stmt, 1); + tree src_type = build_pointer_type (dmr1024_type_node); + + if (TREE_TYPE (src_ptr) != src_type) + src_ptr = build1 (NOP_EXPR, src_type, src_ptr); + + /* The following code will ensure we are sending *src as parameter. */ + tree src = make_ssa_name (TREE_TYPE (src_type)); + gimplify_assign (src, build_simple_mem_ref (src_ptr), &new_seq); + + /* Now we should call the internal builtin RS6000_BIF_DM_EXTRACT512_INTERNAL. */ + if (fncode == RS6000_BIF_DISASSEMBLE_DMR) + num_extract512 = 2; + else + num_extract512 = 1; + + tree extract_decl = rs6000_builtin_decls[RS6000_BIF_DMF_EXTRACT512_INTERNAL]; + + for (unsigned i = 0; i < num_extract512; i++) + { + tree const_arg; + if (fncode == RS6000_BIF_DISASSEMBLE_DMR) + const_arg = build_int_cstu (uint16_type_node, i); + else + const_arg = gimple_call_arg (stmt, 2); + + /* Create call. */ + new_call = gimple_build_call (extract_decl, 2, src, const_arg); + /* Create a tmp reg to denote lhs of call. */ + tree lhs = make_ssa_name (vector_quad_type_node); + + /* lhs = new_call */ + gimple_call_set_lhs (new_call, lhs); + + /* Add gimple stmt to gimple sequence. */ + gimple_seq_add_stmt (&new_seq, new_call); + + /* Now lhs contains the 512-bit value in vector_quad. We have to now + split up the vector_quad into individual vectors. */ + + new_decl = rs6000_builtin_decls[RS6000_BIF_DISASSEMBLE_ACC_INTERNAL]; + tree dst_type = build_pointer_type_for_mode (unsigned_V16QI_type_node, + ptr_mode, true); + + tree dst_base = build1 (NOP_EXPR, dst_type, dst_ptr); + for (unsigned j = 0; j < 4; j++) + { + tree dst = build2 (MEM_REF, unsigned_V16QI_type_node, dst_base, + build_int_cst (dst_type, j * 16 + i * 64)); + tree dstssa = make_ssa_name (unsigned_V16QI_type_node); + new_call = gimple_build_call (new_decl, 2, lhs, + build_int_cstu (uint16_type_node, j)); + gimple_call_set_lhs (new_call, dstssa); + gimple_seq_add_stmt (&new_seq, new_call); + gimplify_assign (dst, dstssa, &new_seq); + } + } + pop_gimplify_context (NULL); + gsi_replace_with_seq (gsi, new_seq, true); + return true; + } + /* Compatibility built-ins; we used to call these __builtin_mma_{dis,}assemble_pair, but now we call them __builtin_vsx_{dis,}assemble_pair. Handle the old versions. */ @@ -1163,7 +1250,7 @@ rs6000_gimple_fold_mma_builtin (gimple_stmt_iterator *gsi, /* If we're disassembling an accumulator into a different type, we need to emit a xxmfacc instruction now, since we cannot do it later. */ - if (fncode == RS6000_BIF_DISASSEMBLE_ACC) + if (fncode == RS6000_BIF_DISASSEMBLE_ACC && !TARGET_DMF) { new_decl = rs6000_builtin_decls[RS6000_BIF_XXMFACC_INTERNAL]; new_call = gimple_build_call (new_decl, 1, src); @@ -1232,27 +1319,49 @@ rs6000_gimple_fold_mma_builtin (gimple_stmt_iterator *gsi, /* Convert this built-in into an internal version that uses pass-by-value arguments. The internal built-in is found in the assoc_bif field. */ - new_decl = rs6000_builtin_decls[rs6000_builtin_info[fncode].assoc_bif]; + size_t new_fncode = rs6000_builtin_info[fncode].assoc_bif; + new_decl = rs6000_builtin_decls[new_fncode]; tree lhs, op[MAX_MMA_OPERANDS]; + tree lhs_type = NULL_TREE; tree acc = gimple_call_arg (stmt, 0); push_gimplify_context (true); - if (bif_is_quad (*bd)) + switch (insn_data[rs6000_builtin_info[new_fncode].icode].operand[0].mode) { - /* This built-in has a pass-by-reference accumulator input, so load it - into a temporary accumulator for use as a pass-by-value input. */ - op[0] = make_ssa_name (vector_quad_type_node); - for (unsigned i = 1; i < nopnds; i++) - op[i] = gimple_call_arg (stmt, i); - gimplify_assign (op[0], build_simple_mem_ref (acc), &new_seq); + case TDOmode: + lhs_type = dmr1024_type_node; + break; + case XOmode: + lhs_type = vector_quad_type_node; + break; + case OOmode: + lhs_type = vector_pair_type_node; + break; + default: + gcc_unreachable (); } - else - { - /* This built-in does not use its pass-by-reference accumulator argument - as an input argument, so remove it from the input list. */ - nopnds--; - for (unsigned i = 0; i < nopnds; i++) - op[i] = gimple_call_arg (stmt, i + 1); + + unsigned nopnds = 0; + for (int i = 0; i < bd->nargs; i++) + { + tree arg = gimple_call_arg (stmt, i); + if (i == 0 && !bif_is_dmr (*bd) && !bif_is_quad (*bd)) + continue; + /* If this is another DMR operand, it is passed in by reference. + The internal built-ins use pass-by-value, so load this operand + into a variable and pass that in as our operand. */ + if (POINTER_TYPE_P (TREE_TYPE (arg)) + && types_compatible_p (TREE_TYPE (TREE_TYPE (arg)), lhs_type)) + { + tree op_mem = build_simple_mem_ref (build1 (NOP_EXPR, + TREE_TYPE (arg), + arg)); + op[nopnds] = make_ssa_name (lhs_type); + gimplify_assign (op[nopnds], op_mem, &new_seq); + } + else + op[nopnds] = arg; + nopnds++; } switch (nopnds) @@ -1284,14 +1393,19 @@ rs6000_gimple_fold_mma_builtin (gimple_stmt_iterator *gsi, new_call = gimple_build_call (new_decl, 7, op[0], op[1], op[2], op[3], op[4], op[5], op[6]); break; + case 8: + new_call = gimple_build_call (new_decl, 8, op[0], op[1], op[2], op[3], + op[4], op[5], op[6], op[7]); + break; + case 9: + new_call = gimple_build_call (new_decl, 9, op[0], op[1], op[2], op[3], + op[4], op[5], op[6], op[7], op[8]); + break; default: gcc_unreachable (); } - if (fncode == RS6000_BIF_BUILD_PAIR || fncode == RS6000_BIF_ASSEMBLE_PAIR_V) - lhs = make_ssa_name (vector_pair_type_node); - else - lhs = make_ssa_name (vector_quad_type_node); + lhs = make_ssa_name (lhs_type); gimple_call_set_lhs (new_call, lhs); gimple_seq_add_stmt (&new_seq, new_call); gimplify_assign (build_simple_mem_ref (acc), lhs, &new_seq); @@ -3008,6 +3122,14 @@ mma_expand_builtin (tree exp, rtx target, insn_code icode, case 7: pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5], op[6]); break; + case 8: + pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5], op[6], + op[7]); + break; + case 9: + pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5], op[6], + op[7], op[8]); + break; default: gcc_unreachable (); } @@ -3550,7 +3672,7 @@ rs6000_expand_builtin (tree exp, rtx target, rtx /* subtarget */, /* Position of first argument (0 for void-returning functions, else 1). */ int k; /* Modes for the return value, if any, and arguments. */ - const int MAX_BUILTIN_ARGS = 6; + const int MAX_BUILTIN_ARGS = 8; machine_mode mode[MAX_BUILTIN_ARGS + 1]; if (void_func) @@ -3685,7 +3807,7 @@ rs6000_expand_builtin (tree exp, rtx target, rtx /* subtarget */, if (bif_is_lxvrze (*bifaddr)) return lxvrze_expand_builtin (target, icode, op, mode[0], mode[1]); - if (bif_is_mma (*bifaddr)) + if (bif_is_mma (*bifaddr) || bif_is_dm (*bifaddr)) return mma_expand_builtin (exp, target, icode, fcode); if (TREE_TYPE (TREE_TYPE (fndecl)) == void_type_node) diff --git a/gcc/config/rs6000/rs6000-builtins.def b/gcc/config/rs6000/rs6000-builtins.def index 85486c70f0d7..f949db410634 100644 --- a/gcc/config/rs6000/rs6000-builtins.def +++ b/gcc/config/rs6000/rs6000-builtins.def @@ -4095,3 +4095,67 @@ const vf __builtin_altivec_unpack_int8_to_fp32 (vui, const int<2>); VUPKINT8TOFP32 altivec_vupkint8tofp32 {} + +[dm] + void __builtin_dmsetdmrz (dmr1024 *); + DMSETDMRZ nothing {dm,dmint} + + dmr1024 __builtin_dmsetdmrz_internal (); + DMSETDMRZ_INTERNAL dmf_dmsetdmrz {dm} + + void __builtin_dmmr (dmr1024 *, dmr1024 *); + DMMR nothing {dm,dmint} + + dmr1024 __builtin_dmmr_internal (dmr1024); + DMMR_INTERNAL movtdo {dm} + + void __builtin_dmxor (dmr1024 *, dmr1024 *); + DMXOR nothing {dm,dmint,dmr} + + dmr1024 __builtin_dmxor_internal (dmr1024, dmr1024); + DMXOR_INTERNAL dmf_dmxor {dm} + + void __builtin_build_dmr (dmr1024 *, vuc, vuc, vuc, vuc, vuc, vuc, vuc, vuc); + BUILD_DMR nothing {dm,dmint} + + dmr1024 __builtin_build_dmr_internal (vuc, vuc, vuc, vuc, vuc, vuc, vuc, vuc); + BUILD_DMR_INTERNAL dmf_build_dmr {dm} + + void __builtin_dmr_extract512 (void *, dmr1024 *, const int<2>); + DMF_EXTRACT512 nothing {dm,dmint} + + v512 __builtin_dmr_extract512_internal (dmr1024, const int<2>); + DMF_EXTRACT512_INTERNAL dm_extract512 {dm} + + void __builtin_disassemble_dmr (void *, dmr1024 *); + DISASSEMBLE_DMR nothing {dm} + + void __builtin_mma_dmxvi8gerx4 (dmr1024 *, v256, vuc); + DMXVI8GERX4 nothing {dm,dmint} + + dmr1024 __builtin_mma_dmxvi8gerx4_internal (v256, vuc); + DMXVI8GERX4_INTERNAL dmf_dmxvi8gerx4 {dm} + + void __builtin_mma_dmxvi8gerx4pp (dmr1024 *, v256, vuc); + DMXVI8GERX4PP nothing {dm,dmint,dmr} + + dmr1024 __builtin_mma_dmxvi8gerx4pp_internal (dmr1024, v256, vuc); + DMXVI8GERX4PP_INTERNAL dmf_dmxvi8gerx4pp {dm} + + void __builtin_mma_pmdmxvi8gerx4 (dmr1024 *, v256, vuc, const int<8>, \ + const int<4>, const int<4>); + PMDMXVI8GERX4 nothing {dm,pair,dmint} + + dmr1024 __builtin_mma_pmdmxvi8gerx4_internal (v256, vuc, const int<8>, \ + const int<4>, const int<4>); + PMDMXVI8GERX4_INTERNAL dmf_pmdmxvi8gerx4 {dm,pair} + + void __builtin_mma_pmdmxvi8gerx4pp (dmr1024 *, v256, vuc, const int<8>, \ + const int<4>, const int<4>); + PMDMXVI8GERX4PP nothing {dm,pair,dmint,dmr} + + dmr1024 __builtin_mma_pmdmxvi8gerx4pp_internal (dmr1024, v256, vuc, \ + const int<8>, const int<4>, \ + const int<4>); + PMDMXVI8GERX4PP_INTERNAL dmf_pmdmxvi8gerx4pp {dm,pair} + diff --git a/gcc/config/rs6000/rs6000.cc b/gcc/config/rs6000/rs6000.cc index 7743c9cdc4a1..9e27dfb40abd 100644 --- a/gcc/config/rs6000/rs6000.cc +++ b/gcc/config/rs6000/rs6000.cc @@ -292,7 +292,8 @@ enum rs6000_reg_type { ALTIVEC_REG_TYPE, FPR_REG_TYPE, SPR_REG_TYPE, - CR_REG_TYPE + CR_REG_TYPE, + DMR_REG_TYPE }; /* Map register class to register type. */ @@ -313,6 +314,7 @@ enum rs6000_reload_reg_type { RELOAD_REG_GPR, /* General purpose registers. */ RELOAD_REG_FPR, /* Traditional floating point regs. */ RELOAD_REG_VMX, /* Altivec (VMX) registers. */ + RELOAD_REG_DMR, /* Dense Math Facility registers. */ RELOAD_REG_ANY, /* OR of GPR, FPR, Altivec masks. */ N_RELOAD_REG }; @@ -321,7 +323,7 @@ enum rs6000_reload_reg_type { into real registers, and skip the ANY class, which is just an OR of the bits. */ #define FIRST_RELOAD_REG_CLASS RELOAD_REG_GPR -#define LAST_RELOAD_REG_CLASS RELOAD_REG_VMX +#define LAST_RELOAD_REG_CLASS RELOAD_REG_DMR /* Map reload register type to a register in the register class. */ struct reload_reg_map_type { @@ -333,6 +335,7 @@ static const struct reload_reg_map_type reload_reg_map[N_RELOAD_REG] = { { "Gpr", FIRST_GPR_REGNO }, /* RELOAD_REG_GPR. */ { "Fpr", FIRST_FPR_REGNO }, /* RELOAD_REG_FPR. */ { "VMX", FIRST_ALTIVEC_REGNO }, /* RELOAD_REG_VMX. */ + { "DMR", FIRST_DMR_REGNO }, /* RELOAD_REG_DMR. */ { "Any", -1 }, /* RELOAD_REG_ANY. */ }; @@ -1226,6 +1229,8 @@ char rs6000_reg_names[][8] = "0", "1", "2", "3", "4", "5", "6", "7", /* vrsave vscr sfp */ "vrsave", "vscr", "sfp", + /* DMRs */ + "0", "1", "2", "3", "4", "5", "6", "7", }; #ifdef TARGET_REGNAMES @@ -1252,6 +1257,8 @@ static const char alt_reg_names[][8] = "%cr0", "%cr1", "%cr2", "%cr3", "%cr4", "%cr5", "%cr6", "%cr7", /* vrsave vscr sfp */ "vrsave", "vscr", "sfp", + /* DMRs */ + "%dm0", "%dm1", "%dm2", "%dm3", "%dm4", "%dm5", "%dm6", "%dm7", }; #endif @@ -1842,6 +1849,9 @@ rs6000_hard_regno_nregs_internal (int regno, machine_mode mode) else if (ALTIVEC_REGNO_P (regno)) reg_size = UNITS_PER_ALTIVEC_WORD; + else if (DMR_REGNO_P (regno)) + reg_size = UNITS_PER_DMR_WORD; + else reg_size = UNITS_PER_WORD; @@ -1858,14 +1868,33 @@ rs6000_hard_regno_mode_ok_uncached (int regno, machine_mode mode) if (COMPLEX_MODE_P (mode)) mode = GET_MODE_INNER (mode); + /* No other types other than XOmode or TDOmode can go in DMRs. */ + if (DMR_REGNO_P (regno) && !(mode ==XOmode || mode == TDOmode)) + return 0; + + /* TDOmode value can be held in either 1 DMR register or 8 VSX + registers. */ + if (mode == TDOmode) + { + if (TARGET_DMF) + return (DMR_REGNO_P (regno) + || (VSX_REGNO_P (regno) + && VSX_REGNO_P (last_regno) + && (regno & 1) == 0)); + else + return 0; + } + /* Vector pair modes need even/odd VSX register pairs. Only allow vector registers. */ if (mode == OOmode) - return (TARGET_MMA && VSX_REGNO_P (regno) && (regno & 1) == 0); + return ((TARGET_MMA || TARGET_DMF) && VSX_REGNO_P (regno) && (regno & 1) == 0); - /* MMA accumulator modes need FPR registers divisible by 4. */ + /* MMA accumulator modes need FPR registers divisible by 4. + If TARGET_DMF is true, XOmode value can be held in a DMR register. */ if (mode == XOmode) - return (TARGET_MMA && FP_REGNO_P (regno) && (regno & 3) == 0); + return ((TARGET_DMF && DMR_REGNO_P (regno)) + || (TARGET_MMA && FP_REGNO_P (regno) && (regno & 3) == 0)); /* PTImode can only go in GPRs. Quad word memory operations require even/odd register combinations, and use PTImode where we need to deal with quad @@ -1982,7 +2011,8 @@ static bool rs6000_modes_tieable_p (machine_mode mode1, machine_mode mode2) { if (mode1 == PTImode || mode1 == OOmode || mode1 == XOmode - || mode2 == PTImode || mode2 == OOmode || mode2 == XOmode) + || mode1 == TDOmode || mode2 == PTImode || mode2 == OOmode + || mode2 == XOmode || mode2 == TDOmode) return mode1 == mode2; if (ALTIVEC_OR_VSX_VECTOR_MODE (mode1)) @@ -2273,6 +2303,7 @@ rs6000_debug_reg_global (void) V4DFmode, OOmode, XOmode, + TDOmode, CCmode, CCUNSmode, CCEQmode, @@ -2308,6 +2339,7 @@ rs6000_debug_reg_global (void) rs6000_debug_reg_print (FIRST_ALTIVEC_REGNO, LAST_ALTIVEC_REGNO, "vs"); + rs6000_debug_reg_print (FIRST_DMR_REGNO, LAST_DMR_REGNO, "dmr"); rs6000_debug_reg_print (LR_REGNO, LR_REGNO, "lr"); rs6000_debug_reg_print (CTR_REGNO, CTR_REGNO, "ctr"); rs6000_debug_reg_print (CR0_REGNO, CR7_REGNO, "cr"); @@ -2328,6 +2360,7 @@ rs6000_debug_reg_global (void) "wr reg_class = %s\n" "wx reg_class = %s\n" "wA reg_class = %s\n" + "wD reg_class = %s\n" "\n", reg_class_names[rs6000_constraints[RS6000_CONSTRAINT_d]], reg_class_names[rs6000_constraints[RS6000_CONSTRAINT_v]], @@ -2335,7 +2368,8 @@ rs6000_debug_reg_global (void) reg_class_names[rs6000_constraints[RS6000_CONSTRAINT_we]], reg_class_names[rs6000_constraints[RS6000_CONSTRAINT_wr]], reg_class_names[rs6000_constraints[RS6000_CONSTRAINT_wx]], - reg_class_names[rs6000_constraints[RS6000_CONSTRAINT_wA]]); + reg_class_names[rs6000_constraints[RS6000_CONSTRAINT_wA]], + reg_class_names[rs6000_constraints[RS6000_CONSTRAINT_wD]]); nl = "\n"; for (m = 0; m < NUM_MACHINE_MODES; ++m) @@ -2632,6 +2666,20 @@ rs6000_setup_reg_addr_masks (void) addr_mask = 0; reg = reload_reg_map[rc].reg; + if (rc == RELOAD_REG_DMR) + { + if (TARGET_DMF && (m2 == XOmode || m2 == TDOmode)) + { + addr_mask = RELOAD_REG_VALID; + reg_addr[m].addr_mask[rc] = addr_mask; + any_addr_mask |= addr_mask; + } + else + reg_addr[m].addr_mask[rc] = 0; + + continue; + } + /* Can mode values go in the GPR/FPR/Altivec registers? */ if (reg >= 0 && rs6000_hard_regno_mode_ok_p[m][reg]) { @@ -2727,10 +2775,12 @@ rs6000_setup_reg_addr_masks (void) /* Vector pairs can do both indexed and offset loads if the instructions are enabled, otherwise they can only do offset loads - since it will be broken into two vector moves. Vector quads can - only do offset loads. */ - else if ((addr_mask != 0) && TARGET_MMA - && (m2 == OOmode || m2 == XOmode)) + since it will be broken into two vector moves. Vector quads and + dmr1024 type can only do offset loads. */ + else if ((addr_mask != 0) + && ((TARGET_MMA && (m2 == OOmode || m2 == XOmode)) + || (TARGET_DMF + && (m2 == TDOmode || m2 == OOmode || m2 == XOmode)))) { addr_mask |= RELOAD_REG_OFFSET; if (rc == RELOAD_REG_FPR || rc == RELOAD_REG_VMX) @@ -2778,6 +2828,9 @@ rs6000_init_hard_regno_mode_ok (bool global_init_p) for (r = FIRST_ALTIVEC_REGNO; r <= LAST_ALTIVEC_REGNO; ++r) rs6000_regno_regclass[r] = ALTIVEC_REGS; + for (r = FIRST_DMR_REGNO; r <= LAST_DMR_REGNO; ++r) + rs6000_regno_regclass[r] = DMR_REGS; + rs6000_regno_regclass[CR0_REGNO] = CR0_REGS; for (r = CR1_REGNO; r <= CR7_REGNO; ++r) rs6000_regno_regclass[r] = CR_REGS; @@ -2806,6 +2859,7 @@ rs6000_init_hard_regno_mode_ok (bool global_init_p) reg_class_to_reg_type[(int)LINK_OR_CTR_REGS] = SPR_REG_TYPE; reg_class_to_reg_type[(int)CR_REGS] = CR_REG_TYPE; reg_class_to_reg_type[(int)CR0_REGS] = CR_REG_TYPE; + reg_class_to_reg_type[(int)DMR_REGS] = DMR_REG_TYPE; if (TARGET_VSX) { @@ -2970,7 +3024,8 @@ rs6000_init_hard_regno_mode_ok (bool global_init_p) wc - Reserved to represent individual CR bits (used in LLVM). wn - always NO_REGS. wr - GPR if 64-bit mode is permitted. - wx - Float register if we can do 32-bit int stores. */ + wx - Float register if we can do 32-bit int stores. + wD - Dense math register if TARGET_DMF is enabled, else float register. */ if (TARGET_HARD_FLOAT) rs6000_constraints[RS6000_CONSTRAINT_d] = FLOAT_REGS; @@ -2978,6 +3033,10 @@ rs6000_init_hard_regno_mode_ok (bool global_init_p) rs6000_constraints[RS6000_CONSTRAINT_v] = ALTIVEC_REGS; if (TARGET_VSX) rs6000_constraints[RS6000_CONSTRAINT_wa] = VSX_REGS; + if (TARGET_DMF) + rs6000_constraints[RS6000_CONSTRAINT_wD] = DMR_REGS; + else if (TARGET_MMA) + rs6000_constraints[RS6000_CONSTRAINT_wD] = FLOAT_REGS; if (TARGET_POWERPC64) { @@ -3160,6 +3219,12 @@ rs6000_init_hard_regno_mode_ok (bool global_init_p) } } + if (TARGET_DMF) + { + reg_addr[TDOmode].reload_load = CODE_FOR_reload_tdo_load; + reg_addr[TDOmode].reload_store = CODE_FOR_reload_tdo_store; + } + /* Precalculate HARD_REGNO_NREGS. */ for (r = 0; HARD_REGISTER_NUM_P (r); ++r) for (m = 0; m < NUM_MACHINE_MODES; ++m) @@ -3186,6 +3251,9 @@ rs6000_init_hard_regno_mode_ok (bool global_init_p) else if (c == FLOAT_REGS) reg_size = UNITS_PER_FP_WORD; + else if (c == DMR_REGS) + reg_size = UNITS_PER_DMR_WORD; + else reg_size = UNITS_PER_WORD; @@ -8682,7 +8750,10 @@ reg_offset_addressing_ok_p (machine_mode mode) underlying vectors support offset addressing. */ case E_OOmode: case E_XOmode: - return TARGET_MMA; + return TARGET_MMA || TARGET_DMF; + + case E_TDOmode: + return TARGET_DMF; case E_SDmode: /* If we can do direct load/stores of SDmode, restrict it to reg+reg @@ -11237,6 +11308,12 @@ rs6000_emit_move (rtx dest, rtx source, machine_mode mode) (mode == OOmode) ? "__vector_pair" : "__vector_quad"); break; + case E_TDOmode: + if (CONST_INT_P (operands[1])) + error ("%qs is an opaque type, and you cannot set it to constants", + "__dmr1024"); + break; + case E_SImode: case E_DImode: /* Use default pattern for address of ELF small data */ @@ -12365,6 +12442,15 @@ rs6000_secondary_reload_memory (rtx addr, addr_mask = (reg_addr[mode].addr_mask[RELOAD_REG_VMX] & ~RELOAD_REG_AND_M16); + /* DMR registers have no load/store instructions; memory access goes + through an intermediate VSX stage: DMR registers are first copied to + VSX registers which are then stored to memory, or value in memory is + first loaded into VSX registers which are then copied ot DMR registers. + Use the DMR addr_mask so the address-code checks below can determine + whether a scratch GPR is needed to simplify the address. */ + else if (rclass == DMR_REGS) + addr_mask = reg_addr[mode].addr_mask[RELOAD_REG_DMR]; + /* If the register allocator hasn't made up its mind yet on the register class to use, settle on defaults to use. */ else if (rclass == NO_REGS) @@ -12693,6 +12779,12 @@ rs6000_secondary_reload_simple_move (enum rs6000_reg_type to_type, || (to_type == SPR_REG_TYPE && from_type == GPR_REG_TYPE))) return true; + /* DMRs can be copied to VSX register, and vice versa. */ + if (TARGET_DMF && (mode == XOmode || mode == TDOmode) + && ((to_type == DMR_REG_TYPE && from_type == VSX_REG_TYPE) + || (to_type == VSX_REG_TYPE && from_type == DMR_REG_TYPE))) + return true; + return false; } @@ -13387,6 +13479,10 @@ rs6000_preferred_reload_class (rtx x, enum reg_class rclass) machine_mode mode = GET_MODE (x); bool is_constant = CONSTANT_P (x); + /* Values cannot be loaded into DMR registers. */ + if (rclass == DMR_REGS) + return NO_REGS; + /* If a mode can't go in FPR/ALTIVEC/VSX registers, don't return a preferred reload class for it. */ if ((rclass == ALTIVEC_REGS || rclass == VSX_REGS) @@ -13483,7 +13579,10 @@ rs6000_preferred_reload_class (rtx x, enum reg_class rclass) return VSX_REGS; if (mode == XOmode) - return FLOAT_REGS; + return (TARGET_DMF ? VSX_REGS : FLOAT_REGS); + + if (mode == TDOmode) + return VSX_REGS; if (GET_MODE_CLASS (mode) == MODE_INT) return GENERAL_REGS; @@ -13647,6 +13746,26 @@ rs6000_secondary_reload_class (enum reg_class rclass, machine_mode mode, && regno >= 0 && CR_REGNO_P (regno)) return NO_REGS; + /* DMR registers can only be moved to/from VSX registers. + - DMR -> VSX or VSX -> DMR: direct, no scratch register needed. + - DMR -> memory or memory -> DMR: must route through VSX; request + VSX_REGS as the intermediate scratch class. + - DMR -> DMR: direct move, no scratch needed. */ + if (rclass == DMR_REGS) + { + /* VSX register or another DMR register: direct move, no scratch. */ + if (regno >= 0 && (VSX_REGNO_P (regno) || DMR_REGNO_P (regno))) + return NO_REGS; + + /* Memory (regno == -1) or unresolved pseudo: need a VSX intermediate. */ + return VSX_REGS; + } + + /* Copying into a VSX register from a DMR register: direct. */ + if ((rclass == VSX_REGS || rclass == FLOAT_REGS || rclass == ALTIVEC_REGS) + && regno >= 0 && DMR_REGNO_P (regno)) + return NO_REGS; + /* Otherwise, we need GENERAL_REGS. */ return GENERAL_REGS; } @@ -14139,6 +14258,15 @@ print_operand (FILE *file, rtx x, int code) output_operand. */ case 'A': + /* Use dense math register if TARGET_DMF enabled. */ + if (TARGET_DMF) + { + if (!REG_P (x) || !DMR_REGNO_P (REGNO (x))) + output_operand_lossage ("invalid %%A value"); + else + fprintf (file, "%d", REGNO (x) - FIRST_DMR_REGNO); + return; + } /* Write the MMA accumulator number associated with VSX register X. */ if (!REG_P (x) || !FP_REGNO_P (REGNO (x)) || (REGNO (x) % 4) != 0) output_operand_lossage ("invalid %%A value"); @@ -14527,6 +14655,27 @@ print_operand (FILE *file, rtx x, int code) ? reg - 32 : reg - FIRST_ALTIVEC_REGNO + 32); +#ifdef TARGET_REGNAMES + if (TARGET_REGNAMES) + fprintf (file, "%%vs%d", vsx_reg); + else +#endif + fprintf (file, "%d", vsx_reg); + } + return; + + case 'W': + /* Like '%x', but prints the VSX register number +2. */ + if (!REG_P (x) || !VSX_REGNO_P (REGNO (x))) + output_operand_lossage ("invalid %%W value"); + else + { + int reg = REGNO (x); + int vsx_reg = (FP_REGNO_P (reg) + ? reg - 32 + : reg - FIRST_ALTIVEC_REGNO + 32); + vsx_reg += 2; + #ifdef TARGET_REGNAMES if (TARGET_REGNAMES) fprintf (file, "%%vs%d", vsx_reg); @@ -20649,6 +20798,8 @@ rs6000_mangle_type (const_tree type) return "u13__vector_pair"; if (type == vector_quad_type_node) return "u13__vector_quad"; + if (type == dmr1024_type_node) + return "u9__dmr1024"; /* For all other types, use the default mangling. */ return NULL; @@ -22761,6 +22912,31 @@ rs6000_debug_address_cost (rtx x, machine_mode mode, } +static int +rs6000_dmr_register_move_cost (machine_mode mode, reg_class_t rclass) +{ + const int base_cost = 2; + HARD_REG_SET vsx_set = (reg_class_contents[rclass] + & reg_class_contents[VSX_REGS]); + + if (TARGET_DMF && !hard_reg_set_empty_p (vsx_set)) + { + /* XOmode can be copied in 1 instruction. */ + if (mode == XOmode) + return base_cost; + + /* TDOmode can be copied in 2 instructions. */ + else if (mode == TDOmode) + return base_cost * 2; + + else + return base_cost * 2 * hard_regno_nregs (FIRST_DMR_REGNO, mode); + } + + return 1000 * 2 * hard_regno_nregs (FIRST_DMR_REGNO, mode); +} + + /* A C expression returning the cost of moving data from a register of class CLASS1 to one of CLASS2. */ @@ -22781,10 +22957,20 @@ rs6000_register_move_cost (machine_mode mode, HARD_REG_SET to_vsx, from_vsx; to_vsx = reg_class_contents[to] & reg_class_contents[VSX_REGS]; from_vsx = reg_class_contents[from] & reg_class_contents[VSX_REGS]; - if (!hard_reg_set_empty_p (to_vsx) - && !hard_reg_set_empty_p (from_vsx) - && (TARGET_VSX - || hard_reg_set_intersect_p (to_vsx, from_vsx))) + + if ((mode == TDOmode || mode==XOmode) && from == DMR_REGS && to == DMR_REGS) + ret = 2 * hard_regno_nregs (FIRST_DMR_REGNO, mode); + + else if (from == DMR_REGS) + ret = rs6000_dmr_register_move_cost (mode, to); + + else if (to == DMR_REGS) + ret = rs6000_dmr_register_move_cost (mode, from); + + else if (!hard_reg_set_empty_p (to_vsx) + && !hard_reg_set_empty_p (from_vsx) + && (TARGET_VSX + || hard_reg_set_intersect_p (to_vsx, from_vsx))) { int reg = FIRST_FPR_REGNO; if (TARGET_VSX @@ -22880,6 +23066,9 @@ rs6000_memory_move_cost (machine_mode mode, reg_class_t rclass, ret = 4 * hard_regno_nregs (32, mode); else if (reg_classes_intersect_p (rclass, ALTIVEC_REGS)) ret = 4 * hard_regno_nregs (FIRST_ALTIVEC_REGNO, mode); + else if (reg_classes_intersect_p (rclass, DMR_REGS)) + ret = (rs6000_dmr_register_move_cost (mode, VSX_REGS) + + rs6000_memory_move_cost (mode, VSX_REGS, false)); else ret = 4 + rs6000_register_move_cost (mode, rclass, GENERAL_REGS); @@ -24088,6 +24277,8 @@ rs6000_compute_pressure_classes (enum reg_class *pressure_classes) if (TARGET_HARD_FLOAT) pressure_classes[n++] = FLOAT_REGS; } + if (TARGET_DMF) + pressure_classes[n++] = DMR_REGS; pressure_classes[n++] = CR_REGS; pressure_classes[n++] = SPECIAL_REGS; @@ -24253,6 +24444,11 @@ rs6000_debugger_regno (unsigned int regno, unsigned int format) if (regno == 64) return 64; + /* Note that the debug format register numbers may be changed + later. */ + if (DMR_REGNO_P (regno)) + return regno - FIRST_DMR_REGNO + 112; + gcc_unreachable (); } @@ -27420,9 +27616,10 @@ rs6000_split_multireg_move (rtx dst, rtx src) mode = GET_MODE (dst); nregs = hard_regno_nregs (reg, mode); - /* If we have a vector quad register for MMA, and this is a load or store, - see if we can use vector paired load/stores. */ - if (mode == XOmode && TARGET_MMA + /* If we have a vector quad register for MMA or DMR register for Dense Math, + and this is a load or store, see if we can use vector paired + load/stores. */ + if ((mode == XOmode || mode == TDOmode) && (TARGET_MMA || TARGET_DMF) && (MEM_P (dst) || MEM_P (src))) { reg_mode = OOmode; @@ -27430,7 +27627,7 @@ rs6000_split_multireg_move (rtx dst, rtx src) } /* If we have a vector pair/quad mode, split it into two/four separate vectors. */ - else if (mode == OOmode || mode == XOmode) + else if (mode == OOmode || mode == XOmode || mode == TDOmode) reg_mode = V1TImode; else if (FP_REGNO_P (reg)) reg_mode = DECIMAL_FLOAT_MODE_P (mode) ? DDmode : @@ -27476,13 +27673,13 @@ rs6000_split_multireg_move (rtx dst, rtx src) return; } - /* The __vector_pair and __vector_quad modes are multi-register + /* The __vector_pair, __vector_quad and __dmr1024 modes are multi-register modes, so if we have to load or store the registers, we have to be careful to properly swap them if we're in little endian mode below. This means the last register gets the first memory location. We also need to be careful of using the right register numbers if we are splitting XO to OO. */ - if (mode == OOmode || mode == XOmode) + if (mode == OOmode || mode == XOmode || mode == TDOmode) { nregs = hard_regno_nregs (reg, mode); int reg_mode_nregs = hard_regno_nregs (reg, reg_mode); @@ -27492,8 +27689,9 @@ rs6000_split_multireg_move (rtx dst, rtx src) unsigned size = GET_MODE_SIZE (reg_mode); /* If we are reading an accumulator register, we have to - deprime it before we can access it. */ - if (TARGET_MMA + deprime it before we can access it, unless we have dense math + registers, which do not need priming/depriming. */ + if (TARGET_MMA && !TARGET_DMF && GET_MODE (src) == XOmode && FP_REGNO_P (REGNO (src))) emit_insn (gen_mma_xxmfacc (src, src)); @@ -27526,8 +27724,9 @@ rs6000_split_multireg_move (rtx dst, rtx src) } /* If we are writing an accumulator register, we have to - prime it after we've written it. */ - if (TARGET_MMA + prime it after we've written it, unless we have dense math + registers, which do not need priming/depriming. */ + if (TARGET_MMA && !TARGET_DMF && GET_MODE (dst) == XOmode && FP_REGNO_P (REGNO (dst))) emit_insn (gen_mma_xxmtacc (dst, dst)); @@ -27595,8 +27794,9 @@ rs6000_split_multireg_move (rtx dst, rtx src) } /* We are writing an accumulator register, so we have to - prime it after we've written it. */ - if (GET_MODE (src) == XOmode) + prime it after we've written it, unless we have dense math + registers, which do not need priming/depriming. */ + if (GET_MODE (src) == XOmode && !TARGET_DMF) emit_insn (gen_mma_xxmtacc (dst, dst)); return; @@ -27608,16 +27808,17 @@ rs6000_split_multireg_move (rtx dst, rtx src) if (REG_P (src) && REG_P (dst) && (REGNO (src) < REGNO (dst))) { /* If we are reading an accumulator register, we have to - deprime it before we can access it. */ - if (TARGET_MMA + deprime it before we can access it, unless we have dense math + registers, which do not need priming/depriming. */ + if (TARGET_MMA && !TARGET_DMF && GET_MODE (src) == XOmode && FP_REGNO_P (REGNO (src))) emit_insn (gen_mma_xxmfacc (src, src)); /* Move register range backwards, if we might have destructive overlap. */ int i; - /* XO/OO are opaque so cannot use subregs. */ - if (mode == OOmode || mode == XOmode ) + /* XO/OO/TDO are opaque so cannot use subregs. */ + if (mode == OOmode || mode == XOmode || mode == TDOmode) { for (i = nregs - 1; i >= 0; i--) { @@ -27636,8 +27837,9 @@ rs6000_split_multireg_move (rtx dst, rtx src) } /* If we are writing an accumulator register, we have to - prime it after we've written it. */ - if (TARGET_MMA + prime it after we've written it, unless we have dense math + registers, which do not need priming/depriming. */ + if (TARGET_MMA && !TARGET_DMF && GET_MODE (dst) == XOmode && FP_REGNO_P (REGNO (dst))) emit_insn (gen_mma_xxmtacc (dst, dst)); } @@ -27773,8 +27975,9 @@ rs6000_split_multireg_move (rtx dst, rtx src) } /* If we are reading an accumulator register, we have to - deprime it before we can access it. */ - if (TARGET_MMA && REG_P (src) + deprime it before we can access it, unless we have dense math + registers, which do not need priming/depriming. */ + if (TARGET_MMA && !TARGET_DMF && REG_P (src) && GET_MODE (src) == XOmode && FP_REGNO_P (REGNO (src))) emit_insn (gen_mma_xxmfacc (src, src)); @@ -27790,8 +27993,8 @@ rs6000_split_multireg_move (rtx dst, rtx src) if (j == 0 && used_update) continue; - /* XO/OO are opaque so cannot use subregs. */ - if (mode == OOmode || mode == XOmode ) + /* XO/OO/TDO are opaque so cannot use subregs. */ + if (mode == OOmode || mode == XOmode || mode == TDOmode) { rtx dst_i = gen_rtx_REG (reg_mode, REGNO (dst) + j); rtx src_i = gen_rtx_REG (reg_mode, REGNO (src) + j); @@ -27805,8 +28008,9 @@ rs6000_split_multireg_move (rtx dst, rtx src) } /* If we are writing an accumulator register, we have to - prime it after we've written it. */ - if (TARGET_MMA && REG_P (dst) + prime it after we've written it, unless we have dense math + registers, which do not need priming/depriming. */ + if (TARGET_MMA && !TARGET_DMF && REG_P (dst) && GET_MODE (dst) == XOmode && FP_REGNO_P (REGNO (dst))) emit_insn (gen_mma_xxmtacc (dst, dst)); @@ -28819,7 +29023,8 @@ rs6000_invalid_conversion (const_tree fromtype, const_tree totype) if (frommode != tomode) { - /* Do not allow conversions to/from XOmode and OOmode types. */ + /* Do not allow conversions to/from XOmode, OOmode and TDOmode + types. */ if (frommode == XOmode) return N_("invalid conversion from type %<__vector_quad%>"); if (tomode == XOmode) @@ -28828,6 +29033,10 @@ rs6000_invalid_conversion (const_tree fromtype, const_tree totype) return N_("invalid conversion from type %<__vector_pair%>"); if (tomode == OOmode) return N_("invalid conversion to type %<__vector_pair%>"); + if (frommode == TDOmode) + return N_("invalid conversion from type %<__dmr1024%>"); + if (tomode == TDOmode) + return N_("invalid conversion to type %<__dmr1024%>"); } /* Conversion allowed. */ @@ -29309,33 +29518,44 @@ constant_generates_xxspltidp (vec_const_128bit_type *vsx_const) return sf_value; } -/* Now we have only two opaque types, they are __vector_quad and - __vector_pair built-in types. They are target specific and - only available when MMA is supported. With MMA supported, it - simply returns true, otherwise it checks if the given gimple - STMT is an assignment, asm or call stmt and uses either of - these two opaque types unexpectedly, if yes, it would raise - an error message and returns true, otherwise it returns false. */ +/* Now we have three opaque types: the __vector_quad, __vector_pair and + __dmr1024 built-in types. They are target specific and each one is + only available when its required ISA support is enabled: + __vector_pair requires MMA or DMF (vector pairs are used by both), + __vector_quad requires MMA, and __dmr1024 requires DMF. Note that + DMF does not imply MMA (e.g. -mcpu=future -mno-mma). With both MMA + and DMF enabled, all three types are usable and this simply returns + false, otherwise it checks if the given gimple STMT is an + assignment, asm or call stmt and uses one of these opaque types + whose ISA support is missing, if yes, it would raise an error + message and returns true, otherwise it returns false. */ bool rs6000_opaque_type_invalid_use_p (gimple *stmt) { - if (TARGET_MMA) + if (TARGET_MMA && TARGET_DMF) return false; - /* If the given TYPE is one MMA opaque type, emit the corresponding - error messages and return true, otherwise return false. */ + /* If the given TYPE is one MMA/DMF opaque type whose required ISA + support is missing, emit the corresponding error message and + return true, otherwise return false. */ auto check_and_error_invalid_use = [](tree type) { tree mv = TYPE_MAIN_VARIANT (type); - if (mv == vector_quad_type_node) + if (!TARGET_MMA && mv == vector_quad_type_node) { error ("type %<__vector_quad%> requires the %qs option", "-mmma"); return true; } - else if (mv == vector_pair_type_node) + else if (!TARGET_MMA && !TARGET_DMF && mv == vector_pair_type_node) + { + error ("type %<__vector_pair%> requires the %qs or %qs option", "-mmma" + , "-mdense-math"); + return true; + } + else if (!TARGET_DMF && mv == dmr1024_type_node) { - error ("type %<__vector_pair%> requires the %qs option", "-mmma"); + error ("type %<__dmr1024%> requires the %qs option", "-mdense-math"); return true; } return false; diff --git a/gcc/config/rs6000/rs6000.h b/gcc/config/rs6000/rs6000.h index 8235e095bcce..bcd4c39d24dc 100644 --- a/gcc/config/rs6000/rs6000.h +++ b/gcc/config/rs6000/rs6000.h @@ -657,6 +657,7 @@ extern unsigned char rs6000_recip_bits[]; #define UNITS_PER_FP_WORD 8 #define UNITS_PER_ALTIVEC_WORD 16 #define UNITS_PER_VSX_WORD 16 +#define UNITS_PER_DMR_WORD 128 /* Type used for ptrdiff_t, as a string used in a declaration. */ #define PTRDIFF_TYPE "int" @@ -752,7 +753,9 @@ enum data_align { align_abi, align_opt, align_both }; RS/6000 has 32 fixed-point registers, 32 floating-point registers, a count register, a link register, and 8 condition register fields, which we view here as separate registers. AltiVec adds 32 vector - registers and a VRsave register. + registers and a VRsave register. The Dense Math Facility, which may be + present in a future processor, introduces 8 DMR registers each having + 1024 bits. In addition, the difference between the frame and argument pointers is a function of the number of registers saved, so we need to have a @@ -767,7 +770,7 @@ enum data_align { align_abi, align_opt, align_both }; Another pseudo (not included in DWARF_FRAME_REGISTERS) is soft frame pointer, which is eventually eliminated in favor of SP or FP. */ -#define FIRST_PSEUDO_REGISTER 111 +#define FIRST_PSEUDO_REGISTER 119 /* Use standard DWARF numbering for DWARF debugging information. */ #define DEBUGGER_REGNO(REGNO) rs6000_debugger_regno ((REGNO), 0) @@ -804,7 +807,9 @@ enum data_align { align_abi, align_opt, align_both }; /* cr0..cr7 */ \ 0, 0, 0, 0, 0, 0, 0, 0, \ /* vrsave vscr sfp */ \ - 1, 1, 1 \ + 1, 1, 1, \ + /* DMRs */ \ + 0, 0, 0, 0, 0, 0, 0, 0 \ } /* Like `CALL_USED_REGISTERS' except this macro doesn't require that @@ -828,7 +833,9 @@ enum data_align { align_abi, align_opt, align_both }; /* cr0..cr7 */ \ 1, 1, 0, 0, 0, 1, 1, 1, \ /* vrsave vscr sfp */ \ - 0, 0, 0 \ + 0, 0, 0, \ + /* DMRs */ \ + 1, 1, 1, 1, 1, 1, 1, 1 \ } #define TOTAL_ALTIVEC_REGS (LAST_ALTIVEC_REGNO - FIRST_ALTIVEC_REGNO + 1) @@ -865,6 +872,7 @@ enum data_align { align_abi, align_opt, align_both }; v2 (not saved; incoming vector arg reg; return value) v19 - v14 (not saved or used for anything) v31 - v20 (saved; order given to save least number) + dmr0 - dmr7 (not saved) vrsave, vscr (fixed) sfp (fixed) */ @@ -907,6 +915,8 @@ enum data_align { align_abi, align_opt, align_both }; 66, \ 83, 82, 81, 80, 79, 78, \ 95, 94, 93, 92, 91, 90, 89, 88, 87, 86, 85, 84, \ + /* DMRs */ \ + 111, 112, 113, 114, 115, 116, 117, 118, \ 108, 109, \ 110 \ } @@ -933,6 +943,9 @@ enum data_align { align_abi, align_opt, align_both }; /* True if register is a VSX register. */ #define VSX_REGNO_P(N) (FP_REGNO_P (N) || ALTIVEC_REGNO_P (N)) +/* True if register is a DMR register. */ +#define DMR_REGNO_P(N) ((N) >= FIRST_DMR_REGNO && (N) <= LAST_DMR_REGNO) + /* Alternate name for any vector register supporting floating point, no matter which instruction set(s) are available. */ #define VFLOAT_REGNO_P(N) \ @@ -972,7 +985,7 @@ enum data_align { align_abi, align_opt, align_both }; /* Modes that are not vectors, but require vector alignment. Treat these like vectors in terms of loads and stores. */ #define VECTOR_ALIGNMENT_P(MODE) \ - (FLOAT128_VECTOR_P (MODE) || (MODE) == OOmode || (MODE) == XOmode) + (FLOAT128_VECTOR_P (MODE) || (MODE) == OOmode || (MODE) == XOmode || (MODE) == TDOmode) #define ALTIVEC_VECTOR_MODE(MODE) \ ((MODE) == V16QImode \ @@ -1070,6 +1083,7 @@ enum reg_class FLOAT_REGS, ALTIVEC_REGS, VSX_REGS, + DMR_REGS, VRSAVE_REGS, VSCR_REGS, GEN_OR_FLOAT_REGS, @@ -1099,6 +1113,7 @@ enum reg_class "FLOAT_REGS", \ "ALTIVEC_REGS", \ "VSX_REGS", \ + "DMR_REGS", \ "VRSAVE_REGS", \ "VSCR_REGS", \ "GEN_OR_FLOAT_REGS", \ @@ -1133,6 +1148,8 @@ enum reg_class { 0x00000000, 0x00000000, 0xffffffff, 0x00000000 }, \ /* VSX_REGS. */ \ { 0x00000000, 0xffffffff, 0xffffffff, 0x00000000 }, \ + /* DMR_REGS. */ \ + { 0x00000000, 0x00000000, 0x00000000, 0x007f8000 }, \ /* VRSAVE_REGS. */ \ { 0x00000000, 0x00000000, 0x00000000, 0x00001000 }, \ /* VSCR_REGS. */ \ @@ -1160,7 +1177,7 @@ enum reg_class /* CA_REGS. */ \ { 0x00000000, 0x00000000, 0x00000000, 0x00000004 }, \ /* ALL_REGS. */ \ - { 0xffffffff, 0xffffffff, 0xffffffff, 0x00007fff } \ + { 0xffffffff, 0xffffffff, 0xffffffff, 0x007fffff } \ } /* The same information, inverted: @@ -1184,6 +1201,7 @@ enum r6000_reg_class_enum { RS6000_CONSTRAINT_wr, /* GPR register if 64-bit */ RS6000_CONSTRAINT_wx, /* FPR register for STFIWX */ RS6000_CONSTRAINT_wA, /* BASE_REGS if 64-bit. */ + RS6000_CONSTRAINT_wD, /* Accumulator registers. */ RS6000_CONSTRAINT_MAX }; @@ -2060,7 +2078,16 @@ extern char rs6000_reg_names[][8]; /* register names (0 vs. %r0). */ &rs6000_reg_names[108][0], /* vrsave */ \ &rs6000_reg_names[109][0], /* vscr */ \ \ - &rs6000_reg_names[110][0] /* sfp */ \ + &rs6000_reg_names[110][0], /* sfp */ \ + \ + &rs6000_reg_names[111][0], /* dmr0 */ \ + &rs6000_reg_names[112][0], /* dmr1 */ \ + &rs6000_reg_names[113][0], /* dmr2 */ \ + &rs6000_reg_names[114][0], /* dmr3 */ \ + &rs6000_reg_names[115][0], /* dmr4 */ \ + &rs6000_reg_names[116][0], /* dmr5 */ \ + &rs6000_reg_names[117][0], /* dmr6 */ \ + &rs6000_reg_names[118][0] /* dmr7 */ \ } /* Table of additional register names to use in user input. */ @@ -2114,6 +2141,8 @@ extern char rs6000_reg_names[][8]; /* register names (0 vs. %r0). */ {"vs52", 84}, {"vs53", 85}, {"vs54", 86}, {"vs55", 87}, \ {"vs56", 88}, {"vs57", 89}, {"vs58", 90}, {"vs59", 91}, \ {"vs60", 92}, {"vs61", 93}, {"vs62", 94}, {"vs63", 95}, \ + {"dmr0", 111}, {"dmr1", 112}, {"dmr2", 113}, {"dmr3", 114}, \ + {"dmr4", 115}, {"dmr5", 116}, {"dmr6", 117}, {"dmr7", 118}, \ } /* This is how to output an element of a case-vector that is relative. */ @@ -2247,6 +2276,7 @@ enum rs6000_builtin_type_index RS6000_BTI_const_str, /* pointer to const char * */ RS6000_BTI_vector_pair, /* unsigned 256-bit types (vector pair). */ RS6000_BTI_vector_quad, /* unsigned 512-bit types (vector quad). */ + RS6000_BTI_dmr1024, /* unsigned 1024-bit type (dmr1024). */ RS6000_BTI_const_ptr_void, /* const pointer to void */ RS6000_BTI_ptr_V16QI, RS6000_BTI_ptr_V1TI, @@ -2285,6 +2315,7 @@ enum rs6000_builtin_type_index RS6000_BTI_ptr_dfloat128, RS6000_BTI_ptr_vector_pair, RS6000_BTI_ptr_vector_quad, + RS6000_BTI_ptr_dmr1024, RS6000_BTI_ptr_long_long, RS6000_BTI_ptr_long_long_unsigned, RS6000_BTI_INTPTI, @@ -2346,6 +2377,7 @@ enum rs6000_builtin_type_index #define const_str_type_node (rs6000_builtin_types[RS6000_BTI_const_str]) #define vector_pair_type_node (rs6000_builtin_types[RS6000_BTI_vector_pair]) #define vector_quad_type_node (rs6000_builtin_types[RS6000_BTI_vector_quad]) +#define dmr1024_type_node (rs6000_builtin_types[RS6000_BTI_dmr1024]) #define pcvoid_type_node (rs6000_builtin_types[RS6000_BTI_const_ptr_void]) #define ptr_V16QI_type_node (rs6000_builtin_types[RS6000_BTI_ptr_V16QI]) #define ptr_V1TI_type_node (rs6000_builtin_types[RS6000_BTI_ptr_V1TI]) @@ -2384,6 +2416,7 @@ enum rs6000_builtin_type_index #define ptr_dfloat128_type_node (rs6000_builtin_types[RS6000_BTI_ptr_dfloat128]) #define ptr_vector_pair_type_node (rs6000_builtin_types[RS6000_BTI_ptr_vector_pair]) #define ptr_vector_quad_type_node (rs6000_builtin_types[RS6000_BTI_ptr_vector_quad]) +#define ptr_dmr1024_type_node (rs6000_builtin_types[RS6000_BTI_ptr_dmr1024]) #define ptr_long_long_integer_type_node (rs6000_builtin_types[RS6000_BTI_ptr_long_long]) #define ptr_long_long_unsigned_type_node (rs6000_builtin_types[RS6000_BTI_ptr_long_long_unsigned]) diff --git a/gcc/config/rs6000/rs6000.md b/gcc/config/rs6000/rs6000.md index 4ec3d3da8a99..e06dcec65889 100644 --- a/gcc/config/rs6000/rs6000.md +++ b/gcc/config/rs6000/rs6000.md @@ -51,6 +51,8 @@ (VRSAVE_REGNO 108) (VSCR_REGNO 109) (FRAME_POINTER_REGNUM 110) + (FIRST_DMR_REGNO 111) + (LAST_DMR_REGNO 118) ]) ;; @@ -221,7 +223,7 @@ vecsimple,veccomplex,vecdiv,veccmp,veccmpsimple,vecperm, vecfloat,vecfdiv,vecdouble,mtvsr,mfvsr,crypto, veclogical,veccmpfx,vecexts,vecmove, - htm,htmsimple,dfp,mma, + htm,htmsimple,dfp,mma,dmf, fused_arith_logical, fused_cmp_isel, fused_carry, @@ -369,7 +371,7 @@ (const (symbol_ref "(enum attr_cpu) rs6000_tune"))) ;; The ISA we implement. -(define_attr "isa" "any,p5,p6,p7,p7v,p8,p8v,p9,p9v,p9kf,p9tf,p10,future" +(define_attr "isa" "any,p5,p6,p7,p7v,p8,p8v,p9,p9v,p9kf,p9tf,p10,future,mma,dmf" (const_string "any")) ;; Is this alternative enabled for the current CPU/ISA/etc.? @@ -425,6 +427,14 @@ (and (eq_attr "isa" "future") (match_test "TARGET_FUTURE")) (const_int 1) + + (and (eq_attr "isa" "mma") + (match_test "TARGET_MMA")) + (const_int 1) + + (and (eq_attr "isa" "dmf") + (match_test "TARGET_DMF")) + (const_int 1) ] (const_int 0))) ;; If this instruction is microcoded on the CELL processor diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi index 4761b07973a7..6bd1e8247add 100644 --- a/gcc/doc/extend.texi +++ b/gcc/doc/extend.texi @@ -18714,6 +18714,7 @@ instructions, but allow the compiler to schedule those calls. * PowerPC Hardware Transactional Memory Built-in Functions:: * PowerPC Atomic Memory Operation Functions:: * PowerPC Matrix-Multiply Assist Built-in Functions:: +* PowerPC Dense Math Facility Built-in Functions:: * PRU Built-in Functions:: * RISC-V Built-in Functions:: * RISC-V Vector Intrinsics:: @@ -27468,6 +27469,43 @@ __vector_pair __builtin_vsx_lxvp (size_t, __vector_pair *); void __builtin_vsx_stxvp (__vector_pair, size_t, __vector_pair *); @end smallexample +Future ISA of PowerPC may add new Matrix-Multiply Assist Plus (MMA+) +instructions. GCC provides support for these instructions through the +following built-in functions which are enabled with the @code{-mmma} option. +The vec_t type below is defined to be a normal vector unsigned char type. +The uint2, uint4 and uint8 parameters are 2-bit, 4-bit and 8-bit unsigned +integer constants respectively. The compiler will verify that they are +constants and that their values are within range. The __dmr1024 type is a +1024-bit integer type. + +The built-in functions supported are: + +@smallexample +void __builtin_mma_dmxvi8gerx4 (__dmr1024 *, __vector_pair, vec_t); +void __builtin_mma_dmxvi8gerx4pp (__dmr1024 *, __vector_pair, vec_t); + +void __builtin_mma_pmdmxvi8gerx4 (__dmr1024 *, __vector_pair, vec_t, uint8, uint4, uint4); +void __builtin_mma_pmdmxvi8gerx4pp (__dmr1024 *, __vector_pair, vec_t, uint8, uint4, uint4); +@end smallexample + +@node PowerPC Dense Math Facility Built-in Functions +@subsection PowerPC Dense Math Facility Built-in Functions + +A future PowerPC processor may provide Dense Math Facility (DMF) +instructions. GCC provides support for these instructions through the +following built-in functions which are enabled with the @code{-mdense-math} +option. The vec_t type below is defined to be a normal vector unsigned char +type. The __dmr1024 type is a 1024 bit integer type. + +The built-in functions supported are: + +@smallexample +void __builtin_dmsetdmrz (__dmr1024 *); +void __builtin_dmmr (__dmr1024 *, __dmr1024 *); +void __builtin_dmxor (__dmr1024 *, __dmr1024 *); +void __builtin_build_dmr (__dmr1024 *, vec_t, vec_t, vec_t, vec_t, vec_t, vec_t, vec_t, vec_t); +@end smallexample + @node PRU Built-in Functions @subsection PRU Built-in Functions diff --git a/gcc/doc/md.texi b/gcc/doc/md.texi index a98a572cc017..4253dc831e85 100644 --- a/gcc/doc/md.texi +++ b/gcc/doc/md.texi @@ -3297,6 +3297,10 @@ Like @code{b}, if @option{-mpowerpc64} is used; otherwise, @code{NO_REGS}. @item wB Signed 5-bit constant integer that can be loaded into an Altivec register. +@item wD +Dense math register if @option{-mdense-math} is used; floating point register if +@option{-mmma} with @option{-mno-dense-math}; otherwise, @code{NO_REGS}. + @item wE Vector constant that can be loaded with the XXSPLTIB instruction. diff --git a/gcc/testsuite/gcc.target/powerpc/dmf-build-dmr.c b/gcc/testsuite/gcc.target/powerpc/dmf-build-dmr.c new file mode 100644 index 000000000000..d5e85e64e27e --- /dev/null +++ b/gcc/testsuite/gcc.target/powerpc/dmf-build-dmr.c @@ -0,0 +1,15 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target powerpc_future_compile_ok } */ +/* { dg-require-effective-target lp64 } */ +/* { dg-options "-mdejagnu-cpu=future -O2" } */ + +typedef unsigned char vec_t __attribute__((vector_size(16))); + +void +foo2 (__dmr1024 *dst, vec_t *src) +{ + __builtin_build_dmr (dst, src[0], src[1], src[2], src[3], src[4], src[5], src[6], src[7]); +} + +/* { dg-final { scan-assembler-times {\mdmxxinstdmr512\M} 2 } } */ +/* { dg-final { scan-assembler-times {\mlxv\M} 8 } } */ diff --git a/gcc/testsuite/gcc.target/powerpc/dmf-builtin.c b/gcc/testsuite/gcc.target/powerpc/dmf-builtin.c new file mode 100644 index 000000000000..02bbebf69f82 --- /dev/null +++ b/gcc/testsuite/gcc.target/powerpc/dmf-builtin.c @@ -0,0 +1,81 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target powerpc_future_compile_ok } */ +/* { dg-require-effective-target lp64 } */ +/* { dg-options "-mdejagnu-cpu=future -O2" } */ + +typedef unsigned char vec_t __attribute__((vector_size(16))); + +void +foo (__dmr1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dmr1024 dmr; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_dmsetdmrz (&dmr); + __builtin_mma_dmxvi8gerx4 (&dmr, vp, vec); + *dst = dmr; +} + +void +bar (__dmr1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dmr1024 dmr = dst[0]; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_dmxvi8gerx4 (&dmr, vp, vec); + dst[1] = dmr; +} + +/* { dg-final { scan-assembler-times {\mdmxvi8gerx4\M} 2 } } */ + +void +foo_1 (__dmr1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dmr1024 dmr; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_dmsetdmrz (&dmr); + __builtin_mma_dmxvi8gerx4pp (&dmr, vp, vec); + *dst = dmr; +} + +void +bar_1 (__dmr1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dmr1024 dmr = dst[0];; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_dmxvi8gerx4pp (&dmr, vp, vec); + dst[1] = dmr; +} + +/* { dg-final { scan-assembler-times {\mdmxvi8gerx4pp\M} 2 } } */ + +void +foo_2 (__dmr1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_pmdmxvi8gerx4 (dst, vp, vec, 255, 15, 2); +} + +/* { dg-final { scan-assembler-times {\mpmdmxvi8gerx4\M} 1 } } */ + +void +foo_3 (__dmr1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_pmdmxvi8gerx4pp (dst, vp, vec, 255, 15, 2); +} + +/* { dg-final { scan-assembler-times {\mpmdmxvi8gerx4pp\M} 1 } } */ + + +void +foo_5 (__dmr1024 *dst, __dmr1024 *src) +{ + __builtin_dmxor (dst, src); +} + +/* { dg-final { scan-assembler-times {\mdmxor\M} 1 } } */ diff --git a/gcc/testsuite/gcc.target/powerpc/dmf-disassemble-dmr.c b/gcc/testsuite/gcc.target/powerpc/dmf-disassemble-dmr.c new file mode 100644 index 000000000000..64a966ea61e6 --- /dev/null +++ b/gcc/testsuite/gcc.target/powerpc/dmf-disassemble-dmr.c @@ -0,0 +1,24 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target powerpc_future_compile_ok } */ +/* { dg-require-effective-target lp64 } */ +/* { dg-options "-mdejagnu-cpu=future -O2" } */ + +typedef unsigned char vec_t __attribute__((vector_size(16))); + +void +bar (vec_t *dst, __dmr1024 *src) +{ + vec_t res[8]; + __builtin_disassemble_dmr (res, src); + dst[0] = res[0]; + dst[2] = res[1]; + dst[4] = res[2]; + dst[6] = res[3]; + dst[8] = res[4]; + dst[10] = res[5]; + dst[12] = res[6]; + dst[14] = res[7]; +} + +/* { dg-final { scan-assembler-times {\mdmxxextfdmr512\M} 2 } } */ +/* { dg-final { scan-assembler-times {\mstxv\M} 8 } } */ diff --git a/gcc/testsuite/gcc.target/powerpc/dmf-extract512.c b/gcc/testsuite/gcc.target/powerpc/dmf-extract512.c new file mode 100644 index 000000000000..2b112c2725cc --- /dev/null +++ b/gcc/testsuite/gcc.target/powerpc/dmf-extract512.c @@ -0,0 +1,20 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target powerpc_future_compile_ok } */ +/* { dg-require-effective-target lp64 } */ +/* { dg-options "-mdejagnu-cpu=future -O2" } */ + +typedef unsigned char vec_t __attribute__((vector_size(16))); + +void +bar (vec_t *dst, __dmr1024 *src) +{ + vec_t res[4]; + __builtin_dmr_extract512 (res, src, 0); + dst[0] = res[0]; + dst[2] = res[1]; + dst[4] = res[2]; + dst[6] = res[3]; +} + +/* { dg-final { scan-assembler-times {\mdmxxextfdmr512\M} 1 } } */ +/* { dg-final { scan-assembler-times {\mstxv\M} 4 } } */ diff --git a/gcc/testsuite/gcc.target/powerpc/dmf-no-mma.c b/gcc/testsuite/gcc.target/powerpc/dmf-no-mma.c new file mode 100644 index 000000000000..3da0c8c601ec --- /dev/null +++ b/gcc/testsuite/gcc.target/powerpc/dmf-no-mma.c @@ -0,0 +1,19 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target powerpc_future_compile_ok } */ +/* { dg-options "-mdejagnu-cpu=future -mdense-math -mno-mma -O2" } */ + +/* Dense Math (-mdense-math) without MMA supports __vector_pair and + __dmr1024 (see dmf-no-mma-1.c), but __vector_quad moves still + require MMA. Verify that a __vector_quad copy under -mno-mma is + diagnosed with a proper error message via the movxo expander + instead of ICEing. The error is emitted during RTL expansion, so + its reported location can be fragile; match it anywhere in the + output. */ + +void +copy_quad (__vector_quad *dst, __vector_quad *src) +{ + *dst = *src; +} + +/* { dg-error "type '__vector_quad' requires the '-mmma' option" "" { target *-*-* } 0 } */ diff --git a/gcc/testsuite/gcc.target/powerpc/dmr1024-alignment.c b/gcc/testsuite/gcc.target/powerpc/dmr1024-alignment.c new file mode 100644 index 000000000000..147556c90e01 --- /dev/null +++ b/gcc/testsuite/gcc.target/powerpc/dmr1024-alignment.c @@ -0,0 +1,33 @@ +/* { dg-do run } */ +/* { dg-require-effective-target hard_float } */ +/* { dg-options "-O2 -mhard-float" } */ + +/* Verify the __dmr1024 opaque type is always registered (regardless of + whether -mdense-math is enabled) and has the expected size and + alignment, following the same model as __vector_pair and + __vector_quad in mma-alignment.c. */ + +#include <stdlib.h> + +struct +{ + int __attribute__ ((__aligned__)) ivar; + __dmr1024 dmr; +} s; + +int +main (void) +{ + /* __dmr1024 holds the contents of a 1,024-bit Dense Math Register. */ + if (sizeof (__dmr1024) != 128) + abort (); + + /* __dmr1024 loads/stores are always broken into two vector-pair + transfers, and there is no support for aligning the stack to + 1,024 bits, so the type alignment is 512 bits (64 bytes), matching + __vector_quad rather than its own 1,024-bit size. */ + if (__alignof__ (s.dmr) != 64) + abort (); + + return 0; +} diff --git a/gcc/testsuite/gcc.target/powerpc/dmr1024-compile.c b/gcc/testsuite/gcc.target/powerpc/dmr1024-compile.c new file mode 100644 index 000000000000..2ae66738f026 --- /dev/null +++ b/gcc/testsuite/gcc.target/powerpc/dmr1024-compile.c @@ -0,0 +1,46 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target powerpc_future_compile_ok } */ +/* { dg-options "-mdejagnu-cpu=future -mdense-math -O2" } */ + +/* Verify that __dmr1024 is accepted as a type with the expected size + and alignment, and that it can be used through pointers without + triggering any diagnostics. Only pointers are passed around here; + actual __dmr1024 value copies are covered by dmr1024-copy.c. */ + +typedef __dmr1024 dmr_t; + +_Static_assert (sizeof (__dmr1024) == 128, "__dmr1024 must be 128 bytes"); +_Static_assert (__alignof__ (__dmr1024) == 64, + "__dmr1024 alignment must be 64 bytes"); + +struct dmr_holder +{ + __dmr1024 dmr; +}; + +extern __dmr1024 global_dmr; +extern const dmr_t *global_dmr_ptr; + +__dmr1024 * +identity (__dmr1024 *p) +{ + return p; +} + +const dmr_t * +identity_const (const dmr_t *p) +{ + return p; +} + +void * +as_void_ptr (__dmr1024 *p) +{ + return (void *) p; +} + +__dmr1024 * +holder_dmr (struct dmr_holder *h) +{ + return &h->dmr; +} diff --git a/gcc/testsuite/gcc.target/powerpc/dmr1024-invalid-use-1.c b/gcc/testsuite/gcc.target/powerpc/dmr1024-invalid-use-1.c new file mode 100644 index 000000000000..dfd72bded7a3 --- /dev/null +++ b/gcc/testsuite/gcc.target/powerpc/dmr1024-invalid-use-1.c @@ -0,0 +1,32 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target powerpc_future_compile_ok } */ +/* { dg-options "-mdejagnu-cpu=future -mdense-math -O2 -std=gnu17" } */ + +/* The __dmr1024 opaque type, like __vector_quad and __vector_pair + (see pr96506-1.c), may not be used as a function parameter. Verify + we flag errors on these uses rather than ICE. + + This must be kept separate from the return-value test + (dmr1024-invalid-use-2.c): the parameter diagnostics below are only + emitted during RTL expansion, which is skipped entirely once a + parse-time error (such as the return-value diagnostic) has been + emitted. */ + +extern void bar0 (); +extern void bar1 (); + +typedef __dmr1024 dmr_t; + +void +foo0 (void) +{ + __dmr1024 v; + bar0 (v); /* { dg-error "invalid use of Dense Math operand of type .__dmr1024. as a function parameter" } */ +} + +void +foo1 (void) +{ + dmr_t v; + bar1 (v); /* { dg-error "invalid use of Dense Math operand of type .__dmr1024. as a function parameter" } */ +} diff --git a/gcc/testsuite/gcc.target/powerpc/dmr1024-invalid-use-2.c b/gcc/testsuite/gcc.target/powerpc/dmr1024-invalid-use-2.c new file mode 100644 index 000000000000..c814cf78827e --- /dev/null +++ b/gcc/testsuite/gcc.target/powerpc/dmr1024-invalid-use-2.c @@ -0,0 +1,23 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target powerpc_future_compile_ok } */ +/* { dg-options "-mdejagnu-cpu=future -mdense-math -O2" } */ + +/* The __dmr1024 opaque type, like __vector_quad and __vector_pair + (see pr96506-2.c), may not be used as a function return value. + Verify we flag errors on these uses rather than ICE. Kept separate + from the function-parameter test (dmr1024-invalid-use-1.c); see the + comment there. */ + +typedef __dmr1024 dmr_t; + +__dmr1024 +foo0 (__dmr1024 *src) +{ /* { dg-error "invalid use of Dense Math type .__dmr1024. as a function return value" } */ + return *src; +} + +dmr_t +foo1 (dmr_t *src) +{ /* { dg-error "invalid use of Dense Math type .__dmr1024. as a function return value" } */ + return *src; +} diff --git a/gcc/testsuite/gcc.target/powerpc/future-dmf-xxacc.c b/gcc/testsuite/gcc.target/powerpc/future-dmf-xxacc.c new file mode 100644 index 000000000000..2242f6693d02 --- /dev/null +++ b/gcc/testsuite/gcc.target/powerpc/future-dmf-xxacc.c @@ -0,0 +1,18 @@ +/* { dg-do compile } */ +/* { dg-options "-mdejagnu-cpu=future -O2" } */ +/* { dg-require-effective-target powerpc_future_compile_ok } */ + +void +foo (__vector_quad *acc) +{ + __builtin_mma_xxmtacc (acc); +} + +void +bar (__vector_quad *acc) +{ + __builtin_mma_xxmfacc (acc); +} + +/* { dg-final { scan-assembler-not {\mxxmtacc\M} } } */ +/* { dg-final { scan-assembler-not {\mxxmfacc\M} } } */ diff --git a/gcc/testsuite/gcc.target/powerpc/future-nodmf-xxacc.c b/gcc/testsuite/gcc.target/powerpc/future-nodmf-xxacc.c new file mode 100644 index 000000000000..a643dc5f6380 --- /dev/null +++ b/gcc/testsuite/gcc.target/powerpc/future-nodmf-xxacc.c @@ -0,0 +1,18 @@ +/* { dg-do compile } */ +/* { dg-options "-mdejagnu-cpu=future -O2 -mno-dense-math" } */ +/* { dg-require-effective-target powerpc_future_compile_ok } */ + +void +foo (__vector_quad *acc) +{ + __builtin_mma_xxmtacc (acc); +} + +void +bar (__vector_quad *acc) +{ + __builtin_mma_xxmfacc (acc); +} + +/* { dg-final { scan-assembler {\mxxmtacc\M} } } */ +/* { dg-final { scan-assembler {\mxxmfacc\M} } } */
