On Sat, Nov 2, 2024 at 8:58 PM Robin Dapp <rdapp....@gmail.com> wrote: > > From: Robin Dapp <rd...@ventanamicro.com> > > This patch adds a zero else operand to masked loads, in particular the > masked gather load builtins that are used for gather vectorization. > > gcc/ChangeLog: > > * config/i386/i386-expand.cc (ix86_expand_special_args_builtin): > Add else-operand handling. > (ix86_expand_builtin): Ditto. > * config/i386/predicates.md (vcvtne2ps2bf_parallel): New > predicate. > (maskload_else_operand): Ditto. > * config/i386/sse.md: Use predicate. > --- > gcc/config/i386/i386-expand.cc | 26 ++++++-- > gcc/config/i386/predicates.md | 4 ++ > gcc/config/i386/sse.md | 112 +++++++++++++++++++++------------ > 3 files changed, 97 insertions(+), 45 deletions(-) > > diff --git a/gcc/config/i386/i386-expand.cc b/gcc/config/i386/i386-expand.cc > index 0de0e842731..6c61f9f87c2 100644 > --- a/gcc/config/i386/i386-expand.cc > +++ b/gcc/config/i386/i386-expand.cc > @@ -12995,10 +12995,11 @@ ix86_expand_special_args_builtin (const struct > builtin_description *d, > { > tree arg; > rtx pat, op; > - unsigned int i, nargs, arg_adjust, memory; > + unsigned int i, nargs, arg_adjust, memory = -1; > unsigned int constant = 100; > bool aligned_mem = false; > - rtx xops[4]; > + rtx xops[4] = {}; > + bool add_els = false; > enum insn_code icode = d->icode; > const struct insn_data_d *insn_p = &insn_data[icode]; > machine_mode tmode = insn_p->operand[0].mode; > @@ -13125,6 +13126,9 @@ ix86_expand_special_args_builtin (const struct > builtin_description *d, > case V4DI_FTYPE_PCV4DI_V4DI: > case V4SI_FTYPE_PCV4SI_V4SI: > case V2DI_FTYPE_PCV2DI_V2DI: > + /* Two actual args but an additional else operand. */ > + add_els = true; > + /* Fallthru. */ > case VOID_FTYPE_INT_INT64: > nargs = 2; > klass = load; > @@ -13397,6 +13401,12 @@ ix86_expand_special_args_builtin (const struct > builtin_description *d, > xops[i]= op; > } > > + if (add_els) > + { > + xops[i] = CONST0_RTX (GET_MODE (xops[0])); > + nargs++; > + } > + > switch (nargs) > { > case 0: > @@ -13653,7 +13663,7 @@ ix86_expand_builtin (tree exp, rtx target, rtx > subtarget, > enum insn_code icode, icode2; > tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0); > tree arg0, arg1, arg2, arg3, arg4; > - rtx op0, op1, op2, op3, op4, pat, pat2, insn; > + rtx op0, op1, op2, op3, op4, opels, pat, pat2, insn; > machine_mode mode0, mode1, mode2, mode3, mode4; > unsigned int fcode = DECL_MD_FUNCTION_CODE (fndecl); > HOST_WIDE_INT bisa, bisa2; > @@ -15560,12 +15570,15 @@ rdseed_step: > op3 = copy_to_reg (op3); > op3 = lowpart_subreg (mode3, op3, GET_MODE (op3)); > } > + > if (!insn_data[icode].operand[5].predicate (op4, mode4)) > { > - error ("the last argument must be scale 1, 2, 4, 8"); > - return const0_rtx; > + error ("the last argument must be scale 1, 2, 4, 8"); > + return const0_rtx; > } > > + opels = CONST0_RTX (GET_MODE (subtarget)); > + > /* Optimize. If mask is known to have all high bits set, > replace op0 with pc_rtx to signal that the instruction > overwrites the whole destination and doesn't use its > @@ -15634,7 +15647,8 @@ rdseed_step: > } > } > > - pat = GEN_FCN (icode) (subtarget, op0, op1, op2, op3, op4); > + pat = GEN_FCN (icode) (subtarget, op0, op1, op2, op3, op4, opels); > + > if (! pat) > return const0_rtx; > emit_insn (pat); > diff --git a/gcc/config/i386/predicates.md b/gcc/config/i386/predicates.md > index 053312bbe27..7c7d8f61f11 100644 > --- a/gcc/config/i386/predicates.md > +++ b/gcc/config/i386/predicates.md > @@ -2346,3 +2346,7 @@ (define_predicate "apx_evex_add_memory_operand" > > return true; > }) > + > +(define_predicate "maskload_else_operand" > + (and (match_code "const_int,const_vector") > + (match_test "op == CONST0_RTX (GET_MODE (op))"))) > diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md > index 36f8567b66f..41c1badbc00 100644 > --- a/gcc/config/i386/sse.md > +++ b/gcc/config/i386/sse.md > @@ -28632,7 +28632,7 @@ (define_insn > "<avx_avx2>_maskstore<ssemodesuffix><avxsizesuffix>" > (set_attr "btver2_decode" "vector") > (set_attr "mode" "<sseinsnmode>")]) > > -(define_expand "maskload<mode><sseintvecmodelower>" > +(define_expand "maskload<mode><sseintvecmodelower>_1" > [(set (match_operand:V48_128_256 0 "register_operand") > (unspec:V48_128_256 > [(match_operand:<sseintvecmode> 2 "register_operand") > @@ -28640,13 +28640,28 @@ (define_expand "maskload<mode><sseintvecmodelower>" > UNSPEC_MASKMOV))] > "TARGET_AVX") > > +(define_expand "maskload<mode><sseintvecmodelower>" > + [(set (match_operand:V48_128_256 0 "register_operand") > + (unspec:V48_128_256 > + [(match_operand:<sseintvecmode> 2 "register_operand") > + (match_operand:V48_128_256 1 "memory_operand") > + (match_operand:V48_128_256 3 "const0_operand")] > + UNSPEC_MASKMOV))] > + "TARGET_AVX" > +{ > + emit_insn (gen_maskload<mode><sseintvecmodelower>_1 (operands[0], > + operands[1], > + operands[2])); > + DONE; > +}) > + > (define_expand "maskload<mode><avx512fmaskmodelower>" > [(set (match_operand:V48_AVX512VL 0 "register_operand") > (vec_merge:V48_AVX512VL > (unspec:V48_AVX512VL > [(match_operand:V48_AVX512VL 1 "memory_operand")] > UNSPEC_MASKLOAD) > - (match_dup 0) > + (match_operand:V48_AVX512VL 3 "const0_operand") > (match_operand:<avx512fmaskmode> 2 "register_operand")))] > "TARGET_AVX512F") > > @@ -28656,8 +28671,9 @@ (define_expand "maskload<mode><avx512fmaskmodelower>" > (unspec:VI12HFBF_AVX512VL > [(match_operand:VI12HFBF_AVX512VL 1 "memory_operand")] > UNSPEC_MASKLOAD) > - (match_dup 0) > - (match_operand:<avx512fmaskmode> 2 "register_operand")))] > + (match_operand:VI12HFBF_AVX512VL 3 "const0_operand") > + (match_operand:<avx512fmaskmode> 2 "register_operand"))) > + ] > "TARGET_AVX512BW") > > (define_expand "maskstore<mode><sseintvecmodelower>" > @@ -29223,20 +29239,22 @@ (define_expand "avx2_gathersi<mode>" > (unspec:VEC_GATHER_MODE > [(match_operand:VEC_GATHER_MODE 1 "register_operand") > (mem:<ssescalarmode> > - (match_par_dup 6 > + (match_par_dup 7 > [(match_operand 2 "vsib_address_operand") > (match_operand:<VEC_GATHER_IDXSI> > 3 "register_operand") > - (match_operand:SI 5 "const1248_operand ")])) > + (match_operand:SI 5 "const1248_operand ") > + (match_operand:VEC_GATHER_MODE 6 > "maskload_else_operand")])) > (mem:BLK (scratch)) > (match_operand:VEC_GATHER_MODE 4 "register_operand")] > UNSPEC_GATHER)) > - (clobber (match_scratch:VEC_GATHER_MODE 7))])] > + (clobber (match_scratch:VEC_GATHER_MODE 8))])] > "TARGET_AVX2" > { > - operands[6] > - = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3], > - operands[5]), UNSPEC_VSIBADDR); > + operands[7] > + = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, operands[2], operands[3], > + operands[5], operands[6]), > + UNSPEC_VSIBADDR); > }) > > (define_insn "*avx2_gathersi<VEC_GATHER_MODE:mode>" > @@ -29247,7 +29265,8 @@ (define_insn "*avx2_gathersi<VEC_GATHER_MODE:mode>" > [(unspec:P > [(match_operand:P 3 "vsib_address_operand" "jb") > (match_operand:<VEC_GATHER_IDXSI> 4 "register_operand" "x") > - (match_operand:SI 6 "const1248_operand")] > + (match_operand:SI 6 "const1248_operand") > + (match_operand:VEC_GATHER_MODE 8 "maskload_else_operand")] > UNSPEC_VSIBADDR)]) > (mem:BLK (scratch)) > (match_operand:VEC_GATHER_MODE 5 "register_operand" "1")] > @@ -29268,7 +29287,8 @@ (define_insn "*avx2_gathersi<VEC_GATHER_MODE:mode>_2" > [(unspec:P > [(match_operand:P 2 "vsib_address_operand" "jb") > (match_operand:<VEC_GATHER_IDXSI> 3 "register_operand" "x") > - (match_operand:SI 5 "const1248_operand")] > + (match_operand:SI 5 "const1248_operand") > + (match_operand:VEC_GATHER_MODE 7 "maskload_else_operand")] > UNSPEC_VSIBADDR)]) > (mem:BLK (scratch)) > (match_operand:VEC_GATHER_MODE 4 "register_operand" "1")] > @@ -29286,20 +29306,22 @@ (define_expand "avx2_gatherdi<mode>" > (unspec:VEC_GATHER_MODE > [(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand") > (mem:<ssescalarmode> > - (match_par_dup 6 > + (match_par_dup 7 > [(match_operand 2 "vsib_address_operand") > (match_operand:<VEC_GATHER_IDXDI> > 3 "register_operand") > - (match_operand:SI 5 "const1248_operand ")])) > + (match_operand:SI 5 "const1248_operand ") > + (match_operand:VEC_GATHER_MODE 6 > "maskload_else_operand")])) > (mem:BLK (scratch)) > (match_operand:<VEC_GATHER_SRCDI> 4 "register_operand")] > UNSPEC_GATHER)) > - (clobber (match_scratch:VEC_GATHER_MODE 7))])] > + (clobber (match_scratch:VEC_GATHER_MODE 8))])] > "TARGET_AVX2" > { > - operands[6] > - = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3], > - operands[5]), UNSPEC_VSIBADDR); > + operands[7] > + = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, operands[2], operands[3], > + operands[5], operands[6]), > + UNSPEC_VSIBADDR); > }) > x86 doesn't define mask_gather_loadmn, so I think you can drop this and all related, only keep the patch I give you in [1] Sorry I didn't make that clear last time.
[1] https://gcc.gnu.org/pipermail/gcc-patches/2024-October/666814.html > (define_insn "*avx2_gatherdi<VEC_GATHER_MODE:mode>" > @@ -29310,7 +29332,8 @@ (define_insn "*avx2_gatherdi<VEC_GATHER_MODE:mode>" > [(unspec:P > [(match_operand:P 3 "vsib_address_operand" "jb") > (match_operand:<VEC_GATHER_IDXDI> 4 "register_operand" "x") > - (match_operand:SI 6 "const1248_operand")] > + (match_operand:SI 6 "const1248_operand") > + (match_operand:VEC_GATHER_MODE 8 "maskload_else_operand")] > UNSPEC_VSIBADDR)]) > (mem:BLK (scratch)) > (match_operand:<VEC_GATHER_SRCDI> 5 "register_operand" "1")] > @@ -29331,7 +29354,8 @@ (define_insn "*avx2_gatherdi<VEC_GATHER_MODE:mode>_2" > [(unspec:P > [(match_operand:P 2 "vsib_address_operand" "jb") > (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "x") > - (match_operand:SI 5 "const1248_operand")] > + (match_operand:SI 5 "const1248_operand") > + (match_operand:VEC_GATHER_MODE 7 "maskload_else_operand")] > UNSPEC_VSIBADDR)]) > (mem:BLK (scratch)) > (match_operand:<VEC_GATHER_SRCDI> 4 "register_operand" "1")] > @@ -29357,7 +29381,8 @@ (define_insn "*avx2_gatherdi<VI4F_256:mode>_3" > [(unspec:P > [(match_operand:P 3 "vsib_address_operand" "jb") > (match_operand:<VEC_GATHER_IDXDI> 4 "register_operand" "x") > - (match_operand:SI 6 "const1248_operand")] > + (match_operand:SI 6 "const1248_operand") > + (match_operand:VI4F_256 8 "maskload_else_operand")] > UNSPEC_VSIBADDR)]) > (mem:BLK (scratch)) > (match_operand:<VEC_GATHER_SRCDI> 5 "register_operand" "1")] > @@ -29381,7 +29406,8 @@ (define_insn "*avx2_gatherdi<VI4F_256:mode>_4" > [(unspec:P > [(match_operand:P 2 "vsib_address_operand" "jb") > (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "x") > - (match_operand:SI 5 "const1248_operand")] > + (match_operand:SI 5 "const1248_operand") > + (match_operand:VI4F_256 7 "maskload_else_operand")] > UNSPEC_VSIBADDR)]) > (mem:BLK (scratch)) > (match_operand:<VEC_GATHER_SRCDI> 4 "register_operand" "1")] > @@ -29402,17 +29428,19 @@ (define_expand "<avx512>_gathersi<mode>" > [(match_operand:VI48F 1 "register_operand") > (match_operand:<avx512fmaskmode> 4 "register_operand") > (mem:<ssescalarmode> > - (match_par_dup 6 > + (match_par_dup 7 > [(match_operand 2 "vsib_address_operand") > (match_operand:<VEC_GATHER_IDXSI> 3 > "register_operand") > - (match_operand:SI 5 "const1248_operand")]))] > + (match_operand:SI 5 "const1248_operand") > + (match_operand:VI48F 6 "maskload_else_operand")]))] > UNSPEC_GATHER)) > - (clobber (match_scratch:<avx512fmaskmode> 7))])] > + (clobber (match_scratch:<avx512fmaskmode> 8))])] > "TARGET_AVX512F" > { > - operands[6] > - = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3], > - operands[5]), UNSPEC_VSIBADDR); > + operands[7] > + = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, operands[2], operands[3], > + operands[5], operands[6]), > + UNSPEC_VSIBADDR); > }) > > (define_insn "*avx512f_gathersi<VI48F:mode>" > @@ -29424,7 +29452,8 @@ (define_insn "*avx512f_gathersi<VI48F:mode>" > [(unspec:P > [(match_operand:P 4 "vsib_address_operand" "Tv") > (match_operand:<VEC_GATHER_IDXSI> 3 "register_operand" "v") > - (match_operand:SI 5 "const1248_operand")] > + (match_operand:SI 5 "const1248_operand") > + (match_operand:VI48F 8 "maskload_else_operand")] > UNSPEC_VSIBADDR)])] > UNSPEC_GATHER)) > (clobber (match_scratch:<avx512fmaskmode> 2 "=&Yk"))] > @@ -29445,7 +29474,8 @@ (define_insn "*avx512f_gathersi<VI48F:mode>_2" > [(unspec:P > [(match_operand:P 3 "vsib_address_operand" "Tv") > (match_operand:<VEC_GATHER_IDXSI> 2 "register_operand" "v") > - (match_operand:SI 4 "const1248_operand")] > + (match_operand:SI 4 "const1248_operand") > + (match_operand:VI48F 7 "maskload_else_operand")] > UNSPEC_VSIBADDR)])] > UNSPEC_GATHER)) > (clobber (match_scratch:<avx512fmaskmode> 1 "=&Yk"))] > @@ -29464,17 +29494,19 @@ (define_expand "<avx512>_gatherdi<mode>" > [(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand") > (match_operand:QI 4 "register_operand") > (mem:<ssescalarmode> > - (match_par_dup 6 > + (match_par_dup 7 > [(match_operand 2 "vsib_address_operand") > (match_operand:<VEC_GATHER_IDXDI> 3 > "register_operand") > - (match_operand:SI 5 "const1248_operand")]))] > + (match_operand:SI 5 "const1248_operand") > + (match_operand:VI48F 6 "maskload_else_operand")]))] > UNSPEC_GATHER)) > - (clobber (match_scratch:QI 7))])] > + (clobber (match_scratch:QI 8))])] > "TARGET_AVX512F" > { > - operands[6] > - = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3], > - operands[5]), UNSPEC_VSIBADDR); > + operands[7] > + = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, operands[2], operands[3], > + operands[5], operands[6]), > + UNSPEC_VSIBADDR); > }) > > (define_insn "*avx512f_gatherdi<VI48F:mode>" > @@ -29486,7 +29518,8 @@ (define_insn "*avx512f_gatherdi<VI48F:mode>" > [(unspec:P > [(match_operand:P 4 "vsib_address_operand" "Tv") > (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "v") > - (match_operand:SI 5 "const1248_operand")] > + (match_operand:SI 5 "const1248_operand") > + (match_operand:VI48F 8 "maskload_else_operand")] > UNSPEC_VSIBADDR)])] > UNSPEC_GATHER)) > (clobber (match_scratch:QI 2 "=&Yk"))] > @@ -29507,7 +29540,8 @@ (define_insn "*avx512f_gatherdi<VI48F:mode>_2" > [(unspec:P > [(match_operand:P 3 "vsib_address_operand" "Tv") > (match_operand:<VEC_GATHER_IDXDI> 2 "register_operand" "v") > - (match_operand:SI 4 "const1248_operand")] > + (match_operand:SI 4 "const1248_operand") > + (match_operand:VI48F 7 "maskload_else_operand")] > UNSPEC_VSIBADDR)])] > UNSPEC_GATHER)) > (clobber (match_scratch:QI 1 "=&Yk"))] > @@ -29544,7 +29578,7 @@ (define_expand "<avx512>_scattersi<mode>" > operands[5] > = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, operands[0], operands[2], > operands[4], operands[1]), > - UNSPEC_VSIBADDR); > + UNSPEC_VSIBADDR); > }) > > (define_insn "*avx512f_scattersi<VI48F:mode>" > -- > 2.47.0 > -- BR, Hongtao