From: Robin Dapp <rd...@ventanamicro.com>
This patch adds a zero else operand to masked loads, in particular the
masked gather load builtins that are used for gather vectorization.
gcc/ChangeLog:
* config/i386/i386-expand.cc (ix86_expand_special_args_builtin):
Add else-operand handling.
(ix86_expand_builtin): Ditto.
* config/i386/predicates.md (vcvtne2ps2bf_parallel): New
predicate.
(maskload_else_operand): Ditto.
* config/i386/sse.md: Use predicate.
---
gcc/config/i386/i386-expand.cc | 26 ++++++--
gcc/config/i386/predicates.md | 4 ++
gcc/config/i386/sse.md | 112 +++++++++++++++++++++------------
3 files changed, 97 insertions(+), 45 deletions(-)
diff --git a/gcc/config/i386/i386-expand.cc b/gcc/config/i386/i386-expand.cc
index 0de0e842731..6c61f9f87c2 100644
--- a/gcc/config/i386/i386-expand.cc
+++ b/gcc/config/i386/i386-expand.cc
@@ -12995,10 +12995,11 @@ ix86_expand_special_args_builtin (const struct
builtin_description *d,
{
tree arg;
rtx pat, op;
- unsigned int i, nargs, arg_adjust, memory;
+ unsigned int i, nargs, arg_adjust, memory = -1;
unsigned int constant = 100;
bool aligned_mem = false;
- rtx xops[4];
+ rtx xops[4] = {};
+ bool add_els = false;
enum insn_code icode = d->icode;
const struct insn_data_d *insn_p = &insn_data[icode];
machine_mode tmode = insn_p->operand[0].mode;
@@ -13125,6 +13126,9 @@ ix86_expand_special_args_builtin (const struct
builtin_description *d,
case V4DI_FTYPE_PCV4DI_V4DI:
case V4SI_FTYPE_PCV4SI_V4SI:
case V2DI_FTYPE_PCV2DI_V2DI:
+ /* Two actual args but an additional else operand. */
+ add_els = true;
+ /* Fallthru. */
case VOID_FTYPE_INT_INT64:
nargs = 2;
klass = load;
@@ -13397,6 +13401,12 @@ ix86_expand_special_args_builtin (const struct
builtin_description *d,
xops[i]= op;
}
+ if (add_els)
+ {
+ xops[i] = CONST0_RTX (GET_MODE (xops[0]));
+ nargs++;
+ }
+
switch (nargs)
{
case 0:
@@ -13653,7 +13663,7 @@ ix86_expand_builtin (tree exp, rtx target, rtx
subtarget,
enum insn_code icode, icode2;
tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
tree arg0, arg1, arg2, arg3, arg4;
- rtx op0, op1, op2, op3, op4, pat, pat2, insn;
+ rtx op0, op1, op2, op3, op4, opels, pat, pat2, insn;
machine_mode mode0, mode1, mode2, mode3, mode4;
unsigned int fcode = DECL_MD_FUNCTION_CODE (fndecl);
HOST_WIDE_INT bisa, bisa2;
@@ -15560,12 +15570,15 @@ rdseed_step:
op3 = copy_to_reg (op3);
op3 = lowpart_subreg (mode3, op3, GET_MODE (op3));
}
+
if (!insn_data[icode].operand[5].predicate (op4, mode4))
{
- error ("the last argument must be scale 1, 2, 4, 8");
- return const0_rtx;
+ error ("the last argument must be scale 1, 2, 4, 8");
+ return const0_rtx;
}
+ opels = CONST0_RTX (GET_MODE (subtarget));
+
/* Optimize. If mask is known to have all high bits set,
replace op0 with pc_rtx to signal that the instruction
overwrites the whole destination and doesn't use its
@@ -15634,7 +15647,8 @@ rdseed_step:
}
}
- pat = GEN_FCN (icode) (subtarget, op0, op1, op2, op3, op4);
+ pat = GEN_FCN (icode) (subtarget, op0, op1, op2, op3, op4, opels);
+
if (! pat)
return const0_rtx;
emit_insn (pat);
diff --git a/gcc/config/i386/predicates.md b/gcc/config/i386/predicates.md
index 053312bbe27..7c7d8f61f11 100644
--- a/gcc/config/i386/predicates.md
+++ b/gcc/config/i386/predicates.md
@@ -2346,3 +2346,7 @@ (define_predicate "apx_evex_add_memory_operand"
return true;
})
+
+(define_predicate "maskload_else_operand"
+ (and (match_code "const_int,const_vector")
+ (match_test "op == CONST0_RTX (GET_MODE (op))")))
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index 36f8567b66f..41c1badbc00 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -28632,7 +28632,7 @@ (define_insn
"<avx_avx2>_maskstore<ssemodesuffix><avxsizesuffix>"
(set_attr "btver2_decode" "vector")
(set_attr "mode" "<sseinsnmode>")])
-(define_expand "maskload<mode><sseintvecmodelower>"
+(define_expand "maskload<mode><sseintvecmodelower>_1"
[(set (match_operand:V48_128_256 0 "register_operand")
(unspec:V48_128_256
[(match_operand:<sseintvecmode> 2 "register_operand")
@@ -28640,13 +28640,28 @@ (define_expand "maskload<mode><sseintvecmodelower>"
UNSPEC_MASKMOV))]
"TARGET_AVX")
+(define_expand "maskload<mode><sseintvecmodelower>"
+ [(set (match_operand:V48_128_256 0 "register_operand")
+ (unspec:V48_128_256
+ [(match_operand:<sseintvecmode> 2 "register_operand")
+ (match_operand:V48_128_256 1 "memory_operand")
+ (match_operand:V48_128_256 3 "const0_operand")]
+ UNSPEC_MASKMOV))]
+ "TARGET_AVX"
+{
+ emit_insn (gen_maskload<mode><sseintvecmodelower>_1 (operands[0],
+ operands[1],
+ operands[2]));
+ DONE;
+})
+
(define_expand "maskload<mode><avx512fmaskmodelower>"
[(set (match_operand:V48_AVX512VL 0 "register_operand")
(vec_merge:V48_AVX512VL
(unspec:V48_AVX512VL
[(match_operand:V48_AVX512VL 1 "memory_operand")]
UNSPEC_MASKLOAD)
- (match_dup 0)
+ (match_operand:V48_AVX512VL 3 "const0_operand")
(match_operand:<avx512fmaskmode> 2 "register_operand")))]
"TARGET_AVX512F")
@@ -28656,8 +28671,9 @@ (define_expand "maskload<mode><avx512fmaskmodelower>"
(unspec:VI12HFBF_AVX512VL
[(match_operand:VI12HFBF_AVX512VL 1 "memory_operand")]
UNSPEC_MASKLOAD)
- (match_dup 0)
- (match_operand:<avx512fmaskmode> 2 "register_operand")))]
+ (match_operand:VI12HFBF_AVX512VL 3 "const0_operand")
+ (match_operand:<avx512fmaskmode> 2 "register_operand")))
+ ]
"TARGET_AVX512BW")
(define_expand "maskstore<mode><sseintvecmodelower>"
@@ -29223,20 +29239,22 @@ (define_expand "avx2_gathersi<mode>"
(unspec:VEC_GATHER_MODE
[(match_operand:VEC_GATHER_MODE 1 "register_operand")
(mem:<ssescalarmode>
- (match_par_dup 6
+ (match_par_dup 7
[(match_operand 2 "vsib_address_operand")
(match_operand:<VEC_GATHER_IDXSI>
3 "register_operand")
- (match_operand:SI 5 "const1248_operand ")]))
+ (match_operand:SI 5 "const1248_operand ")
+ (match_operand:VEC_GATHER_MODE 6
"maskload_else_operand")]))
(mem:BLK (scratch))
(match_operand:VEC_GATHER_MODE 4 "register_operand")]
UNSPEC_GATHER))
- (clobber (match_scratch:VEC_GATHER_MODE 7))])]
+ (clobber (match_scratch:VEC_GATHER_MODE 8))])]
"TARGET_AVX2"
{
- operands[6]
- = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
- operands[5]), UNSPEC_VSIBADDR);
+ operands[7]
+ = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, operands[2], operands[3],
+ operands[5], operands[6]),
+ UNSPEC_VSIBADDR);
})
(define_insn "*avx2_gathersi<VEC_GATHER_MODE:mode>"
@@ -29247,7 +29265,8 @@ (define_insn "*avx2_gathersi<VEC_GATHER_MODE:mode>"
[(unspec:P
[(match_operand:P 3 "vsib_address_operand" "jb")
(match_operand:<VEC_GATHER_IDXSI> 4 "register_operand" "x")
- (match_operand:SI 6 "const1248_operand")]
+ (match_operand:SI 6 "const1248_operand")
+ (match_operand:VEC_GATHER_MODE 8 "maskload_else_operand")]
UNSPEC_VSIBADDR)])
(mem:BLK (scratch))
(match_operand:VEC_GATHER_MODE 5 "register_operand" "1")]
@@ -29268,7 +29287,8 @@ (define_insn "*avx2_gathersi<VEC_GATHER_MODE:mode>_2"
[(unspec:P
[(match_operand:P 2 "vsib_address_operand" "jb")
(match_operand:<VEC_GATHER_IDXSI> 3 "register_operand" "x")
- (match_operand:SI 5 "const1248_operand")]
+ (match_operand:SI 5 "const1248_operand")
+ (match_operand:VEC_GATHER_MODE 7 "maskload_else_operand")]
UNSPEC_VSIBADDR)])
(mem:BLK (scratch))
(match_operand:VEC_GATHER_MODE 4 "register_operand" "1")]
@@ -29286,20 +29306,22 @@ (define_expand "avx2_gatherdi<mode>"
(unspec:VEC_GATHER_MODE
[(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand")
(mem:<ssescalarmode>
- (match_par_dup 6
+ (match_par_dup 7
[(match_operand 2 "vsib_address_operand")
(match_operand:<VEC_GATHER_IDXDI>
3 "register_operand")
- (match_operand:SI 5 "const1248_operand ")]))
+ (match_operand:SI 5 "const1248_operand ")
+ (match_operand:VEC_GATHER_MODE 6
"maskload_else_operand")]))
(mem:BLK (scratch))
(match_operand:<VEC_GATHER_SRCDI> 4 "register_operand")]
UNSPEC_GATHER))
- (clobber (match_scratch:VEC_GATHER_MODE 7))])]
+ (clobber (match_scratch:VEC_GATHER_MODE 8))])]
"TARGET_AVX2"
{
- operands[6]
- = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
- operands[5]), UNSPEC_VSIBADDR);
+ operands[7]
+ = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, operands[2], operands[3],
+ operands[5], operands[6]),
+ UNSPEC_VSIBADDR);
})
(define_insn "*avx2_gatherdi<VEC_GATHER_MODE:mode>"
@@ -29310,7 +29332,8 @@ (define_insn "*avx2_gatherdi<VEC_GATHER_MODE:mode>"
[(unspec:P
[(match_operand:P 3 "vsib_address_operand" "jb")
(match_operand:<VEC_GATHER_IDXDI> 4 "register_operand" "x")
- (match_operand:SI 6 "const1248_operand")]
+ (match_operand:SI 6 "const1248_operand")
+ (match_operand:VEC_GATHER_MODE 8 "maskload_else_operand")]
UNSPEC_VSIBADDR)])
(mem:BLK (scratch))
(match_operand:<VEC_GATHER_SRCDI> 5 "register_operand" "1")]
@@ -29331,7 +29354,8 @@ (define_insn "*avx2_gatherdi<VEC_GATHER_MODE:mode>_2"
[(unspec:P
[(match_operand:P 2 "vsib_address_operand" "jb")
(match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "x")
- (match_operand:SI 5 "const1248_operand")]
+ (match_operand:SI 5 "const1248_operand")
+ (match_operand:VEC_GATHER_MODE 7 "maskload_else_operand")]
UNSPEC_VSIBADDR)])
(mem:BLK (scratch))
(match_operand:<VEC_GATHER_SRCDI> 4 "register_operand" "1")]
@@ -29357,7 +29381,8 @@ (define_insn "*avx2_gatherdi<VI4F_256:mode>_3"
[(unspec:P
[(match_operand:P 3 "vsib_address_operand" "jb")
(match_operand:<VEC_GATHER_IDXDI> 4 "register_operand" "x")
- (match_operand:SI 6 "const1248_operand")]
+ (match_operand:SI 6 "const1248_operand")
+ (match_operand:VI4F_256 8 "maskload_else_operand")]
UNSPEC_VSIBADDR)])
(mem:BLK (scratch))
(match_operand:<VEC_GATHER_SRCDI> 5 "register_operand" "1")]
@@ -29381,7 +29406,8 @@ (define_insn "*avx2_gatherdi<VI4F_256:mode>_4"
[(unspec:P
[(match_operand:P 2 "vsib_address_operand" "jb")
(match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "x")
- (match_operand:SI 5 "const1248_operand")]
+ (match_operand:SI 5 "const1248_operand")
+ (match_operand:VI4F_256 7 "maskload_else_operand")]
UNSPEC_VSIBADDR)])
(mem:BLK (scratch))
(match_operand:<VEC_GATHER_SRCDI> 4 "register_operand" "1")]
@@ -29402,17 +29428,19 @@ (define_expand "<avx512>_gathersi<mode>"
[(match_operand:VI48F 1 "register_operand")
(match_operand:<avx512fmaskmode> 4 "register_operand")
(mem:<ssescalarmode>
- (match_par_dup 6
+ (match_par_dup 7
[(match_operand 2 "vsib_address_operand")
(match_operand:<VEC_GATHER_IDXSI> 3
"register_operand")
- (match_operand:SI 5 "const1248_operand")]))]
+ (match_operand:SI 5 "const1248_operand")
+ (match_operand:VI48F 6 "maskload_else_operand")]))]
UNSPEC_GATHER))
- (clobber (match_scratch:<avx512fmaskmode> 7))])]
+ (clobber (match_scratch:<avx512fmaskmode> 8))])]
"TARGET_AVX512F"
{
- operands[6]
- = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
- operands[5]), UNSPEC_VSIBADDR);
+ operands[7]
+ = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, operands[2], operands[3],
+ operands[5], operands[6]),
+ UNSPEC_VSIBADDR);
})
(define_insn "*avx512f_gathersi<VI48F:mode>"
@@ -29424,7 +29452,8 @@ (define_insn "*avx512f_gathersi<VI48F:mode>"
[(unspec:P
[(match_operand:P 4 "vsib_address_operand" "Tv")
(match_operand:<VEC_GATHER_IDXSI> 3 "register_operand" "v")
- (match_operand:SI 5 "const1248_operand")]
+ (match_operand:SI 5 "const1248_operand")
+ (match_operand:VI48F 8 "maskload_else_operand")]
UNSPEC_VSIBADDR)])]
UNSPEC_GATHER))
(clobber (match_scratch:<avx512fmaskmode> 2 "=&Yk"))]
@@ -29445,7 +29474,8 @@ (define_insn "*avx512f_gathersi<VI48F:mode>_2"
[(unspec:P
[(match_operand:P 3 "vsib_address_operand" "Tv")
(match_operand:<VEC_GATHER_IDXSI> 2 "register_operand" "v")
- (match_operand:SI 4 "const1248_operand")]
+ (match_operand:SI 4 "const1248_operand")
+ (match_operand:VI48F 7 "maskload_else_operand")]
UNSPEC_VSIBADDR)])]
UNSPEC_GATHER))
(clobber (match_scratch:<avx512fmaskmode> 1 "=&Yk"))]
@@ -29464,17 +29494,19 @@ (define_expand "<avx512>_gatherdi<mode>"
[(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand")
(match_operand:QI 4 "register_operand")
(mem:<ssescalarmode>
- (match_par_dup 6
+ (match_par_dup 7
[(match_operand 2 "vsib_address_operand")
(match_operand:<VEC_GATHER_IDXDI> 3
"register_operand")
- (match_operand:SI 5 "const1248_operand")]))]
+ (match_operand:SI 5 "const1248_operand")
+ (match_operand:VI48F 6 "maskload_else_operand")]))]
UNSPEC_GATHER))
- (clobber (match_scratch:QI 7))])]
+ (clobber (match_scratch:QI 8))])]
"TARGET_AVX512F"
{
- operands[6]
- = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
- operands[5]), UNSPEC_VSIBADDR);
+ operands[7]
+ = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, operands[2], operands[3],
+ operands[5], operands[6]),
+ UNSPEC_VSIBADDR);
})
(define_insn "*avx512f_gatherdi<VI48F:mode>"
@@ -29486,7 +29518,8 @@ (define_insn "*avx512f_gatherdi<VI48F:mode>"
[(unspec:P
[(match_operand:P 4 "vsib_address_operand" "Tv")
(match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "v")
- (match_operand:SI 5 "const1248_operand")]
+ (match_operand:SI 5 "const1248_operand")
+ (match_operand:VI48F 8 "maskload_else_operand")]
UNSPEC_VSIBADDR)])]
UNSPEC_GATHER))
(clobber (match_scratch:QI 2 "=&Yk"))]
@@ -29507,7 +29540,8 @@ (define_insn "*avx512f_gatherdi<VI48F:mode>_2"
[(unspec:P
[(match_operand:P 3 "vsib_address_operand" "Tv")
(match_operand:<VEC_GATHER_IDXDI> 2 "register_operand" "v")
- (match_operand:SI 4 "const1248_operand")]
+ (match_operand:SI 4 "const1248_operand")
+ (match_operand:VI48F 7 "maskload_else_operand")]
UNSPEC_VSIBADDR)])]
UNSPEC_GATHER))
(clobber (match_scratch:QI 1 "=&Yk"))]
@@ -29544,7 +29578,7 @@ (define_expand "<avx512>_scattersi<mode>"
operands[5]
= gen_rtx_UNSPEC (Pmode, gen_rtvec (4, operands[0], operands[2],
operands[4], operands[1]),
- UNSPEC_VSIBADDR);
+ UNSPEC_VSIBADDR);
})
(define_insn "*avx512f_scattersi<VI48F:mode>"
--
2.47.0
