Reading YMM registers with all zero bits needs VZEROUPPER on Sandy Bride, Ivy Bridge, Haswell, Broadwell and Alder Lake to avoid SSE <-> AVX transition penalty. Add TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER to generate vzeroupper instruction after loading all-zero YMM/YMM registers and enable it by default.
gcc/ PR target/101456 * config/i386/i386.cc (ix86_avx_u128_mode_needed): Skip the vzeroupper optimization if target needs vzeroupper after reading all-zero YMM/YMM registers. * config/i386/i386.h (TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER): New. * config/i386/x86-tune.def (X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER): New. gcc/testsuite/ PR target/101456 * gcc.target/i386/pr101456-1.c (dg-options): Add -mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper. * gcc.target/i386/pr101456-2.c: Likewise. * gcc.target/i386/pr101456-3.c: New test. * gcc.target/i386/pr101456-4.c: Likewise. --- gcc/config/i386/i386.cc | 51 ++++++++++++---------- gcc/config/i386/i386.h | 2 + gcc/config/i386/x86-tune.def | 5 +++ gcc/testsuite/gcc.target/i386/pr101456-1.c | 2 +- gcc/testsuite/gcc.target/i386/pr101456-2.c | 2 +- gcc/testsuite/gcc.target/i386/pr101456-3.c | 33 ++++++++++++++ gcc/testsuite/gcc.target/i386/pr101456-4.c | 33 ++++++++++++++ 7 files changed, 103 insertions(+), 25 deletions(-) create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-3.c create mode 100644 gcc/testsuite/gcc.target/i386/pr101456-4.c diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc index cf246e74e57..1f8b4caf24c 100644 --- a/gcc/config/i386/i386.cc +++ b/gcc/config/i386/i386.cc @@ -14502,33 +14502,38 @@ ix86_avx_u128_mode_needed (rtx_insn *insn) subrtx_iterator::array_type array; - rtx set = single_set (insn); - if (set) + if (!TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER) { - rtx dest = SET_DEST (set); - rtx src = SET_SRC (set); - if (ix86_check_avx_upper_register (dest)) + /* Perform this vzeroupper optimization if target doesn't need + vzeroupper after reading all-zero YMM/YMM registers. */ + rtx set = single_set (insn); + if (set) { - /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the - source isn't zero. */ - if (standard_sse_constant_p (src, GET_MODE (dest)) != 1) - return AVX_U128_DIRTY; + rtx dest = SET_DEST (set); + rtx src = SET_SRC (set); + if (ix86_check_avx_upper_register (dest)) + { + /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the + source isn't zero. */ + if (standard_sse_constant_p (src, GET_MODE (dest)) != 1) + return AVX_U128_DIRTY; + else + return AVX_U128_ANY; + } else - return AVX_U128_ANY; - } - else - { - FOR_EACH_SUBRTX (iter, array, src, NONCONST) - if (ix86_check_avx_upper_register (*iter)) - { - int status = ix86_avx_u128_mode_source (insn, *iter); - if (status == AVX_U128_DIRTY) - return status; - } - } + { + FOR_EACH_SUBRTX (iter, array, src, NONCONST) + if (ix86_check_avx_upper_register (*iter)) + { + int status = ix86_avx_u128_mode_source (insn, *iter); + if (status == AVX_U128_DIRTY) + return status; + } + } - /* This isn't YMM/ZMM load/store. */ - return AVX_U128_ANY; + /* This isn't YMM/ZMM load/store. */ + return AVX_U128_ANY; + } } /* Require DIRTY mode if a 256bit or 512bit AVX register is referenced. diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h index f41e0908250..98c2e200027 100644 --- a/gcc/config/i386/i386.h +++ b/gcc/config/i386/i386.h @@ -425,6 +425,8 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST]; #define TARGET_AVOID_MFENCE ix86_tune_features[X86_TUNE_AVOID_MFENCE] #define TARGET_EMIT_VZEROUPPER \ ix86_tune_features[X86_TUNE_EMIT_VZEROUPPER] +#define TARGET_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER \ + ix86_tune_features[X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER] #define TARGET_EXPAND_ABS \ ix86_tune_features[X86_TUNE_EXPAND_ABS] #define TARGET_V2DF_REDUCTION_PREFER_HADDPD \ diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def index 82ca0ae63ac..0a068c09202 100644 --- a/gcc/config/i386/x86-tune.def +++ b/gcc/config/i386/x86-tune.def @@ -649,3 +649,8 @@ DEF_TUNE (X86_TUNE_PROMOTE_QI_REGS, "promote_qi_regs", m_NONE) /* X86_TUNE_EMIT_VZEROUPPER: This enables vzeroupper instruction insertion before a transfer of control flow out of the function. */ DEF_TUNE (X86_TUNE_EMIT_VZEROUPPER, "emit_vzeroupper", ~m_KNL) + +/* X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER: This knob generates + vzeroupper instruction after reading all-zero YMM/YMM registers. */ +DEF_TUNE (X86_TUNE_READ_ZERO_YMM_ZMM_NEED_VZEROUPPER, + "read_zero_ymm_zmm_need_vzeroupper", HOST_WIDE_INT_M1U) diff --git a/gcc/testsuite/gcc.target/i386/pr101456-1.c b/gcc/testsuite/gcc.target/i386/pr101456-1.c index 803fc6e0207..7eb74d21439 100644 --- a/gcc/testsuite/gcc.target/i386/pr101456-1.c +++ b/gcc/testsuite/gcc.target/i386/pr101456-1.c @@ -1,5 +1,5 @@ /* { dg-do compile } */ -/* { dg-options "-O2 -march=skylake" } */ +/* { dg-options "-O2 -march=skylake -mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper" } */ #include <x86intrin.h> diff --git a/gcc/testsuite/gcc.target/i386/pr101456-2.c b/gcc/testsuite/gcc.target/i386/pr101456-2.c index 554a0f1702c..9fdc9bd6eb1 100644 --- a/gcc/testsuite/gcc.target/i386/pr101456-2.c +++ b/gcc/testsuite/gcc.target/i386/pr101456-2.c @@ -1,5 +1,5 @@ /* { dg-do compile } */ -/* { dg-options "-O2 -march=skylake" } */ +/* { dg-options "-O2 -march=skylake -mtune-ctrl=^read_zero_ymm_zmm_need_vzeroupper" } */ #include <x86intrin.h> diff --git a/gcc/testsuite/gcc.target/i386/pr101456-3.c b/gcc/testsuite/gcc.target/i386/pr101456-3.c new file mode 100644 index 00000000000..8389d18ed6c --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr101456-3.c @@ -0,0 +1,33 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=skylake -mtune=alderlake" } */ + +#include <x86intrin.h> + +extern __m256 x1; +extern __m256d x2; +extern __m256i x3; + +extern void bar (void); + +void +foo1 (void) +{ + x1 = _mm256_setzero_ps (); + bar (); +} + +void +foo2 (void) +{ + x2 = _mm256_setzero_pd (); + bar (); +} + +void +foo3 (void) +{ + x3 = _mm256_setzero_si256 (); + bar (); +} + +/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */ diff --git a/gcc/testsuite/gcc.target/i386/pr101456-4.c b/gcc/testsuite/gcc.target/i386/pr101456-4.c new file mode 100644 index 00000000000..3e4cdcc4d28 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr101456-4.c @@ -0,0 +1,33 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=haswell" } */ + +#include <x86intrin.h> + +extern __m256 x1; +extern __m256d x2; +extern __m256i x3; + +extern void bar (void); + +void +foo1 (void) +{ + x1 = _mm256_setzero_ps (); + bar (); +} + +void +foo2 (void) +{ + x2 = _mm256_setzero_pd (); + bar (); +} + +void +foo3 (void) +{ + x3 = _mm256_setzero_si256 (); + bar (); +} + +/* { dg-final { scan-assembler-times "vzeroupper" 3 } } */ -- 2.35.1