On Wed, Nov 15, 2023 at 6:42 AM Tamar Christina <tamar.christ...@arm.com> wrote:
>
> Hi All,
>
> This changes unpack instructions to use zip{1,2} when doing a zero-extending
> widening operation. Permutes generally have a higher throughput than the
> widening operations. Zeros are shuffled into the top half of the registers.
>
> The testcase
>
> void d2 (unsigned * restrict a, unsigned short *b, int n)
> {
> for (int i = 0; i < (n & -8); i++)
> a[i] = b[i];
> }
>
> now generates:
>
> movi v1.4s, 0
> .L3:
> ldr q0, [x1], 16
> zip1 v2.8h, v0.8h, v1.8h
> zip2 v0.8h, v0.8h, v1.8h
> stp q2, q0, [x0]
> add x0, x0, 32
> cmp x1, x2
> bne .L3
>
>
> instead of:
>
> .L3:
> ldr q0, [x1], 16
> uxtl v1.4s, v0.4h
> uxtl2 v0.4s, v0.8h
> stp q1, q0, [x0]
> add x0, x0, 32
> cmp x1, x2
> bne .L3
>
> Since we need the extra 0 register we do this only for the vectorizer's lo/hi
> pairs when we know the 0 will be floated outside of the loop.
>
> This gives an 8% speed-up in Imagick in SPECCPU 2017 on Neoverse V2.
>
> Bootstrapped Regtested on aarch64-none-linux-gnu and no issues.
>
> Ok for master?
>
> Thanks,
> Tamar
>
> gcc/ChangeLog:
>
> * config/aarch64/aarch64-simd.md (vec_unpack<su>_lo_<mode,
> vec_unpack<su>_lo_<mode): Split into...
> (vec_unpacku_lo_<mode, vec_unpacks_lo_<mode,
> vec_unpacku_lo_<mode, vec_unpacks_lo_<mode): ...These.
> (aarch64_usubw<mode>_<PERM_EXTEND:perm_hilo>_zip): New.
> (aarch64_uaddw<mode>_<PERM_EXTEND:perm_hilo>_zip): New.
> * config/aarch64/iterators.md (PERM_EXTEND, perm_index): New.
> (perm_hilo): Add UNSPEC_ZIP1, UNSPEC_ZIP2.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.target/aarch64/simd/vmovl_high_1.c: Update codegen.
> * gcc.target/aarch64/uxtl-combine-1.c: New test.
> * gcc.target/aarch64/uxtl-combine-2.c: New test.
> * gcc.target/aarch64/uxtl-combine-3.c: New test.
> * gcc.target/aarch64/uxtl-combine-4.c: New test.
> * gcc.target/aarch64/uxtl-combine-5.c: New test.
> * gcc.target/aarch64/uxtl-combine-6.c: New test.
You have a few typos in the testcases which causes:
ERROR: gcc.target/aarch64/uxtl-combine-4.c: error executing dg-final:
invalid command name "scan-assembler-time"
ERROR: gcc.target/aarch64/uxtl-combine-5.c: error executing dg-final:
invalid command name "scan-assembler-time"
ERROR: gcc.target/aarch64/uxtl-combine-6.c: error executing dg-final:
invalid command name "scan-assembler-time"
They all should be `scan-assembler-times`.
Thanks,
Andrew
>
> --- inline copy of patch --
> diff --git a/gcc/config/aarch64/aarch64-simd.md
> b/gcc/config/aarch64/aarch64-simd.md
> index
> 81ff5bad03d598fa0d48df93d172a28bc0d1d92e..3d811007dd94dcd9176d6021a41a196c12fe9c3f
> 100644
> --- a/gcc/config/aarch64/aarch64-simd.md
> +++ b/gcc/config/aarch64/aarch64-simd.md
> @@ -1988,26 +1988,60 @@ (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
> [(set_attr "type" "neon_shift_imm_long")]
> )
>
> -(define_expand "vec_unpack<su>_hi_<mode>"
> +(define_expand "vec_unpacku_hi_<mode>"
> [(match_operand:<VWIDE> 0 "register_operand")
> - (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
> + (match_operand:VQW 1 "register_operand")]
> + "TARGET_SIMD"
> + {
> + rtx res = gen_reg_rtx (<MODE>mode);
> + rtx tmp = aarch64_gen_shareable_zero (<MODE>mode);
> + if (BYTES_BIG_ENDIAN)
> + emit_insn (gen_aarch64_zip2<mode> (res, tmp, operands[1]));
> + else
> + emit_insn (gen_aarch64_zip2<mode> (res, operands[1], tmp));
> + emit_move_insn (operands[0],
> + simplify_gen_subreg (<VWIDE>mode, res, <MODE>mode, 0));
> + DONE;
> + }
> +)
> +
> +(define_expand "vec_unpacks_hi_<mode>"
> + [(match_operand:<VWIDE> 0 "register_operand")
> + (match_operand:VQW 1 "register_operand")]
> "TARGET_SIMD"
> {
> rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
> - emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
> - operands[1], p));
> + emit_insn (gen_aarch64_simd_vec_unpacks_hi_<mode> (operands[0],
> + operands[1], p));
> + DONE;
> + }
> +)
> +
> +(define_expand "vec_unpacku_lo_<mode>"
> + [(match_operand:<VWIDE> 0 "register_operand")
> + (match_operand:VQW 1 "register_operand")]
> + "TARGET_SIMD"
> + {
> + rtx res = gen_reg_rtx (<MODE>mode);
> + rtx tmp = aarch64_gen_shareable_zero (<MODE>mode);
> + if (BYTES_BIG_ENDIAN)
> + emit_insn (gen_aarch64_zip1<mode> (res, tmp, operands[1]));
> + else
> + emit_insn (gen_aarch64_zip1<mode> (res, operands[1], tmp));
> + emit_move_insn (operands[0],
> + simplify_gen_subreg (<VWIDE>mode, res, <MODE>mode, 0));
> DONE;
> }
> )
>
> -(define_expand "vec_unpack<su>_lo_<mode>"
> +(define_expand "vec_unpacks_lo_<mode>"
> [(match_operand:<VWIDE> 0 "register_operand")
> - (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
> + (match_operand:VQW 1 "register_operand")]
> "TARGET_SIMD"
> {
> rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, false);
> - emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
> - operands[1], p));
> + emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
> + operands[1], p));
> DONE;
> }
> )
> @@ -4735,6 +4769,34 @@ (define_insn
> "aarch64_<ANY_EXTEND:su>subw2<mode>_internal"
> [(set_attr "type" "neon_sub_widen")]
> )
>
> +(define_insn "aarch64_usubw<mode>_<PERM_EXTEND:perm_hilo>_zip"
> + [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
> + (minus:<VWIDE>
> + (match_operand:<VWIDE> 1 "register_operand" "w")
> + (subreg:<VWIDE>
> + (unspec:<MODE> [
> + (match_operand:VQW 2 "register_operand" "w")
> + (match_operand:VQW 3 "aarch64_simd_imm_zero" "Dz")
> + ] PERM_EXTEND) 0)))]
> + "TARGET_SIMD"
> + "usubw<PERM_EXTEND:perm_index>\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vhalftype>"
> + [(set_attr "type" "neon_sub_widen")]
> +)
> +
> +(define_insn "aarch64_uaddw<mode>_<PERM_EXTEND:perm_hilo>_zip"
> + [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
> + (plus:<VWIDE>
> + (subreg:<VWIDE>
> + (unspec:<MODE> [
> + (match_operand:VQW 2 "register_operand" "w")
> + (match_operand:VQW 3 "aarch64_simd_imm_zero" "Dz")
> + ] PERM_EXTEND) 0)
> + (match_operand:<VWIDE> 1 "register_operand" "w")))]
> + "TARGET_SIMD"
> + "uaddw<PERM_EXTEND:perm_index>\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vhalftype>"
> + [(set_attr "type" "neon_add_widen")]
> +)
> +
> (define_insn "aarch64_<ANY_EXTEND:su>addw<mode>"
> [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
> (plus:<VWIDE>
> diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
> index
> f9e2210095ea9d6d9c96971222a7757a2f418c2d..de281671aedc5141c69063f14cf0fbec5adecb04
> 100644
> --- a/gcc/config/aarch64/iterators.md
> +++ b/gcc/config/aarch64/iterators.md
> @@ -2674,6 +2674,9 @@ (define_int_iterator PERMUTEQ [UNSPEC_ZIP1Q UNSPEC_ZIP2Q
> (define_int_iterator OPTAB_PERMUTE [UNSPEC_ZIP1 UNSPEC_ZIP2
> UNSPEC_UZP1 UNSPEC_UZP2])
>
> +;; Permutes for zero extends
> +(define_int_iterator PERM_EXTEND [UNSPEC_ZIP1 UNSPEC_ZIP2])
> +
> (define_int_iterator REVERSE [UNSPEC_REV64 UNSPEC_REV32 UNSPEC_REV16])
>
> (define_int_iterator FRINT [UNSPEC_FRINTZ UNSPEC_FRINTP UNSPEC_FRINTM
> @@ -3496,7 +3499,10 @@ (define_int_attr rev_op [(UNSPEC_REV64 "64")
> (UNSPEC_REV32 "32")
> (UNSPEC_REV16 "16")])
>
> (define_int_attr perm_hilo [(UNSPEC_UNPACKSHI "hi") (UNSPEC_UNPACKUHI "hi")
> - (UNSPEC_UNPACKSLO "lo") (UNSPEC_UNPACKULO "lo")])
> + (UNSPEC_UNPACKSLO "lo") (UNSPEC_UNPACKULO "lo")
> + (UNSPEC_ZIP2 "hi") (UNSPEC_ZIP1 "lo")])
> +
> +(define_int_attr perm_index [(UNSPEC_ZIP2 "2") (UNSPEC_ZIP1 "")])
>
> ;; Return true if the associated optab refers to the high-numbered lanes,
> ;; false if it refers to the low-numbered lanes. The convention is for
> diff --git a/gcc/testsuite/gcc.target/aarch64/simd/vmovl_high_1.c
> b/gcc/testsuite/gcc.target/aarch64/simd/vmovl_high_1.c
> index
> d45bb83e3503d512c443f37a446d30d188719a96..a2d09eaee0de5a3d3409330c5c26a3b5315e84eb
> 100644
> --- a/gcc/testsuite/gcc.target/aarch64/simd/vmovl_high_1.c
> +++ b/gcc/testsuite/gcc.target/aarch64/simd/vmovl_high_1.c
> @@ -22,11 +22,11 @@ FUNC (int32x4_t, int64x2_t, s32)
> /* { dg-final { scan-assembler-times {sxtl2\tv0\.2d, v0\.4s} 1} } */
>
> FUNC (uint8x16_t, uint16x8_t, u8)
> -/* { dg-final { scan-assembler-times {uxtl2\tv0\.8h, v0\.16b} 1} } */
> +/* { dg-final { scan-assembler-times {zip2\tv0\.16b, v0\.16b} 1} } */
>
> FUNC (uint16x8_t, uint32x4_t, u16)
> -/* { dg-final { scan-assembler-times {uxtl2\tv0\.4s, v0\.8h} 1} } */
> +/* { dg-final { scan-assembler-times {zip2\tv0\.8h, v0\.8h} 1} } */
>
> FUNC (uint32x4_t, uint64x2_t, u32)
> -/* { dg-final { scan-assembler-times {uxtl2\tv0\.2d, v0\.4s} 1} } */
> +/* { dg-final { scan-assembler-times {zip2\tv0\.4s, v0\.4s} 1} } */
>
> diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-1.c
> b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-1.c
> new file mode 100755
> index
> 0000000000000000000000000000000000000000..68fa9a09fe55f5a72355e23c90e781a898c5975e
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-1.c
> @@ -0,0 +1,20 @@
> +/* { dg-do assemble } */
> +/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
> +
> +#pragma GCC target "+nosve"
> +
> +#define SIGN unsigned
> +#define TYPE1 char
> +#define TYPE2 short
> +
> +void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n)
> +{
> + for (int i = 0; i < (n & -8); i++)
> + a[i] = b[i];
> +}
> +
> +/* { dg-final { scan-assembler-times {\tzip1\t} 1 } } */
> +/* { dg-final { scan-assembler-times {\tzip2\t} 1 } } */
> +/* { dg-final { scan-assembler-not {\tuxtl\t} } } */
> +/* { dg-final { scan-assembler-not {\tuxtl2\t} } } */
> +
> diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-2.c
> b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-2.c
> new file mode 100755
> index
> 0000000000000000000000000000000000000000..af8a89085cfca800b41970a8410bc91b84a31d07
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-2.c
> @@ -0,0 +1,20 @@
> +/* { dg-do assemble } */
> +/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
> +
> +#pragma GCC target "+nosve"
> +
> +#define SIGN unsigned
> +#define TYPE1 short
> +#define TYPE2 int
> +
> +void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n)
> +{
> + for (int i = 0; i < (n & -8); i++)
> + a[i] = b[i];
> +}
> +
> +/* { dg-final { scan-assembler-times {\tzip1\t} 1 } } */
> +/* { dg-final { scan-assembler-times {\tzip2\t} 1 } } */
> +/* { dg-final { scan-assembler-not {\tuxtl\t} } } */
> +/* { dg-final { scan-assembler-not {\tuxtl2\t} } } */
> +
> diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-3.c
> b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-3.c
> new file mode 100755
> index
> 0000000000000000000000000000000000000000..cdae6d09529b743857a092f53a07111df64775d7
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-3.c
> @@ -0,0 +1,20 @@
> +/* { dg-do assemble } */
> +/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
> +
> +#pragma GCC target "+nosve"
> +
> +#define SIGN unsigned
> +#define TYPE1 int
> +#define TYPE2 long long
> +
> +void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n)
> +{
> + for (int i = 0; i < (n & -8); i++)
> + a[i] = b[i];
> +}
> +
> +/* { dg-final { scan-assembler-times {\tzip1\t} 1 } } */
> +/* { dg-final { scan-assembler-times {\tzip2\t} 1 } } */
> +/* { dg-final { scan-assembler-not {\tuxtl\t} } } */
> +/* { dg-final { scan-assembler-not {\tuxtl2\t} } } */
> +
> diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-4.c
> b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-4.c
> new file mode 100755
> index
> 0000000000000000000000000000000000000000..e1a9c4f5661a36ec7b2c5dc6f0fd85c42fcaac39
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-4.c
> @@ -0,0 +1,20 @@
> +/* { dg-do assemble } */
> +/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
> +
> +#pragma GCC target "+nosve"
> +
> +#define SIGN signed
> +#define TYPE1 char
> +#define TYPE2 short
> +
> +void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n)
> +{
> + for (int i = 0; i < (n & -8); i++)
> + a[i] = b[i];
> +}
> +
> +/* { dg-final { scan-assembler-not {\tzip1\t} } } */
> +/* { dg-final { scan-assembler-not {\tzip2\t} } } */
> +/* { dg-final { scan-assembler-times {\tsxtl\t} 1 } } */
> +/* { dg-final { scan-assembler-time {\tsxtl2\t} 1 } } */
> +
> diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-5.c
> b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-5.c
> new file mode 100755
> index
> 0000000000000000000000000000000000000000..92b09ba4abba80f240ac175be2ef880968534975
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-5.c
> @@ -0,0 +1,20 @@
> +/* { dg-do assemble } */
> +/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
> +
> +#pragma GCC target "+nosve"
> +
> +#define SIGN signed
> +#define TYPE1 short
> +#define TYPE2 int
> +
> +void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n)
> +{
> + for (int i = 0; i < (n & -8); i++)
> + a[i] = b[i];
> +}
> +
> +/* { dg-final { scan-assembler-not {\tzip1\t} } } */
> +/* { dg-final { scan-assembler-not {\tzip2\t} } } */
> +/* { dg-final { scan-assembler-times {\tsxtl\t} 1 } } */
> +/* { dg-final { scan-assembler-time {\tsxtl2\t} 1 } } */
> +
> diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-6.c
> b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-6.c
> new file mode 100755
> index
> 0000000000000000000000000000000000000000..5c6e635f29d1e52f51f5b75a477f7d8744f32ca3
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-6.c
> @@ -0,0 +1,20 @@
> +/* { dg-do assemble } */
> +/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
> +
> +#pragma GCC target "+nosve"
> +
> +#define SIGN signed
> +#define TYPE1 int
> +#define TYPE2 long long
> +
> +void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n)
> +{
> + for (int i = 0; i < (n & -8); i++)
> + a[i] = b[i];
> +}
> +
> +/* { dg-final { scan-assembler-not {\tzip1\t} } } */
> +/* { dg-final { scan-assembler-not {\tzip2\t} } } */
> +/* { dg-final { scan-assembler-times {\tsxtl\t} 1 } } */
> +/* { dg-final { scan-assembler-time {\tsxtl2\t} 1 } } */
> +
>
>
>
>
> --
