On Fri, 16 Feb 2024, Richard Henderson wrote:
> Because non-embedded aarch64 is expected to have AdvSIMD enabled, merely
> double-check with the compiler flags for __ARM_NEON and don't bother with
> a runtime check. Otherwise, model the loop after the x86 SSE2 function,
> and use VADDV to reduce the four vector comparisons.
Commit message will need a refresh (s/VADDV/UMAXV/, and there are no
vector comparisons anymore, "reduce the four vector components" perhaps).
It appears AdvSIMD types do not carry the may_alias attribute, unlike
x86 immintrin.h types. This does not matter for Qemu since it is built
with -fno-strict-aliasing anyway, just mentioning for completeness.
(for aliasing-safe reuse of this code elsewhere, I'd suggest
typedef uint32x4_t uint32x4_a __attribute__((may_alias));
and using the new type in declarations of 'p' and 'e')
Alexander
> Signed-off-by: Richard Henderson <richard.hender...@linaro.org>
> ---
> util/bufferiszero.c | 77 +++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 77 insertions(+)
>
> diff --git a/util/bufferiszero.c b/util/bufferiszero.c
> index 9b338f7be5..77db305bb0 100644
> --- a/util/bufferiszero.c
> +++ b/util/bufferiszero.c
> @@ -214,7 +214,84 @@ bool test_buffer_is_zero_next_accel(void)
> }
> return false;
> }
> +
> +#elif defined(__aarch64__) && defined(__ARM_NEON)
> +#include <arm_neon.h>
> +
> +#define REASSOC_BARRIER(vec0, vec1) asm("" : "+w"(vec0), "+w"(vec1))
> +
> +static bool buffer_is_zero_simd(const void *buf, size_t len)
> +{
> + uint32x4_t t0, t1, t2, t3;
> +
> + /* Align head/tail to 16-byte boundaries. */
> + const uint32x4_t *p = QEMU_ALIGN_PTR_DOWN(buf + 16, 16);
> + const uint32x4_t *e = QEMU_ALIGN_PTR_DOWN(buf + len - 1, 16);
> +
> + /* Unaligned loads at head/tail. */
> + t0 = vld1q_u32(buf) | vld1q_u32(buf + len - 16);
> +
> + /* Collect a partial block at tail end. */
> + t1 = e[-7] | e[-6];
> + t2 = e[-5] | e[-4];
> + t3 = e[-3] | e[-2];
> + t0 |= e[-1];
> + REASSOC_BARRIER(t0, t1);
> + REASSOC_BARRIER(t2, t3);
> + t0 |= t1;
> + t2 |= t3;
> + REASSOC_BARRIER(t0, t2);
> + t0 |= t2;
> +
> + /*
> + * Loop over complete 128-byte blocks.
> + * With the head and tail removed, e - p >= 14, so the loop
> + * must iterate at least once.
> + */
> + do {
> + /*
> + * Reduce via UMAXV. Whatever the actual result,
> + * it will only be zero if all input bytes are zero.
> + */
> + if (unlikely(vmaxvq_u32(t0) != 0)) {
> + return false;
> + }
> +
> + t0 = p[0] | p[1];
> + t1 = p[2] | p[3];
> + t2 = p[4] | p[5];
> + t3 = p[6] | p[7];
> + REASSOC_BARRIER(t0, t1);
> + REASSOC_BARRIER(t2, t3);
> + t0 |= t1;
> + t2 |= t3;
> + REASSOC_BARRIER(t0, t2);
> + t0 |= t2;
> + p += 8;
> + } while (p < e - 7);
> +
> + return vmaxvq_u32(t0) == 0;
> +}
> +
> +static biz_accel_fn const accel_table[] = {
> + buffer_is_zero_int_ge256,
> + buffer_is_zero_simd,
> +};
> +
> +static unsigned accel_index = 1;
> +#define INIT_ACCEL buffer_is_zero_simd
> +
> +bool test_buffer_is_zero_next_accel(void)
> +{
> + if (accel_index != 0) {
> + buffer_is_zero_accel = accel_table[--accel_index];
> + return true;
> + }
> + return false;
> +}
> +
> #else
> +
> bool test_buffer_is_zero_next_accel(void)
> {
> return false;
>
