> Introduce macros as generic interface for address monitoring.
>
> Signed-off-by: Feifei Wang <feifei.wa...@arm.com>
> Reviewed-by: Ruifeng Wang <ruifeng.w...@arm.com>
> ---
> lib/eal/arm/include/rte_pause_64.h | 151 ++++++++++++++++++----------
> lib/eal/include/generic/rte_pause.h | 78 ++++++++++++++
> 2 files changed, 175 insertions(+), 54 deletions(-)
>
> diff --git a/lib/eal/arm/include/rte_pause_64.h
> b/lib/eal/arm/include/rte_pause_64.h
> index e87d10b8cc..205510e044 100644
> --- a/lib/eal/arm/include/rte_pause_64.h
> +++ b/lib/eal/arm/include/rte_pause_64.h
> @@ -31,20 +31,12 @@ static inline void rte_pause(void)
> /* Put processor into low power WFE(Wait For Event) state. */
> #define __WFE() { asm volatile("wfe" : : : "memory"); }
>
> -static __rte_always_inline void
> -rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
> - int memorder)
> -{
> - uint16_t value;
> -
> - assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
> -
> - /*
> - * Atomic exclusive load from addr, it returns the 16-bit content of
> - * *addr while making it 'monitored',when it is written by someone
> - * else, the 'monitored' state is cleared and a event is generated
> - * implicitly to exit WFE.
> - */
> +/*
> + * Atomic exclusive load from addr, it returns the 16-bit content of
> + * *addr while making it 'monitored', when it is written by someone
> + * else, the 'monitored' state is cleared and a event is generated
> + * implicitly to exit WFE.
> + */
> #define __LOAD_EXC_16(src, dst, memorder) { \
> if (memorder == __ATOMIC_RELAXED) { \
> asm volatile("ldxrh %w[tmp], [%x[addr]]" \
> @@ -58,6 +50,52 @@ rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t
> expected,
> : "memory"); \
> } }
>
> +/*
> + * Atomic exclusive load from addr, it returns the 32-bit content of
> + * *addr while making it 'monitored', when it is written by someone
> + * else, the 'monitored' state is cleared and a event is generated
> + * implicitly to exit WFE.
> + */
> +#define __LOAD_EXC_32(src, dst, memorder) { \
> + if (memorder == __ATOMIC_RELAXED) { \
> + asm volatile("ldxr %w[tmp], [%x[addr]]" \
> + : [tmp] "=&r" (dst) \
> + : [addr] "r"(src) \
> + : "memory"); \
> + } else { \
> + asm volatile("ldaxr %w[tmp], [%x[addr]]" \
> + : [tmp] "=&r" (dst) \
> + : [addr] "r"(src) \
> + : "memory"); \
> + } }
> +
> +/*
> + * Atomic exclusive load from addr, it returns the 64-bit content of
> + * *addr while making it 'monitored', when it is written by someone
> + * else, the 'monitored' state is cleared and a event is generated
> + * implicitly to exit WFE.
> + */
> +#define __LOAD_EXC_64(src, dst, memorder) { \
> + if (memorder == __ATOMIC_RELAXED) { \
> + asm volatile("ldxr %x[tmp], [%x[addr]]" \
> + : [tmp] "=&r" (dst) \
> + : [addr] "r"(src) \
> + : "memory"); \
> + } else { \
> + asm volatile("ldaxr %x[tmp], [%x[addr]]" \
> + : [tmp] "=&r" (dst) \
> + : [addr] "r"(src) \
> + : "memory"); \
> + } }
> +
> +static __rte_always_inline void
> +rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
> + int memorder)
> +{
> + uint16_t value;
> +
> + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
> +
> __LOAD_EXC_16(addr, value, memorder)
> if (value != expected) {
> __SEVL()
> @@ -66,7 +104,6 @@ rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t
> expected,
> __LOAD_EXC_16(addr, value, memorder)
> } while (value != expected);
> }
> -#undef __LOAD_EXC_16
> }
>
> static __rte_always_inline void
> @@ -77,25 +114,6 @@ rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t
> expected,
>
> assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
>
> - /*
> - * Atomic exclusive load from addr, it returns the 32-bit content of
> - * *addr while making it 'monitored',when it is written by someone
> - * else, the 'monitored' state is cleared and a event is generated
> - * implicitly to exit WFE.
> - */
> -#define __LOAD_EXC_32(src, dst, memorder) { \
> - if (memorder == __ATOMIC_RELAXED) { \
> - asm volatile("ldxr %w[tmp], [%x[addr]]" \
> - : [tmp] "=&r" (dst) \
> - : [addr] "r"(src) \
> - : "memory"); \
> - } else { \
> - asm volatile("ldaxr %w[tmp], [%x[addr]]" \
> - : [tmp] "=&r" (dst) \
> - : [addr] "r"(src) \
> - : "memory"); \
> - } }
> -
> __LOAD_EXC_32(addr, value, memorder)
> if (value != expected) {
> __SEVL()
> @@ -104,7 +122,6 @@ rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t
> expected,
> __LOAD_EXC_32(addr, value, memorder)
> } while (value != expected);
> }
> -#undef __LOAD_EXC_32
> }
>
> static __rte_always_inline void
> @@ -115,25 +132,6 @@ rte_wait_until_equal_64(volatile uint64_t *addr,
> uint64_t expected,
>
> assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
>
> - /*
> - * Atomic exclusive load from addr, it returns the 64-bit content of
> - * *addr while making it 'monitored',when it is written by someone
> - * else, the 'monitored' state is cleared and a event is generated
> - * implicitly to exit WFE.
> - */
> -#define __LOAD_EXC_64(src, dst, memorder) { \
> - if (memorder == __ATOMIC_RELAXED) { \
> - asm volatile("ldxr %x[tmp], [%x[addr]]" \
> - : [tmp] "=&r" (dst) \
> - : [addr] "r"(src) \
> - : "memory"); \
> - } else { \
> - asm volatile("ldaxr %x[tmp], [%x[addr]]" \
> - : [tmp] "=&r" (dst) \
> - : [addr] "r"(src) \
> - : "memory"); \
> - } }
> -
> __LOAD_EXC_64(addr, value, memorder)
> if (value != expected) {
> __SEVL()
> @@ -143,6 +141,51 @@ rte_wait_until_equal_64(volatile uint64_t *addr,
> uint64_t expected,
> } while (value != expected);
> }
> }
> +
> +#define rte_wait_event_16(addr, mask, expected, cond, memorder)
> \
> +do { \
> + uint16_t value \
> + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); \
> + __LOAD_EXC_16(addr, value, memorder) \
> + if ((value & mask) cond expected) { \
> + __SEVL() \
> + do { \
> + __WFE() \
> + __LOAD_EXC_16(addr, value, memorder) \
> + } while ((value & mask) cond expected); \
> + } \
> +} while (0)
> +
> +#define rte_wait_event_32(addr, mask, expected, cond, memorder)
> \
> +do {
> \
> + uint32_t value \
> + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); \
> + __LOAD_EXC_32(addr, value, memorder) \
> + if ((value & mask) op expected) { \
> + __SEVL() \
> + do { \
> + __WFE() \
> + __LOAD_EXC_32(addr, value, memorder) \
> + } while ((value & mask) cond expected); \
> + } \
> +} while (0)
> +
> +#define rte_wait_event_64(addr, mask, expected, cond, memorder)
> \
> +do {
> \
> + uint64_t value \
> + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); \
> + __LOAD_EXC_64(addr, value, memorder) \
> + if ((value & mask) cond expected) { \
> + __SEVL() \
> + do { \
> + __WFE() \
> + __LOAD_EXC_64(addr, value, memorder) \
> + } while ((value & mask) cond expected); \
> + } \
> +} while (0)
> +
> +#undef __LOAD_EXC_16
> +#undef __LOAD_EXC_32
> #undef __LOAD_EXC_64
>
> #undef __SEVL
> diff --git a/lib/eal/include/generic/rte_pause.h
> b/lib/eal/include/generic/rte_pause.h
> index 668ee4a184..4e32107eca 100644
> --- a/lib/eal/include/generic/rte_pause.h
> +++ b/lib/eal/include/generic/rte_pause.h
> @@ -111,6 +111,84 @@ rte_wait_until_equal_64(volatile uint64_t *addr,
> uint64_t expected,
> while (__atomic_load_n(addr, memorder) != expected)
> rte_pause();
> }
> +
> +/*
> + * Wait until a 16-bit *addr breaks the condition, with a relaxed memory
> + * ordering model meaning the loads around this API can be reordered.
> + *
> + * @param addr
> + * A pointer to the memory location.
> + * @param mask
> + * A mask of value bits in interest
> + * @param expected
> + * A 16-bit expected value to be in the memory location.
> + * @param cond
> + * A symbol representing the condition (==, !=).
> + * @param memorder
> + * Two different memory orders that can be specified:
> + * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
> + * C++11 memory orders with the same names, see the C++11 standard or
> + * the GCC wiki on atomic synchronization for detailed definition.
> + */
Hmm, so now we have 2 APIs doing similar thing:
rte_wait_until_equal_n() and rte_wait_event_n().
Can we probably unite them somehow?
At least make rte_wait_until_equal_n() to use rte_wait_event_n() underneath.
> +#define rte_wait_event_16(addr, mask, expected, cond, memorder)
> \
> +do { \
> + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); \
And why user is not allowed to use __ATOMIC_SEQ_CST here?
BTW, if we expect memorder to always be a constant, might be better
BUILD_BUG_ON()?
> + \
> + while ((__atomic_load_n(addr, memorder) & mask) cond expected) \
> + rte_pause(); \
> +} while (0)
Two thoughts with these macros:
1. It is a goof practise to put () around macro parameters in the macro body.
Will save from a lot of unexpected troubles.
2. I think these 3 macros can be united into one.
Something like:
#define rte_wait_event(addr, mask, expected, cond, memorder) do {\
typeof (*(addr)) val = __atomic_load_n((addr), (memorder)); \
if ((val & (typeof(val))(mask)) cond (typeof(val))(expected)) \
break; \
rte_pause(); \
} while (1);
> +
> +/*
> + * Wait until a 32-bit *addr breaks the condition, with a relaxed memory
> + * ordering model meaning the loads around this API can be reordered.
> + *
> + * @param addr
> + * A pointer to the memory location.
> + * @param mask
> + * A mask of value bits in interest.
> + * @param expected
> + * A 32-bit expected value to be in the memory location.
> + * @param cond
> + * A symbol representing the condition (==, !=).
> + * @param memorder
> + * Two different memory orders that can be specified:
> + * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
> + * C++11 memory orders with the same names, see the C++11 standard or
> + * the GCC wiki on atomic synchronization for detailed definition.
> + */
> +#define rte_wait_event_32(addr, mask, expected, cond, memorder)
> \
> +do { \
> + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); \
> + \
> + while ((__atomic_load_n(addr, memorder) & mask) cond expected) \
> + rte_pause(); \
> +} while (0)
> +
> +/*
> + * Wait until a 64-bit *addr breaks the condition, with a relaxed memory
> + * ordering model meaning the loads around this API can be reordered.
> + *
> + * @param addr
> + * A pointer to the memory location.
> + * @param mask
> + * A mask of value bits in interest
> + * @param expected
> + * A 64-bit expected value to be in the memory location.
> + * @param cond
> + * A symbol representing the condition (==, !=).
> + * @param memorder
> + * Two different memory orders that can be specified:
> + * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
> + * C++11 memory orders with the same names, see the C++11 standard or
> + * the GCC wiki on atomic synchronization for detailed definition.
> + */
> +#define rte_wait_event_64(addr, mask, expected, cond, memorder)
> \
> +do { \
> + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); \
> + \
> + while ((__atomic_load_n(addr, memorder) & mask) cond expected) \
> + rte_pause(); \
> +} while (0)
> #endif
>
> #endif /* _RTE_PAUSE_H_ */
> --
> 2.25.1
