The rte_wait_until_equalxx APIs abstract the functionality of 'polling
for a memory location to become equal to a given value'.
Signed-off-by: Gavin Hu <gavin...@arm.com>
Reviewed-by: Ruifeng Wang <ruifeng.w...@arm.com>
Reviewed-by: Steve Capper <steve.cap...@arm.com>
Reviewed-by: Ola Liljedahl <ola.liljed...@arm.com>
Reviewed-by: Honnappa Nagarahalli <honnappa.nagaraha...@arm.com>
---
.../common/include/arch/arm/rte_atomic_64.h | 4 +
.../common/include/arch/arm/rte_pause_64.h | 106 +++++++++++++++++++++
lib/librte_eal/common/include/generic/rte_pause.h | 39 +++++++-
3 files changed, 148 insertions(+), 1 deletion(-)
diff --git a/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h
b/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h
index 97060e4..8d742c6 100644
--- a/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h
+++ b/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h
@@ -15,8 +15,12 @@ extern "C" {
#include "generic/rte_atomic.h"
+#ifndef dsb
#define dsb(opt) asm volatile("dsb " #opt : : : "memory")
+#endif
+#ifndef dmb
#define dmb(opt) asm volatile("dmb " #opt : : : "memory")
+#endif
#define rte_mb() dsb(sy)
diff --git a/lib/librte_eal/common/include/arch/arm/rte_pause_64.h
b/lib/librte_eal/common/include/arch/arm/rte_pause_64.h
index 93895d3..1f7be0a 100644
--- a/lib/librte_eal/common/include/arch/arm/rte_pause_64.h
+++ b/lib/librte_eal/common/include/arch/arm/rte_pause_64.h
@@ -17,6 +17,112 @@ static inline void rte_pause(void)
asm volatile("yield" ::: "memory");
}
+#ifdef RTE_USE_WFE
+/* Wait for *addr to be updated with expected value */
+static __rte_always_inline void
+rte_wait_until_equal16(volatile uint16_t *addr, uint16_t expected, int
memorder)
+{
+ uint16_t tmp;
+ if (memorder == __ATOMIC_RELAXED)
+ asm volatile(
+ "ldxrh %w[tmp], %w[addr]\n"
+ "cmp %w[tmp], %w[expected]\n"
+ "b.eq 2f\n"
+ "sevl\n"
+ "1: wfe\n"
+ "ldxrh %w[tmp], %w[addr]\n"
+ "cmp %w[tmp], %w[expected]\n"
+ "bne 1b\n"
+ "2:\n"
+ : [tmp] "=&r" (tmp)
+ : [addr] "Q"(*addr), [expected] "r"(expected)
+ : "cc", "memory");
+ else
+ asm volatile(
+ "ldaxrh %w[tmp], %w[addr]\n"
+ "cmp %w[tmp], %w[expected]\n"
+ "b.eq 2f\n"
+ "sevl\n"
+ "1: wfe\n"
+ "ldaxrh %w[tmp], %w[addr]\n"
+ "cmp %w[tmp], %w[expected]\n"
+ "bne 1b\n"
+ "2:\n"
+ : [tmp] "=&r" (tmp)
+ : [addr] "Q"(*addr), [expected] "r"(expected)
+ : "cc", "memory");
+}
+
+static __rte_always_inline void
+rte_wait_until_equal32(volatile uint32_t *addr, uint32_t expected, int
memorder)
+{
+ uint32_t tmp;
+ if (memorder == __ATOMIC_RELAXED)
+ asm volatile(
+ "ldxr %w[tmp], %w[addr]\n"
+ "cmp %w[tmp], %w[expected]\n"
+ "b.eq 2f\n"
+ "sevl\n"
+ "1: wfe\n"
+ "ldxr %w[tmp], %w[addr]\n"
+ "cmp %w[tmp], %w[expected]\n"
+ "bne 1b\n"
+ "2:\n"
+ : [tmp] "=&r" (tmp)
+ : [addr] "Q"(*addr), [expected] "r"(expected)
+ : "cc", "memory");
+ else
+ asm volatile(
+ "ldaxr %w[tmp], %w[addr]\n"
+ "cmp %w[tmp], %w[expected]\n"
+ "b.eq 2f\n"
+ "sevl\n"
+ "1: wfe\n"
+ "ldaxr %w[tmp], %w[addr]\n"
+ "cmp %w[tmp], %w[expected]\n"
+ "bne 1b\n"
+ "2:\n"
+ : [tmp] "=&r" (tmp)
+ : [addr] "Q"(*addr), [expected] "r"(expected)
+ : "cc", "memory");
+}
+
+static __rte_always_inline void
+rte_wait_until_equal64(volatile uint64_t *addr, uint64_t expected, int
memorder)
+{
+ uint64_t tmp;
+ if (memorder == __ATOMIC_RELAXED)
+ asm volatile(
+ "ldxr %x[tmp], %x[addr]\n"
+ "cmp %x[tmp], %x[expected]\n"
+ "b.eq 2f\n"
+ "sevl\n"
+ "1: wfe\n"
+ "ldxr %x[tmp], %x[addr]\n"
+ "cmp %x[tmp], %x[expected]\n"
+ "bne 1b\n"
+ "2:\n"
+ : [tmp] "=&r" (tmp)
+ : [addr] "Q"(*addr), [expected] "r"(expected)
+ : "cc", "memory");
+ else
+ asm volatile(
+ "ldaxr %x[tmp], %x[addr]\n"
+ "cmp %x[tmp], %x[expected]\n"
+ "b.eq 2f\n"
+ "sevl\n"
+ "1: wfe\n"
+ "ldaxr %x[tmp], %x[addr]\n"
+ "cmp %x[tmp], %x[expected]\n"
+ "bne 1b\n"
+ "2:\n"
+ : [tmp] "=&r" (tmp)
+ : [addr] "Q"(*addr), [expected] "r"(expected)
+ : "cc", "memory");
+}
+
+#endif
+
#ifdef __cplusplus
}
#endif
diff --git a/lib/librte_eal/common/include/generic/rte_pause.h
b/lib/librte_eal/common/include/generic/rte_pause.h
index 52bd4db..8f5f025 100644
--- a/lib/librte_eal/common/include/generic/rte_pause.h
+++ b/lib/librte_eal/common/include/generic/rte_pause.h
@@ -4,7 +4,6 @@
#ifndef _RTE_PAUSE_H_
#define _RTE_PAUSE_H_
-
/**
* @file
*
@@ -12,6 +11,10 @@
*
*/
+#include <stdint.h>
+#include <rte_common.h>
+#include <rte_atomic.h>
+
/**
* Pause CPU execution for a short while
*
@@ -20,4 +23,38 @@
*/
static inline void rte_pause(void);
+#if !defined(RTE_USE_WFE)
+#ifdef RTE_USE_C11_MEM_MODEL
+#define __rte_wait_until_equal(addr, expected, memorder) do {\
+ while (__atomic_load_n(addr, memorder) != expected) \
+ rte_pause();\
+} while (0)
+#else
+#define __rte_wait_until_equal(addr, expected, memorder) do {\
+ while (*addr != expected)\
+ rte_pause();\
+ if (memorder != __ATOMIC_RELAXED)\
+ rte_smp_rmb();\
+} while (0)
+#endif
+
+static __rte_always_inline void
+rte_wait_until_equal16(volatile uint16_t *addr, uint16_t expected, int
memorder)
+{
+ __rte_wait_until_equal(addr, expected, memorder);
+}
+
+static __rte_always_inline void
+rte_wait_until_equal32(volatile uint32_t *addr, uint32_t expected, int
memorder)
+{
+ __rte_wait_until_equal(addr, expected, memorder);
+}
+
+static __rte_always_inline void
+rte_wait_until_equal64(volatile uint64_t *addr, uint64_t expected, int
memorder)
+{
+ __rte_wait_until_equal(addr, expected, memorder);
+}
+#endif /* RTE_USE_WFE */
+
#endif /* _RTE_PAUSE_H_ */
--
2.7.4
