Turn cmpxchg() and similar functions into inline C++ template functions.
This produces more robust source as the all the casting the C macros
require is then unnecessary.
Signed-off-by: David Howells <dhowe...@redhat.com>
---
arch/x86/include/asm/atomic.h | 2
arch/x86/include/asm/cmpxchg.h | 222 +++++++++++++---------------------------
2 files changed, 75 insertions(+), 149 deletions(-)
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
index 72759f131cc5..20f225cd47e8 100644
--- a/arch/x86/include/asm/atomic.h
+++ b/arch/x86/include/asm/atomic.h
@@ -190,7 +190,7 @@ static __always_inline int atomic_cmpxchg(atomic_t *v, int
old, int new)
#define atomic_try_cmpxchg atomic_try_cmpxchg
static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new)
{
- return try_cmpxchg(&v->counter, old, new);
+ return try_cmpxchg(&v->counter, *old, new);
}
static inline int atomic_xchg(atomic_t *v, int new)
diff --git a/arch/x86/include/asm/cmpxchg.h b/arch/x86/include/asm/cmpxchg.h
index 5e896c17476d..4bbf947c88a2 100644
--- a/arch/x86/include/asm/cmpxchg.h
+++ b/arch/x86/include/asm/cmpxchg.h
@@ -6,6 +6,12 @@
#include <asm/cpufeatures.h>
#include <asm/alternative.h> /* Provides LOCK_PREFIX */
+enum cmpxchg_lock {
+ __lock_local,
+ __lock_smp,
+ __lock_always
+};
+
/*
* Non-existant functions to indicate usage errors at link time
* (or compile-time if the compiler implements __compiletime_error().
@@ -20,22 +26,6 @@ extern void __add_wrong_size(void)
__compiletime_error("Bad argument size for add");
/*
- * Constants for operation sizes. On 32-bit, the 64-bit size it set to
- * -1 because sizeof will never return -1, thereby making those switch
- * case statements guaranteeed dead code which the compiler will
- * eliminate, and allowing the "missing symbol in the default case" to
- * indicate a usage error.
- */
-#define __X86_CASE_B 1
-#define __X86_CASE_W 2
-#define __X86_CASE_L 4
-#ifdef CONFIG_64BIT
-#define __X86_CASE_Q 8
-#else
-#define __X86_CASE_Q -1 /* sizeof will never return -1
*/
-#endif
-
-/*
* An exchange-type operation, which takes a value and a pointer, and
* returns the old value.
*/
@@ -107,67 +97,82 @@ static inline P xadd(P *ptr, N inc)
return v;
}
+/*
+ * Atomic compare and exchange.
+ */
+template <typename P, typename N>
+static inline bool try_cmpxchg(P *ptr, P &old, N _rep,
+ enum cmpxchg_lock lock = __lock_smp)
+{
+ bool success;
+ P rep = _rep;
+
+ if (sizeof(P) > sizeof(unsigned long))
+ __cmpxchg_wrong_size();
+
+ switch (lock) {
+ case __lock_local:
+ asm volatile("cmpxchg %[rep], %[ptr]"
+ CC_SET(z)
+ : CC_OUT(z) (success),
+ [ptr] "+m" (*ptr),
+ [old] "+a" (old)
+ : [rep] "q" (rep)
+ : "memory");
+ break;
+ case __lock_smp:
+ asm volatile(LOCK_PREFIX "cmpxchg %[rep], %[ptr]"
+ CC_SET(z)
+ : CC_OUT(z) (success),
+ [ptr] "+m" (*ptr),
+ [old] "+a" (old)
+ : [rep] "q" (rep)
+ : "memory");
+ break;
+ case __lock_always:
+ asm volatile("lock; cmpxchg %[rep], %[ptr]"
+ CC_SET(z)
+ : CC_OUT(z) (success),
+ [ptr] "+m" (*ptr),
+ [old] "+a" (old)
+ : [rep] "q" (rep)
+ : "memory");
+ break;
+ }
+
+ return likely(success);
+}
+
+
/*
* Atomic compare and exchange. Compare OLD with MEM, if identical,
* store NEW in MEM. Return the initial value in MEM. Success is
* indicated by comparing RETURN with OLD.
*/
-#define __raw_cmpxchg(ptr, old, new, size, lock) \
-({ \
- __typeof__(*(ptr)) __ret; \
- __typeof__(*(ptr)) __old = (old); \
- __typeof__(*(ptr)) __new = (new); \
- switch (size) { \
- case __X86_CASE_B: \
- { \
- volatile u8 *__ptr = (volatile u8 *)(ptr); \
- asm volatile(lock "cmpxchgb %2,%1" \
- : "=a" (__ret), "+m" (*__ptr) \
- : "q" (__new), "0" (__old) \
- : "memory"); \
- break; \
- } \
- case __X86_CASE_W: \
- { \
- volatile u16 *__ptr = (volatile u16 *)(ptr); \
- asm volatile(lock "cmpxchgw %2,%1" \
- : "=a" (__ret), "+m" (*__ptr) \
- : "r" (__new), "0" (__old) \
- : "memory"); \
- break; \
- } \
- case __X86_CASE_L: \
- { \
- volatile u32 *__ptr = (volatile u32 *)(ptr); \
- asm volatile(lock "cmpxchgl %2,%1" \
- : "=a" (__ret), "+m" (*__ptr) \
- : "r" (__new), "0" (__old) \
- : "memory"); \
- break; \
- } \
- case __X86_CASE_Q: \
- { \
- volatile u64 *__ptr = (volatile u64 *)(ptr); \
- asm volatile(lock "cmpxchgq %2,%1" \
- : "=a" (__ret), "+m" (*__ptr) \
- : "r" (__new), "0" (__old) \
- : "memory"); \
- break; \
- } \
- default: \
- __cmpxchg_wrong_size(); \
- } \
- __ret; \
-})
-#define __cmpxchg(ptr, old, new, size) \
- __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
+template <typename P, typename N>
+static inline P cmpxchg_local(P *ptr, P old, N rep)
+{
+ P ret = old;
+ try_cmpxchg(ptr, ret, rep, __lock_local);
+ return ret;
+}
-#define __sync_cmpxchg(ptr, old, new, size) \
- __raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
+template <typename P, typename N>
+static inline P cmpxchg(P *ptr, P old, N rep)
+{
+ P ret = old;
+ try_cmpxchg(ptr, ret, rep, __lock_smp);
+ return ret;
+}
-#define __cmpxchg_local(ptr, old, new, size) \
- __raw_cmpxchg((ptr), (old), (new), (size), "")
+template <typename P, typename N>
+static inline P sync_cmpxchg(P *ptr, P old, N rep)
+{
+ P ret = old;
+ try_cmpxchg(ptr, ret, rep, __lock_always);
+ return ret;
+}
#ifdef CONFIG_X86_32
# include <asm/cmpxchg_32.h>
@@ -175,85 +180,6 @@ static inline P xadd(P *ptr, N inc)
# include <asm/cmpxchg_64.h>
#endif
-#define cmpxchg(ptr, old, new) \
- __cmpxchg(ptr, old, new, sizeof(*(ptr)))
-
-#define sync_cmpxchg(ptr, old, new) \
- __sync_cmpxchg(ptr, old, new, sizeof(*(ptr)))
-
-#define cmpxchg_local(ptr, old, new) \
- __cmpxchg_local(ptr, old, new, sizeof(*(ptr)))
-
-
-#define __raw_try_cmpxchg(_ptr, _pold, _new, size, lock) \
-({ \
- bool success; \
- __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
- __typeof__(*(_ptr)) __old = *_old; \
- __typeof__(*(_ptr)) __new = (_new); \
- switch (size) { \
- case __X86_CASE_B: \
- { \
- volatile u8 *__ptr = (volatile u8 *)(_ptr); \
- asm volatile(lock "cmpxchgb %[new], %[ptr]" \
- CC_SET(z) \
- : CC_OUT(z) (success), \
- [ptr] "+m" (*__ptr), \
- [old] "+a" (__old) \
- : [new] "q" (__new) \
- : "memory"); \
- break; \
- } \
- case __X86_CASE_W: \
- { \
- volatile u16 *__ptr = (volatile u16 *)(_ptr); \
- asm volatile(lock "cmpxchgw %[new], %[ptr]" \
- CC_SET(z) \
- : CC_OUT(z) (success), \
- [ptr] "+m" (*__ptr), \
- [old] "+a" (__old) \
- : [new] "r" (__new) \
- : "memory"); \
- break; \
- } \
- case __X86_CASE_L: \
- { \
- volatile u32 *__ptr = (volatile u32 *)(_ptr); \
- asm volatile(lock "cmpxchgl %[new], %[ptr]" \
- CC_SET(z) \
- : CC_OUT(z) (success), \
- [ptr] "+m" (*__ptr), \
- [old] "+a" (__old) \
- : [new] "r" (__new) \
- : "memory"); \
- break; \
- } \
- case __X86_CASE_Q: \
- { \
- volatile u64 *__ptr = (volatile u64 *)(_ptr); \
- asm volatile(lock "cmpxchgq %[new], %[ptr]" \
- CC_SET(z) \
- : CC_OUT(z) (success), \
- [ptr] "+m" (*__ptr), \
- [old] "+a" (__old) \
- : [new] "r" (__new) \
- : "memory"); \
- break; \
- } \
- default: \
- __cmpxchg_wrong_size(); \
- } \
- if (unlikely(!success)) \
- *_old = __old; \
- likely(success); \
-})
-
-#define __try_cmpxchg(ptr, pold, new, size) \
- __raw_try_cmpxchg((ptr), (pold), (new), (size), LOCK_PREFIX)
-
-#define try_cmpxchg(ptr, pold, new) \
- __try_cmpxchg((ptr), (pold), (new), sizeof(*(ptr)))
-
#define __cmpxchg_double(pfx, p1, p2, o1, o2, n1, n2) \
({ \
bool __ret; \
