Store the owner CPU number in the lock word so it may be yielded to,
as powerpc's paravirtualised simple spinlocks do.
Signed-off-by: Nicholas Piggin <npig...@gmail.com>
---
arch/powerpc/include/asm/qspinlock.h | 12 ++++++++++--
arch/powerpc/include/asm/qspinlock_types.h | 12 +++++++++++-
arch/powerpc/lib/qspinlock.c | 2 +-
3 files changed, 22 insertions(+), 4 deletions(-)
diff --git a/arch/powerpc/include/asm/qspinlock.h
b/arch/powerpc/include/asm/qspinlock.h
index 2a6f12a2c385..be53702e56fc 100644
--- a/arch/powerpc/include/asm/qspinlock.h
+++ b/arch/powerpc/include/asm/qspinlock.h
@@ -20,8 +20,15 @@ static __always_inline int queued_spin_is_contended(struct
qspinlock *lock)
return !!(READ_ONCE(lock->val) & _Q_TAIL_CPU_MASK);
}
+static __always_inline u32 queued_spin_encode_locked_val(void)
+{
+ /* XXX: make this use lock value in paca like simple spinlocks? */
+ return _Q_LOCKED_VAL | (smp_processor_id() << _Q_OWNER_CPU_OFFSET);
+}
+
static __always_inline int queued_spin_trylock(struct qspinlock *lock)
{
+ u32 new = queued_spin_encode_locked_val();
u32 prev;
asm volatile(
@@ -33,7 +40,7 @@ static __always_inline int queued_spin_trylock(struct
qspinlock *lock)
"\t" PPC_ACQUIRE_BARRIER " \n"
"2: \n"
: "=&r" (prev)
- : "r" (&lock->val), "r" (_Q_LOCKED_VAL),
+ : "r" (&lock->val), "r" (new),
"i" (IS_ENABLED(CONFIG_PPC64))
: "cr0", "memory");
@@ -42,6 +49,7 @@ static __always_inline int queued_spin_trylock(struct
qspinlock *lock)
static __always_inline int __queued_spin_trylock_steal(struct qspinlock *lock)
{
+ u32 new = queued_spin_encode_locked_val();
u32 prev, tmp;
/* Trylock may get ahead of queued nodes if it finds unlocked */
@@ -56,7 +64,7 @@ static __always_inline int __queued_spin_trylock_steal(struct
qspinlock *lock)
"\t" PPC_ACQUIRE_BARRIER " \n"
"2: \n"
: "=&r" (prev), "=&r" (tmp)
- : "r" (&lock->val), "r" (_Q_LOCKED_VAL), "r" (_Q_TAIL_CPU_MASK),
+ : "r" (&lock->val), "r" (new), "r" (_Q_TAIL_CPU_MASK),
"i" (IS_ENABLED(CONFIG_PPC64))
: "cr0", "memory");
diff --git a/arch/powerpc/include/asm/qspinlock_types.h
b/arch/powerpc/include/asm/qspinlock_types.h
index 1911a8a16237..adfeed4aa495 100644
--- a/arch/powerpc/include/asm/qspinlock_types.h
+++ b/arch/powerpc/include/asm/qspinlock_types.h
@@ -29,7 +29,8 @@ typedef struct qspinlock {
* Bitfields in the lock word:
*
* 0: locked bit
- * 1-15: unused bits
+ * 1-14: lock holder cpu
+ * 15: unused bit
* 16: must queue bit
* 17-31: tail cpu (+1)
*/
@@ -40,6 +41,15 @@ typedef struct qspinlock {
#define _Q_LOCKED_BITS 1
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
+/* 0x00007ffe */
+#define _Q_OWNER_CPU_OFFSET 1
+#define _Q_OWNER_CPU_BITS 14
+#define _Q_OWNER_CPU_MASK _Q_SET_MASK(OWNER_CPU)
+
+#if CONFIG_NR_CPUS > (1U << _Q_OWNER_CPU_BITS)
+#error "qspinlock does not support such large CONFIG_NR_CPUS"
+#endif
+
/* 0x00010000 */
#define _Q_MUST_Q_OFFSET 16
#define _Q_MUST_Q_BITS 1
diff --git a/arch/powerpc/lib/qspinlock.c b/arch/powerpc/lib/qspinlock.c
index 979b17ac7bd1..a5b2c0377cf9 100644
--- a/arch/powerpc/lib/qspinlock.c
+++ b/arch/powerpc/lib/qspinlock.c
@@ -55,7 +55,7 @@ static inline int decode_tail_cpu(u32 val)
*/
static __always_inline u32 trylock_clean_tail(struct qspinlock *lock, u32 tail)
{
- u32 newval = _Q_LOCKED_VAL;
+ u32 newval = queued_spin_encode_locked_val();
u32 prev, tmp;
asm volatile(
--
2.37.2
