From: Will Deacon <will.dea...@arm.com>
When contended, architectures may be able to reduce the polling overhead
in ways which aren't expressible using a simple relax() primitive.
This patch allows architectures to hook into the mcs_{lock,unlock}
functions for the contended cases only.
Signed-off-by: Will Deacon <will.dea...@arm.com>
Signed-off-by: Tim Chen <tim.c.c...@linux.intel.com>
---
kernel/locking/mcs_spinlock.c | 42 ++++++++++++++++++++++++++++--------------
1 file changed, 28 insertions(+), 14 deletions(-)
diff --git a/kernel/locking/mcs_spinlock.c b/kernel/locking/mcs_spinlock.c
index c3ee9cf..e12ed32 100644
--- a/kernel/locking/mcs_spinlock.c
+++ b/kernel/locking/mcs_spinlock.c
@@ -16,6 +16,28 @@
#include <linux/mutex.h>
#include <linux/export.h>
+#ifndef arch_mcs_spin_lock_contended
+/*
+ * Using smp_load_acquire() provides a memory barrier that ensures
+ * subsequent operations happen after the lock is acquired.
+ */
+#define arch_mcs_spin_lock_contended(l)
\
+do { \
+ while (!(smp_load_acquire(l))) \
+ arch_mutex_cpu_relax(); \
+} while (0)
+#endif
+
+#ifndef arch_mcs_spin_unlock_contended
+/*
+ * smp_store_release() provides a memory barrier to ensure all
+ * operations in the critical section has been completed before
+ * unlocking.
+ */
+#define arch_mcs_spin_unlock_contended(l) \
+ smp_store_release((l), 1)
+#endif
+
/*
* Note: the smp_load_acquire/smp_store_release pair is not
* sufficient to form a full memory barrier across
@@ -50,13 +72,9 @@ void mcs_spin_lock(struct mcs_spinlock **lock, struct
mcs_spinlock *node)
return;
}
ACCESS_ONCE(prev->next) = node;
- /*
- * Wait until the lock holder passes the lock down.
- * Using smp_load_acquire() provides a memory barrier that
- * ensures subsequent operations happen after the lock is acquired.
- */
- while (!(smp_load_acquire(&node->locked)))
- arch_mutex_cpu_relax();
+
+ /* Wait until the lock holder passes the lock down. */
+ arch_mcs_spin_lock_contended(&node->locked);
}
EXPORT_SYMBOL_GPL(mcs_spin_lock);
@@ -78,12 +96,8 @@ void mcs_spin_unlock(struct mcs_spinlock **lock, struct
mcs_spinlock *node)
while (!(next = ACCESS_ONCE(node->next)))
arch_mutex_cpu_relax();
}
- /*
- * Pass lock to next waiter.
- * smp_store_release() provides a memory barrier to ensure
- * all operations in the critical section has been completed
- * before unlocking.
- */
- smp_store_release(&next->locked, 1);
+
+ /* Pass lock to next waiter. */
+ arch_mcs_spin_unlock_contended(&next->locked);
}
EXPORT_SYMBOL_GPL(mcs_spin_unlock);
--
1.7.11.7
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
