From: Andi Kleen <a...@linux.intel.com>
Convert the rwsem elision to be adaptive. This requires adding
4/8 bytes to the rwsem for the adaptation state. The algorithm
is the same as for other adaptive lock types. The elision
configuration is split for readers and writers.
Signed-off-by: Andi Kleen <a...@linux.intel.com>
---
arch/x86/include/asm/rwsem.h | 22 +++++++++++++++++-----
arch/x86/kernel/rtm-locks.c | 10 ++++++++++
include/linux/rwsem.h | 3 +++
3 files changed, 30 insertions(+), 5 deletions(-)
diff --git a/arch/x86/include/asm/rwsem.h b/arch/x86/include/asm/rwsem.h
index f9297af..da6437c 100644
--- a/arch/x86/include/asm/rwsem.h
+++ b/arch/x86/include/asm/rwsem.h
@@ -39,6 +39,7 @@
#ifdef __KERNEL__
#include <asm/asm.h>
#include <linux/elide.h>
+#include <linux/jump_label.h>
/*
* The bias values and the counter type limits the number of
@@ -58,14 +59,18 @@
#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS +
RWSEM_ACTIVE_BIAS)
-extern bool rwsem_elision;
+extern struct static_key rwsem_elision;
+extern struct elision_config readsem_elision_config;
+extern struct elision_config writesem_elision_config;
/*
* lock for reading
*/
static inline void __down_read(struct rw_semaphore *sem)
{
- if (elide_lock(rwsem_elision, sem->count == 0))
+ if (elide_lock_adapt(rwsem_elision, sem->count == 0,
+ &sem->elision_adapt,
+ &readsem_elision_config))
return;
asm volatile("# beginning down_read\n\t"
LOCK_PREFIX _ASM_INC "(%1)\n\t"
@@ -86,7 +91,9 @@ static inline int __down_read_trylock(struct rw_semaphore
*sem)
{
long result, tmp;
- if (elide_lock(rwsem_elision, sem->count == 0))
+ if (elide_lock_adapt(rwsem_elision, sem->count == 0,
+ &sem->elision_adapt,
+ &readsem_elision_config))
return 1;
asm volatile("# beginning __down_read_trylock\n\t"
" mov %0,%1\n\t"
@@ -111,7 +118,9 @@ static inline void __down_write_nested(struct rw_semaphore
*sem, int subclass)
{
long tmp;
- if (elide_lock(rwsem_elision, sem->count == 0))
+ if (elide_lock_adapt(rwsem_elision, sem->count == 0,
+ &sem->elision_adapt,
+ &readsem_elision_config))
return;
asm volatile("# beginning down_write\n\t"
LOCK_PREFIX " xadd %1,(%2)\n\t"
@@ -139,7 +148,9 @@ static inline int __down_write_trylock(struct rw_semaphore
*sem)
{
long ret;
- if (elide_lock(rwsem_elision, sem->count == 0))
+ if (elide_lock_adapt(rwsem_elision, sem->count == 0,
+ &sem->elision_adapt,
+ &writesem_elision_config))
return 1;
ret = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
RWSEM_ACTIVE_WRITE_BIAS);
@@ -195,6 +206,7 @@ static inline void __up_write(struct rw_semaphore *sem)
*/
static inline void __downgrade_write(struct rw_semaphore *sem)
{
+ /* if (sem->count == 0) return; */
asm volatile("# beginning __downgrade_write\n\t"
LOCK_PREFIX _ASM_ADD "%2,(%1)\n\t"
/*
diff --git a/arch/x86/kernel/rtm-locks.c b/arch/x86/kernel/rtm-locks.c
index 40a0e7d..5a33a12 100644
--- a/arch/x86/kernel/rtm-locks.c
+++ b/arch/x86/kernel/rtm-locks.c
@@ -436,6 +436,15 @@ static unsigned rtm_patch(u8 type, u16 clobbers, void
*ibuf,
struct static_key mutex_elision = STATIC_KEY_INIT_FALSE;
module_param(mutex_elision, static_key, 0644);
+struct static_key rwsem_elision = STATIC_KEY_INIT_FALSE;
+module_param(rwsem_elision, static_key, 0644);
+__read_mostly struct elision_config readsem_elision_config =
+ DEFAULT_ELISION_CONFIG;
+TUNE_ELISION_CONFIG(readsem, readsem_elision_config);
+__read_mostly struct elision_config writesem_elision_config =
+ DEFAULT_ELISION_CONFIG;
+TUNE_ELISION_CONFIG(writesem, writesem_elision_config);
+
void init_rtm_spinlocks(void)
{
if (!boot_cpu_has(X86_FEATURE_RTM))
@@ -464,6 +473,7 @@ void init_rtm_spinlocks(void)
static_key_slow_inc(&rwlock_elision);
static_key_slow_inc(&mutex_elision);
+ static_key_slow_inc(&rwsem_elision);
bitlock_elision = true;
}
diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h
index 436e430..fb96c565 100644
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -28,6 +28,9 @@ struct rw_semaphore {
long count;
raw_spinlock_t wait_lock;
struct list_head wait_list;
+#ifdef CONFIG_RTM_LOCKS
+ short elision_adapt;
+#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
struct lockdep_map dep_map;
#endif
--
1.7.7.6
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
