svn commit: r332394 - head/sys/kern

[Mateusz Guzik]

Author: mjg
Date: Tue Apr 10 22:32:31 2018
New Revision: 332394
URL: https://svnweb.freebsd.org/changeset/base/332394

Log:
  rw: whack avoidable re-reads in try_upgrade

Modified:
  head/sys/kern/kern_rwlock.c

Modified: head/sys/kern/kern_rwlock.c
==============================================================================
--- head/sys/kern/kern_rwlock.c Tue Apr 10 21:14:54 2018        (r332393)
+++ head/sys/kern/kern_rwlock.c Tue Apr 10 22:32:31 2018        (r332394)
@@ -1204,7 +1204,7 @@ __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t v L
 int
 __rw_try_upgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
 {
-       uintptr_t v, x, tid;
+       uintptr_t v, setv, tid;
        struct turnstile *ts;
        int success;
 
@@ -1224,12 +1224,12 @@ __rw_try_upgrade_int(struct rwlock *rw LOCK_FILE_LINE_
         */
        tid = (uintptr_t)curthread;
        success = 0;
+       v = RW_READ_VALUE(rw);
        for (;;) {
-               v = rw->rw_lock;
                if (RW_READERS(v) > 1)
                        break;
                if (!(v & RW_LOCK_WAITERS)) {
-                       success = atomic_cmpset_acq_ptr(&rw->rw_lock, v, tid);
+                       success = atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid);
                        if (!success)
                                continue;
                        break;
@@ -1239,7 +1239,8 @@ __rw_try_upgrade_int(struct rwlock *rw LOCK_FILE_LINE_
                 * Ok, we think we have waiters, so lock the turnstile.
                 */
                ts = turnstile_trywait(&rw->lock_object);
-               v = rw->rw_lock;
+               v = RW_READ_VALUE(rw);
+retry_ts:
                if (RW_READERS(v) > 1) {
                        turnstile_cancel(ts);
                        break;
@@ -1250,16 +1251,16 @@ __rw_try_upgrade_int(struct rwlock *rw LOCK_FILE_LINE_
                 * If we obtain the lock with the flags set, then claim
                 * ownership of the turnstile.
                 */
-               x = rw->rw_lock & RW_LOCK_WAITERS;
-               success = atomic_cmpset_ptr(&rw->rw_lock, v, tid | x);
+               setv = tid | (v & RW_LOCK_WAITERS);
+               success = atomic_fcmpset_ptr(&rw->rw_lock, &v, setv);
                if (success) {
-                       if (x)
+                       if (v & RW_LOCK_WAITERS)
                                turnstile_claim(ts);
                        else
                                turnstile_cancel(ts);
                        break;
                }
-               turnstile_cancel(ts);
+               goto retry_ts;
        }
        LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line);
        if (success) {
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"