Author: jtl
Date: Fri Apr 20 14:47:02 2018
New Revision: 332821
URL: https://svnweb.freebsd.org/changeset/base/332821
Log:
MFC r331309:
If the INP lock is uncontested, avoid taking a reference and jumping
through the lock-switching hoops.
A few of the INP lookup operations that lock INPs after the lookup do
so using this mechanism (to maintain lock ordering):
1. Lock lookup structure.
2. Find INP.
3. Acquire reference on INP.
4. Drop lock on lookup structure.
5. Acquire INP lock.
6. Drop reference on INP.
This change provides a slightly shorter path for cases where the INP
lock is uncontested:
1. Lock lookup structure.
2. Find INP.
3. Try to acquire the INP lock.
4. If successful, drop lock on lookup structure.
Of course, if the INP lock is contested, the functions will need to
revert to the previous way of switching locks safely.
This saves a few atomic operations when the INP lock is uncontested.
Sponsored by: Netflix, Inc.
Modified:
stable/11/sys/netinet/in_pcb.c
stable/11/sys/netinet6/in6_pcb.c
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/sys/netinet/in_pcb.c
==============================================================================
--- stable/11/sys/netinet/in_pcb.c Fri Apr 20 14:34:04 2018
(r332820)
+++ stable/11/sys/netinet/in_pcb.c Fri Apr 20 14:47:02 2018
(r332821)
@@ -1619,6 +1619,7 @@ in_pcblookup_group(struct inpcbinfo *pcbinfo, struct i
struct inpcbhead *head;
struct inpcb *inp, *tmpinp;
u_short fport = fport_arg, lport = lport_arg;
+ bool locked;
/*
* First look for an exact match.
@@ -1805,18 +1806,32 @@ in_pcblookup_group(struct inpcbinfo *pcbinfo, struct i
return (NULL);
found:
- in_pcbref(inp);
- INP_GROUP_UNLOCK(pcbgroup);
- if (lookupflags & INPLOOKUP_WLOCKPCB) {
- INP_WLOCK(inp);
- if (in_pcbrele_wlocked(inp))
- return (NULL);
- } else if (lookupflags & INPLOOKUP_RLOCKPCB) {
- INP_RLOCK(inp);
- if (in_pcbrele_rlocked(inp))
- return (NULL);
- } else
+ if (lookupflags & INPLOOKUP_WLOCKPCB)
+ locked = TRY_INP_WLOCK(inp);
+ else if (lookupflags & INPLOOKUP_RLOCKPCB)
+ locked = TRY_INP_RLOCK(inp);
+ else
panic("%s: locking bug", __func__);
+ if (!locked)
+ in_pcbref(inp);
+ INP_GROUP_UNLOCK(pcbgroup);
+ if (!locked) {
+ if (lookupflags & INPLOOKUP_WLOCKPCB) {
+ INP_WLOCK(inp);
+ if (in_pcbrele_wlocked(inp))
+ return (NULL);
+ } else {
+ INP_RLOCK(inp);
+ if (in_pcbrele_rlocked(inp))
+ return (NULL);
+ }
+ }
+#ifdef INVARIANTS
+ if (lookupflags & INPLOOKUP_WLOCKPCB)
+ INP_WLOCK_ASSERT(inp);
+ else
+ INP_RLOCK_ASSERT(inp);
+#endif
return (inp);
}
#endif /* PCBGROUP */
@@ -1955,23 +1970,38 @@ in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in
struct ifnet *ifp)
{
struct inpcb *inp;
+ bool locked;
INP_HASH_RLOCK(pcbinfo);
inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
(lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
if (inp != NULL) {
- in_pcbref(inp);
- INP_HASH_RUNLOCK(pcbinfo);
- if (lookupflags & INPLOOKUP_WLOCKPCB) {
- INP_WLOCK(inp);
- if (in_pcbrele_wlocked(inp))
- return (NULL);
- } else if (lookupflags & INPLOOKUP_RLOCKPCB) {
- INP_RLOCK(inp);
- if (in_pcbrele_rlocked(inp))
- return (NULL);
- } else
+ if (lookupflags & INPLOOKUP_WLOCKPCB)
+ locked = INP_TRY_WLOCK(inp);
+ else if (lookupflags & INPLOOKUP_RLOCKPCB)
+ locked = INP_TRY_RLOCK(inp);
+ else
panic("%s: locking bug", __func__);
+ if (!locked)
+ in_pcbref(inp);
+ INP_HASH_RUNLOCK(pcbinfo);
+ if (!locked) {
+ if (lookupflags & INPLOOKUP_WLOCKPCB) {
+ INP_WLOCK(inp);
+ if (in_pcbrele_wlocked(inp))
+ return (NULL);
+ } else {
+ INP_RLOCK(inp);
+ if (in_pcbrele_rlocked(inp))
+ return (NULL);
+ }
+ }
+#ifdef INVARIANTS
+ if (lookupflags & INPLOOKUP_WLOCKPCB)
+ INP_WLOCK_ASSERT(inp);
+ else
+ INP_RLOCK_ASSERT(inp);
+#endif
} else
INP_HASH_RUNLOCK(pcbinfo);
return (inp);
Modified: stable/11/sys/netinet6/in6_pcb.c
==============================================================================
--- stable/11/sys/netinet6/in6_pcb.c Fri Apr 20 14:34:04 2018
(r332820)
+++ stable/11/sys/netinet6/in6_pcb.c Fri Apr 20 14:47:02 2018
(r332821)
@@ -866,6 +866,7 @@ in6_pcblookup_group(struct inpcbinfo *pcbinfo, struct
struct inpcbhead *head;
struct inpcb *inp, *tmpinp;
u_short fport = fport_arg, lport = lport_arg;
+ bool locked;
/*
* First look for an exact match.
@@ -1024,18 +1025,32 @@ in6_pcblookup_group(struct inpcbinfo *pcbinfo, struct
return (NULL);
found:
- in_pcbref(inp);
- INP_GROUP_UNLOCK(pcbgroup);
- if (lookupflags & INPLOOKUP_WLOCKPCB) {
- INP_WLOCK(inp);
- if (in_pcbrele_wlocked(inp))
- return (NULL);
- } else if (lookupflags & INPLOOKUP_RLOCKPCB) {
- INP_RLOCK(inp);
- if (in_pcbrele_rlocked(inp))
- return (NULL);
- } else
+ if (lookupflags & INPLOOKUP_WLOCKPCB)
+ locked = INP_TRY_WLOCK(inp);
+ else if (lookupflags & INPLOOKUP_RLOCKPCB)
+ locked = INP_TRY_RLOCK(inp);
+ else
panic("%s: locking buf", __func__);
+ if (!locked)
+ in_pcbref(inp);
+ INP_GROUP_UNLOCK(pcbgroup);
+ if (!locked) {
+ if (lookupflags & INPLOOKUP_WLOCKPCB) {
+ INP_WLOCK(inp);
+ if (in_pcbrele_wlocked(inp))
+ return (NULL);
+ } else {
+ INP_RLOCK(inp);
+ if (in_pcbrele_rlocked(inp))
+ return (NULL);
+ }
+ }
+#ifdef INVARIANTS
+ if (lookupflags & INPLOOKUP_WLOCKPCB)
+ INP_WLOCK_ASSERT(inp);
+ else
+ INP_RLOCK_ASSERT(inp);
+#endif
return (inp);
}
#endif /* PCBGROUP */
@@ -1161,23 +1176,38 @@ in6_pcblookup_hash(struct inpcbinfo *pcbinfo, struct i
struct ifnet *ifp)
{
struct inpcb *inp;
+ bool locked;
INP_HASH_RLOCK(pcbinfo);
inp = in6_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
(lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
if (inp != NULL) {
- in_pcbref(inp);
- INP_HASH_RUNLOCK(pcbinfo);
- if (lookupflags & INPLOOKUP_WLOCKPCB) {
- INP_WLOCK(inp);
- if (in_pcbrele_wlocked(inp))
- return (NULL);
- } else if (lookupflags & INPLOOKUP_RLOCKPCB) {
- INP_RLOCK(inp);
- if (in_pcbrele_rlocked(inp))
- return (NULL);
- } else
+ if (lookupflags & INPLOOKUP_WLOCKPCB)
+ locked = INP_TRY_WLOCK(inp);
+ else if (lookupflags & INPLOOKUP_RLOCKPCB)
+ locked = INP_TRY_RLOCK(inp);
+ else
panic("%s: locking bug", __func__);
+ if (!locked)
+ in_pcbref(inp);
+ INP_HASH_RUNLOCK(pcbinfo);
+ if (!locked) {
+ if (lookupflags & INPLOOKUP_WLOCKPCB) {
+ INP_WLOCK(inp);
+ if (in_pcbrele_wlocked(inp))
+ return (NULL);
+ } else {
+ INP_RLOCK(inp);
+ if (in_pcbrele_rlocked(inp))
+ return (NULL);
+ }
+ }
+#ifdef INVARIANTS
+ if (lookupflags & INPLOOKUP_WLOCKPCB)
+ INP_WLOCK_ASSERT(inp);
+ else
+ INP_RLOCK_ASSERT(inp);
+#endif
} else
INP_HASH_RUNLOCK(pcbinfo);
return (inp);
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
