Author: glebius
Date: Wed Jan 22 05:53:16 2020
New Revision: 356975
URL: https://svnweb.freebsd.org/changeset/base/356975
Log:
Make tcp_output() require network epoch.
Enter the epoch before calling into tcp_output() from those
functions, that didn't do that before.
This eliminates a bunch of epoch recursions in TCP.
Modified:
head/sys/netinet/tcp_output.c
head/sys/netinet/tcp_stacks/bbr.c
head/sys/netinet/tcp_stacks/rack.c
head/sys/netinet/tcp_timer.c
head/sys/netinet/tcp_usrreq.c
head/sys/netinet/toecore.c
Modified: head/sys/netinet/tcp_output.c
==============================================================================
--- head/sys/netinet/tcp_output.c Wed Jan 22 05:51:22 2020
(r356974)
+++ head/sys/netinet/tcp_output.c Wed Jan 22 05:53:16 2020
(r356975)
@@ -193,7 +193,6 @@ cc_after_idle(struct tcpcb *tp)
int
tcp_output(struct tcpcb *tp)
{
- struct epoch_tracker et;
struct socket *so = tp->t_inpcb->inp_socket;
int32_t len;
uint32_t recwin, sendwin;
@@ -233,6 +232,7 @@ tcp_output(struct tcpcb *tp)
const bool hw_tls = false;
#endif
+ NET_EPOCH_ASSERT();
INP_WLOCK_ASSERT(tp->t_inpcb);
#ifdef TCP_OFFLOAD
@@ -1372,7 +1372,6 @@ send:
* m->m_pkthdr.len should have been set before checksum calculation,
* because in6_cksum() need it.
*/
- NET_EPOCH_ENTER(et);
#ifdef INET6
if (isipv6) {
/*
@@ -1458,7 +1457,6 @@ send:
mtu = tp->t_inpcb->inp_route.ro_rt->rt_mtu;
}
#endif /* INET */
- NET_EPOCH_EXIT(et);
out:
/*
Modified: head/sys/netinet/tcp_stacks/bbr.c
==============================================================================
--- head/sys/netinet/tcp_stacks/bbr.c Wed Jan 22 05:51:22 2020
(r356974)
+++ head/sys/netinet/tcp_stacks/bbr.c Wed Jan 22 05:53:16 2020
(r356975)
@@ -12091,7 +12091,6 @@ bbr_window_update_needed(struct tcpcb *tp, struct sock
static int
bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
{
- struct epoch_tracker et;
struct socket *so;
int32_t len;
uint32_t cts;
@@ -13938,7 +13937,6 @@ send:
* m->m_pkthdr.len should have been set before cksum calcuration,
* because in6_cksum() need it.
*/
- NET_EPOCH_ENTER(et);
#ifdef INET6
if (isipv6) {
/*
@@ -14016,7 +14014,6 @@ send:
mtu = inp->inp_route.ro_rt->rt_mtu;
}
#endif /* INET */
- NET_EPOCH_EXIT(et);
out:
if (lgb) {
@@ -14464,6 +14461,8 @@ bbr_output(struct tcpcb *tp)
int32_t ret;
struct timeval tv;
struct tcp_bbr *bbr;
+
+ NET_EPOCH_ASSERT();
bbr = (struct tcp_bbr *)tp->t_fb_ptr;
INP_WLOCK_ASSERT(tp->t_inpcb);
Modified: head/sys/netinet/tcp_stacks/rack.c
==============================================================================
--- head/sys/netinet/tcp_stacks/rack.c Wed Jan 22 05:51:22 2020
(r356974)
+++ head/sys/netinet/tcp_stacks/rack.c Wed Jan 22 05:53:16 2020
(r356975)
@@ -8091,7 +8091,6 @@ old_method:
static int
rack_output(struct tcpcb *tp)
{
- struct epoch_tracker et;
struct socket *so;
uint32_t recwin, sendwin;
uint32_t sb_offset;
@@ -8155,8 +8154,10 @@ rack_output(struct tcpcb *tp)
#ifdef KERN_TLS
hw_tls = (so->so_snd.sb_flags & SB_TLS_IFNET) != 0;
#endif
-
+
+ NET_EPOCH_ASSERT();
INP_WLOCK_ASSERT(inp);
+
#ifdef TCP_OFFLOAD
if (tp->t_flags & TF_TOE)
return (tcp_offload_output(tp));
@@ -9734,7 +9735,6 @@ send:
* m->m_pkthdr.len should have been set before cksum calcuration,
* because in6_cksum() need it.
*/
- NET_EPOCH_ENTER(et);
#ifdef INET6
if (isipv6) {
/*
@@ -9812,7 +9812,6 @@ send:
mtu = inp->inp_route.ro_rt->rt_mtu;
}
#endif /* INET */
- NET_EPOCH_EXIT(et);
out:
if (lgb) {
Modified: head/sys/netinet/tcp_timer.c
==============================================================================
--- head/sys/netinet/tcp_timer.c Wed Jan 22 05:51:22 2020
(r356974)
+++ head/sys/netinet/tcp_timer.c Wed Jan 22 05:53:16 2020
(r356975)
@@ -251,6 +251,7 @@ int tcp_totbackoff = 2559; /* sum of tcp_backoff[] */
void
tcp_timer_delack(void *xtp)
{
+ struct epoch_tracker et;
struct tcpcb *tp = xtp;
struct inpcb *inp;
CURVNET_SET(tp->t_vnet);
@@ -272,8 +273,10 @@ tcp_timer_delack(void *xtp)
}
tp->t_flags |= TF_ACKNOW;
TCPSTAT_INC(tcps_delack);
+ NET_EPOCH_ENTER(et);
(void) tp->t_fb->tfb_tcp_output(tp);
INP_WUNLOCK(inp);
+ NET_EPOCH_EXIT(et);
CURVNET_RESTORE();
}
@@ -570,7 +573,9 @@ tcp_timer_persist(void *xtp)
}
tcp_setpersist(tp);
tp->t_flags |= TF_FORCEDATA;
+ NET_EPOCH_ENTER(et);
(void) tp->t_fb->tfb_tcp_output(tp);
+ NET_EPOCH_EXIT(et);
tp->t_flags &= ~TF_FORCEDATA;
#ifdef TCPDEBUG
@@ -824,9 +829,9 @@ tcp_timer_rexmt(void * xtp)
tp->t_rtttime = 0;
cc_cong_signal(tp, NULL, CC_RTO);
-
+ NET_EPOCH_ENTER(et);
(void) tp->t_fb->tfb_tcp_output(tp);
-
+ NET_EPOCH_EXIT(et);
#ifdef TCPDEBUG
if (tp != NULL && (tp->t_inpcb->inp_socket->so_options & SO_DEBUG))
tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0,
Modified: head/sys/netinet/tcp_usrreq.c
==============================================================================
--- head/sys/netinet/tcp_usrreq.c Wed Jan 22 05:51:22 2020
(r356974)
+++ head/sys/netinet/tcp_usrreq.c Wed Jan 22 05:53:16 2020
(r356975)
@@ -531,6 +531,7 @@ out:
static int
tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
{
+ struct epoch_tracker et;
int error = 0;
struct inpcb *inp;
struct tcpcb *tp = NULL;
@@ -571,7 +572,9 @@ tcp_usr_connect(struct socket *so, struct sockaddr *na
goto out;
#endif
tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp));
+ NET_EPOCH_ENTER(et);
error = tp->t_fb->tfb_tcp_output(tp);
+ NET_EPOCH_EXIT(et);
out:
TCPDEBUG2(PRU_CONNECT);
TCP_PROBE2(debug__user, tp, PRU_CONNECT);
@@ -584,6 +587,7 @@ out:
static int
tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
{
+ struct epoch_tracker et;
int error = 0;
struct inpcb *inp;
struct tcpcb *tp = NULL;
@@ -654,7 +658,9 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *n
(error = tcp_offload_connect(so, nam)) == 0)
goto out;
#endif
+ NET_EPOCH_ENTER(et);
error = tp->t_fb->tfb_tcp_output(tp);
+ NET_EPOCH_EXIT(et);
goto out;
} else {
if ((inp->inp_vflag & INP_IPV6) == 0) {
@@ -677,8 +683,9 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *n
goto out;
#endif
tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp));
+ NET_EPOCH_ENTER(et);
error = tp->t_fb->tfb_tcp_output(tp);
-
+ NET_EPOCH_EXIT(et);
out:
/*
* If the implicit bind in the connect call fails, restore
@@ -882,6 +889,7 @@ out:
static int
tcp_usr_rcvd(struct socket *so, int flags)
{
+ struct epoch_tracker et;
struct inpcb *inp;
struct tcpcb *tp = NULL;
int error = 0;
@@ -911,8 +919,9 @@ tcp_usr_rcvd(struct socket *so, int flags)
tcp_offload_rcvd(tp);
else
#endif
+ NET_EPOCH_ENTER(et);
tp->t_fb->tfb_tcp_output(tp);
-
+ NET_EPOCH_EXIT(et);
out:
TCPDEBUG2(PRU_RCVD);
TCP_PROBE2(debug__user, tp, PRU_RCVD);
@@ -953,8 +962,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf
* We require the pcbinfo "read lock" if we will close the socket
* as part of this call.
*/
- if (flags & PRUS_EOF)
- NET_EPOCH_ENTER(et);
+ NET_EPOCH_ENTER(et);
inp = sotoinpcb(so);
KASSERT(inp != NULL, ("tcp_usr_send: inp == NULL"));
INP_WLOCK(inp);
@@ -1240,14 +1248,14 @@ out:
TCP_PROBE2(debug__user, tp, (flags & PRUS_OOB) ? PRU_SENDOOB :
((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND));
INP_WUNLOCK(inp);
- if (flags & PRUS_EOF)
- NET_EPOCH_EXIT(et);
+ NET_EPOCH_EXIT(et);
return (error);
}
static int
tcp_usr_ready(struct socket *so, struct mbuf *m, int count)
{
+ struct epoch_tracker et;
struct inpcb *inp;
struct tcpcb *tp;
int error;
@@ -1264,8 +1272,11 @@ tcp_usr_ready(struct socket *so, struct mbuf *m, int c
SOCKBUF_LOCK(&so->so_snd);
error = sbready(&so->so_snd, m, count);
SOCKBUF_UNLOCK(&so->so_snd);
- if (error == 0)
+ if (error == 0) {
+ NET_EPOCH_ENTER(et);
error = tp->t_fb->tfb_tcp_output(tp);
+ NET_EPOCH_EXIT(et);
+ }
INP_WUNLOCK(inp);
return (error);
@@ -1921,8 +1932,13 @@ unlock_and_done:
tp->t_flags |= TF_NOPUSH;
else if (tp->t_flags & TF_NOPUSH) {
tp->t_flags &= ~TF_NOPUSH;
- if (TCPS_HAVEESTABLISHED(tp->t_state))
+ if (TCPS_HAVEESTABLISHED(tp->t_state)) {
+ struct epoch_tracker et;
+
+ NET_EPOCH_ENTER(et);
error = tp->t_fb->tfb_tcp_output(tp);
+ NET_EPOCH_EXIT(et);
+ }
}
goto unlock_and_done;
Modified: head/sys/netinet/toecore.c
==============================================================================
--- head/sys/netinet/toecore.c Wed Jan 22 05:51:22 2020 (r356974)
+++ head/sys/netinet/toecore.c Wed Jan 22 05:53:16 2020 (r356975)
@@ -503,6 +503,7 @@ void
toe_connect_failed(struct toedev *tod, struct inpcb *inp, int err)
{
+ NET_EPOCH_ASSERT();
INP_WLOCK_ASSERT(inp);
if (!(inp->inp_flags & INP_DROPPED)) {
@@ -527,7 +528,6 @@ toe_connect_failed(struct toedev *tod, struct inpcb *i
(void) tp->t_fb->tfb_tcp_output(tp);
} else {
- NET_EPOCH_ASSERT();
tp = tcp_drop(tp, err);
if (tp == NULL)
INP_WLOCK(inp); /* re-acquire */
_______________________________________________
[email protected] mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "[email protected]"
