Module Name: src
Committed By: knakahara
Date: Thu Sep 19 04:59:42 UTC 2019
Modified Files:
src/sys/net: if_l2tp.c if_l2tp.h
Log Message:
Divide Tx context of l2tp(4) to improve performance.
It seems l2tp(4) call path is too long for instruction cache. So, dividing
l2tp(4) Tx context improves CPU use efficiency.
After this commit, l2tp(4) throughput gains 10% on my machine(Atom C3000).
To generate a diff of this commit:
cvs rdiff -u -r1.36 -r1.37 src/sys/net/if_l2tp.c
cvs rdiff -u -r1.6 -r1.7 src/sys/net/if_l2tp.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/net/if_l2tp.c
diff -u src/sys/net/if_l2tp.c:1.36 src/sys/net/if_l2tp.c:1.37
--- src/sys/net/if_l2tp.c:1.36 Mon Aug 19 03:24:05 2019
+++ src/sys/net/if_l2tp.c Thu Sep 19 04:59:42 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: if_l2tp.c,v 1.36 2019/08/19 03:24:05 ozaki-r Exp $ */
+/* $NetBSD: if_l2tp.c,v 1.37 2019/09/19 04:59:42 knakahara Exp $ */
/*
* Copyright (c) 2017 Internet Initiative Japan Inc.
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_l2tp.c,v 1.36 2019/08/19 03:24:05 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_l2tp.c,v 1.37 2019/09/19 04:59:42 knakahara Exp $");
#ifdef _KERNEL_OPT
#include "opt_inet.h"
@@ -118,15 +118,20 @@ struct psref_class *lv_psref_class __rea
static void l2tp_ro_init_pc(void *, void *, struct cpu_info *);
static void l2tp_ro_fini_pc(void *, void *, struct cpu_info *);
+static void l2tp_ifq_init_pc(void *, void *, struct cpu_info *);
+
static int l2tp_clone_create(struct if_clone *, int);
static int l2tp_clone_destroy(struct ifnet *);
struct if_clone l2tp_cloner =
IF_CLONE_INITIALIZER("l2tp", l2tp_clone_create, l2tp_clone_destroy);
+static int l2tp_tx_enqueue(struct l2tp_variant *, struct mbuf *);
static int l2tp_output(struct ifnet *, struct mbuf *,
const struct sockaddr *, const struct rtentry *);
+static void l2tp_sendit(struct l2tp_variant *, struct mbuf *);
static void l2tpintr(struct l2tp_variant *);
+static void l2tpintr_softint(void *);
static void l2tp_hash_init(void);
static int l2tp_hash_fini(void);
@@ -225,7 +230,10 @@ l2tp_clone_create(struct if_clone *ifc,
struct l2tp_softc *sc;
struct l2tp_variant *var;
int rv;
-
+ u_int si_flags = SOFTINT_NET;
+#ifdef NET_MPSAFE
+ si_flags |= SOFTINT_MPSAFE;
+#endif
sc = kmem_zalloc(sizeof(struct l2tp_softc), KM_SLEEP);
if_initname(&sc->l2tp_ec.ec_if, ifc->ifc_name, unit);
rv = l2tpattach0(sc);
@@ -248,6 +256,10 @@ l2tp_clone_create(struct if_clone *ifc,
sc->l2tp_ro_percpu = percpu_alloc(sizeof(struct l2tp_ro));
percpu_foreach(sc->l2tp_ro_percpu, l2tp_ro_init_pc, NULL);
+ sc->l2tp_ifq_percpu = percpu_alloc(sizeof(struct ifqueue));
+ percpu_foreach(sc->l2tp_ifq_percpu, l2tp_ifq_init_pc, NULL);
+ sc->l2tp_si = softint_establish(si_flags, l2tpintr_softint, sc);
+
mutex_enter(&l2tp_softcs.lock);
LIST_INSERT_HEAD(&l2tp_softcs.list, sc, l2tp_list);
mutex_exit(&l2tp_softcs.lock);
@@ -326,6 +338,15 @@ l2tp_ro_fini_pc(void *p, void *arg __unu
mutex_obj_free(lro->lr_lock);
}
+void
+l2tp_ifq_init_pc(void *p, void *arg __unused, struct cpu_info *ci __unused)
+{
+ struct ifqueue *ifq = p;
+
+ memset(ifq, 0, sizeof(*ifq));
+ ifq->ifq_maxlen = IFQ_MAXLEN;
+}
+
static int
l2tp_clone_destroy(struct ifnet *ifp)
{
@@ -336,13 +357,17 @@ l2tp_clone_destroy(struct ifnet *ifp)
l2tp_clear_session(sc);
l2tp_delete_tunnel(&sc->l2tp_ec.ec_if);
/*
- * To avoid for l2tp_transmit() to access sc->l2tp_var after free it.
+ * To avoid for l2tp_transmit() and l2tpintr_softint() to access
+ * sc->l2tp_var after free it.
*/
mutex_enter(&sc->l2tp_lock);
var = sc->l2tp_var;
l2tp_variant_update(sc, NULL);
mutex_exit(&sc->l2tp_lock);
+ softint_disestablish(sc->l2tp_si);
+ percpu_free(sc->l2tp_ifq_percpu, sizeof(struct ifqueue));
+
mutex_enter(&l2tp_softcs.lock);
LIST_REMOVE(sc, l2tp_list);
mutex_exit(&l2tp_softcs.lock);
@@ -363,6 +388,37 @@ l2tp_clone_destroy(struct ifnet *ifp)
}
static int
+l2tp_tx_enqueue(struct l2tp_variant *var, struct mbuf *m)
+{
+ struct l2tp_softc *sc;
+ struct ifnet *ifp;
+ struct ifqueue *ifq;
+ int s;
+
+ KASSERT(psref_held(&var->lv_psref, lv_psref_class));
+
+ sc = var->lv_softc;
+ ifp = &sc->l2tp_ec.ec_if;
+
+ s = splsoftnet();
+ ifq = percpu_getref(sc->l2tp_ifq_percpu);
+ if (IF_QFULL(ifq)) {
+ ifp->if_oerrors++;
+ percpu_putref(sc->l2tp_ifq_percpu);
+ splx(s);
+ m_freem(m);
+ return ENOBUFS;
+ }
+
+ IF_ENQUEUE(ifq, m);
+ percpu_putref(sc->l2tp_ifq_percpu);
+ softint_schedule(sc->l2tp_si);
+ /* counter is incremented in l2tpintr() */
+ splx(s);
+ return 0;
+}
+
+static int
l2tp_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
const struct rtentry *rt)
{
@@ -404,17 +460,7 @@ l2tp_output(struct ifnet *ifp, struct mb
}
*mtod(m, int *) = dst->sa_family;
- IFQ_ENQUEUE(&ifp->if_snd, m, error);
- if (error)
- goto end;
-
- /*
- * direct call to avoid infinite loop at l2tpintr()
- */
- l2tpintr(var);
-
- error = 0;
-
+ error = l2tp_tx_enqueue(var, m);
end:
l2tp_putref_variant(var, &psref);
if (error)
@@ -424,12 +470,54 @@ end:
}
static void
+l2tp_sendit(struct l2tp_variant *var, struct mbuf *m)
+{
+ int len;
+ int error;
+ struct l2tp_softc *sc;
+ struct ifnet *ifp;
+
+ KASSERT(psref_held(&var->lv_psref, lv_psref_class));
+
+ sc = var->lv_softc;
+ ifp = &sc->l2tp_ec.ec_if;
+
+ len = m->m_pkthdr.len;
+ m->m_flags &= ~(M_BCAST|M_MCAST);
+ bpf_mtap(ifp, m, BPF_D_OUT);
+
+ switch (var->lv_psrc->sa_family) {
+#ifdef INET
+ case AF_INET:
+ error = in_l2tp_output(var, m);
+ break;
+#endif
+#ifdef INET6
+ case AF_INET6:
+ error = in6_l2tp_output(var, m);
+ break;
+#endif
+ default:
+ m_freem(m);
+ error = ENETDOWN;
+ break;
+ }
+ if (error) {
+ ifp->if_oerrors++;
+ } else {
+ ifp->if_opackets++;
+ ifp->if_obytes += len;
+ }
+}
+
+static void
l2tpintr(struct l2tp_variant *var)
{
struct l2tp_softc *sc;
struct ifnet *ifp;
struct mbuf *m;
- int error;
+ struct ifqueue *ifq;
+ u_int cpuid = cpu_index(curcpu());
KASSERT(psref_held(&var->lv_psref, lv_psref_class));
@@ -438,45 +526,51 @@ l2tpintr(struct l2tp_variant *var)
/* output processing */
if (var->lv_my_sess_id == 0 || var->lv_peer_sess_id == 0) {
- IFQ_PURGE(&ifp->if_snd);
+ ifq = percpu_getref(sc->l2tp_ifq_percpu);
+ IF_PURGE(ifq);
+ percpu_putref(sc->l2tp_ifq_percpu);
+ if (cpuid == 0)
+ IFQ_PURGE(&ifp->if_snd);
return;
}
+ /* Currently, l2tpintr() is always called in softint context. */
+ ifq = percpu_getref(sc->l2tp_ifq_percpu);
for (;;) {
- int len;
-
- IFQ_DEQUEUE(&ifp->if_snd, m);
- if (m == NULL)
- break;
- len = m->m_pkthdr.len;
- m->m_flags &= ~(M_BCAST|M_MCAST);
- bpf_mtap(ifp, m, BPF_D_OUT);
- switch (var->lv_psrc->sa_family) {
-#ifdef INET
- case AF_INET:
- error = in_l2tp_output(var, m);
- break;
-#endif
-#ifdef INET6
- case AF_INET6:
- error = in6_l2tp_output(var, m);
- break;
-#endif
- default:
- m_freem(m);
- error = ENETDOWN;
+ IF_DEQUEUE(ifq, m);
+ if (m != NULL)
+ l2tp_sendit(var, m);
+ else
break;
- }
+ }
+ percpu_putref(sc->l2tp_ifq_percpu);
- if (error)
- ifp->if_oerrors++;
- else {
- ifp->if_opackets++;
- ifp->if_obytes += len;
+ if (cpuid == 0) {
+ for (;;) {
+ IFQ_DEQUEUE(&ifp->if_snd, m);
+ if (m != NULL)
+ l2tp_sendit(var, m);
+ else
+ break;
}
}
}
+static void
+l2tpintr_softint(void *arg)
+{
+ struct l2tp_variant *var;
+ struct psref psref;
+ struct l2tp_softc *sc = arg;
+
+ var = l2tp_getref_variant(sc, &psref);
+ if (var == NULL)
+ return;
+
+ l2tpintr(var);
+ l2tp_putref_variant(var, &psref);
+}
+
void
l2tp_input(struct mbuf *m, struct ifnet *ifp)
{
@@ -578,7 +672,7 @@ l2tp_start(struct ifnet *ifp)
if (var->lv_psrc == NULL || var->lv_pdst == NULL)
return;
- l2tpintr(var);
+ softint_schedule(sc->l2tp_si);
l2tp_putref_variant(var, &psref);
}
@@ -586,7 +680,6 @@ int
l2tp_transmit(struct ifnet *ifp, struct mbuf *m)
{
int error;
- int len;
struct psref psref;
struct l2tp_variant *var;
struct l2tp_softc *sc = container_of(ifp, struct l2tp_softc,
@@ -604,33 +697,9 @@ l2tp_transmit(struct ifnet *ifp, struct
goto out;
}
- len = m->m_pkthdr.len;
m->m_flags &= ~(M_BCAST|M_MCAST);
- bpf_mtap(ifp, m, BPF_D_OUT);
- switch (var->lv_psrc->sa_family) {
-#ifdef INET
- case AF_INET:
- error = in_l2tp_output(var, m);
- break;
-#endif
-#ifdef INET6
- case AF_INET6:
- error = in6_l2tp_output(var, m);
- break;
-#endif
- default:
- m_freem(m);
- error = ENETDOWN;
- break;
- }
-
- if (error)
- ifp->if_oerrors++;
- else {
- ifp->if_opackets++;
- ifp->if_obytes += len;
- }
+ error = l2tp_tx_enqueue(var, m);
out:
l2tp_putref_variant(var, &psref);
return error;
Index: src/sys/net/if_l2tp.h
diff -u src/sys/net/if_l2tp.h:1.6 src/sys/net/if_l2tp.h:1.7
--- src/sys/net/if_l2tp.h:1.6 Fri Oct 19 00:12:56 2018
+++ src/sys/net/if_l2tp.h Thu Sep 19 04:59:42 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: if_l2tp.h,v 1.6 2018/10/19 00:12:56 knakahara Exp $ */
+/* $NetBSD: if_l2tp.h,v 1.7 2019/09/19 04:59:42 knakahara Exp $ */
/*
* Copyright (c) 2017 Internet Initiative Japan Inc.
@@ -107,6 +107,9 @@ struct l2tp_softc {
kmutex_t l2tp_lock; /* writer lock for l2tp_var */
pserialize_t l2tp_psz;
+ void *l2tp_si;
+ percpu_t *l2tp_ifq_percpu;
+
LIST_ENTRY(l2tp_softc) l2tp_list; /* list of all l2tps */
struct pslist_entry l2tp_hash; /* hashed list to lookup by session id */
};
