Module Name: src
Committed By: ozaki-r
Date: Thu Sep 19 04:08:30 UTC 2019
Modified Files:
src/sys/net: route.c route.h
src/sys/netinet: ip_input.c
src/sys/netinet6: ip6_forward.c ip6_input.c
src/sys/netipsec: ipsec_output.c
Log Message:
Avoid having a rtcache directly in a percpu storage
percpu(9) has a certain memory storage for each CPU and provides it by the piece
to users. If the storages went short, percpu(9) enlarges them by allocating new
larger memory areas, replacing old ones with them and destroying the old ones.
A percpu storage referenced by a pointer gotten via percpu_getref can be
destroyed by the mechanism after a running thread sleeps even if percpu_putref
has not been called.
Using rtcache, i.e., packet processing, typically involves sleepable operations
such as rwlock so we must avoid dereferencing a rtcache that is directly stored
in a percpu storage during packet processing. Address this situation by having
just a pointer to a rtcache in a percpu storage instead.
Reviewed by knakahara@ and yamaguchi@
To generate a diff of this commit:
cvs rdiff -u -r1.219 -r1.220 src/sys/net/route.c
cvs rdiff -u -r1.124 -r1.125 src/sys/net/route.h
cvs rdiff -u -r1.390 -r1.391 src/sys/netinet/ip_input.c
cvs rdiff -u -r1.96 -r1.97 src/sys/netinet6/ip6_forward.c
cvs rdiff -u -r1.209 -r1.210 src/sys/netinet6/ip6_input.c
cvs rdiff -u -r1.82 -r1.83 src/sys/netipsec/ipsec_output.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/net/route.c
diff -u src/sys/net/route.c:1.219 src/sys/net/route.c:1.220
--- src/sys/net/route.c:1.219 Fri May 17 03:34:26 2019
+++ src/sys/net/route.c Thu Sep 19 04:08:29 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: route.c,v 1.219 2019/05/17 03:34:26 ozaki-r Exp $ */
+/* $NetBSD: route.c,v 1.220 2019/09/19 04:08:29 ozaki-r Exp $ */
/*-
* Copyright (c) 1998, 2008 The NetBSD Foundation, Inc.
@@ -97,7 +97,7 @@
#endif
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: route.c,v 1.219 2019/05/17 03:34:26 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: route.c,v 1.220 2019/09/19 04:08:29 ozaki-r Exp $");
#include <sys/param.h>
#ifdef RTFLUSH_DEBUG
@@ -2217,6 +2217,29 @@ rtcache_setdst(struct route *ro, const s
return 0;
}
+static void
+rtcache_percpu_init_cpu(void *p, void *arg __unused, struct cpu_info *ci __unused)
+{
+ struct route **rop = p;
+
+ /*
+ * We can't have struct route as percpu data because it can be destroyed
+ * over a memory enlargement processing of percpu.
+ */
+ *rop = kmem_zalloc(sizeof(**rop), KM_SLEEP);
+}
+
+percpu_t *
+rtcache_percpu_alloc(void)
+{
+ percpu_t *pc;
+
+ pc = percpu_alloc(sizeof(struct route *));
+ percpu_foreach(pc, rtcache_percpu_init_cpu, NULL);
+
+ return pc;
+}
+
const struct sockaddr *
rt_settag(struct rtentry *rt, const struct sockaddr *tag)
{
Index: src/sys/net/route.h
diff -u src/sys/net/route.h:1.124 src/sys/net/route.h:1.125
--- src/sys/net/route.h:1.124 Thu Aug 22 21:14:45 2019
+++ src/sys/net/route.h Thu Sep 19 04:08:29 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: route.h,v 1.124 2019/08/22 21:14:45 roy Exp $ */
+/* $NetBSD: route.h,v 1.125 2019/09/19 04:08:29 ozaki-r Exp $ */
/*
* Copyright (c) 1980, 1986, 1993
@@ -42,6 +42,7 @@
#include <sys/rwlock.h>
#include <sys/condvar.h>
#include <sys/pserialize.h>
+#include <sys/percpu.h>
#endif
#include <sys/psref.h>
@@ -507,6 +508,24 @@ struct rtentry *
void rtcache_unref(struct rtentry *, struct route *);
+percpu_t *
+ rtcache_percpu_alloc(void);
+
+static inline struct route *
+rtcache_percpu_getref(percpu_t *pc)
+{
+
+ return *(struct route **)percpu_getref(pc);
+}
+
+static inline void
+rtcache_percpu_putref(percpu_t *pc)
+{
+
+ percpu_putref(pc);
+}
+
+
/* rtsock */
void rt_ieee80211msg(struct ifnet *, int, void *, size_t);
void rt_ifannouncemsg(struct ifnet *, int);
Index: src/sys/netinet/ip_input.c
diff -u src/sys/netinet/ip_input.c:1.390 src/sys/netinet/ip_input.c:1.391
--- src/sys/netinet/ip_input.c:1.390 Sun Sep 15 21:00:15 2019
+++ src/sys/netinet/ip_input.c Thu Sep 19 04:08:29 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: ip_input.c,v 1.390 2019/09/15 21:00:15 bouyer Exp $ */
+/* $NetBSD: ip_input.c,v 1.391 2019/09/19 04:08:29 ozaki-r Exp $ */
/*
* Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -91,7 +91,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.390 2019/09/15 21:00:15 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.391 2019/09/19 04:08:29 ozaki-r Exp $");
#ifdef _KERNEL_OPT
#include "opt_inet.h"
@@ -308,7 +308,7 @@ ip_init(void)
#endif
ipstat_percpu = percpu_alloc(sizeof(uint64_t) * IP_NSTATS);
- ipforward_rt_percpu = percpu_alloc(sizeof(struct route));
+ ipforward_rt_percpu = rtcache_percpu_alloc();
ip_mtudisc_timeout_q = rt_timer_queue_create(ip_mtudisc_timeout);
}
@@ -1184,16 +1184,16 @@ ip_rtaddr(struct in_addr dst, struct psr
sockaddr_in_init(&u.dst4, &dst, 0);
- ro = percpu_getref(ipforward_rt_percpu);
+ ro = rtcache_percpu_getref(ipforward_rt_percpu);
rt = rtcache_lookup(ro, &u.dst);
if (rt == NULL) {
- percpu_putref(ipforward_rt_percpu);
+ rtcache_percpu_putref(ipforward_rt_percpu);
return NULL;
}
ia4_acquire(ifatoia(rt->rt_ifa), psref);
rtcache_unref(rt, ro);
- percpu_putref(ipforward_rt_percpu);
+ rtcache_percpu_putref(ipforward_rt_percpu);
return ifatoia(rt->rt_ifa);
}
@@ -1372,7 +1372,7 @@ ip_forward(struct mbuf *m, int srcrt, st
ro = percpu_getref(ipforward_rt_percpu);
rt = rtcache_lookup(ro, &u.dst);
if (rt == NULL) {
- percpu_putref(ipforward_rt_percpu);
+ rtcache_percpu_putref(ipforward_rt_percpu);
icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, dest, 0);
return;
}
@@ -1444,13 +1444,13 @@ ip_forward(struct mbuf *m, int srcrt, st
m_freem(mcopy);
}
- percpu_putref(ipforward_rt_percpu);
+ rtcache_percpu_putref(ipforward_rt_percpu);
return;
redirect:
error:
if (mcopy == NULL) {
- percpu_putref(ipforward_rt_percpu);
+ rtcache_percpu_putref(ipforward_rt_percpu);
return;
}
@@ -1493,7 +1493,7 @@ error:
*/
if (mcopy)
m_freem(mcopy);
- percpu_putref(ipforward_rt_percpu);
+ rtcache_percpu_putref(ipforward_rt_percpu);
return;
}
icmp_error(mcopy, type, code, dest, destmtu);
Index: src/sys/netinet6/ip6_forward.c
diff -u src/sys/netinet6/ip6_forward.c:1.96 src/sys/netinet6/ip6_forward.c:1.97
--- src/sys/netinet6/ip6_forward.c:1.96 Mon May 13 07:47:59 2019
+++ src/sys/netinet6/ip6_forward.c Thu Sep 19 04:08:29 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: ip6_forward.c,v 1.96 2019/05/13 07:47:59 ozaki-r Exp $ */
+/* $NetBSD: ip6_forward.c,v 1.97 2019/09/19 04:08:29 ozaki-r Exp $ */
/* $KAME: ip6_forward.c,v 1.109 2002/09/11 08:10:17 sakane Exp $ */
/*
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.96 2019/05/13 07:47:59 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.97 2019/09/19 04:08:29 ozaki-r Exp $");
#ifdef _KERNEL_OPT
#include "opt_gateway.h"
@@ -197,7 +197,7 @@ ip6_forward(struct mbuf *m, int srcrt)
}
#endif
- ro = percpu_getref(ip6_forward_rt_percpu);
+ ro = rtcache_percpu_getref(ip6_forward_rt_percpu);
if (srcrt) {
union {
struct sockaddr dst;
@@ -470,7 +470,7 @@ out:
#endif
rtcache_unref(rt, ro);
if (ro != NULL)
- percpu_putref(ip6_forward_rt_percpu);
+ rtcache_percpu_putref(ip6_forward_rt_percpu);
if (rcvif != NULL)
m_put_rcvif_psref(rcvif, &psref);
return;
Index: src/sys/netinet6/ip6_input.c
diff -u src/sys/netinet6/ip6_input.c:1.209 src/sys/netinet6/ip6_input.c:1.210
--- src/sys/netinet6/ip6_input.c:1.209 Sun Sep 15 21:00:15 2019
+++ src/sys/netinet6/ip6_input.c Thu Sep 19 04:08:29 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: ip6_input.c,v 1.209 2019/09/15 21:00:15 bouyer Exp $ */
+/* $NetBSD: ip6_input.c,v 1.210 2019/09/19 04:08:29 ozaki-r Exp $ */
/* $KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $ */
/*
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.209 2019/09/15 21:00:15 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.210 2019/09/19 04:08:29 ozaki-r Exp $");
#ifdef _KERNEL_OPT
#include "opt_gateway.h"
@@ -194,7 +194,7 @@ ip6_init(void)
KASSERT(inet6_pfil_hook != NULL);
ip6stat_percpu = percpu_alloc(sizeof(uint64_t) * IP6_NSTATS);
- ip6_forward_rt_percpu = percpu_alloc(sizeof(struct route));
+ ip6_forward_rt_percpu = rtcache_percpu_alloc();
}
static void
@@ -591,7 +591,7 @@ hbhcheck:
ICMP6_PARAMPROB_HEADER,
(char *)&ip6->ip6_plen - (char *)ip6);
rtcache_unref(rt, ro);
- percpu_putref(ip6_forward_rt_percpu);
+ rtcache_percpu_putref(ip6_forward_rt_percpu);
return;
}
IP6_EXTHDR_GET(hbh, struct ip6_hbh *, m, sizeof(struct ip6_hdr),
@@ -599,7 +599,7 @@ hbhcheck:
if (hbh == NULL) {
IP6_STATINC(IP6_STAT_TOOSHORT);
rtcache_unref(rt, ro);
- percpu_putref(ip6_forward_rt_percpu);
+ rtcache_percpu_putref(ip6_forward_rt_percpu);
return;
}
KASSERT(IP6_HDR_ALIGNED_P(hbh));
@@ -653,7 +653,7 @@ hbhcheck:
if (error != 0) {
rtcache_unref(rt, ro);
- percpu_putref(ip6_forward_rt_percpu);
+ rtcache_percpu_putref(ip6_forward_rt_percpu);
IP6_STATINC(IP6_STAT_CANTFORWARD);
goto bad;
}
@@ -662,7 +662,7 @@ hbhcheck:
goto bad_unref;
} else if (!ours) {
rtcache_unref(rt, ro);
- percpu_putref(ip6_forward_rt_percpu);
+ rtcache_percpu_putref(ip6_forward_rt_percpu);
ip6_forward(m, srcrt);
return;
}
@@ -703,7 +703,7 @@ hbhcheck:
rtcache_unref(rt, ro);
rt = NULL;
}
- percpu_putref(ip6_forward_rt_percpu);
+ rtcache_percpu_putref(ip6_forward_rt_percpu);
rh_present = 0;
frg_present = 0;
Index: src/sys/netipsec/ipsec_output.c
diff -u src/sys/netipsec/ipsec_output.c:1.82 src/sys/netipsec/ipsec_output.c:1.83
--- src/sys/netipsec/ipsec_output.c:1.82 Wed Dec 26 08:58:51 2018
+++ src/sys/netipsec/ipsec_output.c Thu Sep 19 04:08:30 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: ipsec_output.c,v 1.82 2018/12/26 08:58:51 knakahara Exp $ */
+/* $NetBSD: ipsec_output.c,v 1.83 2019/09/19 04:08:30 ozaki-r Exp $ */
/*
* Copyright (c) 2002, 2003 Sam Leffler, Errno Consulting
@@ -29,7 +29,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipsec_output.c,v 1.82 2018/12/26 08:58:51 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipsec_output.c,v 1.83 2019/09/19 04:08:30 ozaki-r Exp $");
#if defined(_KERNEL_OPT)
#include "opt_inet.h"
@@ -113,7 +113,7 @@ ipsec_reinject_ipstack(struct mbuf *m, i
KASSERT(af == AF_INET || af == AF_INET6);
KERNEL_LOCK_UNLESS_NET_MPSAFE();
- ro = percpu_getref(ipsec_rtcache_percpu);
+ ro = rtcache_percpu_getref(ipsec_rtcache_percpu);
switch (af) {
#ifdef INET
case AF_INET:
@@ -131,7 +131,7 @@ ipsec_reinject_ipstack(struct mbuf *m, i
break;
#endif
}
- percpu_putref(ipsec_rtcache_percpu);
+ rtcache_percpu_putref(ipsec_rtcache_percpu);
KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
return rv;
@@ -840,5 +840,5 @@ void
ipsec_output_init(void)
{
- ipsec_rtcache_percpu = percpu_alloc(sizeof(struct route));
+ ipsec_rtcache_percpu = rtcache_percpu_alloc();
}
