Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys Make ipforward_rt and ip6_forward_rt percpu
details: https://anonhg.NetBSD.org/src/rev/515256196e28
branches: trunk
changeset: 817664:515256196e28
user: ozaki-r <ozaki-r%NetBSD.org@localhost>
date: Wed Aug 31 09:14:47 2016 +0000
description:
Make ipforward_rt and ip6_forward_rt percpu
Sharing one rtcache between CPUs is just a bad idea.
Reviewed by knakahara@
diffstat:
sys/netinet/ip_input.c | 30 ++++++++++++++++++++++--------
sys/netinet6/ip6_forward.c | 32 +++++++++++++++++++++-----------
sys/netinet6/ip6_input.c | 18 +++++++++++++-----
3 files changed, 56 insertions(+), 24 deletions(-)
diffs (truncated from 320 to 300 lines):
diff -r 6806faf520ab -r 515256196e28 sys/netinet/ip_input.c
--- a/sys/netinet/ip_input.c Wed Aug 31 09:03:59 2016 +0000
+++ b/sys/netinet/ip_input.c Wed Aug 31 09:14:47 2016 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ip_input.c,v 1.339 2016/08/01 03:15:30 ozaki-r Exp $ */
+/* $NetBSD: ip_input.c,v 1.340 2016/08/31 09:14:47 ozaki-r Exp $ */
/*
* Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -91,7 +91,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.339 2016/08/01 03:15:30 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.340 2016/08/31 09:14:47 ozaki-r Exp $");
#ifdef _KERNEL_OPT
#include "opt_inet.h"
@@ -225,7 +225,7 @@
ipid_state_t * ip_ids __read_mostly;
percpu_t * ipstat_percpu __read_mostly;
-static struct route ipforward_rt __cacheline_aligned;
+static percpu_t *ipforward_rt_percpu __cacheline_aligned;
uint16_t ip_id;
@@ -345,6 +345,10 @@
#endif /* MBUFTRACE */
ipstat_percpu = percpu_alloc(sizeof(uint64_t) * IP_NSTATS);
+
+ ipforward_rt_percpu = percpu_alloc(sizeof(struct route));
+ if (ipforward_rt_percpu == NULL)
+ panic("failed to allocate ipforward_rt_percpu");
}
static struct in_ifaddr *
@@ -1163,11 +1167,14 @@
struct sockaddr dst;
struct sockaddr_in dst4;
} u;
+ struct route *ro;
sockaddr_in_init(&u.dst4, &dst, 0);
SOFTNET_LOCK();
- rt = rtcache_lookup(&ipforward_rt, &u.dst);
+ ro = percpu_getref(ipforward_rt_percpu);
+ rt = rtcache_lookup(ro, &u.dst);
+ percpu_putref(ipforward_rt_percpu);
SOFTNET_UNLOCK();
if (rt == NULL)
return NULL;
@@ -1300,6 +1307,7 @@
struct sockaddr_in dst4;
} u;
uint64_t *ips;
+ struct route *ro;
KASSERTMSG(cpu_softintr_p(), "ip_forward: not in the software "
"interrupt handler; synchronization assumptions violated");
@@ -1331,7 +1339,9 @@
sockaddr_in_init(&u.dst4, &ip->ip_dst, 0);
- if ((rt = rtcache_lookup(&ipforward_rt, &u.dst)) == NULL) {
+ ro = percpu_getref(ipforward_rt_percpu);
+ if ((rt = rtcache_lookup(ro, &u.dst)) == NULL) {
+ percpu_putref(ipforward_rt_percpu);
icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, dest, 0);
SOFTNET_UNLOCK();
return;
@@ -1376,7 +1386,7 @@
}
}
- error = ip_output(m, NULL, &ipforward_rt,
+ error = ip_output(m, NULL, ro,
(IP_FORWARDING | (ip_directedbcast ? IP_ALLOWBROADCAST : 0)),
NULL, NULL);
@@ -1398,17 +1408,19 @@
if (mcopy) {
#ifdef GATEWAY
if (mcopy->m_flags & M_CANFASTFWD)
- ipflow_create(&ipforward_rt, mcopy);
+ ipflow_create(ro, mcopy);
#endif
m_freem(mcopy);
}
+ percpu_putref(ipforward_rt_percpu);
SOFTNET_UNLOCK();
return;
redirect:
error:
if (mcopy == NULL) {
+ percpu_putref(ipforward_rt_percpu);
SOFTNET_UNLOCK();
return;
}
@@ -1432,7 +1444,7 @@
type = ICMP_UNREACH;
code = ICMP_UNREACH_NEEDFRAG;
- if ((rt = rtcache_validate(&ipforward_rt)) != NULL)
+ if ((rt = rtcache_validate(ro)) != NULL)
destmtu = rt->rt_ifp->if_mtu;
#ifdef IPSEC
if (ipsec_used)
@@ -1450,10 +1462,12 @@
*/
if (mcopy)
m_freem(mcopy);
+ percpu_putref(ipforward_rt_percpu);
SOFTNET_UNLOCK();
return;
}
icmp_error(mcopy, type, code, dest, destmtu);
+ percpu_putref(ipforward_rt_percpu);
SOFTNET_UNLOCK();
}
diff -r 6806faf520ab -r 515256196e28 sys/netinet6/ip6_forward.c
--- a/sys/netinet6/ip6_forward.c Wed Aug 31 09:03:59 2016 +0000
+++ b/sys/netinet6/ip6_forward.c Wed Aug 31 09:14:47 2016 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ip6_forward.c,v 1.80 2016/06/28 02:02:56 ozaki-r Exp $ */
+/* $NetBSD: ip6_forward.c,v 1.81 2016/08/31 09:14:47 ozaki-r Exp $ */
/* $KAME: ip6_forward.c,v 1.109 2002/09/11 08:10:17 sakane Exp $ */
/*
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.80 2016/06/28 02:02:56 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.81 2016/08/31 09:14:47 ozaki-r Exp $");
#ifdef _KERNEL_OPT
#include "opt_gateway.h"
@@ -48,6 +48,7 @@
#include <sys/time.h>
#include <sys/kernel.h>
#include <sys/syslog.h>
+#include <sys/percpu.h>
#include <net/if.h>
#include <net/route.h>
@@ -72,7 +73,7 @@
#include <net/net_osdep.h>
-struct route ip6_forward_rt;
+extern percpu_t *ip6_forward_rt_percpu;
extern pfil_head_t *inet6_pfil_hook; /* XXX */
@@ -135,6 +136,7 @@
struct in6_addr src_in6, dst_in6;
struct ifnet *rcvif = NULL;
struct psref psref;
+ struct route *ro;
#ifdef IPSEC
int needipsec = 0;
struct secpolicy *sp = NULL;
@@ -203,6 +205,7 @@
}
#endif /* IPSEC */
+ ro = percpu_getref(ip6_forward_rt_percpu);
if (srcrt) {
union {
struct sockaddr dst;
@@ -210,17 +213,18 @@
} u;
sockaddr_in6_init(&u.dst6, &ip6->ip6_dst, 0, 0, 0);
- if ((rt = rtcache_lookup(&ip6_forward_rt, &u.dst)) == NULL) {
+ if ((rt = rtcache_lookup(ro, &u.dst)) == NULL) {
IP6_STATINC(IP6_STAT_NOROUTE);
/* XXX in6_ifstat_inc(rt->rt_ifp, ifs6_in_noroute) */
if (mcopy) {
icmp6_error(mcopy, ICMP6_DST_UNREACH,
ICMP6_DST_UNREACH_NOROUTE, 0);
}
+ percpu_putref(ip6_forward_rt_percpu);
goto drop;
}
- } else if ((rt = rtcache_validate(&ip6_forward_rt)) == NULL &&
- (rt = rtcache_update(&ip6_forward_rt, 1)) == NULL) {
+ } else if ((rt = rtcache_validate(ro)) == NULL &&
+ (rt = rtcache_update(ro, 1)) == NULL) {
/*
* rtcache_getdst(ip6_forward_rt)->sin6_addr was equal to
* ip6->ip6_dst
@@ -231,9 +235,11 @@
icmp6_error(mcopy, ICMP6_DST_UNREACH,
ICMP6_DST_UNREACH_NOROUTE, 0);
}
+ percpu_putref(ip6_forward_rt_percpu);
goto drop;
}
- dst = satocsin6(rtcache_getdst(&ip6_forward_rt));
+ dst = satocsin6(rtcache_getdst(ro));
+ percpu_putref(ip6_forward_rt_percpu);
/*
* Source scope check: if a packet can't be delivered to its
@@ -318,10 +324,11 @@
*/
if (rt->rt_ifp == rcvif && !srcrt && ip6_sendredirects &&
(rt->rt_flags & (RTF_DYNAMIC|RTF_MODIFIED)) == 0) {
+ ro = percpu_getref(ip6_forward_rt_percpu);
if ((rt->rt_ifp->if_flags & IFF_POINTOPOINT) &&
- nd6_is_addr_neighbor(
- satocsin6(rtcache_getdst(&ip6_forward_rt)),
- rt->rt_ifp)) {
+ nd6_is_addr_neighbor(satocsin6(rtcache_getdst(ro)),
+ rt->rt_ifp)) {
+ percpu_putref(ip6_forward_rt_percpu);
/*
* If the incoming interface is equal to the outgoing
* one, the link attached to the interface is
@@ -341,6 +348,7 @@
ICMP6_DST_UNREACH_ADDR, 0);
goto drop;
}
+ percpu_putref(ip6_forward_rt_percpu);
type = ND_REDIRECT;
}
@@ -409,8 +417,10 @@
IP6_STATINC(IP6_STAT_REDIRECTSENT);
else {
#ifdef GATEWAY
+ ro = percpu_getref(ip6_forward_rt_percpu);
if (m->m_flags & M_CANFASTFWD)
- ip6flow_create(&ip6_forward_rt, m);
+ ip6flow_create(ro, m);
+ percpu_putref(ip6_forward_rt_percpu);
#endif
if (mcopy)
goto freecopy;
diff -r 6806faf520ab -r 515256196e28 sys/netinet6/ip6_input.c
--- a/sys/netinet6/ip6_input.c Wed Aug 31 09:03:59 2016 +0000
+++ b/sys/netinet6/ip6_input.c Wed Aug 31 09:14:47 2016 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ip6_input.c,v 1.166 2016/08/02 04:50:16 knakahara Exp $ */
+/* $NetBSD: ip6_input.c,v 1.167 2016/08/31 09:14:47 ozaki-r Exp $ */
/* $KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $ */
/*
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.166 2016/08/02 04:50:16 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.167 2016/08/31 09:14:47 ozaki-r Exp $");
#ifdef _KERNEL_OPT
#include "opt_gateway.h"
@@ -87,6 +87,7 @@
#include <sys/proc.h>
#include <sys/sysctl.h>
#include <sys/cprng.h>
+#include <sys/percpu.h>
#include <net/if.h>
#include <net/if_types.h>
@@ -143,6 +144,8 @@
percpu_t *ip6stat_percpu;
+percpu_t *ip6_forward_rt_percpu __cacheline_aligned;
+
static void ip6_init2(void);
static void ip6intr(void *);
static struct m_tag *ip6_setdstifaddr(struct mbuf *, const struct in6_ifaddr *);
@@ -194,6 +197,10 @@
KASSERT(inet6_pfil_hook != NULL);
ip6stat_percpu = percpu_alloc(sizeof(uint64_t) * IP6_NSTATS);
+
+ ip6_forward_rt_percpu = percpu_alloc(sizeof(struct route));
+ if (ip6_forward_rt_percpu == NULL)
+ panic("failed to alllocate ip6_forward_rt_percpu");
}
static void
@@ -239,8 +246,6 @@
mutex_exit(softnet_lock);
}
-extern struct route ip6_forward_rt;
-
void
ip6_input(struct mbuf *m, struct ifnet *rcvif)
Home |
Main Index |
Thread Index |
Old Index