[src/trunk]: src/sys/netipsec Add per-CPU rtcache to ipsec_reinject_ipstack

[ozaki-r <ozaki-r%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/c7b785a9f243
branches:  trunk
changeset: 355687:c7b785a9f243
user:      ozaki-r <ozaki-r%NetBSD.org@localhost>
date:      Thu Aug 10 06:11:24 2017 +0000

description:
Add per-CPU rtcache to ipsec_reinject_ipstack

It reduces route lookups and also reduces rtcache lock contentions
when NET_MPSAFE is enabled.

diffstat:

 sys/netipsec/ipsec.c        |   6 ++++--
 sys/netipsec/ipsec.h        |   3 ++-
 sys/netipsec/ipsec_output.c |  19 +++++++++++++++----
 3 files changed, 21 insertions(+), 7 deletions(-)

diffs (114 lines):

diff -r 06b4dc121ec3 -r c7b785a9f243 sys/netipsec/ipsec.c
--- a/sys/netipsec/ipsec.c      Thu Aug 10 06:08:59 2017 +0000
+++ b/sys/netipsec/ipsec.c      Thu Aug 10 06:11:24 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: ipsec.c,v 1.117 2017/08/07 03:18:32 ozaki-r Exp $      */
+/*     $NetBSD: ipsec.c,v 1.118 2017/08/10 06:11:24 ozaki-r Exp $      */
 /*     $FreeBSD: /usr/local/www/cvsroot/FreeBSD/src/sys/netipsec/ipsec.c,v 1.2.2.2 2003/07/01 01:38:13 sam Exp $       */
 /*     $KAME: ipsec.c,v 1.103 2001/05/24 07:14:18 sakane Exp $ */
 
@@ -32,7 +32,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipsec.c,v 1.117 2017/08/07 03:18:32 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipsec.c,v 1.118 2017/08/10 06:11:24 ozaki-r Exp $");
 
 /*
  * IPsec controller part.
@@ -2387,6 +2387,8 @@
 ipsec_attach(void)
 {
 
+       ipsec_output_init();
+
        ipsecstat_percpu = percpu_alloc(sizeof(uint64_t) * IPSEC_NSTATS);
 
        sysctl_net_inet_ipsec_setup(NULL);
diff -r 06b4dc121ec3 -r c7b785a9f243 sys/netipsec/ipsec.h
--- a/sys/netipsec/ipsec.h      Thu Aug 10 06:08:59 2017 +0000
+++ b/sys/netipsec/ipsec.h      Thu Aug 10 06:11:24 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: ipsec.h,v 1.58 2017/08/02 01:28:03 ozaki-r Exp $       */
+/*     $NetBSD: ipsec.h,v 1.59 2017/08/10 06:11:24 ozaki-r Exp $       */
 /*     $FreeBSD: /usr/local/www/cvsroot/FreeBSD/src/sys/netipsec/ipsec.h,v 1.2.4.2 2004/02/14 22:23:23 bms Exp $       */
 /*     $KAME: ipsec.h,v 1.53 2001/11/20 08:32:38 itojun Exp $  */
 
@@ -334,6 +334,7 @@
 void *esp4_ctlinput(int, const struct sockaddr *, void *);
 void *ah4_ctlinput(int, const struct sockaddr *, void *);
 
+void ipsec_output_init(void);
 struct m_tag;
 void ipsec4_common_input(struct mbuf *m, ...);
 int ipsec4_common_input_cb(struct mbuf *, struct secasvar *,
diff -r 06b4dc121ec3 -r c7b785a9f243 sys/netipsec/ipsec_output.c
--- a/sys/netipsec/ipsec_output.c       Thu Aug 10 06:08:59 2017 +0000
+++ b/sys/netipsec/ipsec_output.c       Thu Aug 10 06:11:24 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: ipsec_output.c,v 1.59 2017/08/10 06:08:59 ozaki-r Exp $        */
+/*     $NetBSD: ipsec_output.c,v 1.60 2017/08/10 06:11:24 ozaki-r Exp $        */
 
 /*-
  * Copyright (c) 2002, 2003 Sam Leffler, Errno Consulting
@@ -29,7 +29,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipsec_output.c,v 1.59 2017/08/10 06:08:59 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipsec_output.c,v 1.60 2017/08/10 06:11:24 ozaki-r Exp $");
 
 /*
  * IPsec output processing.
@@ -86,6 +86,7 @@
 
 #include <net/net_osdep.h>             /* ovbcopy() in ipsec6_encapsulate() */
 
+static percpu_t *ipsec_rtcache_percpu __cacheline_aligned;
 
 /*
  * Add a IPSEC_OUT_DONE tag to mark that we have finished the ipsec processing
@@ -112,16 +113,18 @@
 ipsec_reinject_ipstack(struct mbuf *m, int af)
 {
        int rv = -1;
+       struct route *ro;
 
        KASSERT(af == AF_INET || af == AF_INET6);
 
 #ifndef NET_MPSAFE
        KERNEL_LOCK(1, NULL);
 #endif
+       ro = percpu_getref(ipsec_rtcache_percpu);
        switch (af) {
 #ifdef INET
        case AF_INET:
-               rv = ip_output(m, NULL, NULL, IP_RAWOUTPUT|IP_NOIPNEWID,
+               rv = ip_output(m, NULL, ro, IP_RAWOUTPUT|IP_NOIPNEWID,
                    NULL, NULL);
                break;
 #endif
@@ -131,10 +134,11 @@
                 * We don't need massage, IPv6 header fields are always in
                 * net endian.
                 */
-               rv = ip6_output(m, NULL, NULL, 0, NULL, NULL, NULL);
+               rv = ip6_output(m, NULL, ro, 0, NULL, NULL, NULL);
                break;
 #endif
        }
+       percpu_putref(ipsec_rtcache_percpu);
 #ifndef NET_MPSAFE
        KERNEL_UNLOCK_ONE(NULL);
 #endif
@@ -795,3 +799,10 @@
        return error;
 }
 #endif /*INET6*/
+
+void
+ipsec_output_init(void)
+{
+
+       ipsec_rtcache_percpu = percpu_alloc(sizeof(struct route));
+}