[src/trunk]: src/sys Tweak softnet_lock and NET_MPSAFE

[ozaki-r <ozaki-r%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/e73b38372c8e
branches:  trunk
changeset: 821085:e73b38372c8e
user:      ozaki-r <ozaki-r%NetBSD.org@localhost>
date:      Tue Jan 24 07:09:24 2017 +0000

description:
Tweak softnet_lock and NET_MPSAFE

- Don't hold softnet_lock in some functions if NET_MPSAFE
- Add softnet_lock to sysctl_net_inet_icmp_redirtimeout
- Add softnet_lock to expire_upcalls of ip_mroute.c
- Restore softnet_lock for in{,6}_pcbpurgeif{,0} if NET_MPSAFE
- Mark some softnet_lock for future work

diffstat:

 sys/netinet/if_arp.c        |   8 ++++++--
 sys/netinet/igmp.c          |  11 +++++++++--
 sys/netinet/ip_icmp.c       |  10 ++++++++--
 sys/netinet/ip_input.c      |   9 +++++++--
 sys/netinet/ip_mroute.c     |  15 +++++++++------
 sys/netinet/raw_ip.c        |  11 +++++++++--
 sys/netinet/tcp_usrreq.c    |  20 ++++++++++++++------
 sys/netinet/udp_usrreq.c    |  16 +++++++++-------
 sys/netinet6/dccp6_usrreq.c |  16 +++++++++-------
 sys/netinet6/frag6.c        |  17 +++++++++++++++--
 sys/netinet6/in6_ifattach.c |   5 +++--
 sys/netinet6/ip6_mroute.c   |   5 +++--
 sys/netinet6/mld6.c         |  14 +++++++-------
 sys/netinet6/raw_ip6.c      |  16 +++++++++-------
 sys/netinet6/udp6_usrreq.c  |  16 +++++++++-------
 15 files changed, 126 insertions(+), 63 deletions(-)

diffs (truncated from 653 to 300 lines):

diff -r 6cbbfb1c8458 -r e73b38372c8e sys/netinet/if_arp.c
--- a/sys/netinet/if_arp.c      Tue Jan 24 05:22:38 2017 +0000
+++ b/sys/netinet/if_arp.c      Tue Jan 24 07:09:24 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: if_arp.c,v 1.239 2017/01/21 11:07:46 maxv Exp $        */
+/*     $NetBSD: if_arp.c,v 1.240 2017/01/24 07:09:24 ozaki-r Exp $     */
 
 /*-
  * Copyright (c) 1998, 2000, 2008 The NetBSD Foundation, Inc.
@@ -68,7 +68,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_arp.c,v 1.239 2017/01/21 11:07:46 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_arp.c,v 1.240 2017/01/24 07:09:24 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_ddb.h"
@@ -1676,8 +1676,10 @@
        struct dadq *dp;
        char ipbuf[INET_ADDRSTRLEN];
 
+#ifndef NET_MPSAFE
        mutex_enter(softnet_lock);
        KERNEL_LOCK(1, NULL);
+#endif
        mutex_enter(&arp_dad_lock);
 
        /* Sanity check */
@@ -1767,8 +1769,10 @@
 
 done:
        mutex_exit(&arp_dad_lock);
+#ifndef NET_MPSAFE
        KERNEL_UNLOCK_ONE(NULL);
        mutex_exit(softnet_lock);
+#endif
 }
 
 static void
diff -r 6cbbfb1c8458 -r e73b38372c8e sys/netinet/igmp.c
--- a/sys/netinet/igmp.c        Tue Jan 24 05:22:38 2017 +0000
+++ b/sys/netinet/igmp.c        Tue Jan 24 07:09:24 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: igmp.c,v 1.63 2017/01/11 13:08:29 ozaki-r Exp $        */
+/*     $NetBSD: igmp.c,v 1.64 2017/01/24 07:09:24 ozaki-r Exp $        */
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -40,10 +40,11 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: igmp.c,v 1.63 2017/01/11 13:08:29 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: igmp.c,v 1.64 2017/01/24 07:09:24 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_mrouting.h"
+#include "opt_net_mpsafe.h"
 #endif
 
 #include <sys/param.h>
@@ -541,8 +542,10 @@
                return;
        }
 
+#ifndef NET_MPSAFE
        /* XXX: Needed for ip_output(). */
        mutex_enter(softnet_lock);
+#endif
 
        in_multi_lock(RW_WRITER);
        igmp_timers_on = false;
@@ -566,7 +569,9 @@
                inm = in_next_multi(&step);
        }
        in_multi_unlock();
+#ifndef NET_MPSAFE
        mutex_exit(softnet_lock);
+#endif
 }
 
 void
@@ -649,7 +654,9 @@
         * Note: IP_IGMP_MCAST indicates that in_multilock is held.
         * The caller must still acquire softnet_lock for ip_output().
         */
+#ifndef NET_MPSAFE
        KASSERT(mutex_owned(softnet_lock));
+#endif
        ip_output(m, NULL, NULL, IP_IGMP_MCAST, &imo, NULL);
        IGMP_STATINC(IGMP_STAT_SND_REPORTS);
 }
diff -r 6cbbfb1c8458 -r e73b38372c8e sys/netinet/ip_icmp.c
--- a/sys/netinet/ip_icmp.c     Tue Jan 24 05:22:38 2017 +0000
+++ b/sys/netinet/ip_icmp.c     Tue Jan 24 07:09:24 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: ip_icmp.c,v 1.154 2016/12/12 03:55:57 ozaki-r Exp $    */
+/*     $NetBSD: ip_icmp.c,v 1.155 2017/01/24 07:09:24 ozaki-r Exp $    */
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -94,7 +94,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip_icmp.c,v 1.154 2016/12/12 03:55:57 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip_icmp.c,v 1.155 2017/01/24 07:09:24 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_ipsec.h"
@@ -105,6 +105,7 @@
 #include <sys/mbuf.h>
 #include <sys/protosw.h>
 #include <sys/socket.h>
+#include <sys/socketvar.h> /* For softnet_lock */
 #include <sys/kmem.h>
 #include <sys/time.h>
 #include <sys/kernel.h>
@@ -1012,6 +1013,9 @@
                return (EINVAL);
        icmp_redirtimeout = tmp;
 
+       /* XXX NOMPSAFE still need softnet_lock */
+       mutex_enter(softnet_lock);
+
        /*
         * was it a *defined* side-effect that anyone even *reading*
         * this value causes these things to happen?
@@ -1029,6 +1033,8 @@
                    rt_timer_queue_create(icmp_redirtimeout);
        }
 
+       mutex_exit(softnet_lock);
+
        return (0);
 }
 
diff -r 6cbbfb1c8458 -r e73b38372c8e sys/netinet/ip_input.c
--- a/sys/netinet/ip_input.c    Tue Jan 24 05:22:38 2017 +0000
+++ b/sys/netinet/ip_input.c    Tue Jan 24 07:09:24 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: ip_input.c,v 1.347 2016/12/12 03:55:57 ozaki-r Exp $   */
+/*     $NetBSD: ip_input.c,v 1.348 2017/01/24 07:09:24 ozaki-r Exp $   */
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -91,7 +91,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.347 2016/12/12 03:55:57 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.348 2017/01/24 07:09:24 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -866,13 +866,17 @@
 ip_slowtimo(void)
 {
 
+#ifndef NET_MPSAFE
        mutex_enter(softnet_lock);
        KERNEL_LOCK(1, NULL);
+#endif
 
        ip_reass_slowtimo();
 
+#ifndef NET_MPSAFE
        KERNEL_UNLOCK_ONE(NULL);
        mutex_exit(softnet_lock);
+#endif
 }
 
 /*
@@ -1603,6 +1607,7 @@
        if (tmp < 0)
                return (EINVAL);
 
+       /* XXX NOMPSAFE still need softnet_lock */
        mutex_enter(softnet_lock);
 
        ip_mtudisc_timeout = tmp;
diff -r 6cbbfb1c8458 -r e73b38372c8e sys/netinet/ip_mroute.c
--- a/sys/netinet/ip_mroute.c   Tue Jan 24 05:22:38 2017 +0000
+++ b/sys/netinet/ip_mroute.c   Tue Jan 24 07:09:24 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: ip_mroute.c,v 1.145 2017/01/11 13:08:29 ozaki-r Exp $  */
+/*     $NetBSD: ip_mroute.c,v 1.146 2017/01/24 07:09:24 ozaki-r Exp $  */
 
 /*
  * Copyright (c) 1992, 1993
@@ -93,7 +93,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip_mroute.c,v 1.145 2017/01/11 13:08:29 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip_mroute.c,v 1.146 2017/01/24 07:09:24 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -1561,9 +1561,10 @@
 expire_upcalls(void *v)
 {
        int i;
-       int s;
-
-       s = splsoftnet();
+
+       /* XXX NOMPSAFE still need softnet_lock */
+       mutex_enter(softnet_lock);
+       KERNEL_LOCK(1, NULL);
 
        for (i = 0; i < MFCTBLSIZ; i++) {
                struct mfc *rt, *nrt;
@@ -1599,9 +1600,11 @@
                }
        }
 
-       splx(s);
        callout_reset(&expire_upcalls_ch, EXPIRE_TIMEOUT,
            expire_upcalls, NULL);
+
+       KERNEL_UNLOCK_ONE(NULL);
+       mutex_exit(softnet_lock);
 }
 
 /*
diff -r 6cbbfb1c8458 -r e73b38372c8e sys/netinet/raw_ip.c
--- a/sys/netinet/raw_ip.c      Tue Jan 24 05:22:38 2017 +0000
+++ b/sys/netinet/raw_ip.c      Tue Jan 24 07:09:24 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: raw_ip.c,v 1.161 2016/09/29 12:19:47 roy Exp $ */
+/*     $NetBSD: raw_ip.c,v 1.162 2017/01/24 07:09:24 ozaki-r Exp $     */
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -65,13 +65,14 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: raw_ip.c,v 1.161 2016/09/29 12:19:47 roy Exp $");
+__KERNEL_RCSID(0, "$NetBSD: raw_ip.c,v 1.162 2017/01/24 07:09:24 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
 #include "opt_compat_netbsd.h"
 #include "opt_ipsec.h"
 #include "opt_mrouting.h"
+#include "opt_net_mpsafe.h"
 #endif
 
 #include <sys/param.h>
@@ -806,7 +807,13 @@
        s = splsoftnet();
        mutex_enter(softnet_lock);
        in_pcbpurgeif0(&rawcbtable, ifp);
+#ifdef NET_MPSAFE
+       mutex_exit(softnet_lock);
+#endif
        in_purgeif(ifp);
+#ifdef NET_MPSAFE
+       mutex_enter(softnet_lock);
+#endif
        in_pcbpurgeif(&rawcbtable, ifp);
        mutex_exit(softnet_lock);
        splx(s);
diff -r 6cbbfb1c8458 -r e73b38372c8e sys/netinet/tcp_usrreq.c
--- a/sys/netinet/tcp_usrreq.c  Tue Jan 24 05:22:38 2017 +0000
+++ b/sys/netinet/tcp_usrreq.c  Tue Jan 24 07:09:24 2017 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: tcp_usrreq.c,v 1.213 2016/11/18 06:50:04 knakahara Exp $       */
+/*     $NetBSD: tcp_usrreq.c,v 1.214 2017/01/24 07:09:24 ozaki-r Exp $ */
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -99,7 +99,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: tcp_usrreq.c,v 1.213 2016/11/18 06:50:04 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: tcp_usrreq.c,v 1.214 2017/01/24 07:09:24 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -1210,21 +1210,31 @@
 
        s = splsoftnet();
 
-#ifndef NET_MPSAFE
        mutex_enter(softnet_lock);
-#endif
        switch (so->so_proto->pr_domain->dom_family) {
 #ifdef INET
        case PF_INET:
                in_pcbpurgeif0(&tcbtable, ifp);
+#ifdef NET_MPSAFE
+               mutex_exit(softnet_lock);
+#endif
                in_purgeif(ifp);
+#ifdef NET_MPSAFE