[src/trunk]: src/sys Avoid a deadlock between softnet_lock and IFNET_LOCK

[ozaki-r <ozaki-r%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/7fbd806f43a0
branches:  trunk
changeset: 359746:7fbd806f43a0
user:      ozaki-r <ozaki-r%NetBSD.org@localhost>
date:      Sat Feb 24 07:37:08 2018 +0000

description:
Avoid a deadlock between softnet_lock and IFNET_LOCK

A deadlock occurs because there is a violation of the rule of lock ordering;
softnet_lock is held with hodling IFNET_LOCK, which violates the rule.
To avoid the deadlock, replace softnet_lock in in_control and in6_control
with KERNEL_LOCK.

We also need to add some KERNEL_LOCKs to protect the network stack surely.
This is required, for example, for PR kern/51356.

Fix PR kern/53043

diffstat:

 sys/netinet/in.c                                |  9 +++++----
 sys/netinet/ip_input.c                          |  8 ++++----
 sys/netinet/wqinput.c                           |  8 +++++++-
 sys/netinet6/in6.c                              |  9 +++++----
 sys/netinet6/ip6_input.c                        |  8 ++++----
 sys/rump/net/lib/libnetinet/netinet_component.c |  6 ++++--
 6 files changed, 29 insertions(+), 19 deletions(-)

diffs (190 lines):

diff -r ed8103771ae1 -r 7fbd806f43a0 sys/netinet/in.c
--- a/sys/netinet/in.c  Fri Feb 23 21:16:01 2018 +0000
+++ b/sys/netinet/in.c  Sat Feb 24 07:37:08 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: in.c,v 1.218 2018/02/14 14:15:53 maxv Exp $    */
+/*     $NetBSD: in.c,v 1.219 2018/02/24 07:37:09 ozaki-r Exp $ */
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -91,7 +91,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: in.c,v 1.218 2018/02/14 14:15:53 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: in.c,v 1.219 2018/02/24 07:37:09 ozaki-r Exp $");
 
 #include "arp.h"
 
@@ -751,9 +751,10 @@
 {
        int error;
 
-       SOFTNET_LOCK_UNLESS_NET_MPSAFE();
+#ifndef NET_MPSAFE
+       KASSERT(KERNEL_LOCKED_P());
+#endif
        error = in_control0(so, cmd, data, ifp);
-       SOFTNET_UNLOCK_UNLESS_NET_MPSAFE();
 
        return error;
 }
diff -r ed8103771ae1 -r 7fbd806f43a0 sys/netinet/ip_input.c
--- a/sys/netinet/ip_input.c    Fri Feb 23 21:16:01 2018 +0000
+++ b/sys/netinet/ip_input.c    Sat Feb 24 07:37:08 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: ip_input.c,v 1.375 2018/02/09 18:31:52 maxv Exp $      */
+/*     $NetBSD: ip_input.c,v 1.376 2018/02/24 07:37:09 ozaki-r Exp $   */
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -91,7 +91,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.375 2018/02/09 18:31:52 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.376 2018/02/24 07:37:09 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -404,11 +404,11 @@
 
        KASSERT(cpu_softintr_p());
 
-       SOFTNET_LOCK_UNLESS_NET_MPSAFE();
+       SOFTNET_KERNEL_LOCK_UNLESS_NET_MPSAFE();
        while ((m = pktq_dequeue(ip_pktq)) != NULL) {
                ip_input(m);
        }
-       SOFTNET_UNLOCK_UNLESS_NET_MPSAFE();
+       SOFTNET_KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
 }
 
 /*
diff -r ed8103771ae1 -r 7fbd806f43a0 sys/netinet/wqinput.c
--- a/sys/netinet/wqinput.c     Fri Feb 23 21:16:01 2018 +0000
+++ b/sys/netinet/wqinput.c     Sat Feb 24 07:37:08 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: wqinput.c,v 1.3 2017/06/02 19:10:19 para Exp $ */
+/*     $NetBSD: wqinput.c,v 1.4 2018/02/24 07:37:09 ozaki-r Exp $      */
 
 /*-
  * Copyright (c) 2017 Internet Initiative Japan Inc.
@@ -26,6 +26,10 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+#ifdef _KERNEL_OPT
+#include "opt_net_mpsafe.h"
+#endif
+
 #include <sys/param.h>
 #include <sys/kmem.h>
 #include <sys/mbuf.h>
@@ -210,7 +214,9 @@
 
        while ((work = wqinput_work_get(wwl)) != NULL) {
                mutex_enter(softnet_lock);
+               KERNEL_LOCK_UNLESS_NET_MPSAFE();
                wqi->wqi_input(work->ww_mbuf, work->ww_off, work->ww_proto);
+               KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
                mutex_exit(softnet_lock);
 
                pool_put(&wqi->wqi_work_pool, work);
diff -r ed8103771ae1 -r 7fbd806f43a0 sys/netinet6/in6.c
--- a/sys/netinet6/in6.c        Fri Feb 23 21:16:01 2018 +0000
+++ b/sys/netinet6/in6.c        Sat Feb 24 07:37:08 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: in6.c,v 1.259 2018/01/19 08:01:05 ozaki-r Exp $        */
+/*     $NetBSD: in6.c,v 1.260 2018/02/24 07:37:09 ozaki-r Exp $        */
 /*     $KAME: in6.c,v 1.198 2001/07/18 09:12:38 itojun Exp $   */
 
 /*
@@ -62,7 +62,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: in6.c,v 1.259 2018/01/19 08:01:05 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: in6.c,v 1.260 2018/02/24 07:37:09 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -767,9 +767,10 @@
        }
 
        s = splsoftnet();
-       SOFTNET_LOCK_UNLESS_NET_MPSAFE();
+#ifndef NET_MPSAFE
+       KASSERT(KERNEL_LOCKED_P());
+#endif
        error = in6_control1(so , cmd, data, ifp);
-       SOFTNET_UNLOCK_UNLESS_NET_MPSAFE();
        splx(s);
        return error;
 }
diff -r ed8103771ae1 -r 7fbd806f43a0 sys/netinet6/ip6_input.c
--- a/sys/netinet6/ip6_input.c  Fri Feb 23 21:16:01 2018 +0000
+++ b/sys/netinet6/ip6_input.c  Sat Feb 24 07:37:08 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: ip6_input.c,v 1.192 2018/02/14 05:29:39 maxv Exp $     */
+/*     $NetBSD: ip6_input.c,v 1.193 2018/02/24 07:37:09 ozaki-r Exp $  */
 /*     $KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $     */
 
 /*
@@ -62,7 +62,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.192 2018/02/14 05:29:39 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.193 2018/02/24 07:37:09 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_gateway.h"
@@ -218,7 +218,7 @@
 {
        struct mbuf *m;
 
-       SOFTNET_LOCK_UNLESS_NET_MPSAFE();
+       SOFTNET_KERNEL_LOCK_UNLESS_NET_MPSAFE();
        while ((m = pktq_dequeue(ip6_pktq)) != NULL) {
                struct psref psref;
                struct ifnet *rcvif = m_get_rcvif_psref(m, &psref);
@@ -238,7 +238,7 @@
                ip6_input(m, rcvif);
                m_put_rcvif_psref(rcvif, &psref);
        }
-       SOFTNET_UNLOCK_UNLESS_NET_MPSAFE();
+       SOFTNET_KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
 }
 
 void
diff -r ed8103771ae1 -r 7fbd806f43a0 sys/rump/net/lib/libnetinet/netinet_component.c
--- a/sys/rump/net/lib/libnetinet/netinet_component.c   Fri Feb 23 21:16:01 2018 +0000
+++ b/sys/rump/net/lib/libnetinet/netinet_component.c   Sat Feb 24 07:37:08 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: netinet_component.c,v 1.10 2017/12/11 03:29:20 ozaki-r Exp $   */
+/*     $NetBSD: netinet_component.c,v 1.11 2018/02/24 07:37:08 ozaki-r Exp $   */
 
 /*
  * Copyright (c) 2009 Antti Kantee.  All Rights Reserved.
@@ -28,7 +28,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: netinet_component.c,v 1.10 2017/12/11 03:29:20 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: netinet_component.c,v 1.11 2018/02/24 07:37:08 ozaki-r Exp $");
 
 #include <sys/param.h>
 #include <sys/domain.h>
@@ -94,9 +94,11 @@
        sin->sin_len = sizeof(struct sockaddr_in);
        sin->sin_addr.s_addr = inet_addr("127.255.255.255");
 
+       KERNEL_LOCK(1, NULL);
        IFNET_LOCK(lo0ifp);
        in_control(so, SIOCAIFADDR, &ia, lo0ifp);
        IFNET_UNLOCK(lo0ifp);
+       KERNEL_UNLOCK_ONE(NULL);
        if_up(lo0ifp);
        soclose(so);
 }