[src/trunk]: src/sys/kern Fix an IPI deadlock scenario that resulted in a TLB...

[thorpej <thorpej%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/c2a8a0036238
branches:  trunk
changeset: 982173:c2a8a0036238
user:      thorpej <thorpej%NetBSD.org@localhost>
date:      Sat Apr 03 14:56:13 2021 +0000

description:
Fix an IPI deadlock scenario that resulted in a TLB shootdown timeout
panic reported by John Klos on port-alpha:

- pmap_tlb_shootnow(): If we acquire a pmap's activation lock, we will
  have raised the IPL on the current CPU to IPL_SCHED until we drop
  the tlb_lock (due to how nested spin mutexes work).  As such, when
  we release the activation lock, forcibly lower our IPL back to IPL_VM
  so that we can receive and process IPIs while waiting for other CPUs
  to process the shootdowns.
- mutex_vector_enter(): Invoke SPINLOCK_SPIN_HOOK while spinning to acquire
  a spin mutex.  This is a nop on most platforms, but it's important on
  the Alpha.  Without this, IPIs (and thus TLB shootdowns) cannot be
  processed if trying to acquire an IPL_SCHED spin mutex such as those
  used by the scheduler.

...and while we're poking around in here:

- Rework the Alpha SPINLOCK_SPIN_HOOK to only check curcpu()->ci_ipis
  if the current CPU's IPL is >= IPL_CLOCK (thus ensuring that preemption
  is disabled and thus guaranteeing that curcpu() is stable).  (Alpha does
  not yet support kernel preemption, but this is now one less thing that
  would need to be fixed.)

diffstat:

 sys/arch/alpha/alpha/pmap.c   |  11 +++++++++--
 sys/arch/alpha/include/lock.h |  31 ++++++++++++++++---------------
 sys/kern/kern_mutex.c         |   5 +++--
 3 files changed, 28 insertions(+), 19 deletions(-)

diffs (109 lines):

diff -r 340b27230603 -r c2a8a0036238 sys/arch/alpha/alpha/pmap.c
--- a/sys/arch/alpha/alpha/pmap.c       Sat Apr 03 14:39:02 2021 +0000
+++ b/sys/arch/alpha/alpha/pmap.c       Sat Apr 03 14:56:13 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.274 2020/12/29 17:16:15 thorpej Exp $ */
+/* $NetBSD: pmap.c,v 1.275 2021/04/03 14:56:13 thorpej Exp $ */
 
 /*-
  * Copyright (c) 1998, 1999, 2000, 2001, 2007, 2008, 2020
@@ -135,7 +135,7 @@
 
 #include <sys/cdefs.h>                 /* RCS ID & Copyright macro defns */
 
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.274 2020/12/29 17:16:15 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.275 2021/04/03 14:56:13 thorpej Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -962,6 +962,13 @@
        if (activation_locked) {
                KASSERT(tlbctx->t_pmap != NULL);
                PMAP_ACT_UNLOCK(tlbctx->t_pmap);
+               /*
+                * When we acquired the activation lock, we
+                * raised IPL to IPL_SCHED, which blocks out
+                * IPIs.  Force our IPL back down to IPL_VM
+                * so that we can receive IPIs.
+                */
+               alpha_pal_swpipl(IPL_VM);
        }
 
        /*
diff -r 340b27230603 -r c2a8a0036238 sys/arch/alpha/include/lock.h
--- a/sys/arch/alpha/include/lock.h     Sat Apr 03 14:39:02 2021 +0000
+++ b/sys/arch/alpha/include/lock.h     Sat Apr 03 14:56:13 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: lock.h,v 1.30 2019/11/29 20:05:07 riastradh Exp $ */
+/* $NetBSD: lock.h,v 1.31 2021/04/03 14:56:13 thorpej Exp $ */
 
 /*-
  * Copyright (c) 1998, 1999, 2000 The NetBSD Foundation, Inc.
@@ -151,26 +151,27 @@
 #if defined(MULTIPROCESSOR)
 /*
  * On the Alpha, interprocessor interrupts come in at device priority
- * level.  This can cause some problems while waiting for r/w spinlocks
- * from a high'ish priority level: IPIs that come in will not be processed.
- * This can lead to deadlock.
+ * level (ALPHA_PSL_IPL_CLOCK).  This can cause some problems while
+ * waiting for spin locks from a high'ish priority level (like spin
+ * mutexes used by the scheduler): IPIs that come in will not be
+ * processed. This can lead to deadlock.
  *
- * This hook allows IPIs to be processed while a spinlock's interlock
- * is released.
+ * This hook allows IPIs to be processed while spinning.  Note we only
+ * do the special thing if IPIs are blocked (current IPL >= IPL_CLOCK).
+ * IPIs will be processed in the normal fashion otherwise, and checking
+ * this way ensures that preemption is disabled (i.e. curcpu() is stable).
  */
 #define        SPINLOCK_SPIN_HOOK                                              \
 do {                                                                   \
-       struct cpu_info *__ci = curcpu();                               \
-       int __s;                                                        \
+       unsigned long _ipl_ = alpha_pal_rdps() & ALPHA_PSL_IPL_MASK;    \
                                                                        \
-       if (__ci->ci_ipis != 0) {                                       \
-               /* printf("CPU %lu has IPIs pending\n",                 \
-                   __ci->ci_cpuid); */                                 \
-               __s = splhigh();                                                \
-               alpha_ipi_process(__ci, NULL);                          \
-               splx(__s);                                              \
+       if (_ipl_ >= ALPHA_PSL_IPL_CLOCK) {                             \
+               struct cpu_info *__ci = curcpu();                       \
+               if (atomic_load_relaxed(&__ci->ci_ipis) != 0) {         \
+                       alpha_ipi_process(__ci, NULL);                  \
+               }                                                       \
        }                                                               \
-} while (0)
+} while (/*CONSTCOND*/0)
 #define        SPINLOCK_BACKOFF_HOOK   (void)nullop((void *)0)
 #endif /* MULTIPROCESSOR */
 
diff -r 340b27230603 -r c2a8a0036238 sys/kern/kern_mutex.c
--- a/sys/kern/kern_mutex.c     Sat Apr 03 14:39:02 2021 +0000
+++ b/sys/kern/kern_mutex.c     Sat Apr 03 14:56:13 2021 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: kern_mutex.c,v 1.96 2021/03/02 01:15:15 rin Exp $      */
+/*     $NetBSD: kern_mutex.c,v 1.97 2021/04/03 14:56:14 thorpej Exp $  */
 
 /*-
  * Copyright (c) 2002, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc.
@@ -40,7 +40,7 @@
 #define        __MUTEX_PRIVATE
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.96 2021/03/02 01:15:15 rin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.97 2021/04/03 14:56:14 thorpej Exp $");
 
 #include <sys/param.h>
 #include <sys/atomic.h>
@@ -484,6 +484,7 @@
                 */
                do {
                        while (MUTEX_SPINBIT_LOCKED_P(mtx)) {
+                               SPINLOCK_SPIN_HOOK;
                                SPINLOCK_BACKOFF(count);
 #ifdef LOCKDEBUG
                                if (SPINLOCK_SPINOUT(spins))