Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/dev lockstat(4): Membar audit.



details:   https://anonhg.NetBSD.org/src/rev/79a6b890bd3e
branches:  trunk
changeset: 362426:79a6b890bd3e
user:      riastradh <riastradh%NetBSD.org@localhost>
date:      Sun Feb 27 14:16:12 2022 +0000

description:
lockstat(4): Membar audit.

- Serialize updates to lockstat_enabled, lockstat_dev_enabled, and
  lockstat_dtrace_enabled with a new __cpu_simple_lock.

- Use xc_barrier to obviate any need for additional membars in
  lockstat_event.

- Use atomic_load/store_* for access that might not be serialized by
  lockstat_lock or lockstat_enabled_lock.

diffstat:

 external/cddl/osnet/dev/lockstat/lockstat.c |  40 ++++++++++++++---------
 sys/dev/lockstat.c                          |  49 ++++++++++++++++++++--------
 sys/dev/lockstat.h                          |  23 ++++++++++---
 3 files changed, 76 insertions(+), 36 deletions(-)

diffs (269 lines):

diff -r 922a89c71a77 -r 79a6b890bd3e external/cddl/osnet/dev/lockstat/lockstat.c
--- a/external/cddl/osnet/dev/lockstat/lockstat.c       Sun Feb 27 12:27:22 2022 +0000
+++ b/external/cddl/osnet/dev/lockstat/lockstat.c       Sun Feb 27 14:16:12 2022 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: lockstat.c,v 1.10 2019/02/12 14:31:45 rin Exp $        */
+/*     $NetBSD: lockstat.c,v 1.11 2022/02/27 14:16:12 riastradh Exp $  */
 
 /*
  * CDDL HEADER START
@@ -26,7 +26,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.10 2019/02/12 14:31:45 rin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.11 2022/02/27 14:16:12 riastradh Exp $");
 
 #include <sys/types.h>
 #include <sys/proc.h>
@@ -72,11 +72,13 @@
 
        ASSERT(!lockstat_probemap[LS_COMPRESS(probe->lsp_probe)]);
 
-       lockstat_probemap[LS_COMPRESS(probe->lsp_probe)] = id;
        if (lockstat_dtrace_count++ == 0) {
+               LOCKSTAT_ENABLED_UPDATE_BEGIN();
                lockstat_dtrace_enabled = LB_DTRACE;
-               LOCKSTAT_ENABLED_UPDATE();
+               LOCKSTAT_ENABLED_UPDATE_END();
        }
+       atomic_store_relaxed(&lockstat_probemap[LS_COMPRESS(probe->lsp_probe)],
+           id);
 
        return 0;
 }
@@ -89,12 +91,19 @@
 
        ASSERT(lockstat_probemap[LS_COMPRESS(probe->lsp_probe)]);
 
+       atomic_store_relaxed(&lockstat_probemap[LS_COMPRESS(probe->lsp_probe)],
+           0);
        if (--lockstat_dtrace_count == 0) {
+               LOCKSTAT_ENABLED_UPDATE_BEGIN();
                lockstat_dtrace_enabled = 0;
-               LOCKSTAT_ENABLED_UPDATE();
+               LOCKSTAT_ENABLED_UPDATE_END();
+
+               /*
+                * Wait for all lockstat dtrace probe on all CPUs to
+                * finish, now that they've been disabled.
+                */
+               xc_barrier(0);
        }
-
-       lockstat_probemap[LS_COMPRESS(probe->lsp_probe)] = 0;
 }
 
 /*ARGSUSED*/
@@ -149,13 +158,6 @@
        lockstat_destroy
 };
 
-static void
-lockstat_barrier_xc(void *arg0 __unused, void *arg1 __unused)
-{
-
-       membar_consumer();
-}
-
 typedef void (*dtrace_probe_func_t)(dtrace_id_t, uintptr_t, uintptr_t,
     uintptr_t, uintptr_t, uintptr_t);
 
@@ -169,8 +171,14 @@
                return false;
 
        lockstat_probe_func = new;
-       membar_producer();
-       xc_wait(xc_broadcast(0, lockstat_barrier_xc, NULL, NULL));
+
+       /*
+        * Make sure that the probe function is initialized on all CPUs
+        * before we enable the lockstat probe by setting
+        * lockstat_probemap[...].
+        */
+       xc_barrier(0);
+
        return true;
 }
 
diff -r 922a89c71a77 -r 79a6b890bd3e sys/dev/lockstat.c
--- a/sys/dev/lockstat.c        Sun Feb 27 12:27:22 2022 +0000
+++ b/sys/dev/lockstat.c        Sun Feb 27 14:16:12 2022 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: lockstat.c,v 1.27 2020/05/23 23:42:42 ad Exp $ */
+/*     $NetBSD: lockstat.c,v 1.28 2022/02/27 14:16:12 riastradh Exp $  */
 
 /*-
  * Copyright (c) 2006, 2007, 2019 The NetBSD Foundation, Inc.
@@ -41,7 +41,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.27 2020/05/23 23:42:42 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.28 2022/02/27 14:16:12 riastradh Exp $");
 
 #include <sys/types.h>
 #include <sys/param.h>
@@ -54,6 +54,7 @@
 #include <sys/cpu.h>
 #include <sys/syslog.h>
 #include <sys/atomic.h>
+#include <sys/xcall.h>
 
 #include <dev/lockstat.h>
 
@@ -101,6 +102,7 @@
 
 volatile u_int lockstat_enabled;
 volatile u_int lockstat_dev_enabled;
+__cpu_simple_lock_t lockstat_enabled_lock;
 uintptr_t      lockstat_csstart;
 uintptr_t      lockstat_csend;
 uintptr_t      lockstat_csmask;
@@ -154,6 +156,7 @@
        (void)nunits;
 
        __cpu_simple_lock_init(&lockstat_lock);
+       __cpu_simple_lock_init(&lockstat_enabled_lock);
 }
 
 /*
@@ -235,10 +238,26 @@
        lockstat_lockstart = le->le_lockstart;
        lockstat_lockstart = le->le_lockstart;
        lockstat_lockend = le->le_lockend;
-       membar_sync();
+
+       /*
+        * Ensure everything is initialized on all CPUs, by issuing a
+        * null xcall with the side effect of a release barrier on this
+        * CPU and an acquire barrier on all other CPUs, before they
+        * can witness any flags set in lockstat_dev_enabled -- this
+        * way we don't need to add any barriers in lockstat_event.
+        */
+       xc_barrier(0);
+
+       /*
+        * Start timing after the xcall, so we don't spuriously count
+        * xcall communication time, but before flipping the switch, so
+        * we don't dirty sample with locks taken in the timecounter.
+        */
        getnanotime(&lockstat_stime);
-       lockstat_dev_enabled = le->le_mask;
-       LOCKSTAT_ENABLED_UPDATE();
+
+       LOCKSTAT_ENABLED_UPDATE_BEGIN();
+       atomic_store_relaxed(&lockstat_dev_enabled, le->le_mask);
+       LOCKSTAT_ENABLED_UPDATE_END();
 }
 
 /*
@@ -258,13 +277,13 @@
        KASSERT(lockstat_dev_enabled);
 
        /*
-        * Set enabled false, force a write barrier, and wait for other CPUs
-        * to exit lockstat_event().
+        * Disable and wait for other CPUs to exit lockstat_event().
         */
-       lockstat_dev_enabled = 0;
-       LOCKSTAT_ENABLED_UPDATE();
+       LOCKSTAT_ENABLED_UPDATE_BEGIN();
+       atomic_store_relaxed(&lockstat_dev_enabled, 0);
+       LOCKSTAT_ENABLED_UPDATE_END();
        getnanotime(&ts);
-       tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10));
+       xc_barrier(0);
 
        /*
         * Did we run out of buffers while tracing?
@@ -370,12 +389,14 @@
 #ifdef KDTRACE_HOOKS
        uint32_t id;
        CTASSERT((LS_NPROBES & (LS_NPROBES - 1)) == 0);
-       if ((id = lockstat_probemap[LS_COMPRESS(flags)]) != 0)
+       if ((id = atomic_load_relaxed(&lockstat_probemap[LS_COMPRESS(flags)]))
+           != 0)
                (*lockstat_probe_func)(id, lock, callsite, flags, count,
                    cycles);
 #endif
 
-       if ((flags & lockstat_dev_enabled) != flags || count == 0)
+       if ((flags & atomic_load_relaxed(&lockstat_dev_enabled)) != flags ||
+           count == 0)
                return;
        if (lock < lockstat_lockstart || lock > lockstat_lockend)
                return;
@@ -487,7 +508,7 @@
                        error = ENODEV;
                        break;
                }
-               if (lockstat_dev_enabled) {
+               if (atomic_load_relaxed(&lockstat_dev_enabled)) {
                        error = EBUSY;
                        break;
                }
@@ -524,7 +545,7 @@
                break;
 
        case IOC_LOCKSTAT_DISABLE:
-               if (!lockstat_dev_enabled)
+               if (!atomic_load_relaxed(&lockstat_dev_enabled))
                        error = EINVAL;
                else
                        error = lockstat_stop((lsdisable_t *)data);
diff -r 922a89c71a77 -r 79a6b890bd3e sys/dev/lockstat.h
--- a/sys/dev/lockstat.h        Sun Feb 27 12:27:22 2022 +0000
+++ b/sys/dev/lockstat.h        Sun Feb 27 14:16:12 2022 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: lockstat.h,v 1.14 2016/01/24 01:01:11 christos Exp $   */
+/*     $NetBSD: lockstat.h,v 1.15 2022/02/27 14:16:12 riastradh Exp $  */
 
 /*-
  * Copyright (c) 2006 The NetBSD Foundation, Inc.
@@ -38,7 +38,9 @@
 #endif
 
 #include <sys/types.h>
+
 #include <sys/ioccom.h>
+#include <sys/lock.h>
 #include <sys/queue.h>
 #include <sys/time.h>
 
@@ -159,7 +161,7 @@
 #define        LOCKSTAT_TIMER(name)    uint64_t name = 0
 #define        LOCKSTAT_COUNTER(name)  uint64_t name = 0
 #define        LOCKSTAT_FLAG(name)     int name
-#define        LOCKSTAT_ENTER(name)    name = lockstat_enabled
+#define        LOCKSTAT_ENTER(name)    name = atomic_load_relaxed(&lockstat_enabled)
 #define        LOCKSTAT_EXIT(name)
 
 #define        LOCKSTAT_START_TIMER(flag, name)                                \
@@ -214,13 +216,22 @@
 #endif
 
 #if defined(_KERNEL) && NLOCKSTAT > 0
+extern __cpu_simple_lock_t lockstat_enabled_lock;
 extern volatile u_int  lockstat_enabled;
 extern volatile u_int  lockstat_dev_enabled;
 
-#define LOCKSTAT_ENABLED_UPDATE() do { \
-       lockstat_enabled = lockstat_dev_enabled | KDTRACE_LOCKSTAT_ENABLED; \
-       membar_producer(); \
-    } while (/*CONSTCOND*/0)
+#define LOCKSTAT_ENABLED_UPDATE_BEGIN() do                                 \
+{                                                                          \
+       __cpu_simple_lock(&lockstat_enabled_lock);                          \
+} while (/*CONSTCOND*/0)
+
+#define LOCKSTAT_ENABLED_UPDATE_END() do                                   \
+{                                                                          \
+       atomic_store_relaxed(&lockstat_enabled,                             \
+           lockstat_dev_enabled | KDTRACE_LOCKSTAT_ENABLED);               \
+       __cpu_simple_unlock(&lockstat_enabled_lock);                        \
+} while (/*CONSTCOND*/0)
+
 #endif
 
 #endif /* _SYS_LOCKSTAT_H_ */



Home | Main Index | Thread Index | Old Index