Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/dev lockstat(4): Membar audit.
details: https://anonhg.NetBSD.org/src/rev/79a6b890bd3e
branches: trunk
changeset: 362426:79a6b890bd3e
user: riastradh <riastradh%NetBSD.org@localhost>
date: Sun Feb 27 14:16:12 2022 +0000
description:
lockstat(4): Membar audit.
- Serialize updates to lockstat_enabled, lockstat_dev_enabled, and
lockstat_dtrace_enabled with a new __cpu_simple_lock.
- Use xc_barrier to obviate any need for additional membars in
lockstat_event.
- Use atomic_load/store_* for access that might not be serialized by
lockstat_lock or lockstat_enabled_lock.
diffstat:
external/cddl/osnet/dev/lockstat/lockstat.c | 40 ++++++++++++++---------
sys/dev/lockstat.c | 49 ++++++++++++++++++++--------
sys/dev/lockstat.h | 23 ++++++++++---
3 files changed, 76 insertions(+), 36 deletions(-)
diffs (269 lines):
diff -r 922a89c71a77 -r 79a6b890bd3e external/cddl/osnet/dev/lockstat/lockstat.c
--- a/external/cddl/osnet/dev/lockstat/lockstat.c Sun Feb 27 12:27:22 2022 +0000
+++ b/external/cddl/osnet/dev/lockstat/lockstat.c Sun Feb 27 14:16:12 2022 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: lockstat.c,v 1.10 2019/02/12 14:31:45 rin Exp $ */
+/* $NetBSD: lockstat.c,v 1.11 2022/02/27 14:16:12 riastradh Exp $ */
/*
* CDDL HEADER START
@@ -26,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.10 2019/02/12 14:31:45 rin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.11 2022/02/27 14:16:12 riastradh Exp $");
#include <sys/types.h>
#include <sys/proc.h>
@@ -72,11 +72,13 @@
ASSERT(!lockstat_probemap[LS_COMPRESS(probe->lsp_probe)]);
- lockstat_probemap[LS_COMPRESS(probe->lsp_probe)] = id;
if (lockstat_dtrace_count++ == 0) {
+ LOCKSTAT_ENABLED_UPDATE_BEGIN();
lockstat_dtrace_enabled = LB_DTRACE;
- LOCKSTAT_ENABLED_UPDATE();
+ LOCKSTAT_ENABLED_UPDATE_END();
}
+ atomic_store_relaxed(&lockstat_probemap[LS_COMPRESS(probe->lsp_probe)],
+ id);
return 0;
}
@@ -89,12 +91,19 @@
ASSERT(lockstat_probemap[LS_COMPRESS(probe->lsp_probe)]);
+ atomic_store_relaxed(&lockstat_probemap[LS_COMPRESS(probe->lsp_probe)],
+ 0);
if (--lockstat_dtrace_count == 0) {
+ LOCKSTAT_ENABLED_UPDATE_BEGIN();
lockstat_dtrace_enabled = 0;
- LOCKSTAT_ENABLED_UPDATE();
+ LOCKSTAT_ENABLED_UPDATE_END();
+
+ /*
+ * Wait for all lockstat dtrace probe on all CPUs to
+ * finish, now that they've been disabled.
+ */
+ xc_barrier(0);
}
-
- lockstat_probemap[LS_COMPRESS(probe->lsp_probe)] = 0;
}
/*ARGSUSED*/
@@ -149,13 +158,6 @@
lockstat_destroy
};
-static void
-lockstat_barrier_xc(void *arg0 __unused, void *arg1 __unused)
-{
-
- membar_consumer();
-}
-
typedef void (*dtrace_probe_func_t)(dtrace_id_t, uintptr_t, uintptr_t,
uintptr_t, uintptr_t, uintptr_t);
@@ -169,8 +171,14 @@
return false;
lockstat_probe_func = new;
- membar_producer();
- xc_wait(xc_broadcast(0, lockstat_barrier_xc, NULL, NULL));
+
+ /*
+ * Make sure that the probe function is initialized on all CPUs
+ * before we enable the lockstat probe by setting
+ * lockstat_probemap[...].
+ */
+ xc_barrier(0);
+
return true;
}
diff -r 922a89c71a77 -r 79a6b890bd3e sys/dev/lockstat.c
--- a/sys/dev/lockstat.c Sun Feb 27 12:27:22 2022 +0000
+++ b/sys/dev/lockstat.c Sun Feb 27 14:16:12 2022 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: lockstat.c,v 1.27 2020/05/23 23:42:42 ad Exp $ */
+/* $NetBSD: lockstat.c,v 1.28 2022/02/27 14:16:12 riastradh Exp $ */
/*-
* Copyright (c) 2006, 2007, 2019 The NetBSD Foundation, Inc.
@@ -41,7 +41,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.27 2020/05/23 23:42:42 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.28 2022/02/27 14:16:12 riastradh Exp $");
#include <sys/types.h>
#include <sys/param.h>
@@ -54,6 +54,7 @@
#include <sys/cpu.h>
#include <sys/syslog.h>
#include <sys/atomic.h>
+#include <sys/xcall.h>
#include <dev/lockstat.h>
@@ -101,6 +102,7 @@
volatile u_int lockstat_enabled;
volatile u_int lockstat_dev_enabled;
+__cpu_simple_lock_t lockstat_enabled_lock;
uintptr_t lockstat_csstart;
uintptr_t lockstat_csend;
uintptr_t lockstat_csmask;
@@ -154,6 +156,7 @@
(void)nunits;
__cpu_simple_lock_init(&lockstat_lock);
+ __cpu_simple_lock_init(&lockstat_enabled_lock);
}
/*
@@ -235,10 +238,26 @@
lockstat_lockstart = le->le_lockstart;
lockstat_lockstart = le->le_lockstart;
lockstat_lockend = le->le_lockend;
- membar_sync();
+
+ /*
+ * Ensure everything is initialized on all CPUs, by issuing a
+ * null xcall with the side effect of a release barrier on this
+ * CPU and an acquire barrier on all other CPUs, before they
+ * can witness any flags set in lockstat_dev_enabled -- this
+ * way we don't need to add any barriers in lockstat_event.
+ */
+ xc_barrier(0);
+
+ /*
+ * Start timing after the xcall, so we don't spuriously count
+ * xcall communication time, but before flipping the switch, so
+ * we don't dirty sample with locks taken in the timecounter.
+ */
getnanotime(&lockstat_stime);
- lockstat_dev_enabled = le->le_mask;
- LOCKSTAT_ENABLED_UPDATE();
+
+ LOCKSTAT_ENABLED_UPDATE_BEGIN();
+ atomic_store_relaxed(&lockstat_dev_enabled, le->le_mask);
+ LOCKSTAT_ENABLED_UPDATE_END();
}
/*
@@ -258,13 +277,13 @@
KASSERT(lockstat_dev_enabled);
/*
- * Set enabled false, force a write barrier, and wait for other CPUs
- * to exit lockstat_event().
+ * Disable and wait for other CPUs to exit lockstat_event().
*/
- lockstat_dev_enabled = 0;
- LOCKSTAT_ENABLED_UPDATE();
+ LOCKSTAT_ENABLED_UPDATE_BEGIN();
+ atomic_store_relaxed(&lockstat_dev_enabled, 0);
+ LOCKSTAT_ENABLED_UPDATE_END();
getnanotime(&ts);
- tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10));
+ xc_barrier(0);
/*
* Did we run out of buffers while tracing?
@@ -370,12 +389,14 @@
#ifdef KDTRACE_HOOKS
uint32_t id;
CTASSERT((LS_NPROBES & (LS_NPROBES - 1)) == 0);
- if ((id = lockstat_probemap[LS_COMPRESS(flags)]) != 0)
+ if ((id = atomic_load_relaxed(&lockstat_probemap[LS_COMPRESS(flags)]))
+ != 0)
(*lockstat_probe_func)(id, lock, callsite, flags, count,
cycles);
#endif
- if ((flags & lockstat_dev_enabled) != flags || count == 0)
+ if ((flags & atomic_load_relaxed(&lockstat_dev_enabled)) != flags ||
+ count == 0)
return;
if (lock < lockstat_lockstart || lock > lockstat_lockend)
return;
@@ -487,7 +508,7 @@
error = ENODEV;
break;
}
- if (lockstat_dev_enabled) {
+ if (atomic_load_relaxed(&lockstat_dev_enabled)) {
error = EBUSY;
break;
}
@@ -524,7 +545,7 @@
break;
case IOC_LOCKSTAT_DISABLE:
- if (!lockstat_dev_enabled)
+ if (!atomic_load_relaxed(&lockstat_dev_enabled))
error = EINVAL;
else
error = lockstat_stop((lsdisable_t *)data);
diff -r 922a89c71a77 -r 79a6b890bd3e sys/dev/lockstat.h
--- a/sys/dev/lockstat.h Sun Feb 27 12:27:22 2022 +0000
+++ b/sys/dev/lockstat.h Sun Feb 27 14:16:12 2022 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: lockstat.h,v 1.14 2016/01/24 01:01:11 christos Exp $ */
+/* $NetBSD: lockstat.h,v 1.15 2022/02/27 14:16:12 riastradh Exp $ */
/*-
* Copyright (c) 2006 The NetBSD Foundation, Inc.
@@ -38,7 +38,9 @@
#endif
#include <sys/types.h>
+
#include <sys/ioccom.h>
+#include <sys/lock.h>
#include <sys/queue.h>
#include <sys/time.h>
@@ -159,7 +161,7 @@
#define LOCKSTAT_TIMER(name) uint64_t name = 0
#define LOCKSTAT_COUNTER(name) uint64_t name = 0
#define LOCKSTAT_FLAG(name) int name
-#define LOCKSTAT_ENTER(name) name = lockstat_enabled
+#define LOCKSTAT_ENTER(name) name = atomic_load_relaxed(&lockstat_enabled)
#define LOCKSTAT_EXIT(name)
#define LOCKSTAT_START_TIMER(flag, name) \
@@ -214,13 +216,22 @@
#endif
#if defined(_KERNEL) && NLOCKSTAT > 0
+extern __cpu_simple_lock_t lockstat_enabled_lock;
extern volatile u_int lockstat_enabled;
extern volatile u_int lockstat_dev_enabled;
-#define LOCKSTAT_ENABLED_UPDATE() do { \
- lockstat_enabled = lockstat_dev_enabled | KDTRACE_LOCKSTAT_ENABLED; \
- membar_producer(); \
- } while (/*CONSTCOND*/0)
+#define LOCKSTAT_ENABLED_UPDATE_BEGIN() do \
+{ \
+ __cpu_simple_lock(&lockstat_enabled_lock); \
+} while (/*CONSTCOND*/0)
+
+#define LOCKSTAT_ENABLED_UPDATE_END() do \
+{ \
+ atomic_store_relaxed(&lockstat_enabled, \
+ lockstat_dev_enabled | KDTRACE_LOCKSTAT_ENABLED); \
+ __cpu_simple_unlock(&lockstat_enabled_lock); \
+} while (/*CONSTCOND*/0)
+
#endif
#endif /* _SYS_LOCKSTAT_H_ */
Home |
Main Index |
Thread Index |
Old Index