[src/trunk]: src/sys/kern ipi: Membar audit.

[riastradh <riastradh%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/f6ce66016e57
branches:  trunk
changeset: 978632:f6ce66016e57
user:      riastradh <riastradh%NetBSD.org@localhost>
date:      Fri Nov 27 20:11:33 2020 +0000

description:
ipi: Membar audit.

diffstat:

 sys/kern/subr_ipi.c |  42 ++++++++++++++++++++++++++----------------
 1 files changed, 26 insertions(+), 16 deletions(-)

diffs (132 lines):

diff -r 6f656d2e01da -r f6ce66016e57 sys/kern/subr_ipi.c
--- a/sys/kern/subr_ipi.c       Fri Nov 27 20:10:25 2020 +0000
+++ b/sys/kern/subr_ipi.c       Fri Nov 27 20:11:33 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: subr_ipi.c,v 1.8 2020/09/08 16:00:35 riastradh Exp $   */
+/*     $NetBSD: subr_ipi.c,v 1.9 2020/11/27 20:11:33 riastradh Exp $   */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -36,7 +36,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.8 2020/09/08 16:00:35 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.9 2020/11/27 20:11:33 riastradh Exp $");
 
 #include <sys/param.h>
 #include <sys/types.h>
@@ -187,8 +187,11 @@
        KASSERT(ipi_id < IPI_MAXREG);
        KASSERT(kpreempt_disabled());
 
-       /* Mark as pending and send an IPI. */
-       if (membar_consumer(), (ci->ci_ipipend[i] & bitm) == 0) {
+       /* Mark as pending and return true if not previously marked. */
+       if ((atomic_load_acquire(&ci->ci_ipipend[i]) & bitm) == 0) {
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+               membar_exit();
+#endif
                atomic_or_32(&ci->ci_ipipend[i], bitm);
                return true;
        }
@@ -261,6 +264,8 @@
 
 /*
  * put_msg: insert message into the mailbox.
+ *
+ * Caller is responsible for issuing membar_exit first.
  */
 static inline void
 put_msg(ipi_mbox_t *mbox, ipi_msg_t *msg)
@@ -268,8 +273,7 @@
        int count = SPINLOCK_BACKOFF_MIN;
 again:
        for (u_int i = 0; i < IPI_MSG_MAX; i++) {
-               if (__predict_true(mbox->msg[i] == NULL) &&
-                   atomic_cas_ptr(&mbox->msg[i], NULL, msg) == NULL) {
+               if (atomic_cas_ptr(&mbox->msg[i], NULL, msg) == NULL) {
                        return;
                }
        }
@@ -295,12 +299,12 @@
        for (u_int i = 0; i < IPI_BITWORDS; i++) {
                uint32_t pending, bit;
 
-               if (ci->ci_ipipend[i] == 0) {
+               if (atomic_load_relaxed(&ci->ci_ipipend[i]) == 0) {
                        continue;
                }
                pending = atomic_swap_32(&ci->ci_ipipend[i], 0);
 #ifndef __HAVE_ATOMIC_AS_MEMBAR
-               membar_producer();
+               membar_enter();
 #endif
                while ((bit = ffs(pending)) != 0) {
                        const u_int ipi_id = (i << IPI_BITW_SHIFT) | --bit;
@@ -327,10 +331,10 @@
                ipi_msg_t *msg;
 
                /* Get the message. */
-               if ((msg = mbox->msg[i]) == NULL) {
+               if ((msg = atomic_load_acquire(&mbox->msg[i])) == NULL) {
                        continue;
                }
-               mbox->msg[i] = NULL;
+               atomic_store_relaxed(&mbox->msg[i], NULL);
 
                /* Execute the handler. */
                KASSERT(msg->func);
@@ -338,7 +342,7 @@
 
                /* Ack the request. */
 #ifndef __HAVE_ATOMIC_AS_MEMBAR
-               membar_producer();
+               membar_exit();
 #endif
                atomic_dec_uint(&msg->_pending);
        }
@@ -360,7 +364,9 @@
        KASSERT(curcpu() != ci);
 
        msg->_pending = 1;
-       membar_producer();
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+       membar_exit();
+#endif
 
        put_msg(&ipi_mboxes[id], msg);
        ipi_trigger(IPI_SYNCH_ID, ci);
@@ -384,7 +390,9 @@
 
        local = !!kcpuset_isset(target, cpu_index(self));
        msg->_pending = kcpuset_countset(target) - local;
-       membar_producer();
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+       membar_exit();
+#endif
 
        for (CPU_INFO_FOREACH(cii, ci)) {
                cpuid_t id;
@@ -420,7 +428,9 @@
        KASSERT(kpreempt_disabled());
 
        msg->_pending = ncpu - 1;
-       membar_producer();
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+       membar_exit();
+#endif
 
        /* Broadcast IPIs for remote CPUs. */
        for (CPU_INFO_FOREACH(cii, ci)) {
@@ -448,8 +458,8 @@
 {
        int count = SPINLOCK_BACKOFF_MIN;
 
-       while (msg->_pending) {
-               KASSERT(msg->_pending < ncpu);
+       while (atomic_load_acquire(&msg->_pending)) {
+               KASSERT(atomic_load_relaxed(&msg->_pending) < ncpu);
                SPINLOCK_BACKOFF(count);
        }
 }