Re: percpu_foreach() does not execute remotely

[Jason Thorpe <thorpej%me.com@localhost>]

> On Jan 28, 2020, at 9:20 AM, Taylor R Campbell <campbell+netbsd-tech-kern%mumble.net@localhost> wrote:
> 
> OK, that sounds reasonable.  (I misunderstood your first message to
> mean that percpu_foreach_xcall would synchronize with _other activity_
> on each CPU, rather than that it would serialize the xcall with itself
> on different CPUs.)

Something like this.  I haven't tested this yet, but I will shortly.  Only high-pri xcalls are supported in this -- to make low-pri xcalls work, you'd need to change how percpu_cpu_swap() interlocks with percpu_foreach().  High-pri doesn't need to take the percpu_swap_lock because percpu_cpu_swap() protects its critical section with splhigh().

Index: kern/subr_percpu.c
===================================================================
RCS file: /cvsroot/src/sys/kern/subr_percpu.c,v
retrieving revision 1.20
diff -u -p -r1.20 subr_percpu.c
--- kern/subr_percpu.c	5 Dec 2019 03:21:08 -0000	1.20
+++ kern/subr_percpu.c	29 Jan 2020 02:50:48 -0000
@@ -350,7 +350,9 @@ percpu_getptr_remote(percpu_t *pc, struc
 /*
  * percpu_foreach: call the specified callback function for each cpus.
  *
- * => called in thread context.
+ * => must be called from thread context.
+ * => callback executes on **current** CPU (or, really, arbitrary CPU,
+ *    in case of preemption)
  * => caller should not rely on the cpu iteration order.
  * => the callback function should be minimum because it is executed with
  *    holding a global lock, which can block low-priority xcalls.
@@ -368,3 +370,71 @@ percpu_foreach(percpu_t *pc, percpu_call
 	}
 	percpu_traverse_exit();
 }
+
+struct percpu_xcall_ctx {
+	percpu_t	  *ctx_pc;
+	percpu_callback_t  ctx_cb;
+	void		  *ctx_arg;
+};
+
+static void
+percpu_xcfunc(void * const v1, void * const v2 __unused)
+{
+	struct percpu_xcall_ctx * const ctx = v1;
+	struct cpu_info * const ci = curcpu();
+
+	/*
+	 * We are running at the ipl specified in the call to
+	 * percpu_foreach_xcall().  The callback can raise ipl
+	 * if needed to serialize with hard interrupts.
+	 */
+	(*ctx->ctx_cb)(percpu_getptr_remote(ctx->ctx_pc, ci), ctx->ctx_arg, ci);
+}
+
+/*
+ * percpu_foreach_xcall: call the specified callback function for each
+ * cpu.  This version uses an xcall to run the callback on each cpu.
+ *
+ * => must be called from thread context.
+ * => callback executes on **remote** CPU in soft-interrupt context
+ *    (at the specified soft interrupt priority).
+ * => caller should not rely on the cpu iteration order.
+ * => the callback function should be minimum because it is executed
+ *    in soft-interrupt context.  eg. it's illegal for a callback
+ *    function to sleep for memory allocation.
+ */
+void
+percpu_foreach_xcall(percpu_t *pc, int ipl, percpu_callback_t cb, void *arg)
+{
+	struct percpu_xcall_ctx ctx = {
+		.ctx_pc = pc,
+		.ctx_cb = cb,
+		.ctx_arg = arg,
+	};
+	CPU_INFO_ITERATOR cii;
+	struct cpu_info *ci;
+	uint64_t ticket;
+
+	/* don't use a switch statement, because there might be duplicates. */
+	KASSERT(ipl == IPL_SOFTSERIAL || ipl == IPL_SOFTNET ||
+		ipl == IPL_SOFTBIO    || ipl == IPL_SOFTCLOCK);
+
+	/*
+	 * We iterate manually and xc_unicast rather than xc_broadcast in
+	 * order to to serialize the access to the callback argument.
+	 *
+	 * We do not take the percpu_swap_lock because percpu_cpu_swap()
+	 * goes to splhigh during the critical section, thus serializing
+	 * against our xcall softint.
+	 *
+	 * XXX We could use xc_broadcast and serialize invoking the
+	 * callback using a spin mutex, but that seems like it would
+	 * hog the xcall mechanism.  This way at least allows others
+	 * to sneak in and make progress while we do our work.
+	 */
+	for (CPU_INFO_FOREACH(cii, ci)) {
+		ticket = xc_unicast(XC_HIGHPRI_IPL(ipl),
+		    percpu_xcfunc, &ctx, NULL, ci);
+		xc_wait(ticket);
+	}
+}
Index: sys/percpu.h
===================================================================
RCS file: /cvsroot/src/sys/sys/percpu.h,v
retrieving revision 1.3
diff -u -p -r1.3 percpu.h
--- sys/percpu.h	9 Apr 2008 05:11:20 -0000	1.3
+++ sys/percpu.h	29 Jan 2020 02:50:48 -0000
@@ -40,6 +40,7 @@ void	percpu_putref(percpu_t *);
 
 typedef void (*percpu_callback_t)(void *, void *, struct cpu_info *);
 void	percpu_foreach(percpu_t *, percpu_callback_t, void *);
+void	percpu_foreach_xcall(percpu_t *, int, percpu_callback_t, void *);
 
 /* low-level api; don't use unless necessary */
 void	percpu_traverse_enter(void);

-- thorpej