[src/trunk]: src/sys/rump Allow compile-time optimizations to curlwp. This c...

[pooka <pooka%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/80121280d75e
branches:  trunk
changeset: 794472:80121280d75e
user:      pooka <pooka%NetBSD.org@localhost>
date:      Sat Mar 15 15:15:26 2014 +0000

description:
Allow compile-time optimizations to curlwp.  This can have a pretty
staggering impact on performance.  When running sendto() in a loop,
the improvement is 200k more calls per second with an inlined __thread
curlwp as opposed to the default.  In other words, it shaves off hundreds
of CPU cycles per call (~20%).  Even just eliminating the x86_curlwp()
call in favor of an inline gives an improvement of 60k calls per second.

diffstat:

 sys/rump/Makefile.rump                            |   7 +-
 sys/rump/include/machine/cpu.h                    |   7 +-
 sys/rump/include/rump/rumpuser.h                  |   4 +-
 sys/rump/librump/rumpkern/arch/x86/rump_x86_cpu.c |   7 +-
 sys/rump/librump/rumpkern/lwproc.c                |  46 ++++++++++++++--
 sys/rump/librump/rumpkern/rump.c                  |  12 +--
 sys/rump/librump/rumpkern/rump_curlwp.h           |  54 +++++++++++++++++++
 sys/rump/librump/rumpkern/rump_curlwp___thread.h  |  63 +++++++++++++++++++++++
 sys/rump/librump/rumpkern/rump_curlwp_hypercall.h |  43 +++++++++++++++
 sys/rump/librump/rumpkern/rump_private.h          |   6 +-
 sys/rump/librump/rumpkern/scheduler.c             |  14 ++--
 sys/rump/librump/rumpkern/threads.c               |   6 +-
 12 files changed, 233 insertions(+), 36 deletions(-)

diffs (truncated from 527 to 300 lines):

diff -r 275627aa60cc -r 80121280d75e sys/rump/Makefile.rump
--- a/sys/rump/Makefile.rump    Sat Mar 15 14:12:56 2014 +0000
+++ b/sys/rump/Makefile.rump    Sat Mar 15 15:15:26 2014 +0000
@@ -1,4 +1,4 @@
-#      $NetBSD: Makefile.rump,v 1.91 2014/03/13 01:34:06 pooka Exp $
+#      $NetBSD: Makefile.rump,v 1.92 2014/03/15 15:15:26 pooka Exp $
 #
 
 WARNS?=                3       # XXX: src/sys won't compile with -Wsign-compare yet
@@ -8,14 +8,13 @@
 
 # Use NetBSD kernel ABI by default on x86 archs.  Performance-related
 # compile-time options may override this at a later date.
-.if ${MACHINE_ARCH} == "i386" || \
-    ${MACHINE_ARCH} == "x86_64"
+.if (${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "x86_64")
 _RUMP_NATIVEABI= yes
 CPPFLAGS+=     -D_RUMP_NATIVE_ABI
 .else
 _RUMP_NATIVEABI= no
 CPPFLAGS:=     -I${RUMPTOP}/include ${CPPFLAGS}
-CPPFLAGS+=     -D_RUMPKERNEL
+CPPFLAGS+=     -D_RUMPKERNEL -I${RUMPTOP}/librump/rumpkern
 .endif
 
 CPPFLAGS+=     -DMAXUSERS=32
diff -r 275627aa60cc -r 80121280d75e sys/rump/include/machine/cpu.h
--- a/sys/rump/include/machine/cpu.h    Sat Mar 15 14:12:56 2014 +0000
+++ b/sys/rump/include/machine/cpu.h    Sat Mar 15 15:15:26 2014 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpu.h,v 1.19 2014/03/10 23:02:07 pooka Exp $   */
+/*     $NetBSD: cpu.h,v 1.20 2014/03/15 15:15:26 pooka Exp $   */
 
 /*
  * Copyright (c) 2008-2011 Antti Kantee.  All Rights Reserved.
@@ -38,6 +38,8 @@
 #include <sys/cpu_data.h>
 #include <machine/pcb.h>
 
+#include "rump_curlwp.h"
+
 struct cpu_info {
        struct cpu_data ci_data;
        cpuid_t ci_cpuid;
@@ -73,8 +75,7 @@
 void __syncicache(void *, size_t);
 #endif /* __powerpc__ */
 
-struct lwp *rumpuser_curlwp(void);
-#define curlwp rumpuser_curlwp()
+#define curlwp rump_curlwp_fast()
 
 #define curcpu() (curlwp->l_cpu)
 #define cpu_number() (cpu_index(curcpu))
diff -r 275627aa60cc -r 80121280d75e sys/rump/include/rump/rumpuser.h
--- a/sys/rump/include/rump/rumpuser.h  Sat Mar 15 14:12:56 2014 +0000
+++ b/sys/rump/include/rump/rumpuser.h  Sat Mar 15 15:15:26 2014 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: rumpuser.h,v 1.108 2013/05/15 16:00:04 pooka Exp $     */
+/*     $NetBSD: rumpuser.h,v 1.109 2014/03/15 15:15:26 pooka Exp $     */
 
 /*
  * Copyright (c) 2007-2013 Antti Kantee.  All Rights Reserved.
@@ -175,12 +175,14 @@
 void rumpuser_thread_exit(void) __dead;
 int  rumpuser_thread_join(void *);
 
+#if defined(LIBRUMPUSER) || defined(RUMP_CURLWP_PRIVATE)
 enum rumplwpop {
        RUMPUSER_LWP_CREATE, RUMPUSER_LWP_DESTROY,
        RUMPUSER_LWP_SET, RUMPUSER_LWP_CLEAR
 };
 void rumpuser_curlwpop(int, struct lwp *);
 struct lwp *rumpuser_curlwp(void);
+#endif /* LIBRUMPUSER || RUMP_CURLWP_PRIVATE */
 
 struct rumpuser_mtx;
 #define RUMPUSER_MTX_SPIN      0x01
diff -r 275627aa60cc -r 80121280d75e sys/rump/librump/rumpkern/arch/x86/rump_x86_cpu.c
--- a/sys/rump/librump/rumpkern/arch/x86/rump_x86_cpu.c Sat Mar 15 14:12:56 2014 +0000
+++ b/sys/rump/librump/rumpkern/arch/x86/rump_x86_cpu.c Sat Mar 15 15:15:26 2014 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: rump_x86_cpu.c,v 1.1 2014/02/12 22:28:43 pooka Exp $   */
+/*     $NetBSD: rump_x86_cpu.c,v 1.2 2014/03/15 15:15:27 pooka Exp $   */
 
 /*
  * Copyright (c) 2008 Antti Kantee.  All Rights Reserved.
@@ -29,13 +29,14 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: rump_x86_cpu.c,v 1.1 2014/02/12 22:28:43 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: rump_x86_cpu.c,v 1.2 2014/03/15 15:15:27 pooka Exp $");
 
 #include <sys/param.h>
 
 #include <machine/cpu.h>
 
 #include "rump_private.h"
+#include "rump_curlwp.h"
 
 struct cpu_info *cpu_info_list;
 
@@ -65,7 +66,7 @@
 x86_curlwp()
 {
 
-       return rumpuser_curlwp();
+       return rump_curlwp_fast();
 }
 
 void
diff -r 275627aa60cc -r 80121280d75e sys/rump/librump/rumpkern/lwproc.c
--- a/sys/rump/librump/rumpkern/lwproc.c        Sat Mar 15 14:12:56 2014 +0000
+++ b/sys/rump/librump/rumpkern/lwproc.c        Sat Mar 15 15:15:26 2014 +0000
@@ -1,4 +1,4 @@
-/*      $NetBSD: lwproc.c,v 1.26 2013/12/16 15:36:29 pooka Exp $       */
+/*      $NetBSD: lwproc.c,v 1.27 2014/03/15 15:15:27 pooka Exp $       */
 
 /*
  * Copyright (c) 2010, 2011 Antti Kantee.  All Rights Reserved.
@@ -25,8 +25,10 @@
  * SUCH DAMAGE.
  */
 
+#define RUMP_CURLWP_PRIVATE
+
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lwproc.c,v 1.26 2013/12/16 15:36:29 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lwproc.c,v 1.27 2014/03/15 15:15:27 pooka Exp $");
 
 #include <sys/param.h>
 #include <sys/atomic.h>
@@ -42,11 +44,41 @@
 #include <sys/uidinfo.h>
 
 #include <rump/rumpuser.h>
-
 #include "rump_private.h"
+#include "rump_curlwp.h"
 
 struct emul *emul_default = &emul_netbsd;
 
+void
+rump_lwproc_init(void)
+{
+
+       lwproc_curlwpop(RUMPUSER_LWP_CREATE, &lwp0);
+}
+
+struct lwp *
+rump_lwproc_curlwp_hypercall(void)
+{
+
+       return rumpuser_curlwp();
+}
+
+void
+rump_lwproc_curlwp_set(struct lwp *l)
+{
+
+       KASSERT(curlwp == NULL);
+       lwproc_curlwpop(RUMPUSER_LWP_SET, l);
+}
+
+void
+rump_lwproc_curlwp_clear(struct lwp *l)
+{
+
+       KASSERT(l == curlwp);
+       lwproc_curlwpop(RUMPUSER_LWP_CLEAR, l);
+}
+
 static void
 lwproc_proc_free(struct proc *p)
 {
@@ -219,7 +251,7 @@
                kmem_free(l->l_name, MAXCOMLEN);
        lwp_finispecific(l);
 
-       rumpuser_curlwpop(RUMPUSER_LWP_DESTROY, l);
+       lwproc_curlwpop(RUMPUSER_LWP_DESTROY, l);
        membar_exit();
        kmem_free(l, sizeof(*l));
 
@@ -255,7 +287,7 @@
        lwp_initspecific(l);
 
        membar_enter();
-       rumpuser_curlwpop(RUMPUSER_LWP_CREATE, l);
+       lwproc_curlwpop(RUMPUSER_LWP_CREATE, l);
        if (doswitch) {
                rump_lwproc_switch(l);
        }
@@ -364,13 +396,13 @@
        }
 
        KERNEL_UNLOCK_ALL(NULL, &l->l_biglocks);
-       rumpuser_curlwpop(RUMPUSER_LWP_CLEAR, l);
+       lwproc_curlwpop(RUMPUSER_LWP_CLEAR, l);
 
        newlwp->l_cpu = newlwp->l_target_cpu = l->l_cpu;
        newlwp->l_mutex = l->l_mutex;
        newlwp->l_pflag |= LP_RUNNING;
 
-       rumpuser_curlwpop(RUMPUSER_LWP_SET, newlwp);
+       lwproc_curlwpop(RUMPUSER_LWP_SET, newlwp);
        curcpu()->ci_curlwp = newlwp;
        KERNEL_LOCK(newlwp->l_biglocks, NULL);
 
diff -r 275627aa60cc -r 80121280d75e sys/rump/librump/rumpkern/rump.c
--- a/sys/rump/librump/rumpkern/rump.c  Sat Mar 15 14:12:56 2014 +0000
+++ b/sys/rump/librump/rumpkern/rump.c  Sat Mar 15 15:15:26 2014 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: rump.c,v 1.289 2014/03/10 22:44:11 pooka Exp $ */
+/*     $NetBSD: rump.c,v 1.290 2014/03/15 15:15:27 pooka Exp $ */
 
 /*
  * Copyright (c) 2007-2011 Antti Kantee.  All Rights Reserved.
@@ -26,7 +26,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: rump.c,v 1.289 2014/03/10 22:44:11 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: rump.c,v 1.290 2014/03/15 15:15:27 pooka Exp $");
 
 #include <sys/systm.h>
 #define ELFSIZE ARCH_ELFSIZE
@@ -254,12 +254,10 @@
        }
 
        /* init minimal lwp/cpu context */
+       rump_lwproc_init();
        l = &lwp0;
        l->l_cpu = l->l_target_cpu = rump_cpu;
-
-       /* lwp0 isn't created like other threads, so notify hypervisor here */
-       rumpuser_curlwpop(RUMPUSER_LWP_CREATE, l);
-       rumpuser_curlwpop(RUMPUSER_LWP_SET, l);
+       rump_lwproc_curlwp_set(l);
 
        /* retrieve env vars which affect the early stage of bootstrap */
        if (rumpuser_getparam("RUMP_THREADS", buf, sizeof(buf)) == 0) {
@@ -341,7 +339,7 @@
 
        rump_scheduler_init(numcpu);
        /* revert temporary context and schedule a semireal context */
-       rumpuser_curlwpop(RUMPUSER_LWP_CLEAR, l);
+       rump_lwproc_curlwp_clear(l);
        initproc = &proc0; /* borrow proc0 before we get initproc started */
        rump_schedule();
        bootlwp = curlwp;
diff -r 275627aa60cc -r 80121280d75e sys/rump/librump/rumpkern/rump_curlwp.h
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/sys/rump/librump/rumpkern/rump_curlwp.h   Sat Mar 15 15:15:26 2014 +0000
@@ -0,0 +1,54 @@
+/*     $NetBSD: rump_curlwp.h,v 1.1 2014/03/15 15:15:27 pooka Exp $    */
+
+/*-
+ * Copyright (c) 2014 Antti Kantee.  All Rights Reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _SYS_RUMP_CURLWP_H_
+#define _SYS_RUMP_CURLWP_H_
+
+struct lwp *    rump_lwproc_curlwp_hypercall(void);
+
+/* hattrick numbers to avoid someone accidentally using "1" as the value */
+#define RUMP_CURLWP_MODEL_HYPERCALL     10501
+#define RUMP_CURLWP_MODEL___THREAD      20502
+#define RUMP_CURLWP_MODEL_REGISTER      30503
+#define RUMP_CURLWP_MODEL_DEFAULT       RUMP_CURLWP_MODEL_HYPERCALL
+
+#ifndef RUMP_CURLWP_MODEL
+#define RUMP_CURLWP_MODEL RUMP_CURLWP_MODEL_DEFAULT
+#endif
+
+/* provides rump_curlwp_fast() */
+#if RUMP_CURLWP_MODEL == RUMP_CURLWP_MODEL_HYPERCALL