[src/trunk]: src/sys/arch Improve the SVS initialization.

[maxv <maxv%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/235c448685cd
branches:  trunk
changeset: 359719:235c448685cd
user:      maxv <maxv%NetBSD.org@localhost>
date:      Thu Feb 22 09:41:06 2018 +0000

description:
Improve the SVS initialization.

Declare x86_patch_window_open() and x86_patch_window_close(), and globalify
x86_hotpatch().

Introduce svs_enable() in x86/svs.c, that does the SVS hotpatching.

Change svs_init() to take a bool. This function gets called twice; early
when the system just booted (and nothing is initialized), lately when at
least pmap_kernel has been initialized.

diffstat:

 sys/arch/amd64/amd64/machdep.c |   5 +-
 sys/arch/amd64/include/pmap.h  |   3 +-
 sys/arch/x86/include/cpufunc.h |   5 ++-
 sys/arch/x86/x86/patch.c       |  78 +++++++++++++++--------------------------
 sys/arch/x86/x86/svs.c         |  56 +++++++++++++++++++++++++++--
 sys/arch/x86/x86/x86_machdep.c |   7 +--
 6 files changed, 92 insertions(+), 62 deletions(-)

diffs (296 lines):

diff -r b488f471f20f -r 235c448685cd sys/arch/amd64/amd64/machdep.c
--- a/sys/arch/amd64/amd64/machdep.c    Thu Feb 22 08:56:51 2018 +0000
+++ b/sys/arch/amd64/amd64/machdep.c    Thu Feb 22 09:41:06 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: machdep.c,v 1.298 2018/02/11 09:39:36 maxv Exp $       */
+/*     $NetBSD: machdep.c,v 1.299 2018/02/22 09:41:06 maxv Exp $       */
 
 /*
  * Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.298 2018/02/11 09:39:36 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.299 2018/02/22 09:41:06 maxv Exp $");
 
 /* #define XENDEBUG_LOW  */
 
@@ -1598,6 +1598,7 @@
        uvm_lwp_setuarea(&lwp0, lwp0uarea);
 
        cpu_probe(&cpu_info_primary);
+       svs_init(true);
        cpu_init_msrs(&cpu_info_primary, true);
 
        use_pae = 1; /* PAE always enabled in long mode */
diff -r b488f471f20f -r 235c448685cd sys/arch/amd64/include/pmap.h
--- a/sys/arch/amd64/include/pmap.h     Thu Feb 22 08:56:51 2018 +0000
+++ b/sys/arch/amd64/include/pmap.h     Thu Feb 22 09:41:06 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: pmap.h,v 1.43 2018/02/18 14:07:29 maxv Exp $   */
+/*     $NetBSD: pmap.h,v 1.44 2018/02/22 09:41:06 maxv Exp $   */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -221,6 +221,7 @@
 void svs_pmap_sync(struct pmap *, int);
 void svs_lwp_switch(struct lwp *, struct lwp *);
 void svs_pdir_switch(struct pmap *);
+void svs_init(bool);
 extern bool svs_enabled;
 
 #include <x86/pmap.h>
diff -r b488f471f20f -r 235c448685cd sys/arch/x86/include/cpufunc.h
--- a/sys/arch/x86/include/cpufunc.h    Thu Feb 22 08:56:51 2018 +0000
+++ b/sys/arch/x86/include/cpufunc.h    Thu Feb 22 09:41:06 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpufunc.h,v 1.23 2017/10/15 11:31:00 maxv Exp $        */
+/*     $NetBSD: cpufunc.h,v 1.24 2018/02/22 09:41:06 maxv Exp $        */
 
 /*-
  * Copyright (c) 1998, 2007 The NetBSD Foundation, Inc.
@@ -50,6 +50,9 @@
 void   x86_mfence(void);
 void   x86_flush(void);
 #ifndef XEN
+void   x86_hotpatch(uint32_t, const uint8_t *, size_t);
+void   x86_patch_window_open(u_long *, u_long *);
+void   x86_patch_window_close(u_long, u_long);
 void   x86_patch(bool);
 #endif
 void   invlpg(vaddr_t);
diff -r b488f471f20f -r 235c448685cd sys/arch/x86/x86/patch.c
--- a/sys/arch/x86/x86/patch.c  Thu Feb 22 08:56:51 2018 +0000
+++ b/sys/arch/x86/x86/patch.c  Thu Feb 22 09:41:06 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: patch.c,v 1.32 2018/02/22 08:56:52 maxv Exp $  */
+/*     $NetBSD: patch.c,v 1.33 2018/02/22 09:41:06 maxv Exp $  */
 
 /*-
  * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.32 2018/02/22 08:56:52 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.33 2018/02/22 09:41:06 maxv Exp $");
 
 #include "opt_lockdebug.h"
 #ifdef i386
@@ -143,7 +143,7 @@
        }
 }
 
-static void
+void
 x86_hotpatch(uint32_t name, const uint8_t *bytes, size_t size)
 {
        extern char __rodata_hotpatch_start;
@@ -165,6 +165,30 @@
 }
 
 void
+x86_patch_window_open(u_long *psl, u_long *cr0)
+{
+       /* Disable interrupts. */
+       *psl = x86_read_psl();
+       x86_disable_intr();
+
+       /* Disable write protection in supervisor mode. */
+       *cr0 = rcr0();
+       lcr0(*cr0 & ~CR0_WP);
+}
+
+void
+x86_patch_window_close(u_long psl, u_long cr0)
+{
+       /* Write back and invalidate cache, flush pipelines. */
+       wbinvd();
+       x86_flush();
+       x86_write_psl(psl);
+
+       /* Re-enable write protection. */
+       lcr0(cr0);
+}
+
+void
 x86_patch(bool early)
 {
        static bool first, second;
@@ -181,13 +205,7 @@
                second = true;
        }
 
-       /* Disable interrupts. */
-       psl = x86_read_psl();
-       x86_disable_intr();
-
-       /* Disable write protection in supervisor mode. */
-       cr0 = rcr0();
-       lcr0(cr0 & ~CR0_WP);
+       x86_patch_window_open(&psl, &cr0);
 
 #if !defined(GPROF)
        if (!early && ncpu == 1) {
@@ -298,43 +316,5 @@
                x86_hotpatch(HP_NAME_STAC, stac_bytes, sizeof(stac_bytes));
        }
 
-#ifdef SVS
-       if (early && cpu_vendor == CPUVENDOR_INTEL) {
-               extern uint8_t svs_enter, svs_enter_end;
-               extern uint8_t svs_enter_altstack, svs_enter_altstack_end;
-               extern uint8_t svs_leave, svs_leave_end;
-               extern uint8_t svs_leave_altstack, svs_leave_altstack_end;
-               extern bool svs_enabled;
-               uint8_t *bytes;
-               size_t size;
-
-               svs_enabled = true;
-
-               bytes = &svs_enter;
-               size = (size_t)&svs_enter_end - (size_t)&svs_enter;
-               x86_hotpatch(HP_NAME_SVS_ENTER, bytes, size);
-
-               bytes = &svs_enter_altstack;
-               size = (size_t)&svs_enter_altstack_end -
-                   (size_t)&svs_enter_altstack;
-               x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size);
-
-               bytes = &svs_leave;
-               size = (size_t)&svs_leave_end - (size_t)&svs_leave;
-               x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size);
-
-               bytes = &svs_leave_altstack;
-               size = (size_t)&svs_leave_altstack_end -
-                   (size_t)&svs_leave_altstack;
-               x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size);
-       }
-#endif
-
-       /* Write back and invalidate cache, flush pipelines. */
-       wbinvd();
-       x86_flush();
-       x86_write_psl(psl);
-
-       /* Re-enable write protection. */
-       lcr0(cr0);
+       x86_patch_window_close(psl, cr0);
 }
diff -r b488f471f20f -r 235c448685cd sys/arch/x86/x86/svs.c
--- a/sys/arch/x86/x86/svs.c    Thu Feb 22 08:56:51 2018 +0000
+++ b/sys/arch/x86/x86/svs.c    Thu Feb 22 09:41:06 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: svs.c,v 1.4 2018/02/22 08:56:52 maxv Exp $     */
+/*     $NetBSD: svs.c,v 1.5 2018/02/22 09:41:06 maxv Exp $     */
 
 /*
  * Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.4 2018/02/22 08:56:52 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.5 2018/02/22 09:41:06 maxv Exp $");
 
 #include "opt_svs.h"
 
@@ -39,7 +39,9 @@
 #include <sys/proc.h>
 #include <sys/cpu.h>
 
+#include <x86/cputypes.h>
 #include <machine/cpuvar.h>
+#include <machine/frameasm.h>
 
 #include <uvm/uvm.h>
 #include <uvm/uvm_page.h>
@@ -503,12 +505,56 @@
        tlbflushg();
 }
 
-void svs_init(void);
+static void
+svs_enable(void)
+{
+       extern uint8_t svs_enter, svs_enter_end;
+       extern uint8_t svs_enter_altstack, svs_enter_altstack_end;
+       extern uint8_t svs_leave, svs_leave_end;
+       extern uint8_t svs_leave_altstack, svs_leave_altstack_end;
+       u_long psl, cr0;
+       uint8_t *bytes;
+       size_t size;
+
+       svs_enabled = true;
+
+       x86_patch_window_open(&psl, &cr0);
+
+       bytes = &svs_enter;
+       size = (size_t)&svs_enter_end - (size_t)&svs_enter;
+       x86_hotpatch(HP_NAME_SVS_ENTER, bytes, size);
+
+       bytes = &svs_enter_altstack;
+       size = (size_t)&svs_enter_altstack_end -
+           (size_t)&svs_enter_altstack;
+       x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size);
+
+       bytes = &svs_leave;
+       size = (size_t)&svs_leave_end - (size_t)&svs_leave;
+       x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size);
+
+       bytes = &svs_leave_altstack;
+       size = (size_t)&svs_leave_altstack_end -
+           (size_t)&svs_leave_altstack;
+       x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size);
+
+       x86_patch_window_close(psl, cr0);
+}
 
 void
-svs_init(void)
+svs_init(bool early)
 {
-       if (svs_enabled)
+       /*
+        * When early, declare that we want to use SVS, and hotpatch the
+        * entry points. When late, remove PG_G from the page tables.
+        */
+       if (early) {
+               if (cpu_vendor != CPUVENDOR_INTEL) {
+                       return;
+               }
+               svs_enable();
+       } else if (svs_enabled) {
                svs_pgg_update(false);
+       }
 }
 
diff -r b488f471f20f -r 235c448685cd sys/arch/x86/x86/x86_machdep.c
--- a/sys/arch/x86/x86/x86_machdep.c    Thu Feb 22 08:56:51 2018 +0000
+++ b/sys/arch/x86/x86/x86_machdep.c    Thu Feb 22 09:41:06 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: x86_machdep.c,v 1.104 2018/02/22 08:56:52 maxv Exp $   */
+/*     $NetBSD: x86_machdep.c,v 1.105 2018/02/22 09:41:06 maxv Exp $   */
 
 /*-
  * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.104 2018/02/22 08:56:52 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.105 2018/02/22 09:41:06 maxv Exp $");
 
 #include "opt_modular.h"
 #include "opt_physmem.h"
@@ -1092,8 +1092,7 @@
 x86_startup(void)
 {
 #if SVS
-       void svs_init(void);
-       svs_init();
+       svs_init(false);
 #endif
 #if !defined(XEN)
        nmi_init();