Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys Rewrite the FPU code on x86. This greatly simplifies the...



details:   https://anonhg.NetBSD.org/src/rev/fb46d18d84b4
branches:  trunk
changeset: 845682:fb46d18d84b4
user:      maxv <maxv%NetBSD.org@localhost>
date:      Sat Oct 12 06:31:03 2019 +0000

description:
Rewrite the FPU code on x86. This greatly simplifies the logic and removes
the dependency on IPL_HIGH. NVMM is updated accordingly. Posted on
port-amd64 a week ago.

Bump the kernel version to 9.99.16.

diffstat:

 sys/arch/amd64/amd64/amd64_trap.S |    3 +-
 sys/arch/amd64/amd64/genassym.cf  |    5 +-
 sys/arch/amd64/amd64/locore.S     |   27 +--
 sys/arch/amd64/amd64/machdep.c    |   15 +-
 sys/arch/amd64/amd64/spl.S        |    3 +-
 sys/arch/amd64/include/frameasm.h |   16 ++-
 sys/arch/amd64/include/pcb.h      |    3 +-
 sys/arch/amd64/include/proc.h     |    3 +-
 sys/arch/i386/i386/genassym.cf    |    7 +-
 sys/arch/i386/i386/i386_trap.S    |   15 +-
 sys/arch/i386/i386/locore.S       |   29 +---
 sys/arch/i386/i386/machdep.c      |   16 +-
 sys/arch/i386/i386/spl.S          |    5 +-
 sys/arch/i386/include/frameasm.h  |   18 ++-
 sys/arch/i386/include/pcb.h       |    3 +-
 sys/arch/i386/include/proc.h      |    4 +-
 sys/arch/x86/acpi/acpi_wakeup.c   |    8 +-
 sys/arch/x86/include/cpu.h        |    3 +-
 sys/arch/x86/include/fpu.h        |    8 +-
 sys/arch/x86/x86/cpu.c            |   17 +-
 sys/arch/x86/x86/fpu.c            |  279 ++++++++++++-------------------------
 sys/arch/x86/x86/ipi.c            |    6 +-
 sys/arch/x86/x86/vm_machdep.c     |   21 +--
 sys/arch/xen/x86/cpu.c            |   17 +-
 sys/arch/xen/x86/xen_ipi.c        |    8 +-
 sys/dev/nvmm/x86/nvmm_x86_svm.c   |   22 +-
 sys/dev/nvmm/x86/nvmm_x86_vmx.c   |   22 +-
 sys/sys/param.h                   |    4 +-
 28 files changed, 225 insertions(+), 362 deletions(-)

diffs (truncated from 1444 to 300 lines):

diff -r 589a79c664fa -r fb46d18d84b4 sys/arch/amd64/amd64/amd64_trap.S
--- a/sys/arch/amd64/amd64/amd64_trap.S Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/amd64/amd64_trap.S Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: amd64_trap.S,v 1.48 2019/05/18 13:32:12 maxv Exp $     */
+/*     $NetBSD: amd64_trap.S,v 1.49 2019/10/12 06:31:03 maxv Exp $     */
 
 /*
  * Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc.
@@ -672,6 +672,7 @@
        jmp     .Lalltraps_checkast     /* re-check ASTs */
 3:     CHECK_DEFERRED_SWITCH
        jnz     9f
+       HANDLE_DEFERRED_FPU
 
 6:
 #ifdef DIAGNOSTIC
diff -r 589a79c664fa -r fb46d18d84b4 sys/arch/amd64/amd64/genassym.cf
--- a/sys/arch/amd64/amd64/genassym.cf  Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/amd64/genassym.cf  Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-#      $NetBSD: genassym.cf,v 1.76 2019/05/29 16:54:41 maxv Exp $
+#      $NetBSD: genassym.cf,v 1.77 2019/10/12 06:31:03 maxv Exp $
 
 #
 # Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -164,6 +164,7 @@
 
 define MDL_IRET                MDL_IRET
 define MDL_COMPAT32            MDL_COMPAT32
+define MDL_FPU_IN_CPU          MDL_FPU_IN_CPU
 
 define P_FLAG                  offsetof(struct proc, p_flag)
 define P_RASLIST               offsetof(struct proc, p_raslist)
@@ -188,6 +189,7 @@
 define PCB_COMPAT32            PCB_COMPAT32
 define PCB_FS                  offsetof(struct pcb, pcb_fs)
 define PCB_GS                  offsetof(struct pcb, pcb_gs)
+define PCB_SAVEFPU             offsetof(struct pcb, pcb_savefpu)
 
 define TF_RDI                  offsetof(struct trapframe, tf_rdi)
 define TF_RSI                  offsetof(struct trapframe, tf_rsi)
@@ -244,7 +246,6 @@
 define CPU_INFO_NTRAP          offsetof(struct cpu_info, ci_data.cpu_ntrap)
 define CPU_INFO_NINTR          offsetof(struct cpu_info, ci_data.cpu_nintr)
 define CPU_INFO_CURPRIORITY    offsetof(struct cpu_info, ci_schedstate.spc_curpriority)
-define CPU_INFO_FPCURLWP       offsetof(struct cpu_info, ci_fpcurlwp)
 
 define CPU_INFO_GDT            offsetof(struct cpu_info, ci_gdt)
 define CPU_INFO_ILEVEL         offsetof(struct cpu_info, ci_ilevel)
diff -r 589a79c664fa -r fb46d18d84b4 sys/arch/amd64/amd64/locore.S
--- a/sys/arch/amd64/amd64/locore.S     Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/amd64/locore.S     Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: locore.S,v 1.188 2019/10/04 11:47:07 maxv Exp $        */
+/*     $NetBSD: locore.S,v 1.189 2019/10/12 06:31:03 maxv Exp $        */
 
 /*
  * Copyright-o-rama!
@@ -1170,29 +1170,10 @@
        movq    %rax,TF_RIP(%rbx)
 .Lno_RAS:
 
-       /*
-        * Restore cr0 including FPU state (may have CR0_TS set).  Note that
-        * IPL_SCHED prevents from FPU interrupt altering the LWP's saved cr0.
-        */
 #ifndef XENPV
+       /* Raise the IPL to IPL_HIGH. Dropping the priority is deferred until
+        * mi_switch(), when cpu_switchto() returns. XXX Still needed? */
        movl    $IPL_HIGH,CPUVAR(ILEVEL)
-       movl    PCB_CR0(%r14),%ecx      /* has CR0_TS clear */
-       movq    %cr0,%rdx
-
-       /*
-        * If our floating point registers are on a different CPU,
-        * set CR0_TS so we'll trap rather than reuse bogus state.
-        */
-       cmpq    CPUVAR(FPCURLWP),%r12
-       je      .Lskip_TS
-       orq     $CR0_TS,%rcx
-.Lskip_TS:
-
-       /* Reloading CR0 is very expensive - avoid if possible. */
-       cmpq    %rdx,%rcx
-       je      .Lskip_CR0
-       movq    %rcx,%cr0
-.Lskip_CR0:
 
        /* The 32bit LWPs are handled differently. */
        testl   $PCB_COMPAT32,PCB_FLAGS(%r14)
@@ -1305,6 +1286,8 @@
        jne     .Lspl_error
 #endif
 
+       HANDLE_DEFERRED_FPU
+
        /*
         * Decide if we need to take a slow path. That's the case when we
         * want to reload %cs and %ss on a 64bit LWP (MDL_IRET set), or when
diff -r 589a79c664fa -r fb46d18d84b4 sys/arch/amd64/amd64/machdep.c
--- a/sys/arch/amd64/amd64/machdep.c    Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/amd64/machdep.c    Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: machdep.c,v 1.336 2019/08/21 20:30:36 skrll Exp $      */
+/*     $NetBSD: machdep.c,v 1.337 2019/10/12 06:31:03 maxv Exp $       */
 
 /*
  * Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.336 2019/08/21 20:30:36 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.337 2019/10/12 06:31:03 maxv Exp $");
 
 #include "opt_modular.h"
 #include "opt_user_ldt.h"
@@ -442,18 +442,9 @@
        uint64_t zero = 0;
 
        /*
-        * Raise the IPL to IPL_HIGH.
-        * FPU IPIs can alter the LWP's saved cr0.  Dropping the priority
-        * is deferred until mi_switch(), when cpu_switchto() returns.
+        * Raise the IPL to IPL_HIGH. XXX Still needed?
         */
        (void)splhigh();
-       /*
-        * If our floating point registers are on a different CPU,
-        * set CR0_TS so we'll trap rather than reuse bogus state.
-        */
-       if (l != ci->ci_fpcurlwp) {
-               HYPERVISOR_fpu_taskswitch(1);
-       }
 
        /* Update segment registers */
        if (pcb->pcb_flags & PCB_COMPAT32) {
diff -r 589a79c664fa -r fb46d18d84b4 sys/arch/amd64/amd64/spl.S
--- a/sys/arch/amd64/amd64/spl.S        Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/amd64/spl.S        Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: spl.S,v 1.40 2019/02/14 08:18:25 cherry Exp $  */
+/*     $NetBSD: spl.S,v 1.41 2019/10/12 06:31:03 maxv Exp $    */
 
 /*
  * Copyright (c) 2003 Wasabi Systems, Inc.
@@ -397,6 +397,7 @@
 3:
        CHECK_DEFERRED_SWITCH
        jnz     9f
+       HANDLE_DEFERRED_FPU
 6:
        INTRFASTEXIT
 9:
diff -r 589a79c664fa -r fb46d18d84b4 sys/arch/amd64/include/frameasm.h
--- a/sys/arch/amd64/include/frameasm.h Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/include/frameasm.h Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: frameasm.h,v 1.44 2019/05/18 13:32:12 maxv Exp $       */
+/*     $NetBSD: frameasm.h,v 1.45 2019/10/12 06:31:03 maxv Exp $       */
 
 #ifndef _AMD64_MACHINE_FRAMEASM_H
 #define _AMD64_MACHINE_FRAMEASM_H
@@ -246,4 +246,18 @@
 #define CHECK_ASTPENDING(reg)  cmpl    $0, L_MD_ASTPENDING(reg)
 #define CLEAR_ASTPENDING(reg)  movl    $0, L_MD_ASTPENDING(reg)
 
+/*
+ * If the FPU state is not in the CPU, restore it. Executed with interrupts
+ * disabled.
+ *
+ *     %r14 is curlwp, must not be modified
+ *     %rbx must not be modified
+ */
+#define HANDLE_DEFERRED_FPU    \
+       testl   $MDL_FPU_IN_CPU,L_MD_FLAGS(%r14)        ; \
+       jnz     1f                                      ; \
+       call    _C_LABEL(fpu_handle_deferred)           ; \
+       orl     $MDL_FPU_IN_CPU,L_MD_FLAGS(%r14)        ; \
+1:
+
 #endif /* _AMD64_MACHINE_FRAMEASM_H */
diff -r 589a79c664fa -r fb46d18d84b4 sys/arch/amd64/include/pcb.h
--- a/sys/arch/amd64/include/pcb.h      Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/include/pcb.h      Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: pcb.h,v 1.29 2018/07/26 09:29:08 maxv Exp $    */
+/*     $NetBSD: pcb.h,v 1.30 2019/10/12 06:31:03 maxv Exp $    */
 
 /*
  * Copyright (c) 1998 The NetBSD Foundation, Inc.
@@ -96,7 +96,6 @@
 
        uint32_t pcb_unused[8];         /* unused */
 
-       struct cpu_info *pcb_fpcpu;     /* cpu holding our fp state. */
        union savefpu   pcb_savefpu __aligned(64); /* floating point state */
        /* **** DO NOT ADD ANYTHING HERE **** */
 };
diff -r 589a79c664fa -r fb46d18d84b4 sys/arch/amd64/include/proc.h
--- a/sys/arch/amd64/include/proc.h     Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/amd64/include/proc.h     Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: proc.h,v 1.22 2017/02/25 13:34:21 kamil Exp $  */
+/*     $NetBSD: proc.h,v 1.23 2019/10/12 06:31:03 maxv Exp $   */
 
 /*
  * Copyright (c) 1991 Regents of the University of California.
@@ -55,6 +55,7 @@
 
 #define        MDL_COMPAT32            0x0008  /* i386, always return via iret */
 #define        MDL_IRET                0x0010  /* force return via iret, not sysret */
+#define        MDL_FPU_IN_CPU          0x0020  /* the FPU state is in the CPU */
 
 struct mdproc {
        int     md_flags;
diff -r 589a79c664fa -r fb46d18d84b4 sys/arch/i386/i386/genassym.cf
--- a/sys/arch/i386/i386/genassym.cf    Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/i386/i386/genassym.cf    Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-#      $NetBSD: genassym.cf,v 1.113 2019/03/09 08:42:25 maxv Exp $
+#      $NetBSD: genassym.cf,v 1.114 2019/10/12 06:31:03 maxv Exp $
 
 #
 # Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -165,6 +165,7 @@
 define L_FLAG                  offsetof(struct lwp, l_flag)
 define L_PROC                  offsetof(struct lwp, l_proc)
 define L_MD_REGS               offsetof(struct lwp, l_md.md_regs)
+define L_MD_FLAGS              offsetof(struct lwp, l_md.md_flags)
 define L_CTXSWTCH              offsetof(struct lwp, l_ctxswtch)
 define L_MD_ASTPENDING         offsetof(struct lwp, l_md.md_astpending)
 define L_CPU                   offsetof(struct lwp, l_cpu)
@@ -176,6 +177,8 @@
 define P_RASLIST               offsetof(struct proc, p_raslist)
 define P_MD_SYSCALL            offsetof(struct proc, p_md.md_syscall)
 
+define MDL_FPU_IN_CPU          MDL_FPU_IN_CPU
+
 define LW_SYSTEM               LW_SYSTEM
 
 define GUFS_SEL                GUFS_SEL
@@ -200,6 +203,7 @@
 define PCB_FSD                 offsetof(struct pcb, pcb_fsd)
 define PCB_GSD                 offsetof(struct pcb, pcb_gsd)
 define PCB_IOMAP               offsetof(struct pcb, pcb_iomap)
+define PCB_SAVEFPU             offsetof(struct pcb, pcb_savefpu)
 
 define TF_CS                   offsetof(struct trapframe, tf_cs)
 define TF_EIP                  offsetof(struct trapframe, tf_eip)
@@ -251,7 +255,6 @@
 define CPU_INFO_TLBSTATE       offsetof(struct cpu_info, ci_tlbstate)
 define TLBSTATE_VALID          TLBSTATE_VALID
 define CPU_INFO_CURLWP         offsetof(struct cpu_info, ci_curlwp)
-define CPU_INFO_FPCURLWP       offsetof(struct cpu_info, ci_fpcurlwp)
 define CPU_INFO_CURLDT         offsetof(struct cpu_info, ci_curldt)
 define CPU_INFO_IDLELWP        offsetof(struct cpu_info, ci_data.cpu_idlelwp)
 define CPU_INFO_PMAP           offsetof(struct cpu_info, ci_pmap)
diff -r 589a79c664fa -r fb46d18d84b4 sys/arch/i386/i386/i386_trap.S
--- a/sys/arch/i386/i386/i386_trap.S    Sat Oct 12 06:00:52 2019 +0000
+++ b/sys/arch/i386/i386/i386_trap.S    Sat Oct 12 06:31:03 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: i386_trap.S,v 1.19 2019/10/04 15:28:00 maxv Exp $      */
+/*     $NetBSD: i386_trap.S,v 1.20 2019/10/12 06:31:03 maxv Exp $      */
 
 /*
  * Copyright 2002 (c) Wasabi Systems, Inc.
@@ -66,7 +66,7 @@
 
 #if 0
 #include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: i386_trap.S,v 1.19 2019/10/04 15:28:00 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: i386_trap.S,v 1.20 2019/10/12 06:31:03 maxv Exp $");
 #endif
 
 /*
@@ -436,10 +436,10 @@
 
 #ifdef XEN
        STIC(%eax)
-       jz      6f
+       jz      22f
        call    _C_LABEL(stipending)
        testl   %eax,%eax
-       jz      6f
+       jz      22f
        /* process pending interrupts */
        CLI(%eax)
        movl    CPUVAR(ILEVEL),%ebx
@@ -448,15 +448,18 @@
        movl    %ebx,%eax               /* get cpl */
        movl    CPUVAR(XUNMASK)(,%eax,4),%eax
        andl    CPUVAR(XPENDING),%eax   /* any non-masked bits left? */
-       jz      7f
+       jz      11f
        bsrl    %eax,%eax
        btrl    %eax,CPUVAR(XPENDING)
        movl    CPUVAR(XSOURCES)(,%eax,4),%eax
        jmp     *IS_RESUME(%eax)



Home | Main Index | Thread Index | Old Index