[src/trunk]: src/sys/arch/x86/x86 Add kthread_fpu_enter/exit support to x86.

[riastradh <riastradh%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/f9df5f24d0a7
branches:  trunk
changeset: 974447:f9df5f24d0a7
user:      riastradh <riastradh%NetBSD.org@localhost>
date:      Sat Aug 01 02:13:34 2020 +0000

description:
Add kthread_fpu_enter/exit support to x86.

diffstat:

 sys/arch/x86/x86/fpu.c |  71 +++++++++++++++++++++++++++++++++++++++++++------
 1 files changed, 62 insertions(+), 9 deletions(-)

diffs (159 lines):

diff -r beb2d307e716 -r f9df5f24d0a7 sys/arch/x86/x86/fpu.c
--- a/sys/arch/x86/x86/fpu.c    Sat Aug 01 02:13:04 2020 +0000
+++ b/sys/arch/x86/x86/fpu.c    Sat Aug 01 02:13:34 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: fpu.c,v 1.72 2020/07/20 16:43:03 riastradh Exp $       */
+/*     $NetBSD: fpu.c,v 1.73 2020/08/01 02:13:34 riastradh Exp $       */
 
 /*
  * Copyright (c) 2008, 2019 The NetBSD Foundation, Inc.  All
@@ -96,7 +96,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.72 2020/07/20 16:43:03 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.73 2020/08/01 02:13:34 riastradh Exp $");
 
 #include "opt_multiprocessor.h"
 
@@ -107,6 +107,7 @@
 #include <sys/file.h>
 #include <sys/proc.h>
 #include <sys/kernel.h>
+#include <sys/kthread.h>
 #include <sys/sysctl.h>
 #include <sys/xcall.h>
 
@@ -131,13 +132,35 @@
 
 uint32_t x86_fpu_mxcsr_mask __read_mostly = 0;
 
+/*
+ * True if this a thread that is allowed to use the FPU -- either a
+ * user thread, or a system thread with LW_SYSTEM_FPU enabled.
+ */
+static inline bool
+lwp_can_haz_fpu(struct lwp *l)
+{
+
+       return (l->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) != LW_SYSTEM;
+}
+
+/*
+ * True if this is a system thread with its own private FPU state.
+ */
+static inline bool
+lwp_system_fpu_p(struct lwp *l)
+{
+
+       return (l->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) ==
+           (LW_SYSTEM|LW_SYSTEM_FPU);
+}
+
 static inline union savefpu *
 fpu_lwp_area(struct lwp *l)
 {
        struct pcb *pcb = lwp_getpcb(l);
        union savefpu *area = &pcb->pcb_savefpu;
 
-       KASSERT((l->l_flag & LW_SYSTEM) == 0);
+       KASSERT(lwp_can_haz_fpu(l));
        if (l == curlwp) {
                fpu_save();
        }
@@ -155,7 +178,7 @@
 
        s = splvm();
        if (l->l_md.md_flags & MDL_FPU_IN_CPU) {
-               KASSERT((l->l_flag & LW_SYSTEM) == 0);
+               KASSERT(lwp_can_haz_fpu(l));
                fpu_area_save(area, x86_xsave_features);
                l->l_md.md_flags &= ~MDL_FPU_IN_CPU;
        }
@@ -307,7 +330,7 @@
            cpu_index(ci), ci->ci_ilevel);
 
        if (oldlwp->l_md.md_flags & MDL_FPU_IN_CPU) {
-               KASSERT(!(oldlwp->l_flag & LW_SYSTEM));
+               KASSERT(lwp_can_haz_fpu(oldlwp));
                pcb = lwp_getpcb(oldlwp);
                fpu_area_save(&pcb->pcb_savefpu, x86_xsave_features);
                oldlwp->l_md.md_flags &= ~MDL_FPU_IN_CPU;
@@ -322,11 +345,11 @@
        union savefpu *fpu_save;
 
        /* Kernel threads have no FPU. */
-       if (__predict_false(l2->l_flag & LW_SYSTEM)) {
+       if (__predict_false(!lwp_can_haz_fpu(l2))) {
                return;
        }
        /* For init(8). */
-       if (__predict_false(l1->l_flag & LW_SYSTEM)) {
+       if (__predict_false(!lwp_can_haz_fpu(l1))) {
                memset(&pcb2->pcb_savefpu, 0, x86_fpu_save_size);
                return;
        }
@@ -350,6 +373,8 @@
 
 /* -------------------------------------------------------------------------- */
 
+static const union savefpu zero_fpu __aligned(64);
+
 /*
  * fpu_kern_enter()
  *
@@ -369,6 +394,11 @@
        struct cpu_info *ci;
        int s;
 
+       if (lwp_system_fpu_p(l) && !cpu_intr_p()) {
+               KASSERT(!cpu_softintr_p());
+               return;
+       }
+
        s = splvm();
 
        ci = curcpu();
@@ -401,10 +431,16 @@
 void
 fpu_kern_leave(void)
 {
-       static const union savefpu zero_fpu __aligned(64);
-       struct cpu_info *ci = curcpu();
+       struct cpu_info *ci;
        int s;
 
+       if (lwp_system_fpu_p(curlwp) && !cpu_intr_p()) {
+               KASSERT(!cpu_softintr_p());
+               return;
+       }
+
+       ci = curcpu();
+
        KASSERT(ci->ci_ilevel == IPL_VM);
        KASSERT(ci->ci_kfpu_spl != -1);
 
@@ -426,6 +462,23 @@
        splx(s);
 }
 
+void
+kthread_fpu_enter_md(void)
+{
+
+       /* Enable the FPU by clearing CR0_TS.  */
+       clts();
+}
+
+void
+kthread_fpu_exit_md(void)
+{
+
+       /* Zero the FPU state and disable the FPU by setting CR0_TS.  */
+       fpu_area_restore(&zero_fpu, x86_xsave_features);
+       stts();
+}
+
 /* -------------------------------------------------------------------------- */
 
 /*