[src/trunk]: src/sys/dev/nvmm/x86 nvmm-x86: avoid hogging behavior observed r...

[maxv <maxv%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/5ded9166ee32
branches:  trunk
changeset: 1013956:5ded9166ee32
user:      maxv <maxv%NetBSD.org@localhost>
date:      Tue Sep 08 17:02:03 2020 +0000

description:
nvmm-x86: avoid hogging behavior observed recently

When the FPU code got rewritten in NetBSD, the dependency on IPL_HIGH was
eliminated, and I took _vcpu_guest_fpu_enter() out of the VCPU loop since
there was no need to be in the splhigh window.

Later, the code was switched to use the kernel FPU API, API that works at
IPL_VM, not at IPL_NONE.

These two changes mean that the whole VCPU loop is now executing at IPL_VM,
which is not desired, because it introduces a delay in interrupt processing
on the host in certain cases.

Fix this by putting _vcpu_guest_fpu_enter() back inside the VCPU loop.

diffstat:

 sys/dev/nvmm/x86/nvmm_x86_svm.c |  11 ++++-------
 sys/dev/nvmm/x86/nvmm_x86_vmx.c |  11 ++++-------
 2 files changed, 8 insertions(+), 14 deletions(-)

diffs (118 lines):

diff -r 7686310182c4 -r 5ded9166ee32 sys/dev/nvmm/x86/nvmm_x86_svm.c
--- a/sys/dev/nvmm/x86/nvmm_x86_svm.c   Tue Sep 08 17:00:07 2020 +0000
+++ b/sys/dev/nvmm/x86/nvmm_x86_svm.c   Tue Sep 08 17:02:03 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: nvmm_x86_svm.c,v 1.80 2020/09/08 16:58:38 maxv Exp $   */
+/*     $NetBSD: nvmm_x86_svm.c,v 1.81 2020/09/08 17:02:03 maxv Exp $   */
 
 /*
  * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
@@ -29,7 +29,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.80 2020/09/08 16:58:38 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.81 2020/09/08 17:02:03 maxv Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -1328,9 +1328,6 @@
        }
 
        cpudata->gxcr0 = val;
-       if (svm_xcr0_mask != 0) {
-               wrxcr(0, cpudata->gxcr0);
-       }
 
        svm_inkernel_advance(cpudata->vmcb);
        return;
@@ -1516,7 +1513,6 @@
 
        svm_vcpu_guest_dbregs_enter(vcpu);
        svm_vcpu_guest_misc_enter(vcpu);
-       svm_vcpu_guest_fpu_enter(vcpu);
 
        while (1) {
                if (cpudata->gtlb_want_flush) {
@@ -1530,11 +1526,13 @@
                        svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I);
                }
 
+               svm_vcpu_guest_fpu_enter(vcpu);
                svm_clgi();
                machgen = svm_htlb_flush(machdata, cpudata);
                svm_vmrun(cpudata->vmcb_pa, cpudata->gprs);
                svm_htlb_flush_ack(cpudata, machgen);
                svm_stgi();
+               svm_vcpu_guest_fpu_leave(vcpu);
 
                svm_vmcb_cache_default(vmcb);
 
@@ -1622,7 +1620,6 @@
 
        cpudata->gtsc = rdtsc() + vmcb->ctrl.tsc_offset;
 
-       svm_vcpu_guest_fpu_leave(vcpu);
        svm_vcpu_guest_misc_leave(vcpu);
        svm_vcpu_guest_dbregs_leave(vcpu);
 
diff -r 7686310182c4 -r 5ded9166ee32 sys/dev/nvmm/x86/nvmm_x86_vmx.c
--- a/sys/dev/nvmm/x86/nvmm_x86_vmx.c   Tue Sep 08 17:00:07 2020 +0000
+++ b/sys/dev/nvmm/x86/nvmm_x86_vmx.c   Tue Sep 08 17:02:03 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: nvmm_x86_vmx.c,v 1.79 2020/09/08 17:00:07 maxv Exp $   */
+/*     $NetBSD: nvmm_x86_vmx.c,v 1.80 2020/09/08 17:02:03 maxv Exp $   */
 
 /*
  * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
@@ -29,7 +29,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.79 2020/09/08 17:00:07 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.80 2020/09/08 17:02:03 maxv Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -1969,9 +1969,6 @@
        }
 
        cpudata->gxcr0 = val;
-       if (vmx_xcr0_mask != 0) {
-               wrxcr(0, cpudata->gxcr0);
-       }
 
        vmx_inkernel_advance();
        return;
@@ -2228,7 +2225,6 @@
 
        vmx_vcpu_guest_dbregs_enter(vcpu);
        vmx_vcpu_guest_misc_enter(vcpu);
-       vmx_vcpu_guest_fpu_enter(vcpu);
 
        while (1) {
                if (cpudata->gtlb_want_flush) {
@@ -2243,6 +2239,7 @@
                        cpudata->gtsc_want_update = false;
                }
 
+               vmx_vcpu_guest_fpu_enter(vcpu);
                vmx_cli();
                machgen = vmx_htlb_flush(machdata, cpudata);
                lcr2(cpudata->gcr2);
@@ -2254,6 +2251,7 @@
                cpudata->gcr2 = rcr2();
                vmx_htlb_flush_ack(cpudata, machgen);
                vmx_sti();
+               vmx_vcpu_guest_fpu_leave(vcpu);
 
                if (__predict_false(ret != 0)) {
                        vmx_exit_invalid(exit, -1);
@@ -2349,7 +2347,6 @@
 
        cpudata->gtsc = vmx_vmread(VMCS_TSC_OFFSET) + rdtsc();
 
-       vmx_vcpu_guest_fpu_leave(vcpu);
        vmx_vcpu_guest_misc_leave(vcpu);
        vmx_vcpu_guest_dbregs_leave(vcpu);