[src/trunk]: src/sys/dev/nvmm/x86 Optimize NVMM-Intel: keep the VMCS active o...

[maxv <maxv%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/fe6f9956aadd
branches:  trunk
changeset: 449650:fe6f9956aadd
user:      maxv <maxv%NetBSD.org@localhost>
date:      Thu Mar 14 20:29:53 2019 +0000

description:
Optimize NVMM-Intel: keep the VMCS active on the host CPU, and lazy-switch
it on demand only when needed. This allows the CPU to use the cached
version of the guest state, rather than the in-memory copy of it. This is
much more performant.

A VMCS must be active on only one CPU, but one CPU can have several active
VMCSs at the same time.

We keep track of which CPU each VMCS is active on. When we want to execute
a VCPU, we determine whether its VMCS is loaded on another CPU, and if so
send an IPI to ask it to unbusy that VMCS. In most cases the VMCS is
already active on the current CPU, so we don't have to do anything and can
proceed with a fast VMRESUME.

We send IPIs with kpreemption enabled but with a bound LWP, because we
don't want to get context-switched to the CPU we just sent an IPI to.

Overall, with this in place, I see a ~15% performance increase in the
guests on NVMM-Intel.

diffstat:

 sys/dev/nvmm/x86/nvmm_x86_vmx.c |  75 ++++++++++++++++++++++++++++++++++++----
 1 files changed, 67 insertions(+), 8 deletions(-)

diffs (147 lines):

diff -r 7c12e3115864 -r fe6f9956aadd sys/dev/nvmm/x86/nvmm_x86_vmx.c
--- a/sys/dev/nvmm/x86/nvmm_x86_vmx.c   Thu Mar 14 19:51:49 2019 +0000
+++ b/sys/dev/nvmm/x86/nvmm_x86_vmx.c   Thu Mar 14 20:29:53 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: nvmm_x86_vmx.c,v 1.18 2019/03/14 19:26:44 maxv Exp $   */
+/*     $NetBSD: nvmm_x86_vmx.c,v 1.19 2019/03/14 20:29:53 maxv Exp $   */
 
 /*
  * Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.18 2019/03/14 19:26:44 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.19 2019/03/14 20:29:53 maxv Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -648,6 +648,8 @@
        struct vmcs *vmcs;
        paddr_t vmcs_pa;
        size_t vmcs_refcnt;
+       struct cpu_info *vmcs_ci;
+       bool vmcs_launched;
 
        /* MSR bitmap */
        uint8_t *msrbm;
@@ -762,9 +764,35 @@
 }
 
 static void
+vmx_vmclear_ipi(void *arg1, void *arg2)
+{
+       paddr_t vmcs_pa = (paddr_t)arg1;
+       vmx_vmclear(&vmcs_pa);
+}
+
+static void
+vmx_vmclear_remote(struct cpu_info *ci, paddr_t vmcs_pa)
+{
+       uint64_t xc;
+       int bound;
+
+       KASSERT(kpreempt_disabled());
+
+       bound = curlwp_bind();
+       kpreempt_enable();
+
+       xc = xc_unicast(XC_HIGHPRI, vmx_vmclear_ipi, (void *)vmcs_pa, NULL, ci);
+       xc_wait(xc);
+
+       kpreempt_disable();
+       curlwp_bindx(bound);
+}
+
+static void
 vmx_vmcs_enter(struct nvmm_cpu *vcpu)
 {
        struct vmx_cpudata *cpudata = vcpu->cpudata;
+       struct cpu_info *vmcs_ci;
        paddr_t oldpa __diagused;
 
        cpudata->vmcs_refcnt++;
@@ -777,12 +805,22 @@
                return;
        }
 
+       vmcs_ci = cpudata->vmcs_ci;
+       cpudata->vmcs_ci = (void *)0x00FFFFFFFFFFFFFF; /* clobber */
+
        kpreempt_disable();
 
-#ifdef DIAGNOSTIC
-       vmx_vmptrst(&oldpa);
-       KASSERT(oldpa == 0xFFFFFFFFFFFFFFFF);
-#endif
+       if (vmcs_ci == NULL) {
+               /* This VMCS is loaded for the first time. */
+               vmx_vmclear(&cpudata->vmcs_pa);
+               cpudata->vmcs_launched = false;
+       } else if (vmcs_ci != curcpu()) {
+               /* This VMCS is active on a remote CPU. */
+               vmx_vmclear_remote(vmcs_ci, cpudata->vmcs_pa);
+               cpudata->vmcs_launched = false;
+       } else {
+               /* This VMCS is active on curcpu, nothing to do. */
+       }
 
        vmx_vmptrld(&cpudata->vmcs_pa);
 }
@@ -805,6 +843,24 @@
                return;
        }
 
+       cpudata->vmcs_ci = curcpu();
+       kpreempt_enable();
+}
+
+static void
+vmx_vmcs_destroy(struct nvmm_cpu *vcpu)
+{
+       struct vmx_cpudata *cpudata = vcpu->cpudata;
+       paddr_t oldpa __diagused;
+
+       KASSERT(kpreempt_disabled());
+#ifdef DIAGNOSTIC
+       vmx_vmptrst(&oldpa);
+       KASSERT(oldpa == cpudata->vmcs_pa);
+#endif
+       KASSERT(cpudata->vmcs_refcnt == 1);
+       cpudata->vmcs_refcnt--;
+
        vmx_vmclear(&cpudata->vmcs_pa);
        kpreempt_enable();
 }
@@ -1721,11 +1777,12 @@
        uint64_t intstate;
        uint64_t machgen;
        int hcpu, s, ret;
-       bool launched = false;
+       bool launched;
 
        vmx_vmcs_enter(vcpu);
        ci = curcpu();
        hcpu = cpu_number();
+       launched = cpudata->vmcs_launched;
 
        vmx_gtlb_catchup(vcpu, hcpu);
        vmx_htlb_catchup(vcpu, hcpu);
@@ -1860,6 +1917,8 @@
                }
        }
 
+       cpudata->vmcs_launched = launched;
+
        vmx_vcpu_guest_misc_leave(vcpu);
        vmx_vcpu_guest_dbregs_leave(vcpu);
 
@@ -2515,7 +2574,7 @@
 
        vmx_vmcs_enter(vcpu);
        vmx_asid_free(vcpu);
-       vmx_vmcs_leave(vcpu);
+       vmx_vmcs_destroy(vcpu);
 
        kcpuset_destroy(cpudata->htlb_want_flush);