Author: neel
Date: Tue Feb 4 02:45:08 2014
New Revision: 261453
URL: http://svnweb.freebsd.org/changeset/base/261453
Log:
Avoid doing unnecessary nested TLB invalidations.
Prior to this change the cached value of 'pm_eptgen' was tracked per-vcpu
and per-hostcpu. In the degenerate case where 'N' vcpus were sharing
a single hostcpu this could result in 'N - 1' unnecessary TLB invalidations.
Since an 'invept' invalidates mappings for all VPIDs the first 'invept'
is sufficient.
Fix this by moving the 'eptgen[MAXCPU]' array from 'vmxctx' to 'struct vmx'.
If it is known that an 'invept' is going to be done before entering the
guest then it is safe to skip the 'invvpid'. The stat VPU_INVVPID_SAVED
counts the number of 'invvpid' invalidations that were avoided because
they were subsumed by an 'invept'.
Discussed with: grehan
Modified:
head/sys/amd64/vmm/intel/vmx.c
head/sys/amd64/vmm/intel/vmx.h
head/sys/amd64/vmm/intel/vmx_genassym.c
head/sys/amd64/vmm/intel/vmx_support.S
Modified: head/sys/amd64/vmm/intel/vmx.c
==============================================================================
--- head/sys/amd64/vmm/intel/vmx.c Tue Feb 4 02:41:54 2014
(r261452)
+++ head/sys/amd64/vmm/intel/vmx.c Tue Feb 4 02:45:08 2014
(r261453)
@@ -907,7 +907,6 @@ vmx_vminit(struct vm *vm, pmap_t pmap)
panic("vmx_setup_cr4_shadow %d", error);
vmx->ctx[i].pmap = pmap;
- vmx->ctx[i].eptp = vmx->eptp;
}
return (vmx);
@@ -955,20 +954,20 @@ vmx_astpending_trace(struct vmx *vmx, in
#endif
}
+static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations
saved");
+
static void
-vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu)
+vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap)
{
- int lastcpu;
struct vmxstate *vmxstate;
- struct invvpid_desc invvpid_desc = { 0 };
+ struct invvpid_desc invvpid_desc;
vmxstate = &vmx->state[vcpu];
- lastcpu = vmxstate->lastcpu;
- vmxstate->lastcpu = curcpu;
-
- if (lastcpu == curcpu)
+ if (vmxstate->lastcpu == curcpu)
return;
+ vmxstate->lastcpu = curcpu;
+
vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
@@ -991,8 +990,20 @@ vmx_set_pcpu_defaults(struct vmx *vmx, i
* for "all" EP4TAs.
*/
if (vmxstate->vpid != 0) {
- invvpid_desc.vpid = vmxstate->vpid;
- invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
+ if (pmap->pm_eptgen == vmx->eptgen[curcpu]) {
+ invvpid_desc._res1 = 0;
+ invvpid_desc._res2 = 0;
+ invvpid_desc.vpid = vmxstate->vpid;
+ invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
+ } else {
+ /*
+ * The invvpid can be skipped if an invept is going to
+ * be performed before entering the guest. The invept
+ * will invalidate combined mappings tagged with
+ * 'vmx->eptp' for all vpids.
+ */
+ vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1);
+ }
}
}
@@ -1859,8 +1870,6 @@ vmx_run(void *arg, int vcpu, register_t
KASSERT(vmxctx->pmap == pmap,
("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
- KASSERT(vmxctx->eptp == vmx->eptp,
- ("eptp %p different than ctx eptp %#lx", eptp, vmxctx->eptp));
VMPTRLD(vmcs);
@@ -1875,7 +1884,7 @@ vmx_run(void *arg, int vcpu, register_t
vmcs_write(VMCS_HOST_CR3, rcr3());
vmcs_write(VMCS_GUEST_RIP, startrip);
- vmx_set_pcpu_defaults(vmx, vcpu);
+ vmx_set_pcpu_defaults(vmx, vcpu, pmap);
do {
/*
* Interrupts are disabled from this point on until the
@@ -1910,7 +1919,7 @@ vmx_run(void *arg, int vcpu, register_t
vmx_inject_interrupts(vmx, vcpu, vlapic);
vmx_run_trace(vmx, vcpu);
- rc = vmx_enter_guest(vmxctx, launched);
+ rc = vmx_enter_guest(vmxctx, vmx, launched);
enable_intr();
Modified: head/sys/amd64/vmm/intel/vmx.h
==============================================================================
--- head/sys/amd64/vmm/intel/vmx.h Tue Feb 4 02:41:54 2014
(r261452)
+++ head/sys/amd64/vmm/intel/vmx.h Tue Feb 4 02:45:08 2014
(r261453)
@@ -64,16 +64,13 @@ struct vmxctx {
/*
* XXX todo debug registers and fpu state
*/
-
- int inst_fail_status;
- long eptgen[MAXCPU]; /* cached pmap->pm_eptgen */
+ int inst_fail_status;
/*
- * The 'eptp' and the 'pmap' do not change during the lifetime of
- * the VM so it is safe to keep a copy in each vcpu's vmxctx.
+ * The pmap needs to be deactivated in vmx_exit_guest()
+ * so keep a copy of the 'pmap' in each vmxctx.
*/
- vm_paddr_t eptp;
struct pmap *pmap;
};
@@ -113,6 +110,7 @@ struct vmx {
struct vmxstate state[VM_MAXCPU];
uint64_t eptp;
struct vm *vm;
+ long eptgen[MAXCPU]; /* cached pmap->pm_eptgen */
};
CTASSERT((offsetof(struct vmx, vmcs) & PAGE_MASK) == 0);
CTASSERT((offsetof(struct vmx, msr_bitmap) & PAGE_MASK) == 0);
@@ -123,7 +121,7 @@ CTASSERT((offsetof(struct vmx, pir_desc[
#define VMX_VMRESUME_ERROR 1
#define VMX_VMLAUNCH_ERROR 2
#define VMX_INVEPT_ERROR 3
-int vmx_enter_guest(struct vmxctx *ctx, int launched);
+int vmx_enter_guest(struct vmxctx *ctx, struct vmx *vmx, int launched);
void vmx_exit_guest(void);
void vmx_call_isr(uintptr_t entry);
Modified: head/sys/amd64/vmm/intel/vmx_genassym.c
==============================================================================
--- head/sys/amd64/vmm/intel/vmx_genassym.c Tue Feb 4 02:41:54 2014
(r261452)
+++ head/sys/amd64/vmm/intel/vmx_genassym.c Tue Feb 4 02:45:08 2014
(r261453)
@@ -68,10 +68,10 @@ ASSYM(VMXCTX_HOST_RBX, offsetof(struct v
ASSYM(VMXCTX_HOST_RIP, offsetof(struct vmxctx, host_rip));
ASSYM(VMXCTX_INST_FAIL_STATUS, offsetof(struct vmxctx, inst_fail_status));
-ASSYM(VMXCTX_EPTGEN, offsetof(struct vmxctx, eptgen));
-
ASSYM(VMXCTX_PMAP, offsetof(struct vmxctx, pmap));
-ASSYM(VMXCTX_EPTP, offsetof(struct vmxctx, eptp));
+
+ASSYM(VMX_EPTGEN, offsetof(struct vmx, eptgen));
+ASSYM(VMX_EPTP, offsetof(struct vmx, eptp));
ASSYM(VM_FAIL_INVALID, VM_FAIL_INVALID);
ASSYM(VM_FAIL_VALID, VM_FAIL_VALID);
Modified: head/sys/amd64/vmm/intel/vmx_support.S
==============================================================================
--- head/sys/amd64/vmm/intel/vmx_support.S Tue Feb 4 02:41:54 2014
(r261452)
+++ head/sys/amd64/vmm/intel/vmx_support.S Tue Feb 4 02:45:08 2014
(r261453)
@@ -97,7 +97,8 @@
/*
* vmx_enter_guest(struct vmxctx *vmxctx, int launched)
* %rdi: pointer to the 'vmxctx'
- * %esi: launch state of the VMCS
+ * %rsi: pointer to the 'vmx'
+ * %edx: launch state of the VMCS
* Interrupts must be disabled on entry.
*/
ENTRY(vmx_enter_guest)
@@ -114,19 +115,19 @@ ENTRY(vmx_enter_guest)
LK btsl %eax, PM_ACTIVE(%r11)
/*
- * If 'vmxctx->eptgen[curcpu]' is not identical to 'pmap->pm_eptgen'
+ * If 'vmx->eptgen[curcpu]' is not identical to 'pmap->pm_eptgen'
* then we must invalidate all mappings associated with this EPTP.
*/
movq PM_EPTGEN(%r11), %r10
- cmpq %r10, VMXCTX_EPTGEN(%rdi, %rax, 8)
+ cmpq %r10, VMX_EPTGEN(%rsi, %rax, 8)
je guest_restore
- /* Refresh 'vmxctx->eptgen[curcpu]' */
- movq %r10, VMXCTX_EPTGEN(%rdi, %rax, 8)
+ /* Refresh 'vmx->eptgen[curcpu]' */
+ movq %r10, VMX_EPTGEN(%rsi, %rax, 8)
/* Setup the invept descriptor on the host stack */
mov %rsp, %r11
- movq VMXCTX_EPTP(%rdi), %rax
+ movq VMX_EPTP(%rsi), %rax
movq %rax, -16(%r11)
movq $0x0, -8(%r11)
mov $0x1, %eax /* Single context invalidate */
@@ -134,7 +135,7 @@ ENTRY(vmx_enter_guest)
jbe invept_error /* Check invept instruction error */
guest_restore:
- cmpl $0, %esi
+ cmpl $0, %edx
je do_launch
VMX_GUEST_RESTORE
_______________________________________________
svn-src-all@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
