Add arch/x86/kvm/svm/caretaker_vmenter.S and caretaker.h for standalone AMD SVM VMRUN world switch on preserved physical cores.
Signed-off-by: Pasha Tatashin <[email protected]> --- arch/x86/kvm/svm/caretaker.h | 63 +++++++++++++++++++ arch/x86/kvm/svm/caretaker_vmenter.S | 91 ++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 arch/x86/kvm/svm/caretaker.h create mode 100644 arch/x86/kvm/svm/caretaker_vmenter.S diff --git a/arch/x86/kvm/svm/caretaker.h b/arch/x86/kvm/svm/caretaker.h new file mode 100644 index 000000000000..54a0673cb1ea --- /dev/null +++ b/arch/x86/kvm/svm/caretaker.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin <[email protected]> + */ +#ifndef __SVM_CARETAKER_H +#define __SVM_CARETAKER_H + +#ifndef __ASSEMBLY__ +#include <linux/types.h> +#include <linux/kvm_host.h> +#include <linux/oncore.h> +#include "svm.h" +#endif + +#include "../caretaker.h" + +#define CSP_VMCB_OFFSET 0x1000 +#define CSP_HSAVE_PA_OFFSET 0x3000 +#define CSP_VMCB_PA_OFFSET 0x3010 +#define VMCB_RAX_OFFSET 0x5f8 + +#ifndef __ASSEMBLY__ + +struct caretaker_svm_page { + struct caretaker_x86_page common; + + /* Page 1 (4KB): Preserved VMCB */ + struct vmcb vmcb __aligned(PAGE_SIZE); + + /* Page 2 (4KB): Preserved HSAVE area */ + u8 hsave_area[PAGE_SIZE] __aligned(PAGE_SIZE); + u64 hsave_pa; + u64 orig_hsave_pa; + u64 vmcb_pa; + u64 orig_efer; + + /* Preserved MSR and I/O permission bitmaps referenced by VMCB */ + u8 msrpm[MSRPM_SIZE] __aligned(PAGE_SIZE); + u8 iopm[IOPM_SIZE] __aligned(PAGE_SIZE); +} __aligned(PAGE_SIZE); + +static_assert(offsetof(struct caretaker_svm_page, vmcb) == CSP_VMCB_OFFSET); +static_assert(offsetof(struct caretaker_svm_page, hsave_pa) == CSP_HSAVE_PA_OFFSET); +static_assert(offsetof(struct caretaker_svm_page, vmcb_pa) == CSP_VMCB_PA_OFFSET); + +void svm_recalc_intercepts(struct kvm_vcpu *vcpu); + +#ifdef CONFIG_KVM_CARETAKER +int svm_caretaker_enter(void *page); +void svm_caretaker_decode_exit(void *page, struct kvm_caretaker_exit *exit); +void svm_caretaker_register(void); +void svm_caretaker_unregister(void); +void svm_caretaker_init(struct kvm_vcpu *vcpu); +#else +static inline void svm_caretaker_register(void) {} +static inline void svm_caretaker_unregister(void) {} +static inline void svm_caretaker_init(struct kvm_vcpu *vcpu) {} +#endif + +#endif /* !__ASSEMBLY__ */ + +#endif /* __SVM_CARETAKER_H */ diff --git a/arch/x86/kvm/svm/caretaker_vmenter.S b/arch/x86/kvm/svm/caretaker_vmenter.S new file mode 100644 index 000000000000..e0276a34403e --- /dev/null +++ b/arch/x86/kvm/svm/caretaker_vmenter.S @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin <[email protected]> + */ +#include <linux/linkage.h> +#include <linux/cfi_types.h> +#include <asm/asm.h> +#include <asm/bitsperlong.h> +#include <asm/nospec-branch.h> +#include "caretaker.h" +#include "../caretaker_asm.h" + +.section .text.cpu_preserved, "ax" + +/** + * svm_caretaker_enter - Low-level transition to SVM guest mode and back + * @csp: Pointer to struct caretaker_svm_page (%rdi) + */ +SYM_TYPED_FUNC_START(svm_caretaker_enter) + ENDBR + CARETAKER_PUSH_HOST_REGS + movq %cr3, %rax + pushq %rax /* saved_cr3 */ + + clgi + sti + + /* 0. Save host VMCB state (FS, GS, TR, LDTR) to host save area */ + movq CXP_HSAVE_PA(%rdi), %rax + testq %rax, %rax + jz 10f + vmsave %rax +10: + + /* 1. Sync guest RAX to VMCB and load guest state (FS, GS, TR, LDTR) */ + leaq CSP_VMCB_OFFSET(%rdi), %rax + movq CXP_REG_RAX(%rdi), %rbx + movq %rbx, VMCB_RAX_OFFSET(%rax) + movq CSP_VMCB_PA_OFFSET(%rdi), %rax + vmload %rax + + /* 2. Restore guest GPRs */ + CARETAKER_RESTORE_GPRS_NO_RAX %rdi + + /* 3. Run guest */ + pushq %rdi + movq CXP_REG_RDI(%rdi), %rdi + movq (%rsp), %rax + movq CSP_VMCB_PA_OFFSET(%rax), %rax + CARETAKER_CLEAR_CPU_BUFFERS + vmrun %rax + cli + + /* 4. Guest exited: Save guest GPRs */ + xchgq %rdi, (%rsp) /* rdi = csp, stack = guest rdi */ + popq CXP_REG_RDI(%rdi) + + CARETAKER_SAVE_GPRS_NO_RAX %rdi + + /* Stuff Return Stack Buffer to prevent guest RSB poisoning */ + CARETAKER_FILL_RETURN_BUFFER %rax + + /* 5. Save guest state to VMCB */ + movq CSP_VMCB_PA_OFFSET(%rdi), %rax + vmsave %rax + leaq CSP_VMCB_OFFSET(%rdi), %rax + movq VMCB_RAX_OFFSET(%rax), %rbx /* save.rax */ + movq %rbx, CXP_REG_RAX(%rdi) + + /* 5a. Restore host VMCB state (FS, GS, TR, LDTR) from HSAVE */ + movq CXP_HSAVE_PA(%rdi), %rax + testq %rax, %rax + jz 40f + vmload %rax +40: + + /* 5b. Restore host CR3 if clobbered */ + popq %rbx /* saved_cr3 */ + movq %cr3, %rcx + cmpq %rbx, %rcx + je 50f + movq %rbx, %cr3 +50: + stgi + + /* 6. Restore host callee-saved registers & return to C */ + CARETAKER_POP_HOST_REGS + xorq %rax, %rax + ret +SYM_FUNC_END(svm_caretaker_enter) -- 2.55.0.1082.g2b9226bbc0-goog
