The L0/host hypervisor must always redirect traps to the L1/guest hypervisor so extend KVM RISC-V to perform the necessary nested world-switch when redirecting traps.
Signed-off-by: Anup Patel <[email protected]> --- arch/riscv/include/asm/kvm_host.h | 3 + arch/riscv/include/asm/kvm_vcpu_nested.h | 12 ++ arch/riscv/kvm/vcpu_exit.c | 28 +++- arch/riscv/kvm/vcpu_nested.c | 162 +++++++++++++++++++++++ 4 files changed, 201 insertions(+), 4 deletions(-) diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index 3b58953eb4eb..c510564a09a2 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -289,6 +289,9 @@ unsigned long kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu *vcpu, bool read_insn, unsigned long guest_addr, struct kvm_cpu_trap *trap); +void kvm_riscv_vcpu_trap_smode_redirect(struct kvm_vcpu *vcpu, + struct kvm_cpu_trap *trap, + bool prev_priv); void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu, struct kvm_cpu_trap *trap); int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, diff --git a/arch/riscv/include/asm/kvm_vcpu_nested.h b/arch/riscv/include/asm/kvm_vcpu_nested.h index 4234c6e81bb6..6bfb67702610 100644 --- a/arch/riscv/include/asm/kvm_vcpu_nested.h +++ b/arch/riscv/include/asm/kvm_vcpu_nested.h @@ -75,6 +75,18 @@ void kvm_riscv_vcpu_nested_swtlb_reset(struct kvm_vcpu *vcpu); int kvm_riscv_vcpu_nested_swtlb_init(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_nested_swtlb_deinit(struct kvm_vcpu *vcpu); +enum kvm_vcpu_nested_set_virt_event { + NESTED_SET_VIRT_EVENT_TRAP = 0, + NESTED_SET_VIRT_EVENT_SRET +}; + +void kvm_riscv_vcpu_nested_set_virt(struct kvm_vcpu *vcpu, + enum kvm_vcpu_nested_set_virt_event event, + bool virt, bool spvp, bool gva); +void kvm_riscv_vcpu_nested_trap_redirect(struct kvm_vcpu *vcpu, + struct kvm_cpu_trap *trap, + bool prev_priv); + void kvm_riscv_vcpu_nested_reset(struct kvm_vcpu *vcpu); int kvm_riscv_vcpu_nested_init(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_nested_deinit(struct kvm_vcpu *vcpu); diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c index 4f63548e582f..aeec4c4eee06 100644 --- a/arch/riscv/kvm/vcpu_exit.c +++ b/arch/riscv/kvm/vcpu_exit.c @@ -149,19 +149,21 @@ unsigned long kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu *vcpu, } /** - * kvm_riscv_vcpu_trap_redirect -- Redirect trap to Guest + * kvm_riscv_vcpu_trap_smode_redirect -- Redirect S-mode trap to Guest * * @vcpu: The VCPU pointer * @trap: Trap details + * @prev_priv: Previous privilege mode (true: S-mode, false: U-mode) */ -void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu, - struct kvm_cpu_trap *trap) +void kvm_riscv_vcpu_trap_smode_redirect(struct kvm_vcpu *vcpu, + struct kvm_cpu_trap *trap, + bool prev_priv) { unsigned long vsstatus = ncsr_read(CSR_VSSTATUS); /* Change Guest SSTATUS.SPP bit */ vsstatus &= ~SR_SPP; - if (vcpu->arch.guest_context.sstatus & SR_SPP) + if (prev_priv) vsstatus |= SR_SPP; /* Change Guest SSTATUS.SPIE bit */ @@ -187,6 +189,24 @@ void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu, vcpu->arch.guest_context.sstatus |= SR_SPP; } +/** + * kvm_riscv_vcpu_trap_redirect -- Redirect HS-mode trap to Guest + * + * @vcpu: The VCPU pointer + * @trap: Trap details + */ +void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu, + struct kvm_cpu_trap *trap) +{ + bool prev_priv = (vcpu->arch.guest_context.sstatus & SR_SPP) ? true : false; + + /* Update Guest nested state */ + kvm_riscv_vcpu_nested_trap_redirect(vcpu, trap, prev_priv); + + /* Update Guest supervisor state */ + kvm_riscv_vcpu_trap_smode_redirect(vcpu, trap, prev_priv); +} + static inline int vcpu_redirect(struct kvm_vcpu *vcpu, struct kvm_cpu_trap *trap) { int ret = -EFAULT; diff --git a/arch/riscv/kvm/vcpu_nested.c b/arch/riscv/kvm/vcpu_nested.c index 3c30d35b3b39..214206fc28bb 100644 --- a/arch/riscv/kvm/vcpu_nested.c +++ b/arch/riscv/kvm/vcpu_nested.c @@ -3,13 +3,175 @@ * Copyright (c) 2026 Qualcomm Technologies, Inc. */ +#include <linux/smp.h> #include <linux/kvm_host.h> +#include <asm/kvm_nacl.h> +#include <asm/kvm_mmu.h> DEFINE_STATIC_KEY_FALSE(kvm_riscv_nested_available); static bool __read_mostly enable_nested_virt; module_param(enable_nested_virt, bool, 0644); +void kvm_riscv_vcpu_nested_set_virt(struct kvm_vcpu *vcpu, + enum kvm_vcpu_nested_set_virt_event event, + bool virt, bool spvp, bool gva) +{ + struct kvm_vcpu_nested *ns = &vcpu->arch.nested; + struct kvm_vcpu_nested_csr *nsc = &ns->csr; + unsigned long tmp, sr_fs_vs_mask = 0; + int cpu; + + /* If H-extension is not available for VCPU then do nothing */ + if (!riscv_isa_extension_available(vcpu->arch.isa, h)) + return; + + /* Grab the CPU to ensure we remain on same CPU */ + cpu = get_cpu(); + + /* Skip hardware CSR update if no change in virt state */ + if (virt == ns->virt) + goto skip_csr_update; + + /* Update config CSRs (aka hedeleg, hideleg, henvcfg, and hstateeX) */ + kvm_riscv_vcpu_config_load(vcpu, virt); + + /* Update time delta */ + kvm_riscv_vcpu_update_timedelta(vcpu, virt); + + /* Update G-stage page table */ + kvm_riscv_mmu_update_hgatp(vcpu, virt); + + /* Swap hardware vs<xyz> CSRs except vsie and vsstatus */ + nsc->vstvec = ncsr_swap(CSR_VSTVEC, nsc->vstvec); + nsc->vsscratch = ncsr_swap(CSR_VSSCRATCH, nsc->vsscratch); + nsc->vsepc = ncsr_swap(CSR_VSEPC, nsc->vsepc); + nsc->vscause = ncsr_swap(CSR_VSCAUSE, nsc->vscause); + nsc->vstval = ncsr_swap(CSR_VSTVAL, nsc->vstval); + nsc->vsatp = ncsr_swap(CSR_VSATP, nsc->vsatp); + + /* Update vsstatus CSR */ + if (riscv_isa_extension_available(vcpu->arch.isa, f) || + riscv_isa_extension_available(vcpu->arch.isa, d)) + sr_fs_vs_mask |= SR_FS; + if (riscv_isa_extension_available(vcpu->arch.isa, v)) + sr_fs_vs_mask |= SR_VS; + if (virt) { + /* + * Update vsstatus in following manner: + * 1) Swap hardware vsstatus (i.e. virtual-HS mode sstatus) with + * vsstatus in nested virtualization context (i.e. virtual-VS + * mode sstatus) + * 2) Swap host sstatus.[FS|VS] (i.e. HS mode sstatus.[FS|VS]) + * with the vsstatus.[FS|VS] saved in nested virtualization + * context (i.e. virtual-HS mode sstatus.[FS|VS]) + */ + nsc->vsstatus = ncsr_swap(CSR_VSSTATUS, nsc->vsstatus); + tmp = vcpu->arch.guest_context.sstatus & sr_fs_vs_mask; + vcpu->arch.guest_context.sstatus &= ~sr_fs_vs_mask; + vcpu->arch.guest_context.sstatus |= (nsc->vsstatus & sr_fs_vs_mask); + nsc->vsstatus &= ~sr_fs_vs_mask; + nsc->vsstatus |= tmp; + } else { + /* + * Update vsstatus in following manner: + * 1) Swap host sstatus.[FS|VS] (i.e. virtual-HS mode sstatus.[FS|VS]) + * with vsstatus.[FS|VS] saved in the nested virtualization + * context (i.e. HS mode sstatus.[FS|VS]) + * 2) Swap hardware vsstatus (i.e. virtual-VS mode sstatus) with + * vsstatus in nested virtualization context (i.e. virtual-HS + * mode sstatus) + */ + tmp = vcpu->arch.guest_context.sstatus & sr_fs_vs_mask; + vcpu->arch.guest_context.sstatus &= ~sr_fs_vs_mask; + vcpu->arch.guest_context.sstatus |= (nsc->vsstatus & sr_fs_vs_mask); + nsc->vsstatus &= ~sr_fs_vs_mask; + nsc->vsstatus |= tmp; + nsc->vsstatus = ncsr_swap(CSR_VSSTATUS, nsc->vsstatus); + } + +skip_csr_update: + if (event != NESTED_SET_VIRT_EVENT_SRET) { + /* Update guest hstatus.SPV bit */ + nsc->hstatus &= ~HSTATUS_SPV; + nsc->hstatus |= (ns->virt) ? HSTATUS_SPV : 0; + + /* Update guest hstatus.SPVP bit */ + if (ns->virt) { + nsc->hstatus &= ~HSTATUS_SPVP; + if (spvp) + nsc->hstatus |= HSTATUS_SPVP; + } + + /* Update guest hstatus.GVA bit */ + if (event == NESTED_SET_VIRT_EVENT_TRAP) { + nsc->hstatus &= ~HSTATUS_GVA; + nsc->hstatus |= (gva) ? HSTATUS_GVA : 0; + } + } + + /* Update host SRET trapping */ + vcpu->arch.guest_context.hstatus &= ~HSTATUS_VTSR; + if (virt) { + if (nsc->hstatus & HSTATUS_VTSR) + vcpu->arch.guest_context.hstatus |= HSTATUS_VTSR; + } else { + if (nsc->hstatus & HSTATUS_SPV) + vcpu->arch.guest_context.hstatus |= HSTATUS_VTSR; + } + + /* Update host VM trapping */ + vcpu->arch.guest_context.hstatus &= ~HSTATUS_VTVM; + if (virt && (nsc->hstatus & HSTATUS_VTVM)) + vcpu->arch.guest_context.hstatus |= HSTATUS_VTVM; + + /* Update virt flag */ + ns->virt = virt; + + /* Release CPU */ + put_cpu(); +} + +void kvm_riscv_vcpu_nested_trap_redirect(struct kvm_vcpu *vcpu, + struct kvm_cpu_trap *trap, + bool prev_priv) +{ + bool gva; + + /* Do nothing if H-extension is not available for VCPU */ + if (!riscv_isa_extension_available(vcpu->arch.isa, h)) + return; + + /* Determine GVA bit state */ + gva = false; + switch (trap->scause) { + case EXC_INST_MISALIGNED: + case EXC_INST_ACCESS: + case EXC_LOAD_MISALIGNED: + case EXC_LOAD_ACCESS: + case EXC_STORE_MISALIGNED: + case EXC_STORE_ACCESS: + case EXC_INST_PAGE_FAULT: + case EXC_LOAD_PAGE_FAULT: + case EXC_STORE_PAGE_FAULT: + case EXC_INST_GUEST_PAGE_FAULT: + case EXC_LOAD_GUEST_PAGE_FAULT: + case EXC_STORE_GUEST_PAGE_FAULT: + gva = true; + break; + default: + break; + } + + /* Update Guest HTVAL and HTINST */ + vcpu->arch.nested.csr.htval = trap->htval; + vcpu->arch.nested.csr.htinst = trap->htinst; + + /* Turn-off nested virtualization for virtual-HS mode */ + kvm_riscv_vcpu_nested_set_virt(vcpu, NESTED_SET_VIRT_EVENT_TRAP, + false, prev_priv, gva); +} + void kvm_riscv_vcpu_nested_reset(struct kvm_vcpu *vcpu) { struct kvm_vcpu_nested *ns = &vcpu->arch.nested; -- 2.43.0
