From 4ebfa3230b40728638a6acceb709f900f920f921 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 21 Mar 2026 21:24:15 +0000 Subject: [PATCH 1/5] KVM: arm64: pkvm: Move error handling to the end of kvm_hyp_cpu_entry We currently handle CPUs having booted at EL1 in the middle of the kvm_hyp_cpu_entry function. Not only this adversely affects readability, but this is also at a bizarre spot should more error handling be added (which we're about to do). Move the WFE/WFI loop to the end of the function and fix a comment. Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260321212419.2803972-2-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/hyp-init.S | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 0d42eedc7167..5d00bde09201 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -201,14 +201,9 @@ SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu) /* Check that the core was booted in EL2. */ mrs x0, CurrentEL cmp x0, #CurrentEL_EL2 - b.eq 2f + b.ne 1f - /* The core booted in EL1. KVM cannot be initialized on it. */ -1: wfe - wfi - b 1b - -2: msr SPsel, #1 // We want to use SP_EL{1,2} + msr SPsel, #1 // We want to use SP_EL2 init_el2_hcr 0 @@ -222,6 +217,11 @@ SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu) mov x0, x29 ldr x1, =kvm_host_psci_cpu_entry br x1 + + // The core booted in EL1. KVM cannot be initialized on it. +1: wfe + wfi + b 1b SYM_CODE_END(__kvm_hyp_init_cpu) SYM_CODE_START(__kvm_handle_stub_hvc) From 1536a0b1386850b67a9ea840e57b7b475e895fed Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 21 Mar 2026 21:24:16 +0000 Subject: [PATCH 2/5] KVM: arm64: pkvm: Simplify BTI handling on CPU boot In order to perform an indirect branch to kvm_host_psci_cpu_entry() on a BTI-aware system, we first branch to a 'BTI j' landing pad, and from there branch again to the target. While this works, this is really not required: - BLR works with 'BTI c' and 'PACIASP' as the landing pad - Even if LR gets clobbered by BLR, we are going to restore the host's registers, so it is pointless to try and avoid touching LR Given the above, drop the veneer and directly call into C code. If we were to come back from it, we'd directly enter the error handler. Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260321212419.2803972-3-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/host.S | 10 ---------- arch/arm64/kvm/hyp/nvhe/hyp-init.S | 9 +++++---- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index eef15b374abb..465f6f1dd641 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -291,13 +291,3 @@ SYM_CODE_START(__kvm_hyp_host_forward_smc) ret SYM_CODE_END(__kvm_hyp_host_forward_smc) - -/* - * kvm_host_psci_cpu_entry is called through br instruction, which requires - * bti j instruction as compilers (gcc and llvm) doesn't insert bti j for external - * functions, but bti c instead. - */ -SYM_CODE_START(kvm_host_psci_cpu_entry) - bti j - b __kvm_host_psci_cpu_entry -SYM_CODE_END(kvm_host_psci_cpu_entry) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 5d00bde09201..55e0dce65dc5 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -213,12 +213,13 @@ SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu) mov x0, x28 bl ___kvm_hyp_init // Clobbers x0..x2 - /* Leave idmap. */ + /* Leave idmap -- using BLR is OK, LR is restored from host context */ mov x0, x29 - ldr x1, =kvm_host_psci_cpu_entry - br x1 + ldr x1, =__kvm_host_psci_cpu_entry + blr x1 - // The core booted in EL1. KVM cannot be initialized on it. + // The core booted in EL1, or the C code unexpectedly returned. + // Either way, KVM cannot be initialized on it. 1: wfe wfi b 1b From ba64e273eac3d7ec4a2b621b3620c4d3b0399858 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 21 Mar 2026 21:24:17 +0000 Subject: [PATCH 3/5] KVM: arm64: pkvm: Turn __kvm_hyp_init_cpu into an inner label __kvm_hyp_init_cpu really is an internal label for kvm_hyp_cpu_entry and kvm_hyp_cpu_resume. Make it clear that this is what it is, and drop a pointless branch in kvm_hyp_cpu_resume. Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260321212419.2803972-4-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/hyp-init.S | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 55e0dce65dc5..2e80fcbff2df 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -175,7 +175,6 @@ SYM_CODE_END(___kvm_hyp_init) SYM_CODE_START(kvm_hyp_cpu_entry) mov x1, #1 // is_cpu_on = true b __kvm_hyp_init_cpu -SYM_CODE_END(kvm_hyp_cpu_entry) /* * PSCI CPU_SUSPEND / SYSTEM_SUSPEND entry point @@ -184,17 +183,8 @@ SYM_CODE_END(kvm_hyp_cpu_entry) */ SYM_CODE_START(kvm_hyp_cpu_resume) mov x1, #0 // is_cpu_on = false - b __kvm_hyp_init_cpu -SYM_CODE_END(kvm_hyp_cpu_resume) -/* - * Common code for CPU entry points. Initializes EL2 state and - * installs the hypervisor before handing over to a C handler. - * - * x0: struct kvm_nvhe_init_params PA - * x1: bool is_cpu_on - */ -SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu) +SYM_INNER_LABEL(__kvm_hyp_init_cpu, SYM_L_LOCAL) mov x28, x0 // Stash arguments mov x29, x1 @@ -223,7 +213,8 @@ SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu) 1: wfe wfi b 1b -SYM_CODE_END(__kvm_hyp_init_cpu) +SYM_CODE_END(kvm_hyp_cpu_resume) +SYM_CODE_END(kvm_hyp_cpu_entry) SYM_CODE_START(__kvm_handle_stub_hvc) /* From 59c6e12d40a5b05038b68bcdb4690456fee68e8a Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 21 Mar 2026 21:24:18 +0000 Subject: [PATCH 4/5] KVM: arm64: pkvm: Use direct function pointers for cpu_{on,resume} Instead of using a boolean to decide whether a CPU is booting or resuming, just pass an actual function pointer around. This makes the code a bit more straightforward to understand. Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260321212419.2803972-5-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_asm.h | 3 ++- arch/arm64/kvm/hyp/nvhe/hyp-init.S | 9 +++---- arch/arm64/kvm/hyp/nvhe/psci-relay.c | 39 +++++++++++++++++----------- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index a1ad12c72ebf..f4c769857fdf 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -291,7 +291,8 @@ asmlinkage void __noreturn hyp_panic_bad_stack(void); asmlinkage void kvm_unexpected_el2_exception(void); struct kvm_cpu_context; void handle_trap(struct kvm_cpu_context *host_ctxt); -asmlinkage void __noreturn __kvm_host_psci_cpu_entry(bool is_cpu_on); +asmlinkage void __noreturn __kvm_host_psci_cpu_on_entry(void); +asmlinkage void __noreturn __kvm_host_psci_cpu_resume_entry(void); void __noreturn __pkvm_init_finalise(void); void kvm_nvhe_prepare_backtrace(unsigned long fp, unsigned long pc); void kvm_patch_vector_branch(struct alt_instr *alt, diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 2e80fcbff2df..64296b31da73 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -173,7 +173,7 @@ SYM_CODE_END(___kvm_hyp_init) * x0: struct kvm_nvhe_init_params PA */ SYM_CODE_START(kvm_hyp_cpu_entry) - mov x1, #1 // is_cpu_on = true + ldr x29, =__kvm_host_psci_cpu_on_entry b __kvm_hyp_init_cpu /* @@ -182,11 +182,10 @@ SYM_CODE_START(kvm_hyp_cpu_entry) * x0: struct kvm_nvhe_init_params PA */ SYM_CODE_START(kvm_hyp_cpu_resume) - mov x1, #0 // is_cpu_on = false + ldr x29, =__kvm_host_psci_cpu_resume_entry SYM_INNER_LABEL(__kvm_hyp_init_cpu, SYM_L_LOCAL) mov x28, x0 // Stash arguments - mov x29, x1 /* Check that the core was booted in EL2. */ mrs x0, CurrentEL @@ -204,9 +203,7 @@ SYM_INNER_LABEL(__kvm_hyp_init_cpu, SYM_L_LOCAL) bl ___kvm_hyp_init // Clobbers x0..x2 /* Leave idmap -- using BLR is OK, LR is restored from host context */ - mov x0, x29 - ldr x1, =__kvm_host_psci_cpu_entry - blr x1 + blr x29 // The core booted in EL1, or the C code unexpectedly returned. // Either way, KVM cannot be initialized on it. diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c b/arch/arm64/kvm/hyp/nvhe/psci-relay.c index c3e196fb8b18..5aeb5b453a59 100644 --- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c +++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c @@ -200,23 +200,12 @@ static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) __hyp_pa(init_params), 0); } -asmlinkage void __noreturn __kvm_host_psci_cpu_entry(bool is_cpu_on) +static void __noreturn __kvm_host_psci_cpu_entry(unsigned long pc, unsigned long r0) { - struct psci_boot_args *boot_args; - struct kvm_cpu_context *host_ctxt; + struct kvm_cpu_context *host_ctxt = host_data_ptr(host_ctxt); - host_ctxt = host_data_ptr(host_ctxt); - - if (is_cpu_on) - boot_args = this_cpu_ptr(&cpu_on_args); - else - boot_args = this_cpu_ptr(&suspend_args); - - cpu_reg(host_ctxt, 0) = boot_args->r0; - write_sysreg_el2(boot_args->pc, SYS_ELR); - - if (is_cpu_on) - release_boot_args(boot_args); + cpu_reg(host_ctxt, 0) = r0; + write_sysreg_el2(pc, SYS_ELR); write_sysreg_el1(INIT_SCTLR_EL1_MMU_OFF, SYS_SCTLR); write_sysreg(INIT_PSTATE_EL1, SPSR_EL2); @@ -224,6 +213,26 @@ asmlinkage void __noreturn __kvm_host_psci_cpu_entry(bool is_cpu_on) __host_enter(host_ctxt); } +asmlinkage void __noreturn __kvm_host_psci_cpu_on_entry(void) +{ + struct psci_boot_args *boot_args = this_cpu_ptr(&cpu_on_args); + unsigned long pc, r0; + + pc = READ_ONCE(boot_args->pc); + r0 = READ_ONCE(boot_args->r0); + + release_boot_args(boot_args); + + __kvm_host_psci_cpu_entry(pc, r0); +} + +asmlinkage void __noreturn __kvm_host_psci_cpu_resume_entry(void) +{ + struct psci_boot_args *boot_args = this_cpu_ptr(&suspend_args); + + __kvm_host_psci_cpu_entry(boot_args->pc, boot_args->r0); +} + static unsigned long psci_0_1_handler(u64 func_id, struct kvm_cpu_context *host_ctxt) { if (is_psci_0_1(cpu_off, func_id) || is_psci_0_1(migrate, func_id)) From 54a3cc145456272b10c1452fe89e1dcf933d5c39 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 21 Mar 2026 21:24:19 +0000 Subject: [PATCH 5/5] KVM: arm64: Remove extra ISBs when using msr_hcr_el2 The msr_hcr_el2 macro is slightly awkward, as it provides an ISB when CONFIG_AMPERE_ERRATUM_AC04_CPU_23 is present, and none otherwise. Note that this this option is 'default y', meaning that it is likely to be selected. Most instances of msr_hcr_el2 are also immediately followed by an ISB, meaning that in most cases, you end-up with two back-to-back ISBs. This isn't a big deal, but once you have seen that, you can't unsee it. Rework the msr_hcr_el2 macro to always provide the ISB, and drop the superfluous ISBs everywhere else. Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260321212419.2803972-6-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/el2_setup.h | 2 -- arch/arm64/include/asm/sysreg.h | 6 ++---- arch/arm64/kernel/hyp-stub.S | 1 - arch/arm64/kvm/hyp/nvhe/host.S | 1 - 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 85f4c1615472..3e58d6264581 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -50,7 +50,6 @@ * effectively VHE-only or not. */ msr_hcr_el2 x0 // Setup HCR_EL2 as nVHE - isb mov x1, #1 // Write something to FAR_EL1 msr far_el1, x1 isb @@ -64,7 +63,6 @@ .LnE2H0_\@: orr x0, x0, #HCR_E2H msr_hcr_el2 x0 - isb .LnVHE_\@: .endm diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index f4436ecc630c..ca66b8017fa8 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -1114,11 +1114,9 @@ .macro msr_hcr_el2, reg #if IS_ENABLED(CONFIG_AMPERE_ERRATUM_AC04_CPU_23) dsb nsh - msr hcr_el2, \reg - isb -#else - msr hcr_el2, \reg #endif + msr hcr_el2, \reg + isb // Required by AMPERE_ERRATUM_AC04_CPU_23 .endm #else diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S index 085bc9972f6b..634ddc904244 100644 --- a/arch/arm64/kernel/hyp-stub.S +++ b/arch/arm64/kernel/hyp-stub.S @@ -103,7 +103,6 @@ SYM_CODE_START_LOCAL(__finalise_el2) // Engage the VHE magic! mov_q x0, HCR_HOST_VHE_FLAGS msr_hcr_el2 x0 - isb // Use the EL1 allocated stack, per-cpu offset mrs x0, sp_el1 diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index 465f6f1dd641..ff10cafa0ca8 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -125,7 +125,6 @@ SYM_FUNC_START(__hyp_do_panic) mrs x0, hcr_el2 bic x0, x0, #HCR_VM msr_hcr_el2 x0 - isb tlbi vmalls12e1 dsb nsh #endif