diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 998b2a3f615a..144a757e937f 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -50,7 +50,6 @@ * effectively VHE-only or not. */ msr_hcr_el2 x0 // Setup HCR_EL2 as nVHE - isb mov x1, #1 // Write something to FAR_EL1 msr far_el1, x1 isb @@ -64,7 +63,6 @@ .LnE2H0_\@: orr x0, x0, #HCR_E2H msr_hcr_el2 x0 - isb .LnVHE_\@: .endm diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 724319298e71..93371b6138d1 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -301,7 +301,8 @@ asmlinkage void __noreturn hyp_panic_bad_stack(void); asmlinkage void kvm_unexpected_el2_exception(void); struct kvm_cpu_context; void handle_trap(struct kvm_cpu_context *host_ctxt); -asmlinkage void __noreturn __kvm_host_psci_cpu_entry(bool is_cpu_on); +asmlinkage void __noreturn __kvm_host_psci_cpu_on_entry(void); +asmlinkage void __noreturn __kvm_host_psci_cpu_resume_entry(void); void __noreturn __pkvm_init_finalise(void); void kvm_nvhe_prepare_backtrace(unsigned long fp, unsigned long pc); void kvm_patch_vector_branch(struct alt_instr *alt, diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 938cdb248f83..736561480f36 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -1121,11 +1121,9 @@ .macro msr_hcr_el2, reg #if IS_ENABLED(CONFIG_AMPERE_ERRATUM_AC04_CPU_23) dsb nsh - msr hcr_el2, \reg - isb -#else - msr hcr_el2, \reg #endif + msr hcr_el2, \reg + isb // Required by AMPERE_ERRATUM_AC04_CPU_23 .endm #else diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S index 085bc9972f6b..634ddc904244 100644 --- a/arch/arm64/kernel/hyp-stub.S +++ b/arch/arm64/kernel/hyp-stub.S @@ -103,7 +103,6 @@ SYM_CODE_START_LOCAL(__finalise_el2) // Engage the VHE magic! mov_q x0, HCR_HOST_VHE_FLAGS msr_hcr_el2 x0 - isb // Use the EL1 allocated stack, per-cpu offset mrs x0, sp_el1 diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index 3092befcd97c..f337770ec459 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -125,7 +125,6 @@ SYM_FUNC_START(__hyp_do_panic) mrs x0, hcr_el2 bic x0, x0, #HCR_VM msr_hcr_el2 x0 - isb tlbi vmalls12e1 dsb nsh #endif @@ -291,13 +290,3 @@ SYM_CODE_START(__kvm_hyp_host_forward_smc) ret SYM_CODE_END(__kvm_hyp_host_forward_smc) - -/* - * kvm_host_psci_cpu_entry is called through br instruction, which requires - * bti j instruction as compilers (gcc and llvm) doesn't insert bti j for external - * functions, but bti c instead. - */ -SYM_CODE_START(kvm_host_psci_cpu_entry) - bti j - b __kvm_host_psci_cpu_entry -SYM_CODE_END(kvm_host_psci_cpu_entry) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 0d42eedc7167..64296b31da73 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -173,9 +173,8 @@ SYM_CODE_END(___kvm_hyp_init) * x0: struct kvm_nvhe_init_params PA */ SYM_CODE_START(kvm_hyp_cpu_entry) - mov x1, #1 // is_cpu_on = true + ldr x29, =__kvm_host_psci_cpu_on_entry b __kvm_hyp_init_cpu -SYM_CODE_END(kvm_hyp_cpu_entry) /* * PSCI CPU_SUSPEND / SYSTEM_SUSPEND entry point @@ -183,32 +182,17 @@ SYM_CODE_END(kvm_hyp_cpu_entry) * x0: struct kvm_nvhe_init_params PA */ SYM_CODE_START(kvm_hyp_cpu_resume) - mov x1, #0 // is_cpu_on = false - b __kvm_hyp_init_cpu -SYM_CODE_END(kvm_hyp_cpu_resume) + ldr x29, =__kvm_host_psci_cpu_resume_entry -/* - * Common code for CPU entry points. Initializes EL2 state and - * installs the hypervisor before handing over to a C handler. - * - * x0: struct kvm_nvhe_init_params PA - * x1: bool is_cpu_on - */ -SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu) +SYM_INNER_LABEL(__kvm_hyp_init_cpu, SYM_L_LOCAL) mov x28, x0 // Stash arguments - mov x29, x1 /* Check that the core was booted in EL2. */ mrs x0, CurrentEL cmp x0, #CurrentEL_EL2 - b.eq 2f + b.ne 1f - /* The core booted in EL1. KVM cannot be initialized on it. */ -1: wfe - wfi - b 1b - -2: msr SPsel, #1 // We want to use SP_EL{1,2} + msr SPsel, #1 // We want to use SP_EL2 init_el2_hcr 0 @@ -218,11 +202,16 @@ SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu) mov x0, x28 bl ___kvm_hyp_init // Clobbers x0..x2 - /* Leave idmap. */ - mov x0, x29 - ldr x1, =kvm_host_psci_cpu_entry - br x1 -SYM_CODE_END(__kvm_hyp_init_cpu) + /* Leave idmap -- using BLR is OK, LR is restored from host context */ + blr x29 + + // The core booted in EL1, or the C code unexpectedly returned. + // Either way, KVM cannot be initialized on it. +1: wfe + wfi + b 1b +SYM_CODE_END(kvm_hyp_cpu_resume) +SYM_CODE_END(kvm_hyp_cpu_entry) SYM_CODE_START(__kvm_handle_stub_hvc) /* diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c b/arch/arm64/kvm/hyp/nvhe/psci-relay.c index ab4c7bddb163..e20db999e328 100644 --- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c +++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c @@ -201,24 +201,14 @@ static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) __hyp_pa(init_params), 0); } -asmlinkage void __noreturn __kvm_host_psci_cpu_entry(bool is_cpu_on) +static void __noreturn __kvm_host_psci_cpu_entry(unsigned long pc, unsigned long r0) { - struct psci_boot_args *boot_args; - struct kvm_cpu_context *host_ctxt; + struct kvm_cpu_context *host_ctxt = host_data_ptr(host_ctxt); - host_ctxt = host_data_ptr(host_ctxt); trace_hyp_enter(host_ctxt, HYP_REASON_PSCI); - if (is_cpu_on) - boot_args = this_cpu_ptr(&cpu_on_args); - else - boot_args = this_cpu_ptr(&suspend_args); - - cpu_reg(host_ctxt, 0) = boot_args->r0; - write_sysreg_el2(boot_args->pc, SYS_ELR); - - if (is_cpu_on) - release_boot_args(boot_args); + cpu_reg(host_ctxt, 0) = r0; + write_sysreg_el2(pc, SYS_ELR); write_sysreg_el1(INIT_SCTLR_EL1_MMU_OFF, SYS_SCTLR); write_sysreg(INIT_PSTATE_EL1, SPSR_EL2); @@ -227,6 +217,26 @@ asmlinkage void __noreturn __kvm_host_psci_cpu_entry(bool is_cpu_on) __host_enter(host_ctxt); } +asmlinkage void __noreturn __kvm_host_psci_cpu_on_entry(void) +{ + struct psci_boot_args *boot_args = this_cpu_ptr(&cpu_on_args); + unsigned long pc, r0; + + pc = READ_ONCE(boot_args->pc); + r0 = READ_ONCE(boot_args->r0); + + release_boot_args(boot_args); + + __kvm_host_psci_cpu_entry(pc, r0); +} + +asmlinkage void __noreturn __kvm_host_psci_cpu_resume_entry(void) +{ + struct psci_boot_args *boot_args = this_cpu_ptr(&suspend_args); + + __kvm_host_psci_cpu_entry(boot_args->pc, boot_args->r0); +} + static unsigned long psci_0_1_handler(u64 func_id, struct kvm_cpu_context *host_ctxt) { if (is_psci_0_1(cpu_off, func_id) || is_psci_0_1(migrate, func_id))