/* * Copyright (C) 2015 - ARM Ltd * Author: Marc Zyngier * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include #include #include #include __asm__(".arch_extension virt"); /* * Activate the traps, saving the host's fpexc register before * overwriting it. We'll restore it on VM exit. */ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu, u32 *fpexc_host) { u32 val; /* * We are about to set HCPTR.TCP10/11 to trap all floating point * register accesses to HYP, however, the ARM ARM clearly states that * traps are only taken to HYP if the operation would not otherwise * trap to SVC. Therefore, always make sure that for 32-bit guests, * we set FPEXC.EN to prevent traps to SVC, when setting the TCP bits. */ val = read_sysreg(VFP_FPEXC); *fpexc_host = val; if (!(val & FPEXC_EN)) { write_sysreg(val | FPEXC_EN, VFP_FPEXC); isb(); } write_sysreg(vcpu->arch.hcr, HCR); /* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */ write_sysreg(HSTR_T(15), HSTR); write_sysreg(HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11), HCPTR); val = read_sysreg(HDCR); val |= HDCR_TPM | HDCR_TPMCR; /* trap performance monitors */ val |= HDCR_TDRA | HDCR_TDOSA | HDCR_TDA; /* trap debug regs */ write_sysreg(val, HDCR); } static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu) { u32 val; /* * If we pended a virtual abort, preserve it until it gets * cleared. See B1.9.9 (Virtual Abort exception) for details, * but the crucial bit is the zeroing of HCR.VA in the * pseudocode. */ if (vcpu->arch.hcr & HCR_VA) vcpu->arch.hcr = read_sysreg(HCR); write_sysreg(0, HCR); write_sysreg(0, HSTR); val = read_sysreg(HDCR); write_sysreg(val & ~(HDCR_TPM | HDCR_TPMCR), HDCR); write_sysreg(0, HCPTR); } static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu) { struct kvm *kvm = kern_hyp_va(vcpu->kvm); write_sysreg(kvm->arch.vttbr, VTTBR); write_sysreg(vcpu->arch.midr, VPIDR); } static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu) { write_sysreg(0, VTTBR); write_sysreg(read_sysreg(MIDR), VPIDR); } static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu) { if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) __vgic_v3_save_state(vcpu); else __vgic_v2_save_state(vcpu); } static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu) { if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) __vgic_v3_restore_state(vcpu); else __vgic_v2_restore_state(vcpu); } static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu) { u32 hsr = read_sysreg(HSR); u8 ec = hsr >> HSR_EC_SHIFT; u32 hpfar, far; vcpu->arch.fault.hsr = hsr; if (ec == HSR_EC_IABT) far = read_sysreg(HIFAR); else if (ec == HSR_EC_DABT) far = read_sysreg(HDFAR); else return true; /* * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode: * * Abort on the stage 2 translation for a memory access from a * Non-secure PL1 or PL0 mode: * * For any Access flag fault or Translation fault, and also for any * Permission fault on the stage 2 translation of a memory access * made as part of a translation table walk for a stage 1 translation, * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR * is UNKNOWN. */ if (!(hsr & HSR_DABT_S1PTW) && (hsr & HSR_FSC_TYPE) == FSC_PERM) { u64 par, tmp; par = read_sysreg(PAR); write_sysreg(far, ATS1CPR); isb(); tmp = read_sysreg(PAR); write_sysreg(par, PAR); if (unlikely(tmp & 1)) return false; /* Translation failed, back to guest */ hpfar = ((tmp >> 12) & ((1UL << 28) - 1)) << 4; } else { hpfar = read_sysreg(HPFAR); } vcpu->arch.fault.hxfar = far; vcpu->arch.fault.hpfar = hpfar; return true; } int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu) { struct kvm_cpu_context *host_ctxt; struct kvm_cpu_context *guest_ctxt; bool fp_enabled; u64 exit_code; u32 fpexc; vcpu = kern_hyp_va(vcpu); write_sysreg(vcpu, HTPIDR); host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context); guest_ctxt = &vcpu->arch.ctxt; __sysreg_save_state(host_ctxt); __banked_save_state(host_ctxt); __activate_traps(vcpu, &fpexc); __activate_vm(vcpu); __vgic_restore_state(vcpu); __timer_enable_traps(vcpu); __sysreg_restore_state(guest_ctxt); __banked_restore_state(guest_ctxt); /* Jump in the fire! */ again: exit_code = __guest_enter(vcpu, host_ctxt); /* And we're baaack! */ if (exit_code == ARM_EXCEPTION_HVC && !__populate_fault_info(vcpu)) goto again; fp_enabled = __vfp_enabled(); __banked_save_state(guest_ctxt); __sysreg_save_state(guest_ctxt); __timer_disable_traps(vcpu); __vgic_save_state(vcpu); __deactivate_traps(vcpu); __deactivate_vm(vcpu); __banked_restore_state(host_ctxt); __sysreg_restore_state(host_ctxt); if (fp_enabled) { __vfp_save_state(&guest_ctxt->vfp); __vfp_restore_state(&host_ctxt->vfp); } write_sysreg(fpexc, VFP_FPEXC); return exit_code; } static const char * const __hyp_panic_string[] = { [ARM_EXCEPTION_RESET] = "\nHYP panic: RST PC:%08x CPSR:%08x", [ARM_EXCEPTION_UNDEFINED] = "\nHYP panic: UNDEF PC:%08x CPSR:%08x", [ARM_EXCEPTION_SOFTWARE] = "\nHYP panic: SVC PC:%08x CPSR:%08x", [ARM_EXCEPTION_PREF_ABORT] = "\nHYP panic: PABRT PC:%08x CPSR:%08x", [ARM_EXCEPTION_DATA_ABORT] = "\nHYP panic: DABRT PC:%08x ADDR:%08x", [ARM_EXCEPTION_IRQ] = "\nHYP panic: IRQ PC:%08x CPSR:%08x", [ARM_EXCEPTION_FIQ] = "\nHYP panic: FIQ PC:%08x CPSR:%08x", [ARM_EXCEPTION_HVC] = "\nHYP panic: HVC PC:%08x CPSR:%08x", }; void __hyp_text __noreturn __hyp_panic(int cause) { u32 elr = read_special(ELR_hyp); u32 val; if (cause == ARM_EXCEPTION_DATA_ABORT) val = read_sysreg(HDFAR); else val = read_special(SPSR); if (read_sysreg(VTTBR)) { struct kvm_vcpu *vcpu; struct kvm_cpu_context *host_ctxt; vcpu = (struct kvm_vcpu *)read_sysreg(HTPIDR); host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context); __timer_disable_traps(vcpu); __deactivate_traps(vcpu); __deactivate_vm(vcpu); __banked_restore_state(host_ctxt); __sysreg_restore_state(host_ctxt); } /* Call panic for real */ __hyp_do_panic(__hyp_panic_string[cause], elr, val); unreachable(); }