arch/arm/kvm/interrupts.S - kernel/common - Git at Google

 /*
  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  * Author: Christoffer Dall <[email protected]>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2, as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */

 #include <linux/linkage.h>
 #include <linux/const.h>
 #include <asm/unified.h>
 #include <asm/page.h>
 #include <asm/ptrace.h>
 #include <asm/asm-offsets.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_arm.h>
 #include <asm/vfpmacros.h>
 #include "interrupts_head.S"

 	.text

 __kvm_hyp_code_start:
 	.globl __kvm_hyp_code_start

 /********************************************************************
  * Flush per-VMID TLBs
  *
  * void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
  *
  * We rely on the hardware to broadcast the TLB invalidation to all CPUs
  * inside the inner-shareable domain (which is the case for all v7
  * implementations).  If we come across a non-IS SMP implementation, we'll
  * have to use an IPI based mechanism. Until then, we stick to the simple
  * hardware assisted version.
  *
  * As v7 does not support flushing per IPA, just nuke the whole TLB
  * instead, ignoring the ipa value.
  */
 ENTRY(__kvm_tlb_flush_vmid_ipa)
 	push	{r2, r3}

 	dsb	ishst
 	add	r0, r0, #KVM_VTTBR
 	ldrd	r2, r3, [r0]
 	mcrr	p15, 6, r2, r3, c2	@ Write VTTBR
 	isb
 	mcr     p15, 0, r0, c8, c3, 0	@ TLBIALLIS (rt ignored)
 	dsb	ish
 	isb
 	mov	r2, #0
 	mov	r3, #0
 	mcrr	p15, 6, r2, r3, c2	@ Back to VMID #0
 	isb				@ Not necessary if followed by eret

 	pop	{r2, r3}
 	bx	lr
 ENDPROC(__kvm_tlb_flush_vmid_ipa)

 /********************************************************************
  * Flush TLBs and instruction caches of all CPUs inside the inner-shareable
  * domain, for all VMIDs
  *
  * void __kvm_flush_vm_context(void);
  */
 ENTRY(__kvm_flush_vm_context)
 	mov	r0, #0			@ rn parameter for c15 flushes is SBZ

 	/* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */
 	mcr     p15, 4, r0, c8, c3, 4
 	/* Invalidate instruction caches Inner Shareable (ICIALLUIS) */
 	mcr     p15, 0, r0, c7, c1, 0
 	dsb	ish
 	isb				@ Not necessary if followed by eret

 	bx	lr
 ENDPROC(__kvm_flush_vm_context)


 /********************************************************************
  *  Hypervisor world-switch code
  *
  *
  * int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
  */
 ENTRY(__kvm_vcpu_run)
 	@ Save the vcpu pointer
 	mcr	p15, 4, vcpu, c13, c0, 2	@ HTPIDR

 	save_host_regs

 	restore_vgic_state
 	restore_timer_state

 	@ Store hardware CP15 state and load guest state
 	read_cp15_state store_to_vcpu = 0
 	write_cp15_state read_from_vcpu = 1

 	@ If the host kernel has not been configured with VFPv3 support,
 	@ then it is safer if we deny guests from using it as well.
 #ifdef CONFIG_VFPv3
 	@ Set FPEXC_EN so the guest doesn't trap floating point instructions
 	VFPFMRX r2, FPEXC		@ VMRS
 	push	{r2}
 	orr	r2, r2, #FPEXC_EN
 	VFPFMXR FPEXC, r2		@ VMSR
 #endif

 	@ Configure Hyp-role
 	configure_hyp_role vmentry

 	@ Trap coprocessor CRx accesses
 	set_hstr vmentry
 	set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
 	set_hdcr vmentry

 	@ Write configured ID register into MIDR alias
 	ldr	r1, [vcpu, #VCPU_MIDR]
 	mcr	p15, 4, r1, c0, c0, 0

 	@ Write guest view of MPIDR into VMPIDR
 	ldr	r1, [vcpu, #CP15_OFFSET(c0_MPIDR)]
 	mcr	p15, 4, r1, c0, c0, 5

 	@ Set up guest memory translation
 	ldr	r1, [vcpu, #VCPU_KVM]
 	add	r1, r1, #KVM_VTTBR
 	ldrd	r2, r3, [r1]
 	mcrr	p15, 6, r2, r3, c2	@ Write VTTBR

 	@ We're all done, just restore the GPRs and go to the guest
 	restore_guest_regs
 	clrex				@ Clear exclusive monitor
 	eret

 __kvm_vcpu_return:
 	/*
 	 * return convention:
 	 * guest r0, r1, r2 saved on the stack
 	 * r0: vcpu pointer
 	 * r1: exception code
 	 */
 	save_guest_regs

 	@ Set VMID == 0
 	mov	r2, #0
 	mov	r3, #0
 	mcrr	p15, 6, r2, r3, c2	@ Write VTTBR

 	@ Don't trap coprocessor accesses for host kernel
 	set_hstr vmexit
 	set_hdcr vmexit
 	set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))

 #ifdef CONFIG_VFPv3
 	@ Save floating point registers we if let guest use them.
 	tst	r2, #(HCPTR_TCP(10) | HCPTR_TCP(11))
 	bne	after_vfp_restore

 	@ Switch VFP/NEON hardware state to the host's
 	add	r7, vcpu, #VCPU_VFP_GUEST
 	store_vfp_state r7
 	add	r7, vcpu, #VCPU_VFP_HOST
 	ldr	r7, [r7]
 	restore_vfp_state r7

 after_vfp_restore:
 	@ Restore FPEXC_EN which we clobbered on entry
 	pop	{r2}
 	VFPFMXR FPEXC, r2
 #endif

 	@ Reset Hyp-role
 	configure_hyp_role vmexit

 	@ Let host read hardware MIDR
 	mrc	p15, 0, r2, c0, c0, 0
 	mcr	p15, 4, r2, c0, c0, 0

 	@ Back to hardware MPIDR
 	mrc	p15, 0, r2, c0, c0, 5
 	mcr	p15, 4, r2, c0, c0, 5

 	@ Store guest CP15 state and restore host state
 	read_cp15_state store_to_vcpu = 1
 	write_cp15_state read_from_vcpu = 0

 	save_timer_state
 	save_vgic_state

 	restore_host_regs
 	clrex				@ Clear exclusive monitor
 	mov	r0, r1			@ Return the return code
 	mov	r1, #0			@ Clear upper bits in return value
 	bx	lr			@ return to IOCTL

 /********************************************************************
  *  Call function in Hyp mode
  *
  *
  * u64 kvm_call_hyp(void *hypfn, ...);
  *
  * This is not really a variadic function in the classic C-way and care must
  * be taken when calling this to ensure parameters are passed in registers
  * only, since the stack will change between the caller and the callee.
  *
  * Call the function with the first argument containing a pointer to the
  * function you wish to call in Hyp mode, and subsequent arguments will be
  * passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the
  * function pointer can be passed).  The function being called must be mapped
  * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c).  Return values are
  * passed in r0 and r1.
  *
  * A function pointer with a value of 0xffffffff has a special meaning,
  * and is used to implement __hyp_get_vectors in the same way as in
  * arch/arm/kernel/hyp_stub.S.
  *
  * The calling convention follows the standard AAPCS:
  *   r0 - r3: caller save
  *   r12:     caller save
  *   rest:    callee save
  */
 ENTRY(kvm_call_hyp)
 	hvc	#0
 	bx	lr

 /********************************************************************
  * Hypervisor exception vector and handlers
  *
  *
  * The KVM/ARM Hypervisor ABI is defined as follows:
  *
  * Entry to Hyp mode from the host kernel will happen _only_ when an HVC
  * instruction is issued since all traps are disabled when running the host
  * kernel as per the Hyp-mode initialization at boot time.
  *
  * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc
  * below) when the HVC instruction is called from SVC mode (i.e. a guest or the
  * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC
  * instructions are called from within Hyp-mode.
  *
  * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode):
  *    Switching to Hyp mode is done through a simple HVC #0 instruction. The
  *    exception vector code will check that the HVC comes from VMID==0 and if
  *    so will push the necessary state (SPSR, lr_usr) on the Hyp stack.
  *    - r0 contains a pointer to a HYP function
  *    - r1, r2, and r3 contain arguments to the above function.
  *    - The HYP function will be called with its arguments in r0, r1 and r2.
  *    On HYP function return, we return directly to SVC.
  *
  * Note that the above is used to execute code in Hyp-mode from a host-kernel
  * point of view, and is a different concept from performing a world-switch and
  * executing guest code SVC mode (with a VMID != 0).
  */

 /* Handle undef, svc, pabt, or dabt by crashing with a user notice */
 .macro bad_exception exception_code, panic_str
 	push	{r0-r2}
 	mrrc	p15, 6, r0, r1, c2	@ Read VTTBR
 	lsr	r1, r1, #16
 	ands	r1, r1, #0xff
 	beq	99f

 	load_vcpu			@ Load VCPU pointer
 	.if \exception_code == ARM_EXCEPTION_DATA_ABORT
 	mrc	p15, 4, r2, c5, c2, 0	@ HSR
 	mrc	p15, 4, r1, c6, c0, 0	@ HDFAR
 	str	r2, [vcpu, #VCPU_HSR]
 	str	r1, [vcpu, #VCPU_HxFAR]
 	.endif
 	.if \exception_code == ARM_EXCEPTION_PREF_ABORT
 	mrc	p15, 4, r2, c5, c2, 0	@ HSR
 	mrc	p15, 4, r1, c6, c0, 2	@ HIFAR
 	str	r2, [vcpu, #VCPU_HSR]
 	str	r1, [vcpu, #VCPU_HxFAR]
 	.endif
 	mov	r1, #\exception_code
 	b	__kvm_vcpu_return

 	@ We were in the host already. Let's craft a panic-ing return to SVC.
 99:	mrs	r2, cpsr
 	bic	r2, r2, #MODE_MASK
 	orr	r2, r2, #SVC_MODE
 THUMB(	orr	r2, r2, #PSR_T_BIT	)
 	msr	spsr_cxsf, r2
 	mrs	r1, ELR_hyp
 	ldr	r2, =BSYM(panic)
 	msr	ELR_hyp, r2
 	ldr	r0, =\panic_str
 	clrex				@ Clear exclusive monitor
 	eret
 .endm

 	.text

 	.align 5
 __kvm_hyp_vector:
 	.globl __kvm_hyp_vector

 	@ Hyp-mode exception vector
 	W(b)	hyp_reset
 	W(b)	hyp_undef
 	W(b)	hyp_svc
 	W(b)	hyp_pabt
 	W(b)	hyp_dabt
 	W(b)	hyp_hvc
 	W(b)	hyp_irq
 	W(b)	hyp_fiq

 	.align
 hyp_reset:
 	b	hyp_reset

 	.align
 hyp_undef:
 	bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str

 	.align
 hyp_svc:
 	bad_exception ARM_EXCEPTION_HVC, svc_die_str

 	.align
 hyp_pabt:
 	bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str

 	.align
 hyp_dabt:
 	bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str

 	.align
 hyp_hvc:
 	/*
 	 * Getting here is either becuase of a trap from a guest or from calling
 	 * HVC from the host kernel, which means "switch to Hyp mode".
 	 */
 	push	{r0, r1, r2}

 	@ Check syndrome register
 	mrc	p15, 4, r1, c5, c2, 0	@ HSR
 	lsr	r0, r1, #HSR_EC_SHIFT
 #ifdef CONFIG_VFPv3
 	cmp	r0, #HSR_EC_CP_0_13
 	beq	switch_to_guest_vfp
 #endif
 	cmp	r0, #HSR_EC_HVC
 	bne	guest_trap		@ Not HVC instr.

 	/*
 	 * Let's check if the HVC came from VMID 0 and allow simple
 	 * switch to Hyp mode
 	 */
 	mrrc    p15, 6, r0, r2, c2
 	lsr     r2, r2, #16
 	and     r2, r2, #0xff
 	cmp     r2, #0
 	bne	guest_trap		@ Guest called HVC

 host_switch_to_hyp:
 	pop	{r0, r1, r2}

 	/* Check for __hyp_get_vectors */
 	cmp	r0, #-1
 	mrceq	p15, 4, r0, c12, c0, 0	@ get HVBAR
 	beq	1f

 	push	{lr}
 	mrs	lr, SPSR
 	push	{lr}

 	mov	lr, r0
 	mov	r0, r1
 	mov	r1, r2
 	mov	r2, r3

 THUMB(	orr	lr, #1)
 	blx	lr			@ Call the HYP function

 	pop	{lr}
 	msr	SPSR_csxf, lr
 	pop	{lr}
 1:	eret

 guest_trap:
 	load_vcpu			@ Load VCPU pointer to r0
 	str	r1, [vcpu, #VCPU_HSR]

 	@ Check if we need the fault information
 	lsr	r1, r1, #HSR_EC_SHIFT
 	cmp	r1, #HSR_EC_IABT
 	mrceq	p15, 4, r2, c6, c0, 2	@ HIFAR
 	beq	2f
 	cmp	r1, #HSR_EC_DABT
 	bne	1f
 	mrc	p15, 4, r2, c6, c0, 0	@ HDFAR

 2:	str	r2, [vcpu, #VCPU_HxFAR]

 	/*
 	 * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
 	 *
 	 * Abort on the stage 2 translation for a memory access from a
 	 * Non-secure PL1 or PL0 mode:
 	 *
 	 * For any Access flag fault or Translation fault, and also for any
 	 * Permission fault on the stage 2 translation of a memory access
 	 * made as part of a translation table walk for a stage 1 translation,
 	 * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
 	 * is UNKNOWN.
 	 */

 	/* Check for permission fault, and S1PTW */
 	mrc	p15, 4, r1, c5, c2, 0	@ HSR
 	and	r0, r1, #HSR_FSC_TYPE
 	cmp	r0, #FSC_PERM
 	tsteq	r1, #(1 << 7)		@ S1PTW
 	mrcne	p15, 4, r2, c6, c0, 4	@ HPFAR
 	bne	3f

 	/* Preserve PAR */
 	mrrc	p15, 0, r0, r1, c7	@ PAR
 	push	{r0, r1}

 	/* Resolve IPA using the xFAR */
 	mcr	p15, 0, r2, c7, c8, 0	@ ATS1CPR
 	isb
 	mrrc	p15, 0, r0, r1, c7	@ PAR
 	tst	r0, #1
 	bne	4f			@ Failed translation
 	ubfx	r2, r0, #12, #20
 	lsl	r2, r2, #4
 	orr	r2, r2, r1, lsl #24

 	/* Restore PAR */
 	pop	{r0, r1}
 	mcrr	p15, 0, r0, r1, c7	@ PAR

 3:	load_vcpu			@ Load VCPU pointer to r0
 	str	r2, [r0, #VCPU_HPFAR]

 1:	mov	r1, #ARM_EXCEPTION_HVC
 	b	__kvm_vcpu_return

 4:	pop	{r0, r1}		@ Failed translation, return to guest
 	mcrr	p15, 0, r0, r1, c7	@ PAR
 	clrex
 	pop	{r0, r1, r2}
 	eret

 /*
  * If VFPv3 support is not available, then we will not switch the VFP
  * registers; however cp10 and cp11 accesses will still trap and fallback
  * to the regular coprocessor emulation code, which currently will
  * inject an undefined exception to the guest.
  */
 #ifdef CONFIG_VFPv3
 switch_to_guest_vfp:
 	load_vcpu			@ Load VCPU pointer to r0
 	push	{r3-r7}

 	@ NEON/VFP used.  Turn on VFP access.
 	set_hcptr vmexit, (HCPTR_TCP(10) | HCPTR_TCP(11))

 	@ Switch VFP/NEON hardware state to the guest's
 	add	r7, r0, #VCPU_VFP_HOST
 	ldr	r7, [r7]
 	store_vfp_state r7
 	add	r7, r0, #VCPU_VFP_GUEST
 	restore_vfp_state r7

 	pop	{r3-r7}
 	pop	{r0-r2}
 	clrex
 	eret
 #endif

 	.align
 hyp_irq:
 	push	{r0, r1, r2}
 	mov	r1, #ARM_EXCEPTION_IRQ
 	load_vcpu			@ Load VCPU pointer to r0
 	b	__kvm_vcpu_return

 	.align
 hyp_fiq:
 	b	hyp_fiq

 	.ltorg

 __kvm_hyp_code_end:
 	.globl	__kvm_hyp_code_end

 	.section ".rodata"

 und_die_str:
 	.ascii	"unexpected undefined exception in Hyp mode at: %#08x\n"
 pabt_die_str:
 	.ascii	"unexpected prefetch abort in Hyp mode at: %#08x\n"
 dabt_die_str:
 	.ascii	"unexpected data abort in Hyp mode at: %#08x\n"
 svc_die_str:
 	.ascii	"unexpected HVC/SVC trap in Hyp mode at: %#08x\n"
	/*
	* Copyright (C) 2012 - Virtual Open Systems and Columbia University
	* Author: Christoffer Dall <[email protected]>
	*
	* This program is free software; you can redistribute it and/or modify
	* it under the terms of the GNU General Public License, version 2, as
	* published by the Free Software Foundation.
	*
	* This program is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*
	* You should have received a copy of the GNU General Public License
	* along with this program; if not, write to the Free Software
	* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
	*/

	#include <linux/linkage.h>
	#include <linux/const.h>
	#include <asm/unified.h>
	#include <asm/page.h>
	#include <asm/ptrace.h>
	#include <asm/asm-offsets.h>
	#include <asm/kvm_asm.h>
	#include <asm/kvm_arm.h>
	#include <asm/vfpmacros.h>
	#include "interrupts_head.S"

	.text

	__kvm_hyp_code_start:
	.globl __kvm_hyp_code_start

	/********************************************************************
	* Flush per-VMID TLBs
	*
	* void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
	*
	* We rely on the hardware to broadcast the TLB invalidation to all CPUs
	* inside the inner-shareable domain (which is the case for all v7
	* implementations). If we come across a non-IS SMP implementation, we'll
	* have to use an IPI based mechanism. Until then, we stick to the simple
	* hardware assisted version.
	*
	* As v7 does not support flushing per IPA, just nuke the whole TLB
	* instead, ignoring the ipa value.
	*/
	ENTRY(__kvm_tlb_flush_vmid_ipa)
	push {r2, r3}

	dsb ishst
	add r0, r0, #KVM_VTTBR
	ldrd r2, r3, [r0]
	mcrr p15, 6, r2, r3, c2 @ Write VTTBR
	isb
	mcr p15, 0, r0, c8, c3, 0 @ TLBIALLIS (rt ignored)
	dsb ish
	isb
	mov r2, #0
	mov r3, #0
	mcrr p15, 6, r2, r3, c2 @ Back to VMID #0
	isb @ Not necessary if followed by eret

	pop {r2, r3}
	bx lr
	ENDPROC(__kvm_tlb_flush_vmid_ipa)

	/********************************************************************
	* Flush TLBs and instruction caches of all CPUs inside the inner-shareable
	* domain, for all VMIDs
	*
	* void __kvm_flush_vm_context(void);
	*/
	ENTRY(__kvm_flush_vm_context)
	mov r0, #0 @ rn parameter for c15 flushes is SBZ

	/* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */
	mcr p15, 4, r0, c8, c3, 4
	/* Invalidate instruction caches Inner Shareable (ICIALLUIS) */
	mcr p15, 0, r0, c7, c1, 0
	dsb ish
	isb @ Not necessary if followed by eret

	bx lr
	ENDPROC(__kvm_flush_vm_context)


	/********************************************************************
	* Hypervisor world-switch code
	*
	*
	* int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
	*/
	ENTRY(__kvm_vcpu_run)
	@ Save the vcpu pointer
	mcr p15, 4, vcpu, c13, c0, 2 @ HTPIDR

	save_host_regs

	restore_vgic_state
	restore_timer_state

	@ Store hardware CP15 state and load guest state
	read_cp15_state store_to_vcpu = 0
	write_cp15_state read_from_vcpu = 1

	@ If the host kernel has not been configured with VFPv3 support,
	@ then it is safer if we deny guests from using it as well.
	#ifdef CONFIG_VFPv3
	@ Set FPEXC_EN so the guest doesn't trap floating point instructions
	VFPFMRX r2, FPEXC @ VMRS
	push {r2}
	orr r2, r2, #FPEXC_EN
	VFPFMXR FPEXC, r2 @ VMSR
	#endif

	@ Configure Hyp-role
	configure_hyp_role vmentry

	@ Trap coprocessor CRx accesses
	set_hstr vmentry
	set_hcptr vmentry, (HCPTR_TTA \| HCPTR_TCP(10) \| HCPTR_TCP(11))
	set_hdcr vmentry

	@ Write configured ID register into MIDR alias
	ldr r1, [vcpu, #VCPU_MIDR]
	mcr p15, 4, r1, c0, c0, 0

	@ Write guest view of MPIDR into VMPIDR
	ldr r1, [vcpu, #CP15_OFFSET(c0_MPIDR)]
	mcr p15, 4, r1, c0, c0, 5

	@ Set up guest memory translation
	ldr r1, [vcpu, #VCPU_KVM]
	add r1, r1, #KVM_VTTBR
	ldrd r2, r3, [r1]
	mcrr p15, 6, r2, r3, c2 @ Write VTTBR

	@ We're all done, just restore the GPRs and go to the guest
	restore_guest_regs
	clrex @ Clear exclusive monitor
	eret

	__kvm_vcpu_return:
	/*
	* return convention:
	* guest r0, r1, r2 saved on the stack
	* r0: vcpu pointer
	* r1: exception code
	*/
	save_guest_regs

	@ Set VMID == 0
	mov r2, #0
	mov r3, #0
	mcrr p15, 6, r2, r3, c2 @ Write VTTBR

	@ Don't trap coprocessor accesses for host kernel
	set_hstr vmexit
	set_hdcr vmexit
	set_hcptr vmexit, (HCPTR_TTA \| HCPTR_TCP(10) \| HCPTR_TCP(11))

	#ifdef CONFIG_VFPv3
	@ Save floating point registers we if let guest use them.
	tst r2, #(HCPTR_TCP(10) \| HCPTR_TCP(11))
	bne after_vfp_restore

	@ Switch VFP/NEON hardware state to the host's
	add r7, vcpu, #VCPU_VFP_GUEST
	store_vfp_state r7
	add r7, vcpu, #VCPU_VFP_HOST
	ldr r7, [r7]
	restore_vfp_state r7

	after_vfp_restore:
	@ Restore FPEXC_EN which we clobbered on entry
	pop {r2}
	VFPFMXR FPEXC, r2
	#endif

	@ Reset Hyp-role
	configure_hyp_role vmexit

	@ Let host read hardware MIDR
	mrc p15, 0, r2, c0, c0, 0
	mcr p15, 4, r2, c0, c0, 0

	@ Back to hardware MPIDR
	mrc p15, 0, r2, c0, c0, 5
	mcr p15, 4, r2, c0, c0, 5

	@ Store guest CP15 state and restore host state
	read_cp15_state store_to_vcpu = 1
	write_cp15_state read_from_vcpu = 0

	save_timer_state
	save_vgic_state

	restore_host_regs
	clrex @ Clear exclusive monitor
	mov r0, r1 @ Return the return code
	mov r1, #0 @ Clear upper bits in return value
	bx lr @ return to IOCTL

	/********************************************************************
	* Call function in Hyp mode
	*
	*
	* u64 kvm_call_hyp(void *hypfn, ...);
	*
	* This is not really a variadic function in the classic C-way and care must
	* be taken when calling this to ensure parameters are passed in registers
	* only, since the stack will change between the caller and the callee.
	*
	* Call the function with the first argument containing a pointer to the
	* function you wish to call in Hyp mode, and subsequent arguments will be
	* passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the
	* function pointer can be passed). The function being called must be mapped
	* in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are
	* passed in r0 and r1.
	*
	* A function pointer with a value of 0xffffffff has a special meaning,
	* and is used to implement __hyp_get_vectors in the same way as in
	* arch/arm/kernel/hyp_stub.S.
	*
	* The calling convention follows the standard AAPCS:
	* r0 - r3: caller save
	* r12: caller save
	* rest: callee save
	*/
	ENTRY(kvm_call_hyp)
	hvc #0
	bx lr

	/********************************************************************
	* Hypervisor exception vector and handlers
	*
	*
	* The KVM/ARM Hypervisor ABI is defined as follows:
	*
	* Entry to Hyp mode from the host kernel will happen _only_ when an HVC
	* instruction is issued since all traps are disabled when running the host
	* kernel as per the Hyp-mode initialization at boot time.
	*
	* HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc
	* below) when the HVC instruction is called from SVC mode (i.e. a guest or the
	* host kernel) and they cause a trap to the vector page + offset 0x8 when HVC
	* instructions are called from within Hyp-mode.
	*
	* Hyp-ABI: Calling HYP-mode functions from host (in SVC mode):
	* Switching to Hyp mode is done through a simple HVC #0 instruction. The
	* exception vector code will check that the HVC comes from VMID==0 and if
	* so will push the necessary state (SPSR, lr_usr) on the Hyp stack.
	* - r0 contains a pointer to a HYP function
	* - r1, r2, and r3 contain arguments to the above function.
	* - The HYP function will be called with its arguments in r0, r1 and r2.
	* On HYP function return, we return directly to SVC.
	*
	* Note that the above is used to execute code in Hyp-mode from a host-kernel
	* point of view, and is a different concept from performing a world-switch and
	* executing guest code SVC mode (with a VMID != 0).
	*/

	/* Handle undef, svc, pabt, or dabt by crashing with a user notice */
	.macro bad_exception exception_code, panic_str
	push {r0-r2}
	mrrc p15, 6, r0, r1, c2 @ Read VTTBR
	lsr r1, r1, #16
	ands r1, r1, #0xff
	beq 99f

	load_vcpu @ Load VCPU pointer
	.if \exception_code == ARM_EXCEPTION_DATA_ABORT
	mrc p15, 4, r2, c5, c2, 0 @ HSR
	mrc p15, 4, r1, c6, c0, 0 @ HDFAR
	str r2, [vcpu, #VCPU_HSR]
	str r1, [vcpu, #VCPU_HxFAR]
	.endif
	.if \exception_code == ARM_EXCEPTION_PREF_ABORT
	mrc p15, 4, r2, c5, c2, 0 @ HSR
	mrc p15, 4, r1, c6, c0, 2 @ HIFAR
	str r2, [vcpu, #VCPU_HSR]
	str r1, [vcpu, #VCPU_HxFAR]
	.endif
	mov r1, #\exception_code
	b __kvm_vcpu_return

	@ We were in the host already. Let's craft a panic-ing return to SVC.
	99: mrs r2, cpsr
	bic r2, r2, #MODE_MASK
	orr r2, r2, #SVC_MODE
	THUMB( orr r2, r2, #PSR_T_BIT )
	msr spsr_cxsf, r2
	mrs r1, ELR_hyp
	ldr r2, =BSYM(panic)
	msr ELR_hyp, r2
	ldr r0, =\panic_str
	clrex @ Clear exclusive monitor
	eret
	.endm

	.text

	.align 5
	__kvm_hyp_vector:
	.globl __kvm_hyp_vector

	@ Hyp-mode exception vector
	W(b) hyp_reset
	W(b) hyp_undef
	W(b) hyp_svc
	W(b) hyp_pabt
	W(b) hyp_dabt
	W(b) hyp_hvc
	W(b) hyp_irq
	W(b) hyp_fiq

	.align
	hyp_reset:
	b hyp_reset

	.align
	hyp_undef:
	bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str

	.align
	hyp_svc:
	bad_exception ARM_EXCEPTION_HVC, svc_die_str

	.align
	hyp_pabt:
	bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str

	.align
	hyp_dabt:
	bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str

	.align
	hyp_hvc:
	/*
	* Getting here is either becuase of a trap from a guest or from calling
	* HVC from the host kernel, which means "switch to Hyp mode".
	*/
	push {r0, r1, r2}

	@ Check syndrome register
	mrc p15, 4, r1, c5, c2, 0 @ HSR
	lsr r0, r1, #HSR_EC_SHIFT
	#ifdef CONFIG_VFPv3
	cmp r0, #HSR_EC_CP_0_13
	beq switch_to_guest_vfp
	#endif
	cmp r0, #HSR_EC_HVC
	bne guest_trap @ Not HVC instr.

	/*
	* Let's check if the HVC came from VMID 0 and allow simple
	* switch to Hyp mode
	*/
	mrrc p15, 6, r0, r2, c2
	lsr r2, r2, #16
	and r2, r2, #0xff
	cmp r2, #0
	bne guest_trap @ Guest called HVC

	host_switch_to_hyp:
	pop {r0, r1, r2}

	/* Check for __hyp_get_vectors */
	cmp r0, #-1
	mrceq p15, 4, r0, c12, c0, 0 @ get HVBAR
	beq 1f

	push {lr}
	mrs lr, SPSR
	push {lr}

	mov lr, r0
	mov r0, r1
	mov r1, r2
	mov r2, r3

	THUMB( orr lr, #1)
	blx lr @ Call the HYP function

	pop {lr}
	msr SPSR_csxf, lr
	pop {lr}
	1: eret

	guest_trap:
	load_vcpu @ Load VCPU pointer to r0
	str r1, [vcpu, #VCPU_HSR]

	@ Check if we need the fault information
	lsr r1, r1, #HSR_EC_SHIFT
	cmp r1, #HSR_EC_IABT
	mrceq p15, 4, r2, c6, c0, 2 @ HIFAR
	beq 2f
	cmp r1, #HSR_EC_DABT
	bne 1f
	mrc p15, 4, r2, c6, c0, 0 @ HDFAR

	2: str r2, [vcpu, #VCPU_HxFAR]

	/*
	* B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
	*
	* Abort on the stage 2 translation for a memory access from a
	* Non-secure PL1 or PL0 mode:
	*
	* For any Access flag fault or Translation fault, and also for any
	* Permission fault on the stage 2 translation of a memory access
	* made as part of a translation table walk for a stage 1 translation,
	* the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
	* is UNKNOWN.
	*/

	/* Check for permission fault, and S1PTW */
	mrc p15, 4, r1, c5, c2, 0 @ HSR
	and r0, r1, #HSR_FSC_TYPE
	cmp r0, #FSC_PERM
	tsteq r1, #(1 << 7) @ S1PTW
	mrcne p15, 4, r2, c6, c0, 4 @ HPFAR
	bne 3f

	/* Preserve PAR */
	mrrc p15, 0, r0, r1, c7 @ PAR
	push {r0, r1}

	/* Resolve IPA using the xFAR */
	mcr p15, 0, r2, c7, c8, 0 @ ATS1CPR
	isb
	mrrc p15, 0, r0, r1, c7 @ PAR
	tst r0, #1
	bne 4f @ Failed translation
	ubfx r2, r0, #12, #20
	lsl r2, r2, #4
	orr r2, r2, r1, lsl #24

	/* Restore PAR */
	pop {r0, r1}
	mcrr p15, 0, r0, r1, c7 @ PAR

	3: load_vcpu @ Load VCPU pointer to r0
	str r2, [r0, #VCPU_HPFAR]

	1: mov r1, #ARM_EXCEPTION_HVC
	b __kvm_vcpu_return

	4: pop {r0, r1} @ Failed translation, return to guest
	mcrr p15, 0, r0, r1, c7 @ PAR
	clrex
	pop {r0, r1, r2}
	eret

	/*
	* If VFPv3 support is not available, then we will not switch the VFP
	* registers; however cp10 and cp11 accesses will still trap and fallback
	* to the regular coprocessor emulation code, which currently will
	* inject an undefined exception to the guest.
	*/
	#ifdef CONFIG_VFPv3
	switch_to_guest_vfp:
	load_vcpu @ Load VCPU pointer to r0
	push {r3-r7}

	@ NEON/VFP used. Turn on VFP access.
	set_hcptr vmexit, (HCPTR_TCP(10) \| HCPTR_TCP(11))

	@ Switch VFP/NEON hardware state to the guest's
	add r7, r0, #VCPU_VFP_HOST
	ldr r7, [r7]
	store_vfp_state r7
	add r7, r0, #VCPU_VFP_GUEST
	restore_vfp_state r7

	pop {r3-r7}
	pop {r0-r2}
	clrex
	eret
	#endif

	.align
	hyp_irq:
	push {r0, r1, r2}
	mov r1, #ARM_EXCEPTION_IRQ
	load_vcpu @ Load VCPU pointer to r0
	b __kvm_vcpu_return

	.align
	hyp_fiq:
	b hyp_fiq

	.ltorg

	__kvm_hyp_code_end:
	.globl __kvm_hyp_code_end

	.section ".rodata"

	und_die_str:
	.ascii "unexpected undefined exception in Hyp mode at: %#08x\n"
	pabt_die_str:
	.ascii "unexpected prefetch abort in Hyp mode at: %#08x\n"
	dabt_die_str:
	.ascii "unexpected data abort in Hyp mode at: %#08x\n"
	svc_die_str:
	.ascii "unexpected HVC/SVC trap in Hyp mode at: %#08x\n"