1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
|
/*
* Copyright (C) 2012,2013 - ARM Ltd
* Author: Marc Zyngier <marc.zyngier@arm.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/linkage.h>
#include <linux/irqchip/arm-gic.h>
#include <asm/assembler.h>
#include <asm/memory.h>
#include <asm/asm-offsets.h>
#include <asm/fpsimdmacros.h>
#include <asm/kvm.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_mmu.h>
#define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x)
#define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
#define CPU_SPSR_OFFSET(x) CPU_GP_REG_OFFSET(CPU_SPSR + 8*x)
#define CPU_SYSREG_OFFSET(x) (CPU_SYSREGS + 8*x)
.text
.pushsection .hyp.text, "ax"
.align PAGE_SHIFT
__kvm_hyp_code_start:
.globl __kvm_hyp_code_start
.macro save_common_regs
// x2: base address for cpu context
// x3: tmp register
add x3, x2, #CPU_XREG_OFFSET(19)
stp x19, x20, [x3]
stp x21, x22, [x3, #16]
stp x23, x24, [x3, #32]
stp x25, x26, [x3, #48]
stp x27, x28, [x3, #64]
stp x29, lr, [x3, #80]
mrs x19, sp_el0
mrs x20, elr_el2 // EL1 PC
mrs x21, spsr_el2 // EL1 pstate
stp x19, x20, [x3, #96]
str x21, [x3, #112]
mrs x22, sp_el1
mrs x23, elr_el1
mrs x24, spsr_el1
str x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
str x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]
str x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]
.endm
.macro restore_common_regs
// x2: base address for cpu context
// x3: tmp register
ldr x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
ldr x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]
ldr x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]
msr sp_el1, x22
msr elr_el1, x23
msr spsr_el1, x24
add x3, x2, #CPU_XREG_OFFSET(31) // SP_EL0
ldp x19, x20, [x3]
ldr x21, [x3, #16]
msr sp_el0, x19
msr elr_el2, x20 // EL1 PC
msr spsr_el2, x21 // EL1 pstate
add x3, x2, #CPU_XREG_OFFSET(19)
ldp x19, x20, [x3]
ldp x21, x22, [x3, #16]
ldp x23, x24, [x3, #32]
ldp x25, x26, [x3, #48]
ldp x27, x28, [x3, #64]
ldp x29, lr, [x3, #80]
.endm
.macro save_host_regs
save_common_regs
.endm
.macro restore_host_regs
restore_common_regs
.endm
.macro save_fpsimd
// x2: cpu context address
// x3, x4: tmp regs
add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
fpsimd_save x3, 4
.endm
.macro restore_fpsimd
// x2: cpu context address
// x3, x4: tmp regs
add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
fpsimd_restore x3, 4
.endm
.macro save_guest_regs
// x0 is the vcpu address
// x1 is the return code, do not corrupt!
// x2 is the cpu context
// x3 is a tmp register
// Guest's x0-x3 are on the stack
// Compute base to save registers
add x3, x2, #CPU_XREG_OFFSET(4)
stp x4, x5, [x3]
stp x6, x7, [x3, #16]
stp x8, x9, [x3, #32]
stp x10, x11, [x3, #48]
stp x12, x13, [x3, #64]
stp x14, x15, [x3, #80]
stp x16, x17, [x3, #96]
str x18, [x3, #112]
pop x6, x7 // x2, x3
pop x4, x5 // x0, x1
add x3, x2, #CPU_XREG_OFFSET(0)
stp x4, x5, [x3]
stp x6, x7, [x3, #16]
save_common_regs
.endm
.macro restore_guest_regs
// x0 is the vcpu address.
// x2 is the cpu context
// x3 is a tmp register
// Prepare x0-x3 for later restore
add x3, x2, #CPU_XREG_OFFSET(0)
ldp x4, x5, [x3]
ldp x6, x7, [x3, #16]
push x4, x5 // Push x0-x3 on the stack
push x6, x7
// x4-x18
ldp x4, x5, [x3, #32]
ldp x6, x7, [x3, #48]
ldp x8, x9, [x3, #64]
ldp x10, x11, [x3, #80]
ldp x12, x13, [x3, #96]
ldp x14, x15, [x3, #112]
ldp x16, x17, [x3, #128]
ldr x18, [x3, #144]
// x19-x29, lr, sp*, elr*, spsr*
restore_common_regs
// Last bits of the 64bit state
pop x2, x3
pop x0, x1
// Do not touch any register after this!
.endm
/*
* Macros to perform system register save/restore.
*
* Ordering here is absolutely critical, and must be kept consistent
* in {save,restore}_sysregs, {save,restore}_guest_32bit_state,
* and in kvm_asm.h.
*
* In other words, don't touch any of these unless you know what
* you are doing.
*/
.macro save_sysregs
// x2: base address for cpu context
// x3: tmp register
add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)
mrs x4, vmpidr_el2
mrs x5, csselr_el1
mrs x6, sctlr_el1
mrs x7, actlr_el1
mrs x8, cpacr_el1
mrs x9, ttbr0_el1
mrs x10, ttbr1_el1
mrs x11, tcr_el1
mrs x12, esr_el1
mrs x13, afsr0_el1
mrs x14, afsr1_el1
mrs x15, far_el1
mrs x16, mair_el1
mrs x17, vbar_el1
mrs x18, contextidr_el1
mrs x19, tpidr_el0
mrs x20, tpidrro_el0
mrs x21, tpidr_el1
mrs x22, amair_el1
mrs x23, cntkctl_el1
stp x4, x5, [x3]
stp x6, x7, [x3, #16]
stp x8, x9, [x3, #32]
stp x10, x11, [x3, #48]
stp x12, x13, [x3, #64]
stp x14, x15, [x3, #80]
stp x16, x17, [x3, #96]
stp x18, x19, [x3, #112]
stp x20, x21, [x3, #128]
stp x22, x23, [x3, #144]
.endm
.macro restore_sysregs
// x2: base address for cpu context
// x3: tmp register
add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)
ldp x4, x5, [x3]
ldp x6, x7, [x3, #16]
ldp x8, x9, [x3, #32]
ldp x10, x11, [x3, #48]
ldp x12, x13, [x3, #64]
ldp x14, x15, [x3, #80]
ldp x16, x17, [x3, #96]
ldp x18, x19, [x3, #112]
ldp x20, x21, [x3, #128]
ldp x22, x23, [x3, #144]
msr vmpidr_el2, x4
msr csselr_el1, x5
msr sctlr_el1, x6
msr actlr_el1, x7
msr cpacr_el1, x8
msr ttbr0_el1, x9
msr ttbr1_el1, x10
msr tcr_el1, x11
msr esr_el1, x12
msr afsr0_el1, x13
msr afsr1_el1, x14
msr far_el1, x15
msr mair_el1, x16
msr vbar_el1, x17
msr contextidr_el1, x18
msr tpidr_el0, x19
msr tpidrro_el0, x20
msr tpidr_el1, x21
msr amair_el1, x22
msr cntkctl_el1, x23
.endm
.macro activate_traps
ldr x2, [x0, #VCPU_IRQ_LINES]
ldr x1, [x0, #VCPU_HCR_EL2]
orr x2, x2, x1
msr hcr_el2, x2
ldr x2, =(CPTR_EL2_TTA)
msr cptr_el2, x2
ldr x2, =(1 << 15) // Trap CP15 Cr=15
msr hstr_el2, x2
mrs x2, mdcr_el2
and x2, x2, #MDCR_EL2_HPMN_MASK
orr x2, x2, #(MDCR_EL2_TPM | MDCR_EL2_TPMCR)
msr mdcr_el2, x2
.endm
.macro deactivate_traps
mov x2, #HCR_RW
msr hcr_el2, x2
msr cptr_el2, xzr
msr hstr_el2, xzr
mrs x2, mdcr_el2
and x2, x2, #MDCR_EL2_HPMN_MASK
msr mdcr_el2, x2
.endm
.macro activate_vm
ldr x1, [x0, #VCPU_KVM]
kern_hyp_va x1
ldr x2, [x1, #KVM_VTTBR]
msr vttbr_el2, x2
.endm
.macro deactivate_vm
msr vttbr_el2, xzr
.endm
/*
* Save the VGIC CPU state into memory
* x0: Register pointing to VCPU struct
* Do not corrupt x1!!!
*/
.macro save_vgic_state
/* Get VGIC VCTRL base into x2 */
ldr x2, [x0, #VCPU_KVM]
kern_hyp_va x2
ldr x2, [x2, #KVM_VGIC_VCTRL]
kern_hyp_va x2
cbz x2, 2f // disabled
/* Compute the address of struct vgic_cpu */
add x3, x0, #VCPU_VGIC_CPU
/* Save all interesting registers */
ldr w4, [x2, #GICH_HCR]
ldr w5, [x2, #GICH_VMCR]
ldr w6, [x2, #GICH_MISR]
ldr w7, [x2, #GICH_EISR0]
ldr w8, [x2, #GICH_EISR1]
ldr w9, [x2, #GICH_ELRSR0]
ldr w10, [x2, #GICH_ELRSR1]
ldr w11, [x2, #GICH_APR]
str w4, [x3, #VGIC_CPU_HCR]
str w5, [x3, #VGIC_CPU_VMCR]
str w6, [x3, #VGIC_CPU_MISR]
str w7, [x3, #VGIC_CPU_EISR]
str w8, [x3, #(VGIC_CPU_EISR + 4)]
str w9, [x3, #VGIC_CPU_ELRSR]
str w10, [x3, #(VGIC_CPU_ELRSR + 4)]
str w11, [x3, #VGIC_CPU_APR]
/* Clear GICH_HCR */
str wzr, [x2, #GICH_HCR]
/* Save list registers */
add x2, x2, #GICH_LR0
ldr w4, [x3, #VGIC_CPU_NR_LR]
add x3, x3, #VGIC_CPU_LR
1: ldr w5, [x2], #4
str w5, [x3], #4
sub w4, w4, #1
cbnz w4, 1b
2:
.endm
/*
* Restore the VGIC CPU state from memory
* x0: Register pointing to VCPU struct
*/
.macro restore_vgic_state
/* Get VGIC VCTRL base into x2 */
ldr x2, [x0, #VCPU_KVM]
kern_hyp_va x2
ldr x2, [x2, #KVM_VGIC_VCTRL]
kern_hyp_va x2
cbz x2, 2f // disabled
/* Compute the address of struct vgic_cpu */
add x3, x0, #VCPU_VGIC_CPU
/* We only restore a minimal set of registers */
ldr w4, [x3, #VGIC_CPU_HCR]
ldr w5, [x3, #VGIC_CPU_VMCR]
ldr w6, [x3, #VGIC_CPU_APR]
str w4, [x2, #GICH_HCR]
str w5, [x2, #GICH_VMCR]
str w6, [x2, #GICH_APR]
/* Restore list registers */
add x2, x2, #GICH_LR0
ldr w4, [x3, #VGIC_CPU_NR_LR]
add x3, x3, #VGIC_CPU_LR
1: ldr w5, [x3], #4
str w5, [x2], #4
sub w4, w4, #1
cbnz w4, 1b
2:
.endm
__save_sysregs:
save_sysregs
ret
__restore_sysregs:
restore_sysregs
ret
__save_fpsimd:
save_fpsimd
ret
__restore_fpsimd:
restore_fpsimd
ret
/*
* u64 __kvm_vcpu_run(struct kvm_vcpu *vcpu);
*
* This is the world switch. The first half of the function
* deals with entering the guest, and anything from __kvm_vcpu_return
* to the end of the function deals with reentering the host.
* On the enter path, only x0 (vcpu pointer) must be preserved until
* the last moment. On the exit path, x0 (vcpu pointer) and x1 (exception
* code) must both be preserved until the epilogue.
* In both cases, x2 points to the CPU context we're saving/restoring from/to.
*/
ENTRY(__kvm_vcpu_run)
kern_hyp_va x0
msr tpidr_el2, x0 // Save the vcpu register
// Host context
ldr x2, [x0, #VCPU_HOST_CONTEXT]
kern_hyp_va x2
save_host_regs
bl __save_fpsimd
bl __save_sysregs
activate_traps
activate_vm
restore_vgic_state
// Guest context
add x2, x0, #VCPU_CONTEXT
bl __restore_sysregs
bl __restore_fpsimd
restore_guest_regs
// That's it, no more messing around.
eret
__kvm_vcpu_return:
// Assume x0 is the vcpu pointer, x1 the return code
// Guest's x0-x3 are on the stack
// Guest context
add x2, x0, #VCPU_CONTEXT
save_guest_regs
bl __save_fpsimd
bl __save_sysregs
save_vgic_state
deactivate_traps
deactivate_vm
// Host context
ldr x2, [x0, #VCPU_HOST_CONTEXT]
kern_hyp_va x2
bl __restore_sysregs
bl __restore_fpsimd
restore_host_regs
mov x0, x1
ret
END(__kvm_vcpu_run)
// void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
ENTRY(__kvm_tlb_flush_vmid_ipa)
kern_hyp_va x0
ldr x2, [x0, #KVM_VTTBR]
msr vttbr_el2, x2
isb
/*
* We could do so much better if we had the VA as well.
* Instead, we invalidate Stage-2 for this IPA, and the
* whole of Stage-1. Weep...
*/
tlbi ipas2e1is, x1
dsb sy
tlbi vmalle1is
dsb sy
isb
msr vttbr_el2, xzr
ret
ENDPROC(__kvm_tlb_flush_vmid_ipa)
ENTRY(__kvm_flush_vm_context)
tlbi alle1is
ic ialluis
dsb sy
ret
ENDPROC(__kvm_flush_vm_context)
__kvm_hyp_panic:
// Guess the context by looking at VTTBR:
// If zero, then we're already a host.
// Otherwise restore a minimal host context before panicing.
mrs x0, vttbr_el2
cbz x0, 1f
mrs x0, tpidr_el2
deactivate_traps
deactivate_vm
ldr x2, [x0, #VCPU_HOST_CONTEXT]
kern_hyp_va x2
bl __restore_sysregs
1: adr x0, __hyp_panic_str
adr x1, 2f
ldp x2, x3, [x1]
sub x0, x0, x2
add x0, x0, x3
mrs x1, spsr_el2
mrs x2, elr_el2
mrs x3, esr_el2
mrs x4, far_el2
mrs x5, hpfar_el2
mrs x6, par_el1
mrs x7, tpidr_el2
mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
PSR_MODE_EL1h)
msr spsr_el2, lr
ldr lr, =panic
msr elr_el2, lr
eret
.align 3
2: .quad HYP_PAGE_OFFSET
.quad PAGE_OFFSET
ENDPROC(__kvm_hyp_panic)
__hyp_panic_str:
.ascii "HYP panic:\nPS:%08x PC:%p ESR:%p\nFAR:%p HPFAR:%p PAR:%p\nVCPU:%p\n\0"
.align 2
ENTRY(kvm_call_hyp)
hvc #0
ret
ENDPROC(kvm_call_hyp)
.macro invalid_vector label, target
.align 2
\label:
b \target
ENDPROC(\label)
.endm
/* None of these should ever happen */
invalid_vector el2t_sync_invalid, __kvm_hyp_panic
invalid_vector el2t_irq_invalid, __kvm_hyp_panic
invalid_vector el2t_fiq_invalid, __kvm_hyp_panic
invalid_vector el2t_error_invalid, __kvm_hyp_panic
invalid_vector el2h_sync_invalid, __kvm_hyp_panic
invalid_vector el2h_irq_invalid, __kvm_hyp_panic
invalid_vector el2h_fiq_invalid, __kvm_hyp_panic
invalid_vector el2h_error_invalid, __kvm_hyp_panic
invalid_vector el1_sync_invalid, __kvm_hyp_panic
invalid_vector el1_irq_invalid, __kvm_hyp_panic
invalid_vector el1_fiq_invalid, __kvm_hyp_panic
invalid_vector el1_error_invalid, __kvm_hyp_panic
el1_sync: // Guest trapped into EL2
push x0, x1
push x2, x3
mrs x1, esr_el2
lsr x2, x1, #ESR_EL2_EC_SHIFT
cmp x2, #ESR_EL2_EC_HVC64
b.ne el1_trap
mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest
cbnz x3, el1_trap // called HVC
/* Here, we're pretty sure the host called HVC. */
pop x2, x3
pop x0, x1
push lr, xzr
/*
* Compute the function address in EL2, and shuffle the parameters.
*/
kern_hyp_va x0
mov lr, x0
mov x0, x1
mov x1, x2
mov x2, x3
blr lr
pop lr, xzr
eret
el1_trap:
/*
* x1: ESR
* x2: ESR_EC
*/
cmp x2, #ESR_EL2_EC_DABT
mov x0, #ESR_EL2_EC_IABT
ccmp x2, x0, #4, ne
b.ne 1f // Not an abort we care about
/* This is an abort. Check for permission fault */
and x2, x1, #ESR_EL2_FSC_TYPE
cmp x2, #FSC_PERM
b.ne 1f // Not a permission fault
/*
* Check for Stage-1 page table walk, which is guaranteed
* to give a valid HPFAR_EL2.
*/
tbnz x1, #7, 1f // S1PTW is set
/*
* Permission fault, HPFAR_EL2 is invalid.
* Resolve the IPA the hard way using the guest VA.
* Stage-1 translation already validated the memory access rights.
* As such, we can use the EL1 translation regime, and don't have
* to distinguish between EL0 and EL1 access.
*/
mrs x2, far_el2
at s1e1r, x2
isb
/* Read result */
mrs x3, par_el1
tbnz x3, #0, 3f // Bail out if we failed the translation
ubfx x3, x3, #12, #36 // Extract IPA
lsl x3, x3, #4 // and present it like HPFAR
b 2f
1: mrs x3, hpfar_el2
mrs x2, far_el2
2: mrs x0, tpidr_el2
str x1, [x0, #VCPU_ESR_EL2]
str x2, [x0, #VCPU_FAR_EL2]
str x3, [x0, #VCPU_HPFAR_EL2]
mov x1, #ARM_EXCEPTION_TRAP
b __kvm_vcpu_return
/*
* Translation failed. Just return to the guest and
* let it fault again. Another CPU is probably playing
* behind our back.
*/
3: pop x2, x3
pop x0, x1
eret
el1_irq:
push x0, x1
push x2, x3
mrs x0, tpidr_el2
mov x1, #ARM_EXCEPTION_IRQ
b __kvm_vcpu_return
.ltorg
.align 11
ENTRY(__kvm_hyp_vector)
ventry el2t_sync_invalid // Synchronous EL2t
ventry el2t_irq_invalid // IRQ EL2t
ventry el2t_fiq_invalid // FIQ EL2t
ventry el2t_error_invalid // Error EL2t
ventry el2h_sync_invalid // Synchronous EL2h
ventry el2h_irq_invalid // IRQ EL2h
ventry el2h_fiq_invalid // FIQ EL2h
ventry el2h_error_invalid // Error EL2h
ventry el1_sync // Synchronous 64-bit EL1
ventry el1_irq // IRQ 64-bit EL1
ventry el1_fiq_invalid // FIQ 64-bit EL1
ventry el1_error_invalid // Error 64-bit EL1
ventry el1_sync // Synchronous 32-bit EL1
ventry el1_irq // IRQ 32-bit EL1
ventry el1_fiq_invalid // FIQ 32-bit EL1
ventry el1_error_invalid // Error 32-bit EL1
ENDPROC(__kvm_hyp_vector)
__kvm_hyp_code_end:
.globl __kvm_hyp_code_end
.popsection
|