1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
|
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2012 Regents of the University of California
*/
#include <asm/asm-offsets.h>
#include <asm/asm.h>
#include <linux/init.h>
#include <linux/linkage.h>
#include <asm/thread_info.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/csr.h>
#include <asm/hwcap.h>
#include <asm/image.h>
#include "efi-header.S"
#ifdef CONFIG_XIP_KERNEL
.macro XIP_FIXUP_OFFSET reg
REG_L t0, _xip_fixup
add \reg, \reg, t0
.endm
_xip_fixup: .dword CONFIG_PHYS_RAM_BASE - CONFIG_XIP_PHYS_ADDR - XIP_OFFSET
#else
.macro XIP_FIXUP_OFFSET reg
.endm
#endif /* CONFIG_XIP_KERNEL */
__HEAD
ENTRY(_start)
/*
* Image header expected by Linux boot-loaders. The image header data
* structure is described in asm/image.h.
* Do not modify it without modifying the structure and all bootloaders
* that expects this header format!!
*/
#ifdef CONFIG_EFI
/*
* This instruction decodes to "MZ" ASCII required by UEFI.
*/
c.li s4,-13
j _start_kernel
#else
/* jump to start kernel */
j _start_kernel
/* reserved */
.word 0
#endif
.balign 8
#ifdef CONFIG_RISCV_M_MODE
/* Image load offset (0MB) from start of RAM for M-mode */
.dword 0
#else
#if __riscv_xlen == 64
/* Image load offset(2MB) from start of RAM */
.dword 0x200000
#else
/* Image load offset(4MB) from start of RAM */
.dword 0x400000
#endif
#endif
/* Effective size of kernel image */
.dword _end - _start
.dword __HEAD_FLAGS
.word RISCV_HEADER_VERSION
.word 0
.dword 0
.ascii RISCV_IMAGE_MAGIC
.balign 4
.ascii RISCV_IMAGE_MAGIC2
#ifdef CONFIG_EFI
.word pe_head_start - _start
pe_head_start:
__EFI_PE_HEADER
#else
.word 0
#endif
.align 2
#ifdef CONFIG_MMU
relocate:
/* Relocate return address */
la a1, kernel_map
XIP_FIXUP_OFFSET a1
REG_L a1, KERNEL_MAP_VIRT_ADDR(a1)
la a2, _start
sub a1, a1, a2
add ra, ra, a1
/* Point stvec to virtual address of intruction after satp write */
la a2, 1f
add a2, a2, a1
csrw CSR_TVEC, a2
/* Compute satp for kernel page tables, but don't load it yet */
srl a2, a0, PAGE_SHIFT
li a1, SATP_MODE
or a2, a2, a1
/*
* Load trampoline page directory, which will cause us to trap to
* stvec if VA != PA, or simply fall through if VA == PA. We need a
* full fence here because setup_vm() just wrote these PTEs and we need
* to ensure the new translations are in use.
*/
la a0, trampoline_pg_dir
XIP_FIXUP_OFFSET a0
srl a0, a0, PAGE_SHIFT
or a0, a0, a1
sfence.vma
csrw CSR_SATP, a0
.align 2
1:
/* Set trap vector to spin forever to help debug */
la a0, .Lsecondary_park
csrw CSR_TVEC, a0
/* Reload the global pointer */
.option push
.option norelax
la gp, __global_pointer$
.option pop
/*
* Switch to kernel page tables. A full fence is necessary in order to
* avoid using the trampoline translations, which are only correct for
* the first superpage. Fetching the fence is guarnteed to work
* because that first superpage is translated the same way.
*/
csrw CSR_SATP, a2
sfence.vma
ret
#endif /* CONFIG_MMU */
#ifdef CONFIG_SMP
.global secondary_start_sbi
secondary_start_sbi:
/* Mask all interrupts */
csrw CSR_IE, zero
csrw CSR_IP, zero
/* Load the global pointer */
.option push
.option norelax
la gp, __global_pointer$
.option pop
/*
* Disable FPU to detect illegal usage of
* floating point in kernel space
*/
li t0, SR_FS
csrc CSR_STATUS, t0
/* Set trap vector to spin forever to help debug */
la a3, .Lsecondary_park
csrw CSR_TVEC, a3
slli a3, a0, LGREG
la a4, __cpu_up_stack_pointer
XIP_FIXUP_OFFSET a4
la a5, __cpu_up_task_pointer
XIP_FIXUP_OFFSET a5
add a4, a3, a4
add a5, a3, a5
REG_L sp, (a4)
REG_L tp, (a5)
.global secondary_start_common
secondary_start_common:
#ifdef CONFIG_MMU
/* Enable virtual memory and relocate to virtual address */
la a0, swapper_pg_dir
XIP_FIXUP_OFFSET a0
call relocate
#endif
call setup_trap_vector
tail smp_callin
#endif /* CONFIG_SMP */
.align 2
setup_trap_vector:
/* Set trap vector to exception handler */
la a0, handle_exception
csrw CSR_TVEC, a0
/*
* Set sup0 scratch register to 0, indicating to exception vector that
* we are presently executing in kernel.
*/
csrw CSR_SCRATCH, zero
ret
.Lsecondary_park:
/* We lack SMP support or have too many harts, so park this hart */
wfi
j .Lsecondary_park
END(_start)
ENTRY(_start_kernel)
/* Mask all interrupts */
csrw CSR_IE, zero
csrw CSR_IP, zero
#ifdef CONFIG_RISCV_M_MODE
/* flush the instruction cache */
fence.i
/* Reset all registers except ra, a0, a1 */
call reset_regs
/*
* Setup a PMP to permit access to all of memory. Some machines may
* not implement PMPs, so we set up a quick trap handler to just skip
* touching the PMPs on any trap.
*/
la a0, pmp_done
csrw CSR_TVEC, a0
li a0, -1
csrw CSR_PMPADDR0, a0
li a0, (PMP_A_NAPOT | PMP_R | PMP_W | PMP_X)
csrw CSR_PMPCFG0, a0
.align 2
pmp_done:
/*
* The hartid in a0 is expected later on, and we have no firmware
* to hand it to us.
*/
csrr a0, CSR_MHARTID
#endif /* CONFIG_RISCV_M_MODE */
/* Load the global pointer */
.option push
.option norelax
la gp, __global_pointer$
.option pop
/*
* Disable FPU to detect illegal usage of
* floating point in kernel space
*/
li t0, SR_FS
csrc CSR_STATUS, t0
#ifdef CONFIG_SMP
li t0, CONFIG_NR_CPUS
blt a0, t0, .Lgood_cores
tail .Lsecondary_park
.Lgood_cores:
#endif
#ifndef CONFIG_XIP_KERNEL
/* Pick one hart to run the main boot sequence */
la a3, hart_lottery
li a2, 1
amoadd.w a3, a2, (a3)
bnez a3, .Lsecondary_start
#else
/* hart_lottery in flash contains a magic number */
la a3, hart_lottery
mv a2, a3
XIP_FIXUP_OFFSET a2
lw t1, (a3)
amoswap.w t0, t1, (a2)
/* first time here if hart_lottery in RAM is not set */
beq t0, t1, .Lsecondary_start
la sp, _end + THREAD_SIZE
XIP_FIXUP_OFFSET sp
mv s0, a0
call __copy_data
/* Restore a0 copy */
mv a0, s0
#endif
#ifndef CONFIG_XIP_KERNEL
/* Clear BSS for flat non-ELF images */
la a3, __bss_start
la a4, __bss_stop
ble a4, a3, clear_bss_done
clear_bss:
REG_S zero, (a3)
add a3, a3, RISCV_SZPTR
blt a3, a4, clear_bss
clear_bss_done:
#endif
/* Save hart ID and DTB physical address */
mv s0, a0
mv s1, a1
la a2, boot_cpu_hartid
XIP_FIXUP_OFFSET a2
REG_S a0, (a2)
/* Initialize page tables and relocate to virtual addresses */
la sp, init_thread_union + THREAD_SIZE
XIP_FIXUP_OFFSET sp
#ifdef CONFIG_BUILTIN_DTB
la a0, __dtb_start
#else
mv a0, s1
#endif /* CONFIG_BUILTIN_DTB */
call setup_vm
#ifdef CONFIG_MMU
la a0, early_pg_dir
XIP_FIXUP_OFFSET a0
call relocate
#endif /* CONFIG_MMU */
call setup_trap_vector
/* Restore C environment */
la tp, init_task
sw zero, TASK_TI_CPU(tp)
la sp, init_thread_union + THREAD_SIZE
#ifdef CONFIG_KASAN
call kasan_early_init
#endif
/* Start the kernel */
call soc_early_init
tail start_kernel
.Lsecondary_start:
#ifdef CONFIG_SMP
/* Set trap vector to spin forever to help debug */
la a3, .Lsecondary_park
csrw CSR_TVEC, a3
slli a3, a0, LGREG
la a1, __cpu_up_stack_pointer
XIP_FIXUP_OFFSET a1
la a2, __cpu_up_task_pointer
XIP_FIXUP_OFFSET a2
add a1, a3, a1
add a2, a3, a2
/*
* This hart didn't win the lottery, so we wait for the winning hart to
* get far enough along the boot process that it should continue.
*/
.Lwait_for_cpu_up:
/* FIXME: We should WFI to save some energy here. */
REG_L sp, (a1)
REG_L tp, (a2)
beqz sp, .Lwait_for_cpu_up
beqz tp, .Lwait_for_cpu_up
fence
tail secondary_start_common
#endif
END(_start_kernel)
#ifdef CONFIG_RISCV_M_MODE
ENTRY(reset_regs)
li sp, 0
li gp, 0
li tp, 0
li t0, 0
li t1, 0
li t2, 0
li s0, 0
li s1, 0
li a2, 0
li a3, 0
li a4, 0
li a5, 0
li a6, 0
li a7, 0
li s2, 0
li s3, 0
li s4, 0
li s5, 0
li s6, 0
li s7, 0
li s8, 0
li s9, 0
li s10, 0
li s11, 0
li t3, 0
li t4, 0
li t5, 0
li t6, 0
csrw CSR_SCRATCH, 0
#ifdef CONFIG_FPU
csrr t0, CSR_MISA
andi t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D)
beqz t0, .Lreset_regs_done
li t1, SR_FS
csrs CSR_STATUS, t1
fmv.s.x f0, zero
fmv.s.x f1, zero
fmv.s.x f2, zero
fmv.s.x f3, zero
fmv.s.x f4, zero
fmv.s.x f5, zero
fmv.s.x f6, zero
fmv.s.x f7, zero
fmv.s.x f8, zero
fmv.s.x f9, zero
fmv.s.x f10, zero
fmv.s.x f11, zero
fmv.s.x f12, zero
fmv.s.x f13, zero
fmv.s.x f14, zero
fmv.s.x f15, zero
fmv.s.x f16, zero
fmv.s.x f17, zero
fmv.s.x f18, zero
fmv.s.x f19, zero
fmv.s.x f20, zero
fmv.s.x f21, zero
fmv.s.x f22, zero
fmv.s.x f23, zero
fmv.s.x f24, zero
fmv.s.x f25, zero
fmv.s.x f26, zero
fmv.s.x f27, zero
fmv.s.x f28, zero
fmv.s.x f29, zero
fmv.s.x f30, zero
fmv.s.x f31, zero
csrw fcsr, 0
/* note that the caller must clear SR_FS */
#endif /* CONFIG_FPU */
.Lreset_regs_done:
ret
END(reset_regs)
#endif /* CONFIG_RISCV_M_MODE */
__PAGE_ALIGNED_BSS
/* Empty zero page */
.balign PAGE_SIZE
|
