vm.fail
AArch64 EL2 hypervisor for QEMU virt that boots at EL2
1#include "hv/allocator.h"
2#include "hv/arch.h"
3#include "hv/config.h"
4#include "hv/log.h"
5#include "hv/mmio_policy.h"
6#include "hv/panic.h"
7#include "hv/stage2.h"
8#include "hv/string.h"
9#include "hv/sysreg.h"
10#include "hv/trap.h"
11#include "hv/uart.h"
12#include "hv/vcpu.h"
13
14#define S2_ENTRIES 512u
15#define S2_L1_SHIFT 30u
16#define S2_L2_SHIFT 21u
17#define S2_L1_SIZE (1ull << S2_L1_SHIFT)
18#define S2_L2_SIZE (1ull << S2_L2_SHIFT)
19#define S2_DESC_VALID HV_BIT(0)
20#define S2_DESC_TABLE HV_BIT(1)
21#define S2_DESC_AF HV_BIT(10)
22#define S2_DESC_SH_INNER (3ull << 8u)
23#define S2_DESC_MEMATTR_NORMAL (0xfull << 2u)
24#define S2_DESC_MEMATTR_DEVICE (0x0ull << 2u)
25#define S2_DESC_S2AP_R (1ull << 6u)
26#define S2_DESC_S2AP_W (1ull << 7u)
27#define S2_DESC_XN (3ull << 53u)
28
29static uint64_t g_l1[S2_ENTRIES] __attribute__((aligned(65536)));
30static uint64_t g_l2_tables[4][S2_ENTRIES] __attribute__((aligned(4096)));
31static uint32_t g_l2_used;
32
33static uint64_t desc_addr(uint64_t addr)
34{
35 return addr & 0x0000fffffffff000ull;
36}
37
38static bool table_arena_aligned(void)
39{
40 if (!hv_is_aligned_u64((uint64_t)(uintptr_t)g_l1, 65536u)) {
41 return false;
42 }
43 for (uint32_t i = 0; i < HV_ARRAY_SIZE(g_l2_tables); i++) {
44 if (!hv_is_aligned_u64((uint64_t)(uintptr_t)g_l2_tables[i], CONFIG_PAGE_SIZE)) {
45 return false;
46 }
47 }
48 return true;
49}
50
51void stage2_init(void)
52{
53 BUG_ON(!table_arena_aligned());
54 hv_memset(g_l1, 0, sizeof(g_l1));
55 hv_memset(g_l2_tables, 0, sizeof(g_l2_tables));
56 g_l2_used = 0;
57}
58
59static uint64_t *ensure_l2(uint64_t l1_index)
60{
61 if (l1_index >= S2_ENTRIES) {
62 return (uint64_t *)0;
63 }
64 if ((g_l1[l1_index] & S2_DESC_VALID) != 0u) {
65 return (uint64_t *)(uintptr_t)desc_addr(g_l1[l1_index]);
66 }
67 if (g_l2_used >= HV_ARRAY_SIZE(g_l2_tables)) {
68 return (uint64_t *)0;
69 }
70 uint64_t *table = g_l2_tables[g_l2_used++];
71 g_l1[l1_index] = ((uint64_t)(uintptr_t)table & 0x0000fffffffff000ull) |
72 S2_DESC_TABLE | S2_DESC_VALID;
73 return table;
74}
75
76static uint64_t block_desc(paddr_t pa, enum stage2_mem_type type, uint32_t perms)
77{
78 uint64_t desc = (pa & 0x0000ffffffe00000ull) | S2_DESC_VALID | S2_DESC_AF;
79 desc |= (type == S2_MEM_NORMAL) ? (S2_DESC_MEMATTR_NORMAL | S2_DESC_SH_INNER) :
80 S2_DESC_MEMATTR_DEVICE;
81 if ((perms & S2_PERM_R) != 0u) {
82 desc |= S2_DESC_S2AP_R;
83 }
84 if ((perms & S2_PERM_W) != 0u) {
85 desc |= S2_DESC_S2AP_W;
86 }
87 if ((perms & S2_PERM_X) == 0u) {
88 desc |= S2_DESC_XN;
89 }
90 return desc;
91}
92
93static bool descriptor_valid(uint64_t desc)
94{
95 if ((desc & S2_DESC_VALID) == 0u) {
96 return false;
97 }
98 if ((desc & S2_DESC_AF) == 0u) {
99 return false;
100 }
101 if (!hv_is_aligned_u64(desc_addr(desc), S2_L2_SIZE)) {
102 return false;
103 }
104 return true;
105}
106
107static uint64_t *lookup_l2(uint64_t l1_index)
108{
109 if (l1_index >= S2_ENTRIES || (g_l1[l1_index] & S2_DESC_VALID) == 0u) {
110 return (uint64_t *)0;
111 }
112 if ((g_l1[l1_index] & S2_DESC_TABLE) == 0u) {
113 return (uint64_t *)0;
114 }
115 return (uint64_t *)(uintptr_t)desc_addr(g_l1[l1_index]);
116}
117
118static bool l2_table_owned(const uint64_t *table)
119{
120 uintptr_t base = (uintptr_t)&g_l2_tables[0][0];
121 uintptr_t end = (uintptr_t)g_l2_tables + sizeof(g_l2_tables);
122 uintptr_t addr = (uintptr_t)table;
123 return addr >= base && addr < end && hv_is_aligned_u64((uint64_t)addr, CONFIG_PAGE_SIZE);
124}
125
126bool stage2_map_range(ipa_t ipa, paddr_t pa, uint64_t size, enum stage2_mem_type type, uint32_t perms)
127{
128 uint64_t ipa_end;
129 uint64_t pa_end;
130
131 if (size == 0u || perms == 0u) {
132 return false;
133 }
134 if (!hv_is_aligned_u64(ipa, S2_L2_SIZE) || !hv_is_aligned_u64(pa, S2_L2_SIZE) ||
135 !hv_is_aligned_u64(size, S2_L2_SIZE)) {
136 return false;
137 }
138 if (hv_add_overflow_u64(ipa, size, &ipa_end) ||
139 hv_add_overflow_u64(pa, size, &pa_end)) {
140 return false;
141 }
142 if (ipa_end <= ipa || pa_end <= pa) {
143 return false;
144 }
145 if (hv_range_overlaps(pa, size, (uint64_t)(uintptr_t)__image_start,
146 (uint64_t)((uintptr_t)__image_end - (uintptr_t)__image_start))) {
147 return false;
148 }
149
150 for (uint64_t off = 0; off < size; off += S2_L2_SIZE) {
151 uint64_t cur_ipa = ipa + off;
152 uint64_t l1_index = (cur_ipa >> S2_L1_SHIFT) & 0x1ffu;
153 uint64_t l2_index = (cur_ipa >> S2_L2_SHIFT) & 0x1ffu;
154 uint64_t *l2 = ensure_l2(l1_index);
155 if (l2 == (uint64_t *)0 || (l2[l2_index] & S2_DESC_VALID) != 0u) {
156 return false;
157 }
158 uint64_t desc = block_desc(pa + off, type, perms);
159 if (!descriptor_valid(desc)) {
160 return false;
161 }
162 l2[l2_index] = desc;
163 }
164 arch_tlbi_vmalle1is();
165 return true;
166}
167
168bool stage2_unmap_range(ipa_t ipa, uint64_t size)
169{
170 if (size == 0u || !hv_is_aligned_u64(ipa, S2_L2_SIZE) ||
171 !hv_is_aligned_u64(size, S2_L2_SIZE)) {
172 return false;
173 }
174
175 for (uint64_t off = 0; off < size; off += S2_L2_SIZE) {
176 uint64_t cur_ipa = ipa + off;
177 uint64_t l1_index = (cur_ipa >> S2_L1_SHIFT) & 0x1ffu;
178 uint64_t l2_index = (cur_ipa >> S2_L2_SHIFT) & 0x1ffu;
179 uint64_t *l2 = lookup_l2(l1_index);
180 if (l2 == (uint64_t *)0 || (l2[l2_index] & S2_DESC_VALID) == 0u) {
181 return false;
182 }
183 }
184
185 for (uint64_t off = 0; off < size; off += S2_L2_SIZE) {
186 uint64_t cur_ipa = ipa + off;
187 uint64_t l1_index = (cur_ipa >> S2_L1_SHIFT) & 0x1ffu;
188 uint64_t l2_index = (cur_ipa >> S2_L2_SHIFT) & 0x1ffu;
189 uint64_t *l2 = lookup_l2(l1_index);
190 l2[l2_index] = 0u;
191 }
192 arch_tlbi_vmalle1is();
193 return true;
194}
195
196bool stage2_translate(ipa_t ipa, struct stage2_translation *out)
197{
198 uint64_t l1_index = (ipa >> S2_L1_SHIFT) & 0x1ffu;
199 uint64_t l2_index = (ipa >> S2_L2_SHIFT) & 0x1ffu;
200 uint64_t *l2 = lookup_l2(l1_index);
201 uint64_t desc;
202
203 if (out != (struct stage2_translation *)0) {
204 hv_memset(out, 0, sizeof(*out));
205 }
206 if (l2 == (uint64_t *)0 || (l2[l2_index] & S2_DESC_VALID) == 0u) {
207 return false;
208 }
209
210 desc = l2[l2_index];
211 if (out != (struct stage2_translation *)0) {
212 out->mapped = true;
213 out->ipa_base = HV_ALIGN_DOWN(ipa, S2_L2_SIZE);
214 out->pa_base = desc_addr(desc);
215 out->size = S2_L2_SIZE;
216 out->desc = desc;
217 if ((desc & S2_DESC_S2AP_R) != 0u) {
218 out->perms |= S2_PERM_R;
219 }
220 if ((desc & S2_DESC_S2AP_W) != 0u) {
221 out->perms |= S2_PERM_W;
222 }
223 if ((desc & S2_DESC_XN) == 0u) {
224 out->perms |= S2_PERM_X;
225 }
226 out->type = (desc & S2_DESC_MEMATTR_NORMAL) == S2_DESC_MEMATTR_NORMAL ?
227 S2_MEM_NORMAL : S2_MEM_DEVICE;
228 }
229 return true;
230}
231
232ipa_t stage2_ipa_from_abort(uint64_t far_el2, uint64_t hpfar_el2)
233{
234 return ((hpfar_el2 & 0x0000000ffffffff0ull) << 8u) | (far_el2 & 0xfffull);
235}
236
237static bool dfsc_is_translation(uint32_t dfsc)
238{
239 return dfsc >= 0x04u && dfsc <= 0x07u;
240}
241
242enum stage2_fault_intent {
243 S2_FAULT_PAUSE = 0,
244 S2_FAULT_REPAIR,
245 S2_FAULT_MMIO,
246};
247
248static enum stage2_fault_intent classify_fault(ipa_t ipa, uint32_t dfsc)
249{
250 if (mmio_policy_lookup(ipa) != (const struct mmio_policy_region *)0) {
251 return S2_FAULT_MMIO;
252 }
253 if (dfsc_is_translation(dfsc) &&
254 ipa >= CONFIG_GUEST_IPA_BASE &&
255 ipa < CONFIG_GUEST_IPA_BASE + CONFIG_GUEST_RAM_SIZE) {
256 return S2_FAULT_REPAIR;
257 }
258 return S2_FAULT_PAUSE;
259}
260
261bool stage2_handle_abort(struct vcpu *vcpu, const struct trap_frame *frame, bool instruction)
262{
263 ipa_t ipa;
264 ipa_t block_ipa;
265 paddr_t block_pa;
266 uint32_t dfsc;
267 struct log_event ev;
268 enum stage2_fault_intent intent;
269
270 if (vcpu == (struct vcpu *)0 || frame == (const struct trap_frame *)0) {
271 return false;
272 }
273
274 ipa = stage2_ipa_from_abort(frame->far_el2, frame->hpfar_el2);
275 dfsc = (uint32_t)(frame->esr_el2 & 0x3fu);
276 hv_memset(&ev, 0, sizeof(ev));
277 ev.kind = LOG_ABORT;
278 ev.vcpu = vcpu->id;
279 ev.ec = esr_ec(frame->esr_el2);
280 ev.iss = frame->esr_el2 & 0x01ffffffu;
281 ev.elr = frame->elr_el2;
282 ev.far = ipa;
283 ev.hpfar = frame->hpfar_el2;
284 ev.pstate = frame->spsr_el2;
285 ev.reason = instruction ? 406u : 407u;
286
287 intent = classify_fault(ipa, dfsc);
288 if (intent == S2_FAULT_MMIO) {
289 const struct mmio_policy_region *mmio = mmio_policy_lookup(ipa);
290 ev.reason = 410u;
291 ev.name = mmio->name;
292 ev.action = (uint32_t)mmio->action;
293 ev.result = mmio->provenance;
294 if (mmio_policy_emulate(vcpu, ipa, frame->esr_el2)) {
295 ev.reason = 411u;
296 log_event_commit(&ev);
297 arch_advance_guest_pc(vcpu);
298 return true;
299 }
300 log_event_commit(&ev);
301 return false;
302 }
303
304 if (intent != S2_FAULT_REPAIR) {
305 log_event_commit(&ev);
306 return false;
307 }
308
309 block_ipa = HV_ALIGN_DOWN(ipa, S2_L2_SIZE);
310 block_pa = CONFIG_GUEST_RAM_BASE + (block_ipa - CONFIG_GUEST_IPA_BASE);
311 if (!stage2_map_range(block_ipa, block_pa, S2_L2_SIZE, S2_MEM_NORMAL,
312 S2_PERM_R | S2_PERM_W | S2_PERM_X)) {
313 log_event_commit(&ev);
314 return false;
315 }
316
317 ev.reason = instruction ? 408u : 409u;
318 ev.result = block_pa;
319 log_event_commit(&ev);
320 return true;
321}
322
323uint64_t stage2_vttbr(void)
324{
325 return (uint64_t)(uintptr_t)g_l1;
326}
327
328uint64_t stage2_vtcr(void)
329{
330 uint64_t vtcr = 0;
331 vtcr |= 24ull;
332 vtcr |= (1ull << 6u);
333 vtcr |= (1ull << 8u);
334 vtcr |= (1ull << 10u);
335 vtcr |= (3ull << 12u);
336 vtcr |= (2ull << 16u);
337 return vtcr;
338}
339
340void stage2_dump(void)
341{
342 uart_puts("stage2 root=");
343 uart_put_hex64(stage2_vttbr());
344 uart_puts(" vtcr=");
345 uart_put_hex64(stage2_vtcr());
346 uart_puts(" l2_tables=");
347 uart_put_dec64(g_l2_used);
348 uart_puts("\n");
349 for (uint32_t l1 = 0; l1 < S2_ENTRIES; l1++) {
350 if ((g_l1[l1] & S2_DESC_VALID) == 0u) {
351 continue;
352 }
353 uint64_t *l2 = (uint64_t *)(uintptr_t)desc_addr(g_l1[l1]);
354 for (uint32_t l2i = 0; l2i < S2_ENTRIES; l2i++) {
355 if ((l2[l2i] & S2_DESC_VALID) != 0u) {
356 uart_puts("mapping ipa=");
357 uart_put_hex64(((uint64_t)l1 << S2_L1_SHIFT) | ((uint64_t)l2i << S2_L2_SHIFT));
358 uart_puts(" desc=");
359 uart_put_hex64(l2[l2i]);
360 uart_puts("\n");
361 }
362 }
363 }
364}
365
366bool stage2_selftest(void)
367{
368 if (!table_arena_aligned()) {
369 return false;
370 }
371 for (uint32_t i = 0; i < S2_ENTRIES; i++) {
372 if ((g_l1[i] & S2_DESC_VALID) != 0u && (g_l1[i] & S2_DESC_TABLE) == 0u) {
373 return false;
374 }
375 if ((g_l1[i] & S2_DESC_VALID) != 0u) {
376 uint64_t *l2 = (uint64_t *)(uintptr_t)desc_addr(g_l1[i]);
377 if (!l2_table_owned(l2)) {
378 return false;
379 }
380 for (uint32_t j = 0; j < S2_ENTRIES; j++) {
381 if ((l2[j] & S2_DESC_VALID) != 0u && !descriptor_valid(l2[j])) {
382 return false;
383 }
384 }
385 }
386 }
387 return stage2_vttbr() == (uint64_t)(uintptr_t)g_l1;
388}