vm.fail
AArch64 EL2 hypervisor for QEMU virt that boots at EL2
1#include "hv/allocator.h"
2#include "hv/arch.h"
3#include "hv/config.h"
4#include "hv/log.h"
5#include "hv/mmio_policy.h"
6#include "hv/panic.h"
7#include "hv/stage2.h"
8#include "hv/string.h"
9#include "hv/sysreg.h"
10#include "hv/trap.h"
11#include "hv/uart.h"
12#include "hv/vcpu.h"
13
14#define S2_ENTRIES 512u
15#define S2_L1_SHIFT 30u
16#define S2_L2_SHIFT 21u
17#define S2_L1_SIZE (1ull << S2_L1_SHIFT)
18#define S2_L2_SIZE (1ull << S2_L2_SHIFT)
19#define S2_DESC_VALID HV_BIT(0)
20#define S2_DESC_TABLE HV_BIT(1)
21#define S2_DESC_AF HV_BIT(10)
22#define S2_DESC_SH_INNER (3ull << 8u)
23#define S2_DESC_MEMATTR_NORMAL (0xfull << 2u)
24#define S2_DESC_MEMATTR_DEVICE (0x0ull << 2u)
25#define S2_DESC_S2AP_R (1ull << 6u)
26#define S2_DESC_S2AP_W (1ull << 7u)
27#define S2_DESC_XN (3ull << 53u)
28
29static uint64_t g_l1[S2_ENTRIES] __attribute__((aligned(65536)));
30static uint64_t g_l2_tables[4][S2_ENTRIES] __attribute__((aligned(4096)));
31static uint32_t g_l2_used;
32
33static uint64_t desc_addr(uint64_t addr)
34{
35 return addr & 0x0000fffffffff000ull;
36}
37
38static bool table_arena_aligned(void)
39{
40 if (!hv_is_aligned_u64((uint64_t)(uintptr_t)g_l1, 65536u)) {
41 return false;
42 }
43 for (uint32_t i = 0; i < HV_ARRAY_SIZE(g_l2_tables); i++) {
44 if (!hv_is_aligned_u64((uint64_t)(uintptr_t)g_l2_tables[i], CONFIG_PAGE_SIZE)) {
45 return false;
46 }
47 }
48 return true;
49}
50
51void stage2_init(void)
52{
53 BUG_ON(!table_arena_aligned());
54 hv_memset(g_l1, 0, sizeof(g_l1));
55 hv_memset(g_l2_tables, 0, sizeof(g_l2_tables));
56 g_l2_used = 0;
57}
58
59static uint64_t *ensure_l2(uint64_t l1_index)
60{
61 if (l1_index >= S2_ENTRIES) {
62 return (uint64_t *)0;
63 }
64 if ((g_l1[l1_index] & S2_DESC_VALID) != 0u) {
65 return (uint64_t *)(uintptr_t)desc_addr(g_l1[l1_index]);
66 }
67 if (g_l2_used >= HV_ARRAY_SIZE(g_l2_tables)) {
68 return (uint64_t *)0;
69 }
70 uint64_t *table = g_l2_tables[g_l2_used++];
71 g_l1[l1_index] = ((uint64_t)(uintptr_t)table & 0x0000fffffffff000ull) |
72 S2_DESC_TABLE | S2_DESC_VALID;
73 return table;
74}
75
76static uint64_t block_desc(paddr_t pa, enum stage2_mem_type type, uint32_t perms)
77{
78 uint64_t desc = (pa & 0x0000ffffffe00000ull) | S2_DESC_VALID | S2_DESC_AF;
79 desc |= (type == S2_MEM_NORMAL) ? (S2_DESC_MEMATTR_NORMAL | S2_DESC_SH_INNER) :
80 S2_DESC_MEMATTR_DEVICE;
81 if ((perms & S2_PERM_R) != 0u) {
82 desc |= S2_DESC_S2AP_R;
83 }
84 if ((perms & S2_PERM_W) != 0u) {
85 desc |= S2_DESC_S2AP_W;
86 }
87 if ((perms & S2_PERM_X) == 0u) {
88 desc |= S2_DESC_XN;
89 }
90 return desc;
91}
92
93static bool descriptor_valid(uint64_t desc)
94{
95 if ((desc & S2_DESC_VALID) == 0u) {
96 return false;
97 }
98 if ((desc & S2_DESC_AF) == 0u) {
99 return false;
100 }
101 if (!hv_is_aligned_u64(desc_addr(desc), S2_L2_SIZE)) {
102 return false;
103 }
104 return true;
105}
106
107static uint64_t *lookup_l2(uint64_t l1_index)
108{
109 if (l1_index >= S2_ENTRIES || (g_l1[l1_index] & S2_DESC_VALID) == 0u) {
110 return (uint64_t *)0;
111 }
112 if ((g_l1[l1_index] & S2_DESC_TABLE) == 0u) {
113 return (uint64_t *)0;
114 }
115 return (uint64_t *)(uintptr_t)desc_addr(g_l1[l1_index]);
116}
117
118static bool l2_table_owned(const uint64_t *table)
119{
120 uintptr_t base = (uintptr_t)&g_l2_tables[0][0];
121 uintptr_t end = (uintptr_t)g_l2_tables + sizeof(g_l2_tables);
122 uintptr_t addr = (uintptr_t)table;
123 return addr >= base && addr < end && hv_is_aligned_u64((uint64_t)addr, CONFIG_PAGE_SIZE);
124}
125
126bool stage2_map_range(ipa_t ipa, paddr_t pa, uint64_t size, enum stage2_mem_type type, uint32_t perms)
127{
128 uint64_t ipa_end;
129 uint64_t pa_end;
130
131 if (size == 0u || perms == 0u) {
132 return false;
133 }
134 if (!hv_is_aligned_u64(ipa, S2_L2_SIZE) || !hv_is_aligned_u64(pa, S2_L2_SIZE) ||
135 !hv_is_aligned_u64(size, S2_L2_SIZE)) {
136 return false;
137 }
138 if (hv_add_overflow_u64(ipa, size, &ipa_end) ||
139 hv_add_overflow_u64(pa, size, &pa_end)) {
140 return false;
141 }
142 if (ipa_end <= ipa || pa_end <= pa) {
143 return false;
144 }
145 if (hv_range_overlaps(pa, size, (uint64_t)(uintptr_t)__image_start,
146 (uint64_t)((uintptr_t)__image_end - (uintptr_t)__image_start))) {
147 return false;
148 }
149
150 for (uint64_t off = 0; off < size; off += S2_L2_SIZE) {
151 uint64_t cur_ipa = ipa + off;
152 uint64_t l1_index = (cur_ipa >> S2_L1_SHIFT) & 0x1ffu;
153 uint64_t l2_index = (cur_ipa >> S2_L2_SHIFT) & 0x1ffu;
154 uint64_t *l2 = ensure_l2(l1_index);
155 if (l2 == (uint64_t *)0 || (l2[l2_index] & S2_DESC_VALID) != 0u) {
156 return false;
157 }
158 uint64_t desc = block_desc(pa + off, type, perms);
159 if (!descriptor_valid(desc)) {
160 return false;
161 }
162 l2[l2_index] = desc;
163 }
164 arch_tlbi_vmalle1is();
165 return true;
166}
167
168bool stage2_unmap_range(ipa_t ipa, uint64_t size)
169{
170 if (size == 0u || !hv_is_aligned_u64(ipa, S2_L2_SIZE) ||
171 !hv_is_aligned_u64(size, S2_L2_SIZE)) {
172 return false;
173 }
174
175 for (uint64_t off = 0; off < size; off += S2_L2_SIZE) {
176 uint64_t cur_ipa = ipa + off;
177 uint64_t l1_index = (cur_ipa >> S2_L1_SHIFT) & 0x1ffu;
178 uint64_t l2_index = (cur_ipa >> S2_L2_SHIFT) & 0x1ffu;
179 uint64_t *l2 = lookup_l2(l1_index);
180 if (l2 == (uint64_t *)0 || (l2[l2_index] & S2_DESC_VALID) == 0u) {
181 return false;
182 }
183 }
184
185 for (uint64_t off = 0; off < size; off += S2_L2_SIZE) {
186 uint64_t cur_ipa = ipa + off;
187 uint64_t l1_index = (cur_ipa >> S2_L1_SHIFT) & 0x1ffu;
188 uint64_t l2_index = (cur_ipa >> S2_L2_SHIFT) & 0x1ffu;
189 uint64_t *l2 = lookup_l2(l1_index);
190 l2[l2_index] = 0u;
191 }
192 arch_tlbi_vmalle1is();
193 return true;
194}
195
196bool stage2_translate(ipa_t ipa, struct stage2_translation *out)
197{
198 uint64_t l1_index = (ipa >> S2_L1_SHIFT) & 0x1ffu;
199 uint64_t l2_index = (ipa >> S2_L2_SHIFT) & 0x1ffu;
200 uint64_t *l2 = lookup_l2(l1_index);
201 uint64_t desc;
202
203 if (out != (struct stage2_translation *)0) {
204 hv_memset(out, 0, sizeof(*out));
205 }
206 if (l2 == (uint64_t *)0 || (l2[l2_index] & S2_DESC_VALID) == 0u) {
207 return false;
208 }
209
210 desc = l2[l2_index];
211 if (out != (struct stage2_translation *)0) {
212 out->mapped = true;
213 out->ipa_base = HV_ALIGN_DOWN(ipa, S2_L2_SIZE);
214 out->pa_base = desc_addr(desc);
215 out->size = S2_L2_SIZE;
216 out->desc = desc;
217 if ((desc & S2_DESC_S2AP_R) != 0u) {
218 out->perms |= S2_PERM_R;
219 }
220 if ((desc & S2_DESC_S2AP_W) != 0u) {
221 out->perms |= S2_PERM_W;
222 }
223 if ((desc & S2_DESC_XN) == 0u) {
224 out->perms |= S2_PERM_X;
225 }
226 out->type = (desc & S2_DESC_MEMATTR_NORMAL) == S2_DESC_MEMATTR_NORMAL ?
227 S2_MEM_NORMAL : S2_MEM_DEVICE;
228 }
229 return true;
230}
231
232ipa_t stage2_ipa_from_abort(uint64_t far_el2, uint64_t hpfar_el2)
233{
234 return ((hpfar_el2 & 0x0000000ffffffff0ull) << 8u) | (far_el2 & 0xfffull);
235}
236
237static bool dfsc_is_translation(uint32_t dfsc)
238{
239 return dfsc >= 0x04u && dfsc <= 0x07u;
240}
241
242bool stage2_handle_abort(struct vcpu *vcpu, const struct trap_frame *frame, bool instruction)
243{
244 ipa_t ipa;
245 ipa_t block_ipa;
246 paddr_t block_pa;
247 uint32_t dfsc;
248 struct log_event ev;
249
250 if (vcpu == (struct vcpu *)0 || frame == (const struct trap_frame *)0) {
251 return false;
252 }
253
254 ipa = stage2_ipa_from_abort(frame->far_el2, frame->hpfar_el2);
255 dfsc = (uint32_t)(frame->esr_el2 & 0x3fu);
256 hv_memset(&ev, 0, sizeof(ev));
257 ev.kind = LOG_ABORT;
258 ev.vcpu = vcpu->id;
259 ev.ec = esr_ec(frame->esr_el2);
260 ev.iss = frame->esr_el2 & 0x01ffffffu;
261 ev.elr = frame->elr_el2;
262 ev.far = ipa;
263 ev.hpfar = frame->hpfar_el2;
264 ev.pstate = frame->spsr_el2;
265 ev.reason = instruction ? 406u : 407u;
266
267 const struct mmio_policy_region *mmio = mmio_policy_lookup(ipa);
268 if (mmio != (const struct mmio_policy_region *)0) {
269 ev.reason = 410u;
270 ev.name = mmio->name;
271 ev.action = (uint32_t)mmio->action;
272 ev.result = mmio->provenance;
273 if (mmio_policy_emulate(vcpu, ipa, frame->esr_el2)) {
274 ev.reason = 411u;
275 log_event_commit(&ev);
276 arch_advance_guest_pc(vcpu);
277 return true;
278 }
279 log_event_commit(&ev);
280 return false;
281 }
282
283 if (!dfsc_is_translation(dfsc) ||
284 ipa < CONFIG_GUEST_IPA_BASE ||
285 ipa >= CONFIG_GUEST_IPA_BASE + CONFIG_GUEST_RAM_SIZE) {
286 log_event_commit(&ev);
287 return false;
288 }
289
290 block_ipa = HV_ALIGN_DOWN(ipa, S2_L2_SIZE);
291 block_pa = CONFIG_GUEST_RAM_BASE + (block_ipa - CONFIG_GUEST_IPA_BASE);
292 if (!stage2_map_range(block_ipa, block_pa, S2_L2_SIZE, S2_MEM_NORMAL,
293 S2_PERM_R | S2_PERM_W | S2_PERM_X)) {
294 log_event_commit(&ev);
295 return false;
296 }
297
298 ev.reason = instruction ? 408u : 409u;
299 ev.result = block_pa;
300 log_event_commit(&ev);
301 return true;
302}
303
304uint64_t stage2_vttbr(void)
305{
306 return (uint64_t)(uintptr_t)g_l1;
307}
308
309uint64_t stage2_vtcr(void)
310{
311 uint64_t vtcr = 0;
312 vtcr |= 24ull;
313 vtcr |= (1ull << 6u);
314 vtcr |= (1ull << 8u);
315 vtcr |= (1ull << 10u);
316 vtcr |= (3ull << 12u);
317 vtcr |= (2ull << 16u);
318 return vtcr;
319}
320
321void stage2_dump(void)
322{
323 uart_puts("stage2 root=");
324 uart_put_hex64(stage2_vttbr());
325 uart_puts(" vtcr=");
326 uart_put_hex64(stage2_vtcr());
327 uart_puts(" l2_tables=");
328 uart_put_dec64(g_l2_used);
329 uart_puts("\n");
330 for (uint32_t l1 = 0; l1 < S2_ENTRIES; l1++) {
331 if ((g_l1[l1] & S2_DESC_VALID) == 0u) {
332 continue;
333 }
334 uint64_t *l2 = (uint64_t *)(uintptr_t)desc_addr(g_l1[l1]);
335 for (uint32_t l2i = 0; l2i < S2_ENTRIES; l2i++) {
336 if ((l2[l2i] & S2_DESC_VALID) != 0u) {
337 uart_puts("mapping ipa=");
338 uart_put_hex64(((uint64_t)l1 << S2_L1_SHIFT) | ((uint64_t)l2i << S2_L2_SHIFT));
339 uart_puts(" desc=");
340 uart_put_hex64(l2[l2i]);
341 uart_puts("\n");
342 }
343 }
344 }
345}
346
347bool stage2_selftest(void)
348{
349 if (!table_arena_aligned()) {
350 return false;
351 }
352 for (uint32_t i = 0; i < S2_ENTRIES; i++) {
353 if ((g_l1[i] & S2_DESC_VALID) != 0u && (g_l1[i] & S2_DESC_TABLE) == 0u) {
354 return false;
355 }
356 if ((g_l1[i] & S2_DESC_VALID) != 0u) {
357 uint64_t *l2 = (uint64_t *)(uintptr_t)desc_addr(g_l1[i]);
358 if (!l2_table_owned(l2)) {
359 return false;
360 }
361 for (uint32_t j = 0; j < S2_ENTRIES; j++) {
362 if ((l2[j] & S2_DESC_VALID) != 0u && !descriptor_valid(l2[j])) {
363 return false;
364 }
365 }
366 }
367 }
368 return stage2_vttbr() == (uint64_t)(uintptr_t)g_l1;
369}