vm.fail
AArch64 EL2 hypervisor for QEMU virt that boots at EL2
1#include "hv/allocator.h"
2#include "hv/arch.h"
3#include "hv/config.h"
4#include "hv/log.h"
5#include "hv/mmio_policy.h"
6#include "hv/panic.h"
7#include "hv/stage2.h"
8#include "hv/string.h"
9#include "hv/sysreg.h"
10#include "hv/trap.h"
11#include "hv/uart.h"
12#include "hv/vcpu.h"
13
14#define S2_ENTRIES 512u
15#define S2_L1_SHIFT 30u
16#define S2_L2_SHIFT 21u
17#define S2_L1_SIZE (1ull << S2_L1_SHIFT)
18#define S2_L2_SIZE (1ull << S2_L2_SHIFT)
19#define S2_DESC_VALID HV_BIT(0)
20#define S2_DESC_TABLE HV_BIT(1)
21#define S2_DESC_AF HV_BIT(10)
22#define S2_DESC_SH_INNER (3ull << 8u)
23#define S2_DESC_MEMATTR_NORMAL (0xfull << 2u)
24#define S2_DESC_MEMATTR_DEVICE (0x0ull << 2u)
25#define S2_DESC_S2AP_R (1ull << 6u)
26#define S2_DESC_S2AP_W (1ull << 7u)
27#define S2_DESC_XN (3ull << 53u)
28
29static uint64_t g_l1[S2_ENTRIES] __attribute__((aligned(65536)));
30static uint64_t g_l2_tables[4][S2_ENTRIES] __attribute__((aligned(4096)));
31static uint32_t g_l2_used;
32
33static uint64_t desc_addr(uint64_t addr)
34{
35 return addr & 0x0000fffffffff000ull;
36}
37
38void stage2_init(void)
39{
40 hv_memset(g_l1, 0, sizeof(g_l1));
41 hv_memset(g_l2_tables, 0, sizeof(g_l2_tables));
42 g_l2_used = 0;
43}
44
45static uint64_t *ensure_l2(uint64_t l1_index)
46{
47 if (l1_index >= S2_ENTRIES) {
48 return (uint64_t *)0;
49 }
50 if ((g_l1[l1_index] & S2_DESC_VALID) != 0u) {
51 return (uint64_t *)(uintptr_t)desc_addr(g_l1[l1_index]);
52 }
53 if (g_l2_used >= HV_ARRAY_SIZE(g_l2_tables)) {
54 return (uint64_t *)0;
55 }
56 uint64_t *table = g_l2_tables[g_l2_used++];
57 g_l1[l1_index] = ((uint64_t)(uintptr_t)table & 0x0000fffffffff000ull) |
58 S2_DESC_TABLE | S2_DESC_VALID;
59 return table;
60}
61
62static uint64_t block_desc(paddr_t pa, enum stage2_mem_type type, uint32_t perms)
63{
64 uint64_t desc = (pa & 0x0000ffffffe00000ull) | S2_DESC_VALID | S2_DESC_AF;
65 desc |= (type == S2_MEM_NORMAL) ? (S2_DESC_MEMATTR_NORMAL | S2_DESC_SH_INNER) :
66 S2_DESC_MEMATTR_DEVICE;
67 if ((perms & S2_PERM_R) != 0u) {
68 desc |= S2_DESC_S2AP_R;
69 }
70 if ((perms & S2_PERM_W) != 0u) {
71 desc |= S2_DESC_S2AP_W;
72 }
73 if ((perms & S2_PERM_X) == 0u) {
74 desc |= S2_DESC_XN;
75 }
76 return desc;
77}
78
79static bool descriptor_valid(uint64_t desc)
80{
81 if ((desc & S2_DESC_VALID) == 0u) {
82 return false;
83 }
84 if ((desc & S2_DESC_AF) == 0u) {
85 return false;
86 }
87 if (!hv_is_aligned_u64(desc_addr(desc), S2_L2_SIZE)) {
88 return false;
89 }
90 return true;
91}
92
93static uint64_t *lookup_l2(uint64_t l1_index)
94{
95 if (l1_index >= S2_ENTRIES || (g_l1[l1_index] & S2_DESC_VALID) == 0u) {
96 return (uint64_t *)0;
97 }
98 if ((g_l1[l1_index] & S2_DESC_TABLE) == 0u) {
99 return (uint64_t *)0;
100 }
101 return (uint64_t *)(uintptr_t)desc_addr(g_l1[l1_index]);
102}
103
104static bool l2_table_owned(const uint64_t *table)
105{
106 uintptr_t base = (uintptr_t)&g_l2_tables[0][0];
107 uintptr_t end = (uintptr_t)g_l2_tables + sizeof(g_l2_tables);
108 uintptr_t addr = (uintptr_t)table;
109 return addr >= base && addr < end && hv_is_aligned_u64((uint64_t)addr, CONFIG_PAGE_SIZE);
110}
111
112bool stage2_map_range(ipa_t ipa, paddr_t pa, uint64_t size, enum stage2_mem_type type, uint32_t perms)
113{
114 uint64_t ipa_end;
115 uint64_t pa_end;
116
117 if (size == 0u || perms == 0u) {
118 return false;
119 }
120 if (!hv_is_aligned_u64(ipa, S2_L2_SIZE) || !hv_is_aligned_u64(pa, S2_L2_SIZE) ||
121 !hv_is_aligned_u64(size, S2_L2_SIZE)) {
122 return false;
123 }
124 if (hv_add_overflow_u64(ipa, size, &ipa_end) ||
125 hv_add_overflow_u64(pa, size, &pa_end)) {
126 return false;
127 }
128 if (ipa_end <= ipa || pa_end <= pa) {
129 return false;
130 }
131 if (hv_range_overlaps(pa, size, (uint64_t)(uintptr_t)__image_start,
132 (uint64_t)((uintptr_t)__image_end - (uintptr_t)__image_start))) {
133 return false;
134 }
135
136 for (uint64_t off = 0; off < size; off += S2_L2_SIZE) {
137 uint64_t cur_ipa = ipa + off;
138 uint64_t l1_index = (cur_ipa >> S2_L1_SHIFT) & 0x1ffu;
139 uint64_t l2_index = (cur_ipa >> S2_L2_SHIFT) & 0x1ffu;
140 uint64_t *l2 = ensure_l2(l1_index);
141 if (l2 == (uint64_t *)0 || (l2[l2_index] & S2_DESC_VALID) != 0u) {
142 return false;
143 }
144 uint64_t desc = block_desc(pa + off, type, perms);
145 if (!descriptor_valid(desc)) {
146 return false;
147 }
148 l2[l2_index] = desc;
149 }
150 arch_tlbi_vmalle1is();
151 return true;
152}
153
154bool stage2_unmap_range(ipa_t ipa, uint64_t size)
155{
156 if (size == 0u || !hv_is_aligned_u64(ipa, S2_L2_SIZE) ||
157 !hv_is_aligned_u64(size, S2_L2_SIZE)) {
158 return false;
159 }
160
161 for (uint64_t off = 0; off < size; off += S2_L2_SIZE) {
162 uint64_t cur_ipa = ipa + off;
163 uint64_t l1_index = (cur_ipa >> S2_L1_SHIFT) & 0x1ffu;
164 uint64_t l2_index = (cur_ipa >> S2_L2_SHIFT) & 0x1ffu;
165 uint64_t *l2 = lookup_l2(l1_index);
166 if (l2 == (uint64_t *)0 || (l2[l2_index] & S2_DESC_VALID) == 0u) {
167 return false;
168 }
169 }
170
171 for (uint64_t off = 0; off < size; off += S2_L2_SIZE) {
172 uint64_t cur_ipa = ipa + off;
173 uint64_t l1_index = (cur_ipa >> S2_L1_SHIFT) & 0x1ffu;
174 uint64_t l2_index = (cur_ipa >> S2_L2_SHIFT) & 0x1ffu;
175 uint64_t *l2 = lookup_l2(l1_index);
176 l2[l2_index] = 0u;
177 }
178 arch_tlbi_vmalle1is();
179 return true;
180}
181
182bool stage2_translate(ipa_t ipa, struct stage2_translation *out)
183{
184 uint64_t l1_index = (ipa >> S2_L1_SHIFT) & 0x1ffu;
185 uint64_t l2_index = (ipa >> S2_L2_SHIFT) & 0x1ffu;
186 uint64_t *l2 = lookup_l2(l1_index);
187 uint64_t desc;
188
189 if (out != (struct stage2_translation *)0) {
190 hv_memset(out, 0, sizeof(*out));
191 }
192 if (l2 == (uint64_t *)0 || (l2[l2_index] & S2_DESC_VALID) == 0u) {
193 return false;
194 }
195
196 desc = l2[l2_index];
197 if (out != (struct stage2_translation *)0) {
198 out->mapped = true;
199 out->ipa_base = HV_ALIGN_DOWN(ipa, S2_L2_SIZE);
200 out->pa_base = desc_addr(desc);
201 out->size = S2_L2_SIZE;
202 out->desc = desc;
203 if ((desc & S2_DESC_S2AP_R) != 0u) {
204 out->perms |= S2_PERM_R;
205 }
206 if ((desc & S2_DESC_S2AP_W) != 0u) {
207 out->perms |= S2_PERM_W;
208 }
209 if ((desc & S2_DESC_XN) == 0u) {
210 out->perms |= S2_PERM_X;
211 }
212 out->type = (desc & S2_DESC_MEMATTR_NORMAL) == S2_DESC_MEMATTR_NORMAL ?
213 S2_MEM_NORMAL : S2_MEM_DEVICE;
214 }
215 return true;
216}
217
218ipa_t stage2_ipa_from_abort(uint64_t far_el2, uint64_t hpfar_el2)
219{
220 return ((hpfar_el2 & 0x0000000ffffffff0ull) << 8u) | (far_el2 & 0xfffull);
221}
222
223static bool dfsc_is_translation(uint32_t dfsc)
224{
225 return dfsc >= 0x04u && dfsc <= 0x07u;
226}
227
228bool stage2_handle_abort(struct vcpu *vcpu, const struct trap_frame *frame, bool instruction)
229{
230 ipa_t ipa;
231 ipa_t block_ipa;
232 paddr_t block_pa;
233 uint32_t dfsc;
234 struct log_event ev;
235
236 if (vcpu == (struct vcpu *)0 || frame == (const struct trap_frame *)0) {
237 return false;
238 }
239
240 ipa = stage2_ipa_from_abort(frame->far_el2, frame->hpfar_el2);
241 dfsc = (uint32_t)(frame->esr_el2 & 0x3fu);
242 hv_memset(&ev, 0, sizeof(ev));
243 ev.kind = LOG_ABORT;
244 ev.vcpu = vcpu->id;
245 ev.ec = esr_ec(frame->esr_el2);
246 ev.iss = frame->esr_el2 & 0x01ffffffu;
247 ev.elr = frame->elr_el2;
248 ev.far = ipa;
249 ev.hpfar = frame->hpfar_el2;
250 ev.pstate = frame->spsr_el2;
251 ev.reason = instruction ? 406u : 407u;
252
253 const struct mmio_policy_region *mmio = mmio_policy_lookup(ipa);
254 if (mmio != (const struct mmio_policy_region *)0) {
255 ev.reason = 410u;
256 ev.name = mmio->name;
257 ev.action = (uint32_t)mmio->action;
258 if (mmio_policy_emulate(vcpu, ipa, frame->esr_el2)) {
259 ev.reason = 411u;
260 log_event_commit(&ev);
261 arch_advance_guest_pc(vcpu);
262 return true;
263 }
264 log_event_commit(&ev);
265 return false;
266 }
267
268 if (!dfsc_is_translation(dfsc) ||
269 ipa < CONFIG_GUEST_IPA_BASE ||
270 ipa >= CONFIG_GUEST_IPA_BASE + CONFIG_GUEST_RAM_SIZE) {
271 log_event_commit(&ev);
272 return false;
273 }
274
275 block_ipa = HV_ALIGN_DOWN(ipa, S2_L2_SIZE);
276 block_pa = CONFIG_GUEST_RAM_BASE + (block_ipa - CONFIG_GUEST_IPA_BASE);
277 if (!stage2_map_range(block_ipa, block_pa, S2_L2_SIZE, S2_MEM_NORMAL,
278 S2_PERM_R | S2_PERM_W | S2_PERM_X)) {
279 log_event_commit(&ev);
280 return false;
281 }
282
283 ev.reason = instruction ? 408u : 409u;
284 ev.result = block_pa;
285 log_event_commit(&ev);
286 return true;
287}
288
289uint64_t stage2_vttbr(void)
290{
291 return (uint64_t)(uintptr_t)g_l1;
292}
293
294uint64_t stage2_vtcr(void)
295{
296 uint64_t vtcr = 0;
297 vtcr |= 24ull;
298 vtcr |= (1ull << 6u);
299 vtcr |= (1ull << 8u);
300 vtcr |= (1ull << 10u);
301 vtcr |= (3ull << 12u);
302 vtcr |= (2ull << 16u);
303 return vtcr;
304}
305
306void stage2_dump(void)
307{
308 uart_puts("stage2 root=");
309 uart_put_hex64(stage2_vttbr());
310 uart_puts(" vtcr=");
311 uart_put_hex64(stage2_vtcr());
312 uart_puts(" l2_tables=");
313 uart_put_dec64(g_l2_used);
314 uart_puts("\n");
315 for (uint32_t l1 = 0; l1 < S2_ENTRIES; l1++) {
316 if ((g_l1[l1] & S2_DESC_VALID) == 0u) {
317 continue;
318 }
319 uint64_t *l2 = (uint64_t *)(uintptr_t)desc_addr(g_l1[l1]);
320 for (uint32_t l2i = 0; l2i < S2_ENTRIES; l2i++) {
321 if ((l2[l2i] & S2_DESC_VALID) != 0u) {
322 uart_puts("mapping ipa=");
323 uart_put_hex64(((uint64_t)l1 << S2_L1_SHIFT) | ((uint64_t)l2i << S2_L2_SHIFT));
324 uart_puts(" desc=");
325 uart_put_hex64(l2[l2i]);
326 uart_puts("\n");
327 }
328 }
329 }
330}
331
332bool stage2_selftest(void)
333{
334 if (!hv_is_aligned_u64((uint64_t)(uintptr_t)g_l1, 65536u)) {
335 return false;
336 }
337 for (uint32_t i = 0; i < S2_ENTRIES; i++) {
338 if ((g_l1[i] & S2_DESC_VALID) != 0u && (g_l1[i] & S2_DESC_TABLE) == 0u) {
339 return false;
340 }
341 if ((g_l1[i] & S2_DESC_VALID) != 0u) {
342 uint64_t *l2 = (uint64_t *)(uintptr_t)desc_addr(g_l1[i]);
343 if (!l2_table_owned(l2)) {
344 return false;
345 }
346 for (uint32_t j = 0; j < S2_ENTRIES; j++) {
347 if ((l2[j] & S2_DESC_VALID) != 0u && !descriptor_valid(l2[j])) {
348 return false;
349 }
350 }
351 }
352 }
353 return stage2_vttbr() == (uint64_t)(uintptr_t)g_l1;
354}