vm.fail
AArch64 EL2 hypervisor for QEMU virt that boots at EL2
1#include "hv/allocator.h"
2#include "hv/arch.h"
3#include "hv/config.h"
4#include "hv/log.h"
5#include "hv/mmio_policy.h"
6#include "hv/panic.h"
7#include "hv/stage2.h"
8#include "hv/string.h"
9#include "hv/sysreg.h"
10#include "hv/trap.h"
11#include "hv/uart.h"
12#include "hv/vcpu.h"
13
14#define S2_ENTRIES 512u
15#define S2_L1_SHIFT 30u
16#define S2_L2_SHIFT 21u
17#define S2_L1_SIZE (1ull << S2_L1_SHIFT)
18#define S2_L2_SIZE (1ull << S2_L2_SHIFT)
19#define S2_DESC_VALID HV_BIT(0)
20#define S2_DESC_TABLE HV_BIT(1)
21#define S2_DESC_AF HV_BIT(10)
22#define S2_DESC_SH_INNER (3ull << 8u)
23#define S2_DESC_MEMATTR_NORMAL (0xfull << 2u)
24#define S2_DESC_MEMATTR_DEVICE (0x0ull << 2u)
25#define S2_DESC_S2AP_R (1ull << 6u)
26#define S2_DESC_S2AP_W (1ull << 7u)
27#define S2_DESC_XN (3ull << 53u)
28
29static uint64_t g_l1[S2_ENTRIES] __attribute__((aligned(65536)));
30static uint64_t g_l2_tables[4][S2_ENTRIES] __attribute__((aligned(4096)));
31static uint32_t g_l2_used;
32
33static uint64_t desc_addr(uint64_t addr)
34{
35 return addr & 0x0000fffffffff000ull;
36}
37
38void stage2_init(void)
39{
40 hv_memset(g_l1, 0, sizeof(g_l1));
41 hv_memset(g_l2_tables, 0, sizeof(g_l2_tables));
42 g_l2_used = 0;
43}
44
45static uint64_t *ensure_l2(uint64_t l1_index)
46{
47 if (l1_index >= S2_ENTRIES) {
48 return (uint64_t *)0;
49 }
50 if ((g_l1[l1_index] & S2_DESC_VALID) != 0u) {
51 return (uint64_t *)(uintptr_t)desc_addr(g_l1[l1_index]);
52 }
53 if (g_l2_used >= HV_ARRAY_SIZE(g_l2_tables)) {
54 return (uint64_t *)0;
55 }
56 uint64_t *table = g_l2_tables[g_l2_used++];
57 g_l1[l1_index] = ((uint64_t)(uintptr_t)table & 0x0000fffffffff000ull) |
58 S2_DESC_TABLE | S2_DESC_VALID;
59 return table;
60}
61
62static uint64_t block_desc(paddr_t pa, enum stage2_mem_type type, uint32_t perms)
63{
64 uint64_t desc = (pa & 0x0000ffffffe00000ull) | S2_DESC_VALID | S2_DESC_AF;
65 desc |= (type == S2_MEM_NORMAL) ? (S2_DESC_MEMATTR_NORMAL | S2_DESC_SH_INNER) :
66 S2_DESC_MEMATTR_DEVICE;
67 if ((perms & S2_PERM_R) != 0u) {
68 desc |= S2_DESC_S2AP_R;
69 }
70 if ((perms & S2_PERM_W) != 0u) {
71 desc |= S2_DESC_S2AP_W;
72 }
73 if ((perms & S2_PERM_X) == 0u) {
74 desc |= S2_DESC_XN;
75 }
76 return desc;
77}
78
79static bool descriptor_valid(uint64_t desc)
80{
81 if ((desc & S2_DESC_VALID) == 0u) {
82 return false;
83 }
84 if ((desc & S2_DESC_AF) == 0u) {
85 return false;
86 }
87 if (!hv_is_aligned_u64(desc_addr(desc), S2_L2_SIZE)) {
88 return false;
89 }
90 return true;
91}
92
93static uint64_t *lookup_l2(uint64_t l1_index)
94{
95 if (l1_index >= S2_ENTRIES || (g_l1[l1_index] & S2_DESC_VALID) == 0u) {
96 return (uint64_t *)0;
97 }
98 if ((g_l1[l1_index] & S2_DESC_TABLE) == 0u) {
99 return (uint64_t *)0;
100 }
101 return (uint64_t *)(uintptr_t)desc_addr(g_l1[l1_index]);
102}
103
104bool stage2_map_range(ipa_t ipa, paddr_t pa, uint64_t size, enum stage2_mem_type type, uint32_t perms)
105{
106 uint64_t ipa_end;
107 uint64_t pa_end;
108
109 if (size == 0u || perms == 0u) {
110 return false;
111 }
112 if (!hv_is_aligned_u64(ipa, S2_L2_SIZE) || !hv_is_aligned_u64(pa, S2_L2_SIZE) ||
113 !hv_is_aligned_u64(size, S2_L2_SIZE)) {
114 return false;
115 }
116 if (hv_add_overflow_u64(ipa, size, &ipa_end) ||
117 hv_add_overflow_u64(pa, size, &pa_end)) {
118 return false;
119 }
120 if (ipa_end <= ipa || pa_end <= pa) {
121 return false;
122 }
123 if (hv_range_overlaps(pa, size, (uint64_t)(uintptr_t)__image_start,
124 (uint64_t)((uintptr_t)__image_end - (uintptr_t)__image_start))) {
125 return false;
126 }
127
128 for (uint64_t off = 0; off < size; off += S2_L2_SIZE) {
129 uint64_t cur_ipa = ipa + off;
130 uint64_t l1_index = (cur_ipa >> S2_L1_SHIFT) & 0x1ffu;
131 uint64_t l2_index = (cur_ipa >> S2_L2_SHIFT) & 0x1ffu;
132 uint64_t *l2 = ensure_l2(l1_index);
133 if (l2 == (uint64_t *)0 || (l2[l2_index] & S2_DESC_VALID) != 0u) {
134 return false;
135 }
136 uint64_t desc = block_desc(pa + off, type, perms);
137 if (!descriptor_valid(desc)) {
138 return false;
139 }
140 l2[l2_index] = desc;
141 }
142 arch_tlbi_vmalle1is();
143 return true;
144}
145
146bool stage2_unmap_range(ipa_t ipa, uint64_t size)
147{
148 if (size == 0u || !hv_is_aligned_u64(ipa, S2_L2_SIZE) ||
149 !hv_is_aligned_u64(size, S2_L2_SIZE)) {
150 return false;
151 }
152
153 for (uint64_t off = 0; off < size; off += S2_L2_SIZE) {
154 uint64_t cur_ipa = ipa + off;
155 uint64_t l1_index = (cur_ipa >> S2_L1_SHIFT) & 0x1ffu;
156 uint64_t l2_index = (cur_ipa >> S2_L2_SHIFT) & 0x1ffu;
157 uint64_t *l2 = lookup_l2(l1_index);
158 if (l2 == (uint64_t *)0 || (l2[l2_index] & S2_DESC_VALID) == 0u) {
159 return false;
160 }
161 }
162
163 for (uint64_t off = 0; off < size; off += S2_L2_SIZE) {
164 uint64_t cur_ipa = ipa + off;
165 uint64_t l1_index = (cur_ipa >> S2_L1_SHIFT) & 0x1ffu;
166 uint64_t l2_index = (cur_ipa >> S2_L2_SHIFT) & 0x1ffu;
167 uint64_t *l2 = lookup_l2(l1_index);
168 l2[l2_index] = 0u;
169 }
170 arch_tlbi_vmalle1is();
171 return true;
172}
173
174bool stage2_translate(ipa_t ipa, struct stage2_translation *out)
175{
176 uint64_t l1_index = (ipa >> S2_L1_SHIFT) & 0x1ffu;
177 uint64_t l2_index = (ipa >> S2_L2_SHIFT) & 0x1ffu;
178 uint64_t *l2 = lookup_l2(l1_index);
179 uint64_t desc;
180
181 if (out != (struct stage2_translation *)0) {
182 hv_memset(out, 0, sizeof(*out));
183 }
184 if (l2 == (uint64_t *)0 || (l2[l2_index] & S2_DESC_VALID) == 0u) {
185 return false;
186 }
187
188 desc = l2[l2_index];
189 if (out != (struct stage2_translation *)0) {
190 out->mapped = true;
191 out->ipa_base = HV_ALIGN_DOWN(ipa, S2_L2_SIZE);
192 out->pa_base = desc_addr(desc);
193 out->size = S2_L2_SIZE;
194 out->desc = desc;
195 if ((desc & S2_DESC_S2AP_R) != 0u) {
196 out->perms |= S2_PERM_R;
197 }
198 if ((desc & S2_DESC_S2AP_W) != 0u) {
199 out->perms |= S2_PERM_W;
200 }
201 if ((desc & S2_DESC_XN) == 0u) {
202 out->perms |= S2_PERM_X;
203 }
204 out->type = (desc & S2_DESC_MEMATTR_NORMAL) == S2_DESC_MEMATTR_NORMAL ?
205 S2_MEM_NORMAL : S2_MEM_DEVICE;
206 }
207 return true;
208}
209
210ipa_t stage2_ipa_from_abort(uint64_t far_el2, uint64_t hpfar_el2)
211{
212 return ((hpfar_el2 & 0x0000000ffffffff0ull) << 8u) | (far_el2 & 0xfffull);
213}
214
215static bool dfsc_is_translation(uint32_t dfsc)
216{
217 return dfsc >= 0x04u && dfsc <= 0x07u;
218}
219
220bool stage2_handle_abort(struct vcpu *vcpu, const struct trap_frame *frame, bool instruction)
221{
222 ipa_t ipa;
223 ipa_t block_ipa;
224 paddr_t block_pa;
225 uint32_t dfsc;
226 struct log_event ev;
227
228 if (vcpu == (struct vcpu *)0 || frame == (const struct trap_frame *)0) {
229 return false;
230 }
231
232 ipa = stage2_ipa_from_abort(frame->far_el2, frame->hpfar_el2);
233 dfsc = (uint32_t)(frame->esr_el2 & 0x3fu);
234 hv_memset(&ev, 0, sizeof(ev));
235 ev.kind = LOG_ABORT;
236 ev.vcpu = vcpu->id;
237 ev.ec = esr_ec(frame->esr_el2);
238 ev.iss = frame->esr_el2 & 0x01ffffffu;
239 ev.elr = frame->elr_el2;
240 ev.far = ipa;
241 ev.hpfar = frame->hpfar_el2;
242 ev.pstate = frame->spsr_el2;
243 ev.reason = instruction ? 406u : 407u;
244
245 const struct mmio_policy_region *mmio = mmio_policy_lookup(ipa);
246 if (mmio != (const struct mmio_policy_region *)0) {
247 ev.reason = 410u;
248 ev.name = mmio->name;
249 ev.action = (uint32_t)mmio->action;
250 if (mmio_policy_emulate(vcpu, ipa, frame->esr_el2)) {
251 ev.reason = 411u;
252 log_event_commit(&ev);
253 arch_advance_guest_pc(vcpu);
254 return true;
255 }
256 log_event_commit(&ev);
257 return false;
258 }
259
260 if (!dfsc_is_translation(dfsc) ||
261 ipa < CONFIG_GUEST_IPA_BASE ||
262 ipa >= CONFIG_GUEST_IPA_BASE + CONFIG_GUEST_RAM_SIZE) {
263 log_event_commit(&ev);
264 return false;
265 }
266
267 block_ipa = HV_ALIGN_DOWN(ipa, S2_L2_SIZE);
268 block_pa = CONFIG_GUEST_RAM_BASE + (block_ipa - CONFIG_GUEST_IPA_BASE);
269 if (!stage2_map_range(block_ipa, block_pa, S2_L2_SIZE, S2_MEM_NORMAL,
270 S2_PERM_R | S2_PERM_W | S2_PERM_X)) {
271 log_event_commit(&ev);
272 return false;
273 }
274
275 ev.reason = instruction ? 408u : 409u;
276 ev.result = block_pa;
277 log_event_commit(&ev);
278 return true;
279}
280
281uint64_t stage2_vttbr(void)
282{
283 return (uint64_t)(uintptr_t)g_l1;
284}
285
286uint64_t stage2_vtcr(void)
287{
288 uint64_t vtcr = 0;
289 vtcr |= 24ull;
290 vtcr |= (1ull << 6u);
291 vtcr |= (1ull << 8u);
292 vtcr |= (1ull << 10u);
293 vtcr |= (3ull << 12u);
294 vtcr |= (2ull << 16u);
295 return vtcr;
296}
297
298void stage2_dump(void)
299{
300 uart_puts("stage2 root=");
301 uart_put_hex64(stage2_vttbr());
302 uart_puts(" vtcr=");
303 uart_put_hex64(stage2_vtcr());
304 uart_puts(" l2_tables=");
305 uart_put_dec64(g_l2_used);
306 uart_puts("\n");
307 for (uint32_t l1 = 0; l1 < S2_ENTRIES; l1++) {
308 if ((g_l1[l1] & S2_DESC_VALID) == 0u) {
309 continue;
310 }
311 uint64_t *l2 = (uint64_t *)(uintptr_t)desc_addr(g_l1[l1]);
312 for (uint32_t l2i = 0; l2i < S2_ENTRIES; l2i++) {
313 if ((l2[l2i] & S2_DESC_VALID) != 0u) {
314 uart_puts("mapping ipa=");
315 uart_put_hex64(((uint64_t)l1 << S2_L1_SHIFT) | ((uint64_t)l2i << S2_L2_SHIFT));
316 uart_puts(" desc=");
317 uart_put_hex64(l2[l2i]);
318 uart_puts("\n");
319 }
320 }
321 }
322}
323
324bool stage2_selftest(void)
325{
326 if (!hv_is_aligned_u64((uint64_t)(uintptr_t)g_l1, 65536u)) {
327 return false;
328 }
329 for (uint32_t i = 0; i < S2_ENTRIES; i++) {
330 if ((g_l1[i] & S2_DESC_VALID) != 0u && (g_l1[i] & S2_DESC_TABLE) == 0u) {
331 return false;
332 }
333 if ((g_l1[i] & S2_DESC_VALID) != 0u) {
334 uint64_t *l2 = (uint64_t *)(uintptr_t)desc_addr(g_l1[i]);
335 for (uint32_t j = 0; j < S2_ENTRIES; j++) {
336 if ((l2[j] & S2_DESC_VALID) != 0u && !descriptor_valid(l2[j])) {
337 return false;
338 }
339 }
340 }
341 }
342 return stage2_vttbr() == (uint64_t)(uintptr_t)g_l1;
343}