1 /*
2 * Copyright (c) 2021 Maxime Villard, m00nbsd.net
3 * All rights reserved.
4 *
5 * This code is part of the NVMM hypervisor.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/param.h>
30 #include <sys/systm.h>
31 #include <sys/kernel.h>
32 #include <sys/mman.h>
33
34 #include "nvmm.h"
35 #include "nvmm_os.h"
36 #include "nvmm_internal.h"
37
38 os_vmspace_t *
os_vmspace_create(vaddr_t vmin,vaddr_t vmax)39 os_vmspace_create(vaddr_t vmin, vaddr_t vmax)
40 {
41 return uvmspace_alloc(vmin, vmax, false);
42 }
43
44 void
os_vmspace_destroy(os_vmspace_t * vm)45 os_vmspace_destroy(os_vmspace_t *vm)
46 {
47 uvmspace_free(vm);
48 }
49
50 int
os_vmspace_fault(os_vmspace_t * vm,vaddr_t va,vm_prot_t prot)51 os_vmspace_fault(os_vmspace_t *vm, vaddr_t va, vm_prot_t prot)
52 {
53 return uvm_fault(&vm->vm_map, va, prot);
54 }
55
56 os_vmobj_t *
os_vmobj_create(voff_t size)57 os_vmobj_create(voff_t size)
58 {
59 return uao_create(size, 0);
60 }
61
62 void
os_vmobj_ref(os_vmobj_t * vmobj)63 os_vmobj_ref(os_vmobj_t *vmobj)
64 {
65 uao_reference(vmobj);
66 }
67
68 void
os_vmobj_rel(os_vmobj_t * vmobj)69 os_vmobj_rel(os_vmobj_t *vmobj)
70 {
71 uao_detach(vmobj);
72 }
73
74 int
os_vmobj_map(struct vm_map * map,vaddr_t * addr,vsize_t size,os_vmobj_t * vmobj,voff_t offset,bool wired,bool fixed,bool shared,int prot,int maxprot)75 os_vmobj_map(struct vm_map *map, vaddr_t *addr, vsize_t size, os_vmobj_t *vmobj,
76 voff_t offset, bool wired, bool fixed, bool shared, int prot, int maxprot)
77 {
78 uvm_flag_t uflags, uprot, umaxprot;
79 int error;
80
81 /* Convert prot. */
82 uprot = 0;
83 if (prot & PROT_READ)
84 uprot |= UVM_PROT_R;
85 if (prot & PROT_WRITE)
86 uprot |= UVM_PROT_W;
87 if (prot & PROT_EXEC)
88 uprot |= UVM_PROT_X;
89
90 /* Convert maxprot. */
91 umaxprot = 0;
92 if (maxprot & PROT_READ)
93 umaxprot |= UVM_PROT_R;
94 if (maxprot & PROT_WRITE)
95 umaxprot |= UVM_PROT_W;
96 if (maxprot & PROT_EXEC)
97 umaxprot |= UVM_PROT_X;
98
99 uflags = UVM_MAPFLAG(uprot, umaxprot,
100 shared ? UVM_INH_SHARE : UVM_INH_NONE, UVM_ADV_RANDOM,
101 fixed ? (UVM_FLAG_FIXED | UVM_FLAG_UNMAP) : 0);
102
103 if (!fixed) {
104 /* Need to provide a hint. */
105 if (map == os_curproc_map) {
106 *addr = curproc->p_emul->e_vm_default_addr(curproc,
107 (vaddr_t)curproc->p_vmspace->vm_daddr, size,
108 curproc->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN);
109 } else {
110 *addr = 0;
111 }
112 }
113
114 /* Get a reference to the object. */
115 os_vmobj_ref(vmobj);
116
117 /*
118 * Map the object. This consumes the reference on success only. On
119 * failure we must drop the reference manually.
120 */
121 error = uvm_map(map, addr, size, vmobj, offset, 0, uflags);
122 if (error) {
123 /* Drop the ref. */
124 os_vmobj_rel(vmobj);
125 return error;
126 }
127
128 if (wired) {
129 error = uvm_map_pageable(map, *addr, *addr + size, false, 0);
130 if (error) {
131 os_vmobj_unmap(map, *addr, *addr + size, false);
132 return error;
133 }
134 }
135
136 return 0;
137 }
138
139 void
os_vmobj_unmap(struct vm_map * map,vaddr_t start,vaddr_t end,bool wired __unused)140 os_vmobj_unmap(struct vm_map *map, vaddr_t start, vaddr_t end,
141 bool wired __unused)
142 {
143 uvm_unmap(map, start, end);
144 }
145
146 void *
os_pagemem_zalloc(size_t size)147 os_pagemem_zalloc(size_t size)
148 {
149 void *ret;
150
151 ret = (void *)uvm_km_alloc(kernel_map, roundup(size, PAGE_SIZE), 0,
152 UVM_KMF_WIRED | UVM_KMF_ZERO);
153
154 OS_ASSERT((uintptr_t)ret % PAGE_SIZE == 0);
155
156 return ret;
157 }
158
159 void
os_pagemem_free(void * ptr,size_t size)160 os_pagemem_free(void *ptr, size_t size)
161 {
162 uvm_km_free(kernel_map, (vaddr_t)ptr, roundup(size, PAGE_SIZE),
163 UVM_KMF_WIRED);
164 }
165
166 paddr_t
os_pa_zalloc(void)167 os_pa_zalloc(void)
168 {
169 struct vm_page *pg;
170
171 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
172
173 return VM_PAGE_TO_PHYS(pg);
174 }
175
176 void
os_pa_free(paddr_t pa)177 os_pa_free(paddr_t pa)
178 {
179 uvm_pagefree(PHYS_TO_VM_PAGE(pa));
180 }
181
182 int
os_contigpa_zalloc(paddr_t * pa,vaddr_t * va,size_t npages)183 os_contigpa_zalloc(paddr_t *pa, vaddr_t *va, size_t npages)
184 {
185 struct pglist pglist;
186 paddr_t _pa;
187 vaddr_t _va;
188 size_t i;
189 int ret;
190
191 ret = uvm_pglistalloc(npages * PAGE_SIZE, 0, ~0UL, PAGE_SIZE, 0,
192 &pglist, 1, 0);
193 if (ret != 0)
194 return ENOMEM;
195 _pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist));
196 _va = uvm_km_alloc(kernel_map, npages * PAGE_SIZE, 0,
197 UVM_KMF_VAONLY | UVM_KMF_NOWAIT);
198 if (_va == 0)
199 goto error;
200
201 for (i = 0; i < npages; i++) {
202 pmap_kenter_pa(_va + i * PAGE_SIZE, _pa + i * PAGE_SIZE,
203 VM_PROT_READ | VM_PROT_WRITE, PMAP_WRITE_BACK);
204 }
205 pmap_update(pmap_kernel());
206
207 memset((void *)_va, 0, npages * PAGE_SIZE);
208
209 *pa = _pa;
210 *va = _va;
211 return 0;
212
213 error:
214 for (i = 0; i < npages; i++) {
215 uvm_pagefree(PHYS_TO_VM_PAGE(_pa + i * PAGE_SIZE));
216 }
217 return ENOMEM;
218 }
219
220 void
os_contigpa_free(paddr_t pa,vaddr_t va,size_t npages)221 os_contigpa_free(paddr_t pa, vaddr_t va, size_t npages)
222 {
223 size_t i;
224
225 pmap_kremove(va, npages * PAGE_SIZE);
226 pmap_update(pmap_kernel());
227 uvm_km_free(kernel_map, va, npages * PAGE_SIZE, UVM_KMF_VAONLY);
228 for (i = 0; i < npages; i++) {
229 uvm_pagefree(PHYS_TO_VM_PAGE(pa + i * PAGE_SIZE));
230 }
231 }
232
233 /* -------------------------------------------------------------------------- */
234
235 #include <sys/conf.h>
236 #include <sys/device.h>
237 #include <sys/file.h>
238 #include <sys/filedesc.h>
239 #include <sys/module.h>
240
241 #include "ioconf.h"
242
243 static dev_type_open(nbsd_nvmm_open);
244 static int nbsd_nvmm_ioctl(file_t *, u_long, void *);
245 static int nbsd_nvmm_close(file_t *);
246
247 const struct cdevsw nvmm_cdevsw = {
248 .d_open = nbsd_nvmm_open,
249 .d_close = noclose,
250 .d_read = noread,
251 .d_write = nowrite,
252 .d_ioctl = noioctl,
253 .d_stop = nostop,
254 .d_tty = notty,
255 .d_poll = nopoll,
256 .d_mmap = nommap,
257 .d_kqfilter = nokqfilter,
258 .d_discard = nodiscard,
259 .d_flag = D_OTHER | D_MPSAFE
260 };
261
262 static const struct fileops nvmm_fileops = {
263 .fo_read = fbadop_read,
264 .fo_write = fbadop_write,
265 .fo_ioctl = nbsd_nvmm_ioctl,
266 .fo_fcntl = fnullop_fcntl,
267 .fo_poll = fnullop_poll,
268 .fo_stat = fbadop_stat,
269 .fo_close = nbsd_nvmm_close,
270 .fo_kqfilter = fnullop_kqfilter,
271 .fo_restart = fnullop_restart,
272 .fo_mmap = NULL,
273 };
274
275 static int
nbsd_nvmm_open(dev_t dev,int flags,int type,struct lwp * l)276 nbsd_nvmm_open(dev_t dev, int flags, int type, struct lwp *l)
277 {
278 struct nvmm_owner *owner;
279 struct file *fp;
280 int error, fd;
281
282 if (__predict_false(nvmm_impl == NULL))
283 return ENXIO;
284 if (minor(dev) != 0)
285 return EXDEV;
286 if (!(flags & O_CLOEXEC))
287 return EINVAL;
288 error = fd_allocfile(&fp, &fd);
289 if (error)
290 return error;
291
292 if (OFLAGS(flags) & O_WRONLY) {
293 owner = &nvmm_root_owner;
294 } else {
295 owner = os_mem_alloc(sizeof(*owner));
296 owner->pid = l->l_proc->p_pid;
297 }
298
299 return fd_clone(fp, fd, flags, &nvmm_fileops, owner);
300 }
301
302 static int
nbsd_nvmm_ioctl(file_t * fp,u_long cmd,void * data)303 nbsd_nvmm_ioctl(file_t *fp, u_long cmd, void *data)
304 {
305 struct nvmm_owner *owner = fp->f_data;
306
307 OS_ASSERT(owner != NULL);
308
309 return nvmm_ioctl(owner, cmd, data);
310 }
311
312 static int
nbsd_nvmm_close(file_t * fp)313 nbsd_nvmm_close(file_t *fp)
314 {
315 struct nvmm_owner *owner = fp->f_data;
316
317 OS_ASSERT(owner != NULL);
318 nvmm_kill_machines(owner);
319 if (owner != &nvmm_root_owner) {
320 os_mem_free(owner, sizeof(*owner));
321 }
322 fp->f_data = NULL;
323
324 return 0;
325 }
326
327 /* -------------------------------------------------------------------------- */
328
329 static int nvmm_match(device_t, cfdata_t, void *);
330 static void nvmm_attach(device_t, device_t, void *);
331 static int nvmm_detach(device_t, int);
332
333 extern struct cfdriver nvmm_cd;
334
335 CFATTACH_DECL_NEW(nvmm, 0, nvmm_match, nvmm_attach, nvmm_detach, NULL);
336
337 static struct cfdata nvmm_cfdata[] = {
338 {
339 .cf_name = "nvmm",
340 .cf_atname = "nvmm",
341 .cf_unit = 0,
342 .cf_fstate = FSTATE_STAR,
343 .cf_loc = NULL,
344 .cf_flags = 0,
345 .cf_pspec = NULL,
346 },
347 { NULL, NULL, 0, FSTATE_NOTFOUND, NULL, 0, NULL }
348 };
349
350 static int
nvmm_match(device_t self,cfdata_t cfdata,void * arg)351 nvmm_match(device_t self, cfdata_t cfdata, void *arg)
352 {
353 return 1;
354 }
355
356 static void
nvmm_attach(device_t parent,device_t self,void * aux)357 nvmm_attach(device_t parent, device_t self, void *aux)
358 {
359 int error;
360
361 error = nvmm_init();
362 if (error)
363 panic("%s: impossible", __func__);
364 aprint_normal_dev(self, "attached, using backend %s\n",
365 nvmm_impl->name);
366 }
367
368 static int
nvmm_detach(device_t self,int flags)369 nvmm_detach(device_t self, int flags)
370 {
371 if (os_atomic_load_uint(&nmachines) > 0)
372 return EBUSY;
373 nvmm_fini();
374 return 0;
375 }
376
377 void
nvmmattach(int nunits)378 nvmmattach(int nunits)
379 {
380 /* nothing */
381 }
382
383 MODULE(MODULE_CLASS_DRIVER, nvmm, NULL);
384
385 #if defined(_MODULE)
386 CFDRIVER_DECL(nvmm, DV_VIRTUAL, NULL);
387 #endif
388
389 static int
nvmm_modcmd(modcmd_t cmd,void * arg)390 nvmm_modcmd(modcmd_t cmd, void *arg)
391 {
392 #if defined(_MODULE)
393 devmajor_t bmajor = NODEVMAJOR;
394 devmajor_t cmajor = 345;
395 #endif
396 int error;
397
398 switch (cmd) {
399 case MODULE_CMD_INIT:
400 if (nvmm_ident() == NULL) {
401 aprint_error("%s: cpu not supported\n",
402 nvmm_cd.cd_name);
403 return ENOTSUP;
404 }
405 #if defined(_MODULE)
406 error = config_cfdriver_attach(&nvmm_cd);
407 if (error)
408 return error;
409 #endif
410 error = config_cfattach_attach(nvmm_cd.cd_name, &nvmm_ca);
411 if (error) {
412 config_cfdriver_detach(&nvmm_cd);
413 aprint_error("%s: config_cfattach_attach failed\n",
414 nvmm_cd.cd_name);
415 return error;
416 }
417
418 error = config_cfdata_attach(nvmm_cfdata, 1);
419 if (error) {
420 config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca);
421 config_cfdriver_detach(&nvmm_cd);
422 aprint_error("%s: unable to register cfdata\n",
423 nvmm_cd.cd_name);
424 return error;
425 }
426
427 if (config_attach_pseudo(nvmm_cfdata) == NULL) {
428 aprint_error("%s: config_attach_pseudo failed\n",
429 nvmm_cd.cd_name);
430 config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca);
431 config_cfdriver_detach(&nvmm_cd);
432 return ENXIO;
433 }
434
435 #if defined(_MODULE)
436 /* mknod /dev/nvmm c 345 0 */
437 error = devsw_attach(nvmm_cd.cd_name, NULL, &bmajor,
438 &nvmm_cdevsw, &cmajor);
439 if (error) {
440 aprint_error("%s: unable to register devsw\n",
441 nvmm_cd.cd_name);
442 config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca);
443 config_cfdriver_detach(&nvmm_cd);
444 return error;
445 }
446 #endif
447 return 0;
448 case MODULE_CMD_FINI:
449 error = config_cfdata_detach(nvmm_cfdata);
450 if (error)
451 return error;
452 error = config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca);
453 if (error)
454 return error;
455 #if defined(_MODULE)
456 config_cfdriver_detach(&nvmm_cd);
457 devsw_detach(NULL, &nvmm_cdevsw);
458 #endif
459 return 0;
460 case MODULE_CMD_AUTOUNLOAD:
461 return EBUSY;
462 default:
463 return ENOTTY;
464 }
465 }
466