1 /* 2 * Copyright (c) 2018-2021 Maxime Villard, m00nbsd.net 3 * All rights reserved. 4 * 5 * This code is part of the NVMM hypervisor. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 #include <sys/ioctl.h> 31 #include <sys/mman.h> 32 #include <sys/queue.h> 33 34 #include <inttypes.h> 35 #include <stdio.h> 36 #include <stdlib.h> 37 #include <string.h> 38 #include <unistd.h> 39 #include <fcntl.h> 40 #include <errno.h> 41 42 #include "nvmm.h" 43 44 static struct nvmm_capability __capability; 45 46 #ifdef __x86_64__ 47 #include "libnvmm_x86.c" 48 #endif 49 50 #ifdef __DragonFly__ 51 #define LIST_FOREACH_SAFE LIST_FOREACH_MUTABLE 52 #endif 53 54 typedef struct __area { 55 LIST_ENTRY(__area) list; 56 gpaddr_t gpa; 57 uintptr_t hva; 58 size_t size; 59 nvmm_prot_t prot; 60 } area_t; 61 62 typedef LIST_HEAD(, __area) area_list_t; 63 64 static int nvmm_fd = -1; 65 66 /* -------------------------------------------------------------------------- */ 67 68 static bool 69 __area_isvalid(struct nvmm_machine *mach, gpaddr_t gpa, size_t size) 70 { 71 area_list_t *areas = mach->areas; 72 area_t *ent; 73 74 LIST_FOREACH(ent, areas, list) { 75 /* Collision on GPA */ 76 if (gpa >= ent->gpa && gpa < ent->gpa + ent->size) { 77 return false; 78 } 79 if (gpa + size > ent->gpa && 80 gpa + size <= ent->gpa + ent->size) { 81 return false; 82 } 83 if (gpa <= ent->gpa && gpa + size >= ent->gpa + ent->size) { 84 return false; 85 } 86 } 87 88 return true; 89 } 90 91 static int 92 __area_add(struct nvmm_machine *mach, uintptr_t hva, gpaddr_t gpa, size_t size, 93 int prot) 94 { 95 area_list_t *areas = mach->areas; 96 nvmm_prot_t nprot; 97 area_t *area; 98 99 nprot = 0; 100 if (prot & PROT_READ) 101 nprot |= NVMM_PROT_READ; 102 if (prot & PROT_WRITE) 103 nprot |= NVMM_PROT_WRITE; 104 if (prot & PROT_EXEC) 105 nprot |= NVMM_PROT_EXEC; 106 107 if (!__area_isvalid(mach, gpa, size)) { 108 errno = EINVAL; 109 return -1; 110 } 111 112 area = malloc(sizeof(*area)); 113 if (area == NULL) 114 return -1; 115 area->gpa = gpa; 116 area->hva = hva; 117 area->size = size; 118 area->prot = nprot; 119 120 LIST_INSERT_HEAD(areas, area, list); 121 122 return 0; 123 } 124 125 static int 126 __area_delete(struct nvmm_machine *mach, uintptr_t hva, gpaddr_t gpa, 127 size_t size) 128 { 129 area_list_t *areas = mach->areas; 130 area_t *ent, *nxt; 131 132 LIST_FOREACH_SAFE(ent, areas, list, nxt) { 133 if (hva == ent->hva && gpa == ent->gpa && size == ent->size) { 134 LIST_REMOVE(ent, list); 135 free(ent); 136 return 0; 137 } 138 } 139 140 return -1; 141 } 142 143 static void 144 __area_remove_all(struct nvmm_machine *mach) 145 { 146 area_list_t *areas = mach->areas; 147 area_t *ent; 148 149 while ((ent = LIST_FIRST(areas)) != NULL) { 150 LIST_REMOVE(ent, list); 151 free(ent); 152 } 153 154 free(areas); 155 } 156 157 /* -------------------------------------------------------------------------- */ 158 159 int 160 nvmm_init(void) 161 { 162 if (nvmm_fd != -1) 163 return 0; 164 nvmm_fd = open("/dev/nvmm", O_RDONLY | O_CLOEXEC); 165 if (nvmm_fd == -1) 166 return -1; 167 if (nvmm_capability(&__capability) == -1) { 168 close(nvmm_fd); 169 nvmm_fd = -1; 170 return -1; 171 } 172 if (__capability.version != NVMM_KERN_VERSION) { 173 close(nvmm_fd); 174 nvmm_fd = -1; 175 errno = EPROGMISMATCH; 176 return -1; 177 } 178 179 return 0; 180 } 181 182 int 183 nvmm_root_init(void) 184 { 185 if (nvmm_fd != -1) 186 return 0; 187 nvmm_fd = open("/dev/nvmm", O_WRONLY | O_CLOEXEC); 188 if (nvmm_fd == -1) 189 return -1; 190 if (nvmm_capability(&__capability) == -1) { 191 close(nvmm_fd); 192 nvmm_fd = -1; 193 return -1; 194 } 195 if (__capability.version != NVMM_KERN_VERSION) { 196 close(nvmm_fd); 197 nvmm_fd = -1; 198 errno = EPROGMISMATCH; 199 return -1; 200 } 201 202 return 0; 203 } 204 205 int 206 nvmm_capability(struct nvmm_capability *cap) 207 { 208 struct nvmm_ioc_capability args; 209 int ret; 210 211 ret = ioctl(nvmm_fd, NVMM_IOC_CAPABILITY, &args); 212 if (ret == -1) 213 return -1; 214 215 memcpy(cap, &args.cap, sizeof(args.cap)); 216 217 return 0; 218 } 219 220 int 221 nvmm_machine_create(struct nvmm_machine *mach) 222 { 223 struct nvmm_ioc_machine_create args; 224 struct nvmm_comm_page **pages; 225 area_list_t *areas; 226 int ret; 227 228 areas = calloc(1, sizeof(*areas)); 229 if (areas == NULL) 230 return -1; 231 232 pages = calloc(__capability.max_vcpus, sizeof(*pages)); 233 if (pages == NULL) { 234 free(areas); 235 return -1; 236 } 237 238 ret = ioctl(nvmm_fd, NVMM_IOC_MACHINE_CREATE, &args); 239 if (ret == -1) { 240 free(areas); 241 free(pages); 242 return -1; 243 } 244 245 LIST_INIT(areas); 246 247 memset(mach, 0, sizeof(*mach)); 248 mach->machid = args.machid; 249 mach->pages = pages; 250 mach->areas = areas; 251 252 return 0; 253 } 254 255 int 256 nvmm_machine_destroy(struct nvmm_machine *mach) 257 { 258 struct nvmm_ioc_machine_destroy args; 259 int ret; 260 261 args.machid = mach->machid; 262 263 ret = ioctl(nvmm_fd, NVMM_IOC_MACHINE_DESTROY, &args); 264 if (ret == -1) 265 return -1; 266 267 __area_remove_all(mach); 268 free(mach->pages); 269 270 return 0; 271 } 272 273 int 274 nvmm_machine_configure(struct nvmm_machine *mach, uint64_t op, void *conf) 275 { 276 struct nvmm_ioc_machine_configure args; 277 int ret; 278 279 args.machid = mach->machid; 280 args.op = op; 281 args.conf = conf; 282 283 ret = ioctl(nvmm_fd, NVMM_IOC_MACHINE_CONFIGURE, &args); 284 if (ret == -1) 285 return -1; 286 287 return 0; 288 } 289 290 int 291 nvmm_vcpu_create(struct nvmm_machine *mach, nvmm_cpuid_t cpuid, 292 struct nvmm_vcpu *vcpu) 293 { 294 struct nvmm_ioc_vcpu_create args; 295 int ret; 296 297 args.machid = mach->machid; 298 args.cpuid = cpuid; 299 args.comm = NULL; 300 301 ret = ioctl(nvmm_fd, NVMM_IOC_VCPU_CREATE, &args); 302 if (ret == -1) 303 return -1; 304 305 mach->pages[cpuid] = args.comm; 306 307 vcpu->cpuid = cpuid; 308 vcpu->state = &args.comm->state; 309 vcpu->event = &args.comm->event; 310 vcpu->exit = malloc(sizeof(*vcpu->exit)); 311 312 return 0; 313 } 314 315 int 316 nvmm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 317 { 318 struct nvmm_ioc_vcpu_destroy args; 319 int ret; 320 321 args.machid = mach->machid; 322 args.cpuid = vcpu->cpuid; 323 324 ret = ioctl(nvmm_fd, NVMM_IOC_VCPU_DESTROY, &args); 325 if (ret == -1) 326 return -1; 327 328 free(vcpu->exit); 329 330 return 0; 331 } 332 333 int 334 nvmm_vcpu_configure(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 335 uint64_t op, void *conf) 336 { 337 struct nvmm_ioc_vcpu_configure args; 338 int ret; 339 340 switch (op) { 341 case NVMM_VCPU_CONF_CALLBACKS: 342 memcpy(&vcpu->cbs, conf, sizeof(vcpu->cbs)); 343 return 0; 344 } 345 346 args.machid = mach->machid; 347 args.cpuid = vcpu->cpuid; 348 args.op = op; 349 args.conf = conf; 350 351 ret = ioctl(nvmm_fd, NVMM_IOC_VCPU_CONFIGURE, &args); 352 if (ret == -1) 353 return -1; 354 355 return 0; 356 } 357 358 int 359 nvmm_vcpu_setstate(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 360 uint64_t flags) 361 { 362 struct nvmm_comm_page *comm; 363 364 comm = mach->pages[vcpu->cpuid]; 365 comm->state_commit |= flags; 366 comm->state_cached |= flags; 367 368 return 0; 369 } 370 371 int 372 nvmm_vcpu_getstate(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 373 uint64_t flags) 374 { 375 struct nvmm_ioc_vcpu_getstate args; 376 struct nvmm_comm_page *comm; 377 int ret; 378 379 comm = mach->pages[vcpu->cpuid]; 380 381 if (__predict_true((flags & ~comm->state_cached) == 0)) { 382 return 0; 383 } 384 comm->state_wanted = flags & ~comm->state_cached; 385 386 args.machid = mach->machid; 387 args.cpuid = vcpu->cpuid; 388 389 ret = ioctl(nvmm_fd, NVMM_IOC_VCPU_GETSTATE, &args); 390 if (ret == -1) 391 return -1; 392 393 return 0; 394 } 395 396 int 397 nvmm_vcpu_inject(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 398 { 399 struct nvmm_comm_page *comm; 400 401 comm = mach->pages[vcpu->cpuid]; 402 comm->event_commit = true; 403 404 return 0; 405 } 406 407 int 408 nvmm_vcpu_run(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 409 { 410 struct nvmm_ioc_vcpu_run args; 411 int ret; 412 413 args.machid = mach->machid; 414 args.cpuid = vcpu->cpuid; 415 memset(&args.exit, 0, sizeof(args.exit)); 416 417 ret = ioctl(nvmm_fd, NVMM_IOC_VCPU_RUN, &args); 418 if (ret == -1) 419 return -1; 420 421 /* No comm support yet, just copy. */ 422 memcpy(vcpu->exit, &args.exit, sizeof(args.exit)); 423 424 return 0; 425 } 426 427 int 428 nvmm_gpa_map(struct nvmm_machine *mach, uintptr_t hva, gpaddr_t gpa, 429 size_t size, int prot) 430 { 431 struct nvmm_ioc_gpa_map args; 432 int ret; 433 434 ret = __area_add(mach, hva, gpa, size, prot); 435 if (ret == -1) 436 return -1; 437 438 args.machid = mach->machid; 439 args.hva = hva; 440 args.gpa = gpa; 441 args.size = size; 442 args.prot = prot; 443 444 ret = ioctl(nvmm_fd, NVMM_IOC_GPA_MAP, &args); 445 if (ret == -1) { 446 /* Can't recover. */ 447 abort(); 448 } 449 450 return 0; 451 } 452 453 int 454 nvmm_gpa_unmap(struct nvmm_machine *mach, uintptr_t hva, gpaddr_t gpa, 455 size_t size) 456 { 457 struct nvmm_ioc_gpa_unmap args; 458 int ret; 459 460 ret = __area_delete(mach, hva, gpa, size); 461 if (ret == -1) 462 return -1; 463 464 args.machid = mach->machid; 465 args.gpa = gpa; 466 args.size = size; 467 468 ret = ioctl(nvmm_fd, NVMM_IOC_GPA_UNMAP, &args); 469 if (ret == -1) { 470 /* Can't recover. */ 471 abort(); 472 } 473 474 return 0; 475 } 476 477 int 478 nvmm_hva_map(struct nvmm_machine *mach, uintptr_t hva, size_t size) 479 { 480 struct nvmm_ioc_hva_map args; 481 int ret; 482 483 args.machid = mach->machid; 484 args.hva = hva; 485 args.size = size; 486 487 ret = ioctl(nvmm_fd, NVMM_IOC_HVA_MAP, &args); 488 if (ret == -1) 489 return -1; 490 491 return 0; 492 } 493 494 int 495 nvmm_hva_unmap(struct nvmm_machine *mach, uintptr_t hva, size_t size) 496 { 497 struct nvmm_ioc_hva_unmap args; 498 int ret; 499 500 args.machid = mach->machid; 501 args.hva = hva; 502 args.size = size; 503 504 ret = ioctl(nvmm_fd, NVMM_IOC_HVA_UNMAP, &args); 505 if (ret == -1) 506 return -1; 507 508 return 0; 509 } 510 511 /* 512 * nvmm_gva_to_gpa(): architecture-specific. 513 */ 514 515 int 516 nvmm_gpa_to_hva(struct nvmm_machine *mach, gpaddr_t gpa, uintptr_t *hva, 517 nvmm_prot_t *prot) 518 { 519 area_list_t *areas = mach->areas; 520 area_t *ent; 521 522 LIST_FOREACH(ent, areas, list) { 523 if (gpa >= ent->gpa && gpa < ent->gpa + ent->size) { 524 *hva = ent->hva + (gpa - ent->gpa); 525 *prot = ent->prot; 526 return 0; 527 } 528 } 529 530 errno = ENOENT; 531 return -1; 532 } 533 534 /* 535 * nvmm_assist_io(): architecture-specific. 536 */ 537 538 /* 539 * nvmm_assist_mem(): architecture-specific. 540 */ 541 542 int 543 nvmm_ctl(int op, void *data, size_t size) 544 { 545 struct nvmm_ioc_ctl args; 546 int ret; 547 548 args.op = op; 549 args.data = data; 550 args.size = size; 551 552 ret = ioctl(nvmm_fd, NVMM_IOC_CTL, &args); 553 if (ret == -1) 554 return -1; 555 556 return 0; 557 } 558