1 /* $OpenBSD: xen.c,v 1.96 2020/05/29 04:42:25 deraadt Exp $ */ 2 3 /* 4 * Copyright (c) 2015, 2016, 2017 Mike Belopuhov 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/param.h> 20 21 /* Xen requires locked atomic operations */ 22 #ifndef MULTIPROCESSOR 23 #define _XENMPATOMICS 24 #define MULTIPROCESSOR 25 #endif 26 #include <sys/atomic.h> 27 #ifdef _XENMPATOMICS 28 #undef MULTIPROCESSOR 29 #undef _XENMPATOMICS 30 #endif 31 32 #include <sys/systm.h> 33 #include <sys/proc.h> 34 #include <sys/signal.h> 35 #include <sys/signalvar.h> 36 #include <sys/refcnt.h> 37 #include <sys/malloc.h> 38 #include <sys/kernel.h> 39 #include <sys/stdint.h> 40 #include <sys/device.h> 41 #include <sys/task.h> 42 #include <sys/syslog.h> 43 44 #include <machine/bus.h> 45 #include <machine/cpu.h> 46 #include <machine/cpufunc.h> 47 48 #include <uvm/uvm_extern.h> 49 50 #include <machine/i82489var.h> 51 52 #include <dev/pv/pvvar.h> 53 #include <dev/pv/pvreg.h> 54 #include <dev/pv/xenreg.h> 55 #include <dev/pv/xenvar.h> 56 57 /* #define XEN_DEBUG */ 58 59 #ifdef XEN_DEBUG 60 #define DPRINTF(x...) printf(x) 61 #else 62 #define DPRINTF(x...) 63 #endif 64 65 struct xen_softc *xen_sc; 66 67 int xen_init_hypercall(struct xen_softc *); 68 int xen_getfeatures(struct xen_softc *); 69 int xen_init_info_page(struct xen_softc *); 70 int xen_init_cbvec(struct xen_softc *); 71 int xen_init_interrupts(struct xen_softc *); 72 void xen_intr_dispatch(void *); 73 int xen_init_grant_tables(struct xen_softc *); 74 struct xen_gntent * 75 xen_grant_table_grow(struct xen_softc *); 76 int xen_grant_table_alloc(struct xen_softc *, grant_ref_t *); 77 void xen_grant_table_free(struct xen_softc *, grant_ref_t); 78 void xen_grant_table_enter(struct xen_softc *, grant_ref_t, paddr_t, 79 int, int); 80 void xen_grant_table_remove(struct xen_softc *, grant_ref_t); 81 void xen_disable_emulated_devices(struct xen_softc *); 82 83 int xen_match(struct device *, void *, void *); 84 void xen_attach(struct device *, struct device *, void *); 85 void xen_deferred(struct device *); 86 void xen_control(void *); 87 void xen_hotplug(void *); 88 void xen_resume(struct device *); 89 int xen_activate(struct device *, int); 90 int xen_attach_device(struct xen_softc *, struct xen_devlist *, 91 const char *, const char *); 92 int xen_probe_devices(struct xen_softc *); 93 94 int xen_bus_dmamap_create(bus_dma_tag_t, bus_size_t, int, bus_size_t, 95 bus_size_t, int, bus_dmamap_t *); 96 void xen_bus_dmamap_destroy(bus_dma_tag_t, bus_dmamap_t); 97 int xen_bus_dmamap_load(bus_dma_tag_t, bus_dmamap_t, void *, bus_size_t, 98 struct proc *, int); 99 int xen_bus_dmamap_load_mbuf(bus_dma_tag_t, bus_dmamap_t, struct mbuf *, 100 int); 101 void xen_bus_dmamap_unload(bus_dma_tag_t, bus_dmamap_t); 102 void xen_bus_dmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t, 103 bus_size_t, int); 104 105 int xs_attach(struct xen_softc *); 106 107 struct cfdriver xen_cd = { 108 NULL, "xen", DV_DULL 109 }; 110 111 const struct cfattach xen_ca = { 112 sizeof(struct xen_softc), xen_match, xen_attach, NULL, xen_activate 113 }; 114 115 struct bus_dma_tag xen_bus_dma_tag = { 116 NULL, 117 xen_bus_dmamap_create, 118 xen_bus_dmamap_destroy, 119 xen_bus_dmamap_load, 120 xen_bus_dmamap_load_mbuf, 121 NULL, 122 NULL, 123 xen_bus_dmamap_unload, 124 xen_bus_dmamap_sync, 125 _bus_dmamem_alloc, 126 NULL, 127 _bus_dmamem_free, 128 _bus_dmamem_map, 129 _bus_dmamem_unmap, 130 NULL, 131 }; 132 133 int 134 xen_match(struct device *parent, void *match, void *aux) 135 { 136 struct pv_attach_args *pva = aux; 137 struct pvbus_hv *hv = &pva->pva_hv[PVBUS_XEN]; 138 139 if (hv->hv_base == 0) 140 return (0); 141 142 return (1); 143 } 144 145 void 146 xen_attach(struct device *parent, struct device *self, void *aux) 147 { 148 struct pv_attach_args *pva = (struct pv_attach_args *)aux; 149 struct pvbus_hv *hv = &pva->pva_hv[PVBUS_XEN]; 150 struct xen_softc *sc = (struct xen_softc *)self; 151 152 sc->sc_base = hv->hv_base; 153 sc->sc_dmat = pva->pva_dmat; 154 155 if (xen_init_hypercall(sc)) 156 return; 157 158 /* Wire it up to the global */ 159 xen_sc = sc; 160 161 if (xen_getfeatures(sc)) 162 return; 163 164 if (xen_init_info_page(sc)) 165 return; 166 167 xen_init_cbvec(sc); 168 169 if (xen_init_interrupts(sc)) 170 return; 171 172 if (xen_init_grant_tables(sc)) 173 return; 174 175 if (xs_attach(sc)) 176 return; 177 178 xen_probe_devices(sc); 179 180 /* pvbus(4) key/value interface */ 181 hv->hv_kvop = xs_kvop; 182 hv->hv_arg = sc; 183 184 xen_disable_emulated_devices(sc); 185 186 config_mountroot(self, xen_deferred); 187 } 188 189 void 190 xen_deferred(struct device *self) 191 { 192 struct xen_softc *sc = (struct xen_softc *)self; 193 194 if (!(sc->sc_flags & XSF_CBVEC)) { 195 DPRINTF("%s: callback vector hasn't been established\n", 196 sc->sc_dev.dv_xname); 197 return; 198 } 199 200 xen_intr_enable(); 201 202 if (xs_watch(sc, "control", "shutdown", &sc->sc_ctltsk, 203 xen_control, sc)) 204 printf("%s: failed to setup shutdown control watch\n", 205 sc->sc_dev.dv_xname); 206 } 207 208 void 209 xen_control(void *arg) 210 { 211 struct xen_softc *sc = arg; 212 struct xs_transaction xst; 213 char action[128]; 214 int error; 215 216 memset(&xst, 0, sizeof(xst)); 217 xst.xst_id = 0; 218 xst.xst_cookie = sc->sc_xs; 219 220 error = xs_getprop(sc, "control", "shutdown", action, sizeof(action)); 221 if (error) { 222 if (error != ENOENT) 223 printf("%s: failed to process control event\n", 224 sc->sc_dev.dv_xname); 225 return; 226 } 227 228 if (strlen(action) == 0) 229 return; 230 231 /* Acknowledge the event */ 232 xs_setprop(sc, "control", "shutdown", "", 0); 233 234 if (strcmp(action, "halt") == 0 || strcmp(action, "poweroff") == 0) { 235 pvbus_shutdown(&sc->sc_dev); 236 } else if (strcmp(action, "reboot") == 0) { 237 pvbus_reboot(&sc->sc_dev); 238 } else if (strcmp(action, "crash") == 0) { 239 panic("xen told us to do this"); 240 } else if (strcmp(action, "suspend") == 0) { 241 /* Not implemented yet */ 242 } else { 243 printf("%s: unknown shutdown event \"%s\"\n", 244 sc->sc_dev.dv_xname, action); 245 } 246 } 247 248 void 249 xen_resume(struct device *self) 250 { 251 } 252 253 int 254 xen_activate(struct device *self, int act) 255 { 256 int rv = 0; 257 258 switch (act) { 259 case DVACT_RESUME: 260 xen_resume(self); 261 break; 262 } 263 return (rv); 264 } 265 266 int 267 xen_init_hypercall(struct xen_softc *sc) 268 { 269 extern void *xen_hypercall_page; 270 uint32_t regs[4]; 271 paddr_t pa; 272 273 /* Get hypercall page configuration MSR */ 274 CPUID(sc->sc_base + CPUID_OFFSET_XEN_HYPERCALL, 275 regs[0], regs[1], regs[2], regs[3]); 276 277 /* We don't support more than one hypercall page */ 278 if (regs[0] != 1) { 279 printf(": requested %u hypercall pages\n", regs[0]); 280 return (-1); 281 } 282 283 sc->sc_hc = &xen_hypercall_page; 284 285 if (!pmap_extract(pmap_kernel(), (vaddr_t)sc->sc_hc, &pa)) { 286 printf(": hypercall page PA extraction failed\n"); 287 return (-1); 288 } 289 wrmsr(regs[1], pa); 290 291 return (0); 292 } 293 294 int 295 xen_hypercall(struct xen_softc *sc, int op, int argc, ...) 296 { 297 va_list ap; 298 ulong argv[5]; 299 int i; 300 301 if (argc < 0 || argc > 5) 302 return (-1); 303 va_start(ap, argc); 304 for (i = 0; i < argc; i++) 305 argv[i] = (ulong)va_arg(ap, ulong); 306 va_end(ap); 307 return (xen_hypercallv(sc, op, argc, argv)); 308 } 309 310 int 311 xen_hypercallv(struct xen_softc *sc, int op, int argc, ulong *argv) 312 { 313 ulong hcall; 314 int rv = 0; 315 316 hcall = (ulong)sc->sc_hc + op * 32; 317 318 #if defined(XEN_DEBUG) && disabled 319 { 320 int i; 321 322 printf("hypercall %d", op); 323 if (argc > 0) { 324 printf(", args {"); 325 for (i = 0; i < argc; i++) 326 printf(" %#lx", argv[i]); 327 printf(" }\n"); 328 } else 329 printf("\n"); 330 } 331 #endif 332 333 switch (argc) { 334 case 0: { 335 HYPERCALL_RES1; 336 __asm__ volatile ( \ 337 HYPERCALL_LABEL \ 338 : HYPERCALL_OUT1 \ 339 : HYPERCALL_PTR(hcall) \ 340 : HYPERCALL_CLOBBER \ 341 ); 342 HYPERCALL_RET(rv); 343 break; 344 } 345 case 1: { 346 HYPERCALL_RES1; HYPERCALL_RES2; 347 HYPERCALL_ARG1(argv[0]); 348 __asm__ volatile ( \ 349 HYPERCALL_LABEL \ 350 : HYPERCALL_OUT1 HYPERCALL_OUT2 \ 351 : HYPERCALL_IN1 \ 352 , HYPERCALL_PTR(hcall) \ 353 : HYPERCALL_CLOBBER \ 354 ); 355 HYPERCALL_RET(rv); 356 break; 357 } 358 case 2: { 359 HYPERCALL_RES1; HYPERCALL_RES2; HYPERCALL_RES3; 360 HYPERCALL_ARG1(argv[0]); HYPERCALL_ARG2(argv[1]); 361 __asm__ volatile ( \ 362 HYPERCALL_LABEL \ 363 : HYPERCALL_OUT1 HYPERCALL_OUT2 \ 364 HYPERCALL_OUT3 \ 365 : HYPERCALL_IN1 HYPERCALL_IN2 \ 366 , HYPERCALL_PTR(hcall) \ 367 : HYPERCALL_CLOBBER \ 368 ); 369 HYPERCALL_RET(rv); 370 break; 371 } 372 case 3: { 373 HYPERCALL_RES1; HYPERCALL_RES2; HYPERCALL_RES3; 374 HYPERCALL_RES4; 375 HYPERCALL_ARG1(argv[0]); HYPERCALL_ARG2(argv[1]); 376 HYPERCALL_ARG3(argv[2]); 377 __asm__ volatile ( \ 378 HYPERCALL_LABEL \ 379 : HYPERCALL_OUT1 HYPERCALL_OUT2 \ 380 HYPERCALL_OUT3 HYPERCALL_OUT4 \ 381 : HYPERCALL_IN1 HYPERCALL_IN2 \ 382 HYPERCALL_IN3 \ 383 , HYPERCALL_PTR(hcall) \ 384 : HYPERCALL_CLOBBER \ 385 ); 386 HYPERCALL_RET(rv); 387 break; 388 } 389 case 4: { 390 HYPERCALL_RES1; HYPERCALL_RES2; HYPERCALL_RES3; 391 HYPERCALL_RES4; HYPERCALL_RES5; 392 HYPERCALL_ARG1(argv[0]); HYPERCALL_ARG2(argv[1]); 393 HYPERCALL_ARG3(argv[2]); HYPERCALL_ARG4(argv[3]); 394 __asm__ volatile ( \ 395 HYPERCALL_LABEL \ 396 : HYPERCALL_OUT1 HYPERCALL_OUT2 \ 397 HYPERCALL_OUT3 HYPERCALL_OUT4 \ 398 HYPERCALL_OUT5 \ 399 : HYPERCALL_IN1 HYPERCALL_IN2 \ 400 HYPERCALL_IN3 HYPERCALL_IN4 \ 401 , HYPERCALL_PTR(hcall) \ 402 : HYPERCALL_CLOBBER \ 403 ); 404 HYPERCALL_RET(rv); 405 break; 406 } 407 case 5: { 408 HYPERCALL_RES1; HYPERCALL_RES2; HYPERCALL_RES3; 409 HYPERCALL_RES4; HYPERCALL_RES5; HYPERCALL_RES6; 410 HYPERCALL_ARG1(argv[0]); HYPERCALL_ARG2(argv[1]); 411 HYPERCALL_ARG3(argv[2]); HYPERCALL_ARG4(argv[3]); 412 HYPERCALL_ARG5(argv[4]); 413 __asm__ volatile ( \ 414 HYPERCALL_LABEL \ 415 : HYPERCALL_OUT1 HYPERCALL_OUT2 \ 416 HYPERCALL_OUT3 HYPERCALL_OUT4 \ 417 HYPERCALL_OUT5 HYPERCALL_OUT6 \ 418 : HYPERCALL_IN1 HYPERCALL_IN2 \ 419 HYPERCALL_IN3 HYPERCALL_IN4 \ 420 HYPERCALL_IN5 \ 421 , HYPERCALL_PTR(hcall) \ 422 : HYPERCALL_CLOBBER \ 423 ); 424 HYPERCALL_RET(rv); 425 break; 426 } 427 default: 428 DPRINTF("%s: wrong number of arguments: %d\n", __func__, argc); 429 rv = -1; 430 break; 431 } 432 return (rv); 433 } 434 435 int 436 xen_getfeatures(struct xen_softc *sc) 437 { 438 struct xen_feature_info xfi; 439 440 memset(&xfi, 0, sizeof(xfi)); 441 if (xen_hypercall(sc, XC_VERSION, 2, XENVER_get_features, &xfi) < 0) { 442 printf(": failed to fetch features\n"); 443 return (-1); 444 } 445 sc->sc_features = xfi.submap; 446 #ifdef XEN_DEBUG 447 printf(": features %b", sc->sc_features, 448 "\20\014DOM0\013PIRQ\012PVCLOCK\011CBVEC\010GNTFLAGS\007HMA" 449 "\006PTUPD\005PAE4G\004SUPERVISOR\003AUTOPMAP\002WDT\001WPT"); 450 #else 451 printf(": features %#x", sc->sc_features); 452 #endif 453 return (0); 454 } 455 456 #ifdef XEN_DEBUG 457 void 458 xen_print_info_page(void) 459 { 460 struct xen_softc *sc = xen_sc; 461 struct shared_info *s = sc->sc_ipg; 462 struct vcpu_info *v; 463 int i; 464 465 virtio_membar_sync(); 466 for (i = 0; i < XEN_LEGACY_MAX_VCPUS; i++) { 467 v = &s->vcpu_info[i]; 468 if (!v->evtchn_upcall_pending && !v->evtchn_upcall_mask && 469 !v->evtchn_pending_sel && !v->time.version && 470 !v->time.tsc_timestamp && !v->time.system_time && 471 !v->time.tsc_to_system_mul && !v->time.tsc_shift) 472 continue; 473 printf("vcpu%d:\n" 474 " upcall_pending=%02x upcall_mask=%02x pending_sel=%#lx\n" 475 " time version=%u tsc=%llu system=%llu\n" 476 " time mul=%u shift=%d\n", 477 i, v->evtchn_upcall_pending, v->evtchn_upcall_mask, 478 v->evtchn_pending_sel, v->time.version, 479 v->time.tsc_timestamp, v->time.system_time, 480 v->time.tsc_to_system_mul, v->time.tsc_shift); 481 } 482 printf("pending events: "); 483 for (i = 0; i < nitems(s->evtchn_pending); i++) { 484 if (s->evtchn_pending[i] == 0) 485 continue; 486 printf(" %d:%#lx", i, s->evtchn_pending[i]); 487 } 488 printf("\nmasked events: "); 489 for (i = 0; i < nitems(s->evtchn_mask); i++) { 490 if (s->evtchn_mask[i] == 0xffffffffffffffffULL) 491 continue; 492 printf(" %d:%#lx", i, s->evtchn_mask[i]); 493 } 494 printf("\nwc ver=%u sec=%u nsec=%u\n", s->wc_version, s->wc_sec, 495 s->wc_nsec); 496 printf("arch maxpfn=%lu framelist=%lu nmi=%lu\n", s->arch.max_pfn, 497 s->arch.pfn_to_mfn_frame_list, s->arch.nmi_reason); 498 } 499 #endif /* XEN_DEBUG */ 500 501 int 502 xen_init_info_page(struct xen_softc *sc) 503 { 504 struct xen_add_to_physmap xatp; 505 paddr_t pa; 506 507 sc->sc_ipg = malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT | M_ZERO); 508 if (sc->sc_ipg == NULL) { 509 printf(": failed to allocate shared info page\n"); 510 return (-1); 511 } 512 if (!pmap_extract(pmap_kernel(), (vaddr_t)sc->sc_ipg, &pa)) { 513 printf(": shared info page PA extraction failed\n"); 514 free(sc->sc_ipg, M_DEVBUF, PAGE_SIZE); 515 return (-1); 516 } 517 xatp.domid = DOMID_SELF; 518 xatp.idx = 0; 519 xatp.space = XENMAPSPACE_shared_info; 520 xatp.gpfn = atop(pa); 521 if (xen_hypercall(sc, XC_MEMORY, 2, XENMEM_add_to_physmap, &xatp)) { 522 printf(": failed to register shared info page\n"); 523 free(sc->sc_ipg, M_DEVBUF, PAGE_SIZE); 524 return (-1); 525 } 526 return (0); 527 } 528 529 int 530 xen_init_cbvec(struct xen_softc *sc) 531 { 532 struct xen_hvm_param xhp; 533 534 if ((sc->sc_features & XENFEAT_CBVEC) == 0) 535 return (ENOENT); 536 537 xhp.domid = DOMID_SELF; 538 xhp.index = HVM_PARAM_CALLBACK_IRQ; 539 xhp.value = HVM_CALLBACK_VECTOR(LAPIC_XEN_VECTOR); 540 if (xen_hypercall(sc, XC_HVM, 2, HVMOP_set_param, &xhp)) { 541 /* Will retry with the xspd(4) PCI interrupt */ 542 return (ENOENT); 543 } 544 DPRINTF(", idtvec %d", LAPIC_XEN_VECTOR); 545 546 sc->sc_flags |= XSF_CBVEC; 547 548 return (0); 549 } 550 551 int 552 xen_init_interrupts(struct xen_softc *sc) 553 { 554 int i; 555 556 sc->sc_irq = LAPIC_XEN_VECTOR; 557 558 /* 559 * Clear all pending events and mask all interrupts 560 */ 561 for (i = 0; i < nitems(sc->sc_ipg->evtchn_pending); i++) { 562 sc->sc_ipg->evtchn_pending[i] = 0; 563 sc->sc_ipg->evtchn_mask[i] = ~0UL; 564 } 565 566 SLIST_INIT(&sc->sc_intrs); 567 568 mtx_init(&sc->sc_islck, IPL_NET); 569 570 return (0); 571 } 572 573 static int 574 xen_evtchn_hypercall(struct xen_softc *sc, int cmd, void *arg, size_t len) 575 { 576 struct evtchn_op compat; 577 int error; 578 579 error = xen_hypercall(sc, XC_EVTCHN, 2, cmd, arg); 580 if (error == -ENOXENSYS) { 581 memset(&compat, 0, sizeof(compat)); 582 compat.cmd = cmd; 583 memcpy(&compat.u, arg, len); 584 error = xen_hypercall(sc, XC_OEVTCHN, 1, &compat); 585 } 586 return (error); 587 } 588 589 static inline void 590 xen_intsrc_add(struct xen_softc *sc, struct xen_intsrc *xi) 591 { 592 refcnt_init(&xi->xi_refcnt); 593 mtx_enter(&sc->sc_islck); 594 SLIST_INSERT_HEAD(&sc->sc_intrs, xi, xi_entry); 595 mtx_leave(&sc->sc_islck); 596 } 597 598 static inline struct xen_intsrc * 599 xen_intsrc_acquire(struct xen_softc *sc, evtchn_port_t port) 600 { 601 struct xen_intsrc *xi = NULL; 602 603 mtx_enter(&sc->sc_islck); 604 SLIST_FOREACH(xi, &sc->sc_intrs, xi_entry) { 605 if (xi->xi_port == port) { 606 refcnt_take(&xi->xi_refcnt); 607 break; 608 } 609 } 610 mtx_leave(&sc->sc_islck); 611 return (xi); 612 } 613 614 static inline void 615 xen_intsrc_release(struct xen_softc *sc, struct xen_intsrc *xi) 616 { 617 refcnt_rele_wake(&xi->xi_refcnt); 618 } 619 620 static inline struct xen_intsrc * 621 xen_intsrc_remove(struct xen_softc *sc, evtchn_port_t port) 622 { 623 struct xen_intsrc *xi; 624 625 mtx_enter(&sc->sc_islck); 626 SLIST_FOREACH(xi, &sc->sc_intrs, xi_entry) { 627 if (xi->xi_port == port) { 628 SLIST_REMOVE(&sc->sc_intrs, xi, xen_intsrc, xi_entry); 629 break; 630 } 631 } 632 mtx_leave(&sc->sc_islck); 633 if (xi != NULL) 634 refcnt_finalize(&xi->xi_refcnt, "xenisrm"); 635 return (xi); 636 } 637 638 static inline void 639 xen_intr_mask_acquired(struct xen_softc *sc, struct xen_intsrc *xi) 640 { 641 xi->xi_masked = 1; 642 set_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0]); 643 } 644 645 static inline int 646 xen_intr_unmask_release(struct xen_softc *sc, struct xen_intsrc *xi) 647 { 648 struct evtchn_unmask eu; 649 650 xi->xi_masked = 0; 651 if (!test_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0])) { 652 xen_intsrc_release(sc, xi); 653 return (0); 654 } 655 eu.port = xi->xi_port; 656 xen_intsrc_release(sc, xi); 657 return (xen_evtchn_hypercall(sc, EVTCHNOP_unmask, &eu, sizeof(eu))); 658 } 659 660 void 661 xen_intr_ack(void) 662 { 663 struct xen_softc *sc = xen_sc; 664 struct shared_info *s = sc->sc_ipg; 665 struct cpu_info *ci = curcpu(); 666 struct vcpu_info *v = &s->vcpu_info[CPU_INFO_UNIT(ci)]; 667 668 v->evtchn_upcall_pending = 0; 669 virtio_membar_sync(); 670 } 671 672 void 673 xen_intr(void) 674 { 675 struct xen_softc *sc = xen_sc; 676 struct xen_intsrc *xi; 677 struct shared_info *s = sc->sc_ipg; 678 struct cpu_info *ci = curcpu(); 679 struct vcpu_info *v = &s->vcpu_info[CPU_INFO_UNIT(ci)]; 680 ulong pending, selector; 681 int port, bit, row; 682 683 v->evtchn_upcall_pending = 0; 684 selector = atomic_swap_ulong(&v->evtchn_pending_sel, 0); 685 686 for (row = 0; selector > 0; selector >>= 1, row++) { 687 if ((selector & 1) == 0) 688 continue; 689 if ((sc->sc_ipg->evtchn_pending[row] & 690 ~(sc->sc_ipg->evtchn_mask[row])) == 0) 691 continue; 692 pending = atomic_swap_ulong(&sc->sc_ipg->evtchn_pending[row], 693 0) & ~(sc->sc_ipg->evtchn_mask[row]); 694 for (bit = 0; pending > 0; pending >>= 1, bit++) { 695 if ((pending & 1) == 0) 696 continue; 697 port = (row * LONG_BIT) + bit; 698 if ((xi = xen_intsrc_acquire(sc, port)) == NULL) { 699 printf("%s: unhandled interrupt on port %d\n", 700 sc->sc_dev.dv_xname, port); 701 continue; 702 } 703 xi->xi_evcnt.ec_count++; 704 xen_intr_mask_acquired(sc, xi); 705 if (!task_add(xi->xi_taskq, &xi->xi_task)) 706 xen_intsrc_release(sc, xi); 707 } 708 } 709 } 710 711 void 712 xen_intr_schedule(xen_intr_handle_t xih) 713 { 714 struct xen_softc *sc = xen_sc; 715 struct xen_intsrc *xi; 716 717 if ((xi = xen_intsrc_acquire(sc, (evtchn_port_t)xih)) != NULL) { 718 xen_intr_mask_acquired(sc, xi); 719 if (!task_add(xi->xi_taskq, &xi->xi_task)) 720 xen_intsrc_release(sc, xi); 721 } 722 } 723 724 /* 725 * This code achieves two goals: 1) makes sure that *after* masking 726 * the interrupt source we're not getting more task_adds: intr_barrier 727 * will take care of that, and 2) makes sure that the interrupt task 728 * has finished executing the current task and won't be called again: 729 * it sets up a barrier task to await completion of the current task 730 * and relies on the interrupt masking to prevent submission of new 731 * tasks in the future. 732 */ 733 void 734 xen_intr_barrier(xen_intr_handle_t xih) 735 { 736 struct xen_softc *sc = xen_sc; 737 struct xen_intsrc *xi; 738 739 /* 740 * XXX This will need to be revised once intr_barrier starts 741 * using its argument. 742 */ 743 intr_barrier(NULL); 744 745 if ((xi = xen_intsrc_acquire(sc, (evtchn_port_t)xih)) != NULL) { 746 taskq_barrier(xi->xi_taskq); 747 xen_intsrc_release(sc, xi); 748 } 749 } 750 751 void 752 xen_intr_signal(xen_intr_handle_t xih) 753 { 754 struct xen_softc *sc = xen_sc; 755 struct xen_intsrc *xi; 756 struct evtchn_send es; 757 758 if ((xi = xen_intsrc_acquire(sc, (evtchn_port_t)xih)) != NULL) { 759 es.port = xi->xi_port; 760 xen_intsrc_release(sc, xi); 761 xen_evtchn_hypercall(sc, EVTCHNOP_send, &es, sizeof(es)); 762 } 763 } 764 765 int 766 xen_intr_establish(evtchn_port_t port, xen_intr_handle_t *xih, int domain, 767 void (*handler)(void *), void *arg, char *name) 768 { 769 struct xen_softc *sc = xen_sc; 770 struct xen_intsrc *xi; 771 struct evtchn_alloc_unbound eau; 772 #ifdef notyet 773 struct evtchn_bind_vcpu ebv; 774 #endif 775 #if defined(XEN_DEBUG) && disabled 776 struct evtchn_status es; 777 #endif 778 779 if (port && (xi = xen_intsrc_acquire(sc, port)) != NULL) { 780 xen_intsrc_release(sc, xi); 781 DPRINTF("%s: interrupt handler has already been established " 782 "for port %u\n", sc->sc_dev.dv_xname, port); 783 return (-1); 784 } 785 786 xi = malloc(sizeof(*xi), M_DEVBUF, M_NOWAIT | M_ZERO); 787 if (xi == NULL) 788 return (-1); 789 790 xi->xi_port = (evtchn_port_t)*xih; 791 792 xi->xi_handler = handler; 793 xi->xi_ctx = arg; 794 795 xi->xi_taskq = taskq_create(name, 1, IPL_NET, TASKQ_MPSAFE); 796 if (!xi->xi_taskq) { 797 printf("%s: failed to create interrupt task for %s\n", 798 sc->sc_dev.dv_xname, name); 799 free(xi, M_DEVBUF, sizeof(*xi)); 800 return (-1); 801 } 802 task_set(&xi->xi_task, xen_intr_dispatch, xi); 803 804 if (port == 0) { 805 /* We're being asked to allocate a new event port */ 806 memset(&eau, 0, sizeof(eau)); 807 eau.dom = DOMID_SELF; 808 eau.remote_dom = domain; 809 if (xen_evtchn_hypercall(sc, EVTCHNOP_alloc_unbound, &eau, 810 sizeof(eau)) != 0) { 811 DPRINTF("%s: failed to allocate new event port\n", 812 sc->sc_dev.dv_xname); 813 free(xi, M_DEVBUF, sizeof(*xi)); 814 return (-1); 815 } 816 *xih = xi->xi_port = eau.port; 817 } else { 818 *xih = xi->xi_port = port; 819 /* 820 * The Event Channel API didn't open this port, so it is not 821 * responsible for closing it automatically on unbind. 822 */ 823 xi->xi_noclose = 1; 824 } 825 826 #ifdef notyet 827 /* Bind interrupt to VCPU#0 */ 828 memset(&ebv, 0, sizeof(ebv)); 829 ebv.port = xi->xi_port; 830 ebv.vcpu = 0; 831 if (xen_evtchn_hypercall(sc, EVTCHNOP_bind_vcpu, &ebv, sizeof(ebv))) { 832 printf("%s: failed to bind interrupt on port %u to vcpu%d\n", 833 sc->sc_dev.dv_xname, ebv.port, ebv.vcpu); 834 } 835 #endif 836 837 evcount_attach(&xi->xi_evcnt, name, &sc->sc_irq); 838 839 xen_intsrc_add(sc, xi); 840 841 /* Mask the event port */ 842 set_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0]); 843 844 #if defined(XEN_DEBUG) && disabled 845 memset(&es, 0, sizeof(es)); 846 es.dom = DOMID_SELF; 847 es.port = xi->xi_port; 848 if (xen_evtchn_hypercall(sc, EVTCHNOP_status, &es, sizeof(es))) { 849 printf("%s: failed to obtain status for port %d\n", 850 sc->sc_dev.dv_xname, es.port); 851 } 852 printf("%s: port %u bound to vcpu%u", sc->sc_dev.dv_xname, 853 es.port, es.vcpu); 854 if (es.status == EVTCHNSTAT_interdomain) 855 printf(": domain %d port %u\n", es.u.interdomain.dom, 856 es.u.interdomain.port); 857 else if (es.status == EVTCHNSTAT_unbound) 858 printf(": domain %d\n", es.u.unbound.dom); 859 else if (es.status == EVTCHNSTAT_pirq) 860 printf(": pirq %u\n", es.u.pirq); 861 else if (es.status == EVTCHNSTAT_virq) 862 printf(": virq %u\n", es.u.virq); 863 else 864 printf("\n"); 865 #endif 866 867 return (0); 868 } 869 870 int 871 xen_intr_disestablish(xen_intr_handle_t xih) 872 { 873 struct xen_softc *sc = xen_sc; 874 evtchn_port_t port = (evtchn_port_t)xih; 875 struct evtchn_close ec; 876 struct xen_intsrc *xi; 877 878 if ((xi = xen_intsrc_remove(sc, port)) == NULL) 879 return (-1); 880 881 evcount_detach(&xi->xi_evcnt); 882 883 taskq_destroy(xi->xi_taskq); 884 885 set_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0]); 886 clear_bit(xi->xi_port, &sc->sc_ipg->evtchn_pending[0]); 887 888 if (!xi->xi_noclose) { 889 ec.port = xi->xi_port; 890 if (xen_evtchn_hypercall(sc, EVTCHNOP_close, &ec, sizeof(ec))) { 891 DPRINTF("%s: failed to close event port %u\n", 892 sc->sc_dev.dv_xname, xi->xi_port); 893 } 894 } 895 896 free(xi, M_DEVBUF, sizeof(*xi)); 897 return (0); 898 } 899 900 void 901 xen_intr_dispatch(void *arg) 902 { 903 struct xen_softc *sc = xen_sc; 904 struct xen_intsrc *xi = arg; 905 906 if (xi->xi_handler) 907 xi->xi_handler(xi->xi_ctx); 908 909 xen_intr_unmask_release(sc, xi); 910 } 911 912 void 913 xen_intr_enable(void) 914 { 915 struct xen_softc *sc = xen_sc; 916 struct xen_intsrc *xi; 917 struct evtchn_unmask eu; 918 919 mtx_enter(&sc->sc_islck); 920 SLIST_FOREACH(xi, &sc->sc_intrs, xi_entry) { 921 if (!xi->xi_masked) { 922 eu.port = xi->xi_port; 923 if (xen_evtchn_hypercall(sc, EVTCHNOP_unmask, &eu, 924 sizeof(eu))) 925 printf("%s: unmasking port %u failed\n", 926 sc->sc_dev.dv_xname, xi->xi_port); 927 virtio_membar_sync(); 928 if (test_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0])) 929 printf("%s: port %u is still masked\n", 930 sc->sc_dev.dv_xname, xi->xi_port); 931 } 932 } 933 mtx_leave(&sc->sc_islck); 934 } 935 936 void 937 xen_intr_mask(xen_intr_handle_t xih) 938 { 939 struct xen_softc *sc = xen_sc; 940 evtchn_port_t port = (evtchn_port_t)xih; 941 struct xen_intsrc *xi; 942 943 if ((xi = xen_intsrc_acquire(sc, port)) != NULL) { 944 xen_intr_mask_acquired(sc, xi); 945 xen_intsrc_release(sc, xi); 946 } 947 } 948 949 int 950 xen_intr_unmask(xen_intr_handle_t xih) 951 { 952 struct xen_softc *sc = xen_sc; 953 evtchn_port_t port = (evtchn_port_t)xih; 954 struct xen_intsrc *xi; 955 956 if ((xi = xen_intsrc_acquire(sc, port)) != NULL) 957 return (xen_intr_unmask_release(sc, xi)); 958 959 return (0); 960 } 961 962 int 963 xen_init_grant_tables(struct xen_softc *sc) 964 { 965 struct gnttab_query_size gqs; 966 967 gqs.dom = DOMID_SELF; 968 if (xen_hypercall(sc, XC_GNTTAB, 3, GNTTABOP_query_size, &gqs, 1)) { 969 printf(": failed the query for grant table pages\n"); 970 return (-1); 971 } 972 if (gqs.nr_frames == 0 || gqs.nr_frames > gqs.max_nr_frames) { 973 printf(": invalid number of grant table pages: %u/%u\n", 974 gqs.nr_frames, gqs.max_nr_frames); 975 return (-1); 976 } 977 978 sc->sc_gntmax = gqs.max_nr_frames; 979 980 sc->sc_gnt = mallocarray(sc->sc_gntmax + 1, sizeof(struct xen_gntent), 981 M_DEVBUF, M_ZERO | M_NOWAIT); 982 if (sc->sc_gnt == NULL) { 983 printf(": failed to allocate grant table lookup table\n"); 984 return (-1); 985 } 986 987 mtx_init(&sc->sc_gntlck, IPL_NET); 988 989 if (xen_grant_table_grow(sc) == NULL) { 990 free(sc->sc_gnt, M_DEVBUF, sc->sc_gntmax * 991 sizeof(struct xen_gntent)); 992 return (-1); 993 } 994 995 printf(", %d grant table frames", sc->sc_gntmax); 996 997 xen_bus_dma_tag._cookie = sc; 998 999 return (0); 1000 } 1001 1002 struct xen_gntent * 1003 xen_grant_table_grow(struct xen_softc *sc) 1004 { 1005 struct xen_add_to_physmap xatp; 1006 struct xen_gntent *ge; 1007 void *va; 1008 paddr_t pa; 1009 1010 if (sc->sc_gntcnt == sc->sc_gntmax) { 1011 printf("%s: grant table frame allotment limit reached\n", 1012 sc->sc_dev.dv_xname); 1013 return (NULL); 1014 } 1015 1016 va = km_alloc(PAGE_SIZE, &kv_any, &kp_zero, &kd_nowait); 1017 if (va == NULL) 1018 return (NULL); 1019 if (!pmap_extract(pmap_kernel(), (vaddr_t)va, &pa)) { 1020 printf("%s: grant table page PA extraction failed\n", 1021 sc->sc_dev.dv_xname); 1022 km_free(va, PAGE_SIZE, &kv_any, &kp_zero); 1023 return (NULL); 1024 } 1025 1026 mtx_enter(&sc->sc_gntlck); 1027 1028 ge = &sc->sc_gnt[sc->sc_gntcnt]; 1029 ge->ge_table = va; 1030 1031 xatp.domid = DOMID_SELF; 1032 xatp.idx = sc->sc_gntcnt; 1033 xatp.space = XENMAPSPACE_grant_table; 1034 xatp.gpfn = atop(pa); 1035 if (xen_hypercall(sc, XC_MEMORY, 2, XENMEM_add_to_physmap, &xatp)) { 1036 printf("%s: failed to add a grant table page\n", 1037 sc->sc_dev.dv_xname); 1038 km_free(ge->ge_table, PAGE_SIZE, &kv_any, &kp_zero); 1039 mtx_leave(&sc->sc_gntlck); 1040 return (NULL); 1041 } 1042 ge->ge_start = sc->sc_gntcnt * GNTTAB_NEPG; 1043 /* First page has 8 reserved entries */ 1044 ge->ge_reserved = ge->ge_start == 0 ? GNTTAB_NR_RESERVED_ENTRIES : 0; 1045 ge->ge_free = GNTTAB_NEPG - ge->ge_reserved; 1046 ge->ge_next = ge->ge_reserved; 1047 mtx_init(&ge->ge_lock, IPL_NET); 1048 1049 sc->sc_gntcnt++; 1050 mtx_leave(&sc->sc_gntlck); 1051 1052 return (ge); 1053 } 1054 1055 int 1056 xen_grant_table_alloc(struct xen_softc *sc, grant_ref_t *ref) 1057 { 1058 struct xen_gntent *ge; 1059 int i; 1060 1061 /* Start with a previously allocated table page */ 1062 ge = &sc->sc_gnt[sc->sc_gntcnt - 1]; 1063 if (ge->ge_free > 0) { 1064 mtx_enter(&ge->ge_lock); 1065 if (ge->ge_free > 0) 1066 goto search; 1067 mtx_leave(&ge->ge_lock); 1068 } 1069 1070 /* Try other existing table pages */ 1071 for (i = 0; i < sc->sc_gntcnt; i++) { 1072 ge = &sc->sc_gnt[i]; 1073 if (ge->ge_free == 0) 1074 continue; 1075 mtx_enter(&ge->ge_lock); 1076 if (ge->ge_free > 0) 1077 goto search; 1078 mtx_leave(&ge->ge_lock); 1079 } 1080 1081 alloc: 1082 /* Allocate a new table page */ 1083 if ((ge = xen_grant_table_grow(sc)) == NULL) 1084 return (-1); 1085 1086 mtx_enter(&ge->ge_lock); 1087 if (ge->ge_free == 0) { 1088 /* We were not fast enough... */ 1089 mtx_leave(&ge->ge_lock); 1090 goto alloc; 1091 } 1092 1093 search: 1094 for (i = ge->ge_next; 1095 /* Math works here because GNTTAB_NEPG is a power of 2 */ 1096 i != ((ge->ge_next + GNTTAB_NEPG - 1) & (GNTTAB_NEPG - 1)); 1097 i++) { 1098 if (i == GNTTAB_NEPG) 1099 i = 0; 1100 if (ge->ge_reserved && i < ge->ge_reserved) 1101 continue; 1102 if (ge->ge_table[i].frame != 0) 1103 continue; 1104 *ref = ge->ge_start + i; 1105 ge->ge_table[i].flags = GTF_invalid; 1106 ge->ge_table[i].frame = 0xffffffff; /* Mark as taken */ 1107 if ((ge->ge_next = i + 1) == GNTTAB_NEPG) 1108 ge->ge_next = ge->ge_reserved; 1109 ge->ge_free--; 1110 mtx_leave(&ge->ge_lock); 1111 return (0); 1112 } 1113 mtx_leave(&ge->ge_lock); 1114 1115 panic("page full, sc %p gnt %p (%d) ge %p", sc, sc->sc_gnt, 1116 sc->sc_gntcnt, ge); 1117 return (-1); 1118 } 1119 1120 void 1121 xen_grant_table_free(struct xen_softc *sc, grant_ref_t ref) 1122 { 1123 struct xen_gntent *ge; 1124 1125 #ifdef XEN_DEBUG 1126 if (ref > sc->sc_gntcnt * GNTTAB_NEPG) 1127 panic("unmanaged ref %u sc %p gnt %p (%d)", ref, sc, 1128 sc->sc_gnt, sc->sc_gntcnt); 1129 #endif 1130 ge = &sc->sc_gnt[ref / GNTTAB_NEPG]; 1131 mtx_enter(&ge->ge_lock); 1132 #ifdef XEN_DEBUG 1133 if (ref < ge->ge_start || ref > ge->ge_start + GNTTAB_NEPG) { 1134 mtx_leave(&ge->ge_lock); 1135 panic("out of bounds ref %u ge %p start %u sc %p gnt %p", 1136 ref, ge, ge->ge_start, sc, sc->sc_gnt); 1137 } 1138 #endif 1139 ref -= ge->ge_start; 1140 if (ge->ge_table[ref].flags != GTF_invalid) { 1141 mtx_leave(&ge->ge_lock); 1142 panic("reference %u is still in use, flags %#x frame %#x", 1143 ref + ge->ge_start, ge->ge_table[ref].flags, 1144 ge->ge_table[ref].frame); 1145 } 1146 ge->ge_table[ref].frame = 0; 1147 ge->ge_next = ref; 1148 ge->ge_free++; 1149 mtx_leave(&ge->ge_lock); 1150 } 1151 1152 void 1153 xen_grant_table_enter(struct xen_softc *sc, grant_ref_t ref, paddr_t pa, 1154 int domain, int flags) 1155 { 1156 struct xen_gntent *ge; 1157 1158 #ifdef XEN_DEBUG 1159 if (ref > sc->sc_gntcnt * GNTTAB_NEPG) 1160 panic("unmanaged ref %u sc %p gnt %p (%d)", ref, sc, 1161 sc->sc_gnt, sc->sc_gntcnt); 1162 #endif 1163 ge = &sc->sc_gnt[ref / GNTTAB_NEPG]; 1164 #ifdef XEN_DEBUG 1165 if (ref < ge->ge_start || ref > ge->ge_start + GNTTAB_NEPG) { 1166 panic("out of bounds ref %u ge %p start %u sc %p gnt %p", 1167 ref, ge, ge->ge_start, sc, sc->sc_gnt); 1168 } 1169 #endif 1170 ref -= ge->ge_start; 1171 if (ge->ge_table[ref].flags != GTF_invalid) { 1172 panic("reference %u is still in use, flags %#x frame %#x", 1173 ref + ge->ge_start, ge->ge_table[ref].flags, 1174 ge->ge_table[ref].frame); 1175 } 1176 ge->ge_table[ref].frame = atop(pa); 1177 ge->ge_table[ref].domid = domain; 1178 virtio_membar_sync(); 1179 ge->ge_table[ref].flags = GTF_permit_access | flags; 1180 virtio_membar_sync(); 1181 } 1182 1183 void 1184 xen_grant_table_remove(struct xen_softc *sc, grant_ref_t ref) 1185 { 1186 struct xen_gntent *ge; 1187 uint32_t flags, *ptr; 1188 int loop; 1189 1190 #ifdef XEN_DEBUG 1191 if (ref > sc->sc_gntcnt * GNTTAB_NEPG) 1192 panic("unmanaged ref %u sc %p gnt %p (%d)", ref, sc, 1193 sc->sc_gnt, sc->sc_gntcnt); 1194 #endif 1195 ge = &sc->sc_gnt[ref / GNTTAB_NEPG]; 1196 #ifdef XEN_DEBUG 1197 if (ref < ge->ge_start || ref > ge->ge_start + GNTTAB_NEPG) { 1198 panic("out of bounds ref %u ge %p start %u sc %p gnt %p", 1199 ref, ge, ge->ge_start, sc, sc->sc_gnt); 1200 } 1201 #endif 1202 ref -= ge->ge_start; 1203 /* Invalidate the grant reference */ 1204 virtio_membar_sync(); 1205 ptr = (uint32_t *)&ge->ge_table[ref]; 1206 flags = (ge->ge_table[ref].flags & ~(GTF_reading|GTF_writing)) | 1207 (ge->ge_table[ref].domid << 16); 1208 loop = 0; 1209 while (atomic_cas_uint(ptr, flags, GTF_invalid) != flags) { 1210 if (loop++ > 10) { 1211 panic("grant table reference %u is held " 1212 "by domain %d: frame %#x flags %#x", 1213 ref + ge->ge_start, ge->ge_table[ref].domid, 1214 ge->ge_table[ref].frame, ge->ge_table[ref].flags); 1215 } 1216 #if (defined(__amd64__) || defined(__i386__)) 1217 __asm volatile("pause": : : "memory"); 1218 #endif 1219 } 1220 ge->ge_table[ref].frame = 0xffffffff; 1221 } 1222 1223 int 1224 xen_bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, 1225 bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp) 1226 { 1227 struct xen_softc *sc = t->_cookie; 1228 struct xen_gntmap *gm; 1229 int i, error; 1230 1231 if (maxsegsz < PAGE_SIZE) 1232 return (EINVAL); 1233 1234 /* Allocate a dma map structure */ 1235 error = bus_dmamap_create(sc->sc_dmat, size, nsegments, maxsegsz, 1236 boundary, flags, dmamp); 1237 if (error) 1238 return (error); 1239 /* Allocate an array of grant table pa<->ref maps */ 1240 gm = mallocarray(nsegments, sizeof(struct xen_gntmap), M_DEVBUF, 1241 M_ZERO | ((flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK)); 1242 if (gm == NULL) { 1243 bus_dmamap_destroy(sc->sc_dmat, *dmamp); 1244 *dmamp = NULL; 1245 return (ENOMEM); 1246 } 1247 /* Wire it to the dma map */ 1248 (*dmamp)->_dm_cookie = gm; 1249 /* Claim references from the grant table */ 1250 for (i = 0; i < (*dmamp)->_dm_segcnt; i++) { 1251 if (xen_grant_table_alloc(sc, &gm[i].gm_ref)) { 1252 xen_bus_dmamap_destroy(t, *dmamp); 1253 *dmamp = NULL; 1254 return (ENOBUFS); 1255 } 1256 } 1257 return (0); 1258 } 1259 1260 void 1261 xen_bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map) 1262 { 1263 struct xen_softc *sc = t->_cookie; 1264 struct xen_gntmap *gm; 1265 int i; 1266 1267 gm = map->_dm_cookie; 1268 for (i = 0; i < map->_dm_segcnt; i++) { 1269 if (gm[i].gm_ref == 0) 1270 continue; 1271 xen_grant_table_free(sc, gm[i].gm_ref); 1272 } 1273 free(gm, M_DEVBUF, map->_dm_segcnt * sizeof(struct xen_gntmap)); 1274 bus_dmamap_destroy(sc->sc_dmat, map); 1275 } 1276 1277 int 1278 xen_bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 1279 bus_size_t buflen, struct proc *p, int flags) 1280 { 1281 struct xen_softc *sc = t->_cookie; 1282 struct xen_gntmap *gm = map->_dm_cookie; 1283 int i, domain, error; 1284 1285 domain = flags >> 16; 1286 flags &= 0xffff; 1287 error = bus_dmamap_load(sc->sc_dmat, map, buf, buflen, p, flags); 1288 if (error) 1289 return (error); 1290 for (i = 0; i < map->dm_nsegs; i++) { 1291 xen_grant_table_enter(sc, gm[i].gm_ref, map->dm_segs[i].ds_addr, 1292 domain, flags & BUS_DMA_WRITE ? GTF_readonly : 0); 1293 gm[i].gm_paddr = map->dm_segs[i].ds_addr; 1294 map->dm_segs[i].ds_addr = gm[i].gm_ref; 1295 } 1296 return (0); 1297 } 1298 1299 int 1300 xen_bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, 1301 int flags) 1302 { 1303 struct xen_softc *sc = t->_cookie; 1304 struct xen_gntmap *gm = map->_dm_cookie; 1305 int i, domain, error; 1306 1307 domain = flags >> 16; 1308 flags &= 0xffff; 1309 error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m0, flags); 1310 if (error) 1311 return (error); 1312 for (i = 0; i < map->dm_nsegs; i++) { 1313 xen_grant_table_enter(sc, gm[i].gm_ref, map->dm_segs[i].ds_addr, 1314 domain, flags & BUS_DMA_WRITE ? GTF_readonly : 0); 1315 gm[i].gm_paddr = map->dm_segs[i].ds_addr; 1316 map->dm_segs[i].ds_addr = gm[i].gm_ref; 1317 } 1318 return (0); 1319 } 1320 1321 void 1322 xen_bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) 1323 { 1324 struct xen_softc *sc = t->_cookie; 1325 struct xen_gntmap *gm = map->_dm_cookie; 1326 int i; 1327 1328 for (i = 0; i < map->dm_nsegs; i++) { 1329 if (gm[i].gm_paddr == 0) 1330 continue; 1331 xen_grant_table_remove(sc, gm[i].gm_ref); 1332 map->dm_segs[i].ds_addr = gm[i].gm_paddr; 1333 gm[i].gm_paddr = 0; 1334 } 1335 bus_dmamap_unload(sc->sc_dmat, map); 1336 } 1337 1338 void 1339 xen_bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr, 1340 bus_size_t size, int op) 1341 { 1342 if ((op == (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) || 1343 (op == (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE))) 1344 virtio_membar_sync(); 1345 } 1346 1347 static int 1348 xen_attach_print(void *aux, const char *name) 1349 { 1350 struct xen_attach_args *xa = aux; 1351 1352 if (name) 1353 printf("\"%s\" at %s: %s", xa->xa_name, name, xa->xa_node); 1354 1355 return (UNCONF); 1356 } 1357 1358 int 1359 xen_attach_device(struct xen_softc *sc, struct xen_devlist *xdl, 1360 const char *name, const char *unit) 1361 { 1362 struct xen_attach_args xa; 1363 struct xen_device *xdv; 1364 unsigned long long res; 1365 1366 xa.xa_dmat = &xen_bus_dma_tag; 1367 1368 strlcpy(xa.xa_name, name, sizeof(xa.xa_name)); 1369 snprintf(xa.xa_node, sizeof(xa.xa_node), "device/%s/%s", name, unit); 1370 1371 if (xs_getprop(sc, xa.xa_node, "backend", xa.xa_backend, 1372 sizeof(xa.xa_backend))) { 1373 DPRINTF("%s: failed to identify \"backend\" for " 1374 "\"%s\"\n", sc->sc_dev.dv_xname, xa.xa_node); 1375 return (EIO); 1376 } 1377 1378 if (xs_getnum(sc, xa.xa_node, "backend-id", &res) || res > UINT16_MAX) { 1379 DPRINTF("%s: invalid \"backend-id\" for \"%s\"\n", 1380 sc->sc_dev.dv_xname, xa.xa_node); 1381 return (EIO); 1382 } 1383 xa.xa_domid = (uint16_t)res; 1384 1385 xdv = malloc(sizeof(struct xen_device), M_DEVBUF, M_ZERO | M_NOWAIT); 1386 if (xdv == NULL) 1387 return (ENOMEM); 1388 1389 strlcpy(xdv->dv_unit, unit, sizeof(xdv->dv_unit)); 1390 LIST_INSERT_HEAD(&xdl->dl_devs, xdv, dv_entry); 1391 1392 xdv->dv_dev = config_found((struct device *)sc, &xa, xen_attach_print); 1393 1394 return (0); 1395 } 1396 1397 int 1398 xen_probe_devices(struct xen_softc *sc) 1399 { 1400 struct xen_devlist *xdl; 1401 struct xs_transaction xst; 1402 struct iovec *iovp1 = NULL, *iovp2 = NULL; 1403 int i, j, error, iov1_cnt = 0, iov2_cnt = 0; 1404 char path[256]; 1405 1406 memset(&xst, 0, sizeof(xst)); 1407 xst.xst_id = 0; 1408 xst.xst_cookie = sc->sc_xs; 1409 1410 if ((error = xs_cmd(&xst, XS_LIST, "device", &iovp1, &iov1_cnt)) != 0) 1411 return (error); 1412 1413 for (i = 0; i < iov1_cnt; i++) { 1414 if (strcmp("suspend", (char *)iovp1[i].iov_base) == 0) 1415 continue; 1416 snprintf(path, sizeof(path), "device/%s", 1417 (char *)iovp1[i].iov_base); 1418 if ((error = xs_cmd(&xst, XS_LIST, path, &iovp2, 1419 &iov2_cnt)) != 0) 1420 goto out; 1421 if ((xdl = malloc(sizeof(struct xen_devlist), M_DEVBUF, 1422 M_ZERO | M_NOWAIT)) == NULL) { 1423 error = ENOMEM; 1424 goto out; 1425 } 1426 xdl->dl_xen = sc; 1427 strlcpy(xdl->dl_node, (const char *)iovp1[i].iov_base, 1428 XEN_MAX_NODE_LEN); 1429 for (j = 0; j < iov2_cnt; j++) { 1430 error = xen_attach_device(sc, xdl, 1431 (const char *)iovp1[i].iov_base, 1432 (const char *)iovp2[j].iov_base); 1433 if (error) { 1434 printf("%s: failed to attach \"%s/%s\"\n", 1435 sc->sc_dev.dv_xname, path, 1436 (const char *)iovp2[j].iov_base); 1437 goto out; 1438 } 1439 } 1440 /* Setup a watch for every device subtree */ 1441 if (xs_watch(sc, "device", (char *)iovp1[i].iov_base, 1442 &xdl->dl_task, xen_hotplug, xdl)) 1443 printf("%s: failed to setup hotplug watch for \"%s\"\n", 1444 sc->sc_dev.dv_xname, (char *)iovp1[i].iov_base); 1445 SLIST_INSERT_HEAD(&sc->sc_devlists, xdl, dl_entry); 1446 xs_resfree(&xst, iovp2, iov2_cnt); 1447 iovp2 = NULL; 1448 iov2_cnt = 0; 1449 } 1450 1451 out: 1452 if (iovp2) 1453 xs_resfree(&xst, iovp2, iov2_cnt); 1454 xs_resfree(&xst, iovp1, iov1_cnt); 1455 return (error); 1456 } 1457 1458 void 1459 xen_hotplug(void *arg) 1460 { 1461 struct xen_devlist *xdl = arg; 1462 struct xen_softc *sc = xdl->dl_xen; 1463 struct xen_device *xdv, *xvdn; 1464 struct xs_transaction xst; 1465 struct iovec *iovp = NULL; 1466 int error, i, keep, iov_cnt = 0; 1467 char path[256]; 1468 int8_t *seen; 1469 1470 memset(&xst, 0, sizeof(xst)); 1471 xst.xst_id = 0; 1472 xst.xst_cookie = sc->sc_xs; 1473 1474 snprintf(path, sizeof(path), "device/%s", xdl->dl_node); 1475 if ((error = xs_cmd(&xst, XS_LIST, path, &iovp, &iov_cnt)) != 0) 1476 return; 1477 1478 seen = malloc(iov_cnt, M_TEMP, M_ZERO | M_WAITOK); 1479 1480 /* Detect all removed and kept devices */ 1481 LIST_FOREACH_SAFE(xdv, &xdl->dl_devs, dv_entry, xvdn) { 1482 for (i = 0, keep = 0; i < iov_cnt; i++) { 1483 if (!seen[i] && 1484 !strcmp(xdv->dv_unit, (char *)iovp[i].iov_base)) { 1485 seen[i]++; 1486 keep++; 1487 break; 1488 } 1489 } 1490 if (!keep) { 1491 DPRINTF("%s: removing \"%s/%s\"\n", sc->sc_dev.dv_xname, 1492 xdl->dl_node, xdv->dv_unit); 1493 LIST_REMOVE(xdv, dv_entry); 1494 config_detach(xdv->dv_dev, 0); 1495 free(xdv, M_DEVBUF, sizeof(struct xen_device)); 1496 } 1497 } 1498 1499 /* Attach all new devices */ 1500 for (i = 0; i < iov_cnt; i++) { 1501 if (seen[i]) 1502 continue; 1503 DPRINTF("%s: attaching \"%s/%s\"\n", sc->sc_dev.dv_xname, 1504 xdl->dl_node, (const char *)iovp[i].iov_base); 1505 error = xen_attach_device(sc, xdl, xdl->dl_node, 1506 (const char *)iovp[i].iov_base); 1507 if (error) { 1508 printf("%s: failed to attach \"%s/%s\"\n", 1509 sc->sc_dev.dv_xname, path, 1510 (const char *)iovp[i].iov_base); 1511 continue; 1512 } 1513 } 1514 1515 free(seen, M_TEMP, iov_cnt); 1516 1517 xs_resfree(&xst, iovp, iov_cnt); 1518 } 1519 1520 #include <machine/pio.h> 1521 1522 #define XMI_PORT 0x10 1523 #define XMI_MAGIC 0x49d2 1524 #define XMI_UNPLUG_IDE 0x01 1525 #define XMI_UNPLUG_NIC 0x02 1526 #define XMI_UNPLUG_IDESEC 0x04 1527 1528 void 1529 xen_disable_emulated_devices(struct xen_softc *sc) 1530 { 1531 #if defined(__i386__) || defined(__amd64__) 1532 ushort unplug = 0; 1533 1534 if (inw(XMI_PORT) != XMI_MAGIC) { 1535 printf("%s: failed to disable emulated devices\n", 1536 sc->sc_dev.dv_xname); 1537 return; 1538 } 1539 if (sc->sc_unplug & XEN_UNPLUG_IDE) 1540 unplug |= XMI_UNPLUG_IDE; 1541 if (sc->sc_unplug & XEN_UNPLUG_IDESEC) 1542 unplug |= XMI_UNPLUG_IDESEC; 1543 if (sc->sc_unplug & XEN_UNPLUG_NIC) 1544 unplug |= XMI_UNPLUG_NIC; 1545 if (unplug) 1546 outw(XMI_PORT, unplug); 1547 #endif /* __i386__ || __amd64__ */ 1548 } 1549 1550 void 1551 xen_unplug_emulated(void *xsc, int what) 1552 { 1553 struct xen_softc *sc = xsc; 1554 1555 sc->sc_unplug |= what; 1556 } 1557