1 /* $OpenBSD: xen.c,v 1.100 2024/11/27 02:38:35 jsg Exp $ */ 2 3 /* 4 * Copyright (c) 2015, 2016, 2017 Mike Belopuhov 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/param.h> 20 21 /* Xen requires locked atomic operations */ 22 #ifndef MULTIPROCESSOR 23 #define _XENMPATOMICS 24 #define MULTIPROCESSOR 25 #endif 26 #include <sys/atomic.h> 27 #ifdef _XENMPATOMICS 28 #undef MULTIPROCESSOR 29 #undef _XENMPATOMICS 30 #endif 31 32 #include <sys/systm.h> 33 #include <sys/proc.h> 34 #include <sys/refcnt.h> 35 #include <sys/malloc.h> 36 #include <sys/stdint.h> 37 #include <sys/device.h> 38 #include <sys/task.h> 39 40 #include <machine/bus.h> 41 #include <machine/cpu.h> 42 #include <machine/cpufunc.h> 43 44 #include <uvm/uvm_extern.h> 45 46 #include <machine/i82489var.h> 47 48 #include <dev/pv/pvvar.h> 49 #include <dev/pv/pvreg.h> 50 #include <dev/pv/xenreg.h> 51 #include <dev/pv/xenvar.h> 52 53 /* #define XEN_DEBUG */ 54 55 #ifdef XEN_DEBUG 56 #define DPRINTF(x...) printf(x) 57 #else 58 #define DPRINTF(x...) 59 #endif 60 61 struct xen_softc *xen_sc; 62 63 int xen_init_hypercall(struct xen_softc *); 64 int xen_getfeatures(struct xen_softc *); 65 int xen_init_info_page(struct xen_softc *); 66 int xen_init_cbvec(struct xen_softc *); 67 int xen_init_interrupts(struct xen_softc *); 68 void xen_intr_dispatch(void *); 69 int xen_init_grant_tables(struct xen_softc *); 70 struct xen_gntent * 71 xen_grant_table_grow(struct xen_softc *); 72 int xen_grant_table_alloc(struct xen_softc *, grant_ref_t *); 73 void xen_grant_table_free(struct xen_softc *, grant_ref_t); 74 void xen_grant_table_enter(struct xen_softc *, grant_ref_t, paddr_t, 75 int, int); 76 void xen_grant_table_remove(struct xen_softc *, grant_ref_t); 77 void xen_disable_emulated_devices(struct xen_softc *); 78 79 int xen_match(struct device *, void *, void *); 80 void xen_attach(struct device *, struct device *, void *); 81 void xen_deferred(struct device *); 82 void xen_control(void *); 83 void xen_hotplug(void *); 84 void xen_resume(struct device *); 85 int xen_activate(struct device *, int); 86 int xen_attach_device(struct xen_softc *, struct xen_devlist *, 87 const char *, const char *); 88 int xen_probe_devices(struct xen_softc *); 89 90 int xen_bus_dmamap_create(bus_dma_tag_t, bus_size_t, int, bus_size_t, 91 bus_size_t, int, bus_dmamap_t *); 92 void xen_bus_dmamap_destroy(bus_dma_tag_t, bus_dmamap_t); 93 int xen_bus_dmamap_load(bus_dma_tag_t, bus_dmamap_t, void *, bus_size_t, 94 struct proc *, int); 95 int xen_bus_dmamap_load_mbuf(bus_dma_tag_t, bus_dmamap_t, struct mbuf *, 96 int); 97 void xen_bus_dmamap_unload(bus_dma_tag_t, bus_dmamap_t); 98 void xen_bus_dmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t, 99 bus_size_t, int); 100 101 int xs_attach(struct xen_softc *); 102 103 struct cfdriver xen_cd = { 104 NULL, "xen", DV_DULL 105 }; 106 107 const struct cfattach xen_ca = { 108 sizeof(struct xen_softc), xen_match, xen_attach, NULL, xen_activate 109 }; 110 111 struct bus_dma_tag xen_bus_dma_tag = { 112 NULL, 113 xen_bus_dmamap_create, 114 xen_bus_dmamap_destroy, 115 xen_bus_dmamap_load, 116 xen_bus_dmamap_load_mbuf, 117 NULL, 118 NULL, 119 xen_bus_dmamap_unload, 120 xen_bus_dmamap_sync, 121 _bus_dmamem_alloc, 122 NULL, 123 _bus_dmamem_free, 124 _bus_dmamem_map, 125 _bus_dmamem_unmap, 126 NULL, 127 }; 128 129 int 130 xen_match(struct device *parent, void *match, void *aux) 131 { 132 struct pv_attach_args *pva = aux; 133 struct pvbus_hv *hv = &pva->pva_hv[PVBUS_XEN]; 134 135 if (hv->hv_base == 0) 136 return (0); 137 138 return (1); 139 } 140 141 void 142 xen_attach(struct device *parent, struct device *self, void *aux) 143 { 144 struct pv_attach_args *pva = (struct pv_attach_args *)aux; 145 struct pvbus_hv *hv = &pva->pva_hv[PVBUS_XEN]; 146 struct xen_softc *sc = (struct xen_softc *)self; 147 148 sc->sc_base = hv->hv_base; 149 sc->sc_dmat = pva->pva_dmat; 150 151 if (xen_init_hypercall(sc)) 152 return; 153 154 /* Wire it up to the global */ 155 xen_sc = sc; 156 157 if (xen_getfeatures(sc)) 158 return; 159 160 if (xen_init_info_page(sc)) 161 return; 162 163 xen_init_cbvec(sc); 164 165 if (xen_init_interrupts(sc)) 166 return; 167 168 if (xen_init_grant_tables(sc)) 169 return; 170 171 if (xs_attach(sc)) 172 return; 173 174 xen_probe_devices(sc); 175 176 /* pvbus(4) key/value interface */ 177 hv->hv_kvop = xs_kvop; 178 hv->hv_arg = sc; 179 180 xen_disable_emulated_devices(sc); 181 182 config_mountroot(self, xen_deferred); 183 } 184 185 void 186 xen_deferred(struct device *self) 187 { 188 struct xen_softc *sc = (struct xen_softc *)self; 189 190 if (!(sc->sc_flags & XSF_CBVEC)) { 191 DPRINTF("%s: callback vector hasn't been established\n", 192 sc->sc_dev.dv_xname); 193 return; 194 } 195 196 xen_intr_enable(); 197 198 if (xs_watch(sc, "control", "shutdown", &sc->sc_ctltsk, 199 xen_control, sc)) 200 printf("%s: failed to setup shutdown control watch\n", 201 sc->sc_dev.dv_xname); 202 } 203 204 void 205 xen_control(void *arg) 206 { 207 struct xen_softc *sc = arg; 208 struct xs_transaction xst; 209 char action[128]; 210 int error; 211 212 memset(&xst, 0, sizeof(xst)); 213 xst.xst_id = 0; 214 xst.xst_cookie = sc->sc_xs; 215 216 error = xs_getprop(sc, "control", "shutdown", action, sizeof(action)); 217 if (error) { 218 if (error != ENOENT) 219 printf("%s: failed to process control event\n", 220 sc->sc_dev.dv_xname); 221 return; 222 } 223 224 if (strlen(action) == 0) 225 return; 226 227 /* Acknowledge the event */ 228 xs_setprop(sc, "control", "shutdown", "", 0); 229 230 if (strcmp(action, "halt") == 0 || strcmp(action, "poweroff") == 0) { 231 pvbus_shutdown(&sc->sc_dev); 232 } else if (strcmp(action, "reboot") == 0) { 233 pvbus_reboot(&sc->sc_dev); 234 } else if (strcmp(action, "crash") == 0) { 235 panic("xen told us to do this"); 236 } else if (strcmp(action, "suspend") == 0) { 237 /* Not implemented yet */ 238 } else { 239 printf("%s: unknown shutdown event \"%s\"\n", 240 sc->sc_dev.dv_xname, action); 241 } 242 } 243 244 void 245 xen_resume(struct device *self) 246 { 247 } 248 249 int 250 xen_activate(struct device *self, int act) 251 { 252 int rv = 0; 253 254 switch (act) { 255 case DVACT_RESUME: 256 xen_resume(self); 257 break; 258 } 259 return (rv); 260 } 261 262 int 263 xen_init_hypercall(struct xen_softc *sc) 264 { 265 extern void *xen_hypercall_page; 266 uint32_t regs[4]; 267 paddr_t pa; 268 269 /* Get hypercall page configuration MSR */ 270 CPUID(sc->sc_base + CPUID_OFFSET_XEN_HYPERCALL, 271 regs[0], regs[1], regs[2], regs[3]); 272 273 /* We don't support more than one hypercall page */ 274 if (regs[0] != 1) { 275 printf(": requested %u hypercall pages\n", regs[0]); 276 return (-1); 277 } 278 279 sc->sc_hc = &xen_hypercall_page; 280 281 if (!pmap_extract(pmap_kernel(), (vaddr_t)sc->sc_hc, &pa)) { 282 printf(": hypercall page PA extraction failed\n"); 283 return (-1); 284 } 285 wrmsr(regs[1], pa); 286 287 return (0); 288 } 289 290 int 291 xen_hypercall(struct xen_softc *sc, int op, int argc, ...) 292 { 293 va_list ap; 294 ulong argv[5]; 295 int i; 296 297 if (argc < 0 || argc > 5) 298 return (-1); 299 va_start(ap, argc); 300 for (i = 0; i < argc; i++) 301 argv[i] = (ulong)va_arg(ap, ulong); 302 va_end(ap); 303 return (xen_hypercallv(sc, op, argc, argv)); 304 } 305 306 int 307 xen_hypercallv(struct xen_softc *sc, int op, int argc, ulong *argv) 308 { 309 ulong hcall; 310 int rv = 0; 311 312 hcall = (ulong)sc->sc_hc + op * 32; 313 314 #if defined(XEN_DEBUG) && disabled 315 { 316 int i; 317 318 printf("hypercall %d", op); 319 if (argc > 0) { 320 printf(", args {"); 321 for (i = 0; i < argc; i++) 322 printf(" %#lx", argv[i]); 323 printf(" }\n"); 324 } else 325 printf("\n"); 326 } 327 #endif 328 329 switch (argc) { 330 case 0: { 331 HYPERCALL_RES1; 332 __asm__ volatile ( \ 333 HYPERCALL_LABEL \ 334 : HYPERCALL_OUT1 \ 335 : HYPERCALL_PTR(hcall) \ 336 : HYPERCALL_CLOBBER \ 337 ); 338 HYPERCALL_RET(rv); 339 break; 340 } 341 case 1: { 342 HYPERCALL_RES1; HYPERCALL_RES2; 343 HYPERCALL_ARG1(argv[0]); 344 __asm__ volatile ( \ 345 HYPERCALL_LABEL \ 346 : HYPERCALL_OUT1 HYPERCALL_OUT2 \ 347 : HYPERCALL_IN1 \ 348 , HYPERCALL_PTR(hcall) \ 349 : HYPERCALL_CLOBBER \ 350 ); 351 HYPERCALL_RET(rv); 352 break; 353 } 354 case 2: { 355 HYPERCALL_RES1; HYPERCALL_RES2; HYPERCALL_RES3; 356 HYPERCALL_ARG1(argv[0]); HYPERCALL_ARG2(argv[1]); 357 __asm__ volatile ( \ 358 HYPERCALL_LABEL \ 359 : HYPERCALL_OUT1 HYPERCALL_OUT2 \ 360 HYPERCALL_OUT3 \ 361 : HYPERCALL_IN1 HYPERCALL_IN2 \ 362 , HYPERCALL_PTR(hcall) \ 363 : HYPERCALL_CLOBBER \ 364 ); 365 HYPERCALL_RET(rv); 366 break; 367 } 368 case 3: { 369 HYPERCALL_RES1; HYPERCALL_RES2; HYPERCALL_RES3; 370 HYPERCALL_RES4; 371 HYPERCALL_ARG1(argv[0]); HYPERCALL_ARG2(argv[1]); 372 HYPERCALL_ARG3(argv[2]); 373 __asm__ volatile ( \ 374 HYPERCALL_LABEL \ 375 : HYPERCALL_OUT1 HYPERCALL_OUT2 \ 376 HYPERCALL_OUT3 HYPERCALL_OUT4 \ 377 : HYPERCALL_IN1 HYPERCALL_IN2 \ 378 HYPERCALL_IN3 \ 379 , HYPERCALL_PTR(hcall) \ 380 : HYPERCALL_CLOBBER \ 381 ); 382 HYPERCALL_RET(rv); 383 break; 384 } 385 case 4: { 386 HYPERCALL_RES1; HYPERCALL_RES2; HYPERCALL_RES3; 387 HYPERCALL_RES4; HYPERCALL_RES5; 388 HYPERCALL_ARG1(argv[0]); HYPERCALL_ARG2(argv[1]); 389 HYPERCALL_ARG3(argv[2]); HYPERCALL_ARG4(argv[3]); 390 __asm__ volatile ( \ 391 HYPERCALL_LABEL \ 392 : HYPERCALL_OUT1 HYPERCALL_OUT2 \ 393 HYPERCALL_OUT3 HYPERCALL_OUT4 \ 394 HYPERCALL_OUT5 \ 395 : HYPERCALL_IN1 HYPERCALL_IN2 \ 396 HYPERCALL_IN3 HYPERCALL_IN4 \ 397 , HYPERCALL_PTR(hcall) \ 398 : HYPERCALL_CLOBBER \ 399 ); 400 HYPERCALL_RET(rv); 401 break; 402 } 403 case 5: { 404 HYPERCALL_RES1; HYPERCALL_RES2; HYPERCALL_RES3; 405 HYPERCALL_RES4; HYPERCALL_RES5; HYPERCALL_RES6; 406 HYPERCALL_ARG1(argv[0]); HYPERCALL_ARG2(argv[1]); 407 HYPERCALL_ARG3(argv[2]); HYPERCALL_ARG4(argv[3]); 408 HYPERCALL_ARG5(argv[4]); 409 __asm__ volatile ( \ 410 HYPERCALL_LABEL \ 411 : HYPERCALL_OUT1 HYPERCALL_OUT2 \ 412 HYPERCALL_OUT3 HYPERCALL_OUT4 \ 413 HYPERCALL_OUT5 HYPERCALL_OUT6 \ 414 : HYPERCALL_IN1 HYPERCALL_IN2 \ 415 HYPERCALL_IN3 HYPERCALL_IN4 \ 416 HYPERCALL_IN5 \ 417 , HYPERCALL_PTR(hcall) \ 418 : HYPERCALL_CLOBBER \ 419 ); 420 HYPERCALL_RET(rv); 421 break; 422 } 423 default: 424 DPRINTF("%s: wrong number of arguments: %d\n", __func__, argc); 425 rv = -1; 426 break; 427 } 428 return (rv); 429 } 430 431 int 432 xen_getfeatures(struct xen_softc *sc) 433 { 434 struct xen_feature_info xfi; 435 436 memset(&xfi, 0, sizeof(xfi)); 437 if (xen_hypercall(sc, XC_VERSION, 2, XENVER_get_features, &xfi) < 0) { 438 printf(": failed to fetch features\n"); 439 return (-1); 440 } 441 sc->sc_features = xfi.submap; 442 #ifdef XEN_DEBUG 443 printf(": features %b", sc->sc_features, 444 "\20\014DOM0\013PIRQ\012PVCLOCK\011CBVEC\010GNTFLAGS\007HMA" 445 "\006PTUPD\005PAE4G\004SUPERVISOR\003AUTOPMAP\002WDT\001WPT"); 446 #else 447 printf(": features %#x", sc->sc_features); 448 #endif 449 return (0); 450 } 451 452 #ifdef XEN_DEBUG 453 void 454 xen_print_info_page(void) 455 { 456 struct xen_softc *sc = xen_sc; 457 struct shared_info *s = sc->sc_ipg; 458 struct vcpu_info *v; 459 int i; 460 461 virtio_membar_sync(); 462 for (i = 0; i < XEN_LEGACY_MAX_VCPUS; i++) { 463 v = &s->vcpu_info[i]; 464 if (!v->evtchn_upcall_pending && !v->evtchn_upcall_mask && 465 !v->evtchn_pending_sel && !v->time.version && 466 !v->time.tsc_timestamp && !v->time.system_time && 467 !v->time.tsc_to_system_mul && !v->time.tsc_shift) 468 continue; 469 printf("vcpu%d:\n" 470 " upcall_pending=%02x upcall_mask=%02x pending_sel=%#lx\n" 471 " time version=%u tsc=%llu system=%llu\n" 472 " time mul=%u shift=%d\n", 473 i, v->evtchn_upcall_pending, v->evtchn_upcall_mask, 474 v->evtchn_pending_sel, v->time.version, 475 v->time.tsc_timestamp, v->time.system_time, 476 v->time.tsc_to_system_mul, v->time.tsc_shift); 477 } 478 printf("pending events: "); 479 for (i = 0; i < nitems(s->evtchn_pending); i++) { 480 if (s->evtchn_pending[i] == 0) 481 continue; 482 printf(" %d:%#lx", i, s->evtchn_pending[i]); 483 } 484 printf("\nmasked events: "); 485 for (i = 0; i < nitems(s->evtchn_mask); i++) { 486 if (s->evtchn_mask[i] == 0xffffffffffffffffULL) 487 continue; 488 printf(" %d:%#lx", i, s->evtchn_mask[i]); 489 } 490 printf("\nwc ver=%u sec=%u nsec=%u\n", s->wc_version, s->wc_sec, 491 s->wc_nsec); 492 printf("arch maxpfn=%lu framelist=%lu nmi=%lu\n", s->arch.max_pfn, 493 s->arch.pfn_to_mfn_frame_list, s->arch.nmi_reason); 494 } 495 #endif /* XEN_DEBUG */ 496 497 int 498 xen_init_info_page(struct xen_softc *sc) 499 { 500 struct xen_add_to_physmap xatp; 501 paddr_t pa; 502 503 sc->sc_ipg = malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT | M_ZERO); 504 if (sc->sc_ipg == NULL) { 505 printf(": failed to allocate shared info page\n"); 506 return (-1); 507 } 508 if (!pmap_extract(pmap_kernel(), (vaddr_t)sc->sc_ipg, &pa)) { 509 printf(": shared info page PA extraction failed\n"); 510 free(sc->sc_ipg, M_DEVBUF, PAGE_SIZE); 511 return (-1); 512 } 513 xatp.domid = DOMID_SELF; 514 xatp.idx = 0; 515 xatp.space = XENMAPSPACE_shared_info; 516 xatp.gpfn = atop(pa); 517 if (xen_hypercall(sc, XC_MEMORY, 2, XENMEM_add_to_physmap, &xatp)) { 518 printf(": failed to register shared info page\n"); 519 free(sc->sc_ipg, M_DEVBUF, PAGE_SIZE); 520 return (-1); 521 } 522 return (0); 523 } 524 525 int 526 xen_init_cbvec(struct xen_softc *sc) 527 { 528 struct xen_hvm_param xhp; 529 530 if ((sc->sc_features & XENFEAT_CBVEC) == 0) 531 return (ENOENT); 532 533 xhp.domid = DOMID_SELF; 534 xhp.index = HVM_PARAM_CALLBACK_IRQ; 535 xhp.value = HVM_CALLBACK_VECTOR(LAPIC_XEN_VECTOR); 536 if (xen_hypercall(sc, XC_HVM, 2, HVMOP_set_param, &xhp)) { 537 /* Will retry with the xspd(4) PCI interrupt */ 538 return (ENOENT); 539 } 540 DPRINTF(", idtvec %d", LAPIC_XEN_VECTOR); 541 542 sc->sc_flags |= XSF_CBVEC; 543 544 return (0); 545 } 546 547 int 548 xen_init_interrupts(struct xen_softc *sc) 549 { 550 int i; 551 552 sc->sc_irq = LAPIC_XEN_VECTOR; 553 554 /* 555 * Clear all pending events and mask all interrupts 556 */ 557 for (i = 0; i < nitems(sc->sc_ipg->evtchn_pending); i++) { 558 sc->sc_ipg->evtchn_pending[i] = 0; 559 sc->sc_ipg->evtchn_mask[i] = ~0UL; 560 } 561 562 SLIST_INIT(&sc->sc_intrs); 563 564 mtx_init(&sc->sc_islck, IPL_NET); 565 566 return (0); 567 } 568 569 static int 570 xen_evtchn_hypercall(struct xen_softc *sc, int cmd, void *arg, size_t len) 571 { 572 struct evtchn_op compat; 573 int error; 574 575 error = xen_hypercall(sc, XC_EVTCHN, 2, cmd, arg); 576 if (error == -ENOXENSYS) { 577 memset(&compat, 0, sizeof(compat)); 578 compat.cmd = cmd; 579 memcpy(&compat.u, arg, len); 580 error = xen_hypercall(sc, XC_OEVTCHN, 1, &compat); 581 } 582 return (error); 583 } 584 585 static inline void 586 xen_intsrc_add(struct xen_softc *sc, struct xen_intsrc *xi) 587 { 588 refcnt_init(&xi->xi_refcnt); 589 mtx_enter(&sc->sc_islck); 590 SLIST_INSERT_HEAD(&sc->sc_intrs, xi, xi_entry); 591 mtx_leave(&sc->sc_islck); 592 } 593 594 static inline struct xen_intsrc * 595 xen_intsrc_acquire(struct xen_softc *sc, evtchn_port_t port) 596 { 597 struct xen_intsrc *xi = NULL; 598 599 mtx_enter(&sc->sc_islck); 600 SLIST_FOREACH(xi, &sc->sc_intrs, xi_entry) { 601 if (xi->xi_port == port) { 602 refcnt_take(&xi->xi_refcnt); 603 break; 604 } 605 } 606 mtx_leave(&sc->sc_islck); 607 return (xi); 608 } 609 610 static inline void 611 xen_intsrc_release(struct xen_softc *sc, struct xen_intsrc *xi) 612 { 613 refcnt_rele_wake(&xi->xi_refcnt); 614 } 615 616 static inline struct xen_intsrc * 617 xen_intsrc_remove(struct xen_softc *sc, evtchn_port_t port) 618 { 619 struct xen_intsrc *xi; 620 621 mtx_enter(&sc->sc_islck); 622 SLIST_FOREACH(xi, &sc->sc_intrs, xi_entry) { 623 if (xi->xi_port == port) { 624 SLIST_REMOVE(&sc->sc_intrs, xi, xen_intsrc, xi_entry); 625 break; 626 } 627 } 628 mtx_leave(&sc->sc_islck); 629 if (xi != NULL) 630 refcnt_finalize(&xi->xi_refcnt, "xenisrm"); 631 return (xi); 632 } 633 634 static inline void 635 xen_intr_mask_acquired(struct xen_softc *sc, struct xen_intsrc *xi) 636 { 637 xi->xi_masked = 1; 638 set_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0]); 639 } 640 641 static inline int 642 xen_intr_unmask_release(struct xen_softc *sc, struct xen_intsrc *xi) 643 { 644 struct evtchn_unmask eu; 645 646 xi->xi_masked = 0; 647 if (!test_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0])) { 648 xen_intsrc_release(sc, xi); 649 return (0); 650 } 651 eu.port = xi->xi_port; 652 xen_intsrc_release(sc, xi); 653 return (xen_evtchn_hypercall(sc, EVTCHNOP_unmask, &eu, sizeof(eu))); 654 } 655 656 void 657 xen_intr_ack(void) 658 { 659 struct xen_softc *sc = xen_sc; 660 struct shared_info *s = sc->sc_ipg; 661 struct cpu_info *ci = curcpu(); 662 struct vcpu_info *v = &s->vcpu_info[CPU_INFO_UNIT(ci)]; 663 664 v->evtchn_upcall_pending = 0; 665 virtio_membar_sync(); 666 } 667 668 void 669 xen_intr(void) 670 { 671 struct xen_softc *sc = xen_sc; 672 struct xen_intsrc *xi; 673 struct shared_info *s = sc->sc_ipg; 674 struct cpu_info *ci = curcpu(); 675 struct vcpu_info *v = &s->vcpu_info[CPU_INFO_UNIT(ci)]; 676 ulong pending, selector; 677 int port, bit, row; 678 679 v->evtchn_upcall_pending = 0; 680 selector = atomic_swap_ulong(&v->evtchn_pending_sel, 0); 681 682 for (row = 0; selector > 0; selector >>= 1, row++) { 683 if ((selector & 1) == 0) 684 continue; 685 if ((sc->sc_ipg->evtchn_pending[row] & 686 ~(sc->sc_ipg->evtchn_mask[row])) == 0) 687 continue; 688 pending = atomic_swap_ulong(&sc->sc_ipg->evtchn_pending[row], 689 0) & ~(sc->sc_ipg->evtchn_mask[row]); 690 for (bit = 0; pending > 0; pending >>= 1, bit++) { 691 if ((pending & 1) == 0) 692 continue; 693 port = (row * LONG_BIT) + bit; 694 if ((xi = xen_intsrc_acquire(sc, port)) == NULL) { 695 printf("%s: unhandled interrupt on port %d\n", 696 sc->sc_dev.dv_xname, port); 697 continue; 698 } 699 xi->xi_evcnt.ec_count++; 700 xen_intr_mask_acquired(sc, xi); 701 if (!task_add(xi->xi_taskq, &xi->xi_task)) 702 xen_intsrc_release(sc, xi); 703 } 704 } 705 } 706 707 void 708 xen_intr_schedule(xen_intr_handle_t xih) 709 { 710 struct xen_softc *sc = xen_sc; 711 struct xen_intsrc *xi; 712 713 if ((xi = xen_intsrc_acquire(sc, (evtchn_port_t)xih)) != NULL) { 714 xen_intr_mask_acquired(sc, xi); 715 if (!task_add(xi->xi_taskq, &xi->xi_task)) 716 xen_intsrc_release(sc, xi); 717 } 718 } 719 720 /* 721 * This code achieves two goals: 1) makes sure that *after* masking 722 * the interrupt source we're not getting more task_adds: sched_barrier 723 * will take care of that, and 2) makes sure that the interrupt task 724 * has finished executing the current task and won't be called again: 725 * it sets up a barrier task to await completion of the current task 726 * and relies on the interrupt masking to prevent submission of new 727 * tasks in the future. 728 */ 729 void 730 xen_intr_barrier(xen_intr_handle_t xih) 731 { 732 struct xen_softc *sc = xen_sc; 733 struct xen_intsrc *xi; 734 735 sched_barrier(NULL); 736 737 if ((xi = xen_intsrc_acquire(sc, (evtchn_port_t)xih)) != NULL) { 738 taskq_barrier(xi->xi_taskq); 739 xen_intsrc_release(sc, xi); 740 } 741 } 742 743 void 744 xen_intr_signal(xen_intr_handle_t xih) 745 { 746 struct xen_softc *sc = xen_sc; 747 struct xen_intsrc *xi; 748 struct evtchn_send es; 749 750 if ((xi = xen_intsrc_acquire(sc, (evtchn_port_t)xih)) != NULL) { 751 es.port = xi->xi_port; 752 xen_intsrc_release(sc, xi); 753 xen_evtchn_hypercall(sc, EVTCHNOP_send, &es, sizeof(es)); 754 } 755 } 756 757 int 758 xen_intr_establish(evtchn_port_t port, xen_intr_handle_t *xih, int domain, 759 void (*handler)(void *), void *arg, char *name) 760 { 761 struct xen_softc *sc = xen_sc; 762 struct xen_intsrc *xi; 763 struct evtchn_alloc_unbound eau; 764 #ifdef notyet 765 struct evtchn_bind_vcpu ebv; 766 #endif 767 #if defined(XEN_DEBUG) && disabled 768 struct evtchn_status es; 769 #endif 770 771 if (port && (xi = xen_intsrc_acquire(sc, port)) != NULL) { 772 xen_intsrc_release(sc, xi); 773 DPRINTF("%s: interrupt handler has already been established " 774 "for port %u\n", sc->sc_dev.dv_xname, port); 775 return (-1); 776 } 777 778 xi = malloc(sizeof(*xi), M_DEVBUF, M_NOWAIT | M_ZERO); 779 if (xi == NULL) 780 return (-1); 781 782 xi->xi_port = (evtchn_port_t)*xih; 783 784 xi->xi_handler = handler; 785 xi->xi_ctx = arg; 786 787 xi->xi_taskq = taskq_create(name, 1, IPL_NET, TASKQ_MPSAFE); 788 if (!xi->xi_taskq) { 789 printf("%s: failed to create interrupt task for %s\n", 790 sc->sc_dev.dv_xname, name); 791 free(xi, M_DEVBUF, sizeof(*xi)); 792 return (-1); 793 } 794 task_set(&xi->xi_task, xen_intr_dispatch, xi); 795 796 if (port == 0) { 797 /* We're being asked to allocate a new event port */ 798 memset(&eau, 0, sizeof(eau)); 799 eau.dom = DOMID_SELF; 800 eau.remote_dom = domain; 801 if (xen_evtchn_hypercall(sc, EVTCHNOP_alloc_unbound, &eau, 802 sizeof(eau)) != 0) { 803 DPRINTF("%s: failed to allocate new event port\n", 804 sc->sc_dev.dv_xname); 805 free(xi, M_DEVBUF, sizeof(*xi)); 806 return (-1); 807 } 808 *xih = xi->xi_port = eau.port; 809 } else { 810 *xih = xi->xi_port = port; 811 /* 812 * The Event Channel API didn't open this port, so it is not 813 * responsible for closing it automatically on unbind. 814 */ 815 xi->xi_noclose = 1; 816 } 817 818 #ifdef notyet 819 /* Bind interrupt to VCPU#0 */ 820 memset(&ebv, 0, sizeof(ebv)); 821 ebv.port = xi->xi_port; 822 ebv.vcpu = 0; 823 if (xen_evtchn_hypercall(sc, EVTCHNOP_bind_vcpu, &ebv, sizeof(ebv))) { 824 printf("%s: failed to bind interrupt on port %u to vcpu%d\n", 825 sc->sc_dev.dv_xname, ebv.port, ebv.vcpu); 826 } 827 #endif 828 829 evcount_attach(&xi->xi_evcnt, name, &sc->sc_irq); 830 831 xen_intsrc_add(sc, xi); 832 833 /* Mask the event port */ 834 set_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0]); 835 836 #if defined(XEN_DEBUG) && disabled 837 memset(&es, 0, sizeof(es)); 838 es.dom = DOMID_SELF; 839 es.port = xi->xi_port; 840 if (xen_evtchn_hypercall(sc, EVTCHNOP_status, &es, sizeof(es))) { 841 printf("%s: failed to obtain status for port %d\n", 842 sc->sc_dev.dv_xname, es.port); 843 } 844 printf("%s: port %u bound to vcpu%u", sc->sc_dev.dv_xname, 845 es.port, es.vcpu); 846 if (es.status == EVTCHNSTAT_interdomain) 847 printf(": domain %d port %u\n", es.u.interdomain.dom, 848 es.u.interdomain.port); 849 else if (es.status == EVTCHNSTAT_unbound) 850 printf(": domain %d\n", es.u.unbound.dom); 851 else if (es.status == EVTCHNSTAT_pirq) 852 printf(": pirq %u\n", es.u.pirq); 853 else if (es.status == EVTCHNSTAT_virq) 854 printf(": virq %u\n", es.u.virq); 855 else 856 printf("\n"); 857 #endif 858 859 return (0); 860 } 861 862 int 863 xen_intr_disestablish(xen_intr_handle_t xih) 864 { 865 struct xen_softc *sc = xen_sc; 866 evtchn_port_t port = (evtchn_port_t)xih; 867 struct evtchn_close ec; 868 struct xen_intsrc *xi; 869 870 if ((xi = xen_intsrc_remove(sc, port)) == NULL) 871 return (-1); 872 873 evcount_detach(&xi->xi_evcnt); 874 875 taskq_destroy(xi->xi_taskq); 876 877 set_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0]); 878 clear_bit(xi->xi_port, &sc->sc_ipg->evtchn_pending[0]); 879 880 if (!xi->xi_noclose) { 881 ec.port = xi->xi_port; 882 if (xen_evtchn_hypercall(sc, EVTCHNOP_close, &ec, sizeof(ec))) { 883 DPRINTF("%s: failed to close event port %u\n", 884 sc->sc_dev.dv_xname, xi->xi_port); 885 } 886 } 887 888 free(xi, M_DEVBUF, sizeof(*xi)); 889 return (0); 890 } 891 892 void 893 xen_intr_dispatch(void *arg) 894 { 895 struct xen_softc *sc = xen_sc; 896 struct xen_intsrc *xi = arg; 897 898 if (xi->xi_handler) 899 xi->xi_handler(xi->xi_ctx); 900 901 xen_intr_unmask_release(sc, xi); 902 } 903 904 void 905 xen_intr_enable(void) 906 { 907 struct xen_softc *sc = xen_sc; 908 struct xen_intsrc *xi; 909 struct evtchn_unmask eu; 910 911 mtx_enter(&sc->sc_islck); 912 SLIST_FOREACH(xi, &sc->sc_intrs, xi_entry) { 913 if (!xi->xi_masked) { 914 eu.port = xi->xi_port; 915 if (xen_evtchn_hypercall(sc, EVTCHNOP_unmask, &eu, 916 sizeof(eu))) 917 printf("%s: unmasking port %u failed\n", 918 sc->sc_dev.dv_xname, xi->xi_port); 919 virtio_membar_sync(); 920 if (test_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0])) 921 printf("%s: port %u is still masked\n", 922 sc->sc_dev.dv_xname, xi->xi_port); 923 } 924 } 925 mtx_leave(&sc->sc_islck); 926 } 927 928 void 929 xen_intr_mask(xen_intr_handle_t xih) 930 { 931 struct xen_softc *sc = xen_sc; 932 evtchn_port_t port = (evtchn_port_t)xih; 933 struct xen_intsrc *xi; 934 935 if ((xi = xen_intsrc_acquire(sc, port)) != NULL) { 936 xen_intr_mask_acquired(sc, xi); 937 xen_intsrc_release(sc, xi); 938 } 939 } 940 941 int 942 xen_intr_unmask(xen_intr_handle_t xih) 943 { 944 struct xen_softc *sc = xen_sc; 945 evtchn_port_t port = (evtchn_port_t)xih; 946 struct xen_intsrc *xi; 947 948 if ((xi = xen_intsrc_acquire(sc, port)) != NULL) 949 return (xen_intr_unmask_release(sc, xi)); 950 951 return (0); 952 } 953 954 int 955 xen_init_grant_tables(struct xen_softc *sc) 956 { 957 struct gnttab_query_size gqs; 958 959 gqs.dom = DOMID_SELF; 960 if (xen_hypercall(sc, XC_GNTTAB, 3, GNTTABOP_query_size, &gqs, 1)) { 961 printf(": failed the query for grant table pages\n"); 962 return (-1); 963 } 964 if (gqs.nr_frames == 0 || gqs.nr_frames > gqs.max_nr_frames) { 965 printf(": invalid number of grant table pages: %u/%u\n", 966 gqs.nr_frames, gqs.max_nr_frames); 967 return (-1); 968 } 969 970 sc->sc_gntmax = gqs.max_nr_frames; 971 972 sc->sc_gnt = mallocarray(sc->sc_gntmax + 1, sizeof(struct xen_gntent), 973 M_DEVBUF, M_ZERO | M_NOWAIT); 974 if (sc->sc_gnt == NULL) { 975 printf(": failed to allocate grant table lookup table\n"); 976 return (-1); 977 } 978 979 mtx_init(&sc->sc_gntlck, IPL_NET); 980 981 if (xen_grant_table_grow(sc) == NULL) { 982 free(sc->sc_gnt, M_DEVBUF, sc->sc_gntmax * 983 sizeof(struct xen_gntent)); 984 return (-1); 985 } 986 987 printf(", %d grant table frames", sc->sc_gntmax); 988 989 xen_bus_dma_tag._cookie = sc; 990 991 return (0); 992 } 993 994 struct xen_gntent * 995 xen_grant_table_grow(struct xen_softc *sc) 996 { 997 struct xen_add_to_physmap xatp; 998 struct xen_gntent *ge; 999 void *va; 1000 paddr_t pa; 1001 1002 if (sc->sc_gntcnt == sc->sc_gntmax) { 1003 printf("%s: grant table frame allotment limit reached\n", 1004 sc->sc_dev.dv_xname); 1005 return (NULL); 1006 } 1007 1008 va = km_alloc(PAGE_SIZE, &kv_any, &kp_zero, &kd_nowait); 1009 if (va == NULL) 1010 return (NULL); 1011 if (!pmap_extract(pmap_kernel(), (vaddr_t)va, &pa)) { 1012 printf("%s: grant table page PA extraction failed\n", 1013 sc->sc_dev.dv_xname); 1014 km_free(va, PAGE_SIZE, &kv_any, &kp_zero); 1015 return (NULL); 1016 } 1017 1018 mtx_enter(&sc->sc_gntlck); 1019 1020 ge = &sc->sc_gnt[sc->sc_gntcnt]; 1021 ge->ge_table = va; 1022 1023 xatp.domid = DOMID_SELF; 1024 xatp.idx = sc->sc_gntcnt; 1025 xatp.space = XENMAPSPACE_grant_table; 1026 xatp.gpfn = atop(pa); 1027 if (xen_hypercall(sc, XC_MEMORY, 2, XENMEM_add_to_physmap, &xatp)) { 1028 printf("%s: failed to add a grant table page\n", 1029 sc->sc_dev.dv_xname); 1030 km_free(ge->ge_table, PAGE_SIZE, &kv_any, &kp_zero); 1031 mtx_leave(&sc->sc_gntlck); 1032 return (NULL); 1033 } 1034 ge->ge_start = sc->sc_gntcnt * GNTTAB_NEPG; 1035 /* First page has 8 reserved entries */ 1036 ge->ge_reserved = ge->ge_start == 0 ? GNTTAB_NR_RESERVED_ENTRIES : 0; 1037 ge->ge_free = GNTTAB_NEPG - ge->ge_reserved; 1038 ge->ge_next = ge->ge_reserved; 1039 mtx_init(&ge->ge_lock, IPL_NET); 1040 1041 sc->sc_gntcnt++; 1042 mtx_leave(&sc->sc_gntlck); 1043 1044 return (ge); 1045 } 1046 1047 int 1048 xen_grant_table_alloc(struct xen_softc *sc, grant_ref_t *ref) 1049 { 1050 struct xen_gntent *ge; 1051 int i; 1052 1053 /* Start with a previously allocated table page */ 1054 ge = &sc->sc_gnt[sc->sc_gntcnt - 1]; 1055 if (ge->ge_free > 0) { 1056 mtx_enter(&ge->ge_lock); 1057 if (ge->ge_free > 0) 1058 goto search; 1059 mtx_leave(&ge->ge_lock); 1060 } 1061 1062 /* Try other existing table pages */ 1063 for (i = 0; i < sc->sc_gntcnt; i++) { 1064 ge = &sc->sc_gnt[i]; 1065 if (ge->ge_free == 0) 1066 continue; 1067 mtx_enter(&ge->ge_lock); 1068 if (ge->ge_free > 0) 1069 goto search; 1070 mtx_leave(&ge->ge_lock); 1071 } 1072 1073 alloc: 1074 /* Allocate a new table page */ 1075 if ((ge = xen_grant_table_grow(sc)) == NULL) 1076 return (-1); 1077 1078 mtx_enter(&ge->ge_lock); 1079 if (ge->ge_free == 0) { 1080 /* We were not fast enough... */ 1081 mtx_leave(&ge->ge_lock); 1082 goto alloc; 1083 } 1084 1085 search: 1086 for (i = ge->ge_next; 1087 /* Math works here because GNTTAB_NEPG is a power of 2 */ 1088 i != ((ge->ge_next + GNTTAB_NEPG - 1) & (GNTTAB_NEPG - 1)); 1089 i++) { 1090 if (i == GNTTAB_NEPG) 1091 i = 0; 1092 if (ge->ge_reserved && i < ge->ge_reserved) 1093 continue; 1094 if (ge->ge_table[i].frame != 0) 1095 continue; 1096 *ref = ge->ge_start + i; 1097 ge->ge_table[i].flags = GTF_invalid; 1098 ge->ge_table[i].frame = 0xffffffff; /* Mark as taken */ 1099 if ((ge->ge_next = i + 1) == GNTTAB_NEPG) 1100 ge->ge_next = ge->ge_reserved; 1101 ge->ge_free--; 1102 mtx_leave(&ge->ge_lock); 1103 return (0); 1104 } 1105 mtx_leave(&ge->ge_lock); 1106 1107 panic("page full, sc %p gnt %p (%d) ge %p", sc, sc->sc_gnt, 1108 sc->sc_gntcnt, ge); 1109 return (-1); 1110 } 1111 1112 void 1113 xen_grant_table_free(struct xen_softc *sc, grant_ref_t ref) 1114 { 1115 struct xen_gntent *ge; 1116 1117 #ifdef XEN_DEBUG 1118 if (ref > sc->sc_gntcnt * GNTTAB_NEPG) 1119 panic("unmanaged ref %u sc %p gnt %p (%d)", ref, sc, 1120 sc->sc_gnt, sc->sc_gntcnt); 1121 #endif 1122 ge = &sc->sc_gnt[ref / GNTTAB_NEPG]; 1123 mtx_enter(&ge->ge_lock); 1124 #ifdef XEN_DEBUG 1125 if (ref < ge->ge_start || ref > ge->ge_start + GNTTAB_NEPG) { 1126 mtx_leave(&ge->ge_lock); 1127 panic("out of bounds ref %u ge %p start %u sc %p gnt %p", 1128 ref, ge, ge->ge_start, sc, sc->sc_gnt); 1129 } 1130 #endif 1131 ref -= ge->ge_start; 1132 if (ge->ge_table[ref].flags != GTF_invalid) { 1133 mtx_leave(&ge->ge_lock); 1134 panic("reference %u is still in use, flags %#x frame %#x", 1135 ref + ge->ge_start, ge->ge_table[ref].flags, 1136 ge->ge_table[ref].frame); 1137 } 1138 ge->ge_table[ref].frame = 0; 1139 ge->ge_next = ref; 1140 ge->ge_free++; 1141 mtx_leave(&ge->ge_lock); 1142 } 1143 1144 void 1145 xen_grant_table_enter(struct xen_softc *sc, grant_ref_t ref, paddr_t pa, 1146 int domain, int flags) 1147 { 1148 struct xen_gntent *ge; 1149 1150 #ifdef XEN_DEBUG 1151 if (ref > sc->sc_gntcnt * GNTTAB_NEPG) 1152 panic("unmanaged ref %u sc %p gnt %p (%d)", ref, sc, 1153 sc->sc_gnt, sc->sc_gntcnt); 1154 #endif 1155 ge = &sc->sc_gnt[ref / GNTTAB_NEPG]; 1156 #ifdef XEN_DEBUG 1157 if (ref < ge->ge_start || ref > ge->ge_start + GNTTAB_NEPG) { 1158 panic("out of bounds ref %u ge %p start %u sc %p gnt %p", 1159 ref, ge, ge->ge_start, sc, sc->sc_gnt); 1160 } 1161 #endif 1162 ref -= ge->ge_start; 1163 if (ge->ge_table[ref].flags != GTF_invalid) { 1164 panic("reference %u is still in use, flags %#x frame %#x", 1165 ref + ge->ge_start, ge->ge_table[ref].flags, 1166 ge->ge_table[ref].frame); 1167 } 1168 ge->ge_table[ref].frame = atop(pa); 1169 ge->ge_table[ref].domid = domain; 1170 virtio_membar_sync(); 1171 ge->ge_table[ref].flags = GTF_permit_access | flags; 1172 virtio_membar_sync(); 1173 } 1174 1175 void 1176 xen_grant_table_remove(struct xen_softc *sc, grant_ref_t ref) 1177 { 1178 struct xen_gntent *ge; 1179 uint32_t flags, *ptr; 1180 int loop; 1181 1182 #ifdef XEN_DEBUG 1183 if (ref > sc->sc_gntcnt * GNTTAB_NEPG) 1184 panic("unmanaged ref %u sc %p gnt %p (%d)", ref, sc, 1185 sc->sc_gnt, sc->sc_gntcnt); 1186 #endif 1187 ge = &sc->sc_gnt[ref / GNTTAB_NEPG]; 1188 #ifdef XEN_DEBUG 1189 if (ref < ge->ge_start || ref > ge->ge_start + GNTTAB_NEPG) { 1190 panic("out of bounds ref %u ge %p start %u sc %p gnt %p", 1191 ref, ge, ge->ge_start, sc, sc->sc_gnt); 1192 } 1193 #endif 1194 ref -= ge->ge_start; 1195 /* Invalidate the grant reference */ 1196 virtio_membar_sync(); 1197 ptr = (uint32_t *)&ge->ge_table[ref]; 1198 flags = (ge->ge_table[ref].flags & ~(GTF_reading|GTF_writing)) | 1199 (ge->ge_table[ref].domid << 16); 1200 loop = 0; 1201 while (atomic_cas_uint(ptr, flags, GTF_invalid) != flags) { 1202 if (loop++ > 10) { 1203 panic("grant table reference %u is held " 1204 "by domain %d: frame %#x flags %#x", 1205 ref + ge->ge_start, ge->ge_table[ref].domid, 1206 ge->ge_table[ref].frame, ge->ge_table[ref].flags); 1207 } 1208 #if (defined(__amd64__) || defined(__i386__)) 1209 __asm volatile("pause": : : "memory"); 1210 #endif 1211 } 1212 ge->ge_table[ref].frame = 0xffffffff; 1213 } 1214 1215 int 1216 xen_bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, 1217 bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp) 1218 { 1219 struct xen_softc *sc = t->_cookie; 1220 struct xen_gntmap *gm; 1221 int i, error; 1222 1223 if (maxsegsz < PAGE_SIZE) 1224 return (EINVAL); 1225 1226 /* Allocate a dma map structure */ 1227 error = bus_dmamap_create(sc->sc_dmat, size, nsegments, maxsegsz, 1228 boundary, flags, dmamp); 1229 if (error) 1230 return (error); 1231 /* Allocate an array of grant table pa<->ref maps */ 1232 gm = mallocarray(nsegments, sizeof(struct xen_gntmap), M_DEVBUF, 1233 M_ZERO | ((flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK)); 1234 if (gm == NULL) { 1235 bus_dmamap_destroy(sc->sc_dmat, *dmamp); 1236 *dmamp = NULL; 1237 return (ENOMEM); 1238 } 1239 /* Wire it to the dma map */ 1240 (*dmamp)->_dm_cookie = gm; 1241 /* Claim references from the grant table */ 1242 for (i = 0; i < (*dmamp)->_dm_segcnt; i++) { 1243 if (xen_grant_table_alloc(sc, &gm[i].gm_ref)) { 1244 xen_bus_dmamap_destroy(t, *dmamp); 1245 *dmamp = NULL; 1246 return (ENOBUFS); 1247 } 1248 } 1249 return (0); 1250 } 1251 1252 void 1253 xen_bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map) 1254 { 1255 struct xen_softc *sc = t->_cookie; 1256 struct xen_gntmap *gm; 1257 int i; 1258 1259 gm = map->_dm_cookie; 1260 for (i = 0; i < map->_dm_segcnt; i++) { 1261 if (gm[i].gm_ref == 0) 1262 continue; 1263 xen_grant_table_free(sc, gm[i].gm_ref); 1264 } 1265 free(gm, M_DEVBUF, map->_dm_segcnt * sizeof(struct xen_gntmap)); 1266 bus_dmamap_destroy(sc->sc_dmat, map); 1267 } 1268 1269 int 1270 xen_bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 1271 bus_size_t buflen, struct proc *p, int flags) 1272 { 1273 struct xen_softc *sc = t->_cookie; 1274 struct xen_gntmap *gm = map->_dm_cookie; 1275 int i, domain, error; 1276 1277 domain = flags >> 16; 1278 flags &= 0xffff; 1279 error = bus_dmamap_load(sc->sc_dmat, map, buf, buflen, p, flags); 1280 if (error) 1281 return (error); 1282 for (i = 0; i < map->dm_nsegs; i++) { 1283 xen_grant_table_enter(sc, gm[i].gm_ref, map->dm_segs[i].ds_addr, 1284 domain, flags & BUS_DMA_WRITE ? GTF_readonly : 0); 1285 gm[i].gm_paddr = map->dm_segs[i].ds_addr; 1286 map->dm_segs[i].ds_addr = gm[i].gm_ref; 1287 } 1288 return (0); 1289 } 1290 1291 int 1292 xen_bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, 1293 int flags) 1294 { 1295 struct xen_softc *sc = t->_cookie; 1296 struct xen_gntmap *gm = map->_dm_cookie; 1297 int i, domain, error; 1298 1299 domain = flags >> 16; 1300 flags &= 0xffff; 1301 error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m0, flags); 1302 if (error) 1303 return (error); 1304 for (i = 0; i < map->dm_nsegs; i++) { 1305 xen_grant_table_enter(sc, gm[i].gm_ref, map->dm_segs[i].ds_addr, 1306 domain, flags & BUS_DMA_WRITE ? GTF_readonly : 0); 1307 gm[i].gm_paddr = map->dm_segs[i].ds_addr; 1308 map->dm_segs[i].ds_addr = gm[i].gm_ref; 1309 } 1310 return (0); 1311 } 1312 1313 void 1314 xen_bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) 1315 { 1316 struct xen_softc *sc = t->_cookie; 1317 struct xen_gntmap *gm = map->_dm_cookie; 1318 int i; 1319 1320 for (i = 0; i < map->dm_nsegs; i++) { 1321 if (gm[i].gm_paddr == 0) 1322 continue; 1323 xen_grant_table_remove(sc, gm[i].gm_ref); 1324 map->dm_segs[i].ds_addr = gm[i].gm_paddr; 1325 gm[i].gm_paddr = 0; 1326 } 1327 bus_dmamap_unload(sc->sc_dmat, map); 1328 } 1329 1330 void 1331 xen_bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr, 1332 bus_size_t size, int op) 1333 { 1334 if ((op == (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) || 1335 (op == (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE))) 1336 virtio_membar_sync(); 1337 } 1338 1339 static int 1340 xen_attach_print(void *aux, const char *name) 1341 { 1342 struct xen_attach_args *xa = aux; 1343 1344 if (name) 1345 printf("\"%s\" at %s: %s", xa->xa_name, name, xa->xa_node); 1346 1347 return (UNCONF); 1348 } 1349 1350 int 1351 xen_attach_device(struct xen_softc *sc, struct xen_devlist *xdl, 1352 const char *name, const char *unit) 1353 { 1354 struct xen_attach_args xa; 1355 struct xen_device *xdv; 1356 unsigned long long res; 1357 1358 memset(&xa, 0, sizeof(xa)); 1359 xa.xa_dmat = &xen_bus_dma_tag; 1360 1361 strlcpy(xa.xa_name, name, sizeof(xa.xa_name)); 1362 snprintf(xa.xa_node, sizeof(xa.xa_node), "device/%s/%s", name, unit); 1363 1364 if (xs_getprop(sc, xa.xa_node, "backend", xa.xa_backend, 1365 sizeof(xa.xa_backend))) { 1366 DPRINTF("%s: failed to identify \"backend\" for " 1367 "\"%s\"\n", sc->sc_dev.dv_xname, xa.xa_node); 1368 } 1369 1370 if (xs_getnum(sc, xa.xa_node, "backend-id", &res) || res > UINT16_MAX) { 1371 DPRINTF("%s: invalid \"backend-id\" for \"%s\"\n", 1372 sc->sc_dev.dv_xname, xa.xa_node); 1373 } 1374 if (res <= UINT16_MAX) 1375 xa.xa_domid = (uint16_t)res; 1376 1377 xdv = malloc(sizeof(struct xen_device), M_DEVBUF, M_ZERO | M_NOWAIT); 1378 if (xdv == NULL) 1379 return (ENOMEM); 1380 1381 strlcpy(xdv->dv_unit, unit, sizeof(xdv->dv_unit)); 1382 LIST_INSERT_HEAD(&xdl->dl_devs, xdv, dv_entry); 1383 1384 xdv->dv_dev = config_found((struct device *)sc, &xa, xen_attach_print); 1385 1386 return (0); 1387 } 1388 1389 int 1390 xen_probe_devices(struct xen_softc *sc) 1391 { 1392 struct xen_devlist *xdl; 1393 struct xs_transaction xst; 1394 struct iovec *iovp1 = NULL, *iovp2 = NULL; 1395 int i, j, error, iov1_cnt = 0, iov2_cnt = 0; 1396 char path[256]; 1397 1398 memset(&xst, 0, sizeof(xst)); 1399 xst.xst_id = 0; 1400 xst.xst_cookie = sc->sc_xs; 1401 1402 if ((error = xs_cmd(&xst, XS_LIST, "device", &iovp1, &iov1_cnt)) != 0) 1403 return (error); 1404 1405 for (i = 0; i < iov1_cnt; i++) { 1406 if (strcmp("suspend", (char *)iovp1[i].iov_base) == 0) 1407 continue; 1408 snprintf(path, sizeof(path), "device/%s", 1409 (char *)iovp1[i].iov_base); 1410 if ((error = xs_cmd(&xst, XS_LIST, path, &iovp2, 1411 &iov2_cnt)) != 0) 1412 goto out; 1413 if ((xdl = malloc(sizeof(struct xen_devlist), M_DEVBUF, 1414 M_ZERO | M_NOWAIT)) == NULL) { 1415 error = ENOMEM; 1416 goto out; 1417 } 1418 xdl->dl_xen = sc; 1419 strlcpy(xdl->dl_node, (const char *)iovp1[i].iov_base, 1420 XEN_MAX_NODE_LEN); 1421 for (j = 0; j < iov2_cnt; j++) { 1422 error = xen_attach_device(sc, xdl, 1423 (const char *)iovp1[i].iov_base, 1424 (const char *)iovp2[j].iov_base); 1425 if (error) { 1426 printf("%s: failed to attach \"%s/%s\"\n", 1427 sc->sc_dev.dv_xname, path, 1428 (const char *)iovp2[j].iov_base); 1429 continue; 1430 } 1431 } 1432 /* Setup a watch for every device subtree */ 1433 if (xs_watch(sc, "device", (char *)iovp1[i].iov_base, 1434 &xdl->dl_task, xen_hotplug, xdl)) 1435 printf("%s: failed to setup hotplug watch for \"%s\"\n", 1436 sc->sc_dev.dv_xname, (char *)iovp1[i].iov_base); 1437 SLIST_INSERT_HEAD(&sc->sc_devlists, xdl, dl_entry); 1438 xs_resfree(&xst, iovp2, iov2_cnt); 1439 iovp2 = NULL; 1440 iov2_cnt = 0; 1441 } 1442 1443 out: 1444 if (iovp2) 1445 xs_resfree(&xst, iovp2, iov2_cnt); 1446 xs_resfree(&xst, iovp1, iov1_cnt); 1447 return (error); 1448 } 1449 1450 void 1451 xen_hotplug(void *arg) 1452 { 1453 struct xen_devlist *xdl = arg; 1454 struct xen_softc *sc = xdl->dl_xen; 1455 struct xen_device *xdv, *xvdn; 1456 struct xs_transaction xst; 1457 struct iovec *iovp = NULL; 1458 int error, i, keep, iov_cnt = 0; 1459 char path[256]; 1460 int8_t *seen; 1461 1462 memset(&xst, 0, sizeof(xst)); 1463 xst.xst_id = 0; 1464 xst.xst_cookie = sc->sc_xs; 1465 1466 snprintf(path, sizeof(path), "device/%s", xdl->dl_node); 1467 if ((error = xs_cmd(&xst, XS_LIST, path, &iovp, &iov_cnt)) != 0) 1468 return; 1469 1470 seen = malloc(iov_cnt, M_TEMP, M_ZERO | M_WAITOK); 1471 1472 /* Detect all removed and kept devices */ 1473 LIST_FOREACH_SAFE(xdv, &xdl->dl_devs, dv_entry, xvdn) { 1474 for (i = 0, keep = 0; i < iov_cnt; i++) { 1475 if (!seen[i] && 1476 !strcmp(xdv->dv_unit, (char *)iovp[i].iov_base)) { 1477 seen[i]++; 1478 keep++; 1479 break; 1480 } 1481 } 1482 if (!keep) { 1483 DPRINTF("%s: removing \"%s/%s\"\n", sc->sc_dev.dv_xname, 1484 xdl->dl_node, xdv->dv_unit); 1485 LIST_REMOVE(xdv, dv_entry); 1486 config_detach(xdv->dv_dev, 0); 1487 free(xdv, M_DEVBUF, sizeof(struct xen_device)); 1488 } 1489 } 1490 1491 /* Attach all new devices */ 1492 for (i = 0; i < iov_cnt; i++) { 1493 if (seen[i]) 1494 continue; 1495 DPRINTF("%s: attaching \"%s/%s\"\n", sc->sc_dev.dv_xname, 1496 xdl->dl_node, (const char *)iovp[i].iov_base); 1497 error = xen_attach_device(sc, xdl, xdl->dl_node, 1498 (const char *)iovp[i].iov_base); 1499 if (error) { 1500 printf("%s: failed to attach \"%s/%s\"\n", 1501 sc->sc_dev.dv_xname, path, 1502 (const char *)iovp[i].iov_base); 1503 continue; 1504 } 1505 } 1506 1507 free(seen, M_TEMP, iov_cnt); 1508 1509 xs_resfree(&xst, iovp, iov_cnt); 1510 } 1511 1512 #include <machine/pio.h> 1513 1514 #define XMI_PORT 0x10 1515 #define XMI_MAGIC 0x49d2 1516 #define XMI_UNPLUG_IDE 0x01 1517 #define XMI_UNPLUG_NIC 0x02 1518 #define XMI_UNPLUG_IDESEC 0x04 1519 1520 void 1521 xen_disable_emulated_devices(struct xen_softc *sc) 1522 { 1523 #if defined(__i386__) || defined(__amd64__) 1524 ushort unplug = 0; 1525 1526 if (inw(XMI_PORT) != XMI_MAGIC) { 1527 printf("%s: failed to disable emulated devices\n", 1528 sc->sc_dev.dv_xname); 1529 return; 1530 } 1531 if (sc->sc_unplug & XEN_UNPLUG_IDE) 1532 unplug |= XMI_UNPLUG_IDE; 1533 if (sc->sc_unplug & XEN_UNPLUG_IDESEC) 1534 unplug |= XMI_UNPLUG_IDESEC; 1535 if (sc->sc_unplug & XEN_UNPLUG_NIC) 1536 unplug |= XMI_UNPLUG_NIC; 1537 if (unplug) 1538 outw(XMI_PORT, unplug); 1539 #endif /* __i386__ || __amd64__ */ 1540 } 1541 1542 void 1543 xen_unplug_emulated(void *xsc, int what) 1544 { 1545 struct xen_softc *sc = xsc; 1546 1547 sc->sc_unplug |= what; 1548 } 1549