1 /* $NetBSD: xhci.c,v 1.28 2014/11/18 10:18:45 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2013 Jonathan A. Kollasch 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 26 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(0, "$NetBSD: xhci.c,v 1.28 2014/11/18 10:18:45 skrll Exp $"); 31 32 #include "opt_usb.h" 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/kmem.h> 38 #include <sys/malloc.h> 39 #include <sys/device.h> 40 #include <sys/select.h> 41 #include <sys/proc.h> 42 #include <sys/queue.h> 43 #include <sys/mutex.h> 44 #include <sys/condvar.h> 45 #include <sys/bus.h> 46 #include <sys/cpu.h> 47 #include <sys/sysctl.h> 48 49 #include <machine/endian.h> 50 51 #include <dev/usb/usb.h> 52 #include <dev/usb/usbdi.h> 53 #include <dev/usb/usbdivar.h> 54 #include <dev/usb/usbhist.h> 55 #include <dev/usb/usb_mem.h> 56 #include <dev/usb/usb_quirks.h> 57 58 #include <dev/usb/xhcireg.h> 59 #include <dev/usb/xhcivar.h> 60 #include <dev/usb/usbroothub_subr.h> 61 62 63 #ifdef USB_DEBUG 64 #ifndef XHCI_DEBUG 65 #define xhcidebug 0 66 #else 67 static int xhcidebug = 0; 68 69 SYSCTL_SETUP(sysctl_hw_xhci_setup, "sysctl hw.xhci setup") 70 { 71 int err; 72 const struct sysctlnode *rnode; 73 const struct sysctlnode *cnode; 74 75 err = sysctl_createv(clog, 0, NULL, &rnode, 76 CTLFLAG_PERMANENT, CTLTYPE_NODE, "xhci", 77 SYSCTL_DESCR("xhci global controls"), 78 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 79 80 if (err) 81 goto fail; 82 83 /* control debugging printfs */ 84 err = sysctl_createv(clog, 0, &rnode, &cnode, 85 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, 86 "debug", SYSCTL_DESCR("Enable debugging output"), 87 NULL, 0, &xhcidebug, sizeof(xhcidebug), CTL_CREATE, CTL_EOL); 88 if (err) 89 goto fail; 90 91 return; 92 fail: 93 aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, err); 94 } 95 96 #endif /* XHCI_DEBUG */ 97 #endif /* USB_DEBUG */ 98 99 #define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(xhcidebug,N,FMT,A,B,C,D) 100 #define XHCIHIST_FUNC() USBHIST_FUNC() 101 #define XHCIHIST_CALLED(name) USBHIST_CALLED(xhcidebug) 102 103 #define XHCI_DCI_SLOT 0 104 #define XHCI_DCI_EP_CONTROL 1 105 106 #define XHCI_ICI_INPUT_CONTROL 0 107 108 struct xhci_pipe { 109 struct usbd_pipe xp_pipe; 110 }; 111 112 #define XHCI_INTR_ENDPT 1 113 #define XHCI_COMMAND_RING_TRBS 256 114 #define XHCI_EVENT_RING_TRBS 256 115 #define XHCI_EVENT_RING_SEGMENTS 1 116 #define XHCI_TRB_3_ED_BIT XHCI_TRB_3_ISP_BIT 117 118 static usbd_status xhci_open(usbd_pipe_handle); 119 static int xhci_intr1(struct xhci_softc * const); 120 static void xhci_softintr(void *); 121 static void xhci_poll(struct usbd_bus *); 122 static usbd_status xhci_allocm(struct usbd_bus *, usb_dma_t *, uint32_t); 123 static void xhci_freem(struct usbd_bus *, usb_dma_t *); 124 static usbd_xfer_handle xhci_allocx(struct usbd_bus *); 125 static void xhci_freex(struct usbd_bus *, usbd_xfer_handle); 126 static void xhci_get_lock(struct usbd_bus *, kmutex_t **); 127 static usbd_status xhci_new_device(device_t, usbd_bus_handle, int, int, int, 128 struct usbd_port *); 129 130 static usbd_status xhci_configure_endpoint(usbd_pipe_handle); 131 static usbd_status xhci_unconfigure_endpoint(usbd_pipe_handle); 132 static usbd_status xhci_reset_endpoint(usbd_pipe_handle); 133 //static usbd_status xhci_stop_endpoint(usbd_pipe_handle); 134 135 static usbd_status xhci_set_dequeue(usbd_pipe_handle); 136 137 static usbd_status xhci_do_command(struct xhci_softc * const, 138 struct xhci_trb * const, int); 139 static usbd_status xhci_init_slot(struct xhci_softc * const, uint32_t, 140 int, int, int, int); 141 static usbd_status xhci_enable_slot(struct xhci_softc * const, 142 uint8_t * const); 143 static usbd_status xhci_address_device(struct xhci_softc * const, 144 uint64_t, uint8_t, bool); 145 static usbd_status xhci_update_ep0_mps(struct xhci_softc * const, 146 struct xhci_slot * const, u_int); 147 static usbd_status xhci_ring_init(struct xhci_softc * const, 148 struct xhci_ring * const, size_t, size_t); 149 static void xhci_ring_free(struct xhci_softc * const, struct xhci_ring * const); 150 151 static void xhci_noop(usbd_pipe_handle); 152 153 static usbd_status xhci_root_ctrl_transfer(usbd_xfer_handle); 154 static usbd_status xhci_root_ctrl_start(usbd_xfer_handle); 155 static void xhci_root_ctrl_abort(usbd_xfer_handle); 156 static void xhci_root_ctrl_close(usbd_pipe_handle); 157 static void xhci_root_ctrl_done(usbd_xfer_handle); 158 159 static usbd_status xhci_root_intr_transfer(usbd_xfer_handle); 160 static usbd_status xhci_root_intr_start(usbd_xfer_handle); 161 static void xhci_root_intr_abort(usbd_xfer_handle); 162 static void xhci_root_intr_close(usbd_pipe_handle); 163 static void xhci_root_intr_done(usbd_xfer_handle); 164 165 static usbd_status xhci_device_ctrl_transfer(usbd_xfer_handle); 166 static usbd_status xhci_device_ctrl_start(usbd_xfer_handle); 167 static void xhci_device_ctrl_abort(usbd_xfer_handle); 168 static void xhci_device_ctrl_close(usbd_pipe_handle); 169 static void xhci_device_ctrl_done(usbd_xfer_handle); 170 171 static usbd_status xhci_device_intr_transfer(usbd_xfer_handle); 172 static usbd_status xhci_device_intr_start(usbd_xfer_handle); 173 static void xhci_device_intr_abort(usbd_xfer_handle); 174 static void xhci_device_intr_close(usbd_pipe_handle); 175 static void xhci_device_intr_done(usbd_xfer_handle); 176 177 static usbd_status xhci_device_bulk_transfer(usbd_xfer_handle); 178 static usbd_status xhci_device_bulk_start(usbd_xfer_handle); 179 static void xhci_device_bulk_abort(usbd_xfer_handle); 180 static void xhci_device_bulk_close(usbd_pipe_handle); 181 static void xhci_device_bulk_done(usbd_xfer_handle); 182 183 static void xhci_timeout(void *); 184 static void xhci_timeout_task(void *); 185 186 static const struct usbd_bus_methods xhci_bus_methods = { 187 .open_pipe = xhci_open, 188 .soft_intr = xhci_softintr, 189 .do_poll = xhci_poll, 190 .allocm = xhci_allocm, 191 .freem = xhci_freem, 192 .allocx = xhci_allocx, 193 .freex = xhci_freex, 194 .get_lock = xhci_get_lock, 195 .new_device = xhci_new_device, 196 }; 197 198 static const struct usbd_pipe_methods xhci_root_ctrl_methods = { 199 .transfer = xhci_root_ctrl_transfer, 200 .start = xhci_root_ctrl_start, 201 .abort = xhci_root_ctrl_abort, 202 .close = xhci_root_ctrl_close, 203 .cleartoggle = xhci_noop, 204 .done = xhci_root_ctrl_done, 205 }; 206 207 static const struct usbd_pipe_methods xhci_root_intr_methods = { 208 .transfer = xhci_root_intr_transfer, 209 .start = xhci_root_intr_start, 210 .abort = xhci_root_intr_abort, 211 .close = xhci_root_intr_close, 212 .cleartoggle = xhci_noop, 213 .done = xhci_root_intr_done, 214 }; 215 216 217 static const struct usbd_pipe_methods xhci_device_ctrl_methods = { 218 .transfer = xhci_device_ctrl_transfer, 219 .start = xhci_device_ctrl_start, 220 .abort = xhci_device_ctrl_abort, 221 .close = xhci_device_ctrl_close, 222 .cleartoggle = xhci_noop, 223 .done = xhci_device_ctrl_done, 224 }; 225 226 static const struct usbd_pipe_methods xhci_device_isoc_methods = { 227 .cleartoggle = xhci_noop, 228 }; 229 230 static const struct usbd_pipe_methods xhci_device_bulk_methods = { 231 .transfer = xhci_device_bulk_transfer, 232 .start = xhci_device_bulk_start, 233 .abort = xhci_device_bulk_abort, 234 .close = xhci_device_bulk_close, 235 .cleartoggle = xhci_noop, 236 .done = xhci_device_bulk_done, 237 }; 238 239 static const struct usbd_pipe_methods xhci_device_intr_methods = { 240 .transfer = xhci_device_intr_transfer, 241 .start = xhci_device_intr_start, 242 .abort = xhci_device_intr_abort, 243 .close = xhci_device_intr_close, 244 .cleartoggle = xhci_noop, 245 .done = xhci_device_intr_done, 246 }; 247 248 static inline uint32_t 249 xhci_read_4(const struct xhci_softc * const sc, bus_size_t offset) 250 { 251 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, offset); 252 } 253 254 #if 0 /* unused */ 255 static inline void 256 xhci_write_4(const struct xhci_softc * const sc, bus_size_t offset, 257 uint32_t value) 258 { 259 bus_space_write_4(sc->sc_iot, sc->sc_ioh, offset, value); 260 } 261 #endif /* unused */ 262 263 static inline uint32_t 264 xhci_cap_read_4(const struct xhci_softc * const sc, bus_size_t offset) 265 { 266 return bus_space_read_4(sc->sc_iot, sc->sc_cbh, offset); 267 } 268 269 static inline uint32_t 270 xhci_op_read_4(const struct xhci_softc * const sc, bus_size_t offset) 271 { 272 return bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 273 } 274 275 static inline void 276 xhci_op_write_4(const struct xhci_softc * const sc, bus_size_t offset, 277 uint32_t value) 278 { 279 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value); 280 } 281 282 #if 0 /* unused */ 283 static inline uint64_t 284 xhci_op_read_8(const struct xhci_softc * const sc, bus_size_t offset) 285 { 286 uint64_t value; 287 288 if (sc->sc_ac64) { 289 #ifdef XHCI_USE_BUS_SPACE_8 290 value = bus_space_read_8(sc->sc_iot, sc->sc_obh, offset); 291 #else 292 value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 293 value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_obh, 294 offset + 4) << 32; 295 #endif 296 } else { 297 value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 298 } 299 300 return value; 301 } 302 #endif /* unused */ 303 304 static inline void 305 xhci_op_write_8(const struct xhci_softc * const sc, bus_size_t offset, 306 uint64_t value) 307 { 308 if (sc->sc_ac64) { 309 #ifdef XHCI_USE_BUS_SPACE_8 310 bus_space_write_8(sc->sc_iot, sc->sc_obh, offset, value); 311 #else 312 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 0, 313 (value >> 0) & 0xffffffff); 314 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 4, 315 (value >> 32) & 0xffffffff); 316 #endif 317 } else { 318 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value); 319 } 320 } 321 322 static inline uint32_t 323 xhci_rt_read_4(const struct xhci_softc * const sc, bus_size_t offset) 324 { 325 return bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 326 } 327 328 static inline void 329 xhci_rt_write_4(const struct xhci_softc * const sc, bus_size_t offset, 330 uint32_t value) 331 { 332 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value); 333 } 334 335 #if 0 /* unused */ 336 static inline uint64_t 337 xhci_rt_read_8(const struct xhci_softc * const sc, bus_size_t offset) 338 { 339 uint64_t value; 340 341 if (sc->sc_ac64) { 342 #ifdef XHCI_USE_BUS_SPACE_8 343 value = bus_space_read_8(sc->sc_iot, sc->sc_rbh, offset); 344 #else 345 value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 346 value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_rbh, 347 offset + 4) << 32; 348 #endif 349 } else { 350 value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 351 } 352 353 return value; 354 } 355 #endif /* unused */ 356 357 static inline void 358 xhci_rt_write_8(const struct xhci_softc * const sc, bus_size_t offset, 359 uint64_t value) 360 { 361 if (sc->sc_ac64) { 362 #ifdef XHCI_USE_BUS_SPACE_8 363 bus_space_write_8(sc->sc_iot, sc->sc_rbh, offset, value); 364 #else 365 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 0, 366 (value >> 0) & 0xffffffff); 367 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 4, 368 (value >> 32) & 0xffffffff); 369 #endif 370 } else { 371 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value); 372 } 373 } 374 375 #if 0 /* unused */ 376 static inline uint32_t 377 xhci_db_read_4(const struct xhci_softc * const sc, bus_size_t offset) 378 { 379 return bus_space_read_4(sc->sc_iot, sc->sc_dbh, offset); 380 } 381 #endif /* unused */ 382 383 static inline void 384 xhci_db_write_4(const struct xhci_softc * const sc, bus_size_t offset, 385 uint32_t value) 386 { 387 bus_space_write_4(sc->sc_iot, sc->sc_dbh, offset, value); 388 } 389 390 /* --- */ 391 392 static inline uint8_t 393 xhci_ep_get_type(usb_endpoint_descriptor_t * const ed) 394 { 395 u_int eptype; 396 397 switch (UE_GET_XFERTYPE(ed->bmAttributes)) { 398 case UE_CONTROL: 399 eptype = 0x0; 400 break; 401 case UE_ISOCHRONOUS: 402 eptype = 0x1; 403 break; 404 case UE_BULK: 405 eptype = 0x2; 406 break; 407 case UE_INTERRUPT: 408 eptype = 0x3; 409 break; 410 } 411 412 if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) || 413 (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN)) 414 return eptype | 0x4; 415 else 416 return eptype; 417 } 418 419 static u_int 420 xhci_ep_get_dci(usb_endpoint_descriptor_t * const ed) 421 { 422 /* xHCI 1.0 section 4.5.1 */ 423 u_int epaddr = UE_GET_ADDR(ed->bEndpointAddress); 424 u_int in = 0; 425 426 if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) || 427 (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN)) 428 in = 1; 429 430 return epaddr * 2 + in; 431 } 432 433 static inline u_int 434 xhci_dci_to_ici(const u_int i) 435 { 436 return i + 1; 437 } 438 439 static inline void * 440 xhci_slot_get_dcv(struct xhci_softc * const sc, struct xhci_slot * const xs, 441 const u_int dci) 442 { 443 return KERNADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci); 444 } 445 446 #if 0 /* unused */ 447 static inline bus_addr_t 448 xhci_slot_get_dcp(struct xhci_softc * const sc, struct xhci_slot * const xs, 449 const u_int dci) 450 { 451 return DMAADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci); 452 } 453 #endif /* unused */ 454 455 static inline void * 456 xhci_slot_get_icv(struct xhci_softc * const sc, struct xhci_slot * const xs, 457 const u_int ici) 458 { 459 return KERNADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici); 460 } 461 462 static inline bus_addr_t 463 xhci_slot_get_icp(struct xhci_softc * const sc, struct xhci_slot * const xs, 464 const u_int ici) 465 { 466 return DMAADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici); 467 } 468 469 static inline struct xhci_trb * 470 xhci_ring_trbv(struct xhci_ring * const xr, u_int idx) 471 { 472 return KERNADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx); 473 } 474 475 static inline bus_addr_t 476 xhci_ring_trbp(struct xhci_ring * const xr, u_int idx) 477 { 478 return DMAADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx); 479 } 480 481 static inline void 482 xhci_trb_put(struct xhci_trb * const trb, uint64_t parameter, uint32_t status, 483 uint32_t control) 484 { 485 trb->trb_0 = parameter; 486 trb->trb_2 = status; 487 trb->trb_3 = control; 488 } 489 490 /* --- */ 491 492 void 493 xhci_childdet(device_t self, device_t child) 494 { 495 struct xhci_softc * const sc = device_private(self); 496 497 KASSERT(sc->sc_child == child); 498 if (child == sc->sc_child) 499 sc->sc_child = NULL; 500 } 501 502 int 503 xhci_detach(struct xhci_softc *sc, int flags) 504 { 505 int rv = 0; 506 507 if (sc->sc_child != NULL) 508 rv = config_detach(sc->sc_child, flags); 509 510 if (rv != 0) 511 return (rv); 512 513 /* XXX unconfigure/free slots */ 514 515 /* verify: */ 516 xhci_rt_write_4(sc, XHCI_IMAN(0), 0); 517 xhci_op_write_4(sc, XHCI_USBCMD, 0); 518 /* do we need to wait for stop? */ 519 520 xhci_op_write_8(sc, XHCI_CRCR, 0); 521 xhci_ring_free(sc, &sc->sc_cr); 522 cv_destroy(&sc->sc_command_cv); 523 524 xhci_rt_write_4(sc, XHCI_ERSTSZ(0), 0); 525 xhci_rt_write_8(sc, XHCI_ERSTBA(0), 0); 526 xhci_rt_write_8(sc, XHCI_ERDP(0), 0|XHCI_ERDP_LO_BUSY); 527 xhci_ring_free(sc, &sc->sc_er); 528 529 usb_freemem(&sc->sc_bus, &sc->sc_eventst_dma); 530 531 xhci_op_write_8(sc, XHCI_DCBAAP, 0); 532 usb_freemem(&sc->sc_bus, &sc->sc_dcbaa_dma); 533 534 kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) * sc->sc_maxslots); 535 536 mutex_destroy(&sc->sc_lock); 537 mutex_destroy(&sc->sc_intr_lock); 538 539 pool_cache_destroy(sc->sc_xferpool); 540 541 return rv; 542 } 543 544 int 545 xhci_activate(device_t self, enum devact act) 546 { 547 struct xhci_softc * const sc = device_private(self); 548 549 switch (act) { 550 case DVACT_DEACTIVATE: 551 sc->sc_dying = true; 552 return 0; 553 default: 554 return EOPNOTSUPP; 555 } 556 } 557 558 bool 559 xhci_suspend(device_t dv, const pmf_qual_t *qual) 560 { 561 return false; 562 } 563 564 bool 565 xhci_resume(device_t dv, const pmf_qual_t *qual) 566 { 567 return false; 568 } 569 570 bool 571 xhci_shutdown(device_t self, int flags) 572 { 573 return false; 574 } 575 576 577 static void 578 hexdump(const char *msg, const void *base, size_t len) 579 { 580 #if 0 581 size_t cnt; 582 const uint32_t *p; 583 extern paddr_t vtophys(vaddr_t); 584 585 p = base; 586 cnt = 0; 587 588 printf("*** %s (%zu bytes @ %p %p)\n", msg, len, base, 589 (void *)vtophys((vaddr_t)base)); 590 591 while (cnt < len) { 592 if (cnt % 16 == 0) 593 printf("%p: ", p); 594 else if (cnt % 8 == 0) 595 printf(" |"); 596 printf(" %08x", *p++); 597 cnt += 4; 598 if (cnt % 16 == 0) 599 printf("\n"); 600 } 601 #endif 602 } 603 604 605 int 606 xhci_init(struct xhci_softc *sc) 607 { 608 bus_size_t bsz; 609 uint32_t cap, hcs1, hcs2, hcc, dboff, rtsoff; 610 uint32_t ecp, ecr; 611 uint32_t usbcmd, usbsts, pagesize, config; 612 int i; 613 uint16_t hciversion; 614 uint8_t caplength; 615 616 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 617 618 /* XXX Low/Full/High speeds for now */ 619 sc->sc_bus.usbrev = USBREV_2_0; 620 621 cap = xhci_read_4(sc, XHCI_CAPLENGTH); 622 caplength = XHCI_CAP_CAPLENGTH(cap); 623 hciversion = XHCI_CAP_HCIVERSION(cap); 624 625 if ((hciversion < 0x0096) || (hciversion > 0x0100)) { 626 aprint_normal_dev(sc->sc_dev, 627 "xHCI version %x.%x not known to be supported\n", 628 (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff); 629 } else { 630 aprint_verbose_dev(sc->sc_dev, "xHCI version %x.%x\n", 631 (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff); 632 } 633 634 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 0, caplength, 635 &sc->sc_cbh) != 0) { 636 aprint_error_dev(sc->sc_dev, "capability subregion failure\n"); 637 return ENOMEM; 638 } 639 640 hcs1 = xhci_cap_read_4(sc, XHCI_HCSPARAMS1); 641 sc->sc_maxslots = XHCI_HCS1_MAXSLOTS(hcs1); 642 sc->sc_maxintrs = XHCI_HCS1_MAXINTRS(hcs1); 643 sc->sc_maxports = XHCI_HCS1_MAXPORTS(hcs1); 644 hcs2 = xhci_cap_read_4(sc, XHCI_HCSPARAMS2); 645 (void)xhci_cap_read_4(sc, XHCI_HCSPARAMS3); 646 hcc = xhci_cap_read_4(sc, XHCI_HCCPARAMS); 647 648 sc->sc_ac64 = XHCI_HCC_AC64(hcc); 649 sc->sc_ctxsz = XHCI_HCC_CSZ(hcc) ? 64 : 32; 650 aprint_debug_dev(sc->sc_dev, "ac64 %d ctxsz %d\n", sc->sc_ac64, 651 sc->sc_ctxsz); 652 653 aprint_debug_dev(sc->sc_dev, "xECP %x\n", XHCI_HCC_XECP(hcc) * 4); 654 ecp = XHCI_HCC_XECP(hcc) * 4; 655 while (ecp != 0) { 656 ecr = xhci_read_4(sc, ecp); 657 aprint_debug_dev(sc->sc_dev, "ECR %x: %08x\n", ecp, ecr); 658 switch (XHCI_XECP_ID(ecr)) { 659 case XHCI_ID_PROTOCOLS: { 660 uint32_t w0, w4, w8; 661 uint16_t w2; 662 w0 = xhci_read_4(sc, ecp + 0); 663 w2 = (w0 >> 16) & 0xffff; 664 w4 = xhci_read_4(sc, ecp + 4); 665 w8 = xhci_read_4(sc, ecp + 8); 666 aprint_debug_dev(sc->sc_dev, "SP: %08x %08x %08x\n", 667 w0, w4, w8); 668 if (w4 == 0x20425355 && w2 == 0x0300) { 669 sc->sc_ss_port_start = (w8 >> 0) & 0xff;; 670 sc->sc_ss_port_count = (w8 >> 8) & 0xff;; 671 } 672 if (w4 == 0x20425355 && w2 == 0x0200) { 673 sc->sc_hs_port_start = (w8 >> 0) & 0xff; 674 sc->sc_hs_port_count = (w8 >> 8) & 0xff; 675 } 676 break; 677 } 678 default: 679 break; 680 } 681 ecr = xhci_read_4(sc, ecp); 682 if (XHCI_XECP_NEXT(ecr) == 0) { 683 ecp = 0; 684 } else { 685 ecp += XHCI_XECP_NEXT(ecr) * 4; 686 } 687 } 688 689 bsz = XHCI_PORTSC(sc->sc_maxports + 1); 690 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, caplength, bsz, 691 &sc->sc_obh) != 0) { 692 aprint_error_dev(sc->sc_dev, "operational subregion failure\n"); 693 return ENOMEM; 694 } 695 696 dboff = xhci_cap_read_4(sc, XHCI_DBOFF); 697 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, dboff, 698 sc->sc_maxslots * 4, &sc->sc_dbh) != 0) { 699 aprint_error_dev(sc->sc_dev, "doorbell subregion failure\n"); 700 return ENOMEM; 701 } 702 703 rtsoff = xhci_cap_read_4(sc, XHCI_RTSOFF); 704 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, rtsoff, 705 sc->sc_maxintrs * 0x20, &sc->sc_rbh) != 0) { 706 aprint_error_dev(sc->sc_dev, "runtime subregion failure\n"); 707 return ENOMEM; 708 } 709 710 for (i = 0; i < 100; i++) { 711 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 712 if ((usbsts & XHCI_STS_CNR) == 0) 713 break; 714 usb_delay_ms(&sc->sc_bus, 1); 715 } 716 if (i >= 100) 717 return EIO; 718 719 usbcmd = 0; 720 xhci_op_write_4(sc, XHCI_USBCMD, usbcmd); 721 usb_delay_ms(&sc->sc_bus, 1); 722 723 usbcmd = XHCI_CMD_HCRST; 724 xhci_op_write_4(sc, XHCI_USBCMD, usbcmd); 725 for (i = 0; i < 100; i++) { 726 usbcmd = xhci_op_read_4(sc, XHCI_USBCMD); 727 if ((usbcmd & XHCI_CMD_HCRST) == 0) 728 break; 729 usb_delay_ms(&sc->sc_bus, 1); 730 } 731 if (i >= 100) 732 return EIO; 733 734 for (i = 0; i < 100; i++) { 735 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 736 if ((usbsts & XHCI_STS_CNR) == 0) 737 break; 738 usb_delay_ms(&sc->sc_bus, 1); 739 } 740 if (i >= 100) 741 return EIO; 742 743 pagesize = xhci_op_read_4(sc, XHCI_PAGESIZE); 744 aprint_debug_dev(sc->sc_dev, "PAGESIZE 0x%08x\n", pagesize); 745 pagesize = ffs(pagesize); 746 if (pagesize == 0) 747 return EIO; 748 sc->sc_pgsz = 1 << (12 + (pagesize - 1)); 749 aprint_debug_dev(sc->sc_dev, "sc_pgsz 0x%08x\n", (uint32_t)sc->sc_pgsz); 750 aprint_debug_dev(sc->sc_dev, "sc_maxslots 0x%08x\n", 751 (uint32_t)sc->sc_maxslots); 752 753 usbd_status err; 754 755 sc->sc_maxspbuf = XHCI_HCS2_MAXSPBUF(hcs2); 756 aprint_debug_dev(sc->sc_dev, "sc_maxspbuf %d\n", sc->sc_maxspbuf); 757 if (sc->sc_maxspbuf != 0) { 758 err = usb_allocmem(&sc->sc_bus, 759 sizeof(uint64_t) * sc->sc_maxspbuf, sizeof(uint64_t), 760 &sc->sc_spbufarray_dma); 761 if (err) 762 return err; 763 764 sc->sc_spbuf_dma = kmem_zalloc(sizeof(*sc->sc_spbuf_dma) * sc->sc_maxspbuf, KM_SLEEP); 765 uint64_t *spbufarray = KERNADDR(&sc->sc_spbufarray_dma, 0); 766 for (i = 0; i < sc->sc_maxspbuf; i++) { 767 usb_dma_t * const dma = &sc->sc_spbuf_dma[i]; 768 /* allocate contexts */ 769 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, 770 sc->sc_pgsz, dma); 771 if (err) 772 return err; 773 spbufarray[i] = htole64(DMAADDR(dma, 0)); 774 usb_syncmem(dma, 0, sc->sc_pgsz, 775 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 776 } 777 778 usb_syncmem(&sc->sc_spbufarray_dma, 0, 779 sizeof(uint64_t) * sc->sc_maxspbuf, BUS_DMASYNC_PREWRITE); 780 } 781 782 config = xhci_op_read_4(sc, XHCI_CONFIG); 783 config &= ~0xFF; 784 config |= sc->sc_maxslots & 0xFF; 785 xhci_op_write_4(sc, XHCI_CONFIG, config); 786 787 err = xhci_ring_init(sc, &sc->sc_cr, XHCI_COMMAND_RING_TRBS, 788 XHCI_COMMAND_RING_SEGMENTS_ALIGN); 789 if (err) { 790 aprint_error_dev(sc->sc_dev, "command ring init fail\n"); 791 return err; 792 } 793 794 err = xhci_ring_init(sc, &sc->sc_er, XHCI_EVENT_RING_TRBS, 795 XHCI_EVENT_RING_SEGMENTS_ALIGN); 796 if (err) { 797 aprint_error_dev(sc->sc_dev, "event ring init fail\n"); 798 return err; 799 } 800 801 usb_dma_t *dma; 802 size_t size; 803 size_t align; 804 805 dma = &sc->sc_eventst_dma; 806 size = roundup2(XHCI_EVENT_RING_SEGMENTS * XHCI_ERSTE_SIZE, 807 XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN); 808 KASSERT(size <= (512 * 1024)); 809 align = XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN; 810 err = usb_allocmem(&sc->sc_bus, size, align, dma); 811 812 memset(KERNADDR(dma, 0), 0, size); 813 usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE); 814 aprint_debug_dev(sc->sc_dev, "eventst: %s %016jx %p %zx\n", 815 usbd_errstr(err), 816 (uintmax_t)DMAADDR(&sc->sc_eventst_dma, 0), 817 KERNADDR(&sc->sc_eventst_dma, 0), 818 sc->sc_eventst_dma.block->size); 819 820 dma = &sc->sc_dcbaa_dma; 821 size = (1 + sc->sc_maxslots) * sizeof(uint64_t); 822 KASSERT(size <= 2048); 823 align = XHCI_DEVICE_CONTEXT_BASE_ADDRESS_ARRAY_ALIGN; 824 err = usb_allocmem(&sc->sc_bus, size, align, dma); 825 826 memset(KERNADDR(dma, 0), 0, size); 827 if (sc->sc_maxspbuf != 0) { 828 /* 829 * DCBA entry 0 hold the scratchbuf array pointer. 830 */ 831 *(uint64_t *)KERNADDR(dma, 0) = 832 htole64(DMAADDR(&sc->sc_spbufarray_dma, 0)); 833 } 834 usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE); 835 aprint_debug_dev(sc->sc_dev, "dcbaa: %s %016jx %p %zx\n", 836 usbd_errstr(err), 837 (uintmax_t)DMAADDR(&sc->sc_dcbaa_dma, 0), 838 KERNADDR(&sc->sc_dcbaa_dma, 0), 839 sc->sc_dcbaa_dma.block->size); 840 841 sc->sc_slots = kmem_zalloc(sizeof(*sc->sc_slots) * sc->sc_maxslots, 842 KM_SLEEP); 843 844 cv_init(&sc->sc_command_cv, "xhcicmd"); 845 846 struct xhci_erste *erst; 847 erst = KERNADDR(&sc->sc_eventst_dma, 0); 848 erst[0].erste_0 = htole64(xhci_ring_trbp(&sc->sc_er, 0)); 849 erst[0].erste_2 = htole32(XHCI_EVENT_RING_TRBS); 850 erst[0].erste_3 = htole32(0); 851 usb_syncmem(&sc->sc_eventst_dma, 0, 852 XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS, BUS_DMASYNC_PREWRITE); 853 854 xhci_rt_write_4(sc, XHCI_ERSTSZ(0), XHCI_EVENT_RING_SEGMENTS); 855 xhci_rt_write_8(sc, XHCI_ERSTBA(0), DMAADDR(&sc->sc_eventst_dma, 0)); 856 xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(&sc->sc_er, 0) | 857 XHCI_ERDP_LO_BUSY); 858 xhci_op_write_8(sc, XHCI_DCBAAP, DMAADDR(&sc->sc_dcbaa_dma, 0)); 859 xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(&sc->sc_cr, 0) | 860 sc->sc_cr.xr_cs); 861 862 #if 0 863 hexdump("eventst", KERNADDR(&sc->sc_eventst_dma, 0), 864 XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS); 865 #endif 866 867 xhci_rt_write_4(sc, XHCI_IMAN(0), XHCI_IMAN_INTR_ENA); 868 xhci_rt_write_4(sc, XHCI_IMOD(0), 0); 869 870 xhci_op_write_4(sc, XHCI_USBCMD, XHCI_CMD_INTE|XHCI_CMD_RS); /* Go! */ 871 aprint_debug_dev(sc->sc_dev, "USBCMD %08"PRIx32"\n", 872 xhci_op_read_4(sc, XHCI_USBCMD)); 873 874 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTUSB); 875 mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_SCHED); 876 cv_init(&sc->sc_softwake_cv, "xhciab"); 877 878 sc->sc_xferpool = pool_cache_init(sizeof(struct xhci_xfer), 0, 0, 0, 879 "xhcixfer", NULL, IPL_USB, NULL, NULL, NULL); 880 881 /* Set up the bus struct. */ 882 sc->sc_bus.methods = &xhci_bus_methods; 883 sc->sc_bus.pipe_size = sizeof(struct xhci_pipe); 884 885 return USBD_NORMAL_COMPLETION; 886 } 887 888 int 889 xhci_intr(void *v) 890 { 891 struct xhci_softc * const sc = v; 892 int ret = 0; 893 894 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 895 896 if (sc == NULL) 897 return 0; 898 899 mutex_spin_enter(&sc->sc_intr_lock); 900 901 if (sc->sc_dying || !device_has_power(sc->sc_dev)) 902 goto done; 903 904 /* If we get an interrupt while polling, then just ignore it. */ 905 if (sc->sc_bus.use_polling) { 906 #ifdef DIAGNOSTIC 907 DPRINTFN(16, "ignored interrupt while polling", 0, 0, 0, 0); 908 #endif 909 goto done; 910 } 911 912 ret = xhci_intr1(sc); 913 done: 914 mutex_spin_exit(&sc->sc_intr_lock); 915 return ret; 916 } 917 918 int 919 xhci_intr1(struct xhci_softc * const sc) 920 { 921 uint32_t usbsts; 922 uint32_t iman; 923 924 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 925 926 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 927 DPRINTFN(16, "USBSTS %08x", usbsts, 0, 0, 0); 928 #if 0 929 if ((usbsts & (XHCI_STS_EINT|XHCI_STS_PCD)) == 0) { 930 return 0; 931 } 932 #endif 933 xhci_op_write_4(sc, XHCI_USBSTS, 934 usbsts & (2|XHCI_STS_EINT|XHCI_STS_PCD)); /* XXX */ 935 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 936 DPRINTFN(16, "USBSTS %08x", usbsts, 0, 0, 0); 937 938 iman = xhci_rt_read_4(sc, XHCI_IMAN(0)); 939 DPRINTFN(16, "IMAN0 %08x", iman, 0, 0, 0); 940 if ((iman & XHCI_IMAN_INTR_PEND) == 0) { 941 return 0; 942 } 943 xhci_rt_write_4(sc, XHCI_IMAN(0), iman); 944 iman = xhci_rt_read_4(sc, XHCI_IMAN(0)); 945 DPRINTFN(16, "IMAN0 %08x", iman, 0, 0, 0); 946 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 947 DPRINTFN(16, "USBSTS %08x", usbsts, 0, 0, 0); 948 949 sc->sc_bus.no_intrs++; 950 usb_schedsoftintr(&sc->sc_bus); 951 952 return 1; 953 } 954 955 static usbd_status 956 xhci_configure_endpoint(usbd_pipe_handle pipe) 957 { 958 struct xhci_softc * const sc = pipe->device->bus->hci_private; 959 struct xhci_slot * const xs = pipe->device->hci_private; 960 const u_int dci = xhci_ep_get_dci(pipe->endpoint->edesc); 961 usb_endpoint_descriptor_t * const ed = pipe->endpoint->edesc; 962 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); 963 struct xhci_trb trb; 964 usbd_status err; 965 uint32_t *cp; 966 967 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 968 DPRINTFN(4, "dci %u epaddr 0x%02x attr 0x%02x", 969 dci, ed->bEndpointAddress, ed->bmAttributes, 0); 970 971 /* XXX ensure input context is available? */ 972 973 memset(xhci_slot_get_icv(sc, xs, 0), 0, sc->sc_pgsz); 974 975 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 976 cp[0] = htole32(0); 977 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(dci)); 978 979 /* set up input slot context */ 980 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); 981 cp[0] = htole32(XHCI_SCTX_0_CTX_NUM_SET(dci)); 982 cp[1] = htole32(0); 983 cp[2] = htole32(0); 984 cp[3] = htole32(0); 985 986 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(dci)); 987 if (xfertype == UE_INTERRUPT) { 988 cp[0] = htole32( 989 XHCI_EPCTX_0_IVAL_SET(3) /* XXX */ 990 ); 991 cp[1] = htole32( 992 XHCI_EPCTX_1_CERR_SET(3) | 993 XHCI_EPCTX_1_EPTYPE_SET(xhci_ep_get_type(pipe->endpoint->edesc)) | 994 XHCI_EPCTX_1_MAXB_SET(0) | 995 XHCI_EPCTX_1_MAXP_SIZE_SET(8) /* XXX */ 996 ); 997 cp[4] = htole32( 998 XHCI_EPCTX_4_AVG_TRB_LEN_SET(8) 999 ); 1000 } else { 1001 cp[0] = htole32(0); 1002 cp[1] = htole32( 1003 XHCI_EPCTX_1_CERR_SET(3) | 1004 XHCI_EPCTX_1_EPTYPE_SET(xhci_ep_get_type(pipe->endpoint->edesc)) | 1005 XHCI_EPCTX_1_MAXB_SET(0) | 1006 XHCI_EPCTX_1_MAXP_SIZE_SET(512) /* XXX */ 1007 ); 1008 } 1009 *(uint64_t *)(&cp[2]) = htole64( 1010 xhci_ring_trbp(&xs->xs_ep[dci].xe_tr, 0) | 1011 XHCI_EPCTX_2_DCS_SET(1)); 1012 1013 /* sync input contexts before they are read from memory */ 1014 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 1015 hexdump("input control context", xhci_slot_get_icv(sc, xs, 0), 1016 sc->sc_ctxsz * 1); 1017 hexdump("input endpoint context", xhci_slot_get_icv(sc, xs, 1018 xhci_dci_to_ici(dci)), sc->sc_ctxsz * 1); 1019 1020 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); 1021 trb.trb_2 = 0; 1022 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1023 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP); 1024 1025 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1026 1027 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 1028 hexdump("output context", xhci_slot_get_dcv(sc, xs, dci), 1029 sc->sc_ctxsz * 1); 1030 1031 return err; 1032 } 1033 1034 static usbd_status 1035 xhci_unconfigure_endpoint(usbd_pipe_handle pipe) 1036 { 1037 #ifdef USB_DEBUG 1038 struct xhci_slot * const xs = pipe->device->hci_private; 1039 #endif 1040 1041 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1042 DPRINTFN(4, "slot %u", xs->xs_idx, 0, 0, 0); 1043 1044 return USBD_NORMAL_COMPLETION; 1045 } 1046 1047 static usbd_status 1048 xhci_reset_endpoint(usbd_pipe_handle pipe) 1049 { 1050 struct xhci_softc * const sc = pipe->device->bus->hci_private; 1051 struct xhci_slot * const xs = pipe->device->hci_private; 1052 const u_int dci = xhci_ep_get_dci(pipe->endpoint->edesc); 1053 struct xhci_trb trb; 1054 usbd_status err; 1055 1056 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1057 DPRINTFN(4, "dci %u", dci, 0, 0, 0); 1058 1059 trb.trb_0 = 0; 1060 trb.trb_2 = 0; 1061 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1062 XHCI_TRB_3_EP_SET(dci) | 1063 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP); 1064 1065 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1066 1067 return err; 1068 } 1069 1070 #if 0 1071 static usbd_status 1072 xhci_stop_endpoint(usbd_pipe_handle pipe) 1073 { 1074 struct xhci_softc * const sc = pipe->device->bus->hci_private; 1075 struct xhci_slot * const xs = pipe->device->hci_private; 1076 struct xhci_trb trb; 1077 usbd_status err; 1078 const u_int dci = xhci_ep_get_dci(pipe->endpoint->edesc); 1079 1080 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1081 DPRINTFN(4, "dci %u", dci, 0, 0, 0); 1082 1083 trb.trb_0 = 0; 1084 trb.trb_2 = 0; 1085 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1086 XHCI_TRB_3_EP_SET(dci) | 1087 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP); 1088 1089 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1090 1091 return err; 1092 } 1093 #endif 1094 1095 static usbd_status 1096 xhci_set_dequeue(usbd_pipe_handle pipe) 1097 { 1098 struct xhci_softc * const sc = pipe->device->bus->hci_private; 1099 struct xhci_slot * const xs = pipe->device->hci_private; 1100 const u_int dci = xhci_ep_get_dci(pipe->endpoint->edesc); 1101 struct xhci_ring * const xr = &xs->xs_ep[dci].xe_tr; 1102 struct xhci_trb trb; 1103 usbd_status err; 1104 1105 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1106 DPRINTFN(4, "slot %u dci %u", xs->xs_idx, dci, 0, 0); 1107 1108 memset(xr->xr_trb, 0, xr->xr_ntrb * XHCI_TRB_SIZE); 1109 usb_syncmem(&xr->xr_dma, 0, xr->xr_ntrb * XHCI_TRB_SIZE, 1110 BUS_DMASYNC_PREWRITE); 1111 1112 xr->xr_ep = 0; 1113 xr->xr_cs = 1; 1114 1115 trb.trb_0 = xhci_ring_trbp(xr, 0) | 1; /* XXX */ 1116 trb.trb_2 = 0; 1117 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1118 XHCI_TRB_3_EP_SET(dci) | 1119 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE); 1120 1121 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1122 1123 return err; 1124 } 1125 1126 static usbd_status 1127 xhci_open(usbd_pipe_handle pipe) 1128 { 1129 usbd_device_handle const dev = pipe->device; 1130 struct xhci_softc * const sc = dev->bus->hci_private; 1131 usb_endpoint_descriptor_t * const ed = pipe->endpoint->edesc; 1132 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); 1133 1134 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1135 DPRINTFN(1, "addr %d depth %d port %d speed %d", 1136 dev->address, dev->depth, dev->powersrc->portno, dev->speed); 1137 1138 if (sc->sc_dying) 1139 return USBD_IOERROR; 1140 1141 /* Root Hub */ 1142 if (dev->depth == 0 && dev->powersrc->portno == 0 && 1143 dev->speed != USB_SPEED_SUPER) { 1144 switch (ed->bEndpointAddress) { 1145 case USB_CONTROL_ENDPOINT: 1146 pipe->methods = &xhci_root_ctrl_methods; 1147 break; 1148 case UE_DIR_IN | XHCI_INTR_ENDPT: 1149 pipe->methods = &xhci_root_intr_methods; 1150 break; 1151 default: 1152 pipe->methods = NULL; 1153 DPRINTFN(0, "bad bEndpointAddress 0x%02x", 1154 ed->bEndpointAddress, 0, 0, 0); 1155 return USBD_INVAL; 1156 } 1157 return USBD_NORMAL_COMPLETION; 1158 } 1159 1160 switch (xfertype) { 1161 case UE_CONTROL: 1162 pipe->methods = &xhci_device_ctrl_methods; 1163 break; 1164 case UE_ISOCHRONOUS: 1165 pipe->methods = &xhci_device_isoc_methods; 1166 return USBD_INVAL; 1167 break; 1168 case UE_BULK: 1169 pipe->methods = &xhci_device_bulk_methods; 1170 break; 1171 case UE_INTERRUPT: 1172 pipe->methods = &xhci_device_intr_methods; 1173 break; 1174 default: 1175 return USBD_IOERROR; 1176 break; 1177 } 1178 1179 if (ed->bEndpointAddress != USB_CONTROL_ENDPOINT) 1180 xhci_configure_endpoint(pipe); 1181 1182 return USBD_NORMAL_COMPLETION; 1183 } 1184 1185 static void 1186 xhci_rhpsc(struct xhci_softc * const sc, u_int port) 1187 { 1188 usbd_xfer_handle const xfer = sc->sc_intrxfer; 1189 uint8_t *p; 1190 1191 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1192 DPRINTFN(4, "port %u status change", port, 0, 0, 0); 1193 1194 if (xfer == NULL) 1195 return; 1196 1197 if (!(port >= sc->sc_hs_port_start && 1198 port < sc->sc_hs_port_start + sc->sc_hs_port_count)) 1199 return; 1200 1201 port -= sc->sc_hs_port_start; 1202 port += 1; 1203 DPRINTFN(4, "hs port %u status change", port, 0, 0, 0); 1204 1205 p = KERNADDR(&xfer->dmabuf, 0); 1206 memset(p, 0, xfer->length); 1207 p[port/NBBY] |= 1 << (port%NBBY); 1208 xfer->actlen = xfer->length; 1209 xfer->status = USBD_NORMAL_COMPLETION; 1210 usb_transfer_complete(xfer); 1211 } 1212 1213 static void 1214 xhci_handle_event(struct xhci_softc * const sc, 1215 const struct xhci_trb * const trb) 1216 { 1217 uint64_t trb_0; 1218 uint32_t trb_2, trb_3; 1219 1220 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1221 1222 trb_0 = le64toh(trb->trb_0); 1223 trb_2 = le32toh(trb->trb_2); 1224 trb_3 = le32toh(trb->trb_3); 1225 1226 DPRINTFN(14, "event: %p 0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32, 1227 trb, trb_0, trb_2, trb_3); 1228 1229 switch (XHCI_TRB_3_TYPE_GET(trb_3)){ 1230 case XHCI_TRB_EVENT_TRANSFER: { 1231 u_int slot, dci; 1232 struct xhci_slot *xs; 1233 struct xhci_ring *xr; 1234 struct xhci_xfer *xx; 1235 usbd_xfer_handle xfer; 1236 usbd_status err; 1237 1238 slot = XHCI_TRB_3_SLOT_GET(trb_3); 1239 dci = XHCI_TRB_3_EP_GET(trb_3); 1240 1241 xs = &sc->sc_slots[slot]; 1242 xr = &xs->xs_ep[dci].xe_tr; 1243 1244 if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0) { 1245 xx = xr->xr_cookies[(trb_0 - xhci_ring_trbp(xr, 0))/ 1246 sizeof(struct xhci_trb)]; 1247 } else { 1248 xx = (void *)(uintptr_t)(trb_0 & ~0x3); 1249 } 1250 xfer = &xx->xx_xfer; 1251 DPRINTFN(14, "xfer %p", xfer, 0, 0, 0); 1252 1253 if ((trb_3 & XHCI_TRB_3_ED_BIT) != 0) { 1254 DPRINTFN(14, "transfer event data: " 1255 "0x%016"PRIx64" 0x%08"PRIx32" %02x", 1256 trb_0, XHCI_TRB_2_REM_GET(trb_2), 1257 XHCI_TRB_2_ERROR_GET(trb_2), 0); 1258 if ((trb_0 & 0x3) == 0x3) { 1259 xfer->actlen = XHCI_TRB_2_REM_GET(trb_2); 1260 } 1261 } 1262 1263 if (XHCI_TRB_2_ERROR_GET(trb_2) == 1264 XHCI_TRB_ERROR_SUCCESS) { 1265 xfer->actlen = xfer->length - XHCI_TRB_2_REM_GET(trb_2); 1266 err = USBD_NORMAL_COMPLETION; 1267 } else if (XHCI_TRB_2_ERROR_GET(trb_2) == 1268 XHCI_TRB_ERROR_SHORT_PKT) { 1269 xfer->actlen = xfer->length - XHCI_TRB_2_REM_GET(trb_2); 1270 err = USBD_NORMAL_COMPLETION; 1271 } else if (XHCI_TRB_2_ERROR_GET(trb_2) == 1272 XHCI_TRB_ERROR_STALL) { 1273 err = USBD_STALLED; 1274 xr->is_halted = true; 1275 DPRINTFN(1, "ev: xfer done: err %u slot %u dci %u", 1276 XHCI_TRB_2_ERROR_GET(trb_2), slot, dci, 0); 1277 } else { 1278 err = USBD_IOERROR; 1279 } 1280 xfer->status = err; 1281 1282 //mutex_enter(&sc->sc_lock); /* XXX ??? */ 1283 if ((trb_3 & XHCI_TRB_3_ED_BIT) != 0) { 1284 if ((trb_0 & 0x3) == 0x0) { 1285 usb_transfer_complete(xfer); 1286 } 1287 } else { 1288 usb_transfer_complete(xfer); 1289 } 1290 //mutex_exit(&sc->sc_lock); /* XXX ??? */ 1291 1292 } 1293 break; 1294 case XHCI_TRB_EVENT_CMD_COMPLETE: 1295 if (trb_0 == sc->sc_command_addr) { 1296 sc->sc_result_trb.trb_0 = trb_0; 1297 sc->sc_result_trb.trb_2 = trb_2; 1298 sc->sc_result_trb.trb_3 = trb_3; 1299 if (XHCI_TRB_2_ERROR_GET(trb_2) != 1300 XHCI_TRB_ERROR_SUCCESS) { 1301 DPRINTFN(1, "command completion " 1302 "failure: 0x%016"PRIx64" 0x%08"PRIx32" " 1303 "0x%08"PRIx32, trb_0, trb_2, trb_3, 0); 1304 } 1305 cv_signal(&sc->sc_command_cv); 1306 } else { 1307 DPRINTFN(1, "event: %p 0x%016"PRIx64" " 1308 "0x%08"PRIx32" 0x%08"PRIx32, trb, trb_0, 1309 trb_2, trb_3); 1310 } 1311 break; 1312 case XHCI_TRB_EVENT_PORT_STS_CHANGE: 1313 xhci_rhpsc(sc, (uint32_t)((trb_0 >> 24) & 0xff)); 1314 break; 1315 default: 1316 break; 1317 } 1318 } 1319 1320 static void 1321 xhci_softintr(void *v) 1322 { 1323 struct usbd_bus * const bus = v; 1324 struct xhci_softc * const sc = bus->hci_private; 1325 struct xhci_ring * const er = &sc->sc_er; 1326 struct xhci_trb *trb; 1327 int i, j, k; 1328 1329 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1330 1331 i = er->xr_ep; 1332 j = er->xr_cs; 1333 1334 DPRINTFN(16, "xr_ep %d xr_cs %d", i, j, 0, 0); 1335 1336 while (1) { 1337 usb_syncmem(&er->xr_dma, XHCI_TRB_SIZE * i, XHCI_TRB_SIZE, 1338 BUS_DMASYNC_POSTREAD); 1339 trb = &er->xr_trb[i]; 1340 k = (le32toh(trb->trb_3) & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0; 1341 1342 if (j != k) 1343 break; 1344 1345 xhci_handle_event(sc, trb); 1346 1347 i++; 1348 if (i == XHCI_EVENT_RING_TRBS) { 1349 i = 0; 1350 j ^= 1; 1351 } 1352 } 1353 1354 er->xr_ep = i; 1355 er->xr_cs = j; 1356 1357 xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(er, er->xr_ep) | 1358 XHCI_ERDP_LO_BUSY); 1359 1360 DPRINTFN(16, "ends", 0, 0, 0, 0); 1361 1362 return; 1363 } 1364 1365 static void 1366 xhci_poll(struct usbd_bus *bus) 1367 { 1368 struct xhci_softc * const sc = bus->hci_private; 1369 1370 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1371 1372 mutex_spin_enter(&sc->sc_intr_lock); 1373 xhci_intr1(sc); 1374 mutex_spin_exit(&sc->sc_intr_lock); 1375 1376 return; 1377 } 1378 1379 static usbd_status 1380 xhci_allocm(struct usbd_bus *bus, usb_dma_t *dma, uint32_t size) 1381 { 1382 struct xhci_softc * const sc = bus->hci_private; 1383 usbd_status err; 1384 1385 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1386 1387 err = usb_allocmem(&sc->sc_bus, size, 0, dma); 1388 #if 0 1389 if (err == USBD_NOMEM) 1390 err = usb_reserve_allocm(&sc->sc_dma_reserve, dma, size); 1391 #endif 1392 #ifdef XHCI_DEBUG 1393 if (err) 1394 DPRINTFN(1, "usb_allocmem(%u)=%d", err, size, 0, 0); 1395 #endif 1396 1397 return err; 1398 } 1399 1400 static void 1401 xhci_freem(struct usbd_bus *bus, usb_dma_t *dma) 1402 { 1403 struct xhci_softc * const sc = bus->hci_private; 1404 1405 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1406 1407 #if 0 1408 if (dma->block->flags & USB_DMA_RESERVE) { 1409 usb_reserve_freem(&sc->sc_dma_reserve, dma); 1410 return; 1411 } 1412 #endif 1413 usb_freemem(&sc->sc_bus, dma); 1414 } 1415 1416 static usbd_xfer_handle 1417 xhci_allocx(struct usbd_bus *bus) 1418 { 1419 struct xhci_softc * const sc = bus->hci_private; 1420 usbd_xfer_handle xfer; 1421 1422 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1423 1424 xfer = pool_cache_get(sc->sc_xferpool, PR_NOWAIT); 1425 if (xfer != NULL) { 1426 memset(xfer, 0, sizeof(struct xhci_xfer)); 1427 #ifdef DIAGNOSTIC 1428 xfer->busy_free = XFER_BUSY; 1429 #endif 1430 } 1431 1432 return xfer; 1433 } 1434 1435 static void 1436 xhci_freex(struct usbd_bus *bus, usbd_xfer_handle xfer) 1437 { 1438 struct xhci_softc * const sc = bus->hci_private; 1439 1440 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1441 1442 #ifdef DIAGNOSTIC 1443 if (xfer->busy_free != XFER_BUSY) { 1444 DPRINTFN(0, "xfer=%p not busy, 0x%08x", 1445 xfer, xfer->busy_free, 0, 0); 1446 } 1447 xfer->busy_free = XFER_FREE; 1448 #endif 1449 pool_cache_put(sc->sc_xferpool, xfer); 1450 } 1451 1452 static void 1453 xhci_get_lock(struct usbd_bus *bus, kmutex_t **lock) 1454 { 1455 struct xhci_softc * const sc = bus->hci_private; 1456 1457 *lock = &sc->sc_lock; 1458 } 1459 1460 extern u_int32_t usb_cookie_no; 1461 1462 static usbd_status 1463 xhci_new_device(device_t parent, usbd_bus_handle bus, int depth, 1464 int speed, int port, struct usbd_port *up) 1465 { 1466 struct xhci_softc * const sc = bus->hci_private; 1467 usbd_device_handle dev; 1468 usbd_status err; 1469 usb_device_descriptor_t *dd; 1470 struct usbd_device *hub; 1471 struct usbd_device *adev; 1472 int rhport = 0; 1473 struct xhci_slot *xs; 1474 uint32_t *cp; 1475 uint8_t slot; 1476 uint8_t addr; 1477 1478 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1479 DPRINTFN(4, "port=%d depth=%d speed=%d upport %d", 1480 port, depth, speed, up->portno); 1481 1482 dev = malloc(sizeof *dev, M_USB, M_NOWAIT|M_ZERO); 1483 if (dev == NULL) 1484 return USBD_NOMEM; 1485 1486 dev->bus = bus; 1487 1488 /* Set up default endpoint handle. */ 1489 dev->def_ep.edesc = &dev->def_ep_desc; 1490 1491 /* Set up default endpoint descriptor. */ 1492 dev->def_ep_desc.bLength = USB_ENDPOINT_DESCRIPTOR_SIZE; 1493 dev->def_ep_desc.bDescriptorType = UDESC_ENDPOINT; 1494 dev->def_ep_desc.bEndpointAddress = USB_CONTROL_ENDPOINT; 1495 dev->def_ep_desc.bmAttributes = UE_CONTROL; 1496 /* XXX */ 1497 if (speed == USB_SPEED_LOW) 1498 USETW(dev->def_ep_desc.wMaxPacketSize, USB_MAX_IPACKET); 1499 else 1500 USETW(dev->def_ep_desc.wMaxPacketSize, 64); 1501 dev->def_ep_desc.bInterval = 0; 1502 1503 /* doesn't matter, just don't let it uninitialized */ 1504 dev->def_ep.datatoggle = 0; 1505 1506 DPRINTFN(4, "up %p portno %d", up, up->portno, 0, 0); 1507 1508 dev->quirks = &usbd_no_quirk; 1509 dev->address = 0; 1510 dev->ddesc.bMaxPacketSize = 0; 1511 dev->depth = depth; 1512 dev->powersrc = up; 1513 dev->myhub = up->parent; 1514 1515 up->device = dev; 1516 1517 /* Locate root hub port */ 1518 for (adev = dev, hub = dev; 1519 hub != NULL; 1520 adev = hub, hub = hub->myhub) { 1521 DPRINTFN(4, "hub %p", hub, 0, 0, 0); 1522 } 1523 DPRINTFN(4, "hub %p", hub, 0, 0, 0); 1524 1525 if (hub != NULL) { 1526 for (int p = 0; p < hub->hub->hubdesc.bNbrPorts; p++) { 1527 if (hub->hub->ports[p].device == adev) { 1528 rhport = p; 1529 } 1530 } 1531 } else { 1532 rhport = port; 1533 } 1534 if (speed == USB_SPEED_SUPER) { 1535 rhport += sc->sc_ss_port_start - 1; 1536 } else { 1537 rhport += sc->sc_hs_port_start - 1; 1538 } 1539 DPRINTFN(4, "rhport %d", rhport, 0, 0, 0); 1540 1541 dev->speed = speed; 1542 dev->langid = USBD_NOLANG; 1543 dev->cookie.cookie = ++usb_cookie_no; 1544 1545 /* Establish the default pipe. */ 1546 err = usbd_setup_pipe(dev, 0, &dev->def_ep, USBD_DEFAULT_INTERVAL, 1547 &dev->default_pipe); 1548 if (err) { 1549 usbd_remove_device(dev, up); 1550 return (err); 1551 } 1552 1553 dd = &dev->ddesc; 1554 1555 if ((depth == 0) && (port == 0)) { 1556 KASSERT(bus->devices[dev->address] == NULL); 1557 bus->devices[dev->address] = dev; 1558 err = usbd_get_initial_ddesc(dev, dd); 1559 if (err) 1560 return err; 1561 err = usbd_reload_device_desc(dev); 1562 if (err) 1563 return err; 1564 } else { 1565 err = xhci_enable_slot(sc, &slot); 1566 if (err) 1567 return err; 1568 err = xhci_init_slot(sc, slot, depth, speed, port, rhport); 1569 if (err) 1570 return err; 1571 xs = &sc->sc_slots[slot]; 1572 dev->hci_private = xs; 1573 cp = xhci_slot_get_dcv(sc, xs, XHCI_DCI_SLOT); 1574 //hexdump("slot context", cp, sc->sc_ctxsz); 1575 addr = XHCI_SCTX_3_DEV_ADDR_GET(cp[3]); 1576 DPRINTFN(4, "device address %u", addr, 0, 0, 0); 1577 /* XXX ensure we know when the hardware does something 1578 we can't yet cope with */ 1579 KASSERT(addr >= 1 && addr <= 127); 1580 dev->address = addr; 1581 /* XXX dev->address not necessarily unique on bus */ 1582 KASSERT(bus->devices[dev->address] == NULL); 1583 bus->devices[dev->address] = dev; 1584 1585 err = usbd_get_initial_ddesc(dev, dd); 1586 if (err) 1587 return err; 1588 /* 4.8.2.1 */ 1589 if (speed == USB_SPEED_SUPER) 1590 USETW(dev->def_ep_desc.wMaxPacketSize, 1591 (1 << dd->bMaxPacketSize)); 1592 else 1593 USETW(dev->def_ep_desc.wMaxPacketSize, 1594 dd->bMaxPacketSize); 1595 DPRINTFN(4, "bMaxPacketSize %u", dd->bMaxPacketSize, 0, 0, 0); 1596 xhci_update_ep0_mps(sc, xs, 1597 UGETW(dev->def_ep_desc.wMaxPacketSize)); 1598 err = usbd_reload_device_desc(dev); 1599 if (err) 1600 return err; 1601 1602 usbd_kill_pipe(dev->default_pipe); 1603 err = usbd_setup_pipe(dev, 0, &dev->def_ep, 1604 USBD_DEFAULT_INTERVAL, &dev->default_pipe); 1605 } 1606 1607 DPRINTFN(1, "adding unit addr=%d, rev=%02x,", 1608 dev->address, UGETW(dd->bcdUSB), 0, 0); 1609 DPRINTFN(1, " class=%d, subclass=%d, protocol=%d,", 1610 dd->bDeviceClass, dd->bDeviceSubClass, 1611 dd->bDeviceProtocol, 0); 1612 DPRINTFN(1, " mps=%d, len=%d, noconf=%d, speed=%d", 1613 dd->bMaxPacketSize, dd->bLength, dd->bNumConfigurations, 1614 dev->speed); 1615 1616 usbd_add_dev_event(USB_EVENT_DEVICE_ATTACH, dev); 1617 1618 if ((depth == 0) && (port == 0)) { 1619 usbd_attach_roothub(parent, dev); 1620 DPRINTFN(1, "root_hub %p", bus->root_hub, 0, 0, 0); 1621 return USBD_NORMAL_COMPLETION; 1622 } 1623 1624 1625 err = usbd_probe_and_attach(parent, dev, port, dev->address); 1626 if (err) { 1627 usbd_remove_device(dev, up); 1628 return (err); 1629 } 1630 1631 return USBD_NORMAL_COMPLETION; 1632 } 1633 1634 static usbd_status 1635 xhci_ring_init(struct xhci_softc * const sc, struct xhci_ring * const xr, 1636 size_t ntrb, size_t align) 1637 { 1638 usbd_status err; 1639 size_t size = ntrb * XHCI_TRB_SIZE; 1640 1641 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1642 1643 err = usb_allocmem(&sc->sc_bus, size, align, &xr->xr_dma); 1644 if (err) 1645 return err; 1646 mutex_init(&xr->xr_lock, MUTEX_DEFAULT, IPL_SOFTUSB); 1647 xr->xr_cookies = kmem_zalloc(sizeof(*xr->xr_cookies) * ntrb, KM_SLEEP); 1648 xr->xr_trb = xhci_ring_trbv(xr, 0); 1649 xr->xr_ntrb = ntrb; 1650 xr->xr_ep = 0; 1651 xr->xr_cs = 1; 1652 memset(xr->xr_trb, 0, size); 1653 usb_syncmem(&xr->xr_dma, 0, size, BUS_DMASYNC_PREWRITE); 1654 xr->is_halted = false; 1655 1656 return USBD_NORMAL_COMPLETION; 1657 } 1658 1659 static void 1660 xhci_ring_free(struct xhci_softc * const sc, struct xhci_ring * const xr) 1661 { 1662 usb_freemem(&sc->sc_bus, &xr->xr_dma); 1663 mutex_destroy(&xr->xr_lock); 1664 kmem_free(xr->xr_cookies, sizeof(*xr->xr_cookies) * xr->xr_ntrb); 1665 } 1666 1667 static void 1668 xhci_ring_put(struct xhci_softc * const sc, struct xhci_ring * const xr, 1669 void *cookie, struct xhci_trb * const trbs, size_t ntrbs) 1670 { 1671 size_t i; 1672 u_int ri; 1673 u_int cs; 1674 uint64_t parameter; 1675 uint32_t status; 1676 uint32_t control; 1677 1678 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1679 1680 for (i = 0; i < ntrbs; i++) { 1681 DPRINTFN(12, "xr %p trbs %p num %zu", xr, trbs, i, 0); 1682 DPRINTFN(12, " %016"PRIx64" %08"PRIx32" %08"PRIx32, 1683 trbs[i].trb_0, trbs[i].trb_2, trbs[i].trb_3, 0); 1684 KASSERT(XHCI_TRB_3_TYPE_GET(trbs[i].trb_3) != 1685 XHCI_TRB_TYPE_LINK); 1686 } 1687 1688 DPRINTFN(12, "%p xr_ep 0x%x xr_cs %u", xr, xr->xr_ep, xr->xr_cs, 0); 1689 1690 ri = xr->xr_ep; 1691 cs = xr->xr_cs; 1692 1693 /* 1694 * Although the xhci hardware can do scatter/gather dma from 1695 * arbitrary sized buffers, there is a non-obvious restriction 1696 * that a LINK trb is only allowed at the end of a burst of 1697 * transfers - which might be 16kB. 1698 * Arbitrary aligned LINK trb definitely fail on Ivy bridge. 1699 * The simple solution is not to allow a LINK trb in the middle 1700 * of anything - as here. 1701 * XXX: (dsl) There are xhci controllers out there (eg some made by 1702 * ASMedia) that seem to lock up if they process a LINK trb but 1703 * cannot process the linked-to trb yet. 1704 * The code should write the 'cycle' bit on the link trb AFTER 1705 * adding the other trb. 1706 */ 1707 if (ri + ntrbs >= (xr->xr_ntrb - 1)) { 1708 parameter = xhci_ring_trbp(xr, 0); 1709 status = 0; 1710 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | 1711 XHCI_TRB_3_TC_BIT | (cs ? XHCI_TRB_3_CYCLE_BIT : 0); 1712 xhci_trb_put(&xr->xr_trb[ri], htole64(parameter), 1713 htole32(status), htole32(control)); 1714 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * ri, XHCI_TRB_SIZE * 1, 1715 BUS_DMASYNC_PREWRITE); 1716 xr->xr_cookies[ri] = NULL; 1717 xr->xr_ep = 0; 1718 xr->xr_cs ^= 1; 1719 ri = xr->xr_ep; 1720 cs = xr->xr_cs; 1721 } 1722 1723 ri++; 1724 1725 /* Write any subsequent TRB first */ 1726 for (i = 1; i < ntrbs; i++) { 1727 parameter = trbs[i].trb_0; 1728 status = trbs[i].trb_2; 1729 control = trbs[i].trb_3; 1730 1731 if (cs) { 1732 control |= XHCI_TRB_3_CYCLE_BIT; 1733 } else { 1734 control &= ~XHCI_TRB_3_CYCLE_BIT; 1735 } 1736 1737 xhci_trb_put(&xr->xr_trb[ri], htole64(parameter), 1738 htole32(status), htole32(control)); 1739 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * ri, XHCI_TRB_SIZE * 1, 1740 BUS_DMASYNC_PREWRITE); 1741 xr->xr_cookies[ri] = cookie; 1742 ri++; 1743 } 1744 1745 /* Write the first TRB last */ 1746 i = 0; 1747 { 1748 parameter = trbs[i].trb_0; 1749 status = trbs[i].trb_2; 1750 control = trbs[i].trb_3; 1751 1752 if (xr->xr_cs) { 1753 control |= XHCI_TRB_3_CYCLE_BIT; 1754 } else { 1755 control &= ~XHCI_TRB_3_CYCLE_BIT; 1756 } 1757 1758 xhci_trb_put(&xr->xr_trb[xr->xr_ep], htole64(parameter), 1759 htole32(status), htole32(control)); 1760 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * ri, XHCI_TRB_SIZE * 1, 1761 BUS_DMASYNC_PREWRITE); 1762 xr->xr_cookies[xr->xr_ep] = cookie; 1763 } 1764 1765 xr->xr_ep = ri; 1766 xr->xr_cs = cs; 1767 1768 DPRINTFN(12, "%p xr_ep 0x%x xr_cs %u", xr, xr->xr_ep, xr->xr_cs, 0); 1769 } 1770 1771 static usbd_status 1772 xhci_do_command(struct xhci_softc * const sc, struct xhci_trb * const trb, 1773 int timeout) 1774 { 1775 struct xhci_ring * const cr = &sc->sc_cr; 1776 usbd_status err; 1777 1778 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1779 DPRINTFN(12, "input: 0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32, 1780 trb->trb_0, trb->trb_2, trb->trb_3, 0); 1781 1782 mutex_enter(&sc->sc_lock); 1783 1784 KASSERT(sc->sc_command_addr == 0); 1785 sc->sc_command_addr = xhci_ring_trbp(cr, cr->xr_ep); 1786 1787 mutex_enter(&cr->xr_lock); 1788 xhci_ring_put(sc, cr, NULL, trb, 1); 1789 mutex_exit(&cr->xr_lock); 1790 1791 xhci_db_write_4(sc, XHCI_DOORBELL(0), 0); 1792 1793 if (cv_timedwait(&sc->sc_command_cv, &sc->sc_lock, 1794 MAX(1, mstohz(timeout))) == EWOULDBLOCK) { 1795 err = USBD_TIMEOUT; 1796 goto timedout; 1797 } 1798 1799 trb->trb_0 = sc->sc_result_trb.trb_0; 1800 trb->trb_2 = sc->sc_result_trb.trb_2; 1801 trb->trb_3 = sc->sc_result_trb.trb_3; 1802 1803 DPRINTFN(12, "output: 0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32"", 1804 trb->trb_0, trb->trb_2, trb->trb_3, 0); 1805 1806 switch (XHCI_TRB_2_ERROR_GET(trb->trb_2)) { 1807 case XHCI_TRB_ERROR_SUCCESS: 1808 err = USBD_NORMAL_COMPLETION; 1809 break; 1810 default: 1811 case 192 ... 223: 1812 err = USBD_IOERROR; 1813 break; 1814 case 224 ... 255: 1815 err = USBD_NORMAL_COMPLETION; 1816 break; 1817 } 1818 1819 timedout: 1820 sc->sc_command_addr = 0; 1821 mutex_exit(&sc->sc_lock); 1822 return err; 1823 } 1824 1825 static usbd_status 1826 xhci_enable_slot(struct xhci_softc * const sc, uint8_t * const slotp) 1827 { 1828 struct xhci_trb trb; 1829 usbd_status err; 1830 1831 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1832 1833 trb.trb_0 = 0; 1834 trb.trb_2 = 0; 1835 trb.trb_3 = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT); 1836 1837 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1838 if (err != USBD_NORMAL_COMPLETION) { 1839 return err; 1840 } 1841 1842 *slotp = XHCI_TRB_3_SLOT_GET(trb.trb_3); 1843 1844 return err; 1845 } 1846 1847 static usbd_status 1848 xhci_address_device(struct xhci_softc * const sc, 1849 uint64_t icp, uint8_t slot_id, bool bsr) 1850 { 1851 struct xhci_trb trb; 1852 usbd_status err; 1853 1854 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1855 1856 trb.trb_0 = icp; 1857 trb.trb_2 = 0; 1858 trb.trb_3 = XHCI_TRB_3_SLOT_SET(slot_id) | 1859 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) | 1860 (bsr ? XHCI_TRB_3_BSR_BIT : 0); 1861 1862 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1863 return err; 1864 } 1865 1866 static usbd_status 1867 xhci_update_ep0_mps(struct xhci_softc * const sc, 1868 struct xhci_slot * const xs, u_int mps) 1869 { 1870 struct xhci_trb trb; 1871 usbd_status err; 1872 uint32_t * cp; 1873 1874 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1875 DPRINTFN(4, "slot %u mps %u", xs->xs_idx, mps, 0, 0); 1876 1877 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 1878 cp[0] = htole32(0); 1879 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_EP_CONTROL)); 1880 1881 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_EP_CONTROL)); 1882 cp[1] = htole32(XHCI_EPCTX_1_MAXP_SIZE_SET(mps)); 1883 1884 /* sync input contexts before they are read from memory */ 1885 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 1886 hexdump("input context", xhci_slot_get_icv(sc, xs, 0), 1887 sc->sc_ctxsz * 4); 1888 1889 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); 1890 trb.trb_2 = 0; 1891 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1892 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX); 1893 1894 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1895 KASSERT(err == USBD_NORMAL_COMPLETION); /* XXX */ 1896 return err; 1897 } 1898 1899 static void 1900 xhci_set_dcba(struct xhci_softc * const sc, uint64_t dcba, int si) 1901 { 1902 uint64_t * const dcbaa = KERNADDR(&sc->sc_dcbaa_dma, 0); 1903 1904 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1905 DPRINTFN(4, "dcbaa %p dc %016"PRIx64" slot %d", 1906 &dcbaa[si], dcba, si, 0); 1907 1908 dcbaa[si] = htole64(dcba); 1909 usb_syncmem(&sc->sc_dcbaa_dma, si * sizeof(uint64_t), sizeof(uint64_t), 1910 BUS_DMASYNC_PREWRITE); 1911 } 1912 1913 static usbd_status 1914 xhci_init_slot(struct xhci_softc * const sc, uint32_t slot, int depth, 1915 int speed, int port, int rhport) 1916 { 1917 struct xhci_slot *xs; 1918 usbd_status err; 1919 u_int dci; 1920 uint32_t *cp; 1921 uint32_t mps; 1922 uint32_t xspeed; 1923 1924 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1925 DPRINTFN(4, "slot %u depth %d speed %d", 1926 slot, depth, speed, 0); 1927 DPRINTFN(4, " port %d rhport %d", 1928 port, rhport, 0, 0); 1929 1930 switch (speed) { 1931 case USB_SPEED_LOW: 1932 xspeed = 2; 1933 mps = USB_MAX_IPACKET; 1934 break; 1935 case USB_SPEED_FULL: 1936 xspeed = 1; 1937 mps = 64; 1938 break; 1939 case USB_SPEED_HIGH: 1940 xspeed = 3; 1941 mps = USB_2_MAX_CTRL_PACKET; 1942 break; 1943 case USB_SPEED_SUPER: 1944 xspeed = 4; 1945 mps = USB_3_MAX_CTRL_PACKET; 1946 break; 1947 default: 1948 DPRINTFN(0, "impossible speed: %x", speed, 0, 0, 0); 1949 return USBD_INVAL; 1950 } 1951 1952 xs = &sc->sc_slots[slot]; 1953 xs->xs_idx = slot; 1954 1955 /* allocate contexts */ 1956 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz, 1957 &xs->xs_dc_dma); 1958 if (err) 1959 return err; 1960 memset(KERNADDR(&xs->xs_dc_dma, 0), 0, sc->sc_pgsz); 1961 1962 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz, 1963 &xs->xs_ic_dma); 1964 if (err) 1965 return err; 1966 memset(KERNADDR(&xs->xs_ic_dma, 0), 0, sc->sc_pgsz); 1967 1968 for (dci = 0; dci < 32; dci++) { 1969 //CTASSERT(sizeof(xs->xs_ep[dci]) == sizeof(struct xhci_endpoint)); 1970 memset(&xs->xs_ep[dci], 0, sizeof(xs->xs_ep[dci])); 1971 if (dci == XHCI_DCI_SLOT) 1972 continue; 1973 err = xhci_ring_init(sc, &xs->xs_ep[dci].xe_tr, 1974 XHCI_TRANSFER_RING_TRBS, XHCI_TRB_ALIGN); 1975 if (err) { 1976 DPRINTFN(0, "ring init failure", 0, 0, 0, 0); 1977 return err; 1978 } 1979 } 1980 1981 /* set up initial input control context */ 1982 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 1983 cp[0] = htole32(0); 1984 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_EP_CONTROL)| 1985 XHCI_INCTX_1_ADD_MASK(XHCI_DCI_SLOT)); 1986 1987 /* set up input slot context */ 1988 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); 1989 cp[0] = htole32( 1990 XHCI_SCTX_0_CTX_NUM_SET(1) | 1991 XHCI_SCTX_0_SPEED_SET(xspeed) 1992 ); 1993 cp[1] = htole32( 1994 XHCI_SCTX_1_RH_PORT_SET(rhport) 1995 ); 1996 cp[2] = htole32( 1997 XHCI_SCTX_2_IRQ_TARGET_SET(0) 1998 ); 1999 cp[3] = htole32(0); 2000 2001 /* set up input EP0 context */ 2002 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_EP_CONTROL)); 2003 cp[0] = htole32(0); 2004 cp[1] = htole32( 2005 XHCI_EPCTX_1_MAXP_SIZE_SET(mps) | 2006 XHCI_EPCTX_1_EPTYPE_SET(4) | 2007 XHCI_EPCTX_1_CERR_SET(3) 2008 ); 2009 /* can't use xhci_ep_get_dci() yet? */ 2010 *(uint64_t *)(&cp[2]) = htole64( 2011 xhci_ring_trbp(&xs->xs_ep[XHCI_DCI_EP_CONTROL].xe_tr, 0) | 2012 XHCI_EPCTX_2_DCS_SET(1)); 2013 cp[4] = htole32( 2014 XHCI_EPCTX_4_AVG_TRB_LEN_SET(8) 2015 ); 2016 2017 /* sync input contexts before they are read from memory */ 2018 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 2019 hexdump("input context", xhci_slot_get_icv(sc, xs, 0), 2020 sc->sc_ctxsz * 3); 2021 2022 xhci_set_dcba(sc, DMAADDR(&xs->xs_dc_dma, 0), slot); 2023 2024 err = xhci_address_device(sc, xhci_slot_get_icp(sc, xs, 0), slot, 2025 false); 2026 2027 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 2028 hexdump("output context", xhci_slot_get_dcv(sc, xs, 0), 2029 sc->sc_ctxsz * 2); 2030 2031 return err; 2032 } 2033 2034 /* ----- */ 2035 2036 static void 2037 xhci_noop(usbd_pipe_handle pipe) 2038 { 2039 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2040 } 2041 2042 /* root hub descriptors */ 2043 2044 static const usb_device_descriptor_t xhci_devd = { 2045 USB_DEVICE_DESCRIPTOR_SIZE, 2046 UDESC_DEVICE, /* type */ 2047 {0x00, 0x02}, /* USB version */ 2048 UDCLASS_HUB, /* class */ 2049 UDSUBCLASS_HUB, /* subclass */ 2050 UDPROTO_HSHUBSTT, /* protocol */ 2051 64, /* max packet */ 2052 {0},{0},{0x00,0x01}, /* device id */ 2053 1,2,0, /* string indexes */ 2054 1 /* # of configurations */ 2055 }; 2056 2057 static const usb_device_qualifier_t xhci_odevd = { 2058 USB_DEVICE_DESCRIPTOR_SIZE, 2059 UDESC_DEVICE_QUALIFIER, /* type */ 2060 {0x00, 0x02}, /* USB version */ 2061 UDCLASS_HUB, /* class */ 2062 UDSUBCLASS_HUB, /* subclass */ 2063 UDPROTO_FSHUB, /* protocol */ 2064 64, /* max packet */ 2065 1, /* # of configurations */ 2066 0 2067 }; 2068 2069 static const usb_config_descriptor_t xhci_confd = { 2070 USB_CONFIG_DESCRIPTOR_SIZE, 2071 UDESC_CONFIG, 2072 {USB_CONFIG_DESCRIPTOR_SIZE + 2073 USB_INTERFACE_DESCRIPTOR_SIZE + 2074 USB_ENDPOINT_DESCRIPTOR_SIZE}, 2075 1, 2076 1, 2077 0, 2078 UC_ATTR_MBO | UC_SELF_POWERED, 2079 0 /* max power */ 2080 }; 2081 2082 static const usb_interface_descriptor_t xhci_ifcd = { 2083 USB_INTERFACE_DESCRIPTOR_SIZE, 2084 UDESC_INTERFACE, 2085 0, 2086 0, 2087 1, 2088 UICLASS_HUB, 2089 UISUBCLASS_HUB, 2090 UIPROTO_HSHUBSTT, 2091 0 2092 }; 2093 2094 static const usb_endpoint_descriptor_t xhci_endpd = { 2095 USB_ENDPOINT_DESCRIPTOR_SIZE, 2096 UDESC_ENDPOINT, 2097 UE_DIR_IN | XHCI_INTR_ENDPT, 2098 UE_INTERRUPT, 2099 {8, 0}, /* max packet */ 2100 12 2101 }; 2102 2103 static const usb_hub_descriptor_t xhci_hubd = { 2104 USB_HUB_DESCRIPTOR_SIZE, 2105 UDESC_HUB, 2106 0, 2107 {0,0}, 2108 0, 2109 0, 2110 {""}, 2111 {""}, 2112 }; 2113 2114 /* root hub control */ 2115 2116 static usbd_status 2117 xhci_root_ctrl_transfer(usbd_xfer_handle xfer) 2118 { 2119 struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; 2120 usbd_status err; 2121 2122 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2123 2124 /* Insert last in queue. */ 2125 mutex_enter(&sc->sc_lock); 2126 err = usb_insert_transfer(xfer); 2127 mutex_exit(&sc->sc_lock); 2128 if (err) 2129 return err; 2130 2131 /* Pipe isn't running, start first */ 2132 return (xhci_root_ctrl_start(SIMPLEQ_FIRST(&xfer->pipe->queue))); 2133 } 2134 2135 static usbd_status 2136 xhci_root_ctrl_start(usbd_xfer_handle xfer) 2137 { 2138 struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; 2139 usb_port_status_t ps; 2140 usb_device_request_t *req; 2141 void *buf = NULL; 2142 usb_hub_descriptor_t hubd; 2143 usbd_status err; 2144 int len, value, index; 2145 int l, totlen = 0; 2146 int port, i; 2147 uint32_t v; 2148 2149 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2150 2151 if (sc->sc_dying) 2152 return USBD_IOERROR; 2153 2154 req = &xfer->request; 2155 2156 value = UGETW(req->wValue); 2157 index = UGETW(req->wIndex); 2158 len = UGETW(req->wLength); 2159 2160 if (len != 0) 2161 buf = KERNADDR(&xfer->dmabuf, 0); 2162 2163 DPRINTFN(12, "rhreq: %04x %04x %04x %04x", 2164 req->bmRequestType | (req->bRequest << 8), value, index, len); 2165 2166 #define C(x,y) ((x) | ((y) << 8)) 2167 switch(C(req->bRequest, req->bmRequestType)) { 2168 case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE): 2169 case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE): 2170 case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT): 2171 /* 2172 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops 2173 * for the integrated root hub. 2174 */ 2175 break; 2176 case C(UR_GET_CONFIG, UT_READ_DEVICE): 2177 if (len > 0) { 2178 *(uint8_t *)buf = sc->sc_conf; 2179 totlen = 1; 2180 } 2181 break; 2182 case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE): 2183 DPRINTFN(8, "getdesc: wValue=0x%04x", value, 0, 0, 0); 2184 if (len == 0) 2185 break; 2186 switch(value >> 8) { 2187 case UDESC_DEVICE: 2188 if ((value & 0xff) != 0) { 2189 err = USBD_IOERROR; 2190 goto ret; 2191 } 2192 totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE); 2193 memcpy(buf, &xhci_devd, min(l, sizeof(xhci_devd))); 2194 break; 2195 case UDESC_DEVICE_QUALIFIER: 2196 if ((value & 0xff) != 0) { 2197 } 2198 totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE); 2199 memcpy(buf, &xhci_odevd, min(l, sizeof(xhci_odevd))); 2200 break; 2201 case UDESC_OTHER_SPEED_CONFIGURATION: 2202 case UDESC_CONFIG: 2203 if ((value & 0xff) != 0) { 2204 err = USBD_IOERROR; 2205 goto ret; 2206 } 2207 totlen = l = min(len, USB_CONFIG_DESCRIPTOR_SIZE); 2208 memcpy(buf, &xhci_confd, min(l, sizeof(xhci_confd))); 2209 ((usb_config_descriptor_t *)buf)->bDescriptorType = 2210 value >> 8; 2211 buf = (char *)buf + l; 2212 len -= l; 2213 l = min(len, USB_INTERFACE_DESCRIPTOR_SIZE); 2214 totlen += l; 2215 memcpy(buf, &xhci_ifcd, min(l, sizeof(xhci_ifcd))); 2216 buf = (char *)buf + l; 2217 len -= l; 2218 l = min(len, USB_ENDPOINT_DESCRIPTOR_SIZE); 2219 totlen += l; 2220 memcpy(buf, &xhci_endpd, min(l, sizeof(xhci_endpd))); 2221 break; 2222 case UDESC_STRING: 2223 #define sd ((usb_string_descriptor_t *)buf) 2224 switch (value & 0xff) { 2225 case 0: /* Language table */ 2226 totlen = usb_makelangtbl(sd, len); 2227 break; 2228 case 1: /* Vendor */ 2229 totlen = usb_makestrdesc(sd, len, "NetBSD"); 2230 break; 2231 case 2: /* Product */ 2232 totlen = usb_makestrdesc(sd, len, 2233 "xHCI Root Hub"); 2234 break; 2235 } 2236 #undef sd 2237 break; 2238 default: 2239 err = USBD_IOERROR; 2240 goto ret; 2241 } 2242 break; 2243 case C(UR_GET_INTERFACE, UT_READ_INTERFACE): 2244 if (len > 0) { 2245 *(uint8_t *)buf = 0; 2246 totlen = 1; 2247 } 2248 break; 2249 case C(UR_GET_STATUS, UT_READ_DEVICE): 2250 if (len > 1) { 2251 USETW(((usb_status_t *)buf)->wStatus,UDS_SELF_POWERED); 2252 totlen = 2; 2253 } 2254 break; 2255 case C(UR_GET_STATUS, UT_READ_INTERFACE): 2256 case C(UR_GET_STATUS, UT_READ_ENDPOINT): 2257 if (len > 1) { 2258 USETW(((usb_status_t *)buf)->wStatus, 0); 2259 totlen = 2; 2260 } 2261 break; 2262 case C(UR_SET_ADDRESS, UT_WRITE_DEVICE): 2263 if (value >= USB_MAX_DEVICES) { 2264 err = USBD_IOERROR; 2265 goto ret; 2266 } 2267 //sc->sc_addr = value; 2268 break; 2269 case C(UR_SET_CONFIG, UT_WRITE_DEVICE): 2270 if (value != 0 && value != 1) { 2271 err = USBD_IOERROR; 2272 goto ret; 2273 } 2274 sc->sc_conf = value; 2275 break; 2276 case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE): 2277 break; 2278 case C(UR_SET_FEATURE, UT_WRITE_DEVICE): 2279 case C(UR_SET_FEATURE, UT_WRITE_INTERFACE): 2280 case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT): 2281 err = USBD_IOERROR; 2282 goto ret; 2283 case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE): 2284 break; 2285 case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT): 2286 break; 2287 /* Hub requests */ 2288 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE): 2289 break; 2290 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER): 2291 DPRINTFN(4, "UR_CLEAR_PORT_FEATURE port=%d feature=%d", 2292 index, value, 0, 0); 2293 if (index < 1 || index > sc->sc_hs_port_count) { 2294 err = USBD_IOERROR; 2295 goto ret; 2296 } 2297 port = XHCI_PORTSC(sc->sc_hs_port_start - 1 + index); 2298 v = xhci_op_read_4(sc, port); 2299 DPRINTFN(4, "portsc=0x%08x", v, 0, 0, 0); 2300 v &= ~XHCI_PS_CLEAR; 2301 switch (value) { 2302 case UHF_PORT_ENABLE: 2303 xhci_op_write_4(sc, port, v &~ XHCI_PS_PED); 2304 break; 2305 case UHF_PORT_SUSPEND: 2306 err = USBD_IOERROR; 2307 goto ret; 2308 case UHF_PORT_POWER: 2309 break; 2310 case UHF_PORT_TEST: 2311 case UHF_PORT_INDICATOR: 2312 err = USBD_IOERROR; 2313 goto ret; 2314 case UHF_C_PORT_CONNECTION: 2315 xhci_op_write_4(sc, port, v | XHCI_PS_CSC); 2316 break; 2317 case UHF_C_PORT_ENABLE: 2318 case UHF_C_PORT_SUSPEND: 2319 case UHF_C_PORT_OVER_CURRENT: 2320 err = USBD_IOERROR; 2321 goto ret; 2322 case UHF_C_PORT_RESET: 2323 xhci_op_write_4(sc, port, v | XHCI_PS_PRC); 2324 break; 2325 default: 2326 err = USBD_IOERROR; 2327 goto ret; 2328 } 2329 2330 break; 2331 case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE): 2332 if (len == 0) 2333 break; 2334 if ((value & 0xff) != 0) { 2335 err = USBD_IOERROR; 2336 goto ret; 2337 } 2338 hubd = xhci_hubd; 2339 hubd.bNbrPorts = sc->sc_hs_port_count; 2340 USETW(hubd.wHubCharacteristics, UHD_PWR_NO_SWITCH); 2341 hubd.bPwrOn2PwrGood = 200; 2342 for (i = 0, l = sc->sc_maxports; l > 0; i++, l -= 8) 2343 hubd.DeviceRemovable[i++] = 0; /* XXX can't find out? */ 2344 hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i; 2345 l = min(len, hubd.bDescLength); 2346 totlen = l; 2347 memcpy(buf, &hubd, l); 2348 break; 2349 case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE): 2350 if (len != 4) { 2351 err = USBD_IOERROR; 2352 goto ret; 2353 } 2354 memset(buf, 0, len); /* ? XXX */ 2355 totlen = len; 2356 break; 2357 case C(UR_GET_STATUS, UT_READ_CLASS_OTHER): 2358 DPRINTFN(8, "get port status i=%d", index, 0, 0, 0); 2359 if (index < 1 || index > sc->sc_maxports) { 2360 err = USBD_IOERROR; 2361 goto ret; 2362 } 2363 if (len != 4) { 2364 err = USBD_IOERROR; 2365 goto ret; 2366 } 2367 v = xhci_op_read_4(sc, XHCI_PORTSC(sc->sc_hs_port_start - 1 + 2368 index)); 2369 DPRINTFN(4, "READ_CLASS_OTHER GET_STATUS PORTSC %d (%d) %08x", 2370 index, sc->sc_hs_port_start - 1 + index, v, 0); 2371 switch (XHCI_PS_SPEED_GET(v)) { 2372 case 1: 2373 i = UPS_FULL_SPEED; 2374 break; 2375 case 2: 2376 i = UPS_LOW_SPEED; 2377 break; 2378 case 3: 2379 i = UPS_HIGH_SPEED; 2380 break; 2381 default: 2382 i = 0; 2383 break; 2384 } 2385 if (v & XHCI_PS_CCS) i |= UPS_CURRENT_CONNECT_STATUS; 2386 if (v & XHCI_PS_PED) i |= UPS_PORT_ENABLED; 2387 if (v & XHCI_PS_OCA) i |= UPS_OVERCURRENT_INDICATOR; 2388 //if (v & XHCI_PS_SUSP) i |= UPS_SUSPEND; 2389 if (v & XHCI_PS_PR) i |= UPS_RESET; 2390 if (v & XHCI_PS_PP) i |= UPS_PORT_POWER; 2391 USETW(ps.wPortStatus, i); 2392 i = 0; 2393 if (v & XHCI_PS_CSC) i |= UPS_C_CONNECT_STATUS; 2394 if (v & XHCI_PS_PEC) i |= UPS_C_PORT_ENABLED; 2395 if (v & XHCI_PS_OCC) i |= UPS_C_OVERCURRENT_INDICATOR; 2396 if (v & XHCI_PS_PRC) i |= UPS_C_PORT_RESET; 2397 USETW(ps.wPortChange, i); 2398 l = min(len, sizeof ps); 2399 memcpy(buf, &ps, l); 2400 totlen = l; 2401 break; 2402 case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE): 2403 err = USBD_IOERROR; 2404 goto ret; 2405 case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE): 2406 break; 2407 case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER): 2408 if (index < 1 || index > sc->sc_hs_port_count) { 2409 err = USBD_IOERROR; 2410 goto ret; 2411 } 2412 port = XHCI_PORTSC(sc->sc_hs_port_start - 1 + index); 2413 v = xhci_op_read_4(sc, port); 2414 DPRINTFN(4, "portsc=0x%08x", v, 0, 0, 0); 2415 v &= ~XHCI_PS_CLEAR; 2416 switch (value) { 2417 case UHF_PORT_ENABLE: 2418 xhci_op_write_4(sc, port, v | XHCI_PS_PED); 2419 break; 2420 case UHF_PORT_SUSPEND: 2421 /* XXX suspend */ 2422 break; 2423 case UHF_PORT_RESET: 2424 v &= ~ (XHCI_PS_PED | XHCI_PS_PR); 2425 xhci_op_write_4(sc, port, v | XHCI_PS_PR); 2426 /* Wait for reset to complete. */ 2427 usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY); 2428 if (sc->sc_dying) { 2429 err = USBD_IOERROR; 2430 goto ret; 2431 } 2432 v = xhci_op_read_4(sc, port); 2433 if (v & XHCI_PS_PR) { 2434 xhci_op_write_4(sc, port, v & ~XHCI_PS_PR); 2435 usb_delay_ms(&sc->sc_bus, 10); 2436 /* XXX */ 2437 } 2438 break; 2439 case UHF_PORT_POWER: 2440 /* XXX power control */ 2441 break; 2442 /* XXX more */ 2443 case UHF_C_PORT_RESET: 2444 xhci_op_write_4(sc, port, v | XHCI_PS_PRC); 2445 break; 2446 default: 2447 err = USBD_IOERROR; 2448 goto ret; 2449 } 2450 break; 2451 case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER): 2452 case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER): 2453 case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER): 2454 case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER): 2455 break; 2456 default: 2457 err = USBD_IOERROR; 2458 goto ret; 2459 } 2460 xfer->actlen = totlen; 2461 err = USBD_NORMAL_COMPLETION; 2462 ret: 2463 xfer->status = err; 2464 mutex_enter(&sc->sc_lock); 2465 usb_transfer_complete(xfer); 2466 mutex_exit(&sc->sc_lock); 2467 return USBD_IN_PROGRESS; 2468 } 2469 2470 2471 static void 2472 xhci_root_ctrl_abort(usbd_xfer_handle xfer) 2473 { 2474 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2475 /* Nothing to do, all transfers are synchronous. */ 2476 } 2477 2478 2479 static void 2480 xhci_root_ctrl_close(usbd_pipe_handle pipe) 2481 { 2482 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2483 /* Nothing to do. */ 2484 } 2485 2486 static void 2487 xhci_root_ctrl_done(usbd_xfer_handle xfer) 2488 { 2489 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2490 2491 xfer->hcpriv = NULL; 2492 } 2493 2494 /* root hub interrupt */ 2495 2496 static usbd_status 2497 xhci_root_intr_transfer(usbd_xfer_handle xfer) 2498 { 2499 struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; 2500 usbd_status err; 2501 2502 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2503 2504 /* Insert last in queue. */ 2505 mutex_enter(&sc->sc_lock); 2506 err = usb_insert_transfer(xfer); 2507 mutex_exit(&sc->sc_lock); 2508 if (err) 2509 return err; 2510 2511 /* Pipe isn't running, start first */ 2512 return (xhci_root_intr_start(SIMPLEQ_FIRST(&xfer->pipe->queue))); 2513 } 2514 2515 static usbd_status 2516 xhci_root_intr_start(usbd_xfer_handle xfer) 2517 { 2518 struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; 2519 2520 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2521 2522 if (sc->sc_dying) 2523 return USBD_IOERROR; 2524 2525 mutex_enter(&sc->sc_lock); 2526 sc->sc_intrxfer = xfer; 2527 mutex_exit(&sc->sc_lock); 2528 2529 return USBD_IN_PROGRESS; 2530 } 2531 2532 static void 2533 xhci_root_intr_abort(usbd_xfer_handle xfer) 2534 { 2535 struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; 2536 2537 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2538 2539 KASSERT(mutex_owned(&sc->sc_lock)); 2540 KASSERT(xfer->pipe->intrxfer == xfer); 2541 2542 DPRINTFN(1, "remove", 0, 0, 0, 0); 2543 2544 sc->sc_intrxfer = NULL; 2545 2546 xfer->status = USBD_CANCELLED; 2547 usb_transfer_complete(xfer); 2548 } 2549 2550 static void 2551 xhci_root_intr_close(usbd_pipe_handle pipe) 2552 { 2553 struct xhci_softc * const sc = pipe->device->bus->hci_private; 2554 2555 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2556 2557 KASSERT(mutex_owned(&sc->sc_lock)); 2558 2559 sc->sc_intrxfer = NULL; 2560 } 2561 2562 static void 2563 xhci_root_intr_done(usbd_xfer_handle xfer) 2564 { 2565 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2566 2567 xfer->hcpriv = NULL; 2568 } 2569 2570 /* -------------- */ 2571 /* device control */ 2572 2573 static usbd_status 2574 xhci_device_ctrl_transfer(usbd_xfer_handle xfer) 2575 { 2576 struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; 2577 usbd_status err; 2578 2579 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2580 2581 /* Insert last in queue. */ 2582 mutex_enter(&sc->sc_lock); 2583 err = usb_insert_transfer(xfer); 2584 mutex_exit(&sc->sc_lock); 2585 if (err) 2586 return (err); 2587 2588 /* Pipe isn't running, start first */ 2589 return (xhci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->pipe->queue))); 2590 } 2591 2592 static usbd_status 2593 xhci_device_ctrl_start(usbd_xfer_handle xfer) 2594 { 2595 struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; 2596 struct xhci_slot * const xs = xfer->pipe->device->hci_private; 2597 const u_int dci = xhci_ep_get_dci(xfer->pipe->endpoint->edesc); 2598 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 2599 struct xhci_xfer * const xx = (void *)xfer; 2600 usb_device_request_t * const req = &xfer->request; 2601 const bool isread = UT_GET_DIR(req->bmRequestType) == UT_READ; 2602 const uint32_t len = UGETW(req->wLength); 2603 usb_dma_t * const dma = &xfer->dmabuf; 2604 uint64_t parameter; 2605 uint32_t status; 2606 uint32_t control; 2607 u_int i; 2608 2609 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2610 DPRINTFN(12, "req: %04x %04x %04x %04x", 2611 req->bmRequestType | (req->bRequest << 8), UGETW(req->wValue), 2612 UGETW(req->wIndex), UGETW(req->wLength)); 2613 2614 /* XXX */ 2615 if (tr->is_halted) { 2616 xhci_reset_endpoint(xfer->pipe); 2617 tr->is_halted = false; 2618 xhci_set_dequeue(xfer->pipe); 2619 } 2620 2621 /* we rely on the bottom bits for extra info */ 2622 KASSERT(((uintptr_t)xfer & 0x3) == 0x0); 2623 2624 KASSERT((xfer->rqflags & URQ_REQUEST) != 0); 2625 2626 i = 0; 2627 2628 /* setup phase */ 2629 memcpy(¶meter, req, sizeof(*req)); 2630 parameter = le64toh(parameter); 2631 status = XHCI_TRB_2_IRQ_SET(0) | XHCI_TRB_2_BYTES_SET(sizeof(*req)); 2632 control = ((len == 0) ? XHCI_TRB_3_TRT_NONE : 2633 (isread ? XHCI_TRB_3_TRT_IN : XHCI_TRB_3_TRT_OUT)) | 2634 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) | 2635 XHCI_TRB_3_IDT_BIT; 2636 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 2637 2638 if (len == 0) 2639 goto no_data; 2640 2641 /* data phase */ 2642 parameter = DMAADDR(dma, 0); 2643 KASSERT(len <= 0x10000); 2644 status = XHCI_TRB_2_IRQ_SET(0) | 2645 XHCI_TRB_2_TDSZ_SET(1) | 2646 XHCI_TRB_2_BYTES_SET(len); 2647 control = (isread ? XHCI_TRB_3_DIR_IN : 0) | 2648 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE) | 2649 XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_ENT_BIT; 2650 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 2651 2652 parameter = (uintptr_t)xfer | 0x3; 2653 status = XHCI_TRB_2_IRQ_SET(0); 2654 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVENT_DATA) | 2655 XHCI_TRB_3_IOC_BIT; 2656 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 2657 2658 no_data: 2659 parameter = 0; 2660 status = XHCI_TRB_2_IRQ_SET(0); 2661 /* the status stage has inverted direction */ 2662 control = ((isread && (len > 0)) ? 0 : XHCI_TRB_3_DIR_IN) | 2663 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE) | 2664 XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_ENT_BIT; 2665 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 2666 2667 parameter = (uintptr_t)xfer | 0x0; 2668 status = XHCI_TRB_2_IRQ_SET(0); 2669 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVENT_DATA) | 2670 XHCI_TRB_3_IOC_BIT; 2671 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 2672 2673 mutex_enter(&tr->xr_lock); 2674 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 2675 mutex_exit(&tr->xr_lock); 2676 2677 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 2678 2679 if (xfer->timeout && !sc->sc_bus.use_polling) { 2680 callout_reset(&xfer->timeout_handle, mstohz(xfer->timeout), 2681 xhci_timeout, xfer); 2682 } 2683 2684 if (sc->sc_bus.use_polling) { 2685 DPRINTFN(1, "polling", 0, 0, 0, 0); 2686 //xhci_waitintr(sc, xfer); 2687 } 2688 2689 return USBD_IN_PROGRESS; 2690 } 2691 2692 static void 2693 xhci_device_ctrl_done(usbd_xfer_handle xfer) 2694 { 2695 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2696 2697 callout_stop(&xfer->timeout_handle); /* XXX wrong place */ 2698 2699 } 2700 2701 static void 2702 xhci_device_ctrl_abort(usbd_xfer_handle xfer) 2703 { 2704 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2705 } 2706 2707 static void 2708 xhci_device_ctrl_close(usbd_pipe_handle pipe) 2709 { 2710 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2711 } 2712 2713 /* ----------------- */ 2714 /* device isochronus */ 2715 2716 /* ----------- */ 2717 /* device bulk */ 2718 2719 static usbd_status 2720 xhci_device_bulk_transfer(usbd_xfer_handle xfer) 2721 { 2722 struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; 2723 usbd_status err; 2724 2725 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2726 2727 /* Insert last in queue. */ 2728 mutex_enter(&sc->sc_lock); 2729 err = usb_insert_transfer(xfer); 2730 mutex_exit(&sc->sc_lock); 2731 if (err) 2732 return err; 2733 2734 /* 2735 * Pipe isn't running (otherwise err would be USBD_INPROG), 2736 * so start it first. 2737 */ 2738 return (xhci_device_bulk_start(SIMPLEQ_FIRST(&xfer->pipe->queue))); 2739 } 2740 2741 static usbd_status 2742 xhci_device_bulk_start(usbd_xfer_handle xfer) 2743 { 2744 struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; 2745 struct xhci_slot * const xs = xfer->pipe->device->hci_private; 2746 const u_int dci = xhci_ep_get_dci(xfer->pipe->endpoint->edesc); 2747 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 2748 struct xhci_xfer * const xx = (void *)xfer; 2749 const uint32_t len = xfer->length; 2750 usb_dma_t * const dma = &xfer->dmabuf; 2751 uint64_t parameter; 2752 uint32_t status; 2753 uint32_t control; 2754 u_int i = 0; 2755 2756 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2757 2758 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0); 2759 2760 if (sc->sc_dying) 2761 return USBD_IOERROR; 2762 2763 KASSERT((xfer->rqflags & URQ_REQUEST) == 0); 2764 2765 parameter = DMAADDR(dma, 0); 2766 /* 2767 * XXX: (dsl) The physical buffer must not cross a 64k boundary. 2768 * If the user supplied buffer crosses such a boundary then 2 2769 * (or more) TRB should be used. 2770 * If multiple TRB are used the td_size field must be set correctly. 2771 * For v1.0 devices (like ivy bridge) this is the number of usb data 2772 * blocks needed to complete the transfer. 2773 * Setting it to 1 in the last TRB causes an extra zero-length 2774 * data block be sent. 2775 * The earlier documentation differs, I don't know how it behaves. 2776 */ 2777 KASSERT(len <= 0x10000); 2778 status = XHCI_TRB_2_IRQ_SET(0) | 2779 XHCI_TRB_2_TDSZ_SET(1) | 2780 XHCI_TRB_2_BYTES_SET(len); 2781 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) | 2782 XHCI_TRB_3_ISP_BIT | XHCI_TRB_3_IOC_BIT; 2783 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 2784 2785 mutex_enter(&tr->xr_lock); 2786 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 2787 mutex_exit(&tr->xr_lock); 2788 2789 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 2790 2791 if (sc->sc_bus.use_polling) { 2792 DPRINTFN(1, "polling", 0, 0, 0, 0); 2793 //xhci_waitintr(sc, xfer); 2794 } 2795 2796 return USBD_IN_PROGRESS; 2797 } 2798 2799 static void 2800 xhci_device_bulk_done(usbd_xfer_handle xfer) 2801 { 2802 #ifdef USB_DEBUG 2803 struct xhci_slot * const xs = xfer->pipe->device->hci_private; 2804 const u_int dci = xhci_ep_get_dci(xfer->pipe->endpoint->edesc); 2805 #endif 2806 const u_int endpt = xfer->pipe->endpoint->edesc->bEndpointAddress; 2807 const bool isread = UE_GET_DIR(endpt) == UE_DIR_IN; 2808 2809 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2810 2811 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0); 2812 2813 callout_stop(&xfer->timeout_handle); /* XXX wrong place */ 2814 2815 usb_syncmem(&xfer->dmabuf, 0, xfer->length, 2816 isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 2817 2818 2819 } 2820 2821 static void 2822 xhci_device_bulk_abort(usbd_xfer_handle xfer) 2823 { 2824 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2825 } 2826 2827 static void 2828 xhci_device_bulk_close(usbd_pipe_handle pipe) 2829 { 2830 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2831 } 2832 2833 /* --------------- */ 2834 /* device intrrupt */ 2835 2836 static usbd_status 2837 xhci_device_intr_transfer(usbd_xfer_handle xfer) 2838 { 2839 struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; 2840 usbd_status err; 2841 2842 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2843 2844 /* Insert last in queue. */ 2845 mutex_enter(&sc->sc_lock); 2846 err = usb_insert_transfer(xfer); 2847 mutex_exit(&sc->sc_lock); 2848 if (err) 2849 return err; 2850 2851 /* 2852 * Pipe isn't running (otherwise err would be USBD_INPROG), 2853 * so start it first. 2854 */ 2855 return (xhci_device_intr_start(SIMPLEQ_FIRST(&xfer->pipe->queue))); 2856 } 2857 2858 static usbd_status 2859 xhci_device_intr_start(usbd_xfer_handle xfer) 2860 { 2861 struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; 2862 struct xhci_slot * const xs = xfer->pipe->device->hci_private; 2863 const u_int dci = xhci_ep_get_dci(xfer->pipe->endpoint->edesc); 2864 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 2865 struct xhci_xfer * const xx = (void *)xfer; 2866 const uint32_t len = xfer->length; 2867 usb_dma_t * const dma = &xfer->dmabuf; 2868 uint64_t parameter; 2869 uint32_t status; 2870 uint32_t control; 2871 u_int i = 0; 2872 2873 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2874 2875 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0); 2876 2877 if (sc->sc_dying) 2878 return USBD_IOERROR; 2879 2880 KASSERT((xfer->rqflags & URQ_REQUEST) == 0); 2881 2882 parameter = DMAADDR(dma, 0); 2883 KASSERT(len <= 0x10000); 2884 status = XHCI_TRB_2_IRQ_SET(0) | 2885 XHCI_TRB_2_TDSZ_SET(1) | 2886 XHCI_TRB_2_BYTES_SET(len); 2887 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) | 2888 XHCI_TRB_3_ISP_BIT | XHCI_TRB_3_IOC_BIT; 2889 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 2890 2891 mutex_enter(&tr->xr_lock); 2892 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 2893 mutex_exit(&tr->xr_lock); 2894 2895 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 2896 2897 if (sc->sc_bus.use_polling) { 2898 DPRINTFN(1, "polling", 0, 0, 0, 0); 2899 //xhci_waitintr(sc, xfer); 2900 } 2901 2902 return USBD_IN_PROGRESS; 2903 } 2904 2905 static void 2906 xhci_device_intr_done(usbd_xfer_handle xfer) 2907 { 2908 struct xhci_softc * const sc __diagused = 2909 xfer->pipe->device->bus->hci_private; 2910 #ifdef USB_DEBUG 2911 struct xhci_slot * const xs = xfer->pipe->device->hci_private; 2912 const u_int dci = xhci_ep_get_dci(xfer->pipe->endpoint->edesc); 2913 #endif 2914 const u_int endpt = xfer->pipe->endpoint->edesc->bEndpointAddress; 2915 const bool isread = UE_GET_DIR(endpt) == UE_DIR_IN; 2916 2917 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2918 2919 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0); 2920 2921 KASSERT(sc->sc_bus.use_polling || mutex_owned(&sc->sc_lock)); 2922 2923 usb_syncmem(&xfer->dmabuf, 0, xfer->length, 2924 isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 2925 2926 #if 0 2927 device_printf(sc->sc_dev, ""); 2928 for (size_t i = 0; i < xfer->length; i++) { 2929 printf(" %02x", ((uint8_t const *)xfer->buffer)[i]); 2930 } 2931 printf("\n"); 2932 #endif 2933 2934 if (xfer->pipe->repeat) { 2935 xfer->status = xhci_device_intr_start(xfer); 2936 } else { 2937 callout_stop(&xfer->timeout_handle); /* XXX */ 2938 } 2939 2940 } 2941 2942 static void 2943 xhci_device_intr_abort(usbd_xfer_handle xfer) 2944 { 2945 struct xhci_softc * const sc __diagused = 2946 xfer->pipe->device->bus->hci_private; 2947 2948 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2949 2950 KASSERT(mutex_owned(&sc->sc_lock)); 2951 DPRINTFN(15, "%p", xfer, 0, 0, 0); 2952 KASSERT(xfer->pipe->intrxfer == xfer); 2953 xfer->status = USBD_CANCELLED; 2954 usb_transfer_complete(xfer); 2955 } 2956 2957 static void 2958 xhci_device_intr_close(usbd_pipe_handle pipe) 2959 { 2960 //struct xhci_softc * const sc = pipe->device->bus->hci_private; 2961 2962 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2963 DPRINTFN(15, "%p", pipe, 0, 0, 0); 2964 2965 xhci_unconfigure_endpoint(pipe); 2966 } 2967 2968 /* ------------ */ 2969 2970 static void 2971 xhci_timeout(void *addr) 2972 { 2973 struct xhci_xfer * const xx = addr; 2974 usbd_xfer_handle const xfer = &xx->xx_xfer; 2975 struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; 2976 2977 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2978 2979 if (sc->sc_dying) { 2980 return; 2981 } 2982 2983 usb_init_task(&xx->xx_abort_task, xhci_timeout_task, addr, 2984 USB_TASKQ_MPSAFE); 2985 usb_add_task(xx->xx_xfer.pipe->device, &xx->xx_abort_task, 2986 USB_TASKQ_HC); 2987 } 2988 2989 static void 2990 xhci_timeout_task(void *addr) 2991 { 2992 usbd_xfer_handle const xfer = addr; 2993 struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; 2994 2995 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2996 2997 mutex_enter(&sc->sc_lock); 2998 #if 0 2999 xhci_abort_xfer(xfer, USBD_TIMEOUT); 3000 #else 3001 xfer->status = USBD_TIMEOUT; 3002 usb_transfer_complete(xfer); 3003 #endif 3004 mutex_exit(&sc->sc_lock); 3005 } 3006