Lines Matching +full:full +full:- +full:pwr +full:- +full:cycle +full:- +full:in +full:- +full:suspend
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
7 * Redistribution and use in source and binary forms, with or without
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 -s <n>,xhci,{devices}
88 #define XHCI_STREAMS_MAX 1 /* 4-15 in XHCI spec */
90 /* caplength and hci-version registers */
160 uint32_t ccs; /* consumer cycle state */
190 /* device context base address array: maps slot->device context */
198 uint32_t portpmsc; /* port pwr mgmt status & control */
227 uint64_t erstba; /* event ring seg-tbl base addr */
235 int er_enq_idx; /* event ring enqueue index - xHCI */
237 uint32_t er_events_cnt; /* number of events in ER */
238 uint32_t event_pcs; /* producer cycle state flag */
247 * This is referenced from usb_hci->hci_sc; 1 pci_xhci_dev_emu for each
293 #define XHCI_PORTREG_PTR(x,n) &((x)->portregs[(n) - 1])
294 #define XHCI_DEVINST_PTR(x,n) ((x)->devices[(n) - 1])
295 #define XHCI_SLOTDEV_PTR(x,n) ((x)->slots[(n) - 1])
297 #define XHCI_HALTED(sc) ((sc)->opregs.usbsts & XHCI_STS_HCH)
299 #define XHCI_GADDR_SIZE(a) (XHCI_PADDR_SZ - \
300 (((uint64_t) (a)) & (XHCI_PADDR_SZ - 1)))
301 #define XHCI_GADDR(sc,a) paddr_guest2host((sc)->xsc_pi->pi_vmctx, \
357 evtrb->qwTrb0 = port << 24;
358 evtrb->dwTrb2 = XHCI_TRB_2_ERROR_SET(errcode);
359 evtrb->dwTrb3 = XHCI_TRB_3_TYPE_SET(evtype);
369 sc->rtsregs.er_enq_idx = 0;
370 sc->rtsregs.er_events_cnt = 0;
371 sc->rtsregs.event_pcs = 1;
385 do_intr = (sc->opregs.usbcmd & XHCI_CMD_RS) == 0;
387 sc->opregs.usbcmd |= XHCI_CMD_RS;
388 sc->opregs.usbsts &= ~XHCI_STS_HCH;
389 sc->opregs.usbsts |= XHCI_STS_PCD;
402 port->portsc |= XHCI_PS_CSC | XHCI_PS_CCS;
403 port->portsc &= ~XHCI_PS_PLS_MASK;
406 * XHCI 4.19.3 USB2 RxDetect->Polling,
407 * USB3 Polling->U0
409 if (dev->dev_ue->ue_usbver == 2)
410 port->portsc |=
413 port->portsc |=
425 sc->opregs.usbcmd &= ~XHCI_CMD_RS;
426 sc->opregs.usbsts |= XHCI_STS_HCH;
427 sc->opregs.usbsts &= ~XHCI_STS_PCD;
431 cmd |= sc->opregs.usbcmd & XHCI_CMD_RS;
456 if (sc->portregs == NULL)
459 port = (offset - XHCI_PORTREGS_PORT0) / XHCI_PORTREGS_SETSZ;
460 offset = (offset - XHCI_PORTREGS_PORT0) % XHCI_PORTREGS_SETSZ;
487 if ((p->portsc & XHCI_PS_PP) == 0) {
494 oldpls = XHCI_PS_PLS_GET(p->portsc);
497 p->portsc &= XHCI_PS_PED | XHCI_PS_PLS_MASK |
501 p->portsc |= XHCI_PS_CCS;
503 p->portsc |= (value &
513 p->portsc &= ~(value &
535 p->portsc &= ~XHCI_PS_PLS_MASK;
536 p->portsc |= XHCI_PS_PLS_SET(newpls) |
557 p->portpmsc = value;
569 p->porthlpmc = value;
586 assert(sc->opregs.dcbaa_p != NULL);
588 devctx_addr = sc->opregs.dcbaa_p->dcba[slot];
610 if (XHCI_TRB_3_TYPE_GET(curtrb->dwTrb3) == XHCI_TRB_TYPE_LINK) {
612 *guestaddr = curtrb->qwTrb0 & ~0xFUL;
614 next = XHCI_GADDR(sc, curtrb->qwTrb0 & ~0xFUL);
629 sc->rtsregs.intrreg.erdp |= XHCI_ERDP_LO_BUSY;
630 sc->rtsregs.intrreg.iman |= XHCI_IMAN_INTR_PEND;
631 sc->opregs.usbsts |= XHCI_STS_EINT;
634 if ((sc->opregs.usbcmd & XHCI_CMD_INTE) &&
635 (sc->rtsregs.intrreg.iman & XHCI_IMAN_INTR_ENA)) {
636 if (pci_msi_enabled(sc->xsc_pi))
637 pci_generate_msi(sc->xsc_pi, 0);
639 pci_lintr_assert(sc->xsc_pi);
647 if (!pci_msi_enabled(sc->xsc_pi))
648 pci_lintr_assert(sc->xsc_pi);
659 dev_ctx = dev->dev_ctx;
660 ep_ctx = &dev_ctx->ctx_ep[epid];
661 devep = &dev->eps[epid];
662 pstreams = XHCI_EPCTX_0_MAXP_STREAMS_GET(ep_ctx->dwEpCtx0);
665 assert(devep->ep_sctx_trbs == NULL);
667 devep->ep_sctx = XHCI_GADDR(dev->xsc, ep_ctx->qwEpCtx2 &
669 devep->ep_sctx_trbs = calloc(pstreams,
672 devep->ep_sctx_trbs[i].ringaddr =
673 devep->ep_sctx[i].qwSctx0 &
675 devep->ep_sctx_trbs[i].ccs =
676 XHCI_SCTX_0_DCS_GET(devep->ep_sctx[i].qwSctx0);
680 devep->ep_ringaddr = ep_ctx->qwEpCtx2 &
682 devep->ep_ccs = XHCI_EPCTX_2_DCS_GET(ep_ctx->qwEpCtx2);
683 devep->ep_tr = XHCI_GADDR(dev->xsc, devep->ep_ringaddr);
684 DPRINTF(("init_ep tr DCS %x", devep->ep_ccs));
686 devep->ep_MaxPStreams = pstreams;
688 if (devep->ep_xfer == NULL) {
689 devep->ep_xfer = malloc(sizeof(struct usb_data_xfer));
690 USB_DATA_XFER_INIT(devep->ep_xfer);
703 dev_ctx = dev->dev_ctx;
704 ep_ctx = &dev_ctx->ctx_ep[epid];
705 ep_ctx->dwEpCtx0 = (ep_ctx->dwEpCtx0 & ~0x7) | XHCI_ST_EPCTX_DISABLED;
707 devep = &dev->eps[epid];
708 if (devep->ep_MaxPStreams > 0)
709 free(devep->ep_sctx_trbs);
711 if (devep->ep_xfer != NULL) {
712 free(devep->ep_xfer);
713 devep->ep_xfer = NULL;
731 dev->dev_slotstate = XHCI_ST_DISABLED;
749 rts = &sc->rtsregs;
751 erdp = rts->intrreg.erdp & ~0xF;
752 erdp_idx = (erdp - rts->erstba_p[rts->er_deq_seg].qwEvrsTablePtr) /
756 evtrb->qwTrb0, evtrb->dwTrb2, evtrb->dwTrb3));
758 erdp_idx, rts->er_deq_seg, rts->er_enq_idx,
759 rts->er_enq_seg, rts->event_pcs));
761 erdp, rts->erstba_p->qwEvrsTablePtr,
762 rts->erstba_p->dwEvrsTableSize, do_intr));
764 evtrbptr = &rts->erst_p[rts->er_enq_idx];
766 /* TODO: multi-segment table */
767 if (rts->er_events_cnt >= rts->erstba_p->dwEvrsTableSize) {
768 DPRINTF(("pci_xhci[%d] cannot insert event; ring full",
774 if (rts->er_events_cnt == rts->erstba_p->dwEvrsTableSize - 1) {
777 if ((evtrbptr->dwTrb3 & 0x1) == (rts->event_pcs & 0x1)) {
779 DPRINTF(("pci_xhci[%d] insert evt err: ring full",
787 rts->event_pcs;
788 rts->er_events_cnt++;
789 memcpy(&rts->erst_p[rts->er_enq_idx], &errev,
791 rts->er_enq_idx = (rts->er_enq_idx + 1) %
792 rts->erstba_p->dwEvrsTableSize;
799 rts->er_events_cnt++;
802 evtrb->dwTrb3 &= ~XHCI_TRB_3_CYCLE_BIT;
803 evtrb->dwTrb3 |= rts->event_pcs;
805 memcpy(&rts->erst_p[rts->er_enq_idx], evtrb, sizeof(struct xhci_trb));
806 rts->er_enq_idx = (rts->er_enq_idx + 1) %
807 rts->erstba_p->dwEvrsTableSize;
809 if (rts->er_enq_idx == 0)
810 rts->event_pcs ^= 1;
827 if (sc->portregs != NULL)
830 if (dev && dev->dev_slotstate == XHCI_ST_DISABLED) {
832 dev->dev_slotstate = XHCI_ST_ENABLED;
834 dev->hci.hci_address = i;
853 if (sc->portregs == NULL) {
864 if (dev->dev_slotstate == XHCI_ST_DISABLED) {
867 dev->dev_slotstate = XHCI_ST_DISABLED;
886 if (sc->portregs == NULL) {
898 if (!dev || dev->dev_slotstate == XHCI_ST_DISABLED)
901 dev->dev_slotstate = XHCI_ST_DEFAULT;
903 dev->hci.hci_address = 0;
911 dev_ctx->ctx_slot.dwSctx3 = FIELD_REPLACE(
912 dev_ctx->ctx_slot.dwSctx3, XHCI_ST_SLCTX_DEFAULT,
916 dev_ctx->ctx_slot.dwSctx0 = FIELD_REPLACE(
917 dev_ctx->ctx_slot.dwSctx0, 1, 0x1F, 27);
919 /* reset all eps other than ep-0 */
921 ep_ctx = &dev_ctx->ctx_ep[i];
922 ep_ctx->dwEpCtx0 = FIELD_REPLACE( ep_ctx->dwEpCtx0,
944 input_ctx = XHCI_GADDR(sc, trb->qwTrb0 & ~0xFUL);
945 islot_ctx = &input_ctx->ctx_slot;
946 ep0_ctx = &input_ctx->ctx_ep[1];
949 input_ctx->ctx_input.dwInCtx0, input_ctx->ctx_input.dwInCtx1));
951 islot_ctx->dwSctx0, islot_ctx->dwSctx1,
952 islot_ctx->dwSctx2, islot_ctx->dwSctx3));
954 ep0_ctx->dwEpCtx0, ep0_ctx->dwEpCtx1, ep0_ctx->qwEpCtx2,
955 ep0_ctx->dwEpCtx4));
957 /* when setting address: drop-ctx=0, add-ctx=slot+ep0 */
958 if ((input_ctx->ctx_input.dwInCtx0 != 0) ||
959 (input_ctx->ctx_input.dwInCtx1 & 0x03) != 0x03) {
978 dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1,
979 dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3));
984 dev->hci.hci_address = slot;
985 dev->dev_ctx = dev_ctx;
987 if (dev->dev_ue->ue_reset == NULL ||
988 dev->dev_ue->ue_reset(dev->dev_sc) < 0) {
993 memcpy(&dev_ctx->ctx_slot, islot_ctx, sizeof(struct xhci_slot_ctx));
995 dev_ctx->ctx_slot.dwSctx3 =
999 memcpy(&dev_ctx->ctx_ep[1], ep0_ctx, sizeof(struct xhci_endp_ctx));
1000 ep0_ctx = &dev_ctx->ctx_ep[1];
1001 ep0_ctx->dwEpCtx0 = (ep0_ctx->dwEpCtx0 & ~0x7) |
1006 dev->dev_slotstate = XHCI_ST_ADDRESSED;
1010 dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1,
1011 dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3));
1013 ep0_ctx->dwEpCtx0, ep0_ctx->dwEpCtx1, ep0_ctx->qwEpCtx2,
1014 ep0_ctx->dwEpCtx4));
1040 if ((trb->dwTrb3 & XHCI_TRB_3_DCEP_BIT) != 0) {
1041 DPRINTF(("pci_xhci config_ep - deconfigure ep slot %u",
1043 if (dev->dev_ue->ue_stop != NULL)
1044 dev->dev_ue->ue_stop(dev->dev_sc);
1046 dev->dev_slotstate = XHCI_ST_ADDRESSED;
1048 dev->hci.hci_address = 0;
1056 dev_ctx->ctx_slot.dwSctx0 = FIELD_REPLACE(
1057 dev_ctx->ctx_slot.dwSctx0, 1, 0x1F, 27);
1060 dev_ctx->ctx_slot.dwSctx3 = FIELD_REPLACE(
1061 dev_ctx->ctx_slot.dwSctx3, XHCI_ST_SLCTX_ADDRESSED,
1073 if (dev->dev_slotstate < XHCI_ST_ADDRESSED) {
1075 dev->dev_slotstate));
1080 /* In addressed/configured state;
1082 * ep->state = DISABLED
1084 * cp(ep-in, ep-out)
1085 * ep->state = RUNNING
1088 * cp(ep-in, ep-out)
1089 * ep->state = RUNNING
1090 * if input->DisabledCtx[2-31] < 30: (at least 1 ep not disabled)
1091 * slot->state = configured
1094 input_ctx = XHCI_GADDR(sc, trb->qwTrb0 & ~0xFUL);
1095 dev_ctx = dev->dev_ctx;
1097 input_ctx->ctx_input.dwInCtx0, input_ctx->ctx_input.dwInCtx1,
1098 input_ctx->ctx_input.dwInCtx7));
1101 ep_ctx = &dev_ctx->ctx_ep[i];
1103 if (input_ctx->ctx_input.dwInCtx0 &
1105 DPRINTF((" config ep - dropping ep %d", i));
1109 if (input_ctx->ctx_input.dwInCtx1 &
1111 iep_ctx = &input_ctx->ctx_ep[i];
1114 i, iep_ctx->dwEpCtx0, iep_ctx->dwEpCtx1,
1115 iep_ctx->qwEpCtx2, iep_ctx->dwEpCtx4));
1122 ep_ctx->dwEpCtx0 = FIELD_REPLACE(
1123 ep_ctx->dwEpCtx0, XHCI_ST_EPCTX_RUNNING, 0x7, 0);
1128 dev_ctx->ctx_slot.dwSctx3 = FIELD_REPLACE(
1129 dev_ctx->ctx_slot.dwSctx3, XHCI_ST_SLCTX_CONFIGURED, 0x1F, 27);
1130 dev_ctx->ctx_slot.dwSctx0 = FIELD_COPY(
1131 dev_ctx->ctx_slot.dwSctx0, input_ctx->ctx_slot.dwSctx0, 0x1F, 27);
1132 dev->dev_slotstate = XHCI_ST_CONFIGURED;
1136 slot, dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1,
1137 dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3));
1154 epid = XHCI_TRB_3_EP_GET(trb->dwTrb3);
1165 type = XHCI_TRB_3_TYPE_GET(trb->dwTrb3);
1168 (trb->dwTrb3 & XHCI_TRB_3_SUSP_EP_BIT) != 0) {
1169 /* XXX suspend endpoint for 10ms */
1178 devep = &dev->eps[epid];
1179 if (devep->ep_xfer != NULL)
1180 USB_DATA_XFER_RESET(devep->ep_xfer);
1182 dev_ctx = dev->dev_ctx;
1185 ep_ctx = &dev_ctx->ctx_ep[epid];
1187 ep_ctx->dwEpCtx0 = (ep_ctx->dwEpCtx0 & ~0x7) | XHCI_ST_EPCTX_STOPPED;
1189 if (devep->ep_MaxPStreams == 0)
1190 ep_ctx->qwEpCtx2 = devep->ep_ringaddr | devep->ep_ccs;
1193 epid, ep_ctx->dwEpCtx0, ep_ctx->dwEpCtx1, ep_ctx->qwEpCtx2,
1194 ep_ctx->dwEpCtx4));
1197 (dev->dev_ue->ue_reset == NULL ||
1198 dev->dev_ue->ue_reset(dev->dev_sc) < 0)) {
1214 if (devep->ep_MaxPStreams == 0)
1217 if (devep->ep_MaxPStreams > XHCI_STREAMS_MAX)
1220 if (XHCI_EPCTX_0_LSA_GET(ep->dwEpCtx0) == 0) {
1226 if (streamid >= devep->ep_MaxPStreams)
1229 sctx = (struct xhci_stream_ctx *)XHCI_GADDR(sc, ep->qwEpCtx2 & ~0xFUL) +
1231 if (!XHCI_SCTX_0_SCT_GET(sctx->qwSctx0))
1256 DPRINTF(("pci_xhci set_tr: new-tr x%016lx, SCT %u DCS %u",
1257 (trb->qwTrb0 & ~0xF), (uint32_t)((trb->qwTrb0 >> 1) & 0x7),
1258 (uint32_t)(trb->qwTrb0 & 0x1)));
1259 DPRINTF((" stream-id %u, slot %u, epid %u, C %u",
1260 (trb->dwTrb2 >> 16) & 0xFFFF,
1261 XHCI_TRB_3_SLOT_GET(trb->dwTrb3),
1262 XHCI_TRB_3_EP_GET(trb->dwTrb3), trb->dwTrb3 & 0x1));
1264 epid = XHCI_TRB_3_EP_GET(trb->dwTrb3);
1271 dev_ctx = dev->dev_ctx;
1274 ep_ctx = &dev_ctx->ctx_ep[epid];
1275 devep = &dev->eps[epid];
1277 switch (XHCI_EPCTX_0_EPSTATE_GET(ep_ctx->dwEpCtx0)) {
1283 XHCI_EPCTX_0_EPSTATE_GET(ep_ctx->dwEpCtx0)));
1288 streamid = XHCI_TRB_2_STREAM_GET(trb->dwTrb2);
1289 if (devep->ep_MaxPStreams > 0) {
1292 assert(devep->ep_sctx != NULL);
1294 devep->ep_sctx[streamid].qwSctx0 = trb->qwTrb0;
1295 devep->ep_sctx_trbs[streamid].ringaddr =
1296 trb->qwTrb0 & ~0xF;
1297 devep->ep_sctx_trbs[streamid].ccs =
1298 XHCI_EPCTX_2_DCS_GET(trb->qwTrb0);
1305 ep_ctx->qwEpCtx2 = trb->qwTrb0 & ~0xFUL;
1306 devep->ep_ringaddr = ep_ctx->qwEpCtx2 & ~0xFUL;
1307 devep->ep_ccs = trb->qwTrb0 & 0x1;
1308 devep->ep_tr = XHCI_GADDR(sc, devep->ep_ringaddr);
1311 pci_xhci_dump_trb(devep->ep_tr);
1313 ep_ctx->dwEpCtx0 = (ep_ctx->dwEpCtx0 & ~0x7) | XHCI_ST_EPCTX_STOPPED;
1329 input_ctx = XHCI_GADDR(sc, trb->qwTrb0 & ~0xFUL);
1330 islot_ctx = &input_ctx->ctx_slot;
1331 ep0_ctx = &input_ctx->ctx_ep[1];
1334 input_ctx->ctx_input.dwInCtx0, input_ctx->ctx_input.dwInCtx1));
1336 islot_ctx->dwSctx0, islot_ctx->dwSctx1,
1337 islot_ctx->dwSctx2, islot_ctx->dwSctx3));
1339 ep0_ctx->dwEpCtx0, ep0_ctx->dwEpCtx1, ep0_ctx->qwEpCtx2,
1340 ep0_ctx->dwEpCtx4));
1342 /* this command expects drop-ctx=0 & add-ctx=slot+ep0 */
1343 if ((input_ctx->ctx_input.dwInCtx0 != 0) ||
1344 (input_ctx->ctx_input.dwInCtx1 & 0x03) == 0) {
1354 /* assign address to slot; in this emulation, slot_id = address */
1363 dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1,
1364 dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3));
1366 if (input_ctx->ctx_input.dwInCtx1 & 0x01) { /* slot ctx */
1368 dev_ctx->ctx_slot.dwSctx1 = FIELD_COPY(
1369 dev_ctx->ctx_slot.dwSctx1, input_ctx->ctx_slot.dwSctx1,
1373 dev_ctx->ctx_slot.dwSctx2 = FIELD_COPY(
1374 dev_ctx->ctx_slot.dwSctx2, input_ctx->ctx_slot.dwSctx2,
1377 if (input_ctx->ctx_input.dwInCtx1 & 0x02) { /* control ctx */
1379 dev_ctx->ctx_ep[1].dwEpCtx1 = FIELD_COPY(
1380 dev_ctx->ctx_ep[1].dwEpCtx1, ep0_ctx->dwEpCtx1,
1383 ep0_ctx = &dev_ctx->ctx_ep[1];
1388 dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1,
1389 dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3));
1391 ep0_ctx->dwEpCtx0, ep0_ctx->dwEpCtx1, ep0_ctx->qwEpCtx2,
1392 ep0_ctx->dwEpCtx4));
1404 uint32_t ccs; /* cycle state (XHCI 4.9.2) */
1411 sc->opregs.crcr |= XHCI_CRCR_LO_CRR;
1413 trb = sc->opregs.cr_p;
1414 ccs = sc->opregs.crcr & XHCI_CRCR_LO_RCS;
1415 crcr = sc->opregs.crcr & ~0xF;
1418 sc->opregs.cr_p = trb;
1420 type = XHCI_TRB_3_TYPE_GET(trb->dwTrb3);
1422 if ((trb->dwTrb3 & XHCI_TRB_3_CYCLE_BIT) !=
1428 type, trb->qwTrb0, trb->dwTrb2, trb->dwTrb3,
1429 trb->dwTrb3 & XHCI_TRB_3_CYCLE_BIT, ccs));
1439 if (trb->dwTrb3 & XHCI_TRB_3_TC_BIT)
1448 slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3);
1453 slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3);
1458 slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3);
1463 slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3);
1469 slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3);
1475 slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3);
1480 slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3);
1485 slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3);
1528 sc->opregs.crcr = crcr | (sc->opregs.crcr & XHCI_CRCR_LO_CA) | ccs;
1529 sc->opregs.crcr &= ~XHCI_CRCR_LO_CRR;
1564 type = XHCI_TRB_3_TYPE_GET(trb->dwTrb3);
1568 trb->qwTrb0, trb->dwTrb2, trb->dwTrb3));
1586 devep = &dev->eps[epid];
1592 ep_ctx = &dev_ctx->ctx_ep[epid];
1599 for (i = xfer->head; xfer->ndata > 0; ) {
1600 evtrb.qwTrb0 = (uint64_t)xfer->data[i].hci_data;
1602 trbflags = trb->dwTrb3;
1606 i, xfer->data[i].processed, xfer->data[i].blen,
1609 trb->dwTrb3 & XHCI_TRB_3_IOC_BIT ? 1 : 0));
1611 if (!xfer->data[i].processed) {
1612 xfer->head = i;
1616 xfer->ndata--;
1617 edtla += xfer->data[i].bdone;
1619 trb->dwTrb3 = (trb->dwTrb3 & ~0x1) | (xfer->data[i].ccs);
1622 xfer->data[i].streamid, xfer->data[i].trbnext,
1623 xfer->data[i].ccs);
1626 if (!(trb->dwTrb3 & XHCI_TRB_3_IOC_BIT) &&
1628 (trb->dwTrb3 & XHCI_TRB_3_ISP_BIT))) {
1635 XHCI_TRB_2_REM_SET(xfer->data[i].blen);
1642 evtrb.qwTrb0 = trb->qwTrb0;
1668 if (devep->ep_MaxPStreams != 0) {
1669 devep->ep_sctx[streamid].qwSctx0 = (ringaddr & ~0xFUL) |
1672 devep->ep_sctx_trbs[streamid].ringaddr = ringaddr & ~0xFUL;
1673 devep->ep_sctx_trbs[streamid].ccs = ccs & 0x1;
1674 ep_ctx->qwEpCtx2 = (ep_ctx->qwEpCtx2 & ~0x1) | (ccs & 0x1);
1676 DPRINTF(("xhci update ep-ring stream %d, addr %lx",
1677 streamid, devep->ep_sctx[streamid].qwSctx0));
1679 devep->ep_ringaddr = ringaddr & ~0xFUL;
1680 devep->ep_ccs = ccs & 0x1;
1681 devep->ep_tr = XHCI_GADDR(sc, ringaddr & ~0xFUL);
1682 ep_ctx->qwEpCtx2 = (ringaddr & ~0xFUL) | (ccs & 0x1);
1684 DPRINTF(("xhci update ep-ring, addr %lx",
1685 (devep->ep_ringaddr | devep->ep_ccs)));
1701 * Outstanding transfer still in progress (device NAK'd earlier) so retry
1713 ep_ctx->dwEpCtx0 = FIELD_REPLACE(
1714 ep_ctx->dwEpCtx0, XHCI_ST_EPCTX_RUNNING, 0x7, 0);
1719 xfer = devep->ep_xfer;
1723 if (dev->dev_ue->ue_data != NULL) {
1724 err = dev->dev_ue->ue_data(dev->dev_sc, xfer,
1727 if (USB_DATA_GET_ERRCODE(&xfer->data[xfer->head]) ==
1764 ep_ctx->dwEpCtx0 = FIELD_REPLACE(ep_ctx->dwEpCtx0,
1767 xfer = devep->ep_xfer;
1781 trbflags = trb->dwTrb3;
1786 DPRINTF(("Cycle-bit changed trbflags %x, ccs %x",
1795 if (trb->dwTrb3 & XHCI_TRB_3_TC_BIT)
1800 xfer_block->processed = 1;
1805 XHCI_TRB_2_BYTES_GET(trb->dwTrb2) != 8) {
1812 val = trb->qwTrb0;
1813 if (!xfer->ureq)
1814 xfer->ureq = malloc(
1816 memcpy(xfer->ureq, &val,
1821 xfer_block->processed = 1;
1827 DPRINTF(("pci_xhci: trb not supposed to be in "
1837 &trb->qwTrb0 : XHCI_GADDR(sc, trb->qwTrb0)),
1838 trb->dwTrb2 & 0x1FFFF, (void *)addr, ccs);
1849 xfer_block->processed = 1;
1856 xfer_block->processed = 1;
1873 xfer_block->trbnext = addr;
1874 xfer_block->streamid = streamid;
1891 DPRINTF(("pci_xhci[%d]: xfer->ndata %u", __LINE__, xfer->ndata));
1893 if (xfer->ndata <= 0)
1899 if (dev->dev_ue->ue_request != NULL)
1900 usberr = dev->dev_ue->ue_request(dev->dev_sc, xfer);
1921 DPRINTF(("pci_xhci[%d]: event ring full", __LINE__));
1970 devep = &dev->eps[epid];
1975 ep_ctx = &dev_ctx->ctx_ep[epid];
1980 epid, ep_ctx->dwEpCtx0, ep_ctx->dwEpCtx1, ep_ctx->qwEpCtx2,
1981 ep_ctx->dwEpCtx4));
1983 if (ep_ctx->qwEpCtx2 == 0)
1987 if (devep->ep_xfer->ndata > 0) {
1993 if (devep->ep_MaxPStreams != 0) {
2009 sctx_tr = &devep->ep_sctx_trbs[streamid];
2010 ringaddr = sctx_tr->ringaddr;
2011 ccs = sctx_tr->ccs;
2012 trb = XHCI_GADDR(sc, sctx_tr->ringaddr & ~0xFUL);
2014 streamid, ep_ctx->qwEpCtx2 & XHCI_TRB_3_CYCLE_BIT,
2015 trb->dwTrb3 & XHCI_TRB_3_CYCLE_BIT));
2021 ringaddr = devep->ep_ringaddr;
2022 ccs = devep->ep_ccs;
2023 trb = devep->ep_tr;
2025 ep_ctx->qwEpCtx2 & XHCI_TRB_3_CYCLE_BIT,
2026 trb->dwTrb3 & XHCI_TRB_3_CYCLE_BIT));
2029 if (XHCI_TRB_3_TYPE_GET(trb->dwTrb3) == 0) {
2031 ep_ctx->qwEpCtx2, devep->ep_ringaddr, epid));
2044 offset = (offset - sc->dboff) / sizeof(uint32_t);
2056 else if (sc->portregs != NULL)
2067 offset -= sc->rtsoff;
2077 offset -= 0x20; /* start of intrreg */
2079 rts = &sc->rtsregs;
2084 rts->intrreg.iman &= ~XHCI_IMAN_INTR_PEND;
2085 rts->intrreg.iman = (value & XHCI_IMAN_INTR_ENA) |
2086 (rts->intrreg.iman & XHCI_IMAN_INTR_PEND);
2094 rts->intrreg.imod = value;
2098 rts->intrreg.erstsz = value & 0xFFFF;
2103 rts->intrreg.erstba = MASK_64_HI(sc->rtsregs.intrreg.erstba) |
2109 rts->intrreg.erstba = (value << 32) |
2110 MASK_64_LO(sc->rtsregs.intrreg.erstba);
2112 rts->erstba_p = XHCI_GADDR(sc,
2113 sc->rtsregs.intrreg.erstba & ~0x3FUL);
2115 rts->erst_p = XHCI_GADDR(sc,
2116 sc->rtsregs.erstba_p->qwEvrsTablePtr & ~0x3FUL);
2118 rts->er_enq_idx = 0;
2119 rts->er_events_cnt = 0;
2122 rts->erstba_p,
2123 rts->erstba_p->qwEvrsTablePtr,
2124 rts->erstba_p->dwEvrsTableSize));
2129 rts->intrreg.erdp =
2130 MASK_64_HI(sc->rtsregs.intrreg.erdp) |
2131 (rts->intrreg.erdp & XHCI_ERDP_LO_BUSY) |
2134 rts->intrreg.erdp &= ~XHCI_ERDP_LO_BUSY;
2135 rts->intrreg.iman &= ~XHCI_IMAN_INTR_PEND;
2138 rts->er_deq_seg = XHCI_ERDP_LO_SINDEX(value);
2144 rts->intrreg.erdp = (value << 32) |
2145 MASK_64_LO(sc->rtsregs.intrreg.erdp);
2147 if (rts->er_events_cnt > 0) {
2151 erdp = rts->intrreg.erdp & ~0xF;
2152 erdp_i = (erdp - rts->erstba_p->qwEvrsTablePtr) /
2155 if (erdp_i <= rts->er_enq_idx)
2156 rts->er_events_cnt = rts->er_enq_idx - erdp_i;
2158 rts->er_events_cnt =
2159 rts->erstba_p->dwEvrsTableSize -
2160 (erdp_i - rts->er_enq_idx);
2163 erdp, rts->er_events_cnt));
2182 if (sc->portregs == NULL)
2185 port = (offset - XHCI_PORTREGS_PORT0) / XHCI_PORTREGS_SETSZ;
2186 offset = (offset - XHCI_PORTREGS_PORT0) % XHCI_PORTREGS_SETSZ;
2199 reg = portregs->portsc;
2202 reg = portregs->portpmsc;
2205 reg = portregs->portli;
2208 reg = portregs->porthlpmc;
2217 DPRINTF(("pci_xhci: portregs read offset 0x%lx port %u -> 0x%x",
2227 offset -= XHCI_CAPLEN;
2235 sc->opregs.usbcmd = pci_xhci_usbcmd_write(sc, value & 0x3F0F);
2240 sc->opregs.usbsts &= ~(value &
2250 sc->opregs.dnctrl = value & 0xFFFF;
2254 if (sc->opregs.crcr & XHCI_CRCR_LO_CRR) {
2255 sc->opregs.crcr &= ~(XHCI_CRCR_LO_CS|XHCI_CRCR_LO_CA);
2256 sc->opregs.crcr |= value &
2259 sc->opregs.crcr = MASK_64_HI(sc->opregs.crcr) |
2265 if (!(sc->opregs.crcr & XHCI_CRCR_LO_CRR)) {
2266 sc->opregs.crcr = MASK_64_LO(sc->opregs.crcr) |
2269 sc->opregs.cr_p = XHCI_GADDR(sc,
2270 sc->opregs.crcr & ~0xF);
2273 if (sc->opregs.crcr & XHCI_CRCR_LO_CS) {
2277 if (sc->opregs.crcr & XHCI_CRCR_LO_CA) {
2284 sc->opregs.dcbaap = MASK_64_HI(sc->opregs.dcbaap) |
2289 sc->opregs.dcbaap = MASK_64_LO(sc->opregs.dcbaap) |
2291 sc->opregs.dcbaa_p = XHCI_GADDR(sc, sc->opregs.dcbaap & ~0x3FUL);
2294 sc->opregs.dcbaap, (uint64_t)sc->opregs.dcbaa_p));
2298 sc->opregs.config = value & 0x03FF;
2316 sc = pi->pi_arg;
2320 pthread_mutex_lock(&sc->mtx);
2322 WPRINTF(("pci_xhci: write RO-CAPs offset %ld", offset));
2323 else if (offset < sc->dboff)
2325 else if (offset < sc->rtsoff)
2327 else if (offset < sc->regsend)
2332 pthread_mutex_unlock(&sc->mtx);
2342 value = sc->caplength;
2346 value = sc->hcsparams1;
2350 value = sc->hcsparams2;
2354 value = sc->hcsparams3;
2358 value = sc->hccparams1;
2362 value = sc->dboff;
2366 value = sc->rtsoff;
2370 value = sc->hccparams2;
2378 DPRINTF(("pci_xhci: hostcap read offset 0x%lx -> 0x%lx",
2389 offset = (offset - XHCI_CAPLEN);
2393 value = sc->opregs.usbcmd;
2397 value = sc->opregs.usbsts;
2401 value = sc->opregs.pgsz;
2405 value = sc->opregs.dnctrl;
2409 value = sc->opregs.crcr & XHCI_CRCR_LO_CRR;
2417 value = sc->opregs.dcbaap & 0xFFFFFFFF;
2421 value = (sc->opregs.dcbaap >> 32) & 0xFFFFFFFF;
2425 value = sc->opregs.config;
2438 DPRINTF(("pci_xhci: hostop read offset 0x%lx -> 0x%lx",
2457 offset -= sc->rtsoff;
2461 value = sc->rtsregs.mfindex;
2466 offset -= 0x20;
2469 assert(offset < sizeof(sc->rtsregs.intrreg));
2471 p = &sc->rtsregs.intrreg.iman;
2476 DPRINTF(("pci_xhci: rtsregs read offset 0x%lx -> 0x%x",
2487 offset -= sc->regsend;
2492 /* rev major | rev minor | next-cap | cap-id */
2500 /* psic | proto-defined | compat # | compat offset */
2501 value = ((XHCI_MAX_DEVS/2) << 8) | sc->usb2_port_start;
2506 /* rev major | rev minor | next-cap | cap-id */
2514 /* psic | proto-defined | compat # | compat offset */
2515 value = ((XHCI_MAX_DEVS/2) << 8) | sc->usb3_port_start;
2524 DPRINTF(("pci_xhci: xecp read offset 0x%lx -> 0x%x",
2537 sc = pi->pi_arg;
2541 pthread_mutex_lock(&sc->mtx);
2544 else if (offset < sc->dboff)
2546 else if (offset < sc->rtsoff)
2548 else if (offset < sc->regsend)
2550 else if (offset < (sc->regsend + 4*32))
2557 pthread_mutex_unlock(&sc->mtx);
2589 port->portsc &= ~(XHCI_PS_PLS_MASK | XHCI_PS_PR | XHCI_PS_PRC);
2590 port->portsc |= XHCI_PS_PED |
2591 XHCI_PS_SPEED_SET(dev->dev_ue->ue_usbspeed);
2593 if (warm && dev->dev_ue->ue_usbver == 3) {
2594 port->portsc |= XHCI_PS_WRC;
2597 if ((port->portsc & XHCI_PS_PRC) == 0) {
2598 port->portsc |= XHCI_PS_PRC;
2620 port->portsc = XHCI_PS_CCS | /* connected */
2623 if (dev->dev_ue->ue_usbver == 2) {
2624 port->portsc |= XHCI_PS_PLS_SET(UPS_PORT_LS_POLL) |
2625 XHCI_PS_SPEED_SET(dev->dev_ue->ue_usbspeed);
2627 port->portsc |= XHCI_PS_PLS_SET(UPS_PORT_LS_U0) |
2629 XHCI_PS_SPEED_SET(dev->dev_ue->ue_usbspeed);
2632 DPRINTF(("Init port %d 0x%x", portn, port->portsc));
2634 port->portsc = XHCI_PS_PLS_SET(UPS_PORT_LS_RX_DET) | XHCI_PS_PP;
2635 DPRINTF(("Init empty port %d 0x%x", portn, port->portsc));
2655 /* HW endpoint contexts are 0-15; convert to epid based on dir */
2660 dev = hci->hci_sc;
2661 sc = dev->xsc;
2664 if (sc->rtsregs.erstba_p == NULL ||
2665 (sc->opregs.usbcmd & XHCI_CMD_RS) == 0 ||
2666 dev->dev_ctx == NULL)
2669 p = XHCI_PORTREG_PTR(sc, hci->hci_port);
2672 if (XHCI_PS_PLS_GET(p->portsc) == 3) {
2673 p->portsc &= ~XHCI_PS_PLS_MASK;
2674 p->portsc |= XHCI_PS_PLS_SET(UPS_PORT_LS_RESUME);
2675 if ((p->portsc & XHCI_PS_PLC) != 0)
2678 p->portsc |= XHCI_PS_PLC;
2680 pci_xhci_set_evtrb(&evtrb, hci->hci_port,
2687 dev_ctx = dev->dev_ctx;
2688 ep_ctx = &dev_ctx->ctx_ep[epid];
2689 if ((ep_ctx->dwEpCtx0 & 0x7) == XHCI_ST_EPCTX_DISABLED) {
2697 pci_xhci_device_doorbell(sc, hci->hci_port, epid, 0);
2707 DPRINTF(("xhci device event port %d", hci->hci_port));
2774 usb3_port = sc->usb3_port_start;
2775 usb2_port = sc->usb2_port_start;
2777 sc->devices = calloc(XHCI_MAX_DEVS, sizeof(struct pci_xhci_dev_emu *));
2778 sc->slots = calloc(XHCI_MAX_SLOTS, sizeof(struct pci_xhci_dev_emu *));
2788 if (usb2_port == ((sc->usb2_port_start) + XHCI_MAX_DEVS/2) ||
2789 usb3_port == ((sc->usb3_port_start) + XHCI_MAX_DEVS/2)) {
2832 dev->xsc = sc;
2833 dev->hci.hci_sc = dev;
2834 dev->hci.hci_intr = pci_xhci_dev_intr;
2835 dev->hci.hci_event = pci_xhci_dev_event;
2837 if (ue->ue_usbver == 2) {
2838 if (usb2_port == sc->usb2_port_start +
2844 dev->hci.hci_port = usb2_port;
2847 if (usb3_port == sc->usb3_port_start +
2853 dev->hci.hci_port = usb3_port;
2856 XHCI_DEVINST_PTR(sc, dev->hci.hci_port) = dev;
2858 dev->hci.hci_address = 0;
2859 devsc = ue->ue_init(&dev->hci, nvl);
2864 dev->dev_ue = ue;
2865 dev->dev_sc = devsc;
2872 sc->portregs = calloc(XHCI_MAX_DEVS, sizeof(struct pci_xhci_portregs));
2888 free(sc->devices);
2889 free(sc->slots);
2891 return (-1);
2902 return (-1);
2907 pi->pi_arg = sc;
2908 sc->xsc_pi = pi;
2910 sc->usb2_port_start = (XHCI_MAX_DEVS/2) + 1;
2911 sc->usb3_port_start = 1;
2920 sc->caplength = XHCI_SET_CAPLEN(XHCI_CAPLEN) |
2922 sc->hcsparams1 = XHCI_SET_HCSP1_MAXPORTS(XHCI_MAX_DEVS) |
2925 sc->hcsparams2 = XHCI_SET_HCSP2_ERSTMAX(XHCI_ERST_MAX) |
2927 sc->hcsparams3 = 0; /* no latency */
2928 sc->hccparams1 = XHCI_SET_HCCP1_AC64(1) | /* 64-bit addrs */
2929 XHCI_SET_HCCP1_NSS(1) | /* no 2nd-streams */
2932 sc->hccparams2 = XHCI_SET_HCCP2_LEC(1) |
2934 sc->dboff = XHCI_SET_DOORBELL(XHCI_CAPLEN + XHCI_PORTREGS_START +
2937 /* dboff must be 32-bit aligned */
2938 if (sc->dboff & 0x3)
2939 sc->dboff = (sc->dboff + 0x3) & ~0x3;
2941 /* rtsoff must be 32-bytes aligned */
2942 sc->rtsoff = XHCI_SET_RTSOFFSET(sc->dboff + (XHCI_MAX_SLOTS+1) * 32);
2943 if (sc->rtsoff & 0x1F)
2944 sc->rtsoff = (sc->rtsoff + 0x1F) & ~0x1F;
2946 DPRINTF(("pci_xhci dboff: 0x%x, rtsoff: 0x%x", sc->dboff,
2947 sc->rtsoff));
2949 sc->opregs.usbsts = XHCI_STS_HCH;
2950 sc->opregs.pgsz = XHCI_PAGESIZE_4K;
2954 sc->regsend = sc->rtsoff + 0x20 + 32; /* only 1 intrpter */
2958 * value of xecp field is 32-bit offset.
2960 sc->hccparams1 |= XHCI_SET_HCCP1_XECP(sc->regsend/4);
2972 pci_emul_alloc_bar(pi, 0, PCIBAR_MEM32, sc->regsend + 4*32);
2973 DPRINTF(("pci_xhci pci_emu_alloc: %d", sc->regsend + 4*32));
2978 pthread_mutex_init(&sc->mtx, NULL);
3018 if (meta->op == VM_SNAPSHOT_SAVE)
3019 xfer = dev->eps[idx].ep_xfer;
3027 if (meta->op == VM_SNAPSHOT_RESTORE) {
3029 xfer = dev->eps[idx].ep_xfer;
3034 xfer_block = &xfer->data[k];
3036 SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(sc->xsc_pi->pi_vmctx,
3037 xfer_block->buf, XHCI_GADDR_SIZE(xfer_block->buf), true,
3039 SNAPSHOT_VAR_OR_LEAVE(xfer_block->blen, meta, ret, done);
3040 SNAPSHOT_VAR_OR_LEAVE(xfer_block->bdone, meta, ret, done);
3041 SNAPSHOT_VAR_OR_LEAVE(xfer_block->processed, meta, ret, done);
3042 SNAPSHOT_VAR_OR_LEAVE(xfer_block->hci_data, meta, ret, done);
3043 SNAPSHOT_VAR_OR_LEAVE(xfer_block->ccs, meta, ret, done);
3044 SNAPSHOT_VAR_OR_LEAVE(xfer_block->streamid, meta, ret, done);
3045 SNAPSHOT_VAR_OR_LEAVE(xfer_block->trbnext, meta, ret, done);
3048 SNAPSHOT_VAR_OR_LEAVE(xfer->ureq, meta, ret, done);
3049 if (xfer->ureq) {
3050 /* xfer->ureq is not allocated at restore time */
3051 if (meta->op == VM_SNAPSHOT_RESTORE)
3052 xfer->ureq = malloc(sizeof(struct usb_device_request));
3054 SNAPSHOT_BUF_OR_LEAVE(xfer->ureq,
3059 SNAPSHOT_VAR_OR_LEAVE(xfer->ndata, meta, ret, done);
3060 SNAPSHOT_VAR_OR_LEAVE(xfer->head, meta, ret, done);
3061 SNAPSHOT_VAR_OR_LEAVE(xfer->tail, meta, ret, done);
3080 pi = meta->dev_data;
3081 sc = pi->pi_arg;
3083 SNAPSHOT_VAR_OR_LEAVE(sc->caplength, meta, ret, done);
3084 SNAPSHOT_VAR_OR_LEAVE(sc->hcsparams1, meta, ret, done);
3085 SNAPSHOT_VAR_OR_LEAVE(sc->hcsparams2, meta, ret, done);
3086 SNAPSHOT_VAR_OR_LEAVE(sc->hcsparams3, meta, ret, done);
3087 SNAPSHOT_VAR_OR_LEAVE(sc->hccparams1, meta, ret, done);
3088 SNAPSHOT_VAR_OR_LEAVE(sc->dboff, meta, ret, done);
3089 SNAPSHOT_VAR_OR_LEAVE(sc->rtsoff, meta, ret, done);
3090 SNAPSHOT_VAR_OR_LEAVE(sc->hccparams2, meta, ret, done);
3091 SNAPSHOT_VAR_OR_LEAVE(sc->regsend, meta, ret, done);
3094 SNAPSHOT_VAR_OR_LEAVE(sc->opregs.usbcmd, meta, ret, done);
3095 SNAPSHOT_VAR_OR_LEAVE(sc->opregs.usbsts, meta, ret, done);
3096 SNAPSHOT_VAR_OR_LEAVE(sc->opregs.pgsz, meta, ret, done);
3097 SNAPSHOT_VAR_OR_LEAVE(sc->opregs.dnctrl, meta, ret, done);
3098 SNAPSHOT_VAR_OR_LEAVE(sc->opregs.crcr, meta, ret, done);
3099 SNAPSHOT_VAR_OR_LEAVE(sc->opregs.dcbaap, meta, ret, done);
3100 SNAPSHOT_VAR_OR_LEAVE(sc->opregs.config, meta, ret, done);
3103 SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(pi->pi_vmctx, sc->opregs.cr_p,
3104 XHCI_GADDR_SIZE(sc->opregs.cr_p), true, meta, ret, done);
3107 SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(pi->pi_vmctx, sc->opregs.dcbaa_p,
3108 XHCI_GADDR_SIZE(sc->opregs.dcbaa_p), true, meta, ret, done);
3111 SNAPSHOT_VAR_OR_LEAVE(sc->rtsregs.mfindex, meta, ret, done);
3114 SNAPSHOT_VAR_OR_LEAVE(sc->rtsregs.intrreg.iman, meta, ret, done);
3115 SNAPSHOT_VAR_OR_LEAVE(sc->rtsregs.intrreg.imod, meta, ret, done);
3116 SNAPSHOT_VAR_OR_LEAVE(sc->rtsregs.intrreg.erstsz, meta, ret, done);
3117 SNAPSHOT_VAR_OR_LEAVE(sc->rtsregs.intrreg.rsvd, meta, ret, done);
3118 SNAPSHOT_VAR_OR_LEAVE(sc->rtsregs.intrreg.erstba, meta, ret, done);
3119 SNAPSHOT_VAR_OR_LEAVE(sc->rtsregs.intrreg.erdp, meta, ret, done);
3122 SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(pi->pi_vmctx, sc->rtsregs.erstba_p,
3123 XHCI_GADDR_SIZE(sc->rtsregs.erstba_p), true, meta, ret, done);
3126 SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(pi->pi_vmctx, sc->rtsregs.erst_p,
3127 XHCI_GADDR_SIZE(sc->rtsregs.erst_p), true, meta, ret, done);
3129 SNAPSHOT_VAR_OR_LEAVE(sc->rtsregs.er_deq_seg, meta, ret, done);
3130 SNAPSHOT_VAR_OR_LEAVE(sc->rtsregs.er_enq_idx, meta, ret, done);
3131 SNAPSHOT_VAR_OR_LEAVE(sc->rtsregs.er_enq_seg, meta, ret, done);
3132 SNAPSHOT_VAR_OR_LEAVE(sc->rtsregs.er_events_cnt, meta, ret, done);
3133 SNAPSHOT_VAR_OR_LEAVE(sc->rtsregs.event_pcs, meta, ret, done);
3141 if (meta->op == VM_SNAPSHOT_SAVE)
3153 if (meta->op == VM_SNAPSHOT_SAVE) {
3155 strncpy(dname, dev->dev_ue->ue_emu, sizeof(dname) - 1);
3160 if (meta->op == VM_SNAPSHOT_RESTORE) {
3161 dname[sizeof(dname) - 1] = '\0';
3162 if (strcmp(dev->dev_ue->ue_emu, dname)) {
3165 __func__, dname, dev->dev_ue->ue_emu);
3181 SNAPSHOT_VAR_OR_LEAVE(port->portsc, meta, ret, done);
3182 SNAPSHOT_VAR_OR_LEAVE(port->portpmsc, meta, ret, done);
3183 SNAPSHOT_VAR_OR_LEAVE(port->portli, meta, ret, done);
3184 SNAPSHOT_VAR_OR_LEAVE(port->porthlpmc, meta, ret, done);
3188 if (meta->op == VM_SNAPSHOT_SAVE)
3194 if (meta->op == VM_SNAPSHOT_SAVE) {
3196 } else if (meta->op == VM_SNAPSHOT_RESTORE) {
3212 SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(pi->pi_vmctx, dev->dev_ctx,
3213 XHCI_GADDR_SIZE(dev->dev_ctx), true, meta, ret, done);
3215 if (dev->dev_ctx != NULL) {
3223 SNAPSHOT_VAR_OR_LEAVE(dev->dev_slotstate, meta, ret, done);
3225 /* devices[i]->dev_sc */
3226 dev->dev_ue->ue_snapshot(dev->dev_sc, meta);
3228 /* devices[i]->hci */
3229 SNAPSHOT_VAR_OR_LEAVE(dev->hci.hci_address, meta, ret, done);
3230 SNAPSHOT_VAR_OR_LEAVE(dev->hci.hci_port, meta, ret, done);
3233 SNAPSHOT_VAR_OR_LEAVE(sc->usb2_port_start, meta, ret, done);
3234 SNAPSHOT_VAR_OR_LEAVE(sc->usb3_port_start, meta, ret, done);