xref: /openbsd-src/sys/dev/usb/ehci.c (revision 99fd087599a8791921855f21bd7e36130f39aadc)
1 /*	$OpenBSD: ehci.c,v 1.206 2020/02/22 14:01:34 jasper Exp $ */
2 /*	$NetBSD: ehci.c,v 1.66 2004/06/30 03:11:56 mycroft Exp $	*/
3 
4 /*
5  * Copyright (c) 2014-2015 Martin Pieuchot
6  *
7  * Permission to use, copy, modify, and distribute this software for any
8  * purpose with or without fee is hereby granted, provided that the above
9  * copyright notice and this permission notice appear in all copies.
10  *
11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18  */
19 /*
20  * Copyright (c) 2004-2008 The NetBSD Foundation, Inc.
21  * All rights reserved.
22  *
23  * This code is derived from software contributed to The NetBSD Foundation
24  * by Lennart Augustsson (lennart@augustsson.net), Charles M. Hannum and
25  * Jeremy Morse (jeremy.morse@gmail.com).
26  *
27  * Redistribution and use in source and binary forms, with or without
28  * modification, are permitted provided that the following conditions
29  * are met:
30  * 1. Redistributions of source code must retain the above copyright
31  *    notice, this list of conditions and the following disclaimer.
32  * 2. Redistributions in binary form must reproduce the above copyright
33  *    notice, this list of conditions and the following disclaimer in the
34  *    documentation and/or other materials provided with the distribution.
35  *
36  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
37  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
38  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
39  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
40  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
41  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
42  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
43  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
44  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
45  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
46  * POSSIBILITY OF SUCH DAMAGE.
47  */
48 
49 /*
50  * TODO:
51  * 1) The hub driver needs to handle and schedule the transaction translator,
52  *    to assign place in frame where different devices get to go. See chapter
53  *    on hubs in USB 2.0 for details.
54  *
55  * 2) Command failures are not recovered correctly.
56  */
57 
58 #include <sys/param.h>
59 #include <sys/systm.h>
60 #include <sys/kernel.h>
61 #include <sys/rwlock.h>
62 #include <sys/malloc.h>
63 #include <sys/device.h>
64 #include <sys/queue.h>
65 #include <sys/timeout.h>
66 #include <sys/pool.h>
67 #include <sys/endian.h>
68 #include <sys/atomic.h>
69 
70 #include <machine/bus.h>
71 
72 #include <dev/usb/usb.h>
73 #include <dev/usb/usbdi.h>
74 #include <dev/usb/usbdivar.h>
75 #include <dev/usb/usb_mem.h>
76 
77 #include <dev/usb/ehcireg.h>
78 #include <dev/usb/ehcivar.h>
79 
80 struct cfdriver ehci_cd = {
81 	NULL, "ehci", DV_DULL
82 };
83 
84 #ifdef EHCI_DEBUG
85 #define DPRINTF(x)	do { if (ehcidebug) printf x; } while(0)
86 #define DPRINTFN(n,x)	do { if (ehcidebug>(n)) printf x; } while (0)
87 int ehcidebug = 0;
88 #define bitmask_snprintf(q,f,b,l) snprintf((b), (l), "%b", (q), (f))
89 #else
90 #define DPRINTF(x)
91 #define DPRINTFN(n,x)
92 #endif
93 
94 struct pool *ehcixfer;
95 
96 struct ehci_pipe {
97 	struct usbd_pipe pipe;
98 
99 	struct ehci_soft_qh *sqh;
100 	union {
101 		/* Control pipe */
102 		struct {
103 			struct usb_dma reqdma;
104 		} ctl;
105 		/* Iso pipe */
106 		struct {
107 			u_int next_frame;
108 			u_int cur_xfers;
109 		} isoc;
110 	} u;
111 };
112 
113 u_int8_t		ehci_reverse_bits(u_int8_t, int);
114 
115 usbd_status	ehci_open(struct usbd_pipe *);
116 int		ehci_setaddr(struct usbd_device *, int);
117 void		ehci_poll(struct usbd_bus *);
118 void		ehci_softintr(void *);
119 int		ehci_intr1(struct ehci_softc *);
120 void		ehci_check_intr(struct ehci_softc *, struct usbd_xfer *);
121 void		ehci_check_qh_intr(struct ehci_softc *, struct usbd_xfer *);
122 void		ehci_check_itd_intr(struct ehci_softc *, struct usbd_xfer *);
123 void		ehci_idone(struct usbd_xfer *);
124 void		ehci_isoc_idone(struct usbd_xfer *);
125 void		ehci_timeout(void *);
126 void		ehci_timeout_task(void *);
127 void		ehci_intrlist_timeout(void *);
128 
129 struct usbd_xfer *ehci_allocx(struct usbd_bus *);
130 void		ehci_freex(struct usbd_bus *, struct usbd_xfer *);
131 
132 usbd_status	ehci_root_ctrl_transfer(struct usbd_xfer *);
133 usbd_status	ehci_root_ctrl_start(struct usbd_xfer *);
134 void		ehci_root_ctrl_abort(struct usbd_xfer *);
135 void		ehci_root_ctrl_close(struct usbd_pipe *);
136 void		ehci_root_ctrl_done(struct usbd_xfer *);
137 
138 usbd_status	ehci_root_intr_transfer(struct usbd_xfer *);
139 usbd_status	ehci_root_intr_start(struct usbd_xfer *);
140 void		ehci_root_intr_abort(struct usbd_xfer *);
141 void		ehci_root_intr_close(struct usbd_pipe *);
142 void		ehci_root_intr_done(struct usbd_xfer *);
143 
144 usbd_status	ehci_device_ctrl_transfer(struct usbd_xfer *);
145 usbd_status	ehci_device_ctrl_start(struct usbd_xfer *);
146 void		ehci_device_ctrl_abort(struct usbd_xfer *);
147 void		ehci_device_ctrl_close(struct usbd_pipe *);
148 void		ehci_device_ctrl_done(struct usbd_xfer *);
149 
150 usbd_status	ehci_device_bulk_transfer(struct usbd_xfer *);
151 usbd_status	ehci_device_bulk_start(struct usbd_xfer *);
152 void		ehci_device_bulk_abort(struct usbd_xfer *);
153 void		ehci_device_bulk_close(struct usbd_pipe *);
154 void		ehci_device_bulk_done(struct usbd_xfer *);
155 
156 usbd_status	ehci_device_intr_transfer(struct usbd_xfer *);
157 usbd_status	ehci_device_intr_start(struct usbd_xfer *);
158 void		ehci_device_intr_abort(struct usbd_xfer *);
159 void		ehci_device_intr_close(struct usbd_pipe *);
160 void		ehci_device_intr_done(struct usbd_xfer *);
161 
162 usbd_status	ehci_device_isoc_transfer(struct usbd_xfer *);
163 usbd_status	ehci_device_isoc_start(struct usbd_xfer *);
164 void		ehci_device_isoc_abort(struct usbd_xfer *);
165 void		ehci_device_isoc_close(struct usbd_pipe *);
166 void		ehci_device_isoc_done(struct usbd_xfer *);
167 
168 void		ehci_device_clear_toggle(struct usbd_pipe *pipe);
169 
170 void		ehci_pcd(struct ehci_softc *, struct usbd_xfer *);
171 void		ehci_disown(struct ehci_softc *, int, int);
172 
173 struct ehci_soft_qh *ehci_alloc_sqh(struct ehci_softc *);
174 void		ehci_free_sqh(struct ehci_softc *, struct ehci_soft_qh *);
175 
176 struct ehci_soft_qtd *ehci_alloc_sqtd(struct ehci_softc *);
177 void		ehci_free_sqtd(struct ehci_softc *, struct ehci_soft_qtd *);
178 usbd_status	ehci_alloc_sqtd_chain(struct ehci_softc *, u_int,
179 		    struct usbd_xfer *, struct ehci_soft_qtd **, struct ehci_soft_qtd **);
180 void		ehci_free_sqtd_chain(struct ehci_softc *, struct ehci_xfer *);
181 
182 struct ehci_soft_itd *ehci_alloc_itd(struct ehci_softc *);
183 void		ehci_free_itd(struct ehci_softc *, struct ehci_soft_itd *);
184 void		ehci_rem_itd_chain(struct ehci_softc *, struct ehci_xfer *);
185 void		ehci_free_itd_chain(struct ehci_softc *, struct ehci_xfer *);
186 int		ehci_alloc_itd_chain(struct ehci_softc *, struct usbd_xfer *);
187 int		ehci_alloc_sitd_chain(struct ehci_softc *, struct usbd_xfer *);
188 void		ehci_abort_isoc_xfer(struct usbd_xfer *xfer,
189 		    usbd_status status);
190 
191 usbd_status	ehci_device_setintr(struct ehci_softc *, struct ehci_soft_qh *,
192 			    int ival);
193 
194 void		ehci_add_qh(struct ehci_soft_qh *, struct ehci_soft_qh *);
195 void		ehci_rem_qh(struct ehci_softc *, struct ehci_soft_qh *);
196 void		ehci_set_qh_qtd(struct ehci_soft_qh *, struct ehci_soft_qtd *);
197 void		ehci_sync_hc(struct ehci_softc *);
198 
199 void		ehci_close_pipe(struct usbd_pipe *);
200 void		ehci_abort_xfer(struct usbd_xfer *, usbd_status);
201 
202 #ifdef EHCI_DEBUG
203 void		ehci_dump_regs(struct ehci_softc *);
204 void		ehci_dump(void);
205 struct ehci_softc *theehci;
206 void		ehci_dump_link(ehci_link_t, int);
207 void		ehci_dump_sqtds(struct ehci_soft_qtd *);
208 void		ehci_dump_sqtd(struct ehci_soft_qtd *);
209 void		ehci_dump_qtd(struct ehci_qtd *);
210 void		ehci_dump_sqh(struct ehci_soft_qh *);
211 #if notyet
212 void		ehci_dump_itd(struct ehci_soft_itd *);
213 #endif
214 #ifdef DIAGNOSTIC
215 void		ehci_dump_exfer(struct ehci_xfer *);
216 #endif
217 #endif
218 
219 #define EHCI_INTR_ENDPT 1
220 
221 struct usbd_bus_methods ehci_bus_methods = {
222 	.open_pipe = ehci_open,
223 	.dev_setaddr = ehci_setaddr,
224 	.soft_intr = ehci_softintr,
225 	.do_poll = ehci_poll,
226 	.allocx = ehci_allocx,
227 	.freex = ehci_freex,
228 };
229 
230 struct usbd_pipe_methods ehci_root_ctrl_methods = {
231 	.transfer = ehci_root_ctrl_transfer,
232 	.start = ehci_root_ctrl_start,
233 	.abort = ehci_root_ctrl_abort,
234 	.close = ehci_root_ctrl_close,
235 	.done = ehci_root_ctrl_done,
236 };
237 
238 struct usbd_pipe_methods ehci_root_intr_methods = {
239 	.transfer = ehci_root_intr_transfer,
240 	.start = ehci_root_intr_start,
241 	.abort = ehci_root_intr_abort,
242 	.close = ehci_root_intr_close,
243 	.done = ehci_root_intr_done,
244 };
245 
246 struct usbd_pipe_methods ehci_device_ctrl_methods = {
247 	.transfer = ehci_device_ctrl_transfer,
248 	.start = ehci_device_ctrl_start,
249 	.abort = ehci_device_ctrl_abort,
250 	.close = ehci_device_ctrl_close,
251 	.done = ehci_device_ctrl_done,
252 };
253 
254 struct usbd_pipe_methods ehci_device_intr_methods = {
255 	.transfer = ehci_device_intr_transfer,
256 	.start = ehci_device_intr_start,
257 	.abort = ehci_device_intr_abort,
258 	.close = ehci_device_intr_close,
259 	.cleartoggle = ehci_device_clear_toggle,
260 	.done = ehci_device_intr_done,
261 };
262 
263 struct usbd_pipe_methods ehci_device_bulk_methods = {
264 	.transfer = ehci_device_bulk_transfer,
265 	.start = ehci_device_bulk_start,
266 	.abort = ehci_device_bulk_abort,
267 	.close = ehci_device_bulk_close,
268 	.cleartoggle = ehci_device_clear_toggle,
269 	.done = ehci_device_bulk_done,
270 };
271 
272 struct usbd_pipe_methods ehci_device_isoc_methods = {
273 	.transfer = ehci_device_isoc_transfer,
274 	.start = ehci_device_isoc_start,
275 	.abort = ehci_device_isoc_abort,
276 	.close = ehci_device_isoc_close,
277 	.done = ehci_device_isoc_done,
278 };
279 
280 /*
281  * Reverse a number with nbits bits.  Used to evenly distribute lower-level
282  * interrupt heads in the periodic schedule.
283  * Suitable for use with EHCI_IPOLLRATES <= 9.
284  */
285 u_int8_t
286 ehci_reverse_bits(u_int8_t c, int nbits)
287 {
288 	c = ((c >> 1) & 0x55) | ((c << 1) & 0xaa);
289 	c = ((c >> 2) & 0x33) | ((c << 2) & 0xcc);
290 	c = ((c >> 4) & 0x0f) | ((c << 4) & 0xf0);
291 
292 	return c >> (8 - nbits);
293 }
294 
295 usbd_status
296 ehci_init(struct ehci_softc *sc)
297 {
298 	u_int32_t sparams, cparams, hcr;
299 	u_int i, j;
300 	usbd_status err;
301 	struct ehci_soft_qh *sqh;
302 
303 #ifdef EHCI_DEBUG
304 	u_int32_t vers;
305 	theehci = sc;
306 
307 	DPRINTF(("ehci_init: start\n"));
308 
309 	vers = EREAD2(sc, EHCI_HCIVERSION);
310 	DPRINTF(("%s: EHCI version %x.%x\n", sc->sc_bus.bdev.dv_xname,
311 	    vers >> 8, vers & 0xff));
312 #endif
313 
314 	sc->sc_offs = EREAD1(sc, EHCI_CAPLENGTH);
315 
316 	sparams = EREAD4(sc, EHCI_HCSPARAMS);
317 	DPRINTF(("ehci_init: sparams=0x%x\n", sparams));
318 	sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
319 	cparams = EREAD4(sc, EHCI_HCCPARAMS);
320 	DPRINTF(("ehci_init: cparams=0x%x\n", cparams));
321 
322 	/* MUST clear segment register if 64 bit capable. */
323 	if (EHCI_HCC_64BIT(cparams))
324 		EWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
325 
326 	sc->sc_bus.usbrev = USBREV_2_0;
327 
328 	DPRINTF(("%s: resetting\n", sc->sc_bus.bdev.dv_xname));
329 	err = ehci_reset(sc);
330 	if (err)
331 		return (err);
332 
333 	if (ehcixfer == NULL) {
334 		ehcixfer = malloc(sizeof(struct pool), M_DEVBUF, M_NOWAIT);
335 		if (ehcixfer == NULL) {
336 			printf("%s: unable to allocate pool descriptor\n",
337 			    sc->sc_bus.bdev.dv_xname);
338 			return (ENOMEM);
339 		}
340 		pool_init(ehcixfer, sizeof(struct ehci_xfer), 0, IPL_SOFTUSB,
341 		    0, "ehcixfer", NULL);
342 	}
343 
344 	/* frame list size at default, read back what we got and use that */
345 	switch (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD))) {
346 	case 0:
347 		sc->sc_flsize = 1024;
348 		break;
349 	case 1:
350 		sc->sc_flsize = 512;
351 		break;
352 	case 2:
353 		sc->sc_flsize = 256;
354 		break;
355 	case 3:
356 		return (USBD_IOERROR);
357 	}
358 	err = usb_allocmem(&sc->sc_bus, sc->sc_flsize * sizeof(ehci_link_t),
359 	    EHCI_FLALIGN_ALIGN, &sc->sc_fldma);
360 	if (err)
361 		return (err);
362 	DPRINTF(("%s: flsize=%d\n", sc->sc_bus.bdev.dv_xname,sc->sc_flsize));
363 	sc->sc_flist = KERNADDR(&sc->sc_fldma, 0);
364 
365 	for (i = 0; i < sc->sc_flsize; i++)
366 		sc->sc_flist[i] = htole32(EHCI_LINK_TERMINATE);
367 
368 	EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
369 
370 	sc->sc_softitds = mallocarray(sc->sc_flsize,
371 	    sizeof(struct ehci_soft_itd *), M_USB, M_NOWAIT | M_ZERO);
372 	if (sc->sc_softitds == NULL) {
373 		usb_freemem(&sc->sc_bus, &sc->sc_fldma);
374 		return (ENOMEM);
375 	}
376 	LIST_INIT(&sc->sc_freeitds);
377 	TAILQ_INIT(&sc->sc_intrhead);
378 
379 	/* Set up the bus struct. */
380 	sc->sc_bus.methods = &ehci_bus_methods;
381 	sc->sc_bus.pipe_size = sizeof(struct ehci_pipe);
382 
383 	sc->sc_eintrs = EHCI_NORMAL_INTRS;
384 
385 	/*
386 	 * Allocate the interrupt dummy QHs. These are arranged to give poll
387 	 * intervals that are powers of 2 times 1ms.
388 	 */
389 	for (i = 0; i < EHCI_INTRQHS; i++) {
390 		sqh = ehci_alloc_sqh(sc);
391 		if (sqh == NULL) {
392 			err = USBD_NOMEM;
393 			goto bad1;
394 		}
395 		sc->sc_islots[i].sqh = sqh;
396 	}
397 	for (i = 0; i < EHCI_INTRQHS; i++) {
398 		sqh = sc->sc_islots[i].sqh;
399 		if (i == 0) {
400 			/* The last (1ms) QH terminates. */
401 			sqh->qh.qh_link = htole32(EHCI_LINK_TERMINATE);
402 			sqh->next = NULL;
403 		} else {
404 			/* Otherwise the next QH has half the poll interval */
405 			sqh->next = sc->sc_islots[(i + 1) / 2 - 1].sqh;
406 			sqh->qh.qh_link = htole32(sqh->next->physaddr |
407 			    EHCI_LINK_QH);
408 		}
409 		sqh->qh.qh_endp = htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
410 		sqh->qh.qh_endphub = htole32(EHCI_QH_SET_MULT(1));
411 		sqh->qh.qh_curqtd = htole32(EHCI_LINK_TERMINATE);
412 		sqh->qh.qh_qtd.qtd_next = htole32(EHCI_LINK_TERMINATE);
413 		sqh->qh.qh_qtd.qtd_altnext = htole32(EHCI_LINK_TERMINATE);
414 		sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
415 		sqh->sqtd = NULL;
416 		usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
417 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
418 	}
419 	/* Point the frame list at the last level (128ms). */
420 	for (i = 0; i < (1 << (EHCI_IPOLLRATES - 1)); i++)
421 		for (j = i; j < sc->sc_flsize; j += 1 << (EHCI_IPOLLRATES - 1))
422 			sc->sc_flist[j] = htole32(EHCI_LINK_QH | sc->sc_islots[
423 			    EHCI_IQHIDX(EHCI_IPOLLRATES - 1, ehci_reverse_bits(
424 			    i, EHCI_IPOLLRATES - 1))].sqh->physaddr);
425 	usb_syncmem(&sc->sc_fldma, 0, sc->sc_flsize * sizeof(ehci_link_t),
426 	    BUS_DMASYNC_PREWRITE);
427 
428 	/* Allocate dummy QH that starts the async list. */
429 	sqh = ehci_alloc_sqh(sc);
430 	if (sqh == NULL) {
431 		err = USBD_NOMEM;
432 		goto bad1;
433 	}
434 	/* Fill the QH */
435 	sqh->qh.qh_endp =
436 	    htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
437 	sqh->qh.qh_link =
438 	    htole32(sqh->physaddr | EHCI_LINK_QH);
439 	sqh->qh.qh_curqtd = htole32(EHCI_LINK_TERMINATE);
440 	sqh->prev = sqh; /*It's a circular list.. */
441 	sqh->next = sqh;
442 	/* Fill the overlay qTD */
443 	sqh->qh.qh_qtd.qtd_next = htole32(EHCI_LINK_TERMINATE);
444 	sqh->qh.qh_qtd.qtd_altnext = htole32(EHCI_LINK_TERMINATE);
445 	sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
446 	sqh->sqtd = NULL;
447 	usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
448 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
449 
450 	/* Point to async list */
451 	sc->sc_async_head = sqh;
452 	EOWRITE4(sc, EHCI_ASYNCLISTADDR, sqh->physaddr | EHCI_LINK_QH);
453 
454 	timeout_set(&sc->sc_tmo_intrlist, ehci_intrlist_timeout, sc);
455 
456 	rw_init(&sc->sc_doorbell_lock, "ehcidb");
457 
458 	/* Turn on controller */
459 	EOWRITE4(sc, EHCI_USBCMD,
460 	    EHCI_CMD_ITC_2 | /* 2 microframes interrupt delay */
461 	    (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
462 	    EHCI_CMD_ASE |
463 	    EHCI_CMD_PSE |
464 	    EHCI_CMD_RS);
465 
466 	/* Take over port ownership */
467 	EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
468 
469 	for (i = 0; i < 100; i++) {
470 		usb_delay_ms(&sc->sc_bus, 1);
471 		hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
472 		if (!hcr)
473 			break;
474 	}
475 	if (hcr) {
476 		printf("%s: run timeout\n", sc->sc_bus.bdev.dv_xname);
477 		return (USBD_IOERROR);
478 	}
479 
480 	/* Enable interrupts */
481 	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
482 
483 	return (USBD_NORMAL_COMPLETION);
484 
485 #if 0
486  bad2:
487 	ehci_free_sqh(sc, sc->sc_async_head);
488 #endif
489  bad1:
490 	free(sc->sc_softitds, M_USB,
491 	    sc->sc_flsize * sizeof(struct ehci_soft_itd *));
492 	usb_freemem(&sc->sc_bus, &sc->sc_fldma);
493 	return (err);
494 }
495 
496 int
497 ehci_intr(void *v)
498 {
499 	struct ehci_softc *sc = v;
500 
501 	if (sc == NULL || sc->sc_bus.dying)
502 		return (0);
503 
504 	/* If we get an interrupt while polling, then just ignore it. */
505 	if (sc->sc_bus.use_polling) {
506 		u_int32_t intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
507 
508 		if (intrs)
509 			EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
510 		return (0);
511 	}
512 
513 	return (ehci_intr1(sc));
514 }
515 
516 int
517 ehci_intr1(struct ehci_softc *sc)
518 {
519 	u_int32_t intrs, eintrs;
520 
521 	/* In case the interrupt occurs before initialization has completed. */
522 	if (sc == NULL) {
523 #ifdef DIAGNOSTIC
524 		printf("ehci_intr1: sc == NULL\n");
525 #endif
526 		return (0);
527 	}
528 
529 	intrs = EOREAD4(sc, EHCI_USBSTS);
530 	if (intrs == 0xffffffff) {
531 		sc->sc_bus.dying = 1;
532 		return (0);
533 	}
534 	intrs = EHCI_STS_INTRS(intrs);
535 	if (!intrs)
536 		return (0);
537 
538 	eintrs = intrs & sc->sc_eintrs;
539 	if (!eintrs)
540 		return (0);
541 
542 	EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
543 	sc->sc_bus.no_intrs++;
544 
545 	if (eintrs & EHCI_STS_HSE) {
546 		printf("%s: unrecoverable error, controller halted\n",
547 		       sc->sc_bus.bdev.dv_xname);
548 		sc->sc_bus.dying = 1;
549 		return (1);
550 	}
551 	if (eintrs & EHCI_STS_IAA) {
552 		wakeup(&sc->sc_async_head);
553 		eintrs &= ~EHCI_STS_IAA;
554 	}
555 	if (eintrs & (EHCI_STS_INT | EHCI_STS_ERRINT)) {
556 		usb_schedsoftintr(&sc->sc_bus);
557 		eintrs &= ~(EHCI_STS_INT | EHCI_STS_ERRINT);
558 	}
559 	if (eintrs & EHCI_STS_PCD) {
560 		atomic_setbits_int(&sc->sc_flags, EHCIF_PCB_INTR);
561 		usb_schedsoftintr(&sc->sc_bus);
562 		eintrs &= ~EHCI_STS_PCD;
563 	}
564 
565 	if (eintrs != 0) {
566 		/* Block unprocessed interrupts. */
567 		sc->sc_eintrs &= ~eintrs;
568 		EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
569 		printf("%s: blocking intrs 0x%x\n",
570 		       sc->sc_bus.bdev.dv_xname, eintrs);
571 	}
572 
573 	return (1);
574 }
575 
576 void
577 ehci_pcd(struct ehci_softc *sc, struct usbd_xfer *xfer)
578 {
579 	u_char *p;
580 	int i, m;
581 
582 	if (xfer == NULL) {
583 		/* Just ignore the change. */
584 		return;
585 	}
586 
587 	p = KERNADDR(&xfer->dmabuf, 0);
588 	m = min(sc->sc_noport, xfer->length * 8 - 1);
589 	memset(p, 0, xfer->length);
590 	for (i = 1; i <= m; i++) {
591 		/* Pick out CHANGE bits from the status reg. */
592 		if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR)
593 			p[i / 8] |= 1 << (i % 8);
594 	}
595 	xfer->actlen = xfer->length;
596 	xfer->status = USBD_NORMAL_COMPLETION;
597 
598 	usb_transfer_complete(xfer);
599 }
600 
601 /*
602  * Work around the half configured control (default) pipe when setting
603  * the address of a device.
604  *
605  * Because a single QH is setup per endpoint in ehci_open(), and the
606  * control pipe is configured before we could have set the address
607  * of the device or read the wMaxPacketSize of the endpoint, we have
608  * to re-open the pipe twice here.
609  */
610 int
611 ehci_setaddr(struct usbd_device *dev, int addr)
612 {
613 	/* Root Hub */
614 	if (dev->depth == 0)
615 		return (0);
616 
617 	/* Re-establish the default pipe with the new max packet size. */
618 	ehci_close_pipe(dev->default_pipe);
619 	if (ehci_open(dev->default_pipe))
620 		return (EINVAL);
621 
622 	if (usbd_set_address(dev, addr))
623 		return (1);
624 
625 	dev->address = addr;
626 
627 	/* Re-establish the default pipe with the new address. */
628 	ehci_close_pipe(dev->default_pipe);
629 	if (ehci_open(dev->default_pipe))
630 		return (EINVAL);
631 
632 	return (0);
633 }
634 
635 void
636 ehci_softintr(void *v)
637 {
638 	struct ehci_softc *sc = v;
639 	struct ehci_xfer *ex, *nextex;
640 
641 	if (sc->sc_bus.dying)
642 		return;
643 
644 	sc->sc_bus.intr_context++;
645 
646 	if (sc->sc_flags & EHCIF_PCB_INTR) {
647 		atomic_clearbits_int(&sc->sc_flags, EHCIF_PCB_INTR);
648 		ehci_pcd(sc, sc->sc_intrxfer);
649 	}
650 
651 	/*
652 	 * The only explanation I can think of for why EHCI is as brain dead
653 	 * as UHCI interrupt-wise is that Intel was involved in both.
654 	 * An interrupt just tells us that something is done, we have no
655 	 * clue what, so we need to scan through all active transfers. :-(
656 	 */
657 	for (ex = TAILQ_FIRST(&sc->sc_intrhead); ex; ex = nextex) {
658 		nextex = TAILQ_NEXT(ex, inext);
659 		ehci_check_intr(sc, &ex->xfer);
660 	}
661 
662 	/* Schedule a callout to catch any dropped transactions. */
663 	if ((sc->sc_flags & EHCIF_DROPPED_INTR_WORKAROUND) &&
664 	    !TAILQ_EMPTY(&sc->sc_intrhead)) {
665 		timeout_add_sec(&sc->sc_tmo_intrlist, 1);
666 	}
667 
668 	if (sc->sc_softwake) {
669 		sc->sc_softwake = 0;
670 		wakeup(&sc->sc_softwake);
671 	}
672 
673 	sc->sc_bus.intr_context--;
674 }
675 
676 void
677 ehci_check_intr(struct ehci_softc *sc, struct usbd_xfer *xfer)
678 {
679 	int attr = xfer->pipe->endpoint->edesc->bmAttributes;
680 
681 	if (UE_GET_XFERTYPE(attr) == UE_ISOCHRONOUS)
682 		ehci_check_itd_intr(sc, xfer);
683 	else
684 		ehci_check_qh_intr(sc, xfer);
685 }
686 
687 void
688 ehci_check_qh_intr(struct ehci_softc *sc, struct usbd_xfer *xfer)
689 {
690 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
691 	struct ehci_soft_qtd *sqtd, *lsqtd = ex->sqtdend;
692 	uint32_t status;
693 
694 	KASSERT(ex->sqtdstart != NULL && ex->sqtdend != NULL);
695 
696 	usb_syncmem(&lsqtd->dma,
697 	    lsqtd->offs + offsetof(struct ehci_qtd, qtd_status),
698 	    sizeof(lsqtd->qtd.qtd_status),
699 	    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
700 
701 	/*
702 	 * If the last TD is still active we need to check whether there
703 	 * is a an error somewhere in the middle, or whether there was a
704 	 * short packet (SPD and not ACTIVE).
705 	 */
706 	if (letoh32(lsqtd->qtd.qtd_status) & EHCI_QTD_ACTIVE) {
707 		DPRINTFN(12, ("ehci_check_intr: active ex=%p\n", ex));
708 		for (sqtd = ex->sqtdstart; sqtd != lsqtd; sqtd=sqtd->nextqtd) {
709 			usb_syncmem(&sqtd->dma,
710 			    sqtd->offs + offsetof(struct ehci_qtd, qtd_status),
711 			    sizeof(sqtd->qtd.qtd_status),
712 			    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
713 			status = letoh32(sqtd->qtd.qtd_status);
714 			usb_syncmem(&sqtd->dma,
715 			    sqtd->offs + offsetof(struct ehci_qtd, qtd_status),
716 			    sizeof(sqtd->qtd.qtd_status), BUS_DMASYNC_PREREAD);
717 			/* If there's an active QTD the xfer isn't done. */
718 			if (status & EHCI_QTD_ACTIVE)
719 				break;
720 			/* Any kind of error makes the xfer done. */
721 			if (status & EHCI_QTD_HALTED)
722 				goto done;
723 			/* We want short packets, and it is short: it's done */
724 			if (EHCI_QTD_GET_BYTES(status) != 0)
725 				goto done;
726 		}
727 		DPRINTFN(12, ("ehci_check_intr: ex=%p std=%p still active\n",
728 			      ex, ex->sqtdstart));
729 		usb_syncmem(&lsqtd->dma,
730 		    lsqtd->offs + offsetof(struct ehci_qtd, qtd_status),
731 		    sizeof(lsqtd->qtd.qtd_status), BUS_DMASYNC_PREREAD);
732 		return;
733 	}
734  done:
735 	TAILQ_REMOVE(&sc->sc_intrhead, ex, inext);
736 	timeout_del(&xfer->timeout_handle);
737 	usb_rem_task(xfer->pipe->device, &xfer->abort_task);
738 	ehci_idone(xfer);
739 }
740 
741 void
742 ehci_check_itd_intr(struct ehci_softc *sc, struct usbd_xfer *xfer)
743 {
744 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
745 	struct ehci_soft_itd *itd = ex->itdend;
746 	int i;
747 
748 	if (xfer != SIMPLEQ_FIRST(&xfer->pipe->queue))
749 		return;
750 
751 	KASSERT(ex->itdstart != NULL && ex->itdend != NULL);
752 
753 	/* Check no active transfers in last itd, meaning we're finished */
754 	if (xfer->device->speed == USB_SPEED_HIGH) {
755 		usb_syncmem(&itd->dma,
756 		    itd->offs + offsetof(struct ehci_itd, itd_ctl),
757 		    sizeof(itd->itd.itd_ctl),
758 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
759 
760 		for (i = 0; i < 8; i++) {
761 			if (letoh32(itd->itd.itd_ctl[i]) & EHCI_ITD_ACTIVE)
762 				return;
763 		}
764 	} else {
765 		usb_syncmem(&itd->dma,
766 		    itd->offs + offsetof(struct ehci_sitd, sitd_trans),
767 		    sizeof(itd->sitd.sitd_trans),
768 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
769 
770 		if (le32toh(itd->sitd.sitd_trans) & EHCI_SITD_ACTIVE)
771 			return;
772 	}
773 
774 	/* All descriptor(s) inactive, it's done */
775 	TAILQ_REMOVE(&sc->sc_intrhead, ex, inext);
776 	timeout_del(&xfer->timeout_handle);
777 	usb_rem_task(xfer->pipe->device, &xfer->abort_task);
778 	ehci_isoc_idone(xfer);
779 }
780 
781 void
782 ehci_isoc_idone(struct usbd_xfer *xfer)
783 {
784 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
785 	struct ehci_soft_itd *itd;
786 	int i, len, uframes, nframes = 0, actlen = 0;
787 	uint32_t status = 0;
788 
789 	if (xfer->status == USBD_CANCELLED || xfer->status == USBD_TIMEOUT)
790 		return;
791 
792 	if (xfer->device->speed == USB_SPEED_HIGH) {
793 		switch (xfer->pipe->endpoint->edesc->bInterval) {
794 		case 0:
795 			panic("isoc xfer suddenly has 0 bInterval, invalid");
796 		case 1:
797 			uframes = 1;
798 			break;
799 		case 2:
800 			uframes = 2;
801 			break;
802 		case 3:
803 			uframes = 4;
804 			break;
805 		default:
806 			uframes = 8;
807 			break;
808 		}
809 
810 		for (itd = ex->itdstart; itd != NULL; itd = itd->xfer_next) {
811 			usb_syncmem(&itd->dma,
812 			    itd->offs + offsetof(struct ehci_itd, itd_ctl),
813 			    sizeof(itd->itd.itd_ctl), BUS_DMASYNC_POSTWRITE |
814 			    BUS_DMASYNC_POSTREAD);
815 
816 			for (i = 0; i < 8; i += uframes) {
817 				/* XXX - driver didn't fill in the frame full
818 				 *   of uframes. This leads to scheduling
819 				 *   inefficiencies, but working around
820 				 *   this doubles complexity of tracking
821 				 *   an xfer.
822 				 */
823 				if (nframes >= xfer->nframes)
824 					break;
825 
826 				status = letoh32(itd->itd.itd_ctl[i]);
827 				len = EHCI_ITD_GET_LEN(status);
828 				if (EHCI_ITD_GET_STATUS(status) != 0)
829 					len = 0; /*No valid data on error*/
830 
831 				xfer->frlengths[nframes++] = len;
832 				actlen += len;
833 			}
834 		}
835 	} else {
836 		for (itd = ex->itdstart; itd != NULL; itd = itd->xfer_next) {
837 			usb_syncmem(&itd->dma,
838 			    itd->offs + offsetof(struct ehci_sitd, sitd_trans),
839 			    sizeof(itd->sitd.sitd_trans),
840 			    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
841 
842 			status = le32toh(itd->sitd.sitd_trans);
843 			len = EHCI_SITD_GET_LEN(status);
844 			if (xfer->frlengths[nframes] >= len)
845 				len = xfer->frlengths[nframes] - len;
846 			else
847 				len = 0;
848 
849 			xfer->frlengths[nframes++] = len;
850 			actlen += len;
851 	    	}
852 	}
853 
854 #ifdef DIAGNOSTIC
855 	ex->isdone = 1;
856 #endif
857 	xfer->actlen = actlen;
858 	xfer->status = USBD_NORMAL_COMPLETION;
859 	usb_transfer_complete(xfer);
860 }
861 
862 void
863 ehci_idone(struct usbd_xfer *xfer)
864 {
865 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
866 	struct ehci_soft_qtd *sqtd;
867 	u_int32_t status = 0, nstatus = 0;
868 	int actlen, cerr;
869 
870 #ifdef DIAGNOSTIC
871 	{
872 		int s = splhigh();
873 		if (ex->isdone) {
874 			splx(s);
875 			printf("ehci_idone: ex=%p is done!\n", ex);
876 			return;
877 		}
878 		ex->isdone = 1;
879 		splx(s);
880 	}
881 #endif
882 	if (xfer->status == USBD_CANCELLED || xfer->status == USBD_TIMEOUT)
883 		return;
884 
885 	actlen = 0;
886 	for (sqtd = ex->sqtdstart; sqtd != NULL; sqtd = sqtd->nextqtd) {
887 		usb_syncmem(&sqtd->dma, sqtd->offs, sizeof(sqtd->qtd),
888 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
889 		nstatus = letoh32(sqtd->qtd.qtd_status);
890 		if (nstatus & EHCI_QTD_ACTIVE)
891 			break;
892 
893 		status = nstatus;
894 		/* halt is ok if descriptor is last, and complete */
895 		if (sqtd->qtd.qtd_next == htole32(EHCI_LINK_TERMINATE) &&
896 		    EHCI_QTD_GET_BYTES(status) == 0)
897 			status &= ~EHCI_QTD_HALTED;
898 		if (EHCI_QTD_GET_PID(status) !=	EHCI_QTD_PID_SETUP)
899 			actlen += sqtd->len - EHCI_QTD_GET_BYTES(status);
900 	}
901 
902 	cerr = EHCI_QTD_GET_CERR(status);
903 	DPRINTFN(/*10*/2, ("ehci_idone: len=%d, actlen=%d, cerr=%d, "
904 	    "status=0x%x\n", xfer->length, actlen, cerr, status));
905 	xfer->actlen = actlen;
906 	if ((status & EHCI_QTD_HALTED) != 0) {
907 		if ((status & EHCI_QTD_BABBLE) == 0 && cerr > 0)
908 			xfer->status = USBD_STALLED;
909 		else
910 			xfer->status = USBD_IOERROR; /* more info XXX */
911 	} else
912 		xfer->status = USBD_NORMAL_COMPLETION;
913 
914 	/* XXX transfer_complete memcpys out transfer data (for in endpoints)
915 	 * during this call, before methods->done is called: dma sync required
916 	 * beforehand? */
917 	usb_transfer_complete(xfer);
918 	DPRINTFN(/*12*/2, ("ehci_idone: ex=%p done\n", ex));
919 }
920 
921 void
922 ehci_poll(struct usbd_bus *bus)
923 {
924 	struct ehci_softc *sc = (struct ehci_softc *)bus;
925 
926 	if (EOREAD4(sc, EHCI_USBSTS) & sc->sc_eintrs)
927 		ehci_intr1(sc);
928 }
929 
930 int
931 ehci_detach(struct device *self, int flags)
932 {
933 	struct ehci_softc *sc = (struct ehci_softc *)self;
934 	int rv;
935 
936 	rv = config_detach_children(self, flags);
937 	if (rv != 0)
938 		return (rv);
939 
940 	timeout_del(&sc->sc_tmo_intrlist);
941 
942 	ehci_reset(sc);
943 
944 	usb_delay_ms(&sc->sc_bus, 300); /* XXX let stray task complete */
945 
946 	free(sc->sc_softitds, M_USB,
947 	    sc->sc_flsize * sizeof(struct ehci_soft_itd *));
948 	usb_freemem(&sc->sc_bus, &sc->sc_fldma);
949 	/* XXX free other data structures XXX */
950 
951 	return (rv);
952 }
953 
954 
955 int
956 ehci_activate(struct device *self, int act)
957 {
958 	struct ehci_softc *sc = (struct ehci_softc *)self;
959 	u_int32_t cmd, hcr, cparams;
960 	int i, rv = 0;
961 
962 	switch (act) {
963 	case DVACT_SUSPEND:
964 		rv = config_activate_children(self, act);
965 
966 #ifdef DIAGNOSTIC
967 		if (!TAILQ_EMPTY(&sc->sc_intrhead)) {
968 			printf("%s: interrupt list not empty\n",
969 			    sc->sc_bus.bdev.dv_xname);
970 			return (-1);
971 		}
972 #endif
973 
974 		sc->sc_bus.use_polling++;
975 
976 		for (i = 1; i <= sc->sc_noport; i++) {
977 			cmd = EOREAD4(sc, EHCI_PORTSC(i));
978 			if ((cmd & (EHCI_PS_PO|EHCI_PS_PE)) == EHCI_PS_PE)
979 				EOWRITE4(sc, EHCI_PORTSC(i),
980 				    cmd | EHCI_PS_SUSP);
981 		}
982 
983 		/*
984 		 * First tell the host to stop processing Asynchronous
985 		 * and Periodic schedules.
986 		 */
987 		cmd = EOREAD4(sc, EHCI_USBCMD) & ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
988 		EOWRITE4(sc, EHCI_USBCMD, cmd);
989 		for (i = 0; i < 100; i++) {
990 			usb_delay_ms(&sc->sc_bus, 1);
991 			hcr = EOREAD4(sc, EHCI_USBSTS) &
992 			    (EHCI_STS_ASS | EHCI_STS_PSS);
993 			if (hcr == 0)
994 				break;
995 		}
996 		if (hcr != 0)
997 			printf("%s: disable schedules timeout\n",
998 			    sc->sc_bus.bdev.dv_xname);
999 
1000 		/*
1001 		 * Then reset the host as if it was a shutdown.
1002 		 *
1003 		 * All USB devices are disconnected/reconnected during
1004 		 * a suspend/resume cycle so keep it simple.
1005 		 */
1006 		ehci_reset(sc);
1007 
1008 		sc->sc_bus.use_polling--;
1009 		break;
1010 	case DVACT_RESUME:
1011 		sc->sc_bus.use_polling++;
1012 
1013 		ehci_reset(sc);
1014 
1015 		cparams = EREAD4(sc, EHCI_HCCPARAMS);
1016 		/* MUST clear segment register if 64 bit capable. */
1017 		if (EHCI_HCC_64BIT(cparams))
1018 			EWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
1019 
1020 		EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
1021 		EOWRITE4(sc, EHCI_ASYNCLISTADDR,
1022 	  	    sc->sc_async_head->physaddr | EHCI_LINK_QH);
1023 
1024 		hcr = 0;
1025 		for (i = 1; i <= sc->sc_noport; i++) {
1026 			cmd = EOREAD4(sc, EHCI_PORTSC(i));
1027 			if ((cmd & (EHCI_PS_PO|EHCI_PS_SUSP)) == EHCI_PS_SUSP) {
1028 				EOWRITE4(sc, EHCI_PORTSC(i), cmd | EHCI_PS_FPR);
1029 				hcr = 1;
1030 			}
1031 		}
1032 
1033 		if (hcr) {
1034 			usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);
1035 			for (i = 1; i <= sc->sc_noport; i++) {
1036 				cmd = EOREAD4(sc, EHCI_PORTSC(i));
1037 				if ((cmd & (EHCI_PS_PO|EHCI_PS_SUSP)) ==
1038 				   EHCI_PS_SUSP)
1039 					EOWRITE4(sc, EHCI_PORTSC(i),
1040 					   cmd & ~EHCI_PS_FPR);
1041 			}
1042 		}
1043 
1044 		/* Turn on controller */
1045 		EOWRITE4(sc, EHCI_USBCMD,
1046 		    EHCI_CMD_ITC_2 | /* 2 microframes interrupt delay */
1047 		    (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
1048 		    EHCI_CMD_ASE |
1049 		    EHCI_CMD_PSE |
1050 		    EHCI_CMD_RS);
1051 
1052 		/* Take over port ownership */
1053 		EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
1054 		for (i = 0; i < 100; i++) {
1055 			usb_delay_ms(&sc->sc_bus, 1);
1056 			hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1057 			if (!hcr)
1058 				break;
1059 		}
1060 
1061 		if (hcr) {
1062 			printf("%s: run timeout\n", sc->sc_bus.bdev.dv_xname);
1063 			/* XXX should we bail here? */
1064 		}
1065 
1066 		EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1067 
1068 		usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);
1069 
1070 		sc->sc_bus.use_polling--;
1071 		rv = config_activate_children(self, act);
1072 		break;
1073 	case DVACT_POWERDOWN:
1074 		rv = config_activate_children(self, act);
1075 		ehci_reset(sc);
1076 		break;
1077 	default:
1078 		rv = config_activate_children(self, act);
1079 		break;
1080 	}
1081 	return (rv);
1082 }
1083 
1084 usbd_status
1085 ehci_reset(struct ehci_softc *sc)
1086 {
1087 	u_int32_t hcr, usbmode;
1088 	int i;
1089 
1090 	EOWRITE4(sc, EHCI_USBCMD, 0);	/* Halt controller */
1091 	for (i = 0; i < 100; i++) {
1092 		usb_delay_ms(&sc->sc_bus, 1);
1093 		hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1094 		if (hcr)
1095 			break;
1096 	}
1097 
1098 	if (!hcr)
1099 		printf("%s: halt timeout\n", sc->sc_bus.bdev.dv_xname);
1100 
1101 	if (sc->sc_flags & EHCIF_USBMODE)
1102 		usbmode = EOREAD4(sc, EHCI_USBMODE);
1103 
1104 	EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
1105 	for (i = 0; i < 100; i++) {
1106 		usb_delay_ms(&sc->sc_bus, 1);
1107 		hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
1108 		if (!hcr)
1109 			break;
1110 	}
1111 
1112 	if (hcr) {
1113 		printf("%s: reset timeout\n", sc->sc_bus.bdev.dv_xname);
1114 		return (USBD_IOERROR);
1115 	}
1116 
1117 	if (sc->sc_flags & EHCIF_USBMODE)
1118 		EOWRITE4(sc, EHCI_USBMODE, usbmode);
1119 
1120 	return (USBD_NORMAL_COMPLETION);
1121 }
1122 
1123 struct usbd_xfer *
1124 ehci_allocx(struct usbd_bus *bus)
1125 {
1126 	struct ehci_xfer *ex;
1127 
1128 	ex = pool_get(ehcixfer, PR_NOWAIT | PR_ZERO);
1129 #ifdef DIAGNOSTIC
1130 	if (ex != NULL)
1131 		ex->isdone = 1;
1132 #endif
1133 	return ((struct usbd_xfer *)ex);
1134 }
1135 
1136 void
1137 ehci_freex(struct usbd_bus *bus, struct usbd_xfer *xfer)
1138 {
1139 	struct ehci_xfer *ex = (struct ehci_xfer*)xfer;
1140 
1141 #ifdef DIAGNOSTIC
1142 	if (!ex->isdone) {
1143 		printf("%s: !isdone\n", __func__);
1144 		return;
1145 	}
1146 #endif
1147 	pool_put(ehcixfer, ex);
1148 }
1149 
1150 void
1151 ehci_device_clear_toggle(struct usbd_pipe *pipe)
1152 {
1153 	struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1154 
1155 #ifdef DIAGNOSTIC
1156 	if ((epipe->sqh->qh.qh_qtd.qtd_status & htole32(EHCI_QTD_ACTIVE)) != 0)
1157 		panic("ehci_device_clear_toggle: queue active");
1158 #endif
1159 	epipe->sqh->qh.qh_qtd.qtd_status &= htole32(~EHCI_QTD_TOGGLE_MASK);
1160 }
1161 
1162 #ifdef EHCI_DEBUG
1163 void
1164 ehci_dump_regs(struct ehci_softc *sc)
1165 {
1166 	int i;
1167 
1168 	printf("cmd=0x%08x, sts=0x%08x, ien=0x%08x\n",
1169 	    EOREAD4(sc, EHCI_USBCMD),
1170 	    EOREAD4(sc, EHCI_USBSTS),
1171 	    EOREAD4(sc, EHCI_USBINTR));
1172 	printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
1173 	    EOREAD4(sc, EHCI_FRINDEX),
1174 	    EOREAD4(sc, EHCI_CTRLDSSEGMENT),
1175 	    EOREAD4(sc, EHCI_PERIODICLISTBASE),
1176 	    EOREAD4(sc, EHCI_ASYNCLISTADDR));
1177 	for (i = 1; i <= sc->sc_noport; i++)
1178 		printf("port %d status=0x%08x\n", i,
1179 		    EOREAD4(sc, EHCI_PORTSC(i)));
1180 }
1181 
1182 /*
1183  * Unused function - this is meant to be called from a kernel
1184  * debugger.
1185  */
1186 void
1187 ehci_dump(void)
1188 {
1189 	ehci_dump_regs(theehci);
1190 }
1191 
1192 void
1193 ehci_dump_link(ehci_link_t link, int type)
1194 {
1195 	link = letoh32(link);
1196 	printf("0x%08x", link);
1197 	if (link & EHCI_LINK_TERMINATE)
1198 		printf("<T>");
1199 	else {
1200 		printf("<");
1201 		if (type) {
1202 			switch (EHCI_LINK_TYPE(link)) {
1203 			case EHCI_LINK_ITD:
1204 				printf("ITD");
1205 				break;
1206 			case EHCI_LINK_QH:
1207 				printf("QH");
1208 				break;
1209 			case EHCI_LINK_SITD:
1210 				printf("SITD");
1211 				break;
1212 			case EHCI_LINK_FSTN:
1213 				printf("FSTN");
1214 				break;
1215 			}
1216 		}
1217 		printf(">");
1218 	}
1219 }
1220 
1221 void
1222 ehci_dump_sqtds(struct ehci_soft_qtd *sqtd)
1223 {
1224 	int i;
1225 	u_int32_t stop;
1226 
1227 	stop = 0;
1228 	for (i = 0; sqtd && i < 20 && !stop; sqtd = sqtd->nextqtd, i++) {
1229 		ehci_dump_sqtd(sqtd);
1230 		usb_syncmem(&sqtd->dma,
1231 		    sqtd->offs + offsetof(struct ehci_qtd, qtd_next),
1232 		    sizeof(sqtd->qtd),
1233 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1234 		stop = sqtd->qtd.qtd_next & htole32(EHCI_LINK_TERMINATE);
1235 		usb_syncmem(&sqtd->dma,
1236 		    sqtd->offs + offsetof(struct ehci_qtd, qtd_next),
1237 		    sizeof(sqtd->qtd), BUS_DMASYNC_PREREAD);
1238 	}
1239 	if (!stop)
1240 		printf("dump aborted, too many TDs\n");
1241 }
1242 
1243 void
1244 ehci_dump_sqtd(struct ehci_soft_qtd *sqtd)
1245 {
1246 	usb_syncmem(&sqtd->dma, sqtd->offs,
1247 	    sizeof(sqtd->qtd), BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1248 	printf("QTD(%p) at 0x%08x:\n", sqtd, sqtd->physaddr);
1249 	ehci_dump_qtd(&sqtd->qtd);
1250 	usb_syncmem(&sqtd->dma, sqtd->offs,
1251 	    sizeof(sqtd->qtd), BUS_DMASYNC_PREREAD);
1252 }
1253 
1254 void
1255 ehci_dump_qtd(struct ehci_qtd *qtd)
1256 {
1257 	u_int32_t s;
1258 	char sbuf[128];
1259 
1260 	printf("  next="); ehci_dump_link(qtd->qtd_next, 0);
1261 	printf(" altnext="); ehci_dump_link(qtd->qtd_altnext, 0);
1262 	printf("\n");
1263 	s = letoh32(qtd->qtd_status);
1264 	bitmask_snprintf(EHCI_QTD_GET_STATUS(s), "\20\10ACTIVE\7HALTED"
1265 	    "\6BUFERR\5BABBLE\4XACTERR\3MISSED\2SPLIT\1PING",
1266 	    sbuf, sizeof(sbuf));
1267 	printf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
1268 	    s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
1269 	    EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
1270 	printf("    cerr=%d pid=%d stat=0x%s\n", EHCI_QTD_GET_CERR(s),
1271 	    EHCI_QTD_GET_PID(s), sbuf);
1272 	for (s = 0; s < 5; s++)
1273 		printf("  buffer[%d]=0x%08x\n", s, letoh32(qtd->qtd_buffer[s]));
1274 }
1275 
1276 void
1277 ehci_dump_sqh(struct ehci_soft_qh *sqh)
1278 {
1279 	struct ehci_qh *qh = &sqh->qh;
1280 	u_int32_t endp, endphub;
1281 
1282 	usb_syncmem(&sqh->dma, sqh->offs,
1283 	    sizeof(sqh->qh), BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1284 	printf("QH(%p) at 0x%08x:\n", sqh, sqh->physaddr);
1285 	printf("  link="); ehci_dump_link(qh->qh_link, 1); printf("\n");
1286 	endp = letoh32(qh->qh_endp);
1287 	printf("  endp=0x%08x\n", endp);
1288 	printf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
1289 	    EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
1290 	    EHCI_QH_GET_ENDPT(endp),  EHCI_QH_GET_EPS(endp),
1291 	    EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
1292 	printf("    mpl=0x%x ctl=%d nrl=%d\n",
1293 	    EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
1294 	    EHCI_QH_GET_NRL(endp));
1295 	endphub = letoh32(qh->qh_endphub);
1296 	printf("  endphub=0x%08x\n", endphub);
1297 	printf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
1298 	    EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
1299 	    EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
1300 	    EHCI_QH_GET_MULT(endphub));
1301 	printf("  curqtd="); ehci_dump_link(qh->qh_curqtd, 0); printf("\n");
1302 	printf("Overlay qTD:\n");
1303 	ehci_dump_qtd(&qh->qh_qtd);
1304 	usb_syncmem(&sqh->dma, sqh->offs,
1305 	    sizeof(sqh->qh), BUS_DMASYNC_PREREAD);
1306 }
1307 
1308 #if notyet
1309 void
1310 ehci_dump_itd(struct ehci_soft_itd *itd)
1311 {
1312 	ehci_isoc_trans_t t;
1313 	ehci_isoc_bufr_ptr_t b, b2, b3;
1314 	int i;
1315 
1316 	printf("ITD: next phys=%X\n", itd->itd.itd_next);
1317 
1318 	for (i = 0; i < 8; i++) {
1319 		t = letoh32(itd->itd.itd_ctl[i]);
1320 		printf("ITDctl %d: stat=%X len=%X ioc=%X pg=%X offs=%X\n", i,
1321 		    EHCI_ITD_GET_STATUS(t), EHCI_ITD_GET_LEN(t),
1322 		    EHCI_ITD_GET_IOC(t), EHCI_ITD_GET_PG(t),
1323 		    EHCI_ITD_GET_OFFS(t));
1324 	}
1325 	printf("ITDbufr: ");
1326 	for (i = 0; i < 7; i++)
1327 		printf("%X,", EHCI_ITD_GET_BPTR(letoh32(itd->itd.itd_bufr[i])));
1328 
1329 	b = letoh32(itd->itd.itd_bufr[0]);
1330 	b2 = letoh32(itd->itd.itd_bufr[1]);
1331 	b3 = letoh32(itd->itd.itd_bufr[2]);
1332 	printf("\nep=%X daddr=%X dir=%d maxpkt=%X multi=%X\n",
1333 	    EHCI_ITD_GET_EP(b), EHCI_ITD_GET_DADDR(b), EHCI_ITD_GET_DIR(b2),
1334 	    EHCI_ITD_GET_MAXPKT(b2), EHCI_ITD_GET_MULTI(b3));
1335 }
1336 #endif
1337 
1338 #ifdef DIAGNOSTIC
1339 void
1340 ehci_dump_exfer(struct ehci_xfer *ex)
1341 {
1342 	printf("ehci_dump_exfer: ex=%p sqtdstart=%p end=%p itdstart=%p end=%p "
1343 	    "isdone=%d\n", ex, ex->sqtdstart, ex->sqtdend, ex->itdstart,
1344 	    ex->itdend, ex->isdone);
1345 }
1346 #endif
1347 
1348 #endif /* EHCI_DEBUG */
1349 
1350 usbd_status
1351 ehci_open(struct usbd_pipe *pipe)
1352 {
1353 	struct usbd_device *dev = pipe->device;
1354 	struct ehci_softc *sc = (struct ehci_softc *)dev->bus;
1355 	usb_endpoint_descriptor_t *ed = pipe->endpoint->edesc;
1356 	u_int8_t addr = dev->address;
1357 	u_int8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes);
1358 	struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1359 	struct ehci_soft_qh *sqh;
1360 	usbd_status err;
1361 	int s;
1362 	int ival, speed, naks;
1363 	int hshubaddr, hshubport;
1364 
1365 	DPRINTFN(1, ("ehci_open: pipe=%p, addr=%d, endpt=%d\n",
1366 	    pipe, addr, ed->bEndpointAddress));
1367 
1368 	if (sc->sc_bus.dying)
1369 		return (USBD_IOERROR);
1370 
1371 	if (dev->myhsport) {
1372 		hshubaddr = dev->myhsport->parent->address;
1373 		hshubport = dev->myhsport->portno;
1374 	} else {
1375 		hshubaddr = 0;
1376 		hshubport = 0;
1377 	}
1378 
1379 	/* Root Hub */
1380 	if (pipe->device->depth == 0) {
1381 		switch (ed->bEndpointAddress) {
1382 		case USB_CONTROL_ENDPOINT:
1383 			pipe->methods = &ehci_root_ctrl_methods;
1384 			break;
1385 		case UE_DIR_IN | EHCI_INTR_ENDPT:
1386 			pipe->methods = &ehci_root_intr_methods;
1387 			break;
1388 		default:
1389 			return (USBD_INVAL);
1390 		}
1391 		return (USBD_NORMAL_COMPLETION);
1392 	}
1393 
1394 	/* XXX All this stuff is only valid for async. */
1395 	switch (dev->speed) {
1396 	case USB_SPEED_LOW:
1397 		speed = EHCI_QH_SPEED_LOW;
1398 		break;
1399 	case USB_SPEED_FULL:
1400 		speed = EHCI_QH_SPEED_FULL;
1401 		break;
1402 	case USB_SPEED_HIGH:
1403 		speed = EHCI_QH_SPEED_HIGH;
1404 		break;
1405 	default:
1406 		panic("ehci_open: bad device speed %d", dev->speed);
1407 	}
1408 
1409 	/*
1410 	 * NAK reload count:
1411 	 * must be zero with using periodic transfer.
1412 	 * Linux 4.20's driver (ehci-q.c) sets 4, we use same value.
1413 	 */
1414 	naks = ((xfertype == UE_CONTROL) || (xfertype == UE_BULK)) ? 4 : 0;
1415 
1416 	/* Allocate sqh for everything, save isoc xfers */
1417 	if (xfertype != UE_ISOCHRONOUS) {
1418 		sqh = ehci_alloc_sqh(sc);
1419 		if (sqh == NULL)
1420 			return (USBD_NOMEM);
1421 		/* qh_link filled when the QH is added */
1422 		sqh->qh.qh_endp = htole32(
1423 		    EHCI_QH_SET_ADDR(addr) |
1424 		    EHCI_QH_SET_ENDPT(UE_GET_ADDR(ed->bEndpointAddress)) |
1425 		    EHCI_QH_SET_EPS(speed) |
1426 		    (xfertype == UE_CONTROL ? EHCI_QH_DTC : 0) |
1427 		    EHCI_QH_SET_MPL(UGETW(ed->wMaxPacketSize)) |
1428 		    (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_CONTROL ?
1429 		    EHCI_QH_CTL : 0) |
1430 		    EHCI_QH_SET_NRL(naks)
1431 		);
1432 		/*
1433 		 * To reduce conflict with split isochronous transfer,
1434 		 * schedule (split) interrupt transfer at latter half of
1435 		 * 1ms frame:
1436 		 *
1437 		 *         |<-------------- H-Frame -------------->|
1438 		 *         .H0  :H1   H2   H3   H4   H5   H6   H7  .H0" :H1"
1439 		 *         .    :                                  .    :
1440 		 * [HS]    .    :          SS        CS   CS'  CS" .    :
1441 		 * [FS/LS] .    :               |<== >>>> >>>|     .    :
1442 		 *         .    :                                  .    :
1443 		 *         .B7' :B0   B1   B2   B3   B4   B5   B6  .B7  :B0"
1444 		 *              |<-------------- B-Frame -------------->|
1445 		 *
1446 		 */
1447 		sqh->qh.qh_endphub = htole32(
1448 		    EHCI_QH_SET_MULT(1) |
1449 		    EHCI_QH_SET_SMASK(xfertype == UE_INTERRUPT ? 0x08 : 0)
1450 		);
1451 		if (speed != EHCI_QH_SPEED_HIGH) {
1452 			sqh->qh.qh_endphub |= htole32(
1453 			    EHCI_QH_SET_HUBA(hshubaddr) |
1454 			    EHCI_QH_SET_PORT(hshubport) |
1455 			    EHCI_QH_SET_CMASK(0xe0)
1456 			);
1457 		}
1458 		sqh->qh.qh_curqtd = htole32(EHCI_LINK_TERMINATE);
1459 		/* Fill the overlay qTD */
1460 		sqh->qh.qh_qtd.qtd_next = htole32(EHCI_LINK_TERMINATE);
1461 		sqh->qh.qh_qtd.qtd_altnext = htole32(EHCI_LINK_TERMINATE);
1462 		sqh->qh.qh_qtd.qtd_status =
1463 		    htole32(EHCI_QTD_SET_TOGGLE(pipe->endpoint->savedtoggle));
1464 
1465 		usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
1466 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1467 		epipe->sqh = sqh;
1468 	} /*xfertype == UE_ISOC*/
1469 
1470 	switch (xfertype) {
1471 	case UE_CONTROL:
1472 		err = usb_allocmem(&sc->sc_bus, sizeof(usb_device_request_t),
1473 		    0, &epipe->u.ctl.reqdma);
1474 		if (err) {
1475 			ehci_free_sqh(sc, sqh);
1476 			return (err);
1477 		}
1478 		pipe->methods = &ehci_device_ctrl_methods;
1479 		s = splusb();
1480 		ehci_add_qh(sqh, sc->sc_async_head);
1481 		splx(s);
1482 		break;
1483 	case UE_BULK:
1484 		pipe->methods = &ehci_device_bulk_methods;
1485 		s = splusb();
1486 		ehci_add_qh(sqh, sc->sc_async_head);
1487 		splx(s);
1488 		break;
1489 	case UE_INTERRUPT:
1490 		pipe->methods = &ehci_device_intr_methods;
1491 		ival = pipe->interval;
1492 		if (ival == USBD_DEFAULT_INTERVAL)
1493 			ival = ed->bInterval;
1494 		s = splusb();
1495 		err = ehci_device_setintr(sc, sqh, ival);
1496 		splx(s);
1497 		return (err);
1498 	case UE_ISOCHRONOUS:
1499 		switch (speed) {
1500 		case EHCI_QH_SPEED_HIGH:
1501 		case EHCI_QH_SPEED_FULL:
1502 			pipe->methods = &ehci_device_isoc_methods;
1503 			break;
1504 		case EHCI_QH_SPEED_LOW:
1505 		default:
1506 			return (USBD_INVAL);
1507 		}
1508 		/* Spec page 271 says intervals > 16 are invalid */
1509 		if (ed->bInterval == 0 || ed->bInterval > 16) {
1510 			printf("ehci: opening pipe with invalid bInterval\n");
1511 			return (USBD_INVAL);
1512 		}
1513 		if (UGETW(ed->wMaxPacketSize) == 0) {
1514 			printf("ehci: zero length endpoint open request\n");
1515 			return (USBD_INVAL);
1516 		}
1517 		epipe->u.isoc.next_frame = 0;
1518 		epipe->u.isoc.cur_xfers = 0;
1519 		break;
1520 	default:
1521 		DPRINTF(("ehci: bad xfer type %d\n", xfertype));
1522 		return (USBD_INVAL);
1523 	}
1524 	return (USBD_NORMAL_COMPLETION);
1525 }
1526 
1527 /*
1528  * Add an ED to the schedule.  Called at splusb().
1529  * If in the async schedule, it will always have a next.
1530  * If in the intr schedule it may not.
1531  */
1532 void
1533 ehci_add_qh(struct ehci_soft_qh *sqh, struct ehci_soft_qh *head)
1534 {
1535 	splsoftassert(IPL_SOFTUSB);
1536 
1537 	usb_syncmem(&head->dma, head->offs + offsetof(struct ehci_qh, qh_link),
1538 	    sizeof(head->qh.qh_link), BUS_DMASYNC_POSTWRITE);
1539 	sqh->next = head->next;
1540 	sqh->prev = head;
1541 	sqh->qh.qh_link = head->qh.qh_link;
1542 	usb_syncmem(&sqh->dma, sqh->offs + offsetof(struct ehci_qh, qh_link),
1543 	    sizeof(sqh->qh.qh_link), BUS_DMASYNC_PREWRITE);
1544 	head->next = sqh;
1545 	if (sqh->next)
1546 		sqh->next->prev = sqh;
1547 	head->qh.qh_link = htole32(sqh->physaddr | EHCI_LINK_QH);
1548 	usb_syncmem(&head->dma, head->offs + offsetof(struct ehci_qh, qh_link),
1549 	    sizeof(head->qh.qh_link), BUS_DMASYNC_PREWRITE);
1550 }
1551 
1552 /*
1553  * Remove an ED from the schedule.  Called at splusb().
1554  * Will always have a 'next' if it's in the async list as it's circular.
1555  */
1556 void
1557 ehci_rem_qh(struct ehci_softc *sc, struct ehci_soft_qh *sqh)
1558 {
1559 	splsoftassert(IPL_SOFTUSB);
1560 	/* XXX */
1561 	usb_syncmem(&sqh->dma, sqh->offs + offsetof(struct ehci_qh, qh_link),
1562 	    sizeof(sqh->qh.qh_link), BUS_DMASYNC_POSTWRITE);
1563 	sqh->prev->qh.qh_link = sqh->qh.qh_link;
1564 	sqh->prev->next = sqh->next;
1565 	if (sqh->next)
1566 		sqh->next->prev = sqh->prev;
1567 	usb_syncmem(&sqh->prev->dma,
1568 	    sqh->prev->offs + offsetof(struct ehci_qh, qh_link),
1569 	    sizeof(sqh->prev->qh.qh_link), BUS_DMASYNC_PREWRITE);
1570 
1571 	ehci_sync_hc(sc);
1572 }
1573 
1574 void
1575 ehci_set_qh_qtd(struct ehci_soft_qh *sqh, struct ehci_soft_qtd *sqtd)
1576 {
1577 	int i;
1578 	u_int32_t status;
1579 
1580 	/* Save toggle bit and ping status. */
1581 	usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
1582 	    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1583 	status = sqh->qh.qh_qtd.qtd_status &
1584 	    htole32(EHCI_QTD_TOGGLE_MASK |
1585 		EHCI_QTD_SET_STATUS(EHCI_QTD_PINGSTATE));
1586 	/* Set HALTED to make hw leave it alone. */
1587 	sqh->qh.qh_qtd.qtd_status =
1588 	    htole32(EHCI_QTD_SET_STATUS(EHCI_QTD_HALTED));
1589 	usb_syncmem(&sqh->dma,
1590 	    sqh->offs + offsetof(struct ehci_qh, qh_qtd.qtd_status),
1591 	    sizeof(sqh->qh.qh_qtd.qtd_status),
1592 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1593 	sqh->qh.qh_curqtd = 0;
1594 	sqh->qh.qh_qtd.qtd_next = htole32(sqtd->physaddr);
1595 	sqh->qh.qh_qtd.qtd_altnext = htole32(EHCI_LINK_TERMINATE);
1596 	for (i = 0; i < EHCI_QTD_NBUFFERS; i++)
1597 		sqh->qh.qh_qtd.qtd_buffer[i] = 0;
1598 	sqh->sqtd = sqtd;
1599 	usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
1600 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1601 	/* Set !HALTED && !ACTIVE to start execution, preserve some fields */
1602 	sqh->qh.qh_qtd.qtd_status = status;
1603 	usb_syncmem(&sqh->dma,
1604 	    sqh->offs + offsetof(struct ehci_qh, qh_qtd.qtd_status),
1605 	    sizeof(sqh->qh.qh_qtd.qtd_status),
1606 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1607 }
1608 
1609 /*
1610  * Ensure that the HC has released all references to the QH.  We do this
1611  * by asking for a Async Advance Doorbell interrupt and then we wait for
1612  * the interrupt.
1613  * To make this easier we first obtain exclusive use of the doorbell.
1614  */
1615 void
1616 ehci_sync_hc(struct ehci_softc *sc)
1617 {
1618 	int s, error;
1619 	int tries = 0;
1620 
1621 	if (sc->sc_bus.dying) {
1622 		return;
1623 	}
1624 
1625 	/* get doorbell */
1626 	rw_enter_write(&sc->sc_doorbell_lock);
1627 	s = splhardusb();
1628 	do {
1629 		EOWRITE4(sc, EHCI_USBCMD, EOREAD4(sc, EHCI_USBCMD) |
1630 		    EHCI_CMD_IAAD);
1631 		error = tsleep_nsec(&sc->sc_async_head, PZERO, "ehcidi",
1632 		    MSEC_TO_NSEC(500));
1633 	} while (error && ++tries < 10);
1634 	splx(s);
1635 	/* release doorbell */
1636 	rw_exit_write(&sc->sc_doorbell_lock);
1637 #ifdef DIAGNOSTIC
1638 	if (error)
1639 		printf("ehci_sync_hc: tsleep() = %d\n", error);
1640 #endif
1641 }
1642 
1643 void
1644 ehci_rem_itd_chain(struct ehci_softc *sc, struct ehci_xfer *ex)
1645 {
1646 	struct ehci_soft_itd *itd, *prev = NULL;
1647 
1648 	splsoftassert(IPL_SOFTUSB);
1649 
1650 	KASSERT(ex->itdstart != NULL && ex->itdend != NULL);
1651 
1652 	for (itd = ex->itdstart; itd != NULL; itd = itd->xfer_next) {
1653 		prev = itd->u.frame_list.prev;
1654 		/* Unlink itd from hardware chain, or frame array */
1655 		if (prev == NULL) { /* We're at the table head */
1656 			sc->sc_softitds[itd->slot] = itd->u.frame_list.next;
1657 			sc->sc_flist[itd->slot] = itd->itd.itd_next;
1658 			usb_syncmem(&sc->sc_fldma,
1659 			    sizeof(uint32_t) * itd->slot, sizeof(uint32_t),
1660 			    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1661 
1662 			if (itd->u.frame_list.next != NULL)
1663 				itd->u.frame_list.next->u.frame_list.prev =
1664 				    NULL;
1665 		} else {
1666 			/* XXX this part is untested... */
1667 			prev->itd.itd_next = itd->itd.itd_next;
1668 			usb_syncmem(&itd->dma,
1669 			    itd->offs + offsetof(struct ehci_itd, itd_next),
1670 			    sizeof(itd->itd.itd_next), BUS_DMASYNC_PREWRITE);
1671 
1672 			prev->u.frame_list.next = itd->u.frame_list.next;
1673 			if (itd->u.frame_list.next != NULL)
1674 				itd->u.frame_list.next->u.frame_list.prev =
1675 				    prev;
1676 		}
1677 	}
1678 }
1679 
1680 void
1681 ehci_free_itd_chain(struct ehci_softc *sc, struct ehci_xfer *ex)
1682 {
1683 	struct ehci_soft_itd *itd, *prev = NULL;
1684 
1685 	splsoftassert(IPL_SOFTUSB);
1686 
1687 	KASSERT(ex->itdstart != NULL && ex->itdend != NULL);
1688 
1689 	for (itd = ex->itdstart; itd != NULL; itd = itd->xfer_next) {
1690 		if (prev != NULL)
1691 			ehci_free_itd(sc, prev);
1692 		prev = itd;
1693 	}
1694 	if (prev)
1695 		ehci_free_itd(sc, prev);
1696 	ex->itdstart = NULL;
1697 	ex->itdend = NULL;
1698 }
1699 
1700 /*
1701  * Data structures and routines to emulate the root hub.
1702  */
1703 usb_device_descriptor_t ehci_devd = {
1704 	USB_DEVICE_DESCRIPTOR_SIZE,
1705 	UDESC_DEVICE,		/* type */
1706 	{0x00, 0x02},		/* USB version */
1707 	UDCLASS_HUB,		/* class */
1708 	UDSUBCLASS_HUB,		/* subclass */
1709 	UDPROTO_HSHUBSTT,	/* protocol */
1710 	64,			/* max packet */
1711 	{0},{0},{0x00,0x01},	/* device id */
1712 	1,2,0,			/* string indicies */
1713 	1			/* # of configurations */
1714 };
1715 
1716 usb_device_qualifier_t ehci_odevd = {
1717 	USB_DEVICE_DESCRIPTOR_SIZE,
1718 	UDESC_DEVICE_QUALIFIER,	/* type */
1719 	{0x00, 0x02},		/* USB version */
1720 	UDCLASS_HUB,		/* class */
1721 	UDSUBCLASS_HUB,		/* subclass */
1722 	UDPROTO_FSHUB,		/* protocol */
1723 	64,			/* max packet */
1724 	1,			/* # of configurations */
1725 	0
1726 };
1727 
1728 usb_config_descriptor_t ehci_confd = {
1729 	USB_CONFIG_DESCRIPTOR_SIZE,
1730 	UDESC_CONFIG,
1731 	{USB_CONFIG_DESCRIPTOR_SIZE +
1732 	 USB_INTERFACE_DESCRIPTOR_SIZE +
1733 	 USB_ENDPOINT_DESCRIPTOR_SIZE},
1734 	1,
1735 	1,
1736 	0,
1737 	UC_BUS_POWERED | UC_SELF_POWERED,
1738 	0			/* max power */
1739 };
1740 
1741 usb_interface_descriptor_t ehci_ifcd = {
1742 	USB_INTERFACE_DESCRIPTOR_SIZE,
1743 	UDESC_INTERFACE,
1744 	0,
1745 	0,
1746 	1,
1747 	UICLASS_HUB,
1748 	UISUBCLASS_HUB,
1749 	UIPROTO_HSHUBSTT,
1750 	0
1751 };
1752 
1753 usb_endpoint_descriptor_t ehci_endpd = {
1754 	USB_ENDPOINT_DESCRIPTOR_SIZE,
1755 	UDESC_ENDPOINT,
1756 	UE_DIR_IN | EHCI_INTR_ENDPT,
1757 	UE_INTERRUPT,
1758 	{8, 0},			/* max packet */
1759 	12
1760 };
1761 
1762 usb_hub_descriptor_t ehci_hubd = {
1763 	USB_HUB_DESCRIPTOR_SIZE,
1764 	UDESC_HUB,
1765 	0,
1766 	{0,0},
1767 	0,
1768 	0,
1769 	{0},
1770 };
1771 
1772 /*
1773  * Simulate a hardware hub by handling all the necessary requests.
1774  */
1775 usbd_status
1776 ehci_root_ctrl_transfer(struct usbd_xfer *xfer)
1777 {
1778 	usbd_status err;
1779 
1780 	/* Insert last in queue. */
1781 	err = usb_insert_transfer(xfer);
1782 	if (err)
1783 		return (err);
1784 
1785 	/* Pipe isn't running, start first */
1786 	return (ehci_root_ctrl_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
1787 }
1788 
1789 usbd_status
1790 ehci_root_ctrl_start(struct usbd_xfer *xfer)
1791 {
1792 	struct ehci_softc *sc = (struct ehci_softc *)xfer->device->bus;
1793 	usb_device_request_t *req;
1794 	void *buf = NULL;
1795 	int port, i;
1796 	int s, len, value, index, l, totlen = 0;
1797 	usb_port_status_t ps;
1798 	usb_hub_descriptor_t hubd;
1799 	usbd_status err;
1800 	u_int32_t v;
1801 
1802 	if (sc->sc_bus.dying)
1803 		return (USBD_IOERROR);
1804 
1805 #ifdef DIAGNOSTIC
1806 	if (!(xfer->rqflags & URQ_REQUEST))
1807 		/* XXX panic */
1808 		return (USBD_INVAL);
1809 #endif
1810 	req = &xfer->request;
1811 
1812 	DPRINTFN(4,("ehci_root_ctrl_start: type=0x%02x request=%02x\n",
1813 		    req->bmRequestType, req->bRequest));
1814 
1815 	len = UGETW(req->wLength);
1816 	value = UGETW(req->wValue);
1817 	index = UGETW(req->wIndex);
1818 
1819 	if (len != 0)
1820 		buf = KERNADDR(&xfer->dmabuf, 0);
1821 
1822 #define C(x,y) ((x) | ((y) << 8))
1823 	switch(C(req->bRequest, req->bmRequestType)) {
1824 	case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
1825 	case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
1826 	case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
1827 		/*
1828 		 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
1829 		 * for the integrated root hub.
1830 		 */
1831 		break;
1832 	case C(UR_GET_CONFIG, UT_READ_DEVICE):
1833 		if (len > 0) {
1834 			*(u_int8_t *)buf = sc->sc_conf;
1835 			totlen = 1;
1836 		}
1837 		break;
1838 	case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
1839 		DPRINTFN(8,("ehci_root_ctrl_start: wValue=0x%04x\n", value));
1840 		switch(value >> 8) {
1841 		case UDESC_DEVICE:
1842 			if ((value & 0xff) != 0) {
1843 				err = USBD_IOERROR;
1844 				goto ret;
1845 			}
1846 			totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
1847 			USETW(ehci_devd.idVendor, sc->sc_id_vendor);
1848 			memcpy(buf, &ehci_devd, l);
1849 			break;
1850 		/*
1851 		 * We can't really operate at another speed, but the spec says
1852 		 * we need this descriptor.
1853 		 */
1854 		case UDESC_DEVICE_QUALIFIER:
1855 			if ((value & 0xff) != 0) {
1856 				err = USBD_IOERROR;
1857 				goto ret;
1858 			}
1859 			totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
1860 			memcpy(buf, &ehci_odevd, l);
1861 			break;
1862 		/*
1863 		 * We can't really operate at another speed, but the spec says
1864 		 * we need this descriptor.
1865 		 */
1866 		case UDESC_OTHER_SPEED_CONFIGURATION:
1867 		case UDESC_CONFIG:
1868 			if ((value & 0xff) != 0) {
1869 				err = USBD_IOERROR;
1870 				goto ret;
1871 			}
1872 			totlen = l = min(len, USB_CONFIG_DESCRIPTOR_SIZE);
1873 			memcpy(buf, &ehci_confd, l);
1874 			((usb_config_descriptor_t *)buf)->bDescriptorType =
1875 			    value >> 8;
1876 			buf = (char *)buf + l;
1877 			len -= l;
1878 			l = min(len, USB_INTERFACE_DESCRIPTOR_SIZE);
1879 			totlen += l;
1880 			memcpy(buf, &ehci_ifcd, l);
1881 			buf = (char *)buf + l;
1882 			len -= l;
1883 			l = min(len, USB_ENDPOINT_DESCRIPTOR_SIZE);
1884 			totlen += l;
1885 			memcpy(buf, &ehci_endpd, l);
1886 			break;
1887 		case UDESC_STRING:
1888 			if (len == 0)
1889 				break;
1890 			*(u_int8_t *)buf = 0;
1891 			totlen = 1;
1892 			switch (value & 0xff) {
1893 			case 0: /* Language table */
1894 				totlen = usbd_str(buf, len, "\001");
1895 				break;
1896 			case 1: /* Vendor */
1897 				totlen = usbd_str(buf, len, sc->sc_vendor);
1898 				break;
1899 			case 2: /* Product */
1900 				totlen = usbd_str(buf, len, "EHCI root hub");
1901 				break;
1902 			}
1903 			break;
1904 		default:
1905 			err = USBD_IOERROR;
1906 			goto ret;
1907 		}
1908 		break;
1909 	case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
1910 		if (len > 0) {
1911 			*(u_int8_t *)buf = 0;
1912 			totlen = 1;
1913 		}
1914 		break;
1915 	case C(UR_GET_STATUS, UT_READ_DEVICE):
1916 		if (len > 1) {
1917 			USETW(((usb_status_t *)buf)->wStatus,UDS_SELF_POWERED);
1918 			totlen = 2;
1919 		}
1920 		break;
1921 	case C(UR_GET_STATUS, UT_READ_INTERFACE):
1922 	case C(UR_GET_STATUS, UT_READ_ENDPOINT):
1923 		if (len > 1) {
1924 			USETW(((usb_status_t *)buf)->wStatus, 0);
1925 			totlen = 2;
1926 		}
1927 		break;
1928 	case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
1929 		if (value >= USB_MAX_DEVICES) {
1930 			err = USBD_IOERROR;
1931 			goto ret;
1932 		}
1933 		break;
1934 	case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
1935 		if (value != 0 && value != 1) {
1936 			err = USBD_IOERROR;
1937 			goto ret;
1938 		}
1939 		sc->sc_conf = value;
1940 		break;
1941 	case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
1942 		break;
1943 	case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
1944 	case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
1945 	case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
1946 		err = USBD_IOERROR;
1947 		goto ret;
1948 	case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
1949 		break;
1950 	case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
1951 		break;
1952 	/* Hub requests */
1953 	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
1954 		break;
1955 	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
1956 		DPRINTFN(8, ("ehci_root_ctrl_start: UR_CLEAR_PORT_FEATURE "
1957 		    "port=%d feature=%d\n", index, value));
1958 		if (index < 1 || index > sc->sc_noport) {
1959 			err = USBD_IOERROR;
1960 			goto ret;
1961 		}
1962 		port = EHCI_PORTSC(index);
1963 		v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
1964 		switch(value) {
1965 		case UHF_PORT_ENABLE:
1966 			EOWRITE4(sc, port, v &~ EHCI_PS_PE);
1967 			break;
1968 		case UHF_PORT_SUSPEND:
1969 			EOWRITE4(sc, port, v &~ EHCI_PS_SUSP);
1970 			break;
1971 		case UHF_PORT_POWER:
1972 			EOWRITE4(sc, port, v &~ EHCI_PS_PP);
1973 			break;
1974 		case UHF_PORT_TEST:
1975 			DPRINTFN(2,("ehci_root_ctrl_start: "
1976 			    "clear port test %d\n", index));
1977 			break;
1978 		case UHF_PORT_INDICATOR:
1979 			DPRINTFN(2,("ehci_root_ctrl_start: "
1980 			    "clear port index %d\n", index));
1981 			EOWRITE4(sc, port, v &~ EHCI_PS_PIC);
1982 			break;
1983 		case UHF_C_PORT_CONNECTION:
1984 			EOWRITE4(sc, port, v | EHCI_PS_CSC);
1985 			break;
1986 		case UHF_C_PORT_ENABLE:
1987 			EOWRITE4(sc, port, v | EHCI_PS_PEC);
1988 			break;
1989 		case UHF_C_PORT_SUSPEND:
1990 			/* how? */
1991 			break;
1992 		case UHF_C_PORT_OVER_CURRENT:
1993 			EOWRITE4(sc, port, v | EHCI_PS_OCC);
1994 			break;
1995 		case UHF_C_PORT_RESET:
1996 			sc->sc_isreset = 0;
1997 			break;
1998 		default:
1999 			err = USBD_IOERROR;
2000 			goto ret;
2001 		}
2002 		break;
2003 	case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
2004 		if ((value & 0xff) != 0) {
2005 			err = USBD_IOERROR;
2006 			goto ret;
2007 		}
2008 		hubd = ehci_hubd;
2009 		hubd.bNbrPorts = sc->sc_noport;
2010 		v = EREAD4(sc, EHCI_HCSPARAMS);
2011 		USETW(hubd.wHubCharacteristics,
2012 		    (EHCI_HCS_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_NO_SWITCH) |
2013 		    (EHCI_HCS_P_INDICATOR(v) ? UHD_PORT_IND : 0));
2014 		hubd.bPwrOn2PwrGood = 200; /* XXX can't find out? */
2015 		for (i = 0, l = sc->sc_noport; l > 0; i++, l -= 8, v >>= 8)
2016 			hubd.DeviceRemovable[i++] = 0; /* XXX can't find out? */
2017 		hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i;
2018 		l = min(len, hubd.bDescLength);
2019 		totlen = l;
2020 		memcpy(buf, &hubd, l);
2021 		break;
2022 	case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
2023 		if (len != 4) {
2024 			err = USBD_IOERROR;
2025 			goto ret;
2026 		}
2027 		memset(buf, 0, len); /* ? XXX */
2028 		totlen = len;
2029 		break;
2030 	case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
2031 		DPRINTFN(8,("ehci_root_ctrl_start: get port status i=%d\n",
2032 		    index));
2033 		if (index < 1 || index > sc->sc_noport) {
2034 			err = USBD_IOERROR;
2035 			goto ret;
2036 		}
2037 		if (len != 4) {
2038 			err = USBD_IOERROR;
2039 			goto ret;
2040 		}
2041 		v = EOREAD4(sc, EHCI_PORTSC(index));
2042 		DPRINTFN(8,("ehci_root_ctrl_start: port status=0x%04x\n", v));
2043 		i = UPS_HIGH_SPEED;
2044 		if (v & EHCI_PS_CS)	i |= UPS_CURRENT_CONNECT_STATUS;
2045 		if (v & EHCI_PS_PE)	i |= UPS_PORT_ENABLED;
2046 		if (v & EHCI_PS_SUSP)	i |= UPS_SUSPEND;
2047 		if (v & EHCI_PS_OCA)	i |= UPS_OVERCURRENT_INDICATOR;
2048 		if (v & EHCI_PS_PR)	i |= UPS_RESET;
2049 		if (v & EHCI_PS_PP)	i |= UPS_PORT_POWER;
2050 		USETW(ps.wPortStatus, i);
2051 		i = 0;
2052 		if (v & EHCI_PS_CSC)	i |= UPS_C_CONNECT_STATUS;
2053 		if (v & EHCI_PS_PEC)	i |= UPS_C_PORT_ENABLED;
2054 		if (v & EHCI_PS_OCC)	i |= UPS_C_OVERCURRENT_INDICATOR;
2055 		if (sc->sc_isreset)	i |= UPS_C_PORT_RESET;
2056 		USETW(ps.wPortChange, i);
2057 		l = min(len, sizeof(ps));
2058 		memcpy(buf, &ps, l);
2059 		totlen = l;
2060 		break;
2061 	case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
2062 		err = USBD_IOERROR;
2063 		goto ret;
2064 	case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
2065 		break;
2066 	case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
2067 		if (index < 1 || index > sc->sc_noport) {
2068 			err = USBD_IOERROR;
2069 			goto ret;
2070 		}
2071 		port = EHCI_PORTSC(index);
2072 		v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
2073 		switch(value) {
2074 		case UHF_PORT_ENABLE:
2075 			EOWRITE4(sc, port, v | EHCI_PS_PE);
2076 			break;
2077 		case UHF_PORT_SUSPEND:
2078 			EOWRITE4(sc, port, v | EHCI_PS_SUSP);
2079 			break;
2080 		case UHF_PORT_DISOWN_TO_1_1:
2081 			/* enter to Port Reset State */
2082 			v &= ~EHCI_PS_PE;
2083 			EOWRITE4(sc, port, v | EHCI_PS_PR);
2084 			ehci_disown(sc, index, 0);
2085 			break;
2086 		case UHF_PORT_RESET:
2087 			DPRINTFN(5,("ehci_root_ctrl_start: reset port %d\n",
2088 			    index));
2089 			if (EHCI_PS_IS_LOWSPEED(v)) {
2090 				/* Low speed device, give up ownership. */
2091 				ehci_disown(sc, index, 1);
2092 				break;
2093 			}
2094 			/* Start reset sequence. */
2095 			v &= ~ (EHCI_PS_PE | EHCI_PS_PR);
2096 			EOWRITE4(sc, port, v | EHCI_PS_PR);
2097 			/* Wait for reset to complete. */
2098 			usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY);
2099 			if (sc->sc_bus.dying) {
2100 				err = USBD_IOERROR;
2101 				goto ret;
2102 			}
2103 			/* Terminate reset sequence. */
2104 			v = EOREAD4(sc, port);
2105 			EOWRITE4(sc, port, v & ~EHCI_PS_PR);
2106 			/* Wait for HC to complete reset. */
2107 			usb_delay_ms(&sc->sc_bus, EHCI_PORT_RESET_COMPLETE);
2108 			if (sc->sc_bus.dying) {
2109 				err = USBD_IOERROR;
2110 				goto ret;
2111 			}
2112 			v = EOREAD4(sc, port);
2113 			DPRINTF(("ehci after reset, status=0x%08x\n", v));
2114 			if (v & EHCI_PS_PR) {
2115 				printf("%s: port reset timeout\n",
2116 				    sc->sc_bus.bdev.dv_xname);
2117 				err = USBD_IOERROR;
2118 				goto ret;
2119 			}
2120 			if (!(v & EHCI_PS_PE)) {
2121 				/* Not a high speed device, give up ownership.*/
2122 				ehci_disown(sc, index, 0);
2123 				break;
2124 			}
2125 			sc->sc_isreset = 1;
2126 			DPRINTF(("ehci port %d reset, status = 0x%08x\n",
2127 			    index, v));
2128 			break;
2129 		case UHF_PORT_POWER:
2130 			DPRINTFN(2,("ehci_root_ctrl_start: "
2131 			    "set port power %d\n", index));
2132 			EOWRITE4(sc, port, v | EHCI_PS_PP);
2133 			break;
2134 		case UHF_PORT_TEST:
2135 			DPRINTFN(2,("ehci_root_ctrl_start: "
2136 			    "set port test %d\n", index));
2137 			break;
2138 		case UHF_PORT_INDICATOR:
2139 			DPRINTFN(2,("ehci_root_ctrl_start: "
2140 			    "set port ind %d\n", index));
2141 			EOWRITE4(sc, port, v | EHCI_PS_PIC);
2142 			break;
2143 		default:
2144 			err = USBD_IOERROR;
2145 			goto ret;
2146 		}
2147 		break;
2148 	case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
2149 	case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
2150 	case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
2151 	case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
2152 		break;
2153 	default:
2154 		err = USBD_IOERROR;
2155 		goto ret;
2156 	}
2157 	xfer->actlen = totlen;
2158 	err = USBD_NORMAL_COMPLETION;
2159  ret:
2160 	xfer->status = err;
2161 	s = splusb();
2162 	usb_transfer_complete(xfer);
2163 	splx(s);
2164 	return (err);
2165 }
2166 
2167 void
2168 ehci_disown(struct ehci_softc *sc, int index, int lowspeed)
2169 {
2170 	int port;
2171 	u_int32_t v;
2172 
2173 	port = EHCI_PORTSC(index);
2174 	v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
2175 	EOWRITE4(sc, port, v | EHCI_PS_PO);
2176 }
2177 
2178 /* Abort a root control request. */
2179 void
2180 ehci_root_ctrl_abort(struct usbd_xfer *xfer)
2181 {
2182 	/* Nothing to do, all transfers are synchronous. */
2183 }
2184 
2185 /* Close the root pipe. */
2186 void
2187 ehci_root_ctrl_close(struct usbd_pipe *pipe)
2188 {
2189 	/* Nothing to do. */
2190 }
2191 
2192 void
2193 ehci_root_intr_done(struct usbd_xfer *xfer)
2194 {
2195 }
2196 
2197 usbd_status
2198 ehci_root_intr_transfer(struct usbd_xfer *xfer)
2199 {
2200 	usbd_status err;
2201 
2202 	/* Insert last in queue. */
2203 	err = usb_insert_transfer(xfer);
2204 	if (err)
2205 		return (err);
2206 
2207 	/* Pipe isn't running, start first */
2208 	return (ehci_root_intr_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
2209 }
2210 
2211 usbd_status
2212 ehci_root_intr_start(struct usbd_xfer *xfer)
2213 {
2214 	struct ehci_softc *sc = (struct ehci_softc *)xfer->device->bus;
2215 
2216 	if (sc->sc_bus.dying)
2217 		return (USBD_IOERROR);
2218 
2219 	sc->sc_intrxfer = xfer;
2220 
2221 	return (USBD_IN_PROGRESS);
2222 }
2223 
2224 void
2225 ehci_root_intr_abort(struct usbd_xfer *xfer)
2226 {
2227 	struct ehci_softc *sc = (struct ehci_softc *)xfer->device->bus;
2228 	int s;
2229 
2230 	sc->sc_intrxfer = NULL;
2231 
2232 	xfer->status = USBD_CANCELLED;
2233 	s = splusb();
2234 	usb_transfer_complete(xfer);
2235 	splx(s);
2236 }
2237 
2238 void
2239 ehci_root_intr_close(struct usbd_pipe *pipe)
2240 {
2241 }
2242 
2243 void
2244 ehci_root_ctrl_done(struct usbd_xfer *xfer)
2245 {
2246 }
2247 
2248 struct ehci_soft_qh *
2249 ehci_alloc_sqh(struct ehci_softc *sc)
2250 {
2251 	struct ehci_soft_qh *sqh = NULL;
2252 	usbd_status err;
2253 	int i, offs;
2254 	struct usb_dma dma;
2255 	int s;
2256 
2257 	s = splusb();
2258 	if (sc->sc_freeqhs == NULL) {
2259 		DPRINTFN(2, ("ehci_alloc_sqh: allocating chunk\n"));
2260 		err = usb_allocmem(&sc->sc_bus, EHCI_SQH_SIZE * EHCI_SQH_CHUNK,
2261 		    EHCI_PAGE_SIZE, &dma);
2262 		if (err)
2263 			goto out;
2264 		for (i = 0; i < EHCI_SQH_CHUNK; i++) {
2265 			offs = i * EHCI_SQH_SIZE;
2266 			sqh = KERNADDR(&dma, offs);
2267 			sqh->physaddr = DMAADDR(&dma, offs);
2268 			sqh->dma = dma;
2269 			sqh->offs = offs;
2270 			sqh->next = sc->sc_freeqhs;
2271 			sc->sc_freeqhs = sqh;
2272 		}
2273 	}
2274 	sqh = sc->sc_freeqhs;
2275 	sc->sc_freeqhs = sqh->next;
2276 	memset(&sqh->qh, 0, sizeof(struct ehci_qh));
2277 	sqh->next = NULL;
2278 	sqh->prev = NULL;
2279 
2280 out:
2281 	splx(s);
2282 	return (sqh);
2283 }
2284 
2285 void
2286 ehci_free_sqh(struct ehci_softc *sc, struct ehci_soft_qh *sqh)
2287 {
2288 	int s;
2289 
2290 	s = splusb();
2291 	sqh->next = sc->sc_freeqhs;
2292 	sc->sc_freeqhs = sqh;
2293 	splx(s);
2294 }
2295 
2296 struct ehci_soft_qtd *
2297 ehci_alloc_sqtd(struct ehci_softc *sc)
2298 {
2299 	struct ehci_soft_qtd *sqtd = NULL;
2300 	usbd_status err;
2301 	int i, offs;
2302 	struct usb_dma dma;
2303 	int s;
2304 
2305 	s = splusb();
2306 	if (sc->sc_freeqtds == NULL) {
2307 		DPRINTFN(2, ("ehci_alloc_sqtd: allocating chunk\n"));
2308 		err = usb_allocmem(&sc->sc_bus, EHCI_SQTD_SIZE*EHCI_SQTD_CHUNK,
2309 		    EHCI_PAGE_SIZE, &dma);
2310 		if (err)
2311 			goto out;
2312 		for(i = 0; i < EHCI_SQTD_CHUNK; i++) {
2313 			offs = i * EHCI_SQTD_SIZE;
2314 			sqtd = KERNADDR(&dma, offs);
2315 			sqtd->physaddr = DMAADDR(&dma, offs);
2316 			sqtd->dma = dma;
2317 			sqtd->offs = offs;
2318 			sqtd->nextqtd = sc->sc_freeqtds;
2319 			sc->sc_freeqtds = sqtd;
2320 		}
2321 	}
2322 
2323 	sqtd = sc->sc_freeqtds;
2324 	sc->sc_freeqtds = sqtd->nextqtd;
2325 	memset(&sqtd->qtd, 0, sizeof(struct ehci_qtd));
2326 	sqtd->nextqtd = NULL;
2327 
2328 out:
2329 	splx(s);
2330 	return (sqtd);
2331 }
2332 
2333 void
2334 ehci_free_sqtd(struct ehci_softc *sc, struct ehci_soft_qtd *sqtd)
2335 {
2336 	int s;
2337 
2338 	s = splusb();
2339 	sqtd->nextqtd = sc->sc_freeqtds;
2340 	sc->sc_freeqtds = sqtd;
2341 	splx(s);
2342 }
2343 
2344 usbd_status
2345 ehci_alloc_sqtd_chain(struct ehci_softc *sc, u_int alen, struct usbd_xfer *xfer,
2346     struct ehci_soft_qtd **sp, struct ehci_soft_qtd **ep)
2347 {
2348 	struct ehci_soft_qtd *next, *cur;
2349 	ehci_physaddr_t dataphys, dataphyspage, dataphyslastpage, nextphys;
2350 	u_int32_t qtdstatus;
2351 	u_int len, curlen;
2352 	int mps, i, iscontrol, forceshort;
2353 	int rd = usbd_xfer_isread(xfer);
2354 	struct usb_dma *dma = &xfer->dmabuf;
2355 
2356 	DPRINTFN(alen<4*4096,("ehci_alloc_sqtd_chain: start len=%d\n", alen));
2357 
2358 	len = alen;
2359 	iscontrol = UE_GET_XFERTYPE(xfer->pipe->endpoint->edesc->bmAttributes) ==
2360 	    UE_CONTROL;
2361 
2362 	dataphys = DMAADDR(dma, 0);
2363 	dataphyslastpage = EHCI_PAGE(dataphys + len - 1);
2364 	qtdstatus = EHCI_QTD_ACTIVE |
2365 	    EHCI_QTD_SET_PID(rd ? EHCI_QTD_PID_IN : EHCI_QTD_PID_OUT) |
2366 	    EHCI_QTD_SET_CERR(3); /* IOC and BYTES set below */
2367 	mps = UGETW(xfer->pipe->endpoint->edesc->wMaxPacketSize);
2368 	forceshort = ((xfer->flags & USBD_FORCE_SHORT_XFER) || len == 0) &&
2369 	    len % mps == 0;
2370 	/*
2371 	 * The control transfer data stage always starts with a toggle of 1.
2372 	 * For other transfers we let the hardware track the toggle state.
2373 	 */
2374 	if (iscontrol)
2375 		qtdstatus |= EHCI_QTD_SET_TOGGLE(1);
2376 
2377 	cur = ehci_alloc_sqtd(sc);
2378 	*sp = cur;
2379 	if (cur == NULL)
2380 		goto nomem;
2381 
2382 	usb_syncmem(dma, 0, alen,
2383 	    rd ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
2384 	for (;;) {
2385 		dataphyspage = EHCI_PAGE(dataphys);
2386 		/* The EHCI hardware can handle at most 5 pages. */
2387 		if (dataphyslastpage - dataphyspage <
2388 		    EHCI_QTD_NBUFFERS * EHCI_PAGE_SIZE) {
2389 			/* we can handle it in this QTD */
2390 			curlen = len;
2391 		} else {
2392 			/* must use multiple TDs, fill as much as possible. */
2393 			curlen = EHCI_QTD_NBUFFERS * EHCI_PAGE_SIZE -
2394 				 EHCI_PAGE_OFFSET(dataphys);
2395 #ifdef DIAGNOSTIC
2396 			if (curlen > len) {
2397 				printf("ehci_alloc_sqtd_chain: curlen=%u "
2398 				    "len=%u offs=0x%x\n", curlen, len,
2399 				    EHCI_PAGE_OFFSET(dataphys));
2400 				printf("lastpage=0x%x page=0x%x phys=0x%x\n",
2401 				    dataphyslastpage, dataphyspage, dataphys);
2402 				curlen = len;
2403 			}
2404 #endif
2405 			/* the length must be a multiple of the max size */
2406 			curlen -= curlen % mps;
2407 			DPRINTFN(1,("ehci_alloc_sqtd_chain: multiple QTDs, "
2408 			    "curlen=%u\n", curlen));
2409 #ifdef DIAGNOSTIC
2410 			if (curlen == 0)
2411 				panic("ehci_alloc_std: curlen == 0");
2412 #endif
2413 		}
2414 
2415 		DPRINTFN(4,("ehci_alloc_sqtd_chain: dataphys=0x%08x "
2416 		    "dataphyslastpage=0x%08x len=%u curlen=%u\n",
2417 		    dataphys, dataphyslastpage, len, curlen));
2418 		len -= curlen;
2419 
2420 		/*
2421 		 * Allocate another transfer if there's more data left,
2422 		 * or if force last short transfer flag is set and we're
2423 		 * allocating a multiple of the max packet size.
2424 		 */
2425 		if (len != 0 || forceshort) {
2426 			next = ehci_alloc_sqtd(sc);
2427 			if (next == NULL)
2428 				goto nomem;
2429 			nextphys = htole32(next->physaddr);
2430 		} else {
2431 			next = NULL;
2432 			nextphys = htole32(EHCI_LINK_TERMINATE);
2433 		}
2434 
2435 		for (i = 0; i * EHCI_PAGE_SIZE <
2436 		    curlen + EHCI_PAGE_OFFSET(dataphys); i++) {
2437 			ehci_physaddr_t a = dataphys + i * EHCI_PAGE_SIZE;
2438 			if (i != 0) /* use offset only in first buffer */
2439 				a = EHCI_PAGE(a);
2440 #ifdef DIAGNOSTIC
2441 			if (i >= EHCI_QTD_NBUFFERS) {
2442 				printf("ehci_alloc_sqtd_chain: i=%d\n", i);
2443 				goto nomem;
2444 			}
2445 #endif
2446 			cur->qtd.qtd_buffer[i] = htole32(a);
2447 			cur->qtd.qtd_buffer_hi[i] = 0;
2448 		}
2449 		cur->nextqtd = next;
2450 		cur->qtd.qtd_next = cur->qtd.qtd_altnext = nextphys;
2451 		cur->qtd.qtd_status = htole32(qtdstatus |
2452 		    EHCI_QTD_SET_BYTES(curlen));
2453 		cur->len = curlen;
2454 		DPRINTFN(10,("ehci_alloc_sqtd_chain: cbp=0x%08x end=0x%08x\n",
2455 		    dataphys, dataphys + curlen));
2456 		DPRINTFN(10,("ehci_alloc_sqtd_chain: curlen=%u\n", curlen));
2457 		if (iscontrol) {
2458 			/*
2459 			 * adjust the toggle based on the number of packets
2460 			 * in this qtd
2461 			 */
2462 			if ((((curlen + mps - 1) / mps) & 1) || curlen == 0)
2463 				qtdstatus ^= EHCI_QTD_TOGGLE_MASK;
2464 		}
2465 		if (len == 0) {
2466 			if (! forceshort)
2467 				break;
2468 			forceshort = 0;
2469 		}
2470 		usb_syncmem(&cur->dma, cur->offs, sizeof(cur->qtd),
2471 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2472 		DPRINTFN(10,("ehci_alloc_sqtd_chain: extend chain\n"));
2473 		dataphys += curlen;
2474 		cur = next;
2475 	}
2476 	cur->qtd.qtd_status |= htole32(EHCI_QTD_IOC);
2477 	usb_syncmem(&cur->dma, cur->offs, sizeof(cur->qtd),
2478 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2479 	*ep = cur;
2480 
2481 	DPRINTFN(10,("ehci_alloc_sqtd_chain: return sqtd=%p sqtdend=%p\n",
2482 	    *sp, *ep));
2483 
2484 	return (USBD_NORMAL_COMPLETION);
2485 
2486  nomem:
2487 	/* XXX free chain */
2488 	DPRINTFN(-1,("ehci_alloc_sqtd_chain: no memory\n"));
2489 	return (USBD_NOMEM);
2490 }
2491 
2492 void
2493 ehci_free_sqtd_chain(struct ehci_softc *sc, struct ehci_xfer *ex)
2494 {
2495 	struct ehci_pipe *epipe = (struct ehci_pipe *)ex->xfer.pipe;
2496 	struct ehci_soft_qtd *sqtd, *next;
2497 
2498 	DPRINTFN(10,("ehci_free_sqtd_chain: sqtd=%p\n", ex->sqtdstart));
2499 
2500 	for (sqtd = ex->sqtdstart; sqtd != NULL; sqtd = next) {
2501 		next = sqtd->nextqtd;
2502 		ehci_free_sqtd(sc, sqtd);
2503 	}
2504 	ex->sqtdstart = ex->sqtdend = NULL;
2505 	epipe->sqh->sqtd = NULL;
2506 }
2507 
2508 struct ehci_soft_itd *
2509 ehci_alloc_itd(struct ehci_softc *sc)
2510 {
2511 	struct ehci_soft_itd *itd, *freeitd;
2512 	usbd_status err;
2513 	int i, s, offs, frindex, previndex;
2514 	struct usb_dma dma;
2515 
2516 	s = splusb();
2517 
2518 	/* Find an itd that wasn't freed this frame or last frame. This can
2519 	 * discard itds that were freed before frindex wrapped around
2520 	 * XXX - can this lead to thrashing? Could fix by enabling wrap-around
2521 	 *       interrupt and fiddling with list when that happens */
2522 	frindex = (EOREAD4(sc, EHCI_FRINDEX) + 1) >> 3;
2523 	previndex = (frindex != 0) ? frindex - 1 : sc->sc_flsize;
2524 
2525 	freeitd = NULL;
2526 	LIST_FOREACH(itd, &sc->sc_freeitds, u.free_list) {
2527 		if (itd == NULL)
2528 			break;
2529 		if (itd->slot != frindex && itd->slot != previndex) {
2530 			freeitd = itd;
2531 			break;
2532 		}
2533 	}
2534 
2535 	if (freeitd == NULL) {
2536 		err = usb_allocmem(&sc->sc_bus, EHCI_ITD_SIZE * EHCI_ITD_CHUNK,
2537 		    EHCI_PAGE_SIZE, &dma);
2538 		if (err) {
2539 			splx(s);
2540 			return (NULL);
2541 		}
2542 
2543 		for (i = 0; i < EHCI_ITD_CHUNK; i++) {
2544 			offs = i * EHCI_ITD_SIZE;
2545 			itd = KERNADDR(&dma, offs);
2546 			itd->physaddr = DMAADDR(&dma, offs);
2547 			itd->dma = dma;
2548 			itd->offs = offs;
2549 			LIST_INSERT_HEAD(&sc->sc_freeitds, itd, u.free_list);
2550 		}
2551 		freeitd = LIST_FIRST(&sc->sc_freeitds);
2552 	}
2553 
2554 	itd = freeitd;
2555 	LIST_REMOVE(itd, u.free_list);
2556 	memset(&itd->itd, 0, sizeof(struct ehci_itd));
2557 	usb_syncmem(&itd->dma, itd->offs + offsetof(struct ehci_itd, itd_next),
2558 	    sizeof(itd->itd.itd_next), BUS_DMASYNC_PREWRITE |
2559 	    BUS_DMASYNC_PREREAD);
2560 
2561 	itd->u.frame_list.next = NULL;
2562 	itd->u.frame_list.prev = NULL;
2563 	itd->xfer_next = NULL;
2564 	itd->slot = 0;
2565 	splx(s);
2566 
2567 	return (itd);
2568 }
2569 
2570 void
2571 ehci_free_itd(struct ehci_softc *sc, struct ehci_soft_itd *itd)
2572 {
2573 	int s;
2574 
2575 	s = splusb();
2576 	LIST_INSERT_HEAD(&sc->sc_freeitds, itd, u.free_list);
2577 	splx(s);
2578 }
2579 
2580 /*
2581  * Close a reqular pipe.
2582  * Assumes that there are no pending transactions.
2583  */
2584 void
2585 ehci_close_pipe(struct usbd_pipe *pipe)
2586 {
2587 	struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
2588 	struct ehci_softc *sc = (struct ehci_softc *)pipe->device->bus;
2589 	struct ehci_soft_qh *sqh = epipe->sqh;
2590 	int s;
2591 
2592 	s = splusb();
2593 	ehci_rem_qh(sc, sqh);
2594 	splx(s);
2595 	pipe->endpoint->savedtoggle =
2596 	    EHCI_QTD_GET_TOGGLE(letoh32(sqh->qh.qh_qtd.qtd_status));
2597 	ehci_free_sqh(sc, epipe->sqh);
2598 }
2599 
2600 /*
2601  * Abort a device request.
2602  * If this routine is called at splusb() it guarantees that the request
2603  * will be removed from the hardware scheduling and that the callback
2604  * for it will be called with USBD_CANCELLED status.
2605  * It's impossible to guarantee that the requested transfer will not
2606  * have happened since the hardware runs concurrently.
2607  * If the transaction has already happened we rely on the ordinary
2608  * interrupt processing to process it.
2609  */
2610 void
2611 ehci_abort_xfer(struct usbd_xfer *xfer, usbd_status status)
2612 {
2613 	struct ehci_softc *sc = (struct ehci_softc *)xfer->device->bus;
2614 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2615 	struct ehci_xfer *ex = (struct ehci_xfer*)xfer;
2616 	struct ehci_soft_qh *sqh = epipe->sqh;
2617 	struct ehci_soft_qtd *sqtd;
2618 	int s;
2619 
2620 	if (sc->sc_bus.dying || xfer->status == USBD_NOT_STARTED) {
2621 		s = splusb();
2622 		if (xfer->status != USBD_NOT_STARTED)
2623 			TAILQ_REMOVE(&sc->sc_intrhead, ex, inext);
2624 		xfer->status = status;	/* make software ignore it */
2625 		timeout_del(&xfer->timeout_handle);
2626 		usb_rem_task(xfer->device, &xfer->abort_task);
2627 #ifdef DIAGNOSTIC
2628 		ex->isdone = 1;
2629 #endif
2630 		usb_transfer_complete(xfer);
2631 		splx(s);
2632 		return;
2633 	}
2634 
2635 	if (xfer->device->bus->intr_context)
2636 		panic("ehci_abort_xfer: not in process context");
2637 
2638 	/*
2639 	 * If an abort is already in progress then just wait for it to
2640 	 * complete and return.
2641 	 */
2642 	if (ex->ehci_xfer_flags & EHCI_XFER_ABORTING) {
2643 		DPRINTFN(2, ("ehci_abort_xfer: already aborting\n"));
2644 		/* No need to wait if we're aborting from a timeout. */
2645 		if (status == USBD_TIMEOUT)
2646 			return;
2647 		/* Override the status which might be USBD_TIMEOUT. */
2648 		xfer->status = status;
2649 		DPRINTFN(2, ("ehci_abort_xfer: waiting for abort to finish\n"));
2650 		ex->ehci_xfer_flags |= EHCI_XFER_ABORTWAIT;
2651 		while (ex->ehci_xfer_flags & EHCI_XFER_ABORTING)
2652 			tsleep_nsec(&ex->ehci_xfer_flags, PZERO, "ehciaw", INFSLP);
2653 		return;
2654 	}
2655 
2656 	/*
2657 	 * Step 1: Make interrupt routine and timeouts ignore xfer.
2658 	 */
2659 	s = splusb();
2660 	ex->ehci_xfer_flags |= EHCI_XFER_ABORTING;
2661 	xfer->status = status;	/* make software ignore it */
2662 	TAILQ_REMOVE(&sc->sc_intrhead, ex, inext);
2663 	timeout_del(&xfer->timeout_handle);
2664 	usb_rem_task(xfer->device, &xfer->abort_task);
2665 	splx(s);
2666 
2667 	/*
2668 	 * Step 2: Deactivate all of the qTDs that we will be removing,
2669 	 * otherwise the queue head may go active again.
2670 	 */
2671 	usb_syncmem(&sqh->dma,
2672 	    sqh->offs + offsetof(struct ehci_qh, qh_qtd.qtd_status),
2673 	    sizeof(sqh->qh.qh_qtd.qtd_status),
2674 	    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2675 	sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
2676 	usb_syncmem(&sqh->dma,
2677 	    sqh->offs + offsetof(struct ehci_qh, qh_qtd.qtd_status),
2678 	    sizeof(sqh->qh.qh_qtd.qtd_status),
2679 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2680 
2681 	for (sqtd = ex->sqtdstart; sqtd != NULL; sqtd = sqtd->nextqtd) {
2682 		usb_syncmem(&sqtd->dma,
2683 		    sqtd->offs + offsetof(struct ehci_qtd, qtd_status),
2684 		    sizeof(sqtd->qtd.qtd_status),
2685 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2686 		sqtd->qtd.qtd_status = htole32(EHCI_QTD_HALTED);
2687 		usb_syncmem(&sqtd->dma,
2688 		    sqtd->offs + offsetof(struct ehci_qtd, qtd_status),
2689 		    sizeof(sqtd->qtd.qtd_status),
2690 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2691 	}
2692 	ehci_sync_hc(sc);
2693 
2694 	/*
2695 	 * Step 3: Make sure the soft interrupt routine has run. This
2696 	 * should remove any completed items off the queue.
2697 	 * The hardware has no reference to completed items (TDs).
2698 	 * It's safe to remove them at any time.
2699 	 */
2700 	s = splusb();
2701 	sc->sc_softwake = 1;
2702 	usb_schedsoftintr(&sc->sc_bus);
2703 	tsleep_nsec(&sc->sc_softwake, PZERO, "ehciab", INFSLP);
2704 
2705 #ifdef DIAGNOSTIC
2706 	ex->isdone = 1;
2707 #endif
2708 	/* Do the wakeup first to avoid touching the xfer after the callback. */
2709 	ex->ehci_xfer_flags &= ~EHCI_XFER_ABORTING;
2710 	if (ex->ehci_xfer_flags & EHCI_XFER_ABORTWAIT) {
2711 		ex->ehci_xfer_flags &= ~EHCI_XFER_ABORTWAIT;
2712 		wakeup(&ex->ehci_xfer_flags);
2713 	}
2714 	usb_transfer_complete(xfer);
2715 
2716 	splx(s);
2717 }
2718 
2719 void
2720 ehci_abort_isoc_xfer(struct usbd_xfer *xfer, usbd_status status)
2721 {
2722 	struct ehci_softc *sc = (struct ehci_softc *)xfer->device->bus;
2723 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
2724 	ehci_isoc_trans_t trans_status;
2725 	struct ehci_soft_itd *itd;
2726 	int i;
2727 
2728 	splsoftassert(IPL_SOFTUSB);
2729 
2730 	if (sc->sc_bus.dying || xfer->status == USBD_NOT_STARTED) {
2731 		if (xfer->status != USBD_NOT_STARTED)
2732 			TAILQ_REMOVE(&sc->sc_intrhead, ex, inext);
2733 		xfer->status = status;
2734 		timeout_del(&xfer->timeout_handle);
2735 		usb_rem_task(xfer->device, &xfer->abort_task);
2736 		usb_transfer_complete(xfer);
2737 		return;
2738 	}
2739 
2740 	/* Transfer is already done. */
2741 	if (xfer->status != USBD_IN_PROGRESS) {
2742 		DPRINTF(("%s: already done \n", __func__));
2743 		return;
2744 	}
2745 
2746 
2747 #ifdef DIAGNOSTIC
2748 	ex->isdone = 1;
2749 #endif
2750 	xfer->status = status;
2751 	TAILQ_REMOVE(&sc->sc_intrhead, ex, inext);
2752 	timeout_del(&xfer->timeout_handle);
2753 	usb_rem_task(xfer->device, &xfer->abort_task);
2754 
2755 	if (xfer->device->speed == USB_SPEED_HIGH) {
2756 		for (itd = ex->itdstart; itd != NULL; itd = itd->xfer_next) {
2757 			usb_syncmem(&itd->dma,
2758 			    itd->offs + offsetof(struct ehci_itd, itd_ctl),
2759 			    sizeof(itd->itd.itd_ctl),
2760 			    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2761 
2762 			for (i = 0; i < 8; i++) {
2763 				trans_status = le32toh(itd->itd.itd_ctl[i]);
2764 				trans_status &= ~EHCI_ITD_ACTIVE;
2765 				itd->itd.itd_ctl[i] = htole32(trans_status);
2766 			}
2767 
2768 			usb_syncmem(&itd->dma,
2769 			    itd->offs + offsetof(struct ehci_itd, itd_ctl),
2770 			    sizeof(itd->itd.itd_ctl),
2771 			    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2772 		}
2773 	} else {
2774 		for (itd = ex->itdstart; itd != NULL; itd = itd->xfer_next) {
2775 			usb_syncmem(&itd->dma,
2776 			    itd->offs + offsetof(struct ehci_sitd, sitd_trans),
2777 			    sizeof(itd->sitd.sitd_trans),
2778 			    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2779 
2780 			trans_status = le32toh(itd->sitd.sitd_trans);
2781 			trans_status &= ~EHCI_SITD_ACTIVE;
2782 			itd->sitd.sitd_trans = htole32(trans_status);
2783 
2784 			usb_syncmem(&itd->dma,
2785 			    itd->offs + offsetof(struct ehci_sitd, sitd_trans),
2786 			    sizeof(itd->sitd.sitd_trans),
2787 			    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2788 		}
2789 	}
2790 
2791 	sc->sc_softwake = 1;
2792 	usb_schedsoftintr(&sc->sc_bus);
2793 	tsleep_nsec(&sc->sc_softwake, PZERO, "ehciab", INFSLP);
2794 
2795 	usb_transfer_complete(xfer);
2796 }
2797 
2798 void
2799 ehci_timeout(void *addr)
2800 {
2801 	struct usbd_xfer *xfer = addr;
2802 	struct ehci_softc *sc = (struct ehci_softc *)xfer->device->bus;
2803 
2804 	if (sc->sc_bus.dying) {
2805 		ehci_timeout_task(addr);
2806 		return;
2807 	}
2808 
2809 	usb_init_task(&xfer->abort_task, ehci_timeout_task, addr,
2810 	    USB_TASK_TYPE_ABORT);
2811 	usb_add_task(xfer->device, &xfer->abort_task);
2812 }
2813 
2814 void
2815 ehci_timeout_task(void *addr)
2816 {
2817 	struct usbd_xfer *xfer = addr;
2818 	int s;
2819 
2820 	s = splusb();
2821 	ehci_abort_xfer(xfer, USBD_TIMEOUT);
2822 	splx(s);
2823 }
2824 
2825 /*
2826  * Some EHCI chips from VIA / ATI seem to trigger interrupts before writing
2827  * back the qTD status, or miss signalling occasionally under heavy load.
2828  * If the host machine is too fast, we can miss transaction completion - when
2829  * we scan the active list the transaction still seems to be active. This
2830  * generally exhibits itself as a umass stall that never recovers.
2831  *
2832  * We work around this behaviour by setting up this callback after any softintr
2833  * that completes with transactions still pending, giving us another chance to
2834  * check for completion after the writeback has taken place.
2835  */
2836 void
2837 ehci_intrlist_timeout(void *arg)
2838 {
2839 	struct ehci_softc *sc = arg;
2840 	int s;
2841 
2842 	if (sc->sc_bus.dying)
2843 		return;
2844 
2845 	s = splusb();
2846 	DPRINTFN(1, ("ehci_intrlist_timeout\n"));
2847 	usb_schedsoftintr(&sc->sc_bus);
2848 	splx(s);
2849 }
2850 
2851 usbd_status
2852 ehci_device_ctrl_transfer(struct usbd_xfer *xfer)
2853 {
2854 	usbd_status err;
2855 
2856 	/* Insert last in queue. */
2857 	err = usb_insert_transfer(xfer);
2858 	if (err)
2859 		return (err);
2860 
2861 	/* Pipe isn't running, start first */
2862 	return (ehci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
2863 }
2864 
2865 usbd_status
2866 ehci_device_ctrl_start(struct usbd_xfer *xfer)
2867 {
2868 	struct ehci_softc *sc = (struct ehci_softc *)xfer->device->bus;
2869 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2870 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
2871 	usb_device_request_t *req = &xfer->request;
2872 	struct ehci_soft_qtd *setup, *stat, *next;
2873 	struct ehci_soft_qh *sqh;
2874 	u_int len = UGETW(req->wLength);
2875 	usbd_status err;
2876 	int s;
2877 
2878 	KASSERT(xfer->rqflags & URQ_REQUEST);
2879 
2880 	if (sc->sc_bus.dying)
2881 		return (USBD_IOERROR);
2882 
2883 	setup = ehci_alloc_sqtd(sc);
2884 	if (setup == NULL) {
2885 		err = USBD_NOMEM;
2886 		goto bad1;
2887 	}
2888 	stat = ehci_alloc_sqtd(sc);
2889 	if (stat == NULL) {
2890 		err = USBD_NOMEM;
2891 		goto bad2;
2892 	}
2893 
2894 	sqh = epipe->sqh;
2895 
2896 	/* Set up data transaction */
2897 	if (len != 0) {
2898 		struct ehci_soft_qtd *end;
2899 
2900 		err = ehci_alloc_sqtd_chain(sc, len, xfer, &next, &end);
2901 		if (err)
2902 			goto bad3;
2903 		end->qtd.qtd_status &= htole32(~EHCI_QTD_IOC);
2904 		end->nextqtd = stat;
2905 		end->qtd.qtd_next =
2906 		    end->qtd.qtd_altnext = htole32(stat->physaddr);
2907 		usb_syncmem(&end->dma, end->offs, sizeof(end->qtd),
2908 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2909 	} else {
2910 		next = stat;
2911 	}
2912 
2913 	memcpy(KERNADDR(&epipe->u.ctl.reqdma, 0), req, sizeof(*req));
2914 	usb_syncmem(&epipe->u.ctl.reqdma, 0, sizeof *req, BUS_DMASYNC_PREWRITE);
2915 
2916 	/* Clear toggle */
2917 	setup->qtd.qtd_status = htole32(
2918 	    EHCI_QTD_ACTIVE |
2919 	    EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
2920 	    EHCI_QTD_SET_CERR(3) |
2921 	    EHCI_QTD_SET_TOGGLE(0) |
2922 	    EHCI_QTD_SET_BYTES(sizeof(*req)));
2923 	setup->qtd.qtd_buffer[0] = htole32(DMAADDR(&epipe->u.ctl.reqdma, 0));
2924 	setup->qtd.qtd_buffer_hi[0] = 0;
2925 	setup->nextqtd = next;
2926 	setup->qtd.qtd_next = setup->qtd.qtd_altnext = htole32(next->physaddr);
2927 	setup->len = sizeof(*req);
2928 	usb_syncmem(&setup->dma, setup->offs, sizeof(setup->qtd),
2929 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2930 
2931 	stat->qtd.qtd_status = htole32(
2932 	    EHCI_QTD_ACTIVE |
2933 	    EHCI_QTD_SET_PID(usbd_xfer_isread(xfer) ?
2934 		EHCI_QTD_PID_OUT : EHCI_QTD_PID_IN) |
2935 	    EHCI_QTD_SET_CERR(3) |
2936 	    EHCI_QTD_SET_TOGGLE(1) |
2937 	    EHCI_QTD_IOC);
2938 	stat->qtd.qtd_buffer[0] = 0; /* XXX not needed? */
2939 	stat->qtd.qtd_buffer_hi[0] = 0; /* XXX not needed? */
2940 	stat->nextqtd = NULL;
2941 	stat->qtd.qtd_next = stat->qtd.qtd_altnext = htole32(EHCI_LINK_TERMINATE);
2942 	stat->len = 0;
2943 	usb_syncmem(&stat->dma, stat->offs, sizeof(stat->qtd),
2944 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2945 
2946 	ex->sqtdstart = setup;
2947 	ex->sqtdend = stat;
2948 #ifdef DIAGNOSTIC
2949 	if (!ex->isdone) {
2950 		printf("%s: not done, ex=%p\n", __func__, ex);
2951 	}
2952 	ex->isdone = 0;
2953 #endif
2954 
2955 	/* Insert qTD in QH list. */
2956 	s = splusb();
2957 	ehci_set_qh_qtd(sqh, setup);
2958 	if (xfer->timeout && !sc->sc_bus.use_polling) {
2959 		timeout_del(&xfer->timeout_handle);
2960 		timeout_set(&xfer->timeout_handle, ehci_timeout, xfer);
2961 		timeout_add_msec(&xfer->timeout_handle, xfer->timeout);
2962 	}
2963 	TAILQ_INSERT_TAIL(&sc->sc_intrhead, ex, inext);
2964 	xfer->status = USBD_IN_PROGRESS;
2965 	splx(s);
2966 
2967 	return (USBD_IN_PROGRESS);
2968 
2969  bad3:
2970 	ehci_free_sqtd(sc, stat);
2971  bad2:
2972 	ehci_free_sqtd(sc, setup);
2973  bad1:
2974 	xfer->status = err;
2975 	usb_transfer_complete(xfer);
2976 	return (err);
2977 }
2978 
2979 void
2980 ehci_device_ctrl_done(struct usbd_xfer *xfer)
2981 {
2982 	struct ehci_softc *sc = (struct ehci_softc *)xfer->device->bus;
2983 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
2984 
2985 	KASSERT(xfer->rqflags & URQ_REQUEST);
2986 
2987 	if (xfer->status != USBD_NOMEM) {
2988 		ehci_free_sqtd_chain(sc, ex);
2989 	}
2990 }
2991 
2992 void
2993 ehci_device_ctrl_abort(struct usbd_xfer *xfer)
2994 {
2995 	ehci_abort_xfer(xfer, USBD_CANCELLED);
2996 }
2997 
2998 void
2999 ehci_device_ctrl_close(struct usbd_pipe *pipe)
3000 {
3001 	ehci_close_pipe(pipe);
3002 }
3003 
3004 usbd_status
3005 ehci_device_bulk_transfer(struct usbd_xfer *xfer)
3006 {
3007 	usbd_status err;
3008 
3009 	/* Insert last in queue. */
3010 	err = usb_insert_transfer(xfer);
3011 	if (err)
3012 		return (err);
3013 
3014 	/* Pipe isn't running, start first */
3015 	return (ehci_device_bulk_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
3016 }
3017 
3018 usbd_status
3019 ehci_device_bulk_start(struct usbd_xfer *xfer)
3020 {
3021 	struct ehci_softc *sc = (struct ehci_softc *)xfer->device->bus;
3022 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3023 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
3024 	struct ehci_soft_qtd *data, *dataend;
3025 	struct ehci_soft_qh *sqh;
3026 	usbd_status err;
3027 	int s;
3028 
3029 	KASSERT(!(xfer->rqflags & URQ_REQUEST));
3030 
3031 	if (sc->sc_bus.dying)
3032 		return (USBD_IOERROR);
3033 
3034 	sqh = epipe->sqh;
3035 
3036 	err = ehci_alloc_sqtd_chain(sc, xfer->length, xfer, &data, &dataend);
3037 	if (err) {
3038 		xfer->status = err;
3039 		usb_transfer_complete(xfer);
3040 		return (err);
3041 	}
3042 
3043 	/* Set up interrupt info. */
3044 	ex->sqtdstart = data;
3045 	ex->sqtdend = dataend;
3046 #ifdef DIAGNOSTIC
3047 	if (!ex->isdone) {
3048 		printf("ehci_device_bulk_start: not done, ex=%p\n", ex);
3049 	}
3050 	ex->isdone = 0;
3051 #endif
3052 
3053 	s = splusb();
3054 	ehci_set_qh_qtd(sqh, data);
3055 	if (xfer->timeout && !sc->sc_bus.use_polling) {
3056 		timeout_del(&xfer->timeout_handle);
3057 		timeout_set(&xfer->timeout_handle, ehci_timeout, xfer);
3058 		timeout_add_msec(&xfer->timeout_handle, xfer->timeout);
3059 	}
3060 	TAILQ_INSERT_TAIL(&sc->sc_intrhead, ex, inext);
3061 	xfer->status = USBD_IN_PROGRESS;
3062 	splx(s);
3063 
3064 	return (USBD_IN_PROGRESS);
3065 }
3066 
3067 void
3068 ehci_device_bulk_abort(struct usbd_xfer *xfer)
3069 {
3070 	ehci_abort_xfer(xfer, USBD_CANCELLED);
3071 }
3072 
3073 /*
3074  * Close a device bulk pipe.
3075  */
3076 void
3077 ehci_device_bulk_close(struct usbd_pipe *pipe)
3078 {
3079 	ehci_close_pipe(pipe);
3080 }
3081 
3082 void
3083 ehci_device_bulk_done(struct usbd_xfer *xfer)
3084 {
3085 	struct ehci_softc *sc = (struct ehci_softc *)xfer->device->bus;
3086 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
3087 
3088 	if (xfer->status != USBD_NOMEM) {
3089 		ehci_free_sqtd_chain(sc, ex);
3090 	}
3091 }
3092 
3093 usbd_status
3094 ehci_device_setintr(struct ehci_softc *sc, struct ehci_soft_qh *sqh, int ival)
3095 {
3096 	struct ehci_soft_islot *isp;
3097 	int islot, lev;
3098 
3099 	/* Find a poll rate that is large enough. */
3100 	for (lev = EHCI_IPOLLRATES - 1; lev > 0; lev--)
3101 		if (EHCI_ILEV_IVAL(lev) <= ival)
3102 			break;
3103 
3104 	/* Pick an interrupt slot at the right level. */
3105 	/* XXX could do better than picking at random */
3106 	islot = EHCI_IQHIDX(lev, arc4random());
3107 
3108 	sqh->islot = islot;
3109 	isp = &sc->sc_islots[islot];
3110 	ehci_add_qh(sqh, isp->sqh);
3111 
3112 	return (USBD_NORMAL_COMPLETION);
3113 }
3114 
3115 usbd_status
3116 ehci_device_intr_transfer(struct usbd_xfer *xfer)
3117 {
3118 	usbd_status err;
3119 
3120 	/* Insert last in queue. */
3121 	err = usb_insert_transfer(xfer);
3122 	if (err)
3123 		return (err);
3124 
3125 	/*
3126 	 * Pipe isn't running (otherwise err would be USBD_INPROG),
3127 	 * so start it first.
3128 	 */
3129 	return (ehci_device_intr_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
3130 }
3131 
3132 usbd_status
3133 ehci_device_intr_start(struct usbd_xfer *xfer)
3134 {
3135 	struct ehci_softc *sc = (struct ehci_softc *)xfer->device->bus;
3136 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
3137 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3138 	struct ehci_soft_qtd *data, *dataend;
3139 	struct ehci_soft_qh *sqh;
3140 	usbd_status err;
3141 	int s;
3142 
3143 	KASSERT(!(xfer->rqflags & URQ_REQUEST));
3144 
3145 	if (sc->sc_bus.dying)
3146 		return (USBD_IOERROR);
3147 
3148 	sqh = epipe->sqh;
3149 
3150 	err = ehci_alloc_sqtd_chain(sc, xfer->length, xfer, &data, &dataend);
3151 	if (err) {
3152 		xfer->status = err;
3153 		usb_transfer_complete(xfer);
3154 		return (err);
3155 	}
3156 
3157 	/* Set up interrupt info. */
3158 	ex->sqtdstart = data;
3159 	ex->sqtdend = dataend;
3160 #ifdef DIAGNOSTIC
3161 	if (!ex->isdone)
3162 		printf("ehci_device_intr_start: not done, ex=%p\n", ex);
3163 	ex->isdone = 0;
3164 #endif
3165 
3166 	s = splusb();
3167 	ehci_set_qh_qtd(sqh, data);
3168 	if (xfer->timeout && !sc->sc_bus.use_polling) {
3169 		timeout_del(&xfer->timeout_handle);
3170 		timeout_set(&xfer->timeout_handle, ehci_timeout, xfer);
3171 		timeout_add_msec(&xfer->timeout_handle, xfer->timeout);
3172 	}
3173 	TAILQ_INSERT_TAIL(&sc->sc_intrhead, ex, inext);
3174 	xfer->status = USBD_IN_PROGRESS;
3175 	splx(s);
3176 
3177 	return (USBD_IN_PROGRESS);
3178 }
3179 
3180 void
3181 ehci_device_intr_abort(struct usbd_xfer *xfer)
3182 {
3183 	KASSERT(!xfer->pipe->repeat || xfer->pipe->intrxfer == xfer);
3184 
3185 	/*
3186 	 * XXX - abort_xfer uses ehci_sync_hc, which syncs via the advance
3187 	 *       async doorbell. That's dependant on the async list, wheras
3188 	 *       intr xfers are periodic, should not use this?
3189 	 */
3190 	ehci_abort_xfer(xfer, USBD_CANCELLED);
3191 }
3192 
3193 void
3194 ehci_device_intr_close(struct usbd_pipe *pipe)
3195 {
3196 	ehci_close_pipe(pipe);
3197 }
3198 
3199 void
3200 ehci_device_intr_done(struct usbd_xfer *xfer)
3201 {
3202 	struct ehci_softc *sc = (struct ehci_softc *)xfer->device->bus;
3203 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3204 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
3205 	struct ehci_soft_qtd *data, *dataend;
3206 	struct ehci_soft_qh *sqh;
3207 	usbd_status err;
3208 	int s;
3209 
3210 	if (xfer->pipe->repeat) {
3211 		ehci_free_sqtd_chain(sc, ex);
3212 
3213 		usb_syncmem(&xfer->dmabuf, 0, xfer->length,
3214 		    usbd_xfer_isread(xfer) ?
3215 		    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
3216 		sqh = epipe->sqh;
3217 
3218 		err = ehci_alloc_sqtd_chain(sc, xfer->length, xfer, &data, &dataend);
3219 		if (err) {
3220 			xfer->status = err;
3221 			return;
3222 		}
3223 
3224 		/* Set up interrupt info. */
3225 		ex->sqtdstart = data;
3226 		ex->sqtdend = dataend;
3227 #ifdef DIAGNOSTIC
3228 		if (!ex->isdone) {
3229 			printf("ehci_device_intr_done: not done, ex=%p\n",
3230 					ex);
3231 		}
3232 		ex->isdone = 0;
3233 #endif
3234 
3235 		s = splusb();
3236 		ehci_set_qh_qtd(sqh, data);
3237 		if (xfer->timeout && !sc->sc_bus.use_polling) {
3238 			timeout_del(&xfer->timeout_handle);
3239 			timeout_set(&xfer->timeout_handle, ehci_timeout, xfer);
3240 			timeout_add_msec(&xfer->timeout_handle, xfer->timeout);
3241 		}
3242 		TAILQ_INSERT_TAIL(&sc->sc_intrhead, ex, inext);
3243 		xfer->status = USBD_IN_PROGRESS;
3244 		splx(s);
3245 	} else if (xfer->status != USBD_NOMEM) {
3246 		ehci_free_sqtd_chain(sc, ex);
3247 	}
3248 }
3249 
3250 usbd_status
3251 ehci_device_isoc_transfer(struct usbd_xfer *xfer)
3252 {
3253 	usbd_status err;
3254 
3255 	err = usb_insert_transfer(xfer);
3256 	if (err && err != USBD_IN_PROGRESS)
3257 		return (err);
3258 
3259 	return (ehci_device_isoc_start(xfer));
3260 }
3261 
3262 usbd_status
3263 ehci_device_isoc_start(struct usbd_xfer *xfer)
3264 {
3265 	struct ehci_softc *sc = (struct ehci_softc *)xfer->device->bus;
3266 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3267 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
3268 	usb_endpoint_descriptor_t *ed = xfer->pipe->endpoint->edesc;
3269 	uint8_t ival = ed->bInterval;
3270 	struct ehci_soft_itd *itd;
3271 	int s, frindex;
3272 	uint32_t link;
3273 
3274 	KASSERT(!(xfer->rqflags & URQ_REQUEST));
3275 	KASSERT(ival > 0 && ival <= 16);
3276 
3277 	/*
3278 	 * To allow continuous transfers, above we start all transfers
3279 	 * immediately. However, we're still going to get usbd_start_next call
3280 	 * this when another xfer completes. So, check if this is already
3281 	 * in progress or not
3282 	 */
3283 	if (ex->itdstart != NULL)
3284 		return (USBD_IN_PROGRESS);
3285 
3286 	if (sc->sc_bus.dying)
3287 		return (USBD_IOERROR);
3288 
3289 	/* Why would you do that anyway? */
3290 	if (sc->sc_bus.use_polling)
3291 		return (USBD_INVAL);
3292 
3293 	/*
3294 	 * To avoid complication, don't allow a request right now that'll span
3295 	 * the entire frame table. To within 4 frames, to allow some leeway
3296 	 * on either side of where the hc currently is.
3297 	 */
3298 	if ((1 << (ival - 1)) * xfer->nframes >= (sc->sc_flsize - 4) * 8)
3299 		return (USBD_INVAL);
3300 
3301 	/*
3302 	 * Step 1: Allocate and initialize itds.
3303 	 */
3304 	if (xfer->device->speed == USB_SPEED_HIGH) {
3305 		if (ehci_alloc_itd_chain(sc, xfer))
3306 			return (USBD_INVAL);
3307 
3308 		link = EHCI_LINK_ITD;
3309 	} else {
3310 		if (ehci_alloc_sitd_chain(sc, xfer))
3311 			return (USBD_INVAL);
3312 
3313 		link = EHCI_LINK_SITD;
3314 	}
3315 
3316 #ifdef DIAGNOSTIC
3317 	if (!ex->isdone) {
3318 		printf("%s: not done, ex=%p\n", __func__, ex);
3319 	}
3320 	ex->isdone = 0;
3321 #endif
3322 
3323 	/*
3324 	 * Part 2: Transfer descriptors have now been set up, now they must
3325 	 * be scheduled into the period frame list. Erk. Not wanting to
3326 	 * complicate matters, transfer is denied if the transfer spans
3327 	 * more than the period frame list.
3328 	 */
3329 	s = splusb();
3330 
3331 	/* Start inserting frames */
3332 	if (epipe->u.isoc.cur_xfers > 0) {
3333 		frindex = epipe->u.isoc.next_frame;
3334 	} else {
3335 		frindex = EOREAD4(sc, EHCI_FRINDEX);
3336 		frindex = frindex >> 3; /* Erase microframe index */
3337 		frindex += 2;
3338 	}
3339 
3340 	if (frindex >= sc->sc_flsize)
3341 		frindex &= (sc->sc_flsize - 1);
3342 
3343 	/* What's the frame interval? */
3344 	ival = (1 << (ival - 1));
3345 	if (ival / 8 == 0)
3346 		ival = 1;
3347 	else
3348 		ival /= 8;
3349 
3350 	/* Abuse the fact that itd_next == sitd_next. */
3351 	for (itd = ex->itdstart; itd != NULL; itd = itd->xfer_next) {
3352 		itd->itd.itd_next = sc->sc_flist[frindex];
3353 		if (itd->itd.itd_next == 0)
3354 			itd->itd.itd_next = htole32(EHCI_LINK_TERMINATE);
3355 
3356 		sc->sc_flist[frindex] = htole32(link | itd->physaddr);
3357 		itd->u.frame_list.next = sc->sc_softitds[frindex];
3358 		sc->sc_softitds[frindex] = itd;
3359 		if (itd->u.frame_list.next != NULL)
3360 			itd->u.frame_list.next->u.frame_list.prev = itd;
3361 		itd->slot = frindex;
3362 		itd->u.frame_list.prev = NULL;
3363 
3364 		frindex += ival;
3365 		if (frindex >= sc->sc_flsize)
3366 			frindex -= sc->sc_flsize;
3367 	}
3368 
3369 	epipe->u.isoc.cur_xfers++;
3370 	epipe->u.isoc.next_frame = frindex;
3371 
3372 	TAILQ_INSERT_TAIL(&sc->sc_intrhead, ex, inext);
3373 	xfer->status = USBD_IN_PROGRESS;
3374 	xfer->done = 0;
3375 	splx(s);
3376 
3377 	return (USBD_IN_PROGRESS);
3378 }
3379 
3380 int
3381 ehci_alloc_itd_chain(struct ehci_softc *sc, struct usbd_xfer *xfer)
3382 {
3383 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
3384 	usb_endpoint_descriptor_t *ed = xfer->pipe->endpoint->edesc;
3385 	const uint32_t mps = UGETW(ed->wMaxPacketSize);
3386 	struct ehci_soft_itd *itd = NULL, *pitd = NULL;
3387 	int i, j, nframes, uframes, ufrperframe;
3388 	int offs = 0, trans_count = 0;
3389 
3390 	/*
3391 	 * How many itds do we need?  One per transfer if interval >= 8
3392 	 * microframes, fewer if we use multiple microframes per frame.
3393 	 */
3394 	switch (ed->bInterval) {
3395 	case 1:
3396 		ufrperframe = 8;
3397 		break;
3398 	case 2:
3399 		ufrperframe = 4;
3400 		break;
3401 	case 3:
3402 		ufrperframe = 2;
3403 		break;
3404 	default:
3405 		ufrperframe = 1;
3406 		break;
3407 	}
3408 	nframes = (xfer->nframes + (ufrperframe - 1)) / ufrperframe;
3409 	uframes = 8 / ufrperframe;
3410 	if (nframes == 0)
3411 		return (1);
3412 
3413 	for (i = 0; i < nframes; i++) {
3414 		uint32_t froffs = offs;
3415 
3416 		itd = ehci_alloc_itd(sc);
3417 		if (itd == NULL) {
3418 			ehci_free_itd_chain(sc, ex);
3419 			return (1);
3420 		}
3421 
3422 		if (pitd != NULL)
3423 			pitd->xfer_next = itd;
3424 		else
3425 			ex->itdstart = itd;
3426 
3427 		/*
3428 		 * Step 1.5, initialize uframes
3429 		 */
3430 		for (j = 0; j < 8; j += uframes) {
3431 			/* Calculate which page in the list this starts in */
3432 			int addr = DMAADDR(&xfer->dmabuf, froffs);
3433 			addr = EHCI_PAGE_OFFSET(addr) + (offs - froffs);
3434 			addr = EHCI_PAGE(addr) / EHCI_PAGE_SIZE;
3435 
3436 			/* This gets the initial offset into the first page,
3437 			 * looks how far further along the current uframe
3438 			 * offset is. Works out how many pages that is.
3439 			 */
3440 			itd->itd.itd_ctl[j] = htole32(
3441 			    EHCI_ITD_ACTIVE |
3442 			    EHCI_ITD_SET_LEN(xfer->frlengths[trans_count]) |
3443 			    EHCI_ITD_SET_PG(addr) |
3444 			    EHCI_ITD_SET_OFFS(DMAADDR(&xfer->dmabuf, offs))
3445 			);
3446 
3447 			offs += xfer->frlengths[trans_count];
3448 			trans_count++;
3449 
3450 			if (trans_count >= xfer->nframes) { /*Set IOC*/
3451 				itd->itd.itd_ctl[j] |= htole32(EHCI_ITD_IOC);
3452 				break;
3453 			}
3454 		}
3455 
3456 		/* Step 1.75, set buffer pointers. To simplify matters, all
3457 		 * pointers are filled out for the next 7 hardware pages in
3458 		 * the dma block, so no need to worry what pages to cover
3459 		 * and what to not.
3460 		 */
3461 
3462 		for (j = 0; j < 7; j++) {
3463 			/*
3464 			 * Don't try to lookup a page that's past the end
3465 			 * of buffer
3466 			 */
3467 			int page_offs = EHCI_PAGE(froffs +
3468 			    (EHCI_PAGE_SIZE * j));
3469 
3470 			if (page_offs >= xfer->dmabuf.block->size)
3471 				break;
3472 
3473 			long long page = DMAADDR(&xfer->dmabuf, page_offs);
3474 			page = EHCI_PAGE(page);
3475 			itd->itd.itd_bufr[j] = htole32(page);
3476 			itd->itd.itd_bufr_hi[j] = htole32(page >> 32);
3477 		}
3478 
3479 		/*
3480 		 * Other special values
3481 		 */
3482 		itd->itd.itd_bufr[0] |= htole32(
3483 		    EHCI_ITD_SET_ENDPT(UE_GET_ADDR(ed->bEndpointAddress)) |
3484 		    EHCI_ITD_SET_DADDR(xfer->pipe->device->address)
3485 		);
3486 
3487 		itd->itd.itd_bufr[1] |= htole32(
3488 		    (usbd_xfer_isread(xfer) ? EHCI_ITD_SET_DIR(1) : 0) |
3489 		    EHCI_ITD_SET_MAXPKT(UE_GET_SIZE(mps))
3490 		);
3491 		/* FIXME: handle invalid trans */
3492 		itd->itd.itd_bufr[2] |= htole32(
3493 		    EHCI_ITD_SET_MULTI(UE_GET_TRANS(mps)+1)
3494 		);
3495 
3496 		pitd = itd;
3497 	}
3498 
3499 	ex->itdend = itd;
3500 
3501 	return (0);
3502 }
3503 
3504 int
3505 ehci_alloc_sitd_chain(struct ehci_softc *sc, struct usbd_xfer *xfer)
3506 {
3507 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
3508 	struct usbd_device *hshub = xfer->device->myhsport->parent;
3509 	usb_endpoint_descriptor_t *ed = xfer->pipe->endpoint->edesc;
3510 	struct ehci_soft_itd *itd = NULL, *pitd = NULL;
3511 	uint8_t smask, cmask, tp, uf;
3512 	int i, nframes, offs = 0;
3513 	uint32_t endp;
3514 
3515 	nframes = xfer->nframes;
3516 	if (nframes == 0)
3517 		return (1);
3518 
3519 	endp = EHCI_SITD_SET_ENDPT(UE_GET_ADDR(ed->bEndpointAddress)) |
3520 	    EHCI_SITD_SET_ADDR(xfer->device->address) |
3521 	    EHCI_SITD_SET_PORT(xfer->device->myhsport->portno) |
3522 	    EHCI_SITD_SET_HUBA(hshub->address);
3523 
3524 	if (usbd_xfer_isread(xfer))
3525 		endp |= EHCI_SITD_SET_DIR(1);
3526 
3527 	for (i = 0; i < nframes; i++) {
3528 		uint32_t addr = DMAADDR(&xfer->dmabuf, offs);
3529 		uint32_t page = EHCI_PAGE(addr + xfer->frlengths[i] - 1);
3530 
3531 		itd = ehci_alloc_itd(sc);
3532 		if (itd == NULL) {
3533 			ehci_free_itd_chain(sc, ex);
3534 			return (1);
3535 		}
3536 		if (pitd)
3537 			pitd->xfer_next = itd;
3538 		else
3539 			ex->itdstart = itd;
3540 
3541 		itd->sitd.sitd_endp = htole32(endp);
3542 		itd->sitd.sitd_back = htole32(EHCI_LINK_TERMINATE);
3543 		itd->sitd.sitd_trans = htole32(
3544 		    EHCI_SITD_ACTIVE |
3545 		    EHCI_SITD_SET_LEN(xfer->frlengths[i]) |
3546 		    ((i == nframes - 1) ? EHCI_SITD_IOC : 0)
3547 		);
3548 
3549 		uf = max(1, ((xfer->frlengths[i] + 187) / 188));
3550 
3551 		/*
3552 		 * Since we do not yet budget and schedule micro-frames
3553 		 * we assume there is no other transfer using the same
3554 		 * TT.
3555 		 */
3556 		if (usbd_xfer_isread(xfer)) {
3557 			smask = 0x01;
3558 			cmask = ((1 << (uf + 2)) - 1) << 2;
3559 		} else {
3560 			/* Is the payload is greater than 188 bytes? */
3561 			if (uf == 1)
3562 				tp = EHCI_SITD_TP_ALL;
3563 			else
3564 				tp = EHCI_SITD_TP_BEGIN;
3565 
3566 			page |=	EHCI_SITD_SET_TCOUNT(uf) | EHCI_SITD_SET_TP(tp);
3567 			smask = (1 << uf) - 1;
3568 			cmask = 0x00;
3569 		}
3570 
3571 		itd->sitd.sitd_sched = htole32(
3572 		    EHCI_SITD_SET_SMASK(smask) | EHCI_SITD_SET_CMASK(cmask)
3573 		);
3574 		itd->sitd.sitd_bufr[0] = htole32(addr);
3575 		itd->sitd.sitd_bufr[1] = htole32(page);
3576 
3577 		offs += xfer->frlengths[i];
3578 		pitd = itd;
3579 	}
3580 
3581 	ex->itdend = itd;
3582 
3583 	return (0);
3584 }
3585 
3586 void
3587 ehci_device_isoc_abort(struct usbd_xfer *xfer)
3588 {
3589 	int s;
3590 
3591 	s = splusb();
3592 	ehci_abort_isoc_xfer(xfer, USBD_CANCELLED);
3593 	splx(s);
3594 }
3595 
3596 void
3597 ehci_device_isoc_close(struct usbd_pipe *pipe)
3598 {
3599 }
3600 
3601 void
3602 ehci_device_isoc_done(struct usbd_xfer *xfer)
3603 {
3604 	struct ehci_softc *sc = (struct ehci_softc *)xfer->device->bus;
3605 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3606 	struct ehci_xfer *ex = (struct ehci_xfer *)xfer;
3607 	int s;
3608 
3609 	s = splusb();
3610 	epipe->u.isoc.cur_xfers--;
3611 	if (xfer->status != USBD_NOMEM) {
3612 		ehci_rem_itd_chain(sc, ex);
3613 		ehci_free_itd_chain(sc, ex);
3614 	}
3615 	splx(s);
3616 }
3617