xref: /netbsd-src/sys/dev/usb/ehci.c (revision a6f3f22f245acb8ee3bbf6871d7dce989204fa97)
1 /*	$NetBSD: ehci.c,v 1.245 2015/09/11 06:51:43 skrll Exp $ */
2 
3 /*
4  * Copyright (c) 2004-2012 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Lennart Augustsson (lennart@augustsson.net), Charles M. Hannum,
9  * Jeremy Morse (jeremy.morse@gmail.com), Jared D. McNeill
10  * (jmcneill@invisible.ca) and Matthew R. Green (mrg@eterna.com.au).
11  *
12  * Redistribution and use in source and binary forms, with or without
13  * modification, are permitted provided that the following conditions
14  * are met:
15  * 1. Redistributions of source code must retain the above copyright
16  *    notice, this list of conditions and the following disclaimer.
17  * 2. Redistributions in binary form must reproduce the above copyright
18  *    notice, this list of conditions and the following disclaimer in the
19  *    documentation and/or other materials provided with the distribution.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
25  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31  * POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 /*
35  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
36  *
37  * The EHCI 1.0 spec can be found at
38  * http://www.intel.com/technology/usb/spec.htm
39  * and the USB 2.0 spec at
40  * http://www.usb.org/developers/docs/
41  *
42  */
43 
44 /*
45  * TODO:
46  * 1) hold off explorations by companion controllers until ehci has started.
47  *
48  * 2) The hub driver needs to handle and schedule the transaction translator,
49  *    to assign place in frame where different devices get to go. See chapter
50  *    on hubs in USB 2.0 for details.
51  *
52  * 3) Command failures are not recovered correctly.
53  */
54 
55 #include <sys/cdefs.h>
56 __KERNEL_RCSID(0, "$NetBSD: ehci.c,v 1.245 2015/09/11 06:51:43 skrll Exp $");
57 
58 #include "ohci.h"
59 #include "uhci.h"
60 
61 #ifdef _KERNEL_OPT
62 #include "opt_usb.h"
63 #endif
64 
65 #include <sys/param.h>
66 
67 #include <sys/bus.h>
68 #include <sys/cpu.h>
69 #include <sys/device.h>
70 #include <sys/kernel.h>
71 #include <sys/kmem.h>
72 #include <sys/mutex.h>
73 #include <sys/proc.h>
74 #include <sys/queue.h>
75 #include <sys/select.h>
76 #include <sys/sysctl.h>
77 #include <sys/systm.h>
78 
79 #include <machine/endian.h>
80 
81 #include <dev/usb/usb.h>
82 #include <dev/usb/usbdi.h>
83 #include <dev/usb/usbdivar.h>
84 #include <dev/usb/usbhist.h>
85 #include <dev/usb/usb_mem.h>
86 #include <dev/usb/usb_quirks.h>
87 #include <dev/usb/usbroothub_subr.h>
88 
89 #include <dev/usb/ehcireg.h>
90 #include <dev/usb/ehcivar.h>
91 
92 
93 #ifdef USB_DEBUG
94 #ifndef EHCI_DEBUG
95 #define ehcidebug 0
96 #else
97 static int ehcidebug = 0;
98 
99 SYSCTL_SETUP(sysctl_hw_ehci_setup, "sysctl hw.ehci setup")
100 {
101 	int err;
102 	const struct sysctlnode *rnode;
103 	const struct sysctlnode *cnode;
104 
105 	err = sysctl_createv(clog, 0, NULL, &rnode,
106 	    CTLFLAG_PERMANENT, CTLTYPE_NODE, "ehci",
107 	    SYSCTL_DESCR("ehci global controls"),
108 	    NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
109 
110 	if (err)
111 		goto fail;
112 
113 	/* control debugging printfs */
114 	err = sysctl_createv(clog, 0, &rnode, &cnode,
115 	    CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT,
116 	    "debug", SYSCTL_DESCR("Enable debugging output"),
117 	    NULL, 0, &ehcidebug, sizeof(ehcidebug), CTL_CREATE, CTL_EOL);
118 	if (err)
119 		goto fail;
120 
121 	return;
122 fail:
123 	aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, err);
124 }
125 
126 #endif /* EHCI_DEBUG */
127 #endif /* USB_DEBUG */
128 
129 struct ehci_pipe {
130 	struct usbd_pipe pipe;
131 	int nexttoggle;
132 
133 	ehci_soft_qh_t *sqh;
134 	union {
135 		ehci_soft_qtd_t *qtd;
136 		/* ehci_soft_itd_t *itd; */
137 	} tail;
138 	union {
139 		/* Control pipe */
140 		struct {
141 			usb_dma_t reqdma;
142 		} ctl;
143 		/* Interrupt pipe */
144 		struct {
145 			u_int length;
146 		} intr;
147 		/* Bulk pipe */
148 		struct {
149 			u_int length;
150 		} bulk;
151 		/* Iso pipe */
152 		struct {
153 			u_int next_frame;
154 			u_int cur_xfers;
155 		} isoc;
156 	} u;
157 };
158 
159 Static usbd_status	ehci_open(usbd_pipe_handle);
160 Static void		ehci_poll(struct usbd_bus *);
161 Static void		ehci_softintr(void *);
162 Static int		ehci_intr1(ehci_softc_t *);
163 Static void		ehci_waitintr(ehci_softc_t *, usbd_xfer_handle);
164 Static void		ehci_check_intr(ehci_softc_t *, struct ehci_xfer *);
165 Static void		ehci_check_qh_intr(ehci_softc_t *, struct ehci_xfer *);
166 Static void		ehci_check_itd_intr(ehci_softc_t *, struct ehci_xfer *);
167 Static void		ehci_idone(struct ehci_xfer *);
168 Static void		ehci_timeout(void *);
169 Static void		ehci_timeout_task(void *);
170 Static void		ehci_intrlist_timeout(void *);
171 Static void		ehci_doorbell(void *);
172 Static void		ehci_pcd(void *);
173 
174 Static usbd_status	ehci_allocm(struct usbd_bus *, usb_dma_t *, u_int32_t);
175 Static void		ehci_freem(struct usbd_bus *, usb_dma_t *);
176 
177 Static usbd_xfer_handle	ehci_allocx(struct usbd_bus *);
178 Static void		ehci_freex(struct usbd_bus *, usbd_xfer_handle);
179 Static void		ehci_get_lock(struct usbd_bus *, kmutex_t **);
180 
181 Static usbd_status	ehci_root_ctrl_transfer(usbd_xfer_handle);
182 Static usbd_status	ehci_root_ctrl_start(usbd_xfer_handle);
183 Static void		ehci_root_ctrl_abort(usbd_xfer_handle);
184 Static void		ehci_root_ctrl_close(usbd_pipe_handle);
185 Static void		ehci_root_ctrl_done(usbd_xfer_handle);
186 
187 Static usbd_status	ehci_root_intr_transfer(usbd_xfer_handle);
188 Static usbd_status	ehci_root_intr_start(usbd_xfer_handle);
189 Static void		ehci_root_intr_abort(usbd_xfer_handle);
190 Static void		ehci_root_intr_close(usbd_pipe_handle);
191 Static void		ehci_root_intr_done(usbd_xfer_handle);
192 
193 Static usbd_status	ehci_device_ctrl_transfer(usbd_xfer_handle);
194 Static usbd_status	ehci_device_ctrl_start(usbd_xfer_handle);
195 Static void		ehci_device_ctrl_abort(usbd_xfer_handle);
196 Static void		ehci_device_ctrl_close(usbd_pipe_handle);
197 Static void		ehci_device_ctrl_done(usbd_xfer_handle);
198 
199 Static usbd_status	ehci_device_bulk_transfer(usbd_xfer_handle);
200 Static usbd_status	ehci_device_bulk_start(usbd_xfer_handle);
201 Static void		ehci_device_bulk_abort(usbd_xfer_handle);
202 Static void		ehci_device_bulk_close(usbd_pipe_handle);
203 Static void		ehci_device_bulk_done(usbd_xfer_handle);
204 
205 Static usbd_status	ehci_device_intr_transfer(usbd_xfer_handle);
206 Static usbd_status	ehci_device_intr_start(usbd_xfer_handle);
207 Static void		ehci_device_intr_abort(usbd_xfer_handle);
208 Static void		ehci_device_intr_close(usbd_pipe_handle);
209 Static void		ehci_device_intr_done(usbd_xfer_handle);
210 
211 Static usbd_status	ehci_device_isoc_transfer(usbd_xfer_handle);
212 Static usbd_status	ehci_device_isoc_start(usbd_xfer_handle);
213 Static void		ehci_device_isoc_abort(usbd_xfer_handle);
214 Static void		ehci_device_isoc_close(usbd_pipe_handle);
215 Static void		ehci_device_isoc_done(usbd_xfer_handle);
216 
217 Static void		ehci_device_clear_toggle(usbd_pipe_handle pipe);
218 Static void		ehci_noop(usbd_pipe_handle pipe);
219 
220 Static void		ehci_disown(ehci_softc_t *, int, int);
221 
222 Static ehci_soft_qh_t  *ehci_alloc_sqh(ehci_softc_t *);
223 Static void		ehci_free_sqh(ehci_softc_t *, ehci_soft_qh_t *);
224 
225 Static ehci_soft_qtd_t  *ehci_alloc_sqtd(ehci_softc_t *);
226 Static void		ehci_free_sqtd(ehci_softc_t *, ehci_soft_qtd_t *);
227 Static usbd_status	ehci_alloc_sqtd_chain(struct ehci_pipe *,
228 			    ehci_softc_t *, int, int, usbd_xfer_handle,
229 			    ehci_soft_qtd_t **, ehci_soft_qtd_t **);
230 Static void		ehci_free_sqtd_chain(ehci_softc_t *, ehci_soft_qtd_t *,
231 					    ehci_soft_qtd_t *);
232 
233 Static ehci_soft_itd_t	*ehci_alloc_itd(ehci_softc_t *sc);
234 Static void		ehci_free_itd(ehci_softc_t *sc, ehci_soft_itd_t *itd);
235 Static void 		ehci_rem_free_itd_chain(ehci_softc_t *sc,
236 						struct ehci_xfer *exfer);
237 Static void 		ehci_abort_isoc_xfer(usbd_xfer_handle xfer,
238 						usbd_status status);
239 
240 Static usbd_status	ehci_device_request(usbd_xfer_handle xfer);
241 
242 Static usbd_status	ehci_device_setintr(ehci_softc_t *, ehci_soft_qh_t *,
243 			    int ival);
244 
245 Static void		ehci_add_qh(ehci_softc_t *, ehci_soft_qh_t *,
246 				    ehci_soft_qh_t *);
247 Static void		ehci_rem_qh(ehci_softc_t *, ehci_soft_qh_t *,
248 				    ehci_soft_qh_t *);
249 Static void		ehci_set_qh_qtd(ehci_soft_qh_t *, ehci_soft_qtd_t *);
250 Static void		ehci_sync_hc(ehci_softc_t *);
251 
252 Static void		ehci_close_pipe(usbd_pipe_handle, ehci_soft_qh_t *);
253 Static void		ehci_abort_xfer(usbd_xfer_handle, usbd_status);
254 
255 #ifdef EHCI_DEBUG
256 Static ehci_softc_t 	*theehci;
257 void			ehci_dump(void);
258 #endif
259 
260 #ifdef EHCI_DEBUG
261 Static void		ehci_dump_regs(ehci_softc_t *);
262 Static void		ehci_dump_sqtds(ehci_soft_qtd_t *);
263 Static void		ehci_dump_sqtd(ehci_soft_qtd_t *);
264 Static void		ehci_dump_qtd(ehci_qtd_t *);
265 Static void		ehci_dump_sqh(ehci_soft_qh_t *);
266 Static void		ehci_dump_sitd(struct ehci_soft_itd *itd);
267 Static void		ehci_dump_itd(struct ehci_soft_itd *);
268 Static void		ehci_dump_exfer(struct ehci_xfer *);
269 #endif
270 
271 #define EHCI_NULL htole32(EHCI_LINK_TERMINATE)
272 
273 #define EHCI_INTR_ENDPT 1
274 
275 #define ehci_add_intr_list(sc, ex) \
276 	TAILQ_INSERT_TAIL(&(sc)->sc_intrhead, (ex), inext);
277 #define ehci_del_intr_list(sc, ex) \
278 	do { \
279 		TAILQ_REMOVE(&sc->sc_intrhead, (ex), inext); \
280 		(ex)->inext.tqe_prev = NULL; \
281 	} while (0)
282 #define ehci_active_intr_list(ex) ((ex)->inext.tqe_prev != NULL)
283 
284 Static const struct usbd_bus_methods ehci_bus_methods = {
285 	.open_pipe =	ehci_open,
286 	.soft_intr =	ehci_softintr,
287 	.do_poll =	ehci_poll,
288 	.allocm =	ehci_allocm,
289 	.freem =	ehci_freem,
290 	.allocx =	ehci_allocx,
291 	.freex =	ehci_freex,
292 	.get_lock =	ehci_get_lock,
293 	.new_device =	NULL,
294 };
295 
296 Static const struct usbd_pipe_methods ehci_root_ctrl_methods = {
297 	.transfer =	ehci_root_ctrl_transfer,
298 	.start =	ehci_root_ctrl_start,
299 	.abort =	ehci_root_ctrl_abort,
300 	.close =	ehci_root_ctrl_close,
301 	.cleartoggle =	ehci_noop,
302 	.done =		ehci_root_ctrl_done,
303 };
304 
305 Static const struct usbd_pipe_methods ehci_root_intr_methods = {
306 	.transfer =	ehci_root_intr_transfer,
307 	.start =	ehci_root_intr_start,
308 	.abort =	ehci_root_intr_abort,
309 	.close =	ehci_root_intr_close,
310 	.cleartoggle =	ehci_noop,
311 	.done =		ehci_root_intr_done,
312 };
313 
314 Static const struct usbd_pipe_methods ehci_device_ctrl_methods = {
315 	.transfer =	ehci_device_ctrl_transfer,
316 	.start =	ehci_device_ctrl_start,
317 	.abort =	ehci_device_ctrl_abort,
318 	.close =	ehci_device_ctrl_close,
319 	.cleartoggle =	ehci_noop,
320 	.done =		ehci_device_ctrl_done,
321 };
322 
323 Static const struct usbd_pipe_methods ehci_device_intr_methods = {
324 	.transfer =	ehci_device_intr_transfer,
325 	.start =	ehci_device_intr_start,
326 	.abort =	ehci_device_intr_abort,
327 	.close =	ehci_device_intr_close,
328 	.cleartoggle =	ehci_device_clear_toggle,
329 	.done =		ehci_device_intr_done,
330 };
331 
332 Static const struct usbd_pipe_methods ehci_device_bulk_methods = {
333 	.transfer =	ehci_device_bulk_transfer,
334 	.start =	ehci_device_bulk_start,
335 	.abort =	ehci_device_bulk_abort,
336 	.close =	ehci_device_bulk_close,
337 	.cleartoggle =	ehci_device_clear_toggle,
338 	.done =		ehci_device_bulk_done,
339 };
340 
341 Static const struct usbd_pipe_methods ehci_device_isoc_methods = {
342 	.transfer =	ehci_device_isoc_transfer,
343 	.start =	ehci_device_isoc_start,
344 	.abort =	ehci_device_isoc_abort,
345 	.close =	ehci_device_isoc_close,
346 	.cleartoggle =	ehci_noop,
347 	.done =		ehci_device_isoc_done,
348 };
349 
350 static const uint8_t revbits[EHCI_MAX_POLLRATE] = {
351 0x00,0x40,0x20,0x60,0x10,0x50,0x30,0x70,0x08,0x48,0x28,0x68,0x18,0x58,0x38,0x78,
352 0x04,0x44,0x24,0x64,0x14,0x54,0x34,0x74,0x0c,0x4c,0x2c,0x6c,0x1c,0x5c,0x3c,0x7c,
353 0x02,0x42,0x22,0x62,0x12,0x52,0x32,0x72,0x0a,0x4a,0x2a,0x6a,0x1a,0x5a,0x3a,0x7a,
354 0x06,0x46,0x26,0x66,0x16,0x56,0x36,0x76,0x0e,0x4e,0x2e,0x6e,0x1e,0x5e,0x3e,0x7e,
355 0x01,0x41,0x21,0x61,0x11,0x51,0x31,0x71,0x09,0x49,0x29,0x69,0x19,0x59,0x39,0x79,
356 0x05,0x45,0x25,0x65,0x15,0x55,0x35,0x75,0x0d,0x4d,0x2d,0x6d,0x1d,0x5d,0x3d,0x7d,
357 0x03,0x43,0x23,0x63,0x13,0x53,0x33,0x73,0x0b,0x4b,0x2b,0x6b,0x1b,0x5b,0x3b,0x7b,
358 0x07,0x47,0x27,0x67,0x17,0x57,0x37,0x77,0x0f,0x4f,0x2f,0x6f,0x1f,0x5f,0x3f,0x7f,
359 };
360 
361 usbd_status
362 ehci_init(ehci_softc_t *sc)
363 {
364 	u_int32_t vers, sparams, cparams, hcr;
365 	u_int i;
366 	usbd_status err;
367 	ehci_soft_qh_t *sqh;
368 	u_int ncomp;
369 
370 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
371 #ifdef EHCI_DEBUG
372 	theehci = sc;
373 #endif
374 
375 	mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTUSB);
376 	mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_USB);
377 	cv_init(&sc->sc_softwake_cv, "ehciab");
378 	cv_init(&sc->sc_doorbell, "ehcidi");
379 
380 	sc->sc_xferpool = pool_cache_init(sizeof(struct ehci_xfer), 0, 0, 0,
381 	    "ehcixfer", NULL, IPL_USB, NULL, NULL, NULL);
382 
383 	sc->sc_doorbell_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
384 	    ehci_doorbell, sc);
385 	KASSERT(sc->sc_doorbell_si != NULL);
386 	sc->sc_pcd_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
387 	    ehci_pcd, sc);
388 	KASSERT(sc->sc_pcd_si != NULL);
389 
390 	sc->sc_offs = EREAD1(sc, EHCI_CAPLENGTH);
391 
392 	vers = EREAD2(sc, EHCI_HCIVERSION);
393 	aprint_verbose("%s: EHCI version %x.%x\n", device_xname(sc->sc_dev),
394 	       vers >> 8, vers & 0xff);
395 
396 	sparams = EREAD4(sc, EHCI_HCSPARAMS);
397 	USBHIST_LOG(ehcidebug, "sparams=%#x", sparams, 0, 0, 0);
398 	sc->sc_npcomp = EHCI_HCS_N_PCC(sparams);
399 	ncomp = EHCI_HCS_N_CC(sparams);
400 	if (ncomp != sc->sc_ncomp) {
401 		aprint_verbose("%s: wrong number of companions (%d != %d)\n",
402 			       device_xname(sc->sc_dev), ncomp, sc->sc_ncomp);
403 #if NOHCI == 0 || NUHCI == 0
404 		aprint_error("%s: ohci or uhci probably not configured\n",
405 			     device_xname(sc->sc_dev));
406 #endif
407 		if (ncomp < sc->sc_ncomp)
408 			sc->sc_ncomp = ncomp;
409 	}
410 	if (sc->sc_ncomp > 0) {
411 		KASSERT(!(sc->sc_flags & EHCIF_ETTF));
412 		aprint_normal("%s: companion controller%s, %d port%s each:",
413 		    device_xname(sc->sc_dev), sc->sc_ncomp!=1 ? "s" : "",
414 		    EHCI_HCS_N_PCC(sparams),
415 		    EHCI_HCS_N_PCC(sparams)!=1 ? "s" : "");
416 		for (i = 0; i < sc->sc_ncomp; i++)
417 			aprint_normal(" %s", device_xname(sc->sc_comps[i]));
418 		aprint_normal("\n");
419 	}
420 	sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
421 	cparams = EREAD4(sc, EHCI_HCCPARAMS);
422 	USBHIST_LOG(ehcidebug, "cparams=%#x", cparams, 0, 0, 0);
423 	sc->sc_hasppc = EHCI_HCS_PPC(sparams);
424 
425 	if (EHCI_HCC_64BIT(cparams)) {
426 		/* MUST clear segment register if 64 bit capable. */
427 		EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
428 	}
429 
430 	sc->sc_bus.usbrev = USBREV_2_0;
431 
432 	usb_setup_reserve(sc->sc_dev, &sc->sc_dma_reserve, sc->sc_bus.dmatag,
433 	    USB_MEM_RESERVE);
434 
435 	/* Reset the controller */
436 	USBHIST_LOG(ehcidebug, "resetting", 0, 0, 0, 0);
437 	EOWRITE4(sc, EHCI_USBCMD, 0);	/* Halt controller */
438 	usb_delay_ms(&sc->sc_bus, 1);
439 	EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
440 	for (i = 0; i < 100; i++) {
441 		usb_delay_ms(&sc->sc_bus, 1);
442 		hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
443 		if (!hcr)
444 			break;
445 	}
446 	if (hcr) {
447 		aprint_error("%s: reset timeout\n", device_xname(sc->sc_dev));
448 		return (USBD_IOERROR);
449 	}
450 	if (sc->sc_vendor_init)
451 		sc->sc_vendor_init(sc);
452 
453 	/* XXX need proper intr scheduling */
454 	sc->sc_rand = 96;
455 
456 	/* frame list size at default, read back what we got and use that */
457 	switch (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD))) {
458 	case 0: sc->sc_flsize = 1024; break;
459 	case 1: sc->sc_flsize = 512; break;
460 	case 2: sc->sc_flsize = 256; break;
461 	case 3: return (USBD_IOERROR);
462 	}
463 	err = usb_allocmem(&sc->sc_bus, sc->sc_flsize * sizeof(ehci_link_t),
464 	    EHCI_FLALIGN_ALIGN, &sc->sc_fldma);
465 	if (err)
466 		return (err);
467 	USBHIST_LOG(ehcidebug, "flsize=%d", sc->sc_flsize, 0, 0, 0);
468 	sc->sc_flist = KERNADDR(&sc->sc_fldma, 0);
469 
470 	for (i = 0; i < sc->sc_flsize; i++) {
471 		sc->sc_flist[i] = EHCI_NULL;
472 	}
473 
474 	EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
475 
476 	sc->sc_softitds = kmem_zalloc(sc->sc_flsize * sizeof(ehci_soft_itd_t *),
477 				     KM_SLEEP);
478 	if (sc->sc_softitds == NULL)
479 		return ENOMEM;
480 	LIST_INIT(&sc->sc_freeitds);
481 	TAILQ_INIT(&sc->sc_intrhead);
482 
483 	/* Set up the bus struct. */
484 	sc->sc_bus.methods = &ehci_bus_methods;
485 	sc->sc_bus.pipe_size = sizeof(struct ehci_pipe);
486 
487 	sc->sc_eintrs = EHCI_NORMAL_INTRS;
488 
489 	/*
490 	 * Allocate the interrupt dummy QHs. These are arranged to give poll
491 	 * intervals that are powers of 2 times 1ms.
492 	 */
493 	for (i = 0; i < EHCI_INTRQHS; i++) {
494 		sqh = ehci_alloc_sqh(sc);
495 		if (sqh == NULL) {
496 			err = USBD_NOMEM;
497 			goto bad1;
498 		}
499 		sc->sc_islots[i].sqh = sqh;
500 	}
501 	for (i = 0; i < EHCI_INTRQHS; i++) {
502 		sqh = sc->sc_islots[i].sqh;
503 		if (i == 0) {
504 			/* The last (1ms) QH terminates. */
505 			sqh->qh.qh_link = EHCI_NULL;
506 			sqh->next = NULL;
507 		} else {
508 			/* Otherwise the next QH has half the poll interval */
509 			sqh->next = sc->sc_islots[(i + 1) / 2 - 1].sqh;
510 			sqh->qh.qh_link = htole32(sqh->next->physaddr |
511 			    EHCI_LINK_QH);
512 		}
513 		sqh->qh.qh_endp = htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
514 		sqh->qh.qh_endphub = htole32(EHCI_QH_SET_MULT(1));
515 		sqh->qh.qh_curqtd = EHCI_NULL;
516 		sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
517 		sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
518 		sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
519 		sqh->sqtd = NULL;
520 		usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
521 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
522 	}
523 	/* Point the frame list at the last level (128ms). */
524 	for (i = 0; i < sc->sc_flsize; i++) {
525 		int j;
526 
527 		j = (i & ~(EHCI_MAX_POLLRATE-1)) |
528 		    revbits[i & (EHCI_MAX_POLLRATE-1)];
529 		sc->sc_flist[j] = htole32(EHCI_LINK_QH |
530 		    sc->sc_islots[EHCI_IQHIDX(EHCI_IPOLLRATES - 1,
531 		    i)].sqh->physaddr);
532 	}
533 	usb_syncmem(&sc->sc_fldma, 0, sc->sc_flsize * sizeof(ehci_link_t),
534 	    BUS_DMASYNC_PREWRITE);
535 
536 	/* Allocate dummy QH that starts the async list. */
537 	sqh = ehci_alloc_sqh(sc);
538 	if (sqh == NULL) {
539 		err = USBD_NOMEM;
540 		goto bad1;
541 	}
542 	/* Fill the QH */
543 	sqh->qh.qh_endp =
544 	    htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
545 	sqh->qh.qh_link =
546 	    htole32(sqh->physaddr | EHCI_LINK_QH);
547 	sqh->qh.qh_curqtd = EHCI_NULL;
548 	sqh->next = NULL;
549 	/* Fill the overlay qTD */
550 	sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
551 	sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
552 	sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
553 	sqh->sqtd = NULL;
554 	usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
555 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
556 #ifdef EHCI_DEBUG
557 	USBHIST_LOGN(ehcidebug, 5, "--- dump start ---", 0, 0, 0, 0);
558 	ehci_dump_sqh(sqh);
559 	USBHIST_LOGN(ehcidebug, 5, "--- dump end ---", 0, 0, 0, 0);
560 #endif
561 
562 	/* Point to async list */
563 	sc->sc_async_head = sqh;
564 	EOWRITE4(sc, EHCI_ASYNCLISTADDR, sqh->physaddr | EHCI_LINK_QH);
565 
566 	callout_init(&sc->sc_tmo_intrlist, CALLOUT_MPSAFE);
567 
568 	/* Turn on controller */
569 	EOWRITE4(sc, EHCI_USBCMD,
570 		 EHCI_CMD_ITC_2 | /* 2 microframes interrupt delay */
571 		 (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
572 		 EHCI_CMD_ASE |
573 		 EHCI_CMD_PSE |
574 		 EHCI_CMD_RS);
575 
576 	/* Take over port ownership */
577 	EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
578 
579 	for (i = 0; i < 100; i++) {
580 		usb_delay_ms(&sc->sc_bus, 1);
581 		hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
582 		if (!hcr)
583 			break;
584 	}
585 	if (hcr) {
586 		aprint_error("%s: run timeout\n", device_xname(sc->sc_dev));
587 		return (USBD_IOERROR);
588 	}
589 
590 	/* Enable interrupts */
591 	USBHIST_LOG(ehcidebug, "enabling interupts", 0, 0, 0, 0);
592 	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
593 
594 	return (USBD_NORMAL_COMPLETION);
595 
596 #if 0
597  bad2:
598 	ehci_free_sqh(sc, sc->sc_async_head);
599 #endif
600  bad1:
601 	usb_freemem(&sc->sc_bus, &sc->sc_fldma);
602 	return (err);
603 }
604 
605 int
606 ehci_intr(void *v)
607 {
608 	ehci_softc_t *sc = v;
609 	int ret = 0;
610 
611 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
612 
613 	if (sc == NULL)
614 		return 0;
615 
616 	mutex_spin_enter(&sc->sc_intr_lock);
617 
618 	if (sc->sc_dying || !device_has_power(sc->sc_dev))
619 		goto done;
620 
621 	/* If we get an interrupt while polling, then just ignore it. */
622 	if (sc->sc_bus.use_polling) {
623 		u_int32_t intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
624 
625 		if (intrs)
626 			EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
627 #ifdef DIAGNOSTIC
628 		USBHIST_LOGN(ehcidebug, 16,
629 		    "ignored interrupt while polling", 0, 0, 0, 0);
630 #endif
631 		goto done;
632 	}
633 
634 	ret = ehci_intr1(sc);
635 
636 done:
637 	mutex_spin_exit(&sc->sc_intr_lock);
638 	return ret;
639 }
640 
641 Static int
642 ehci_intr1(ehci_softc_t *sc)
643 {
644 	u_int32_t intrs, eintrs;
645 
646 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
647 
648 	/* In case the interrupt occurs before initialization has completed. */
649 	if (sc == NULL) {
650 #ifdef DIAGNOSTIC
651 		printf("ehci_intr1: sc == NULL\n");
652 #endif
653 		return (0);
654 	}
655 
656 	KASSERT(mutex_owned(&sc->sc_intr_lock));
657 
658 	intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
659 	if (!intrs)
660 		return (0);
661 
662 	eintrs = intrs & sc->sc_eintrs;
663 	USBHIST_LOG(ehcidebug, "sc=%p intrs=%#x(%#x) eintrs=%#x",
664 	    sc, intrs, EOREAD4(sc, EHCI_USBSTS), eintrs);
665 	if (!eintrs)
666 		return (0);
667 
668 	EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
669 	sc->sc_bus.no_intrs++;
670 	if (eintrs & EHCI_STS_IAA) {
671 		USBHIST_LOG(ehcidebug, "door bell", 0, 0, 0, 0);
672 		kpreempt_disable();
673 		KASSERT(sc->sc_doorbell_si != NULL);
674 		softint_schedule(sc->sc_doorbell_si);
675 		kpreempt_enable();
676 		eintrs &= ~EHCI_STS_IAA;
677 	}
678 	if (eintrs & (EHCI_STS_INT | EHCI_STS_ERRINT)) {
679 		USBHIST_LOG(ehcidebug, "INT=%d  ERRINT=%d",
680 		    eintrs & EHCI_STS_INT ? 1 : 0,
681 		    eintrs & EHCI_STS_ERRINT ? 1 : 0, 0, 0);
682 		usb_schedsoftintr(&sc->sc_bus);
683 		eintrs &= ~(EHCI_STS_INT | EHCI_STS_ERRINT);
684 	}
685 	if (eintrs & EHCI_STS_HSE) {
686 		printf("%s: unrecoverable error, controller halted\n",
687 		       device_xname(sc->sc_dev));
688 		/* XXX what else */
689 	}
690 	if (eintrs & EHCI_STS_PCD) {
691 		kpreempt_disable();
692 		KASSERT(sc->sc_pcd_si != NULL);
693 		softint_schedule(sc->sc_pcd_si);
694 		kpreempt_enable();
695 		eintrs &= ~EHCI_STS_PCD;
696 	}
697 
698 	if (eintrs != 0) {
699 		/* Block unprocessed interrupts. */
700 		sc->sc_eintrs &= ~eintrs;
701 		EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
702 		printf("%s: blocking intrs 0x%x\n",
703 		       device_xname(sc->sc_dev), eintrs);
704 	}
705 
706 	return (1);
707 }
708 
709 Static void
710 ehci_doorbell(void *addr)
711 {
712 	ehci_softc_t *sc = addr;
713 
714 	mutex_enter(&sc->sc_lock);
715 	cv_broadcast(&sc->sc_doorbell);
716 	mutex_exit(&sc->sc_lock);
717 }
718 
719 Static void
720 ehci_pcd(void *addr)
721 {
722 	ehci_softc_t *sc = addr;
723 	usbd_xfer_handle xfer;
724 	u_char *p;
725 	int i, m;
726 
727 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
728 
729 	mutex_enter(&sc->sc_lock);
730 	xfer = sc->sc_intrxfer;
731 
732 	if (xfer == NULL) {
733 		/* Just ignore the change. */
734 		goto done;
735 	}
736 
737 	p = KERNADDR(&xfer->dmabuf, 0);
738 	m = min(sc->sc_noport, xfer->length * 8 - 1);
739 	memset(p, 0, xfer->length);
740 	for (i = 1; i <= m; i++) {
741 		/* Pick out CHANGE bits from the status reg. */
742 		if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR)
743 			p[i/8] |= 1 << (i%8);
744 		if (i % 8 == 7)
745 			USBHIST_LOG(ehcidebug, "change(%d)=0x%02x", i / 8,
746 			    p[i/8], 0, 0);
747 	}
748 	xfer->actlen = xfer->length;
749 	xfer->status = USBD_NORMAL_COMPLETION;
750 
751 	usb_transfer_complete(xfer);
752 
753 done:
754 	mutex_exit(&sc->sc_lock);
755 }
756 
757 Static void
758 ehci_softintr(void *v)
759 {
760 	struct usbd_bus *bus = v;
761 	ehci_softc_t *sc = bus->hci_private;
762 	struct ehci_xfer *ex, *nextex;
763 
764 	KASSERT(sc->sc_bus.use_polling || mutex_owned(&sc->sc_lock));
765 
766 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
767 
768 	/*
769 	 * The only explanation I can think of for why EHCI is as brain dead
770 	 * as UHCI interrupt-wise is that Intel was involved in both.
771 	 * An interrupt just tells us that something is done, we have no
772 	 * clue what, so we need to scan through all active transfers. :-(
773 	 */
774 	for (ex = TAILQ_FIRST(&sc->sc_intrhead); ex; ex = nextex) {
775 		nextex = TAILQ_NEXT(ex, inext);
776 		ehci_check_intr(sc, ex);
777 	}
778 
779 	/* Schedule a callout to catch any dropped transactions. */
780 	if ((sc->sc_flags & EHCIF_DROPPED_INTR_WORKAROUND) &&
781 	    !TAILQ_EMPTY(&sc->sc_intrhead))
782 		callout_reset(&sc->sc_tmo_intrlist,
783 		    hz, ehci_intrlist_timeout, sc);
784 
785 	if (sc->sc_softwake) {
786 		sc->sc_softwake = 0;
787 		cv_broadcast(&sc->sc_softwake_cv);
788 	}
789 }
790 
791 /* Check for an interrupt. */
792 Static void
793 ehci_check_intr(ehci_softc_t *sc, struct ehci_xfer *ex)
794 {
795 	int attr;
796 
797 	USBHIST_FUNC();	USBHIST_CALLED(ehcidebug);
798 	USBHIST_LOG(ehcidebug, "ex = %p", ex, 0, 0, 0);
799 
800 	KASSERT(sc->sc_bus.use_polling || mutex_owned(&sc->sc_lock));
801 
802 	attr = ex->xfer.pipe->endpoint->edesc->bmAttributes;
803 	if (UE_GET_XFERTYPE(attr) == UE_ISOCHRONOUS)
804 		ehci_check_itd_intr(sc, ex);
805 	else
806 		ehci_check_qh_intr(sc, ex);
807 
808 	return;
809 }
810 
811 Static void
812 ehci_check_qh_intr(ehci_softc_t *sc, struct ehci_xfer *ex)
813 {
814 	ehci_soft_qtd_t *sqtd, *lsqtd;
815 	__uint32_t status;
816 
817 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
818 
819 	KASSERT(sc->sc_bus.use_polling || mutex_owned(&sc->sc_lock));
820 
821 	if (ex->sqtdstart == NULL) {
822 		printf("ehci_check_qh_intr: not valid sqtd\n");
823 		return;
824 	}
825 
826 	lsqtd = ex->sqtdend;
827 #ifdef DIAGNOSTIC
828 	if (lsqtd == NULL) {
829 		printf("ehci_check_qh_intr: lsqtd==0\n");
830 		return;
831 	}
832 #endif
833 	/*
834 	 * If the last TD is still active we need to check whether there
835 	 * is an error somewhere in the middle, or whether there was a
836 	 * short packet (SPD and not ACTIVE).
837 	 */
838 	usb_syncmem(&lsqtd->dma,
839 	    lsqtd->offs + offsetof(ehci_qtd_t, qtd_status),
840 	    sizeof(lsqtd->qtd.qtd_status),
841 	    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
842 	status = le32toh(lsqtd->qtd.qtd_status);
843 	usb_syncmem(&lsqtd->dma,
844 	    lsqtd->offs + offsetof(ehci_qtd_t, qtd_status),
845 	    sizeof(lsqtd->qtd.qtd_status), BUS_DMASYNC_PREREAD);
846 	if (status & EHCI_QTD_ACTIVE) {
847 		USBHIST_LOGN(ehcidebug, 10, "active ex=%p", ex, 0, 0, 0);
848 		for (sqtd = ex->sqtdstart; sqtd != lsqtd; sqtd=sqtd->nextqtd) {
849 			usb_syncmem(&sqtd->dma,
850 			    sqtd->offs + offsetof(ehci_qtd_t, qtd_status),
851 			    sizeof(sqtd->qtd.qtd_status),
852 			    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
853 			status = le32toh(sqtd->qtd.qtd_status);
854 			usb_syncmem(&sqtd->dma,
855 			    sqtd->offs + offsetof(ehci_qtd_t, qtd_status),
856 			    sizeof(sqtd->qtd.qtd_status), BUS_DMASYNC_PREREAD);
857 			/* If there's an active QTD the xfer isn't done. */
858 			if (status & EHCI_QTD_ACTIVE)
859 				break;
860 			/* Any kind of error makes the xfer done. */
861 			if (status & EHCI_QTD_HALTED)
862 				goto done;
863 			/* Handle short packets */
864 			if (EHCI_QTD_GET_BYTES(status) != 0) {
865 				usbd_pipe_handle pipe = ex->xfer.pipe;
866 				usb_endpoint_descriptor_t *ed =
867 				    pipe->endpoint->edesc;
868 				uint8_t xt = UE_GET_XFERTYPE(ed->bmAttributes);
869 
870 				/*
871 				 * If we get here for a control transfer then
872 				 * we need to let the hardware complete the
873 				 * status phase.  That is, we're not done
874 				 * quite yet.
875 				 *
876 				 * Otherwise, we're done.
877 				 */
878 				if (xt == UE_CONTROL) {
879 					break;
880 				}
881 				goto done;
882 			}
883 		}
884 		USBHIST_LOGN(ehcidebug, 10, "ex=%p std=%p still active",
885 		    ex, ex->sqtdstart, 0, 0);
886 #ifdef EHCI_DEBUG
887 		USBHIST_LOGN(ehcidebug, 5, "--- still active start ---", 0, 0, 0, 0);
888 		ehci_dump_sqtds(ex->sqtdstart);
889 		USBHIST_LOGN(ehcidebug, 5, "--- still active end ---", 0, 0, 0, 0);
890 #endif
891 
892 		return;
893 	}
894  done:
895 	USBHIST_LOGN(ehcidebug, 10, "ex=%p done", ex, 0, 0, 0);
896 	callout_stop(&ex->xfer.timeout_handle);
897 	ehci_idone(ex);
898 }
899 
900 Static void
901 ehci_check_itd_intr(ehci_softc_t *sc, struct ehci_xfer *ex)
902 {
903 	ehci_soft_itd_t *itd;
904 	int i;
905 
906 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
907 
908 	KASSERT(mutex_owned(&sc->sc_lock));
909 
910 	if (&ex->xfer != SIMPLEQ_FIRST(&ex->xfer.pipe->queue))
911 		return;
912 
913 	if (ex->itdstart == NULL) {
914 		printf("ehci_check_itd_intr: not valid itd\n");
915 		return;
916 	}
917 
918 	itd = ex->itdend;
919 #ifdef DIAGNOSTIC
920 	if (itd == NULL) {
921 		printf("ehci_check_itd_intr: itdend == 0\n");
922 		return;
923 	}
924 #endif
925 
926 	/*
927 	 * check no active transfers in last itd, meaning we're finished
928 	 */
929 
930 	usb_syncmem(&itd->dma, itd->offs + offsetof(ehci_itd_t, itd_ctl),
931 		    sizeof(itd->itd.itd_ctl), BUS_DMASYNC_POSTWRITE |
932 		    BUS_DMASYNC_POSTREAD);
933 
934 	for (i = 0; i < EHCI_ITD_NUFRAMES; i++) {
935 		if (le32toh(itd->itd.itd_ctl[i]) & EHCI_ITD_ACTIVE)
936 			break;
937 	}
938 
939 	if (i == EHCI_ITD_NUFRAMES) {
940 		goto done; /* All 8 descriptors inactive, it's done */
941 	}
942 
943 	USBHIST_LOGN(ehcidebug, 10, "ex %p itd %p still active", ex,
944 	    ex->itdstart, 0, 0);
945 	return;
946 done:
947 	USBHIST_LOG(ehcidebug, "ex %p done", ex, 0, 0, 0);
948 	callout_stop(&ex->xfer.timeout_handle);
949 	ehci_idone(ex);
950 }
951 
952 Static void
953 ehci_idone(struct ehci_xfer *ex)
954 {
955 	usbd_xfer_handle xfer = &ex->xfer;
956 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
957 	struct ehci_softc *sc = xfer->pipe->device->bus->hci_private;
958 	ehci_soft_qtd_t *sqtd, *lsqtd;
959 	u_int32_t status = 0, nstatus = 0;
960 	int actlen;
961 
962 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
963 
964 	KASSERT(sc->sc_bus.use_polling || mutex_owned(&sc->sc_lock));
965 
966 	USBHIST_LOG(ehcidebug, "ex=%p", ex, 0, 0, 0);
967 
968 #ifdef DIAGNOSTIC
969 	if (ex->isdone) {
970 		printf("ehci_idone: ex=%p is done!\n", ex);
971 #ifdef EHCI_DEBUG
972 		ehci_dump_exfer(ex);
973 #endif
974 		return;
975 	}
976 	ex->isdone = 1;
977 #endif
978 
979 	if (xfer->status == USBD_CANCELLED ||
980 	    xfer->status == USBD_TIMEOUT) {
981 		USBHIST_LOG(ehcidebug, "aborted xfer=%p", xfer, 0, 0, 0);
982 		return;
983 	}
984 
985 	USBHIST_LOG(ehcidebug, "xfer=%p, pipe=%p ready", xfer, epipe, 0, 0);
986 #ifdef EHCI_DEBUG
987 	ehci_dump_sqtds(ex->sqtdstart);
988 #endif
989 
990 	/* The transfer is done, compute actual length and status. */
991 
992 	if (UE_GET_XFERTYPE(xfer->pipe->endpoint->edesc->bmAttributes)
993 				== UE_ISOCHRONOUS) {
994 		/* Isoc transfer */
995 		struct ehci_soft_itd *itd;
996 		int i, nframes, len, uframes;
997 
998 		nframes = 0;
999 		actlen = 0;
1000 
1001 		i = xfer->pipe->endpoint->edesc->bInterval;
1002 		uframes = min(1 << (i - 1), USB_UFRAMES_PER_FRAME);
1003 
1004 		for (itd = ex->itdstart; itd != NULL; itd = itd->xfer_next) {
1005 			usb_syncmem(&itd->dma,itd->offs + offsetof(ehci_itd_t,itd_ctl),
1006 			    sizeof(itd->itd.itd_ctl), BUS_DMASYNC_POSTWRITE |
1007 			    BUS_DMASYNC_POSTREAD);
1008 
1009 			for (i = 0; i < EHCI_ITD_NUFRAMES; i += uframes) {
1010 				/* XXX - driver didn't fill in the frame full
1011 				 *   of uframes. This leads to scheduling
1012 				 *   inefficiencies, but working around
1013 				 *   this doubles complexity of tracking
1014 				 *   an xfer.
1015 				 */
1016 				if (nframes >= xfer->nframes)
1017 					break;
1018 
1019 				status = le32toh(itd->itd.itd_ctl[i]);
1020 				len = EHCI_ITD_GET_LEN(status);
1021 				if (EHCI_ITD_GET_STATUS(status) != 0)
1022 					len = 0; /*No valid data on error*/
1023 
1024 				xfer->frlengths[nframes++] = len;
1025 				actlen += len;
1026 			}
1027 
1028 			if (nframes >= xfer->nframes)
1029 				break;
1030 	    	}
1031 
1032 		xfer->actlen = actlen;
1033 		xfer->status = USBD_NORMAL_COMPLETION;
1034 		goto end;
1035 	}
1036 
1037 	/* Continue processing xfers using queue heads */
1038 
1039 	lsqtd = ex->sqtdend;
1040 	actlen = 0;
1041 	for (sqtd = ex->sqtdstart; sqtd != lsqtd->nextqtd;
1042 	     sqtd = sqtd->nextqtd) {
1043 		usb_syncmem(&sqtd->dma, sqtd->offs, sizeof(sqtd->qtd),
1044 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1045 		nstatus = le32toh(sqtd->qtd.qtd_status);
1046 		if (nstatus & EHCI_QTD_ACTIVE)
1047 			break;
1048 
1049 		status = nstatus;
1050 		if (EHCI_QTD_GET_PID(status) != EHCI_QTD_PID_SETUP)
1051 			actlen += sqtd->len - EHCI_QTD_GET_BYTES(status);
1052 	}
1053 
1054 
1055 	/*
1056 	 * If there are left over TDs we need to update the toggle.
1057 	 * The default pipe doesn't need it since control transfers
1058 	 * start the toggle at 0 every time.
1059 	 * For a short transfer we need to update the toggle for the missing
1060 	 * packets within the qTD.
1061 	 */
1062 	if ((sqtd != lsqtd->nextqtd || EHCI_QTD_GET_BYTES(status)) &&
1063 	    xfer->pipe->device->default_pipe != xfer->pipe) {
1064 		USBHIST_LOG(ehcidebug,
1065 		    "toggle update status=0x%08x nstatus=0x%08x",
1066 		    status, nstatus, 0, 0);
1067 #if 0
1068 		ehci_dump_sqh(epipe->sqh);
1069 		ehci_dump_sqtds(ex->sqtdstart);
1070 #endif
1071 		epipe->nexttoggle = EHCI_QTD_GET_TOGGLE(nstatus);
1072 	}
1073 
1074 	USBHIST_LOG(ehcidebug, "len=%d actlen=%d status=0x%08x", xfer->length,
1075 	    actlen, status, 0);
1076 	xfer->actlen = actlen;
1077 	if (status & EHCI_QTD_HALTED) {
1078 #ifdef EHCI_DEBUG
1079 		USBHIST_LOG(ehcidebug, "halted addr=%d endpt=0x%02x",
1080 		    xfer->pipe->device->address,
1081 		    xfer->pipe->endpoint->edesc->bEndpointAddress, 0, 0);
1082 		USBHIST_LOG(ehcidebug, "cerr=%d pid=%d stat=%#x",
1083 		    EHCI_QTD_GET_CERR(status), EHCI_QTD_GET_PID(status),
1084 		    status, 0);
1085 		USBHIST_LOG(ehcidebug,
1086 		    "active =%d halted=%d buferr=%d babble=%d",
1087 		    status & EHCI_QTD_ACTIVE ? 1 : 0,
1088 		    status & EHCI_QTD_HALTED ? 1 : 0,
1089 		    status & EHCI_QTD_BUFERR ? 1 : 0,
1090 		    status & EHCI_QTD_BABBLE ? 1 : 0);
1091 
1092 		USBHIST_LOG(ehcidebug,
1093 		    "xacterr=%d missed=%d split =%d ping  =%d",
1094 		    status & EHCI_QTD_XACTERR ? 1 : 0,
1095 		    status & EHCI_QTD_MISSEDMICRO ? 1 : 0,
1096 		    status & EHCI_QTD_SPLITXSTATE ? 1 : 0,
1097 		    status & EHCI_QTD_PINGSTATE ? 1 : 0);
1098 
1099 		ehci_dump_sqh(epipe->sqh);
1100 		ehci_dump_sqtds(ex->sqtdstart);
1101 #endif
1102 		/* low&full speed has an extra error flag */
1103 		if (EHCI_QH_GET_EPS(epipe->sqh->qh.qh_endp) !=
1104 		    EHCI_QH_SPEED_HIGH)
1105 			status &= EHCI_QTD_STATERRS | EHCI_QTD_PINGSTATE;
1106 		else
1107 			status &= EHCI_QTD_STATERRS;
1108 		if (status == 0) /* no other errors means a stall */ {
1109 			xfer->status = USBD_STALLED;
1110 		} else {
1111 			xfer->status = USBD_IOERROR; /* more info XXX */
1112 		}
1113 		/* XXX need to reset TT on missed microframe */
1114 		if (status & EHCI_QTD_MISSEDMICRO) {
1115 			printf("%s: missed microframe, TT reset not "
1116 			    "implemented, hub might be inoperational\n",
1117 			    device_xname(sc->sc_dev));
1118 		}
1119 	} else {
1120 		xfer->status = USBD_NORMAL_COMPLETION;
1121 	}
1122 
1123     end:
1124 	/* XXX transfer_complete memcpys out transfer data (for in endpoints)
1125 	 * during this call, before methods->done is called: dma sync required
1126 	 * beforehand? */
1127 	usb_transfer_complete(xfer);
1128 	USBHIST_LOG(ehcidebug, "ex=%p done", ex, 0, 0, 0);
1129 }
1130 
1131 /*
1132  * Wait here until controller claims to have an interrupt.
1133  * Then call ehci_intr and return.  Use timeout to avoid waiting
1134  * too long.
1135  */
1136 Static void
1137 ehci_waitintr(ehci_softc_t *sc, usbd_xfer_handle xfer)
1138 {
1139 	int timo;
1140 	u_int32_t intrs;
1141 
1142 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
1143 
1144 	xfer->status = USBD_IN_PROGRESS;
1145 	for (timo = xfer->timeout; timo >= 0; timo--) {
1146 		usb_delay_ms(&sc->sc_bus, 1);
1147 		if (sc->sc_dying)
1148 			break;
1149 		intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS)) &
1150 			sc->sc_eintrs;
1151 		USBHIST_LOG(ehcidebug, "0x%04x", intrs, 0, 0, 0);
1152 #ifdef EHCI_DEBUG
1153 		if (ehcidebug > 15)
1154 			ehci_dump_regs(sc);
1155 #endif
1156 		if (intrs) {
1157 			mutex_spin_enter(&sc->sc_intr_lock);
1158 			ehci_intr1(sc);
1159 			mutex_spin_exit(&sc->sc_intr_lock);
1160 			if (xfer->status != USBD_IN_PROGRESS)
1161 				return;
1162 		}
1163 	}
1164 
1165 	/* Timeout */
1166 	USBHIST_LOG(ehcidebug, "timeout", 0, 0, 0, 0);
1167 	xfer->status = USBD_TIMEOUT;
1168 	mutex_enter(&sc->sc_lock);
1169 	usb_transfer_complete(xfer);
1170 	mutex_exit(&sc->sc_lock);
1171 	/* XXX should free TD */
1172 }
1173 
1174 Static void
1175 ehci_poll(struct usbd_bus *bus)
1176 {
1177 	ehci_softc_t *sc = bus->hci_private;
1178 
1179 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
1180 
1181 #ifdef EHCI_DEBUG
1182 	static int last;
1183 	int new;
1184 	new = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
1185 	if (new != last) {
1186 		USBHIST_LOG(ehcidebug, "intrs=0x%04x", new, 0, 0, 0);
1187 		last = new;
1188 	}
1189 #endif
1190 
1191 	if (EOREAD4(sc, EHCI_USBSTS) & sc->sc_eintrs) {
1192 		mutex_spin_enter(&sc->sc_intr_lock);
1193 		ehci_intr1(sc);
1194 		mutex_spin_exit(&sc->sc_intr_lock);
1195 	}
1196 }
1197 
1198 void
1199 ehci_childdet(device_t self, device_t child)
1200 {
1201 	struct ehci_softc *sc = device_private(self);
1202 
1203 	KASSERT(sc->sc_child == child);
1204 	sc->sc_child = NULL;
1205 }
1206 
1207 int
1208 ehci_detach(struct ehci_softc *sc, int flags)
1209 {
1210 	int rv = 0;
1211 
1212 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
1213 
1214 	if (sc->sc_child != NULL)
1215 		rv = config_detach(sc->sc_child, flags);
1216 
1217 	if (rv != 0)
1218 		return (rv);
1219 
1220 	callout_halt(&sc->sc_tmo_intrlist, NULL);
1221 	callout_destroy(&sc->sc_tmo_intrlist);
1222 
1223 	/* XXX free other data structures XXX */
1224 	if (sc->sc_softitds)
1225 		kmem_free(sc->sc_softitds,
1226 		    sc->sc_flsize * sizeof(ehci_soft_itd_t *));
1227 	cv_destroy(&sc->sc_doorbell);
1228 	cv_destroy(&sc->sc_softwake_cv);
1229 
1230 #if 0
1231 	/* XXX destroyed in ehci_pci.c as it controls ehci_intr access */
1232 
1233 	softint_disestablish(sc->sc_doorbell_si);
1234 	softint_disestablish(sc->sc_pcd_si);
1235 
1236 	mutex_destroy(&sc->sc_lock);
1237 	mutex_destroy(&sc->sc_intr_lock);
1238 #endif
1239 
1240 	pool_cache_destroy(sc->sc_xferpool);
1241 
1242 	EOWRITE4(sc, EHCI_CONFIGFLAG, 0);
1243 
1244 	return (rv);
1245 }
1246 
1247 
1248 int
1249 ehci_activate(device_t self, enum devact act)
1250 {
1251 	struct ehci_softc *sc = device_private(self);
1252 
1253 	switch (act) {
1254 	case DVACT_DEACTIVATE:
1255 		sc->sc_dying = 1;
1256 		return 0;
1257 	default:
1258 		return EOPNOTSUPP;
1259 	}
1260 }
1261 
1262 /*
1263  * Handle suspend/resume.
1264  *
1265  * We need to switch to polling mode here, because this routine is
1266  * called from an interrupt context.  This is all right since we
1267  * are almost suspended anyway.
1268  *
1269  * Note that this power handler isn't to be registered directly; the
1270  * bus glue needs to call out to it.
1271  */
1272 bool
1273 ehci_suspend(device_t dv, const pmf_qual_t *qual)
1274 {
1275 	ehci_softc_t *sc = device_private(dv);
1276 	int i;
1277 	uint32_t cmd, hcr;
1278 
1279 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
1280 
1281 	mutex_spin_enter(&sc->sc_intr_lock);
1282 	sc->sc_bus.use_polling++;
1283 	mutex_spin_exit(&sc->sc_intr_lock);
1284 
1285 	for (i = 1; i <= sc->sc_noport; i++) {
1286 		cmd = EOREAD4(sc, EHCI_PORTSC(i)) & ~EHCI_PS_CLEAR;
1287 		if ((cmd & EHCI_PS_PO) == 0 && (cmd & EHCI_PS_PE) == EHCI_PS_PE)
1288 			EOWRITE4(sc, EHCI_PORTSC(i), cmd | EHCI_PS_SUSP);
1289 	}
1290 
1291 	sc->sc_cmd = EOREAD4(sc, EHCI_USBCMD);
1292 
1293 	cmd = sc->sc_cmd & ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
1294 	EOWRITE4(sc, EHCI_USBCMD, cmd);
1295 
1296 	for (i = 0; i < 100; i++) {
1297 		hcr = EOREAD4(sc, EHCI_USBSTS) & (EHCI_STS_ASS | EHCI_STS_PSS);
1298 		if (hcr == 0)
1299 			break;
1300 
1301 		usb_delay_ms(&sc->sc_bus, 1);
1302 	}
1303 	if (hcr != 0)
1304 		printf("%s: reset timeout\n", device_xname(dv));
1305 
1306 	cmd &= ~EHCI_CMD_RS;
1307 	EOWRITE4(sc, EHCI_USBCMD, cmd);
1308 
1309 	for (i = 0; i < 100; i++) {
1310 		hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1311 		if (hcr == EHCI_STS_HCH)
1312 			break;
1313 
1314 		usb_delay_ms(&sc->sc_bus, 1);
1315 	}
1316 	if (hcr != EHCI_STS_HCH)
1317 		printf("%s: config timeout\n", device_xname(dv));
1318 
1319 	mutex_spin_enter(&sc->sc_intr_lock);
1320 	sc->sc_bus.use_polling--;
1321 	mutex_spin_exit(&sc->sc_intr_lock);
1322 
1323 	return true;
1324 }
1325 
1326 bool
1327 ehci_resume(device_t dv, const pmf_qual_t *qual)
1328 {
1329 	ehci_softc_t *sc = device_private(dv);
1330 	int i;
1331 	uint32_t cmd, hcr;
1332 
1333 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
1334 
1335 	/* restore things in case the bios sucks */
1336 	EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
1337 	EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
1338 	EOWRITE4(sc, EHCI_ASYNCLISTADDR,
1339 	    sc->sc_async_head->physaddr | EHCI_LINK_QH);
1340 
1341 	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs & ~EHCI_INTR_PCIE);
1342 
1343 	EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
1344 
1345 	hcr = 0;
1346 	for (i = 1; i <= sc->sc_noport; i++) {
1347 		cmd = EOREAD4(sc, EHCI_PORTSC(i)) & ~EHCI_PS_CLEAR;
1348 		if ((cmd & EHCI_PS_PO) == 0 &&
1349 		    (cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP) {
1350 			EOWRITE4(sc, EHCI_PORTSC(i), cmd | EHCI_PS_FPR);
1351 			hcr = 1;
1352 		}
1353 	}
1354 
1355 	if (hcr) {
1356 		usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);
1357 
1358 		for (i = 1; i <= sc->sc_noport; i++) {
1359 			cmd = EOREAD4(sc, EHCI_PORTSC(i)) & ~EHCI_PS_CLEAR;
1360 			if ((cmd & EHCI_PS_PO) == 0 &&
1361 			    (cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)
1362 				EOWRITE4(sc, EHCI_PORTSC(i),
1363 				    cmd & ~EHCI_PS_FPR);
1364 		}
1365 	}
1366 
1367 	EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
1368 	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1369 
1370 	for (i = 0; i < 100; i++) {
1371 		hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1372 		if (hcr != EHCI_STS_HCH)
1373 			break;
1374 
1375 		usb_delay_ms(&sc->sc_bus, 1);
1376 	}
1377 	if (hcr == EHCI_STS_HCH)
1378 		printf("%s: config timeout\n", device_xname(dv));
1379 
1380 	return true;
1381 }
1382 
1383 /*
1384  * Shut down the controller when the system is going down.
1385  */
1386 bool
1387 ehci_shutdown(device_t self, int flags)
1388 {
1389 	ehci_softc_t *sc = device_private(self);
1390 
1391 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
1392 
1393 	EOWRITE4(sc, EHCI_USBCMD, 0);	/* Halt controller */
1394 	EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
1395 	return true;
1396 }
1397 
1398 Static usbd_status
1399 ehci_allocm(struct usbd_bus *bus, usb_dma_t *dma, u_int32_t size)
1400 {
1401 	struct ehci_softc *sc = bus->hci_private;
1402 	usbd_status err;
1403 
1404 	err = usb_allocmem_flags(&sc->sc_bus, size, 0, dma, USBMALLOC_MULTISEG);
1405 #ifdef EHCI_DEBUG
1406 	if (err)
1407 		printf("ehci_allocm: usb_allocmem_flags()= %s (%d)\n",
1408 			usbd_errstr(err), err);
1409 #endif
1410 	if (err == USBD_NOMEM)
1411 		err = usb_reserve_allocm(&sc->sc_dma_reserve, dma, size);
1412 #ifdef EHCI_DEBUG
1413 	if (err)
1414 		printf("ehci_allocm: usb_reserve_allocm()= %s (%d)\n",
1415 			usbd_errstr(err), err);
1416 #endif
1417 	return (err);
1418 }
1419 
1420 Static void
1421 ehci_freem(struct usbd_bus *bus, usb_dma_t *dma)
1422 {
1423 	struct ehci_softc *sc = bus->hci_private;
1424 
1425 	if (dma->block->flags & USB_DMA_RESERVE) {
1426 		usb_reserve_freem(&sc->sc_dma_reserve,
1427 		    dma);
1428 		return;
1429 	}
1430 	usb_freemem(&sc->sc_bus, dma);
1431 }
1432 
1433 Static usbd_xfer_handle
1434 ehci_allocx(struct usbd_bus *bus)
1435 {
1436 	struct ehci_softc *sc = bus->hci_private;
1437 	usbd_xfer_handle xfer;
1438 
1439 	xfer = pool_cache_get(sc->sc_xferpool, PR_NOWAIT);
1440 	if (xfer != NULL) {
1441 		memset(xfer, 0, sizeof(struct ehci_xfer));
1442 #ifdef DIAGNOSTIC
1443 		EXFER(xfer)->isdone = 1;
1444 		xfer->busy_free = XFER_BUSY;
1445 #endif
1446 	}
1447 	return (xfer);
1448 }
1449 
1450 Static void
1451 ehci_freex(struct usbd_bus *bus, usbd_xfer_handle xfer)
1452 {
1453 	struct ehci_softc *sc = bus->hci_private;
1454 
1455 #ifdef DIAGNOSTIC
1456 	if (xfer->busy_free != XFER_BUSY) {
1457 		printf("ehci_freex: xfer=%p not busy, 0x%08x\n", xfer,
1458 		       xfer->busy_free);
1459 	}
1460 	xfer->busy_free = XFER_FREE;
1461 	if (!EXFER(xfer)->isdone) {
1462 		printf("ehci_freex: !isdone\n");
1463 	}
1464 #endif
1465 	pool_cache_put(sc->sc_xferpool, xfer);
1466 }
1467 
1468 Static void
1469 ehci_get_lock(struct usbd_bus *bus, kmutex_t **lock)
1470 {
1471 	struct ehci_softc *sc = bus->hci_private;
1472 
1473 	*lock = &sc->sc_lock;
1474 }
1475 
1476 Static void
1477 ehci_device_clear_toggle(usbd_pipe_handle pipe)
1478 {
1479 	struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1480 
1481 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
1482 
1483 	USBHIST_LOG(ehcidebug, "epipe=%p status=0x%08x",
1484 	    epipe, epipe->sqh->qh.qh_qtd.qtd_status, 0, 0);
1485 #ifdef EHCI_DEBUG
1486 	if (ehcidebug)
1487 		usbd_dump_pipe(pipe);
1488 #endif
1489 	epipe->nexttoggle = 0;
1490 }
1491 
1492 Static void
1493 ehci_noop(usbd_pipe_handle pipe)
1494 {
1495 }
1496 
1497 #ifdef EHCI_DEBUG
1498 /*
1499  * Unused function - this is meant to be called from a kernel
1500  * debugger.
1501  */
1502 void
1503 ehci_dump(void)
1504 {
1505 	ehci_softc_t *sc = theehci;
1506 	int i;
1507 	printf("cmd=0x%08x, sts=0x%08x, ien=0x%08x\n",
1508 	    EOREAD4(sc, EHCI_USBCMD),
1509 	    EOREAD4(sc, EHCI_USBSTS),
1510 	    EOREAD4(sc, EHCI_USBINTR));
1511 	printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
1512 	    EOREAD4(sc, EHCI_FRINDEX),
1513 	    EOREAD4(sc, EHCI_CTRLDSSEGMENT),
1514 	    EOREAD4(sc, EHCI_PERIODICLISTBASE),
1515 	    EOREAD4(sc, EHCI_ASYNCLISTADDR));
1516 	for (i = 1; i <= sc->sc_noport; i++)
1517 		printf("port %d status=0x%08x\n", i,
1518 		    EOREAD4(sc, EHCI_PORTSC(i)));
1519 }
1520 
1521 Static void
1522 ehci_dump_regs(ehci_softc_t *sc)
1523 {
1524 	int i;
1525 
1526 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
1527 
1528 	USBHIST_LOG(ehcidebug,
1529 	    "cmd     = 0x%08x  sts      = 0x%08x  ien      = 0x%08x",
1530 	    EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS),
1531 	    EOREAD4(sc, EHCI_USBINTR), 0);
1532 	USBHIST_LOG(ehcidebug,
1533 	    "frindex = 0x%08x  ctrdsegm = 0x%08x  periodic = 0x%08x  "
1534 	    "async   = 0x%08x",
1535 	    EOREAD4(sc, EHCI_FRINDEX), EOREAD4(sc, EHCI_CTRLDSSEGMENT),
1536 	    EOREAD4(sc, EHCI_PERIODICLISTBASE),
1537 	    EOREAD4(sc, EHCI_ASYNCLISTADDR));
1538 	for (i = 1; i <= sc->sc_noport; i += 2) {
1539 		if (i == sc->sc_noport) {
1540 			USBHIST_LOG(ehcidebug,
1541 			    "port %d status = 0x%08x", i,
1542 			    EOREAD4(sc, EHCI_PORTSC(i)), 0, 0);
1543 		} else {
1544 			USBHIST_LOG(ehcidebug,
1545 			    "port %d status = 0x%08x  port %d status = 0x%08x",
1546 			    i, EOREAD4(sc, EHCI_PORTSC(i)),
1547 			    i+1, EOREAD4(sc, EHCI_PORTSC(i+1)));
1548 		}
1549 	}
1550 }
1551 
1552 #ifdef EHCI_DEBUG
1553 #define ehci_dump_link(link, type) do {					\
1554 	USBHIST_LOG(ehcidebug, "    link 0x%08x (T = %d):",		\
1555 	    link,							\
1556 	    link & EHCI_LINK_TERMINATE ? 1 : 0, 0, 0);			\
1557 	if (type) {							\
1558 		USBHIST_LOG(ehcidebug,					\
1559 		    "        ITD  = %d  QH   = %d  SITD = %d  FSTN = %d",\
1560 		    EHCI_LINK_TYPE(link) == EHCI_LINK_ITD ? 1 : 0,	\
1561 		    EHCI_LINK_TYPE(link) == EHCI_LINK_QH ? 1 : 0,	\
1562 		    EHCI_LINK_TYPE(link) == EHCI_LINK_SITD ? 1 : 0,	\
1563 		    EHCI_LINK_TYPE(link) == EHCI_LINK_FSTN ? 1 : 0);	\
1564 	}								\
1565 } while(0)
1566 #else
1567 #define ehci_dump_link(link, type)
1568 #endif
1569 
1570 Static void
1571 ehci_dump_sqtds(ehci_soft_qtd_t *sqtd)
1572 {
1573 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
1574 	int i;
1575 	uint32_t stop = 0;
1576 
1577 	for (i = 0; sqtd && i < 20 && !stop; sqtd = sqtd->nextqtd, i++) {
1578 		ehci_dump_sqtd(sqtd);
1579 		usb_syncmem(&sqtd->dma,
1580 		    sqtd->offs + offsetof(ehci_qtd_t, qtd_next),
1581 		    sizeof(sqtd->qtd),
1582 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1583 		stop = sqtd->qtd.qtd_next & htole32(EHCI_LINK_TERMINATE);
1584 		usb_syncmem(&sqtd->dma,
1585 		    sqtd->offs + offsetof(ehci_qtd_t, qtd_next),
1586 		    sizeof(sqtd->qtd), BUS_DMASYNC_PREREAD);
1587 	}
1588 	if (!stop)
1589 		USBHIST_LOG(ehcidebug,
1590 		    "dump aborted, too many TDs", 0, 0, 0, 0);
1591 }
1592 
1593 Static void
1594 ehci_dump_sqtd(ehci_soft_qtd_t *sqtd)
1595 {
1596 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
1597 
1598 	usb_syncmem(&sqtd->dma, sqtd->offs,
1599 	    sizeof(sqtd->qtd), BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1600 
1601 	USBHIST_LOGN(ehcidebug, 10,
1602 	    "QTD(%p) at 0x%08x:", sqtd, sqtd->physaddr, 0, 0);
1603 	ehci_dump_qtd(&sqtd->qtd);
1604 
1605 	usb_syncmem(&sqtd->dma, sqtd->offs,
1606 	    sizeof(sqtd->qtd), BUS_DMASYNC_PREREAD);
1607 }
1608 
1609 Static void
1610 ehci_dump_qtd(ehci_qtd_t *qtd)
1611 {
1612 	USBHIST_FUNC();	USBHIST_CALLED(ehcidebug);
1613 
1614 #ifdef USBHIST
1615 	uint32_t s = le32toh(qtd->qtd_status);
1616 #endif
1617 
1618 	USBHIST_LOGN(ehcidebug, 10,
1619 	    "     next = 0x%08x  altnext = 0x%08x  status = 0x%08x",
1620 	    qtd->qtd_next, qtd->qtd_altnext, s, 0);
1621 	USBHIST_LOGN(ehcidebug, 10,
1622 	    "   toggle = %d ioc = %d bytes = %#x "
1623 	    "c_page = %#x", EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_IOC(s),
1624 	    EHCI_QTD_GET_BYTES(s), EHCI_QTD_GET_C_PAGE(s));
1625 	USBHIST_LOGN(ehcidebug, 10,
1626 	    "     cerr = %d pid = %d stat  = %x",
1627 	    EHCI_QTD_GET_CERR(s), EHCI_QTD_GET_PID(s), EHCI_QTD_GET_STATUS(s),
1628 	    0);
1629 	USBHIST_LOGN(ehcidebug, 10,
1630 	    "active =%d halted=%d buferr=%d babble=%d",
1631 	    s & EHCI_QTD_ACTIVE ? 1 : 0,
1632 	    s & EHCI_QTD_HALTED ? 1 : 0,
1633 	    s & EHCI_QTD_BUFERR ? 1 : 0,
1634 	    s & EHCI_QTD_BABBLE ? 1 : 0);
1635 	USBHIST_LOGN(ehcidebug, 10,
1636 	    "xacterr=%d missed=%d split =%d ping  =%d",
1637 	    s & EHCI_QTD_XACTERR ? 1 : 0,
1638 	    s & EHCI_QTD_MISSEDMICRO ? 1 : 0,
1639 	    s & EHCI_QTD_SPLITXSTATE ? 1 : 0,
1640 	    s & EHCI_QTD_PINGSTATE ? 1 : 0);
1641 	USBHIST_LOGN(ehcidebug, 10,
1642 	    "buffer[0] = %#x  buffer[1] = %#x  "
1643 	    "buffer[2] = %#x  buffer[3] = %#x",
1644 	    le32toh(qtd->qtd_buffer[0]), le32toh(qtd->qtd_buffer[1]),
1645 	    le32toh(qtd->qtd_buffer[2]), le32toh(qtd->qtd_buffer[3]));
1646 	USBHIST_LOGN(ehcidebug, 10,
1647 	    "buffer[4] = %#x", le32toh(qtd->qtd_buffer[4]), 0, 0, 0);
1648 }
1649 
1650 Static void
1651 ehci_dump_sqh(ehci_soft_qh_t *sqh)
1652 {
1653 #ifdef USBHIST
1654 	ehci_qh_t *qh = &sqh->qh;
1655 	ehci_link_t link;
1656 #endif
1657 	u_int32_t endp, endphub;
1658 	USBHIST_FUNC();	USBHIST_CALLED(ehcidebug);
1659 
1660 	usb_syncmem(&sqh->dma, sqh->offs,
1661 	    sizeof(sqh->qh), BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1662 
1663 	USBHIST_LOGN(ehcidebug, 10,
1664 	    "QH(%p) at %#x:", sqh, sqh->physaddr, 0, 0);
1665 	link = le32toh(qh->qh_link);
1666 	ehci_dump_link(link, true);
1667 
1668 	endp = le32toh(qh->qh_endp);
1669 	USBHIST_LOGN(ehcidebug, 10,
1670 	    "    endp = %#x", endp, 0, 0, 0);
1671 	USBHIST_LOGN(ehcidebug, 10,
1672 	    "        addr = 0x%02x  inact = %d  endpt = %d  eps = %d",
1673 	    EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
1674 	    EHCI_QH_GET_ENDPT(endp), EHCI_QH_GET_EPS(endp));
1675 	USBHIST_LOGN(ehcidebug, 10,
1676 	    "        dtc  = %d     hrecl = %d",
1677 	    EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp), 0, 0);
1678 	USBHIST_LOGN(ehcidebug, 10,
1679 	    "        ctl  = %d     nrl   = %d  mpl   = %#x(%d)",
1680 	    EHCI_QH_GET_CTL(endp),EHCI_QH_GET_NRL(endp),
1681 	    EHCI_QH_GET_MPL(endp), EHCI_QH_GET_MPL(endp));
1682 
1683 	endphub = le32toh(qh->qh_endphub);
1684 	USBHIST_LOGN(ehcidebug, 10,
1685 	    " endphub = %#x", endphub, 0, 0, 0);
1686 	USBHIST_LOGN(ehcidebug, 10,
1687 	    "      smask = 0x%02x  cmask = 0x%02x",
1688 	    EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub), 1, 0);
1689 	USBHIST_LOGN(ehcidebug, 10,
1690 	    "      huba  = 0x%02x  port  = %d  mult = %d",
1691 	    EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
1692 	    EHCI_QH_GET_MULT(endphub), 0);
1693 
1694 	link = le32toh(qh->qh_curqtd);
1695 	ehci_dump_link(link, false);
1696 	USBHIST_LOGN(ehcidebug, 10, "Overlay qTD:", 0, 0, 0, 0);
1697 	ehci_dump_qtd(&qh->qh_qtd);
1698 
1699 	usb_syncmem(&sqh->dma, sqh->offs,
1700 	    sizeof(sqh->qh), BUS_DMASYNC_PREREAD);
1701 }
1702 
1703 Static void
1704 ehci_dump_itd(struct ehci_soft_itd *itd)
1705 {
1706 	ehci_isoc_trans_t t;
1707 	ehci_isoc_bufr_ptr_t b, b2, b3;
1708 	int i;
1709 
1710 	USBHIST_FUNC();	USBHIST_CALLED(ehcidebug);
1711 
1712 	USBHIST_LOG(ehcidebug, "ITD: next phys = %#x", itd->itd.itd_next, 0,
1713 	    0, 0);
1714 
1715 	for (i = 0; i < EHCI_ITD_NUFRAMES; i++) {
1716 		t = le32toh(itd->itd.itd_ctl[i]);
1717 		USBHIST_LOG(ehcidebug, "ITDctl %d: stat = %x len = %x",
1718 		    i, EHCI_ITD_GET_STATUS(t), EHCI_ITD_GET_LEN(t), 0);
1719 		USBHIST_LOG(ehcidebug, "     ioc = %x pg = %x offs = %x",
1720 		    EHCI_ITD_GET_IOC(t), EHCI_ITD_GET_PG(t),
1721 		    EHCI_ITD_GET_OFFS(t), 0);
1722 	}
1723 	USBHIST_LOG(ehcidebug, "ITDbufr: ", 0, 0, 0, 0);
1724 	for (i = 0; i < EHCI_ITD_NBUFFERS; i++)
1725 		USBHIST_LOG(ehcidebug, "      %x",
1726 		    EHCI_ITD_GET_BPTR(le32toh(itd->itd.itd_bufr[i])), 0, 0, 0);
1727 
1728 	b = le32toh(itd->itd.itd_bufr[0]);
1729 	b2 = le32toh(itd->itd.itd_bufr[1]);
1730 	b3 = le32toh(itd->itd.itd_bufr[2]);
1731 	USBHIST_LOG(ehcidebug, "     ep = %x daddr = %x dir = %d",
1732 	    EHCI_ITD_GET_EP(b), EHCI_ITD_GET_DADDR(b), EHCI_ITD_GET_DIR(b2), 0);
1733 	USBHIST_LOG(ehcidebug, "     maxpkt = %x multi = %x",
1734 	    EHCI_ITD_GET_MAXPKT(b2), EHCI_ITD_GET_MULTI(b3), 0, 0);
1735 }
1736 
1737 Static void
1738 ehci_dump_sitd(struct ehci_soft_itd *itd)
1739 {
1740 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
1741 
1742 	USBHIST_LOG(ehcidebug, "SITD %p next = %p prev = %p",
1743 	    itd, itd->u.frame_list.next, itd->u.frame_list.prev, 0);
1744 	USBHIST_LOG(ehcidebug, "        xfernext=%p physaddr=%X slot=%d",
1745 	    itd->xfer_next, itd->physaddr, itd->slot, 0);
1746 }
1747 
1748 Static void
1749 ehci_dump_exfer(struct ehci_xfer *ex)
1750 {
1751 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
1752 
1753 	USBHIST_LOG(ehcidebug, "ex = %p sqtdstart = %p end = %p",
1754 	    ex, ex->sqtdstart, ex->sqtdend, 0);
1755 	USBHIST_LOG(ehcidebug, "     itdstart = %p end = %p isdone = %d",
1756 	    ex->itdstart, ex->itdend, ex->isdone, 0);
1757 }
1758 #endif
1759 
1760 Static usbd_status
1761 ehci_open(usbd_pipe_handle pipe)
1762 {
1763 	usbd_device_handle dev = pipe->device;
1764 	ehci_softc_t *sc = dev->bus->hci_private;
1765 	usb_endpoint_descriptor_t *ed = pipe->endpoint->edesc;
1766 	u_int8_t addr = dev->address;
1767 	u_int8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes);
1768 	struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1769 	ehci_soft_qh_t *sqh;
1770 	usbd_status err;
1771 	int ival, speed, naks;
1772 	int hshubaddr, hshubport;
1773 
1774 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
1775 
1776 	USBHIST_LOG(ehcidebug, "pipe=%p, addr=%d, endpt=%d (%d)",
1777 	    pipe, addr, ed->bEndpointAddress, sc->sc_addr);
1778 
1779 	if (dev->myhsport) {
1780 		/*
1781 		 * When directly attached FS/LS device while doing embedded
1782 		 * transaction translations and we are the hub, set the hub
1783 		 * address to 0 (us).
1784 		 */
1785 		if (!(sc->sc_flags & EHCIF_ETTF)
1786 		    || (dev->myhsport->parent->address != sc->sc_addr)) {
1787 			hshubaddr = dev->myhsport->parent->address;
1788 		} else {
1789 			hshubaddr = 0;
1790 		}
1791 		hshubport = dev->myhsport->portno;
1792 	} else {
1793 		hshubaddr = 0;
1794 		hshubport = 0;
1795 	}
1796 
1797 	if (sc->sc_dying)
1798 		return (USBD_IOERROR);
1799 
1800 	/* toggle state needed for bulk endpoints */
1801 	epipe->nexttoggle = pipe->endpoint->datatoggle;
1802 
1803 	if (addr == sc->sc_addr) {
1804 		switch (ed->bEndpointAddress) {
1805 		case USB_CONTROL_ENDPOINT:
1806 			pipe->methods = &ehci_root_ctrl_methods;
1807 			break;
1808 		case UE_DIR_IN | EHCI_INTR_ENDPT:
1809 			pipe->methods = &ehci_root_intr_methods;
1810 			break;
1811 		default:
1812 			USBHIST_LOG(ehcidebug,
1813 			    "bad bEndpointAddress 0x%02x",
1814 			    ed->bEndpointAddress, 0, 0, 0);
1815 			return (USBD_INVAL);
1816 		}
1817 		return (USBD_NORMAL_COMPLETION);
1818 	}
1819 
1820 	/* XXX All this stuff is only valid for async. */
1821 	switch (dev->speed) {
1822 	case USB_SPEED_LOW:  speed = EHCI_QH_SPEED_LOW;  break;
1823 	case USB_SPEED_FULL: speed = EHCI_QH_SPEED_FULL; break;
1824 	case USB_SPEED_HIGH: speed = EHCI_QH_SPEED_HIGH; break;
1825 	default: panic("ehci_open: bad device speed %d", dev->speed);
1826 	}
1827 	if (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_ISOCHRONOUS) {
1828 		aprint_error_dev(sc->sc_dev, "error opening low/full speed "
1829 		    "isoc endpoint.\n");
1830 		aprint_normal_dev(sc->sc_dev, "a low/full speed device is "
1831 		    "attached to a USB2 hub, and transaction translations are "
1832 		    "not yet supported.\n");
1833 		aprint_normal_dev(sc->sc_dev, "reattach the device to the "
1834 		    "root hub instead.\n");
1835 		USBHIST_LOG(ehcidebug, "hshubaddr=%d hshubport=%d",
1836 			    hshubaddr, hshubport, 0, 0);
1837 		return USBD_INVAL;
1838 	}
1839 
1840 	/*
1841 	 * For interrupt transfer, nak throttling must be disabled, but for
1842 	 * the other transfer type, nak throttling should be enabled from the
1843 	 * viewpoint that avoids the memory thrashing.
1844 	 */
1845 	naks = (xfertype == UE_INTERRUPT) ? 0
1846 	    : ((speed == EHCI_QH_SPEED_HIGH) ? 4 : 0);
1847 
1848 	/* Allocate sqh for everything, save isoc xfers */
1849 	if (xfertype != UE_ISOCHRONOUS) {
1850 		sqh = ehci_alloc_sqh(sc);
1851 		if (sqh == NULL)
1852 			return (USBD_NOMEM);
1853 		/* qh_link filled when the QH is added */
1854 		sqh->qh.qh_endp = htole32(
1855 		    EHCI_QH_SET_ADDR(addr) |
1856 		    EHCI_QH_SET_ENDPT(UE_GET_ADDR(ed->bEndpointAddress)) |
1857 		    EHCI_QH_SET_EPS(speed) |
1858 		    EHCI_QH_DTC |
1859 		    EHCI_QH_SET_MPL(UGETW(ed->wMaxPacketSize)) |
1860 		    (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_CONTROL ?
1861 		     EHCI_QH_CTL : 0) |
1862 		    EHCI_QH_SET_NRL(naks)
1863 		    );
1864 		sqh->qh.qh_endphub = htole32(
1865 		    EHCI_QH_SET_MULT(1) |
1866 		    EHCI_QH_SET_SMASK(xfertype == UE_INTERRUPT ? 0x02 : 0)
1867 		    );
1868 		if (speed != EHCI_QH_SPEED_HIGH)
1869 			sqh->qh.qh_endphub |= htole32(
1870 			    EHCI_QH_SET_PORT(hshubport) |
1871 			    EHCI_QH_SET_HUBA(hshubaddr) |
1872 			    EHCI_QH_SET_CMASK(0x08) /* XXX */
1873 			);
1874 		sqh->qh.qh_curqtd = EHCI_NULL;
1875 		/* Fill the overlay qTD */
1876 		sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
1877 		sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
1878 		sqh->qh.qh_qtd.qtd_status = htole32(0);
1879 
1880 		usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
1881 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1882 		epipe->sqh = sqh;
1883 	} else {
1884 		sqh = NULL;
1885 	} /*xfertype == UE_ISOC*/
1886 
1887 	switch (xfertype) {
1888 	case UE_CONTROL:
1889 		err = usb_allocmem(&sc->sc_bus, sizeof(usb_device_request_t),
1890 				   0, &epipe->u.ctl.reqdma);
1891 #ifdef EHCI_DEBUG
1892 		if (err)
1893 			printf("ehci_open: usb_allocmem()=%d\n", err);
1894 #endif
1895 		if (err)
1896 			goto bad;
1897 		pipe->methods = &ehci_device_ctrl_methods;
1898 		mutex_enter(&sc->sc_lock);
1899 		ehci_add_qh(sc, sqh, sc->sc_async_head);
1900 		mutex_exit(&sc->sc_lock);
1901 		break;
1902 	case UE_BULK:
1903 		pipe->methods = &ehci_device_bulk_methods;
1904 		mutex_enter(&sc->sc_lock);
1905 		ehci_add_qh(sc, sqh, sc->sc_async_head);
1906 		mutex_exit(&sc->sc_lock);
1907 		break;
1908 	case UE_INTERRUPT:
1909 		pipe->methods = &ehci_device_intr_methods;
1910 		ival = pipe->interval;
1911 		if (ival == USBD_DEFAULT_INTERVAL) {
1912 			if (speed == EHCI_QH_SPEED_HIGH) {
1913 				if (ed->bInterval > 16) {
1914 					/*
1915 					 * illegal with high-speed, but there
1916 					 * were documentation bugs in the spec,
1917 					 * so be generous
1918 					 */
1919 					ival = 256;
1920 				} else
1921 					ival = (1 << (ed->bInterval - 1)) / 8;
1922 			} else
1923 				ival = ed->bInterval;
1924 		}
1925 		err = ehci_device_setintr(sc, sqh, ival);
1926 		if (err)
1927 			goto bad;
1928 		break;
1929 	case UE_ISOCHRONOUS:
1930 		pipe->methods = &ehci_device_isoc_methods;
1931 		if (ed->bInterval == 0 || ed->bInterval > 16) {
1932 			printf("ehci: opening pipe with invalid bInterval\n");
1933 			err = USBD_INVAL;
1934 			goto bad;
1935 		}
1936 		if (UGETW(ed->wMaxPacketSize) == 0) {
1937 			printf("ehci: zero length endpoint open request\n");
1938 			err = USBD_INVAL;
1939 			goto bad;
1940 		}
1941 		epipe->u.isoc.next_frame = 0;
1942 		epipe->u.isoc.cur_xfers = 0;
1943 		break;
1944 	default:
1945 		USBHIST_LOG(ehcidebug, "bad xfer type %d", xfertype, 0, 0, 0);
1946 		err = USBD_INVAL;
1947 		goto bad;
1948 	}
1949 	return (USBD_NORMAL_COMPLETION);
1950 
1951  bad:
1952 	if (sqh != NULL)
1953 		ehci_free_sqh(sc, sqh);
1954 	return (err);
1955 }
1956 
1957 /*
1958  * Add an ED to the schedule.  Called with USB lock held.
1959  */
1960 Static void
1961 ehci_add_qh(ehci_softc_t *sc, ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
1962 {
1963 
1964 	KASSERT(mutex_owned(&sc->sc_lock));
1965 
1966 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
1967 
1968 	usb_syncmem(&head->dma, head->offs + offsetof(ehci_qh_t, qh_link),
1969 	    sizeof(head->qh.qh_link), BUS_DMASYNC_POSTWRITE);
1970 
1971 	sqh->next = head->next;
1972 	sqh->qh.qh_link = head->qh.qh_link;
1973 
1974 	usb_syncmem(&sqh->dma, sqh->offs + offsetof(ehci_qh_t, qh_link),
1975 	    sizeof(sqh->qh.qh_link), BUS_DMASYNC_PREWRITE);
1976 
1977 	head->next = sqh;
1978 	head->qh.qh_link = htole32(sqh->physaddr | EHCI_LINK_QH);
1979 
1980 	usb_syncmem(&head->dma, head->offs + offsetof(ehci_qh_t, qh_link),
1981 	    sizeof(head->qh.qh_link), BUS_DMASYNC_PREWRITE);
1982 
1983 #ifdef EHCI_DEBUG
1984 	ehci_dump_sqh(sqh);
1985 #endif
1986 }
1987 
1988 /*
1989  * Remove an ED from the schedule.  Called with USB lock held.
1990  */
1991 Static void
1992 ehci_rem_qh(ehci_softc_t *sc, ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
1993 {
1994 	ehci_soft_qh_t *p;
1995 
1996 	KASSERT(mutex_owned(&sc->sc_lock));
1997 
1998 	/* XXX */
1999 	for (p = head; p != NULL && p->next != sqh; p = p->next)
2000 		;
2001 	if (p == NULL)
2002 		panic("ehci_rem_qh: ED not found");
2003 	usb_syncmem(&sqh->dma, sqh->offs + offsetof(ehci_qh_t, qh_link),
2004 	    sizeof(sqh->qh.qh_link), BUS_DMASYNC_POSTWRITE);
2005 	p->next = sqh->next;
2006 	p->qh.qh_link = sqh->qh.qh_link;
2007 	usb_syncmem(&p->dma, p->offs + offsetof(ehci_qh_t, qh_link),
2008 	    sizeof(p->qh.qh_link), BUS_DMASYNC_PREWRITE);
2009 
2010 	ehci_sync_hc(sc);
2011 }
2012 
2013 Static void
2014 ehci_set_qh_qtd(ehci_soft_qh_t *sqh, ehci_soft_qtd_t *sqtd)
2015 {
2016 	int i;
2017 	u_int32_t status;
2018 
2019 	/* Save toggle bit and ping status. */
2020 	usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
2021 	    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2022 	status = sqh->qh.qh_qtd.qtd_status &
2023 	    htole32(EHCI_QTD_TOGGLE_MASK |
2024 		    EHCI_QTD_SET_STATUS(EHCI_QTD_PINGSTATE));
2025 	/* Set HALTED to make hw leave it alone. */
2026 	sqh->qh.qh_qtd.qtd_status =
2027 	    htole32(EHCI_QTD_SET_STATUS(EHCI_QTD_HALTED));
2028 	usb_syncmem(&sqh->dma,
2029 	    sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
2030 	    sizeof(sqh->qh.qh_qtd.qtd_status),
2031 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2032 	sqh->qh.qh_curqtd = 0;
2033 	sqh->qh.qh_qtd.qtd_next = htole32(sqtd->physaddr);
2034 	sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
2035 	for (i = 0; i < EHCI_QTD_NBUFFERS; i++)
2036 		sqh->qh.qh_qtd.qtd_buffer[i] = 0;
2037 	sqh->sqtd = sqtd;
2038 	usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
2039 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2040 	/* Set !HALTED && !ACTIVE to start execution, preserve some fields */
2041 	sqh->qh.qh_qtd.qtd_status = status;
2042 	usb_syncmem(&sqh->dma,
2043 	    sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
2044 	    sizeof(sqh->qh.qh_qtd.qtd_status),
2045 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2046 }
2047 
2048 /*
2049  * Ensure that the HC has released all references to the QH.  We do this
2050  * by asking for a Async Advance Doorbell interrupt and then we wait for
2051  * the interrupt.
2052  * To make this easier we first obtain exclusive use of the doorbell.
2053  */
2054 Static void
2055 ehci_sync_hc(ehci_softc_t *sc)
2056 {
2057 	int error __diagused;
2058 
2059 	KASSERT(mutex_owned(&sc->sc_lock));
2060 
2061 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
2062 
2063 	if (sc->sc_dying) {
2064 		USBHIST_LOG(ehcidebug, "dying", 0, 0, 0, 0);
2065 		return;
2066 	}
2067 	/* ask for doorbell */
2068 	EOWRITE4(sc, EHCI_USBCMD, EOREAD4(sc, EHCI_USBCMD) | EHCI_CMD_IAAD);
2069 	USBHIST_LOG(ehcidebug, "cmd = 0x%08x sts = 0x%08x",
2070 		    EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS), 0, 0);
2071 
2072 	error = cv_timedwait(&sc->sc_doorbell, &sc->sc_lock, hz); /* bell wait */
2073 
2074 	USBHIST_LOG(ehcidebug, "cmd = 0x%08x sts = 0x%08x",
2075 		    EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS), 0, 0);
2076 #ifdef DIAGNOSTIC
2077 	if (error)
2078 		printf("ehci_sync_hc: cv_timedwait() = %d\n", error);
2079 #endif
2080 }
2081 
2082 Static void
2083 ehci_rem_free_itd_chain(ehci_softc_t *sc, struct ehci_xfer *exfer)
2084 {
2085 	struct ehci_soft_itd *itd, *prev;
2086 
2087 	prev = NULL;
2088 
2089 	if (exfer->itdstart == NULL || exfer->itdend == NULL)
2090 		panic("ehci isoc xfer being freed, but with no itd chain");
2091 
2092 	for (itd = exfer->itdstart; itd != NULL; itd = itd->xfer_next) {
2093 		prev = itd->u.frame_list.prev;
2094 		/* Unlink itd from hardware chain, or frame array */
2095 		if (prev == NULL) { /* We're at the table head */
2096 			sc->sc_softitds[itd->slot] = itd->u.frame_list.next;
2097 			sc->sc_flist[itd->slot] = itd->itd.itd_next;
2098 			usb_syncmem(&sc->sc_fldma,
2099 			    sizeof(ehci_link_t) * itd->slot,
2100                 	    sizeof(ehci_link_t),
2101 			    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2102 
2103 			if (itd->u.frame_list.next != NULL)
2104 				itd->u.frame_list.next->u.frame_list.prev = NULL;
2105 		} else {
2106 			/* XXX this part is untested... */
2107 			prev->itd.itd_next = itd->itd.itd_next;
2108 			usb_syncmem(&itd->dma,
2109 			    itd->offs + offsetof(ehci_itd_t, itd_next),
2110                 	    sizeof(itd->itd.itd_next), BUS_DMASYNC_PREWRITE);
2111 
2112 			prev->u.frame_list.next = itd->u.frame_list.next;
2113 			if (itd->u.frame_list.next != NULL)
2114 				itd->u.frame_list.next->u.frame_list.prev = prev;
2115 		}
2116 	}
2117 
2118 	prev = NULL;
2119 	for (itd = exfer->itdstart; itd != NULL; itd = itd->xfer_next) {
2120 		if (prev != NULL)
2121 			ehci_free_itd(sc, prev);
2122 		prev = itd;
2123 	}
2124 	if (prev)
2125 		ehci_free_itd(sc, prev);
2126 	exfer->itdstart = NULL;
2127 	exfer->itdend = NULL;
2128 }
2129 
2130 /***********/
2131 
2132 /*
2133  * Data structures and routines to emulate the root hub.
2134  */
2135 Static usb_device_descriptor_t ehci_devd = {
2136 	USB_DEVICE_DESCRIPTOR_SIZE,
2137 	UDESC_DEVICE,		/* type */
2138 	{0x00, 0x02},		/* USB version */
2139 	UDCLASS_HUB,		/* class */
2140 	UDSUBCLASS_HUB,		/* subclass */
2141 	UDPROTO_HSHUBSTT,	/* protocol */
2142 	64,			/* max packet */
2143 	{0},{0},{0x00,0x01},	/* device id */
2144 	1,2,0,			/* string indicies */
2145 	1			/* # of configurations */
2146 };
2147 
2148 Static const usb_device_qualifier_t ehci_odevd = {
2149 	USB_DEVICE_DESCRIPTOR_SIZE,
2150 	UDESC_DEVICE_QUALIFIER,	/* type */
2151 	{0x00, 0x02},		/* USB version */
2152 	UDCLASS_HUB,		/* class */
2153 	UDSUBCLASS_HUB,		/* subclass */
2154 	UDPROTO_FSHUB,		/* protocol */
2155 	64,			/* max packet */
2156 	1,			/* # of configurations */
2157 	0
2158 };
2159 
2160 Static const usb_config_descriptor_t ehci_confd = {
2161 	USB_CONFIG_DESCRIPTOR_SIZE,
2162 	UDESC_CONFIG,
2163 	{USB_CONFIG_DESCRIPTOR_SIZE +
2164 	 USB_INTERFACE_DESCRIPTOR_SIZE +
2165 	 USB_ENDPOINT_DESCRIPTOR_SIZE},
2166 	1,
2167 	1,
2168 	0,
2169 	UC_ATTR_MBO | UC_SELF_POWERED,
2170 	0			/* max power */
2171 };
2172 
2173 Static const usb_interface_descriptor_t ehci_ifcd = {
2174 	USB_INTERFACE_DESCRIPTOR_SIZE,
2175 	UDESC_INTERFACE,
2176 	0,
2177 	0,
2178 	1,
2179 	UICLASS_HUB,
2180 	UISUBCLASS_HUB,
2181 	UIPROTO_HSHUBSTT,
2182 	0
2183 };
2184 
2185 Static const usb_endpoint_descriptor_t ehci_endpd = {
2186 	USB_ENDPOINT_DESCRIPTOR_SIZE,
2187 	UDESC_ENDPOINT,
2188 	UE_DIR_IN | EHCI_INTR_ENDPT,
2189 	UE_INTERRUPT,
2190 	{8, 0},			/* max packet */
2191 	12
2192 };
2193 
2194 Static const usb_hub_descriptor_t ehci_hubd = {
2195 	USB_HUB_DESCRIPTOR_SIZE,
2196 	UDESC_HUB,
2197 	0,
2198 	{0,0},
2199 	0,
2200 	0,
2201 	{""},
2202 	{""},
2203 };
2204 
2205 /*
2206  * Simulate a hardware hub by handling all the necessary requests.
2207  */
2208 Static usbd_status
2209 ehci_root_ctrl_transfer(usbd_xfer_handle xfer)
2210 {
2211 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
2212 	usbd_status err;
2213 
2214 	/* Insert last in queue. */
2215 	mutex_enter(&sc->sc_lock);
2216 	err = usb_insert_transfer(xfer);
2217 	mutex_exit(&sc->sc_lock);
2218 	if (err)
2219 		return (err);
2220 
2221 	/* Pipe isn't running, start first */
2222 	return (ehci_root_ctrl_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
2223 }
2224 
2225 Static usbd_status
2226 ehci_root_ctrl_start(usbd_xfer_handle xfer)
2227 {
2228 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
2229 	usb_device_request_t *req;
2230 	void *buf = NULL;
2231 	int port, i;
2232 	int len, value, index, l, totlen = 0;
2233 	usb_port_status_t ps;
2234 	usb_hub_descriptor_t hubd;
2235 	usbd_status err;
2236 	u_int32_t v;
2237 
2238 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
2239 
2240 	if (sc->sc_dying)
2241 		return (USBD_IOERROR);
2242 
2243 #ifdef DIAGNOSTIC
2244 	if (!(xfer->rqflags & URQ_REQUEST))
2245 		/* XXX panic */
2246 		return (USBD_INVAL);
2247 #endif
2248 	req = &xfer->request;
2249 
2250 	USBHIST_LOG(ehcidebug, "type=0x%02x request=%02x",
2251 		    req->bmRequestType, req->bRequest, 0, 0);
2252 
2253 	len = UGETW(req->wLength);
2254 	value = UGETW(req->wValue);
2255 	index = UGETW(req->wIndex);
2256 
2257 	if (len != 0)
2258 		buf = KERNADDR(&xfer->dmabuf, 0);
2259 
2260 #define C(x,y) ((x) | ((y) << 8))
2261 	switch(C(req->bRequest, req->bmRequestType)) {
2262 	case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
2263 	case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
2264 	case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
2265 		/*
2266 		 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
2267 		 * for the integrated root hub.
2268 		 */
2269 		break;
2270 	case C(UR_GET_CONFIG, UT_READ_DEVICE):
2271 		if (len > 0) {
2272 			*(u_int8_t *)buf = sc->sc_conf;
2273 			totlen = 1;
2274 		}
2275 		break;
2276 	case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
2277 		USBHIST_LOG(ehcidebug, "wValue=0x%04x", value, 0, 0, 0);
2278 		if (len == 0)
2279 			break;
2280 		switch(value >> 8) {
2281 		case UDESC_DEVICE:
2282 			if ((value & 0xff) != 0) {
2283 				err = USBD_IOERROR;
2284 				goto ret;
2285 			}
2286 			totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
2287 			USETW(ehci_devd.idVendor, sc->sc_id_vendor);
2288 			memcpy(buf, &ehci_devd, l);
2289 			break;
2290 		/*
2291 		 * We can't really operate at another speed, but the spec says
2292 		 * we need this descriptor.
2293 		 */
2294 		case UDESC_DEVICE_QUALIFIER:
2295 			if ((value & 0xff) != 0) {
2296 				err = USBD_IOERROR;
2297 				goto ret;
2298 			}
2299 			totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
2300 			memcpy(buf, &ehci_odevd, l);
2301 			break;
2302 		/*
2303 		 * We can't really operate at another speed, but the spec says
2304 		 * we need this descriptor.
2305 		 */
2306 		case UDESC_OTHER_SPEED_CONFIGURATION:
2307 		case UDESC_CONFIG:
2308 			if ((value & 0xff) != 0) {
2309 				err = USBD_IOERROR;
2310 				goto ret;
2311 			}
2312 			totlen = l = min(len, USB_CONFIG_DESCRIPTOR_SIZE);
2313 			memcpy(buf, &ehci_confd, l);
2314 			((usb_config_descriptor_t *)buf)->bDescriptorType =
2315 				value >> 8;
2316 			buf = (char *)buf + l;
2317 			len -= l;
2318 			l = min(len, USB_INTERFACE_DESCRIPTOR_SIZE);
2319 			totlen += l;
2320 			memcpy(buf, &ehci_ifcd, l);
2321 			buf = (char *)buf + l;
2322 			len -= l;
2323 			l = min(len, USB_ENDPOINT_DESCRIPTOR_SIZE);
2324 			totlen += l;
2325 			memcpy(buf, &ehci_endpd, l);
2326 			break;
2327 		case UDESC_STRING:
2328 #define sd ((usb_string_descriptor_t *)buf)
2329 			switch (value & 0xff) {
2330 			case 0: /* Language table */
2331 				totlen = usb_makelangtbl(sd, len);
2332 				break;
2333 			case 1: /* Vendor */
2334 				totlen = usb_makestrdesc(sd, len,
2335 							 sc->sc_vendor);
2336 				break;
2337 			case 2: /* Product */
2338 				totlen = usb_makestrdesc(sd, len,
2339 							 "EHCI root hub");
2340 				break;
2341 			}
2342 #undef sd
2343 			break;
2344 		default:
2345 			err = USBD_IOERROR;
2346 			goto ret;
2347 		}
2348 		break;
2349 	case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
2350 		if (len > 0) {
2351 			*(u_int8_t *)buf = 0;
2352 			totlen = 1;
2353 		}
2354 		break;
2355 	case C(UR_GET_STATUS, UT_READ_DEVICE):
2356 		if (len > 1) {
2357 			USETW(((usb_status_t *)buf)->wStatus,UDS_SELF_POWERED);
2358 			totlen = 2;
2359 		}
2360 		break;
2361 	case C(UR_GET_STATUS, UT_READ_INTERFACE):
2362 	case C(UR_GET_STATUS, UT_READ_ENDPOINT):
2363 		if (len > 1) {
2364 			USETW(((usb_status_t *)buf)->wStatus, 0);
2365 			totlen = 2;
2366 		}
2367 		break;
2368 	case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
2369 		if (value >= USB_MAX_DEVICES) {
2370 			err = USBD_IOERROR;
2371 			goto ret;
2372 		}
2373 		sc->sc_addr = value;
2374 		break;
2375 	case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
2376 		if (value != 0 && value != 1) {
2377 			err = USBD_IOERROR;
2378 			goto ret;
2379 		}
2380 		sc->sc_conf = value;
2381 		break;
2382 	case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
2383 		break;
2384 	case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
2385 	case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
2386 	case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
2387 		err = USBD_IOERROR;
2388 		goto ret;
2389 	case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
2390 		break;
2391 	case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
2392 		break;
2393 	/* Hub requests */
2394 	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
2395 		break;
2396 	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
2397 		USBHIST_LOG(ehcidebug,
2398 		    "UR_CLEAR_PORT_FEATURE port=%d feature=%d", index, value,
2399 		    0, 0);
2400 		if (index < 1 || index > sc->sc_noport) {
2401 			err = USBD_IOERROR;
2402 			goto ret;
2403 		}
2404 		port = EHCI_PORTSC(index);
2405 		v = EOREAD4(sc, port);
2406 		USBHIST_LOG(ehcidebug, "portsc=0x%08x", v, 0, 0, 0);
2407 		v &= ~EHCI_PS_CLEAR;
2408 		switch(value) {
2409 		case UHF_PORT_ENABLE:
2410 			EOWRITE4(sc, port, v &~ EHCI_PS_PE);
2411 			break;
2412 		case UHF_PORT_SUSPEND:
2413 			if (!(v & EHCI_PS_SUSP)) /* not suspended */
2414 				break;
2415 			v &= ~EHCI_PS_SUSP;
2416 			EOWRITE4(sc, port, v | EHCI_PS_FPR);
2417 			/* see USB2 spec ch. 7.1.7.7 */
2418 			usb_delay_ms(&sc->sc_bus, 20);
2419 			EOWRITE4(sc, port, v);
2420 			usb_delay_ms(&sc->sc_bus, 2);
2421 #ifdef DEBUG
2422 			v = EOREAD4(sc, port);
2423 			if (v & (EHCI_PS_FPR | EHCI_PS_SUSP))
2424 				printf("ehci: resume failed: %x\n", v);
2425 #endif
2426 			break;
2427 		case UHF_PORT_POWER:
2428 			if (sc->sc_hasppc)
2429 				EOWRITE4(sc, port, v &~ EHCI_PS_PP);
2430 			break;
2431 		case UHF_PORT_TEST:
2432 			USBHIST_LOG(ehcidebug, "clear port test "
2433 				    "%d", index, 0, 0, 0);
2434 			break;
2435 		case UHF_PORT_INDICATOR:
2436 			USBHIST_LOG(ehcidebug, "clear port ind "
2437 				    "%d", index, 0, 0, 0);
2438 			EOWRITE4(sc, port, v &~ EHCI_PS_PIC);
2439 			break;
2440 		case UHF_C_PORT_CONNECTION:
2441 			EOWRITE4(sc, port, v | EHCI_PS_CSC);
2442 			break;
2443 		case UHF_C_PORT_ENABLE:
2444 			EOWRITE4(sc, port, v | EHCI_PS_PEC);
2445 			break;
2446 		case UHF_C_PORT_SUSPEND:
2447 			/* how? */
2448 			break;
2449 		case UHF_C_PORT_OVER_CURRENT:
2450 			EOWRITE4(sc, port, v | EHCI_PS_OCC);
2451 			break;
2452 		case UHF_C_PORT_RESET:
2453 			sc->sc_isreset[index] = 0;
2454 			break;
2455 		default:
2456 			err = USBD_IOERROR;
2457 			goto ret;
2458 		}
2459 #if 0
2460 		switch(value) {
2461 		case UHF_C_PORT_CONNECTION:
2462 		case UHF_C_PORT_ENABLE:
2463 		case UHF_C_PORT_SUSPEND:
2464 		case UHF_C_PORT_OVER_CURRENT:
2465 		case UHF_C_PORT_RESET:
2466 		default:
2467 			break;
2468 		}
2469 #endif
2470 		break;
2471 	case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
2472 		if (len == 0)
2473 			break;
2474 		if ((value & 0xff) != 0) {
2475 			err = USBD_IOERROR;
2476 			goto ret;
2477 		}
2478 		hubd = ehci_hubd;
2479 		hubd.bNbrPorts = sc->sc_noport;
2480 		v = EOREAD4(sc, EHCI_HCSPARAMS);
2481 		USETW(hubd.wHubCharacteristics,
2482 		    EHCI_HCS_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_NO_SWITCH |
2483 		    EHCI_HCS_P_INDICATOR(EREAD4(sc, EHCI_HCSPARAMS))
2484 			? UHD_PORT_IND : 0);
2485 		hubd.bPwrOn2PwrGood = 200; /* XXX can't find out? */
2486 		for (i = 0, l = sc->sc_noport; l > 0; i++, l -= 8, v >>= 8)
2487 			hubd.DeviceRemovable[i++] = 0; /* XXX can't find out? */
2488 		hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i;
2489 		l = min(len, hubd.bDescLength);
2490 		totlen = l;
2491 		memcpy(buf, &hubd, l);
2492 		break;
2493 	case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
2494 		if (len != 4) {
2495 			err = USBD_IOERROR;
2496 			goto ret;
2497 		}
2498 		memset(buf, 0, len); /* ? XXX */
2499 		totlen = len;
2500 		break;
2501 	case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
2502 		USBHIST_LOG(ehcidebug, "get port status i=%d", index, 0, 0, 0);
2503 		if (index < 1 || index > sc->sc_noport) {
2504 			err = USBD_IOERROR;
2505 			goto ret;
2506 		}
2507 		if (len != 4) {
2508 			err = USBD_IOERROR;
2509 			goto ret;
2510 		}
2511 		v = EOREAD4(sc, EHCI_PORTSC(index));
2512 		USBHIST_LOG(ehcidebug, "port status=0x%04x", v, 0, 0, 0);
2513 
2514 		i = UPS_HIGH_SPEED;
2515 		if (sc->sc_flags & EHCIF_ETTF) {
2516 			/*
2517 			 * If we are doing embedded transaction translation,
2518 			 * then directly attached LS/FS devices are reset by
2519 			 * the EHCI controller itself.  PSPD is encoded
2520 			 * the same way as in USBSTATUS.
2521 			 */
2522 			i = __SHIFTOUT(v, EHCI_PS_PSPD) * UPS_LOW_SPEED;
2523 		}
2524 		if (v & EHCI_PS_CS)	i |= UPS_CURRENT_CONNECT_STATUS;
2525 		if (v & EHCI_PS_PE)	i |= UPS_PORT_ENABLED;
2526 		if (v & EHCI_PS_SUSP)	i |= UPS_SUSPEND;
2527 		if (v & EHCI_PS_OCA)	i |= UPS_OVERCURRENT_INDICATOR;
2528 		if (v & EHCI_PS_PR)	i |= UPS_RESET;
2529 		if (v & EHCI_PS_PP)	i |= UPS_PORT_POWER;
2530 		if (sc->sc_vendor_port_status)
2531 			i = sc->sc_vendor_port_status(sc, v, i);
2532 		USETW(ps.wPortStatus, i);
2533 		i = 0;
2534 		if (v & EHCI_PS_CSC)	i |= UPS_C_CONNECT_STATUS;
2535 		if (v & EHCI_PS_PEC)	i |= UPS_C_PORT_ENABLED;
2536 		if (v & EHCI_PS_OCC)	i |= UPS_C_OVERCURRENT_INDICATOR;
2537 		if (sc->sc_isreset[index]) i |= UPS_C_PORT_RESET;
2538 		USETW(ps.wPortChange, i);
2539 		l = min(len, sizeof ps);
2540 		memcpy(buf, &ps, l);
2541 		totlen = l;
2542 		break;
2543 	case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
2544 		err = USBD_IOERROR;
2545 		goto ret;
2546 	case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
2547 		break;
2548 	case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
2549 		if (index < 1 || index > sc->sc_noport) {
2550 			err = USBD_IOERROR;
2551 			goto ret;
2552 		}
2553 		port = EHCI_PORTSC(index);
2554 		v = EOREAD4(sc, port);
2555 		USBHIST_LOG(ehcidebug, "portsc=0x%08x", v, 0, 0, 0);
2556 		v &= ~EHCI_PS_CLEAR;
2557 		switch(value) {
2558 		case UHF_PORT_ENABLE:
2559 			EOWRITE4(sc, port, v | EHCI_PS_PE);
2560 			break;
2561 		case UHF_PORT_SUSPEND:
2562 			EOWRITE4(sc, port, v | EHCI_PS_SUSP);
2563 			break;
2564 		case UHF_PORT_RESET:
2565 			USBHIST_LOG(ehcidebug, "reset port %d", index, 0, 0, 0);
2566 			if (EHCI_PS_IS_LOWSPEED(v)
2567 			    && sc->sc_ncomp > 0
2568 			    && !(sc->sc_flags & EHCIF_ETTF)) {
2569 				/*
2570 				 * Low speed device on non-ETTF controller or
2571 				 * unaccompanied controller, give up ownership.
2572 				 */
2573 				ehci_disown(sc, index, 1);
2574 				break;
2575 			}
2576 			/* Start reset sequence. */
2577 			v &= ~ (EHCI_PS_PE | EHCI_PS_PR);
2578 			EOWRITE4(sc, port, v | EHCI_PS_PR);
2579 			/* Wait for reset to complete. */
2580 			usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY);
2581 			if (sc->sc_dying) {
2582 				err = USBD_IOERROR;
2583 				goto ret;
2584 			}
2585 			/*
2586 			 * An embedded transaction translator will automatically
2587 			 * terminate the reset sequence so there's no need to
2588 			 * it.
2589 			 */
2590 			v = EOREAD4(sc, port);
2591 			if (v & EHCI_PS_PR) {
2592 				/* Terminate reset sequence. */
2593 				EOWRITE4(sc, port, v & ~EHCI_PS_PR);
2594 				/* Wait for HC to complete reset. */
2595 				usb_delay_ms(&sc->sc_bus,
2596 				    EHCI_PORT_RESET_COMPLETE);
2597 				if (sc->sc_dying) {
2598 					err = USBD_IOERROR;
2599 					goto ret;
2600 				}
2601 			}
2602 
2603 			v = EOREAD4(sc, port);
2604 			USBHIST_LOG(ehcidebug,
2605 			    "ehci after reset, status=0x%08x", v, 0, 0, 0);
2606 			if (v & EHCI_PS_PR) {
2607 				printf("%s: port reset timeout\n",
2608 				       device_xname(sc->sc_dev));
2609 				return (USBD_TIMEOUT);
2610 			}
2611 			if (!(v & EHCI_PS_PE)) {
2612 				/* Not a high speed device, give up ownership.*/
2613 				ehci_disown(sc, index, 0);
2614 				break;
2615 			}
2616 			sc->sc_isreset[index] = 1;
2617 			USBHIST_LOG(ehcidebug,
2618 			    "ehci port %d reset, status = 0x%08x", index, v, 0,
2619 			    0);
2620 			break;
2621 		case UHF_PORT_POWER:
2622 			USBHIST_LOG(ehcidebug,
2623 			    "set port power %d (has PPC = %d)", index,
2624 			    sc->sc_hasppc, 0, 0);
2625 			if (sc->sc_hasppc)
2626 				EOWRITE4(sc, port, v | EHCI_PS_PP);
2627 			break;
2628 		case UHF_PORT_TEST:
2629 			USBHIST_LOG(ehcidebug, "set port test %d",
2630 				index, 0, 0, 0);
2631 			break;
2632 		case UHF_PORT_INDICATOR:
2633 			USBHIST_LOG(ehcidebug, "set port ind %d",
2634 				index, 0, 0, 0);
2635 			EOWRITE4(sc, port, v | EHCI_PS_PIC);
2636 			break;
2637 		default:
2638 			err = USBD_IOERROR;
2639 			goto ret;
2640 		}
2641 		break;
2642 	case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
2643 	case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
2644 	case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
2645 	case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
2646 		break;
2647 	default:
2648 		err = USBD_IOERROR;
2649 		goto ret;
2650 	}
2651 	xfer->actlen = totlen;
2652 	err = USBD_NORMAL_COMPLETION;
2653  ret:
2654 	mutex_enter(&sc->sc_lock);
2655 	xfer->status = err;
2656 	usb_transfer_complete(xfer);
2657 	mutex_exit(&sc->sc_lock);
2658 	return (USBD_IN_PROGRESS);
2659 }
2660 
2661 Static void
2662 ehci_disown(ehci_softc_t *sc, int index, int lowspeed)
2663 {
2664 	int port;
2665 	u_int32_t v;
2666 
2667 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
2668 
2669 	USBHIST_LOG(ehcidebug, "index=%d lowspeed=%d", index, lowspeed, 0, 0);
2670 #ifdef DIAGNOSTIC
2671 	if (sc->sc_npcomp != 0) {
2672 		int i = (index-1) / sc->sc_npcomp;
2673 		if (i >= sc->sc_ncomp)
2674 			printf("%s: strange port\n",
2675 			       device_xname(sc->sc_dev));
2676 		else
2677 			printf("%s: handing over %s speed device on "
2678 			       "port %d to %s\n",
2679 			       device_xname(sc->sc_dev),
2680 			       lowspeed ? "low" : "full",
2681 			       index, device_xname(sc->sc_comps[i]));
2682 	} else {
2683 		printf("%s: npcomp == 0\n", device_xname(sc->sc_dev));
2684 	}
2685 #endif
2686 	port = EHCI_PORTSC(index);
2687 	v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
2688 	EOWRITE4(sc, port, v | EHCI_PS_PO);
2689 }
2690 
2691 /* Abort a root control request. */
2692 Static void
2693 ehci_root_ctrl_abort(usbd_xfer_handle xfer)
2694 {
2695 	/* Nothing to do, all transfers are synchronous. */
2696 }
2697 
2698 /* Close the root pipe. */
2699 Static void
2700 ehci_root_ctrl_close(usbd_pipe_handle pipe)
2701 {
2702 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
2703 	/* Nothing to do. */
2704 }
2705 
2706 Static void
2707 ehci_root_ctrl_done(usbd_xfer_handle xfer)
2708 {
2709 	xfer->hcpriv = NULL;
2710 }
2711 
2712 Static usbd_status
2713 ehci_root_intr_transfer(usbd_xfer_handle xfer)
2714 {
2715 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
2716 	usbd_status err;
2717 
2718 	/* Insert last in queue. */
2719 	mutex_enter(&sc->sc_lock);
2720 	err = usb_insert_transfer(xfer);
2721 	mutex_exit(&sc->sc_lock);
2722 	if (err)
2723 		return (err);
2724 
2725 	/* Pipe isn't running, start first */
2726 	return (ehci_root_intr_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
2727 }
2728 
2729 Static usbd_status
2730 ehci_root_intr_start(usbd_xfer_handle xfer)
2731 {
2732 	usbd_pipe_handle pipe = xfer->pipe;
2733 	ehci_softc_t *sc = pipe->device->bus->hci_private;
2734 
2735 	if (sc->sc_dying)
2736 		return (USBD_IOERROR);
2737 
2738 	mutex_enter(&sc->sc_lock);
2739 	sc->sc_intrxfer = xfer;
2740 	mutex_exit(&sc->sc_lock);
2741 
2742 	return (USBD_IN_PROGRESS);
2743 }
2744 
2745 /* Abort a root interrupt request. */
2746 Static void
2747 ehci_root_intr_abort(usbd_xfer_handle xfer)
2748 {
2749 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
2750 
2751 	KASSERT(mutex_owned(&sc->sc_lock));
2752 	KASSERT(xfer->pipe->intrxfer == xfer);
2753 
2754 	sc->sc_intrxfer = NULL;
2755 
2756 	xfer->status = USBD_CANCELLED;
2757 	usb_transfer_complete(xfer);
2758 }
2759 
2760 /* Close the root pipe. */
2761 Static void
2762 ehci_root_intr_close(usbd_pipe_handle pipe)
2763 {
2764 	ehci_softc_t *sc = pipe->device->bus->hci_private;
2765 
2766 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
2767 
2768 	KASSERT(mutex_owned(&sc->sc_lock));
2769 
2770 	sc->sc_intrxfer = NULL;
2771 }
2772 
2773 Static void
2774 ehci_root_intr_done(usbd_xfer_handle xfer)
2775 {
2776 	xfer->hcpriv = NULL;
2777 }
2778 
2779 /************************/
2780 
2781 Static ehci_soft_qh_t *
2782 ehci_alloc_sqh(ehci_softc_t *sc)
2783 {
2784 	ehci_soft_qh_t *sqh;
2785 	usbd_status err;
2786 	int i, offs;
2787 	usb_dma_t dma;
2788 
2789 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
2790 
2791 	if (sc->sc_freeqhs == NULL) {
2792 		USBHIST_LOG(ehcidebug, "allocating chunk", 0, 0, 0, 0);
2793 		err = usb_allocmem(&sc->sc_bus, EHCI_SQH_SIZE * EHCI_SQH_CHUNK,
2794 			  EHCI_PAGE_SIZE, &dma);
2795 #ifdef EHCI_DEBUG
2796 		if (err)
2797 			printf("ehci_alloc_sqh: usb_allocmem()=%d\n", err);
2798 #endif
2799 		if (err)
2800 			return (NULL);
2801 		for(i = 0; i < EHCI_SQH_CHUNK; i++) {
2802 			offs = i * EHCI_SQH_SIZE;
2803 			sqh = KERNADDR(&dma, offs);
2804 			sqh->physaddr = DMAADDR(&dma, offs);
2805 			sqh->dma = dma;
2806 			sqh->offs = offs;
2807 			sqh->next = sc->sc_freeqhs;
2808 			sc->sc_freeqhs = sqh;
2809 		}
2810 	}
2811 	sqh = sc->sc_freeqhs;
2812 	sc->sc_freeqhs = sqh->next;
2813 	memset(&sqh->qh, 0, sizeof(ehci_qh_t));
2814 	sqh->next = NULL;
2815 	return (sqh);
2816 }
2817 
2818 Static void
2819 ehci_free_sqh(ehci_softc_t *sc, ehci_soft_qh_t *sqh)
2820 {
2821 	sqh->next = sc->sc_freeqhs;
2822 	sc->sc_freeqhs = sqh;
2823 }
2824 
2825 Static ehci_soft_qtd_t *
2826 ehci_alloc_sqtd(ehci_softc_t *sc)
2827 {
2828 	ehci_soft_qtd_t *sqtd = NULL;
2829 	usbd_status err;
2830 	int i, offs;
2831 	usb_dma_t dma;
2832 
2833 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
2834 
2835 	if (sc->sc_freeqtds == NULL) {
2836 		USBHIST_LOG(ehcidebug, "allocating chunk", 0, 0, 0, 0);
2837 
2838 		err = usb_allocmem(&sc->sc_bus, EHCI_SQTD_SIZE*EHCI_SQTD_CHUNK,
2839 			  EHCI_PAGE_SIZE, &dma);
2840 #ifdef EHCI_DEBUG
2841 		if (err)
2842 			printf("ehci_alloc_sqtd: usb_allocmem()=%d\n", err);
2843 #endif
2844 		if (err)
2845 			goto done;
2846 
2847 		for(i = 0; i < EHCI_SQTD_CHUNK; i++) {
2848 			offs = i * EHCI_SQTD_SIZE;
2849 			sqtd = KERNADDR(&dma, offs);
2850 			sqtd->physaddr = DMAADDR(&dma, offs);
2851 			sqtd->dma = dma;
2852 			sqtd->offs = offs;
2853 
2854 			sqtd->nextqtd = sc->sc_freeqtds;
2855 			sc->sc_freeqtds = sqtd;
2856 		}
2857 	}
2858 
2859 	sqtd = sc->sc_freeqtds;
2860 	sc->sc_freeqtds = sqtd->nextqtd;
2861 	memset(&sqtd->qtd, 0, sizeof(ehci_qtd_t));
2862 	sqtd->nextqtd = NULL;
2863 	sqtd->xfer = NULL;
2864 
2865 done:
2866 	return (sqtd);
2867 }
2868 
2869 Static void
2870 ehci_free_sqtd(ehci_softc_t *sc, ehci_soft_qtd_t *sqtd)
2871 {
2872 
2873 	KASSERT(sc->sc_bus.use_polling || mutex_owned(&sc->sc_lock));
2874 
2875 	sqtd->nextqtd = sc->sc_freeqtds;
2876 	sc->sc_freeqtds = sqtd;
2877 }
2878 
2879 Static usbd_status
2880 ehci_alloc_sqtd_chain(struct ehci_pipe *epipe, ehci_softc_t *sc,
2881 		     int alen, int rd, usbd_xfer_handle xfer,
2882 		     ehci_soft_qtd_t **sp, ehci_soft_qtd_t **ep)
2883 {
2884 	ehci_soft_qtd_t *next, *cur;
2885 	ehci_physaddr_t nextphys;
2886 	u_int32_t qtdstatus;
2887 	int len, curlen, mps;
2888 	int i, tog;
2889 	int pages, pageoffs;
2890 	bus_size_t curoffs;
2891 	vaddr_t va, va_offs;
2892 	usb_dma_t *dma = &xfer->dmabuf;
2893 	u_int16_t flags = xfer->flags;
2894 	paddr_t a;
2895 
2896 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
2897 
2898 	USBHIST_LOG(ehcidebug, "start len=%d", alen, 0, 0, 0);
2899 
2900 	len = alen;
2901 	qtdstatus = EHCI_QTD_ACTIVE |
2902 	    EHCI_QTD_SET_PID(rd ? EHCI_QTD_PID_IN : EHCI_QTD_PID_OUT) |
2903 	    EHCI_QTD_SET_CERR(3)
2904 	    /* IOC set below */
2905 	    /* BYTES set below */
2906 	    ;
2907 	mps = UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize);
2908 	tog = epipe->nexttoggle;
2909 	qtdstatus |= EHCI_QTD_SET_TOGGLE(tog);
2910 
2911 	cur = ehci_alloc_sqtd(sc);
2912 	*sp = cur;
2913 	if (cur == NULL)
2914 		goto nomem;
2915 
2916 	usb_syncmem(dma, 0, alen,
2917 	    rd ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
2918 	curoffs = 0;
2919 	for (;;) {
2920 		/* The EHCI hardware can handle at most 5 pages. */
2921 		va_offs = (vaddr_t)KERNADDR(dma, curoffs);
2922 		va_offs = EHCI_PAGE_OFFSET(va_offs);
2923 		if (len-curoffs < EHCI_QTD_NBUFFERS*EHCI_PAGE_SIZE - va_offs) {
2924 			/* we can handle it in this QTD */
2925 			curlen = len - curoffs;
2926 		} else {
2927 			/* must use multiple TDs, fill as much as possible. */
2928 			curlen = EHCI_QTD_NBUFFERS * EHCI_PAGE_SIZE - va_offs;
2929 
2930 			/* the length must be a multiple of the max size */
2931 			curlen -= curlen % mps;
2932 			USBHIST_LOG(ehcidebug, "multiple QTDs, "
2933 				    "curlen=%d", curlen, 0, 0, 0);
2934 #ifdef DIAGNOSTIC
2935 			if (curlen == 0)
2936 				panic("ehci_alloc_sqtd_chain: curlen == 0");
2937 #endif
2938 		}
2939 		USBHIST_LOG(ehcidebug, "len=%d curlen=%d curoffs=%zu",
2940 			len, curlen, (size_t)curoffs, 0);
2941 
2942 		/*
2943 		 * Allocate another transfer if there's more data left,
2944 		 * or if force last short transfer flag is set and we're
2945 		 * allocating a multiple of the max packet size.
2946 		 */
2947 
2948 		if (curoffs + curlen != len ||
2949 		    ((curlen % mps) == 0 && !rd && curlen != 0 &&
2950 		     (flags & USBD_FORCE_SHORT_XFER))) {
2951 			next = ehci_alloc_sqtd(sc);
2952 			if (next == NULL)
2953 				goto nomem;
2954 			nextphys = htole32(next->physaddr);
2955 		} else {
2956 			next = NULL;
2957 			nextphys = EHCI_NULL;
2958 		}
2959 
2960 		/* Find number of pages we'll be using, insert dma addresses */
2961 		pages = EHCI_PAGE(curlen + EHCI_PAGE_SIZE -1) >> 12;
2962 		KASSERT(pages <= EHCI_QTD_NBUFFERS);
2963 		pageoffs = EHCI_PAGE(curoffs);
2964 		for (i = 0; i < pages; i++) {
2965 			a = DMAADDR(dma, pageoffs + i * EHCI_PAGE_SIZE);
2966 			cur->qtd.qtd_buffer[i] = htole32(a & 0xFFFFF000);
2967 			/* Cast up to avoid compiler warnings */
2968 			cur->qtd.qtd_buffer_hi[i] = htole32((uint64_t)a >> 32);
2969 		}
2970 
2971 		/* First buffer pointer requires a page offset to start at */
2972 		va = (vaddr_t)KERNADDR(dma, curoffs);
2973 		cur->qtd.qtd_buffer[0] |= htole32(EHCI_PAGE_OFFSET(va));
2974 
2975 		cur->nextqtd = next;
2976 		cur->qtd.qtd_next = cur->qtd.qtd_altnext = nextphys;
2977 		cur->qtd.qtd_status =
2978 		    htole32(qtdstatus | EHCI_QTD_SET_BYTES(curlen));
2979 		cur->xfer = xfer;
2980 		cur->len = curlen;
2981 
2982 		USBHIST_LOG(ehcidebug, "cbp=0x%08zx end=0x%08zx",
2983 		    (size_t)curoffs, (size_t)(curoffs + curlen), 0, 0);
2984 
2985 		/* adjust the toggle based on the number of packets in this
2986 		   qtd */
2987 		if (((curlen + mps - 1) / mps) & 1) {
2988 			tog ^= 1;
2989 			qtdstatus ^= EHCI_QTD_TOGGLE_MASK;
2990 		}
2991 		if (next == NULL)
2992 			break;
2993 		usb_syncmem(&cur->dma, cur->offs, sizeof(cur->qtd),
2994 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2995 		USBHIST_LOG(ehcidebug, "extend chain", 0, 0, 0, 0);
2996 		if (len)
2997 			curoffs += curlen;
2998 		cur = next;
2999 	}
3000 	cur->qtd.qtd_status |= htole32(EHCI_QTD_IOC);
3001 	usb_syncmem(&cur->dma, cur->offs, sizeof(cur->qtd),
3002 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3003 	*ep = cur;
3004 	epipe->nexttoggle = tog;
3005 
3006 	USBHIST_LOG(ehcidebug, "return sqtd=%p sqtdend=%p",
3007 	    *sp, *ep, 0, 0);
3008 
3009 	return (USBD_NORMAL_COMPLETION);
3010 
3011  nomem:
3012 	/* XXX free chain */
3013 	USBHIST_LOG(ehcidebug, "no memory", 0, 0, 0, 0);
3014 	return (USBD_NOMEM);
3015 }
3016 
3017 Static void
3018 ehci_free_sqtd_chain(ehci_softc_t *sc, ehci_soft_qtd_t *sqtd,
3019 		    ehci_soft_qtd_t *sqtdend)
3020 {
3021 	ehci_soft_qtd_t *p;
3022 	int i;
3023 
3024 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3025 
3026 	USBHIST_LOG(ehcidebug, "sqtd=%p sqtdend=%p",
3027 	    sqtd, sqtdend, 0, 0);
3028 
3029 	for (i = 0; sqtd != sqtdend; sqtd = p, i++) {
3030 		p = sqtd->nextqtd;
3031 		ehci_free_sqtd(sc, sqtd);
3032 	}
3033 }
3034 
3035 Static ehci_soft_itd_t *
3036 ehci_alloc_itd(ehci_softc_t *sc)
3037 {
3038 	struct ehci_soft_itd *itd, *freeitd;
3039 	usbd_status err;
3040 	int i, offs, frindex, previndex;
3041 	usb_dma_t dma;
3042 
3043 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3044 
3045 	mutex_enter(&sc->sc_lock);
3046 
3047 	/* Find an itd that wasn't freed this frame or last frame. This can
3048 	 * discard itds that were freed before frindex wrapped around
3049 	 * XXX - can this lead to thrashing? Could fix by enabling wrap-around
3050 	 *       interrupt and fiddling with list when that happens */
3051 	frindex = (EOREAD4(sc, EHCI_FRINDEX) + 1) >> 3;
3052 	previndex = (frindex != 0) ? frindex - 1 : sc->sc_flsize;
3053 
3054 	freeitd = NULL;
3055 	LIST_FOREACH(itd, &sc->sc_freeitds, u.free_list) {
3056 		if (itd == NULL)
3057 			break;
3058 		if (itd->slot != frindex && itd->slot != previndex) {
3059 			freeitd = itd;
3060 			break;
3061 		}
3062 	}
3063 
3064 	if (freeitd == NULL) {
3065 		USBHIST_LOG(ehcidebug, "allocating chunk", 0, 0, 0, 0);
3066 		err = usb_allocmem(&sc->sc_bus, EHCI_ITD_SIZE * EHCI_ITD_CHUNK,
3067 				EHCI_PAGE_SIZE, &dma);
3068 
3069 		if (err) {
3070 			USBHIST_LOG(ehcidebug,
3071 			    "alloc returned %d", err, 0, 0, 0);
3072 			mutex_exit(&sc->sc_lock);
3073 			return NULL;
3074 		}
3075 
3076 		for (i = 0; i < EHCI_ITD_CHUNK; i++) {
3077 			offs = i * EHCI_ITD_SIZE;
3078 			itd = KERNADDR(&dma, offs);
3079 			itd->physaddr = DMAADDR(&dma, offs);
3080 	 		itd->dma = dma;
3081 			itd->offs = offs;
3082 			LIST_INSERT_HEAD(&sc->sc_freeitds, itd, u.free_list);
3083 		}
3084 		freeitd = LIST_FIRST(&sc->sc_freeitds);
3085 	}
3086 
3087 	itd = freeitd;
3088 	LIST_REMOVE(itd, u.free_list);
3089 	memset(&itd->itd, 0, sizeof(ehci_itd_t));
3090 	usb_syncmem(&itd->dma, itd->offs + offsetof(ehci_itd_t, itd_next),
3091                     sizeof(itd->itd.itd_next), BUS_DMASYNC_PREWRITE |
3092                     BUS_DMASYNC_PREREAD);
3093 
3094 	itd->u.frame_list.next = NULL;
3095 	itd->u.frame_list.prev = NULL;
3096 	itd->xfer_next = NULL;
3097 	itd->slot = 0;
3098 
3099 	mutex_exit(&sc->sc_lock);
3100 
3101 	return itd;
3102 }
3103 
3104 Static void
3105 ehci_free_itd(ehci_softc_t *sc, ehci_soft_itd_t *itd)
3106 {
3107 
3108 	KASSERT(mutex_owned(&sc->sc_lock));
3109 
3110 	LIST_INSERT_HEAD(&sc->sc_freeitds, itd, u.free_list);
3111 }
3112 
3113 /****************/
3114 
3115 /*
3116  * Close a reqular pipe.
3117  * Assumes that there are no pending transactions.
3118  */
3119 Static void
3120 ehci_close_pipe(usbd_pipe_handle pipe, ehci_soft_qh_t *head)
3121 {
3122 	struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
3123 	ehci_softc_t *sc = pipe->device->bus->hci_private;
3124 	ehci_soft_qh_t *sqh = epipe->sqh;
3125 
3126 	KASSERT(mutex_owned(&sc->sc_lock));
3127 
3128 	ehci_rem_qh(sc, sqh, head);
3129 	ehci_free_sqh(sc, epipe->sqh);
3130 }
3131 
3132 /*
3133  * Abort a device request.
3134  * If this routine is called at splusb() it guarantees that the request
3135  * will be removed from the hardware scheduling and that the callback
3136  * for it will be called with USBD_CANCELLED status.
3137  * It's impossible to guarantee that the requested transfer will not
3138  * have happened since the hardware runs concurrently.
3139  * If the transaction has already happened we rely on the ordinary
3140  * interrupt processing to process it.
3141  * XXX This is most probably wrong.
3142  * XXXMRG this doesn't make sense anymore.
3143  */
3144 Static void
3145 ehci_abort_xfer(usbd_xfer_handle xfer, usbd_status status)
3146 {
3147 #define exfer EXFER(xfer)
3148 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3149 	ehci_softc_t *sc = epipe->pipe.device->bus->hci_private;
3150 	ehci_soft_qh_t *sqh = epipe->sqh;
3151 	ehci_soft_qtd_t *sqtd;
3152 	ehci_physaddr_t cur;
3153 	u_int32_t qhstatus;
3154 	int hit;
3155 	int wake;
3156 
3157 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3158 
3159 	USBHIST_LOG(ehcidebug, "xfer=%p pipe=%p", xfer, epipe, 0, 0);
3160 
3161 	KASSERT(mutex_owned(&sc->sc_lock));
3162 
3163 	if (sc->sc_dying) {
3164 		/* If we're dying, just do the software part. */
3165 		xfer->status = status;	/* make software ignore it */
3166 		callout_stop(&xfer->timeout_handle);
3167 		usb_transfer_complete(xfer);
3168 		return;
3169 	}
3170 
3171 	if (cpu_intr_p() || cpu_softintr_p())
3172 		panic("ehci_abort_xfer: not in process context");
3173 
3174 	/*
3175 	 * If an abort is already in progress then just wait for it to
3176 	 * complete and return.
3177 	 */
3178 	if (xfer->hcflags & UXFER_ABORTING) {
3179 		USBHIST_LOG(ehcidebug, "already aborting", 0, 0, 0, 0);
3180 #ifdef DIAGNOSTIC
3181 		if (status == USBD_TIMEOUT)
3182 			printf("ehci_abort_xfer: TIMEOUT while aborting\n");
3183 #endif
3184 		/* Override the status which might be USBD_TIMEOUT. */
3185 		xfer->status = status;
3186 		USBHIST_LOG(ehcidebug, "waiting for abort to finish",
3187 			0, 0, 0, 0);
3188 		xfer->hcflags |= UXFER_ABORTWAIT;
3189 		while (xfer->hcflags & UXFER_ABORTING)
3190 			cv_wait(&xfer->hccv, &sc->sc_lock);
3191 		return;
3192 	}
3193 	xfer->hcflags |= UXFER_ABORTING;
3194 
3195 	/*
3196 	 * Step 1: Make interrupt routine and hardware ignore xfer.
3197 	 */
3198 	xfer->status = status;	/* make software ignore it */
3199 	callout_stop(&xfer->timeout_handle);
3200 
3201 	usb_syncmem(&sqh->dma,
3202 	    sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
3203 	    sizeof(sqh->qh.qh_qtd.qtd_status),
3204 	    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
3205 	qhstatus = sqh->qh.qh_qtd.qtd_status;
3206 	sqh->qh.qh_qtd.qtd_status = qhstatus | htole32(EHCI_QTD_HALTED);
3207 	usb_syncmem(&sqh->dma,
3208 	    sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
3209 	    sizeof(sqh->qh.qh_qtd.qtd_status),
3210 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3211 	for (sqtd = exfer->sqtdstart; ; sqtd = sqtd->nextqtd) {
3212 		usb_syncmem(&sqtd->dma,
3213 		    sqtd->offs + offsetof(ehci_qtd_t, qtd_status),
3214 		    sizeof(sqtd->qtd.qtd_status),
3215 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
3216 		sqtd->qtd.qtd_status |= htole32(EHCI_QTD_HALTED);
3217 		usb_syncmem(&sqtd->dma,
3218 		    sqtd->offs + offsetof(ehci_qtd_t, qtd_status),
3219 		    sizeof(sqtd->qtd.qtd_status),
3220 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3221 		if (sqtd == exfer->sqtdend)
3222 			break;
3223 	}
3224 
3225 	/*
3226 	 * Step 2: Wait until we know hardware has finished any possible
3227 	 * use of the xfer.  Also make sure the soft interrupt routine
3228 	 * has run.
3229 	 */
3230 	ehci_sync_hc(sc);
3231 	sc->sc_softwake = 1;
3232 	usb_schedsoftintr(&sc->sc_bus);
3233 	cv_wait(&sc->sc_softwake_cv, &sc->sc_lock);
3234 
3235 	/*
3236 	 * Step 3: Remove any vestiges of the xfer from the hardware.
3237 	 * The complication here is that the hardware may have executed
3238 	 * beyond the xfer we're trying to abort.  So as we're scanning
3239 	 * the TDs of this xfer we check if the hardware points to
3240 	 * any of them.
3241 	 */
3242 
3243 	usb_syncmem(&sqh->dma,
3244 	    sqh->offs + offsetof(ehci_qh_t, qh_curqtd),
3245 	    sizeof(sqh->qh.qh_curqtd),
3246 	    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
3247 	cur = EHCI_LINK_ADDR(le32toh(sqh->qh.qh_curqtd));
3248 	hit = 0;
3249 	for (sqtd = exfer->sqtdstart; ; sqtd = sqtd->nextqtd) {
3250 		hit |= cur == sqtd->physaddr;
3251 		if (sqtd == exfer->sqtdend)
3252 			break;
3253 	}
3254 	sqtd = sqtd->nextqtd;
3255 	/* Zap curqtd register if hardware pointed inside the xfer. */
3256 	if (hit && sqtd != NULL) {
3257 		USBHIST_LOG(ehcidebug, "cur=0x%08x", sqtd->physaddr, 0, 0, 0);
3258 		sqh->qh.qh_curqtd = htole32(sqtd->physaddr); /* unlink qTDs */
3259 		usb_syncmem(&sqh->dma,
3260 		    sqh->offs + offsetof(ehci_qh_t, qh_curqtd),
3261 		    sizeof(sqh->qh.qh_curqtd),
3262 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3263 		sqh->qh.qh_qtd.qtd_status = qhstatus;
3264 		usb_syncmem(&sqh->dma,
3265 		    sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
3266 		    sizeof(sqh->qh.qh_qtd.qtd_status),
3267 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3268 	} else {
3269 		USBHIST_LOG(ehcidebug, "no hit", 0, 0, 0, 0);
3270 	}
3271 
3272 	/*
3273 	 * Step 4: Execute callback.
3274 	 */
3275 #ifdef DIAGNOSTIC
3276 	exfer->isdone = 1;
3277 #endif
3278 	wake = xfer->hcflags & UXFER_ABORTWAIT;
3279 	xfer->hcflags &= ~(UXFER_ABORTING | UXFER_ABORTWAIT);
3280 	usb_transfer_complete(xfer);
3281 	if (wake) {
3282 		cv_broadcast(&xfer->hccv);
3283 	}
3284 
3285 	KASSERT(mutex_owned(&sc->sc_lock));
3286 #undef exfer
3287 }
3288 
3289 Static void
3290 ehci_abort_isoc_xfer(usbd_xfer_handle xfer, usbd_status status)
3291 {
3292 	ehci_isoc_trans_t trans_status;
3293 	struct ehci_pipe *epipe;
3294 	struct ehci_xfer *exfer;
3295 	ehci_softc_t *sc;
3296 	struct ehci_soft_itd *itd;
3297 	int i, wake;
3298 
3299 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3300 
3301 	epipe = (struct ehci_pipe *) xfer->pipe;
3302 	exfer = EXFER(xfer);
3303 	sc = epipe->pipe.device->bus->hci_private;
3304 
3305 	USBHIST_LOG(ehcidebug, "xfer %p pipe %p", xfer, epipe, 0, 0);
3306 
3307 	KASSERT(mutex_owned(&sc->sc_lock));
3308 
3309 	if (sc->sc_dying) {
3310 		xfer->status = status;
3311 		callout_stop(&xfer->timeout_handle);
3312 		usb_transfer_complete(xfer);
3313 		return;
3314 	}
3315 
3316 	if (xfer->hcflags & UXFER_ABORTING) {
3317 		USBHIST_LOG(ehcidebug, "already aborting", 0, 0, 0, 0);
3318 
3319 #ifdef DIAGNOSTIC
3320 		if (status == USBD_TIMEOUT)
3321 			printf("ehci_abort_isoc_xfer: TIMEOUT while aborting\n");
3322 #endif
3323 
3324 		xfer->status = status;
3325 		USBHIST_LOG(ehcidebug,
3326 		    "waiting for abort to finish", 0, 0, 0, 0);
3327 		xfer->hcflags |= UXFER_ABORTWAIT;
3328 		while (xfer->hcflags & UXFER_ABORTING)
3329 			cv_wait(&xfer->hccv, &sc->sc_lock);
3330 		goto done;
3331 	}
3332 	xfer->hcflags |= UXFER_ABORTING;
3333 
3334 	xfer->status = status;
3335 	callout_stop(&xfer->timeout_handle);
3336 
3337 	for (itd = exfer->itdstart; itd != NULL; itd = itd->xfer_next) {
3338 		usb_syncmem(&itd->dma,
3339 		    itd->offs + offsetof(ehci_itd_t, itd_ctl),
3340 		    sizeof(itd->itd.itd_ctl),
3341 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
3342 
3343 		for (i = 0; i < 8; i++) {
3344 			trans_status = le32toh(itd->itd.itd_ctl[i]);
3345 			trans_status &= ~EHCI_ITD_ACTIVE;
3346 			itd->itd.itd_ctl[i] = htole32(trans_status);
3347 		}
3348 
3349 		usb_syncmem(&itd->dma,
3350 		    itd->offs + offsetof(ehci_itd_t, itd_ctl),
3351 		    sizeof(itd->itd.itd_ctl),
3352 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3353 	}
3354 
3355         sc->sc_softwake = 1;
3356         usb_schedsoftintr(&sc->sc_bus);
3357 	cv_wait(&sc->sc_softwake_cv, &sc->sc_lock);
3358 
3359 #ifdef DIAGNOSTIC
3360 	exfer->isdone = 1;
3361 #endif
3362 	wake = xfer->hcflags & UXFER_ABORTWAIT;
3363 	xfer->hcflags &= ~(UXFER_ABORTING | UXFER_ABORTWAIT);
3364 	usb_transfer_complete(xfer);
3365 	if (wake) {
3366 		cv_broadcast(&xfer->hccv);
3367 	}
3368 
3369 done:
3370 	KASSERT(mutex_owned(&sc->sc_lock));
3371 	return;
3372 }
3373 
3374 Static void
3375 ehci_timeout(void *addr)
3376 {
3377 	struct ehci_xfer *exfer = addr;
3378 	struct ehci_pipe *epipe = (struct ehci_pipe *)exfer->xfer.pipe;
3379 	ehci_softc_t *sc = epipe->pipe.device->bus->hci_private;
3380 
3381 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3382 
3383 	USBHIST_LOG(ehcidebug, "exfer %p", exfer, 0, 0, 0);
3384 #ifdef EHCI_DEBUG
3385 	if (ehcidebug > 1)
3386 		usbd_dump_pipe(exfer->xfer.pipe);
3387 #endif
3388 
3389 	if (sc->sc_dying) {
3390 		mutex_enter(&sc->sc_lock);
3391 		ehci_abort_xfer(&exfer->xfer, USBD_TIMEOUT);
3392 		mutex_exit(&sc->sc_lock);
3393 		return;
3394 	}
3395 
3396 	/* Execute the abort in a process context. */
3397 	usb_init_task(&exfer->abort_task, ehci_timeout_task, addr,
3398 	    USB_TASKQ_MPSAFE);
3399 	usb_add_task(exfer->xfer.pipe->device, &exfer->abort_task,
3400 	    USB_TASKQ_HC);
3401 }
3402 
3403 Static void
3404 ehci_timeout_task(void *addr)
3405 {
3406 	usbd_xfer_handle xfer = addr;
3407 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
3408 
3409 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3410 
3411 	USBHIST_LOG(ehcidebug, "xfer=%p", xfer, 0, 0, 0);
3412 
3413 	mutex_enter(&sc->sc_lock);
3414 	ehci_abort_xfer(xfer, USBD_TIMEOUT);
3415 	mutex_exit(&sc->sc_lock);
3416 }
3417 
3418 /************************/
3419 
3420 Static usbd_status
3421 ehci_device_ctrl_transfer(usbd_xfer_handle xfer)
3422 {
3423 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
3424 	usbd_status err;
3425 
3426 	/* Insert last in queue. */
3427 	mutex_enter(&sc->sc_lock);
3428 	err = usb_insert_transfer(xfer);
3429 	mutex_exit(&sc->sc_lock);
3430 	if (err)
3431 		return (err);
3432 
3433 	/* Pipe isn't running, start first */
3434 	return (ehci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
3435 }
3436 
3437 Static usbd_status
3438 ehci_device_ctrl_start(usbd_xfer_handle xfer)
3439 {
3440 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
3441 	usbd_status err;
3442 
3443 	if (sc->sc_dying)
3444 		return (USBD_IOERROR);
3445 
3446 #ifdef DIAGNOSTIC
3447 	if (!(xfer->rqflags & URQ_REQUEST)) {
3448 		/* XXX panic */
3449 		printf("ehci_device_ctrl_transfer: not a request\n");
3450 		return (USBD_INVAL);
3451 	}
3452 #endif
3453 
3454 	err = ehci_device_request(xfer);
3455 	if (err) {
3456 		return (err);
3457 	}
3458 
3459 	if (sc->sc_bus.use_polling)
3460 		ehci_waitintr(sc, xfer);
3461 
3462 	return (USBD_IN_PROGRESS);
3463 }
3464 
3465 Static void
3466 ehci_device_ctrl_done(usbd_xfer_handle xfer)
3467 {
3468 	struct ehci_xfer *ex = EXFER(xfer);
3469 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
3470 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3471 	usb_device_request_t *req = &xfer->request;
3472 	int len = UGETW(req->wLength);
3473 	int rd = req->bmRequestType & UT_READ;
3474 
3475 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3476 
3477 	USBHIST_LOG(ehcidebug, "xfer=%p", xfer, 0, 0, 0);
3478 
3479 	KASSERT(sc->sc_bus.use_polling || mutex_owned(&sc->sc_lock));
3480 
3481 #ifdef DIAGNOSTIC
3482 	if (!(xfer->rqflags & URQ_REQUEST)) {
3483 		panic("ehci_ctrl_done: not a request");
3484 	}
3485 #endif
3486 
3487 	if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
3488 		ehci_del_intr_list(sc, ex);	/* remove from active list */
3489 		ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
3490 		usb_syncmem(&epipe->u.ctl.reqdma, 0, sizeof *req,
3491 		    BUS_DMASYNC_POSTWRITE);
3492 		if (len)
3493 			usb_syncmem(&xfer->dmabuf, 0, len,
3494 			    rd ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
3495 	}
3496 
3497 	USBHIST_LOG(ehcidebug, "length=%d", xfer->actlen, 0, 0, 0);
3498 }
3499 
3500 /* Abort a device control request. */
3501 Static void
3502 ehci_device_ctrl_abort(usbd_xfer_handle xfer)
3503 {
3504 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3505 
3506 	USBHIST_LOG(ehcidebug, "xfer=%p", xfer, 0, 0, 0);
3507 	ehci_abort_xfer(xfer, USBD_CANCELLED);
3508 }
3509 
3510 /* Close a device control pipe. */
3511 Static void
3512 ehci_device_ctrl_close(usbd_pipe_handle pipe)
3513 {
3514 	ehci_softc_t *sc = pipe->device->bus->hci_private;
3515 	/*struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;*/
3516 
3517 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3518 
3519 	KASSERT(mutex_owned(&sc->sc_lock));
3520 
3521 	USBHIST_LOG(ehcidebug, "pipe=%p", pipe, 0, 0, 0);
3522 
3523 	ehci_close_pipe(pipe, sc->sc_async_head);
3524 }
3525 
3526 Static usbd_status
3527 ehci_device_request(usbd_xfer_handle xfer)
3528 {
3529 #define exfer EXFER(xfer)
3530 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3531 	usb_device_request_t *req = &xfer->request;
3532 	usbd_device_handle dev = epipe->pipe.device;
3533 	ehci_softc_t *sc = dev->bus->hci_private;
3534 	ehci_soft_qtd_t *setup, *stat, *next;
3535 	ehci_soft_qh_t *sqh;
3536 	int isread;
3537 	int len;
3538 	usbd_status err;
3539 
3540 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3541 
3542 	isread = req->bmRequestType & UT_READ;
3543 	len = UGETW(req->wLength);
3544 
3545 	USBHIST_LOG(ehcidebug, "type=0x%02x, request=0x%02x, "
3546 	    "wValue=0x%04x, wIndex=0x%04x",
3547 	    req->bmRequestType, req->bRequest, UGETW(req->wValue),
3548 	    UGETW(req->wIndex));
3549 	USBHIST_LOG(ehcidebug, "len=%d, addr=%d, endpt=%d",
3550 	    len, dev->address,
3551 	    epipe->pipe.endpoint->edesc->bEndpointAddress, 0);
3552 
3553 	setup = ehci_alloc_sqtd(sc);
3554 	if (setup == NULL) {
3555 		err = USBD_NOMEM;
3556 		goto bad1;
3557 	}
3558 	stat = ehci_alloc_sqtd(sc);
3559 	if (stat == NULL) {
3560 		err = USBD_NOMEM;
3561 		goto bad2;
3562 	}
3563 
3564 	mutex_enter(&sc->sc_lock);
3565 
3566 	sqh = epipe->sqh;
3567 
3568 	KASSERTMSG(EHCI_QH_GET_ADDR(le32toh(sqh->qh.qh_endp)) == dev->address,
3569 	    "address QH %d pipe %d\n",
3570 	    EHCI_QH_GET_ADDR(le32toh(sqh->qh.qh_endp)), dev->address);
3571 	KASSERTMSG(EHCI_QH_GET_MPL(le32toh(sqh->qh.qh_endp)) ==
3572 	    UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize),
3573 	    "MPS QH %d pipe %d\n",
3574 	    EHCI_QH_GET_MPL(le32toh(sqh->qh.qh_endp)),
3575 	    UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize));
3576 
3577 	/* Set up data transaction */
3578 	if (len != 0) {
3579 		ehci_soft_qtd_t *end;
3580 
3581 		/* Start toggle at 1. */
3582 		epipe->nexttoggle = 1;
3583 		err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
3584 			  &next, &end);
3585 		if (err)
3586 			goto bad3;
3587 		end->qtd.qtd_status &= htole32(~EHCI_QTD_IOC);
3588 		end->nextqtd = stat;
3589 		end->qtd.qtd_next = end->qtd.qtd_altnext =
3590 		    htole32(stat->physaddr);
3591 		usb_syncmem(&end->dma, end->offs, sizeof(end->qtd),
3592 		   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3593 	} else {
3594 		next = stat;
3595 	}
3596 
3597 	memcpy(KERNADDR(&epipe->u.ctl.reqdma, 0), req, sizeof *req);
3598 	usb_syncmem(&epipe->u.ctl.reqdma, 0, sizeof *req, BUS_DMASYNC_PREWRITE);
3599 
3600 	/* Clear toggle */
3601 	setup->qtd.qtd_status = htole32(
3602 	    EHCI_QTD_ACTIVE |
3603 	    EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
3604 	    EHCI_QTD_SET_CERR(3) |
3605 	    EHCI_QTD_SET_TOGGLE(0) |
3606 	    EHCI_QTD_SET_BYTES(sizeof *req)
3607 	    );
3608 	setup->qtd.qtd_buffer[0] = htole32(DMAADDR(&epipe->u.ctl.reqdma, 0));
3609 	setup->qtd.qtd_buffer_hi[0] = 0;
3610 	setup->nextqtd = next;
3611 	setup->qtd.qtd_next = setup->qtd.qtd_altnext = htole32(next->physaddr);
3612 	setup->xfer = xfer;
3613 	setup->len = sizeof *req;
3614 	usb_syncmem(&setup->dma, setup->offs, sizeof(setup->qtd),
3615 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3616 
3617 	stat->qtd.qtd_status = htole32(
3618 	    EHCI_QTD_ACTIVE |
3619 	    EHCI_QTD_SET_PID(isread ? EHCI_QTD_PID_OUT : EHCI_QTD_PID_IN) |
3620 	    EHCI_QTD_SET_CERR(3) |
3621 	    EHCI_QTD_SET_TOGGLE(1) |
3622 	    EHCI_QTD_IOC
3623 	    );
3624 	stat->qtd.qtd_buffer[0] = 0; /* XXX not needed? */
3625 	stat->qtd.qtd_buffer_hi[0] = 0; /* XXX not needed? */
3626 	stat->nextqtd = NULL;
3627 	stat->qtd.qtd_next = stat->qtd.qtd_altnext = EHCI_NULL;
3628 	stat->xfer = xfer;
3629 	stat->len = 0;
3630 	usb_syncmem(&stat->dma, stat->offs, sizeof(stat->qtd),
3631 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3632 
3633 #ifdef EHCI_DEBUG
3634 	USBHIST_LOGN(ehcidebug, 5, "dump:", 0, 0, 0, 0);
3635 	ehci_dump_sqh(sqh);
3636 	ehci_dump_sqtds(setup);
3637 #endif
3638 
3639 	exfer->sqtdstart = setup;
3640 	exfer->sqtdend = stat;
3641 #ifdef DIAGNOSTIC
3642 	if (!exfer->isdone) {
3643 		printf("ehci_device_request: not done, exfer=%p\n", exfer);
3644 	}
3645 	exfer->isdone = 0;
3646 #endif
3647 
3648 	/* Insert qTD in QH list. */
3649 	ehci_set_qh_qtd(sqh, setup); /* also does usb_syncmem(sqh) */
3650 	if (xfer->timeout && !sc->sc_bus.use_polling) {
3651 		callout_reset(&xfer->timeout_handle, mstohz(xfer->timeout),
3652 		    ehci_timeout, xfer);
3653 	}
3654 	ehci_add_intr_list(sc, exfer);
3655 	xfer->status = USBD_IN_PROGRESS;
3656 	mutex_exit(&sc->sc_lock);
3657 
3658 #ifdef EHCI_DEBUG
3659 	USBHIST_LOGN(ehcidebug, 10, "status=%x, dump:",
3660 	    EOREAD4(sc, EHCI_USBSTS), 0, 0, 0);
3661 //	delay(10000);
3662 	ehci_dump_regs(sc);
3663 	ehci_dump_sqh(sc->sc_async_head);
3664 	ehci_dump_sqh(sqh);
3665 	ehci_dump_sqtds(setup);
3666 #endif
3667 
3668 	return (USBD_NORMAL_COMPLETION);
3669 
3670  bad3:
3671 	mutex_exit(&sc->sc_lock);
3672 	ehci_free_sqtd(sc, stat);
3673  bad2:
3674 	ehci_free_sqtd(sc, setup);
3675  bad1:
3676 	USBHIST_LOG(ehcidebug, "no memory", 0, 0, 0, 0);
3677 	mutex_enter(&sc->sc_lock);
3678 	xfer->status = err;
3679 	usb_transfer_complete(xfer);
3680 	mutex_exit(&sc->sc_lock);
3681 	return (err);
3682 #undef exfer
3683 }
3684 
3685 /*
3686  * Some EHCI chips from VIA seem to trigger interrupts before writing back the
3687  * qTD status, or miss signalling occasionally under heavy load.  If the host
3688  * machine is too fast, we we can miss transaction completion - when we scan
3689  * the active list the transaction still seems to be active.  This generally
3690  * exhibits itself as a umass stall that never recovers.
3691  *
3692  * We work around this behaviour by setting up this callback after any softintr
3693  * that completes with transactions still pending, giving us another chance to
3694  * check for completion after the writeback has taken place.
3695  */
3696 Static void
3697 ehci_intrlist_timeout(void *arg)
3698 {
3699 	ehci_softc_t *sc = arg;
3700 
3701 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3702 
3703 	usb_schedsoftintr(&sc->sc_bus);
3704 }
3705 
3706 /************************/
3707 
3708 Static usbd_status
3709 ehci_device_bulk_transfer(usbd_xfer_handle xfer)
3710 {
3711 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
3712 	usbd_status err;
3713 
3714 	/* Insert last in queue. */
3715 	mutex_enter(&sc->sc_lock);
3716 	err = usb_insert_transfer(xfer);
3717 	mutex_exit(&sc->sc_lock);
3718 	if (err)
3719 		return (err);
3720 
3721 	/* Pipe isn't running, start first */
3722 	return (ehci_device_bulk_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
3723 }
3724 
3725 Static usbd_status
3726 ehci_device_bulk_start(usbd_xfer_handle xfer)
3727 {
3728 #define exfer EXFER(xfer)
3729 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3730 	usbd_device_handle dev = epipe->pipe.device;
3731 	ehci_softc_t *sc = dev->bus->hci_private;
3732 	ehci_soft_qtd_t *data, *dataend;
3733 	ehci_soft_qh_t *sqh;
3734 	usbd_status err;
3735 	int len, isread, endpt;
3736 
3737 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3738 
3739 	USBHIST_LOG(ehcidebug, "xfer=%p len=%d flags=%d",
3740 	    xfer, xfer->length, xfer->flags, 0);
3741 
3742 	if (sc->sc_dying)
3743 		return (USBD_IOERROR);
3744 
3745 #ifdef DIAGNOSTIC
3746 	if (xfer->rqflags & URQ_REQUEST)
3747 		panic("ehci_device_bulk_start: a request");
3748 #endif
3749 
3750 	mutex_enter(&sc->sc_lock);
3751 
3752 	len = xfer->length;
3753 	endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3754 	isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3755 	sqh = epipe->sqh;
3756 
3757 	epipe->u.bulk.length = len;
3758 
3759 	err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer, &data,
3760 				   &dataend);
3761 	if (err) {
3762 		USBHIST_LOG(ehcidebug, "no memory", 0, 0, 0, 0);
3763 		xfer->status = err;
3764 		usb_transfer_complete(xfer);
3765 		mutex_exit(&sc->sc_lock);
3766 		return (err);
3767 	}
3768 
3769 #ifdef EHCI_DEBUG
3770 	USBHIST_LOGN(ehcidebug, 5, "data(1):", 0, 0, 0, 0);
3771 	ehci_dump_sqh(sqh);
3772 	ehci_dump_sqtds(data);
3773 #endif
3774 
3775 	/* Set up interrupt info. */
3776 	exfer->sqtdstart = data;
3777 	exfer->sqtdend = dataend;
3778 #ifdef DIAGNOSTIC
3779 	if (!exfer->isdone) {
3780 		printf("ehci_device_bulk_start: not done, ex=%p\n", exfer);
3781 	}
3782 	exfer->isdone = 0;
3783 #endif
3784 
3785 	ehci_set_qh_qtd(sqh, data); /* also does usb_syncmem(sqh) */
3786 	if (xfer->timeout && !sc->sc_bus.use_polling) {
3787 		callout_reset(&xfer->timeout_handle, mstohz(xfer->timeout),
3788 		    ehci_timeout, xfer);
3789 	}
3790 	ehci_add_intr_list(sc, exfer);
3791 	xfer->status = USBD_IN_PROGRESS;
3792 	mutex_exit(&sc->sc_lock);
3793 
3794 #ifdef EHCI_DEBUG
3795 	USBHIST_LOGN(ehcidebug, 5, "data(2)", 0, 0, 0, 0);
3796 //	delay(10000);
3797 	USBHIST_LOGN(ehcidebug, 5, "data(3)", 0, 0, 0, 0);
3798 	ehci_dump_regs(sc);
3799 #if 0
3800 	printf("async_head:\n");
3801 	ehci_dump_sqh(sc->sc_async_head);
3802 #endif
3803 	USBHIST_LOG(ehcidebug, "sqh:", 0, 0, 0, 0);
3804 	ehci_dump_sqh(sqh);
3805 	ehci_dump_sqtds(data);
3806 #endif
3807 
3808 	if (sc->sc_bus.use_polling)
3809 		ehci_waitintr(sc, xfer);
3810 
3811 	return (USBD_IN_PROGRESS);
3812 #undef exfer
3813 }
3814 
3815 Static void
3816 ehci_device_bulk_abort(usbd_xfer_handle xfer)
3817 {
3818 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3819 
3820 	USBHIST_LOG(ehcidebug, "xfer %p", xfer, 0, 0, 0);
3821 	ehci_abort_xfer(xfer, USBD_CANCELLED);
3822 }
3823 
3824 /*
3825  * Close a device bulk pipe.
3826  */
3827 Static void
3828 ehci_device_bulk_close(usbd_pipe_handle pipe)
3829 {
3830 	ehci_softc_t *sc = pipe->device->bus->hci_private;
3831 	struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
3832 
3833 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3834 
3835 	KASSERT(mutex_owned(&sc->sc_lock));
3836 
3837 	USBHIST_LOG(ehcidebug, "pipe=%p", pipe, 0, 0, 0);
3838 	pipe->endpoint->datatoggle = epipe->nexttoggle;
3839 	ehci_close_pipe(pipe, sc->sc_async_head);
3840 }
3841 
3842 Static void
3843 ehci_device_bulk_done(usbd_xfer_handle xfer)
3844 {
3845 	struct ehci_xfer *ex = EXFER(xfer);
3846 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
3847 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3848 	int endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3849 	int rd = UE_GET_DIR(endpt) == UE_DIR_IN;
3850 
3851 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3852 
3853 	USBHIST_LOG(ehcidebug, "xfer=%p, actlen=%d",
3854 	    xfer, xfer->actlen, 0, 0);
3855 
3856 	KASSERT(mutex_owned(&sc->sc_lock));
3857 
3858 	if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
3859 		ehci_del_intr_list(sc, ex);	/* remove from active list */
3860 		ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
3861 		usb_syncmem(&xfer->dmabuf, 0, xfer->length,
3862 		    rd ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
3863 	}
3864 
3865 	USBHIST_LOG(ehcidebug, "length=%d", xfer->actlen, 0, 0, 0);
3866 }
3867 
3868 /************************/
3869 
3870 Static usbd_status
3871 ehci_device_setintr(ehci_softc_t *sc, ehci_soft_qh_t *sqh, int ival)
3872 {
3873 	struct ehci_soft_islot *isp;
3874 	int islot, lev;
3875 
3876 	/* Find a poll rate that is large enough. */
3877 	for (lev = EHCI_IPOLLRATES - 1; lev > 0; lev--)
3878 		if (EHCI_ILEV_IVAL(lev) <= ival)
3879 			break;
3880 
3881 	/* Pick an interrupt slot at the right level. */
3882 	/* XXX could do better than picking at random */
3883 	sc->sc_rand = (sc->sc_rand + 191) % sc->sc_flsize;
3884 	islot = EHCI_IQHIDX(lev, sc->sc_rand);
3885 
3886 	sqh->islot = islot;
3887 	isp = &sc->sc_islots[islot];
3888 	mutex_enter(&sc->sc_lock);
3889 	ehci_add_qh(sc, sqh, isp->sqh);
3890 	mutex_exit(&sc->sc_lock);
3891 
3892 	return (USBD_NORMAL_COMPLETION);
3893 }
3894 
3895 Static usbd_status
3896 ehci_device_intr_transfer(usbd_xfer_handle xfer)
3897 {
3898 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
3899 	usbd_status err;
3900 
3901 	/* Insert last in queue. */
3902 	mutex_enter(&sc->sc_lock);
3903 	err = usb_insert_transfer(xfer);
3904 	mutex_exit(&sc->sc_lock);
3905 	if (err)
3906 		return (err);
3907 
3908 	/*
3909 	 * Pipe isn't running (otherwise err would be USBD_INPROG),
3910 	 * so start it first.
3911 	 */
3912 	return (ehci_device_intr_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
3913 }
3914 
3915 Static usbd_status
3916 ehci_device_intr_start(usbd_xfer_handle xfer)
3917 {
3918 #define exfer EXFER(xfer)
3919 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3920 	usbd_device_handle dev = xfer->pipe->device;
3921 	ehci_softc_t *sc = dev->bus->hci_private;
3922 	ehci_soft_qtd_t *data, *dataend;
3923 	ehci_soft_qh_t *sqh;
3924 	usbd_status err;
3925 	int len, isread, endpt;
3926 
3927 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
3928 
3929 	USBHIST_LOG(ehcidebug, "xfer=%p len=%d flags=%d",
3930 	    xfer, xfer->length, xfer->flags, 0);
3931 
3932 	if (sc->sc_dying)
3933 		return (USBD_IOERROR);
3934 
3935 #ifdef DIAGNOSTIC
3936 	if (xfer->rqflags & URQ_REQUEST)
3937 		panic("ehci_device_intr_start: a request");
3938 #endif
3939 
3940 	mutex_enter(&sc->sc_lock);
3941 
3942 	len = xfer->length;
3943 	endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3944 	isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3945 	sqh = epipe->sqh;
3946 
3947 	epipe->u.intr.length = len;
3948 
3949 	err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer, &data,
3950 	    &dataend);
3951 	if (err) {
3952 		USBHIST_LOG(ehcidebug, "no memory", 0, 0, 0, 0);
3953 		xfer->status = err;
3954 		usb_transfer_complete(xfer);
3955 		mutex_exit(&sc->sc_lock);
3956 		return (err);
3957 	}
3958 
3959 #ifdef EHCI_DEBUG
3960 	USBHIST_LOGN(ehcidebug, 5, "data(1)", 0, 0, 0, 0);
3961 	ehci_dump_sqh(sqh);
3962 	ehci_dump_sqtds(data);
3963 #endif
3964 
3965 	/* Set up interrupt info. */
3966 	exfer->sqtdstart = data;
3967 	exfer->sqtdend = dataend;
3968 #ifdef DIAGNOSTIC
3969 	if (!exfer->isdone) {
3970 		printf("ehci_device_intr_start: not done, ex=%p\n", exfer);
3971 	}
3972 	exfer->isdone = 0;
3973 #endif
3974 
3975 	ehci_set_qh_qtd(sqh, data); /* also does usb_syncmem(sqh) */
3976 	if (xfer->timeout && !sc->sc_bus.use_polling) {
3977 		callout_reset(&xfer->timeout_handle, mstohz(xfer->timeout),
3978 		    ehci_timeout, xfer);
3979 	}
3980 	ehci_add_intr_list(sc, exfer);
3981 	xfer->status = USBD_IN_PROGRESS;
3982 	mutex_exit(&sc->sc_lock);
3983 
3984 #ifdef EHCI_DEBUG
3985 	USBHIST_LOGN(ehcidebug, 5, "data(2)", 0, 0, 0, 0);
3986 //	delay(10000);
3987 	USBHIST_LOGN(ehcidebug, 5, "data(3)", 0, 0, 0, 0);
3988 	ehci_dump_regs(sc);
3989 	USBHIST_LOGN(ehcidebug, 5, "sqh:", 0, 0, 0, 0);
3990 	ehci_dump_sqh(sqh);
3991 	ehci_dump_sqtds(data);
3992 #endif
3993 
3994 	if (sc->sc_bus.use_polling)
3995 		ehci_waitintr(sc, xfer);
3996 
3997 	return (USBD_IN_PROGRESS);
3998 #undef exfer
3999 }
4000 
4001 Static void
4002 ehci_device_intr_abort(usbd_xfer_handle xfer)
4003 {
4004 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
4005 
4006 	USBHIST_LOG(ehcidebug, "xfer=%p", xfer, 0, 0, 0);
4007 	KASSERT(xfer->pipe->intrxfer == xfer);
4008 
4009 	/*
4010 	 * XXX - abort_xfer uses ehci_sync_hc, which syncs via the advance
4011 	 *       async doorbell. That's dependent on the async list, wheras
4012 	 *       intr xfers are periodic, should not use this?
4013 	 */
4014 	ehci_abort_xfer(xfer, USBD_CANCELLED);
4015 }
4016 
4017 Static void
4018 ehci_device_intr_close(usbd_pipe_handle pipe)
4019 {
4020 	ehci_softc_t *sc = pipe->device->bus->hci_private;
4021 	struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
4022 	struct ehci_soft_islot *isp;
4023 
4024 	KASSERT(mutex_owned(&sc->sc_lock));
4025 
4026 	isp = &sc->sc_islots[epipe->sqh->islot];
4027 	ehci_close_pipe(pipe, isp->sqh);
4028 }
4029 
4030 Static void
4031 ehci_device_intr_done(usbd_xfer_handle xfer)
4032 {
4033 #define exfer EXFER(xfer)
4034 	struct ehci_xfer *ex = EXFER(xfer);
4035 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
4036 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
4037 	ehci_soft_qtd_t *data, *dataend;
4038 	ehci_soft_qh_t *sqh;
4039 	usbd_status err;
4040 	int len, isread, endpt;
4041 
4042 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
4043 
4044 	USBHIST_LOG(ehcidebug, "xfer=%p, actlen=%d",
4045 	    xfer, xfer->actlen, 0, 0);
4046 
4047 	KASSERT(sc->sc_bus.use_polling || mutex_owned(&sc->sc_lock));
4048 
4049 	if (xfer->pipe->repeat) {
4050 		ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
4051 
4052 		len = epipe->u.intr.length;
4053 		xfer->length = len;
4054 		endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
4055 		isread = UE_GET_DIR(endpt) == UE_DIR_IN;
4056 		usb_syncmem(&xfer->dmabuf, 0, len,
4057 		    isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
4058 		sqh = epipe->sqh;
4059 
4060 		err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
4061 		    &data, &dataend);
4062 		if (err) {
4063 			USBHIST_LOG(ehcidebug, "no memory", 0, 0, 0, 0);
4064 			xfer->status = err;
4065 			return;
4066 		}
4067 
4068 		/* Set up interrupt info. */
4069 		exfer->sqtdstart = data;
4070 		exfer->sqtdend = dataend;
4071 #ifdef DIAGNOSTIC
4072 		if (!exfer->isdone) {
4073 			USBHIST_LOG(ehcidebug, "marked not done, ex = %p",
4074 				exfer, 0, 0, 0);
4075 			printf("ehci_device_intr_done: not done, ex=%p\n",
4076 			    exfer);
4077 		}
4078 		exfer->isdone = 0;
4079 #endif
4080 
4081 		ehci_set_qh_qtd(sqh, data); /* also does usb_syncmem(sqh) */
4082 		if (xfer->timeout && !sc->sc_bus.use_polling) {
4083 			callout_reset(&xfer->timeout_handle,
4084 			    mstohz(xfer->timeout), ehci_timeout, xfer);
4085 		}
4086 
4087 		xfer->status = USBD_IN_PROGRESS;
4088 	} else if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
4089 		ehci_del_intr_list(sc, ex); /* remove from active list */
4090 		ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
4091 		endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
4092 		isread = UE_GET_DIR(endpt) == UE_DIR_IN;
4093 		usb_syncmem(&xfer->dmabuf, 0, xfer->length,
4094 		    isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
4095 	}
4096 #undef exfer
4097 }
4098 
4099 /************************/
4100 
4101 Static usbd_status
4102 ehci_device_isoc_transfer(usbd_xfer_handle xfer)
4103 {
4104 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
4105 	usbd_status err;
4106 
4107 	mutex_enter(&sc->sc_lock);
4108 	err = usb_insert_transfer(xfer);
4109 	mutex_exit(&sc->sc_lock);
4110 	if (err && err != USBD_IN_PROGRESS)
4111 		return err;
4112 
4113 	return ehci_device_isoc_start(xfer);
4114 }
4115 
4116 Static usbd_status
4117 ehci_device_isoc_start(usbd_xfer_handle xfer)
4118 {
4119 	struct ehci_pipe *epipe;
4120 	ehci_softc_t *sc;
4121 	struct ehci_xfer *exfer;
4122 	ehci_soft_itd_t *itd, *prev, *start, *stop;
4123 	usb_dma_t *dma_buf;
4124 	int i, j, k, frames, uframes, ufrperframe;
4125 	int trans_count, offs, total_length;
4126 	int frindex;
4127 
4128 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
4129 
4130 	start = NULL;
4131 	prev = NULL;
4132 	itd = NULL;
4133 	trans_count = 0;
4134 	total_length = 0;
4135 	exfer = (struct ehci_xfer *) xfer;
4136 	sc = xfer->pipe->device->bus->hci_private;
4137 	epipe = (struct ehci_pipe *)xfer->pipe;
4138 
4139 	/*
4140 	 * To allow continuous transfers, above we start all transfers
4141 	 * immediately. However, we're still going to get usbd_start_next call
4142 	 * this when another xfer completes. So, check if this is already
4143 	 * in progress or not
4144 	 */
4145 
4146 	if (exfer->itdstart != NULL)
4147 		return USBD_IN_PROGRESS;
4148 
4149 	USBHIST_LOG(ehcidebug, "xfer %p len %d flags %d",
4150 	    xfer, xfer->length, xfer->flags, 0);
4151 
4152 	if (sc->sc_dying)
4153 		return USBD_IOERROR;
4154 
4155 	/*
4156 	 * To avoid complication, don't allow a request right now that'll span
4157 	 * the entire frame table. To within 4 frames, to allow some leeway
4158 	 * on either side of where the hc currently is.
4159 	 */
4160 	if ((1 << (epipe->pipe.endpoint->edesc->bInterval)) *
4161 			xfer->nframes >= (sc->sc_flsize - 4) * 8) {
4162 		USBHIST_LOG(ehcidebug,
4163 		    "isoc descriptor spans entire frametable", 0, 0, 0, 0);
4164 		printf("ehci: isoc descriptor requested that spans the entire frametable, too many frames\n");
4165 		return USBD_INVAL;
4166 	}
4167 
4168 #ifdef DIAGNOSTIC
4169 	if (xfer->rqflags & URQ_REQUEST)
4170 		panic("ehci_device_isoc_start: request");
4171 
4172 	if (!exfer->isdone) {
4173 		USBHIST_LOG(ehcidebug, "marked not done, ex = %p", exfer,
4174 			0, 0, 0);
4175 		printf("ehci_device_isoc_start: not done, ex = %p\n", exfer);
4176 	}
4177 	exfer->isdone = 0;
4178 #endif
4179 
4180 	/*
4181 	 * Step 1: Allocate and initialize itds, how many do we need?
4182 	 * One per transfer if interval >= 8 microframes, fewer if we use
4183 	 * multiple microframes per frame.
4184 	 */
4185 
4186 	i = epipe->pipe.endpoint->edesc->bInterval;
4187 	if (i > 16 || i == 0) {
4188 		/* Spec page 271 says intervals > 16 are invalid */
4189 		USBHIST_LOG(ehcidebug, "bInterval %d invalid", i, 0, 0, 0);
4190 		return USBD_INVAL;
4191 	}
4192 
4193 	ufrperframe = max(1, USB_UFRAMES_PER_FRAME / (1 << (i - 1)));
4194 	frames = (xfer->nframes + (ufrperframe - 1)) / ufrperframe;
4195 	uframes = USB_UFRAMES_PER_FRAME / ufrperframe;
4196 
4197 	if (frames == 0) {
4198 		USBHIST_LOG(ehcidebug, "frames == 0", 0, 0, 0, 0);
4199 		return USBD_INVAL;
4200 	}
4201 
4202 	dma_buf = &xfer->dmabuf;
4203 	offs = 0;
4204 
4205 	for (i = 0; i < frames; i++) {
4206 		int froffs = offs;
4207 		itd = ehci_alloc_itd(sc);
4208 
4209 		if (prev != NULL) {
4210 			prev->itd.itd_next =
4211 			    htole32(itd->physaddr | EHCI_LINK_ITD);
4212 			usb_syncmem(&itd->dma,
4213 			    itd->offs + offsetof(ehci_itd_t, itd_next),
4214                 	    sizeof(itd->itd.itd_next), BUS_DMASYNC_POSTWRITE);
4215 
4216 			prev->xfer_next = itd;
4217 	    	} else {
4218 			start = itd;
4219 		}
4220 
4221 		/*
4222 		 * Step 1.5, initialize uframes
4223 		 */
4224 		for (j = 0; j < EHCI_ITD_NUFRAMES; j += uframes) {
4225 			/* Calculate which page in the list this starts in */
4226 			int addr = DMAADDR(dma_buf, froffs);
4227 			addr = EHCI_PAGE_OFFSET(addr);
4228 			addr += (offs - froffs);
4229 			addr = EHCI_PAGE(addr);
4230 			addr /= EHCI_PAGE_SIZE;
4231 
4232 			/* This gets the initial offset into the first page,
4233 			 * looks how far further along the current uframe
4234 			 * offset is. Works out how many pages that is.
4235 			 */
4236 
4237 			itd->itd.itd_ctl[j] = htole32 ( EHCI_ITD_ACTIVE |
4238 			    EHCI_ITD_SET_LEN(xfer->frlengths[trans_count]) |
4239 			    EHCI_ITD_SET_PG(addr) |
4240 			    EHCI_ITD_SET_OFFS(EHCI_PAGE_OFFSET(DMAADDR(dma_buf,offs))));
4241 
4242 			total_length += xfer->frlengths[trans_count];
4243 			offs += xfer->frlengths[trans_count];
4244 			trans_count++;
4245 
4246 			if (trans_count >= xfer->nframes) { /*Set IOC*/
4247 				itd->itd.itd_ctl[j] |= htole32(EHCI_ITD_IOC);
4248 				break;
4249 			}
4250 		}
4251 
4252 		/* Step 1.75, set buffer pointers. To simplify matters, all
4253 		 * pointers are filled out for the next 7 hardware pages in
4254 		 * the dma block, so no need to worry what pages to cover
4255 		 * and what to not.
4256 		 */
4257 
4258 		for (j = 0; j < EHCI_ITD_NBUFFERS; j++) {
4259 			/*
4260 			 * Don't try to lookup a page that's past the end
4261 			 * of buffer
4262 			 */
4263 			int page_offs = EHCI_PAGE(froffs + (EHCI_PAGE_SIZE * j));
4264 			if (page_offs >= dma_buf->block->size)
4265 				break;
4266 
4267 			unsigned long long page = DMAADDR(dma_buf, page_offs);
4268 			page = EHCI_PAGE(page);
4269 			itd->itd.itd_bufr[j] =
4270 			    htole32(EHCI_ITD_SET_BPTR(page));
4271 			itd->itd.itd_bufr_hi[j] =
4272 			    htole32(page >> 32);
4273 		}
4274 
4275 		/*
4276 		 * Other special values
4277 		 */
4278 
4279 		k = epipe->pipe.endpoint->edesc->bEndpointAddress;
4280 		itd->itd.itd_bufr[0] |= htole32(EHCI_ITD_SET_EP(UE_GET_ADDR(k)) |
4281 		    EHCI_ITD_SET_DADDR(epipe->pipe.device->address));
4282 
4283 		k = (UE_GET_DIR(epipe->pipe.endpoint->edesc->bEndpointAddress))
4284 		    ? 1 : 0;
4285 		j = UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize);
4286 		itd->itd.itd_bufr[1] |= htole32(EHCI_ITD_SET_DIR(k) |
4287 		    EHCI_ITD_SET_MAXPKT(UE_GET_SIZE(j)));
4288 
4289 		/* FIXME: handle invalid trans */
4290 		itd->itd.itd_bufr[2] |=
4291 		    htole32(EHCI_ITD_SET_MULTI(UE_GET_TRANS(j)+1));
4292 
4293 		usb_syncmem(&itd->dma,
4294 		    itd->offs + offsetof(ehci_itd_t, itd_next),
4295                     sizeof(ehci_itd_t),
4296 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
4297 
4298 		prev = itd;
4299 	} /* End of frame */
4300 
4301 	stop = itd;
4302 	stop->xfer_next = NULL;
4303 	exfer->isoc_len = total_length;
4304 
4305 	usb_syncmem(&exfer->xfer.dmabuf, 0, total_length,
4306 		BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4307 
4308 	/*
4309 	 * Part 2: Transfer descriptors have now been set up, now they must
4310 	 * be scheduled into the period frame list. Erk. Not wanting to
4311 	 * complicate matters, transfer is denied if the transfer spans
4312 	 * more than the period frame list.
4313 	 */
4314 
4315 	mutex_enter(&sc->sc_lock);
4316 
4317 	/* Start inserting frames */
4318 	if (epipe->u.isoc.cur_xfers > 0) {
4319 		frindex = epipe->u.isoc.next_frame;
4320 	} else {
4321 		frindex = EOREAD4(sc, EHCI_FRINDEX);
4322 		frindex = frindex >> 3; /* Erase microframe index */
4323 		frindex += 2;
4324 	}
4325 
4326 	if (frindex >= sc->sc_flsize)
4327 		frindex &= (sc->sc_flsize - 1);
4328 
4329 	/* What's the frame interval? */
4330 	i = (1 << (epipe->pipe.endpoint->edesc->bInterval - 1));
4331 	if (i / USB_UFRAMES_PER_FRAME == 0)
4332 		i = 1;
4333 	else
4334 		i /= USB_UFRAMES_PER_FRAME;
4335 
4336 	itd = start;
4337 	for (j = 0; j < frames; j++) {
4338 		if (itd == NULL)
4339 			panic("ehci: unexpectedly ran out of isoc itds, isoc_start");
4340 
4341 		itd->itd.itd_next = sc->sc_flist[frindex];
4342 		if (itd->itd.itd_next == 0)
4343 			/* FIXME: frindex table gets initialized to NULL
4344 			 * or EHCI_NULL? */
4345 			itd->itd.itd_next = EHCI_NULL;
4346 
4347 		usb_syncmem(&itd->dma,
4348 		    itd->offs + offsetof(ehci_itd_t, itd_next),
4349                     sizeof(itd->itd.itd_next),
4350 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
4351 
4352 		sc->sc_flist[frindex] = htole32(EHCI_LINK_ITD | itd->physaddr);
4353 
4354 		usb_syncmem(&sc->sc_fldma,
4355 		    sizeof(ehci_link_t) * frindex,
4356                     sizeof(ehci_link_t),
4357 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
4358 
4359 		itd->u.frame_list.next = sc->sc_softitds[frindex];
4360 		sc->sc_softitds[frindex] = itd;
4361 		if (itd->u.frame_list.next != NULL)
4362 			itd->u.frame_list.next->u.frame_list.prev = itd;
4363 		itd->slot = frindex;
4364 		itd->u.frame_list.prev = NULL;
4365 
4366 		frindex += i;
4367 		if (frindex >= sc->sc_flsize)
4368 			frindex -= sc->sc_flsize;
4369 
4370 		itd = itd->xfer_next;
4371 	}
4372 
4373 	epipe->u.isoc.cur_xfers++;
4374 	epipe->u.isoc.next_frame = frindex;
4375 
4376 	exfer->itdstart = start;
4377 	exfer->itdend = stop;
4378 	exfer->sqtdstart = NULL;
4379 	exfer->sqtdend = NULL;
4380 
4381 	ehci_add_intr_list(sc, exfer);
4382 	xfer->status = USBD_IN_PROGRESS;
4383 	xfer->done = 0;
4384 	mutex_exit(&sc->sc_lock);
4385 
4386 	if (sc->sc_bus.use_polling) {
4387 		printf("Starting ehci isoc xfer with polling. Bad idea?\n");
4388 		ehci_waitintr(sc, xfer);
4389 	}
4390 
4391 	return USBD_IN_PROGRESS;
4392 }
4393 
4394 Static void
4395 ehci_device_isoc_abort(usbd_xfer_handle xfer)
4396 {
4397 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
4398 
4399 	USBHIST_LOG(ehcidebug, "xfer = %p", xfer, 0, 0, 0);
4400 	ehci_abort_isoc_xfer(xfer, USBD_CANCELLED);
4401 }
4402 
4403 Static void
4404 ehci_device_isoc_close(usbd_pipe_handle pipe)
4405 {
4406 	USBHIST_FUNC(); USBHIST_CALLED(ehcidebug);
4407 
4408 	USBHIST_LOG(ehcidebug, "nothing in the pipe to free?", 0, 0, 0, 0);
4409 }
4410 
4411 Static void
4412 ehci_device_isoc_done(usbd_xfer_handle xfer)
4413 {
4414 	struct ehci_xfer *exfer;
4415 	ehci_softc_t *sc;
4416 	struct ehci_pipe *epipe;
4417 
4418 	exfer = EXFER(xfer);
4419 	sc = xfer->pipe->device->bus->hci_private;
4420 	epipe = (struct ehci_pipe *) xfer->pipe;
4421 
4422 	KASSERT(mutex_owned(&sc->sc_lock));
4423 
4424 	epipe->u.isoc.cur_xfers--;
4425 	if (xfer->status != USBD_NOMEM && ehci_active_intr_list(exfer)) {
4426 		ehci_del_intr_list(sc, exfer);
4427 		ehci_rem_free_itd_chain(sc, exfer);
4428 	}
4429 
4430 	usb_syncmem(&xfer->dmabuf, 0, xfer->length, BUS_DMASYNC_POSTWRITE |
4431                     BUS_DMASYNC_POSTREAD);
4432 
4433 }
4434