xref: /netbsd-src/sys/dev/usb/ehci.c (revision c2f76ff004a2cb67efe5b12d97bd3ef7fe89e18d)
1 /*	$NetBSD: ehci.c,v 1.173 2011/01/18 15:05:03 jmcneill Exp $ */
2 
3 /*
4  * Copyright (c) 2004-2008 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Lennart Augustsson (lennart@augustsson.net), Charles M. Hannum and
9  * Jeremy Morse (jeremy.morse@gmail.com).
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  * POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 /*
34  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
35  *
36  * The EHCI 1.0 spec can be found at
37  * http://www.intel.com/technology/usb/spec.htm
38  * and the USB 2.0 spec at
39  * http://www.usb.org/developers/docs/
40  *
41  */
42 
43 /*
44  * TODO:
45  * 1) hold off explorations by companion controllers until ehci has started.
46  *
47  * 2) The hub driver needs to handle and schedule the transaction translator,
48  *    to assign place in frame where different devices get to go. See chapter
49  *    on hubs in USB 2.0 for details.
50  *
51  * 3) Command failures are not recovered correctly.
52  */
53 
54 #include <sys/cdefs.h>
55 __KERNEL_RCSID(0, "$NetBSD: ehci.c,v 1.173 2011/01/18 15:05:03 jmcneill Exp $");
56 
57 #include "ohci.h"
58 #include "uhci.h"
59 
60 #include <sys/param.h>
61 #include <sys/systm.h>
62 #include <sys/kernel.h>
63 #include <sys/malloc.h>
64 #include <sys/device.h>
65 #include <sys/select.h>
66 #include <sys/proc.h>
67 #include <sys/queue.h>
68 #include <sys/mutex.h>
69 #include <sys/bus.h>
70 
71 #include <machine/endian.h>
72 
73 #include <dev/usb/usb.h>
74 #include <dev/usb/usbdi.h>
75 #include <dev/usb/usbdivar.h>
76 #include <dev/usb/usb_mem.h>
77 #include <dev/usb/usb_quirks.h>
78 
79 #include <dev/usb/ehcireg.h>
80 #include <dev/usb/ehcivar.h>
81 #include <dev/usb/usbroothub_subr.h>
82 
83 #ifdef EHCI_DEBUG
84 #define DPRINTF(x)	do { if (ehcidebug) printf x; } while(0)
85 #define DPRINTFN(n,x)	do { if (ehcidebug>(n)) printf x; } while (0)
86 int ehcidebug = 0;
87 #else
88 #define DPRINTF(x)
89 #define DPRINTFN(n,x)
90 #endif
91 
92 struct ehci_pipe {
93 	struct usbd_pipe pipe;
94 	int nexttoggle;
95 
96 	ehci_soft_qh_t *sqh;
97 	union {
98 		ehci_soft_qtd_t *qtd;
99 		/* ehci_soft_itd_t *itd; */
100 	} tail;
101 	union {
102 		/* Control pipe */
103 		struct {
104 			usb_dma_t reqdma;
105 			u_int length;
106 		} ctl;
107 		/* Interrupt pipe */
108 		struct {
109 			u_int length;
110 		} intr;
111 		/* Bulk pipe */
112 		struct {
113 			u_int length;
114 		} bulk;
115 		/* Iso pipe */
116 		struct {
117 			u_int next_frame;
118 			u_int cur_xfers;
119 		} isoc;
120 	} u;
121 };
122 
123 Static usbd_status	ehci_open(usbd_pipe_handle);
124 Static void		ehci_poll(struct usbd_bus *);
125 Static void		ehci_softintr(void *);
126 Static int		ehci_intr1(ehci_softc_t *);
127 Static void		ehci_waitintr(ehci_softc_t *, usbd_xfer_handle);
128 Static void		ehci_check_intr(ehci_softc_t *, struct ehci_xfer *);
129 Static void		ehci_check_qh_intr(ehci_softc_t *, struct ehci_xfer *);
130 Static void		ehci_check_itd_intr(ehci_softc_t *, struct ehci_xfer *);
131 Static void		ehci_idone(struct ehci_xfer *);
132 Static void		ehci_timeout(void *);
133 Static void		ehci_timeout_task(void *);
134 Static void		ehci_intrlist_timeout(void *);
135 
136 Static usbd_status	ehci_allocm(struct usbd_bus *, usb_dma_t *, u_int32_t);
137 Static void		ehci_freem(struct usbd_bus *, usb_dma_t *);
138 
139 Static usbd_xfer_handle	ehci_allocx(struct usbd_bus *);
140 Static void		ehci_freex(struct usbd_bus *, usbd_xfer_handle);
141 
142 Static usbd_status	ehci_root_ctrl_transfer(usbd_xfer_handle);
143 Static usbd_status	ehci_root_ctrl_start(usbd_xfer_handle);
144 Static void		ehci_root_ctrl_abort(usbd_xfer_handle);
145 Static void		ehci_root_ctrl_close(usbd_pipe_handle);
146 Static void		ehci_root_ctrl_done(usbd_xfer_handle);
147 
148 Static usbd_status	ehci_root_intr_transfer(usbd_xfer_handle);
149 Static usbd_status	ehci_root_intr_start(usbd_xfer_handle);
150 Static void		ehci_root_intr_abort(usbd_xfer_handle);
151 Static void		ehci_root_intr_close(usbd_pipe_handle);
152 Static void		ehci_root_intr_done(usbd_xfer_handle);
153 
154 Static usbd_status	ehci_device_ctrl_transfer(usbd_xfer_handle);
155 Static usbd_status	ehci_device_ctrl_start(usbd_xfer_handle);
156 Static void		ehci_device_ctrl_abort(usbd_xfer_handle);
157 Static void		ehci_device_ctrl_close(usbd_pipe_handle);
158 Static void		ehci_device_ctrl_done(usbd_xfer_handle);
159 
160 Static usbd_status	ehci_device_bulk_transfer(usbd_xfer_handle);
161 Static usbd_status	ehci_device_bulk_start(usbd_xfer_handle);
162 Static void		ehci_device_bulk_abort(usbd_xfer_handle);
163 Static void		ehci_device_bulk_close(usbd_pipe_handle);
164 Static void		ehci_device_bulk_done(usbd_xfer_handle);
165 
166 Static usbd_status	ehci_device_intr_transfer(usbd_xfer_handle);
167 Static usbd_status	ehci_device_intr_start(usbd_xfer_handle);
168 Static void		ehci_device_intr_abort(usbd_xfer_handle);
169 Static void		ehci_device_intr_close(usbd_pipe_handle);
170 Static void		ehci_device_intr_done(usbd_xfer_handle);
171 
172 Static usbd_status	ehci_device_isoc_transfer(usbd_xfer_handle);
173 Static usbd_status	ehci_device_isoc_start(usbd_xfer_handle);
174 Static void		ehci_device_isoc_abort(usbd_xfer_handle);
175 Static void		ehci_device_isoc_close(usbd_pipe_handle);
176 Static void		ehci_device_isoc_done(usbd_xfer_handle);
177 
178 Static void		ehci_device_clear_toggle(usbd_pipe_handle pipe);
179 Static void		ehci_noop(usbd_pipe_handle pipe);
180 
181 Static void		ehci_pcd(ehci_softc_t *, usbd_xfer_handle);
182 Static void		ehci_disown(ehci_softc_t *, int, int);
183 
184 Static ehci_soft_qh_t  *ehci_alloc_sqh(ehci_softc_t *);
185 Static void		ehci_free_sqh(ehci_softc_t *, ehci_soft_qh_t *);
186 
187 Static ehci_soft_qtd_t  *ehci_alloc_sqtd(ehci_softc_t *);
188 Static void		ehci_free_sqtd(ehci_softc_t *, ehci_soft_qtd_t *);
189 Static usbd_status	ehci_alloc_sqtd_chain(struct ehci_pipe *,
190 			    ehci_softc_t *, int, int, usbd_xfer_handle,
191 			    ehci_soft_qtd_t **, ehci_soft_qtd_t **);
192 Static void		ehci_free_sqtd_chain(ehci_softc_t *, ehci_soft_qtd_t *,
193 					    ehci_soft_qtd_t *);
194 
195 Static ehci_soft_itd_t	*ehci_alloc_itd(ehci_softc_t *sc);
196 Static void		ehci_free_itd(ehci_softc_t *sc, ehci_soft_itd_t *itd);
197 Static void 		ehci_rem_free_itd_chain(ehci_softc_t *sc,
198 						struct ehci_xfer *exfer);
199 Static void 		ehci_abort_isoc_xfer(usbd_xfer_handle xfer,
200 						usbd_status status);
201 
202 Static usbd_status	ehci_device_request(usbd_xfer_handle xfer);
203 
204 Static usbd_status	ehci_device_setintr(ehci_softc_t *, ehci_soft_qh_t *,
205 			    int ival);
206 
207 Static void		ehci_add_qh(ehci_soft_qh_t *, ehci_soft_qh_t *);
208 Static void		ehci_rem_qh(ehci_softc_t *, ehci_soft_qh_t *,
209 				    ehci_soft_qh_t *);
210 Static void		ehci_set_qh_qtd(ehci_soft_qh_t *, ehci_soft_qtd_t *);
211 Static void		ehci_sync_hc(ehci_softc_t *);
212 
213 Static void		ehci_close_pipe(usbd_pipe_handle, ehci_soft_qh_t *);
214 Static void		ehci_abort_xfer(usbd_xfer_handle, usbd_status);
215 
216 #ifdef EHCI_DEBUG
217 Static void		ehci_dump_regs(ehci_softc_t *);
218 void			ehci_dump(void);
219 Static ehci_softc_t 	*theehci;
220 Static void		ehci_dump_link(ehci_link_t, int);
221 Static void		ehci_dump_sqtds(ehci_soft_qtd_t *);
222 Static void		ehci_dump_sqtd(ehci_soft_qtd_t *);
223 Static void		ehci_dump_qtd(ehci_qtd_t *);
224 Static void		ehci_dump_sqh(ehci_soft_qh_t *);
225 #if notyet
226 Static void		ehci_dump_sitd(struct ehci_soft_itd *itd);
227 Static void		ehci_dump_itd(struct ehci_soft_itd *);
228 #endif
229 #ifdef DIAGNOSTIC
230 Static void		ehci_dump_exfer(struct ehci_xfer *);
231 #endif
232 #endif
233 
234 #define EHCI_NULL htole32(EHCI_LINK_TERMINATE)
235 
236 #define EHCI_INTR_ENDPT 1
237 
238 #define ehci_add_intr_list(sc, ex) \
239 	TAILQ_INSERT_TAIL(&(sc)->sc_intrhead, (ex), inext);
240 #define ehci_del_intr_list(sc, ex) \
241 	do { \
242 		TAILQ_REMOVE(&sc->sc_intrhead, (ex), inext); \
243 		(ex)->inext.tqe_prev = NULL; \
244 	} while (0)
245 #define ehci_active_intr_list(ex) ((ex)->inext.tqe_prev != NULL)
246 
247 Static const struct usbd_bus_methods ehci_bus_methods = {
248 	ehci_open,
249 	ehci_softintr,
250 	ehci_poll,
251 	ehci_allocm,
252 	ehci_freem,
253 	ehci_allocx,
254 	ehci_freex,
255 };
256 
257 Static const struct usbd_pipe_methods ehci_root_ctrl_methods = {
258 	ehci_root_ctrl_transfer,
259 	ehci_root_ctrl_start,
260 	ehci_root_ctrl_abort,
261 	ehci_root_ctrl_close,
262 	ehci_noop,
263 	ehci_root_ctrl_done,
264 };
265 
266 Static const struct usbd_pipe_methods ehci_root_intr_methods = {
267 	ehci_root_intr_transfer,
268 	ehci_root_intr_start,
269 	ehci_root_intr_abort,
270 	ehci_root_intr_close,
271 	ehci_noop,
272 	ehci_root_intr_done,
273 };
274 
275 Static const struct usbd_pipe_methods ehci_device_ctrl_methods = {
276 	ehci_device_ctrl_transfer,
277 	ehci_device_ctrl_start,
278 	ehci_device_ctrl_abort,
279 	ehci_device_ctrl_close,
280 	ehci_noop,
281 	ehci_device_ctrl_done,
282 };
283 
284 Static const struct usbd_pipe_methods ehci_device_intr_methods = {
285 	ehci_device_intr_transfer,
286 	ehci_device_intr_start,
287 	ehci_device_intr_abort,
288 	ehci_device_intr_close,
289 	ehci_device_clear_toggle,
290 	ehci_device_intr_done,
291 };
292 
293 Static const struct usbd_pipe_methods ehci_device_bulk_methods = {
294 	ehci_device_bulk_transfer,
295 	ehci_device_bulk_start,
296 	ehci_device_bulk_abort,
297 	ehci_device_bulk_close,
298 	ehci_device_clear_toggle,
299 	ehci_device_bulk_done,
300 };
301 
302 Static const struct usbd_pipe_methods ehci_device_isoc_methods = {
303 	ehci_device_isoc_transfer,
304 	ehci_device_isoc_start,
305 	ehci_device_isoc_abort,
306 	ehci_device_isoc_close,
307 	ehci_noop,
308 	ehci_device_isoc_done,
309 };
310 
311 static const uint8_t revbits[EHCI_MAX_POLLRATE] = {
312 0x00,0x40,0x20,0x60,0x10,0x50,0x30,0x70,0x08,0x48,0x28,0x68,0x18,0x58,0x38,0x78,
313 0x04,0x44,0x24,0x64,0x14,0x54,0x34,0x74,0x0c,0x4c,0x2c,0x6c,0x1c,0x5c,0x3c,0x7c,
314 0x02,0x42,0x22,0x62,0x12,0x52,0x32,0x72,0x0a,0x4a,0x2a,0x6a,0x1a,0x5a,0x3a,0x7a,
315 0x06,0x46,0x26,0x66,0x16,0x56,0x36,0x76,0x0e,0x4e,0x2e,0x6e,0x1e,0x5e,0x3e,0x7e,
316 0x01,0x41,0x21,0x61,0x11,0x51,0x31,0x71,0x09,0x49,0x29,0x69,0x19,0x59,0x39,0x79,
317 0x05,0x45,0x25,0x65,0x15,0x55,0x35,0x75,0x0d,0x4d,0x2d,0x6d,0x1d,0x5d,0x3d,0x7d,
318 0x03,0x43,0x23,0x63,0x13,0x53,0x33,0x73,0x0b,0x4b,0x2b,0x6b,0x1b,0x5b,0x3b,0x7b,
319 0x07,0x47,0x27,0x67,0x17,0x57,0x37,0x77,0x0f,0x4f,0x2f,0x6f,0x1f,0x5f,0x3f,0x7f,
320 };
321 
322 usbd_status
323 ehci_init(ehci_softc_t *sc)
324 {
325 	u_int32_t vers, sparams, cparams, hcr;
326 	u_int i;
327 	usbd_status err;
328 	ehci_soft_qh_t *sqh;
329 	u_int ncomp;
330 
331 	DPRINTF(("ehci_init: start\n"));
332 #ifdef EHCI_DEBUG
333 	theehci = sc;
334 #endif
335 
336 	sc->sc_offs = EREAD1(sc, EHCI_CAPLENGTH);
337 
338 	vers = EREAD2(sc, EHCI_HCIVERSION);
339 	aprint_verbose("%s: EHCI version %x.%x\n", device_xname(sc->sc_dev),
340 	       vers >> 8, vers & 0xff);
341 
342 	sparams = EREAD4(sc, EHCI_HCSPARAMS);
343 	DPRINTF(("ehci_init: sparams=0x%x\n", sparams));
344 	sc->sc_npcomp = EHCI_HCS_N_PCC(sparams);
345 	ncomp = EHCI_HCS_N_CC(sparams);
346 	if (ncomp != sc->sc_ncomp) {
347 		aprint_verbose("%s: wrong number of companions (%d != %d)\n",
348 			       device_xname(sc->sc_dev), ncomp, sc->sc_ncomp);
349 #if NOHCI == 0 || NUHCI == 0
350 		aprint_error("%s: ohci or uhci probably not configured\n",
351 			     device_xname(sc->sc_dev));
352 #endif
353 		if (ncomp < sc->sc_ncomp)
354 			sc->sc_ncomp = ncomp;
355 	}
356 	if (sc->sc_ncomp > 0) {
357 		KASSERT(!(sc->sc_flags & EHCIF_ETTF));
358 		aprint_normal("%s: companion controller%s, %d port%s each:",
359 		    device_xname(sc->sc_dev), sc->sc_ncomp!=1 ? "s" : "",
360 		    EHCI_HCS_N_PCC(sparams),
361 		    EHCI_HCS_N_PCC(sparams)!=1 ? "s" : "");
362 		for (i = 0; i < sc->sc_ncomp; i++)
363 			aprint_normal(" %s", device_xname(sc->sc_comps[i]));
364 		aprint_normal("\n");
365 	}
366 	sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
367 	cparams = EREAD4(sc, EHCI_HCCPARAMS);
368 	DPRINTF(("ehci_init: cparams=0x%x\n", cparams));
369 	sc->sc_hasppc = EHCI_HCS_PPC(sparams);
370 
371 	if (EHCI_HCC_64BIT(cparams)) {
372 		/* MUST clear segment register if 64 bit capable. */
373 		EWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
374 	}
375 
376 	sc->sc_bus.usbrev = USBREV_2_0;
377 
378 	usb_setup_reserve(sc->sc_dev, &sc->sc_dma_reserve, sc->sc_bus.dmatag,
379 	    USB_MEM_RESERVE);
380 
381 	/* Reset the controller */
382 	DPRINTF(("%s: resetting\n", device_xname(sc->sc_dev)));
383 	EOWRITE4(sc, EHCI_USBCMD, 0);	/* Halt controller */
384 	usb_delay_ms(&sc->sc_bus, 1);
385 	EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
386 	for (i = 0; i < 100; i++) {
387 		usb_delay_ms(&sc->sc_bus, 1);
388 		hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
389 		if (!hcr)
390 			break;
391 	}
392 	if (hcr) {
393 		aprint_error("%s: reset timeout\n", device_xname(sc->sc_dev));
394 		return (USBD_IOERROR);
395 	}
396 	if (sc->sc_vendor_init)
397 		sc->sc_vendor_init(sc);
398 
399 	/*
400 	 * If we are doing embedded transaction translation function, force
401 	 * the controller to host mode.
402 	 */
403 	if (sc->sc_flags & EHCIF_ETTF) {
404 		uint32_t usbmode = EREAD4(sc, EHCI_USBMODE);
405 		usbmode &= ~EHCI_USBMODE_CM;
406 		usbmode |= EHCI_USBMODE_CM_HOST;
407 		EWRITE4(sc, EHCI_USBMODE, usbmode);
408 	}
409 
410 	/* XXX need proper intr scheduling */
411 	sc->sc_rand = 96;
412 
413 	/* frame list size at default, read back what we got and use that */
414 	switch (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD))) {
415 	case 0: sc->sc_flsize = 1024; break;
416 	case 1: sc->sc_flsize = 512; break;
417 	case 2: sc->sc_flsize = 256; break;
418 	case 3: return (USBD_IOERROR);
419 	}
420 	err = usb_allocmem(&sc->sc_bus, sc->sc_flsize * sizeof(ehci_link_t),
421 	    EHCI_FLALIGN_ALIGN, &sc->sc_fldma);
422 	if (err)
423 		return (err);
424 	DPRINTF(("%s: flsize=%d\n", device_xname(sc->sc_dev),sc->sc_flsize));
425 	sc->sc_flist = KERNADDR(&sc->sc_fldma, 0);
426 
427 	for (i = 0; i < sc->sc_flsize; i++) {
428 		sc->sc_flist[i] = EHCI_NULL;
429 	}
430 
431 	EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
432 
433 	sc->sc_softitds = malloc(sc->sc_flsize * sizeof(ehci_soft_itd_t *),
434 					M_USB, M_NOWAIT | M_ZERO);
435 	if (sc->sc_softitds == NULL)
436 		return ENOMEM;
437 	LIST_INIT(&sc->sc_freeitds);
438 	TAILQ_INIT(&sc->sc_intrhead);
439 	mutex_init(&sc->sc_intrhead_lock, MUTEX_DEFAULT, IPL_USB);
440 
441 	/* Set up the bus struct. */
442 	sc->sc_bus.methods = &ehci_bus_methods;
443 	sc->sc_bus.pipe_size = sizeof(struct ehci_pipe);
444 
445 	sc->sc_eintrs = EHCI_NORMAL_INTRS;
446 
447 	/*
448 	 * Allocate the interrupt dummy QHs. These are arranged to give poll
449 	 * intervals that are powers of 2 times 1ms.
450 	 */
451 	for (i = 0; i < EHCI_INTRQHS; i++) {
452 		sqh = ehci_alloc_sqh(sc);
453 		if (sqh == NULL) {
454 			err = USBD_NOMEM;
455 			goto bad1;
456 		}
457 		sc->sc_islots[i].sqh = sqh;
458 	}
459 	for (i = 0; i < EHCI_INTRQHS; i++) {
460 		sqh = sc->sc_islots[i].sqh;
461 		if (i == 0) {
462 			/* The last (1ms) QH terminates. */
463 			sqh->qh.qh_link = EHCI_NULL;
464 			sqh->next = NULL;
465 		} else {
466 			/* Otherwise the next QH has half the poll interval */
467 			sqh->next = sc->sc_islots[(i + 1) / 2 - 1].sqh;
468 			sqh->qh.qh_link = htole32(sqh->next->physaddr |
469 			    EHCI_LINK_QH);
470 		}
471 		sqh->qh.qh_endp = htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
472 		sqh->qh.qh_curqtd = EHCI_NULL;
473 		sqh->next = NULL;
474 		sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
475 		sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
476 		sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
477 		sqh->sqtd = NULL;
478 		usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
479 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
480 	}
481 	/* Point the frame list at the last level (128ms). */
482 	for (i = 0; i < sc->sc_flsize; i++) {
483 		int j;
484 
485 		j = (i & ~(EHCI_MAX_POLLRATE-1)) |
486 		    revbits[i & (EHCI_MAX_POLLRATE-1)];
487 		sc->sc_flist[j] = htole32(EHCI_LINK_QH |
488 		    sc->sc_islots[EHCI_IQHIDX(EHCI_IPOLLRATES - 1,
489 		    i)].sqh->physaddr);
490 	}
491 	usb_syncmem(&sc->sc_fldma, 0, sc->sc_flsize * sizeof(ehci_link_t),
492 	    BUS_DMASYNC_PREWRITE);
493 
494 	/* Allocate dummy QH that starts the async list. */
495 	sqh = ehci_alloc_sqh(sc);
496 	if (sqh == NULL) {
497 		err = USBD_NOMEM;
498 		goto bad1;
499 	}
500 	/* Fill the QH */
501 	sqh->qh.qh_endp =
502 	    htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
503 	sqh->qh.qh_link =
504 	    htole32(sqh->physaddr | EHCI_LINK_QH);
505 	sqh->qh.qh_curqtd = EHCI_NULL;
506 	sqh->next = NULL;
507 	/* Fill the overlay qTD */
508 	sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
509 	sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
510 	sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
511 	sqh->sqtd = NULL;
512 	usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
513 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
514 #ifdef EHCI_DEBUG
515 	if (ehcidebug) {
516 		ehci_dump_sqh(sqh);
517 	}
518 #endif
519 
520 	/* Point to async list */
521 	sc->sc_async_head = sqh;
522 	EOWRITE4(sc, EHCI_ASYNCLISTADDR, sqh->physaddr | EHCI_LINK_QH);
523 
524 	callout_init(&(sc->sc_tmo_intrlist), 0);
525 
526 	mutex_init(&sc->sc_doorbell_lock, MUTEX_DEFAULT, IPL_NONE);
527 
528 	/* Turn on controller */
529 	EOWRITE4(sc, EHCI_USBCMD,
530 		 EHCI_CMD_ITC_2 | /* 2 microframes interrupt delay */
531 		 (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
532 		 EHCI_CMD_ASE |
533 		 EHCI_CMD_PSE |
534 		 EHCI_CMD_RS);
535 
536 	/* Take over port ownership */
537 	EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
538 
539 	for (i = 0; i < 100; i++) {
540 		usb_delay_ms(&sc->sc_bus, 1);
541 		hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
542 		if (!hcr)
543 			break;
544 	}
545 	if (hcr) {
546 		aprint_error("%s: run timeout\n", device_xname(sc->sc_dev));
547 		return (USBD_IOERROR);
548 	}
549 
550 	/* Enable interrupts */
551 	DPRINTFN(1,("ehci_init: enabling\n"));
552 	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
553 
554 	return (USBD_NORMAL_COMPLETION);
555 
556 #if 0
557  bad2:
558 	ehci_free_sqh(sc, sc->sc_async_head);
559 #endif
560  bad1:
561 	usb_freemem(&sc->sc_bus, &sc->sc_fldma);
562 	return (err);
563 }
564 
565 int
566 ehci_intr(void *v)
567 {
568 	ehci_softc_t *sc = v;
569 
570 	if (sc == NULL || sc->sc_dying || !device_has_power(sc->sc_dev))
571 		return (0);
572 
573 	/* If we get an interrupt while polling, then just ignore it. */
574 	if (sc->sc_bus.use_polling) {
575 		u_int32_t intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
576 
577 		if (intrs)
578 			EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
579 #ifdef DIAGNOSTIC
580 		DPRINTFN(16, ("ehci_intr: ignored interrupt while polling\n"));
581 #endif
582 		return (0);
583 	}
584 
585 	return (ehci_intr1(sc));
586 }
587 
588 Static int
589 ehci_intr1(ehci_softc_t *sc)
590 {
591 	u_int32_t intrs, eintrs;
592 
593 	DPRINTFN(20,("ehci_intr1: enter\n"));
594 
595 	/* In case the interrupt occurs before initialization has completed. */
596 	if (sc == NULL) {
597 #ifdef DIAGNOSTIC
598 		printf("ehci_intr1: sc == NULL\n");
599 #endif
600 		return (0);
601 	}
602 
603 	intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
604 	if (!intrs)
605 		return (0);
606 
607 	eintrs = intrs & sc->sc_eintrs;
608 	DPRINTFN(7, ("ehci_intr1: sc=%p intrs=0x%x(0x%x) eintrs=0x%x\n",
609 		     sc, (u_int)intrs, EOREAD4(sc, EHCI_USBSTS),
610 		     (u_int)eintrs));
611 	if (!eintrs)
612 		return (0);
613 
614 	EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
615 	sc->sc_bus.intr_context++;
616 	sc->sc_bus.no_intrs++;
617 	if (eintrs & EHCI_STS_IAA) {
618 		DPRINTF(("ehci_intr1: door bell\n"));
619 		wakeup(&sc->sc_async_head);
620 		eintrs &= ~EHCI_STS_IAA;
621 	}
622 	if (eintrs & (EHCI_STS_INT | EHCI_STS_ERRINT)) {
623 		DPRINTFN(5,("ehci_intr1: %s %s\n",
624 			    eintrs & EHCI_STS_INT ? "INT" : "",
625 			    eintrs & EHCI_STS_ERRINT ? "ERRINT" : ""));
626 		usb_schedsoftintr(&sc->sc_bus);
627 		eintrs &= ~(EHCI_STS_INT | EHCI_STS_ERRINT);
628 	}
629 	if (eintrs & EHCI_STS_HSE) {
630 		printf("%s: unrecoverable error, controller halted\n",
631 		       device_xname(sc->sc_dev));
632 		/* XXX what else */
633 	}
634 	if (eintrs & EHCI_STS_PCD) {
635 		ehci_pcd(sc, sc->sc_intrxfer);
636 		eintrs &= ~EHCI_STS_PCD;
637 	}
638 
639 	sc->sc_bus.intr_context--;
640 
641 	if (eintrs != 0) {
642 		/* Block unprocessed interrupts. */
643 		sc->sc_eintrs &= ~eintrs;
644 		EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
645 		printf("%s: blocking intrs 0x%x\n",
646 		       device_xname(sc->sc_dev), eintrs);
647 	}
648 
649 	return (1);
650 }
651 
652 
653 Static void
654 ehci_pcd(ehci_softc_t *sc, usbd_xfer_handle xfer)
655 {
656 	usbd_pipe_handle pipe;
657 	u_char *p;
658 	int i, m;
659 
660 	if (xfer == NULL) {
661 		/* Just ignore the change. */
662 		return;
663 	}
664 
665 	pipe = xfer->pipe;
666 
667 	p = KERNADDR(&xfer->dmabuf, 0);
668 	m = min(sc->sc_noport, xfer->length * 8 - 1);
669 	memset(p, 0, xfer->length);
670 	for (i = 1; i <= m; i++) {
671 		/* Pick out CHANGE bits from the status reg. */
672 		if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR)
673 			p[i/8] |= 1 << (i%8);
674 	}
675 	DPRINTF(("ehci_pcd: change=0x%02x\n", *p));
676 	xfer->actlen = xfer->length;
677 	xfer->status = USBD_NORMAL_COMPLETION;
678 
679 	usb_transfer_complete(xfer);
680 }
681 
682 Static void
683 ehci_softintr(void *v)
684 {
685 	struct usbd_bus *bus = v;
686 	ehci_softc_t *sc = bus->hci_private;
687 	struct ehci_xfer *ex, *nextex;
688 
689 	DPRINTFN(10,("%s: ehci_softintr (%d)\n", device_xname(sc->sc_dev),
690 		     sc->sc_bus.intr_context));
691 
692 	sc->sc_bus.intr_context++;
693 
694 	/*
695 	 * The only explanation I can think of for why EHCI is as brain dead
696 	 * as UHCI interrupt-wise is that Intel was involved in both.
697 	 * An interrupt just tells us that something is done, we have no
698 	 * clue what, so we need to scan through all active transfers. :-(
699 	 */
700 	for (ex = TAILQ_FIRST(&sc->sc_intrhead); ex; ex = nextex) {
701 		nextex = TAILQ_NEXT(ex, inext);
702 		ehci_check_intr(sc, ex);
703 	}
704 
705 	/* Schedule a callout to catch any dropped transactions. */
706 	if ((sc->sc_flags & EHCIF_DROPPED_INTR_WORKAROUND) &&
707 	    !TAILQ_EMPTY(&sc->sc_intrhead))
708 		callout_reset(&(sc->sc_tmo_intrlist),
709 		    (hz), (ehci_intrlist_timeout), (sc));
710 
711 #ifdef USB_USE_SOFTINTR
712 	if (sc->sc_softwake) {
713 		sc->sc_softwake = 0;
714 		wakeup(&sc->sc_softwake);
715 	}
716 #endif /* USB_USE_SOFTINTR */
717 
718 	sc->sc_bus.intr_context--;
719 }
720 
721 /* Check for an interrupt. */
722 Static void
723 ehci_check_intr(ehci_softc_t *sc, struct ehci_xfer *ex)
724 {
725 	int attr;
726 
727 	DPRINTFN(/*15*/2, ("ehci_check_intr: ex=%p\n", ex));
728 
729 	attr = ex->xfer.pipe->endpoint->edesc->bmAttributes;
730 	if (UE_GET_XFERTYPE(attr) == UE_ISOCHRONOUS)
731 		ehci_check_itd_intr(sc, ex);
732 	else
733 		ehci_check_qh_intr(sc, ex);
734 
735 	return;
736 }
737 
738 Static void
739 ehci_check_qh_intr(ehci_softc_t *sc, struct ehci_xfer *ex)
740 {
741 	ehci_soft_qtd_t *sqtd, *lsqtd;
742 	__uint32_t status;
743 
744 	if (ex->sqtdstart == NULL) {
745 		printf("ehci_check_qh_intr: not valid sqtd\n");
746 		return;
747 	}
748 
749 	lsqtd = ex->sqtdend;
750 #ifdef DIAGNOSTIC
751 	if (lsqtd == NULL) {
752 		printf("ehci_check_qh_intr: lsqtd==0\n");
753 		return;
754 	}
755 #endif
756 	/*
757 	 * If the last TD is still active we need to check whether there
758 	 * is a an error somewhere in the middle, or whether there was a
759 	 * short packet (SPD and not ACTIVE).
760 	 */
761 	usb_syncmem(&lsqtd->dma,
762 	    lsqtd->offs + offsetof(ehci_qtd_t, qtd_status),
763 	    sizeof(lsqtd->qtd.qtd_status),
764 	    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
765 	if (le32toh(lsqtd->qtd.qtd_status) & EHCI_QTD_ACTIVE) {
766 		DPRINTFN(12, ("ehci_check_intr: active ex=%p\n", ex));
767 		for (sqtd = ex->sqtdstart; sqtd != lsqtd; sqtd=sqtd->nextqtd) {
768 			usb_syncmem(&sqtd->dma,
769 			    sqtd->offs + offsetof(ehci_qtd_t, qtd_status),
770 			    sizeof(sqtd->qtd.qtd_status),
771 			    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
772 			status = le32toh(sqtd->qtd.qtd_status);
773 			usb_syncmem(&sqtd->dma,
774 			    sqtd->offs + offsetof(ehci_qtd_t, qtd_status),
775 			    sizeof(sqtd->qtd.qtd_status), BUS_DMASYNC_PREREAD);
776 			/* If there's an active QTD the xfer isn't done. */
777 			if (status & EHCI_QTD_ACTIVE)
778 				break;
779 			/* Any kind of error makes the xfer done. */
780 			if (status & EHCI_QTD_HALTED)
781 				goto done;
782 			/* We want short packets, and it is short: it's done */
783 			if (EHCI_QTD_GET_BYTES(status) != 0)
784 				goto done;
785 		}
786 		DPRINTFN(12, ("ehci_check_intr: ex=%p std=%p still active\n",
787 			      ex, ex->sqtdstart));
788 		usb_syncmem(&lsqtd->dma,
789 		    lsqtd->offs + offsetof(ehci_qtd_t, qtd_status),
790 		    sizeof(lsqtd->qtd.qtd_status), BUS_DMASYNC_PREREAD);
791 		return;
792 	}
793  done:
794 	DPRINTFN(12, ("ehci_check_intr: ex=%p done\n", ex));
795 	callout_stop(&ex->xfer.timeout_handle);
796 	ehci_idone(ex);
797 }
798 
799 Static void
800 ehci_check_itd_intr(ehci_softc_t *sc, struct ehci_xfer *ex) {
801 	ehci_soft_itd_t *itd;
802 	int i;
803 
804 	if (&ex->xfer != SIMPLEQ_FIRST(&ex->xfer.pipe->queue))
805 		return;
806 
807 	if (ex->itdstart == NULL) {
808 		printf("ehci_check_itd_intr: not valid itd\n");
809 		return;
810 	}
811 
812 	itd = ex->itdend;
813 #ifdef DIAGNOSTIC
814 	if (itd == NULL) {
815 		printf("ehci_check_itd_intr: itdend == 0\n");
816 		return;
817 	}
818 #endif
819 
820 	/*
821 	 * check no active transfers in last itd, meaning we're finished
822 	 */
823 
824 	usb_syncmem(&itd->dma, itd->offs + offsetof(ehci_itd_t, itd_ctl),
825 		    sizeof(itd->itd.itd_ctl), BUS_DMASYNC_POSTWRITE |
826 		    BUS_DMASYNC_POSTREAD);
827 
828 	for (i = 0; i < EHCI_ITD_NUFRAMES; i++) {
829 		if (le32toh(itd->itd.itd_ctl[i]) & EHCI_ITD_ACTIVE)
830 			break;
831 	}
832 
833 	if (i == EHCI_ITD_NUFRAMES) {
834 		goto done; /* All 8 descriptors inactive, it's done */
835 	}
836 
837 	DPRINTFN(12, ("ehci_check_itd_intr: ex %p itd %p still active\n", ex,
838 			ex->itdstart));
839 	return;
840 done:
841 	DPRINTFN(12, ("ehci_check_itd_intr: ex=%p done\n", ex));
842 	callout_stop(&ex->xfer.timeout_handle);
843 	ehci_idone(ex);
844 }
845 
846 Static void
847 ehci_idone(struct ehci_xfer *ex)
848 {
849 	usbd_xfer_handle xfer = &ex->xfer;
850 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
851 	ehci_soft_qtd_t *sqtd, *lsqtd;
852 	u_int32_t status = 0, nstatus = 0;
853 	int actlen;
854 
855 	DPRINTFN(/*12*/2, ("ehci_idone: ex=%p\n", ex));
856 #ifdef DIAGNOSTIC
857 	{
858 		int s = splhigh();
859 		if (ex->isdone) {
860 			splx(s);
861 #ifdef EHCI_DEBUG
862 			printf("ehci_idone: ex is done!\n   ");
863 			ehci_dump_exfer(ex);
864 #else
865 			printf("ehci_idone: ex=%p is done!\n", ex);
866 #endif
867 			return;
868 		}
869 		ex->isdone = 1;
870 		splx(s);
871 	}
872 #endif
873 	if (xfer->status == USBD_CANCELLED ||
874 	    xfer->status == USBD_TIMEOUT) {
875 		DPRINTF(("ehci_idone: aborted xfer=%p\n", xfer));
876 		return;
877 	}
878 
879 #ifdef EHCI_DEBUG
880 	DPRINTFN(/*10*/2, ("ehci_idone: xfer=%p, pipe=%p ready\n", xfer, epipe));
881 	if (ehcidebug > 10)
882 		ehci_dump_sqtds(ex->sqtdstart);
883 #endif
884 
885 	/* The transfer is done, compute actual length and status. */
886 
887 	if (UE_GET_XFERTYPE(xfer->pipe->endpoint->edesc->bmAttributes)
888 				== UE_ISOCHRONOUS) {
889 		/* Isoc transfer */
890 		struct ehci_soft_itd *itd;
891 		int i, nframes, len, uframes;
892 
893 		nframes = 0;
894 		actlen = 0;
895 
896 		i = xfer->pipe->endpoint->edesc->bInterval;
897 		uframes = min(1 << (i - 1), USB_UFRAMES_PER_FRAME);
898 
899 		for (itd = ex->itdstart; itd != NULL; itd = itd->xfer_next) {
900 			usb_syncmem(&itd->dma,itd->offs + offsetof(ehci_itd_t,itd_ctl),
901 			    sizeof(itd->itd.itd_ctl), BUS_DMASYNC_POSTWRITE |
902 			    BUS_DMASYNC_POSTREAD);
903 
904 			for (i = 0; i < EHCI_ITD_NUFRAMES; i += uframes) {
905 				/* XXX - driver didn't fill in the frame full
906 				 *   of uframes. This leads to scheduling
907 				 *   inefficiencies, but working around
908 				 *   this doubles complexity of tracking
909 				 *   an xfer.
910 				 */
911 				if (nframes >= xfer->nframes)
912 					break;
913 
914 				status = le32toh(itd->itd.itd_ctl[i]);
915 				len = EHCI_ITD_GET_LEN(status);
916 				if (EHCI_ITD_GET_STATUS(status) != 0)
917 					len = 0; /*No valid data on error*/
918 
919 				xfer->frlengths[nframes++] = len;
920 				actlen += len;
921 			}
922 
923 			if (nframes >= xfer->nframes)
924 				break;
925 	    	}
926 
927 		xfer->actlen = actlen;
928 		xfer->status = USBD_NORMAL_COMPLETION;
929 		goto end;
930 	}
931 
932 	/* Continue processing xfers using queue heads */
933 
934 	lsqtd = ex->sqtdend;
935 	actlen = 0;
936 	for (sqtd = ex->sqtdstart; sqtd != lsqtd->nextqtd; sqtd = sqtd->nextqtd) {
937 		usb_syncmem(&sqtd->dma, sqtd->offs, sizeof(sqtd->qtd),
938 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
939 		nstatus = le32toh(sqtd->qtd.qtd_status);
940 		if (nstatus & EHCI_QTD_ACTIVE)
941 			break;
942 
943 		status = nstatus;
944 		if (EHCI_QTD_GET_PID(status) != EHCI_QTD_PID_SETUP)
945 			actlen += sqtd->len - EHCI_QTD_GET_BYTES(status);
946 	}
947 
948 
949 	/*
950 	 * If there are left over TDs we need to update the toggle.
951 	 * The default pipe doesn't need it since control transfers
952 	 * start the toggle at 0 every time.
953 	 * For a short transfer we need to update the toggle for the missing
954 	 * packets within the qTD.
955 	 */
956 	if ((sqtd != lsqtd->nextqtd || EHCI_QTD_GET_BYTES(status)) &&
957 	    xfer->pipe->device->default_pipe != xfer->pipe) {
958 		DPRINTFN(2, ("ehci_idone: need toggle update "
959 			     "status=%08x nstatus=%08x\n", status, nstatus));
960 #if 0
961 		ehci_dump_sqh(epipe->sqh);
962 		ehci_dump_sqtds(ex->sqtdstart);
963 #endif
964 		epipe->nexttoggle = EHCI_QTD_GET_TOGGLE(nstatus);
965 	}
966 
967 	DPRINTFN(/*10*/2, ("ehci_idone: len=%d, actlen=%d, status=0x%x\n",
968 			   xfer->length, actlen, status));
969 	xfer->actlen = actlen;
970 	if (status & EHCI_QTD_HALTED) {
971 #ifdef EHCI_DEBUG
972 		char sbuf[128];
973 
974 		snprintb(sbuf, sizeof(sbuf),
975 		    "\20\7HALTED\6BUFERR\5BABBLE\4XACTERR\3MISSED\1PINGSTATE",
976 		    (u_int32_t)status);
977 
978 		DPRINTFN(2, ("ehci_idone: error, addr=%d, endpt=0x%02x, "
979 			  "status 0x%s\n",
980 			  xfer->pipe->device->address,
981 			  xfer->pipe->endpoint->edesc->bEndpointAddress,
982 			  sbuf));
983 		if (ehcidebug > 2) {
984 			ehci_dump_sqh(epipe->sqh);
985 			ehci_dump_sqtds(ex->sqtdstart);
986 		}
987 #endif
988 		/* low&full speed has an extra error flag */
989 		if (EHCI_QH_GET_EPS(epipe->sqh->qh.qh_endp) !=
990 		    EHCI_QH_SPEED_HIGH)
991 			status &= EHCI_QTD_STATERRS | EHCI_QTD_PINGSTATE;
992 		else
993 			status &= EHCI_QTD_STATERRS;
994 		if (status == 0) /* no other errors means a stall */ {
995 			xfer->status = USBD_STALLED;
996 		} else {
997 			xfer->status = USBD_IOERROR; /* more info XXX */
998 		}
999 		/* XXX need to reset TT on missed microframe */
1000 		if (status & EHCI_QTD_MISSEDMICRO) {
1001 			ehci_softc_t *sc =
1002 			    xfer->pipe->device->bus->hci_private;
1003 
1004 			printf("%s: missed microframe, TT reset not "
1005 			    "implemented, hub might be inoperational\n",
1006 			    device_xname(sc->sc_dev));
1007 		}
1008 	} else {
1009 		xfer->status = USBD_NORMAL_COMPLETION;
1010 	}
1011 
1012     end:
1013 	/* XXX transfer_complete memcpys out transfer data (for in endpoints)
1014 	 * during this call, before methods->done is called: dma sync required
1015 	 * beforehand? */
1016 	usb_transfer_complete(xfer);
1017 	DPRINTFN(/*12*/2, ("ehci_idone: ex=%p done\n", ex));
1018 }
1019 
1020 /*
1021  * Wait here until controller claims to have an interrupt.
1022  * Then call ehci_intr and return.  Use timeout to avoid waiting
1023  * too long.
1024  */
1025 Static void
1026 ehci_waitintr(ehci_softc_t *sc, usbd_xfer_handle xfer)
1027 {
1028 	int timo;
1029 	u_int32_t intrs;
1030 
1031 	xfer->status = USBD_IN_PROGRESS;
1032 	for (timo = xfer->timeout; timo >= 0; timo--) {
1033 		usb_delay_ms(&sc->sc_bus, 1);
1034 		if (sc->sc_dying)
1035 			break;
1036 		intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS)) &
1037 			sc->sc_eintrs;
1038 		DPRINTFN(15,("ehci_waitintr: 0x%04x\n", intrs));
1039 #ifdef EHCI_DEBUG
1040 		if (ehcidebug > 15)
1041 			ehci_dump_regs(sc);
1042 #endif
1043 		if (intrs) {
1044 			ehci_intr1(sc);
1045 			if (xfer->status != USBD_IN_PROGRESS)
1046 				return;
1047 		}
1048 	}
1049 
1050 	/* Timeout */
1051 	DPRINTF(("ehci_waitintr: timeout\n"));
1052 	xfer->status = USBD_TIMEOUT;
1053 	usb_transfer_complete(xfer);
1054 	/* XXX should free TD */
1055 }
1056 
1057 Static void
1058 ehci_poll(struct usbd_bus *bus)
1059 {
1060 	ehci_softc_t *sc = bus->hci_private;
1061 #ifdef EHCI_DEBUG
1062 	static int last;
1063 	int new;
1064 	new = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
1065 	if (new != last) {
1066 		DPRINTFN(10,("ehci_poll: intrs=0x%04x\n", new));
1067 		last = new;
1068 	}
1069 #endif
1070 
1071 	if (EOREAD4(sc, EHCI_USBSTS) & sc->sc_eintrs)
1072 		ehci_intr1(sc);
1073 }
1074 
1075 void
1076 ehci_childdet(device_t self, device_t child)
1077 {
1078 	struct ehci_softc *sc = device_private(self);
1079 
1080 	KASSERT(sc->sc_child == child);
1081 	sc->sc_child = NULL;
1082 }
1083 
1084 int
1085 ehci_detach(struct ehci_softc *sc, int flags)
1086 {
1087 	int rv = 0;
1088 
1089 	if (sc->sc_child != NULL)
1090 		rv = config_detach(sc->sc_child, flags);
1091 
1092 	if (rv != 0)
1093 		return (rv);
1094 
1095 	callout_stop(&sc->sc_tmo_intrlist);
1096 
1097 	usb_delay_ms(&sc->sc_bus, 300); /* XXX let stray task complete */
1098 
1099 	/* XXX free other data structures XXX */
1100 	mutex_destroy(&sc->sc_doorbell_lock);
1101 	mutex_destroy(&sc->sc_intrhead_lock);
1102 
1103 	EOWRITE4(sc, EHCI_CONFIGFLAG, 0);
1104 
1105 	return (rv);
1106 }
1107 
1108 
1109 int
1110 ehci_activate(device_t self, enum devact act)
1111 {
1112 	struct ehci_softc *sc = device_private(self);
1113 
1114 	switch (act) {
1115 	case DVACT_DEACTIVATE:
1116 		sc->sc_dying = 1;
1117 		return 0;
1118 	default:
1119 		return EOPNOTSUPP;
1120 	}
1121 }
1122 
1123 /*
1124  * Handle suspend/resume.
1125  *
1126  * We need to switch to polling mode here, because this routine is
1127  * called from an interrupt context.  This is all right since we
1128  * are almost suspended anyway.
1129  *
1130  * Note that this power handler isn't to be registered directly; the
1131  * bus glue needs to call out to it.
1132  */
1133 bool
1134 ehci_suspend(device_t dv, const pmf_qual_t *qual)
1135 {
1136 	ehci_softc_t *sc = device_private(dv);
1137 	int i, s;
1138 	uint32_t cmd, hcr;
1139 
1140 	s = splhardusb();
1141 
1142 	sc->sc_bus.use_polling++;
1143 
1144 	for (i = 1; i <= sc->sc_noport; i++) {
1145 		cmd = EOREAD4(sc, EHCI_PORTSC(i)) & ~EHCI_PS_CLEAR;
1146 		if ((cmd & EHCI_PS_PO) == 0 && (cmd & EHCI_PS_PE) == EHCI_PS_PE)
1147 			EOWRITE4(sc, EHCI_PORTSC(i), cmd | EHCI_PS_SUSP);
1148 	}
1149 
1150 	sc->sc_cmd = EOREAD4(sc, EHCI_USBCMD);
1151 
1152 	cmd = sc->sc_cmd & ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
1153 	EOWRITE4(sc, EHCI_USBCMD, cmd);
1154 
1155 	for (i = 0; i < 100; i++) {
1156 		hcr = EOREAD4(sc, EHCI_USBSTS) & (EHCI_STS_ASS | EHCI_STS_PSS);
1157 		if (hcr == 0)
1158 			break;
1159 
1160 		usb_delay_ms(&sc->sc_bus, 1);
1161 	}
1162 	if (hcr != 0)
1163 		printf("%s: reset timeout\n", device_xname(dv));
1164 
1165 	cmd &= ~EHCI_CMD_RS;
1166 	EOWRITE4(sc, EHCI_USBCMD, cmd);
1167 
1168 	for (i = 0; i < 100; i++) {
1169 		hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1170 		if (hcr == EHCI_STS_HCH)
1171 			break;
1172 
1173 		usb_delay_ms(&sc->sc_bus, 1);
1174 	}
1175 	if (hcr != EHCI_STS_HCH)
1176 		printf("%s: config timeout\n", device_xname(dv));
1177 
1178 	sc->sc_bus.use_polling--;
1179 	splx(s);
1180 
1181 	return true;
1182 }
1183 
1184 bool
1185 ehci_resume(device_t dv, const pmf_qual_t *qual)
1186 {
1187 	ehci_softc_t *sc = device_private(dv);
1188 	int i;
1189 	uint32_t cmd, hcr;
1190 
1191 	/* restore things in case the bios sucks */
1192 	EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
1193 	EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
1194 	EOWRITE4(sc, EHCI_ASYNCLISTADDR,
1195 	    sc->sc_async_head->physaddr | EHCI_LINK_QH);
1196 
1197 	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs & ~EHCI_INTR_PCIE);
1198 
1199 	EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
1200 
1201 	hcr = 0;
1202 	for (i = 1; i <= sc->sc_noport; i++) {
1203 		cmd = EOREAD4(sc, EHCI_PORTSC(i)) & ~EHCI_PS_CLEAR;
1204 		if ((cmd & EHCI_PS_PO) == 0 &&
1205 		    (cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP) {
1206 			EOWRITE4(sc, EHCI_PORTSC(i), cmd | EHCI_PS_FPR);
1207 			hcr = 1;
1208 		}
1209 	}
1210 
1211 	if (hcr) {
1212 		usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);
1213 
1214 		for (i = 1; i <= sc->sc_noport; i++) {
1215 			cmd = EOREAD4(sc, EHCI_PORTSC(i)) & ~EHCI_PS_CLEAR;
1216 			if ((cmd & EHCI_PS_PO) == 0 &&
1217 			    (cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)
1218 				EOWRITE4(sc, EHCI_PORTSC(i),
1219 				    cmd & ~EHCI_PS_FPR);
1220 		}
1221 	}
1222 
1223 	EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
1224 	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1225 
1226 	for (i = 0; i < 100; i++) {
1227 		hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1228 		if (hcr != EHCI_STS_HCH)
1229 			break;
1230 
1231 		usb_delay_ms(&sc->sc_bus, 1);
1232 	}
1233 	if (hcr == EHCI_STS_HCH)
1234 		printf("%s: config timeout\n", device_xname(dv));
1235 
1236 	return true;
1237 }
1238 
1239 /*
1240  * Shut down the controller when the system is going down.
1241  */
1242 bool
1243 ehci_shutdown(device_t self, int flags)
1244 {
1245 	ehci_softc_t *sc = device_private(self);
1246 
1247 	DPRINTF(("ehci_shutdown: stopping the HC\n"));
1248 	EOWRITE4(sc, EHCI_USBCMD, 0);	/* Halt controller */
1249 	EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
1250 	return true;
1251 }
1252 
1253 Static usbd_status
1254 ehci_allocm(struct usbd_bus *bus, usb_dma_t *dma, u_int32_t size)
1255 {
1256 	struct ehci_softc *sc = bus->hci_private;
1257 	usbd_status err;
1258 
1259 	err = usb_allocmem(&sc->sc_bus, size, 0, dma);
1260 	if (err == USBD_NOMEM)
1261 		err = usb_reserve_allocm(&sc->sc_dma_reserve, dma, size);
1262 #ifdef EHCI_DEBUG
1263 	if (err)
1264 		printf("ehci_allocm: usb_allocmem()=%d\n", err);
1265 #endif
1266 	return (err);
1267 }
1268 
1269 Static void
1270 ehci_freem(struct usbd_bus *bus, usb_dma_t *dma)
1271 {
1272 	struct ehci_softc *sc = bus->hci_private;
1273 
1274 	if (dma->block->flags & USB_DMA_RESERVE) {
1275 		usb_reserve_freem(&sc->sc_dma_reserve,
1276 		    dma);
1277 		return;
1278 	}
1279 	usb_freemem(&sc->sc_bus, dma);
1280 }
1281 
1282 Static usbd_xfer_handle
1283 ehci_allocx(struct usbd_bus *bus)
1284 {
1285 	struct ehci_softc *sc = bus->hci_private;
1286 	usbd_xfer_handle xfer;
1287 
1288 	xfer = SIMPLEQ_FIRST(&sc->sc_free_xfers);
1289 	if (xfer != NULL) {
1290 		SIMPLEQ_REMOVE_HEAD(&sc->sc_free_xfers, next);
1291 #ifdef DIAGNOSTIC
1292 		if (xfer->busy_free != XFER_FREE) {
1293 			printf("ehci_allocx: xfer=%p not free, 0x%08x\n", xfer,
1294 			       xfer->busy_free);
1295 		}
1296 #endif
1297 	} else {
1298 		xfer = malloc(sizeof(struct ehci_xfer), M_USB, M_NOWAIT);
1299 	}
1300 	if (xfer != NULL) {
1301 		memset(xfer, 0, sizeof(struct ehci_xfer));
1302 #ifdef DIAGNOSTIC
1303 		EXFER(xfer)->isdone = 1;
1304 		xfer->busy_free = XFER_BUSY;
1305 #endif
1306 	}
1307 	return (xfer);
1308 }
1309 
1310 Static void
1311 ehci_freex(struct usbd_bus *bus, usbd_xfer_handle xfer)
1312 {
1313 	struct ehci_softc *sc = bus->hci_private;
1314 
1315 #ifdef DIAGNOSTIC
1316 	if (xfer->busy_free != XFER_BUSY) {
1317 		printf("ehci_freex: xfer=%p not busy, 0x%08x\n", xfer,
1318 		       xfer->busy_free);
1319 	}
1320 	xfer->busy_free = XFER_FREE;
1321 	if (!EXFER(xfer)->isdone) {
1322 		printf("ehci_freex: !isdone\n");
1323 	}
1324 #endif
1325 	SIMPLEQ_INSERT_HEAD(&sc->sc_free_xfers, xfer, next);
1326 }
1327 
1328 Static void
1329 ehci_device_clear_toggle(usbd_pipe_handle pipe)
1330 {
1331 	struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1332 
1333 	DPRINTF(("ehci_device_clear_toggle: epipe=%p status=0x%x\n",
1334 		 epipe, epipe->sqh->qh.qh_qtd.qtd_status));
1335 #ifdef EHCI_DEBUG
1336 	if (ehcidebug)
1337 		usbd_dump_pipe(pipe);
1338 #endif
1339 	epipe->nexttoggle = 0;
1340 }
1341 
1342 Static void
1343 ehci_noop(usbd_pipe_handle pipe)
1344 {
1345 }
1346 
1347 #ifdef EHCI_DEBUG
1348 Static void
1349 ehci_dump_regs(ehci_softc_t *sc)
1350 {
1351 	int i;
1352 	printf("cmd=0x%08x, sts=0x%08x, ien=0x%08x\n",
1353 	       EOREAD4(sc, EHCI_USBCMD),
1354 	       EOREAD4(sc, EHCI_USBSTS),
1355 	       EOREAD4(sc, EHCI_USBINTR));
1356 	printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
1357 	       EOREAD4(sc, EHCI_FRINDEX),
1358 	       EOREAD4(sc, EHCI_CTRLDSSEGMENT),
1359 	       EOREAD4(sc, EHCI_PERIODICLISTBASE),
1360 	       EOREAD4(sc, EHCI_ASYNCLISTADDR));
1361 	for (i = 1; i <= sc->sc_noport; i++)
1362 		printf("port %d status=0x%08x\n", i,
1363 		       EOREAD4(sc, EHCI_PORTSC(i)));
1364 }
1365 
1366 /*
1367  * Unused function - this is meant to be called from a kernel
1368  * debugger.
1369  */
1370 void
1371 ehci_dump(void)
1372 {
1373 	ehci_dump_regs(theehci);
1374 }
1375 
1376 Static void
1377 ehci_dump_link(ehci_link_t link, int type)
1378 {
1379 	link = le32toh(link);
1380 	printf("0x%08x", link);
1381 	if (link & EHCI_LINK_TERMINATE)
1382 		printf("<T>");
1383 	else {
1384 		printf("<");
1385 		if (type) {
1386 			switch (EHCI_LINK_TYPE(link)) {
1387 			case EHCI_LINK_ITD: printf("ITD"); break;
1388 			case EHCI_LINK_QH: printf("QH"); break;
1389 			case EHCI_LINK_SITD: printf("SITD"); break;
1390 			case EHCI_LINK_FSTN: printf("FSTN"); break;
1391 			}
1392 		}
1393 		printf(">");
1394 	}
1395 }
1396 
1397 Static void
1398 ehci_dump_sqtds(ehci_soft_qtd_t *sqtd)
1399 {
1400 	int i;
1401 	u_int32_t stop;
1402 
1403 	stop = 0;
1404 	for (i = 0; sqtd && i < 20 && !stop; sqtd = sqtd->nextqtd, i++) {
1405 		ehci_dump_sqtd(sqtd);
1406 		usb_syncmem(&sqtd->dma,
1407 		    sqtd->offs + offsetof(ehci_qtd_t, qtd_next),
1408 		    sizeof(sqtd->qtd),
1409 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1410 		stop = sqtd->qtd.qtd_next & htole32(EHCI_LINK_TERMINATE);
1411 		usb_syncmem(&sqtd->dma,
1412 		    sqtd->offs + offsetof(ehci_qtd_t, qtd_next),
1413 		    sizeof(sqtd->qtd), BUS_DMASYNC_PREREAD);
1414 	}
1415 	if (sqtd)
1416 		printf("dump aborted, too many TDs\n");
1417 }
1418 
1419 Static void
1420 ehci_dump_sqtd(ehci_soft_qtd_t *sqtd)
1421 {
1422 	usb_syncmem(&sqtd->dma, sqtd->offs,
1423 	    sizeof(sqtd->qtd), BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1424 	printf("QTD(%p) at 0x%08x:\n", sqtd, sqtd->physaddr);
1425 	ehci_dump_qtd(&sqtd->qtd);
1426 	usb_syncmem(&sqtd->dma, sqtd->offs,
1427 	    sizeof(sqtd->qtd), BUS_DMASYNC_PREREAD);
1428 }
1429 
1430 Static void
1431 ehci_dump_qtd(ehci_qtd_t *qtd)
1432 {
1433 	u_int32_t s;
1434 	char sbuf[128];
1435 
1436 	printf("  next="); ehci_dump_link(qtd->qtd_next, 0);
1437 	printf(" altnext="); ehci_dump_link(qtd->qtd_altnext, 0);
1438 	printf("\n");
1439 	s = le32toh(qtd->qtd_status);
1440 	snprintb(sbuf, sizeof(sbuf),
1441 	    "\20\10ACTIVE\7HALTED\6BUFERR\5BABBLE\4XACTERR"
1442 	    "\3MISSED\2SPLIT\1PING", EHCI_QTD_GET_STATUS(s));
1443 	printf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
1444 	       s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
1445 	       EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
1446 	printf("    cerr=%d pid=%d stat=0x%s\n", EHCI_QTD_GET_CERR(s),
1447 	       EHCI_QTD_GET_PID(s), sbuf);
1448 	for (s = 0; s < 5; s++)
1449 		printf("  buffer[%d]=0x%08x\n", s, le32toh(qtd->qtd_buffer[s]));
1450 }
1451 
1452 Static void
1453 ehci_dump_sqh(ehci_soft_qh_t *sqh)
1454 {
1455 	ehci_qh_t *qh = &sqh->qh;
1456 	u_int32_t endp, endphub;
1457 
1458 	usb_syncmem(&sqh->dma, sqh->offs,
1459 	    sizeof(sqh->qh), BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1460 	printf("QH(%p) at 0x%08x:\n", sqh, sqh->physaddr);
1461 	printf("  link="); ehci_dump_link(qh->qh_link, 1); printf("\n");
1462 	endp = le32toh(qh->qh_endp);
1463 	printf("  endp=0x%08x\n", endp);
1464 	printf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
1465 	       EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
1466 	       EHCI_QH_GET_ENDPT(endp),  EHCI_QH_GET_EPS(endp),
1467 	       EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
1468 	printf("    mpl=0x%x ctl=%d nrl=%d\n",
1469 	       EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
1470 	       EHCI_QH_GET_NRL(endp));
1471 	endphub = le32toh(qh->qh_endphub);
1472 	printf("  endphub=0x%08x\n", endphub);
1473 	printf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
1474 	       EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
1475 	       EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
1476 	       EHCI_QH_GET_MULT(endphub));
1477 	printf("  curqtd="); ehci_dump_link(qh->qh_curqtd, 0); printf("\n");
1478 	printf("Overlay qTD:\n");
1479 	ehci_dump_qtd(&qh->qh_qtd);
1480 	usb_syncmem(&sqh->dma, sqh->offs,
1481 	    sizeof(sqh->qh), BUS_DMASYNC_PREREAD);
1482 }
1483 
1484 #if notyet
1485 Static void
1486 ehci_dump_itd(struct ehci_soft_itd *itd)
1487 {
1488 	ehci_isoc_trans_t t;
1489 	ehci_isoc_bufr_ptr_t b, b2, b3;
1490 	int i;
1491 
1492 	printf("ITD: next phys=%X\n", itd->itd.itd_next);
1493 
1494 	for (i = 0; i < EHCI_ITD_NUFRAMES; i++) {
1495 		t = le32toh(itd->itd.itd_ctl[i]);
1496 		printf("ITDctl %d: stat=%X len=%X ioc=%X pg=%X offs=%X\n", i,
1497 		    EHCI_ITD_GET_STATUS(t), EHCI_ITD_GET_LEN(t),
1498 		    EHCI_ITD_GET_IOC(t), EHCI_ITD_GET_PG(t),
1499 		    EHCI_ITD_GET_OFFS(t));
1500 	}
1501 	printf("ITDbufr: ");
1502 	for (i = 0; i < EHCI_ITD_NBUFFERS; i++)
1503 		printf("%X,", EHCI_ITD_GET_BPTR(le32toh(itd->itd.itd_bufr[i])));
1504 
1505 	b = le32toh(itd->itd.itd_bufr[0]);
1506 	b2 = le32toh(itd->itd.itd_bufr[1]);
1507 	b3 = le32toh(itd->itd.itd_bufr[2]);
1508 	printf("\nep=%X daddr=%X dir=%d maxpkt=%X multi=%X\n",
1509 	    EHCI_ITD_GET_EP(b), EHCI_ITD_GET_DADDR(b), EHCI_ITD_GET_DIR(b2),
1510 	    EHCI_ITD_GET_MAXPKT(b2), EHCI_ITD_GET_MULTI(b3));
1511 }
1512 
1513 Static void
1514 ehci_dump_sitd(struct ehci_soft_itd *itd)
1515 {
1516 	printf("SITD %p next=%p prev=%p xfernext=%p physaddr=%X slot=%d\n",
1517 			itd, itd->u.frame_list.next, itd->u.frame_list.prev,
1518 			itd->xfer_next, itd->physaddr, itd->slot);
1519 }
1520 #endif
1521 
1522 #ifdef DIAGNOSTIC
1523 Static void
1524 ehci_dump_exfer(struct ehci_xfer *ex)
1525 {
1526 	printf("ehci_dump_exfer: ex=%p sqtdstart=%p end=%p itdstart=%p end=%p isdone=%d\n", ex, ex->sqtdstart, ex->sqtdend, ex->itdstart, ex->itdend, ex->isdone);
1527 }
1528 #endif
1529 #endif
1530 
1531 Static usbd_status
1532 ehci_open(usbd_pipe_handle pipe)
1533 {
1534 	usbd_device_handle dev = pipe->device;
1535 	ehci_softc_t *sc = dev->bus->hci_private;
1536 	usb_endpoint_descriptor_t *ed = pipe->endpoint->edesc;
1537 	u_int8_t addr = dev->address;
1538 	u_int8_t xfertype = ed->bmAttributes & UE_XFERTYPE;
1539 	struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1540 	ehci_soft_qh_t *sqh;
1541 	usbd_status err;
1542 	int s;
1543 	int ival, speed, naks;
1544 	int hshubaddr, hshubport;
1545 
1546 	DPRINTFN(1, ("ehci_open: pipe=%p, addr=%d, endpt=%d (%d)\n",
1547 		     pipe, addr, ed->bEndpointAddress, sc->sc_addr));
1548 
1549 	if (dev->myhsport) {
1550 		/*
1551 		 * When directly attached FS/LS device while doing embedded
1552 		 * transaction translations and we are the hub, set the hub
1553 		 * adddress to 0 (us).
1554 		 */
1555 		if (!(sc->sc_flags & EHCIF_ETTF)
1556 		    || (dev->myhsport->parent->address != sc->sc_addr)) {
1557 			hshubaddr = dev->myhsport->parent->address;
1558 		} else {
1559 			hshubaddr = 0;
1560 		}
1561 		hshubport = dev->myhsport->portno;
1562 	} else {
1563 		hshubaddr = 0;
1564 		hshubport = 0;
1565 	}
1566 
1567 	if (sc->sc_dying)
1568 		return (USBD_IOERROR);
1569 
1570 	epipe->nexttoggle = 0;
1571 
1572 	if (addr == sc->sc_addr) {
1573 		switch (ed->bEndpointAddress) {
1574 		case USB_CONTROL_ENDPOINT:
1575 			pipe->methods = &ehci_root_ctrl_methods;
1576 			break;
1577 		case UE_DIR_IN | EHCI_INTR_ENDPT:
1578 			pipe->methods = &ehci_root_intr_methods;
1579 			break;
1580 		default:
1581 			DPRINTF(("ehci_open: bad bEndpointAddress 0x%02x\n",
1582 			    ed->bEndpointAddress));
1583 			return (USBD_INVAL);
1584 		}
1585 		return (USBD_NORMAL_COMPLETION);
1586 	}
1587 
1588 	/* XXX All this stuff is only valid for async. */
1589 	switch (dev->speed) {
1590 	case USB_SPEED_LOW:  speed = EHCI_QH_SPEED_LOW;  break;
1591 	case USB_SPEED_FULL: speed = EHCI_QH_SPEED_FULL; break;
1592 	case USB_SPEED_HIGH: speed = EHCI_QH_SPEED_HIGH; break;
1593 	default: panic("ehci_open: bad device speed %d", dev->speed);
1594 	}
1595 	if (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_ISOCHRONOUS) {
1596 		aprint_error_dev(sc->sc_dev, "error opening low/full speed "
1597 		    "isoc endpoint.\n");
1598 		aprint_normal_dev(sc->sc_dev, "a low/full speed device is "
1599 		    "attached to a USB2 hub, and transaction translations are "
1600 		    "not yet supported.\n");
1601 		aprint_normal_dev(sc->sc_dev, "reattach the device to the "
1602 		    "root hub instead.\n");
1603 		DPRINTFN(1,("ehci_open: hshubaddr=%d hshubport=%d\n",
1604 			    hshubaddr, hshubport));
1605 		return USBD_INVAL;
1606 	}
1607 
1608 	/*
1609 	 * For interrupt transfer, nak throttling must be disabled, but for
1610 	 * the other transfer type, nak throttling should be enabled from the
1611 	 * veiwpoint that avoids the memory thrashing.
1612 	 */
1613 	naks = (xfertype == UE_INTERRUPT) ? 0
1614 	    : ((speed == EHCI_QH_SPEED_HIGH) ? 4 : 0);
1615 
1616 	/* Allocate sqh for everything, save isoc xfers */
1617 	if (xfertype != UE_ISOCHRONOUS) {
1618 		sqh = ehci_alloc_sqh(sc);
1619 		if (sqh == NULL)
1620 			return (USBD_NOMEM);
1621 		/* qh_link filled when the QH is added */
1622 		sqh->qh.qh_endp = htole32(
1623 		    EHCI_QH_SET_ADDR(addr) |
1624 		    EHCI_QH_SET_ENDPT(UE_GET_ADDR(ed->bEndpointAddress)) |
1625 		    EHCI_QH_SET_EPS(speed) |
1626 		    EHCI_QH_DTC |
1627 		    EHCI_QH_SET_MPL(UGETW(ed->wMaxPacketSize)) |
1628 		    (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_CONTROL ?
1629 		     EHCI_QH_CTL : 0) |
1630 		    EHCI_QH_SET_NRL(naks)
1631 		    );
1632 		sqh->qh.qh_endphub = htole32(
1633 		    EHCI_QH_SET_MULT(1) |
1634 		    EHCI_QH_SET_SMASK(xfertype == UE_INTERRUPT ? 0x02 : 0)
1635 		    );
1636 		if (speed != EHCI_QH_SPEED_HIGH)
1637 			sqh->qh.qh_endphub |= htole32(
1638 			    EHCI_QH_SET_PORT(hshubport) |
1639 			    EHCI_QH_SET_HUBA(hshubaddr) |
1640 			    EHCI_QH_SET_CMASK(0x08) /* XXX */
1641 			);
1642 		sqh->qh.qh_curqtd = EHCI_NULL;
1643 		/* Fill the overlay qTD */
1644 		sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
1645 		sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
1646 		sqh->qh.qh_qtd.qtd_status = htole32(0);
1647 
1648 		usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
1649 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1650 		epipe->sqh = sqh;
1651 	} else {
1652 		sqh = NULL;
1653 	} /*xfertype == UE_ISOC*/
1654 
1655 	switch (xfertype) {
1656 	case UE_CONTROL:
1657 		err = usb_allocmem(&sc->sc_bus, sizeof(usb_device_request_t),
1658 				   0, &epipe->u.ctl.reqdma);
1659 #ifdef EHCI_DEBUG
1660 		if (err)
1661 			printf("ehci_open: usb_allocmem()=%d\n", err);
1662 #endif
1663 		if (err)
1664 			goto bad;
1665 		pipe->methods = &ehci_device_ctrl_methods;
1666 		s = splusb();
1667 		ehci_add_qh(sqh, sc->sc_async_head);
1668 		splx(s);
1669 		break;
1670 	case UE_BULK:
1671 		pipe->methods = &ehci_device_bulk_methods;
1672 		s = splusb();
1673 		ehci_add_qh(sqh, sc->sc_async_head);
1674 		splx(s);
1675 		break;
1676 	case UE_INTERRUPT:
1677 		pipe->methods = &ehci_device_intr_methods;
1678 		ival = pipe->interval;
1679 		if (ival == USBD_DEFAULT_INTERVAL) {
1680 			if (speed == EHCI_QH_SPEED_HIGH) {
1681 				if (ed->bInterval > 16) {
1682 					/*
1683 					 * illegal with high-speed, but there
1684 					 * were documentation bugs in the spec,
1685 					 * so be generous
1686 					 */
1687 					ival = 256;
1688 				} else
1689 					ival = (1 << (ed->bInterval - 1)) / 8;
1690 			} else
1691 				ival = ed->bInterval;
1692 		}
1693 		err = ehci_device_setintr(sc, sqh, ival);
1694 		if (err)
1695 			goto bad;
1696 		break;
1697 	case UE_ISOCHRONOUS:
1698 		pipe->methods = &ehci_device_isoc_methods;
1699 		if (ed->bInterval == 0 || ed->bInterval > 16) {
1700 			printf("ehci: opening pipe with invalid bInterval\n");
1701 			err = USBD_INVAL;
1702 			goto bad;
1703 		}
1704 		if (UGETW(ed->wMaxPacketSize) == 0) {
1705 			printf("ehci: zero length endpoint open request\n");
1706 			err = USBD_INVAL;
1707 			goto bad;
1708 		}
1709 		epipe->u.isoc.next_frame = 0;
1710 		epipe->u.isoc.cur_xfers = 0;
1711 		break;
1712 	default:
1713 		DPRINTF(("ehci: bad xfer type %d\n", xfertype));
1714 		err = USBD_INVAL;
1715 		goto bad;
1716 	}
1717 	return (USBD_NORMAL_COMPLETION);
1718 
1719  bad:
1720 	if (sqh != NULL)
1721 		ehci_free_sqh(sc, sqh);
1722 	return (err);
1723 }
1724 
1725 /*
1726  * Add an ED to the schedule.  Called at splusb().
1727  */
1728 Static void
1729 ehci_add_qh(ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
1730 {
1731 	SPLUSBCHECK;
1732 
1733 	usb_syncmem(&head->dma, head->offs + offsetof(ehci_qh_t, qh_link),
1734 	    sizeof(head->qh.qh_link), BUS_DMASYNC_POSTWRITE);
1735 	sqh->next = head->next;
1736 	sqh->qh.qh_link = head->qh.qh_link;
1737 	usb_syncmem(&sqh->dma, sqh->offs + offsetof(ehci_qh_t, qh_link),
1738 	    sizeof(sqh->qh.qh_link), BUS_DMASYNC_PREWRITE);
1739 	head->next = sqh;
1740 	head->qh.qh_link = htole32(sqh->physaddr | EHCI_LINK_QH);
1741 	usb_syncmem(&head->dma, head->offs + offsetof(ehci_qh_t, qh_link),
1742 	    sizeof(head->qh.qh_link), BUS_DMASYNC_PREWRITE);
1743 
1744 #ifdef EHCI_DEBUG
1745 	if (ehcidebug > 5) {
1746 		printf("ehci_add_qh:\n");
1747 		ehci_dump_sqh(sqh);
1748 	}
1749 #endif
1750 }
1751 
1752 /*
1753  * Remove an ED from the schedule.  Called at splusb().
1754  */
1755 Static void
1756 ehci_rem_qh(ehci_softc_t *sc, ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
1757 {
1758 	ehci_soft_qh_t *p;
1759 
1760 	SPLUSBCHECK;
1761 	/* XXX */
1762 	for (p = head; p != NULL && p->next != sqh; p = p->next)
1763 		;
1764 	if (p == NULL)
1765 		panic("ehci_rem_qh: ED not found");
1766 	usb_syncmem(&sqh->dma, sqh->offs + offsetof(ehci_qh_t, qh_link),
1767 	    sizeof(sqh->qh.qh_link), BUS_DMASYNC_POSTWRITE);
1768 	p->next = sqh->next;
1769 	p->qh.qh_link = sqh->qh.qh_link;
1770 	usb_syncmem(&p->dma, p->offs + offsetof(ehci_qh_t, qh_link),
1771 	    sizeof(p->qh.qh_link), BUS_DMASYNC_PREWRITE);
1772 
1773 	ehci_sync_hc(sc);
1774 }
1775 
1776 Static void
1777 ehci_set_qh_qtd(ehci_soft_qh_t *sqh, ehci_soft_qtd_t *sqtd)
1778 {
1779 	int i;
1780 	u_int32_t status;
1781 
1782 	/* Save toggle bit and ping status. */
1783 	usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
1784 	    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1785 	status = sqh->qh.qh_qtd.qtd_status &
1786 	    htole32(EHCI_QTD_TOGGLE_MASK |
1787 		    EHCI_QTD_SET_STATUS(EHCI_QTD_PINGSTATE));
1788 	/* Set HALTED to make hw leave it alone. */
1789 	sqh->qh.qh_qtd.qtd_status =
1790 	    htole32(EHCI_QTD_SET_STATUS(EHCI_QTD_HALTED));
1791 	usb_syncmem(&sqh->dma,
1792 	    sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
1793 	    sizeof(sqh->qh.qh_qtd.qtd_status),
1794 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1795 	sqh->qh.qh_curqtd = 0;
1796 	sqh->qh.qh_qtd.qtd_next = htole32(sqtd->physaddr);
1797 	sqh->qh.qh_qtd.qtd_altnext = 0;
1798 	for (i = 0; i < EHCI_QTD_NBUFFERS; i++)
1799 		sqh->qh.qh_qtd.qtd_buffer[i] = 0;
1800 	sqh->sqtd = sqtd;
1801 	usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
1802 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1803 	/* Set !HALTED && !ACTIVE to start execution, preserve some fields */
1804 	sqh->qh.qh_qtd.qtd_status = status;
1805 	usb_syncmem(&sqh->dma,
1806 	    sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
1807 	    sizeof(sqh->qh.qh_qtd.qtd_status),
1808 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1809 }
1810 
1811 /*
1812  * Ensure that the HC has released all references to the QH.  We do this
1813  * by asking for a Async Advance Doorbell interrupt and then we wait for
1814  * the interrupt.
1815  * To make this easier we first obtain exclusive use of the doorbell.
1816  */
1817 Static void
1818 ehci_sync_hc(ehci_softc_t *sc)
1819 {
1820 	int s, error;
1821 
1822 	if (sc->sc_dying) {
1823 		DPRINTFN(2,("ehci_sync_hc: dying\n"));
1824 		return;
1825 	}
1826 	DPRINTFN(2,("ehci_sync_hc: enter\n"));
1827 	mutex_enter(&sc->sc_doorbell_lock);	/* get doorbell */
1828 	s = splhardusb();
1829 	/* ask for doorbell */
1830 	EOWRITE4(sc, EHCI_USBCMD, EOREAD4(sc, EHCI_USBCMD) | EHCI_CMD_IAAD);
1831 	DPRINTFN(1,("ehci_sync_hc: cmd=0x%08x sts=0x%08x\n",
1832 		    EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS)));
1833 	error = tsleep(&sc->sc_async_head, PZERO, "ehcidi", hz); /* bell wait */
1834 	DPRINTFN(1,("ehci_sync_hc: cmd=0x%08x sts=0x%08x\n",
1835 		    EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS)));
1836 	splx(s);
1837 	mutex_exit(&sc->sc_doorbell_lock);	/* release doorbell */
1838 #ifdef DIAGNOSTIC
1839 	if (error)
1840 		printf("ehci_sync_hc: tsleep() = %d\n", error);
1841 #endif
1842 	DPRINTFN(2,("ehci_sync_hc: exit\n"));
1843 }
1844 
1845 /*Call at splusb*/
1846 Static void
1847 ehci_rem_free_itd_chain(ehci_softc_t *sc, struct ehci_xfer *exfer)
1848 {
1849 	struct ehci_soft_itd *itd, *prev;
1850 
1851 	prev = NULL;
1852 
1853 	if (exfer->itdstart == NULL || exfer->itdend == NULL)
1854 		panic("ehci isoc xfer being freed, but with no itd chain\n");
1855 
1856 	for (itd = exfer->itdstart; itd != NULL; itd = itd->xfer_next) {
1857 		prev = itd->u.frame_list.prev;
1858 		/* Unlink itd from hardware chain, or frame array */
1859 		if (prev == NULL) { /* We're at the table head */
1860 			sc->sc_softitds[itd->slot] = itd->u.frame_list.next;
1861 			sc->sc_flist[itd->slot] = itd->itd.itd_next;
1862 			usb_syncmem(&sc->sc_fldma,
1863 			    sizeof(ehci_link_t) * itd->slot,
1864                 	    sizeof(ehci_link_t),
1865 			    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1866 
1867 			if (itd->u.frame_list.next != NULL)
1868 				itd->u.frame_list.next->u.frame_list.prev = NULL;
1869 		} else {
1870 			/* XXX this part is untested... */
1871 			prev->itd.itd_next = itd->itd.itd_next;
1872 			usb_syncmem(&itd->dma,
1873 			    itd->offs + offsetof(ehci_itd_t, itd_next),
1874                 	    sizeof(itd->itd.itd_next), BUS_DMASYNC_PREWRITE);
1875 
1876 			prev->u.frame_list.next = itd->u.frame_list.next;
1877 			if (itd->u.frame_list.next != NULL)
1878 				itd->u.frame_list.next->u.frame_list.prev = prev;
1879 		}
1880 	}
1881 
1882 	prev = NULL;
1883 	for (itd = exfer->itdstart; itd != NULL; itd = itd->xfer_next) {
1884 		if (prev != NULL)
1885 			ehci_free_itd(sc, prev);
1886 		prev = itd;
1887 	}
1888 	if (prev)
1889 		ehci_free_itd(sc, prev);
1890 	exfer->itdstart = NULL;
1891 	exfer->itdend = NULL;
1892 }
1893 
1894 /***********/
1895 
1896 /*
1897  * Data structures and routines to emulate the root hub.
1898  */
1899 Static usb_device_descriptor_t ehci_devd = {
1900 	USB_DEVICE_DESCRIPTOR_SIZE,
1901 	UDESC_DEVICE,		/* type */
1902 	{0x00, 0x02},		/* USB version */
1903 	UDCLASS_HUB,		/* class */
1904 	UDSUBCLASS_HUB,		/* subclass */
1905 	UDPROTO_HSHUBSTT,	/* protocol */
1906 	64,			/* max packet */
1907 	{0},{0},{0x00,0x01},	/* device id */
1908 	1,2,0,			/* string indicies */
1909 	1			/* # of configurations */
1910 };
1911 
1912 Static const usb_device_qualifier_t ehci_odevd = {
1913 	USB_DEVICE_DESCRIPTOR_SIZE,
1914 	UDESC_DEVICE_QUALIFIER,	/* type */
1915 	{0x00, 0x02},		/* USB version */
1916 	UDCLASS_HUB,		/* class */
1917 	UDSUBCLASS_HUB,		/* subclass */
1918 	UDPROTO_FSHUB,		/* protocol */
1919 	64,			/* max packet */
1920 	1,			/* # of configurations */
1921 	0
1922 };
1923 
1924 Static const usb_config_descriptor_t ehci_confd = {
1925 	USB_CONFIG_DESCRIPTOR_SIZE,
1926 	UDESC_CONFIG,
1927 	{USB_CONFIG_DESCRIPTOR_SIZE +
1928 	 USB_INTERFACE_DESCRIPTOR_SIZE +
1929 	 USB_ENDPOINT_DESCRIPTOR_SIZE},
1930 	1,
1931 	1,
1932 	0,
1933 	UC_ATTR_MBO | UC_SELF_POWERED,
1934 	0			/* max power */
1935 };
1936 
1937 Static const usb_interface_descriptor_t ehci_ifcd = {
1938 	USB_INTERFACE_DESCRIPTOR_SIZE,
1939 	UDESC_INTERFACE,
1940 	0,
1941 	0,
1942 	1,
1943 	UICLASS_HUB,
1944 	UISUBCLASS_HUB,
1945 	UIPROTO_HSHUBSTT,
1946 	0
1947 };
1948 
1949 Static const usb_endpoint_descriptor_t ehci_endpd = {
1950 	USB_ENDPOINT_DESCRIPTOR_SIZE,
1951 	UDESC_ENDPOINT,
1952 	UE_DIR_IN | EHCI_INTR_ENDPT,
1953 	UE_INTERRUPT,
1954 	{8, 0},			/* max packet */
1955 	12
1956 };
1957 
1958 Static const usb_hub_descriptor_t ehci_hubd = {
1959 	USB_HUB_DESCRIPTOR_SIZE,
1960 	UDESC_HUB,
1961 	0,
1962 	{0,0},
1963 	0,
1964 	0,
1965 	{""},
1966 	{""},
1967 };
1968 
1969 /*
1970  * Simulate a hardware hub by handling all the necessary requests.
1971  */
1972 Static usbd_status
1973 ehci_root_ctrl_transfer(usbd_xfer_handle xfer)
1974 {
1975 	usbd_status err;
1976 
1977 	/* Insert last in queue. */
1978 	err = usb_insert_transfer(xfer);
1979 	if (err)
1980 		return (err);
1981 
1982 	/* Pipe isn't running, start first */
1983 	return (ehci_root_ctrl_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
1984 }
1985 
1986 Static usbd_status
1987 ehci_root_ctrl_start(usbd_xfer_handle xfer)
1988 {
1989 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
1990 	usb_device_request_t *req;
1991 	void *buf = NULL;
1992 	int port, i;
1993 	int s, len, value, index, l, totlen = 0;
1994 	usb_port_status_t ps;
1995 	usb_hub_descriptor_t hubd;
1996 	usbd_status err;
1997 	u_int32_t v;
1998 
1999 	if (sc->sc_dying)
2000 		return (USBD_IOERROR);
2001 
2002 #ifdef DIAGNOSTIC
2003 	if (!(xfer->rqflags & URQ_REQUEST))
2004 		/* XXX panic */
2005 		return (USBD_INVAL);
2006 #endif
2007 	req = &xfer->request;
2008 
2009 	DPRINTFN(4,("ehci_root_ctrl_start: type=0x%02x request=%02x\n",
2010 		    req->bmRequestType, req->bRequest));
2011 
2012 	len = UGETW(req->wLength);
2013 	value = UGETW(req->wValue);
2014 	index = UGETW(req->wIndex);
2015 
2016 	if (len != 0)
2017 		buf = KERNADDR(&xfer->dmabuf, 0);
2018 
2019 #define C(x,y) ((x) | ((y) << 8))
2020 	switch(C(req->bRequest, req->bmRequestType)) {
2021 	case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
2022 	case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
2023 	case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
2024 		/*
2025 		 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
2026 		 * for the integrated root hub.
2027 		 */
2028 		break;
2029 	case C(UR_GET_CONFIG, UT_READ_DEVICE):
2030 		if (len > 0) {
2031 			*(u_int8_t *)buf = sc->sc_conf;
2032 			totlen = 1;
2033 		}
2034 		break;
2035 	case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
2036 		DPRINTFN(8,("ehci_root_ctrl_start: wValue=0x%04x\n", value));
2037 		if (len == 0)
2038 			break;
2039 		switch(value >> 8) {
2040 		case UDESC_DEVICE:
2041 			if ((value & 0xff) != 0) {
2042 				err = USBD_IOERROR;
2043 				goto ret;
2044 			}
2045 			totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
2046 			USETW(ehci_devd.idVendor, sc->sc_id_vendor);
2047 			memcpy(buf, &ehci_devd, l);
2048 			break;
2049 		/*
2050 		 * We can't really operate at another speed, but the spec says
2051 		 * we need this descriptor.
2052 		 */
2053 		case UDESC_DEVICE_QUALIFIER:
2054 			if ((value & 0xff) != 0) {
2055 				err = USBD_IOERROR;
2056 				goto ret;
2057 			}
2058 			totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
2059 			memcpy(buf, &ehci_odevd, l);
2060 			break;
2061 		/*
2062 		 * We can't really operate at another speed, but the spec says
2063 		 * we need this descriptor.
2064 		 */
2065 		case UDESC_OTHER_SPEED_CONFIGURATION:
2066 		case UDESC_CONFIG:
2067 			if ((value & 0xff) != 0) {
2068 				err = USBD_IOERROR;
2069 				goto ret;
2070 			}
2071 			totlen = l = min(len, USB_CONFIG_DESCRIPTOR_SIZE);
2072 			memcpy(buf, &ehci_confd, l);
2073 			((usb_config_descriptor_t *)buf)->bDescriptorType =
2074 				value >> 8;
2075 			buf = (char *)buf + l;
2076 			len -= l;
2077 			l = min(len, USB_INTERFACE_DESCRIPTOR_SIZE);
2078 			totlen += l;
2079 			memcpy(buf, &ehci_ifcd, l);
2080 			buf = (char *)buf + l;
2081 			len -= l;
2082 			l = min(len, USB_ENDPOINT_DESCRIPTOR_SIZE);
2083 			totlen += l;
2084 			memcpy(buf, &ehci_endpd, l);
2085 			break;
2086 		case UDESC_STRING:
2087 #define sd ((usb_string_descriptor_t *)buf)
2088 			switch (value & 0xff) {
2089 			case 0: /* Language table */
2090 				totlen = usb_makelangtbl(sd, len);
2091 				break;
2092 			case 1: /* Vendor */
2093 				totlen = usb_makestrdesc(sd, len,
2094 							 sc->sc_vendor);
2095 				break;
2096 			case 2: /* Product */
2097 				totlen = usb_makestrdesc(sd, len,
2098 							 "EHCI root hub");
2099 				break;
2100 			}
2101 #undef sd
2102 			break;
2103 		default:
2104 			err = USBD_IOERROR;
2105 			goto ret;
2106 		}
2107 		break;
2108 	case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
2109 		if (len > 0) {
2110 			*(u_int8_t *)buf = 0;
2111 			totlen = 1;
2112 		}
2113 		break;
2114 	case C(UR_GET_STATUS, UT_READ_DEVICE):
2115 		if (len > 1) {
2116 			USETW(((usb_status_t *)buf)->wStatus,UDS_SELF_POWERED);
2117 			totlen = 2;
2118 		}
2119 		break;
2120 	case C(UR_GET_STATUS, UT_READ_INTERFACE):
2121 	case C(UR_GET_STATUS, UT_READ_ENDPOINT):
2122 		if (len > 1) {
2123 			USETW(((usb_status_t *)buf)->wStatus, 0);
2124 			totlen = 2;
2125 		}
2126 		break;
2127 	case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
2128 		if (value >= USB_MAX_DEVICES) {
2129 			err = USBD_IOERROR;
2130 			goto ret;
2131 		}
2132 		sc->sc_addr = value;
2133 		break;
2134 	case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
2135 		if (value != 0 && value != 1) {
2136 			err = USBD_IOERROR;
2137 			goto ret;
2138 		}
2139 		sc->sc_conf = value;
2140 		break;
2141 	case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
2142 		break;
2143 	case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
2144 	case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
2145 	case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
2146 		err = USBD_IOERROR;
2147 		goto ret;
2148 	case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
2149 		break;
2150 	case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
2151 		break;
2152 	/* Hub requests */
2153 	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
2154 		break;
2155 	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
2156 		DPRINTFN(4, ("ehci_root_ctrl_start: UR_CLEAR_PORT_FEATURE "
2157 			     "port=%d feature=%d\n",
2158 			     index, value));
2159 		if (index < 1 || index > sc->sc_noport) {
2160 			err = USBD_IOERROR;
2161 			goto ret;
2162 		}
2163 		port = EHCI_PORTSC(index);
2164 		v = EOREAD4(sc, port);
2165 		DPRINTFN(4, ("ehci_root_ctrl_start: portsc=0x%08x\n", v));
2166 		v &= ~EHCI_PS_CLEAR;
2167 		switch(value) {
2168 		case UHF_PORT_ENABLE:
2169 			EOWRITE4(sc, port, v &~ EHCI_PS_PE);
2170 			break;
2171 		case UHF_PORT_SUSPEND:
2172 			if (!(v & EHCI_PS_SUSP)) /* not suspended */
2173 				break;
2174 			v &= ~EHCI_PS_SUSP;
2175 			EOWRITE4(sc, port, v | EHCI_PS_FPR);
2176 			/* see USB2 spec ch. 7.1.7.7 */
2177 			usb_delay_ms(&sc->sc_bus, 20);
2178 			EOWRITE4(sc, port, v);
2179 			usb_delay_ms(&sc->sc_bus, 2);
2180 #ifdef DEBUG
2181 			v = EOREAD4(sc, port);
2182 			if (v & (EHCI_PS_FPR | EHCI_PS_SUSP))
2183 				printf("ehci: resume failed: %x\n", v);
2184 #endif
2185 			break;
2186 		case UHF_PORT_POWER:
2187 			if (sc->sc_hasppc)
2188 				EOWRITE4(sc, port, v &~ EHCI_PS_PP);
2189 			break;
2190 		case UHF_PORT_TEST:
2191 			DPRINTFN(2,("ehci_root_ctrl_start: clear port test "
2192 				    "%d\n", index));
2193 			break;
2194 		case UHF_PORT_INDICATOR:
2195 			DPRINTFN(2,("ehci_root_ctrl_start: clear port ind "
2196 				    "%d\n", index));
2197 			EOWRITE4(sc, port, v &~ EHCI_PS_PIC);
2198 			break;
2199 		case UHF_C_PORT_CONNECTION:
2200 			EOWRITE4(sc, port, v | EHCI_PS_CSC);
2201 			break;
2202 		case UHF_C_PORT_ENABLE:
2203 			EOWRITE4(sc, port, v | EHCI_PS_PEC);
2204 			break;
2205 		case UHF_C_PORT_SUSPEND:
2206 			/* how? */
2207 			break;
2208 		case UHF_C_PORT_OVER_CURRENT:
2209 			EOWRITE4(sc, port, v | EHCI_PS_OCC);
2210 			break;
2211 		case UHF_C_PORT_RESET:
2212 			sc->sc_isreset[index] = 0;
2213 			break;
2214 		default:
2215 			err = USBD_IOERROR;
2216 			goto ret;
2217 		}
2218 #if 0
2219 		switch(value) {
2220 		case UHF_C_PORT_CONNECTION:
2221 		case UHF_C_PORT_ENABLE:
2222 		case UHF_C_PORT_SUSPEND:
2223 		case UHF_C_PORT_OVER_CURRENT:
2224 		case UHF_C_PORT_RESET:
2225 		default:
2226 			break;
2227 		}
2228 #endif
2229 		break;
2230 	case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
2231 		if (len == 0)
2232 			break;
2233 		if ((value & 0xff) != 0) {
2234 			err = USBD_IOERROR;
2235 			goto ret;
2236 		}
2237 		hubd = ehci_hubd;
2238 		hubd.bNbrPorts = sc->sc_noport;
2239 		v = EOREAD4(sc, EHCI_HCSPARAMS);
2240 		USETW(hubd.wHubCharacteristics,
2241 		    EHCI_HCS_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_NO_SWITCH |
2242 		    EHCI_HCS_P_INDICATOR(EREAD4(sc, EHCI_HCSPARAMS))
2243 			? UHD_PORT_IND : 0);
2244 		hubd.bPwrOn2PwrGood = 200; /* XXX can't find out? */
2245 		for (i = 0, l = sc->sc_noport; l > 0; i++, l -= 8, v >>= 8)
2246 			hubd.DeviceRemovable[i++] = 0; /* XXX can't find out? */
2247 		hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i;
2248 		l = min(len, hubd.bDescLength);
2249 		totlen = l;
2250 		memcpy(buf, &hubd, l);
2251 		break;
2252 	case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
2253 		if (len != 4) {
2254 			err = USBD_IOERROR;
2255 			goto ret;
2256 		}
2257 		memset(buf, 0, len); /* ? XXX */
2258 		totlen = len;
2259 		break;
2260 	case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
2261 		DPRINTFN(8,("ehci_root_ctrl_start: get port status i=%d\n",
2262 			    index));
2263 		if (index < 1 || index > sc->sc_noport) {
2264 			err = USBD_IOERROR;
2265 			goto ret;
2266 		}
2267 		if (len != 4) {
2268 			err = USBD_IOERROR;
2269 			goto ret;
2270 		}
2271 		v = EOREAD4(sc, EHCI_PORTSC(index));
2272 		DPRINTFN(8,("ehci_root_ctrl_start: port status=0x%04x\n",
2273 			    v));
2274 
2275 		if (sc->sc_flags & EHCIF_ETTF) {
2276 			/*
2277 			 * If we are doing embedded transaction translation,
2278 			 * then directly attached LS/FS devices are reset by
2279 			 * the EHCI controller itself.  PSPD is encoded
2280 			 * the same way as in USBSTATUS.
2281 			 */
2282 			i = __SHIFTOUT(v, EHCI_PS_PSPD) * UPS_LOW_SPEED;
2283 		} else {
2284 			i = UPS_HIGH_SPEED;
2285 		}
2286 		if (v & EHCI_PS_CS)	i |= UPS_CURRENT_CONNECT_STATUS;
2287 		if (v & EHCI_PS_PE)	i |= UPS_PORT_ENABLED;
2288 		if (v & EHCI_PS_SUSP)	i |= UPS_SUSPEND;
2289 		if (v & EHCI_PS_OCA)	i |= UPS_OVERCURRENT_INDICATOR;
2290 		if (v & EHCI_PS_PR)	i |= UPS_RESET;
2291 		if (v & EHCI_PS_PP)	i |= UPS_PORT_POWER;
2292 		if (sc->sc_vendor_port_status)
2293 			i = sc->sc_vendor_port_status(sc, v, i);
2294 		USETW(ps.wPortStatus, i);
2295 		i = 0;
2296 		if (v & EHCI_PS_CSC)	i |= UPS_C_CONNECT_STATUS;
2297 		if (v & EHCI_PS_PEC)	i |= UPS_C_PORT_ENABLED;
2298 		if (v & EHCI_PS_OCC)	i |= UPS_C_OVERCURRENT_INDICATOR;
2299 		if (sc->sc_isreset[index]) i |= UPS_C_PORT_RESET;
2300 		USETW(ps.wPortChange, i);
2301 		l = min(len, sizeof ps);
2302 		memcpy(buf, &ps, l);
2303 		totlen = l;
2304 		break;
2305 	case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
2306 		err = USBD_IOERROR;
2307 		goto ret;
2308 	case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
2309 		break;
2310 	case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
2311 		if (index < 1 || index > sc->sc_noport) {
2312 			err = USBD_IOERROR;
2313 			goto ret;
2314 		}
2315 		port = EHCI_PORTSC(index);
2316 		v = EOREAD4(sc, port);
2317 		DPRINTFN(4, ("ehci_root_ctrl_start: portsc=0x%08x\n", v));
2318 		v &= ~EHCI_PS_CLEAR;
2319 		switch(value) {
2320 		case UHF_PORT_ENABLE:
2321 			EOWRITE4(sc, port, v | EHCI_PS_PE);
2322 			break;
2323 		case UHF_PORT_SUSPEND:
2324 			EOWRITE4(sc, port, v | EHCI_PS_SUSP);
2325 			break;
2326 		case UHF_PORT_RESET:
2327 			DPRINTFN(5,("ehci_root_ctrl_start: reset port %d\n",
2328 				    index));
2329 			if (EHCI_PS_IS_LOWSPEED(v)
2330 			    && sc->sc_ncomp > 0
2331 			    && !(sc->sc_flags & EHCIF_ETTF)) {
2332 				/*
2333 				 * Low speed device on non-ETTF controller or
2334 				 * unaccompanied controller, give up ownership.
2335 				 */
2336 				ehci_disown(sc, index, 1);
2337 				break;
2338 			}
2339 			/* Start reset sequence. */
2340 			v &= ~ (EHCI_PS_PE | EHCI_PS_PR);
2341 			EOWRITE4(sc, port, v | EHCI_PS_PR);
2342 			/* Wait for reset to complete. */
2343 			usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY);
2344 			if (sc->sc_dying) {
2345 				err = USBD_IOERROR;
2346 				goto ret;
2347 			}
2348 			/*
2349 			 * An embedded transaction translater will automatically
2350 			 * terminate the reset sequence so there's no need to
2351 			 * it.
2352 			 */
2353 			if (!(sc->sc_flags & EHCIF_ETTF)) {
2354 				/* Terminate reset sequence. */
2355 				v = EOREAD4(sc, port);
2356 				EOWRITE4(sc, port, v & ~EHCI_PS_PR);
2357 				/* Wait for HC to complete reset. */
2358 				usb_delay_ms(&sc->sc_bus,
2359 				    EHCI_PORT_RESET_COMPLETE);
2360 				if (sc->sc_dying) {
2361 					err = USBD_IOERROR;
2362 					goto ret;
2363 				}
2364 			}
2365 
2366 			v = EOREAD4(sc, port);
2367 			DPRINTF(("ehci after reset, status=0x%08x\n", v));
2368 			if (v & EHCI_PS_PR) {
2369 				printf("%s: port reset timeout\n",
2370 				       device_xname(sc->sc_dev));
2371 				return (USBD_TIMEOUT);
2372 			}
2373 			if (!(v & EHCI_PS_PE)) {
2374 				/* Not a high speed device, give up ownership.*/
2375 				ehci_disown(sc, index, 0);
2376 				break;
2377 			}
2378 			sc->sc_isreset[index] = 1;
2379 			DPRINTF(("ehci port %d reset, status = 0x%08x\n",
2380 				 index, v));
2381 			break;
2382 		case UHF_PORT_POWER:
2383 			DPRINTFN(2,("ehci_root_ctrl_start: set port power "
2384 				    "%d (has PPC = %d)\n", index,
2385 				    sc->sc_hasppc));
2386 			if (sc->sc_hasppc)
2387 				EOWRITE4(sc, port, v | EHCI_PS_PP);
2388 			break;
2389 		case UHF_PORT_TEST:
2390 			DPRINTFN(2,("ehci_root_ctrl_start: set port test "
2391 				    "%d\n", index));
2392 			break;
2393 		case UHF_PORT_INDICATOR:
2394 			DPRINTFN(2,("ehci_root_ctrl_start: set port ind "
2395 				    "%d\n", index));
2396 			EOWRITE4(sc, port, v | EHCI_PS_PIC);
2397 			break;
2398 		default:
2399 			err = USBD_IOERROR;
2400 			goto ret;
2401 		}
2402 		break;
2403 	case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
2404 	case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
2405 	case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
2406 	case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
2407 		break;
2408 	default:
2409 		err = USBD_IOERROR;
2410 		goto ret;
2411 	}
2412 	xfer->actlen = totlen;
2413 	err = USBD_NORMAL_COMPLETION;
2414  ret:
2415 	xfer->status = err;
2416 	s = splusb();
2417 	usb_transfer_complete(xfer);
2418 	splx(s);
2419 	return (USBD_IN_PROGRESS);
2420 }
2421 
2422 Static void
2423 ehci_disown(ehci_softc_t *sc, int index, int lowspeed)
2424 {
2425 	int port;
2426 	u_int32_t v;
2427 
2428 	DPRINTF(("ehci_disown: index=%d lowspeed=%d\n", index, lowspeed));
2429 #ifdef DIAGNOSTIC
2430 	if (sc->sc_npcomp != 0) {
2431 		int i = (index-1) / sc->sc_npcomp;
2432 		if (i >= sc->sc_ncomp)
2433 			printf("%s: strange port\n",
2434 			       device_xname(sc->sc_dev));
2435 		else
2436 			printf("%s: handing over %s speed device on "
2437 			       "port %d to %s\n",
2438 			       device_xname(sc->sc_dev),
2439 			       lowspeed ? "low" : "full",
2440 			       index, device_xname(sc->sc_comps[i]));
2441 	} else {
2442 		printf("%s: npcomp == 0\n", device_xname(sc->sc_dev));
2443 	}
2444 #endif
2445 	port = EHCI_PORTSC(index);
2446 	v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
2447 	EOWRITE4(sc, port, v | EHCI_PS_PO);
2448 }
2449 
2450 /* Abort a root control request. */
2451 Static void
2452 ehci_root_ctrl_abort(usbd_xfer_handle xfer)
2453 {
2454 	/* Nothing to do, all transfers are synchronous. */
2455 }
2456 
2457 /* Close the root pipe. */
2458 Static void
2459 ehci_root_ctrl_close(usbd_pipe_handle pipe)
2460 {
2461 	DPRINTF(("ehci_root_ctrl_close\n"));
2462 	/* Nothing to do. */
2463 }
2464 
2465 Static void
2466 ehci_root_intr_done(usbd_xfer_handle xfer)
2467 {
2468 	xfer->hcpriv = NULL;
2469 }
2470 
2471 Static usbd_status
2472 ehci_root_intr_transfer(usbd_xfer_handle xfer)
2473 {
2474 	usbd_status err;
2475 
2476 	/* Insert last in queue. */
2477 	err = usb_insert_transfer(xfer);
2478 	if (err)
2479 		return (err);
2480 
2481 	/* Pipe isn't running, start first */
2482 	return (ehci_root_intr_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
2483 }
2484 
2485 Static usbd_status
2486 ehci_root_intr_start(usbd_xfer_handle xfer)
2487 {
2488 	usbd_pipe_handle pipe = xfer->pipe;
2489 	ehci_softc_t *sc = pipe->device->bus->hci_private;
2490 
2491 	if (sc->sc_dying)
2492 		return (USBD_IOERROR);
2493 
2494 	sc->sc_intrxfer = xfer;
2495 
2496 	return (USBD_IN_PROGRESS);
2497 }
2498 
2499 /* Abort a root interrupt request. */
2500 Static void
2501 ehci_root_intr_abort(usbd_xfer_handle xfer)
2502 {
2503 	int s;
2504 
2505 	if (xfer->pipe->intrxfer == xfer) {
2506 		DPRINTF(("ehci_root_intr_abort: remove\n"));
2507 		xfer->pipe->intrxfer = NULL;
2508 	}
2509 	xfer->status = USBD_CANCELLED;
2510 	s = splusb();
2511 	usb_transfer_complete(xfer);
2512 	splx(s);
2513 }
2514 
2515 /* Close the root pipe. */
2516 Static void
2517 ehci_root_intr_close(usbd_pipe_handle pipe)
2518 {
2519 	ehci_softc_t *sc = pipe->device->bus->hci_private;
2520 
2521 	DPRINTF(("ehci_root_intr_close\n"));
2522 
2523 	sc->sc_intrxfer = NULL;
2524 }
2525 
2526 Static void
2527 ehci_root_ctrl_done(usbd_xfer_handle xfer)
2528 {
2529 	xfer->hcpriv = NULL;
2530 }
2531 
2532 /************************/
2533 
2534 Static ehci_soft_qh_t *
2535 ehci_alloc_sqh(ehci_softc_t *sc)
2536 {
2537 	ehci_soft_qh_t *sqh;
2538 	usbd_status err;
2539 	int i, offs;
2540 	usb_dma_t dma;
2541 
2542 	if (sc->sc_freeqhs == NULL) {
2543 		DPRINTFN(2, ("ehci_alloc_sqh: allocating chunk\n"));
2544 		err = usb_allocmem(&sc->sc_bus, EHCI_SQH_SIZE * EHCI_SQH_CHUNK,
2545 			  EHCI_PAGE_SIZE, &dma);
2546 #ifdef EHCI_DEBUG
2547 		if (err)
2548 			printf("ehci_alloc_sqh: usb_allocmem()=%d\n", err);
2549 #endif
2550 		if (err)
2551 			return (NULL);
2552 		for(i = 0; i < EHCI_SQH_CHUNK; i++) {
2553 			offs = i * EHCI_SQH_SIZE;
2554 			sqh = KERNADDR(&dma, offs);
2555 			sqh->physaddr = DMAADDR(&dma, offs);
2556 			sqh->dma = dma;
2557 			sqh->offs = offs;
2558 			sqh->next = sc->sc_freeqhs;
2559 			sc->sc_freeqhs = sqh;
2560 		}
2561 	}
2562 	sqh = sc->sc_freeqhs;
2563 	sc->sc_freeqhs = sqh->next;
2564 	memset(&sqh->qh, 0, sizeof(ehci_qh_t));
2565 	sqh->next = NULL;
2566 	return (sqh);
2567 }
2568 
2569 Static void
2570 ehci_free_sqh(ehci_softc_t *sc, ehci_soft_qh_t *sqh)
2571 {
2572 	sqh->next = sc->sc_freeqhs;
2573 	sc->sc_freeqhs = sqh;
2574 }
2575 
2576 Static ehci_soft_qtd_t *
2577 ehci_alloc_sqtd(ehci_softc_t *sc)
2578 {
2579 	ehci_soft_qtd_t *sqtd;
2580 	usbd_status err;
2581 	int i, offs;
2582 	usb_dma_t dma;
2583 	int s;
2584 
2585 	if (sc->sc_freeqtds == NULL) {
2586 		DPRINTFN(2, ("ehci_alloc_sqtd: allocating chunk\n"));
2587 		err = usb_allocmem(&sc->sc_bus, EHCI_SQTD_SIZE*EHCI_SQTD_CHUNK,
2588 			  EHCI_PAGE_SIZE, &dma);
2589 #ifdef EHCI_DEBUG
2590 		if (err)
2591 			printf("ehci_alloc_sqtd: usb_allocmem()=%d\n", err);
2592 #endif
2593 		if (err)
2594 			return (NULL);
2595 		s = splusb();
2596 		for(i = 0; i < EHCI_SQTD_CHUNK; i++) {
2597 			offs = i * EHCI_SQTD_SIZE;
2598 			sqtd = KERNADDR(&dma, offs);
2599 			sqtd->physaddr = DMAADDR(&dma, offs);
2600 			sqtd->dma = dma;
2601 			sqtd->offs = offs;
2602 			sqtd->nextqtd = sc->sc_freeqtds;
2603 			sc->sc_freeqtds = sqtd;
2604 		}
2605 		splx(s);
2606 	}
2607 
2608 	s = splusb();
2609 	sqtd = sc->sc_freeqtds;
2610 	sc->sc_freeqtds = sqtd->nextqtd;
2611 	memset(&sqtd->qtd, 0, sizeof(ehci_qtd_t));
2612 	sqtd->nextqtd = NULL;
2613 	sqtd->xfer = NULL;
2614 	splx(s);
2615 
2616 	return (sqtd);
2617 }
2618 
2619 Static void
2620 ehci_free_sqtd(ehci_softc_t *sc, ehci_soft_qtd_t *sqtd)
2621 {
2622 	int s;
2623 
2624 	s = splusb();
2625 	sqtd->nextqtd = sc->sc_freeqtds;
2626 	sc->sc_freeqtds = sqtd;
2627 	splx(s);
2628 }
2629 
2630 Static usbd_status
2631 ehci_alloc_sqtd_chain(struct ehci_pipe *epipe, ehci_softc_t *sc,
2632 		     int alen, int rd, usbd_xfer_handle xfer,
2633 		     ehci_soft_qtd_t **sp, ehci_soft_qtd_t **ep)
2634 {
2635 	ehci_soft_qtd_t *next, *cur;
2636 	ehci_physaddr_t dataphys, dataphyspage, dataphyslastpage, nextphys;
2637 	u_int32_t qtdstatus;
2638 	int len, curlen, mps;
2639 	int i, tog;
2640 	usb_dma_t *dma = &xfer->dmabuf;
2641 	u_int16_t flags = xfer->flags;
2642 
2643 	DPRINTFN(alen<4*4096,("ehci_alloc_sqtd_chain: start len=%d\n", alen));
2644 
2645 	len = alen;
2646 	dataphys = DMAADDR(dma, 0);
2647 	dataphyslastpage = EHCI_PAGE(dataphys + len - 1);
2648 	qtdstatus = EHCI_QTD_ACTIVE |
2649 	    EHCI_QTD_SET_PID(rd ? EHCI_QTD_PID_IN : EHCI_QTD_PID_OUT) |
2650 	    EHCI_QTD_SET_CERR(3)
2651 	    /* IOC set below */
2652 	    /* BYTES set below */
2653 	    ;
2654 	mps = UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize);
2655 	tog = epipe->nexttoggle;
2656 	qtdstatus |= EHCI_QTD_SET_TOGGLE(tog);
2657 
2658 	cur = ehci_alloc_sqtd(sc);
2659 	*sp = cur;
2660 	if (cur == NULL)
2661 		goto nomem;
2662 
2663 	usb_syncmem(dma, 0, alen,
2664 	    rd ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
2665 	for (;;) {
2666 		dataphyspage = EHCI_PAGE(dataphys);
2667 		/* The EHCI hardware can handle at most 5 pages. */
2668 		if (dataphyslastpage - dataphyspage <
2669 		    EHCI_QTD_NBUFFERS * EHCI_PAGE_SIZE) {
2670 			/* we can handle it in this QTD */
2671 			curlen = len;
2672 		} else {
2673 			/* must use multiple TDs, fill as much as possible. */
2674 			curlen = EHCI_QTD_NBUFFERS * EHCI_PAGE_SIZE -
2675 				 EHCI_PAGE_OFFSET(dataphys);
2676 #ifdef DIAGNOSTIC
2677 			if (curlen > len) {
2678 				printf("ehci_alloc_sqtd_chain: curlen=0x%x "
2679 				       "len=0x%x offs=0x%x\n", curlen, len,
2680 				       EHCI_PAGE_OFFSET(dataphys));
2681 				printf("lastpage=0x%x page=0x%x phys=0x%x\n",
2682 				       dataphyslastpage, dataphyspage,
2683 				       dataphys);
2684 				curlen = len;
2685 			}
2686 #endif
2687 			/* the length must be a multiple of the max size */
2688 			curlen -= curlen % mps;
2689 			DPRINTFN(1,("ehci_alloc_sqtd_chain: multiple QTDs, "
2690 				    "curlen=%d\n", curlen));
2691 #ifdef DIAGNOSTIC
2692 			if (curlen == 0)
2693 				panic("ehci_alloc_sqtd_chain: curlen == 0");
2694 #endif
2695 		}
2696 		DPRINTFN(4,("ehci_alloc_sqtd_chain: dataphys=0x%08x "
2697 			    "dataphyslastpage=0x%08x len=%d curlen=%d\n",
2698 			    dataphys, dataphyslastpage,
2699 			    len, curlen));
2700 		len -= curlen;
2701 
2702 		/*
2703 		 * Allocate another transfer if there's more data left,
2704 		 * or if force last short transfer flag is set and we're
2705 		 * allocating a multiple of the max packet size.
2706 		 */
2707 		if (len != 0 ||
2708 		    ((curlen % mps) == 0 && !rd && curlen != 0 &&
2709 		     (flags & USBD_FORCE_SHORT_XFER))) {
2710 			next = ehci_alloc_sqtd(sc);
2711 			if (next == NULL)
2712 				goto nomem;
2713 			nextphys = htole32(next->physaddr);
2714 		} else {
2715 			next = NULL;
2716 			nextphys = EHCI_NULL;
2717 		}
2718 
2719 		for (i = 0; i * EHCI_PAGE_SIZE <
2720 		            curlen + EHCI_PAGE_OFFSET(dataphys); i++) {
2721 			ehci_physaddr_t a = dataphys + i * EHCI_PAGE_SIZE;
2722 			if (i != 0) /* use offset only in first buffer */
2723 				a = EHCI_PAGE(a);
2724 			cur->qtd.qtd_buffer[i] = htole32(a);
2725 			cur->qtd.qtd_buffer_hi[i] = 0;
2726 #ifdef DIAGNOSTIC
2727 			if (i >= EHCI_QTD_NBUFFERS) {
2728 				printf("ehci_alloc_sqtd_chain: i=%d\n", i);
2729 				goto nomem;
2730 			}
2731 #endif
2732 		}
2733 		cur->nextqtd = next;
2734 		cur->qtd.qtd_next = cur->qtd.qtd_altnext = nextphys;
2735 		cur->qtd.qtd_status =
2736 		    htole32(qtdstatus | EHCI_QTD_SET_BYTES(curlen));
2737 		cur->xfer = xfer;
2738 		cur->len = curlen;
2739 
2740 		DPRINTFN(10,("ehci_alloc_sqtd_chain: cbp=0x%08x end=0x%08x\n",
2741 			    dataphys, dataphys + curlen));
2742 		/* adjust the toggle based on the number of packets in this
2743 		   qtd */
2744 		if (((curlen + mps - 1) / mps) & 1) {
2745 			tog ^= 1;
2746 			qtdstatus ^= EHCI_QTD_TOGGLE_MASK;
2747 		}
2748 		if (next == NULL)
2749 			break;
2750 		usb_syncmem(&cur->dma, cur->offs, sizeof(cur->qtd),
2751 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2752 		DPRINTFN(10,("ehci_alloc_sqtd_chain: extend chain\n"));
2753 		dataphys += curlen;
2754 		cur = next;
2755 	}
2756 	cur->qtd.qtd_status |= htole32(EHCI_QTD_IOC);
2757 	usb_syncmem(&cur->dma, cur->offs, sizeof(cur->qtd),
2758 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2759 	*ep = cur;
2760 	epipe->nexttoggle = tog;
2761 
2762 	DPRINTFN(10,("ehci_alloc_sqtd_chain: return sqtd=%p sqtdend=%p\n",
2763 		     *sp, *ep));
2764 
2765 	return (USBD_NORMAL_COMPLETION);
2766 
2767  nomem:
2768 	/* XXX free chain */
2769 	DPRINTFN(-1,("ehci_alloc_sqtd_chain: no memory\n"));
2770 	return (USBD_NOMEM);
2771 }
2772 
2773 Static void
2774 ehci_free_sqtd_chain(ehci_softc_t *sc, ehci_soft_qtd_t *sqtd,
2775 		    ehci_soft_qtd_t *sqtdend)
2776 {
2777 	ehci_soft_qtd_t *p;
2778 	int i;
2779 
2780 	DPRINTFN(10,("ehci_free_sqtd_chain: sqtd=%p sqtdend=%p\n",
2781 		     sqtd, sqtdend));
2782 
2783 	for (i = 0; sqtd != sqtdend; sqtd = p, i++) {
2784 		p = sqtd->nextqtd;
2785 		ehci_free_sqtd(sc, sqtd);
2786 	}
2787 }
2788 
2789 Static ehci_soft_itd_t *
2790 ehci_alloc_itd(ehci_softc_t *sc)
2791 {
2792 	struct ehci_soft_itd *itd, *freeitd;
2793 	usbd_status err;
2794 	int i, s, offs, frindex, previndex;
2795 	usb_dma_t dma;
2796 
2797 	s = splusb();
2798 
2799 	/* Find an itd that wasn't freed this frame or last frame. This can
2800 	 * discard itds that were freed before frindex wrapped around
2801 	 * XXX - can this lead to thrashing? Could fix by enabling wrap-around
2802 	 *       interrupt and fiddling with list when that happens */
2803 	frindex = (EOREAD4(sc, EHCI_FRINDEX) + 1) >> 3;
2804 	previndex = (frindex != 0) ? frindex - 1 : sc->sc_flsize;
2805 
2806 	freeitd = NULL;
2807 	LIST_FOREACH(itd, &sc->sc_freeitds, u.free_list) {
2808 		if (itd == NULL)
2809 			break;
2810 		if (itd->slot != frindex && itd->slot != previndex) {
2811 			freeitd = itd;
2812 			break;
2813 		}
2814 	}
2815 
2816 	if (freeitd == NULL) {
2817 		DPRINTFN(2, ("ehci_alloc_itd allocating chunk\n"));
2818 		err = usb_allocmem(&sc->sc_bus, EHCI_ITD_SIZE * EHCI_ITD_CHUNK,
2819 				EHCI_PAGE_SIZE, &dma);
2820 
2821 		if (err) {
2822 			DPRINTF(("ehci_alloc_itd, alloc returned %d\n", err));
2823 			return NULL;
2824 		}
2825 
2826 		for (i = 0; i < EHCI_ITD_CHUNK; i++) {
2827 			offs = i * EHCI_ITD_SIZE;
2828 			itd = KERNADDR(&dma, offs);
2829 			itd->physaddr = DMAADDR(&dma, offs);
2830 	 		itd->dma = dma;
2831 			itd->offs = offs;
2832 			LIST_INSERT_HEAD(&sc->sc_freeitds, itd, u.free_list);
2833 		}
2834 		freeitd = LIST_FIRST(&sc->sc_freeitds);
2835 	}
2836 
2837 	itd = freeitd;
2838 	LIST_REMOVE(itd, u.free_list);
2839 	memset(&itd->itd, 0, sizeof(ehci_itd_t));
2840 	usb_syncmem(&itd->dma, itd->offs + offsetof(ehci_itd_t, itd_next),
2841                     sizeof(itd->itd.itd_next), BUS_DMASYNC_PREWRITE |
2842                     BUS_DMASYNC_PREREAD);
2843 
2844 	itd->u.frame_list.next = NULL;
2845 	itd->u.frame_list.prev = NULL;
2846 	itd->xfer_next = NULL;
2847 	itd->slot = 0;
2848 	splx(s);
2849 
2850 	return itd;
2851 }
2852 
2853 Static void
2854 ehci_free_itd(ehci_softc_t *sc, ehci_soft_itd_t *itd)
2855 {
2856 	int s;
2857 
2858 	s = splusb();
2859 	LIST_INSERT_HEAD(&sc->sc_freeitds, itd, u.free_list);
2860 	splx(s);
2861 }
2862 
2863 /****************/
2864 
2865 /*
2866  * Close a reqular pipe.
2867  * Assumes that there are no pending transactions.
2868  */
2869 Static void
2870 ehci_close_pipe(usbd_pipe_handle pipe, ehci_soft_qh_t *head)
2871 {
2872 	struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
2873 	ehci_softc_t *sc = pipe->device->bus->hci_private;
2874 	ehci_soft_qh_t *sqh = epipe->sqh;
2875 	int s;
2876 
2877 	s = splusb();
2878 	ehci_rem_qh(sc, sqh, head);
2879 	splx(s);
2880 	ehci_free_sqh(sc, epipe->sqh);
2881 }
2882 
2883 /*
2884  * Abort a device request.
2885  * If this routine is called at splusb() it guarantees that the request
2886  * will be removed from the hardware scheduling and that the callback
2887  * for it will be called with USBD_CANCELLED status.
2888  * It's impossible to guarantee that the requested transfer will not
2889  * have happened since the hardware runs concurrently.
2890  * If the transaction has already happened we rely on the ordinary
2891  * interrupt processing to process it.
2892  * XXX This is most probably wrong.
2893  */
2894 Static void
2895 ehci_abort_xfer(usbd_xfer_handle xfer, usbd_status status)
2896 {
2897 #define exfer EXFER(xfer)
2898 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2899 	ehci_softc_t *sc = epipe->pipe.device->bus->hci_private;
2900 	ehci_soft_qh_t *sqh = epipe->sqh;
2901 	ehci_soft_qtd_t *sqtd;
2902 	ehci_physaddr_t cur;
2903 	u_int32_t qhstatus;
2904 	int s;
2905 	int hit;
2906 	int wake;
2907 
2908 	DPRINTF(("ehci_abort_xfer: xfer=%p pipe=%p\n", xfer, epipe));
2909 
2910 	if (sc->sc_dying) {
2911 		/* If we're dying, just do the software part. */
2912 		s = splusb();
2913 		xfer->status = status;	/* make software ignore it */
2914 		callout_stop(&xfer->timeout_handle);
2915 		usb_transfer_complete(xfer);
2916 		splx(s);
2917 		return;
2918 	}
2919 
2920 	if (xfer->device->bus->intr_context)
2921 		panic("ehci_abort_xfer: not in process context");
2922 
2923 	/*
2924 	 * If an abort is already in progress then just wait for it to
2925 	 * complete and return.
2926 	 */
2927 	if (xfer->hcflags & UXFER_ABORTING) {
2928 		DPRINTFN(2, ("ehci_abort_xfer: already aborting\n"));
2929 #ifdef DIAGNOSTIC
2930 		if (status == USBD_TIMEOUT)
2931 			printf("ehci_abort_xfer: TIMEOUT while aborting\n");
2932 #endif
2933 		/* Override the status which might be USBD_TIMEOUT. */
2934 		xfer->status = status;
2935 		DPRINTFN(2, ("ehci_abort_xfer: waiting for abort to finish\n"));
2936 		xfer->hcflags |= UXFER_ABORTWAIT;
2937 		while (xfer->hcflags & UXFER_ABORTING)
2938 			tsleep(&xfer->hcflags, PZERO, "ehciaw", 0);
2939 		return;
2940 	}
2941 	xfer->hcflags |= UXFER_ABORTING;
2942 
2943 	/*
2944 	 * Step 1: Make interrupt routine and hardware ignore xfer.
2945 	 */
2946 	s = splusb();
2947 	xfer->status = status;	/* make software ignore it */
2948 	callout_stop(&xfer->timeout_handle);
2949 
2950 	usb_syncmem(&sqh->dma,
2951 	    sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
2952 	    sizeof(sqh->qh.qh_qtd.qtd_status),
2953 	    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2954 	qhstatus = sqh->qh.qh_qtd.qtd_status;
2955 	sqh->qh.qh_qtd.qtd_status = qhstatus | htole32(EHCI_QTD_HALTED);
2956 	usb_syncmem(&sqh->dma,
2957 	    sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
2958 	    sizeof(sqh->qh.qh_qtd.qtd_status),
2959 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2960 	for (sqtd = exfer->sqtdstart; ; sqtd = sqtd->nextqtd) {
2961 		usb_syncmem(&sqtd->dma,
2962 		    sqtd->offs + offsetof(ehci_qtd_t, qtd_status),
2963 		    sizeof(sqtd->qtd.qtd_status),
2964 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2965 		sqtd->qtd.qtd_status |= htole32(EHCI_QTD_HALTED);
2966 		usb_syncmem(&sqtd->dma,
2967 		    sqtd->offs + offsetof(ehci_qtd_t, qtd_status),
2968 		    sizeof(sqtd->qtd.qtd_status),
2969 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2970 		if (sqtd == exfer->sqtdend)
2971 			break;
2972 	}
2973 	splx(s);
2974 
2975 	/*
2976 	 * Step 2: Wait until we know hardware has finished any possible
2977 	 * use of the xfer.  Also make sure the soft interrupt routine
2978 	 * has run.
2979 	 */
2980 	ehci_sync_hc(sc);
2981 	s = splusb();
2982 #ifdef USB_USE_SOFTINTR
2983 	sc->sc_softwake = 1;
2984 #endif /* USB_USE_SOFTINTR */
2985 	usb_schedsoftintr(&sc->sc_bus);
2986 #ifdef USB_USE_SOFTINTR
2987 	tsleep(&sc->sc_softwake, PZERO, "ehciab", 0);
2988 #endif /* USB_USE_SOFTINTR */
2989 	splx(s);
2990 
2991 	/*
2992 	 * Step 3: Remove any vestiges of the xfer from the hardware.
2993 	 * The complication here is that the hardware may have executed
2994 	 * beyond the xfer we're trying to abort.  So as we're scanning
2995 	 * the TDs of this xfer we check if the hardware points to
2996 	 * any of them.
2997 	 */
2998 	s = splusb();		/* XXX why? */
2999 
3000 	usb_syncmem(&sqh->dma,
3001 	    sqh->offs + offsetof(ehci_qh_t, qh_curqtd),
3002 	    sizeof(sqh->qh.qh_curqtd),
3003 	    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
3004 	cur = EHCI_LINK_ADDR(le32toh(sqh->qh.qh_curqtd));
3005 	hit = 0;
3006 	for (sqtd = exfer->sqtdstart; ; sqtd = sqtd->nextqtd) {
3007 		hit |= cur == sqtd->physaddr;
3008 		if (sqtd == exfer->sqtdend)
3009 			break;
3010 	}
3011 	sqtd = sqtd->nextqtd;
3012 	/* Zap curqtd register if hardware pointed inside the xfer. */
3013 	if (hit && sqtd != NULL) {
3014 		DPRINTFN(1,("ehci_abort_xfer: cur=0x%08x\n", sqtd->physaddr));
3015 		sqh->qh.qh_curqtd = htole32(sqtd->physaddr); /* unlink qTDs */
3016 		usb_syncmem(&sqh->dma,
3017 		    sqh->offs + offsetof(ehci_qh_t, qh_curqtd),
3018 		    sizeof(sqh->qh.qh_curqtd),
3019 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3020 		sqh->qh.qh_qtd.qtd_status = qhstatus;
3021 		usb_syncmem(&sqh->dma,
3022 		    sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
3023 		    sizeof(sqh->qh.qh_qtd.qtd_status),
3024 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3025 	} else {
3026 		DPRINTFN(1,("ehci_abort_xfer: no hit\n"));
3027 	}
3028 
3029 	/*
3030 	 * Step 4: Execute callback.
3031 	 */
3032 #ifdef DIAGNOSTIC
3033 	exfer->isdone = 1;
3034 #endif
3035 	wake = xfer->hcflags & UXFER_ABORTWAIT;
3036 	xfer->hcflags &= ~(UXFER_ABORTING | UXFER_ABORTWAIT);
3037 	usb_transfer_complete(xfer);
3038 	if (wake)
3039 		wakeup(&xfer->hcflags);
3040 
3041 	splx(s);
3042 #undef exfer
3043 }
3044 
3045 Static void
3046 ehci_abort_isoc_xfer(usbd_xfer_handle xfer, usbd_status status)
3047 {
3048 	ehci_isoc_trans_t trans_status;
3049 	struct ehci_pipe *epipe;
3050 	struct ehci_xfer *exfer;
3051 	ehci_softc_t *sc;
3052 	struct ehci_soft_itd *itd;
3053 	int s, i, wake;
3054 
3055 	epipe = (struct ehci_pipe *) xfer->pipe;
3056 	exfer = EXFER(xfer);
3057 	sc = epipe->pipe.device->bus->hci_private;
3058 
3059 	DPRINTF(("ehci_abort_isoc_xfer: xfer %p pipe %p\n", xfer, epipe));
3060 
3061 	if (sc->sc_dying) {
3062 		s = splusb();
3063 		xfer->status = status;
3064 		callout_stop(&xfer->timeout_handle);
3065 		usb_transfer_complete(xfer);
3066 		splx(s);
3067 		return;
3068 	}
3069 
3070 	if (xfer->hcflags & UXFER_ABORTING) {
3071 		DPRINTFN(2, ("ehci_abort_isoc_xfer: already aborting\n"));
3072 
3073 #ifdef DIAGNOSTIC
3074 		if (status == USBD_TIMEOUT)
3075 			printf("ehci_abort_xfer: TIMEOUT while aborting\n");
3076 #endif
3077 
3078 		xfer->status = status;
3079 		DPRINTFN(2, ("ehci_abort_xfer: waiting for abort to finish\n"));
3080 		xfer->hcflags |= UXFER_ABORTWAIT;
3081 		while (xfer->hcflags & UXFER_ABORTING)
3082 			tsleep(&xfer->hcflags, PZERO, "ehciiaw", 0);
3083 		return;
3084 	}
3085 	xfer->hcflags |= UXFER_ABORTING;
3086 
3087 	xfer->status = status;
3088 	callout_stop(&xfer->timeout_handle);
3089 
3090 	s = splusb();
3091 	for (itd = exfer->itdstart; itd != NULL; itd = itd->xfer_next) {
3092 		usb_syncmem(&itd->dma,
3093 		    itd->offs + offsetof(ehci_itd_t, itd_ctl),
3094 		    sizeof(itd->itd.itd_ctl),
3095 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
3096 
3097 		for (i = 0; i < 8; i++) {
3098 			trans_status = le32toh(itd->itd.itd_ctl[i]);
3099 			trans_status &= ~EHCI_ITD_ACTIVE;
3100 			itd->itd.itd_ctl[i] = htole32(trans_status);
3101 		}
3102 
3103 		usb_syncmem(&itd->dma,
3104 		    itd->offs + offsetof(ehci_itd_t, itd_ctl),
3105 		    sizeof(itd->itd.itd_ctl),
3106 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3107 	}
3108 	splx(s);
3109 
3110         s = splusb();
3111 #ifdef USB_USE_SOFTINTR
3112         sc->sc_softwake = 1;
3113 #endif /* USB_USE_SOFTINTR */
3114         usb_schedsoftintr(&sc->sc_bus);
3115 #ifdef USB_USE_SOFTINTR
3116         tsleep(&sc->sc_softwake, PZERO, "ehciab", 0);
3117 #endif /* USB_USE_SOFTINTR */
3118         splx(s);
3119 
3120 #ifdef DIAGNOSTIC
3121 	exfer->isdone = 1;
3122 #endif
3123 	wake = xfer->hcflags & UXFER_ABORTWAIT;
3124 	xfer->hcflags &= ~(UXFER_ABORTING | UXFER_ABORTWAIT);
3125 	usb_transfer_complete(xfer);
3126 	if (wake)
3127 		wakeup(&xfer->hcflags);
3128 
3129 	return;
3130 }
3131 
3132 Static void
3133 ehci_timeout(void *addr)
3134 {
3135 	struct ehci_xfer *exfer = addr;
3136 	struct ehci_pipe *epipe = (struct ehci_pipe *)exfer->xfer.pipe;
3137 	ehci_softc_t *sc = epipe->pipe.device->bus->hci_private;
3138 
3139 	DPRINTF(("ehci_timeout: exfer=%p\n", exfer));
3140 #ifdef EHCI_DEBUG
3141 	if (ehcidebug > 1)
3142 		usbd_dump_pipe(exfer->xfer.pipe);
3143 #endif
3144 
3145 	if (sc->sc_dying) {
3146 		ehci_abort_xfer(&exfer->xfer, USBD_TIMEOUT);
3147 		return;
3148 	}
3149 
3150 	/* Execute the abort in a process context. */
3151 	usb_init_task(&exfer->abort_task, ehci_timeout_task, addr);
3152 	usb_add_task(exfer->xfer.pipe->device, &exfer->abort_task,
3153 	    USB_TASKQ_HC);
3154 }
3155 
3156 Static void
3157 ehci_timeout_task(void *addr)
3158 {
3159 	usbd_xfer_handle xfer = addr;
3160 	int s;
3161 
3162 	DPRINTF(("ehci_timeout_task: xfer=%p\n", xfer));
3163 
3164 	s = splusb();
3165 	ehci_abort_xfer(xfer, USBD_TIMEOUT);
3166 	splx(s);
3167 }
3168 
3169 /************************/
3170 
3171 Static usbd_status
3172 ehci_device_ctrl_transfer(usbd_xfer_handle xfer)
3173 {
3174 	usbd_status err;
3175 
3176 	/* Insert last in queue. */
3177 	err = usb_insert_transfer(xfer);
3178 	if (err)
3179 		return (err);
3180 
3181 	/* Pipe isn't running, start first */
3182 	return (ehci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
3183 }
3184 
3185 Static usbd_status
3186 ehci_device_ctrl_start(usbd_xfer_handle xfer)
3187 {
3188 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
3189 	usbd_status err;
3190 
3191 	if (sc->sc_dying)
3192 		return (USBD_IOERROR);
3193 
3194 #ifdef DIAGNOSTIC
3195 	if (!(xfer->rqflags & URQ_REQUEST)) {
3196 		/* XXX panic */
3197 		printf("ehci_device_ctrl_transfer: not a request\n");
3198 		return (USBD_INVAL);
3199 	}
3200 #endif
3201 
3202 	err = ehci_device_request(xfer);
3203 	if (err)
3204 		return (err);
3205 
3206 	if (sc->sc_bus.use_polling)
3207 		ehci_waitintr(sc, xfer);
3208 	return (USBD_IN_PROGRESS);
3209 }
3210 
3211 Static void
3212 ehci_device_ctrl_done(usbd_xfer_handle xfer)
3213 {
3214 	struct ehci_xfer *ex = EXFER(xfer);
3215 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
3216 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3217 	usb_device_request_t *req = &xfer->request;
3218 	int len = UGETW(req->wLength);
3219 	int rd = req->bmRequestType & UT_READ;
3220 
3221 	DPRINTFN(10,("ehci_ctrl_done: xfer=%p\n", xfer));
3222 
3223 #ifdef DIAGNOSTIC
3224 	if (!(xfer->rqflags & URQ_REQUEST)) {
3225 		panic("ehci_ctrl_done: not a request");
3226 	}
3227 #endif
3228 
3229 	mutex_enter(&sc->sc_intrhead_lock);
3230 	if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
3231 		ehci_del_intr_list(sc, ex);	/* remove from active list */
3232 		ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
3233 		usb_syncmem(&epipe->u.ctl.reqdma, 0, sizeof *req,
3234 		    BUS_DMASYNC_POSTWRITE);
3235 		if (len)
3236 			usb_syncmem(&xfer->dmabuf, 0, len,
3237 			    rd ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
3238 	}
3239 	mutex_exit(&sc->sc_intrhead_lock);
3240 
3241 	DPRINTFN(5, ("ehci_ctrl_done: length=%d\n", xfer->actlen));
3242 }
3243 
3244 /* Abort a device control request. */
3245 Static void
3246 ehci_device_ctrl_abort(usbd_xfer_handle xfer)
3247 {
3248 	DPRINTF(("ehci_device_ctrl_abort: xfer=%p\n", xfer));
3249 	ehci_abort_xfer(xfer, USBD_CANCELLED);
3250 }
3251 
3252 /* Close a device control pipe. */
3253 Static void
3254 ehci_device_ctrl_close(usbd_pipe_handle pipe)
3255 {
3256 	ehci_softc_t *sc = pipe->device->bus->hci_private;
3257 	/*struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;*/
3258 
3259 	DPRINTF(("ehci_device_ctrl_close: pipe=%p\n", pipe));
3260 	ehci_close_pipe(pipe, sc->sc_async_head);
3261 }
3262 
3263 Static usbd_status
3264 ehci_device_request(usbd_xfer_handle xfer)
3265 {
3266 #define exfer EXFER(xfer)
3267 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3268 	usb_device_request_t *req = &xfer->request;
3269 	usbd_device_handle dev = epipe->pipe.device;
3270 	ehci_softc_t *sc = dev->bus->hci_private;
3271 	int addr = dev->address;
3272 	ehci_soft_qtd_t *setup, *stat, *next;
3273 	ehci_soft_qh_t *sqh;
3274 	int isread;
3275 	int len;
3276 	usbd_status err;
3277 	int s;
3278 
3279 	isread = req->bmRequestType & UT_READ;
3280 	len = UGETW(req->wLength);
3281 
3282 	DPRINTFN(3,("ehci_device_request: type=0x%02x, request=0x%02x, "
3283 		    "wValue=0x%04x, wIndex=0x%04x len=%d, addr=%d, endpt=%d\n",
3284 		    req->bmRequestType, req->bRequest, UGETW(req->wValue),
3285 		    UGETW(req->wIndex), len, addr,
3286 		    epipe->pipe.endpoint->edesc->bEndpointAddress));
3287 
3288 	setup = ehci_alloc_sqtd(sc);
3289 	if (setup == NULL) {
3290 		err = USBD_NOMEM;
3291 		goto bad1;
3292 	}
3293 	stat = ehci_alloc_sqtd(sc);
3294 	if (stat == NULL) {
3295 		err = USBD_NOMEM;
3296 		goto bad2;
3297 	}
3298 
3299 	sqh = epipe->sqh;
3300 	epipe->u.ctl.length = len;
3301 
3302 	/* Update device address and length since they may have changed
3303 	   during the setup of the control pipe in usbd_new_device(). */
3304 	/* XXX This only needs to be done once, but it's too early in open. */
3305 	/* XXXX Should not touch ED here! */
3306 	sqh->qh.qh_endp =
3307 	    (sqh->qh.qh_endp & htole32(~(EHCI_QH_ADDRMASK | EHCI_QH_MPLMASK))) |
3308 	    htole32(
3309 	     EHCI_QH_SET_ADDR(addr) |
3310 	     EHCI_QH_SET_MPL(UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize))
3311 	    );
3312 
3313 	/* Set up data transaction */
3314 	if (len != 0) {
3315 		ehci_soft_qtd_t *end;
3316 
3317 		/* Start toggle at 1. */
3318 		epipe->nexttoggle = 1;
3319 		err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
3320 			  &next, &end);
3321 		if (err)
3322 			goto bad3;
3323 		end->qtd.qtd_status &= htole32(~EHCI_QTD_IOC);
3324 		end->nextqtd = stat;
3325 		end->qtd.qtd_next =
3326 		end->qtd.qtd_altnext = htole32(stat->physaddr);
3327 		usb_syncmem(&end->dma, end->offs, sizeof(end->qtd),
3328 		   BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3329 	} else {
3330 		next = stat;
3331 	}
3332 
3333 	memcpy(KERNADDR(&epipe->u.ctl.reqdma, 0), req, sizeof *req);
3334 	usb_syncmem(&epipe->u.ctl.reqdma, 0, sizeof *req, BUS_DMASYNC_PREWRITE);
3335 
3336 	/* Clear toggle */
3337 	setup->qtd.qtd_status = htole32(
3338 	    EHCI_QTD_ACTIVE |
3339 	    EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
3340 	    EHCI_QTD_SET_CERR(3) |
3341 	    EHCI_QTD_SET_TOGGLE(0) |
3342 	    EHCI_QTD_SET_BYTES(sizeof *req)
3343 	    );
3344 	setup->qtd.qtd_buffer[0] = htole32(DMAADDR(&epipe->u.ctl.reqdma, 0));
3345 	setup->qtd.qtd_buffer_hi[0] = 0;
3346 	setup->nextqtd = next;
3347 	setup->qtd.qtd_next = setup->qtd.qtd_altnext = htole32(next->physaddr);
3348 	setup->xfer = xfer;
3349 	setup->len = sizeof *req;
3350 	usb_syncmem(&setup->dma, setup->offs, sizeof(setup->qtd),
3351 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3352 
3353 	stat->qtd.qtd_status = htole32(
3354 	    EHCI_QTD_ACTIVE |
3355 	    EHCI_QTD_SET_PID(isread ? EHCI_QTD_PID_OUT : EHCI_QTD_PID_IN) |
3356 	    EHCI_QTD_SET_CERR(3) |
3357 	    EHCI_QTD_SET_TOGGLE(1) |
3358 	    EHCI_QTD_IOC
3359 	    );
3360 	stat->qtd.qtd_buffer[0] = 0; /* XXX not needed? */
3361 	stat->qtd.qtd_buffer_hi[0] = 0; /* XXX not needed? */
3362 	stat->nextqtd = NULL;
3363 	stat->qtd.qtd_next = stat->qtd.qtd_altnext = EHCI_NULL;
3364 	stat->xfer = xfer;
3365 	stat->len = 0;
3366 	usb_syncmem(&stat->dma, stat->offs, sizeof(stat->qtd),
3367 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3368 
3369 #ifdef EHCI_DEBUG
3370 	if (ehcidebug > 5) {
3371 		DPRINTF(("ehci_device_request:\n"));
3372 		ehci_dump_sqh(sqh);
3373 		ehci_dump_sqtds(setup);
3374 	}
3375 #endif
3376 
3377 	exfer->sqtdstart = setup;
3378 	exfer->sqtdend = stat;
3379 #ifdef DIAGNOSTIC
3380 	if (!exfer->isdone) {
3381 		printf("ehci_device_request: not done, exfer=%p\n", exfer);
3382 	}
3383 	exfer->isdone = 0;
3384 #endif
3385 
3386 	/* Insert qTD in QH list. */
3387 	s = splusb();
3388 	ehci_set_qh_qtd(sqh, setup); /* also does usb_syncmem(sqh) */
3389 	if (xfer->timeout && !sc->sc_bus.use_polling) {
3390 		callout_reset(&(xfer->timeout_handle), (mstohz(xfer->timeout)),
3391 		    (ehci_timeout), (xfer));
3392 	}
3393 	mutex_enter(&sc->sc_intrhead_lock);
3394 	ehci_add_intr_list(sc, exfer);
3395 	mutex_exit(&sc->sc_intrhead_lock);
3396 	xfer->status = USBD_IN_PROGRESS;
3397 	splx(s);
3398 
3399 #ifdef EHCI_DEBUG
3400 	if (ehcidebug > 10) {
3401 		DPRINTF(("ehci_device_request: status=%x\n",
3402 			 EOREAD4(sc, EHCI_USBSTS)));
3403 		delay(10000);
3404 		ehci_dump_regs(sc);
3405 		ehci_dump_sqh(sc->sc_async_head);
3406 		ehci_dump_sqh(sqh);
3407 		ehci_dump_sqtds(setup);
3408 	}
3409 #endif
3410 
3411 	return (USBD_NORMAL_COMPLETION);
3412 
3413  bad3:
3414 	ehci_free_sqtd(sc, stat);
3415  bad2:
3416 	ehci_free_sqtd(sc, setup);
3417  bad1:
3418 	DPRINTFN(-1,("ehci_device_request: no memory\n"));
3419 	xfer->status = err;
3420 	usb_transfer_complete(xfer);
3421 	return (err);
3422 #undef exfer
3423 }
3424 
3425 /*
3426  * Some EHCI chips from VIA seem to trigger interrupts before writing back the
3427  * qTD status, or miss signalling occasionally under heavy load.  If the host
3428  * machine is too fast, we we can miss transaction completion - when we scan
3429  * the active list the transaction still seems to be active.  This generally
3430  * exhibits itself as a umass stall that never recovers.
3431  *
3432  * We work around this behaviour by setting up this callback after any softintr
3433  * that completes with transactions still pending, giving us another chance to
3434  * check for completion after the writeback has taken place.
3435  */
3436 Static void
3437 ehci_intrlist_timeout(void *arg)
3438 {
3439 	ehci_softc_t *sc = arg;
3440 	int s = splusb();
3441 
3442 	DPRINTF(("ehci_intrlist_timeout\n"));
3443 	usb_schedsoftintr(&sc->sc_bus);
3444 
3445 	splx(s);
3446 }
3447 
3448 /************************/
3449 
3450 Static usbd_status
3451 ehci_device_bulk_transfer(usbd_xfer_handle xfer)
3452 {
3453 	usbd_status err;
3454 
3455 	/* Insert last in queue. */
3456 	err = usb_insert_transfer(xfer);
3457 	if (err)
3458 		return (err);
3459 
3460 	/* Pipe isn't running, start first */
3461 	return (ehci_device_bulk_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
3462 }
3463 
3464 Static usbd_status
3465 ehci_device_bulk_start(usbd_xfer_handle xfer)
3466 {
3467 #define exfer EXFER(xfer)
3468 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3469 	usbd_device_handle dev = epipe->pipe.device;
3470 	ehci_softc_t *sc = dev->bus->hci_private;
3471 	ehci_soft_qtd_t *data, *dataend;
3472 	ehci_soft_qh_t *sqh;
3473 	usbd_status err;
3474 	int len, isread, endpt;
3475 	int s;
3476 
3477 	DPRINTFN(2, ("ehci_device_bulk_start: xfer=%p len=%d flags=%d\n",
3478 		     xfer, xfer->length, xfer->flags));
3479 
3480 	if (sc->sc_dying)
3481 		return (USBD_IOERROR);
3482 
3483 #ifdef DIAGNOSTIC
3484 	if (xfer->rqflags & URQ_REQUEST)
3485 		panic("ehci_device_bulk_start: a request");
3486 #endif
3487 
3488 	len = xfer->length;
3489 	endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3490 	isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3491 	sqh = epipe->sqh;
3492 
3493 	epipe->u.bulk.length = len;
3494 
3495 	err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer, &data,
3496 				   &dataend);
3497 	if (err) {
3498 		DPRINTFN(-1,("ehci_device_bulk_transfer: no memory\n"));
3499 		xfer->status = err;
3500 		usb_transfer_complete(xfer);
3501 		return (err);
3502 	}
3503 
3504 #ifdef EHCI_DEBUG
3505 	if (ehcidebug > 5) {
3506 		DPRINTF(("ehci_device_bulk_start: data(1)\n"));
3507 		ehci_dump_sqh(sqh);
3508 		ehci_dump_sqtds(data);
3509 	}
3510 #endif
3511 
3512 	/* Set up interrupt info. */
3513 	exfer->sqtdstart = data;
3514 	exfer->sqtdend = dataend;
3515 #ifdef DIAGNOSTIC
3516 	if (!exfer->isdone) {
3517 		printf("ehci_device_bulk_start: not done, ex=%p\n", exfer);
3518 	}
3519 	exfer->isdone = 0;
3520 #endif
3521 
3522 	s = splusb();
3523 	ehci_set_qh_qtd(sqh, data); /* also does usb_syncmem(sqh) */
3524 	if (xfer->timeout && !sc->sc_bus.use_polling) {
3525 		callout_reset(&(xfer->timeout_handle), (mstohz(xfer->timeout)),
3526 		    (ehci_timeout), (xfer));
3527 	}
3528 	mutex_enter(&sc->sc_intrhead_lock);
3529 	ehci_add_intr_list(sc, exfer);
3530 	mutex_exit(&sc->sc_intrhead_lock);
3531 	xfer->status = USBD_IN_PROGRESS;
3532 	splx(s);
3533 
3534 #ifdef EHCI_DEBUG
3535 	if (ehcidebug > 10) {
3536 		DPRINTF(("ehci_device_bulk_start: data(2)\n"));
3537 		delay(10000);
3538 		DPRINTF(("ehci_device_bulk_start: data(3)\n"));
3539 		ehci_dump_regs(sc);
3540 #if 0
3541 		printf("async_head:\n");
3542 		ehci_dump_sqh(sc->sc_async_head);
3543 #endif
3544 		printf("sqh:\n");
3545 		ehci_dump_sqh(sqh);
3546 		ehci_dump_sqtds(data);
3547 	}
3548 #endif
3549 
3550 	if (sc->sc_bus.use_polling)
3551 		ehci_waitintr(sc, xfer);
3552 
3553 	return (USBD_IN_PROGRESS);
3554 #undef exfer
3555 }
3556 
3557 Static void
3558 ehci_device_bulk_abort(usbd_xfer_handle xfer)
3559 {
3560 	DPRINTF(("ehci_device_bulk_abort: xfer=%p\n", xfer));
3561 	ehci_abort_xfer(xfer, USBD_CANCELLED);
3562 }
3563 
3564 /*
3565  * Close a device bulk pipe.
3566  */
3567 Static void
3568 ehci_device_bulk_close(usbd_pipe_handle pipe)
3569 {
3570 	ehci_softc_t *sc = pipe->device->bus->hci_private;
3571 
3572 	DPRINTF(("ehci_device_bulk_close: pipe=%p\n", pipe));
3573 	ehci_close_pipe(pipe, sc->sc_async_head);
3574 }
3575 
3576 Static void
3577 ehci_device_bulk_done(usbd_xfer_handle xfer)
3578 {
3579 	struct ehci_xfer *ex = EXFER(xfer);
3580 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
3581 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3582 	int endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3583 	int rd = UE_GET_DIR(endpt) == UE_DIR_IN;
3584 
3585 	DPRINTFN(10,("ehci_bulk_done: xfer=%p, actlen=%d\n",
3586 		     xfer, xfer->actlen));
3587 
3588 	mutex_enter(&sc->sc_intrhead_lock);
3589 	if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
3590 		ehci_del_intr_list(sc, ex);	/* remove from active list */
3591 		ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
3592 		usb_syncmem(&xfer->dmabuf, 0, xfer->length,
3593 		    rd ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
3594 	}
3595 	mutex_exit(&sc->sc_intrhead_lock);
3596 
3597 	DPRINTFN(5, ("ehci_bulk_done: length=%d\n", xfer->actlen));
3598 }
3599 
3600 /************************/
3601 
3602 Static usbd_status
3603 ehci_device_setintr(ehci_softc_t *sc, ehci_soft_qh_t *sqh, int ival)
3604 {
3605 	struct ehci_soft_islot *isp;
3606 	int islot, lev;
3607 
3608 	/* Find a poll rate that is large enough. */
3609 	for (lev = EHCI_IPOLLRATES - 1; lev > 0; lev--)
3610 		if (EHCI_ILEV_IVAL(lev) <= ival)
3611 			break;
3612 
3613 	/* Pick an interrupt slot at the right level. */
3614 	/* XXX could do better than picking at random */
3615 	sc->sc_rand = (sc->sc_rand + 191) % sc->sc_flsize;
3616 	islot = EHCI_IQHIDX(lev, sc->sc_rand);
3617 
3618 	sqh->islot = islot;
3619 	isp = &sc->sc_islots[islot];
3620 	ehci_add_qh(sqh, isp->sqh);
3621 
3622 	return (USBD_NORMAL_COMPLETION);
3623 }
3624 
3625 Static usbd_status
3626 ehci_device_intr_transfer(usbd_xfer_handle xfer)
3627 {
3628 	usbd_status err;
3629 
3630 	/* Insert last in queue. */
3631 	err = usb_insert_transfer(xfer);
3632 	if (err)
3633 		return (err);
3634 
3635 	/*
3636 	 * Pipe isn't running (otherwise err would be USBD_INPROG),
3637 	 * so start it first.
3638 	 */
3639 	return (ehci_device_intr_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
3640 }
3641 
3642 Static usbd_status
3643 ehci_device_intr_start(usbd_xfer_handle xfer)
3644 {
3645 #define exfer EXFER(xfer)
3646 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3647 	usbd_device_handle dev = xfer->pipe->device;
3648 	ehci_softc_t *sc = dev->bus->hci_private;
3649 	ehci_soft_qtd_t *data, *dataend;
3650 	ehci_soft_qh_t *sqh;
3651 	usbd_status err;
3652 	int len, isread, endpt;
3653 	int s;
3654 
3655 	DPRINTFN(2, ("ehci_device_intr_start: xfer=%p len=%d flags=%d\n",
3656 	    xfer, xfer->length, xfer->flags));
3657 
3658 	if (sc->sc_dying)
3659 		return (USBD_IOERROR);
3660 
3661 #ifdef DIAGNOSTIC
3662 	if (xfer->rqflags & URQ_REQUEST)
3663 		panic("ehci_device_intr_start: a request");
3664 #endif
3665 
3666 	len = xfer->length;
3667 	endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3668 	isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3669 	sqh = epipe->sqh;
3670 
3671 	epipe->u.intr.length = len;
3672 
3673 	err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer, &data,
3674 	    &dataend);
3675 	if (err) {
3676 		DPRINTFN(-1, ("ehci_device_intr_start: no memory\n"));
3677 		xfer->status = err;
3678 		usb_transfer_complete(xfer);
3679 		return (err);
3680 	}
3681 
3682 #ifdef EHCI_DEBUG
3683 	if (ehcidebug > 5) {
3684 		DPRINTF(("ehci_device_intr_start: data(1)\n"));
3685 		ehci_dump_sqh(sqh);
3686 		ehci_dump_sqtds(data);
3687 	}
3688 #endif
3689 
3690 	/* Set up interrupt info. */
3691 	exfer->sqtdstart = data;
3692 	exfer->sqtdend = dataend;
3693 #ifdef DIAGNOSTIC
3694 	if (!exfer->isdone) {
3695 		printf("ehci_device_intr_start: not done, ex=%p\n", exfer);
3696 	}
3697 	exfer->isdone = 0;
3698 #endif
3699 
3700 	s = splusb();
3701 	ehci_set_qh_qtd(sqh, data); /* also does usb_syncmem(sqh) */
3702 	if (xfer->timeout && !sc->sc_bus.use_polling) {
3703 		callout_reset(&(xfer->timeout_handle), (mstohz(xfer->timeout)),
3704 		    (ehci_timeout), (xfer));
3705 	}
3706 	mutex_enter(&sc->sc_intrhead_lock);
3707 	ehci_add_intr_list(sc, exfer);
3708 	mutex_exit(&sc->sc_intrhead_lock);
3709 	xfer->status = USBD_IN_PROGRESS;
3710 	splx(s);
3711 
3712 #ifdef EHCI_DEBUG
3713 	if (ehcidebug > 10) {
3714 		DPRINTF(("ehci_device_intr_start: data(2)\n"));
3715 		delay(10000);
3716 		DPRINTF(("ehci_device_intr_start: data(3)\n"));
3717 		ehci_dump_regs(sc);
3718 		printf("sqh:\n");
3719 		ehci_dump_sqh(sqh);
3720 		ehci_dump_sqtds(data);
3721 	}
3722 #endif
3723 
3724 	if (sc->sc_bus.use_polling)
3725 		ehci_waitintr(sc, xfer);
3726 
3727 	return (USBD_IN_PROGRESS);
3728 #undef exfer
3729 }
3730 
3731 Static void
3732 ehci_device_intr_abort(usbd_xfer_handle xfer)
3733 {
3734 	DPRINTFN(1, ("ehci_device_intr_abort: xfer=%p\n", xfer));
3735 	if (xfer->pipe->intrxfer == xfer) {
3736 		DPRINTFN(1, ("echi_device_intr_abort: remove\n"));
3737 		xfer->pipe->intrxfer = NULL;
3738 	}
3739 	/*
3740 	 * XXX - abort_xfer uses ehci_sync_hc, which syncs via the advance
3741 	 *       async doorbell. That's dependant on the async list, wheras
3742 	 *       intr xfers are periodic, should not use this?
3743 	 */
3744 	ehci_abort_xfer(xfer, USBD_CANCELLED);
3745 }
3746 
3747 Static void
3748 ehci_device_intr_close(usbd_pipe_handle pipe)
3749 {
3750 	ehci_softc_t *sc = pipe->device->bus->hci_private;
3751 	struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
3752 	struct ehci_soft_islot *isp;
3753 
3754 	isp = &sc->sc_islots[epipe->sqh->islot];
3755 	ehci_close_pipe(pipe, isp->sqh);
3756 }
3757 
3758 Static void
3759 ehci_device_intr_done(usbd_xfer_handle xfer)
3760 {
3761 #define exfer EXFER(xfer)
3762 	struct ehci_xfer *ex = EXFER(xfer);
3763 	ehci_softc_t *sc = xfer->pipe->device->bus->hci_private;
3764 	struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3765 	ehci_soft_qtd_t *data, *dataend;
3766 	ehci_soft_qh_t *sqh;
3767 	usbd_status err;
3768 	int len, isread, endpt, s;
3769 
3770 	DPRINTFN(10, ("ehci_device_intr_done: xfer=%p, actlen=%d\n",
3771 	    xfer, xfer->actlen));
3772 
3773 	mutex_enter(&sc->sc_intrhead_lock);
3774 	if (xfer->pipe->repeat) {
3775 		ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
3776 
3777 		len = epipe->u.intr.length;
3778 		xfer->length = len;
3779 		endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3780 		isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3781 		usb_syncmem(&xfer->dmabuf, 0, len,
3782 		    isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
3783 		sqh = epipe->sqh;
3784 
3785 		err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
3786 		    &data, &dataend);
3787 		if (err) {
3788 			DPRINTFN(-1, ("ehci_device_intr_done: no memory\n"));
3789 			xfer->status = err;
3790 			mutex_exit(&sc->sc_intrhead_lock);
3791 			return;
3792 		}
3793 
3794 		/* Set up interrupt info. */
3795 		exfer->sqtdstart = data;
3796 		exfer->sqtdend = dataend;
3797 #ifdef DIAGNOSTIC
3798 		if (!exfer->isdone) {
3799 			printf("ehci_device_intr_done: not done, ex=%p\n",
3800 			    exfer);
3801 		}
3802 		exfer->isdone = 0;
3803 #endif
3804 
3805 		s = splusb();
3806 		ehci_set_qh_qtd(sqh, data); /* also does usb_syncmem(sqh) */
3807 		if (xfer->timeout && !sc->sc_bus.use_polling) {
3808 			callout_reset(&(xfer->timeout_handle),
3809 			    (mstohz(xfer->timeout)), (ehci_timeout), (xfer));
3810 		}
3811 		splx(s);
3812 
3813 		xfer->status = USBD_IN_PROGRESS;
3814 	} else if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
3815 		ehci_del_intr_list(sc, ex); /* remove from active list */
3816 		ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
3817 		endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3818 		isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3819 		usb_syncmem(&xfer->dmabuf, 0, xfer->length,
3820 		    isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
3821 	}
3822 	mutex_exit(&sc->sc_intrhead_lock);
3823 #undef exfer
3824 }
3825 
3826 /************************/
3827 
3828 Static usbd_status
3829 ehci_device_isoc_transfer(usbd_xfer_handle xfer)
3830 {
3831 	usbd_status err;
3832 
3833 	err = usb_insert_transfer(xfer);
3834 	if (err && err != USBD_IN_PROGRESS)
3835 		return err;
3836 
3837 	return ehci_device_isoc_start(xfer);
3838 }
3839 
3840 Static usbd_status
3841 ehci_device_isoc_start(usbd_xfer_handle xfer)
3842 {
3843 	struct ehci_pipe *epipe;
3844 	usbd_device_handle dev;
3845 	ehci_softc_t *sc;
3846 	struct ehci_xfer *exfer;
3847 	ehci_soft_itd_t *itd, *prev, *start, *stop;
3848 	usb_dma_t *dma_buf;
3849 	int i, j, k, frames, uframes, ufrperframe;
3850 	int s, trans_count, offs, total_length;
3851 	int frindex;
3852 
3853 	start = NULL;
3854 	prev = NULL;
3855 	itd = NULL;
3856 	trans_count = 0;
3857 	total_length = 0;
3858 	exfer = (struct ehci_xfer *) xfer;
3859 	sc = xfer->pipe->device->bus->hci_private;
3860 	dev = xfer->pipe->device;
3861 	epipe = (struct ehci_pipe *)xfer->pipe;
3862 
3863 	/*
3864 	 * To allow continuous transfers, above we start all transfers
3865 	 * immediately. However, we're still going to get usbd_start_next call
3866 	 * this when another xfer completes. So, check if this is already
3867 	 * in progress or not
3868 	 */
3869 
3870 	if (exfer->itdstart != NULL)
3871 		return USBD_IN_PROGRESS;
3872 
3873 	DPRINTFN(2, ("ehci_device_isoc_start: xfer %p len %d flags %d\n",
3874 			xfer, xfer->length, xfer->flags));
3875 
3876 	if (sc->sc_dying)
3877 		return USBD_IOERROR;
3878 
3879 	/*
3880 	 * To avoid complication, don't allow a request right now that'll span
3881 	 * the entire frame table. To within 4 frames, to allow some leeway
3882 	 * on either side of where the hc currently is.
3883 	 */
3884 	if ((1 << (epipe->pipe.endpoint->edesc->bInterval)) *
3885 			xfer->nframes >= (sc->sc_flsize - 4) * 8) {
3886 		printf("ehci: isoc descriptor requested that spans the entire frametable, too many frames\n");
3887 		return USBD_INVAL;
3888 	}
3889 
3890 #ifdef DIAGNOSTIC
3891 	if (xfer->rqflags & URQ_REQUEST)
3892 		panic("ehci_device_isoc_start: request\n");
3893 
3894 	if (!exfer->isdone)
3895 		printf("ehci_device_isoc_start: not done, ex = %p\n", exfer);
3896 	exfer->isdone = 0;
3897 #endif
3898 
3899 	/*
3900 	 * Step 1: Allocate and initialize itds, how many do we need?
3901 	 * One per transfer if interval >= 8 microframes, fewer if we use
3902 	 * multiple microframes per frame.
3903 	 */
3904 
3905 	i = epipe->pipe.endpoint->edesc->bInterval;
3906 	if (i > 16 || i == 0) {
3907 		/* Spec page 271 says intervals > 16 are invalid */
3908 		DPRINTF(("ehci_device_isoc_start: bInvertal %d invalid\n", i));
3909 		return USBD_INVAL;
3910 	}
3911 
3912 	ufrperframe = max(1, USB_UFRAMES_PER_FRAME / (1 << (i - 1)));
3913 	frames = (xfer->nframes + (ufrperframe - 1)) / ufrperframe;
3914 	uframes = USB_UFRAMES_PER_FRAME / ufrperframe;
3915 
3916 	if (frames == 0) {
3917 		DPRINTF(("ehci_device_isoc_start: frames == 0\n"));
3918 		return USBD_INVAL;
3919 	}
3920 
3921 	dma_buf = &xfer->dmabuf;
3922 	offs = 0;
3923 
3924 	for (i = 0; i < frames; i++) {
3925 		int froffs = offs;
3926 		itd = ehci_alloc_itd(sc);
3927 
3928 		if (prev != NULL) {
3929 			prev->itd.itd_next =
3930 			    htole32(itd->physaddr | EHCI_LINK_ITD);
3931 			usb_syncmem(&itd->dma,
3932 			    itd->offs + offsetof(ehci_itd_t, itd_next),
3933                 	    sizeof(itd->itd.itd_next), BUS_DMASYNC_POSTWRITE);
3934 
3935 			prev->xfer_next = itd;
3936 	    	} else {
3937 			start = itd;
3938 		}
3939 
3940 		/*
3941 		 * Step 1.5, initialize uframes
3942 		 */
3943 		for (j = 0; j < EHCI_ITD_NUFRAMES; j += uframes) {
3944 			/* Calculate which page in the list this starts in */
3945 			int addr = DMAADDR(dma_buf, froffs);
3946 			addr = EHCI_PAGE_OFFSET(addr);
3947 			addr += (offs - froffs);
3948 			addr = EHCI_PAGE(addr);
3949 			addr /= EHCI_PAGE_SIZE;
3950 
3951 			/* This gets the initial offset into the first page,
3952 			 * looks how far further along the current uframe
3953 			 * offset is. Works out how many pages that is.
3954 			 */
3955 
3956 			itd->itd.itd_ctl[j] = htole32 ( EHCI_ITD_ACTIVE |
3957 			    EHCI_ITD_SET_LEN(xfer->frlengths[trans_count]) |
3958 			    EHCI_ITD_SET_PG(addr) |
3959 			    EHCI_ITD_SET_OFFS(EHCI_PAGE_OFFSET(DMAADDR(dma_buf,offs))));
3960 
3961 			total_length += xfer->frlengths[trans_count];
3962 			offs += xfer->frlengths[trans_count];
3963 			trans_count++;
3964 
3965 			if (trans_count >= xfer->nframes) { /*Set IOC*/
3966 				itd->itd.itd_ctl[j] |= htole32(EHCI_ITD_IOC);
3967 				break;
3968 			}
3969 		}
3970 
3971 		/* Step 1.75, set buffer pointers. To simplify matters, all
3972 		 * pointers are filled out for the next 7 hardware pages in
3973 		 * the dma block, so no need to worry what pages to cover
3974 		 * and what to not.
3975 		 */
3976 
3977 		for (j = 0; j < EHCI_ITD_NBUFFERS; j++) {
3978 			/*
3979 			 * Don't try to lookup a page that's past the end
3980 			 * of buffer
3981 			 */
3982 			int page_offs = EHCI_PAGE(froffs + (EHCI_PAGE_SIZE * j));
3983 			if (page_offs >= dma_buf->block->size)
3984 				break;
3985 
3986 			long long page = DMAADDR(dma_buf, page_offs);
3987 			page = EHCI_PAGE(page);
3988 			itd->itd.itd_bufr[j] =
3989 			    htole32(EHCI_ITD_SET_BPTR(page));
3990 			itd->itd.itd_bufr_hi[j] =
3991 			    htole32(page >> 32);
3992 		}
3993 
3994 		/*
3995 		 * Other special values
3996 		 */
3997 
3998 		k = epipe->pipe.endpoint->edesc->bEndpointAddress;
3999 		itd->itd.itd_bufr[0] |= htole32(EHCI_ITD_SET_EP(UE_GET_ADDR(k)) |
4000 		    EHCI_ITD_SET_DADDR(epipe->pipe.device->address));
4001 
4002 		k = (UE_GET_DIR(epipe->pipe.endpoint->edesc->bEndpointAddress))
4003 		    ? 1 : 0;
4004 		j = UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize);
4005 		itd->itd.itd_bufr[1] |= htole32(EHCI_ITD_SET_DIR(k) |
4006 		    EHCI_ITD_SET_MAXPKT(UE_GET_SIZE(j)));
4007 
4008 		/* FIXME: handle invalid trans */
4009 		itd->itd.itd_bufr[2] |=
4010 		    htole32(EHCI_ITD_SET_MULTI(UE_GET_TRANS(j)+1));
4011 
4012 		usb_syncmem(&itd->dma,
4013 		    itd->offs + offsetof(ehci_itd_t, itd_next),
4014                     sizeof(ehci_itd_t),
4015 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
4016 
4017 		prev = itd;
4018 	} /* End of frame */
4019 
4020 	stop = itd;
4021 	stop->xfer_next = NULL;
4022 	exfer->isoc_len = total_length;
4023 
4024 	usb_syncmem(&exfer->xfer.dmabuf, 0, total_length,
4025 		BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4026 
4027 	/*
4028 	 * Part 2: Transfer descriptors have now been set up, now they must
4029 	 * be scheduled into the period frame list. Erk. Not wanting to
4030 	 * complicate matters, transfer is denied if the transfer spans
4031 	 * more than the period frame list.
4032 	 */
4033 
4034 	s = splusb();
4035 
4036 	/* Start inserting frames */
4037 	if (epipe->u.isoc.cur_xfers > 0) {
4038 		frindex = epipe->u.isoc.next_frame;
4039 	} else {
4040 		frindex = EOREAD4(sc, EHCI_FRINDEX);
4041 		frindex = frindex >> 3; /* Erase microframe index */
4042 		frindex += 2;
4043 	}
4044 
4045 	if (frindex >= sc->sc_flsize)
4046 		frindex &= (sc->sc_flsize - 1);
4047 
4048 	/* What's the frame interval? */
4049 	i = (1 << (epipe->pipe.endpoint->edesc->bInterval - 1));
4050 	if (i / USB_UFRAMES_PER_FRAME == 0)
4051 		i = 1;
4052 	else
4053 		i /= USB_UFRAMES_PER_FRAME;
4054 
4055 	itd = start;
4056 	for (j = 0; j < frames; j++) {
4057 		if (itd == NULL)
4058 			panic("ehci: unexpectedly ran out of isoc itds, isoc_start\n");
4059 
4060 		itd->itd.itd_next = sc->sc_flist[frindex];
4061 		if (itd->itd.itd_next == 0)
4062 			/* FIXME: frindex table gets initialized to NULL
4063 			 * or EHCI_NULL? */
4064 			itd->itd.itd_next = EHCI_NULL;
4065 
4066 		usb_syncmem(&itd->dma,
4067 		    itd->offs + offsetof(ehci_itd_t, itd_next),
4068                     sizeof(itd->itd.itd_next),
4069 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
4070 
4071 		sc->sc_flist[frindex] = htole32(EHCI_LINK_ITD | itd->physaddr);
4072 
4073 		usb_syncmem(&sc->sc_fldma,
4074 		    sizeof(ehci_link_t) * frindex,
4075                     sizeof(ehci_link_t),
4076 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
4077 
4078 		itd->u.frame_list.next = sc->sc_softitds[frindex];
4079 		sc->sc_softitds[frindex] = itd;
4080 		if (itd->u.frame_list.next != NULL)
4081 			itd->u.frame_list.next->u.frame_list.prev = itd;
4082 		itd->slot = frindex;
4083 		itd->u.frame_list.prev = NULL;
4084 
4085 		frindex += i;
4086 		if (frindex >= sc->sc_flsize)
4087 			frindex -= sc->sc_flsize;
4088 
4089 		itd = itd->xfer_next;
4090 	}
4091 
4092 	epipe->u.isoc.cur_xfers++;
4093 	epipe->u.isoc.next_frame = frindex;
4094 
4095 	exfer->itdstart = start;
4096 	exfer->itdend = stop;
4097 	exfer->sqtdstart = NULL;
4098 	exfer->sqtdstart = NULL;
4099 
4100 	mutex_enter(&sc->sc_intrhead_lock);
4101 	ehci_add_intr_list(sc, exfer);
4102 	mutex_exit(&sc->sc_intrhead_lock);
4103 	xfer->status = USBD_IN_PROGRESS;
4104 	xfer->done = 0;
4105 	splx(s);
4106 
4107 	if (sc->sc_bus.use_polling) {
4108 		printf("Starting ehci isoc xfer with polling. Bad idea?\n");
4109 		ehci_waitintr(sc, xfer);
4110 	}
4111 
4112 	return USBD_IN_PROGRESS;
4113 }
4114 
4115 Static void
4116 ehci_device_isoc_abort(usbd_xfer_handle xfer)
4117 {
4118 	DPRINTFN(1, ("ehci_device_isoc_abort: xfer = %p\n", xfer));
4119 	ehci_abort_isoc_xfer(xfer, USBD_CANCELLED);
4120 }
4121 
4122 Static void
4123 ehci_device_isoc_close(usbd_pipe_handle pipe)
4124 {
4125 	DPRINTFN(1, ("ehci_device_isoc_close: nothing in the pipe to free?\n"));
4126 }
4127 
4128 Static void
4129 ehci_device_isoc_done(usbd_xfer_handle xfer)
4130 {
4131 	struct ehci_xfer *exfer;
4132 	ehci_softc_t *sc;
4133 	struct ehci_pipe *epipe;
4134 	int s;
4135 
4136 	exfer = EXFER(xfer);
4137 	sc = xfer->pipe->device->bus->hci_private;
4138 	epipe = (struct ehci_pipe *) xfer->pipe;
4139 
4140 	s = splusb();
4141 	epipe->u.isoc.cur_xfers--;
4142 	mutex_enter(&sc->sc_intrhead_lock);
4143 	if (xfer->status != USBD_NOMEM && ehci_active_intr_list(exfer)) {
4144 		ehci_del_intr_list(sc, exfer);
4145 		ehci_rem_free_itd_chain(sc, exfer);
4146 	}
4147 	mutex_exit(&sc->sc_intrhead_lock);
4148 	splx(s);
4149 
4150 	usb_syncmem(&xfer->dmabuf, 0, xfer->length, BUS_DMASYNC_POSTWRITE |
4151                     BUS_DMASYNC_POSTREAD);
4152 
4153 }
4154