1 /*
2 * from 9front ../port/usbxhci.c
3 */
4
5 #include "u.h"
6 #include "../port/lib.h"
7 #include "mem.h"
8 #include "dat.h"
9 #include "fns.h"
10 #include "io.h"
11 #include "pci.h"
12 #include "../port/error.h"
13 #include "usb.h"
14
15 extern void dmaflush(int, void*, ulong);
16
17 enum {
18 /* Capability Registers */
19 CAPLENGTH = 0x00/4, // 1
20 HCIVERSION = 0x02/4, // 2
21 HCSPARAMS1 = 0x04/4,
22 HCSPARAMS2 = 0x08/4,
23 HCSPARAMS3 = 0x0C/4,
24
25 HCCPARAMS = 0x10/4,
26 AC64 = 1<<0,
27 BNC = 1<<1,
28 CSZ = 1<<2,
29 PPC = 1<<3,
30 PIND = 1<<4,
31 LHRC = 1<<5,
32 LTC = 1<<6,
33 NSS = 1<<7,
34
35 DBOFF = 0x14/4,
36 RTSOFF = 0x18/4,
37
38 HCCPARAMS2 = 0x1C/4,
39
40 /* Operational Registers */
41 USBCMD = 0x00/4, /* USB Command Register */
42 RUNSTOP = 1<<0, /* Run/Stop - RW */
43 HCRST = 1<<1, /* Host Controller Reset - RW */
44 INTE = 1<<2, /* Interrupter Enable - RW */
45 HSEE = 1<<3, /* Host System Error Enable - RW */
46 LHCRST = 1<<7, /* Light Host Controller Reset - RO/RW */
47 CSS = 1<<8, /* Controller Save State - RW */
48 CRS = 1<<9, /* Controller Restore State - RW */
49 EWE = 1<<10, /* Enable Wrap Event - RW */
50 EU3S = 1<<11, /* Enable U3 MFINDEX Stop - RW */
51
52 USBSTS = 0x04/4, /* USB Status Register */
53 HCH = 1<<0, /* HCHalted - RO */
54 HSE = 1<<2, /* Host System Error - RW1C */
55 EINT = 1<<3, /* Event Interrupt - RW1C */
56 PCD = 1<<4, /* Port Change Detect - RW1C */
57 SSS = 1<<8, /* Save State Status - RO */
58 RSS = 1<<9, /* Restore State Status - RO */
59 SRE = 1<<10, /* Save/Restore Error - RW1C */
60 CNR = 1<<11, /* Controller Not Ready - RO */
61 HCE = 1<<12, /* Host Controller Error - RO */
62
63 PAGESIZE = 0x08/4, /* Page Size - RO */
64
65 DNCTRL = 0x14/4, /* Device Notification Control Register - RW */
66
67 CRCR = 0x18/4, /* Command Ring Control Register - RW */
68 RCS = 1<<0, /* Ring Cycle State - RW */
69 CS = 1<<1, /* Command Stop - RW1S */
70 CA = 1<<2, /* Command Abort - RW1S */
71 CRR = 1<<3, /* Command Ring Running - RO */
72
73 DCBAAP = 0x30/4, // 8
74
75 CONFIG = 0x38/4, /* Configure Register (MaxSlotEn[7:0]) */
76
77 /* Port Register Set */
78 PORTSC = 0x00/4, /* Port status and Control Register */
79 CCS = 1<<0, /* Current Connect Status - ROS */
80 PED = 1<<1, /* Port Enable/Disabled - RW1CS */
81 OCA = 1<<3, /* Over-current Active - RO */
82 PR = 1<<4, /* Port Reset - RW1S */
83 PLS = 15<<5, /* Port Link State - RWS */
84 PP = 1<<9, /* Port Power - RWS */
85 PS = 15<<10, /* Port Speed - ROS */
86 PIC = 3<<14, /* Port Indicator Control - RWS */
87 LWS = 1<<16, /* Port Link Write Strobe - RW */
88 CSC = 1<<17, /* Connect Status Change - RW1CS */
89 PEC = 1<<18, /* Port Enabled/Disabled Change - RW1CS */
90 WRC = 1<<19, /* Warm Port Reset Change - RW1CS */
91 OCC = 1<<20, /* Over-current Change - RW1CS */
92 PRC = 1<<21, /* Port Reset Change - RW1CS */
93 PLC = 1<<22, /* Port Link State Change - RW1CS */
94 CEC = 1<<23, /* Port Config Error Change - RW1CS */
95 CAS = 1<<24, /* Cold Attach Status - RO */
96 WCE = 1<<25, /* Wake on Connect Enable - RWS */
97 WDE = 1<<26, /* Wake on Disconnect Enable - RWS */
98 WOE = 1<<27, /* Wake on Over-current Enable - RWS */
99 DR = 1<<30, /* Device Removable - RO */
100 WPR = 1<<31, /* Warm Port Reset - RW1S */
101
102 PORTPMSC = 0x04/4,
103 PORTLI = 0x08/4,
104
105 /* Host Controller Runtime Register */
106 MFINDEX = 0x0000/4, /* Microframe Index */
107 IR0 = 0x0020/4, /* Interrupt Register Set 0 */
108
109 /* Interrupter Registers */
110 IMAN = 0x00/4, /* Interrupter Management */
111 IMOD = 0x04/4, /* Interrupter Moderation */
112 ERSTSZ = 0x08/4, /* Event Ring Segment Table Size */
113 ERSTBA = 0x10/4, /* Event Ring Segment Table Base Address */
114 ERDP = 0x18/4, /* Event Ring Dequeue Pointer */
115
116 /* TRB flags */
117 TR_ENT = 1<<1,
118 TR_ISP = 1<<2,
119 TR_NS = 1<<3,
120 TR_CH = 1<<4,
121 TR_IOC = 1<<5,
122 TR_IDT = 1<<6,
123 TR_BEI = 1<<9,
124
125 /* TRB types */
126 TR_RESERVED = 0<<10,
127 TR_NORMAL = 1<<10,
128 TR_SETUPSTAGE = 2<<10,
129 TR_DATASTAGE = 3<<10,
130 TR_STATUSSTAGE = 4<<10,
131 TR_ISOCH = 5<<10,
132 TR_LINK = 6<<10,
133 TR_EVENTDATA = 7<<10,
134 TR_NOOP = 8<<10,
135
136 CR_ENABLESLOT = 9<<10,
137 CR_DISABLESLOT = 10<<10,
138 CR_ADDRESSDEV = 11<<10,
139 CR_CONFIGEP = 12<<10,
140 CR_EVALCTX = 13<<10,
141 CR_RESETEP = 14<<10,
142 CR_STOPEP = 15<<10,
143 CR_SETTRDQP = 16<<10,
144 CR_RESETDEV = 17<<10,
145 CR_FORCECMD = 18<<10,
146 CR_NEGBW = 19<<10,
147 CR_SETLAT = 20<<10,
148 CR_GETPORTBW = 21<<10,
149 CR_FORCEHDR = 22<<10,
150 CR_NOOP = 23<<10,
151
152 ER_TRANSFER = 32<<10,
153 ER_CMDCOMPL = 33<<10,
154 ER_PORTSC = 34<<10,
155 ER_BWREQ = 35<<10,
156 ER_DOORBELL = 36<<10,
157 ER_HCE = 37<<10,
158 ER_DEVNOTE = 38<<10,
159 ER_MFINDEXWRAP = 39<<10,
160 };
161
162 typedef struct Ctlr Ctlr;
163 typedef struct Wait Wait;
164 typedef struct Ring Ring;
165 typedef struct Slot Slot;
166 typedef struct Epio Epio;
167 typedef struct Port Port;
168
169 struct Wait
170 {
171 Wait *next;
172 Ring *ring;
173 u32int *td;
174 u32int er[4];
175 Rendez *z;
176 };
177
178 struct Ring
179 {
180 int id;
181
182 Slot *slot;
183
184 u32int *base;
185
186 u32int mask;
187 u32int shift;
188
189 u32int rp;
190 u32int wp;
191
192 u32int *ctx;
193 u32int *doorbell;
194
195 int stopped;
196
197 Wait *pending;
198 Lock;
199 };
200
201 struct Slot
202 {
203 int id;
204
205 int confval; // bConfigurationValue of SET_CONFIGURATION
206 int iface; // bInterfaceNumber of SET_INTERFACE
207 int altc; // bAlternateSetting of SET_INTERFACE
208
209 Ctlr *ctlr;
210 Udev *dev;
211
212 u32int *ibase;
213 u32int *obase;
214
215 /* endpoint rings */
216 int nep;
217 Ring epr[32];
218 };
219
220 struct Port
221 {
222 char spec[4];
223 int proto;
224
225 u32int *reg;
226 };
227
228 struct Ctlr
229 {
230 Pcidev *pcidev;
231
232 u32int *mmio;
233
234 u32int *opr; /* operational registers */
235 u32int *rts; /* runtime registers */
236 u32int *dba; /* doorbell array */
237
238 u64int *dcba; /* device context base array */
239
240 u64int *sba; /* scratchpad buffer array */
241 void *sbp; /* scratchpad buffer pages */
242
243 u32int *erst[1]; /* event ring segment table */
244 Ring er[1]; /* event ring segment */
245 Ring cr[1]; /* command ring segment */
246 QLock cmdlock;
247
248 u32int µframe;
249
250 QLock slotlock;
251 Slot **slot; /* slots by slot id */
252 Port *port;
253
254 u32int hccparams;
255
256 int csz;
257 int pagesize;
258 int nscratch;
259 int nintrs;
260 int nslots;
261
262 Rendez recover;
263 void *active;
264 uintptr base;
265 };
266
267 struct Epio
268 {
269 QLock;
270
271 Ring *ring;
272 Block *b;
273
274 /* iso */
275 u32int frame;
276 u32int period;
277 u32int incr;
278 u32int tdsz;
279
280 int nleft;
281 };
282
283 static char Ebadlen[] = "bad usb request length";
284 static char Enotconfig[] = "usb endpoint not configured";
285 static char Erecover[] = "xhci controller needs reset";
286
287 static char*
288 ctlrcmd(Ctlr *ctlr, u32int c, u32int s, u64int p, u32int *er);
289
290 static void
setrptr(u32int * reg,u64int pa)291 setrptr(u32int *reg, u64int pa)
292 {
293 coherence();
294 reg[0] = pa;
295 reg[1] = pa>>32;
296 }
297
298 static u32int
frame(Ctlr * ctlr)299 µframe(Ctlr *ctlr)
300 {
301 u32int µ;
302 do {
303 µ = (ctlr->rts[MFINDEX] & (1<<14)-1) |
304 (ctlr->µframe & ~((1<<14)-1));
305 } while((int)(µ - ctlr->µframe) < 0);
306 return µ;
307 }
308
309 static void
freering(Ring * r)310 freering(Ring *r)
311 {
312 if(r == nil)
313 return;
314 if(r->base != nil){
315 dmaflush(0, r->base, 4*4<<r->shift);
316 free(r->base);
317 }
318 memset(r, 0, sizeof(*r));
319 }
320
321 static Ring*
initring(Ring * r,int shift)322 initring(Ring *r, int shift)
323 {
324 r->id = 0;
325 r->ctx = nil;
326 r->slot = nil;
327 r->doorbell = nil;
328 r->pending = nil;
329 r->stopped = 0;
330 r->shift = shift;
331 r->mask = (1<<shift)-1;
332 r->rp = r->wp = 0;
333 r->base = mallocalign(4*4<<shift, 64, 0, 64*1024);
334 if(r->base == nil){
335 freering(r);
336 error(Enomem);
337 }
338 dmaflush(1, r->base, 4*4<<shift);
339 return r;
340 }
341
342 static void
flushring(Ring * r)343 flushring(Ring *r)
344 {
345 Rendez *z;
346 Wait *w;
347
348 while((w = r->pending) != nil){
349 r->pending = w->next;
350 w->next = nil;
351 if((z = w->z) != nil){
352 w->z = nil;
353 wakeup(z);
354 }
355 }
356 }
357
358 static u64int
resetring(Ring * r)359 resetring(Ring *r)
360 {
361 u64int pa;
362
363 ilock(r);
364 flushring(r);
365 r->rp = r->wp;
366 pa = PCIWADDR(&r->base[4*(r->wp & r->mask)]) | ((~r->wp>>r->shift) & 1);
367 iunlock(r);
368
369 return pa;
370 }
371
372 static u32int*
xecp(Ctlr * ctlr,uchar id,u32int * p)373 xecp(Ctlr *ctlr, uchar id, u32int *p)
374 {
375 u32int x, *e;
376
377 e = &ctlr->mmio[ctlr->pcidev->mem[0].size/4];
378 if(p == nil){
379 p = ctlr->mmio;
380 x = ctlr->hccparams>>16;
381 } else {
382 assert(p < e);
383 x = (*p>>8) & 255;
384 }
385 while(x != 0){
386 p += x;
387 if(p >= e)
388 break;
389 x = *p;
390 if((x & 255) == id)
391 return p;
392 x >>= 8;
393 x &= 255;
394 }
395 return nil;
396 }
397
398 static void
handoff(Ctlr * ctlr)399 handoff(Ctlr *ctlr)
400 {
401 u32int *r;
402 int i;
403
404 if((r = xecp(ctlr, 1, nil)) == nil)
405 return;
406 if(getconf("*noxhcihandoff") == nil){
407 r[0] |= 1<<24; /* request ownership */
408 for(i = 0; (r[0] & (1<<16)) != 0 && i<100; i++)
409 tsleep(&up->sleep, return0, nil, 10);
410 }
411 /* disable SMI interrupts */
412 r[1] &= 7<<1 | 255<<5 | 7<<17 | 7<<29;
413
414 /* clear BIOS ownership in case of timeout */
415 r[0] &= ~(1<<16);
416 }
417
418 static void
shutdown(Hci * hp)419 shutdown(Hci *hp)
420 {
421 Ctlr *ctlr = hp->aux;
422 int i;
423
424 ctlr->opr[USBCMD] = 0;
425 for(i=0; (ctlr->opr[USBSTS] & HCH) == 0 && i < 10; i++)
426 delay(10);
427 pciintrdisable(ctlr->pcidev->tbdf, hp->interrupt, hp);
428 pcidisable(ctlr->pcidev);
429 }
430
431 static void
release(Ctlr * ctlr)432 release(Ctlr *ctlr)
433 {
434 int i;
435
436 freering(ctlr->cr);
437 for(i=0; i<nelem(ctlr->er); i++){
438 freering(&ctlr->er[i]);
439 free(ctlr->erst[i]);
440 ctlr->erst[i] = nil;
441 }
442 free(ctlr->port), ctlr->port = nil;
443 free(ctlr->slot), ctlr->slot = nil;
444 free(ctlr->dcba), ctlr->dcba = nil;
445 free(ctlr->sba), ctlr->sba = nil;
446 if(ctlr->sbp != nil){
447 dmaflush(0, ctlr->sbp, ctlr->nscratch*ctlr->pagesize);
448 free(ctlr->sbp);
449 ctlr->sbp = nil;
450 }
451 }
452
453 static void recover(void *arg);
454
455 static void
init(Hci * hp)456 init(Hci *hp)
457 {
458 Ctlr *ctlr;
459 Port *pp;
460 u32int *x;
461 uchar *p;
462 int i, j;
463
464 ctlr = hp->aux;
465 pcienable(ctlr->pcidev);
466 if(ctlr->mmio[CAPLENGTH] == -1){
467 pcidisable(ctlr->pcidev);
468 error("controller vanished");
469 }
470
471 ctlr->opr = &ctlr->mmio[(ctlr->mmio[CAPLENGTH]&0xFF)/4];
472 ctlr->dba = &ctlr->mmio[ctlr->mmio[DBOFF]/4];
473 ctlr->rts = &ctlr->mmio[ctlr->mmio[RTSOFF]/4];
474
475 ctlr->hccparams = ctlr->mmio[HCCPARAMS];
476 handoff(ctlr);
477 xhcireset(BUSBNO(hp->tbdf)<<20 | BUSDNO(hp->tbdf)<<15 | BUSFNO(hp->tbdf)<<12);
478
479 for(i=0; (ctlr->opr[USBSTS] & CNR) != 0 && i<100; i++)
480 tsleep(&up->sleep, return0, nil, 10);
481 if(i == 100)
482 print("%s: controller not ready, status %ux\n", hp->type, ctlr->opr[USBSTS]);
483
484 ctlr->opr[USBCMD] = HCRST;
485 delay(1);
486 for(i=0; (ctlr->opr[USBSTS] & (CNR|HCH)) != HCH && i<100; i++)
487 tsleep(&up->sleep, return0, nil, 10);
488 if(i == 100)
489 print("%s: controller not halted, status %ux\n", hp->type, ctlr->opr[USBSTS]);
490
491 pcisetbme(ctlr->pcidev);
492 pciintrenable(ctlr->pcidev->tbdf, hp->interrupt, hp);
493
494 if(waserror()){
495 shutdown(hp);
496 release(ctlr);
497 nexterror();
498 }
499
500 ctlr->csz = (ctlr->hccparams & CSZ) != 0;
501 ctlr->pagesize = (ctlr->opr[PAGESIZE] & 0xFFFF) << 12;
502
503 ctlr->nscratch = (ctlr->mmio[HCSPARAMS2] >> 27) & 0x1F | (ctlr->mmio[HCSPARAMS2] >> 16) & 0x3E0;
504 ctlr->nintrs = (ctlr->mmio[HCSPARAMS1] >> 8) & 0x7FF;
505 ctlr->nslots = (ctlr->mmio[HCSPARAMS1] >> 0) & 0xFF;
506
507 hp->highspeed = 1;
508 hp->superspeed = 0;
509 hp->nports = (ctlr->mmio[HCSPARAMS1] >> 24) & 0xFF;
510 ctlr->port = malloc(hp->nports * sizeof(Port));
511 if(ctlr->port == nil)
512 error(Enomem);
513 for(i=0; i<hp->nports; i++)
514 ctlr->port[i].reg = &ctlr->opr[0x400/4 + i*4];
515
516 x = nil;
517 while((x = xecp(ctlr, 2, x)) != nil){
518 i = x[2]&255;
519 j = (x[2]>>8)&255;
520 while(j--){
521 if(i < 1 || i > hp->nports)
522 break;
523 pp = &ctlr->port[i-1];
524 pp->proto = x[0]>>16;
525 memmove(pp->spec, &x[1], 4);
526 if(memcmp(pp->spec, "USB ", 4) == 0 && pp->proto >= 0x0300)
527 hp->superspeed |= 1<<(i-1);
528 i++;
529 }
530 }
531
532 ctlr->slot = malloc((1+ctlr->nslots)*sizeof(ctlr->slot[0]));
533 ctlr->dcba = mallocalign((1+ctlr->nslots)*sizeof(ctlr->dcba[0]), 64, 0, ctlr->pagesize);
534 if(ctlr->slot == nil || ctlr->dcba == nil)
535 error(Enomem);
536 if(ctlr->nscratch != 0){
537 ctlr->sba = mallocalign(ctlr->nscratch*8, 64, 0, ctlr->pagesize);
538 ctlr->sbp = mallocalign(ctlr->nscratch*ctlr->pagesize, ctlr->pagesize, 0, 0);
539 if(ctlr->sba == nil || ctlr->sbp == nil)
540 error(Enomem);
541 for(i=0, p = ctlr->sbp; i<ctlr->nscratch; i++, p += ctlr->pagesize){
542 memset(p, 0, ctlr->pagesize);
543 ctlr->sba[i] = PCIWADDR(p);
544 }
545 dmaflush(1, ctlr->sbp, ctlr->nscratch*ctlr->pagesize);
546 dmaflush(1, ctlr->sba, ctlr->nscratch*8);
547 ctlr->dcba[0] = PCIWADDR(ctlr->sba);
548 } else {
549 ctlr->dcba[0] = 0;
550 }
551 for(i=1; i<=ctlr->nslots; i++)
552 ctlr->dcba[i] = 0;
553
554 ctlr->opr[CONFIG] = (ctlr->opr[CONFIG] & 0xFFFFFC00) | ctlr->nslots; /* MaxSlotsEn */
555
556 dmaflush(1, ctlr->dcba, (1+ctlr->nslots)*sizeof(ctlr->dcba[0]));
557 setrptr(&ctlr->opr[DCBAAP], PCIWADDR(ctlr->dcba));
558
559 initring(ctlr->cr, 8); /* 256 entries */
560 ctlr->cr->id = 0;
561 ctlr->cr->doorbell = &ctlr->dba[0];
562 setrptr(&ctlr->opr[CRCR], resetring(ctlr->cr));
563
564 for(i=0; i<ctlr->nintrs; i++){
565 u32int *irs = &ctlr->rts[IR0 + i*8];
566
567 if(i >= nelem(ctlr->er)){
568 irs[ERSTSZ] = 0; /* disable ring */
569 irs[IMAN] = 1;
570 irs[IMOD] = 0;
571 setrptr(&irs[ERSTBA], 0);
572 setrptr(&irs[ERDP], 0);
573 continue;
574 }
575
576 /* allocate and link into event ring segment table */
577 initring(&ctlr->er[i], 8); /* 256 entries */
578 ctlr->erst[i] = mallocalign(4*4, 64, 0, 0);
579 if(ctlr->erst[i] == nil)
580 error(Enomem);
581 *((u64int*)ctlr->erst[i]) = PCIWADDR(ctlr->er[i].base);
582 ctlr->erst[i][2] = ctlr->er[i].mask+1;
583 ctlr->erst[i][3] = 0;
584 dmaflush(1, ctlr->erst[i], 4*4);
585
586 irs[ERSTSZ] = 1; /* just one segment */
587 irs[IMAN] = 3;
588 irs[IMOD] = 0;
589 setrptr(&irs[ERSTBA], PCIWADDR(ctlr->erst[i]));
590 setrptr(&irs[ERDP], PCIWADDR(ctlr->er[i].base) | (1<<3));
591 }
592 poperror();
593
594 ctlr->µframe = 0;
595 ctlr->opr[USBSTS] = ctlr->opr[USBSTS] & (HSE|EINT|PCD|SRE);
596 coherence();
597
598 ctlr->opr[USBCMD] = RUNSTOP|INTE|HSEE|EWE;
599 for(i=0; (ctlr->opr[USBSTS] & (CNR|HCH)) != 0 && i<100; i++)
600 tsleep(&up->sleep, return0, nil, 10);
601
602 kproc("xhcirecover", recover, hp);
603 }
604
605 static int
needrecover(void * arg)606 needrecover(void *arg)
607 {
608 Ctlr *ctlr = arg;
609 return ctlr->er->stopped ||
610 (ctlr->opr[USBSTS] & (HCH|HCE|HSE)) != 0;
611 }
612
613 static void
recover(void * arg)614 recover(void *arg)
615 {
616 Hci *hp = arg;
617 Ctlr *ctlr = hp->aux;
618
619 while(waserror())
620 ;
621 while(!needrecover(ctlr))
622 tsleep(&ctlr->recover, needrecover, ctlr, 10);
623 shutdown(hp);
624
625 /*
626 * flush all transactions and wait until all devices have
627 * been detached by usbd.
628 */
629 for(;;){
630 int i, j, active;
631
632 ilock(ctlr->cr);
633 ctlr->cr->stopped = 1;
634 flushring(ctlr->cr);
635 iunlock(ctlr->cr);
636
637 active = 0;
638 qlock(&ctlr->slotlock);
639 for(i=1; i<=ctlr->nslots; i++){
640 Slot *slot = ctlr->slot[i];
641 if(slot == nil)
642 continue;
643 active++;
644 for(j=0; j < slot->nep; j++){
645 Ring *ring = &slot->epr[j];
646 if(ring->base == nil)
647 continue;
648 ilock(ring);
649 ring->stopped = 1;
650 flushring(ring);
651 iunlock(ring);
652 }
653 }
654 qunlock(&ctlr->slotlock);
655 if(active == 0)
656 break;
657
658 tsleep(&up->sleep, return0, nil, 100);
659 }
660
661 qlock(&ctlr->slotlock);
662 qlock(&ctlr->cmdlock);
663
664 release(ctlr);
665 if(waserror()) {
666 print("xhci recovery failed: %s\n", up->errstr);
667 } else {
668 init(hp);
669 poperror();
670 }
671
672 qunlock(&ctlr->cmdlock);
673 qunlock(&ctlr->slotlock);
674
675 pexit("", 1);
676 }
677
678 static void
dump(Hci *)679 dump(Hci *)
680 {
681 }
682
683 static void
queuetd(Ring * r,u32int c,u32int s,u64int p,Wait * w)684 queuetd(Ring *r, u32int c, u32int s, u64int p, Wait *w)
685 {
686 u32int *td, x;
687
688 x = r->wp++;
689 if((x & r->mask) == r->mask){
690 td = r->base + 4*(x & r->mask);
691 *(u64int*)td = PCIWADDR(r->base);
692 td[2] = 0;
693 td[3] = ((~x>>r->shift)&1) | (1<<1) | TR_LINK;
694 dmaflush(1, td, 4*4);
695 x = r->wp++;
696 }
697 td = r->base + 4*(x & r->mask);
698 if(w != nil){
699 w->er[0] = w->er[1] = w->er[2] = w->er[3] = 0;
700 w->ring = r;
701 w->td = td;
702 w->z = &up->sleep;
703
704 ilock(r);
705 w->next = r->pending;
706 r->pending = w;
707 iunlock(r);
708 }
709 coherence();
710 *(u64int*)td = p;
711 td[2] = s;
712 td[3] = ((~x>>r->shift)&1) | c;
713 dmaflush(1, td, 4*4);
714 }
715
716 static char *ccerrtab[] = {
717 [2] "Data Buffer Error",
718 [3] "Babble Detected Error",
719 [4] "USB Transaction Error",
720 [5] "TRB Error",
721 [6] "Stall Error",
722 [7] "Resume Error",
723 [8] "Bandwidth Error",
724 [9] "No Slots Available",
725 [10] "Invalid Stream Type",
726 [11] "Slot Not Enabled",
727 [12] "Endpoint Not Enabled",
728 [13] "Short Packet",
729 [14] "Ring Underrun",
730 [15] "Ring Overrun",
731 [16] "VF Event Ring Full",
732 [17] "Parameter Error",
733 [18] "Bandwidth Overrun Error",
734 [19] "Context State Error",
735 [20] "No Ping Response",
736 [21] "Event Ring Full",
737 [22] "Incompatible Device",
738 [23] "Missed Service Error",
739 [24] "Command Ring Stopped",
740 [25] "Command Aborted",
741 [26] "Stopped",
742 [27] "Stopped - Length Invalid",
743 [29] "Max Exit Latency Too Large",
744 [31] "Isoch Buffer Overrun",
745 [32] "Event Lost Error",
746 [33] "Undefined Error",
747 [34] "Invalid Stream ID",
748 [35] "Secondary Bandwidth Error",
749 [36] "Split Transaction Error",
750 };
751
752 static char*
ccerrstr(u32int cc)753 ccerrstr(u32int cc)
754 {
755 char *s;
756
757 if(cc == 1 || cc == 13)
758 return nil;
759 if(cc < nelem(ccerrtab) && ccerrtab[cc] != nil)
760 s = ccerrtab[cc];
761 else
762 s = "???";
763 return s;
764 }
765
766 static int
waitdone(void * a)767 waitdone(void *a)
768 {
769 return ((Wait*)a)->z == nil;
770 }
771
772 static char*
waittd(Ctlr * ctlr,Wait * w,int tmout)773 waittd(Ctlr *ctlr, Wait *w, int tmout)
774 {
775 Ring *r = w->ring;
776
777 coherence();
778 *r->doorbell = r->id;
779
780 while(waserror()){
781 if(r->stopped) {
782 ctlr->er->stopped = 1;
783 wakeup(&ctlr->recover);
784
785 /* wait for rescue */
786 tmout = 0;
787 continue;
788 }
789
790 if(r == ctlr->cr)
791 ctlr->opr[CRCR] |= CA;
792 else
793 ctlrcmd(ctlr, CR_STOPEP | (r->id<<16) | (r->slot->id<<24), 0, 0, nil);
794 r->stopped = 1;
795
796 /* time to abort the transaction */
797 tmout = 5000;
798 }
799 if(tmout > 0){
800 tsleep(&up->sleep, waitdone, w, tmout);
801 if(!waitdone(w))
802 error("timed out");
803 } else {
804 while(!waitdone(w))
805 sleep(&up->sleep, waitdone, w);
806 }
807 poperror();
808 return ccerrstr(w->er[2]>>24);
809 }
810
811 static char*
ctlrcmd(Ctlr * ctlr,u32int c,u32int s,u64int p,u32int * er)812 ctlrcmd(Ctlr *ctlr, u32int c, u32int s, u64int p, u32int *er)
813 {
814 Wait w[1];
815 char *err;
816
817 qlock(&ctlr->cmdlock);
818 if(needrecover(ctlr)){
819 qunlock(&ctlr->cmdlock);
820 return Erecover;
821 }
822 ctlr->cr->stopped = 0;
823 queuetd(ctlr->cr, c, s, p, w);
824 err = waittd(ctlr, w, 5000);
825 qunlock(&ctlr->cmdlock);
826
827 if(er != nil)
828 memmove(er, w->er, 4*4);
829
830 return err;
831 }
832
833 static void
completering(Ring * r,u32int * er)834 completering(Ring *r, u32int *er)
835 {
836 Wait *w, **wp;
837 u32int *td, x;
838 u64int pa;
839
840 pa = (*(u64int*)er) & ~15ULL;
841 ilock(r);
842
843 for(x = r->rp; (int)(r->wp - x) > 0;){
844 td = &r->base[4*(x++ & r->mask)];
845 if((u64int)PCIWADDR(td) == pa){
846 r->rp = x;
847 break;
848 }
849 }
850
851 wp = &r->pending;
852 while(w = *wp){
853 if((u64int)PCIWADDR(w->td) == pa){
854 Rendez *z = w->z;
855
856 memmove(w->er, er, 4*4);
857 *wp = w->next;
858 w->next = nil;
859
860 if(z != nil){
861 w->z = nil;
862 wakeup(z);
863 }
864 break;
865 } else {
866 wp = &w->next;
867 }
868 }
869
870 iunlock(r);
871 }
872
873 static void
interrupt(Ureg *,void * arg)874 interrupt(Ureg*, void *arg)
875 {
876 Hci *hp = arg;
877 Ctlr *ctlr = hp->aux;
878 Ring *ring = ctlr->er;
879 Slot *slot;
880 u32int *irs, *td, x;
881
882 if(ring->base == nil)
883 return;
884
885 irs = &ctlr->rts[IR0];
886 x = irs[IMAN];
887 if(x & 1) irs[IMAN] = x & 3;
888
889 for(x = ring->rp;; x=++ring->rp){
890 td = ring->base + 4*(x & ring->mask);
891 dmaflush(0, td, 4*4);
892
893 if((((x>>ring->shift)^td[3])&1) == 0)
894 break;
895
896 switch(td[3] & 0xFC00){
897 case ER_CMDCOMPL:
898 completering(ctlr->cr, td);
899 break;
900 case ER_TRANSFER:
901 x = td[3]>>24;
902 if(x == 0 || x > ctlr->nslots)
903 break;
904 slot = ctlr->slot[x];
905 if(slot == nil)
906 break;
907 completering(&slot->epr[(td[3]>>16)-1&31], td);
908 break;
909 case ER_MFINDEXWRAP:
910 ctlr->µframe = (ctlr->rts[MFINDEX] & (1<<14)-1) |
911 (ctlr->µframe+(1<<14) & ~((1<<14)-1));
912 break;
913 case ER_HCE:
914 iprint("xhci: host controller error: %ux %ux %ux %ux\n",
915 td[0], td[1], td[2], td[3]);
916 ctlr->er->stopped = 1;
917 wakeup(&ctlr->recover);
918 return;
919 case ER_PORTSC:
920 break;
921 case ER_BWREQ:
922 case ER_DOORBELL:
923 case ER_DEVNOTE:
924 default:
925 iprint("xhci: event %ud: %ux %ux %ux %ux\n",
926 x, td[0], td[1], td[2], td[3]);
927 }
928 }
929
930 setrptr(&irs[ERDP], PCIWADDR(td) | (1<<3));
931 }
932
933 static void
freeslot(void * arg)934 freeslot(void *arg)
935 {
936 Slot *slot;
937
938 if(arg == nil)
939 return;
940 slot = arg;
941 if(slot->id != 0){
942 Ctlr *ctlr = slot->ctlr;
943 qlock(&ctlr->slotlock);
944 if(ctlr->slot != nil && ctlr->slot[slot->id] == slot){
945 ctlrcmd(ctlr, CR_DISABLESLOT | (slot->id<<24), 0, 0, nil);
946 dmaflush(0, slot->obase, 32*32 << ctlr->csz);
947 ctlr->dcba[slot->id] = 0;
948 dmaflush(1, &ctlr->dcba[slot->id], sizeof(ctlr->dcba[0]));
949 ctlr->slot[slot->id] = nil;
950 }
951 qunlock(&ctlr->slotlock);
952 }
953 freering(&slot->epr[0]);
954 free(slot->ibase);
955 free(slot->obase);
956 free(slot);
957 }
958
959 static Slot*
allocslot(Ctlr * ctlr,Udev * dev)960 allocslot(Ctlr *ctlr, Udev *dev)
961 {
962 u32int r[4];
963 Slot *slot;
964 char *err;
965
966 slot = malloc(sizeof(Slot));
967 if(slot == nil)
968 error(Enomem);
969
970 slot->ctlr = ctlr;
971 slot->dev = dev;
972 slot->nep = 0;
973 slot->id = 0;
974
975 slot->confval = 0;
976 slot->iface = 0;
977 slot->altc = 0;
978
979 qlock(&ctlr->slotlock);
980 if(waserror()){
981 qunlock(&ctlr->slotlock);
982 freeslot(slot);
983 nexterror();
984 }
985 if(ctlr->slot == nil)
986 error(Erecover);
987 slot->ibase = mallocalign(32*33 << ctlr->csz, 64, 0, ctlr->pagesize);
988 slot->obase = mallocalign(32*32 << ctlr->csz, 64, 0, ctlr->pagesize);
989 if(slot->ibase == nil || slot->obase == nil)
990 error(Enomem);
991
992 if((err = ctlrcmd(ctlr, CR_ENABLESLOT, 0, 0, r)) != nil)
993 error(err);
994 slot->id = r[3]>>24;
995 if(slot->id <= 0 || slot->id > ctlr->nslots || ctlr->slot[slot->id] != nil){
996 slot->id = 0;
997 error("bad slot id from controller");
998 }
999 poperror();
1000
1001 dmaflush(1, slot->obase, 32*32 << ctlr->csz);
1002 ctlr->dcba[slot->id] = PCIWADDR(slot->obase);
1003 dmaflush(1, &ctlr->dcba[slot->id], sizeof(ctlr->dcba[0]));
1004
1005 ctlr->slot[slot->id] = slot;
1006
1007 qunlock(&ctlr->slotlock);
1008
1009 return slot;
1010 }
1011
1012 static void
setdebug(Hci *,int)1013 setdebug(Hci *, int)
1014 {
1015 }
1016
1017 static void
epclose(Ep * ep)1018 epclose(Ep *ep)
1019 {
1020 Ctlr *ctlr;
1021 Slot *slot;
1022 Ring *ring;
1023 Epio *io;
1024
1025 if(ep->dev->isroot)
1026 return;
1027
1028 io = ep->aux;
1029 if(io == nil)
1030 return;
1031 ep->aux = nil;
1032
1033 ctlr = ep->hp->aux;
1034 slot = ep->dev->aux;
1035
1036 if(ep->nb > 0 && (io[OREAD].ring != nil || io[OWRITE].ring != nil)){
1037 u32int *w;
1038
1039 /* input control context */
1040 w = slot->ibase;
1041 memset(w, 0, 32<<ctlr->csz);
1042 w[1] = 1;
1043 if((ring = io[OREAD].ring) != nil){
1044 w[0] |= 1 << ring->id;
1045 if(ring->id == slot->nep)
1046 slot->nep--;
1047 ctlrcmd(ctlr, CR_STOPEP | (ring->id<<16) | (slot->id<<24), 0, 0, nil);
1048 }
1049 if((ring = io[OWRITE].ring) != nil){
1050 w[0] |= 1 << ring->id;
1051 if(ring->id == slot->nep)
1052 slot->nep--;
1053 ctlrcmd(ctlr, CR_STOPEP | (ring->id<<16) | (slot->id<<24), 0, 0, nil);
1054 }
1055
1056 /* (input) slot context */
1057 w += 8<<ctlr->csz;
1058 w[0] = (w[0] & ~(0x1F<<27)) | slot->nep<<27;
1059
1060 /* (input) ep context */
1061 w += ep->nb*2*8<<ctlr->csz;
1062 memset(w, 0, 2*32<<ctlr->csz);
1063
1064 dmaflush(1, slot->ibase, 32*33 << ctlr->csz);
1065 ctlrcmd(ctlr, CR_CONFIGEP | (slot->id<<24), 0, PCIWADDR(slot->ibase), nil);
1066 dmaflush(0, slot->obase, 32*32 << ctlr->csz);
1067
1068 freering(io[OREAD].ring);
1069 freering(io[OWRITE].ring);
1070 }
1071 freeb(io[OREAD].b);
1072 freeb(io[OWRITE].b);
1073 free(io);
1074 }
1075
1076 static void
initepctx(u32int * w,Ring * r,Ep * ep)1077 initepctx(u32int *w, Ring *r, Ep *ep)
1078 {
1079 int ival;
1080
1081 if(ep->dev->speed == Lowspeed || ep->dev->speed == Fullspeed){
1082 for(ival=3; ival < 11 && (1<<ival) < ep->pollival; ival++)
1083 ;
1084 } else {
1085 for(ival=0; ival < 15 && (1<<ival) < ep->pollival; ival++)
1086 ;
1087 }
1088 w[0] = ival<<16;
1089 w[1] = ((ep->ttype-Tctl) | (r->id&1)<<2)<<3 | (ep->ntds-1)<<8 | ep->maxpkt<<16;
1090 if(ep->ttype != Tiso)
1091 w[1] |= 3<<1;
1092 *((u64int*)&w[2]) = PCIWADDR(r->base) | 1;
1093 w[4] = 2*ep->maxpkt;
1094 if(ep->ttype == Tintr || ep->ttype == Tiso)
1095 w[4] |= (ep->maxpkt*ep->ntds)<<16;
1096 }
1097
1098 static void
initisoio(Epio * io,Ep * ep)1099 initisoio(Epio *io, Ep *ep)
1100 {
1101 if(io->ring == nil)
1102 return;
1103 io->frame = 0;
1104 io->period = ep->pollival<<3*(ep->dev->speed == Fullspeed);
1105 io->incr = (ep->hz*io->period<<8)/8000;
1106 io->tdsz = (io->incr+255>>8)*ep->samplesz;
1107 io->b = allocb((io->ring->mask+1)*io->tdsz);
1108 }
1109
1110 static void
initep(Ep * ep)1111 initep(Ep *ep)
1112 {
1113 Epio *io;
1114 Ctlr *ctlr;
1115 Slot *slot;
1116 Ring *ring;
1117 u32int *w;
1118 char *err;
1119
1120 io = ep->aux;
1121 ctlr = ep->hp->aux;
1122 slot = ep->dev->aux;
1123
1124 io[OREAD].ring = io[OWRITE].ring = nil;
1125 if(ep->nb == 0){
1126 io[OWRITE].ring = &slot->epr[0];
1127 return;
1128 }
1129
1130 /* (input) control context */
1131 w = slot->ibase;
1132 memset(w, 0, 32<<ctlr->csz);
1133 w[1] = 1;
1134 w[31] = slot->altc<<16 | slot->iface<<8 | slot->confval;
1135
1136 if(waserror()){
1137 freering(io[OWRITE].ring), io[OWRITE].ring = nil;
1138 freering(io[OREAD].ring), io[OREAD].ring = nil;
1139 nexterror();
1140 }
1141 if(ep->mode != OREAD){
1142 ring = initring(io[OWRITE].ring = &slot->epr[ep->nb*2-1], 8);
1143 ring->id = ep->nb*2;
1144 if(ring->id > slot->nep)
1145 slot->nep = ring->id;
1146 ring->slot = slot;
1147 ring->doorbell = &ctlr->dba[slot->id];
1148 ring->ctx = &slot->obase[ring->id*8<<ctlr->csz];
1149 w[1] |= 1 << ring->id;
1150 }
1151 if(ep->mode != OWRITE){
1152 ring = initring(io[OREAD].ring = &slot->epr[ep->nb*2], 8);
1153 ring->id = ep->nb*2+1;
1154 if(ring->id > slot->nep)
1155 slot->nep = ring->id;
1156 ring->slot = slot;
1157 ring->doorbell = &ctlr->dba[slot->id];
1158 ring->ctx = &slot->obase[ring->id*8<<ctlr->csz];
1159 w[1] |= 1 << ring->id;
1160 }
1161
1162 /* (input) slot context */
1163 w += 8<<ctlr->csz;
1164 w[0] = (w[0] & ~(0x1F<<27)) | slot->nep<<27;
1165 if(!ep->dev->ishub)
1166 w[0] &= ~(1<<25); // MTT
1167
1168 /* (input) ep context */
1169 w += ep->nb*2*8<<ctlr->csz;
1170 if(io[OWRITE].ring != nil){
1171 memset(w, 0, 5*4);
1172 initepctx(w, io[OWRITE].ring, ep);
1173 }
1174
1175 w += 8<<ctlr->csz;
1176 if(io[OREAD].ring != nil){
1177 memset(w, 0, 5*4);
1178 initepctx(w, io[OREAD].ring, ep);
1179 }
1180
1181 dmaflush(1, slot->ibase, 32*33 << ctlr->csz);
1182 err = ctlrcmd(ctlr, CR_CONFIGEP | (slot->id<<24), 0, PCIWADDR(slot->ibase), nil);
1183 dmaflush(0, slot->obase, 32*32 << ctlr->csz);
1184 if(err != nil)
1185 error(err);
1186
1187 if(ep->ttype == Tiso){
1188 initisoio(io+OWRITE, ep);
1189 initisoio(io+OREAD, ep);
1190 }
1191 poperror();
1192 }
1193
1194 static int
speedid(int speed)1195 speedid(int speed)
1196 {
1197 switch(speed){
1198 case Fullspeed: return 1;
1199 case Lowspeed: return 2;
1200 case Highspeed: return 3;
1201 case Superspeed: return 4;
1202 }
1203 return 0;
1204 }
1205
1206 static void
epopen(Ep * ep)1207 epopen(Ep *ep)
1208 {
1209 Ctlr *ctlr = ep->hp->aux;
1210 Slot *slot, *hub;
1211 Ring *ring;
1212 Epio *io;
1213 Udev *dev;
1214 char *err;
1215 u32int *w;
1216 int i;
1217
1218 if(ep->dev->isroot)
1219 return;
1220 if(needrecover(ctlr))
1221 error(Erecover);
1222 io = malloc(sizeof(Epio)*2);
1223 if(io == nil)
1224 error(Enomem);
1225 ep->aux = io;
1226 if(waserror()){
1227 epclose(ep);
1228 nexterror();
1229 }
1230 dev = ep->dev;
1231 slot = dev->aux;
1232 if(slot != nil && slot->dev == dev){
1233 initep(ep);
1234 poperror();
1235 return;
1236 }
1237
1238 /* first open has to be control endpoint */
1239 if(ep->nb != 0)
1240 error(Egreg);
1241
1242 slot = allocslot(ctlr, dev);
1243 if(waserror()){
1244 freeslot(slot);
1245 nexterror();
1246 }
1247
1248 /* allocate control ep 0 ring */
1249 ring = initring(io[OWRITE].ring = &slot->epr[0], 4);
1250 ring->id = 1;
1251 slot->nep = 1;
1252 ring->slot = slot;
1253 ring->doorbell = &ctlr->dba[slot->id];
1254 ring->ctx = &slot->obase[8];
1255
1256 /* (input) control context */
1257 w = slot->ibase;
1258 memset(w, 0, 3*32<<ctlr->csz);
1259 w[1] = 3; /* A0, A1 */
1260
1261 /* (input) slot context */
1262 w += 8<<ctlr->csz;
1263 w[2] = w[3] = 0;
1264 w[0] = dev->routestr | speedid(dev->speed)<<20 |
1265 (dev->speed == Highspeed && dev->ishub != 0)<<25 | // MTT
1266 (dev->ishub != 0)<<26 | slot->nep<<27;
1267 w[1] = dev->rootport<<16;
1268
1269 /* find the parent hub that this device is conected to */
1270 qlock(&ctlr->slotlock);
1271 for(i=1; i<=ctlr->nslots; i++){
1272 hub = ctlr->slot[i];
1273 if(hub == nil || hub->dev == nil || hub->dev->aux != hub)
1274 continue;
1275 if(hub == slot || hub->dev == dev)
1276 continue;
1277 if(!hub->dev->ishub)
1278 continue;
1279 if(hub->dev->addr != dev->hub)
1280 continue;
1281 if(hub->dev->rootport != dev->rootport)
1282 continue;
1283
1284 if(dev->speed < Highspeed && hub->dev->speed == Highspeed){
1285 w[0] |= 1<<25; // MTT
1286 w[2] = hub->id | dev->port<<8;
1287 }
1288 break;
1289 }
1290 qunlock(&ctlr->slotlock);
1291
1292 /* (input) ep context 0 */
1293 w += 8<<ctlr->csz;
1294 initepctx(w, io[OWRITE].ring, ep);
1295
1296 dmaflush(1, slot->ibase, 32*33 << ctlr->csz);
1297 err = ctlrcmd(ctlr, CR_ADDRESSDEV | (slot->id<<24), 0, PCIWADDR(slot->ibase), nil);
1298 dmaflush(0, slot->obase, 32*32 << ctlr->csz);
1299 if(err != nil)
1300 error(err);
1301
1302 /* (output) slot context */
1303 w = slot->obase;
1304
1305 dev->addr = w[3] & 0xFF;
1306
1307 dev->aux = slot;
1308 dev->free = freeslot;
1309
1310 poperror();
1311 poperror();
1312 }
1313
1314 static long
isoread(Ep *,uchar *,long)1315 isoread(Ep *, uchar *, long)
1316 {
1317 error(Egreg);
1318 return 0;
1319 }
1320
1321 static long
isowrite(Ep * ep,uchar * p,long n)1322 isowrite(Ep *ep, uchar *p, long n)
1323 {
1324 uchar *s, *d;
1325 Ctlr *ctlr;
1326 Epio *io;
1327 u32int i, µ;
1328 long m;
1329
1330 s = p;
1331 io = (Epio*)ep->aux + OWRITE;
1332 qlock(io);
1333 if(waserror()){
1334 qunlock(io);
1335 nexterror();
1336 }
1337 µ = io->period;
1338 ctlr = ep->hp->aux;
1339 if(needrecover(ctlr))
1340 error(Erecover);
1341 for(i = io->frame;; i++){
1342 for(;;){
1343 m = (int)(io->ring->wp - io->ring->rp);
1344 if(m <= 0)
1345 i = (80 + µframe(ctlr))/µ;
1346 if(m < io->ring->mask)
1347 break;
1348 *io->ring->doorbell = io->ring->id;
1349 tsleep(&up->sleep, return0, nil, 5);
1350 }
1351 m = ((io->incr + (i*io->incr&255))>>8)*ep->samplesz;
1352 d = io->b->rp + (i&io->ring->mask)*io->tdsz;
1353 m -= io->nleft, d += io->nleft;
1354 if(n < m){
1355 memmove(d, p, n);
1356 p += n;
1357 io->nleft += n;
1358 break;
1359 }
1360 memmove(d, p, m);
1361 p += m, n -= m;
1362 m += io->nleft, d -= io->nleft;
1363 io->nleft = 0;
1364 dmaflush(1, d, m);
1365 queuetd(io->ring, TR_ISOCH | (i*µ/8 & 0x7ff)<<20 | TR_IOC, m, PCIWADDR(d), nil);
1366 }
1367 io->frame = i;
1368 while(io->ring->rp != io->ring->wp){
1369 int d = (int)(i*µ - µframe(ctlr))/8;
1370 //d -= ep->sampledelay*1000 / ep->hz;
1371 if(d < 5)
1372 break;
1373 *io->ring->doorbell = io->ring->id;
1374 tsleep(&up->sleep, return0, nil, d);
1375 }
1376 qunlock(io);
1377 poperror();
1378
1379 return p - s;
1380 }
1381
1382 static char*
unstall(Ep * ep,Ring * r)1383 unstall(Ep *ep, Ring *r)
1384 {
1385 char *err;
1386
1387 switch(r->ctx[0]&7){
1388 case 2: /* halted */
1389 case 4: /* error */
1390 ep->clrhalt = 1;
1391 }
1392 if(ep->clrhalt){
1393 ep->clrhalt = 0;
1394 err = ctlrcmd(r->slot->ctlr, CR_RESETEP | (r->id<<16) | (r->slot->id<<24), 0, 0, nil);
1395 dmaflush(0, r->ctx, 8*4 << r->slot->ctlr->csz);
1396 if(err != nil)
1397 return err;
1398 r->stopped = 1;
1399 }
1400 if(r->stopped){
1401 err = ctlrcmd(r->slot->ctlr, CR_SETTRDQP | (r->id<<16) | (r->slot->id<<24), 0, resetring(r), nil);
1402 dmaflush(0, r->ctx, 8*4 << r->slot->ctlr->csz);
1403 if(err != nil)
1404 return err;
1405 r->stopped = 0;
1406 }
1407 if(r->wp - r->rp >= r->mask)
1408 return "Ring Full";
1409 return nil;
1410 }
1411
1412 static long
epread(Ep * ep,void * va,long n)1413 epread(Ep *ep, void *va, long n)
1414 {
1415 Epio *io;
1416 Ctlr *ctlr;
1417 uchar *p;
1418 char *err;
1419 Wait w[1];
1420
1421 if(ep->dev->isroot)
1422 error(Egreg);
1423
1424 p = va;
1425 if(ep->ttype == Tctl){
1426 io = (Epio*)ep->aux + OREAD;
1427 qlock(io);
1428 if(io->b == nil || BLEN(io->b) == 0){
1429 qunlock(io);
1430 return 0;
1431 }
1432 if(n > BLEN(io->b))
1433 n = BLEN(io->b);
1434 memmove(p, io->b->rp, n);
1435 io->b->rp += n;
1436 qunlock(io);
1437 return n;
1438 } else if(ep->ttype == Tiso)
1439 return isoread(ep, p, n);
1440
1441 if((uintptr)p <= KZERO){
1442 Block *b;
1443
1444 b = allocb(n);
1445 if(waserror()){
1446 freeb(b);
1447 nexterror();
1448 }
1449 n = epread(ep, b->rp, n);
1450 memmove(p, b->rp, n);
1451 freeb(b);
1452 poperror();
1453 return n;
1454 }
1455
1456 ctlr = (Ctlr*)ep->hp->aux;
1457 io = (Epio*)ep->aux + OREAD;
1458 qlock(io);
1459 if(waserror()){
1460 dmaflush(0, io->ring->ctx, 8*4 << ctlr->csz);
1461 qunlock(io);
1462 nexterror();
1463 }
1464
1465 if((err = unstall(ep, io->ring)) != nil)
1466 error(err);
1467
1468 dmaflush(1, p, n);
1469 queuetd(io->ring, TR_NORMAL | TR_IOC, n, PCIWADDR(p), w);
1470 err = waittd(ctlr, w, ep->tmout);
1471 dmaflush(0, p, n);
1472 if(err != nil)
1473 error(err);
1474
1475 qunlock(io);
1476 poperror();
1477
1478 n -= (w->er[2] & 0xFFFFFF);
1479 if(n < 0)
1480 n = 0;
1481
1482 return n;
1483 }
1484
1485 static long
epwrite(Ep * ep,void * va,long n)1486 epwrite(Ep *ep, void *va, long n)
1487 {
1488 Wait w[3];
1489 Ctlr *ctlr;
1490 Epio *io;
1491 uchar *p;
1492 char *err;
1493
1494 if(ep->dev->isroot)
1495 error(Egreg);
1496
1497 p = va;
1498 if(ep->ttype == Tctl){
1499 int dir, len;
1500 Ring *ring;
1501 Slot *slot;
1502
1503 if(n < 8)
1504 error(Eshort);
1505
1506 if(p[0] == 0x00 && p[1] == 0x05)
1507 return n;
1508
1509 ctlr = (Ctlr*)ep->hp->aux;
1510 io = (Epio*)ep->aux + OREAD;
1511 ring = io[OWRITE-OREAD].ring;
1512 slot = ring->slot;
1513 qlock(io);
1514 if(waserror()){
1515 ilock(ring);
1516 ring->pending = nil;
1517 iunlock(ring);
1518 dmaflush(0, ring->ctx, 8*4 << ctlr->csz);
1519 qunlock(io);
1520 nexterror();
1521 }
1522 if(io->b != nil){
1523 freeb(io->b);
1524 io->b = nil;
1525 }
1526 len = GET2(&p[6]);
1527 dir = (p[0] & Rd2h) != 0;
1528 if(len > 0){
1529 io->b = allocb(len);
1530 if(dir == 0){ /* out */
1531 assert(len >= n-8);
1532 memmove(io->b->wp, p+8, n-8);
1533 } else {
1534 memset(io->b->wp, 0, len);
1535 io->b->wp += len;
1536 }
1537 }
1538 if((err = unstall(ep, ring)) != nil)
1539 error(err);
1540
1541 if((ring->ctx[1]>>16) != ep->maxpkt){
1542 u32int *w = slot->ibase;
1543 w[0] = 0;
1544 w[1] = 1<<ring->id;
1545 w += (ring->id+1)*8<<ctlr->csz;
1546 initepctx(w, ring, ep);
1547 dmaflush(1, slot->ibase, 32*33 << ctlr->csz);
1548 err = ctlrcmd(ctlr, CR_EVALCTX | (slot->id<<24), 0, PCIWADDR(slot->ibase), nil);
1549 dmaflush(0, slot->obase, 32*32 << ctlr->csz);
1550 if(err != nil)
1551 error(err);
1552 }
1553
1554 queuetd(ring, TR_SETUPSTAGE | (len > 0 ? 2+dir : 0)<<16 | TR_IDT | TR_IOC, 8,
1555 p[0] | p[1]<<8 | GET2(&p[2])<<16 |
1556 (u64int)(GET2(&p[4]) | len<<16)<<32, &w[0]);
1557 if(len > 0){
1558 dmaflush(1, io->b->rp, len);
1559 queuetd(ring, TR_DATASTAGE | dir<<16 | TR_IOC, len,
1560 PCIWADDR(io->b->rp), &w[1]);
1561 }
1562 queuetd(ring, TR_STATUSSTAGE | (len == 0 || !dir)<<16 | TR_IOC, 0, 0, &w[2]);
1563
1564 if((err = waittd(ctlr, &w[0], ep->tmout)) != nil)
1565 error(err);
1566 if(len > 0){
1567 if((err = waittd(ctlr, &w[1], ep->tmout)) != nil)
1568 error(err);
1569 if(dir != 0){
1570 dmaflush(0, io->b->rp, len);
1571 io->b->wp -= (w[1].er[2] & 0xFFFFFF);
1572 if(io->b->wp < io->b->rp)
1573 io->b->wp = io->b->rp;
1574 }
1575 }
1576 if((err = waittd(ctlr, &w[2], ep->tmout)) != nil)
1577 error(err);
1578
1579 if(p[0] == 0x00 && p[1] == 0x09){
1580 slot->confval = GET2(&p[2]);
1581 } else if(p[0] == 0x01 && p[1] == 0x0d){
1582 slot->altc = GET2(&p[2]);
1583 slot->iface = GET2(&p[4]);
1584 }
1585
1586 qunlock(io);
1587 poperror();
1588
1589 return n;
1590 } else if(ep->ttype == Tiso)
1591 return isowrite(ep, p, n);
1592
1593 if((uintptr)p <= KZERO){
1594 Block *b;
1595
1596 b = allocb(n);
1597 if(waserror()){
1598 freeb(b);
1599 nexterror();
1600 }
1601 memmove(b->wp, p, n);
1602 n = epwrite(ep, b->wp, n);
1603 freeb(b);
1604 poperror();
1605 return n;
1606 }
1607
1608 ctlr = (Ctlr*)ep->hp->aux;
1609 io = (Epio*)ep->aux + OWRITE;
1610 qlock(io);
1611 if(waserror()){
1612 dmaflush(0, io->ring->ctx, 8*4 << ctlr->csz);
1613 qunlock(io);
1614 nexterror();
1615 }
1616
1617 if((err = unstall(ep, io->ring)) != nil)
1618 error(err);
1619
1620 dmaflush(1, p, n);
1621 queuetd(io->ring, TR_NORMAL | TR_IOC, n, PCIWADDR(p), w);
1622 if((err = waittd(ctlr, w, ep->tmout)) != nil)
1623 error(err);
1624
1625 qunlock(io);
1626 poperror();
1627
1628 return n;
1629 }
1630
1631 static char*
seprintep(char * s,char *,Ep *)1632 seprintep(char *s, char*, Ep*)
1633 {
1634 return s;
1635 }
1636
1637 static int
portstatus(Hci * hp,int port)1638 portstatus(Hci *hp, int port)
1639 {
1640 Ctlr *ctlr = hp->aux;
1641 u32int psc, ps;
1642
1643 if(ctlr->port == nil || needrecover(ctlr))
1644 return 0;
1645
1646 ps = 0;
1647 psc = ctlr->port[port-1].reg[PORTSC];
1648 if(psc & CCS) ps |= HPpresent;
1649 if(psc & PED) ps |= HPenable;
1650 if(psc & OCA) ps |= HPovercurrent;
1651 if(psc & PR) ps |= HPreset;
1652
1653 if((hp->superspeed & (1<<(port-1))) != 0){
1654 ps |= psc & (PLS|PP);
1655 if(psc & CSC) ps |= 1<<0+16;
1656 if(psc & OCC) ps |= 1<<3+16;
1657 if(psc & PRC) ps |= 1<<4+16;
1658 if(psc & WRC) ps |= 1<<5+16;
1659 if(psc & PLC) ps |= 1<<6+16;
1660 if(psc & CEC) ps |= 1<<7+16;
1661 } else {
1662 if((ps & HPreset) == 0){
1663 switch((psc>>10)&15){
1664 case 1:
1665 /* full speed */
1666 break;
1667 case 2:
1668 ps |= HPslow;
1669 break;
1670 case 3:
1671 ps |= HPhigh;
1672 break;
1673 }
1674 }
1675 if(psc & PP) ps |= HPpower;
1676 if(psc & CSC) ps |= HPstatuschg;
1677 if(psc & PRC) ps |= HPchange;
1678 }
1679
1680 return ps;
1681 }
1682
1683 static int
portenable(Hci *,int,int)1684 portenable(Hci*, int, int)
1685 {
1686 return 0;
1687 }
1688
1689 static int
portreset(Hci * hp,int port,int on)1690 portreset(Hci *hp, int port, int on)
1691 {
1692 Ctlr *ctlr = hp->aux;
1693
1694 if(ctlr->port == nil || needrecover(ctlr))
1695 return 0;
1696
1697 if(on){
1698 ctlr->port[port-1].reg[PORTSC] |= PR;
1699 tsleep(&up->sleep, return0, nil, 200);
1700 }
1701 return 0;
1702 }
1703
1704
1705 static Ctlr *ctlrs[Nhcis];
1706
1707 static void
scanpci(void)1708 scanpci(void)
1709 {
1710 static int already = 0;
1711 int i;
1712 uintpci io;
1713 Ctlr *ctlr;
1714 Pcidev *p;
1715 u32int *mmio;
1716
1717 if(already)
1718 return;
1719 already = 1;
1720 p = nil;
1721 while ((p = pcimatch(p, 0, 0)) != nil) {
1722 /*
1723 * Find XHCI controllers (Programming Interface = 0x30).
1724 */
1725 if(p->ccrb != Pcibcserial || p->ccru != Pciscusb || p->ccrp != 0x30)
1726 continue;
1727 io = p->mem[0].bar & ~0x0f;
1728 if(io == 0)
1729 continue;
1730 print("usbxhci: %#x %#x: port %llux size %#x irq %d\n",
1731 p->vid, p->did, io, p->mem[0].size, p->intl);
1732 mmio = (u32int*)mmukmapx(VIRTPCI, io, p->mem[0].size);
1733 if(mmio == nil){
1734 print("usbxhci: cannot map registers\n");
1735 continue;
1736 }
1737 ctlr = malloc(sizeof(Ctlr));
1738 if(ctlr == nil){
1739 print("usbxhci: no memory\n");
1740 continue;
1741 }
1742 ctlr->base = io;
1743 ctlr->active = nil;
1744 ctlr->pcidev = p;
1745 ctlr->mmio = mmio;
1746 for(i = 0; i < nelem(ctlrs); i++)
1747 if(ctlrs[i] == nil){
1748 ctlrs[i] = ctlr;
1749 break;
1750 }
1751 if(i >= nelem(ctlrs))
1752 print("xhci: bug: more than %d controllers\n", nelem(ctlrs));
1753 }
1754 }
1755
1756 static int
reset(Hci * hp)1757 reset(Hci *hp)
1758 {
1759 Ctlr *ctlr;
1760 int i;
1761
1762 if(getconf("*nousbxhci"))
1763 return -1;
1764
1765 scanpci();
1766
1767 /*
1768 * Any adapter matches if no hp->port is supplied,
1769 * otherwise the ports must match.
1770 */
1771 for(i = 0; i < nelem(ctlrs) && ctlrs[i] != nil; i++){
1772 ctlr = ctlrs[i];
1773 if(ctlr->active == nil)
1774 if(hp->port == 0 || hp->port == ctlr->base){
1775 ctlr->active = hp;
1776 goto Found;
1777 }
1778 }
1779 return -1;
1780
1781 Found:
1782 hp->aux = ctlr;
1783 hp->port = ctlr->base;
1784 hp->irq = ctlr->pcidev->intl;
1785 hp->tbdf = ctlr->pcidev->tbdf;
1786
1787 hp->init = init;
1788 hp->dump = dump;
1789 hp->interrupt = interrupt;
1790 hp->epopen = epopen;
1791 hp->epclose = epclose;
1792 hp->epread = epread;
1793 hp->epwrite = epwrite;
1794 hp->seprintep = seprintep;
1795 hp->portenable = portenable;
1796 hp->portreset = portreset;
1797 hp->portstatus = portstatus;
1798 hp->shutdown = shutdown;
1799 hp->debug = setdebug;
1800 hp->type = "xhci";
1801
1802 return 0;
1803 }
1804
1805 void
usbxhcilink(void)1806 usbxhcilink(void)
1807 {
1808 addhcitype("xhci", reset);
1809 }
1810
1811 void
dmaflush(int clean,void * p,ulong len)1812 dmaflush(int clean, void *p, ulong len)
1813 {
1814 uintptr s = (uintptr)p;
1815 uintptr e = (uintptr)p + len;
1816
1817 if(clean){
1818 s &= ~(BLOCKALIGN-1);
1819 e += BLOCKALIGN-1;
1820 e &= ~(BLOCKALIGN-1);
1821 cachedwbse((void*)s, e - s);
1822 return;
1823 }
1824 if(s & BLOCKALIGN-1){
1825 s &= ~(BLOCKALIGN-1);
1826 cachedwbinvse((void*)s, BLOCKALIGN);
1827 s += BLOCKALIGN;
1828 }
1829 if(e & BLOCKALIGN-1){
1830 e &= ~(BLOCKALIGN-1);
1831 if(e < s)
1832 return;
1833 cachedwbinvse((void*)e, BLOCKALIGN);
1834 }
1835 if(s < e)
1836 cachedinvse((void*)s, e - s);
1837 }
1838