1 /*
2 * virtio ethernet driver implementing the legacy interface:
3 * http://docs.oasis-open.org/virtio/virtio/v1.0/virtio-v1.0.html
4 */
5 #include "u.h"
6 #include "../port/lib.h"
7 #include "mem.h"
8 #include "dat.h"
9 #include "fns.h"
10 #include "io.h"
11 #include "ureg.h"
12 #include "../port/error.h"
13
14 #include "../port/sd.h"
15
16 typedef struct Vring Vring;
17 typedef struct Vdesc Vdesc;
18 typedef struct Vused Vused;
19 typedef struct Vqueue Vqueue;
20 typedef struct Vdev Vdev;
21
22 typedef struct ScsiCfg ScsiCfg;
23
24 /* device types */
25 enum {
26 TypBlk = 2,
27 TypSCSI = 8,
28 };
29
30 /* status flags */
31 enum {
32 Acknowledge = 1,
33 Driver = 2,
34 DriverOk = 4,
35 Failed = 0x80,
36 };
37
38 /* virtio ports */
39 enum {
40 Devfeat = 0,
41 Drvfeat = 4,
42 Qaddr = 8,
43 Qsize = 12,
44 Qselect = 14,
45 Qnotify = 16,
46 Status = 18,
47 Isr = 19,
48
49 Devspec = 20,
50 };
51
52 /* descriptor flags */
53 enum {
54 Next = 1,
55 Write = 2,
56 Indirect = 4,
57 };
58
59 /* struct sizes */
60 enum {
61 VringSize = 4,
62 };
63
64 struct Vring
65 {
66 u16int flags;
67 u16int idx;
68 };
69
70 struct Vdesc
71 {
72 u64int addr;
73 u32int len;
74 u16int flags;
75 u16int next;
76 };
77
78 struct Vused
79 {
80 u32int id;
81 u32int len;
82 };
83
84 struct Vqueue
85 {
86 Lock;
87
88 Vdev *dev;
89 int idx;
90
91 int size;
92
93 int free;
94 int nfree;
95
96 Vdesc *desc;
97
98 Vring *avail;
99 u16int *availent;
100 u16int *availevent;
101
102 Vring *used;
103 Vused *usedent;
104 u16int *usedevent;
105 u16int lastused;
106
107 void *rock[];
108 };
109
110 struct Vdev
111 {
112 int typ;
113
114 Pcidev *pci;
115
116 ulong port;
117 ulong feat;
118
119 int nqueue;
120 Vqueue *queue[16];
121
122 void *cfg; /* device specific config (for scsi) */
123
124 Vdev *next;
125 };
126
127 enum {
128 CDBSIZE = 32,
129 SENSESIZE = 96,
130 };
131
132 struct ScsiCfg
133 {
134 u32int num_queues;
135 u32int seg_max;
136 u32int max_sectors;
137 u32int cmd_per_lun;
138 u32int event_info_size;
139 u32int sense_size;
140 u32int cdb_size;
141 u16int max_channel;
142 u16int max_target;
143 u32int max_lun;
144 };
145
146 static Vqueue*
mkvqueue(int size)147 mkvqueue(int size)
148 {
149 Vqueue *q;
150 uchar *p;
151 int i;
152
153 q = malloc(sizeof(*q) + sizeof(void*)*size);
154 p = mallocalign(
155 PGROUND(sizeof(Vdesc)*size +
156 VringSize +
157 sizeof(u16int)*size +
158 sizeof(u16int)) +
159 PGROUND(VringSize +
160 sizeof(Vused)*size +
161 sizeof(u16int)),
162 BY2PG, 0, 0);
163 if(p == nil || q == nil){
164 print("virtio: no memory for Vqueue\n");
165 free(p);
166 free(q);
167 return nil;
168 }
169
170 q->desc = (void*)p;
171 p += sizeof(Vdesc)*size;
172 q->avail = (void*)p;
173 p += VringSize;
174 q->availent = (void*)p;
175 p += sizeof(u16int)*size;
176 q->availevent = (void*)p;
177 p += sizeof(u16int);
178
179 p = (uchar*)PGROUND((uintptr)p);
180 q->used = (void*)p;
181 p += VringSize;
182 q->usedent = (void*)p;
183 p += sizeof(Vused)*size;
184 q->usedevent = (void*)p;
185
186 q->free = -1;
187 q->nfree = q->size = size;
188 for(i=0; i<size; i++){
189 q->desc[i].next = q->free;
190 q->free = i;
191 }
192
193 return q;
194 }
195
196 static Vdev*
viopnpdevs(int typ)197 viopnpdevs(int typ)
198 {
199 Vdev *vd, *h, *t;
200 Vqueue *q;
201 Pcidev *p;
202 int n, i, size;
203
204 h = t = nil;
205 for(p = nil; p = pcimatch(p, 0x1AF4, 0);){
206 if((p->did < 0x1000) || (p->did > 0x103F))
207 continue;
208 if(p->rid != 0)
209 continue;
210 //if((p->mem[0].bar & 1) == 0) /* zero on gce */
211 // continue;
212 if(pcicfgr16(p, 0x2E) != typ)
213 continue;
214 if((vd = malloc(sizeof(*vd))) == nil){
215 print("virtio: no memory for Vdev\n");
216 break;
217 }
218 vd->port = p->mem[0].bar & ~3;
219 size = p->mem[0].size;
220 if(vd->port == 0){ /* gce */
221 vd->port = 0xc000;
222 size = 0x40;
223 }
224 if(ioalloc(vd->port, size, 0, "virtio") < 0){
225 print("virtio: port %lux in use\n", vd->port);
226 free(vd);
227 continue;
228 }
229 vd->typ = typ;
230 vd->pci = p;
231
232 /* reset */
233 outb(vd->port+Status, 0);
234
235 vd->feat = inl(vd->port+Devfeat);
236 outb(vd->port+Status, Acknowledge|Driver);
237 for(i=0; i<nelem(vd->queue); i++){
238 outs(vd->port+Qselect, i);
239 n = ins(vd->port+Qsize);
240 if(n == 0 || (n & (n-1)) != 0)
241 break;
242 if((q = mkvqueue(n)) == nil)
243 break;
244 q->dev = vd;
245 q->idx = i;
246 vd->queue[i] = q;
247 coherence();
248 outl(vd->port+Qaddr, PADDR(vd->queue[i]->desc)/BY2PG);
249 }
250 vd->nqueue = i;
251
252 if(h == nil)
253 h = vd;
254 else
255 t->next = vd;
256 t = vd;
257 }
258
259 return h;
260 }
261
262 struct Rock {
263 int done;
264 Rendez *sleep;
265 };
266
267 static void
vqinterrupt(Vqueue * q)268 vqinterrupt(Vqueue *q)
269 {
270 int id, free, m;
271 struct Rock *r;
272 Rendez *z;
273
274 m = q->size-1;
275
276 ilock(q);
277 while((q->lastused ^ q->used->idx) & m){
278 id = q->usedent[q->lastused++ & m].id;
279 if(r = q->rock[id]){
280 q->rock[id] = nil;
281 z = r->sleep;
282 r->done = 1; /* hands off */
283 if(z != nil)
284 wakeup(z);
285 }
286 do {
287 free = id;
288 id = q->desc[free].next;
289 q->desc[free].next = q->free;
290 q->free = free;
291 q->nfree++;
292 } while(q->desc[free].flags & Next);
293 }
294 iunlock(q);
295 }
296
297 static void
viointerrupt(Ureg *,void * arg)298 viointerrupt(Ureg *, void *arg)
299 {
300 Vdev *vd = arg;
301
302 if(inb(vd->port+Isr) & 1)
303 vqinterrupt(vd->queue[vd->typ == TypSCSI ? 2 : 0]);
304 }
305
306 static int
viodone(void * arg)307 viodone(void *arg)
308 {
309 return ((struct Rock*)arg)->done;
310 }
311
312 static void
vqio(Vqueue * q,int head)313 vqio(Vqueue *q, int head)
314 {
315 struct Rock rock;
316
317 rock.done = 0;
318 rock.sleep = &up->sleep;
319 q->rock[head] = &rock;
320 q->availent[q->avail->idx & (q->size-1)] = head;
321 coherence();
322 q->avail->idx++;
323 iunlock(q);
324 if((q->used->flags & 1) == 0)
325 outs(q->dev->port+Qnotify, q->idx);
326 while(!rock.done){
327 while(waserror())
328 ;
329 tsleep(rock.sleep, viodone, &rock, 1000);
330 poperror();
331
332 if(!rock.done)
333 vqinterrupt(q);
334 }
335 }
336
337 static int
vioblkreq(Vdev * vd,int typ,void * a,long count,long secsize,uvlong lba)338 vioblkreq(Vdev *vd, int typ, void *a, long count, long secsize, uvlong lba)
339 {
340 int need, free, head;
341 Vqueue *q;
342 Vdesc *d;
343
344 u8int status;
345 struct Vioblkreqhdr {
346 u32int typ;
347 u32int prio;
348 u64int lba;
349 } req;
350
351 need = 2;
352 if(a != nil)
353 need = 3;
354
355 status = -1;
356 req.typ = typ;
357 req.prio = 0;
358 req.lba = lba;
359
360 q = vd->queue[0];
361 ilock(q);
362 while(q->nfree < need){
363 iunlock(q);
364
365 if(!waserror())
366 tsleep(&up->sleep, return0, 0, 500);
367 poperror();
368
369 ilock(q);
370 }
371
372 head = free = q->free;
373
374 d = &q->desc[free]; free = d->next;
375 d->addr = PADDR(&req);
376 d->len = sizeof(req);
377 d->flags = Next;
378
379 if(a != nil){
380 d = &q->desc[free]; free = d->next;
381 d->addr = PADDR(a);
382 d->len = secsize*count;
383 d->flags = typ ? Next : (Write|Next);
384 }
385
386 d = &q->desc[free]; free = d->next;
387 d->addr = PADDR(&status);
388 d->len = sizeof(status);
389 d->flags = Write;
390
391 q->free = free;
392 q->nfree -= need;
393
394 /* queue io, unlock and wait for completion */
395 vqio(q, head);
396
397 return status;
398 }
399
400 static int
vioscsireq(SDreq * r)401 vioscsireq(SDreq *r)
402 {
403 u8int resp[4+4+2+2+SENSESIZE];
404 u8int req[8+8+3+CDBSIZE];
405 int free, head;
406 u32int len;
407 Vqueue *q;
408 Vdesc *d;
409 Vdev *vd;
410 SDunit *u;
411 ScsiCfg *cfg;
412
413 u = r->unit;
414 vd = u->dev->ctlr;
415 cfg = vd->cfg;
416
417 memset(resp, 0, sizeof(resp));
418 memset(req, 0, sizeof(req));
419 req[0] = 1;
420 req[1] = u->subno;
421 req[2] = r->lun>>8;
422 req[3] = r->lun&0xFF;
423 *(u64int*)(&req[8]) = (uintptr)r;
424
425 memmove(&req[8+8+3], r->cmd, r->clen);
426
427 q = vd->queue[2];
428 ilock(q);
429 while(q->nfree < 3){
430 iunlock(q);
431
432 if(!waserror())
433 tsleep(&up->sleep, return0, 0, 500);
434 poperror();
435
436 ilock(q);
437 }
438
439 head = free = q->free;
440
441 d = &q->desc[free]; free = d->next;
442 d->addr = PADDR(req);
443 d->len = 8+8+3+cfg->cdb_size;
444 d->flags = Next;
445
446 if(r->write && r->dlen > 0){
447 d = &q->desc[free]; free = d->next;
448 d->addr = PADDR(r->data);
449 d->len = r->dlen;
450 d->flags = Next;
451 }
452
453 d = &q->desc[free]; free = d->next;
454 d->addr = PADDR(resp);
455 d->len = 4+4+2+2+cfg->sense_size;
456 d->flags = Write;
457
458 if(!r->write && r->dlen > 0){
459 d->flags |= Next;
460
461 d = &q->desc[free]; free = d->next;
462 d->addr = PADDR(r->data);
463 d->len = r->dlen;
464 d->flags = Write;
465 }
466
467 q->free = free;
468 q->nfree -= 2 + (r->dlen > 0);
469
470 /* queue io, unlock and wait for completion */
471 vqio(q, head);
472
473 /* response+status */
474 r->status = resp[10];
475 if(resp[11] != 0)
476 r->status = SDcheck;
477
478 /* sense_len */
479 len = *((u32int*)&resp[0]);
480 if(len > 0){
481 if(len > sizeof(r->sense))
482 len = sizeof(r->sense);
483 memmove(r->sense, &resp[4+4+2+2], len);
484 r->flags |= SDvalidsense;
485 }
486
487 /* data residue */
488 len = *((u32int*)&resp[4]);
489 if(len > r->dlen)
490 r->rlen = 0;
491 else
492 r->rlen = r->dlen - len;
493
494 return r->status;
495
496 }
497
498 static long
viobio(SDunit * u,int lun,int write,void * a,long count,uvlong lba)499 viobio(SDunit *u, int lun, int write, void *a, long count, uvlong lba)
500 {
501 long ss, cc, max, ret;
502 Vdev *vd;
503
504 vd = u->dev->ctlr;
505 if(vd->typ == TypSCSI)
506 return scsibio(u, lun, write, a, count, lba);
507
508 max = 32;
509 ss = u->secsize;
510 ret = 0;
511 while(count > 0){
512 if((cc = count) > max)
513 cc = max;
514 if(vioblkreq(vd, write != 0, (uchar*)a + ret, cc, ss, lba) != 0)
515 error(Eio);
516 ret += cc*ss;
517 count -= cc;
518 lba += cc;
519 }
520 return ret;
521 }
522
523 enum {
524 SDread,
525 SDwrite,
526 };
527
528 static int
viorio(SDreq * r)529 viorio(SDreq *r)
530 {
531 int i, count, rw;
532 uvlong lba;
533 SDunit *u;
534 Vdev *vd;
535
536 u = r->unit;
537 vd = u->dev->ctlr;
538 if(vd->typ == TypSCSI)
539 return vioscsireq(r);
540 if(r->cmd[0] == 0x35 || r->cmd[0] == 0x91){
541 if(vioblkreq(vd, 4, nil, 0, 0, 0) != 0)
542 return sdsetsense(r, SDcheck, 3, 0xc, 2);
543 return sdsetsense(r, SDok, 0, 0, 0);
544 }
545 if((i = sdfakescsi(r, nil, 0)) != SDnostatus)
546 return r->status = i;
547 if((i = sdfakescsirw(r, &lba, &count, &rw)) != SDnostatus)
548 return i;
549 r->rlen = viobio(u, r->lun, rw == SDwrite, r->data, count, lba);
550 return r->status = SDok;
551 }
552
553 static int
vioonline(SDunit * u)554 vioonline(SDunit *u)
555 {
556 uvlong cap;
557 Vdev *vd;
558
559 vd = u->dev->ctlr;
560 if(vd->typ == TypSCSI)
561 return scsionline(u);
562
563 cap = inl(vd->port+Devspec+4);
564 cap <<= 32;
565 cap |= inl(vd->port+Devspec);
566 if(u->sectors != cap){
567 u->sectors = cap;
568 u->secsize = 512;
569 return 2;
570 }
571 return 1;
572 }
573
574 static int
vioverify(SDunit * u)575 vioverify(SDunit *u)
576 {
577 Vdev *vd;
578
579 vd = u->dev->ctlr;
580 if(vd->typ == TypSCSI)
581 return scsiverify(u);
582
583 return 1;
584 }
585
586 SDifc sdvirtioifc;
587
588 static int
vioenable(SDev * sd)589 vioenable(SDev *sd)
590 {
591 char name[32];
592 Vdev *vd;
593
594 vd = sd->ctlr;
595 pcisetbme(vd->pci);
596 snprint(name, sizeof(name), "%s (%s)", sd->name, sd->ifc->name);
597 intrenable(vd->pci->intl, viointerrupt, vd, vd->pci->tbdf, name);
598 outb(vd->port+Status, inb(vd->port+Status) | DriverOk);
599 return 1;
600 }
601
602 static int
viodisable(SDev * sd)603 viodisable(SDev *sd)
604 {
605 char name[32];
606 Vdev *vd;
607
608 vd = sd->ctlr;
609 snprint(name, sizeof(name), "%s (%s)", sd->name, sd->ifc->name);
610 intrdisable(vd->pci->intl, viointerrupt, vd, vd->pci->tbdf, name);
611 pciclrbme(vd->pci);
612 return 1;
613 }
614
615 static SDev*
viopnp(void)616 viopnp(void)
617 {
618 SDev *s, *h, *t;
619 Vdev *vd;
620 int id;
621
622 h = t = nil;
623
624 id = 'F';
625 for(vd = viopnpdevs(TypBlk); vd; vd = vd->next){
626 if(vd->nqueue != 1)
627 continue;
628
629 if((s = malloc(sizeof(*s))) == nil)
630 break;
631 s->ctlr = vd;
632 s->idno = id++;
633 s->ifc = &sdvirtioifc;
634 s->nunit = 1;
635 if(h)
636 t->next = s;
637 else
638 h = s;
639 t = s;
640 }
641
642 id = '0';
643 for(vd = viopnpdevs(TypSCSI); vd; vd = vd->next){
644 ScsiCfg *cfg;
645
646 if(vd->nqueue < 3)
647 continue;
648
649 if((cfg = malloc(sizeof(*cfg))) == nil)
650 break;
651 cfg->num_queues = inl(vd->port+Devspec+4*0);
652 cfg->seg_max = inl(vd->port+Devspec+4*1);
653 cfg->max_sectors = inl(vd->port+Devspec+4*2);
654 cfg->cmd_per_lun = inl(vd->port+Devspec+4*3);
655 cfg->event_info_size = inl(vd->port+Devspec+4*4);
656 cfg->sense_size = inl(vd->port+Devspec+4*5);
657 cfg->cdb_size = inl(vd->port+Devspec+4*6);
658 cfg->max_channel = ins(vd->port+Devspec+4*7);
659 cfg->max_target = ins(vd->port+Devspec+4*7+2);
660 cfg->max_lun = inl(vd->port+Devspec+4*8);
661
662 if(cfg->max_target == 0){
663 free(cfg);
664 continue;
665 }
666 if((cfg->cdb_size > CDBSIZE) || (cfg->sense_size > SENSESIZE)){
667 print("sdvirtio: cdb %ud or sense size %ud too big\n",
668 cfg->cdb_size, cfg->sense_size);
669 free(cfg);
670 continue;
671 }
672 vd->cfg = cfg;
673
674 if((s = malloc(sizeof(*s))) == nil)
675 break;
676 s->ctlr = vd;
677 s->idno = id++;
678 s->ifc = &sdvirtioifc;
679 s->nunit = cfg->max_target;
680 if(h)
681 t->next = s;
682 else
683 h = s;
684 t = s;
685 }
686
687 return h;
688 }
689
690 SDifc sdvirtioifc = {
691 "virtio", /* name */
692
693 viopnp, /* pnp */
694 nil, /* legacy */
695 vioenable, /* enable */
696 viodisable, /* disable */
697
698 vioverify, /* verify */
699 vioonline, /* online */
700 viorio, /* rio */
701 nil, /* rctl */
702 nil, /* wctl */
703
704 viobio, /* bio */
705 nil, /* probe */
706 nil, /* clear */
707 nil, /* rtopctl */
708 nil, /* wtopctl */
709 };
710