1 /*
2 * virtio ethernet driver implementing the legacy interface:
3 * http://docs.oasis-open.org/virtio/virtio/v1.0/virtio-v1.0.html
4 */
5 #include "u.h"
6 #include "../port/lib.h"
7 #include "mem.h"
8 #include "dat.h"
9 #include "fns.h"
10 #include "io.h"
11 #include "ureg.h"
12 #include "../port/error.h"
13
14 #include "../port/sd.h"
15
16 typedef struct Vring Vring;
17 typedef struct Vdesc Vdesc;
18 typedef struct Vused Vused;
19 typedef struct Vqueue Vqueue;
20 typedef struct Vdev Vdev;
21
22 typedef struct ScsiCfg ScsiCfg;
23
24 /* device types */
25 enum {
26 TypBlk = 2,
27 TypSCSI = 8,
28 };
29
30 /* status flags */
31 enum {
32 Acknowledge = 1,
33 Driver = 2,
34 DriverOk = 4,
35 Failed = 0x80,
36 };
37
38 /* virtio ports */
39 enum {
40 Devfeat = 0,
41 Drvfeat = 4,
42 Qaddr = 8,
43 Qsize = 12,
44 Qselect = 14,
45 Qnotify = 16,
46 Status = 18,
47 Isr = 19,
48
49 Devspec = 20,
50 };
51
52 /* descriptor flags */
53 enum {
54 _Next = 1,
55 Write = 2,
56 Indirect = 4,
57 };
58
59 /* struct sizes */
60 enum {
61 VringSize = 4,
62 };
63
64 struct Vring
65 {
66 u16int flags;
67 u16int idx;
68 };
69
70 struct Vdesc
71 {
72 u64int addr;
73 u32int len;
74 u16int flags;
75 u16int next;
76 };
77
78 struct Vused
79 {
80 u32int id;
81 u32int len;
82 };
83
84 struct Vqueue
85 {
86 Lock;
87
88 Vdev *dev;
89 int idx;
90
91 int size;
92
93 int free;
94 int nfree;
95
96 Vdesc *desc;
97
98 Vring *avail;
99 u16int *availent;
100 u16int *availevent;
101
102 Vring *used;
103 Vused *usedent;
104 u16int *usedevent;
105 u16int lastused;
106
107 void *rock[];
108 };
109
110 struct Vdev
111 {
112 int typ;
113
114 Pcidev *pci;
115 void* vector;
116
117 ulong port;
118 ulong feat;
119
120 int nqueue;
121 Vqueue *queue[16];
122
123 void *cfg; /* device specific config (for scsi) */
124
125 Vdev *next;
126 };
127
128 enum {
129 CDBSIZE = 32,
130 SENSESIZE = 96,
131 };
132
133 struct ScsiCfg
134 {
135 u32int num_queues;
136 u32int seg_max;
137 u32int max_sectors;
138 u32int cmd_per_lun;
139 u32int event_info_size;
140 u32int sense_size;
141 u32int cdb_size;
142 u16int max_channel;
143 u16int max_target;
144 u32int max_lun;
145 };
146
147 static Vqueue*
mkvqueue(int size)148 mkvqueue(int size)
149 {
150 Vqueue *q;
151 uchar *p;
152 int i;
153
154 q = malloc(sizeof(*q) + sizeof(void*)*size);
155 p = mallocalign(
156 ROUNDUP(sizeof(Vdesc)*size +
157 VringSize +
158 sizeof(u16int)*size +
159 sizeof(u16int),
160 4*KiB) +
161 ROUNDUP(VringSize +
162 sizeof(Vused)*size +
163 sizeof(u16int),
164 4*KiB),
165 PGSZ, 0, 0);
166 if(p == nil || q == nil){
167 print("virtio: no memory for Vqueue\n");
168 free(p);
169 free(q);
170 return nil;
171 }
172
173 q->desc = (void*)p;
174 p += sizeof(Vdesc)*size;
175 q->avail = (void*)p;
176 p += VringSize;
177 q->availent = (void*)p;
178 p += sizeof(u16int)*size;
179 q->availevent = (void*)p;
180 p += sizeof(u16int);
181
182 p = (uchar*)ROUNDUP((uintptr)p, 4*KiB);
183 q->used = (void*)p;
184 p += VringSize;
185 q->usedent = (void*)p;
186 p += sizeof(Vused)*size;
187 q->usedevent = (void*)p;
188
189 q->free = -1;
190 q->nfree = q->size = size;
191 for(i=0; i<size; i++){
192 q->desc[i].next = q->free;
193 q->free = i;
194 }
195
196 return q;
197 }
198
199 static Vdev*
viopnpdevs(int typ)200 viopnpdevs(int typ)
201 {
202 Vdev *vd, *h, *t;
203 Vqueue *q;
204 Pcidev *p;
205 int n, i, size;
206
207 h = t = nil;
208 for(p = nil; p = pcimatch(p, 0x1AF4, 0);){
209 if((p->did < 0x1000) || (p->did > 0x103F))
210 continue;
211 if(p->rid != 0)
212 continue;
213 //if((p->mem[0].bar & 1) == 0) /* zero on gce */
214 // continue;
215 if(pcicfgr16(p, 0x2E) != typ)
216 continue;
217 if((vd = malloc(sizeof(*vd))) == nil){
218 print("virtio: no memory for Vdev\n");
219 break;
220 }
221 vd->port = p->mem[0].bar & ~3;
222 size = p->mem[0].size;
223 if(vd->port == 0){ /* gce */
224 vd->port = 0xc000;
225 size = 0x40;
226 }
227 if(ioalloc(vd->port, size, 0, "virtio") < 0){
228 print("virtio: port %lux in use\n", vd->port);
229 free(vd);
230 continue;
231 }
232 vd->typ = typ;
233 vd->pci = p;
234
235 /* reset */
236 outb(vd->port+Status, 0);
237
238 vd->feat = inl(vd->port+Devfeat);
239 outb(vd->port+Status, Acknowledge|Driver);
240 for(i=0; i<nelem(vd->queue); i++){
241 outs(vd->port+Qselect, i);
242 n = ins(vd->port+Qsize);
243 if(n == 0 || (n & (n-1)) != 0)
244 break;
245 if((q = mkvqueue(n)) == nil)
246 break;
247 q->dev = vd;
248 q->idx = i;
249 vd->queue[i] = q;
250 coherence();
251 outl(vd->port+Qaddr, PADDR(vd->queue[i]->desc)/PGSZ);
252 }
253 vd->nqueue = i;
254
255 if(h == nil)
256 h = vd;
257 else
258 t->next = vd;
259 t = vd;
260 }
261
262 return h;
263 }
264
265 struct Rock {
266 int done;
267 Rendez *sleep;
268 };
269
270 static void
vqinterrupt(Vqueue * q)271 vqinterrupt(Vqueue *q)
272 {
273 int id, free, m;
274 struct Rock *r;
275 Rendez *z;
276
277 m = q->size-1;
278
279 ilock(q);
280 while((q->lastused ^ q->used->idx) & m){
281 id = q->usedent[q->lastused++ & m].id;
282 if(r = q->rock[id]){
283 q->rock[id] = nil;
284 z = r->sleep;
285 r->done = 1; /* hands off */
286 if(z != nil)
287 wakeup(z);
288 }
289 do {
290 free = id;
291 id = q->desc[free].next;
292 q->desc[free].next = q->free;
293 q->free = free;
294 q->nfree++;
295 } while(q->desc[free].flags & _Next);
296 }
297 iunlock(q);
298 }
299
300 static void
viointerrupt(Ureg *,void * arg)301 viointerrupt(Ureg *, void *arg)
302 {
303 Vdev *vd = arg;
304
305 if(inb(vd->port+Isr) & 1)
306 vqinterrupt(vd->queue[vd->typ == TypSCSI ? 2 : 0]);
307 }
308
309 static int
viodone(void * arg)310 viodone(void *arg)
311 {
312 return ((struct Rock*)arg)->done;
313 }
314
315 static void
vqio(Vqueue * q,int head)316 vqio(Vqueue *q, int head)
317 {
318 struct Rock rock;
319
320 rock.done = 0;
321 rock.sleep = &up->sleep;
322 q->rock[head] = &rock;
323 q->availent[q->avail->idx & (q->size-1)] = head;
324 coherence();
325 q->avail->idx++;
326 iunlock(q);
327 if((q->used->flags & 1) == 0)
328 outs(q->dev->port+Qnotify, q->idx);
329 while(!rock.done){
330 while(waserror())
331 ;
332 tsleep(rock.sleep, viodone, &rock, 1000);
333 poperror();
334
335 if(!rock.done)
336 vqinterrupt(q);
337 }
338 }
339
340 static int
vioblkreq(Vdev * vd,int typ,void * a,long count,long secsize,uvlong lba)341 vioblkreq(Vdev *vd, int typ, void *a, long count, long secsize, uvlong lba)
342 {
343 int need, free, head;
344 Vqueue *q;
345 Vdesc *d;
346
347 u8int status;
348 struct Vioblkreqhdr {
349 u32int typ;
350 u32int prio;
351 u64int lba;
352 } req;
353
354 need = 2;
355 if(a != nil)
356 need = 3;
357
358 status = -1;
359 req.typ = typ;
360 req.prio = 0;
361 req.lba = lba;
362
363 q = vd->queue[0];
364 ilock(q);
365 while(q->nfree < need){
366 iunlock(q);
367
368 if(!waserror())
369 tsleep(&up->sleep, return0, 0, 500);
370 poperror();
371
372 ilock(q);
373 }
374
375 head = free = q->free;
376
377 d = &q->desc[free]; free = d->next;
378 d->addr = PADDR(&req);
379 d->len = sizeof(req);
380 d->flags = _Next;
381
382 if(a != nil){
383 d = &q->desc[free]; free = d->next;
384 d->addr = PADDR(a);
385 d->len = secsize*count;
386 d->flags = typ ? _Next : (Write|_Next);
387 }
388
389 d = &q->desc[free]; free = d->next;
390 d->addr = PADDR(&status);
391 d->len = sizeof(status);
392 d->flags = Write;
393
394 q->free = free;
395 q->nfree -= need;
396
397 /* queue io, unlock and wait for completion */
398 vqio(q, head);
399
400 return status;
401 }
402
403 static int
vioscsireq(SDreq * r)404 vioscsireq(SDreq *r)
405 {
406 u8int resp[4+4+2+2+SENSESIZE];
407 u8int req[8+8+3+CDBSIZE];
408 int free, head;
409 u32int len;
410 Vqueue *q;
411 Vdesc *d;
412 Vdev *vd;
413 SDunit *u;
414 ScsiCfg *cfg;
415
416 u = r->unit;
417 vd = u->dev->ctlr;
418 cfg = vd->cfg;
419
420 memset(resp, 0, sizeof(resp));
421 memset(req, 0, sizeof(req));
422 req[0] = 1;
423 req[1] = u->subno;
424 req[2] = r->lun>>8;
425 req[3] = r->lun&0xFF;
426 *(u64int*)(&req[8]) = (uintptr)r;
427
428 memmove(&req[8+8+3], r->cmd, r->clen);
429
430 q = vd->queue[2];
431 ilock(q);
432 while(q->nfree < 3){
433 iunlock(q);
434
435 if(!waserror())
436 tsleep(&up->sleep, return0, 0, 500);
437 poperror();
438
439 ilock(q);
440 }
441
442 head = free = q->free;
443
444 d = &q->desc[free]; free = d->next;
445 d->addr = PADDR(req);
446 d->len = 8+8+3+cfg->cdb_size;
447 d->flags = _Next;
448
449 if(r->write && r->dlen > 0){
450 d = &q->desc[free]; free = d->next;
451 d->addr = PADDR(r->data);
452 d->len = r->dlen;
453 d->flags = _Next;
454 }
455
456 d = &q->desc[free]; free = d->next;
457 d->addr = PADDR(resp);
458 d->len = 4+4+2+2+cfg->sense_size;
459 d->flags = Write;
460
461 if(!r->write && r->dlen > 0){
462 d->flags |= _Next;
463
464 d = &q->desc[free]; free = d->next;
465 d->addr = PADDR(r->data);
466 d->len = r->dlen;
467 d->flags = Write;
468 }
469
470 q->free = free;
471 q->nfree -= 2 + (r->dlen > 0);
472
473 /* queue io, unlock and wait for completion */
474 vqio(q, head);
475
476 /* response+status */
477 r->status = resp[10];
478 if(resp[11] != 0)
479 r->status = SDcheck;
480
481 /* sense_len */
482 len = *((u32int*)&resp[0]);
483 if(len > 0){
484 if(len > sizeof(r->sense))
485 len = sizeof(r->sense);
486 memmove(r->sense, &resp[4+4+2+2], len);
487 r->flags |= SDvalidsense;
488 }
489
490 /* data residue */
491 len = *((u32int*)&resp[4]);
492 if(len > r->dlen)
493 r->rlen = 0;
494 else
495 r->rlen = r->dlen - len;
496
497 return r->status;
498
499 }
500
501 static long
viobio(SDunit * u,int lun,int write,void * a,long count,uvlong lba)502 viobio(SDunit *u, int lun, int write, void *a, long count, uvlong lba)
503 {
504 long ss, cc, max, ret;
505 Vdev *vd;
506
507 vd = u->dev->ctlr;
508 if(vd->typ == TypSCSI)
509 return scsibio(u, lun, write, a, count, lba);
510
511 max = 32;
512 ss = u->secsize;
513 ret = 0;
514 while(count > 0){
515 if((cc = count) > max)
516 cc = max;
517 if(vioblkreq(vd, write != 0, (uchar*)a + ret, cc, ss, lba) != 0)
518 error(Eio);
519 ret += cc*ss;
520 count -= cc;
521 lba += cc;
522 }
523 return ret;
524 }
525
526 enum {
527 SDread,
528 SDwrite,
529 };
530
531 static int
viorio(SDreq * r)532 viorio(SDreq *r)
533 {
534 int i, count, rw;
535 uvlong lba;
536 SDunit *u;
537 Vdev *vd;
538
539 u = r->unit;
540 vd = u->dev->ctlr;
541 if(vd->typ == TypSCSI)
542 return vioscsireq(r);
543 if(r->cmd[0] == 0x35 || r->cmd[0] == 0x91){
544 if(vioblkreq(vd, 4, nil, 0, 0, 0) != 0)
545 return sdsetsense(r, SDcheck, 3, 0xc, 2);
546 return sdsetsense(r, SDok, 0, 0, 0);
547 }
548 if((i = sdfakescsi(r, nil, 0)) != SDnostatus)
549 return r->status = i;
550 if((i = sdfakescsirw(r, &lba, &count, &rw)) != SDnostatus)
551 return i;
552 r->rlen = viobio(u, r->lun, rw == SDwrite, r->data, count, lba);
553 return r->status = SDok;
554 }
555
556 static int
vioonline(SDunit * u)557 vioonline(SDunit *u)
558 {
559 uvlong cap;
560 Vdev *vd;
561
562 vd = u->dev->ctlr;
563 if(vd->typ == TypSCSI)
564 return scsionline(u);
565
566 cap = inl(vd->port+Devspec+4);
567 cap <<= 32;
568 cap |= inl(vd->port+Devspec);
569 if(u->sectors != cap){
570 u->sectors = cap;
571 u->secsize = 512;
572 return 2;
573 }
574 return 1;
575 }
576
577 static int
vioverify(SDunit * u)578 vioverify(SDunit *u)
579 {
580 Vdev *vd;
581
582 vd = u->dev->ctlr;
583 if(vd->typ == TypSCSI)
584 return scsiverify(u);
585
586 return 1;
587 }
588
589 SDifc sdvirtioifc;
590
591 static int
vioenable(SDev * sd)592 vioenable(SDev *sd)
593 {
594 char name[32];
595 Vdev *vd;
596
597 vd = sd->ctlr;
598 pcisetbme(vd->pci);
599 snprint(name, sizeof(name), "%s (%s)", sd->name, sd->ifc->name);
600 vd->vector = intrenable(vd->pci->intl, viointerrupt, vd, vd->pci->tbdf, name);
601 outb(vd->port+Status, inb(vd->port+Status) | DriverOk);
602 return 1;
603 }
604
605 static int
viodisable(SDev * sd)606 viodisable(SDev *sd)
607 {
608 Vdev *vd;
609
610 vd = sd->ctlr;
611 intrdisable(vd->vector);
612 pciclrbme(vd->pci);
613 return 1;
614 }
615
616 static SDev*
viopnp(void)617 viopnp(void)
618 {
619 SDev *s, *h, *t;
620 Vdev *vd;
621 int id;
622
623 h = t = nil;
624
625 id = 'F';
626 for(vd = viopnpdevs(TypBlk); vd; vd = vd->next){
627 if(vd->nqueue == 0)
628 continue;
629
630 if((s = malloc(sizeof(*s))) == nil)
631 break;
632 s->ctlr = vd;
633 s->idno = id++;
634 s->ifc = &sdvirtioifc;
635 s->nunit = 1;
636 if(h)
637 t->next = s;
638 else
639 h = s;
640 t = s;
641 }
642
643 id = '0';
644 for(vd = viopnpdevs(TypSCSI); vd; vd = vd->next){
645 ScsiCfg *cfg;
646
647 if(vd->nqueue < 3)
648 continue;
649
650 if((cfg = malloc(sizeof(*cfg))) == nil)
651 break;
652 cfg->num_queues = inl(vd->port+Devspec+4*0);
653 cfg->seg_max = inl(vd->port+Devspec+4*1);
654 cfg->max_sectors = inl(vd->port+Devspec+4*2);
655 cfg->cmd_per_lun = inl(vd->port+Devspec+4*3);
656 cfg->event_info_size = inl(vd->port+Devspec+4*4);
657 cfg->sense_size = inl(vd->port+Devspec+4*5);
658 cfg->cdb_size = inl(vd->port+Devspec+4*6);
659 cfg->max_channel = ins(vd->port+Devspec+4*7);
660 cfg->max_target = ins(vd->port+Devspec+4*7+2);
661 cfg->max_lun = inl(vd->port+Devspec+4*8);
662
663 if(cfg->max_target == 0){
664 free(cfg);
665 continue;
666 }
667 if((cfg->cdb_size > CDBSIZE) || (cfg->sense_size > SENSESIZE)){
668 print("sdvirtio: cdb %ud or sense size %ud too big\n",
669 cfg->cdb_size, cfg->sense_size);
670 free(cfg);
671 continue;
672 }
673 vd->cfg = cfg;
674
675 if((s = malloc(sizeof(*s))) == nil)
676 break;
677 s->ctlr = vd;
678 s->idno = id++;
679 s->ifc = &sdvirtioifc;
680 s->nunit = cfg->max_target;
681 if(h)
682 t->next = s;
683 else
684 h = s;
685 t = s;
686 }
687
688 return h;
689 }
690
691 SDifc sdvirtioifc = {
692 "virtio", /* name */
693
694 viopnp, /* pnp */
695 nil, /* legacy */
696 vioenable, /* enable */
697 viodisable, /* disable */
698
699 vioverify, /* verify */
700 vioonline, /* online */
701 viorio, /* rio */
702 nil, /* rctl */
703 nil, /* wctl */
704
705 viobio, /* bio */
706 nil, /* probe */
707 nil, /* clear */
708 nil, /* rtopctl */
709 nil, /* wtopctl */
710 };
711