1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020-2021 Xilinx, Inc.
3 */
4
5 #include <pthread.h>
6 #include <unistd.h>
7 #include <sys/ioctl.h>
8
9 #include <rte_errno.h>
10 #include <rte_malloc.h>
11 #include <rte_vdpa.h>
12 #include <rte_vfio.h>
13 #include <rte_vhost.h>
14
15 #include <vdpa_driver.h>
16
17 #include "efx.h"
18 #include "sfc_vdpa_ops.h"
19 #include "sfc_vdpa.h"
20
21 /* These protocol features are needed to enable notifier ctrl */
22 #define SFC_VDPA_PROTOCOL_FEATURES \
23 ((1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK) | \
24 (1ULL << VHOST_USER_PROTOCOL_F_BACKEND_REQ) | \
25 (1ULL << VHOST_USER_PROTOCOL_F_BACKEND_SEND_FD) | \
26 (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
27 (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) | \
28 (1ULL << VHOST_USER_PROTOCOL_F_MQ))
29
30 /*
31 * Set of features which are enabled by default.
32 * Protocol feature bit is needed to enable notification notifier ctrl.
33 */
34 #define SFC_VDPA_DEFAULT_FEATURES \
35 ((1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
36 (1ULL << VIRTIO_NET_F_MQ))
37
38 #define SFC_VDPA_MSIX_IRQ_SET_BUF_LEN \
39 (sizeof(struct vfio_irq_set) + \
40 sizeof(int) * (SFC_VDPA_MAX_QUEUE_PAIRS * 2 + 1))
41
42 /* It will be used for target VF when calling function is not PF */
43 #define SFC_VDPA_VF_NULL 0xFFFF
44
45 static int
sfc_vdpa_get_device_features(struct sfc_vdpa_ops_data * ops_data)46 sfc_vdpa_get_device_features(struct sfc_vdpa_ops_data *ops_data)
47 {
48 int rc;
49 uint64_t dev_features;
50 efx_nic_t *nic;
51
52 nic = sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle)->nic;
53
54 rc = efx_virtio_get_features(nic, EFX_VIRTIO_DEVICE_TYPE_NET,
55 &dev_features);
56 if (rc != 0) {
57 sfc_vdpa_err(ops_data->dev_handle,
58 "could not read device feature: %s",
59 rte_strerror(rc));
60 return rc;
61 }
62
63 ops_data->dev_features = dev_features;
64
65 sfc_vdpa_info(ops_data->dev_handle,
66 "device supported virtio features : 0x%" PRIx64,
67 ops_data->dev_features);
68
69 return 0;
70 }
71
72 static uint64_t
hva_to_gpa(int vid,uint64_t hva)73 hva_to_gpa(int vid, uint64_t hva)
74 {
75 struct rte_vhost_memory *vhost_mem = NULL;
76 struct rte_vhost_mem_region *mem_reg = NULL;
77 uint32_t i;
78 uint64_t gpa = 0;
79
80 if (rte_vhost_get_mem_table(vid, &vhost_mem) < 0)
81 goto error;
82
83 for (i = 0; i < vhost_mem->nregions; i++) {
84 mem_reg = &vhost_mem->regions[i];
85
86 if (hva >= mem_reg->host_user_addr &&
87 hva < mem_reg->host_user_addr + mem_reg->size) {
88 gpa = (hva - mem_reg->host_user_addr) +
89 mem_reg->guest_phys_addr;
90 break;
91 }
92 }
93
94 error:
95 free(vhost_mem);
96 return gpa;
97 }
98
99 static int
sfc_vdpa_enable_vfio_intr(struct sfc_vdpa_ops_data * ops_data)100 sfc_vdpa_enable_vfio_intr(struct sfc_vdpa_ops_data *ops_data)
101 {
102 int rc;
103 int *irq_fd_ptr;
104 int vfio_dev_fd;
105 uint32_t i, num_vring;
106 struct rte_vhost_vring vring;
107 struct vfio_irq_set *irq_set;
108 struct rte_pci_device *pci_dev;
109 char irq_set_buf[SFC_VDPA_MSIX_IRQ_SET_BUF_LEN];
110 void *dev;
111
112 num_vring = rte_vhost_get_vring_num(ops_data->vid);
113 dev = ops_data->dev_handle;
114 vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
115 pci_dev = sfc_vdpa_adapter_by_dev_handle(dev)->pdev;
116
117 irq_set = (struct vfio_irq_set *)irq_set_buf;
118 irq_set->argsz = sizeof(irq_set_buf);
119 irq_set->count = num_vring + 1;
120 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
121 VFIO_IRQ_SET_ACTION_TRIGGER;
122 irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
123 irq_set->start = 0;
124 irq_fd_ptr = (int *)&irq_set->data;
125 irq_fd_ptr[RTE_INTR_VEC_ZERO_OFFSET] =
126 rte_intr_fd_get(pci_dev->intr_handle);
127
128 for (i = 0; i < num_vring; i++) {
129 rc = rte_vhost_get_vhost_vring(ops_data->vid, i, &vring);
130 if (rc)
131 return -1;
132
133 irq_fd_ptr[RTE_INTR_VEC_RXTX_OFFSET + i] = vring.callfd;
134 }
135
136 rc = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
137 if (rc) {
138 sfc_vdpa_err(ops_data->dev_handle,
139 "error enabling MSI-X interrupts: %s",
140 strerror(errno));
141 return -1;
142 }
143
144 return 0;
145 }
146
147 static int
sfc_vdpa_disable_vfio_intr(struct sfc_vdpa_ops_data * ops_data)148 sfc_vdpa_disable_vfio_intr(struct sfc_vdpa_ops_data *ops_data)
149 {
150 int rc;
151 int vfio_dev_fd;
152 struct vfio_irq_set irq_set;
153 void *dev;
154
155 dev = ops_data->dev_handle;
156 vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
157
158 irq_set.argsz = sizeof(irq_set);
159 irq_set.count = 0;
160 irq_set.flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
161 irq_set.index = VFIO_PCI_MSIX_IRQ_INDEX;
162 irq_set.start = 0;
163
164 rc = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, &irq_set);
165 if (rc) {
166 sfc_vdpa_err(ops_data->dev_handle,
167 "error disabling MSI-X interrupts: %s",
168 strerror(errno));
169 return -1;
170 }
171
172 return 0;
173 }
174
175 static int
sfc_vdpa_get_vring_info(struct sfc_vdpa_ops_data * ops_data,int vq_num,struct sfc_vdpa_vring_info * vring)176 sfc_vdpa_get_vring_info(struct sfc_vdpa_ops_data *ops_data,
177 int vq_num, struct sfc_vdpa_vring_info *vring)
178 {
179 int rc;
180 uint64_t gpa;
181 struct rte_vhost_vring vq;
182
183 rc = rte_vhost_get_vhost_vring(ops_data->vid, vq_num, &vq);
184 if (rc < 0) {
185 sfc_vdpa_err(ops_data->dev_handle,
186 "get vhost vring failed: %s", rte_strerror(rc));
187 return rc;
188 }
189
190 gpa = hva_to_gpa(ops_data->vid, (uint64_t)(uintptr_t)vq.desc);
191 if (gpa == 0) {
192 sfc_vdpa_err(ops_data->dev_handle,
193 "fail to get GPA for descriptor ring.");
194 return -1;
195 }
196 vring->desc = gpa;
197
198 gpa = hva_to_gpa(ops_data->vid, (uint64_t)(uintptr_t)vq.avail);
199 if (gpa == 0) {
200 sfc_vdpa_err(ops_data->dev_handle,
201 "fail to get GPA for available ring.");
202 return -1;
203 }
204 vring->avail = gpa;
205
206 gpa = hva_to_gpa(ops_data->vid, (uint64_t)(uintptr_t)vq.used);
207 if (gpa == 0) {
208 sfc_vdpa_err(ops_data->dev_handle,
209 "fail to get GPA for used ring.");
210 return -1;
211 }
212 vring->used = gpa;
213
214 vring->size = vq.size;
215
216 rc = rte_vhost_get_vring_base(ops_data->vid, vq_num,
217 &vring->last_avail_idx,
218 &vring->last_used_idx);
219
220 return rc;
221 }
222
223 static int
sfc_vdpa_virtq_start(struct sfc_vdpa_ops_data * ops_data,int vq_num)224 sfc_vdpa_virtq_start(struct sfc_vdpa_ops_data *ops_data, int vq_num)
225 {
226 int rc;
227 uint32_t doorbell;
228 efx_virtio_vq_t *vq;
229 struct sfc_vdpa_vring_info vring;
230 efx_virtio_vq_cfg_t vq_cfg;
231 efx_virtio_vq_dyncfg_t vq_dyncfg;
232
233 vq = ops_data->vq_cxt[vq_num].vq;
234 if (vq == NULL)
235 return -1;
236
237 rc = sfc_vdpa_get_vring_info(ops_data, vq_num, &vring);
238 if (rc < 0) {
239 sfc_vdpa_err(ops_data->dev_handle,
240 "get vring info failed: %s", rte_strerror(rc));
241 goto fail_vring_info;
242 }
243
244 vq_cfg.evvc_target_vf = SFC_VDPA_VF_NULL;
245
246 /* even virtqueue for RX and odd for TX */
247 if (vq_num % 2) {
248 vq_cfg.evvc_type = EFX_VIRTIO_VQ_TYPE_NET_TXQ;
249 sfc_vdpa_info(ops_data->dev_handle,
250 "configure virtqueue # %d (TXQ)", vq_num);
251 } else {
252 vq_cfg.evvc_type = EFX_VIRTIO_VQ_TYPE_NET_RXQ;
253 sfc_vdpa_info(ops_data->dev_handle,
254 "configure virtqueue # %d (RXQ)", vq_num);
255 }
256
257 vq_cfg.evvc_vq_num = vq_num;
258 vq_cfg.evvc_desc_tbl_addr = vring.desc;
259 vq_cfg.evvc_avail_ring_addr = vring.avail;
260 vq_cfg.evvc_used_ring_addr = vring.used;
261 vq_cfg.evvc_vq_size = vring.size;
262
263 vq_dyncfg.evvd_vq_used_idx = vring.last_used_idx;
264 vq_dyncfg.evvd_vq_avail_idx = vring.last_avail_idx;
265
266 /* MSI-X vector is function-relative */
267 vq_cfg.evvc_msix_vector = RTE_INTR_VEC_RXTX_OFFSET + vq_num;
268 if (ops_data->vdpa_context == SFC_VDPA_AS_VF)
269 vq_cfg.evvc_pas_id = 0;
270 vq_cfg.evcc_features = ops_data->dev_features &
271 ops_data->req_features;
272
273 /* Start virtqueue */
274 rc = efx_virtio_qstart(vq, &vq_cfg, &vq_dyncfg);
275 if (rc != 0) {
276 sfc_vdpa_err(ops_data->dev_handle,
277 "virtqueue start failed: %s",
278 rte_strerror(rc));
279 goto fail_virtio_qstart;
280 }
281
282 sfc_vdpa_info(ops_data->dev_handle,
283 "virtqueue started successfully for vq_num %d", vq_num);
284
285 rc = efx_virtio_get_doorbell_offset(vq, &doorbell);
286 if (rc != 0) {
287 sfc_vdpa_err(ops_data->dev_handle,
288 "failed to get doorbell offset: %s",
289 rte_strerror(rc));
290 goto fail_doorbell;
291 }
292
293 /*
294 * Cache the bar_offset here for each VQ here, it will come
295 * in handy when sfc_vdpa_get_notify_area() is invoked.
296 */
297 ops_data->vq_cxt[vq_num].doorbell = (void *)(uintptr_t)doorbell;
298 ops_data->vq_cxt[vq_num].enable = B_TRUE;
299
300 return rc;
301
302 fail_doorbell:
303 fail_virtio_qstart:
304 efx_virtio_qdestroy(vq);
305 fail_vring_info:
306 return rc;
307 }
308
309 static int
sfc_vdpa_virtq_stop(struct sfc_vdpa_ops_data * ops_data,int vq_num)310 sfc_vdpa_virtq_stop(struct sfc_vdpa_ops_data *ops_data, int vq_num)
311 {
312 int rc;
313 efx_virtio_vq_dyncfg_t vq_idx;
314 efx_virtio_vq_t *vq;
315
316 if (ops_data->vq_cxt[vq_num].enable != B_TRUE)
317 return -1;
318
319 vq = ops_data->vq_cxt[vq_num].vq;
320 if (vq == NULL)
321 return -1;
322
323 /* stop the vq */
324 rc = efx_virtio_qstop(vq, &vq_idx);
325 if (rc == 0) {
326 ops_data->vq_cxt[vq_num].cidx = vq_idx.evvd_vq_used_idx;
327 ops_data->vq_cxt[vq_num].pidx = vq_idx.evvd_vq_avail_idx;
328 }
329 ops_data->vq_cxt[vq_num].enable = B_FALSE;
330
331 return rc;
332 }
333
334 static int
sfc_vdpa_configure(struct sfc_vdpa_ops_data * ops_data)335 sfc_vdpa_configure(struct sfc_vdpa_ops_data *ops_data)
336 {
337 int rc, i;
338 int nr_vring;
339 int max_vring_cnt;
340 efx_virtio_vq_t *vq;
341 efx_nic_t *nic;
342 void *dev;
343
344 dev = ops_data->dev_handle;
345 nic = sfc_vdpa_adapter_by_dev_handle(dev)->nic;
346
347 SFC_EFX_ASSERT(ops_data->state == SFC_VDPA_STATE_INITIALIZED);
348
349 ops_data->state = SFC_VDPA_STATE_CONFIGURING;
350
351 nr_vring = rte_vhost_get_vring_num(ops_data->vid);
352 max_vring_cnt =
353 (sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count * 2);
354
355 /* number of vring should not be more than supported max vq count */
356 if (nr_vring > max_vring_cnt) {
357 sfc_vdpa_err(dev,
358 "nr_vring (%d) is > max vring count (%d)",
359 nr_vring, max_vring_cnt);
360 goto fail_vring_num;
361 }
362
363 rc = sfc_vdpa_dma_map(ops_data, true);
364 if (rc) {
365 sfc_vdpa_err(dev,
366 "DMA map failed: %s", rte_strerror(rc));
367 goto fail_dma_map;
368 }
369
370 for (i = 0; i < nr_vring; i++) {
371 rc = efx_virtio_qcreate(nic, &vq);
372 if ((rc != 0) || (vq == NULL)) {
373 sfc_vdpa_err(dev,
374 "virtqueue create failed: %s",
375 rte_strerror(rc));
376 goto fail_vq_create;
377 }
378
379 /* store created virtqueue context */
380 ops_data->vq_cxt[i].vq = vq;
381 }
382
383 ops_data->vq_count = i;
384
385 ops_data->state = SFC_VDPA_STATE_CONFIGURED;
386
387 return 0;
388
389 fail_vq_create:
390 sfc_vdpa_dma_map(ops_data, false);
391
392 fail_dma_map:
393 fail_vring_num:
394 ops_data->state = SFC_VDPA_STATE_INITIALIZED;
395
396 return -1;
397 }
398
399 static void
sfc_vdpa_close(struct sfc_vdpa_ops_data * ops_data)400 sfc_vdpa_close(struct sfc_vdpa_ops_data *ops_data)
401 {
402 int i;
403
404 if (ops_data->state != SFC_VDPA_STATE_CONFIGURED)
405 return;
406
407 ops_data->state = SFC_VDPA_STATE_CLOSING;
408
409 for (i = 0; i < ops_data->vq_count; i++) {
410 if (ops_data->vq_cxt[i].vq == NULL)
411 continue;
412
413 efx_virtio_qdestroy(ops_data->vq_cxt[i].vq);
414 }
415
416 sfc_vdpa_dma_map(ops_data, false);
417
418 ops_data->state = SFC_VDPA_STATE_INITIALIZED;
419 }
420
421 static void
sfc_vdpa_stop(struct sfc_vdpa_ops_data * ops_data)422 sfc_vdpa_stop(struct sfc_vdpa_ops_data *ops_data)
423 {
424 int i;
425 int rc;
426
427 if (ops_data->state != SFC_VDPA_STATE_STARTED)
428 return;
429
430 ops_data->state = SFC_VDPA_STATE_STOPPING;
431
432 for (i = 0; i < ops_data->vq_count; i++) {
433 rc = sfc_vdpa_virtq_stop(ops_data, i);
434 if (rc != 0)
435 continue;
436 }
437
438 sfc_vdpa_disable_vfio_intr(ops_data);
439
440 sfc_vdpa_filter_remove(ops_data);
441
442 ops_data->state = SFC_VDPA_STATE_CONFIGURED;
443 }
444
445 static int
sfc_vdpa_start(struct sfc_vdpa_ops_data * ops_data)446 sfc_vdpa_start(struct sfc_vdpa_ops_data *ops_data)
447 {
448 int i, j;
449 int rc;
450
451 SFC_EFX_ASSERT(ops_data->state == SFC_VDPA_STATE_CONFIGURED);
452
453 sfc_vdpa_log_init(ops_data->dev_handle, "entry");
454
455 ops_data->state = SFC_VDPA_STATE_STARTING;
456
457 sfc_vdpa_log_init(ops_data->dev_handle, "enable interrupts");
458 rc = sfc_vdpa_enable_vfio_intr(ops_data);
459 if (rc < 0) {
460 sfc_vdpa_err(ops_data->dev_handle,
461 "vfio intr allocation failed: %s",
462 rte_strerror(rc));
463 goto fail_enable_vfio_intr;
464 }
465
466 rte_vhost_get_negotiated_features(ops_data->vid,
467 &ops_data->req_features);
468
469 sfc_vdpa_info(ops_data->dev_handle,
470 "negotiated feature : 0x%" PRIx64,
471 ops_data->req_features);
472
473 for (i = 0; i < ops_data->vq_count; i++) {
474 sfc_vdpa_log_init(ops_data->dev_handle,
475 "starting vq# %d", i);
476 rc = sfc_vdpa_virtq_start(ops_data, i);
477 if (rc != 0)
478 goto fail_vq_start;
479 }
480
481 ops_data->vq_count = i;
482
483 sfc_vdpa_log_init(ops_data->dev_handle,
484 "configure MAC filters");
485 rc = sfc_vdpa_filter_config(ops_data);
486 if (rc != 0) {
487 sfc_vdpa_err(ops_data->dev_handle,
488 "MAC filter config failed: %s",
489 rte_strerror(rc));
490 goto fail_filter_cfg;
491 }
492
493 ops_data->state = SFC_VDPA_STATE_STARTED;
494
495 sfc_vdpa_log_init(ops_data->dev_handle, "done");
496
497 return 0;
498
499 fail_filter_cfg:
500 /* remove already created filters */
501 sfc_vdpa_filter_remove(ops_data);
502 fail_vq_start:
503 /* stop already started virtqueues */
504 for (j = 0; j < i; j++)
505 sfc_vdpa_virtq_stop(ops_data, j);
506 sfc_vdpa_disable_vfio_intr(ops_data);
507
508 fail_enable_vfio_intr:
509 ops_data->state = SFC_VDPA_STATE_CONFIGURED;
510
511 return rc;
512 }
513
514 static int
sfc_vdpa_get_queue_num(struct rte_vdpa_device * vdpa_dev,uint32_t * queue_num)515 sfc_vdpa_get_queue_num(struct rte_vdpa_device *vdpa_dev, uint32_t *queue_num)
516 {
517 struct sfc_vdpa_ops_data *ops_data;
518 void *dev;
519
520 ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
521 if (ops_data == NULL)
522 return -1;
523
524 dev = ops_data->dev_handle;
525 *queue_num = sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count;
526
527 sfc_vdpa_info(dev, "vDPA ops get_queue_num :: supported queue num : %u",
528 *queue_num);
529
530 return 0;
531 }
532
533 static int
sfc_vdpa_get_features(struct rte_vdpa_device * vdpa_dev,uint64_t * features)534 sfc_vdpa_get_features(struct rte_vdpa_device *vdpa_dev, uint64_t *features)
535 {
536 struct sfc_vdpa_ops_data *ops_data;
537
538 ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
539 if (ops_data == NULL)
540 return -1;
541
542 *features = ops_data->drv_features;
543
544 sfc_vdpa_info(ops_data->dev_handle,
545 "vDPA ops get_feature :: features : 0x%" PRIx64,
546 *features);
547
548 return 0;
549 }
550
551 static int
sfc_vdpa_get_protocol_features(struct rte_vdpa_device * vdpa_dev,uint64_t * features)552 sfc_vdpa_get_protocol_features(struct rte_vdpa_device *vdpa_dev,
553 uint64_t *features)
554 {
555 struct sfc_vdpa_ops_data *ops_data;
556
557 ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
558 if (ops_data == NULL)
559 return -1;
560
561 *features = SFC_VDPA_PROTOCOL_FEATURES;
562
563 sfc_vdpa_info(ops_data->dev_handle,
564 "vDPA ops get_protocol_feature :: features : 0x%" PRIx64,
565 *features);
566
567 return 0;
568 }
569
570 static uint32_t
sfc_vdpa_notify_ctrl(void * arg)571 sfc_vdpa_notify_ctrl(void *arg)
572 {
573 struct sfc_vdpa_ops_data *ops_data;
574 int vid;
575
576 ops_data = arg;
577 if (ops_data == NULL)
578 return 0;
579
580 sfc_vdpa_adapter_lock(sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle));
581
582 vid = ops_data->vid;
583
584 if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
585 sfc_vdpa_info(ops_data->dev_handle,
586 "vDPA (%s): Notifier could not get configured",
587 ops_data->vdpa_dev->device->name);
588
589 sfc_vdpa_adapter_unlock(sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle));
590
591 return 0;
592 }
593
594 static int
sfc_vdpa_setup_notify_ctrl(struct sfc_vdpa_ops_data * ops_data)595 sfc_vdpa_setup_notify_ctrl(struct sfc_vdpa_ops_data *ops_data)
596 {
597 int ret;
598
599 ops_data->is_notify_thread_started = false;
600
601 /*
602 * Use rte_vhost_host_notifier_ctrl in a thread to avoid
603 * dead lock scenario when multiple VFs are used in single vdpa
604 * application and multiple VFs are passed to a single VM.
605 */
606 ret = rte_thread_create_internal_control(&ops_data->notify_tid,
607 "sfc-vdpa", sfc_vdpa_notify_ctrl, ops_data);
608 if (ret != 0) {
609 sfc_vdpa_err(ops_data->dev_handle,
610 "failed to create notify_ctrl thread: %s",
611 rte_strerror(ret));
612 return -1;
613 }
614 ops_data->is_notify_thread_started = true;
615
616 return 0;
617 }
618
619 static int
sfc_vdpa_dev_config(int vid)620 sfc_vdpa_dev_config(int vid)
621 {
622 struct rte_vdpa_device *vdpa_dev;
623 int rc;
624 struct sfc_vdpa_ops_data *ops_data;
625
626 vdpa_dev = rte_vhost_get_vdpa_device(vid);
627
628 ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
629 if (ops_data == NULL) {
630 SFC_VDPA_GENERIC_LOG(ERR,
631 "invalid vDPA device : %p, vid : %d",
632 vdpa_dev, vid);
633 return -1;
634 }
635
636 sfc_vdpa_log_init(ops_data->dev_handle, "entry");
637
638 ops_data->vid = vid;
639
640 sfc_vdpa_adapter_lock(sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle));
641
642 sfc_vdpa_log_init(ops_data->dev_handle, "configuring");
643 rc = sfc_vdpa_configure(ops_data);
644 if (rc != 0)
645 goto fail_vdpa_config;
646
647 sfc_vdpa_log_init(ops_data->dev_handle, "starting");
648 rc = sfc_vdpa_start(ops_data);
649 if (rc != 0)
650 goto fail_vdpa_start;
651
652 rc = sfc_vdpa_setup_notify_ctrl(ops_data);
653 if (rc != 0)
654 goto fail_vdpa_notify;
655
656 sfc_vdpa_adapter_unlock(sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle));
657
658 sfc_vdpa_log_init(ops_data->dev_handle, "done");
659
660 return 0;
661
662 fail_vdpa_notify:
663 sfc_vdpa_stop(ops_data);
664
665 fail_vdpa_start:
666 sfc_vdpa_close(ops_data);
667
668 fail_vdpa_config:
669 sfc_vdpa_adapter_unlock(sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle));
670
671 return -1;
672 }
673
674 static int
sfc_vdpa_dev_close(int vid)675 sfc_vdpa_dev_close(int vid)
676 {
677 int ret;
678 struct rte_vdpa_device *vdpa_dev;
679 struct sfc_vdpa_ops_data *ops_data;
680
681 vdpa_dev = rte_vhost_get_vdpa_device(vid);
682
683 ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
684 if (ops_data == NULL) {
685 SFC_VDPA_GENERIC_LOG(ERR,
686 "invalid vDPA device : %p, vid : %d",
687 vdpa_dev, vid);
688 return -1;
689 }
690
691 sfc_vdpa_adapter_lock(sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle));
692 if (ops_data->is_notify_thread_started == true) {
693 ret = pthread_cancel((pthread_t)ops_data->notify_tid.opaque_id);
694 if (ret != 0) {
695 sfc_vdpa_err(ops_data->dev_handle,
696 "failed to cancel notify_ctrl thread: %s",
697 rte_strerror(ret));
698 }
699
700 ret = rte_thread_join(ops_data->notify_tid, NULL);
701 if (ret != 0) {
702 sfc_vdpa_err(ops_data->dev_handle,
703 "failed to join terminated notify_ctrl thread: %s",
704 rte_strerror(ret));
705 }
706 }
707 ops_data->is_notify_thread_started = false;
708
709 sfc_vdpa_stop(ops_data);
710 sfc_vdpa_close(ops_data);
711
712 sfc_vdpa_adapter_unlock(sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle));
713
714 return 0;
715 }
716
717 static int
sfc_vdpa_set_vring_state(int vid,int vring,int state)718 sfc_vdpa_set_vring_state(int vid, int vring, int state)
719 {
720 struct sfc_vdpa_ops_data *ops_data;
721 struct rte_vdpa_device *vdpa_dev;
722 efx_rc_t rc;
723 int vring_max;
724 void *dev;
725
726 vdpa_dev = rte_vhost_get_vdpa_device(vid);
727
728 ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
729 if (ops_data == NULL)
730 return -1;
731
732 dev = ops_data->dev_handle;
733
734 sfc_vdpa_info(dev,
735 "vDPA ops set_vring_state: vid: %d, vring: %d, state:%d",
736 vid, vring, state);
737
738 vring_max = (sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count * 2);
739
740 if (vring < 0 || vring > vring_max) {
741 sfc_vdpa_err(dev, "received invalid vring id : %d to set state",
742 vring);
743 return -1;
744 }
745
746 /*
747 * Skip if device is not yet started. virtqueues state can be
748 * changed once it is created and other configurations are done.
749 */
750 if (ops_data->state != SFC_VDPA_STATE_STARTED)
751 return 0;
752
753 if (ops_data->vq_cxt[vring].enable == state)
754 return 0;
755
756 if (state == 0) {
757 rc = sfc_vdpa_virtq_stop(ops_data, vring);
758 if (rc != 0) {
759 sfc_vdpa_err(dev, "virtqueue stop failed: %s",
760 rte_strerror(rc));
761 }
762 } else {
763 rc = sfc_vdpa_virtq_start(ops_data, vring);
764 if (rc != 0) {
765 sfc_vdpa_err(dev, "virtqueue start failed: %s",
766 rte_strerror(rc));
767 }
768 }
769
770 return rc;
771 }
772
773 static int
sfc_vdpa_set_features(int vid)774 sfc_vdpa_set_features(int vid)
775 {
776 RTE_SET_USED(vid);
777
778 return -1;
779 }
780
781 static int
sfc_vdpa_get_vfio_device_fd(int vid)782 sfc_vdpa_get_vfio_device_fd(int vid)
783 {
784 struct rte_vdpa_device *vdpa_dev;
785 struct sfc_vdpa_ops_data *ops_data;
786 int vfio_dev_fd;
787 void *dev;
788
789 vdpa_dev = rte_vhost_get_vdpa_device(vid);
790
791 ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
792 if (ops_data == NULL)
793 return -1;
794
795 dev = ops_data->dev_handle;
796 vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
797
798 sfc_vdpa_info(dev, "vDPA ops get_vfio_device_fd :: vfio fd : %d",
799 vfio_dev_fd);
800
801 return vfio_dev_fd;
802 }
803
804 static int
sfc_vdpa_get_notify_area(int vid,int qid,uint64_t * offset,uint64_t * size)805 sfc_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
806 {
807 int ret;
808 efx_nic_t *nic;
809 int vfio_dev_fd;
810 volatile void *doorbell;
811 struct rte_pci_device *pci_dev;
812 struct rte_vdpa_device *vdpa_dev;
813 struct sfc_vdpa_ops_data *ops_data;
814 struct vfio_region_info reg = { .argsz = sizeof(reg) };
815 const efx_nic_cfg_t *encp;
816 int max_vring_cnt;
817 int64_t len;
818 void *dev;
819
820 vdpa_dev = rte_vhost_get_vdpa_device(vid);
821
822 ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
823 if (ops_data == NULL)
824 return -1;
825
826 dev = ops_data->dev_handle;
827
828 vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
829 max_vring_cnt =
830 (sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count * 2);
831
832 nic = sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle)->nic;
833 encp = efx_nic_cfg_get(nic);
834
835 if (qid >= max_vring_cnt) {
836 sfc_vdpa_err(dev, "invalid qid : %d", qid);
837 return -1;
838 }
839
840 reg.index = sfc_vdpa_adapter_by_dev_handle(dev)->mem_bar.esb_rid;
841 ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, ®);
842 if (ret != 0) {
843 sfc_vdpa_err(dev, "could not get device region info: %s",
844 strerror(errno));
845 return ret;
846 }
847
848 /* Use bar_offset that was cached during sfc_vdpa_virtq_start() */
849 *offset = reg.offset + (uint64_t)ops_data->vq_cxt[qid].doorbell;
850
851 len = (1U << encp->enc_vi_window_shift) / 2;
852 if (len >= sysconf(_SC_PAGESIZE)) {
853 *size = sysconf(_SC_PAGESIZE);
854 } else {
855 sfc_vdpa_err(dev, "invalid VI window size : 0x%" PRIx64, len);
856 return -1;
857 }
858
859 sfc_vdpa_info(dev, "vDPA ops get_notify_area :: offset : 0x%" PRIx64,
860 *offset);
861
862 pci_dev = sfc_vdpa_adapter_by_dev_handle(dev)->pdev;
863 doorbell = (uint8_t *)pci_dev->mem_resource[reg.index].addr + *offset;
864
865 /*
866 * virtio-net driver in VM sends queue notifications before
867 * vDPA has a chance to setup the queues and notification area,
868 * and hence the HW misses these doorbell notifications.
869 * Since, it is safe to send duplicate doorbell, send another
870 * doorbell from vDPA driver as workaround for this timing issue.
871 */
872 rte_write16(qid, doorbell);
873
874 return 0;
875 }
876
877 static struct rte_vdpa_dev_ops sfc_vdpa_ops = {
878 .get_queue_num = sfc_vdpa_get_queue_num,
879 .get_features = sfc_vdpa_get_features,
880 .get_protocol_features = sfc_vdpa_get_protocol_features,
881 .dev_conf = sfc_vdpa_dev_config,
882 .dev_close = sfc_vdpa_dev_close,
883 .set_vring_state = sfc_vdpa_set_vring_state,
884 .set_features = sfc_vdpa_set_features,
885 .get_vfio_device_fd = sfc_vdpa_get_vfio_device_fd,
886 .get_notify_area = sfc_vdpa_get_notify_area,
887 };
888
889 struct sfc_vdpa_ops_data *
sfc_vdpa_device_init(void * dev_handle,enum sfc_vdpa_context context)890 sfc_vdpa_device_init(void *dev_handle, enum sfc_vdpa_context context)
891 {
892 struct sfc_vdpa_ops_data *ops_data;
893 struct rte_pci_device *pci_dev;
894 int rc;
895
896 /* Create vDPA ops context */
897 ops_data = rte_zmalloc("vdpa", sizeof(struct sfc_vdpa_ops_data), 0);
898 if (ops_data == NULL)
899 return NULL;
900
901 ops_data->vdpa_context = context;
902 ops_data->dev_handle = dev_handle;
903
904 pci_dev = sfc_vdpa_adapter_by_dev_handle(dev_handle)->pdev;
905
906 /* Register vDPA Device */
907 sfc_vdpa_log_init(dev_handle, "register vDPA device");
908 ops_data->vdpa_dev =
909 rte_vdpa_register_device(&pci_dev->device, &sfc_vdpa_ops);
910 if (ops_data->vdpa_dev == NULL) {
911 sfc_vdpa_err(dev_handle, "vDPA device registration failed");
912 goto fail_register_device;
913 }
914
915 /* Read supported device features */
916 sfc_vdpa_log_init(dev_handle, "get device feature");
917 rc = sfc_vdpa_get_device_features(ops_data);
918 if (rc != 0)
919 goto fail_get_dev_feature;
920
921 /* Driver features are superset of device supported feature
922 * and any additional features supported by the driver.
923 */
924 ops_data->drv_features =
925 ops_data->dev_features | SFC_VDPA_DEFAULT_FEATURES;
926
927 ops_data->state = SFC_VDPA_STATE_INITIALIZED;
928
929 return ops_data;
930
931 fail_get_dev_feature:
932 rte_vdpa_unregister_device(ops_data->vdpa_dev);
933
934 fail_register_device:
935 rte_free(ops_data);
936 return NULL;
937 }
938
939 void
sfc_vdpa_device_fini(struct sfc_vdpa_ops_data * ops_data)940 sfc_vdpa_device_fini(struct sfc_vdpa_ops_data *ops_data)
941 {
942 rte_vdpa_unregister_device(ops_data->vdpa_dev);
943
944 rte_free(ops_data);
945 }
946