1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2019 Mellanox Technologies, Ltd
3 */
4 #include <string.h>
5 #include <unistd.h>
6 #include <sys/eventfd.h>
7
8 #include <rte_malloc.h>
9 #include <rte_errno.h>
10 #include <rte_io.h>
11
12 #include <mlx5_common.h>
13
14 #include "mlx5_vdpa_utils.h"
15 #include "mlx5_vdpa.h"
16
17
18 static void
mlx5_vdpa_virtq_kick_handler(void * cb_arg)19 mlx5_vdpa_virtq_kick_handler(void *cb_arg)
20 {
21 struct mlx5_vdpa_virtq *virtq = cb_arg;
22 struct mlx5_vdpa_priv *priv = virtq->priv;
23 uint64_t buf;
24 int nbytes;
25 int retry;
26
27 pthread_mutex_lock(&virtq->virtq_lock);
28 if (priv->state != MLX5_VDPA_STATE_CONFIGURED && !virtq->enable) {
29 pthread_mutex_unlock(&virtq->virtq_lock);
30 DRV_LOG(ERR, "device %d queue %d down, skip kick handling",
31 priv->vid, virtq->index);
32 return;
33 }
34 if (rte_intr_fd_get(virtq->intr_handle) < 0) {
35 pthread_mutex_unlock(&virtq->virtq_lock);
36 return;
37 }
38 for (retry = 0; retry < 3; ++retry) {
39 nbytes = read(rte_intr_fd_get(virtq->intr_handle), &buf,
40 8);
41 if (nbytes < 0) {
42 if (errno == EINTR ||
43 errno == EWOULDBLOCK ||
44 errno == EAGAIN)
45 continue;
46 DRV_LOG(ERR, "Failed to read kickfd of virtq %d: %s.",
47 virtq->index, strerror(errno));
48 }
49 break;
50 }
51 if (nbytes < 0) {
52 pthread_mutex_unlock(&virtq->virtq_lock);
53 return;
54 }
55 rte_spinlock_lock(&priv->db_lock);
56 rte_write32(virtq->index, priv->virtq_db_addr);
57 rte_spinlock_unlock(&priv->db_lock);
58 pthread_mutex_unlock(&virtq->virtq_lock);
59 if (priv->state != MLX5_VDPA_STATE_CONFIGURED && !virtq->enable) {
60 DRV_LOG(ERR, "device %d queue %d down, skip kick handling.",
61 priv->vid, virtq->index);
62 return;
63 }
64 if (virtq->notifier_state == MLX5_VDPA_NOTIFIER_STATE_DISABLED) {
65 if (rte_vhost_host_notifier_ctrl(priv->vid, virtq->index, true))
66 virtq->notifier_state = MLX5_VDPA_NOTIFIER_STATE_ERR;
67 else
68 virtq->notifier_state =
69 MLX5_VDPA_NOTIFIER_STATE_ENABLED;
70 DRV_LOG(INFO, "Virtq %u notifier state is %s.", virtq->index,
71 virtq->notifier_state ==
72 MLX5_VDPA_NOTIFIER_STATE_ENABLED ? "enabled" :
73 "disabled");
74 }
75 DRV_LOG(DEBUG, "Ring virtq %u doorbell.", virtq->index);
76 }
77
78 /* Virtq must be locked before calling this function. */
79 static void
mlx5_vdpa_virtq_unregister_intr_handle(struct mlx5_vdpa_virtq * virtq)80 mlx5_vdpa_virtq_unregister_intr_handle(struct mlx5_vdpa_virtq *virtq)
81 {
82 int ret = -EAGAIN;
83
84 if (!virtq->intr_handle)
85 return;
86 if (rte_intr_fd_get(virtq->intr_handle) >= 0) {
87 while (ret == -EAGAIN) {
88 ret = rte_intr_callback_unregister(virtq->intr_handle,
89 mlx5_vdpa_virtq_kick_handler, virtq);
90 if (ret == -EAGAIN) {
91 DRV_LOG(DEBUG, "Try again to unregister fd %d of virtq %hu interrupt",
92 rte_intr_fd_get(virtq->intr_handle),
93 virtq->index);
94 pthread_mutex_unlock(&virtq->virtq_lock);
95 usleep(MLX5_VDPA_INTR_RETRIES_USEC);
96 pthread_mutex_lock(&virtq->virtq_lock);
97 }
98 }
99 (void)rte_intr_fd_set(virtq->intr_handle, -1);
100 }
101 rte_intr_instance_free(virtq->intr_handle);
102 virtq->intr_handle = NULL;
103 }
104
105 void
mlx5_vdpa_virtq_unreg_intr_handle_all(struct mlx5_vdpa_priv * priv)106 mlx5_vdpa_virtq_unreg_intr_handle_all(struct mlx5_vdpa_priv *priv)
107 {
108 uint32_t i;
109 struct mlx5_vdpa_virtq *virtq;
110
111 for (i = 0; i < priv->nr_virtqs; i++) {
112 virtq = &priv->virtqs[i];
113 pthread_mutex_lock(&virtq->virtq_lock);
114 mlx5_vdpa_virtq_unregister_intr_handle(virtq);
115 pthread_mutex_unlock(&virtq->virtq_lock);
116 }
117 }
118
119 static void
mlx5_vdpa_vq_destroy(struct mlx5_vdpa_virtq * virtq)120 mlx5_vdpa_vq_destroy(struct mlx5_vdpa_virtq *virtq)
121 {
122 /* Clean pre-created resource in dev removal only */
123 claim_zero(mlx5_devx_cmd_destroy(virtq->virtq));
124 virtq->index = 0;
125 virtq->virtq = NULL;
126 virtq->configured = 0;
127 }
128
129 /* Release cached VQ resources. */
130 void
mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv * priv)131 mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv *priv)
132 {
133 unsigned int i, j;
134
135 mlx5_vdpa_steer_unset(priv);
136 for (i = 0; i < priv->caps.max_num_virtio_queues; i++) {
137 struct mlx5_vdpa_virtq *virtq = &priv->virtqs[i];
138
139 pthread_mutex_lock(&virtq->virtq_lock);
140 if (virtq->virtq)
141 mlx5_vdpa_vq_destroy(virtq);
142 for (j = 0; j < RTE_DIM(virtq->umems); ++j) {
143 if (virtq->umems[j].obj) {
144 claim_zero(mlx5_glue->devx_umem_dereg
145 (virtq->umems[j].obj));
146 virtq->umems[j].obj = NULL;
147 }
148 if (virtq->umems[j].buf) {
149 rte_free(virtq->umems[j].buf);
150 virtq->umems[j].buf = NULL;
151 }
152 virtq->umems[j].size = 0;
153 }
154 if (virtq->eqp.fw_qp)
155 mlx5_vdpa_event_qp_destroy(&virtq->eqp);
156 pthread_mutex_unlock(&virtq->virtq_lock);
157 }
158 }
159
160 void
mlx5_vdpa_virtq_unset(struct mlx5_vdpa_virtq * virtq)161 mlx5_vdpa_virtq_unset(struct mlx5_vdpa_virtq *virtq)
162 {
163 int ret;
164
165 mlx5_vdpa_virtq_unregister_intr_handle(virtq);
166 if (virtq->configured) {
167 ret = mlx5_vdpa_virtq_stop(virtq->priv, virtq->index);
168 if (ret)
169 DRV_LOG(WARNING, "Failed to stop virtq %d.",
170 virtq->index);
171 }
172 mlx5_vdpa_vq_destroy(virtq);
173 virtq->notifier_state = MLX5_VDPA_NOTIFIER_STATE_DISABLED;
174 }
175
176 void
mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv * priv,bool release_resource)177 mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv,
178 bool release_resource)
179 {
180 struct mlx5_vdpa_virtq *virtq;
181 uint32_t i, max_virtq, valid_vq_num;
182
183 valid_vq_num = ((priv->queues * 2) < priv->caps.max_num_virtio_queues) ?
184 (priv->queues * 2) : priv->caps.max_num_virtio_queues;
185 max_virtq = (release_resource &&
186 (valid_vq_num) > priv->nr_virtqs) ?
187 (valid_vq_num) : priv->nr_virtqs;
188 for (i = 0; i < max_virtq; i++) {
189 virtq = &priv->virtqs[i];
190 pthread_mutex_lock(&virtq->virtq_lock);
191 mlx5_vdpa_virtq_unset(virtq);
192 virtq->enable = 0;
193 if (!release_resource && i < valid_vq_num)
194 mlx5_vdpa_virtq_single_resource_prepare(
195 priv, i);
196 pthread_mutex_unlock(&virtq->virtq_lock);
197 }
198 if (!release_resource && priv->queues &&
199 mlx5_vdpa_is_modify_virtq_supported(priv))
200 if (mlx5_vdpa_steer_update(priv, true))
201 mlx5_vdpa_steer_unset(priv);
202 priv->features = 0;
203 priv->nr_virtqs = 0;
204 }
205
206 int
mlx5_vdpa_virtq_modify(struct mlx5_vdpa_virtq * virtq,int state)207 mlx5_vdpa_virtq_modify(struct mlx5_vdpa_virtq *virtq, int state)
208 {
209 struct mlx5_devx_virtq_attr attr = {
210 .mod_fields_bitmap = MLX5_VIRTQ_MODIFY_TYPE_STATE,
211 .state = state ? MLX5_VIRTQ_STATE_RDY :
212 MLX5_VIRTQ_STATE_SUSPEND,
213 .queue_index = virtq->index,
214 };
215
216 return mlx5_devx_cmd_modify_virtq(virtq->virtq, &attr);
217 }
218
219 int
mlx5_vdpa_virtq_stop(struct mlx5_vdpa_priv * priv,int index)220 mlx5_vdpa_virtq_stop(struct mlx5_vdpa_priv *priv, int index)
221 {
222 struct mlx5_vdpa_virtq *virtq = &priv->virtqs[index];
223 int ret;
224
225 if (virtq->stopped || !virtq->configured)
226 return 0;
227 ret = mlx5_vdpa_virtq_modify(virtq, 0);
228 if (ret)
229 return -1;
230 virtq->stopped = 1;
231 DRV_LOG(DEBUG, "vid %u virtq %u was stopped.", priv->vid, index);
232 return mlx5_vdpa_virtq_query(priv, index);
233 }
234
235 int
mlx5_vdpa_virtq_query(struct mlx5_vdpa_priv * priv,int index)236 mlx5_vdpa_virtq_query(struct mlx5_vdpa_priv *priv, int index)
237 {
238 struct mlx5_devx_virtq_attr attr = {0};
239 struct mlx5_vdpa_virtq *virtq = &priv->virtqs[index];
240 int ret;
241
242 if (mlx5_devx_cmd_query_virtq(virtq->virtq, &attr)) {
243 DRV_LOG(ERR, "Failed to query virtq %d.", index);
244 return -1;
245 }
246 DRV_LOG(INFO, "Query vid %d vring %d: hw_available_idx=%d, "
247 "hw_used_index=%d", priv->vid, index,
248 attr.hw_available_index, attr.hw_used_index);
249 ret = rte_vhost_set_vring_base(priv->vid, index,
250 attr.hw_available_index,
251 attr.hw_used_index);
252 if (ret) {
253 DRV_LOG(ERR, "Failed to set virtq %d base.", index);
254 return -1;
255 }
256 if (attr.state == MLX5_VIRTQ_STATE_ERROR)
257 DRV_LOG(WARNING, "vid %d vring %d hw error=%hhu.",
258 priv->vid, index, attr.error_type);
259 return 0;
260 }
261
262 static uint64_t
mlx5_vdpa_hva_to_gpa(struct rte_vhost_memory * mem,uint64_t hva)263 mlx5_vdpa_hva_to_gpa(struct rte_vhost_memory *mem, uint64_t hva)
264 {
265 struct rte_vhost_mem_region *reg;
266 uint32_t i;
267 uint64_t gpa = 0;
268
269 for (i = 0; i < mem->nregions; i++) {
270 reg = &mem->regions[i];
271 if (hva >= reg->host_user_addr &&
272 hva < reg->host_user_addr + reg->size) {
273 gpa = hva - reg->host_user_addr + reg->guest_phys_addr;
274 break;
275 }
276 }
277 return gpa;
278 }
279
280 static int
mlx5_vdpa_virtq_sub_objs_prepare(struct mlx5_vdpa_priv * priv,struct mlx5_devx_virtq_attr * attr,struct rte_vhost_vring * vq,int index,bool is_prepare)281 mlx5_vdpa_virtq_sub_objs_prepare(struct mlx5_vdpa_priv *priv,
282 struct mlx5_devx_virtq_attr *attr,
283 struct rte_vhost_vring *vq,
284 int index, bool is_prepare)
285 {
286 struct mlx5_vdpa_virtq *virtq = &priv->virtqs[index];
287 uint64_t gpa;
288 int ret;
289 unsigned int i;
290 uint16_t last_avail_idx = 0;
291 uint16_t last_used_idx = 0;
292
293 if (virtq->virtq)
294 attr->mod_fields_bitmap = MLX5_VIRTQ_MODIFY_TYPE_STATE |
295 MLX5_VIRTQ_MODIFY_TYPE_ADDR |
296 MLX5_VIRTQ_MODIFY_TYPE_HW_AVAILABLE_INDEX |
297 MLX5_VIRTQ_MODIFY_TYPE_HW_USED_INDEX |
298 MLX5_VIRTQ_MODIFY_TYPE_VERSION_1_0 |
299 MLX5_VIRTQ_MODIFY_TYPE_Q_TYPE |
300 MLX5_VIRTQ_MODIFY_TYPE_Q_MKEY |
301 MLX5_VIRTQ_MODIFY_TYPE_QUEUE_FEATURE_BIT_MASK |
302 MLX5_VIRTQ_MODIFY_TYPE_EVENT_MODE;
303 attr->tso_ipv4 = is_prepare ? 1 :
304 !!(priv->features & (1ULL << VIRTIO_NET_F_HOST_TSO4));
305 attr->tso_ipv6 = is_prepare ? 1 :
306 !!(priv->features & (1ULL << VIRTIO_NET_F_HOST_TSO6));
307 attr->tx_csum = is_prepare ? 1 :
308 !!(priv->features & (1ULL << VIRTIO_NET_F_CSUM));
309 attr->rx_csum = is_prepare ? 1 :
310 !!(priv->features & (1ULL << VIRTIO_NET_F_GUEST_CSUM));
311 attr->virtio_version_1_0 = is_prepare ? 1 :
312 !!(priv->features & (1ULL << VIRTIO_F_VERSION_1));
313 attr->q_type =
314 (priv->features & (1ULL << VIRTIO_F_RING_PACKED)) ?
315 MLX5_VIRTQ_TYPE_PACKED : MLX5_VIRTQ_TYPE_SPLIT;
316 /*
317 * No need event QPs creation when the guest in poll mode or when the
318 * capability allows it.
319 */
320 attr->event_mode = is_prepare || vq->callfd != -1 ||
321 !(priv->caps.event_mode & (1 << MLX5_VIRTQ_EVENT_MODE_NO_MSIX)) ?
322 MLX5_VIRTQ_EVENT_MODE_QP : MLX5_VIRTQ_EVENT_MODE_NO_MSIX;
323 if (attr->event_mode == MLX5_VIRTQ_EVENT_MODE_QP) {
324 ret = mlx5_vdpa_event_qp_prepare(priv, vq->size,
325 vq->callfd, virtq, !virtq->virtq);
326 if (ret) {
327 DRV_LOG(ERR,
328 "Failed to create event QPs for virtq %d.",
329 index);
330 return -1;
331 }
332 attr->mod_fields_bitmap |= MLX5_VIRTQ_MODIFY_TYPE_EVENT_MODE;
333 attr->qp_id = virtq->eqp.fw_qp->id;
334 } else {
335 DRV_LOG(INFO, "Virtq %d is, for sure, working by poll mode, no"
336 " need event QPs and event mechanism.", index);
337 }
338 if (priv->caps.queue_counters_valid) {
339 if (!virtq->counters)
340 virtq->counters = mlx5_devx_cmd_create_virtio_q_counters
341 (priv->cdev->ctx);
342 if (!virtq->counters) {
343 DRV_LOG(ERR, "Failed to create virtq couners for virtq"
344 " %d.", index);
345 return -1;
346 }
347 attr->counters_obj_id = virtq->counters->id;
348 }
349 /* Setup 3 UMEMs for each virtq. */
350 if (!virtq->virtq) {
351 for (i = 0; i < RTE_DIM(virtq->umems); ++i) {
352 uint32_t size;
353 void *buf;
354 struct mlx5dv_devx_umem *obj;
355
356 size =
357 priv->caps.umems[i].a * vq->size + priv->caps.umems[i].b;
358 if (virtq->umems[i].size == size &&
359 virtq->umems[i].obj != NULL) {
360 /* Reuse registered memory. */
361 memset(virtq->umems[i].buf, 0, size);
362 goto reuse;
363 }
364 if (virtq->umems[i].obj)
365 claim_zero(mlx5_glue->devx_umem_dereg
366 (virtq->umems[i].obj));
367 rte_free(virtq->umems[i].buf);
368 virtq->umems[i].size = 0;
369 virtq->umems[i].obj = NULL;
370 virtq->umems[i].buf = NULL;
371 buf = rte_zmalloc(__func__,
372 size, 4096);
373 if (buf == NULL) {
374 DRV_LOG(ERR, "Cannot allocate umem %d memory for virtq."
375 " %u.", i, index);
376 return -1;
377 }
378 obj = mlx5_glue->devx_umem_reg(priv->cdev->ctx,
379 buf, size, IBV_ACCESS_LOCAL_WRITE);
380 if (obj == NULL) {
381 DRV_LOG(ERR, "Failed to register umem %d for virtq %u.",
382 i, index);
383 rte_free(buf);
384 return -1;
385 }
386 virtq->umems[i].size = size;
387 virtq->umems[i].buf = buf;
388 virtq->umems[i].obj = obj;
389 reuse:
390 attr->umems[i].id = virtq->umems[i].obj->umem_id;
391 attr->umems[i].offset = 0;
392 attr->umems[i].size = virtq->umems[i].size;
393 }
394 }
395 if (!is_prepare && attr->q_type == MLX5_VIRTQ_TYPE_SPLIT) {
396 gpa = mlx5_vdpa_hva_to_gpa(priv->vmem_info.vmem,
397 (uint64_t)(uintptr_t)vq->desc);
398 if (!gpa) {
399 DRV_LOG(ERR, "Failed to get descriptor ring GPA.");
400 return -1;
401 }
402 attr->desc_addr = gpa;
403 gpa = mlx5_vdpa_hva_to_gpa(priv->vmem_info.vmem,
404 (uint64_t)(uintptr_t)vq->used);
405 if (!gpa) {
406 DRV_LOG(ERR, "Failed to get GPA for used ring.");
407 return -1;
408 }
409 attr->used_addr = gpa;
410 gpa = mlx5_vdpa_hva_to_gpa(priv->vmem_info.vmem,
411 (uint64_t)(uintptr_t)vq->avail);
412 if (!gpa) {
413 DRV_LOG(ERR, "Failed to get GPA for available ring.");
414 return -1;
415 }
416 attr->available_addr = gpa;
417 }
418 if (!is_prepare) {
419 ret = rte_vhost_get_vring_base(priv->vid,
420 index, &last_avail_idx, &last_used_idx);
421 if (ret) {
422 last_avail_idx = 0;
423 last_used_idx = 0;
424 DRV_LOG(WARNING, "Couldn't get vring base, idx are set to 0.");
425 } else {
426 DRV_LOG(INFO, "vid %d: Init last_avail_idx=%d, last_used_idx=%d for "
427 "virtq %d.", priv->vid, last_avail_idx,
428 last_used_idx, index);
429 }
430 }
431 attr->hw_available_index = last_avail_idx;
432 attr->hw_used_index = last_used_idx;
433 attr->q_size = vq->size;
434 attr->mkey = is_prepare ? 0 : priv->gpa_mkey_index;
435 attr->tis_id = priv->tiss[(index / 2) % priv->num_lag_ports]->id;
436 attr->queue_index = index;
437 attr->pd = priv->cdev->pdn;
438 attr->hw_latency_mode = priv->hw_latency_mode;
439 attr->hw_max_latency_us = priv->hw_max_latency_us;
440 attr->hw_max_pending_comp = priv->hw_max_pending_comp;
441 if (attr->hw_latency_mode || attr->hw_max_latency_us ||
442 attr->hw_max_pending_comp)
443 attr->mod_fields_bitmap |= MLX5_VIRTQ_MODIFY_TYPE_QUEUE_PERIOD;
444 return 0;
445 }
446
447 bool
mlx5_vdpa_virtq_single_resource_prepare(struct mlx5_vdpa_priv * priv,int index)448 mlx5_vdpa_virtq_single_resource_prepare(struct mlx5_vdpa_priv *priv,
449 int index)
450 {
451 struct mlx5_devx_virtq_attr attr = {0};
452 struct mlx5_vdpa_virtq *virtq;
453 struct rte_vhost_vring vq = {
454 .size = priv->queue_size,
455 .callfd = -1,
456 };
457 int ret;
458
459 virtq = &priv->virtqs[index];
460 virtq->index = index;
461 virtq->vq_size = vq.size;
462 virtq->configured = 0;
463 virtq->virtq = NULL;
464 ret = mlx5_vdpa_virtq_sub_objs_prepare(priv, &attr, &vq, index, true);
465 if (ret) {
466 DRV_LOG(ERR,
467 "Cannot prepare setup resource for virtq %d.", index);
468 return true;
469 }
470 if (mlx5_vdpa_is_modify_virtq_supported(priv)) {
471 virtq->virtq =
472 mlx5_devx_cmd_create_virtq(priv->cdev->ctx, &attr);
473 virtq->priv = priv;
474 if (!virtq->virtq)
475 return true;
476 virtq->rx_csum = attr.rx_csum;
477 virtq->virtio_version_1_0 = attr.virtio_version_1_0;
478 virtq->event_mode = attr.event_mode;
479 }
480 return false;
481 }
482
483 bool
mlx5_vdpa_is_modify_virtq_supported(struct mlx5_vdpa_priv * priv)484 mlx5_vdpa_is_modify_virtq_supported(struct mlx5_vdpa_priv *priv)
485 {
486 return (priv->caps.vnet_modify_ext &&
487 priv->caps.virtio_net_q_addr_modify &&
488 priv->caps.virtio_q_index_modify) ? true : false;
489 }
490
491 static int
mlx5_vdpa_virtq_doorbell_setup(struct mlx5_vdpa_virtq * virtq,struct rte_vhost_vring * vq,int index)492 mlx5_vdpa_virtq_doorbell_setup(struct mlx5_vdpa_virtq *virtq,
493 struct rte_vhost_vring *vq, int index)
494 {
495 virtq->intr_handle = mlx5_os_interrupt_handler_create(
496 RTE_INTR_INSTANCE_F_SHARED, false,
497 vq->kickfd, mlx5_vdpa_virtq_kick_handler, virtq);
498 if (virtq->intr_handle == NULL) {
499 DRV_LOG(ERR, "Fail to allocate intr_handle for virtq %d.", index);
500 return -1;
501 }
502 return 0;
503 }
504
505 int
mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv * priv,int index,bool reg_kick)506 mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv, int index, bool reg_kick)
507 {
508 struct mlx5_vdpa_virtq *virtq = &priv->virtqs[index];
509 struct rte_vhost_vring vq;
510 struct mlx5_devx_virtq_attr attr = {0};
511 int ret;
512 uint16_t event_num = MLX5_EVENT_TYPE_OBJECT_CHANGE;
513 uint64_t cookie;
514
515 ret = rte_vhost_get_vhost_vring(priv->vid, index, &vq);
516 if (ret)
517 return -1;
518 if (vq.size == 0)
519 return 0;
520 virtq->priv = priv;
521 virtq->stopped = 0;
522 ret = mlx5_vdpa_virtq_sub_objs_prepare(priv, &attr,
523 &vq, index, false);
524 if (ret) {
525 DRV_LOG(ERR, "Failed to setup update virtq attr %d.",
526 index);
527 goto error;
528 }
529 if (!virtq->virtq) {
530 virtq->index = index;
531 virtq->vq_size = vq.size;
532 virtq->virtq = mlx5_devx_cmd_create_virtq(priv->cdev->ctx,
533 &attr);
534 if (!virtq->virtq)
535 goto error;
536 attr.mod_fields_bitmap = MLX5_VIRTQ_MODIFY_TYPE_STATE;
537 }
538 attr.state = MLX5_VIRTQ_STATE_RDY;
539 ret = mlx5_devx_cmd_modify_virtq(virtq->virtq, &attr);
540 if (ret) {
541 DRV_LOG(ERR, "Failed to modify virtq %d.", index);
542 goto error;
543 }
544 claim_zero(rte_vhost_enable_guest_notification(priv->vid, index, 1));
545 virtq->rx_csum = attr.rx_csum;
546 virtq->virtio_version_1_0 = attr.virtio_version_1_0;
547 virtq->event_mode = attr.event_mode;
548 virtq->configured = 1;
549 rte_spinlock_lock(&priv->db_lock);
550 rte_write32(virtq->index, priv->virtq_db_addr);
551 rte_spinlock_unlock(&priv->db_lock);
552 /* Setup doorbell mapping. */
553 if (reg_kick) {
554 if (mlx5_vdpa_virtq_doorbell_setup(virtq, &vq, index)) {
555 DRV_LOG(ERR, "Failed to register virtq %d interrupt.",
556 index);
557 goto error;
558 }
559 }
560 /* Subscribe virtq error event. */
561 virtq->version++;
562 cookie = ((uint64_t)virtq->version << 32) + index;
563 ret = mlx5_glue->devx_subscribe_devx_event(priv->err_chnl,
564 virtq->virtq->obj,
565 sizeof(event_num),
566 &event_num, cookie);
567 if (ret) {
568 DRV_LOG(ERR, "Failed to subscribe device %d virtq %d error event.",
569 priv->vid, index);
570 rte_errno = errno;
571 goto error;
572 }
573 /* Initial notification to ask Qemu handling completed buffers. */
574 if (virtq->eqp.cq.callfd != -1)
575 eventfd_write(virtq->eqp.cq.callfd, (eventfd_t)1);
576 DRV_LOG(DEBUG, "vid %u virtq %u was created successfully.", priv->vid,
577 index);
578 return 0;
579 error:
580 mlx5_vdpa_virtq_unset(virtq);
581 return -1;
582 }
583
584 static int
mlx5_vdpa_features_validate(struct mlx5_vdpa_priv * priv)585 mlx5_vdpa_features_validate(struct mlx5_vdpa_priv *priv)
586 {
587 if (priv->features & (1ULL << VIRTIO_F_RING_PACKED)) {
588 if (!(priv->caps.virtio_queue_type & (1 <<
589 MLX5_VIRTQ_TYPE_PACKED))) {
590 DRV_LOG(ERR, "Failed to configure PACKED mode for vdev "
591 "%d - it was not reported by HW/driver"
592 " capability.", priv->vid);
593 return -ENOTSUP;
594 }
595 }
596 if (priv->features & (1ULL << VIRTIO_NET_F_HOST_TSO4)) {
597 if (!priv->caps.tso_ipv4) {
598 DRV_LOG(ERR, "Failed to enable TSO4 for vdev %d - TSO4"
599 " was not reported by HW/driver capability.",
600 priv->vid);
601 return -ENOTSUP;
602 }
603 }
604 if (priv->features & (1ULL << VIRTIO_NET_F_HOST_TSO6)) {
605 if (!priv->caps.tso_ipv6) {
606 DRV_LOG(ERR, "Failed to enable TSO6 for vdev %d - TSO6"
607 " was not reported by HW/driver capability.",
608 priv->vid);
609 return -ENOTSUP;
610 }
611 }
612 if (priv->features & (1ULL << VIRTIO_NET_F_CSUM)) {
613 if (!priv->caps.tx_csum) {
614 DRV_LOG(ERR, "Failed to enable CSUM for vdev %d - CSUM"
615 " was not reported by HW/driver capability.",
616 priv->vid);
617 return -ENOTSUP;
618 }
619 }
620 if (priv->features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) {
621 if (!priv->caps.rx_csum) {
622 DRV_LOG(ERR, "Failed to enable GUEST CSUM for vdev %d"
623 " GUEST CSUM was not reported by HW/driver "
624 "capability.", priv->vid);
625 return -ENOTSUP;
626 }
627 }
628 if (priv->features & (1ULL << VIRTIO_F_VERSION_1)) {
629 if (!priv->caps.virtio_version_1_0) {
630 DRV_LOG(ERR, "Failed to enable version 1 for vdev %d "
631 "version 1 was not reported by HW/driver"
632 " capability.", priv->vid);
633 return -ENOTSUP;
634 }
635 }
636 return 0;
637 }
638
639 static bool
mlx5_vdpa_is_pre_created_vq_mismatch(struct mlx5_vdpa_priv * priv,struct mlx5_vdpa_virtq * virtq)640 mlx5_vdpa_is_pre_created_vq_mismatch(struct mlx5_vdpa_priv *priv,
641 struct mlx5_vdpa_virtq *virtq)
642 {
643 struct rte_vhost_vring vq;
644 uint32_t event_mode;
645
646 if (virtq->rx_csum !=
647 !!(priv->features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)))
648 return true;
649 if (virtq->virtio_version_1_0 !=
650 !!(priv->features & (1ULL << VIRTIO_F_VERSION_1)))
651 return true;
652 if (rte_vhost_get_vhost_vring(priv->vid, virtq->index, &vq))
653 return true;
654 if (vq.size != virtq->vq_size)
655 return true;
656 event_mode = vq.callfd != -1 || !(priv->caps.event_mode &
657 (1 << MLX5_VIRTQ_EVENT_MODE_NO_MSIX)) ?
658 MLX5_VIRTQ_EVENT_MODE_QP : MLX5_VIRTQ_EVENT_MODE_NO_MSIX;
659 if (virtq->event_mode != event_mode)
660 return true;
661 return false;
662 }
663
664 int
mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv * priv)665 mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv)
666 {
667 int ret = rte_vhost_get_negotiated_features(priv->vid, &priv->features);
668 uint16_t nr_vring = rte_vhost_get_vring_num(priv->vid);
669 RTE_ATOMIC(uint32_t) remaining_cnt = 0;
670 RTE_ATOMIC(uint32_t) err_cnt = 0;
671 uint32_t task_num = 0;
672 uint32_t i, thrd_idx, data[1];
673 struct mlx5_vdpa_virtq *virtq;
674 struct rte_vhost_vring vq;
675
676 if (ret || mlx5_vdpa_features_validate(priv)) {
677 DRV_LOG(ERR, "Failed to configure negotiated features.");
678 return -1;
679 }
680 if ((priv->features & (1ULL << VIRTIO_NET_F_CSUM)) == 0 &&
681 ((priv->features & (1ULL << VIRTIO_NET_F_HOST_TSO4)) > 0 ||
682 (priv->features & (1ULL << VIRTIO_NET_F_HOST_TSO6)) > 0)) {
683 /* Packet may be corrupted if TSO is enabled without CSUM. */
684 DRV_LOG(INFO, "TSO is enabled without CSUM, force CSUM.");
685 priv->features |= (1ULL << VIRTIO_NET_F_CSUM);
686 }
687 if (nr_vring > priv->caps.max_num_virtio_queues) {
688 DRV_LOG(ERR, "Do not support more than %d virtqs(%d).",
689 (int)priv->caps.max_num_virtio_queues,
690 (int)nr_vring);
691 return -1;
692 }
693 priv->nr_virtqs = nr_vring;
694 if (priv->use_c_thread) {
695 uint32_t main_task_idx[nr_vring];
696
697 for (i = 0; i < nr_vring; i++) {
698 virtq = &priv->virtqs[i];
699 if (!virtq->enable)
700 continue;
701 if (priv->queues && virtq->virtq) {
702 if (mlx5_vdpa_is_pre_created_vq_mismatch(priv, virtq)) {
703 mlx5_vdpa_prepare_virtq_destroy(priv);
704 i = 0;
705 virtq = &priv->virtqs[i];
706 if (!virtq->enable)
707 continue;
708 }
709 }
710 thrd_idx = i % (conf_thread_mng.max_thrds + 1);
711 if (!thrd_idx) {
712 main_task_idx[task_num] = i;
713 task_num++;
714 continue;
715 }
716 thrd_idx = priv->last_c_thrd_idx + 1;
717 if (thrd_idx >= conf_thread_mng.max_thrds)
718 thrd_idx = 0;
719 priv->last_c_thrd_idx = thrd_idx;
720 data[0] = i;
721 if (mlx5_vdpa_task_add(priv, thrd_idx,
722 MLX5_VDPA_TASK_SETUP_VIRTQ,
723 &remaining_cnt, &err_cnt,
724 (void **)&data, 1)) {
725 DRV_LOG(ERR, "Fail to add "
726 "task setup virtq (%d).", i);
727 main_task_idx[task_num] = i;
728 task_num++;
729 }
730 }
731 for (i = 0; i < task_num; i++) {
732 virtq = &priv->virtqs[main_task_idx[i]];
733 pthread_mutex_lock(&virtq->virtq_lock);
734 if (mlx5_vdpa_virtq_setup(priv,
735 main_task_idx[i], false)) {
736 pthread_mutex_unlock(&virtq->virtq_lock);
737 goto error;
738 }
739 virtq->enable = 1;
740 pthread_mutex_unlock(&virtq->virtq_lock);
741 }
742 if (mlx5_vdpa_c_thread_wait_bulk_tasks_done(&remaining_cnt,
743 &err_cnt, 2000)) {
744 DRV_LOG(ERR,
745 "Failed to wait virt-queue setup tasks ready.");
746 goto error;
747 }
748 for (i = 0; i < nr_vring; i++) {
749 /* Setup doorbell mapping in order for Qume. */
750 virtq = &priv->virtqs[i];
751 pthread_mutex_lock(&virtq->virtq_lock);
752 if (!virtq->enable || !virtq->configured) {
753 pthread_mutex_unlock(&virtq->virtq_lock);
754 continue;
755 }
756 if (rte_vhost_get_vhost_vring(priv->vid, i, &vq)) {
757 pthread_mutex_unlock(&virtq->virtq_lock);
758 goto error;
759 }
760 if (mlx5_vdpa_virtq_doorbell_setup(virtq, &vq, i)) {
761 pthread_mutex_unlock(&virtq->virtq_lock);
762 DRV_LOG(ERR,
763 "Failed to register virtq %d interrupt.", i);
764 goto error;
765 }
766 pthread_mutex_unlock(&virtq->virtq_lock);
767 }
768 } else {
769 for (i = 0; i < nr_vring; i++) {
770 virtq = &priv->virtqs[i];
771 if (!virtq->enable)
772 continue;
773 if (priv->queues && virtq->virtq) {
774 if (mlx5_vdpa_is_pre_created_vq_mismatch(priv,
775 virtq)) {
776 mlx5_vdpa_prepare_virtq_destroy(
777 priv);
778 i = 0;
779 virtq = &priv->virtqs[i];
780 if (!virtq->enable)
781 continue;
782 }
783 }
784 pthread_mutex_lock(&virtq->virtq_lock);
785 if (mlx5_vdpa_virtq_setup(priv, i, true)) {
786 pthread_mutex_unlock(
787 &virtq->virtq_lock);
788 goto error;
789 }
790 virtq->enable = 1;
791 pthread_mutex_unlock(&virtq->virtq_lock);
792 }
793 }
794 return 0;
795 error:
796 mlx5_vdpa_virtqs_release(priv, true);
797 return -1;
798 }
799
800 static int
mlx5_vdpa_virtq_is_modified(struct mlx5_vdpa_priv * priv,struct mlx5_vdpa_virtq * virtq)801 mlx5_vdpa_virtq_is_modified(struct mlx5_vdpa_priv *priv,
802 struct mlx5_vdpa_virtq *virtq)
803 {
804 struct rte_vhost_vring vq;
805 int ret = rte_vhost_get_vhost_vring(priv->vid, virtq->index, &vq);
806
807 if (ret)
808 return -1;
809 if (vq.size != virtq->vq_size || vq.kickfd !=
810 rte_intr_fd_get(virtq->intr_handle))
811 return 1;
812 if (virtq->eqp.cq.cq_obj.cq) {
813 if (vq.callfd != virtq->eqp.cq.callfd)
814 return 1;
815 } else if (vq.callfd != -1) {
816 return 1;
817 }
818 return 0;
819 }
820
821 int
mlx5_vdpa_virtq_enable(struct mlx5_vdpa_priv * priv,int index,int enable)822 mlx5_vdpa_virtq_enable(struct mlx5_vdpa_priv *priv, int index, int enable)
823 {
824 struct mlx5_vdpa_virtq *virtq = &priv->virtqs[index];
825 int ret;
826
827 DRV_LOG(INFO, "Update virtq %d status %sable -> %sable.", index,
828 virtq->enable ? "en" : "dis", enable ? "en" : "dis");
829 if (priv->state == MLX5_VDPA_STATE_PROBED) {
830 virtq->enable = !!enable;
831 return 0;
832 }
833 if (virtq->enable == !!enable) {
834 if (!enable)
835 return 0;
836 ret = mlx5_vdpa_virtq_is_modified(priv, virtq);
837 if (ret < 0) {
838 DRV_LOG(ERR, "Virtq %d modify check failed.", index);
839 return -1;
840 }
841 if (ret == 0)
842 return 0;
843 DRV_LOG(INFO, "Virtq %d was modified, recreate it.", index);
844 }
845 if (virtq->configured) {
846 virtq->enable = 0;
847 if (is_virtq_recvq(virtq->index, priv->nr_virtqs)) {
848 ret = mlx5_vdpa_steer_update(priv, false);
849 if (ret)
850 DRV_LOG(WARNING, "Failed to disable steering "
851 "for virtq %d.", index);
852 }
853 mlx5_vdpa_virtq_unset(virtq);
854 } else {
855 if (virtq->virtq &&
856 mlx5_vdpa_is_pre_created_vq_mismatch(priv, virtq))
857 DRV_LOG(WARNING,
858 "Configuration mismatch dummy virtq %d.", index);
859 }
860 if (enable) {
861 ret = mlx5_vdpa_virtq_setup(priv, index, true);
862 if (ret) {
863 DRV_LOG(ERR, "Failed to setup virtq %d.", index);
864 return ret;
865 }
866 virtq->enable = 1;
867 if (is_virtq_recvq(virtq->index, priv->nr_virtqs)) {
868 ret = mlx5_vdpa_steer_update(priv, false);
869 if (ret)
870 DRV_LOG(WARNING, "Failed to enable steering "
871 "for virtq %d.", index);
872 }
873 }
874 return 0;
875 }
876
877 int
mlx5_vdpa_virtq_stats_get(struct mlx5_vdpa_priv * priv,int qid,struct rte_vdpa_stat * stats,unsigned int n)878 mlx5_vdpa_virtq_stats_get(struct mlx5_vdpa_priv *priv, int qid,
879 struct rte_vdpa_stat *stats, unsigned int n)
880 {
881 struct mlx5_vdpa_virtq *virtq = &priv->virtqs[qid];
882 struct mlx5_devx_virtio_q_couners_attr *attr = &virtq->stats;
883 int ret;
884
885 if (!virtq->counters) {
886 DRV_LOG(ERR, "Failed to read virtq %d statistics - virtq "
887 "is invalid.", qid);
888 return -EINVAL;
889 }
890 ret = mlx5_devx_cmd_query_virtio_q_counters(virtq->counters, attr);
891 if (ret) {
892 DRV_LOG(ERR, "Failed to read virtq %d stats from HW.", qid);
893 return ret;
894 }
895 ret = (int)RTE_MIN(n, (unsigned int)MLX5_VDPA_STATS_MAX);
896 if (ret == MLX5_VDPA_STATS_RECEIVED_DESCRIPTORS)
897 return ret;
898 stats[MLX5_VDPA_STATS_RECEIVED_DESCRIPTORS] = (struct rte_vdpa_stat) {
899 .id = MLX5_VDPA_STATS_RECEIVED_DESCRIPTORS,
900 .value = attr->received_desc - virtq->reset.received_desc,
901 };
902 if (ret == MLX5_VDPA_STATS_COMPLETED_DESCRIPTORS)
903 return ret;
904 stats[MLX5_VDPA_STATS_COMPLETED_DESCRIPTORS] = (struct rte_vdpa_stat) {
905 .id = MLX5_VDPA_STATS_COMPLETED_DESCRIPTORS,
906 .value = attr->completed_desc - virtq->reset.completed_desc,
907 };
908 if (ret == MLX5_VDPA_STATS_BAD_DESCRIPTOR_ERRORS)
909 return ret;
910 stats[MLX5_VDPA_STATS_BAD_DESCRIPTOR_ERRORS] = (struct rte_vdpa_stat) {
911 .id = MLX5_VDPA_STATS_BAD_DESCRIPTOR_ERRORS,
912 .value = attr->bad_desc_errors - virtq->reset.bad_desc_errors,
913 };
914 if (ret == MLX5_VDPA_STATS_EXCEED_MAX_CHAIN)
915 return ret;
916 stats[MLX5_VDPA_STATS_EXCEED_MAX_CHAIN] = (struct rte_vdpa_stat) {
917 .id = MLX5_VDPA_STATS_EXCEED_MAX_CHAIN,
918 .value = attr->exceed_max_chain - virtq->reset.exceed_max_chain,
919 };
920 if (ret == MLX5_VDPA_STATS_INVALID_BUFFER)
921 return ret;
922 stats[MLX5_VDPA_STATS_INVALID_BUFFER] = (struct rte_vdpa_stat) {
923 .id = MLX5_VDPA_STATS_INVALID_BUFFER,
924 .value = attr->invalid_buffer - virtq->reset.invalid_buffer,
925 };
926 if (ret == MLX5_VDPA_STATS_COMPLETION_ERRORS)
927 return ret;
928 stats[MLX5_VDPA_STATS_COMPLETION_ERRORS] = (struct rte_vdpa_stat) {
929 .id = MLX5_VDPA_STATS_COMPLETION_ERRORS,
930 .value = attr->error_cqes - virtq->reset.error_cqes,
931 };
932 return ret;
933 }
934
935 int
mlx5_vdpa_virtq_stats_reset(struct mlx5_vdpa_priv * priv,int qid)936 mlx5_vdpa_virtq_stats_reset(struct mlx5_vdpa_priv *priv, int qid)
937 {
938 struct mlx5_vdpa_virtq *virtq = &priv->virtqs[qid];
939 int ret;
940
941 if (virtq->counters == NULL) /* VQ not enabled. */
942 return 0;
943 ret = mlx5_devx_cmd_query_virtio_q_counters(virtq->counters,
944 &virtq->reset);
945 if (ret)
946 DRV_LOG(ERR, "Failed to read virtq %d reset stats from HW.",
947 qid);
948 return ret;
949 }
950