1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2019 Mellanox Technologies, Ltd 3 */ 4 #include <unistd.h> 5 #include <stdint.h> 6 #include <sched.h> 7 #include <fcntl.h> 8 #include <sys/eventfd.h> 9 10 #include <rte_malloc.h> 11 #include <rte_memory.h> 12 #include <rte_errno.h> 13 #include <rte_lcore.h> 14 #include <rte_atomic.h> 15 #include <rte_common.h> 16 #include <rte_io.h> 17 #include <rte_alarm.h> 18 19 #include <mlx5_common.h> 20 #include <mlx5_common_os.h> 21 #include <mlx5_common_devx.h> 22 #include <mlx5_glue.h> 23 24 #include "mlx5_vdpa_utils.h" 25 #include "mlx5_vdpa.h" 26 27 28 #define MLX5_VDPA_ERROR_TIME_SEC 3u 29 30 void 31 mlx5_vdpa_event_qp_global_release(struct mlx5_vdpa_priv *priv) 32 { 33 if (priv->uar) { 34 mlx5_glue->devx_free_uar(priv->uar); 35 priv->uar = NULL; 36 } 37 #ifdef HAVE_IBV_DEVX_EVENT 38 if (priv->eventc) { 39 union { 40 struct mlx5dv_devx_async_event_hdr event_resp; 41 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) 42 + 128]; 43 } out; 44 45 /* Clean all pending events. */ 46 while (mlx5_glue->devx_get_event(priv->eventc, &out.event_resp, 47 sizeof(out.buf)) >= 48 (ssize_t)sizeof(out.event_resp.cookie)) 49 ; 50 mlx5_os_devx_destroy_event_channel(priv->eventc); 51 priv->eventc = NULL; 52 } 53 #endif 54 } 55 56 /* Prepare all the global resources for all the event objects.*/ 57 static int 58 mlx5_vdpa_event_qp_global_prepare(struct mlx5_vdpa_priv *priv) 59 { 60 int flags, ret; 61 62 if (priv->eventc) 63 return 0; 64 priv->eventc = mlx5_os_devx_create_event_channel(priv->ctx, 65 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA); 66 if (!priv->eventc) { 67 rte_errno = errno; 68 DRV_LOG(ERR, "Failed to create event channel %d.", 69 rte_errno); 70 goto error; 71 } 72 flags = fcntl(priv->eventc->fd, F_GETFL); 73 ret = fcntl(priv->eventc->fd, F_SETFL, flags | O_NONBLOCK); 74 if (ret) { 75 DRV_LOG(ERR, "Failed to change event channel FD."); 76 goto error; 77 } 78 /* 79 * This PMD always claims the write memory barrier on UAR 80 * registers writings, it is safe to allocate UAR with any 81 * memory mapping type. 82 */ 83 priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1); 84 if (!priv->uar) { 85 rte_errno = errno; 86 DRV_LOG(ERR, "Failed to allocate UAR."); 87 goto error; 88 } 89 return 0; 90 error: 91 mlx5_vdpa_event_qp_global_release(priv); 92 return -1; 93 } 94 95 static void 96 mlx5_vdpa_cq_destroy(struct mlx5_vdpa_cq *cq) 97 { 98 mlx5_devx_cq_destroy(&cq->cq_obj); 99 memset(cq, 0, sizeof(*cq)); 100 } 101 102 static inline void __rte_unused 103 mlx5_vdpa_cq_arm(struct mlx5_vdpa_priv *priv, struct mlx5_vdpa_cq *cq) 104 { 105 uint32_t arm_sn = cq->arm_sn << MLX5_CQ_SQN_OFFSET; 106 uint32_t cq_ci = cq->cq_ci & MLX5_CI_MASK; 107 uint32_t doorbell_hi = arm_sn | MLX5_CQ_DBR_CMD_ALL | cq_ci; 108 uint64_t doorbell = ((uint64_t)doorbell_hi << 32) | cq->cq_obj.cq->id; 109 uint64_t db_be = rte_cpu_to_be_64(doorbell); 110 uint32_t *addr = RTE_PTR_ADD(priv->uar->base_addr, MLX5_CQ_DOORBELL); 111 112 rte_io_wmb(); 113 cq->cq_obj.db_rec[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(doorbell_hi); 114 rte_wmb(); 115 #ifdef RTE_ARCH_64 116 *(uint64_t *)addr = db_be; 117 #else 118 *(uint32_t *)addr = db_be; 119 rte_io_wmb(); 120 *((uint32_t *)addr + 1) = db_be >> 32; 121 #endif 122 cq->arm_sn++; 123 cq->armed = 1; 124 } 125 126 static int 127 mlx5_vdpa_cq_create(struct mlx5_vdpa_priv *priv, uint16_t log_desc_n, 128 int callfd, struct mlx5_vdpa_cq *cq) 129 { 130 struct mlx5_devx_cq_attr attr = { 131 .use_first_only = 1, 132 .uar_page_id = priv->uar->page_id, 133 }; 134 uint16_t event_nums[1] = {0}; 135 int ret; 136 137 ret = mlx5_devx_cq_create(priv->ctx, &cq->cq_obj, log_desc_n, &attr, 138 SOCKET_ID_ANY); 139 if (ret) 140 goto error; 141 cq->cq_ci = 0; 142 cq->log_desc_n = log_desc_n; 143 rte_spinlock_init(&cq->sl); 144 /* Subscribe CQ event to the event channel controlled by the driver. */ 145 ret = mlx5_os_devx_subscribe_devx_event(priv->eventc, 146 cq->cq_obj.cq->obj, 147 sizeof(event_nums), event_nums, 148 (uint64_t)(uintptr_t)cq); 149 if (ret) { 150 DRV_LOG(ERR, "Failed to subscribe CQE event."); 151 rte_errno = errno; 152 goto error; 153 } 154 cq->callfd = callfd; 155 /* Init CQ to ones to be in HW owner in the start. */ 156 cq->cq_obj.cqes[0].op_own = MLX5_CQE_OWNER_MASK; 157 cq->cq_obj.cqes[0].wqe_counter = rte_cpu_to_be_16(UINT16_MAX); 158 /* First arming. */ 159 mlx5_vdpa_cq_arm(priv, cq); 160 return 0; 161 error: 162 mlx5_vdpa_cq_destroy(cq); 163 return -1; 164 } 165 166 static inline uint32_t 167 mlx5_vdpa_cq_poll(struct mlx5_vdpa_cq *cq) 168 { 169 struct mlx5_vdpa_event_qp *eqp = 170 container_of(cq, struct mlx5_vdpa_event_qp, cq); 171 const unsigned int cq_size = 1 << cq->log_desc_n; 172 union { 173 struct { 174 uint16_t wqe_counter; 175 uint8_t rsvd5; 176 uint8_t op_own; 177 }; 178 uint32_t word; 179 } last_word; 180 uint16_t next_wqe_counter = cq->cq_ci; 181 uint16_t cur_wqe_counter; 182 uint16_t comp; 183 184 last_word.word = rte_read32(&cq->cq_obj.cqes[0].wqe_counter); 185 cur_wqe_counter = rte_be_to_cpu_16(last_word.wqe_counter); 186 comp = cur_wqe_counter + (uint16_t)1 - next_wqe_counter; 187 if (comp) { 188 cq->cq_ci += comp; 189 MLX5_ASSERT(MLX5_CQE_OPCODE(last_word.op_own) != 190 MLX5_CQE_INVALID); 191 if (unlikely(!(MLX5_CQE_OPCODE(last_word.op_own) == 192 MLX5_CQE_RESP_ERR || 193 MLX5_CQE_OPCODE(last_word.op_own) == 194 MLX5_CQE_REQ_ERR))) 195 cq->errors++; 196 rte_io_wmb(); 197 /* Ring CQ doorbell record. */ 198 cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci); 199 rte_io_wmb(); 200 /* Ring SW QP doorbell record. */ 201 eqp->db_rec[0] = rte_cpu_to_be_32(cq->cq_ci + cq_size); 202 } 203 return comp; 204 } 205 206 static void 207 mlx5_vdpa_arm_all_cqs(struct mlx5_vdpa_priv *priv) 208 { 209 struct mlx5_vdpa_cq *cq; 210 int i; 211 212 for (i = 0; i < priv->nr_virtqs; i++) { 213 cq = &priv->virtqs[i].eqp.cq; 214 if (cq->cq_obj.cq && !cq->armed) 215 mlx5_vdpa_cq_arm(priv, cq); 216 } 217 } 218 219 static void 220 mlx5_vdpa_timer_sleep(struct mlx5_vdpa_priv *priv, uint32_t max) 221 { 222 if (priv->event_mode == MLX5_VDPA_EVENT_MODE_DYNAMIC_TIMER) { 223 switch (max) { 224 case 0: 225 priv->timer_delay_us += priv->event_us; 226 break; 227 case 1: 228 break; 229 default: 230 priv->timer_delay_us /= max; 231 break; 232 } 233 } 234 if (priv->timer_delay_us) 235 usleep(priv->timer_delay_us); 236 else 237 /* Give-up CPU to improve polling threads scheduling. */ 238 sched_yield(); 239 } 240 241 static void * 242 mlx5_vdpa_poll_handle(void *arg) 243 { 244 struct mlx5_vdpa_priv *priv = arg; 245 int i; 246 struct mlx5_vdpa_cq *cq; 247 uint32_t max; 248 uint64_t current_tic; 249 250 pthread_mutex_lock(&priv->timer_lock); 251 while (!priv->timer_on) 252 pthread_cond_wait(&priv->timer_cond, &priv->timer_lock); 253 pthread_mutex_unlock(&priv->timer_lock); 254 priv->timer_delay_us = priv->event_mode == 255 MLX5_VDPA_EVENT_MODE_DYNAMIC_TIMER ? 256 MLX5_VDPA_DEFAULT_TIMER_DELAY_US : 257 priv->event_us; 258 while (1) { 259 max = 0; 260 pthread_mutex_lock(&priv->vq_config_lock); 261 for (i = 0; i < priv->nr_virtqs; i++) { 262 cq = &priv->virtqs[i].eqp.cq; 263 if (cq->cq_obj.cq && !cq->armed) { 264 uint32_t comp = mlx5_vdpa_cq_poll(cq); 265 266 if (comp) { 267 /* Notify guest for descs consuming. */ 268 if (cq->callfd != -1) 269 eventfd_write(cq->callfd, 270 (eventfd_t)1); 271 if (comp > max) 272 max = comp; 273 } 274 } 275 } 276 current_tic = rte_rdtsc(); 277 if (!max) { 278 /* No traffic ? stop timer and load interrupts. */ 279 if (current_tic - priv->last_traffic_tic >= 280 rte_get_timer_hz() * priv->no_traffic_time_s) { 281 DRV_LOG(DEBUG, "Device %s traffic was stopped.", 282 priv->vdev->device->name); 283 mlx5_vdpa_arm_all_cqs(priv); 284 pthread_mutex_unlock(&priv->vq_config_lock); 285 pthread_mutex_lock(&priv->timer_lock); 286 priv->timer_on = 0; 287 while (!priv->timer_on) 288 pthread_cond_wait(&priv->timer_cond, 289 &priv->timer_lock); 290 pthread_mutex_unlock(&priv->timer_lock); 291 priv->timer_delay_us = priv->event_mode == 292 MLX5_VDPA_EVENT_MODE_DYNAMIC_TIMER ? 293 MLX5_VDPA_DEFAULT_TIMER_DELAY_US : 294 priv->event_us; 295 continue; 296 } 297 } else { 298 priv->last_traffic_tic = current_tic; 299 } 300 pthread_mutex_unlock(&priv->vq_config_lock); 301 mlx5_vdpa_timer_sleep(priv, max); 302 } 303 return NULL; 304 } 305 306 static void 307 mlx5_vdpa_interrupt_handler(void *cb_arg) 308 { 309 struct mlx5_vdpa_priv *priv = cb_arg; 310 #ifdef HAVE_IBV_DEVX_EVENT 311 union { 312 struct mlx5dv_devx_async_event_hdr event_resp; 313 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128]; 314 } out; 315 316 pthread_mutex_lock(&priv->vq_config_lock); 317 while (mlx5_glue->devx_get_event(priv->eventc, &out.event_resp, 318 sizeof(out.buf)) >= 319 (ssize_t)sizeof(out.event_resp.cookie)) { 320 struct mlx5_vdpa_cq *cq = (struct mlx5_vdpa_cq *) 321 (uintptr_t)out.event_resp.cookie; 322 struct mlx5_vdpa_event_qp *eqp = container_of(cq, 323 struct mlx5_vdpa_event_qp, cq); 324 struct mlx5_vdpa_virtq *virtq = container_of(eqp, 325 struct mlx5_vdpa_virtq, eqp); 326 327 if (!virtq->enable) 328 continue; 329 mlx5_vdpa_cq_poll(cq); 330 /* Notify guest for descs consuming. */ 331 if (cq->callfd != -1) 332 eventfd_write(cq->callfd, (eventfd_t)1); 333 if (priv->event_mode == MLX5_VDPA_EVENT_MODE_ONLY_INTERRUPT) { 334 mlx5_vdpa_cq_arm(priv, cq); 335 pthread_mutex_unlock(&priv->vq_config_lock); 336 return; 337 } 338 /* Don't arm again - timer will take control. */ 339 DRV_LOG(DEBUG, "Device %s virtq %d cq %d event was captured." 340 " Timer is %s, cq ci is %u.\n", 341 priv->vdev->device->name, 342 (int)virtq->index, cq->cq_obj.cq->id, 343 priv->timer_on ? "on" : "off", cq->cq_ci); 344 cq->armed = 0; 345 } 346 #endif 347 348 /* Traffic detected: make sure timer is on. */ 349 priv->last_traffic_tic = rte_rdtsc(); 350 pthread_mutex_lock(&priv->timer_lock); 351 if (!priv->timer_on) { 352 priv->timer_on = 1; 353 pthread_cond_signal(&priv->timer_cond); 354 } 355 pthread_mutex_unlock(&priv->timer_lock); 356 pthread_mutex_unlock(&priv->vq_config_lock); 357 } 358 359 static void 360 mlx5_vdpa_err_interrupt_handler(void *cb_arg __rte_unused) 361 { 362 #ifdef HAVE_IBV_DEVX_EVENT 363 struct mlx5_vdpa_priv *priv = cb_arg; 364 union { 365 struct mlx5dv_devx_async_event_hdr event_resp; 366 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128]; 367 } out; 368 uint32_t vq_index, i, version; 369 struct mlx5_vdpa_virtq *virtq; 370 uint64_t sec; 371 372 pthread_mutex_lock(&priv->vq_config_lock); 373 while (mlx5_glue->devx_get_event(priv->err_chnl, &out.event_resp, 374 sizeof(out.buf)) >= 375 (ssize_t)sizeof(out.event_resp.cookie)) { 376 vq_index = out.event_resp.cookie & UINT32_MAX; 377 version = out.event_resp.cookie >> 32; 378 if (vq_index >= priv->nr_virtqs) { 379 DRV_LOG(ERR, "Invalid device %s error event virtq %d.", 380 priv->vdev->device->name, vq_index); 381 continue; 382 } 383 virtq = &priv->virtqs[vq_index]; 384 if (!virtq->enable || virtq->version != version) 385 continue; 386 if (rte_rdtsc() / rte_get_tsc_hz() < MLX5_VDPA_ERROR_TIME_SEC) 387 continue; 388 virtq->stopped = true; 389 /* Query error info. */ 390 if (mlx5_vdpa_virtq_query(priv, vq_index)) 391 goto log; 392 /* Disable vq. */ 393 if (mlx5_vdpa_virtq_enable(priv, vq_index, 0)) { 394 DRV_LOG(ERR, "Failed to disable virtq %d.", vq_index); 395 goto log; 396 } 397 /* Retry if error happens less than N times in 3 seconds. */ 398 sec = (rte_rdtsc() - virtq->err_time[0]) / rte_get_tsc_hz(); 399 if (sec > MLX5_VDPA_ERROR_TIME_SEC) { 400 /* Retry. */ 401 if (mlx5_vdpa_virtq_enable(priv, vq_index, 1)) 402 DRV_LOG(ERR, "Failed to enable virtq %d.", 403 vq_index); 404 else 405 DRV_LOG(WARNING, "Recover virtq %d: %u.", 406 vq_index, ++virtq->n_retry); 407 } else { 408 /* Retry timeout, give up. */ 409 DRV_LOG(ERR, "Device %s virtq %d failed to recover.", 410 priv->vdev->device->name, vq_index); 411 } 412 log: 413 /* Shift in current time to error time log end. */ 414 for (i = 1; i < RTE_DIM(virtq->err_time); i++) 415 virtq->err_time[i - 1] = virtq->err_time[i]; 416 virtq->err_time[RTE_DIM(virtq->err_time) - 1] = rte_rdtsc(); 417 } 418 pthread_mutex_unlock(&priv->vq_config_lock); 419 #endif 420 } 421 422 int 423 mlx5_vdpa_err_event_setup(struct mlx5_vdpa_priv *priv) 424 { 425 int ret; 426 int flags; 427 428 /* Setup device event channel. */ 429 priv->err_chnl = mlx5_glue->devx_create_event_channel(priv->ctx, 0); 430 if (!priv->err_chnl) { 431 rte_errno = errno; 432 DRV_LOG(ERR, "Failed to create device event channel %d.", 433 rte_errno); 434 goto error; 435 } 436 flags = fcntl(priv->err_chnl->fd, F_GETFL); 437 ret = fcntl(priv->err_chnl->fd, F_SETFL, flags | O_NONBLOCK); 438 if (ret) { 439 DRV_LOG(ERR, "Failed to change device event channel FD."); 440 goto error; 441 } 442 priv->err_intr_handle.fd = priv->err_chnl->fd; 443 priv->err_intr_handle.type = RTE_INTR_HANDLE_EXT; 444 if (rte_intr_callback_register(&priv->err_intr_handle, 445 mlx5_vdpa_err_interrupt_handler, 446 priv)) { 447 priv->err_intr_handle.fd = 0; 448 DRV_LOG(ERR, "Failed to register error interrupt for device %d.", 449 priv->vid); 450 goto error; 451 } else { 452 DRV_LOG(DEBUG, "Registered error interrupt for device%d.", 453 priv->vid); 454 } 455 return 0; 456 error: 457 mlx5_vdpa_err_event_unset(priv); 458 return -1; 459 } 460 461 void 462 mlx5_vdpa_err_event_unset(struct mlx5_vdpa_priv *priv) 463 { 464 int retries = MLX5_VDPA_INTR_RETRIES; 465 int ret = -EAGAIN; 466 467 if (!priv->err_intr_handle.fd) 468 return; 469 while (retries-- && ret == -EAGAIN) { 470 ret = rte_intr_callback_unregister(&priv->err_intr_handle, 471 mlx5_vdpa_err_interrupt_handler, 472 priv); 473 if (ret == -EAGAIN) { 474 DRV_LOG(DEBUG, "Try again to unregister fd %d " 475 "of error interrupt, retries = %d.", 476 priv->err_intr_handle.fd, retries); 477 rte_pause(); 478 } 479 } 480 memset(&priv->err_intr_handle, 0, sizeof(priv->err_intr_handle)); 481 if (priv->err_chnl) { 482 #ifdef HAVE_IBV_DEVX_EVENT 483 union { 484 struct mlx5dv_devx_async_event_hdr event_resp; 485 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 486 128]; 487 } out; 488 489 /* Clean all pending events. */ 490 while (mlx5_glue->devx_get_event(priv->err_chnl, 491 &out.event_resp, sizeof(out.buf)) >= 492 (ssize_t)sizeof(out.event_resp.cookie)) 493 ; 494 #endif 495 mlx5_glue->devx_destroy_event_channel(priv->err_chnl); 496 priv->err_chnl = NULL; 497 } 498 } 499 500 int 501 mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv) 502 { 503 int ret; 504 rte_cpuset_t cpuset; 505 pthread_attr_t attr; 506 char name[16]; 507 const struct sched_param sp = { 508 .sched_priority = sched_get_priority_max(SCHED_RR), 509 }; 510 511 if (!priv->eventc) 512 /* All virtqs are in poll mode. */ 513 return 0; 514 if (priv->event_mode != MLX5_VDPA_EVENT_MODE_ONLY_INTERRUPT) { 515 pthread_mutex_init(&priv->timer_lock, NULL); 516 pthread_cond_init(&priv->timer_cond, NULL); 517 priv->timer_on = 0; 518 pthread_attr_init(&attr); 519 ret = pthread_attr_setschedpolicy(&attr, SCHED_RR); 520 if (ret) { 521 DRV_LOG(ERR, "Failed to set thread sched policy = RR."); 522 return -1; 523 } 524 ret = pthread_attr_setschedparam(&attr, &sp); 525 if (ret) { 526 DRV_LOG(ERR, "Failed to set thread priority."); 527 return -1; 528 } 529 ret = pthread_create(&priv->timer_tid, &attr, 530 mlx5_vdpa_poll_handle, (void *)priv); 531 if (ret) { 532 DRV_LOG(ERR, "Failed to create timer thread."); 533 return -1; 534 } 535 CPU_ZERO(&cpuset); 536 if (priv->event_core != -1) 537 CPU_SET(priv->event_core, &cpuset); 538 else 539 cpuset = rte_lcore_cpuset(rte_get_main_lcore()); 540 ret = pthread_setaffinity_np(priv->timer_tid, 541 sizeof(cpuset), &cpuset); 542 if (ret) { 543 DRV_LOG(ERR, "Failed to set thread affinity."); 544 goto error; 545 } 546 snprintf(name, sizeof(name), "vDPA-mlx5-%d", priv->vid); 547 ret = pthread_setname_np(priv->timer_tid, name); 548 if (ret) { 549 DRV_LOG(ERR, "Failed to set timer thread name."); 550 return -1; 551 } 552 } 553 priv->intr_handle.fd = priv->eventc->fd; 554 priv->intr_handle.type = RTE_INTR_HANDLE_EXT; 555 if (rte_intr_callback_register(&priv->intr_handle, 556 mlx5_vdpa_interrupt_handler, priv)) { 557 priv->intr_handle.fd = 0; 558 DRV_LOG(ERR, "Failed to register CQE interrupt %d.", rte_errno); 559 goto error; 560 } 561 return 0; 562 error: 563 mlx5_vdpa_cqe_event_unset(priv); 564 return -1; 565 } 566 567 void 568 mlx5_vdpa_cqe_event_unset(struct mlx5_vdpa_priv *priv) 569 { 570 int retries = MLX5_VDPA_INTR_RETRIES; 571 int ret = -EAGAIN; 572 void *status; 573 574 if (priv->intr_handle.fd) { 575 while (retries-- && ret == -EAGAIN) { 576 ret = rte_intr_callback_unregister(&priv->intr_handle, 577 mlx5_vdpa_interrupt_handler, 578 priv); 579 if (ret == -EAGAIN) { 580 DRV_LOG(DEBUG, "Try again to unregister fd %d " 581 "of CQ interrupt, retries = %d.", 582 priv->intr_handle.fd, retries); 583 rte_pause(); 584 } 585 } 586 memset(&priv->intr_handle, 0, sizeof(priv->intr_handle)); 587 } 588 if (priv->timer_tid) { 589 pthread_cancel(priv->timer_tid); 590 pthread_join(priv->timer_tid, &status); 591 } 592 priv->timer_tid = 0; 593 } 594 595 void 596 mlx5_vdpa_event_qp_destroy(struct mlx5_vdpa_event_qp *eqp) 597 { 598 if (eqp->sw_qp) 599 claim_zero(mlx5_devx_cmd_destroy(eqp->sw_qp)); 600 if (eqp->umem_obj) 601 claim_zero(mlx5_glue->devx_umem_dereg(eqp->umem_obj)); 602 if (eqp->umem_buf) 603 rte_free(eqp->umem_buf); 604 if (eqp->fw_qp) 605 claim_zero(mlx5_devx_cmd_destroy(eqp->fw_qp)); 606 mlx5_vdpa_cq_destroy(&eqp->cq); 607 memset(eqp, 0, sizeof(*eqp)); 608 } 609 610 static int 611 mlx5_vdpa_qps2rts(struct mlx5_vdpa_event_qp *eqp) 612 { 613 if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_RST2INIT_QP, 614 eqp->sw_qp->id)) { 615 DRV_LOG(ERR, "Failed to modify FW QP to INIT state(%u).", 616 rte_errno); 617 return -1; 618 } 619 if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_RST2INIT_QP, 620 eqp->fw_qp->id)) { 621 DRV_LOG(ERR, "Failed to modify SW QP to INIT state(%u).", 622 rte_errno); 623 return -1; 624 } 625 if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_INIT2RTR_QP, 626 eqp->sw_qp->id)) { 627 DRV_LOG(ERR, "Failed to modify FW QP to RTR state(%u).", 628 rte_errno); 629 return -1; 630 } 631 if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_INIT2RTR_QP, 632 eqp->fw_qp->id)) { 633 DRV_LOG(ERR, "Failed to modify SW QP to RTR state(%u).", 634 rte_errno); 635 return -1; 636 } 637 if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_RTR2RTS_QP, 638 eqp->sw_qp->id)) { 639 DRV_LOG(ERR, "Failed to modify FW QP to RTS state(%u).", 640 rte_errno); 641 return -1; 642 } 643 if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_RTR2RTS_QP, 644 eqp->fw_qp->id)) { 645 DRV_LOG(ERR, "Failed to modify SW QP to RTS state(%u).", 646 rte_errno); 647 return -1; 648 } 649 return 0; 650 } 651 652 int 653 mlx5_vdpa_event_qp_create(struct mlx5_vdpa_priv *priv, uint16_t desc_n, 654 int callfd, struct mlx5_vdpa_event_qp *eqp) 655 { 656 struct mlx5_devx_qp_attr attr = {0}; 657 uint16_t log_desc_n = rte_log2_u32(desc_n); 658 uint32_t umem_size = (1 << log_desc_n) * MLX5_WSEG_SIZE + 659 sizeof(*eqp->db_rec) * 2; 660 661 if (mlx5_vdpa_event_qp_global_prepare(priv)) 662 return -1; 663 if (mlx5_vdpa_cq_create(priv, log_desc_n, callfd, &eqp->cq)) 664 return -1; 665 attr.pd = priv->pdn; 666 attr.ts_format = mlx5_ts_format_conv(priv->qp_ts_format); 667 eqp->fw_qp = mlx5_devx_cmd_create_qp(priv->ctx, &attr); 668 if (!eqp->fw_qp) { 669 DRV_LOG(ERR, "Failed to create FW QP(%u).", rte_errno); 670 goto error; 671 } 672 eqp->umem_buf = rte_zmalloc(__func__, umem_size, 4096); 673 if (!eqp->umem_buf) { 674 DRV_LOG(ERR, "Failed to allocate memory for SW QP."); 675 rte_errno = ENOMEM; 676 goto error; 677 } 678 eqp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx, 679 (void *)(uintptr_t)eqp->umem_buf, 680 umem_size, 681 IBV_ACCESS_LOCAL_WRITE); 682 if (!eqp->umem_obj) { 683 DRV_LOG(ERR, "Failed to register umem for SW QP."); 684 goto error; 685 } 686 attr.uar_index = priv->uar->page_id; 687 attr.cqn = eqp->cq.cq_obj.cq->id; 688 attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE)); 689 attr.rq_size = 1 << log_desc_n; 690 attr.log_rq_stride = rte_log2_u32(MLX5_WSEG_SIZE); 691 attr.sq_size = 0; /* No need SQ. */ 692 attr.dbr_umem_valid = 1; 693 attr.wq_umem_id = eqp->umem_obj->umem_id; 694 attr.wq_umem_offset = 0; 695 attr.dbr_umem_id = eqp->umem_obj->umem_id; 696 attr.dbr_address = (1 << log_desc_n) * MLX5_WSEG_SIZE; 697 attr.ts_format = mlx5_ts_format_conv(priv->qp_ts_format); 698 eqp->sw_qp = mlx5_devx_cmd_create_qp(priv->ctx, &attr); 699 if (!eqp->sw_qp) { 700 DRV_LOG(ERR, "Failed to create SW QP(%u).", rte_errno); 701 goto error; 702 } 703 eqp->db_rec = RTE_PTR_ADD(eqp->umem_buf, (uintptr_t)attr.dbr_address); 704 if (mlx5_vdpa_qps2rts(eqp)) 705 goto error; 706 /* First ringing. */ 707 rte_write32(rte_cpu_to_be_32(1 << log_desc_n), &eqp->db_rec[0]); 708 return 0; 709 error: 710 mlx5_vdpa_event_qp_destroy(eqp); 711 return -1; 712 } 713