1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 #include <stdint.h> 5 6 #include <rte_errno.h> 7 #include <rte_common.h> 8 #include <rte_eal_paging.h> 9 10 #include <mlx5_glue.h> 11 #include <mlx5_common_os.h> 12 13 #include "mlx5_prm.h" 14 #include "mlx5_devx_cmds.h" 15 #include "mlx5_common_log.h" 16 #include "mlx5_malloc.h" 17 #include "mlx5_common.h" 18 #include "mlx5_common_devx.h" 19 20 /** 21 * Destroy DevX Completion Queue. 22 * 23 * @param[in] cq 24 * DevX CQ to destroy. 25 */ 26 void 27 mlx5_devx_cq_destroy(struct mlx5_devx_cq *cq) 28 { 29 if (cq->cq) 30 claim_zero(mlx5_devx_cmd_destroy(cq->cq)); 31 if (cq->umem_obj) 32 claim_zero(mlx5_os_umem_dereg(cq->umem_obj)); 33 if (cq->umem_buf) 34 mlx5_free((void *)(uintptr_t)cq->umem_buf); 35 } 36 37 /* Mark all CQEs initially as invalid. */ 38 static void 39 mlx5_cq_init(struct mlx5_devx_cq *cq_obj, uint16_t cq_size) 40 { 41 volatile struct mlx5_cqe *cqe = cq_obj->cqes; 42 uint16_t i; 43 44 for (i = 0; i < cq_size; i++, cqe++) 45 cqe->op_own = (MLX5_CQE_INVALID << 4) | MLX5_CQE_OWNER_MASK; 46 } 47 48 /** 49 * Create Completion Queue using DevX API. 50 * 51 * Get a pointer to partially initialized attributes structure, and updates the 52 * following fields: 53 * q_umem_valid 54 * q_umem_id 55 * q_umem_offset 56 * db_umem_valid 57 * db_umem_id 58 * db_umem_offset 59 * eqn 60 * log_cq_size 61 * log_page_size 62 * All other fields are updated by caller. 63 * 64 * @param[in] ctx 65 * Context returned from mlx5 open_device() glue function. 66 * @param[in/out] cq_obj 67 * Pointer to CQ to create. 68 * @param[in] log_desc_n 69 * Log of number of descriptors in queue. 70 * @param[in] attr 71 * Pointer to CQ attributes structure. 72 * @param[in] socket 73 * Socket to use for allocation. 74 * 75 * @return 76 * 0 on success, a negative errno value otherwise and rte_errno is set. 77 */ 78 int 79 mlx5_devx_cq_create(void *ctx, struct mlx5_devx_cq *cq_obj, uint16_t log_desc_n, 80 struct mlx5_devx_cq_attr *attr, int socket) 81 { 82 struct mlx5_devx_obj *cq = NULL; 83 struct mlx5dv_devx_umem *umem_obj = NULL; 84 void *umem_buf = NULL; 85 size_t page_size = rte_mem_page_size(); 86 size_t alignment = MLX5_CQE_BUF_ALIGNMENT; 87 uint32_t umem_size, umem_dbrec; 88 uint32_t eqn; 89 uint16_t cq_size = 1 << log_desc_n; 90 int ret; 91 92 if (page_size == (size_t)-1 || alignment == (size_t)-1) { 93 DRV_LOG(ERR, "Failed to get page_size."); 94 rte_errno = ENOMEM; 95 return -rte_errno; 96 } 97 /* Query first EQN. */ 98 ret = mlx5_glue->devx_query_eqn(ctx, 0, &eqn); 99 if (ret) { 100 rte_errno = errno; 101 DRV_LOG(ERR, "Failed to query event queue number."); 102 return -rte_errno; 103 } 104 /* Allocate memory buffer for CQEs and doorbell record. */ 105 umem_size = sizeof(struct mlx5_cqe) * cq_size; 106 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); 107 umem_size += MLX5_DBR_SIZE; 108 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size, 109 alignment, socket); 110 if (!umem_buf) { 111 DRV_LOG(ERR, "Failed to allocate memory for CQ."); 112 rte_errno = ENOMEM; 113 return -rte_errno; 114 } 115 /* Register allocated buffer in user space with DevX. */ 116 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, umem_size, 117 IBV_ACCESS_LOCAL_WRITE); 118 if (!umem_obj) { 119 DRV_LOG(ERR, "Failed to register umem for CQ."); 120 rte_errno = errno; 121 goto error; 122 } 123 /* Fill attributes for CQ object creation. */ 124 attr->q_umem_valid = 1; 125 attr->q_umem_id = mlx5_os_get_umem_id(umem_obj); 126 attr->q_umem_offset = 0; 127 attr->db_umem_valid = 1; 128 attr->db_umem_id = attr->q_umem_id; 129 attr->db_umem_offset = umem_dbrec; 130 attr->eqn = eqn; 131 attr->log_cq_size = log_desc_n; 132 attr->log_page_size = rte_log2_u32(page_size); 133 /* Create completion queue object with DevX. */ 134 cq = mlx5_devx_cmd_create_cq(ctx, attr); 135 if (!cq) { 136 DRV_LOG(ERR, "Can't create DevX CQ object."); 137 rte_errno = ENOMEM; 138 goto error; 139 } 140 cq_obj->umem_buf = umem_buf; 141 cq_obj->umem_obj = umem_obj; 142 cq_obj->cq = cq; 143 cq_obj->db_rec = RTE_PTR_ADD(cq_obj->umem_buf, umem_dbrec); 144 /* Mark all CQEs initially as invalid. */ 145 mlx5_cq_init(cq_obj, cq_size); 146 return 0; 147 error: 148 ret = rte_errno; 149 if (umem_obj) 150 claim_zero(mlx5_os_umem_dereg(umem_obj)); 151 if (umem_buf) 152 mlx5_free((void *)(uintptr_t)umem_buf); 153 rte_errno = ret; 154 return -rte_errno; 155 } 156 157 /** 158 * Destroy DevX Send Queue. 159 * 160 * @param[in] sq 161 * DevX SQ to destroy. 162 */ 163 void 164 mlx5_devx_sq_destroy(struct mlx5_devx_sq *sq) 165 { 166 if (sq->sq) 167 claim_zero(mlx5_devx_cmd_destroy(sq->sq)); 168 if (sq->umem_obj) 169 claim_zero(mlx5_os_umem_dereg(sq->umem_obj)); 170 if (sq->umem_buf) 171 mlx5_free((void *)(uintptr_t)sq->umem_buf); 172 } 173 174 /** 175 * Create Send Queue using DevX API. 176 * 177 * Get a pointer to partially initialized attributes structure, and updates the 178 * following fields: 179 * wq_type 180 * wq_umem_valid 181 * wq_umem_id 182 * wq_umem_offset 183 * dbr_umem_valid 184 * dbr_umem_id 185 * dbr_addr 186 * log_wq_stride 187 * log_wq_sz 188 * log_wq_pg_sz 189 * All other fields are updated by caller. 190 * 191 * @param[in] ctx 192 * Context returned from mlx5 open_device() glue function. 193 * @param[in/out] sq_obj 194 * Pointer to SQ to create. 195 * @param[in] log_wqbb_n 196 * Log of number of WQBBs in queue. 197 * @param[in] attr 198 * Pointer to SQ attributes structure. 199 * @param[in] socket 200 * Socket to use for allocation. 201 * 202 * @return 203 * 0 on success, a negative errno value otherwise and rte_errno is set. 204 */ 205 int 206 mlx5_devx_sq_create(void *ctx, struct mlx5_devx_sq *sq_obj, uint16_t log_wqbb_n, 207 struct mlx5_devx_create_sq_attr *attr, int socket) 208 { 209 struct mlx5_devx_obj *sq = NULL; 210 struct mlx5dv_devx_umem *umem_obj = NULL; 211 void *umem_buf = NULL; 212 size_t alignment = MLX5_WQE_BUF_ALIGNMENT; 213 uint32_t umem_size, umem_dbrec; 214 uint16_t sq_size = 1 << log_wqbb_n; 215 int ret; 216 217 if (alignment == (size_t)-1) { 218 DRV_LOG(ERR, "Failed to get WQE buf alignment."); 219 rte_errno = ENOMEM; 220 return -rte_errno; 221 } 222 /* Allocate memory buffer for WQEs and doorbell record. */ 223 umem_size = MLX5_WQE_SIZE * sq_size; 224 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); 225 umem_size += MLX5_DBR_SIZE; 226 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size, 227 alignment, socket); 228 if (!umem_buf) { 229 DRV_LOG(ERR, "Failed to allocate memory for SQ."); 230 rte_errno = ENOMEM; 231 return -rte_errno; 232 } 233 /* Register allocated buffer in user space with DevX. */ 234 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, umem_size, 235 IBV_ACCESS_LOCAL_WRITE); 236 if (!umem_obj) { 237 DRV_LOG(ERR, "Failed to register umem for SQ."); 238 rte_errno = errno; 239 goto error; 240 } 241 /* Fill attributes for SQ object creation. */ 242 attr->wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 243 attr->wq_attr.wq_umem_valid = 1; 244 attr->wq_attr.wq_umem_id = mlx5_os_get_umem_id(umem_obj); 245 attr->wq_attr.wq_umem_offset = 0; 246 attr->wq_attr.dbr_umem_valid = 1; 247 attr->wq_attr.dbr_umem_id = attr->wq_attr.wq_umem_id; 248 attr->wq_attr.dbr_addr = umem_dbrec; 249 attr->wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE); 250 attr->wq_attr.log_wq_sz = log_wqbb_n; 251 attr->wq_attr.log_wq_pg_sz = MLX5_LOG_PAGE_SIZE; 252 /* Create send queue object with DevX. */ 253 sq = mlx5_devx_cmd_create_sq(ctx, attr); 254 if (!sq) { 255 DRV_LOG(ERR, "Can't create DevX SQ object."); 256 rte_errno = ENOMEM; 257 goto error; 258 } 259 sq_obj->umem_buf = umem_buf; 260 sq_obj->umem_obj = umem_obj; 261 sq_obj->sq = sq; 262 sq_obj->db_rec = RTE_PTR_ADD(sq_obj->umem_buf, umem_dbrec); 263 return 0; 264 error: 265 ret = rte_errno; 266 if (umem_obj) 267 claim_zero(mlx5_os_umem_dereg(umem_obj)); 268 if (umem_buf) 269 mlx5_free((void *)(uintptr_t)umem_buf); 270 rte_errno = ret; 271 return -rte_errno; 272 } 273 274 /** 275 * Destroy DevX Queue Pair. 276 * 277 * @param[in] qp 278 * DevX QP to destroy. 279 */ 280 void 281 mlx5_devx_qp_destroy(struct mlx5_devx_qp *qp) 282 { 283 if (qp->qp) 284 claim_zero(mlx5_devx_cmd_destroy(qp->qp)); 285 if (qp->umem_obj) 286 claim_zero(mlx5_os_umem_dereg(qp->umem_obj)); 287 if (qp->umem_buf) 288 mlx5_free((void *)(uintptr_t)qp->umem_buf); 289 } 290 291 /** 292 * Create Queue Pair using DevX API. 293 * 294 * Get a pointer to partially initialized attributes structure, and updates the 295 * following fields: 296 * wq_umem_id 297 * wq_umem_offset 298 * dbr_umem_valid 299 * dbr_umem_id 300 * dbr_address 301 * log_page_size 302 * All other fields are updated by caller. 303 * 304 * @param[in] ctx 305 * Context returned from mlx5 open_device() glue function. 306 * @param[in/out] qp_obj 307 * Pointer to QP to create. 308 * @param[in] log_wqbb_n 309 * Log of number of WQBBs in queue. 310 * @param[in] attr 311 * Pointer to QP attributes structure. 312 * @param[in] socket 313 * Socket to use for allocation. 314 * 315 * @return 316 * 0 on success, a negative errno value otherwise and rte_errno is set. 317 */ 318 int 319 mlx5_devx_qp_create(void *ctx, struct mlx5_devx_qp *qp_obj, uint16_t log_wqbb_n, 320 struct mlx5_devx_qp_attr *attr, int socket) 321 { 322 struct mlx5_devx_obj *qp = NULL; 323 struct mlx5dv_devx_umem *umem_obj = NULL; 324 void *umem_buf = NULL; 325 size_t alignment = MLX5_WQE_BUF_ALIGNMENT; 326 uint32_t umem_size, umem_dbrec; 327 uint16_t qp_size = 1 << log_wqbb_n; 328 int ret; 329 330 if (alignment == (size_t)-1) { 331 DRV_LOG(ERR, "Failed to get WQE buf alignment."); 332 rte_errno = ENOMEM; 333 return -rte_errno; 334 } 335 /* Allocate memory buffer for WQEs and doorbell record. */ 336 umem_size = MLX5_WQE_SIZE * qp_size; 337 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); 338 umem_size += MLX5_DBR_SIZE; 339 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size, 340 alignment, socket); 341 if (!umem_buf) { 342 DRV_LOG(ERR, "Failed to allocate memory for QP."); 343 rte_errno = ENOMEM; 344 return -rte_errno; 345 } 346 /* Register allocated buffer in user space with DevX. */ 347 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, umem_size, 348 IBV_ACCESS_LOCAL_WRITE); 349 if (!umem_obj) { 350 DRV_LOG(ERR, "Failed to register umem for QP."); 351 rte_errno = errno; 352 goto error; 353 } 354 /* Fill attributes for SQ object creation. */ 355 attr->wq_umem_id = mlx5_os_get_umem_id(umem_obj); 356 attr->wq_umem_offset = 0; 357 attr->dbr_umem_valid = 1; 358 attr->dbr_umem_id = attr->wq_umem_id; 359 attr->dbr_address = umem_dbrec; 360 attr->log_page_size = MLX5_LOG_PAGE_SIZE; 361 /* Create send queue object with DevX. */ 362 qp = mlx5_devx_cmd_create_qp(ctx, attr); 363 if (!qp) { 364 DRV_LOG(ERR, "Can't create DevX QP object."); 365 rte_errno = ENOMEM; 366 goto error; 367 } 368 qp_obj->umem_buf = umem_buf; 369 qp_obj->umem_obj = umem_obj; 370 qp_obj->qp = qp; 371 qp_obj->db_rec = RTE_PTR_ADD(qp_obj->umem_buf, umem_dbrec); 372 return 0; 373 error: 374 ret = rte_errno; 375 if (umem_obj) 376 claim_zero(mlx5_os_umem_dereg(umem_obj)); 377 if (umem_buf) 378 mlx5_free((void *)(uintptr_t)umem_buf); 379 rte_errno = ret; 380 return -rte_errno; 381 } 382 383 /** 384 * Destroy DevX Receive Queue. 385 * 386 * @param[in] rq 387 * DevX RQ to destroy. 388 */ 389 void 390 mlx5_devx_rq_destroy(struct mlx5_devx_rq *rq) 391 { 392 if (rq->rq) 393 claim_zero(mlx5_devx_cmd_destroy(rq->rq)); 394 if (rq->umem_obj) 395 claim_zero(mlx5_os_umem_dereg(rq->umem_obj)); 396 if (rq->umem_buf) 397 mlx5_free((void *)(uintptr_t)rq->umem_buf); 398 } 399 400 /** 401 * Create Receive Queue using DevX API. 402 * 403 * Get a pointer to partially initialized attributes structure, and updates the 404 * following fields: 405 * wq_umem_valid 406 * wq_umem_id 407 * wq_umem_offset 408 * dbr_umem_valid 409 * dbr_umem_id 410 * dbr_addr 411 * log_wq_pg_sz 412 * All other fields are updated by caller. 413 * 414 * @param[in] ctx 415 * Context returned from mlx5 open_device() glue function. 416 * @param[in/out] rq_obj 417 * Pointer to RQ to create. 418 * @param[in] wqe_size 419 * Size of WQE structure. 420 * @param[in] log_wqbb_n 421 * Log of number of WQBBs in queue. 422 * @param[in] attr 423 * Pointer to RQ attributes structure. 424 * @param[in] socket 425 * Socket to use for allocation. 426 * 427 * @return 428 * 0 on success, a negative errno value otherwise and rte_errno is set. 429 */ 430 int 431 mlx5_devx_rq_create(void *ctx, struct mlx5_devx_rq *rq_obj, uint32_t wqe_size, 432 uint16_t log_wqbb_n, 433 struct mlx5_devx_create_rq_attr *attr, int socket) 434 { 435 struct mlx5_devx_obj *rq = NULL; 436 struct mlx5dv_devx_umem *umem_obj = NULL; 437 void *umem_buf = NULL; 438 size_t alignment = MLX5_WQE_BUF_ALIGNMENT; 439 uint32_t umem_size, umem_dbrec; 440 uint16_t rq_size = 1 << log_wqbb_n; 441 int ret; 442 443 if (alignment == (size_t)-1) { 444 DRV_LOG(ERR, "Failed to get WQE buf alignment."); 445 rte_errno = ENOMEM; 446 return -rte_errno; 447 } 448 /* Allocate memory buffer for WQEs and doorbell record. */ 449 umem_size = wqe_size * rq_size; 450 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); 451 umem_size += MLX5_DBR_SIZE; 452 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size, 453 alignment, socket); 454 if (!umem_buf) { 455 DRV_LOG(ERR, "Failed to allocate memory for RQ."); 456 rte_errno = ENOMEM; 457 return -rte_errno; 458 } 459 /* Register allocated buffer in user space with DevX. */ 460 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, 461 umem_size, 0); 462 if (!umem_obj) { 463 DRV_LOG(ERR, "Failed to register umem for RQ."); 464 rte_errno = errno; 465 goto error; 466 } 467 /* Fill attributes for RQ object creation. */ 468 attr->wq_attr.wq_umem_valid = 1; 469 attr->wq_attr.wq_umem_id = mlx5_os_get_umem_id(umem_obj); 470 attr->wq_attr.wq_umem_offset = 0; 471 attr->wq_attr.dbr_umem_valid = 1; 472 attr->wq_attr.dbr_umem_id = attr->wq_attr.wq_umem_id; 473 attr->wq_attr.dbr_addr = umem_dbrec; 474 attr->wq_attr.log_wq_pg_sz = MLX5_LOG_PAGE_SIZE; 475 /* Create receive queue object with DevX. */ 476 rq = mlx5_devx_cmd_create_rq(ctx, attr, socket); 477 if (!rq) { 478 DRV_LOG(ERR, "Can't create DevX RQ object."); 479 rte_errno = ENOMEM; 480 goto error; 481 } 482 rq_obj->umem_buf = umem_buf; 483 rq_obj->umem_obj = umem_obj; 484 rq_obj->rq = rq; 485 rq_obj->db_rec = RTE_PTR_ADD(rq_obj->umem_buf, umem_dbrec); 486 return 0; 487 error: 488 ret = rte_errno; 489 if (umem_obj) 490 claim_zero(mlx5_os_umem_dereg(umem_obj)); 491 if (umem_buf) 492 mlx5_free((void *)(uintptr_t)umem_buf); 493 rte_errno = ret; 494 return -rte_errno; 495 } 496 497 498 /** 499 * Change QP state to RTS. 500 * 501 * @param[in] qp 502 * DevX QP to change. 503 * @param[in] remote_qp_id 504 * The remote QP ID for MLX5_CMD_OP_INIT2RTR_QP operation. 505 * 506 * @return 507 * 0 on success, a negative errno value otherwise and rte_errno is set. 508 */ 509 int 510 mlx5_devx_qp2rts(struct mlx5_devx_qp *qp, uint32_t remote_qp_id) 511 { 512 if (mlx5_devx_cmd_modify_qp_state(qp->qp, MLX5_CMD_OP_RST2INIT_QP, 513 remote_qp_id)) { 514 DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).", 515 rte_errno); 516 return -1; 517 } 518 if (mlx5_devx_cmd_modify_qp_state(qp->qp, MLX5_CMD_OP_INIT2RTR_QP, 519 remote_qp_id)) { 520 DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).", 521 rte_errno); 522 return -1; 523 } 524 if (mlx5_devx_cmd_modify_qp_state(qp->qp, MLX5_CMD_OP_RTR2RTS_QP, 525 remote_qp_id)) { 526 DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).", 527 rte_errno); 528 return -1; 529 } 530 return 0; 531 } 532