1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 */ 4 5 #include <infiniband/verbs.h> 6 7 #include "spdk/log.h" 8 #include "spdk/util.h" 9 #include "spdk/likely.h" 10 #include "spdk/thread.h" 11 #include "spdk/tree.h" 12 13 #include "spdk_internal/rdma_utils.h" 14 #include "mlx5_priv.h" 15 #include "mlx5_ifc.h" 16 17 #define MLX5_UMR_POOL_VALID_FLAGS_MASK (~(SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO | SPDK_MLX5_MKEY_POOL_FLAG_SIGNATURE)) 18 #define MLX5_CRYPTO_BSF_P_TYPE_CRYPTO (0x1) 19 #define MLX5_CRYPTO_BSF_SIZE_64B (0x2) 20 21 #define MLX5_SIG_BSF_SIZE_32B (0x1) 22 /* Transaction Format Selector */ 23 #define MLX5_SIG_BSF_TFS_CRC32C (64) 24 #define MLX5_SIG_BSF_TFS_SHIFT (24) 25 /* Transaction Init/Check_gen bits */ 26 #define MLX5_SIG_BSF_EXT_M_T_CHECK_GEN (1u << 24) 27 #define MLX5_SIG_BSF_EXT_M_T_INIT (1u << 25) 28 #define MLX5_SIG_BSF_EXT_W_T_CHECK_GEN (1u << 28) 29 #define MLX5_SIG_BSF_EXT_W_T_INIT (1u << 29) 30 31 RB_HEAD(mlx5_mkeys_tree, spdk_mlx5_mkey_pool_obj); 32 33 struct mlx5_relaxed_ordering_caps { 34 bool relaxed_ordering_write_pci_enabled; 35 bool relaxed_ordering_write; 36 bool relaxed_ordering_read; 37 bool relaxed_ordering_write_umr; 38 bool relaxed_ordering_read_umr; 39 }; 40 41 struct mlx5_mkey_attr { 42 uint64_t addr; 43 uint64_t size; 44 uint32_t log_entity_size; 45 struct mlx5_wqe_data_seg *klm; 46 uint32_t klm_count; 47 /* Size of bsf in octowords. If 0 then bsf is disabled */ 48 uint32_t bsf_octowords; 49 bool crypto_en; 50 bool relaxed_ordering_write; 51 bool relaxed_ordering_read; 52 }; 53 54 struct mlx5_mkey { 55 struct mlx5dv_devx_obj *devx_obj; 56 uint32_t mkey; 57 uint64_t addr; 58 }; 59 60 struct spdk_mlx5_mkey_pool { 61 struct ibv_pd *pd; 62 struct spdk_mempool *mpool; 63 struct mlx5_mkeys_tree tree; 64 struct mlx5_mkey **mkeys; 65 uint32_t num_mkeys; 66 uint32_t refcnt; 67 uint32_t flags; 68 TAILQ_ENTRY(spdk_mlx5_mkey_pool) link; 69 }; 70 71 static int 72 mlx5_key_obj_compare(struct spdk_mlx5_mkey_pool_obj *key1, struct spdk_mlx5_mkey_pool_obj *key2) 73 { 74 return key1->mkey < key2->mkey ? -1 : key1->mkey > key2->mkey; 75 } 76 77 RB_GENERATE_STATIC(mlx5_mkeys_tree, spdk_mlx5_mkey_pool_obj, node, mlx5_key_obj_compare); 78 79 static TAILQ_HEAD(mlx5_mkey_pool_head, 80 spdk_mlx5_mkey_pool) g_mkey_pools = TAILQ_HEAD_INITIALIZER(g_mkey_pools); 81 static pthread_mutex_t g_mkey_pool_lock = PTHREAD_MUTEX_INITIALIZER; 82 83 #define SPDK_KLM_MAX_TRANSLATION_ENTRIES_NUM 128 84 85 static struct mlx5_mkey * 86 mlx5_mkey_create(struct ibv_pd *pd, struct mlx5_mkey_attr *attr) 87 { 88 struct mlx5_wqe_data_seg *klms = attr->klm; 89 uint32_t klm_count = attr->klm_count; 90 int in_size_dw = DEVX_ST_SZ_DW(create_mkey_in) + 91 (klm_count ? SPDK_ALIGN_CEIL(klm_count, 4) : 0) * DEVX_ST_SZ_DW(klm); 92 uint32_t in[in_size_dw]; 93 uint32_t out[DEVX_ST_SZ_DW(create_mkey_out)] = {0}; 94 void *mkc; 95 uint32_t translation_size; 96 struct mlx5_mkey *cmkey; 97 struct ibv_context *ctx = pd->context; 98 uint32_t pd_id = 0; 99 uint32_t i; 100 uint8_t *klm; 101 102 cmkey = calloc(1, sizeof(*cmkey)); 103 if (!cmkey) { 104 SPDK_ERRLOG("failed to alloc cross_mkey\n"); 105 return NULL; 106 } 107 108 memset(in, 0, in_size_dw * 4); 109 DEVX_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY); 110 mkc = DEVX_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 111 112 if (klm_count > 0) { 113 klm = (uint8_t *)DEVX_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 114 translation_size = SPDK_ALIGN_CEIL(klm_count, 4); 115 116 for (i = 0; i < klm_count; i++) { 117 DEVX_SET(klm, klm, byte_count, klms[i].byte_count); 118 DEVX_SET(klm, klm, mkey, klms[i].lkey); 119 DEVX_SET64(klm, klm, address, klms[i].addr); 120 klms += DEVX_ST_SZ_BYTES(klm); 121 } 122 123 for (; i < translation_size; i++) { 124 DEVX_SET(klm, klms, byte_count, 0x0); 125 DEVX_SET(klm, klms, mkey, 0x0); 126 DEVX_SET64(klm, klms, address, 0x0); 127 klm += DEVX_ST_SZ_BYTES(klm); 128 } 129 } 130 131 DEVX_SET(mkc, mkc, access_mode_1_0, attr->log_entity_size ? 132 MLX5_MKC_ACCESS_MODE_KLMFBS : 133 MLX5_MKC_ACCESS_MODE_KLMS); 134 DEVX_SET(mkc, mkc, log_page_size, attr->log_entity_size); 135 136 mlx5_get_pd_id(pd, &pd_id); 137 DEVX_SET(create_mkey_in, in, translations_octword_actual_size, klm_count); 138 if (klm_count == 0) { 139 DEVX_SET(mkc, mkc, free, 0x1); 140 } 141 DEVX_SET(mkc, mkc, lw, 0x1); 142 DEVX_SET(mkc, mkc, lr, 0x1); 143 DEVX_SET(mkc, mkc, rw, 0x1); 144 DEVX_SET(mkc, mkc, rr, 0x1); 145 DEVX_SET(mkc, mkc, umr_en, 1); 146 DEVX_SET(mkc, mkc, qpn, 0xffffff); 147 DEVX_SET(mkc, mkc, pd, pd_id); 148 DEVX_SET(mkc, mkc, translations_octword_size, 149 SPDK_KLM_MAX_TRANSLATION_ENTRIES_NUM); 150 DEVX_SET(mkc, mkc, relaxed_ordering_write, 151 attr->relaxed_ordering_write); 152 DEVX_SET(mkc, mkc, relaxed_ordering_read, 153 attr->relaxed_ordering_read); 154 DEVX_SET64(mkc, mkc, start_addr, attr->addr); 155 DEVX_SET64(mkc, mkc, len, attr->size); 156 DEVX_SET(mkc, mkc, mkey_7_0, 0x42); 157 if (attr->crypto_en) { 158 DEVX_SET(mkc, mkc, crypto_en, 1); 159 } 160 if (attr->bsf_octowords) { 161 DEVX_SET(mkc, mkc, bsf_en, 1); 162 DEVX_SET(mkc, mkc, bsf_octword_size, attr->bsf_octowords); 163 } 164 165 cmkey->devx_obj = mlx5dv_devx_obj_create(ctx, in, sizeof(in), out, 166 sizeof(out)); 167 if (!cmkey->devx_obj) { 168 SPDK_ERRLOG("mlx5dv_devx_obj_create() failed to create mkey, errno:%d\n", errno); 169 goto out_err; 170 } 171 172 cmkey->mkey = DEVX_GET(create_mkey_out, out, mkey_index) << 8 | 0x42; 173 return cmkey; 174 175 out_err: 176 free(cmkey); 177 return NULL; 178 } 179 180 static int 181 mlx5_mkey_destroy(struct mlx5_mkey *mkey) 182 { 183 int ret = 0; 184 185 if (mkey->devx_obj) { 186 ret = mlx5dv_devx_obj_destroy(mkey->devx_obj); 187 } 188 189 free(mkey); 190 191 return ret; 192 } 193 194 static int 195 mlx5_query_relaxed_ordering_caps(struct ibv_context *context, 196 struct mlx5_relaxed_ordering_caps *caps) 197 { 198 uint8_t in[DEVX_ST_SZ_BYTES(query_hca_cap_in)] = {}; 199 uint8_t out[DEVX_ST_SZ_BYTES(query_hca_cap_out)] = {}; 200 int ret; 201 202 DEVX_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 203 DEVX_SET(query_hca_cap_in, in, op_mod, 204 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE_CAP_2); 205 ret = mlx5dv_devx_general_cmd(context, in, sizeof(in), 206 out, sizeof(out)); 207 if (ret) { 208 return ret; 209 } 210 211 caps->relaxed_ordering_write_pci_enabled = DEVX_GET(query_hca_cap_out, 212 out, capability.cmd_hca_cap.relaxed_ordering_write_pci_enabled); 213 caps->relaxed_ordering_write = DEVX_GET(query_hca_cap_out, out, 214 capability.cmd_hca_cap.relaxed_ordering_write); 215 caps->relaxed_ordering_read = DEVX_GET(query_hca_cap_out, out, 216 capability.cmd_hca_cap.relaxed_ordering_read); 217 caps->relaxed_ordering_write_umr = DEVX_GET(query_hca_cap_out, 218 out, capability.cmd_hca_cap.relaxed_ordering_write_umr); 219 caps->relaxed_ordering_read_umr = DEVX_GET(query_hca_cap_out, 220 out, capability.cmd_hca_cap.relaxed_ordering_read_umr); 221 return 0; 222 } 223 224 static int 225 mlx5_mkey_pool_create_mkey(struct mlx5_mkey **_mkey, struct ibv_pd *pd, 226 struct mlx5_relaxed_ordering_caps *caps, uint32_t flags) 227 { 228 struct mlx5_mkey *mkey; 229 struct mlx5_mkey_attr mkey_attr = {}; 230 uint32_t bsf_size = 0; 231 232 mkey_attr.addr = 0; 233 mkey_attr.size = 0; 234 mkey_attr.log_entity_size = 0; 235 mkey_attr.relaxed_ordering_write = caps->relaxed_ordering_write; 236 mkey_attr.relaxed_ordering_read = caps->relaxed_ordering_read; 237 mkey_attr.klm_count = 0; 238 mkey_attr.klm = NULL; 239 if (flags & SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO) { 240 mkey_attr.crypto_en = true; 241 bsf_size += 64; 242 } 243 if (flags & SPDK_MLX5_MKEY_POOL_FLAG_SIGNATURE) { 244 bsf_size += 64; 245 } 246 mkey_attr.bsf_octowords = bsf_size / 16; 247 248 mkey = mlx5_mkey_create(pd, &mkey_attr); 249 if (!mkey) { 250 SPDK_ERRLOG("Failed to create mkey on dev %s\n", pd->context->device->name); 251 return -EINVAL; 252 } 253 *_mkey = mkey; 254 255 return 0; 256 } 257 258 static void 259 mlx5_set_mkey_in_pool(struct spdk_mempool *mp, void *cb_arg, void *_mkey, unsigned obj_idx) 260 { 261 struct spdk_mlx5_mkey_pool_obj *mkey = _mkey; 262 struct spdk_mlx5_mkey_pool *pool = cb_arg; 263 264 assert(obj_idx < pool->num_mkeys); 265 assert(pool->mkeys[obj_idx] != NULL); 266 mkey->mkey = pool->mkeys[obj_idx]->mkey; 267 mkey->pool_flag = pool->flags & 0xf; 268 mkey->sig.sigerr_count = 1; 269 mkey->sig.sigerr = false; 270 271 RB_INSERT(mlx5_mkeys_tree, &pool->tree, mkey); 272 } 273 274 static const char *g_mkey_pool_names[] = { 275 [SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO] = "crypto", 276 [SPDK_MLX5_MKEY_POOL_FLAG_SIGNATURE] = "signature", 277 }; 278 279 static void 280 mlx5_mkey_pool_destroy(struct spdk_mlx5_mkey_pool *pool) 281 { 282 uint32_t i; 283 284 if (pool->mpool) { 285 spdk_mempool_free(pool->mpool); 286 } 287 if (pool->mkeys) { 288 for (i = 0; i < pool->num_mkeys; i++) { 289 if (pool->mkeys[i]) { 290 mlx5_mkey_destroy(pool->mkeys[i]); 291 pool->mkeys[i] = NULL; 292 } 293 } 294 free(pool->mkeys); 295 } 296 TAILQ_REMOVE(&g_mkey_pools, pool, link); 297 free(pool); 298 } 299 300 static int 301 mlx5_mkey_pools_init(struct spdk_mlx5_mkey_pool_param *params, struct ibv_pd *pd) 302 { 303 struct spdk_mlx5_mkey_pool *new_pool; 304 struct mlx5_mkey **mkeys; 305 struct mlx5_relaxed_ordering_caps caps; 306 uint32_t j, pdn; 307 int rc; 308 char pool_name[32]; 309 310 new_pool = calloc(1, sizeof(*new_pool)); 311 if (!new_pool) { 312 rc = -ENOMEM; 313 goto err; 314 } 315 TAILQ_INSERT_TAIL(&g_mkey_pools, new_pool, link); 316 rc = mlx5_query_relaxed_ordering_caps(pd->context, &caps); 317 if (rc) { 318 SPDK_ERRLOG("Failed to get relaxed ordering capabilities, dev %s\n", 319 pd->context->device->dev_name); 320 goto err; 321 } 322 mkeys = calloc(params->mkey_count, sizeof(struct mlx5_mkey *)); 323 if (!mkeys) { 324 rc = -ENOMEM; 325 goto err; 326 } 327 new_pool->mkeys = mkeys; 328 new_pool->num_mkeys = params->mkey_count; 329 new_pool->pd = pd; 330 new_pool->flags = params->flags; 331 for (j = 0; j < params->mkey_count; j++) { 332 rc = mlx5_mkey_pool_create_mkey(&mkeys[j], pd, &caps, params->flags); 333 if (rc) { 334 goto err; 335 } 336 } 337 rc = mlx5_get_pd_id(pd, &pdn); 338 if (rc) { 339 SPDK_ERRLOG("Failed to get pdn, pd %p\n", pd); 340 goto err; 341 } 342 rc = snprintf(pool_name, 32, "%s_%s_%04u", pd->context->device->name, 343 g_mkey_pool_names[new_pool->flags], pdn); 344 if (rc < 0) { 345 goto err; 346 } 347 RB_INIT(&new_pool->tree); 348 new_pool->mpool = spdk_mempool_create_ctor(pool_name, params->mkey_count, 349 sizeof(struct spdk_mlx5_mkey_pool_obj), 350 params->cache_per_thread, SPDK_ENV_NUMA_ID_ANY, 351 mlx5_set_mkey_in_pool, new_pool); 352 if (!new_pool->mpool) { 353 SPDK_ERRLOG("Failed to create mempool\n"); 354 rc = -ENOMEM; 355 goto err; 356 } 357 358 return 0; 359 360 err: 361 mlx5_mkey_pool_destroy(new_pool); 362 363 return rc; 364 } 365 366 static struct spdk_mlx5_mkey_pool * 367 mlx5_mkey_pool_get(struct ibv_pd *pd, uint32_t flags) 368 { 369 struct spdk_mlx5_mkey_pool *pool; 370 371 TAILQ_FOREACH(pool, &g_mkey_pools, link) { 372 if (pool->pd == pd && pool->flags == flags) { 373 return pool; 374 } 375 } 376 377 return NULL; 378 } 379 380 int 381 spdk_mlx5_mkey_pool_init(struct spdk_mlx5_mkey_pool_param *params, struct ibv_pd *pd) 382 { 383 int rc; 384 385 if (!pd) { 386 return -EINVAL; 387 } 388 389 if (!params || !params->mkey_count) { 390 return -EINVAL; 391 } 392 if ((params->flags & MLX5_UMR_POOL_VALID_FLAGS_MASK) != 0) { 393 SPDK_ERRLOG("Invalid flags %x\n", params->flags); 394 return -EINVAL; 395 } 396 if ((params->flags & (SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO | SPDK_MLX5_MKEY_POOL_FLAG_SIGNATURE)) == 397 (SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO | SPDK_MLX5_MKEY_POOL_FLAG_SIGNATURE)) { 398 SPDK_ERRLOG("Both crypto and signature capabilities are not supported\n"); 399 return -EINVAL; 400 } 401 if (params->cache_per_thread > params->mkey_count || !params->cache_per_thread) { 402 params->cache_per_thread = params->mkey_count * 3 / 4 / spdk_env_get_core_count(); 403 } 404 405 pthread_mutex_lock(&g_mkey_pool_lock); 406 if (mlx5_mkey_pool_get(pd, params->flags) != NULL) { 407 pthread_mutex_unlock(&g_mkey_pool_lock); 408 return -EEXIST; 409 } 410 411 rc = mlx5_mkey_pools_init(params, pd); 412 pthread_mutex_unlock(&g_mkey_pool_lock); 413 414 return rc; 415 } 416 417 int 418 spdk_mlx5_mkey_pool_destroy(uint32_t flags, struct ibv_pd *pd) 419 { 420 struct spdk_mlx5_mkey_pool *pool; 421 int rc = 0; 422 423 if (!pd) { 424 return -EINVAL; 425 } 426 427 if ((flags & MLX5_UMR_POOL_VALID_FLAGS_MASK) != 0) { 428 SPDK_ERRLOG("Invalid flags %x\n", flags); 429 return -EINVAL; 430 } 431 432 pthread_mutex_lock(&g_mkey_pool_lock); 433 pool = mlx5_mkey_pool_get(pd, flags); 434 if (!pool) { 435 SPDK_ERRLOG("Cant find a pool for PD %p, flags %x\n", pd, flags); 436 pthread_mutex_unlock(&g_mkey_pool_lock); 437 return -ENODEV; 438 } 439 if (pool->refcnt) { 440 SPDK_WARNLOG("Can't delete pool pd %p, dev %s\n", pool->pd, pool->pd->context->device->dev_name); 441 rc = -EAGAIN; 442 } else { 443 mlx5_mkey_pool_destroy(pool); 444 } 445 pthread_mutex_unlock(&g_mkey_pool_lock); 446 447 return rc; 448 } 449 450 struct spdk_mlx5_mkey_pool * 451 spdk_mlx5_mkey_pool_get_ref(struct ibv_pd *pd, uint32_t flags) 452 { 453 struct spdk_mlx5_mkey_pool *pool; 454 455 if ((flags & MLX5_UMR_POOL_VALID_FLAGS_MASK) != 0) { 456 SPDK_ERRLOG("Invalid flags %x\n", flags); 457 return NULL; 458 } 459 460 pthread_mutex_lock(&g_mkey_pool_lock); 461 pool = mlx5_mkey_pool_get(pd, flags); 462 if (pool) { 463 pool->refcnt++; 464 } 465 pthread_mutex_unlock(&g_mkey_pool_lock); 466 467 return pool; 468 } 469 470 void 471 spdk_mlx5_mkey_pool_put_ref(struct spdk_mlx5_mkey_pool *pool) 472 { 473 pthread_mutex_lock(&g_mkey_pool_lock); 474 pool->refcnt--; 475 pthread_mutex_unlock(&g_mkey_pool_lock); 476 } 477 478 int 479 spdk_mlx5_mkey_pool_get_bulk(struct spdk_mlx5_mkey_pool *pool, 480 struct spdk_mlx5_mkey_pool_obj **mkeys, uint32_t mkeys_count) 481 { 482 assert(pool->mpool); 483 484 return spdk_mempool_get_bulk(pool->mpool, (void **)mkeys, mkeys_count); 485 } 486 487 void 488 spdk_mlx5_mkey_pool_put_bulk(struct spdk_mlx5_mkey_pool *pool, 489 struct spdk_mlx5_mkey_pool_obj **mkeys, uint32_t mkeys_count) 490 { 491 assert(pool->mpool); 492 493 spdk_mempool_put_bulk(pool->mpool, (void **)mkeys, mkeys_count); 494 } 495 496 static inline void 497 _mlx5_set_umr_ctrl_seg_mtt(struct mlx5_wqe_umr_ctrl_seg *ctrl, uint32_t klms_octowords, 498 uint64_t mkey_mask) 499 { 500 ctrl->flags |= MLX5_WQE_UMR_CTRL_FLAG_INLINE; 501 ctrl->klm_octowords = htobe16(klms_octowords); 502 /* 503 * Going to modify two properties of KLM mkey: 504 * 1. 'free' field: change this mkey from in free to in use 505 * 2. 'len' field: to include the total bytes in iovec 506 */ 507 mkey_mask |= MLX5_WQE_UMR_CTRL_MKEY_MASK_FREE | MLX5_WQE_UMR_CTRL_MKEY_MASK_LEN; 508 509 ctrl->mkey_mask |= htobe64(mkey_mask); 510 } 511 512 static inline void 513 mlx5_set_umr_ctrl_seg_mtt(struct mlx5_wqe_umr_ctrl_seg *ctrl, uint32_t klms_octowords) 514 { 515 _mlx5_set_umr_ctrl_seg_mtt(ctrl, klms_octowords, 0); 516 } 517 518 static inline void 519 mlx5_set_umr_ctrl_seg_mtt_sig(struct mlx5_wqe_umr_ctrl_seg *ctrl, uint32_t klms_octowords) 520 { 521 _mlx5_set_umr_ctrl_seg_mtt(ctrl, klms_octowords, MLX5_WQE_UMR_CTRL_MKEY_MASK_SIG_ERR); 522 } 523 524 static inline void 525 mlx5_set_umr_ctrl_seg_bsf_size(struct mlx5_wqe_umr_ctrl_seg *ctrl, int bsf_size) 526 { 527 ctrl->bsf_octowords = htobe16(SPDK_ALIGN_CEIL(SPDK_CEIL_DIV(bsf_size, 16), 4)); 528 } 529 530 static inline void 531 mlx5_set_umr_mkey_seg_mtt(struct mlx5_wqe_mkey_context_seg *mkey, 532 struct spdk_mlx5_umr_attr *umr_attr) 533 { 534 mkey->len = htobe64(umr_attr->umr_len); 535 } 536 537 static void 538 mlx5_set_umr_mkey_seg(struct mlx5_wqe_mkey_context_seg *mkey, 539 struct spdk_mlx5_umr_attr *umr_attr) 540 { 541 memset(mkey, 0, 64); 542 mlx5_set_umr_mkey_seg_mtt(mkey, umr_attr); 543 } 544 545 static void 546 mlx5_set_umr_mkey_seg_sig(struct mlx5_wqe_mkey_context_seg *mkey, 547 struct spdk_mlx5_umr_sig_attr *sig_attr) 548 { 549 mkey->flags_pd = htobe32((sig_attr->sigerr_count & 1) << 26); 550 } 551 552 static inline void 553 mlx5_set_umr_inline_klm_seg(struct mlx5_wqe_umr_klm_seg *klm, struct ibv_sge *sge) 554 { 555 klm->byte_count = htobe32(sge->length); 556 klm->mkey = htobe32(sge->lkey); 557 klm->address = htobe64(sge->addr); 558 } 559 560 static void * 561 mlx5_build_inline_mtt(struct mlx5_hw_qp *qp, uint32_t *to_end, struct mlx5_wqe_umr_klm_seg *dst_klm, 562 struct spdk_mlx5_umr_attr *umr_attr) 563 { 564 struct ibv_sge *src_sge = umr_attr->sge; 565 int num_wqebbs = umr_attr->sge_count / 4; 566 int tail = umr_attr->sge_count & 0x3; 567 int i; 568 569 for (i = 0; i < num_wqebbs; i++) { 570 mlx5_set_umr_inline_klm_seg(&dst_klm[0], src_sge++); 571 mlx5_set_umr_inline_klm_seg(&dst_klm[1], src_sge++); 572 mlx5_set_umr_inline_klm_seg(&dst_klm[2], src_sge++); 573 mlx5_set_umr_inline_klm_seg(&dst_klm[3], src_sge++); 574 /* sizeof(*dst_klm) * 4 == MLX5_SEND_WQE_BB */ 575 dst_klm = mlx5_qp_get_next_wqebb(qp, to_end, dst_klm); 576 } 577 578 if (!tail) { 579 return dst_klm; 580 } 581 582 for (i = 0; i < tail; i++) { 583 mlx5_set_umr_inline_klm_seg(&dst_klm[i], src_sge++); 584 } 585 586 /* Fill PAD entries to make whole mtt aligned to 64B(MLX5_SEND_WQE_BB) */ 587 memset(&dst_klm[i], 0, MLX5_SEND_WQE_BB - sizeof(struct mlx5_wqe_umr_klm_seg) * tail); 588 589 return mlx5_qp_get_next_wqebb(qp, to_end, dst_klm); 590 } 591 592 static inline void 593 mlx5_set_umr_crypto_bsf_seg(struct mlx5_crypto_bsf_seg *bsf, struct spdk_mlx5_umr_crypto_attr *attr, 594 uint32_t raw_data_size, uint8_t bsf_size) 595 { 596 uint64_t *iv = (void *)bsf->xts_initial_tweak; 597 598 memset(bsf, 0, sizeof(*bsf)); 599 switch (attr->tweak_mode) { 600 case SPDK_MLX5_CRYPTO_KEY_TWEAK_MODE_SIMPLE_LBA_LE: 601 iv[0] = htole64(attr->xts_iv); 602 iv[1] = 0; 603 break; 604 case SPDK_MLX5_CRYPTO_KEY_TWEAK_MODE_SIMPLE_LBA_BE: 605 iv[0] = 0; 606 iv[1] = htobe64(attr->xts_iv); 607 break; 608 default: 609 assert(false && "unsupported tweak mode"); 610 break; 611 } 612 613 bsf->size_type = (bsf_size << 6) | MLX5_CRYPTO_BSF_P_TYPE_CRYPTO; 614 bsf->enc_order = attr->enc_order; 615 bsf->raw_data_size = htobe32(raw_data_size); 616 bsf->crypto_block_size_pointer = attr->bs_selector; 617 bsf->dek_pointer = htobe32(attr->dek_obj_id); 618 *((uint64_t *)bsf->keytag) = attr->keytag; 619 } 620 621 static inline uint8_t 622 mlx5_get_crc32c_tfs(uint32_t seed) 623 { 624 assert(seed == 0 || seed == 0xffffffff); 625 return MLX5_SIG_BSF_TFS_CRC32C | !seed; 626 } 627 628 static inline void 629 mlx5_set_umr_sig_bsf_seg(struct mlx5_sig_bsf_seg *bsf, 630 struct spdk_mlx5_umr_sig_attr *attr) 631 { 632 uint8_t bsf_size = MLX5_SIG_BSF_SIZE_32B; 633 uint32_t tfs_psv; 634 uint32_t init_gen; 635 636 memset(bsf, 0, sizeof(*bsf)); 637 bsf->basic.bsf_size_sbs = (bsf_size << 6); 638 bsf->basic.raw_data_size = htobe32(attr->raw_data_size); 639 bsf->basic.check_byte_mask = 0xff; 640 641 tfs_psv = mlx5_get_crc32c_tfs(attr->seed); 642 tfs_psv = tfs_psv << MLX5_SIG_BSF_TFS_SHIFT; 643 tfs_psv |= attr->psv_index & 0xffffff; 644 645 if (attr->domain == SPDK_MLX5_UMR_SIG_DOMAIN_WIRE) { 646 bsf->ext.w_tfs_psv = htobe32(tfs_psv); 647 init_gen = attr->init ? MLX5_SIG_BSF_EXT_W_T_INIT : 0; 648 if (attr->check_gen) { 649 init_gen |= MLX5_SIG_BSF_EXT_W_T_CHECK_GEN; 650 } 651 bsf->ext.t_init_gen_pro_size = htobe32(init_gen); 652 } else { 653 bsf->ext.m_tfs_psv = htobe32(tfs_psv); 654 init_gen = attr->init ? MLX5_SIG_BSF_EXT_M_T_INIT : 0; 655 if (attr->check_gen) { 656 init_gen |= MLX5_SIG_BSF_EXT_M_T_CHECK_GEN; 657 } 658 bsf->ext.t_init_gen_pro_size = htobe32(init_gen); 659 } 660 } 661 662 static inline void 663 mlx5_umr_configure_with_wrap_around_crypto(struct spdk_mlx5_qp *qp, 664 struct spdk_mlx5_umr_attr *umr_attr, struct spdk_mlx5_umr_crypto_attr *crypto_attr, uint64_t wr_id, 665 uint32_t flags, uint32_t wqe_size, uint32_t umr_wqe_n_bb, uint32_t mtt_size) 666 { 667 struct mlx5_hw_qp *hw = &qp->hw; 668 struct mlx5_wqe_ctrl_seg *ctrl; 669 struct mlx5_wqe_ctrl_seg *gen_ctrl; 670 struct mlx5_wqe_umr_ctrl_seg *umr_ctrl; 671 struct mlx5_wqe_mkey_context_seg *mkey; 672 struct mlx5_wqe_umr_klm_seg *klm; 673 struct mlx5_crypto_bsf_seg *bsf; 674 uint8_t fm_ce_se; 675 uint32_t pi, to_end; 676 677 fm_ce_se = mlx5_qp_fm_ce_se_update(qp, (uint8_t)flags); 678 679 ctrl = (struct mlx5_wqe_ctrl_seg *)mlx5_qp_get_wqe_bb(hw); 680 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 681 to_end = (hw->sq_wqe_cnt - pi) * MLX5_SEND_WQE_BB; 682 683 /* 684 * sizeof(gen_ctrl) + sizeof(umr_ctrl) == MLX5_SEND_WQE_BB, 685 * so do not need to worry about wqe buffer wrap around. 686 * 687 * build genenal ctrl segment 688 */ 689 gen_ctrl = ctrl; 690 mlx5_set_ctrl_seg(gen_ctrl, hw->sq_pi, MLX5_OPCODE_UMR, 0, 691 hw->qp_num, fm_ce_se, 692 SPDK_CEIL_DIV(wqe_size, 16), 0, 693 htobe32(umr_attr->mkey)); 694 695 /* build umr ctrl segment */ 696 umr_ctrl = (struct mlx5_wqe_umr_ctrl_seg *)(gen_ctrl + 1); 697 memset(umr_ctrl, 0, sizeof(*umr_ctrl)); 698 mlx5_set_umr_ctrl_seg_mtt(umr_ctrl, mtt_size); 699 mlx5_set_umr_ctrl_seg_bsf_size(umr_ctrl, sizeof(struct mlx5_crypto_bsf_seg)); 700 701 /* build mkey context segment */ 702 mkey = mlx5_qp_get_next_wqebb(hw, &to_end, ctrl); 703 mlx5_set_umr_mkey_seg(mkey, umr_attr); 704 705 klm = mlx5_qp_get_next_wqebb(hw, &to_end, mkey); 706 bsf = mlx5_build_inline_mtt(hw, &to_end, klm, umr_attr); 707 708 mlx5_set_umr_crypto_bsf_seg(bsf, crypto_attr, umr_attr->umr_len, MLX5_CRYPTO_BSF_SIZE_64B); 709 710 mlx5_qp_wqe_submit(qp, ctrl, umr_wqe_n_bb, pi); 711 712 mlx5_qp_set_comp(qp, pi, wr_id, fm_ce_se, umr_wqe_n_bb); 713 assert(qp->tx_available >= umr_wqe_n_bb); 714 qp->tx_available -= umr_wqe_n_bb; 715 } 716 717 static inline void 718 mlx5_umr_configure_full_crypto(struct spdk_mlx5_qp *dv_qp, struct spdk_mlx5_umr_attr *umr_attr, 719 struct spdk_mlx5_umr_crypto_attr *crypto_attr, uint64_t wr_id, 720 uint32_t flags, uint32_t wqe_size, uint32_t umr_wqe_n_bb, 721 uint32_t mtt_size) 722 { 723 struct mlx5_hw_qp *hw = &dv_qp->hw; 724 struct mlx5_wqe_ctrl_seg *ctrl; 725 struct mlx5_wqe_ctrl_seg *gen_ctrl; 726 struct mlx5_wqe_umr_ctrl_seg *umr_ctrl; 727 struct mlx5_wqe_mkey_context_seg *mkey; 728 struct mlx5_wqe_umr_klm_seg *klm; 729 struct mlx5_crypto_bsf_seg *bsf; 730 uint8_t fm_ce_se; 731 uint32_t pi; 732 uint32_t i; 733 734 fm_ce_se = mlx5_qp_fm_ce_se_update(dv_qp, (uint8_t)flags); 735 736 ctrl = (struct mlx5_wqe_ctrl_seg *)mlx5_qp_get_wqe_bb(hw); 737 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 738 gen_ctrl = ctrl; 739 mlx5_set_ctrl_seg(gen_ctrl, hw->sq_pi, MLX5_OPCODE_UMR, 0, 740 hw->qp_num, fm_ce_se, 741 SPDK_CEIL_DIV(wqe_size, 16), 0, 742 htobe32(umr_attr->mkey)); 743 744 /* build umr ctrl segment */ 745 umr_ctrl = (struct mlx5_wqe_umr_ctrl_seg *)(gen_ctrl + 1); 746 memset(umr_ctrl, 0, sizeof(*umr_ctrl)); 747 mlx5_set_umr_ctrl_seg_mtt(umr_ctrl, mtt_size); 748 mlx5_set_umr_ctrl_seg_bsf_size(umr_ctrl, sizeof(struct mlx5_crypto_bsf_seg)); 749 750 /* build mkey context segment */ 751 mkey = (struct mlx5_wqe_mkey_context_seg *)(umr_ctrl + 1); 752 mlx5_set_umr_mkey_seg(mkey, umr_attr); 753 754 klm = (struct mlx5_wqe_umr_klm_seg *)(mkey + 1); 755 for (i = 0; i < umr_attr->sge_count; i++) { 756 mlx5_set_umr_inline_klm_seg(klm, &umr_attr->sge[i]); 757 /* sizeof(*klm) * 4 == MLX5_SEND_WQE_BB */ 758 klm = klm + 1; 759 } 760 /* fill PAD if existing */ 761 /* PAD entries is to make whole mtt aligned to 64B(MLX5_SEND_WQE_BB), 762 * So it will not happen wrap around during fill PAD entries. */ 763 for (; i < mtt_size; i++) { 764 memset(klm, 0, sizeof(*klm)); 765 klm = klm + 1; 766 } 767 768 bsf = (struct mlx5_crypto_bsf_seg *)klm; 769 mlx5_set_umr_crypto_bsf_seg(bsf, crypto_attr, umr_attr->umr_len, MLX5_CRYPTO_BSF_SIZE_64B); 770 771 mlx5_qp_wqe_submit(dv_qp, ctrl, umr_wqe_n_bb, pi); 772 773 mlx5_qp_set_comp(dv_qp, pi, wr_id, fm_ce_se, umr_wqe_n_bb); 774 assert(dv_qp->tx_available >= umr_wqe_n_bb); 775 dv_qp->tx_available -= umr_wqe_n_bb; 776 } 777 778 int 779 spdk_mlx5_umr_configure_crypto(struct spdk_mlx5_qp *qp, struct spdk_mlx5_umr_attr *umr_attr, 780 struct spdk_mlx5_umr_crypto_attr *crypto_attr, uint64_t wr_id, uint32_t flags) 781 { 782 struct mlx5_hw_qp *hw = &qp->hw; 783 uint32_t pi, to_end, umr_wqe_n_bb; 784 uint32_t wqe_size, mtt_size; 785 uint32_t inline_klm_size; 786 787 if (!spdk_unlikely(umr_attr->sge_count)) { 788 return -EINVAL; 789 } 790 791 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 792 to_end = (hw->sq_wqe_cnt - pi) * MLX5_SEND_WQE_BB; 793 794 /* 795 * UMR WQE LAYOUT: 796 * ----------------------------------------------------------------------- 797 * | gen_ctrl | umr_ctrl | mkey_ctx | inline klm mtt | inline crypto bsf | 798 * ----------------------------------------------------------------------- 799 * 16bytes 48bytes 64bytes sge_count*16 bytes 64 bytes 800 * 801 * Note: size of inline klm mtt should be aligned to 64 bytes. 802 */ 803 wqe_size = sizeof(struct mlx5_wqe_ctrl_seg) + sizeof(struct mlx5_wqe_umr_ctrl_seg) + 804 sizeof(struct mlx5_wqe_mkey_context_seg); 805 mtt_size = SPDK_ALIGN_CEIL(umr_attr->sge_count, 4); 806 inline_klm_size = mtt_size * sizeof(struct mlx5_wqe_umr_klm_seg); 807 wqe_size += inline_klm_size; 808 wqe_size += sizeof(struct mlx5_crypto_bsf_seg); 809 810 umr_wqe_n_bb = SPDK_CEIL_DIV(wqe_size, MLX5_SEND_WQE_BB); 811 if (spdk_unlikely(umr_wqe_n_bb > qp->tx_available)) { 812 return -ENOMEM; 813 } 814 if (spdk_unlikely(umr_attr->sge_count > qp->max_send_sge)) { 815 return -E2BIG; 816 } 817 818 if (spdk_unlikely(to_end < wqe_size)) { 819 mlx5_umr_configure_with_wrap_around_crypto(qp, umr_attr, crypto_attr, wr_id, flags, wqe_size, 820 umr_wqe_n_bb, 821 mtt_size); 822 } else { 823 mlx5_umr_configure_full_crypto(qp, umr_attr, crypto_attr, wr_id, flags, wqe_size, umr_wqe_n_bb, 824 mtt_size); 825 } 826 827 return 0; 828 } 829 830 static inline void 831 mlx5_umr_configure_with_wrap_around_sig(struct spdk_mlx5_qp *dv_qp, 832 struct spdk_mlx5_umr_attr *umr_attr, 833 struct spdk_mlx5_umr_sig_attr *sig_attr, uint64_t wr_id, 834 uint32_t flags, uint32_t wqe_size, uint32_t umr_wqe_n_bb, uint32_t mtt_size) 835 { 836 struct mlx5_hw_qp *hw = &dv_qp->hw; 837 struct mlx5_wqe_ctrl_seg *ctrl; 838 struct mlx5_wqe_ctrl_seg *gen_ctrl; 839 struct mlx5_wqe_umr_ctrl_seg *umr_ctrl; 840 struct mlx5_wqe_mkey_context_seg *mkey; 841 struct mlx5_wqe_umr_klm_seg *klm; 842 struct mlx5_sig_bsf_seg *bsf; 843 uint8_t fm_ce_se; 844 uint32_t pi, to_end; 845 846 fm_ce_se = mlx5_qp_fm_ce_se_update(dv_qp, (uint8_t)flags); 847 848 ctrl = (struct mlx5_wqe_ctrl_seg *)mlx5_qp_get_wqe_bb(hw); 849 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 850 to_end = (hw->sq_wqe_cnt - pi) * MLX5_SEND_WQE_BB; 851 852 /* 853 * sizeof(gen_ctrl) + sizeof(umr_ctrl) == MLX5_SEND_WQE_BB, 854 * so do not need to worry about wqe buffer wrap around. 855 * 856 * build genenal ctrl segment 857 */ 858 gen_ctrl = ctrl; 859 mlx5_set_ctrl_seg(gen_ctrl, hw->sq_pi, MLX5_OPCODE_UMR, 0, 860 hw->qp_num, fm_ce_se, 861 SPDK_CEIL_DIV(wqe_size, 16), 0, 862 htobe32(umr_attr->mkey)); 863 864 /* build umr ctrl segment */ 865 umr_ctrl = (struct mlx5_wqe_umr_ctrl_seg *)(gen_ctrl + 1); 866 memset(umr_ctrl, 0, sizeof(*umr_ctrl)); 867 mlx5_set_umr_ctrl_seg_mtt_sig(umr_ctrl, mtt_size); 868 mlx5_set_umr_ctrl_seg_bsf_size(umr_ctrl, sizeof(struct mlx5_sig_bsf_seg)); 869 870 /* build mkey context segment */ 871 mkey = mlx5_qp_get_next_wqebb(hw, &to_end, ctrl); 872 mlx5_set_umr_mkey_seg(mkey, umr_attr); 873 mlx5_set_umr_mkey_seg_sig(mkey, sig_attr); 874 875 klm = mlx5_qp_get_next_wqebb(hw, &to_end, mkey); 876 bsf = mlx5_build_inline_mtt(hw, &to_end, klm, umr_attr); 877 878 mlx5_set_umr_sig_bsf_seg(bsf, sig_attr); 879 880 mlx5_qp_wqe_submit(dv_qp, ctrl, umr_wqe_n_bb, pi); 881 882 mlx5_qp_set_comp(dv_qp, pi, wr_id, fm_ce_se, umr_wqe_n_bb); 883 assert(dv_qp->tx_available >= umr_wqe_n_bb); 884 dv_qp->tx_available -= umr_wqe_n_bb; 885 } 886 887 static inline void 888 mlx5_umr_configure_full_sig(struct spdk_mlx5_qp *dv_qp, struct spdk_mlx5_umr_attr *umr_attr, 889 struct spdk_mlx5_umr_sig_attr *sig_attr, uint64_t wr_id, 890 uint32_t flags, uint32_t wqe_size, uint32_t umr_wqe_n_bb, 891 uint32_t mtt_size) 892 { 893 struct mlx5_hw_qp *hw = &dv_qp->hw; 894 struct mlx5_wqe_ctrl_seg *ctrl; 895 struct mlx5_wqe_ctrl_seg *gen_ctrl; 896 struct mlx5_wqe_umr_ctrl_seg *umr_ctrl; 897 struct mlx5_wqe_mkey_context_seg *mkey; 898 struct mlx5_wqe_umr_klm_seg *klm; 899 struct mlx5_sig_bsf_seg *bsf; 900 uint8_t fm_ce_se; 901 uint32_t pi; 902 uint32_t i; 903 904 fm_ce_se = mlx5_qp_fm_ce_se_update(dv_qp, (uint8_t)flags); 905 906 ctrl = (struct mlx5_wqe_ctrl_seg *)mlx5_qp_get_wqe_bb(hw); 907 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 908 gen_ctrl = ctrl; 909 mlx5_set_ctrl_seg(gen_ctrl, hw->sq_pi, MLX5_OPCODE_UMR, 0, 910 hw->qp_num, fm_ce_se, 911 SPDK_CEIL_DIV(wqe_size, 16), 0, 912 htobe32(umr_attr->mkey)); 913 914 /* build umr ctrl segment */ 915 umr_ctrl = (struct mlx5_wqe_umr_ctrl_seg *)(gen_ctrl + 1); 916 memset(umr_ctrl, 0, sizeof(*umr_ctrl)); 917 mlx5_set_umr_ctrl_seg_mtt_sig(umr_ctrl, mtt_size); 918 mlx5_set_umr_ctrl_seg_bsf_size(umr_ctrl, sizeof(struct mlx5_sig_bsf_seg)); 919 920 /* build mkey context segment */ 921 mkey = (struct mlx5_wqe_mkey_context_seg *)(umr_ctrl + 1); 922 memset(mkey, 0, sizeof(*mkey)); 923 mlx5_set_umr_mkey_seg_mtt(mkey, umr_attr); 924 mlx5_set_umr_mkey_seg_sig(mkey, sig_attr); 925 926 klm = (struct mlx5_wqe_umr_klm_seg *)(mkey + 1); 927 for (i = 0; i < umr_attr->sge_count; i++) { 928 mlx5_set_umr_inline_klm_seg(klm, &umr_attr->sge[i]); 929 /* sizeof(*klm) * 4 == MLX5_SEND_WQE_BB */ 930 klm = klm + 1; 931 } 932 /* fill PAD if existing */ 933 /* PAD entries is to make whole mtt aligned to 64B(MLX5_SEND_WQE_BB), 934 * So it will not happen warp around during fill PAD entries. */ 935 for (; i < mtt_size; i++) { 936 memset(klm, 0, sizeof(*klm)); 937 klm = klm + 1; 938 } 939 940 bsf = (struct mlx5_sig_bsf_seg *)klm; 941 mlx5_set_umr_sig_bsf_seg(bsf, sig_attr); 942 943 mlx5_qp_wqe_submit(dv_qp, ctrl, umr_wqe_n_bb, pi); 944 945 mlx5_qp_set_comp(dv_qp, pi, wr_id, fm_ce_se, umr_wqe_n_bb); 946 assert(dv_qp->tx_available >= umr_wqe_n_bb); 947 dv_qp->tx_available -= umr_wqe_n_bb; 948 } 949 950 int 951 spdk_mlx5_umr_configure_sig(struct spdk_mlx5_qp *qp, struct spdk_mlx5_umr_attr *umr_attr, 952 struct spdk_mlx5_umr_sig_attr *sig_attr, uint64_t wr_id, uint32_t flags) 953 { 954 struct mlx5_hw_qp *hw = &qp->hw; 955 uint32_t pi, to_end, umr_wqe_n_bb; 956 uint32_t wqe_size, mtt_size; 957 uint32_t inline_klm_size; 958 959 if (!spdk_unlikely(umr_attr->sge_count)) { 960 return -EINVAL; 961 } 962 963 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 964 to_end = (hw->sq_wqe_cnt - pi) * MLX5_SEND_WQE_BB; 965 966 /* 967 * UMR WQE LAYOUT: 968 * ----------------------------------------------------------------------- 969 * | gen_ctrl | umr_ctrl | mkey_ctx | inline klm mtt | inline sig bsf | 970 * ----------------------------------------------------------------------- 971 * 16bytes 48bytes 64bytes sg_count*16 bytes 64 bytes 972 * 973 * Note: size of inline klm mtt should be aligned to 64 bytes. 974 */ 975 wqe_size = sizeof(struct mlx5_wqe_ctrl_seg) + sizeof(struct mlx5_wqe_umr_ctrl_seg) + 976 sizeof(struct mlx5_wqe_mkey_context_seg); 977 mtt_size = SPDK_ALIGN_CEIL(umr_attr->sge_count, 4); 978 inline_klm_size = mtt_size * sizeof(struct mlx5_wqe_umr_klm_seg); 979 wqe_size += inline_klm_size; 980 wqe_size += sizeof(struct mlx5_sig_bsf_seg); 981 982 umr_wqe_n_bb = SPDK_CEIL_DIV(wqe_size, MLX5_SEND_WQE_BB); 983 if (spdk_unlikely(umr_wqe_n_bb > qp->tx_available)) { 984 return -ENOMEM; 985 } 986 if (spdk_unlikely(umr_attr->sge_count > qp->max_send_sge)) { 987 return -E2BIG; 988 } 989 990 if (spdk_unlikely(to_end < wqe_size)) { 991 mlx5_umr_configure_with_wrap_around_sig(qp, umr_attr, sig_attr, wr_id, flags, wqe_size, 992 umr_wqe_n_bb, mtt_size); 993 } else { 994 mlx5_umr_configure_full_sig(qp, umr_attr, sig_attr, wr_id, flags, wqe_size, umr_wqe_n_bb, 995 mtt_size); 996 } 997 998 return 0; 999 } 1000 1001 static struct mlx5dv_devx_obj * 1002 mlx5_cmd_create_psv(struct ibv_context *context, uint32_t pdn, uint32_t *psv_index) 1003 { 1004 uint32_t out[DEVX_ST_SZ_DW(create_psv_out)] = {}; 1005 uint32_t in[DEVX_ST_SZ_DW(create_psv_in)] = {}; 1006 struct mlx5dv_devx_obj *obj; 1007 1008 assert(context); 1009 assert(psv_index); 1010 1011 DEVX_SET(create_psv_in, in, opcode, MLX5_CMD_OP_CREATE_PSV); 1012 DEVX_SET(create_psv_in, in, pd, pdn); 1013 DEVX_SET(create_psv_in, in, num_psv, 1); 1014 1015 obj = mlx5dv_devx_obj_create(context, in, sizeof(in), out, sizeof(out)); 1016 if (obj) { 1017 *psv_index = DEVX_GET(create_psv_out, out, psv0_index); 1018 } 1019 1020 return obj; 1021 } 1022 1023 struct spdk_mlx5_psv * 1024 spdk_mlx5_create_psv(struct ibv_pd *pd) 1025 { 1026 uint32_t pdn; 1027 struct spdk_mlx5_psv *psv; 1028 int err; 1029 1030 assert(pd); 1031 1032 err = mlx5_get_pd_id(pd, &pdn); 1033 if (err) { 1034 return NULL; 1035 } 1036 1037 psv = calloc(1, sizeof(*psv)); 1038 if (!psv) { 1039 return NULL; 1040 } 1041 1042 psv->devx_obj = mlx5_cmd_create_psv(pd->context, pdn, &psv->index); 1043 if (!psv->devx_obj) { 1044 free(psv); 1045 return NULL; 1046 } 1047 1048 return psv; 1049 } 1050 1051 int 1052 spdk_mlx5_destroy_psv(struct spdk_mlx5_psv *psv) 1053 { 1054 int ret; 1055 1056 ret = mlx5dv_devx_obj_destroy(psv->devx_obj); 1057 if (!ret) { 1058 free(psv); 1059 } 1060 1061 return ret; 1062 } 1063 1064 int 1065 spdk_mlx5_qp_set_psv(struct spdk_mlx5_qp *qp, uint32_t psv_index, uint32_t crc_seed, uint64_t wr_id, 1066 uint32_t flags) 1067 { 1068 struct mlx5_hw_qp *hw = &qp->hw; 1069 uint32_t pi, wqe_size, wqe_n_bb; 1070 struct mlx5_wqe_ctrl_seg *ctrl; 1071 struct mlx5_wqe_ctrl_seg *gen_ctrl; 1072 struct mlx5_wqe_set_psv_seg *psv; 1073 uint8_t fm_ce_se; 1074 uint64_t transient_signature = (uint64_t)crc_seed << 32; 1075 1076 wqe_size = sizeof(struct mlx5_wqe_ctrl_seg) + sizeof(struct mlx5_wqe_set_psv_seg); 1077 /* The size of SET_PSV WQE is constant and smaller than WQE BB. */ 1078 assert(wqe_size < MLX5_SEND_WQE_BB); 1079 wqe_n_bb = 1; 1080 if (spdk_unlikely(wqe_n_bb > qp->tx_available)) { 1081 return -ENOMEM; 1082 } 1083 1084 fm_ce_se = mlx5_qp_fm_ce_se_update(qp, (uint8_t)flags); 1085 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 1086 ctrl = (struct mlx5_wqe_ctrl_seg *)mlx5_qp_get_wqe_bb(hw); 1087 gen_ctrl = ctrl; 1088 mlx5_set_ctrl_seg(gen_ctrl, hw->sq_pi, MLX5_OPCODE_SET_PSV, 0, hw->qp_num, fm_ce_se, 1089 SPDK_CEIL_DIV(wqe_size, 16), 0, 0); 1090 1091 /* build umr PSV segment */ 1092 psv = (struct mlx5_wqe_set_psv_seg *)(gen_ctrl + 1); 1093 /* Zeroing the set_psv segment and WQE padding. */ 1094 memset(psv, 0, MLX5_SEND_WQE_BB - sizeof(struct mlx5_wqe_ctrl_seg)); 1095 psv->psv_index = htobe32(psv_index); 1096 psv->transient_signature = htobe64(transient_signature); 1097 1098 mlx5_qp_wqe_submit(qp, ctrl, wqe_n_bb, pi); 1099 mlx5_qp_set_comp(qp, pi, wr_id, fm_ce_se, wqe_n_bb); 1100 assert(qp->tx_available >= wqe_n_bb); 1101 qp->tx_available -= wqe_n_bb; 1102 1103 return 0; 1104 } 1105