1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 */ 4 5 #include <infiniband/verbs.h> 6 7 #include "spdk/log.h" 8 #include "spdk/util.h" 9 #include "spdk/likely.h" 10 #include "spdk/thread.h" 11 #include "spdk/tree.h" 12 13 #include "spdk_internal/rdma_utils.h" 14 #include "mlx5_priv.h" 15 #include "mlx5_ifc.h" 16 17 #define MLX5_UMR_POOL_VALID_FLAGS_MASK (~(SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO | SPDK_MLX5_MKEY_POOL_FLAG_SIGNATURE)) 18 #define MLX5_CRYPTO_BSF_P_TYPE_CRYPTO (0x1) 19 #define MLX5_CRYPTO_BSF_SIZE_64B (0x2) 20 21 #define MLX5_SIG_BSF_SIZE_32B (0x1) 22 /* Transaction Format Selector */ 23 #define MLX5_SIG_BSF_TFS_CRC32C (64) 24 #define MLX5_SIG_BSF_TFS_SHIFT (24) 25 /* Transaction Init/Check_gen bits */ 26 #define MLX5_SIG_BSF_EXT_M_T_CHECK_GEN (1u << 24) 27 #define MLX5_SIG_BSF_EXT_M_T_INIT (1u << 25) 28 #define MLX5_SIG_BSF_EXT_W_T_CHECK_GEN (1u << 28) 29 #define MLX5_SIG_BSF_EXT_W_T_INIT (1u << 29) 30 31 RB_HEAD(mlx5_mkeys_tree, spdk_mlx5_mkey_pool_obj); 32 33 struct mlx5_relaxed_ordering_caps { 34 bool relaxed_ordering_write_pci_enabled; 35 bool relaxed_ordering_write; 36 bool relaxed_ordering_read; 37 bool relaxed_ordering_write_umr; 38 bool relaxed_ordering_read_umr; 39 }; 40 41 struct mlx5_mkey_attr { 42 uint64_t addr; 43 uint64_t size; 44 uint32_t log_entity_size; 45 struct mlx5_wqe_data_seg *klm; 46 uint32_t klm_count; 47 /* Size of bsf in octowords. If 0 then bsf is disabled */ 48 uint32_t bsf_octowords; 49 bool crypto_en; 50 bool relaxed_ordering_write; 51 bool relaxed_ordering_read; 52 }; 53 54 struct mlx5_mkey { 55 struct mlx5dv_devx_obj *devx_obj; 56 uint32_t mkey; 57 uint64_t addr; 58 }; 59 60 struct spdk_mlx5_mkey_pool { 61 struct ibv_pd *pd; 62 struct spdk_mempool *mpool; 63 struct mlx5_mkeys_tree tree; 64 struct mlx5_mkey **mkeys; 65 uint32_t num_mkeys; 66 uint32_t refcnt; 67 uint32_t flags; 68 TAILQ_ENTRY(spdk_mlx5_mkey_pool) link; 69 }; 70 71 static bool g_umr_implementer_registered; 72 73 static int 74 mlx5_key_obj_compare(struct spdk_mlx5_mkey_pool_obj *key1, struct spdk_mlx5_mkey_pool_obj *key2) 75 { 76 return key1->mkey < key2->mkey ? -1 : key1->mkey > key2->mkey; 77 } 78 79 RB_GENERATE_STATIC(mlx5_mkeys_tree, spdk_mlx5_mkey_pool_obj, node, mlx5_key_obj_compare); 80 81 static TAILQ_HEAD(mlx5_mkey_pool_head, 82 spdk_mlx5_mkey_pool) g_mkey_pools = TAILQ_HEAD_INITIALIZER(g_mkey_pools); 83 static pthread_mutex_t g_mkey_pool_lock = PTHREAD_MUTEX_INITIALIZER; 84 85 #define SPDK_KLM_MAX_TRANSLATION_ENTRIES_NUM 128 86 87 static struct mlx5_mkey * 88 mlx5_mkey_create(struct ibv_pd *pd, struct mlx5_mkey_attr *attr) 89 { 90 struct mlx5_wqe_data_seg *klms = attr->klm; 91 uint32_t klm_count = attr->klm_count; 92 int in_size_dw = DEVX_ST_SZ_DW(create_mkey_in) + 93 (klm_count ? SPDK_ALIGN_CEIL(klm_count, 4) : 0) * DEVX_ST_SZ_DW(klm); 94 uint32_t in[in_size_dw]; 95 uint32_t out[DEVX_ST_SZ_DW(create_mkey_out)] = {0}; 96 void *mkc; 97 uint32_t translation_size; 98 struct mlx5_mkey *cmkey; 99 struct ibv_context *ctx = pd->context; 100 uint32_t pd_id = 0; 101 uint32_t i; 102 uint8_t *klm; 103 104 cmkey = calloc(1, sizeof(*cmkey)); 105 if (!cmkey) { 106 SPDK_ERRLOG("failed to alloc cross_mkey\n"); 107 return NULL; 108 } 109 110 memset(in, 0, in_size_dw * 4); 111 DEVX_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY); 112 mkc = DEVX_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 113 114 if (klm_count > 0) { 115 klm = (uint8_t *)DEVX_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 116 translation_size = SPDK_ALIGN_CEIL(klm_count, 4); 117 118 for (i = 0; i < klm_count; i++) { 119 DEVX_SET(klm, klm, byte_count, klms[i].byte_count); 120 DEVX_SET(klm, klm, mkey, klms[i].lkey); 121 DEVX_SET64(klm, klm, address, klms[i].addr); 122 klms += DEVX_ST_SZ_BYTES(klm); 123 } 124 125 for (; i < translation_size; i++) { 126 DEVX_SET(klm, klms, byte_count, 0x0); 127 DEVX_SET(klm, klms, mkey, 0x0); 128 DEVX_SET64(klm, klms, address, 0x0); 129 klm += DEVX_ST_SZ_BYTES(klm); 130 } 131 } 132 133 DEVX_SET(mkc, mkc, access_mode_1_0, attr->log_entity_size ? 134 MLX5_MKC_ACCESS_MODE_KLMFBS : 135 MLX5_MKC_ACCESS_MODE_KLMS); 136 DEVX_SET(mkc, mkc, log_page_size, attr->log_entity_size); 137 138 mlx5_get_pd_id(pd, &pd_id); 139 DEVX_SET(create_mkey_in, in, translations_octword_actual_size, klm_count); 140 if (klm_count == 0) { 141 DEVX_SET(mkc, mkc, free, 0x1); 142 } 143 DEVX_SET(mkc, mkc, lw, 0x1); 144 DEVX_SET(mkc, mkc, lr, 0x1); 145 DEVX_SET(mkc, mkc, rw, 0x1); 146 DEVX_SET(mkc, mkc, rr, 0x1); 147 DEVX_SET(mkc, mkc, umr_en, 1); 148 DEVX_SET(mkc, mkc, qpn, 0xffffff); 149 DEVX_SET(mkc, mkc, pd, pd_id); 150 DEVX_SET(mkc, mkc, translations_octword_size, 151 SPDK_KLM_MAX_TRANSLATION_ENTRIES_NUM); 152 DEVX_SET(mkc, mkc, relaxed_ordering_write, 153 attr->relaxed_ordering_write); 154 DEVX_SET(mkc, mkc, relaxed_ordering_read, 155 attr->relaxed_ordering_read); 156 DEVX_SET64(mkc, mkc, start_addr, attr->addr); 157 DEVX_SET64(mkc, mkc, len, attr->size); 158 DEVX_SET(mkc, mkc, mkey_7_0, 0x42); 159 if (attr->crypto_en) { 160 DEVX_SET(mkc, mkc, crypto_en, 1); 161 } 162 if (attr->bsf_octowords) { 163 DEVX_SET(mkc, mkc, bsf_en, 1); 164 DEVX_SET(mkc, mkc, bsf_octword_size, attr->bsf_octowords); 165 } 166 167 cmkey->devx_obj = mlx5dv_devx_obj_create(ctx, in, sizeof(in), out, 168 sizeof(out)); 169 if (!cmkey->devx_obj) { 170 SPDK_ERRLOG("mlx5dv_devx_obj_create() failed to create mkey, errno:%d\n", errno); 171 goto out_err; 172 } 173 174 cmkey->mkey = DEVX_GET(create_mkey_out, out, mkey_index) << 8 | 0x42; 175 return cmkey; 176 177 out_err: 178 free(cmkey); 179 return NULL; 180 } 181 182 static int 183 mlx5_mkey_destroy(struct mlx5_mkey *mkey) 184 { 185 int ret = 0; 186 187 if (mkey->devx_obj) { 188 ret = mlx5dv_devx_obj_destroy(mkey->devx_obj); 189 } 190 191 free(mkey); 192 193 return ret; 194 } 195 196 static int 197 mlx5_query_relaxed_ordering_caps(struct ibv_context *context, 198 struct mlx5_relaxed_ordering_caps *caps) 199 { 200 uint8_t in[DEVX_ST_SZ_BYTES(query_hca_cap_in)] = {}; 201 uint8_t out[DEVX_ST_SZ_BYTES(query_hca_cap_out)] = {}; 202 int ret; 203 204 DEVX_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 205 DEVX_SET(query_hca_cap_in, in, op_mod, 206 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE_CAP_2); 207 ret = mlx5dv_devx_general_cmd(context, in, sizeof(in), 208 out, sizeof(out)); 209 if (ret) { 210 return ret; 211 } 212 213 caps->relaxed_ordering_write_pci_enabled = DEVX_GET(query_hca_cap_out, 214 out, capability.cmd_hca_cap.relaxed_ordering_write_pci_enabled); 215 caps->relaxed_ordering_write = DEVX_GET(query_hca_cap_out, out, 216 capability.cmd_hca_cap.relaxed_ordering_write); 217 caps->relaxed_ordering_read = DEVX_GET(query_hca_cap_out, out, 218 capability.cmd_hca_cap.relaxed_ordering_read); 219 caps->relaxed_ordering_write_umr = DEVX_GET(query_hca_cap_out, 220 out, capability.cmd_hca_cap.relaxed_ordering_write_umr); 221 caps->relaxed_ordering_read_umr = DEVX_GET(query_hca_cap_out, 222 out, capability.cmd_hca_cap.relaxed_ordering_read_umr); 223 return 0; 224 } 225 226 static int 227 mlx5_mkey_pool_create_mkey(struct mlx5_mkey **_mkey, struct ibv_pd *pd, 228 struct mlx5_relaxed_ordering_caps *caps, uint32_t flags) 229 { 230 struct mlx5_mkey *mkey; 231 struct mlx5_mkey_attr mkey_attr = {}; 232 uint32_t bsf_size = 0; 233 234 mkey_attr.addr = 0; 235 mkey_attr.size = 0; 236 mkey_attr.log_entity_size = 0; 237 mkey_attr.relaxed_ordering_write = caps->relaxed_ordering_write; 238 mkey_attr.relaxed_ordering_read = caps->relaxed_ordering_read; 239 mkey_attr.klm_count = 0; 240 mkey_attr.klm = NULL; 241 if (flags & SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO) { 242 mkey_attr.crypto_en = true; 243 bsf_size += 64; 244 } 245 if (flags & SPDK_MLX5_MKEY_POOL_FLAG_SIGNATURE) { 246 bsf_size += 64; 247 } 248 mkey_attr.bsf_octowords = bsf_size / 16; 249 250 mkey = mlx5_mkey_create(pd, &mkey_attr); 251 if (!mkey) { 252 SPDK_ERRLOG("Failed to create mkey on dev %s\n", pd->context->device->name); 253 return -EINVAL; 254 } 255 *_mkey = mkey; 256 257 return 0; 258 } 259 260 static void 261 mlx5_set_mkey_in_pool(struct spdk_mempool *mp, void *cb_arg, void *_mkey, unsigned obj_idx) 262 { 263 struct spdk_mlx5_mkey_pool_obj *mkey = _mkey; 264 struct spdk_mlx5_mkey_pool *pool = cb_arg; 265 266 assert(obj_idx < pool->num_mkeys); 267 assert(pool->mkeys[obj_idx] != NULL); 268 mkey->mkey = pool->mkeys[obj_idx]->mkey; 269 mkey->pool_flag = pool->flags & 0xf; 270 mkey->sig.sigerr_count = 1; 271 mkey->sig.sigerr = false; 272 273 RB_INSERT(mlx5_mkeys_tree, &pool->tree, mkey); 274 } 275 276 static const char *g_mkey_pool_names[] = { 277 [SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO] = "crypto", 278 [SPDK_MLX5_MKEY_POOL_FLAG_SIGNATURE] = "signature", 279 }; 280 281 static void 282 mlx5_mkey_pool_destroy(struct spdk_mlx5_mkey_pool *pool) 283 { 284 uint32_t i; 285 286 if (pool->mpool) { 287 spdk_mempool_free(pool->mpool); 288 } 289 if (pool->mkeys) { 290 for (i = 0; i < pool->num_mkeys; i++) { 291 if (pool->mkeys[i]) { 292 mlx5_mkey_destroy(pool->mkeys[i]); 293 pool->mkeys[i] = NULL; 294 } 295 } 296 free(pool->mkeys); 297 } 298 TAILQ_REMOVE(&g_mkey_pools, pool, link); 299 free(pool); 300 } 301 302 static int 303 mlx5_mkey_pools_init(struct spdk_mlx5_mkey_pool_param *params, struct ibv_pd *pd) 304 { 305 struct spdk_mlx5_mkey_pool *new_pool; 306 struct mlx5_mkey **mkeys; 307 struct mlx5_relaxed_ordering_caps caps; 308 uint32_t j, pdn; 309 int rc; 310 char pool_name[32]; 311 312 new_pool = calloc(1, sizeof(*new_pool)); 313 if (!new_pool) { 314 rc = -ENOMEM; 315 goto err; 316 } 317 TAILQ_INSERT_TAIL(&g_mkey_pools, new_pool, link); 318 rc = mlx5_query_relaxed_ordering_caps(pd->context, &caps); 319 if (rc) { 320 SPDK_ERRLOG("Failed to get relaxed ordering capabilities, dev %s\n", 321 pd->context->device->dev_name); 322 goto err; 323 } 324 mkeys = calloc(params->mkey_count, sizeof(struct mlx5_mkey *)); 325 if (!mkeys) { 326 rc = -ENOMEM; 327 goto err; 328 } 329 new_pool->mkeys = mkeys; 330 new_pool->num_mkeys = params->mkey_count; 331 new_pool->pd = pd; 332 new_pool->flags = params->flags; 333 for (j = 0; j < params->mkey_count; j++) { 334 rc = mlx5_mkey_pool_create_mkey(&mkeys[j], pd, &caps, params->flags); 335 if (rc) { 336 goto err; 337 } 338 } 339 rc = mlx5_get_pd_id(pd, &pdn); 340 if (rc) { 341 SPDK_ERRLOG("Failed to get pdn, pd %p\n", pd); 342 goto err; 343 } 344 rc = snprintf(pool_name, 32, "%s_%s_%04u", pd->context->device->name, 345 g_mkey_pool_names[new_pool->flags], pdn); 346 if (rc < 0) { 347 goto err; 348 } 349 RB_INIT(&new_pool->tree); 350 new_pool->mpool = spdk_mempool_create_ctor(pool_name, params->mkey_count, 351 sizeof(struct spdk_mlx5_mkey_pool_obj), 352 params->cache_per_thread, SPDK_ENV_NUMA_ID_ANY, 353 mlx5_set_mkey_in_pool, new_pool); 354 if (!new_pool->mpool) { 355 SPDK_ERRLOG("Failed to create mempool\n"); 356 rc = -ENOMEM; 357 goto err; 358 } 359 360 return 0; 361 362 err: 363 mlx5_mkey_pool_destroy(new_pool); 364 365 return rc; 366 } 367 368 static struct spdk_mlx5_mkey_pool * 369 mlx5_mkey_pool_get(struct ibv_pd *pd, uint32_t flags) 370 { 371 struct spdk_mlx5_mkey_pool *pool; 372 373 TAILQ_FOREACH(pool, &g_mkey_pools, link) { 374 if (pool->pd == pd && pool->flags == flags) { 375 return pool; 376 } 377 } 378 379 return NULL; 380 } 381 382 int 383 spdk_mlx5_mkey_pool_init(struct spdk_mlx5_mkey_pool_param *params, struct ibv_pd *pd) 384 { 385 int rc; 386 387 if (!pd) { 388 return -EINVAL; 389 } 390 391 if (!params || !params->mkey_count) { 392 return -EINVAL; 393 } 394 if ((params->flags & MLX5_UMR_POOL_VALID_FLAGS_MASK) != 0) { 395 SPDK_ERRLOG("Invalid flags %x\n", params->flags); 396 return -EINVAL; 397 } 398 if ((params->flags & (SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO | SPDK_MLX5_MKEY_POOL_FLAG_SIGNATURE)) == 399 (SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO | SPDK_MLX5_MKEY_POOL_FLAG_SIGNATURE)) { 400 SPDK_ERRLOG("Both crypto and signature capabilities are not supported\n"); 401 return -EINVAL; 402 } 403 if (params->cache_per_thread > params->mkey_count || !params->cache_per_thread) { 404 params->cache_per_thread = params->mkey_count * 3 / 4 / spdk_env_get_core_count(); 405 } 406 407 pthread_mutex_lock(&g_mkey_pool_lock); 408 if (mlx5_mkey_pool_get(pd, params->flags) != NULL) { 409 pthread_mutex_unlock(&g_mkey_pool_lock); 410 return -EEXIST; 411 } 412 413 rc = mlx5_mkey_pools_init(params, pd); 414 pthread_mutex_unlock(&g_mkey_pool_lock); 415 416 return rc; 417 } 418 419 int 420 spdk_mlx5_mkey_pool_destroy(uint32_t flags, struct ibv_pd *pd) 421 { 422 struct spdk_mlx5_mkey_pool *pool; 423 int rc = 0; 424 425 if (!pd) { 426 return -EINVAL; 427 } 428 429 if ((flags & MLX5_UMR_POOL_VALID_FLAGS_MASK) != 0) { 430 SPDK_ERRLOG("Invalid flags %x\n", flags); 431 return -EINVAL; 432 } 433 434 pthread_mutex_lock(&g_mkey_pool_lock); 435 pool = mlx5_mkey_pool_get(pd, flags); 436 if (!pool) { 437 SPDK_ERRLOG("Cant find a pool for PD %p, flags %x\n", pd, flags); 438 pthread_mutex_unlock(&g_mkey_pool_lock); 439 return -ENODEV; 440 } 441 if (pool->refcnt) { 442 SPDK_WARNLOG("Can't delete pool pd %p, dev %s\n", pool->pd, pool->pd->context->device->dev_name); 443 rc = -EAGAIN; 444 } else { 445 mlx5_mkey_pool_destroy(pool); 446 } 447 pthread_mutex_unlock(&g_mkey_pool_lock); 448 449 return rc; 450 } 451 452 struct spdk_mlx5_mkey_pool * 453 spdk_mlx5_mkey_pool_get_ref(struct ibv_pd *pd, uint32_t flags) 454 { 455 struct spdk_mlx5_mkey_pool *pool; 456 457 if ((flags & MLX5_UMR_POOL_VALID_FLAGS_MASK) != 0) { 458 SPDK_ERRLOG("Invalid flags %x\n", flags); 459 return NULL; 460 } 461 462 pthread_mutex_lock(&g_mkey_pool_lock); 463 pool = mlx5_mkey_pool_get(pd, flags); 464 if (pool) { 465 pool->refcnt++; 466 } 467 pthread_mutex_unlock(&g_mkey_pool_lock); 468 469 return pool; 470 } 471 472 void 473 spdk_mlx5_mkey_pool_put_ref(struct spdk_mlx5_mkey_pool *pool) 474 { 475 pthread_mutex_lock(&g_mkey_pool_lock); 476 pool->refcnt--; 477 pthread_mutex_unlock(&g_mkey_pool_lock); 478 } 479 480 int 481 spdk_mlx5_mkey_pool_get_bulk(struct spdk_mlx5_mkey_pool *pool, 482 struct spdk_mlx5_mkey_pool_obj **mkeys, uint32_t mkeys_count) 483 { 484 assert(pool->mpool); 485 486 return spdk_mempool_get_bulk(pool->mpool, (void **)mkeys, mkeys_count); 487 } 488 489 void 490 spdk_mlx5_mkey_pool_put_bulk(struct spdk_mlx5_mkey_pool *pool, 491 struct spdk_mlx5_mkey_pool_obj **mkeys, uint32_t mkeys_count) 492 { 493 assert(pool->mpool); 494 495 spdk_mempool_put_bulk(pool->mpool, (void **)mkeys, mkeys_count); 496 } 497 498 static inline void 499 _mlx5_set_umr_ctrl_seg_mtt(struct mlx5_wqe_umr_ctrl_seg *ctrl, uint32_t klms_octowords, 500 uint64_t mkey_mask) 501 { 502 ctrl->flags |= MLX5_WQE_UMR_CTRL_FLAG_INLINE; 503 ctrl->klm_octowords = htobe16(klms_octowords); 504 /* 505 * Going to modify two properties of KLM mkey: 506 * 1. 'free' field: change this mkey from in free to in use 507 * 2. 'len' field: to include the total bytes in iovec 508 */ 509 mkey_mask |= MLX5_WQE_UMR_CTRL_MKEY_MASK_FREE | MLX5_WQE_UMR_CTRL_MKEY_MASK_LEN; 510 511 ctrl->mkey_mask |= htobe64(mkey_mask); 512 } 513 514 static inline void 515 mlx5_set_umr_ctrl_seg_mtt(struct mlx5_wqe_umr_ctrl_seg *ctrl, uint32_t klms_octowords) 516 { 517 _mlx5_set_umr_ctrl_seg_mtt(ctrl, klms_octowords, 0); 518 } 519 520 static inline void 521 mlx5_set_umr_ctrl_seg_mtt_sig(struct mlx5_wqe_umr_ctrl_seg *ctrl, uint32_t klms_octowords) 522 { 523 _mlx5_set_umr_ctrl_seg_mtt(ctrl, klms_octowords, MLX5_WQE_UMR_CTRL_MKEY_MASK_SIG_ERR); 524 } 525 526 static inline void 527 mlx5_set_umr_ctrl_seg_bsf_size(struct mlx5_wqe_umr_ctrl_seg *ctrl, int bsf_size) 528 { 529 ctrl->bsf_octowords = htobe16(SPDK_ALIGN_CEIL(SPDK_CEIL_DIV(bsf_size, 16), 4)); 530 } 531 532 static inline void 533 mlx5_set_umr_mkey_seg_mtt(struct mlx5_wqe_mkey_context_seg *mkey, 534 struct spdk_mlx5_umr_attr *umr_attr) 535 { 536 mkey->len = htobe64(umr_attr->umr_len); 537 } 538 539 static void 540 mlx5_set_umr_mkey_seg(struct mlx5_wqe_mkey_context_seg *mkey, 541 struct spdk_mlx5_umr_attr *umr_attr) 542 { 543 memset(mkey, 0, 64); 544 mlx5_set_umr_mkey_seg_mtt(mkey, umr_attr); 545 } 546 547 static void 548 mlx5_set_umr_mkey_seg_sig(struct mlx5_wqe_mkey_context_seg *mkey, 549 struct spdk_mlx5_umr_sig_attr *sig_attr) 550 { 551 mkey->flags_pd = htobe32((sig_attr->sigerr_count & 1) << 26); 552 } 553 554 static inline void 555 mlx5_set_umr_inline_klm_seg(struct mlx5_wqe_umr_klm_seg *klm, struct ibv_sge *sge) 556 { 557 klm->byte_count = htobe32(sge->length); 558 klm->mkey = htobe32(sge->lkey); 559 klm->address = htobe64(sge->addr); 560 } 561 562 static void * 563 mlx5_build_inline_mtt(struct mlx5_hw_qp *qp, uint32_t *to_end, struct mlx5_wqe_umr_klm_seg *dst_klm, 564 struct spdk_mlx5_umr_attr *umr_attr) 565 { 566 struct ibv_sge *src_sge = umr_attr->sge; 567 int num_wqebbs = umr_attr->sge_count / 4; 568 int tail = umr_attr->sge_count & 0x3; 569 int i; 570 571 for (i = 0; i < num_wqebbs; i++) { 572 mlx5_set_umr_inline_klm_seg(&dst_klm[0], src_sge++); 573 mlx5_set_umr_inline_klm_seg(&dst_klm[1], src_sge++); 574 mlx5_set_umr_inline_klm_seg(&dst_klm[2], src_sge++); 575 mlx5_set_umr_inline_klm_seg(&dst_klm[3], src_sge++); 576 /* sizeof(*dst_klm) * 4 == MLX5_SEND_WQE_BB */ 577 dst_klm = mlx5_qp_get_next_wqebb(qp, to_end, dst_klm); 578 } 579 580 if (!tail) { 581 return dst_klm; 582 } 583 584 for (i = 0; i < tail; i++) { 585 mlx5_set_umr_inline_klm_seg(&dst_klm[i], src_sge++); 586 } 587 588 /* Fill PAD entries to make whole mtt aligned to 64B(MLX5_SEND_WQE_BB) */ 589 memset(&dst_klm[i], 0, MLX5_SEND_WQE_BB - sizeof(struct mlx5_wqe_umr_klm_seg) * tail); 590 591 return mlx5_qp_get_next_wqebb(qp, to_end, dst_klm); 592 } 593 594 static inline void 595 mlx5_set_umr_crypto_bsf_seg(struct mlx5_crypto_bsf_seg *bsf, struct spdk_mlx5_umr_crypto_attr *attr, 596 uint32_t raw_data_size, uint8_t bsf_size) 597 { 598 uint64_t *iv = (void *)bsf->xts_initial_tweak; 599 600 memset(bsf, 0, sizeof(*bsf)); 601 switch (attr->tweak_mode) { 602 case SPDK_MLX5_CRYPTO_KEY_TWEAK_MODE_SIMPLE_LBA_LE: 603 iv[0] = htole64(attr->xts_iv); 604 iv[1] = 0; 605 break; 606 case SPDK_MLX5_CRYPTO_KEY_TWEAK_MODE_SIMPLE_LBA_BE: 607 iv[0] = 0; 608 iv[1] = htobe64(attr->xts_iv); 609 break; 610 default: 611 assert(false && "unsupported tweak mode"); 612 break; 613 } 614 615 bsf->size_type = (bsf_size << 6) | MLX5_CRYPTO_BSF_P_TYPE_CRYPTO; 616 bsf->enc_order = attr->enc_order; 617 bsf->raw_data_size = htobe32(raw_data_size); 618 bsf->crypto_block_size_pointer = attr->bs_selector; 619 bsf->dek_pointer = htobe32(attr->dek_obj_id); 620 *((uint64_t *)bsf->keytag) = attr->keytag; 621 } 622 623 static inline uint8_t 624 mlx5_get_crc32c_tfs(uint32_t seed) 625 { 626 assert(seed == 0 || seed == 0xffffffff); 627 return MLX5_SIG_BSF_TFS_CRC32C | !seed; 628 } 629 630 static inline void 631 mlx5_set_umr_sig_bsf_seg(struct mlx5_sig_bsf_seg *bsf, 632 struct spdk_mlx5_umr_sig_attr *attr) 633 { 634 uint8_t bsf_size = MLX5_SIG_BSF_SIZE_32B; 635 uint32_t tfs_psv; 636 uint32_t init_gen; 637 638 memset(bsf, 0, sizeof(*bsf)); 639 bsf->basic.bsf_size_sbs = (bsf_size << 6); 640 bsf->basic.raw_data_size = htobe32(attr->raw_data_size); 641 bsf->basic.check_byte_mask = 0xff; 642 643 tfs_psv = mlx5_get_crc32c_tfs(attr->seed); 644 tfs_psv = tfs_psv << MLX5_SIG_BSF_TFS_SHIFT; 645 tfs_psv |= attr->psv_index & 0xffffff; 646 647 if (attr->domain == SPDK_MLX5_UMR_SIG_DOMAIN_WIRE) { 648 bsf->ext.w_tfs_psv = htobe32(tfs_psv); 649 init_gen = attr->init ? MLX5_SIG_BSF_EXT_W_T_INIT : 0; 650 if (attr->check_gen) { 651 init_gen |= MLX5_SIG_BSF_EXT_W_T_CHECK_GEN; 652 } 653 bsf->ext.t_init_gen_pro_size = htobe32(init_gen); 654 } else { 655 bsf->ext.m_tfs_psv = htobe32(tfs_psv); 656 init_gen = attr->init ? MLX5_SIG_BSF_EXT_M_T_INIT : 0; 657 if (attr->check_gen) { 658 init_gen |= MLX5_SIG_BSF_EXT_M_T_CHECK_GEN; 659 } 660 bsf->ext.t_init_gen_pro_size = htobe32(init_gen); 661 } 662 } 663 664 static inline void 665 mlx5_umr_configure_with_wrap_around_crypto(struct spdk_mlx5_qp *qp, 666 struct spdk_mlx5_umr_attr *umr_attr, struct spdk_mlx5_umr_crypto_attr *crypto_attr, uint64_t wr_id, 667 uint32_t flags, uint32_t wqe_size, uint32_t umr_wqe_n_bb, uint32_t mtt_size) 668 { 669 struct mlx5_hw_qp *hw = &qp->hw; 670 struct mlx5_wqe_ctrl_seg *ctrl; 671 struct mlx5_wqe_ctrl_seg *gen_ctrl; 672 struct mlx5_wqe_umr_ctrl_seg *umr_ctrl; 673 struct mlx5_wqe_mkey_context_seg *mkey; 674 struct mlx5_wqe_umr_klm_seg *klm; 675 struct mlx5_crypto_bsf_seg *bsf; 676 uint8_t fm_ce_se; 677 uint32_t pi, to_end; 678 679 fm_ce_se = mlx5_qp_fm_ce_se_update(qp, (uint8_t)flags); 680 681 ctrl = (struct mlx5_wqe_ctrl_seg *)mlx5_qp_get_wqe_bb(hw); 682 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 683 to_end = (hw->sq_wqe_cnt - pi) * MLX5_SEND_WQE_BB; 684 685 /* 686 * sizeof(gen_ctrl) + sizeof(umr_ctrl) == MLX5_SEND_WQE_BB, 687 * so do not need to worry about wqe buffer wrap around. 688 * 689 * build genenal ctrl segment 690 */ 691 gen_ctrl = ctrl; 692 mlx5_set_ctrl_seg(gen_ctrl, hw->sq_pi, MLX5_OPCODE_UMR, 0, 693 hw->qp_num, fm_ce_se, 694 SPDK_CEIL_DIV(wqe_size, 16), 0, 695 htobe32(umr_attr->mkey)); 696 697 /* build umr ctrl segment */ 698 umr_ctrl = (struct mlx5_wqe_umr_ctrl_seg *)(gen_ctrl + 1); 699 memset(umr_ctrl, 0, sizeof(*umr_ctrl)); 700 mlx5_set_umr_ctrl_seg_mtt(umr_ctrl, mtt_size); 701 mlx5_set_umr_ctrl_seg_bsf_size(umr_ctrl, sizeof(struct mlx5_crypto_bsf_seg)); 702 703 /* build mkey context segment */ 704 mkey = mlx5_qp_get_next_wqebb(hw, &to_end, ctrl); 705 mlx5_set_umr_mkey_seg(mkey, umr_attr); 706 707 klm = mlx5_qp_get_next_wqebb(hw, &to_end, mkey); 708 bsf = mlx5_build_inline_mtt(hw, &to_end, klm, umr_attr); 709 710 mlx5_set_umr_crypto_bsf_seg(bsf, crypto_attr, umr_attr->umr_len, MLX5_CRYPTO_BSF_SIZE_64B); 711 712 mlx5_qp_wqe_submit(qp, ctrl, umr_wqe_n_bb, pi); 713 714 mlx5_qp_set_comp(qp, pi, wr_id, fm_ce_se, umr_wqe_n_bb); 715 assert(qp->tx_available >= umr_wqe_n_bb); 716 qp->tx_available -= umr_wqe_n_bb; 717 } 718 719 static inline void 720 mlx5_umr_configure_full_crypto(struct spdk_mlx5_qp *dv_qp, struct spdk_mlx5_umr_attr *umr_attr, 721 struct spdk_mlx5_umr_crypto_attr *crypto_attr, uint64_t wr_id, 722 uint32_t flags, uint32_t wqe_size, uint32_t umr_wqe_n_bb, 723 uint32_t mtt_size) 724 { 725 struct mlx5_hw_qp *hw = &dv_qp->hw; 726 struct mlx5_wqe_ctrl_seg *ctrl; 727 struct mlx5_wqe_ctrl_seg *gen_ctrl; 728 struct mlx5_wqe_umr_ctrl_seg *umr_ctrl; 729 struct mlx5_wqe_mkey_context_seg *mkey; 730 struct mlx5_wqe_umr_klm_seg *klm; 731 struct mlx5_crypto_bsf_seg *bsf; 732 uint8_t fm_ce_se; 733 uint32_t pi; 734 uint32_t i; 735 736 fm_ce_se = mlx5_qp_fm_ce_se_update(dv_qp, (uint8_t)flags); 737 738 ctrl = (struct mlx5_wqe_ctrl_seg *)mlx5_qp_get_wqe_bb(hw); 739 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 740 gen_ctrl = ctrl; 741 mlx5_set_ctrl_seg(gen_ctrl, hw->sq_pi, MLX5_OPCODE_UMR, 0, 742 hw->qp_num, fm_ce_se, 743 SPDK_CEIL_DIV(wqe_size, 16), 0, 744 htobe32(umr_attr->mkey)); 745 746 /* build umr ctrl segment */ 747 umr_ctrl = (struct mlx5_wqe_umr_ctrl_seg *)(gen_ctrl + 1); 748 memset(umr_ctrl, 0, sizeof(*umr_ctrl)); 749 mlx5_set_umr_ctrl_seg_mtt(umr_ctrl, mtt_size); 750 mlx5_set_umr_ctrl_seg_bsf_size(umr_ctrl, sizeof(struct mlx5_crypto_bsf_seg)); 751 752 /* build mkey context segment */ 753 mkey = (struct mlx5_wqe_mkey_context_seg *)(umr_ctrl + 1); 754 mlx5_set_umr_mkey_seg(mkey, umr_attr); 755 756 klm = (struct mlx5_wqe_umr_klm_seg *)(mkey + 1); 757 for (i = 0; i < umr_attr->sge_count; i++) { 758 mlx5_set_umr_inline_klm_seg(klm, &umr_attr->sge[i]); 759 /* sizeof(*klm) * 4 == MLX5_SEND_WQE_BB */ 760 klm = klm + 1; 761 } 762 /* fill PAD if existing */ 763 /* PAD entries is to make whole mtt aligned to 64B(MLX5_SEND_WQE_BB), 764 * So it will not happen wrap around during fill PAD entries. */ 765 for (; i < mtt_size; i++) { 766 memset(klm, 0, sizeof(*klm)); 767 klm = klm + 1; 768 } 769 770 bsf = (struct mlx5_crypto_bsf_seg *)klm; 771 mlx5_set_umr_crypto_bsf_seg(bsf, crypto_attr, umr_attr->umr_len, MLX5_CRYPTO_BSF_SIZE_64B); 772 773 mlx5_qp_wqe_submit(dv_qp, ctrl, umr_wqe_n_bb, pi); 774 775 mlx5_qp_set_comp(dv_qp, pi, wr_id, fm_ce_se, umr_wqe_n_bb); 776 assert(dv_qp->tx_available >= umr_wqe_n_bb); 777 dv_qp->tx_available -= umr_wqe_n_bb; 778 } 779 780 int 781 spdk_mlx5_umr_configure_crypto(struct spdk_mlx5_qp *qp, struct spdk_mlx5_umr_attr *umr_attr, 782 struct spdk_mlx5_umr_crypto_attr *crypto_attr, uint64_t wr_id, uint32_t flags) 783 { 784 struct mlx5_hw_qp *hw = &qp->hw; 785 uint32_t pi, to_end, umr_wqe_n_bb; 786 uint32_t wqe_size, mtt_size; 787 uint32_t inline_klm_size; 788 789 if (!spdk_unlikely(umr_attr->sge_count)) { 790 return -EINVAL; 791 } 792 793 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 794 to_end = (hw->sq_wqe_cnt - pi) * MLX5_SEND_WQE_BB; 795 796 /* 797 * UMR WQE LAYOUT: 798 * ----------------------------------------------------------------------- 799 * | gen_ctrl | umr_ctrl | mkey_ctx | inline klm mtt | inline crypto bsf | 800 * ----------------------------------------------------------------------- 801 * 16bytes 48bytes 64bytes sge_count*16 bytes 64 bytes 802 * 803 * Note: size of inline klm mtt should be aligned to 64 bytes. 804 */ 805 wqe_size = sizeof(struct mlx5_wqe_ctrl_seg) + sizeof(struct mlx5_wqe_umr_ctrl_seg) + 806 sizeof(struct mlx5_wqe_mkey_context_seg); 807 mtt_size = SPDK_ALIGN_CEIL(umr_attr->sge_count, 4); 808 inline_klm_size = mtt_size * sizeof(struct mlx5_wqe_umr_klm_seg); 809 wqe_size += inline_klm_size; 810 wqe_size += sizeof(struct mlx5_crypto_bsf_seg); 811 812 umr_wqe_n_bb = SPDK_CEIL_DIV(wqe_size, MLX5_SEND_WQE_BB); 813 if (spdk_unlikely(umr_wqe_n_bb > qp->tx_available)) { 814 return -ENOMEM; 815 } 816 if (spdk_unlikely(umr_attr->sge_count > qp->max_send_sge)) { 817 return -E2BIG; 818 } 819 820 if (spdk_unlikely(to_end < wqe_size)) { 821 mlx5_umr_configure_with_wrap_around_crypto(qp, umr_attr, crypto_attr, wr_id, flags, wqe_size, 822 umr_wqe_n_bb, 823 mtt_size); 824 } else { 825 mlx5_umr_configure_full_crypto(qp, umr_attr, crypto_attr, wr_id, flags, wqe_size, umr_wqe_n_bb, 826 mtt_size); 827 } 828 829 return 0; 830 } 831 832 static inline void 833 mlx5_umr_configure_with_wrap_around_sig(struct spdk_mlx5_qp *dv_qp, 834 struct spdk_mlx5_umr_attr *umr_attr, 835 struct spdk_mlx5_umr_sig_attr *sig_attr, uint64_t wr_id, 836 uint32_t flags, uint32_t wqe_size, uint32_t umr_wqe_n_bb, uint32_t mtt_size) 837 { 838 struct mlx5_hw_qp *hw = &dv_qp->hw; 839 struct mlx5_wqe_ctrl_seg *ctrl; 840 struct mlx5_wqe_ctrl_seg *gen_ctrl; 841 struct mlx5_wqe_umr_ctrl_seg *umr_ctrl; 842 struct mlx5_wqe_mkey_context_seg *mkey; 843 struct mlx5_wqe_umr_klm_seg *klm; 844 struct mlx5_sig_bsf_seg *bsf; 845 uint8_t fm_ce_se; 846 uint32_t pi, to_end; 847 848 fm_ce_se = mlx5_qp_fm_ce_se_update(dv_qp, (uint8_t)flags); 849 850 ctrl = (struct mlx5_wqe_ctrl_seg *)mlx5_qp_get_wqe_bb(hw); 851 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 852 to_end = (hw->sq_wqe_cnt - pi) * MLX5_SEND_WQE_BB; 853 854 /* 855 * sizeof(gen_ctrl) + sizeof(umr_ctrl) == MLX5_SEND_WQE_BB, 856 * so do not need to worry about wqe buffer wrap around. 857 * 858 * build genenal ctrl segment 859 */ 860 gen_ctrl = ctrl; 861 mlx5_set_ctrl_seg(gen_ctrl, hw->sq_pi, MLX5_OPCODE_UMR, 0, 862 hw->qp_num, fm_ce_se, 863 SPDK_CEIL_DIV(wqe_size, 16), 0, 864 htobe32(umr_attr->mkey)); 865 866 /* build umr ctrl segment */ 867 umr_ctrl = (struct mlx5_wqe_umr_ctrl_seg *)(gen_ctrl + 1); 868 memset(umr_ctrl, 0, sizeof(*umr_ctrl)); 869 mlx5_set_umr_ctrl_seg_mtt_sig(umr_ctrl, mtt_size); 870 mlx5_set_umr_ctrl_seg_bsf_size(umr_ctrl, sizeof(struct mlx5_sig_bsf_seg)); 871 872 /* build mkey context segment */ 873 mkey = mlx5_qp_get_next_wqebb(hw, &to_end, ctrl); 874 mlx5_set_umr_mkey_seg(mkey, umr_attr); 875 mlx5_set_umr_mkey_seg_sig(mkey, sig_attr); 876 877 klm = mlx5_qp_get_next_wqebb(hw, &to_end, mkey); 878 bsf = mlx5_build_inline_mtt(hw, &to_end, klm, umr_attr); 879 880 mlx5_set_umr_sig_bsf_seg(bsf, sig_attr); 881 882 mlx5_qp_wqe_submit(dv_qp, ctrl, umr_wqe_n_bb, pi); 883 884 mlx5_qp_set_comp(dv_qp, pi, wr_id, fm_ce_se, umr_wqe_n_bb); 885 assert(dv_qp->tx_available >= umr_wqe_n_bb); 886 dv_qp->tx_available -= umr_wqe_n_bb; 887 } 888 889 static inline void 890 mlx5_umr_configure_full_sig(struct spdk_mlx5_qp *dv_qp, struct spdk_mlx5_umr_attr *umr_attr, 891 struct spdk_mlx5_umr_sig_attr *sig_attr, uint64_t wr_id, 892 uint32_t flags, uint32_t wqe_size, uint32_t umr_wqe_n_bb, 893 uint32_t mtt_size) 894 { 895 struct mlx5_hw_qp *hw = &dv_qp->hw; 896 struct mlx5_wqe_ctrl_seg *ctrl; 897 struct mlx5_wqe_ctrl_seg *gen_ctrl; 898 struct mlx5_wqe_umr_ctrl_seg *umr_ctrl; 899 struct mlx5_wqe_mkey_context_seg *mkey; 900 struct mlx5_wqe_umr_klm_seg *klm; 901 struct mlx5_sig_bsf_seg *bsf; 902 uint8_t fm_ce_se; 903 uint32_t pi; 904 uint32_t i; 905 906 fm_ce_se = mlx5_qp_fm_ce_se_update(dv_qp, (uint8_t)flags); 907 908 ctrl = (struct mlx5_wqe_ctrl_seg *)mlx5_qp_get_wqe_bb(hw); 909 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 910 gen_ctrl = ctrl; 911 mlx5_set_ctrl_seg(gen_ctrl, hw->sq_pi, MLX5_OPCODE_UMR, 0, 912 hw->qp_num, fm_ce_se, 913 SPDK_CEIL_DIV(wqe_size, 16), 0, 914 htobe32(umr_attr->mkey)); 915 916 /* build umr ctrl segment */ 917 umr_ctrl = (struct mlx5_wqe_umr_ctrl_seg *)(gen_ctrl + 1); 918 memset(umr_ctrl, 0, sizeof(*umr_ctrl)); 919 mlx5_set_umr_ctrl_seg_mtt_sig(umr_ctrl, mtt_size); 920 mlx5_set_umr_ctrl_seg_bsf_size(umr_ctrl, sizeof(struct mlx5_sig_bsf_seg)); 921 922 /* build mkey context segment */ 923 mkey = (struct mlx5_wqe_mkey_context_seg *)(umr_ctrl + 1); 924 memset(mkey, 0, sizeof(*mkey)); 925 mlx5_set_umr_mkey_seg_mtt(mkey, umr_attr); 926 mlx5_set_umr_mkey_seg_sig(mkey, sig_attr); 927 928 klm = (struct mlx5_wqe_umr_klm_seg *)(mkey + 1); 929 for (i = 0; i < umr_attr->sge_count; i++) { 930 mlx5_set_umr_inline_klm_seg(klm, &umr_attr->sge[i]); 931 /* sizeof(*klm) * 4 == MLX5_SEND_WQE_BB */ 932 klm = klm + 1; 933 } 934 /* fill PAD if existing */ 935 /* PAD entries is to make whole mtt aligned to 64B(MLX5_SEND_WQE_BB), 936 * So it will not happen warp around during fill PAD entries. */ 937 for (; i < mtt_size; i++) { 938 memset(klm, 0, sizeof(*klm)); 939 klm = klm + 1; 940 } 941 942 bsf = (struct mlx5_sig_bsf_seg *)klm; 943 mlx5_set_umr_sig_bsf_seg(bsf, sig_attr); 944 945 mlx5_qp_wqe_submit(dv_qp, ctrl, umr_wqe_n_bb, pi); 946 947 mlx5_qp_set_comp(dv_qp, pi, wr_id, fm_ce_se, umr_wqe_n_bb); 948 assert(dv_qp->tx_available >= umr_wqe_n_bb); 949 dv_qp->tx_available -= umr_wqe_n_bb; 950 } 951 952 int 953 spdk_mlx5_umr_configure_sig(struct spdk_mlx5_qp *qp, struct spdk_mlx5_umr_attr *umr_attr, 954 struct spdk_mlx5_umr_sig_attr *sig_attr, uint64_t wr_id, uint32_t flags) 955 { 956 struct mlx5_hw_qp *hw = &qp->hw; 957 uint32_t pi, to_end, umr_wqe_n_bb; 958 uint32_t wqe_size, mtt_size; 959 uint32_t inline_klm_size; 960 961 if (!spdk_unlikely(umr_attr->sge_count)) { 962 return -EINVAL; 963 } 964 965 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 966 to_end = (hw->sq_wqe_cnt - pi) * MLX5_SEND_WQE_BB; 967 968 /* 969 * UMR WQE LAYOUT: 970 * ----------------------------------------------------------------------- 971 * | gen_ctrl | umr_ctrl | mkey_ctx | inline klm mtt | inline sig bsf | 972 * ----------------------------------------------------------------------- 973 * 16bytes 48bytes 64bytes sg_count*16 bytes 64 bytes 974 * 975 * Note: size of inline klm mtt should be aligned to 64 bytes. 976 */ 977 wqe_size = sizeof(struct mlx5_wqe_ctrl_seg) + sizeof(struct mlx5_wqe_umr_ctrl_seg) + 978 sizeof(struct mlx5_wqe_mkey_context_seg); 979 mtt_size = SPDK_ALIGN_CEIL(umr_attr->sge_count, 4); 980 inline_klm_size = mtt_size * sizeof(struct mlx5_wqe_umr_klm_seg); 981 wqe_size += inline_klm_size; 982 wqe_size += sizeof(struct mlx5_sig_bsf_seg); 983 984 umr_wqe_n_bb = SPDK_CEIL_DIV(wqe_size, MLX5_SEND_WQE_BB); 985 if (spdk_unlikely(umr_wqe_n_bb > qp->tx_available)) { 986 return -ENOMEM; 987 } 988 if (spdk_unlikely(umr_attr->sge_count > qp->max_send_sge)) { 989 return -E2BIG; 990 } 991 992 if (spdk_unlikely(to_end < wqe_size)) { 993 mlx5_umr_configure_with_wrap_around_sig(qp, umr_attr, sig_attr, wr_id, flags, wqe_size, 994 umr_wqe_n_bb, mtt_size); 995 } else { 996 mlx5_umr_configure_full_sig(qp, umr_attr, sig_attr, wr_id, flags, wqe_size, umr_wqe_n_bb, 997 mtt_size); 998 } 999 1000 return 0; 1001 } 1002 1003 static inline void 1004 mlx5_umr_configure_full(struct spdk_mlx5_qp *dv_qp, struct spdk_mlx5_umr_attr *umr_attr, 1005 uint64_t wr_id, uint32_t flags, uint32_t wqe_size, uint32_t umr_wqe_n_bb, 1006 uint32_t mtt_size) 1007 { 1008 struct mlx5_hw_qp *hw = &dv_qp->hw; 1009 struct mlx5_wqe_ctrl_seg *ctrl; 1010 struct mlx5_wqe_ctrl_seg *gen_ctrl; 1011 struct mlx5_wqe_umr_ctrl_seg *umr_ctrl; 1012 struct mlx5_wqe_mkey_context_seg *mkey; 1013 struct mlx5_wqe_umr_klm_seg *klm; 1014 uint8_t fm_ce_se; 1015 uint32_t pi; 1016 uint32_t i; 1017 1018 fm_ce_se = mlx5_qp_fm_ce_se_update(dv_qp, (uint8_t)flags); 1019 1020 ctrl = (struct mlx5_wqe_ctrl_seg *)mlx5_qp_get_wqe_bb(hw); 1021 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 1022 1023 gen_ctrl = ctrl; 1024 mlx5_set_ctrl_seg(gen_ctrl, hw->sq_pi, MLX5_OPCODE_UMR, 0, 1025 hw->qp_num, fm_ce_se, 1026 SPDK_CEIL_DIV(wqe_size, 16), 0, 1027 htobe32(umr_attr->mkey)); 1028 1029 /* build umr ctrl segment */ 1030 umr_ctrl = (struct mlx5_wqe_umr_ctrl_seg *)(gen_ctrl + 1); 1031 memset(umr_ctrl, 0, sizeof(*umr_ctrl)); 1032 mlx5_set_umr_ctrl_seg_mtt(umr_ctrl, mtt_size); 1033 1034 /* build mkey context segment */ 1035 mkey = (struct mlx5_wqe_mkey_context_seg *)(umr_ctrl + 1); 1036 mlx5_set_umr_mkey_seg(mkey, umr_attr); 1037 1038 klm = (struct mlx5_wqe_umr_klm_seg *)(mkey + 1); 1039 for (i = 0; i < umr_attr->sge_count; i++) { 1040 mlx5_set_umr_inline_klm_seg(klm, &umr_attr->sge[i]); 1041 /* sizeof(*klm) * 4 == MLX5_SEND_WQE_BB */ 1042 klm = klm + 1; 1043 } 1044 /* fill PAD if existing */ 1045 /* PAD entries is to make whole mtt aligned to 64B(MLX5_SEND_WQE_BB), 1046 * So it will not happen warp around during fill PAD entries. */ 1047 for (; i < mtt_size; i++) { 1048 memset(klm, 0, sizeof(*klm)); 1049 klm = klm + 1; 1050 } 1051 1052 mlx5_qp_wqe_submit(dv_qp, ctrl, umr_wqe_n_bb, pi); 1053 1054 mlx5_qp_set_comp(dv_qp, pi, wr_id, fm_ce_se, umr_wqe_n_bb); 1055 assert(dv_qp->tx_available >= umr_wqe_n_bb); 1056 dv_qp->tx_available -= umr_wqe_n_bb; 1057 } 1058 1059 static inline void 1060 mlx5_umr_configure_with_wrap_around(struct spdk_mlx5_qp *dv_qp, struct spdk_mlx5_umr_attr *umr_attr, 1061 uint64_t wr_id, uint32_t flags, uint32_t wqe_size, uint32_t umr_wqe_n_bb, 1062 uint32_t mtt_size) 1063 { 1064 struct mlx5_hw_qp *hw = &dv_qp->hw; 1065 struct mlx5_wqe_ctrl_seg *ctrl; 1066 struct mlx5_wqe_ctrl_seg *gen_ctrl; 1067 struct mlx5_wqe_umr_ctrl_seg *umr_ctrl; 1068 struct mlx5_wqe_mkey_context_seg *mkey; 1069 struct mlx5_wqe_umr_klm_seg *klm; 1070 uint8_t fm_ce_se; 1071 uint32_t pi, to_end; 1072 1073 fm_ce_se = mlx5_qp_fm_ce_se_update(dv_qp, (uint8_t)flags); 1074 1075 ctrl = (struct mlx5_wqe_ctrl_seg *)mlx5_qp_get_wqe_bb(hw); 1076 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 1077 to_end = (hw->sq_wqe_cnt - pi) * MLX5_SEND_WQE_BB; 1078 /* 1079 * sizeof(gen_ctrl) + sizeof(umr_ctrl) == MLX5_SEND_WQE_BB, 1080 * so do not need to worry about wqe buffer wrap around. 1081 * 1082 * build genenal ctrl segment 1083 */ 1084 gen_ctrl = ctrl; 1085 mlx5_set_ctrl_seg(gen_ctrl, hw->sq_pi, MLX5_OPCODE_UMR, 0, 1086 hw->qp_num, fm_ce_se, 1087 SPDK_CEIL_DIV(wqe_size, 16), 0, 1088 htobe32(umr_attr->mkey)); 1089 1090 /* build umr ctrl segment */ 1091 umr_ctrl = (struct mlx5_wqe_umr_ctrl_seg *)(gen_ctrl + 1); 1092 memset(umr_ctrl, 0, sizeof(*umr_ctrl)); 1093 mlx5_set_umr_ctrl_seg_mtt(umr_ctrl, mtt_size); 1094 1095 /* build mkey context segment */ 1096 mkey = mlx5_qp_get_next_wqebb(hw, &to_end, ctrl); 1097 mlx5_set_umr_mkey_seg(mkey, umr_attr); 1098 1099 klm = mlx5_qp_get_next_wqebb(hw, &to_end, mkey); 1100 mlx5_build_inline_mtt(hw, &to_end, klm, umr_attr); 1101 1102 mlx5_qp_wqe_submit(dv_qp, ctrl, umr_wqe_n_bb, pi); 1103 1104 mlx5_qp_set_comp(dv_qp, pi, wr_id, fm_ce_se, umr_wqe_n_bb); 1105 assert(dv_qp->tx_available >= umr_wqe_n_bb); 1106 dv_qp->tx_available -= umr_wqe_n_bb; 1107 } 1108 1109 int 1110 spdk_mlx5_umr_configure(struct spdk_mlx5_qp *qp, struct spdk_mlx5_umr_attr *umr_attr, 1111 uint64_t wr_id, uint32_t flags) 1112 { 1113 struct mlx5_hw_qp *hw = &qp->hw; 1114 uint32_t pi, to_end, umr_wqe_n_bb; 1115 uint32_t wqe_size, mtt_size; 1116 uint32_t inline_klm_size; 1117 1118 if (!spdk_unlikely(umr_attr->sge_count)) { 1119 return -EINVAL; 1120 } 1121 1122 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 1123 to_end = (hw->sq_wqe_cnt - pi) * MLX5_SEND_WQE_BB; 1124 1125 /* 1126 * UMR WQE LAYOUT: 1127 * --------------------------------------------------- 1128 * | gen_ctrl | umr_ctrl | mkey_ctx | inline klm mtt | 1129 * --------------------------------------------------- 1130 * 16bytes 48bytes 64bytes sg_count*16 bytes 1131 * 1132 * Note: size of inline klm mtt should be aligned to 64 bytes. 1133 */ 1134 wqe_size = sizeof(struct mlx5_wqe_ctrl_seg) + sizeof(struct mlx5_wqe_umr_ctrl_seg) + sizeof( 1135 struct mlx5_wqe_mkey_context_seg); 1136 mtt_size = SPDK_ALIGN_CEIL(umr_attr->sge_count, 4); 1137 inline_klm_size = mtt_size * sizeof(union mlx5_wqe_umr_inline_seg); 1138 wqe_size += inline_klm_size; 1139 1140 umr_wqe_n_bb = SPDK_CEIL_DIV(wqe_size, MLX5_SEND_WQE_BB); 1141 if (spdk_unlikely(umr_wqe_n_bb > qp->tx_available)) { 1142 return -ENOMEM; 1143 } 1144 if (spdk_unlikely(umr_attr->sge_count > qp->max_send_sge)) { 1145 return -E2BIG; 1146 } 1147 1148 if (spdk_unlikely(to_end < wqe_size)) { 1149 mlx5_umr_configure_with_wrap_around(qp, umr_attr, wr_id, flags, wqe_size, umr_wqe_n_bb, 1150 mtt_size); 1151 } else { 1152 mlx5_umr_configure_full(qp, umr_attr, wr_id, flags, wqe_size, umr_wqe_n_bb, mtt_size); 1153 } 1154 1155 return 0; 1156 } 1157 1158 static struct mlx5dv_devx_obj * 1159 mlx5_cmd_create_psv(struct ibv_context *context, uint32_t pdn, uint32_t *psv_index) 1160 { 1161 uint32_t out[DEVX_ST_SZ_DW(create_psv_out)] = {}; 1162 uint32_t in[DEVX_ST_SZ_DW(create_psv_in)] = {}; 1163 struct mlx5dv_devx_obj *obj; 1164 1165 assert(context); 1166 assert(psv_index); 1167 1168 DEVX_SET(create_psv_in, in, opcode, MLX5_CMD_OP_CREATE_PSV); 1169 DEVX_SET(create_psv_in, in, pd, pdn); 1170 DEVX_SET(create_psv_in, in, num_psv, 1); 1171 1172 obj = mlx5dv_devx_obj_create(context, in, sizeof(in), out, sizeof(out)); 1173 if (obj) { 1174 *psv_index = DEVX_GET(create_psv_out, out, psv0_index); 1175 } 1176 1177 return obj; 1178 } 1179 1180 struct spdk_mlx5_psv * 1181 spdk_mlx5_create_psv(struct ibv_pd *pd) 1182 { 1183 uint32_t pdn; 1184 struct spdk_mlx5_psv *psv; 1185 int err; 1186 1187 assert(pd); 1188 1189 err = mlx5_get_pd_id(pd, &pdn); 1190 if (err) { 1191 return NULL; 1192 } 1193 1194 psv = calloc(1, sizeof(*psv)); 1195 if (!psv) { 1196 return NULL; 1197 } 1198 1199 psv->devx_obj = mlx5_cmd_create_psv(pd->context, pdn, &psv->index); 1200 if (!psv->devx_obj) { 1201 free(psv); 1202 return NULL; 1203 } 1204 1205 return psv; 1206 } 1207 1208 int 1209 spdk_mlx5_destroy_psv(struct spdk_mlx5_psv *psv) 1210 { 1211 int ret; 1212 1213 ret = mlx5dv_devx_obj_destroy(psv->devx_obj); 1214 if (!ret) { 1215 free(psv); 1216 } 1217 1218 return ret; 1219 } 1220 1221 int 1222 spdk_mlx5_qp_set_psv(struct spdk_mlx5_qp *qp, uint32_t psv_index, uint32_t crc_seed, uint64_t wr_id, 1223 uint32_t flags) 1224 { 1225 struct mlx5_hw_qp *hw = &qp->hw; 1226 uint32_t pi, wqe_size, wqe_n_bb; 1227 struct mlx5_wqe_ctrl_seg *ctrl; 1228 struct mlx5_wqe_ctrl_seg *gen_ctrl; 1229 struct mlx5_wqe_set_psv_seg *psv; 1230 uint8_t fm_ce_se; 1231 uint64_t transient_signature = (uint64_t)crc_seed << 32; 1232 1233 wqe_size = sizeof(struct mlx5_wqe_ctrl_seg) + sizeof(struct mlx5_wqe_set_psv_seg); 1234 /* The size of SET_PSV WQE is constant and smaller than WQE BB. */ 1235 assert(wqe_size < MLX5_SEND_WQE_BB); 1236 wqe_n_bb = 1; 1237 if (spdk_unlikely(wqe_n_bb > qp->tx_available)) { 1238 return -ENOMEM; 1239 } 1240 1241 fm_ce_se = mlx5_qp_fm_ce_se_update(qp, (uint8_t)flags); 1242 pi = hw->sq_pi & (hw->sq_wqe_cnt - 1); 1243 ctrl = (struct mlx5_wqe_ctrl_seg *)mlx5_qp_get_wqe_bb(hw); 1244 gen_ctrl = ctrl; 1245 mlx5_set_ctrl_seg(gen_ctrl, hw->sq_pi, MLX5_OPCODE_SET_PSV, 0, hw->qp_num, fm_ce_se, 1246 SPDK_CEIL_DIV(wqe_size, 16), 0, 0); 1247 1248 /* build umr PSV segment */ 1249 psv = (struct mlx5_wqe_set_psv_seg *)(gen_ctrl + 1); 1250 /* Zeroing the set_psv segment and WQE padding. */ 1251 memset(psv, 0, MLX5_SEND_WQE_BB - sizeof(struct mlx5_wqe_ctrl_seg)); 1252 psv->psv_index = htobe32(psv_index); 1253 psv->transient_signature = htobe64(transient_signature); 1254 1255 mlx5_qp_wqe_submit(qp, ctrl, wqe_n_bb, pi); 1256 mlx5_qp_set_comp(qp, pi, wr_id, fm_ce_se, wqe_n_bb); 1257 assert(qp->tx_available >= wqe_n_bb); 1258 qp->tx_available -= wqe_n_bb; 1259 1260 return 0; 1261 } 1262 1263 void 1264 spdk_mlx5_umr_implementer_register(bool registered) 1265 { 1266 g_umr_implementer_registered = registered; 1267 } 1268 1269 bool 1270 spdk_mlx5_umr_implementer_is_registered(void) 1271 { 1272 return g_umr_implementer_registered; 1273 } 1274