1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. 3 * All rights reserved. 4 */ 5 6 #ifndef SPDK_MLX5_H 7 #define SPDK_MLX5_H 8 9 #include <infiniband/mlx5dv.h> 10 11 #include "spdk/tree.h" 12 13 #define SPDK_MLX5_DEV_MAX_NAME_LEN 64 14 15 /* API for low level PRM based mlx5 driver implementation. Some terminology: 16 * PRM - Programming Reference Manual 17 * QP - Queue Pair 18 * SQ - Submission Queue 19 * CQ - Completion Queue 20 * WQE - Work Queue Element 21 * WQEBB - Work Queue Element Build Block (64 bytes) 22 * CQE - Completion Queue Entry 23 * BSF - Byte Stream Format - part of UMR WQ which describes specific data properties such as encryption or signature 24 * UMR - User Memory Region 25 * DEK - Data Encryption Key 26 */ 27 28 #define SPDK_MLX5_VENDOR_ID_MELLANOX 0x2c9 29 30 struct spdk_mlx5_crypto_dek_legacy; 31 struct spdk_mlx5_crypto_keytag; 32 33 enum { 34 /** Error Completion Event - generate CQE on error for every CTRL segment, even one without CQ_UPDATE bit. 35 * Don't generate CQE in other cases. Default behaviour */ 36 SPDK_MLX5_WQE_CTRL_CE_CQ_ECE = 3 << 2, 37 /** Do not generate IBV_WC_WR_FLUSH_ERR for non-signaled CTRL segments. Completions are generated only for 38 * signaled (CQ_UPDATE) CTRL segments and the first error */ 39 SPDK_MLX5_WQE_CTRL_CE_CQ_NO_FLUSH_ERROR = 1 << 2, 40 /** Always generate CQE for CTRL segment WQE */ 41 SPDK_MLX5_WQE_CTRL_CE_CQ_UPDATE = MLX5_WQE_CTRL_CQ_UPDATE, 42 SPDK_MLX5_WQE_CTRL_CE_MASK = 3 << 2, 43 SPDK_MLX5_WQE_CTRL_SOLICITED = MLX5_WQE_CTRL_SOLICITED, 44 /** WQE starts execution only after all previous Read/Atomic WQEs complete */ 45 SPDK_MLX5_WQE_CTRL_FENCE = MLX5_WQE_CTRL_FENCE, 46 /** WQE starts execution after all local WQEs (memory operation, gather) complete */ 47 SPDK_MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE = MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE, 48 /** WQE starts execution only after all previous WQEs complete */ 49 SPDK_MLX5_WQE_CTRL_STRONG_ORDERING = 3 << 5, 50 }; 51 52 struct spdk_mlx5_crypto_dek_create_attr { 53 /* Data Encryption Key in binary form */ 54 char *dek; 55 /* Length of the dek */ 56 size_t dek_len; 57 }; 58 59 struct spdk_mlx5_cq; 60 struct spdk_mlx5_qp; 61 62 struct spdk_mlx5_cq_attr { 63 uint32_t cqe_cnt; 64 uint32_t cqe_size; 65 void *cq_context; 66 struct ibv_comp_channel *comp_channel; 67 int comp_vector; 68 }; 69 70 struct spdk_mlx5_qp_attr { 71 struct ibv_qp_cap cap; 72 bool sigall; 73 /* If set then CQ_UPDATE will be cleared for every ctrl WQE and only last ctrl WQE before ringing the doorbell 74 * will be updated with CQ_UPDATE flag */ 75 bool siglast; 76 }; 77 78 struct spdk_mlx5_cq_completion { 79 union { 80 uint64_t wr_id; 81 uint32_t mkey; /* applicable if status == MLX5_CQE_SYNDROME_SIGERR */ 82 }; 83 int status; 84 }; 85 86 struct spdk_mlx5_mkey_pool; 87 88 enum spdk_mlx5_mkey_pool_flags { 89 SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO = 1 << 0, 90 /* Max number of pools of different types */ 91 SPDK_MLX5_MKEY_POOL_FLAG_COUNT = 1, 92 }; 93 94 struct spdk_mlx5_mkey_pool_param { 95 uint32_t mkey_count; 96 uint32_t cache_per_thread; 97 /* enum spdk_mlx5_mkey_pool_flags */ 98 uint32_t flags; 99 }; 100 101 struct spdk_mlx5_mkey_pool_obj { 102 uint32_t mkey; 103 /* Determines which pool the mkey belongs to. See \ref spdk_mlx5_mkey_pool_flags */ 104 uint8_t pool_flag; 105 RB_ENTRY(spdk_mlx5_mkey_pool_obj) node; 106 struct { 107 uint32_t sigerr_count; 108 bool sigerr; 109 } sig; 110 }; 111 112 struct spdk_mlx5_umr_attr { 113 struct ibv_sge *sge; 114 uint32_t mkey; /* User Memory Region key to configure */ 115 uint32_t umr_len; 116 uint16_t sge_count; 117 }; 118 119 enum spdk_mlx5_encryption_order { 120 SPDK_MLX5_ENCRYPTION_ORDER_ENCRYPTED_WIRE_SIGNATURE = 0x0, 121 SPDK_MLX5_ENCRYPTION_ORDER_ENCRYPTED_MEMORY_SIGNATURE = 0x1, 122 SPDK_MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE = 0x2, 123 SPDK_MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY = 0x3, 124 }; 125 126 enum spdk_mlx5_block_size_selector { 127 SPDK_MLX5_BLOCK_SIZE_SELECTOR_RESERVED = 0, 128 SPDK_MLX5_BLOCK_SIZE_SELECTOR_512 = 1, 129 SPDK_MLX5_BLOCK_SIZE_SELECTOR_520 = 2, 130 SPDK_MLX5_BLOCK_SIZE_SELECTOR_4096 = 3, 131 SPDK_MLX5_BLOCK_SIZE_SELECTOR_4160 = 4, 132 }; 133 134 enum spdk_mlx5_crypto_key_tweak_mode { 135 SPDK_MLX5_CRYPTO_KEY_TWEAK_MODE_SIMPLE_LBA_BE = 0, 136 SPDK_MLX5_CRYPTO_KEY_TWEAK_MODE_SIMPLE_LBA_LE = 1, 137 }; 138 139 struct spdk_mlx5_crypto_dek_data { 140 /** low level devx obj id which represents the DEK */ 141 uint32_t dek_obj_id; 142 /** Crypto key tweak mode */ 143 enum spdk_mlx5_crypto_key_tweak_mode tweak_mode; 144 }; 145 146 struct spdk_mlx5_umr_crypto_attr { 147 uint8_t enc_order; /* see \ref enum spdk_mlx5_encryption_order */ 148 uint8_t bs_selector; /* see \ref enum spdk_mlx5_block_size_selector */ 149 uint8_t tweak_mode; /* see \ref enum spdk_mlx5_crypto_key_tweak_mode */ 150 /* Low level ID of the Data Encryption Key */ 151 uint32_t dek_obj_id; 152 uint64_t xts_iv; 153 uint64_t keytag; /* Must match DEK's keytag or 0 */ 154 }; 155 156 struct spdk_mlx5_device_crypto_caps { 157 bool wrapped_crypto_operational; 158 bool wrapped_crypto_going_to_commissioning; 159 bool wrapped_import_method_aes_xts; 160 bool single_block_le_tweak; 161 bool multi_block_be_tweak; 162 bool multi_block_le_tweak; 163 }; 164 165 struct spdk_mlx5_device_caps { 166 /* Content of this structure is valid only if crypto_supported is true */ 167 struct spdk_mlx5_device_crypto_caps crypto; 168 bool crypto_supported; 169 }; 170 171 /** 172 * Query device capabilities 173 * 174 * \param context Context of a device to query 175 * \param caps Device capabilities 176 * \return 0 on success, negated errno on failure. 177 */ 178 int spdk_mlx5_device_query_caps(struct ibv_context *context, struct spdk_mlx5_device_caps *caps); 179 180 /** 181 * Create Completion Queue 182 * 183 * \note: CQ and all associated qpairs must be accessed in scope of a single thread 184 * \note: CQ size must be enough to hold completions of all connected qpairs 185 * 186 * \param pd Protection Domain 187 * \param cq_attr Attributes to be used to create CQ 188 * \param cq_out Pointer created CQ 189 * \return 0 on success, negated errno on failure. \b cq_out is set only on success result 190 */ 191 int spdk_mlx5_cq_create(struct ibv_pd *pd, struct spdk_mlx5_cq_attr *cq_attr, 192 struct spdk_mlx5_cq **cq_out); 193 194 /** 195 * Destroy Completion Queue 196 * 197 * \param cq CQ created with \ref spdk_mlx5_cq_create 198 */ 199 int spdk_mlx5_cq_destroy(struct spdk_mlx5_cq *cq); 200 201 /** 202 * Create loopback qpair suitable for RDMA operations 203 * 204 * \param pd Protection Domain 205 * \param cq Completion Queue to bind QP to 206 * \param qp_attr Attributes to be used to create QP 207 * \param qp_out Pointer created QP 208 * \return 0 on success, negated errno on failure. \b qp_out is set only on success result 209 */ 210 int spdk_mlx5_qp_create(struct ibv_pd *pd, struct spdk_mlx5_cq *cq, 211 struct spdk_mlx5_qp_attr *qp_attr, struct spdk_mlx5_qp **qp_out); 212 213 /** 214 * Changes internal qpair state to error causing all unprocessed Work Requests to be completed with IBV_WC_WR_FLUSH_ERR 215 * status code. 216 * 217 * \param qp qpair pointer 218 * \return 0 on success, negated errno on failure 219 */ 220 int spdk_mlx5_qp_set_error_state(struct spdk_mlx5_qp *qp); 221 222 /** 223 * Get original verbs qp 224 * 225 * \param qp mlx5 qp 226 * \return Pointer to the underlying ibv_verbs qp 227 */ 228 struct ibv_qp *spdk_mlx5_qp_get_verbs_qp(struct spdk_mlx5_qp *qp); 229 230 /** 231 * Destroy qpair 232 * 233 * \param qp QP created with \ref spdk_mlx5_qp_create 234 */ 235 void spdk_mlx5_qp_destroy(struct spdk_mlx5_qp *qp); 236 237 /** 238 * Poll Completion Queue, save up to \b max_completions into \b comp array 239 * 240 * \param cq Completion Queue 241 * \param comp Array of completions to be filled by this function 242 * \param max_completions 243 * \return 0 on success, negated errno on failure 244 */ 245 int spdk_mlx5_cq_poll_completions(struct spdk_mlx5_cq *cq, 246 struct spdk_mlx5_cq_completion *comp, int max_completions); 247 248 /** 249 * Ring Send Queue doorbell, submits all previously posted WQEs to HW 250 * 251 * \param qp qpair pointer 252 */ 253 void spdk_mlx5_qp_complete_send(struct spdk_mlx5_qp *qp); 254 255 /** 256 * Submit RDMA_WRITE operations on the qpair 257 * 258 * \param qp qpair pointer 259 * \param sge Memory layout of the local data to be written 260 * \param sge_count Number of \b sge entries 261 * \param dstaddr Remote address to write \b sge to 262 * \param rkey Remote memory key 263 * \param wrid wrid which is returned in the CQE 264 * \param flags SPDK_MLX5_WQE_CTRL_CE_CQ_UPDATE to have a signaled completion; Any of SPDK_MLX5_WQE_CTRL_FENCE* or 0 265 * \return 0 on success, negated errno on failure 266 */ 267 int spdk_mlx5_qp_rdma_write(struct spdk_mlx5_qp *qp, struct ibv_sge *sge, uint32_t sge_count, 268 uint64_t dstaddr, uint32_t rkey, uint64_t wrid, uint32_t flags); 269 270 /** 271 * Submit RDMA_WRITE operations on the qpair 272 * 273 * \param qp qpair pointer 274 * \param sge Memory layout of the local buffers for reading remote data 275 * \param sge_count Number of \b sge entries 276 * \param dstaddr Remote address to read into \b sge 277 * \param rkey Remote memory key 278 * \param wrid wrid which is returned in the CQE 279 * \param flags SPDK_MLX5_WQE_CTRL_CE_CQ_UPDATE to have a signaled completion; Any of SPDK_MLX5_WQE_CTRL_FENCE* or 0 280 * \return 0 on success, negated errno on failure 281 */ 282 int spdk_mlx5_qp_rdma_read(struct spdk_mlx5_qp *qp, struct ibv_sge *sge, uint32_t sge_count, 283 uint64_t dstaddr, uint32_t rkey, uint64_t wrid, uint32_t flags); 284 285 /** 286 * Configure User Memory Region obtained using \ref spdk_mlx5_mkey_pool_get_bulk with crypto capabilities. 287 * 288 * Besides crypto capabilities, it allows to gather memory chunks into virtually contig (from the NIC point of view) 289 * memory space with start address 0. The user must ensure that \b qp's capacity is enough to perform this operation. 290 * It only works if the UMR pool was created with crypto capabilities. 291 * 292 * \param qp Qpair to be used for UMR configuration. If RDMA operation which references this UMR is used on the same \b qp 293 * then it is not necessary to wait for the UMR configuration to complete. Instead, first RDMA operation after UMR 294 * configuration must have flag SPDK_MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE set to 1 295 * \param umr_attr Common UMR attributes, describe memory layout 296 * \param crypto_attr Crypto UMR attributes 297 * \param wr_id wrid which is returned in the CQE 298 * \param flags SPDK_MLX5_WQE_CTRL_CE_CQ_UPDATE to have a signaled completion; Any of SPDK_MLX5_WQE_CTRL_FENCE* or 0 299 * \return 0 on success, negated errno on failure 300 */ 301 int spdk_mlx5_umr_configure_crypto(struct spdk_mlx5_qp *qp, struct spdk_mlx5_umr_attr *umr_attr, 302 struct spdk_mlx5_umr_crypto_attr *crypto_attr, uint64_t wr_id, uint32_t flags); 303 304 /** 305 * Return a NULL terminated array of devices which support crypto operation on Nvidia NICs 306 * 307 * \param dev_num The size of the array or 0 308 * \return Array of contexts. This array must be released with \b spdk_mlx5_crypto_devs_release 309 */ 310 struct ibv_context **spdk_mlx5_crypto_devs_get(int *dev_num); 311 312 /** 313 * Releases array of devices allocated by \b spdk_mlx5_crypto_devs_get 314 * 315 * \param rdma_devs Array of device to be released 316 */ 317 void spdk_mlx5_crypto_devs_release(struct ibv_context **rdma_devs); 318 319 /** 320 * Create a keytag which contains DEKs per each crypto device in the system 321 * 322 * \param attr Crypto attributes 323 * \param out Keytag 324 * \return 0 on success, negated errno of failure 325 */ 326 int spdk_mlx5_crypto_keytag_create(struct spdk_mlx5_crypto_dek_create_attr *attr, 327 struct spdk_mlx5_crypto_keytag **out); 328 329 /** 330 * Destroy a keytag created using \b spdk_mlx5_crypto_keytag_create 331 * 332 * \param keytag Keytag pointer 333 */ 334 void spdk_mlx5_crypto_keytag_destroy(struct spdk_mlx5_crypto_keytag *keytag); 335 336 /** 337 * Get Data Encryption Key data 338 * 339 * \param keytag Keytag with DEKs 340 * \param pd Protection Domain which is going to be used to register UMR. 341 * \param dek_obj_id Low level DEK ID, can be used to configure crypto UMR 342 * \param data DEK data to be filled by this function 343 * \return 0 on success, negated errno on failure 344 */ 345 int spdk_mlx5_crypto_get_dek_data(struct spdk_mlx5_crypto_keytag *keytag, struct ibv_pd *pd, 346 struct spdk_mlx5_crypto_dek_data *data); 347 348 /** 349 * Specify which devices are allowed to be used for crypto operation. 350 * 351 * If the user doesn't call this function then all devices which support crypto will be used. 352 * This function copies devices names. In order to free allocated memory, the user must call 353 * this function with either NULL \b dev_names or with \b devs_count equal 0. This way can also 354 * be used to allow all devices. 355 * 356 * Subsequent calls with non-NULL \b dev_names and non-zero \b devs_count current copied dev_names array. 357 * 358 * This function is not thread safe. 359 * 360 * \param dev_names Array of devices names which are allowed to be used for crypto operations 361 * \param devs_count Size of \b devs_count array 362 * \return 0 on success, negated errno on failure 363 */ 364 int spdk_mlx5_crypto_devs_allow(const char *const dev_names[], size_t devs_count); 365 366 /** 367 * Creates a pool of memory keys for a given \b PD. If params::flags has SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO enabled, 368 * then a device associated with PD must support crypto operations. 369 * 370 * Can be called several times for different PDs. Has no effect if a pool for \b PD with the same \b flags already exists 371 * 372 * \param params Parameter of the memory pool 373 * \param pd Protection Domain 374 * \return 0 on success, errno on failure 375 */ 376 int spdk_mlx5_mkey_pool_init(struct spdk_mlx5_mkey_pool_param *params, struct ibv_pd *pd); 377 378 /** 379 * Destroy mkey pools with the given \b flags and \b pd which was created by \ref spdk_mlx5_mkey_pool_init. 380 * 381 * The pool reference must be released by \ref spdk_mlx5_mkey_pool_put_ref before calling this function. 382 * 383 * \param pd Protection Domain 384 * \param flags Specifies type of the pool to delete. 385 * \return 0 on success, negated errno on failure 386 */ 387 int spdk_mlx5_mkey_pool_destroy(uint32_t flags, struct ibv_pd *pd); 388 389 /** 390 * Get a reference to mkey pool specified by PD, increment internal reference counter. 391 * 392 * \param pd PD to get a mkey pool for 393 * \param flags Required mkey pool flags, see \ref enum spdk_mlx5_mkey_pool_flags 394 * \return Pointer to the mkey pool on success or NULL on error 395 */ 396 struct spdk_mlx5_mkey_pool *spdk_mlx5_mkey_pool_get_ref(struct ibv_pd *pd, uint32_t flags); 397 398 /** 399 * Put the mkey pool reference. 400 * 401 * The pool is NOT destroyed if even reference counter reaches 0 402 * 403 * \param pool Mkey pool pointer 404 */ 405 void spdk_mlx5_mkey_pool_put_ref(struct spdk_mlx5_mkey_pool *pool); 406 407 /** 408 * Get several mkeys from the pool 409 * 410 * \param pool mkey pool 411 * \param mkeys array of mkey pointers to be filled by this function 412 * \param mkeys_count number of mkeys to get from the pool 413 * \return 0 on success, errno on failure 414 */ 415 int spdk_mlx5_mkey_pool_get_bulk(struct spdk_mlx5_mkey_pool *pool, 416 struct spdk_mlx5_mkey_pool_obj **mkeys, uint32_t mkeys_count); 417 418 /** 419 * Return mkeys to the pool 420 * 421 * \param pool mkey pool 422 * \param mkeys array of mkey pointers to be returned to the pool 423 * \param mkeys_count number of mkeys to be returned to the pool 424 */ 425 void spdk_mlx5_mkey_pool_put_bulk(struct spdk_mlx5_mkey_pool *pool, 426 struct spdk_mlx5_mkey_pool_obj **mkeys, uint32_t mkeys_count); 427 428 #endif /* SPDK_MLX5_H */ 429