1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 */ 4 5 #include <rdma/rdma_cma.h> 6 #include <infiniband/verbs.h> 7 #include <infiniband/mlx5dv.h> 8 9 #include "spdk/stdinc.h" 10 #include "spdk/queue.h" 11 #include "spdk/log.h" 12 #include "spdk/likely.h" 13 #include "spdk/util.h" 14 #include "spdk_internal/mlx5.h" 15 #include "spdk_internal/rdma_utils.h" 16 #include "mlx5_ifc.h" 17 18 #define MLX5_VENDOR_ID_MELLANOX 0x2c9 19 20 /* Plaintext key sizes */ 21 /* 64b keytag */ 22 #define SPDK_MLX5_AES_XTS_KEYTAG_SIZE 8 23 /* key1_128b + key2_128b */ 24 #define SPDK_MLX5_AES_XTS_128_DEK_BYTES 32 25 /* key1_256b + key2_256b */ 26 #define SPDK_MLX5_AES_XTS_256_DEK_BYTES 64 27 /* key1_128b + key2_128b + 64b_keytag */ 28 #define SPDK_MLX5_AES_XTS_128_DEK_BYTES_WITH_KEYTAG (SPDK_MLX5_AES_XTS_128_DEK_BYTES + SPDK_MLX5_AES_XTS_KEYTAG_SIZE) 29 /* key1_256b + key2_256b + 64b_keytag */ 30 #define SPDK_MLX5_AES_XTS_256_DEK_BYTES_WITH_KEYTAG (SPDK_MLX5_AES_XTS_256_DEK_BYTES + SPDK_MLX5_AES_XTS_KEYTAG_SIZE) 31 32 static char **g_allowed_devices; 33 static size_t g_allowed_devices_count; 34 35 struct spdk_mlx5_crypto_dek { 36 struct mlx5dv_dek *dek_obj; 37 struct ibv_pd *pd; 38 struct ibv_context *context; 39 }; 40 41 struct spdk_mlx5_crypto_keytag { 42 struct spdk_mlx5_crypto_dek *deks; 43 uint32_t deks_num; 44 bool has_keytag; 45 char keytag[8]; 46 }; 47 48 static void 49 mlx5_crypto_devs_free(void) 50 { 51 size_t i; 52 53 if (!g_allowed_devices) { 54 return; 55 } 56 57 for (i = 0; i < g_allowed_devices_count; i++) { 58 free(g_allowed_devices[i]); 59 } 60 free(g_allowed_devices); 61 g_allowed_devices = NULL; 62 g_allowed_devices_count = 0; 63 } 64 65 static bool 66 mlx5_crypto_dev_allowed(const char *dev) 67 { 68 size_t i; 69 70 if (!g_allowed_devices || !g_allowed_devices_count) { 71 return true; 72 } 73 74 for (i = 0; i < g_allowed_devices_count; i++) { 75 if (strcmp(g_allowed_devices[i], dev) == 0) { 76 return true; 77 } 78 } 79 80 return false; 81 } 82 83 int 84 spdk_mlx5_crypto_devs_allow(const char *const dev_names[], size_t devs_count) 85 { 86 size_t i; 87 88 mlx5_crypto_devs_free(); 89 90 if (!dev_names || !devs_count) { 91 return 0; 92 } 93 94 g_allowed_devices = calloc(devs_count, sizeof(char *)); 95 if (!g_allowed_devices) { 96 return -ENOMEM; 97 } 98 for (i = 0; i < devs_count; i++) { 99 g_allowed_devices[i] = strndup(dev_names[i], SPDK_MLX5_DEV_MAX_NAME_LEN); 100 if (!g_allowed_devices[i]) { 101 mlx5_crypto_devs_free(); 102 return -ENOMEM; 103 } 104 g_allowed_devices_count++; 105 } 106 107 return 0; 108 } 109 110 struct ibv_context ** 111 spdk_mlx5_crypto_devs_get(int *dev_num) 112 { 113 struct ibv_context **rdma_devs, **rdma_devs_out = NULL, *dev; 114 struct ibv_device_attr dev_attr; 115 struct ibv_port_attr port_attr; 116 struct spdk_mlx5_device_caps dev_caps; 117 uint8_t in[DEVX_ST_SZ_BYTES(query_nic_vport_context_in)]; 118 uint8_t out[DEVX_ST_SZ_BYTES(query_nic_vport_context_out)]; 119 uint8_t devx_v; 120 int num_rdma_devs = 0, i, rc; 121 int num_crypto_devs = 0; 122 123 /* query all devices, save mlx5 with crypto support */ 124 rdma_devs = rdma_get_devices(&num_rdma_devs); 125 if (!rdma_devs || !num_rdma_devs) { 126 *dev_num = 0; 127 return NULL; 128 } 129 130 rdma_devs_out = calloc(num_rdma_devs + 1, sizeof(*rdma_devs_out)); 131 if (!rdma_devs_out) { 132 SPDK_ERRLOG("Memory allocation failed\n"); 133 return NULL; 134 } 135 136 for (i = 0; i < num_rdma_devs; i++) { 137 dev = rdma_devs[i]; 138 rc = ibv_query_device(dev, &dev_attr); 139 if (rc) { 140 SPDK_ERRLOG("Failed to query dev %s, skipping\n", dev->device->name); 141 continue; 142 } 143 if (dev_attr.vendor_id != MLX5_VENDOR_ID_MELLANOX) { 144 SPDK_DEBUGLOG(mlx5, "dev %s is not Mellanox device, skipping\n", dev->device->name); 145 continue; 146 } 147 148 if (!mlx5_crypto_dev_allowed(dev->device->name)) { 149 continue; 150 } 151 152 rc = ibv_query_port(dev, 1, &port_attr); 153 if (rc) { 154 SPDK_ERRLOG("Failed to query port attributes for device %s, rc %d\n", dev->device->name, rc); 155 continue; 156 } 157 158 if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) { 159 /* Port may be ethernet but roce is still disabled */ 160 memset(in, 0, sizeof(in)); 161 memset(out, 0, sizeof(out)); 162 DEVX_SET(query_nic_vport_context_in, in, opcode, MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT); 163 rc = mlx5dv_devx_general_cmd(dev, in, sizeof(in), out, sizeof(out)); 164 if (rc) { 165 SPDK_ERRLOG("Failed to get VPORT context for device %s. Assuming ROCE is disabled\n", 166 dev->device->name); 167 continue; 168 } 169 170 devx_v = DEVX_GET(query_nic_vport_context_out, out, nic_vport_context.roce_en); 171 if (!devx_v) { 172 SPDK_ERRLOG("Device %s, RoCE disabled\n", dev->device->name); 173 continue; 174 } 175 } 176 177 memset(&dev_caps, 0, sizeof(dev_caps)); 178 rc = spdk_mlx5_device_query_caps(dev, &dev_caps); 179 if (rc) { 180 SPDK_ERRLOG("Failed to query mlx5 dev %s, skipping\n", dev->device->name); 181 continue; 182 } 183 if (!dev_caps.crypto_supported) { 184 SPDK_WARNLOG("dev %s crypto engine doesn't support crypto\n", dev->device->name); 185 continue; 186 } 187 if (!(dev_caps.crypto.single_block_le_tweak || dev_caps.crypto.multi_block_le_tweak || 188 dev_caps.crypto.multi_block_be_tweak)) { 189 SPDK_WARNLOG("dev %s crypto engine doesn't support AES_XTS\n", dev->device->name); 190 continue; 191 } 192 if (dev_caps.crypto.wrapped_import_method_aes_xts) { 193 SPDK_WARNLOG("dev %s uses wrapped import method which is not supported by mlx5 lib\n", 194 dev->device->name); 195 continue; 196 } 197 198 rdma_devs_out[num_crypto_devs++] = dev; 199 } 200 201 if (!num_crypto_devs) { 202 SPDK_DEBUGLOG(mlx5, "Found no mlx5 crypto devices\n"); 203 goto err_out; 204 } 205 206 rdma_free_devices(rdma_devs); 207 *dev_num = num_crypto_devs; 208 209 return rdma_devs_out; 210 211 err_out: 212 free(rdma_devs_out); 213 rdma_free_devices(rdma_devs); 214 *dev_num = 0; 215 return NULL; 216 } 217 218 void 219 spdk_mlx5_crypto_devs_release(struct ibv_context **rdma_devs) 220 { 221 if (rdma_devs) { 222 free(rdma_devs); 223 } 224 } 225 226 int 227 spdk_mlx5_device_query_caps(struct ibv_context *context, struct spdk_mlx5_device_caps *caps) 228 { 229 uint16_t opmod = MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE | 230 HCA_CAP_OPMOD_GET_CUR; 231 uint32_t out[DEVX_ST_SZ_DW(query_hca_cap_out)] = {}; 232 uint32_t in[DEVX_ST_SZ_DW(query_hca_cap_in)] = {}; 233 int rc; 234 235 DEVX_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 236 DEVX_SET(query_hca_cap_in, in, op_mod, opmod); 237 238 rc = mlx5dv_devx_general_cmd(context, in, sizeof(in), out, sizeof(out)); 239 if (rc) { 240 return rc; 241 } 242 243 caps->crypto_supported = DEVX_GET(query_hca_cap_out, out, capability.cmd_hca_cap.crypto); 244 if (!caps->crypto_supported) { 245 return 0; 246 } 247 248 caps->crypto.single_block_le_tweak = DEVX_GET(query_hca_cap_out, 249 out, capability.cmd_hca_cap.aes_xts_single_block_le_tweak); 250 caps->crypto.multi_block_be_tweak = DEVX_GET(query_hca_cap_out, out, 251 capability.cmd_hca_cap.aes_xts_multi_block_be_tweak); 252 caps->crypto.multi_block_le_tweak = DEVX_GET(query_hca_cap_out, out, 253 capability.cmd_hca_cap.aes_xts_multi_block_le_tweak); 254 255 opmod = MLX5_SET_HCA_CAP_OP_MOD_CRYPTO | HCA_CAP_OPMOD_GET_CUR; 256 memset(&out, 0, sizeof(out)); 257 memset(&in, 0, sizeof(in)); 258 259 DEVX_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 260 DEVX_SET(query_hca_cap_in, in, op_mod, opmod); 261 262 rc = mlx5dv_devx_general_cmd(context, in, sizeof(in), out, sizeof(out)); 263 if (rc) { 264 return rc; 265 } 266 267 caps->crypto.wrapped_crypto_operational = DEVX_GET(query_hca_cap_out, out, 268 capability.crypto_caps.wrapped_crypto_operational); 269 caps->crypto.wrapped_crypto_going_to_commissioning = DEVX_GET(query_hca_cap_out, out, 270 capability.crypto_caps .wrapped_crypto_going_to_commissioning); 271 caps->crypto.wrapped_import_method_aes_xts = (DEVX_GET(query_hca_cap_out, out, 272 capability.crypto_caps.wrapped_import_method) & 273 MLX5_CRYPTO_CAPS_WRAPPED_IMPORT_METHOD_AES) != 0; 274 275 return 0; 276 } 277 278 void 279 spdk_mlx5_crypto_keytag_destroy(struct spdk_mlx5_crypto_keytag *keytag) 280 { 281 struct spdk_mlx5_crypto_dek *dek; 282 uint32_t i; 283 284 if (!keytag) { 285 return; 286 } 287 288 for (i = 0; i < keytag->deks_num; i++) { 289 dek = &keytag->deks[i]; 290 if (dek->dek_obj) { 291 mlx5dv_dek_destroy(dek->dek_obj); 292 } 293 if (dek->pd) { 294 spdk_rdma_utils_put_pd(dek->pd); 295 } 296 } 297 spdk_memset_s(keytag->keytag, sizeof(keytag->keytag), 0, sizeof(keytag->keytag)); 298 free(keytag->deks); 299 free(keytag); 300 } 301 302 int 303 spdk_mlx5_crypto_keytag_create(struct spdk_mlx5_crypto_dek_create_attr *attr, 304 struct spdk_mlx5_crypto_keytag **out) 305 { 306 struct spdk_mlx5_crypto_dek *dek; 307 struct spdk_mlx5_crypto_keytag *keytag; 308 struct ibv_context **devs; 309 struct mlx5dv_dek_init_attr init_attr = {}; 310 struct mlx5dv_dek_attr query_attr; 311 int num_devs = 0, i, rc; 312 bool has_keytag; 313 314 315 if (!attr || !attr->dek) { 316 return -EINVAL; 317 } 318 switch (attr->dek_len) { 319 case SPDK_MLX5_AES_XTS_128_DEK_BYTES_WITH_KEYTAG: 320 init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_128; 321 has_keytag = true; 322 SPDK_DEBUGLOG(mlx5, "128b AES_XTS with keytag\n"); 323 break; 324 case SPDK_MLX5_AES_XTS_256_DEK_BYTES_WITH_KEYTAG: 325 init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_256; 326 has_keytag = true; 327 SPDK_DEBUGLOG(mlx5, "256b AES_XTS with keytag\n"); 328 break; 329 case SPDK_MLX5_AES_XTS_128_DEK_BYTES: 330 init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_128; 331 has_keytag = false; 332 SPDK_DEBUGLOG(mlx5, "128b AES_XTS\n"); 333 break; 334 case SPDK_MLX5_AES_XTS_256_DEK_BYTES: 335 init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_256; 336 has_keytag = false; 337 SPDK_DEBUGLOG(mlx5, "256b AES_XTS\n"); 338 break; 339 default: 340 SPDK_ERRLOG("Invalid key length %zu. The following keys are supported:\n" 341 "128b key + key2, %u bytes;\n" 342 "256b key + key2, %u bytes\n" 343 "128b key + key2 + keytag, %u bytes\n" 344 "256b lye + key2 + keytag, %u bytes\n", 345 attr->dek_len, SPDK_MLX5_AES_XTS_128_DEK_BYTES, MLX5DV_CRYPTO_KEY_SIZE_256, 346 SPDK_MLX5_AES_XTS_128_DEK_BYTES_WITH_KEYTAG, SPDK_MLX5_AES_XTS_256_DEK_BYTES_WITH_KEYTAG); 347 return -EINVAL; 348 } 349 350 devs = spdk_mlx5_crypto_devs_get(&num_devs); 351 if (!devs || !num_devs) { 352 SPDK_DEBUGLOG(mlx5, "No crypto devices found\n"); 353 return -ENOTSUP; 354 } 355 356 keytag = calloc(1, sizeof(*keytag)); 357 if (!keytag) { 358 SPDK_ERRLOG("Memory allocation failed\n"); 359 spdk_mlx5_crypto_devs_release(devs); 360 return -ENOMEM; 361 } 362 keytag->deks = calloc(num_devs, sizeof(struct spdk_mlx5_crypto_dek)); 363 if (!keytag->deks) { 364 SPDK_ERRLOG("Memory allocation failed\n"); 365 spdk_mlx5_crypto_devs_release(devs); 366 free(keytag); 367 return -ENOMEM; 368 } 369 370 for (i = 0; i < num_devs; i++) { 371 keytag->deks_num++; 372 dek = &keytag->deks[i]; 373 dek->pd = spdk_rdma_utils_get_pd(devs[i]); 374 if (!dek->pd) { 375 SPDK_ERRLOG("Failed to get PD on device %s\n", devs[i]->device->name); 376 rc = -EINVAL; 377 goto err_out; 378 } 379 dek->context = devs[i]; 380 381 init_attr.pd = dek->pd; 382 init_attr.has_keytag = has_keytag; 383 init_attr.key_purpose = MLX5DV_CRYPTO_KEY_PURPOSE_AES_XTS; 384 init_attr.comp_mask = MLX5DV_DEK_INIT_ATTR_CRYPTO_LOGIN; 385 init_attr.crypto_login = NULL; 386 memcpy(init_attr.key, attr->dek, attr->dek_len); 387 388 dek->dek_obj = mlx5dv_dek_create(dek->context, &init_attr); 389 spdk_memset_s(init_attr.key, sizeof(init_attr.key), 0, sizeof(init_attr.key)); 390 if (!dek->dek_obj) { 391 SPDK_ERRLOG("mlx5dv_dek_create failed on dev %s, errno %d\n", dek->context->device->name, errno); 392 rc = -EINVAL; 393 goto err_out; 394 } 395 396 memset(&query_attr, 0, sizeof(query_attr)); 397 rc = mlx5dv_dek_query(dek->dek_obj, &query_attr); 398 if (rc) { 399 SPDK_ERRLOG("Failed to query DEK on dev %s, rc %d\n", dek->context->device->name, rc); 400 goto err_out; 401 } 402 if (query_attr.state != MLX5DV_DEK_STATE_READY) { 403 SPDK_ERRLOG("DEK on dev %s state %d\n", dek->context->device->name, query_attr.state); 404 rc = -EINVAL; 405 goto err_out; 406 } 407 } 408 409 if (has_keytag) { 410 /* Save keytag, it will be used to configure crypto MKEY */ 411 keytag->has_keytag = true; 412 memcpy(keytag->keytag, attr->dek + attr->dek_len - SPDK_MLX5_AES_XTS_KEYTAG_SIZE, 413 SPDK_MLX5_AES_XTS_KEYTAG_SIZE); 414 } 415 416 spdk_mlx5_crypto_devs_release(devs); 417 *out = keytag; 418 419 return 0; 420 421 err_out: 422 spdk_mlx5_crypto_keytag_destroy(keytag); 423 spdk_mlx5_crypto_devs_release(devs); 424 425 return rc; 426 } 427 428 static inline struct spdk_mlx5_crypto_dek * 429 mlx5_crypto_get_dek_by_pd(struct spdk_mlx5_crypto_keytag *keytag, struct ibv_pd *pd) 430 { 431 struct spdk_mlx5_crypto_dek *dek; 432 uint32_t i; 433 434 for (i = 0; i < keytag->deks_num; i++) { 435 dek = &keytag->deks[i]; 436 if (dek->pd == pd) { 437 return dek; 438 } 439 } 440 441 return NULL; 442 } 443 444 int 445 spdk_mlx5_crypto_set_attr(struct mlx5dv_crypto_attr *attr_out, 446 struct spdk_mlx5_crypto_keytag *keytag, struct ibv_pd *pd, 447 uint32_t block_size, uint64_t iv, bool encrypt_on_tx) 448 { 449 struct spdk_mlx5_crypto_dek *dek; 450 enum mlx5dv_block_size bs; 451 452 dek = mlx5_crypto_get_dek_by_pd(keytag, pd); 453 if (spdk_unlikely(!dek)) { 454 SPDK_ERRLOG("No DEK for pd %p (dev %s)\n", pd, pd->context->device->name); 455 return -EINVAL; 456 } 457 458 switch (block_size) { 459 case 512: 460 bs = MLX5DV_BLOCK_SIZE_512; 461 break; 462 case 520: 463 bs = MLX5DV_BLOCK_SIZE_520; 464 break; 465 case 4048: 466 bs = MLX5DV_BLOCK_SIZE_4048; 467 break; 468 case 4096: 469 bs = MLX5DV_BLOCK_SIZE_4096; 470 break; 471 case 4160: 472 bs = MLX5DV_BLOCK_SIZE_4160; 473 break; 474 default: 475 SPDK_ERRLOG("Unsupported block size %u\n", block_size); 476 return -EINVAL; 477 } 478 479 memset(attr_out, 0, sizeof(*attr_out)); 480 attr_out->dek = dek->dek_obj; 481 attr_out->crypto_standard = MLX5DV_CRYPTO_STANDARD_AES_XTS; 482 attr_out->data_unit_size = bs; 483 attr_out->encrypt_on_tx = encrypt_on_tx; 484 memcpy(attr_out->initial_tweak, &iv, sizeof(iv)); 485 if (keytag->has_keytag) { 486 memcpy(attr_out->keytag, keytag->keytag, sizeof(keytag->keytag)); 487 } 488 489 return 0; 490 } 491 492 SPDK_LOG_REGISTER_COMPONENT(mlx5) 493