1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 */ 4 5 #include <rdma/rdma_cma.h> 6 #include <infiniband/verbs.h> 7 #include <infiniband/mlx5dv.h> 8 9 #include "spdk/stdinc.h" 10 #include "spdk/queue.h" 11 #include "spdk/log.h" 12 #include "spdk/likely.h" 13 #include "spdk/util.h" 14 #include "spdk_internal/mlx5.h" 15 #include "spdk_internal/rdma_utils.h" 16 17 #define MLX5_VENDOR_ID_MELLANOX 0x2c9 18 19 /* Plaintext key sizes */ 20 /* 64b keytag */ 21 #define SPDK_MLX5_AES_XTS_KEYTAG_SIZE 8 22 /* key1_128b + key2_128b */ 23 #define SPDK_MLX5_AES_XTS_128_DEK_BYTES 32 24 /* key1_256b + key2_256b */ 25 #define SPDK_MLX5_AES_XTS_256_DEK_BYTES 64 26 /* key1_128b + key2_128b + 64b_keytag */ 27 #define SPDK_MLX5_AES_XTS_128_DEK_BYTES_WITH_KEYTAG (SPDK_MLX5_AES_XTS_128_DEK_BYTES + SPDK_MLX5_AES_XTS_KEYTAG_SIZE) 28 /* key1_256b + key2_256b + 64b_keytag */ 29 #define SPDK_MLX5_AES_XTS_256_DEK_BYTES_WITH_KEYTAG (SPDK_MLX5_AES_XTS_256_DEK_BYTES + SPDK_MLX5_AES_XTS_KEYTAG_SIZE) 30 31 static char **g_allowed_devices; 32 static size_t g_allowed_devices_count; 33 34 struct spdk_mlx5_crypto_dek { 35 struct mlx5dv_dek *dek_obj; 36 struct ibv_pd *pd; 37 struct ibv_context *context; 38 }; 39 40 struct spdk_mlx5_crypto_keytag { 41 struct spdk_mlx5_crypto_dek *deks; 42 uint32_t deks_num; 43 bool has_keytag; 44 char keytag[8]; 45 }; 46 47 static void 48 mlx5_crypto_devs_free(void) 49 { 50 size_t i; 51 52 if (!g_allowed_devices) { 53 return; 54 } 55 56 for (i = 0; i < g_allowed_devices_count; i++) { 57 free(g_allowed_devices[i]); 58 } 59 free(g_allowed_devices); 60 g_allowed_devices = NULL; 61 g_allowed_devices_count = 0; 62 } 63 64 static bool 65 mlx5_crypto_dev_allowed(const char *dev) 66 { 67 size_t i; 68 69 if (!g_allowed_devices || !g_allowed_devices_count) { 70 return true; 71 } 72 73 for (i = 0; i < g_allowed_devices_count; i++) { 74 if (strcmp(g_allowed_devices[i], dev) == 0) { 75 return true; 76 } 77 } 78 79 return false; 80 } 81 82 int 83 spdk_mlx5_crypto_devs_allow(const char *const dev_names[], size_t devs_count) 84 { 85 size_t i; 86 87 mlx5_crypto_devs_free(); 88 89 if (!dev_names || !devs_count) { 90 return 0; 91 } 92 93 g_allowed_devices = calloc(devs_count, sizeof(char *)); 94 if (!g_allowed_devices) { 95 return -ENOMEM; 96 } 97 for (i = 0; i < devs_count; i++) { 98 g_allowed_devices[i] = strndup(dev_names[i], SPDK_MLX5_DEV_MAX_NAME_LEN); 99 if (!g_allowed_devices[i]) { 100 mlx5_crypto_devs_free(); 101 return -ENOMEM; 102 } 103 g_allowed_devices_count++; 104 } 105 106 return 0; 107 } 108 109 struct ibv_context ** 110 spdk_mlx5_crypto_devs_get(int *dev_num) 111 { 112 struct ibv_context **rdma_devs, **rdma_devs_out = NULL, *dev; 113 struct ibv_device_attr dev_attr; 114 struct mlx5dv_context dv_dev_attr; 115 int num_rdma_devs = 0, i, rc; 116 int num_crypto_devs = 0; 117 118 /* query all devices, save mlx5 with crypto support */ 119 rdma_devs = rdma_get_devices(&num_rdma_devs); 120 if (!rdma_devs || !num_rdma_devs) { 121 *dev_num = 0; 122 return NULL; 123 } 124 125 rdma_devs_out = calloc(num_rdma_devs + 1, sizeof(*rdma_devs_out)); 126 if (!rdma_devs_out) { 127 SPDK_ERRLOG("Memory allocation failed\n"); 128 return NULL; 129 } 130 131 for (i = 0; i < num_rdma_devs; i++) { 132 dev = rdma_devs[i]; 133 134 rc = ibv_query_device(dev, &dev_attr); 135 if (rc) { 136 SPDK_ERRLOG("Failed to query dev %s, skipping\n", dev->device->name); 137 continue; 138 } 139 if (dev_attr.vendor_id != MLX5_VENDOR_ID_MELLANOX) { 140 SPDK_DEBUGLOG(mlx5, "dev %s is not Mellanox device, skipping\n", dev->device->name); 141 continue; 142 } 143 144 if (!mlx5_crypto_dev_allowed(dev->device->name)) { 145 continue; 146 } 147 148 memset(&dv_dev_attr, 0, sizeof(dv_dev_attr)); 149 dv_dev_attr.comp_mask |= MLX5DV_CONTEXT_MASK_CRYPTO_OFFLOAD; 150 rc = mlx5dv_query_device(dev, &dv_dev_attr); 151 if (rc) { 152 SPDK_ERRLOG("Failed to query mlx5 dev %s, skipping\n", dev->device->name); 153 continue; 154 } 155 if (!(dv_dev_attr.crypto_caps.flags & MLX5DV_CRYPTO_CAPS_CRYPTO)) { 156 SPDK_DEBUGLOG(mlx5, "dev %s crypto engine doesn't support crypto, skipping\n", dev->device->name); 157 continue; 158 } 159 if (!(dv_dev_attr.crypto_caps.crypto_engines & (MLX5DV_CRYPTO_ENGINES_CAP_AES_XTS | 160 MLX5DV_CRYPTO_ENGINES_CAP_AES_XTS_SINGLE_BLOCK))) { 161 SPDK_DEBUGLOG(mlx5, "dev %s crypto engine doesn't support AES_XTS, skipping\n", dev->device->name); 162 continue; 163 } 164 if (dv_dev_attr.crypto_caps.wrapped_import_method & 165 MLX5DV_CRYPTO_WRAPPED_IMPORT_METHOD_CAP_AES_XTS) { 166 SPDK_WARNLOG("dev %s uses wrapped import method (0x%x) which is not supported by mlx5 accel module\n", 167 dev->device->name, dv_dev_attr.crypto_caps.wrapped_import_method); 168 continue; 169 } 170 171 SPDK_NOTICELOG("Crypto dev %s\n", dev->device->name); 172 rdma_devs_out[num_crypto_devs++] = dev; 173 } 174 175 if (!num_crypto_devs) { 176 SPDK_DEBUGLOG(mlx5, "Found no mlx5 crypto devices\n"); 177 goto err_out; 178 } 179 180 rdma_free_devices(rdma_devs); 181 *dev_num = num_crypto_devs; 182 183 return rdma_devs_out; 184 185 err_out: 186 free(rdma_devs_out); 187 rdma_free_devices(rdma_devs); 188 *dev_num = 0; 189 return NULL; 190 } 191 192 void 193 spdk_mlx5_crypto_devs_release(struct ibv_context **rdma_devs) 194 { 195 if (rdma_devs) { 196 free(rdma_devs); 197 } 198 } 199 200 void 201 spdk_mlx5_crypto_keytag_destroy(struct spdk_mlx5_crypto_keytag *keytag) 202 { 203 struct spdk_mlx5_crypto_dek *dek; 204 uint32_t i; 205 206 if (!keytag) { 207 return; 208 } 209 210 for (i = 0; i < keytag->deks_num; i++) { 211 dek = &keytag->deks[i]; 212 if (dek->dek_obj) { 213 mlx5dv_dek_destroy(dek->dek_obj); 214 } 215 if (dek->pd) { 216 spdk_rdma_utils_put_pd(dek->pd); 217 } 218 } 219 spdk_memset_s(keytag->keytag, sizeof(keytag->keytag), 0, sizeof(keytag->keytag)); 220 free(keytag->deks); 221 free(keytag); 222 } 223 224 int 225 spdk_mlx5_crypto_keytag_create(struct spdk_mlx5_crypto_dek_create_attr *attr, 226 struct spdk_mlx5_crypto_keytag **out) 227 { 228 struct spdk_mlx5_crypto_dek *dek; 229 struct spdk_mlx5_crypto_keytag *keytag; 230 struct ibv_context **devs; 231 struct mlx5dv_dek_init_attr init_attr = {}; 232 struct mlx5dv_dek_attr query_attr; 233 int num_devs = 0, i, rc; 234 bool has_keytag; 235 236 237 if (!attr || !attr->dek) { 238 return -EINVAL; 239 } 240 switch (attr->dek_len) { 241 case SPDK_MLX5_AES_XTS_128_DEK_BYTES_WITH_KEYTAG: 242 init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_128; 243 has_keytag = true; 244 SPDK_DEBUGLOG(mlx5, "128b AES_XTS with keytag\n"); 245 break; 246 case SPDK_MLX5_AES_XTS_256_DEK_BYTES_WITH_KEYTAG: 247 init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_256; 248 has_keytag = true; 249 SPDK_DEBUGLOG(mlx5, "256b AES_XTS with keytag\n"); 250 break; 251 case SPDK_MLX5_AES_XTS_128_DEK_BYTES: 252 init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_128; 253 has_keytag = false; 254 SPDK_DEBUGLOG(mlx5, "128b AES_XTS\n"); 255 break; 256 case SPDK_MLX5_AES_XTS_256_DEK_BYTES: 257 init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_256; 258 has_keytag = false; 259 SPDK_DEBUGLOG(mlx5, "256b AES_XTS\n"); 260 break; 261 default: 262 SPDK_ERRLOG("Invalid key length %zu. The following keys are supported:\n" 263 "128b key + key2, %u bytes;\n" 264 "256b key + key2, %u bytes\n" 265 "128b key + key2 + keytag, %u bytes\n" 266 "256b lye + key2 + keytag, %u bytes\n", 267 attr->dek_len, SPDK_MLX5_AES_XTS_128_DEK_BYTES, MLX5DV_CRYPTO_KEY_SIZE_256, 268 SPDK_MLX5_AES_XTS_128_DEK_BYTES_WITH_KEYTAG, SPDK_MLX5_AES_XTS_256_DEK_BYTES_WITH_KEYTAG); 269 return -EINVAL; 270 } 271 272 devs = spdk_mlx5_crypto_devs_get(&num_devs); 273 if (!devs || !num_devs) { 274 SPDK_DEBUGLOG(mlx5, "No crypto devices found\n"); 275 return -ENOTSUP; 276 } 277 278 keytag = calloc(1, sizeof(*keytag)); 279 if (!keytag) { 280 SPDK_ERRLOG("Memory allocation failed\n"); 281 spdk_mlx5_crypto_devs_release(devs); 282 return -ENOMEM; 283 } 284 keytag->deks = calloc(num_devs, sizeof(struct spdk_mlx5_crypto_dek)); 285 if (!keytag->deks) { 286 SPDK_ERRLOG("Memory allocation failed\n"); 287 spdk_mlx5_crypto_devs_release(devs); 288 free(keytag); 289 return -ENOMEM; 290 } 291 292 for (i = 0; i < num_devs; i++) { 293 keytag->deks_num++; 294 dek = &keytag->deks[i]; 295 dek->pd = spdk_rdma_utils_get_pd(devs[i]); 296 if (!dek->pd) { 297 SPDK_ERRLOG("Failed to get PD on device %s\n", devs[i]->device->name); 298 rc = -EINVAL; 299 goto err_out; 300 } 301 dek->context = devs[i]; 302 303 init_attr.pd = dek->pd; 304 init_attr.has_keytag = has_keytag; 305 init_attr.key_purpose = MLX5DV_CRYPTO_KEY_PURPOSE_AES_XTS; 306 init_attr.comp_mask = MLX5DV_DEK_INIT_ATTR_CRYPTO_LOGIN; 307 init_attr.crypto_login = NULL; 308 memcpy(init_attr.key, attr->dek, attr->dek_len); 309 310 dek->dek_obj = mlx5dv_dek_create(dek->context, &init_attr); 311 spdk_memset_s(init_attr.key, sizeof(init_attr.key), 0, sizeof(init_attr.key)); 312 if (!dek->dek_obj) { 313 SPDK_ERRLOG("mlx5dv_dek_create failed on dev %s, errno %d\n", dek->context->device->name, errno); 314 rc = -EINVAL; 315 goto err_out; 316 } 317 318 memset(&query_attr, 0, sizeof(query_attr)); 319 rc = mlx5dv_dek_query(dek->dek_obj, &query_attr); 320 if (rc) { 321 SPDK_ERRLOG("Failed to query DEK on dev %s, rc %d\n", dek->context->device->name, rc); 322 goto err_out; 323 } 324 if (query_attr.state != MLX5DV_DEK_STATE_READY) { 325 SPDK_ERRLOG("DEK on dev %s state %d\n", dek->context->device->name, query_attr.state); 326 rc = -EINVAL; 327 goto err_out; 328 } 329 } 330 331 if (has_keytag) { 332 /* Save keytag, it will be used to configure crypto MKEY */ 333 keytag->has_keytag = true; 334 memcpy(keytag->keytag, attr->dek + attr->dek_len - SPDK_MLX5_AES_XTS_KEYTAG_SIZE, 335 SPDK_MLX5_AES_XTS_KEYTAG_SIZE); 336 } 337 338 spdk_mlx5_crypto_devs_release(devs); 339 *out = keytag; 340 341 return 0; 342 343 err_out: 344 spdk_mlx5_crypto_keytag_destroy(keytag); 345 spdk_mlx5_crypto_devs_release(devs); 346 347 return rc; 348 } 349 350 static inline struct spdk_mlx5_crypto_dek * 351 mlx5_crypto_get_dek_by_pd(struct spdk_mlx5_crypto_keytag *keytag, struct ibv_pd *pd) 352 { 353 struct spdk_mlx5_crypto_dek *dek; 354 uint32_t i; 355 356 for (i = 0; i < keytag->deks_num; i++) { 357 dek = &keytag->deks[i]; 358 if (dek->pd == pd) { 359 return dek; 360 } 361 } 362 363 return NULL; 364 } 365 366 int 367 spdk_mlx5_crypto_set_attr(struct mlx5dv_crypto_attr *attr_out, 368 struct spdk_mlx5_crypto_keytag *keytag, struct ibv_pd *pd, 369 uint32_t block_size, uint64_t iv, bool encrypt_on_tx) 370 { 371 struct spdk_mlx5_crypto_dek *dek; 372 enum mlx5dv_block_size bs; 373 374 dek = mlx5_crypto_get_dek_by_pd(keytag, pd); 375 if (spdk_unlikely(!dek)) { 376 SPDK_ERRLOG("No DEK for pd %p (dev %s)\n", pd, pd->context->device->name); 377 return -EINVAL; 378 } 379 380 switch (block_size) { 381 case 512: 382 bs = MLX5DV_BLOCK_SIZE_512; 383 break; 384 case 520: 385 bs = MLX5DV_BLOCK_SIZE_520; 386 break; 387 case 4048: 388 bs = MLX5DV_BLOCK_SIZE_4048; 389 break; 390 case 4096: 391 bs = MLX5DV_BLOCK_SIZE_4096; 392 break; 393 case 4160: 394 bs = MLX5DV_BLOCK_SIZE_4160; 395 break; 396 default: 397 SPDK_ERRLOG("Unsupported block size %u\n", block_size); 398 return -EINVAL; 399 } 400 401 memset(attr_out, 0, sizeof(*attr_out)); 402 attr_out->dek = dek->dek_obj; 403 attr_out->crypto_standard = MLX5DV_CRYPTO_STANDARD_AES_XTS; 404 attr_out->data_unit_size = bs; 405 attr_out->encrypt_on_tx = encrypt_on_tx; 406 memcpy(attr_out->initial_tweak, &iv, sizeof(iv)); 407 if (keytag->has_keytag) { 408 memcpy(attr_out->keytag, keytag->keytag, sizeof(keytag->keytag)); 409 } 410 411 return 0; 412 } 413 414 SPDK_LOG_REGISTER_COMPONENT(mlx5) 415