1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 */ 4 5 #include <rdma/rdma_cma.h> 6 #include <infiniband/verbs.h> 7 #include <infiniband/mlx5dv.h> 8 9 #include "spdk/stdinc.h" 10 #include "spdk/queue.h" 11 #include "spdk/log.h" 12 #include "spdk/likely.h" 13 #include "spdk/util.h" 14 #include "spdk_internal/mlx5.h" 15 #include "spdk_internal/rdma.h" 16 17 #define MLX5_VENDOR_ID_MELLANOX 0x2c9 18 19 /* Plaintext key sizes */ 20 /* 64b keytag */ 21 #define SPDK_MLX5_AES_XTS_KEYTAG_SIZE 8 22 /* key1_128b + key2_128b */ 23 #define SPDK_MLX5_AES_XTS_128_DEK_BYTES 32 24 /* key1_256b + key2_256b */ 25 #define SPDK_MLX5_AES_XTS_256_DEK_BYTES 64 26 /* key1_128b + key2_128b + 64b_keytag */ 27 #define SPDK_MLX5_AES_XTS_128_DEK_BYTES_WITH_KEYTAG (SPDK_MLX5_AES_XTS_128_DEK_BYTES + SPDK_MLX5_AES_XTS_KEYTAG_SIZE) 28 /* key1_256b + key2_256b + 64b_keytag */ 29 #define SPDK_MLX5_AES_XTS_256_DEK_BYTES_WITH_KEYTAG (SPDK_MLX5_AES_XTS_256_DEK_BYTES + SPDK_MLX5_AES_XTS_KEYTAG_SIZE) 30 31 struct spdk_mlx5_crypto_dek { 32 struct mlx5dv_dek *dek_obj; 33 struct ibv_pd *pd; 34 struct ibv_context *context; 35 }; 36 37 struct spdk_mlx5_crypto_keytag { 38 struct spdk_mlx5_crypto_dek *deks; 39 uint32_t deks_num; 40 bool has_keytag; 41 char keytag[8]; 42 }; 43 44 struct ibv_context ** 45 spdk_mlx5_crypto_devs_get(int *dev_num) 46 { 47 struct ibv_context **rdma_devs, **rdma_devs_out = NULL, *dev; 48 struct ibv_device_attr dev_attr; 49 struct mlx5dv_context dv_dev_attr; 50 int num_rdma_devs = 0, i, rc; 51 int num_crypto_devs = 0; 52 53 /* query all devices, save mlx5 with crypto support */ 54 rdma_devs = rdma_get_devices(&num_rdma_devs); 55 if (!rdma_devs || !num_rdma_devs) { 56 *dev_num = 0; 57 return NULL; 58 } 59 60 rdma_devs_out = calloc(num_rdma_devs + 1, sizeof(*rdma_devs_out)); 61 if (!rdma_devs_out) { 62 SPDK_ERRLOG("Memory allocation failed\n"); 63 return NULL; 64 } 65 66 for (i = 0; i < num_rdma_devs; i++) { 67 dev = rdma_devs[i]; 68 69 rc = ibv_query_device(dev, &dev_attr); 70 if (rc) { 71 SPDK_ERRLOG("Failed to query dev %s, skipping\n", dev->device->name); 72 continue; 73 } 74 if (dev_attr.vendor_id != MLX5_VENDOR_ID_MELLANOX) { 75 SPDK_DEBUGLOG(mlx5, "dev %s is not Mellanox device, skipping\n", dev->device->name); 76 continue; 77 } 78 79 memset(&dv_dev_attr, 0, sizeof(dv_dev_attr)); 80 dv_dev_attr.comp_mask |= MLX5DV_CONTEXT_MASK_CRYPTO_OFFLOAD; 81 rc = mlx5dv_query_device(dev, &dv_dev_attr); 82 if (rc) { 83 SPDK_ERRLOG("Failed to query mlx5 dev %s, skipping\n", dev->device->name); 84 continue; 85 } 86 if (!(dv_dev_attr.crypto_caps.flags & MLX5DV_CRYPTO_CAPS_CRYPTO)) { 87 SPDK_DEBUGLOG(mlx5, "dev %s crypto engine doesn't support crypto, skipping\n", dev->device->name); 88 continue; 89 } 90 if (!(dv_dev_attr.crypto_caps.crypto_engines & (MLX5DV_CRYPTO_ENGINES_CAP_AES_XTS | 91 MLX5DV_CRYPTO_ENGINES_CAP_AES_XTS_SINGLE_BLOCK))) { 92 SPDK_DEBUGLOG(mlx5, "dev %s crypto engine doesn't support AES_XTS, skipping\n", dev->device->name); 93 continue; 94 } 95 if (dv_dev_attr.crypto_caps.wrapped_import_method & 96 MLX5DV_CRYPTO_WRAPPED_IMPORT_METHOD_CAP_AES_XTS) { 97 SPDK_WARNLOG("dev %s uses wrapped import method (0x%x) which is not supported by mlx5 accel module\n", 98 dev->device->name, dv_dev_attr.crypto_caps.wrapped_import_method); 99 continue; 100 } 101 102 SPDK_NOTICELOG("Crypto dev %s\n", dev->device->name); 103 rdma_devs_out[num_crypto_devs++] = dev; 104 } 105 106 if (!num_crypto_devs) { 107 SPDK_DEBUGLOG(mlx5, "Found no mlx5 crypto devices\n"); 108 goto err_out; 109 } 110 111 rdma_free_devices(rdma_devs); 112 *dev_num = num_crypto_devs; 113 114 return rdma_devs_out; 115 116 err_out: 117 free(rdma_devs_out); 118 rdma_free_devices(rdma_devs); 119 *dev_num = 0; 120 return NULL; 121 } 122 123 void 124 spdk_mlx5_crypto_devs_release(struct ibv_context **rdma_devs) 125 { 126 if (rdma_devs) { 127 free(rdma_devs); 128 } 129 } 130 131 void 132 spdk_mlx5_crypto_keytag_destroy(struct spdk_mlx5_crypto_keytag *keytag) 133 { 134 struct spdk_mlx5_crypto_dek *dek; 135 uint32_t i; 136 137 if (!keytag) { 138 return; 139 } 140 141 for (i = 0; i < keytag->deks_num; i++) { 142 dek = &keytag->deks[i]; 143 if (dek->dek_obj) { 144 mlx5dv_dek_destroy(dek->dek_obj); 145 } 146 if (dek->pd) { 147 spdk_rdma_put_pd(dek->pd); 148 } 149 } 150 spdk_memset_s(keytag->keytag, sizeof(keytag->keytag), 0, sizeof(keytag->keytag)); 151 free(keytag->deks); 152 free(keytag); 153 } 154 155 int 156 spdk_mlx5_crypto_keytag_create(struct spdk_mlx5_crypto_dek_create_attr *attr, 157 struct spdk_mlx5_crypto_keytag **out) 158 { 159 struct spdk_mlx5_crypto_dek *dek; 160 struct spdk_mlx5_crypto_keytag *keytag; 161 struct ibv_context **devs; 162 struct mlx5dv_dek_init_attr init_attr = {}; 163 struct mlx5dv_dek_attr query_attr; 164 int num_devs = 0, i, rc; 165 bool has_keytag; 166 167 168 if (!attr || !attr->dek) { 169 return -EINVAL; 170 } 171 switch (attr->dek_len) { 172 case SPDK_MLX5_AES_XTS_128_DEK_BYTES_WITH_KEYTAG: 173 init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_128; 174 has_keytag = true; 175 SPDK_DEBUGLOG(mlx5, "128b AES_XTS with keytag\n"); 176 break; 177 case SPDK_MLX5_AES_XTS_256_DEK_BYTES_WITH_KEYTAG: 178 init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_256; 179 has_keytag = true; 180 SPDK_DEBUGLOG(mlx5, "256b AES_XTS with keytag\n"); 181 break; 182 case SPDK_MLX5_AES_XTS_128_DEK_BYTES: 183 init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_128; 184 has_keytag = false; 185 SPDK_DEBUGLOG(mlx5, "128b AES_XTS\n"); 186 break; 187 case SPDK_MLX5_AES_XTS_256_DEK_BYTES: 188 init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_256; 189 has_keytag = false; 190 SPDK_DEBUGLOG(mlx5, "256b AES_XTS\n"); 191 break; 192 default: 193 SPDK_ERRLOG("Invalid key length %zu. The following keys are supported:\n" 194 "128b key + key2, %u bytes;\n" 195 "256b key + key2, %u bytes\n" 196 "128b key + key2 + keytag, %u bytes\n" 197 "256b lye + key2 + keytag, %u bytes\n", 198 attr->dek_len, SPDK_MLX5_AES_XTS_128_DEK_BYTES, MLX5DV_CRYPTO_KEY_SIZE_256, 199 SPDK_MLX5_AES_XTS_128_DEK_BYTES_WITH_KEYTAG, SPDK_MLX5_AES_XTS_256_DEK_BYTES_WITH_KEYTAG); 200 return -EINVAL; 201 } 202 203 devs = spdk_mlx5_crypto_devs_get(&num_devs); 204 if (!devs || !num_devs) { 205 SPDK_DEBUGLOG(mlx5, "No crypto devices found\n"); 206 return -ENOTSUP; 207 } 208 209 keytag = calloc(1, sizeof(*keytag)); 210 if (!keytag) { 211 SPDK_ERRLOG("Memory allocation failed\n"); 212 spdk_mlx5_crypto_devs_release(devs); 213 return -ENOMEM; 214 } 215 keytag->deks = calloc(num_devs, sizeof(struct spdk_mlx5_crypto_dek)); 216 if (!keytag->deks) { 217 SPDK_ERRLOG("Memory allocation failed\n"); 218 spdk_mlx5_crypto_devs_release(devs); 219 free(keytag); 220 return -ENOMEM; 221 } 222 223 for (i = 0; i < num_devs; i++) { 224 keytag->deks_num++; 225 dek = &keytag->deks[i]; 226 dek->pd = spdk_rdma_get_pd(devs[i]); 227 if (!dek->pd) { 228 SPDK_ERRLOG("Failed to get PD on device %s\n", devs[i]->device->name); 229 rc = -EINVAL; 230 goto err_out; 231 } 232 dek->context = devs[i]; 233 234 init_attr.pd = dek->pd; 235 init_attr.has_keytag = has_keytag; 236 init_attr.key_purpose = MLX5DV_CRYPTO_KEY_PURPOSE_AES_XTS; 237 init_attr.comp_mask = MLX5DV_DEK_INIT_ATTR_CRYPTO_LOGIN; 238 init_attr.crypto_login = NULL; 239 memcpy(init_attr.key, attr->dek, attr->dek_len); 240 241 dek->dek_obj = mlx5dv_dek_create(dek->context, &init_attr); 242 spdk_memset_s(init_attr.key, sizeof(init_attr.key), 0, sizeof(init_attr.key)); 243 if (!dek->dek_obj) { 244 SPDK_ERRLOG("mlx5dv_dek_create failed on dev %s, errno %d\n", dek->context->device->name, errno); 245 rc = -EINVAL; 246 goto err_out; 247 } 248 249 memset(&query_attr, 0, sizeof(query_attr)); 250 rc = mlx5dv_dek_query(dek->dek_obj, &query_attr); 251 if (rc) { 252 SPDK_ERRLOG("Failed to query DEK on dev %s, rc %d\n", dek->context->device->name, rc); 253 goto err_out; 254 } 255 if (query_attr.state != MLX5DV_DEK_STATE_READY) { 256 SPDK_ERRLOG("DEK on dev %s state %d\n", dek->context->device->name, query_attr.state); 257 rc = -EINVAL; 258 goto err_out; 259 } 260 } 261 262 if (has_keytag) { 263 /* Save keytag, it will be used to configure crypto MKEY */ 264 keytag->has_keytag = true; 265 memcpy(keytag->keytag, attr->dek + attr->dek_len - SPDK_MLX5_AES_XTS_KEYTAG_SIZE, 266 SPDK_MLX5_AES_XTS_KEYTAG_SIZE); 267 } 268 269 spdk_mlx5_crypto_devs_release(devs); 270 *out = keytag; 271 272 return 0; 273 274 err_out: 275 spdk_mlx5_crypto_keytag_destroy(keytag); 276 spdk_mlx5_crypto_devs_release(devs); 277 278 return rc; 279 } 280 281 static inline struct spdk_mlx5_crypto_dek * 282 mlx5_crypto_get_dek_by_pd(struct spdk_mlx5_crypto_keytag *keytag, struct ibv_pd *pd) 283 { 284 struct spdk_mlx5_crypto_dek *dek; 285 uint32_t i; 286 287 for (i = 0; i < keytag->deks_num; i++) { 288 dek = &keytag->deks[i]; 289 if (dek->pd == pd) { 290 return dek; 291 } 292 } 293 294 return NULL; 295 } 296 297 int 298 spdk_mlx5_crypto_set_attr(struct mlx5dv_crypto_attr *attr_out, 299 struct spdk_mlx5_crypto_keytag *keytag, struct ibv_pd *pd, 300 uint32_t block_size, uint64_t iv, bool encrypt_on_tx) 301 { 302 struct spdk_mlx5_crypto_dek *dek; 303 enum mlx5dv_block_size bs; 304 305 dek = mlx5_crypto_get_dek_by_pd(keytag, pd); 306 if (spdk_unlikely(!dek)) { 307 SPDK_ERRLOG("No DEK for pd %p (dev %s)\n", pd, pd->context->device->name); 308 return -EINVAL; 309 } 310 311 switch (block_size) { 312 case 512: 313 bs = MLX5DV_BLOCK_SIZE_512; 314 break; 315 case 520: 316 bs = MLX5DV_BLOCK_SIZE_520; 317 break; 318 case 4048: 319 bs = MLX5DV_BLOCK_SIZE_4048; 320 break; 321 case 4096: 322 bs = MLX5DV_BLOCK_SIZE_4096; 323 break; 324 case 4160: 325 bs = MLX5DV_BLOCK_SIZE_4160; 326 break; 327 default: 328 SPDK_ERRLOG("Unsupported block size %u\n", block_size); 329 return -EINVAL; 330 } 331 332 memset(attr_out, 0, sizeof(*attr_out)); 333 attr_out->dek = dek->dek_obj; 334 attr_out->crypto_standard = MLX5DV_CRYPTO_STANDARD_AES_XTS; 335 attr_out->data_unit_size = bs; 336 attr_out->encrypt_on_tx = encrypt_on_tx; 337 memcpy(attr_out->initial_tweak, &iv, sizeof(iv)); 338 if (keytag->has_keytag) { 339 memcpy(attr_out->keytag, keytag->keytag, sizeof(keytag->keytag)); 340 } 341 342 return 0; 343 } 344 345 SPDK_LOG_REGISTER_COMPONENT(mlx5) 346