1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2021-2024 Advanced Micro Devices, Inc. 3 */ 4 5 #include <inttypes.h> 6 7 #include <rte_common.h> 8 #include <rte_malloc.h> 9 #include <rte_bitops.h> 10 11 #include "ionic_crypto.h" 12 13 static int 14 iocpt_cq_init(struct iocpt_cq *cq, uint16_t num_descs) 15 { 16 if (!rte_is_power_of_2(num_descs) || 17 num_descs < IOCPT_MIN_RING_DESC || 18 num_descs > IOCPT_MAX_RING_DESC) { 19 IOCPT_PRINT(ERR, "%u descriptors (min: %u max: %u)", 20 num_descs, IOCPT_MIN_RING_DESC, IOCPT_MAX_RING_DESC); 21 return -EINVAL; 22 } 23 24 cq->num_descs = num_descs; 25 cq->size_mask = num_descs - 1; 26 cq->tail_idx = 0; 27 cq->done_color = 1; 28 29 return 0; 30 } 31 32 static void 33 iocpt_cq_map(struct iocpt_cq *cq, void *base, rte_iova_t base_pa) 34 { 35 cq->base = base; 36 cq->base_pa = base_pa; 37 } 38 39 uint32_t 40 iocpt_cq_service(struct iocpt_cq *cq, uint32_t work_to_do, 41 iocpt_cq_cb cb, void *cb_arg) 42 { 43 uint32_t work_done = 0; 44 45 if (work_to_do == 0) 46 return 0; 47 48 while (cb(cq, cq->tail_idx, cb_arg)) { 49 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 50 if (cq->tail_idx == 0) 51 cq->done_color = !cq->done_color; 52 53 if (++work_done == work_to_do) 54 break; 55 } 56 57 return work_done; 58 } 59 60 static int 61 iocpt_q_init(struct iocpt_queue *q, uint8_t type, uint32_t index, 62 uint16_t num_descs, uint16_t num_segs, uint32_t socket_id) 63 { 64 uint32_t ring_size; 65 66 if (!rte_is_power_of_2(num_descs)) 67 return -EINVAL; 68 69 ring_size = rte_log2_u32(num_descs); 70 if (ring_size < 2 || ring_size > 16) 71 return -EINVAL; 72 73 q->type = type; 74 q->index = index; 75 q->num_descs = num_descs; 76 q->num_segs = num_segs; 77 q->size_mask = num_descs - 1; 78 q->head_idx = 0; 79 q->tail_idx = 0; 80 81 q->info = rte_calloc_socket("iocpt", 82 num_descs * num_segs, sizeof(void *), 83 rte_mem_page_size(), socket_id); 84 if (q->info == NULL) { 85 IOCPT_PRINT(ERR, "Cannot allocate queue info"); 86 return -ENOMEM; 87 } 88 89 return 0; 90 } 91 92 static void 93 iocpt_q_map(struct iocpt_queue *q, void *base, rte_iova_t base_pa) 94 { 95 q->base = base; 96 q->base_pa = base_pa; 97 } 98 99 static void 100 iocpt_q_sg_map(struct iocpt_queue *q, void *base, rte_iova_t base_pa) 101 { 102 q->sg_base = base; 103 q->sg_base_pa = base_pa; 104 } 105 106 static void 107 iocpt_q_free(struct iocpt_queue *q) 108 { 109 if (q->info != NULL) { 110 rte_free(q->info); 111 q->info = NULL; 112 } 113 } 114 115 static const struct rte_memzone * 116 iocpt_dma_zone_reserve(const char *type_name, uint16_t qid, size_t size, 117 unsigned int align, int socket_id) 118 { 119 char zone_name[RTE_MEMZONE_NAMESIZE]; 120 const struct rte_memzone *mz; 121 int err; 122 123 err = snprintf(zone_name, sizeof(zone_name), 124 "iocpt_%s_%u", type_name, qid); 125 if (err >= RTE_MEMZONE_NAMESIZE) { 126 IOCPT_PRINT(ERR, "Name %s too long", type_name); 127 return NULL; 128 } 129 130 mz = rte_memzone_lookup(zone_name); 131 if (mz != NULL) 132 return mz; 133 134 return rte_memzone_reserve_aligned(zone_name, size, socket_id, 135 RTE_MEMZONE_IOVA_CONTIG, align); 136 } 137 138 static int 139 iocpt_commonq_alloc(struct iocpt_dev *dev, 140 uint8_t type, 141 size_t struct_size, 142 uint32_t socket_id, 143 uint32_t index, 144 const char *type_name, 145 uint16_t flags, 146 uint16_t num_descs, 147 uint16_t num_segs, 148 uint16_t desc_size, 149 uint16_t cq_desc_size, 150 uint16_t sg_desc_size, 151 struct iocpt_common_q **comq) 152 { 153 struct iocpt_common_q *new; 154 uint32_t q_size, cq_size, sg_size, total_size; 155 void *q_base, *cq_base, *sg_base; 156 rte_iova_t q_base_pa = 0; 157 rte_iova_t cq_base_pa = 0; 158 rte_iova_t sg_base_pa = 0; 159 size_t page_size = rte_mem_page_size(); 160 int err; 161 162 *comq = NULL; 163 164 q_size = num_descs * desc_size; 165 cq_size = num_descs * cq_desc_size; 166 sg_size = num_descs * sg_desc_size; 167 168 /* 169 * Note: aligning q_size/cq_size is not enough due to cq_base address 170 * aligning as q_base could be not aligned to the page. 171 * Adding page_size. 172 */ 173 total_size = RTE_ALIGN(q_size, page_size) + 174 RTE_ALIGN(cq_size, page_size) + page_size; 175 if (flags & IOCPT_Q_F_SG) 176 total_size += RTE_ALIGN(sg_size, page_size) + page_size; 177 178 new = rte_zmalloc_socket("iocpt", struct_size, 179 RTE_CACHE_LINE_SIZE, socket_id); 180 if (new == NULL) { 181 IOCPT_PRINT(ERR, "Cannot allocate queue structure"); 182 return -ENOMEM; 183 } 184 185 new->dev = dev; 186 187 err = iocpt_q_init(&new->q, type, index, num_descs, num_segs, 188 socket_id); 189 if (err != 0) { 190 IOCPT_PRINT(ERR, "Queue initialization failed"); 191 goto err_free_q; 192 } 193 194 err = iocpt_cq_init(&new->cq, num_descs); 195 if (err != 0) { 196 IOCPT_PRINT(ERR, "Completion queue initialization failed"); 197 goto err_deinit_q; 198 } 199 200 new->base_z = iocpt_dma_zone_reserve(type_name, index, total_size, 201 IONIC_ALIGN, socket_id); 202 if (new->base_z == NULL) { 203 IOCPT_PRINT(ERR, "Cannot reserve queue DMA memory"); 204 err = -ENOMEM; 205 goto err_deinit_cq; 206 } 207 208 new->base = new->base_z->addr; 209 new->base_pa = new->base_z->iova; 210 211 q_base = new->base; 212 q_base_pa = new->base_pa; 213 iocpt_q_map(&new->q, q_base, q_base_pa); 214 215 cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, page_size); 216 cq_base_pa = RTE_ALIGN(q_base_pa + q_size, page_size); 217 iocpt_cq_map(&new->cq, cq_base, cq_base_pa); 218 219 if (flags & IOCPT_Q_F_SG) { 220 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size, 221 page_size); 222 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, page_size); 223 iocpt_q_sg_map(&new->q, sg_base, sg_base_pa); 224 } 225 226 IOCPT_PRINT(DEBUG, "q_base_pa %#jx cq_base_pa %#jx sg_base_pa %#jx", 227 q_base_pa, cq_base_pa, sg_base_pa); 228 229 *comq = new; 230 231 return 0; 232 233 err_deinit_cq: 234 err_deinit_q: 235 iocpt_q_free(&new->q); 236 err_free_q: 237 rte_free(new); 238 return err; 239 } 240 241 struct ionic_doorbell * 242 iocpt_db_map(struct iocpt_dev *dev, struct iocpt_queue *q) 243 { 244 return dev->db_pages + q->hw_type; 245 } 246 247 static int 248 iocpt_adminq_alloc(struct iocpt_dev *dev) 249 { 250 struct iocpt_admin_q *aq; 251 uint16_t num_descs = IOCPT_ADMINQ_LENGTH; 252 uint16_t flags = 0; 253 int err; 254 255 err = iocpt_commonq_alloc(dev, 256 IOCPT_QTYPE_ADMINQ, 257 sizeof(struct iocpt_admin_q), 258 rte_socket_id(), 259 0, 260 "admin", 261 flags, 262 num_descs, 263 1, 264 sizeof(struct iocpt_admin_cmd), 265 sizeof(struct iocpt_admin_comp), 266 0, 267 (struct iocpt_common_q **)&aq); 268 if (err != 0) 269 return err; 270 271 aq->flags = flags; 272 273 dev->adminq = aq; 274 275 return 0; 276 } 277 278 static int 279 iocpt_adminq_init(struct iocpt_dev *dev) 280 { 281 return iocpt_dev_adminq_init(dev); 282 } 283 284 static void 285 iocpt_adminq_deinit(struct iocpt_dev *dev) 286 { 287 dev->adminq->flags &= ~IOCPT_Q_F_INITED; 288 } 289 290 static void 291 iocpt_adminq_free(struct iocpt_admin_q *aq) 292 { 293 if (aq->base_z != NULL) { 294 rte_memzone_free(aq->base_z); 295 aq->base_z = NULL; 296 aq->base = NULL; 297 aq->base_pa = 0; 298 } 299 300 iocpt_q_free(&aq->q); 301 302 rte_free(aq); 303 } 304 305 static int 306 iocpt_alloc_objs(struct iocpt_dev *dev) 307 { 308 int err; 309 310 IOCPT_PRINT(DEBUG, "Crypto: %s", dev->name); 311 312 rte_spinlock_init(&dev->adminq_lock); 313 rte_spinlock_init(&dev->adminq_service_lock); 314 315 err = iocpt_adminq_alloc(dev); 316 if (err != 0) { 317 IOCPT_PRINT(ERR, "Cannot allocate admin queue"); 318 err = -ENOMEM; 319 goto err_out; 320 } 321 322 dev->info_sz = RTE_ALIGN(sizeof(*dev->info), rte_mem_page_size()); 323 dev->info_z = iocpt_dma_zone_reserve("info", 0, dev->info_sz, 324 IONIC_ALIGN, dev->socket_id); 325 if (dev->info_z == NULL) { 326 IOCPT_PRINT(ERR, "Cannot allocate dev info memory"); 327 err = -ENOMEM; 328 goto err_free_adminq; 329 } 330 331 dev->info = dev->info_z->addr; 332 dev->info_pa = dev->info_z->iova; 333 334 return 0; 335 336 err_free_adminq: 337 iocpt_adminq_free(dev->adminq); 338 dev->adminq = NULL; 339 err_out: 340 return err; 341 } 342 343 static int 344 iocpt_init(struct iocpt_dev *dev) 345 { 346 int err; 347 348 /* Uses dev_cmds */ 349 err = iocpt_dev_init(dev, dev->info_pa); 350 if (err != 0) 351 return err; 352 353 err = iocpt_adminq_init(dev); 354 if (err != 0) 355 return err; 356 357 dev->state |= IOCPT_DEV_F_INITED; 358 359 return 0; 360 } 361 362 void 363 iocpt_configure(struct iocpt_dev *dev) 364 { 365 RTE_SET_USED(dev); 366 } 367 368 void 369 iocpt_deinit(struct iocpt_dev *dev) 370 { 371 IOCPT_PRINT_CALL(); 372 373 if (!(dev->state & IOCPT_DEV_F_INITED)) 374 return; 375 376 iocpt_adminq_deinit(dev); 377 378 dev->state &= ~IOCPT_DEV_F_INITED; 379 } 380 381 static void 382 iocpt_free_objs(struct iocpt_dev *dev) 383 { 384 IOCPT_PRINT_CALL(); 385 386 if (dev->adminq != NULL) { 387 iocpt_adminq_free(dev->adminq); 388 dev->adminq = NULL; 389 } 390 391 if (dev->info != NULL) { 392 rte_memzone_free(dev->info_z); 393 dev->info_z = NULL; 394 dev->info = NULL; 395 dev->info_pa = 0; 396 } 397 } 398 399 static int 400 iocpt_devargs(struct rte_devargs *devargs, struct iocpt_dev *dev) 401 { 402 RTE_SET_USED(devargs); 403 RTE_SET_USED(dev); 404 405 return 0; 406 } 407 408 int 409 iocpt_probe(void *bus_dev, struct rte_device *rte_dev, 410 struct iocpt_dev_bars *bars, const struct iocpt_dev_intf *intf, 411 uint8_t driver_id, uint8_t socket_id) 412 { 413 struct rte_cryptodev_pmd_init_params init_params = { 414 "iocpt", 415 sizeof(struct iocpt_dev), 416 socket_id, 417 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 418 }; 419 struct rte_cryptodev *cdev; 420 struct iocpt_dev *dev; 421 uint32_t i, sig; 422 int err; 423 424 /* Check structs (trigger error at compilation time) */ 425 iocpt_struct_size_checks(); 426 427 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 428 IOCPT_PRINT(ERR, "Multi-process not supported"); 429 err = -EPERM; 430 goto err; 431 } 432 433 cdev = rte_cryptodev_pmd_create(rte_dev->name, rte_dev, &init_params); 434 if (cdev == NULL) { 435 IOCPT_PRINT(ERR, "Out of memory"); 436 err = -ENOMEM; 437 goto err; 438 } 439 440 dev = cdev->data->dev_private; 441 dev->crypto_dev = cdev; 442 dev->bus_dev = bus_dev; 443 dev->intf = intf; 444 dev->driver_id = driver_id; 445 dev->socket_id = socket_id; 446 447 for (i = 0; i < bars->num_bars; i++) { 448 struct ionic_dev_bar *bar = &bars->bar[i]; 449 450 IOCPT_PRINT(DEBUG, 451 "bar[%u] = { .va = %p, .pa = %#jx, .len = %lu }", 452 i, bar->vaddr, bar->bus_addr, bar->len); 453 if (bar->vaddr == NULL) { 454 IOCPT_PRINT(ERR, "Null bar found, aborting"); 455 err = -EFAULT; 456 goto err_destroy_crypto_dev; 457 } 458 459 dev->bars.bar[i].vaddr = bar->vaddr; 460 dev->bars.bar[i].bus_addr = bar->bus_addr; 461 dev->bars.bar[i].len = bar->len; 462 } 463 dev->bars.num_bars = bars->num_bars; 464 465 err = iocpt_devargs(rte_dev->devargs, dev); 466 if (err != 0) { 467 IOCPT_PRINT(ERR, "Cannot parse device arguments"); 468 goto err_destroy_crypto_dev; 469 } 470 471 err = iocpt_setup_bars(dev); 472 if (err != 0) { 473 IOCPT_PRINT(ERR, "Cannot setup BARs: %d, aborting", err); 474 goto err_destroy_crypto_dev; 475 } 476 477 sig = ioread32(&dev->dev_info->signature); 478 if (sig != IOCPT_DEV_INFO_SIGNATURE) { 479 IOCPT_PRINT(ERR, "Incompatible firmware signature %#x", sig); 480 err = -EFAULT; 481 goto err_destroy_crypto_dev; 482 } 483 484 for (i = 0; i < IOCPT_FWVERS_BUFLEN; i++) 485 dev->fw_version[i] = ioread8(&dev->dev_info->fw_version[i]); 486 dev->fw_version[IOCPT_FWVERS_BUFLEN - 1] = '\0'; 487 IOCPT_PRINT(DEBUG, "%s firmware: %s", dev->name, dev->fw_version); 488 489 err = iocpt_dev_identify(dev); 490 if (err != 0) { 491 IOCPT_PRINT(ERR, "Cannot identify device: %d, aborting", 492 err); 493 goto err_destroy_crypto_dev; 494 } 495 496 err = iocpt_alloc_objs(dev); 497 if (err != 0) { 498 IOCPT_PRINT(ERR, "Cannot alloc device objects: %d", err); 499 goto err_destroy_crypto_dev; 500 } 501 502 err = iocpt_init(dev); 503 if (err != 0) { 504 IOCPT_PRINT(ERR, "Cannot init device: %d, aborting", err); 505 goto err_free_objs; 506 } 507 508 return 0; 509 510 err_free_objs: 511 iocpt_free_objs(dev); 512 err_destroy_crypto_dev: 513 rte_cryptodev_pmd_destroy(cdev); 514 err: 515 return err; 516 } 517 518 int 519 iocpt_remove(struct rte_device *rte_dev) 520 { 521 struct rte_cryptodev *cdev; 522 struct iocpt_dev *dev; 523 524 cdev = rte_cryptodev_pmd_get_named_dev(rte_dev->name); 525 if (cdev == NULL) { 526 IOCPT_PRINT(DEBUG, "Cannot find device %s", rte_dev->name); 527 return -ENODEV; 528 } 529 530 dev = cdev->data->dev_private; 531 532 iocpt_deinit(dev); 533 534 iocpt_dev_reset(dev); 535 536 iocpt_free_objs(dev); 537 538 rte_cryptodev_pmd_destroy(cdev); 539 540 return 0; 541 } 542 543 RTE_LOG_REGISTER_DEFAULT(iocpt_logtype, NOTICE); 544