1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD. 3 */ 4 #include <stdbool.h> 5 #include <unistd.h> 6 7 #include <rte_common.h> 8 #include <rte_errno.h> 9 #include <rte_pci.h> 10 #include <rte_bus_pci.h> 11 #include <rte_cryptodev.h> 12 #include <rte_cryptodev_pmd.h> 13 #include <rte_eal.h> 14 15 #include "virtio_cryptodev.h" 16 #include "virtqueue.h" 17 #include "virtio_crypto_algs.h" 18 #include "virtio_crypto_capabilities.h" 19 20 int virtio_crypto_logtype_init; 21 int virtio_crypto_logtype_session; 22 int virtio_crypto_logtype_rx; 23 int virtio_crypto_logtype_tx; 24 int virtio_crypto_logtype_driver; 25 26 static int virtio_crypto_dev_configure(struct rte_cryptodev *dev, 27 struct rte_cryptodev_config *config); 28 static int virtio_crypto_dev_start(struct rte_cryptodev *dev); 29 static void virtio_crypto_dev_stop(struct rte_cryptodev *dev); 30 static int virtio_crypto_dev_close(struct rte_cryptodev *dev); 31 static void virtio_crypto_dev_info_get(struct rte_cryptodev *dev, 32 struct rte_cryptodev_info *dev_info); 33 static void virtio_crypto_dev_stats_get(struct rte_cryptodev *dev, 34 struct rte_cryptodev_stats *stats); 35 static void virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev); 36 static int virtio_crypto_qp_setup(struct rte_cryptodev *dev, 37 uint16_t queue_pair_id, 38 const struct rte_cryptodev_qp_conf *qp_conf, 39 int socket_id); 40 static int virtio_crypto_qp_release(struct rte_cryptodev *dev, 41 uint16_t queue_pair_id); 42 static void virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev); 43 static unsigned int virtio_crypto_sym_get_session_private_size( 44 struct rte_cryptodev *dev); 45 static void virtio_crypto_sym_clear_session(struct rte_cryptodev *dev, 46 struct rte_cryptodev_sym_session *sess); 47 static int virtio_crypto_sym_configure_session(struct rte_cryptodev *dev, 48 struct rte_crypto_sym_xform *xform, 49 struct rte_cryptodev_sym_session *session, 50 struct rte_mempool *mp); 51 52 /* 53 * The set of PCI devices this driver supports 54 */ 55 static const struct rte_pci_id pci_id_virtio_crypto_map[] = { 56 { RTE_PCI_DEVICE(VIRTIO_CRYPTO_PCI_VENDORID, 57 VIRTIO_CRYPTO_PCI_DEVICEID) }, 58 { .vendor_id = 0, /* sentinel */ }, 59 }; 60 61 static const struct rte_cryptodev_capabilities virtio_capabilities[] = { 62 VIRTIO_SYM_CAPABILITIES, 63 RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() 64 }; 65 66 uint8_t cryptodev_virtio_driver_id; 67 68 #define NUM_ENTRY_SYM_CREATE_SESSION 4 69 70 static int 71 virtio_crypto_send_command(struct virtqueue *vq, 72 struct virtio_crypto_op_ctrl_req *ctrl, uint8_t *cipher_key, 73 uint8_t *auth_key, struct virtio_crypto_session *session) 74 { 75 uint8_t idx = 0; 76 uint8_t needed = 1; 77 uint32_t head = 0; 78 uint32_t len_cipher_key = 0; 79 uint32_t len_auth_key = 0; 80 uint32_t len_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req); 81 uint32_t len_session_input = sizeof(struct virtio_crypto_session_input); 82 uint32_t len_total = 0; 83 uint32_t input_offset = 0; 84 void *virt_addr_started = NULL; 85 phys_addr_t phys_addr_started; 86 struct vring_desc *desc; 87 uint32_t desc_offset; 88 struct virtio_crypto_session_input *input; 89 int ret; 90 91 PMD_INIT_FUNC_TRACE(); 92 93 if (session == NULL) { 94 VIRTIO_CRYPTO_SESSION_LOG_ERR("session is NULL."); 95 return -EINVAL; 96 } 97 /* cipher only is supported, it is available if auth_key is NULL */ 98 if (!cipher_key) { 99 VIRTIO_CRYPTO_SESSION_LOG_ERR("cipher key is NULL."); 100 return -EINVAL; 101 } 102 103 head = vq->vq_desc_head_idx; 104 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_desc_head_idx = %d, vq = %p", 105 head, vq); 106 107 if (vq->vq_free_cnt < needed) { 108 VIRTIO_CRYPTO_SESSION_LOG_ERR("Not enough entry"); 109 return -ENOSPC; 110 } 111 112 /* calculate the length of cipher key */ 113 if (cipher_key) { 114 switch (ctrl->u.sym_create_session.op_type) { 115 case VIRTIO_CRYPTO_SYM_OP_CIPHER: 116 len_cipher_key 117 = ctrl->u.sym_create_session.u.cipher 118 .para.keylen; 119 break; 120 case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING: 121 len_cipher_key 122 = ctrl->u.sym_create_session.u.chain 123 .para.cipher_param.keylen; 124 break; 125 default: 126 VIRTIO_CRYPTO_SESSION_LOG_ERR("invalid op type"); 127 return -EINVAL; 128 } 129 } 130 131 /* calculate the length of auth key */ 132 if (auth_key) { 133 len_auth_key = 134 ctrl->u.sym_create_session.u.chain.para.u.mac_param 135 .auth_key_len; 136 } 137 138 /* 139 * malloc memory to store indirect vring_desc entries, including 140 * ctrl request, cipher key, auth key, session input and desc vring 141 */ 142 desc_offset = len_ctrl_req + len_cipher_key + len_auth_key 143 + len_session_input; 144 virt_addr_started = rte_malloc(NULL, 145 desc_offset + NUM_ENTRY_SYM_CREATE_SESSION 146 * sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE); 147 if (virt_addr_started == NULL) { 148 VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap memory"); 149 return -ENOSPC; 150 } 151 phys_addr_started = rte_malloc_virt2iova(virt_addr_started); 152 153 /* address to store indirect vring desc entries */ 154 desc = (struct vring_desc *) 155 ((uint8_t *)virt_addr_started + desc_offset); 156 157 /* ctrl req part */ 158 memcpy(virt_addr_started, ctrl, len_ctrl_req); 159 desc[idx].addr = phys_addr_started; 160 desc[idx].len = len_ctrl_req; 161 desc[idx].flags = VRING_DESC_F_NEXT; 162 desc[idx].next = idx + 1; 163 idx++; 164 len_total += len_ctrl_req; 165 input_offset += len_ctrl_req; 166 167 /* cipher key part */ 168 if (len_cipher_key > 0) { 169 memcpy((uint8_t *)virt_addr_started + len_total, 170 cipher_key, len_cipher_key); 171 172 desc[idx].addr = phys_addr_started + len_total; 173 desc[idx].len = len_cipher_key; 174 desc[idx].flags = VRING_DESC_F_NEXT; 175 desc[idx].next = idx + 1; 176 idx++; 177 len_total += len_cipher_key; 178 input_offset += len_cipher_key; 179 } 180 181 /* auth key part */ 182 if (len_auth_key > 0) { 183 memcpy((uint8_t *)virt_addr_started + len_total, 184 auth_key, len_auth_key); 185 186 desc[idx].addr = phys_addr_started + len_total; 187 desc[idx].len = len_auth_key; 188 desc[idx].flags = VRING_DESC_F_NEXT; 189 desc[idx].next = idx + 1; 190 idx++; 191 len_total += len_auth_key; 192 input_offset += len_auth_key; 193 } 194 195 /* input part */ 196 input = (struct virtio_crypto_session_input *) 197 ((uint8_t *)virt_addr_started + input_offset); 198 input->status = VIRTIO_CRYPTO_ERR; 199 input->session_id = ~0ULL; 200 desc[idx].addr = phys_addr_started + len_total; 201 desc[idx].len = len_session_input; 202 desc[idx].flags = VRING_DESC_F_WRITE; 203 idx++; 204 205 /* use a single desc entry */ 206 vq->vq_ring.desc[head].addr = phys_addr_started + desc_offset; 207 vq->vq_ring.desc[head].len = idx * sizeof(struct vring_desc); 208 vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT; 209 vq->vq_free_cnt--; 210 211 vq->vq_desc_head_idx = vq->vq_ring.desc[head].next; 212 213 vq_update_avail_ring(vq, head); 214 vq_update_avail_idx(vq); 215 216 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d", 217 vq->vq_queue_index); 218 219 virtqueue_notify(vq); 220 221 rte_rmb(); 222 while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) { 223 rte_rmb(); 224 usleep(100); 225 } 226 227 while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) { 228 uint32_t idx, desc_idx, used_idx; 229 struct vring_used_elem *uep; 230 231 used_idx = (uint32_t)(vq->vq_used_cons_idx 232 & (vq->vq_nentries - 1)); 233 uep = &vq->vq_ring.used->ring[used_idx]; 234 idx = (uint32_t) uep->id; 235 desc_idx = idx; 236 237 while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) { 238 desc_idx = vq->vq_ring.desc[desc_idx].next; 239 vq->vq_free_cnt++; 240 } 241 242 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx; 243 vq->vq_desc_head_idx = idx; 244 245 vq->vq_used_cons_idx++; 246 vq->vq_free_cnt++; 247 } 248 249 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n" 250 "vq->vq_desc_head_idx=%d", 251 vq->vq_free_cnt, vq->vq_desc_head_idx); 252 253 /* get the result */ 254 if (input->status != VIRTIO_CRYPTO_OK) { 255 VIRTIO_CRYPTO_SESSION_LOG_ERR("Something wrong on backend! " 256 "status=%u, session_id=%" PRIu64 "", 257 input->status, input->session_id); 258 rte_free(virt_addr_started); 259 ret = -1; 260 } else { 261 session->session_id = input->session_id; 262 263 VIRTIO_CRYPTO_SESSION_LOG_INFO("Create session successfully, " 264 "session_id=%" PRIu64 "", input->session_id); 265 rte_free(virt_addr_started); 266 ret = 0; 267 } 268 269 return ret; 270 } 271 272 void 273 virtio_crypto_queue_release(struct virtqueue *vq) 274 { 275 struct virtio_crypto_hw *hw; 276 277 PMD_INIT_FUNC_TRACE(); 278 279 if (vq) { 280 hw = vq->hw; 281 /* Select and deactivate the queue */ 282 VTPCI_OPS(hw)->del_queue(hw, vq); 283 284 rte_memzone_free(vq->mz); 285 rte_mempool_free(vq->mpool); 286 rte_free(vq); 287 } 288 } 289 290 #define MPOOL_MAX_NAME_SZ 32 291 292 int 293 virtio_crypto_queue_setup(struct rte_cryptodev *dev, 294 int queue_type, 295 uint16_t vtpci_queue_idx, 296 uint16_t nb_desc, 297 int socket_id, 298 struct virtqueue **pvq) 299 { 300 char vq_name[VIRTQUEUE_MAX_NAME_SZ]; 301 char mpool_name[MPOOL_MAX_NAME_SZ]; 302 const struct rte_memzone *mz; 303 unsigned int vq_size, size; 304 struct virtio_crypto_hw *hw = dev->data->dev_private; 305 struct virtqueue *vq = NULL; 306 uint32_t i = 0; 307 uint32_t j; 308 309 PMD_INIT_FUNC_TRACE(); 310 311 VIRTIO_CRYPTO_INIT_LOG_DBG("setting up queue: %u", vtpci_queue_idx); 312 313 /* 314 * Read the virtqueue size from the Queue Size field 315 * Always power of 2 and if 0 virtqueue does not exist 316 */ 317 vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx); 318 if (vq_size == 0) { 319 VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue does not exist"); 320 return -EINVAL; 321 } 322 VIRTIO_CRYPTO_INIT_LOG_DBG("vq_size: %u", vq_size); 323 324 if (!rte_is_power_of_2(vq_size)) { 325 VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue size is not powerof 2"); 326 return -EINVAL; 327 } 328 329 if (queue_type == VTCRYPTO_DATAQ) { 330 snprintf(vq_name, sizeof(vq_name), "dev%d_dataqueue%d", 331 dev->data->dev_id, vtpci_queue_idx); 332 snprintf(mpool_name, sizeof(mpool_name), 333 "dev%d_dataqueue%d_mpool", 334 dev->data->dev_id, vtpci_queue_idx); 335 } else if (queue_type == VTCRYPTO_CTRLQ) { 336 snprintf(vq_name, sizeof(vq_name), "dev%d_controlqueue", 337 dev->data->dev_id); 338 snprintf(mpool_name, sizeof(mpool_name), 339 "dev%d_controlqueue_mpool", 340 dev->data->dev_id); 341 } 342 size = RTE_ALIGN_CEIL(sizeof(*vq) + 343 vq_size * sizeof(struct vq_desc_extra), 344 RTE_CACHE_LINE_SIZE); 345 vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE, 346 socket_id); 347 if (vq == NULL) { 348 VIRTIO_CRYPTO_INIT_LOG_ERR("Can not allocate virtqueue"); 349 return -ENOMEM; 350 } 351 352 if (queue_type == VTCRYPTO_DATAQ) { 353 /* pre-allocate a mempool and use it in the data plane to 354 * improve performance 355 */ 356 vq->mpool = rte_mempool_lookup(mpool_name); 357 if (vq->mpool == NULL) 358 vq->mpool = rte_mempool_create(mpool_name, 359 vq_size, 360 sizeof(struct virtio_crypto_op_cookie), 361 RTE_CACHE_LINE_SIZE, 0, 362 NULL, NULL, NULL, NULL, socket_id, 363 0); 364 if (!vq->mpool) { 365 VIRTIO_CRYPTO_DRV_LOG_ERR("Virtio Crypto PMD " 366 "Cannot create mempool"); 367 goto mpool_create_err; 368 } 369 for (i = 0; i < vq_size; i++) { 370 vq->vq_descx[i].cookie = 371 rte_zmalloc("crypto PMD op cookie pointer", 372 sizeof(struct virtio_crypto_op_cookie), 373 RTE_CACHE_LINE_SIZE); 374 if (vq->vq_descx[i].cookie == NULL) { 375 VIRTIO_CRYPTO_DRV_LOG_ERR("Failed to " 376 "alloc mem for cookie"); 377 goto cookie_alloc_err; 378 } 379 } 380 } 381 382 vq->hw = hw; 383 vq->dev_id = dev->data->dev_id; 384 vq->vq_queue_index = vtpci_queue_idx; 385 vq->vq_nentries = vq_size; 386 387 /* 388 * Using part of the vring entries is permitted, but the maximum 389 * is vq_size 390 */ 391 if (nb_desc == 0 || nb_desc > vq_size) 392 nb_desc = vq_size; 393 vq->vq_free_cnt = nb_desc; 394 395 /* 396 * Reserve a memzone for vring elements 397 */ 398 size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN); 399 vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN); 400 VIRTIO_CRYPTO_INIT_LOG_DBG("%s vring_size: %d, rounded_vring_size: %d", 401 (queue_type == VTCRYPTO_DATAQ) ? "dataq" : "ctrlq", 402 size, vq->vq_ring_size); 403 404 mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, 405 socket_id, 0, VIRTIO_PCI_VRING_ALIGN); 406 if (mz == NULL) { 407 if (rte_errno == EEXIST) 408 mz = rte_memzone_lookup(vq_name); 409 if (mz == NULL) { 410 VIRTIO_CRYPTO_INIT_LOG_ERR("not enough memory"); 411 goto mz_reserve_err; 412 } 413 } 414 415 /* 416 * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit, 417 * and only accepts 32 bit page frame number. 418 * Check if the allocated physical memory exceeds 16TB. 419 */ 420 if ((mz->phys_addr + vq->vq_ring_size - 1) 421 >> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) { 422 VIRTIO_CRYPTO_INIT_LOG_ERR("vring address shouldn't be " 423 "above 16TB!"); 424 goto vring_addr_err; 425 } 426 427 memset(mz->addr, 0, sizeof(mz->len)); 428 vq->mz = mz; 429 vq->vq_ring_mem = mz->phys_addr; 430 vq->vq_ring_virt_mem = mz->addr; 431 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_mem(physical): 0x%"PRIx64, 432 (uint64_t)mz->phys_addr); 433 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_virt_mem: 0x%"PRIx64, 434 (uint64_t)(uintptr_t)mz->addr); 435 436 *pvq = vq; 437 438 return 0; 439 440 vring_addr_err: 441 rte_memzone_free(mz); 442 mz_reserve_err: 443 cookie_alloc_err: 444 rte_mempool_free(vq->mpool); 445 if (i != 0) { 446 for (j = 0; j < i; j++) 447 rte_free(vq->vq_descx[j].cookie); 448 } 449 mpool_create_err: 450 rte_free(vq); 451 return -ENOMEM; 452 } 453 454 static int 455 virtio_crypto_ctrlq_setup(struct rte_cryptodev *dev, uint16_t queue_idx) 456 { 457 int ret; 458 struct virtqueue *vq; 459 struct virtio_crypto_hw *hw = dev->data->dev_private; 460 461 /* if virtio device has started, do not touch the virtqueues */ 462 if (dev->data->dev_started) 463 return 0; 464 465 PMD_INIT_FUNC_TRACE(); 466 467 ret = virtio_crypto_queue_setup(dev, VTCRYPTO_CTRLQ, queue_idx, 468 0, SOCKET_ID_ANY, &vq); 469 if (ret < 0) { 470 VIRTIO_CRYPTO_INIT_LOG_ERR("control vq initialization failed"); 471 return ret; 472 } 473 474 hw->cvq = vq; 475 476 return 0; 477 } 478 479 static void 480 virtio_crypto_free_queues(struct rte_cryptodev *dev) 481 { 482 unsigned int i; 483 struct virtio_crypto_hw *hw = dev->data->dev_private; 484 485 PMD_INIT_FUNC_TRACE(); 486 487 /* control queue release */ 488 virtio_crypto_queue_release(hw->cvq); 489 490 /* data queue release */ 491 for (i = 0; i < hw->max_dataqueues; i++) 492 virtio_crypto_queue_release(dev->data->queue_pairs[i]); 493 } 494 495 static int 496 virtio_crypto_dev_close(struct rte_cryptodev *dev __rte_unused) 497 { 498 return 0; 499 } 500 501 /* 502 * dev_ops for virtio, bare necessities for basic operation 503 */ 504 static struct rte_cryptodev_ops virtio_crypto_dev_ops = { 505 /* Device related operations */ 506 .dev_configure = virtio_crypto_dev_configure, 507 .dev_start = virtio_crypto_dev_start, 508 .dev_stop = virtio_crypto_dev_stop, 509 .dev_close = virtio_crypto_dev_close, 510 .dev_infos_get = virtio_crypto_dev_info_get, 511 512 .stats_get = virtio_crypto_dev_stats_get, 513 .stats_reset = virtio_crypto_dev_stats_reset, 514 515 .queue_pair_setup = virtio_crypto_qp_setup, 516 .queue_pair_release = virtio_crypto_qp_release, 517 518 /* Crypto related operations */ 519 .sym_session_get_size = virtio_crypto_sym_get_session_private_size, 520 .sym_session_configure = virtio_crypto_sym_configure_session, 521 .sym_session_clear = virtio_crypto_sym_clear_session 522 }; 523 524 static void 525 virtio_crypto_update_stats(struct rte_cryptodev *dev, 526 struct rte_cryptodev_stats *stats) 527 { 528 unsigned int i; 529 struct virtio_crypto_hw *hw = dev->data->dev_private; 530 531 PMD_INIT_FUNC_TRACE(); 532 533 if (stats == NULL) { 534 VIRTIO_CRYPTO_DRV_LOG_ERR("invalid pointer"); 535 return; 536 } 537 538 for (i = 0; i < hw->max_dataqueues; i++) { 539 const struct virtqueue *data_queue 540 = dev->data->queue_pairs[i]; 541 if (data_queue == NULL) 542 continue; 543 544 stats->enqueued_count += data_queue->packets_sent_total; 545 stats->enqueue_err_count += data_queue->packets_sent_failed; 546 547 stats->dequeued_count += data_queue->packets_received_total; 548 stats->dequeue_err_count 549 += data_queue->packets_received_failed; 550 } 551 } 552 553 static void 554 virtio_crypto_dev_stats_get(struct rte_cryptodev *dev, 555 struct rte_cryptodev_stats *stats) 556 { 557 PMD_INIT_FUNC_TRACE(); 558 559 virtio_crypto_update_stats(dev, stats); 560 } 561 562 static void 563 virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev) 564 { 565 unsigned int i; 566 struct virtio_crypto_hw *hw = dev->data->dev_private; 567 568 PMD_INIT_FUNC_TRACE(); 569 570 for (i = 0; i < hw->max_dataqueues; i++) { 571 struct virtqueue *data_queue = dev->data->queue_pairs[i]; 572 if (data_queue == NULL) 573 continue; 574 575 data_queue->packets_sent_total = 0; 576 data_queue->packets_sent_failed = 0; 577 578 data_queue->packets_received_total = 0; 579 data_queue->packets_received_failed = 0; 580 } 581 } 582 583 static int 584 virtio_crypto_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, 585 const struct rte_cryptodev_qp_conf *qp_conf, 586 int socket_id) 587 { 588 int ret; 589 struct virtqueue *vq; 590 591 PMD_INIT_FUNC_TRACE(); 592 593 /* if virtio dev is started, do not touch the virtqueues */ 594 if (dev->data->dev_started) 595 return 0; 596 597 ret = virtio_crypto_queue_setup(dev, VTCRYPTO_DATAQ, queue_pair_id, 598 qp_conf->nb_descriptors, socket_id, &vq); 599 if (ret < 0) { 600 VIRTIO_CRYPTO_INIT_LOG_ERR( 601 "virtio crypto data queue initialization failed\n"); 602 return ret; 603 } 604 605 dev->data->queue_pairs[queue_pair_id] = vq; 606 607 return 0; 608 } 609 610 static int 611 virtio_crypto_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) 612 { 613 struct virtqueue *vq 614 = (struct virtqueue *)dev->data->queue_pairs[queue_pair_id]; 615 616 PMD_INIT_FUNC_TRACE(); 617 618 if (vq == NULL) { 619 VIRTIO_CRYPTO_DRV_LOG_DBG("vq already freed"); 620 return 0; 621 } 622 623 virtio_crypto_queue_release(vq); 624 return 0; 625 } 626 627 static int 628 virtio_negotiate_features(struct virtio_crypto_hw *hw, uint64_t req_features) 629 { 630 uint64_t host_features; 631 632 PMD_INIT_FUNC_TRACE(); 633 634 /* Prepare guest_features: feature that driver wants to support */ 635 VIRTIO_CRYPTO_INIT_LOG_DBG("guest_features before negotiate = %" PRIx64, 636 req_features); 637 638 /* Read device(host) feature bits */ 639 host_features = VTPCI_OPS(hw)->get_features(hw); 640 VIRTIO_CRYPTO_INIT_LOG_DBG("host_features before negotiate = %" PRIx64, 641 host_features); 642 643 /* 644 * Negotiate features: Subset of device feature bits are written back 645 * guest feature bits. 646 */ 647 hw->guest_features = req_features; 648 hw->guest_features = vtpci_cryptodev_negotiate_features(hw, 649 host_features); 650 VIRTIO_CRYPTO_INIT_LOG_DBG("features after negotiate = %" PRIx64, 651 hw->guest_features); 652 653 if (hw->modern) { 654 if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) { 655 VIRTIO_CRYPTO_INIT_LOG_ERR( 656 "VIRTIO_F_VERSION_1 features is not enabled."); 657 return -1; 658 } 659 vtpci_cryptodev_set_status(hw, 660 VIRTIO_CONFIG_STATUS_FEATURES_OK); 661 if (!(vtpci_cryptodev_get_status(hw) & 662 VIRTIO_CONFIG_STATUS_FEATURES_OK)) { 663 VIRTIO_CRYPTO_INIT_LOG_ERR("failed to set FEATURES_OK " 664 "status!"); 665 return -1; 666 } 667 } 668 669 hw->req_guest_features = req_features; 670 671 return 0; 672 } 673 674 /* reset device and renegotiate features if needed */ 675 static int 676 virtio_crypto_init_device(struct rte_cryptodev *cryptodev, 677 uint64_t req_features) 678 { 679 struct virtio_crypto_hw *hw = cryptodev->data->dev_private; 680 struct virtio_crypto_config local_config; 681 struct virtio_crypto_config *config = &local_config; 682 683 PMD_INIT_FUNC_TRACE(); 684 685 /* Reset the device although not necessary at startup */ 686 vtpci_cryptodev_reset(hw); 687 688 /* Tell the host we've noticed this device. */ 689 vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); 690 691 /* Tell the host we've known how to drive the device. */ 692 vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); 693 if (virtio_negotiate_features(hw, req_features) < 0) 694 return -1; 695 696 /* Get status of the device */ 697 vtpci_read_cryptodev_config(hw, 698 offsetof(struct virtio_crypto_config, status), 699 &config->status, sizeof(config->status)); 700 if (config->status != VIRTIO_CRYPTO_S_HW_READY) { 701 VIRTIO_CRYPTO_DRV_LOG_ERR("accelerator hardware is " 702 "not ready"); 703 return -1; 704 } 705 706 /* Get number of data queues */ 707 vtpci_read_cryptodev_config(hw, 708 offsetof(struct virtio_crypto_config, max_dataqueues), 709 &config->max_dataqueues, 710 sizeof(config->max_dataqueues)); 711 hw->max_dataqueues = config->max_dataqueues; 712 713 VIRTIO_CRYPTO_INIT_LOG_DBG("hw->max_dataqueues=%d", 714 hw->max_dataqueues); 715 716 return 0; 717 } 718 719 /* 720 * This function is based on probe() function 721 * It returns 0 on success. 722 */ 723 static int 724 crypto_virtio_create(const char *name, struct rte_pci_device *pci_dev, 725 struct rte_cryptodev_pmd_init_params *init_params) 726 { 727 struct rte_cryptodev *cryptodev; 728 struct virtio_crypto_hw *hw; 729 730 PMD_INIT_FUNC_TRACE(); 731 732 cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device, 733 init_params); 734 if (cryptodev == NULL) 735 return -ENODEV; 736 737 cryptodev->driver_id = cryptodev_virtio_driver_id; 738 cryptodev->dev_ops = &virtio_crypto_dev_ops; 739 740 cryptodev->enqueue_burst = virtio_crypto_pkt_tx_burst; 741 cryptodev->dequeue_burst = virtio_crypto_pkt_rx_burst; 742 743 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 744 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING; 745 746 hw = cryptodev->data->dev_private; 747 hw->dev_id = cryptodev->data->dev_id; 748 hw->virtio_dev_capabilities = virtio_capabilities; 749 750 VIRTIO_CRYPTO_INIT_LOG_DBG("dev %d vendorID=0x%x deviceID=0x%x", 751 cryptodev->data->dev_id, pci_dev->id.vendor_id, 752 pci_dev->id.device_id); 753 754 /* pci device init */ 755 if (vtpci_cryptodev_init(pci_dev, hw)) 756 return -1; 757 758 if (virtio_crypto_init_device(cryptodev, 759 VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0) 760 return -1; 761 762 return 0; 763 } 764 765 static int 766 virtio_crypto_dev_uninit(struct rte_cryptodev *cryptodev) 767 { 768 struct virtio_crypto_hw *hw = cryptodev->data->dev_private; 769 770 PMD_INIT_FUNC_TRACE(); 771 772 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 773 return -EPERM; 774 775 if (cryptodev->data->dev_started) { 776 virtio_crypto_dev_stop(cryptodev); 777 virtio_crypto_dev_close(cryptodev); 778 } 779 780 cryptodev->dev_ops = NULL; 781 cryptodev->enqueue_burst = NULL; 782 cryptodev->dequeue_burst = NULL; 783 784 /* release control queue */ 785 virtio_crypto_queue_release(hw->cvq); 786 787 rte_free(cryptodev->data); 788 cryptodev->data = NULL; 789 790 VIRTIO_CRYPTO_DRV_LOG_INFO("dev_uninit completed"); 791 792 return 0; 793 } 794 795 static int 796 virtio_crypto_dev_configure(struct rte_cryptodev *cryptodev, 797 struct rte_cryptodev_config *config __rte_unused) 798 { 799 struct virtio_crypto_hw *hw = cryptodev->data->dev_private; 800 801 PMD_INIT_FUNC_TRACE(); 802 803 if (virtio_crypto_init_device(cryptodev, 804 VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0) 805 return -1; 806 807 /* setup control queue 808 * [0, 1, ... ,(config->max_dataqueues - 1)] are data queues 809 * config->max_dataqueues is the control queue 810 */ 811 if (virtio_crypto_ctrlq_setup(cryptodev, hw->max_dataqueues) < 0) { 812 VIRTIO_CRYPTO_INIT_LOG_ERR("control queue setup error"); 813 return -1; 814 } 815 virtio_crypto_ctrlq_start(cryptodev); 816 817 return 0; 818 } 819 820 static void 821 virtio_crypto_dev_stop(struct rte_cryptodev *dev) 822 { 823 struct virtio_crypto_hw *hw = dev->data->dev_private; 824 825 PMD_INIT_FUNC_TRACE(); 826 VIRTIO_CRYPTO_DRV_LOG_DBG("virtio_dev_stop"); 827 828 vtpci_cryptodev_reset(hw); 829 830 virtio_crypto_dev_free_mbufs(dev); 831 virtio_crypto_free_queues(dev); 832 833 dev->data->dev_started = 0; 834 } 835 836 static int 837 virtio_crypto_dev_start(struct rte_cryptodev *dev) 838 { 839 struct virtio_crypto_hw *hw = dev->data->dev_private; 840 841 if (dev->data->dev_started) 842 return 0; 843 844 /* Do final configuration before queue engine starts */ 845 virtio_crypto_dataq_start(dev); 846 vtpci_cryptodev_reinit_complete(hw); 847 848 dev->data->dev_started = 1; 849 850 return 0; 851 } 852 853 static void 854 virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev) 855 { 856 uint32_t i; 857 struct virtio_crypto_hw *hw = dev->data->dev_private; 858 859 for (i = 0; i < hw->max_dataqueues; i++) { 860 VIRTIO_CRYPTO_INIT_LOG_DBG("Before freeing dataq[%d] used " 861 "and unused buf", i); 862 VIRTQUEUE_DUMP((struct virtqueue *) 863 dev->data->queue_pairs[i]); 864 865 VIRTIO_CRYPTO_INIT_LOG_DBG("queue_pairs[%d]=%p", 866 i, dev->data->queue_pairs[i]); 867 868 virtqueue_detatch_unused(dev->data->queue_pairs[i]); 869 870 VIRTIO_CRYPTO_INIT_LOG_DBG("After freeing dataq[%d] used and " 871 "unused buf", i); 872 VIRTQUEUE_DUMP( 873 (struct virtqueue *)dev->data->queue_pairs[i]); 874 } 875 } 876 877 static unsigned int 878 virtio_crypto_sym_get_session_private_size( 879 struct rte_cryptodev *dev __rte_unused) 880 { 881 PMD_INIT_FUNC_TRACE(); 882 883 return RTE_ALIGN_CEIL(sizeof(struct virtio_crypto_session), 16); 884 } 885 886 static int 887 virtio_crypto_check_sym_session_paras( 888 struct rte_cryptodev *dev) 889 { 890 struct virtio_crypto_hw *hw; 891 892 PMD_INIT_FUNC_TRACE(); 893 894 if (unlikely(dev == NULL)) { 895 VIRTIO_CRYPTO_SESSION_LOG_ERR("dev is NULL"); 896 return -1; 897 } 898 if (unlikely(dev->data == NULL)) { 899 VIRTIO_CRYPTO_SESSION_LOG_ERR("dev->data is NULL"); 900 return -1; 901 } 902 hw = dev->data->dev_private; 903 if (unlikely(hw == NULL)) { 904 VIRTIO_CRYPTO_SESSION_LOG_ERR("hw is NULL"); 905 return -1; 906 } 907 if (unlikely(hw->cvq == NULL)) { 908 VIRTIO_CRYPTO_SESSION_LOG_ERR("vq is NULL"); 909 return -1; 910 } 911 912 return 0; 913 } 914 915 static int 916 virtio_crypto_check_sym_clear_session_paras( 917 struct rte_cryptodev *dev, 918 struct rte_cryptodev_sym_session *sess) 919 { 920 PMD_INIT_FUNC_TRACE(); 921 922 if (sess == NULL) { 923 VIRTIO_CRYPTO_SESSION_LOG_ERR("sym_session is NULL"); 924 return -1; 925 } 926 927 return virtio_crypto_check_sym_session_paras(dev); 928 } 929 930 #define NUM_ENTRY_SYM_CLEAR_SESSION 2 931 932 static void 933 virtio_crypto_sym_clear_session( 934 struct rte_cryptodev *dev, 935 struct rte_cryptodev_sym_session *sess) 936 { 937 struct virtio_crypto_hw *hw; 938 struct virtqueue *vq; 939 struct virtio_crypto_session *session; 940 struct virtio_crypto_op_ctrl_req *ctrl; 941 struct vring_desc *desc; 942 uint8_t *status; 943 uint8_t needed = 1; 944 uint32_t head; 945 uint8_t *malloc_virt_addr; 946 uint64_t malloc_phys_addr; 947 uint8_t len_inhdr = sizeof(struct virtio_crypto_inhdr); 948 uint32_t len_op_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req); 949 uint32_t desc_offset = len_op_ctrl_req + len_inhdr; 950 951 PMD_INIT_FUNC_TRACE(); 952 953 if (virtio_crypto_check_sym_clear_session_paras(dev, sess) < 0) 954 return; 955 956 hw = dev->data->dev_private; 957 vq = hw->cvq; 958 session = (struct virtio_crypto_session *)get_sym_session_private_data( 959 sess, cryptodev_virtio_driver_id); 960 if (session == NULL) { 961 VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid session parameter"); 962 return; 963 } 964 965 VIRTIO_CRYPTO_SESSION_LOG_INFO("vq->vq_desc_head_idx = %d, " 966 "vq = %p", vq->vq_desc_head_idx, vq); 967 968 if (vq->vq_free_cnt < needed) { 969 VIRTIO_CRYPTO_SESSION_LOG_ERR( 970 "vq->vq_free_cnt = %d is less than %d, " 971 "not enough", vq->vq_free_cnt, needed); 972 return; 973 } 974 975 /* 976 * malloc memory to store information of ctrl request op, 977 * returned status and desc vring 978 */ 979 malloc_virt_addr = rte_malloc(NULL, len_op_ctrl_req + len_inhdr 980 + NUM_ENTRY_SYM_CLEAR_SESSION 981 * sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE); 982 if (malloc_virt_addr == NULL) { 983 VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap room"); 984 return; 985 } 986 malloc_phys_addr = rte_malloc_virt2iova(malloc_virt_addr); 987 988 /* assign ctrl request op part */ 989 ctrl = (struct virtio_crypto_op_ctrl_req *)malloc_virt_addr; 990 ctrl->header.opcode = VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION; 991 /* default data virtqueue is 0 */ 992 ctrl->header.queue_id = 0; 993 ctrl->u.destroy_session.session_id = session->session_id; 994 995 /* status part */ 996 status = &(((struct virtio_crypto_inhdr *) 997 ((uint8_t *)malloc_virt_addr + len_op_ctrl_req))->status); 998 *status = VIRTIO_CRYPTO_ERR; 999 1000 /* indirect desc vring part */ 1001 desc = (struct vring_desc *)((uint8_t *)malloc_virt_addr 1002 + desc_offset); 1003 1004 /* ctrl request part */ 1005 desc[0].addr = malloc_phys_addr; 1006 desc[0].len = len_op_ctrl_req; 1007 desc[0].flags = VRING_DESC_F_NEXT; 1008 desc[0].next = 1; 1009 1010 /* status part */ 1011 desc[1].addr = malloc_phys_addr + len_op_ctrl_req; 1012 desc[1].len = len_inhdr; 1013 desc[1].flags = VRING_DESC_F_WRITE; 1014 1015 /* use only a single desc entry */ 1016 head = vq->vq_desc_head_idx; 1017 vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT; 1018 vq->vq_ring.desc[head].addr = malloc_phys_addr + desc_offset; 1019 vq->vq_ring.desc[head].len 1020 = NUM_ENTRY_SYM_CLEAR_SESSION 1021 * sizeof(struct vring_desc); 1022 vq->vq_free_cnt -= needed; 1023 1024 vq->vq_desc_head_idx = vq->vq_ring.desc[head].next; 1025 1026 vq_update_avail_ring(vq, head); 1027 vq_update_avail_idx(vq); 1028 1029 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d", 1030 vq->vq_queue_index); 1031 1032 virtqueue_notify(vq); 1033 1034 rte_rmb(); 1035 while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) { 1036 rte_rmb(); 1037 usleep(100); 1038 } 1039 1040 while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) { 1041 uint32_t idx, desc_idx, used_idx; 1042 struct vring_used_elem *uep; 1043 1044 used_idx = (uint32_t)(vq->vq_used_cons_idx 1045 & (vq->vq_nentries - 1)); 1046 uep = &vq->vq_ring.used->ring[used_idx]; 1047 idx = (uint32_t) uep->id; 1048 desc_idx = idx; 1049 while (vq->vq_ring.desc[desc_idx].flags 1050 & VRING_DESC_F_NEXT) { 1051 desc_idx = vq->vq_ring.desc[desc_idx].next; 1052 vq->vq_free_cnt++; 1053 } 1054 1055 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx; 1056 vq->vq_desc_head_idx = idx; 1057 vq->vq_used_cons_idx++; 1058 vq->vq_free_cnt++; 1059 } 1060 1061 if (*status != VIRTIO_CRYPTO_OK) { 1062 VIRTIO_CRYPTO_SESSION_LOG_ERR("Close session failed " 1063 "status=%"PRIu32", session_id=%"PRIu64"", 1064 *status, session->session_id); 1065 rte_free(malloc_virt_addr); 1066 return; 1067 } 1068 1069 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n" 1070 "vq->vq_desc_head_idx=%d", 1071 vq->vq_free_cnt, vq->vq_desc_head_idx); 1072 1073 VIRTIO_CRYPTO_SESSION_LOG_INFO("Close session %"PRIu64" successfully ", 1074 session->session_id); 1075 1076 memset(session, 0, sizeof(struct virtio_crypto_session)); 1077 struct rte_mempool *sess_mp = rte_mempool_from_obj(session); 1078 set_sym_session_private_data(sess, cryptodev_virtio_driver_id, NULL); 1079 rte_mempool_put(sess_mp, session); 1080 rte_free(malloc_virt_addr); 1081 } 1082 1083 static struct rte_crypto_cipher_xform * 1084 virtio_crypto_get_cipher_xform(struct rte_crypto_sym_xform *xform) 1085 { 1086 do { 1087 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) 1088 return &xform->cipher; 1089 1090 xform = xform->next; 1091 } while (xform); 1092 1093 return NULL; 1094 } 1095 1096 static struct rte_crypto_auth_xform * 1097 virtio_crypto_get_auth_xform(struct rte_crypto_sym_xform *xform) 1098 { 1099 do { 1100 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) 1101 return &xform->auth; 1102 1103 xform = xform->next; 1104 } while (xform); 1105 1106 return NULL; 1107 } 1108 1109 /** Get xform chain order */ 1110 static int 1111 virtio_crypto_get_chain_order(struct rte_crypto_sym_xform *xform) 1112 { 1113 if (xform == NULL) 1114 return -1; 1115 1116 /* Cipher Only */ 1117 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 1118 xform->next == NULL) 1119 return VIRTIO_CRYPTO_CMD_CIPHER; 1120 1121 /* Authentication Only */ 1122 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1123 xform->next == NULL) 1124 return VIRTIO_CRYPTO_CMD_AUTH; 1125 1126 /* Authenticate then Cipher */ 1127 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1128 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) 1129 return VIRTIO_CRYPTO_CMD_HASH_CIPHER; 1130 1131 /* Cipher then Authenticate */ 1132 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 1133 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 1134 return VIRTIO_CRYPTO_CMD_CIPHER_HASH; 1135 1136 return -1; 1137 } 1138 1139 static int 1140 virtio_crypto_sym_pad_cipher_param( 1141 struct virtio_crypto_cipher_session_para *para, 1142 struct rte_crypto_cipher_xform *cipher_xform) 1143 { 1144 switch (cipher_xform->algo) { 1145 case RTE_CRYPTO_CIPHER_AES_CBC: 1146 para->algo = VIRTIO_CRYPTO_CIPHER_AES_CBC; 1147 break; 1148 default: 1149 VIRTIO_CRYPTO_SESSION_LOG_ERR("Crypto: Unsupported " 1150 "Cipher alg %u", cipher_xform->algo); 1151 return -1; 1152 } 1153 1154 para->keylen = cipher_xform->key.length; 1155 switch (cipher_xform->op) { 1156 case RTE_CRYPTO_CIPHER_OP_ENCRYPT: 1157 para->op = VIRTIO_CRYPTO_OP_ENCRYPT; 1158 break; 1159 case RTE_CRYPTO_CIPHER_OP_DECRYPT: 1160 para->op = VIRTIO_CRYPTO_OP_DECRYPT; 1161 break; 1162 default: 1163 VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported cipher operation " 1164 "parameter"); 1165 return -1; 1166 } 1167 1168 return 0; 1169 } 1170 1171 static int 1172 virtio_crypto_sym_pad_auth_param( 1173 struct virtio_crypto_op_ctrl_req *ctrl, 1174 struct rte_crypto_auth_xform *auth_xform) 1175 { 1176 uint32_t *algo; 1177 struct virtio_crypto_alg_chain_session_para *para = 1178 &(ctrl->u.sym_create_session.u.chain.para); 1179 1180 switch (ctrl->u.sym_create_session.u.chain.para.hash_mode) { 1181 case VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN: 1182 algo = &(para->u.hash_param.algo); 1183 break; 1184 case VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH: 1185 algo = &(para->u.mac_param.algo); 1186 break; 1187 default: 1188 VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported hash mode %u " 1189 "specified", 1190 ctrl->u.sym_create_session.u.chain.para.hash_mode); 1191 return -1; 1192 } 1193 1194 switch (auth_xform->algo) { 1195 case RTE_CRYPTO_AUTH_SHA1_HMAC: 1196 *algo = VIRTIO_CRYPTO_MAC_HMAC_SHA1; 1197 break; 1198 default: 1199 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1200 "Crypto: Undefined Hash algo %u specified", 1201 auth_xform->algo); 1202 return -1; 1203 } 1204 1205 return 0; 1206 } 1207 1208 static int 1209 virtio_crypto_sym_pad_op_ctrl_req( 1210 struct virtio_crypto_op_ctrl_req *ctrl, 1211 struct rte_crypto_sym_xform *xform, bool is_chainned, 1212 uint8_t *cipher_key_data, uint8_t *auth_key_data, 1213 struct virtio_crypto_session *session) 1214 { 1215 int ret; 1216 struct rte_crypto_auth_xform *auth_xform = NULL; 1217 struct rte_crypto_cipher_xform *cipher_xform = NULL; 1218 1219 /* Get cipher xform from crypto xform chain */ 1220 cipher_xform = virtio_crypto_get_cipher_xform(xform); 1221 if (cipher_xform) { 1222 if (cipher_xform->key.length > VIRTIO_CRYPTO_MAX_KEY_SIZE) { 1223 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1224 "cipher key size cannot be longer than %u", 1225 VIRTIO_CRYPTO_MAX_KEY_SIZE); 1226 return -1; 1227 } 1228 if (cipher_xform->iv.length > VIRTIO_CRYPTO_MAX_IV_SIZE) { 1229 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1230 "cipher IV size cannot be longer than %u", 1231 VIRTIO_CRYPTO_MAX_IV_SIZE); 1232 return -1; 1233 } 1234 if (is_chainned) 1235 ret = virtio_crypto_sym_pad_cipher_param( 1236 &ctrl->u.sym_create_session.u.chain.para 1237 .cipher_param, cipher_xform); 1238 else 1239 ret = virtio_crypto_sym_pad_cipher_param( 1240 &ctrl->u.sym_create_session.u.cipher.para, 1241 cipher_xform); 1242 1243 if (ret < 0) { 1244 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1245 "pad cipher parameter failed"); 1246 return -1; 1247 } 1248 1249 memcpy(cipher_key_data, cipher_xform->key.data, 1250 cipher_xform->key.length); 1251 1252 session->iv.offset = cipher_xform->iv.offset; 1253 session->iv.length = cipher_xform->iv.length; 1254 } 1255 1256 /* Get auth xform from crypto xform chain */ 1257 auth_xform = virtio_crypto_get_auth_xform(xform); 1258 if (auth_xform) { 1259 /* FIXME: support VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */ 1260 struct virtio_crypto_alg_chain_session_para *para = 1261 &(ctrl->u.sym_create_session.u.chain.para); 1262 if (auth_xform->key.length) { 1263 if (auth_xform->key.length > 1264 VIRTIO_CRYPTO_MAX_KEY_SIZE) { 1265 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1266 "auth key size cannot be longer than %u", 1267 VIRTIO_CRYPTO_MAX_KEY_SIZE); 1268 return -1; 1269 } 1270 para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH; 1271 para->u.mac_param.auth_key_len = 1272 (uint32_t)auth_xform->key.length; 1273 para->u.mac_param.hash_result_len = 1274 auth_xform->digest_length; 1275 memcpy(auth_key_data, auth_xform->key.data, 1276 auth_xform->key.length); 1277 } else { 1278 para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN; 1279 para->u.hash_param.hash_result_len = 1280 auth_xform->digest_length; 1281 } 1282 1283 ret = virtio_crypto_sym_pad_auth_param(ctrl, auth_xform); 1284 if (ret < 0) { 1285 VIRTIO_CRYPTO_SESSION_LOG_ERR("pad auth parameter " 1286 "failed"); 1287 return -1; 1288 } 1289 } 1290 1291 return 0; 1292 } 1293 1294 static int 1295 virtio_crypto_check_sym_configure_session_paras( 1296 struct rte_cryptodev *dev, 1297 struct rte_crypto_sym_xform *xform, 1298 struct rte_cryptodev_sym_session *sym_sess, 1299 struct rte_mempool *mempool) 1300 { 1301 if (unlikely(xform == NULL) || unlikely(sym_sess == NULL) || 1302 unlikely(mempool == NULL)) { 1303 VIRTIO_CRYPTO_SESSION_LOG_ERR("NULL pointer"); 1304 return -1; 1305 } 1306 1307 if (virtio_crypto_check_sym_session_paras(dev) < 0) 1308 return -1; 1309 1310 return 0; 1311 } 1312 1313 static int 1314 virtio_crypto_sym_configure_session( 1315 struct rte_cryptodev *dev, 1316 struct rte_crypto_sym_xform *xform, 1317 struct rte_cryptodev_sym_session *sess, 1318 struct rte_mempool *mempool) 1319 { 1320 int ret; 1321 struct virtio_crypto_session crypto_sess; 1322 void *session_private = &crypto_sess; 1323 struct virtio_crypto_session *session; 1324 struct virtio_crypto_op_ctrl_req *ctrl_req; 1325 enum virtio_crypto_cmd_id cmd_id; 1326 uint8_t cipher_key_data[VIRTIO_CRYPTO_MAX_KEY_SIZE] = {0}; 1327 uint8_t auth_key_data[VIRTIO_CRYPTO_MAX_KEY_SIZE] = {0}; 1328 struct virtio_crypto_hw *hw; 1329 struct virtqueue *control_vq; 1330 1331 PMD_INIT_FUNC_TRACE(); 1332 1333 ret = virtio_crypto_check_sym_configure_session_paras(dev, xform, 1334 sess, mempool); 1335 if (ret < 0) { 1336 VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid parameters"); 1337 return ret; 1338 } 1339 1340 if (rte_mempool_get(mempool, &session_private)) { 1341 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1342 "Couldn't get object from session mempool"); 1343 return -ENOMEM; 1344 } 1345 1346 session = (struct virtio_crypto_session *)session_private; 1347 memset(session, 0, sizeof(struct virtio_crypto_session)); 1348 ctrl_req = &session->ctrl; 1349 ctrl_req->header.opcode = VIRTIO_CRYPTO_CIPHER_CREATE_SESSION; 1350 /* FIXME: support multiqueue */ 1351 ctrl_req->header.queue_id = 0; 1352 1353 hw = dev->data->dev_private; 1354 control_vq = hw->cvq; 1355 1356 cmd_id = virtio_crypto_get_chain_order(xform); 1357 if (cmd_id == VIRTIO_CRYPTO_CMD_CIPHER_HASH) 1358 ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order 1359 = VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 1360 if (cmd_id == VIRTIO_CRYPTO_CMD_HASH_CIPHER) 1361 ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order 1362 = VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; 1363 1364 switch (cmd_id) { 1365 case VIRTIO_CRYPTO_CMD_CIPHER_HASH: 1366 case VIRTIO_CRYPTO_CMD_HASH_CIPHER: 1367 ctrl_req->u.sym_create_session.op_type 1368 = VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING; 1369 1370 ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req, 1371 xform, true, cipher_key_data, auth_key_data, session); 1372 if (ret < 0) { 1373 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1374 "padding sym op ctrl req failed"); 1375 goto error_out; 1376 } 1377 ret = virtio_crypto_send_command(control_vq, ctrl_req, 1378 cipher_key_data, auth_key_data, session); 1379 if (ret < 0) { 1380 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1381 "create session failed: %d", ret); 1382 goto error_out; 1383 } 1384 break; 1385 case VIRTIO_CRYPTO_CMD_CIPHER: 1386 ctrl_req->u.sym_create_session.op_type 1387 = VIRTIO_CRYPTO_SYM_OP_CIPHER; 1388 ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req, xform, 1389 false, cipher_key_data, auth_key_data, session); 1390 if (ret < 0) { 1391 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1392 "padding sym op ctrl req failed"); 1393 goto error_out; 1394 } 1395 ret = virtio_crypto_send_command(control_vq, ctrl_req, 1396 cipher_key_data, NULL, session); 1397 if (ret < 0) { 1398 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1399 "create session failed: %d", ret); 1400 goto error_out; 1401 } 1402 break; 1403 default: 1404 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1405 "Unsupported operation chain order parameter"); 1406 goto error_out; 1407 } 1408 1409 set_sym_session_private_data(sess, dev->driver_id, 1410 session_private); 1411 1412 return 0; 1413 1414 error_out: 1415 return -1; 1416 } 1417 1418 static void 1419 virtio_crypto_dev_info_get(struct rte_cryptodev *dev, 1420 struct rte_cryptodev_info *info) 1421 { 1422 struct virtio_crypto_hw *hw = dev->data->dev_private; 1423 1424 PMD_INIT_FUNC_TRACE(); 1425 1426 if (info != NULL) { 1427 info->driver_id = cryptodev_virtio_driver_id; 1428 info->feature_flags = dev->feature_flags; 1429 info->max_nb_queue_pairs = hw->max_dataqueues; 1430 /* No limit of number of sessions */ 1431 info->sym.max_nb_sessions = 0; 1432 info->capabilities = hw->virtio_dev_capabilities; 1433 } 1434 } 1435 1436 static int 1437 crypto_virtio_pci_probe( 1438 struct rte_pci_driver *pci_drv __rte_unused, 1439 struct rte_pci_device *pci_dev) 1440 { 1441 struct rte_cryptodev_pmd_init_params init_params = { 1442 .name = "", 1443 .socket_id = pci_dev->device.numa_node, 1444 .private_data_size = sizeof(struct virtio_crypto_hw) 1445 }; 1446 char name[RTE_CRYPTODEV_NAME_MAX_LEN]; 1447 1448 VIRTIO_CRYPTO_DRV_LOG_DBG("Found Crypto device at %02x:%02x.%x", 1449 pci_dev->addr.bus, 1450 pci_dev->addr.devid, 1451 pci_dev->addr.function); 1452 1453 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 1454 1455 return crypto_virtio_create(name, pci_dev, &init_params); 1456 } 1457 1458 static int 1459 crypto_virtio_pci_remove( 1460 struct rte_pci_device *pci_dev __rte_unused) 1461 { 1462 struct rte_cryptodev *cryptodev; 1463 char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; 1464 1465 if (pci_dev == NULL) 1466 return -EINVAL; 1467 1468 rte_pci_device_name(&pci_dev->addr, cryptodev_name, 1469 sizeof(cryptodev_name)); 1470 1471 cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name); 1472 if (cryptodev == NULL) 1473 return -ENODEV; 1474 1475 return virtio_crypto_dev_uninit(cryptodev); 1476 } 1477 1478 static struct rte_pci_driver rte_virtio_crypto_driver = { 1479 .id_table = pci_id_virtio_crypto_map, 1480 .drv_flags = 0, 1481 .probe = crypto_virtio_pci_probe, 1482 .remove = crypto_virtio_pci_remove 1483 }; 1484 1485 static struct cryptodev_driver virtio_crypto_drv; 1486 1487 RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_VIRTIO_PMD, rte_virtio_crypto_driver); 1488 RTE_PMD_REGISTER_CRYPTO_DRIVER(virtio_crypto_drv, 1489 rte_virtio_crypto_driver.driver, 1490 cryptodev_virtio_driver_id); 1491 1492 RTE_INIT(virtio_crypto_init_log) 1493 { 1494 virtio_crypto_logtype_init = rte_log_register("pmd.crypto.virtio.init"); 1495 if (virtio_crypto_logtype_init >= 0) 1496 rte_log_set_level(virtio_crypto_logtype_init, RTE_LOG_NOTICE); 1497 1498 virtio_crypto_logtype_session = 1499 rte_log_register("pmd.crypto.virtio.session"); 1500 if (virtio_crypto_logtype_session >= 0) 1501 rte_log_set_level(virtio_crypto_logtype_session, 1502 RTE_LOG_NOTICE); 1503 1504 virtio_crypto_logtype_rx = rte_log_register("pmd.crypto.virtio.rx"); 1505 if (virtio_crypto_logtype_rx >= 0) 1506 rte_log_set_level(virtio_crypto_logtype_rx, RTE_LOG_NOTICE); 1507 1508 virtio_crypto_logtype_tx = rte_log_register("pmd.crypto.virtio.tx"); 1509 if (virtio_crypto_logtype_tx >= 0) 1510 rte_log_set_level(virtio_crypto_logtype_tx, RTE_LOG_NOTICE); 1511 1512 virtio_crypto_logtype_driver = 1513 rte_log_register("pmd.crypto.virtio.driver"); 1514 if (virtio_crypto_logtype_driver >= 0) 1515 rte_log_set_level(virtio_crypto_logtype_driver, RTE_LOG_NOTICE); 1516 } 1517