1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD. 3 */ 4 #include <stdbool.h> 5 #include <unistd.h> 6 7 #include <rte_common.h> 8 #include <rte_errno.h> 9 #include <rte_pci.h> 10 #include <rte_bus_pci.h> 11 #include <rte_cryptodev.h> 12 #include <rte_cryptodev_pmd.h> 13 #include <rte_eal.h> 14 15 #include "virtio_cryptodev.h" 16 #include "virtqueue.h" 17 #include "virtio_crypto_algs.h" 18 #include "virtio_crypto_capabilities.h" 19 20 int virtio_crypto_logtype_init; 21 int virtio_crypto_logtype_session; 22 int virtio_crypto_logtype_rx; 23 int virtio_crypto_logtype_tx; 24 int virtio_crypto_logtype_driver; 25 26 static int virtio_crypto_dev_configure(struct rte_cryptodev *dev, 27 struct rte_cryptodev_config *config); 28 static int virtio_crypto_dev_start(struct rte_cryptodev *dev); 29 static void virtio_crypto_dev_stop(struct rte_cryptodev *dev); 30 static int virtio_crypto_dev_close(struct rte_cryptodev *dev); 31 static void virtio_crypto_dev_info_get(struct rte_cryptodev *dev, 32 struct rte_cryptodev_info *dev_info); 33 static void virtio_crypto_dev_stats_get(struct rte_cryptodev *dev, 34 struct rte_cryptodev_stats *stats); 35 static void virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev); 36 static int virtio_crypto_qp_setup(struct rte_cryptodev *dev, 37 uint16_t queue_pair_id, 38 const struct rte_cryptodev_qp_conf *qp_conf, 39 int socket_id, 40 struct rte_mempool *session_pool); 41 static int virtio_crypto_qp_release(struct rte_cryptodev *dev, 42 uint16_t queue_pair_id); 43 static void virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev); 44 static unsigned int virtio_crypto_sym_get_session_private_size( 45 struct rte_cryptodev *dev); 46 static void virtio_crypto_sym_clear_session(struct rte_cryptodev *dev, 47 struct rte_cryptodev_sym_session *sess); 48 static int virtio_crypto_sym_configure_session(struct rte_cryptodev *dev, 49 struct rte_crypto_sym_xform *xform, 50 struct rte_cryptodev_sym_session *session, 51 struct rte_mempool *mp); 52 53 /* 54 * The set of PCI devices this driver supports 55 */ 56 static const struct rte_pci_id pci_id_virtio_crypto_map[] = { 57 { RTE_PCI_DEVICE(VIRTIO_CRYPTO_PCI_VENDORID, 58 VIRTIO_CRYPTO_PCI_DEVICEID) }, 59 { .vendor_id = 0, /* sentinel */ }, 60 }; 61 62 static const struct rte_cryptodev_capabilities virtio_capabilities[] = { 63 VIRTIO_SYM_CAPABILITIES, 64 RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() 65 }; 66 67 uint8_t cryptodev_virtio_driver_id; 68 69 #define NUM_ENTRY_SYM_CREATE_SESSION 4 70 71 static int 72 virtio_crypto_send_command(struct virtqueue *vq, 73 struct virtio_crypto_op_ctrl_req *ctrl, uint8_t *cipher_key, 74 uint8_t *auth_key, struct virtio_crypto_session *session) 75 { 76 uint8_t idx = 0; 77 uint8_t needed = 1; 78 uint32_t head = 0; 79 uint32_t len_cipher_key = 0; 80 uint32_t len_auth_key = 0; 81 uint32_t len_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req); 82 uint32_t len_session_input = sizeof(struct virtio_crypto_session_input); 83 uint32_t len_total = 0; 84 uint32_t input_offset = 0; 85 void *virt_addr_started = NULL; 86 phys_addr_t phys_addr_started; 87 struct vring_desc *desc; 88 uint32_t desc_offset; 89 struct virtio_crypto_session_input *input; 90 int ret; 91 92 PMD_INIT_FUNC_TRACE(); 93 94 if (session == NULL) { 95 VIRTIO_CRYPTO_SESSION_LOG_ERR("session is NULL."); 96 return -EINVAL; 97 } 98 /* cipher only is supported, it is available if auth_key is NULL */ 99 if (!cipher_key) { 100 VIRTIO_CRYPTO_SESSION_LOG_ERR("cipher key is NULL."); 101 return -EINVAL; 102 } 103 104 head = vq->vq_desc_head_idx; 105 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_desc_head_idx = %d, vq = %p", 106 head, vq); 107 108 if (vq->vq_free_cnt < needed) { 109 VIRTIO_CRYPTO_SESSION_LOG_ERR("Not enough entry"); 110 return -ENOSPC; 111 } 112 113 /* calculate the length of cipher key */ 114 if (cipher_key) { 115 switch (ctrl->u.sym_create_session.op_type) { 116 case VIRTIO_CRYPTO_SYM_OP_CIPHER: 117 len_cipher_key 118 = ctrl->u.sym_create_session.u.cipher 119 .para.keylen; 120 break; 121 case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING: 122 len_cipher_key 123 = ctrl->u.sym_create_session.u.chain 124 .para.cipher_param.keylen; 125 break; 126 default: 127 VIRTIO_CRYPTO_SESSION_LOG_ERR("invalid op type"); 128 return -EINVAL; 129 } 130 } 131 132 /* calculate the length of auth key */ 133 if (auth_key) { 134 len_auth_key = 135 ctrl->u.sym_create_session.u.chain.para.u.mac_param 136 .auth_key_len; 137 } 138 139 /* 140 * malloc memory to store indirect vring_desc entries, including 141 * ctrl request, cipher key, auth key, session input and desc vring 142 */ 143 desc_offset = len_ctrl_req + len_cipher_key + len_auth_key 144 + len_session_input; 145 virt_addr_started = rte_malloc(NULL, 146 desc_offset + NUM_ENTRY_SYM_CREATE_SESSION 147 * sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE); 148 if (virt_addr_started == NULL) { 149 VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap memory"); 150 return -ENOSPC; 151 } 152 phys_addr_started = rte_malloc_virt2iova(virt_addr_started); 153 154 /* address to store indirect vring desc entries */ 155 desc = (struct vring_desc *) 156 ((uint8_t *)virt_addr_started + desc_offset); 157 158 /* ctrl req part */ 159 memcpy(virt_addr_started, ctrl, len_ctrl_req); 160 desc[idx].addr = phys_addr_started; 161 desc[idx].len = len_ctrl_req; 162 desc[idx].flags = VRING_DESC_F_NEXT; 163 desc[idx].next = idx + 1; 164 idx++; 165 len_total += len_ctrl_req; 166 input_offset += len_ctrl_req; 167 168 /* cipher key part */ 169 if (len_cipher_key > 0) { 170 memcpy((uint8_t *)virt_addr_started + len_total, 171 cipher_key, len_cipher_key); 172 173 desc[idx].addr = phys_addr_started + len_total; 174 desc[idx].len = len_cipher_key; 175 desc[idx].flags = VRING_DESC_F_NEXT; 176 desc[idx].next = idx + 1; 177 idx++; 178 len_total += len_cipher_key; 179 input_offset += len_cipher_key; 180 } 181 182 /* auth key part */ 183 if (len_auth_key > 0) { 184 memcpy((uint8_t *)virt_addr_started + len_total, 185 auth_key, len_auth_key); 186 187 desc[idx].addr = phys_addr_started + len_total; 188 desc[idx].len = len_auth_key; 189 desc[idx].flags = VRING_DESC_F_NEXT; 190 desc[idx].next = idx + 1; 191 idx++; 192 len_total += len_auth_key; 193 input_offset += len_auth_key; 194 } 195 196 /* input part */ 197 input = (struct virtio_crypto_session_input *) 198 ((uint8_t *)virt_addr_started + input_offset); 199 input->status = VIRTIO_CRYPTO_ERR; 200 input->session_id = ~0ULL; 201 desc[idx].addr = phys_addr_started + len_total; 202 desc[idx].len = len_session_input; 203 desc[idx].flags = VRING_DESC_F_WRITE; 204 idx++; 205 206 /* use a single desc entry */ 207 vq->vq_ring.desc[head].addr = phys_addr_started + desc_offset; 208 vq->vq_ring.desc[head].len = idx * sizeof(struct vring_desc); 209 vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT; 210 vq->vq_free_cnt--; 211 212 vq->vq_desc_head_idx = vq->vq_ring.desc[head].next; 213 214 vq_update_avail_ring(vq, head); 215 vq_update_avail_idx(vq); 216 217 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d", 218 vq->vq_queue_index); 219 220 virtqueue_notify(vq); 221 222 rte_rmb(); 223 while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) { 224 rte_rmb(); 225 usleep(100); 226 } 227 228 while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) { 229 uint32_t idx, desc_idx, used_idx; 230 struct vring_used_elem *uep; 231 232 used_idx = (uint32_t)(vq->vq_used_cons_idx 233 & (vq->vq_nentries - 1)); 234 uep = &vq->vq_ring.used->ring[used_idx]; 235 idx = (uint32_t) uep->id; 236 desc_idx = idx; 237 238 while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) { 239 desc_idx = vq->vq_ring.desc[desc_idx].next; 240 vq->vq_free_cnt++; 241 } 242 243 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx; 244 vq->vq_desc_head_idx = idx; 245 246 vq->vq_used_cons_idx++; 247 vq->vq_free_cnt++; 248 } 249 250 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n" 251 "vq->vq_desc_head_idx=%d", 252 vq->vq_free_cnt, vq->vq_desc_head_idx); 253 254 /* get the result */ 255 if (input->status != VIRTIO_CRYPTO_OK) { 256 VIRTIO_CRYPTO_SESSION_LOG_ERR("Something wrong on backend! " 257 "status=%u, session_id=%" PRIu64 "", 258 input->status, input->session_id); 259 rte_free(virt_addr_started); 260 ret = -1; 261 } else { 262 session->session_id = input->session_id; 263 264 VIRTIO_CRYPTO_SESSION_LOG_INFO("Create session successfully, " 265 "session_id=%" PRIu64 "", input->session_id); 266 rte_free(virt_addr_started); 267 ret = 0; 268 } 269 270 return ret; 271 } 272 273 void 274 virtio_crypto_queue_release(struct virtqueue *vq) 275 { 276 struct virtio_crypto_hw *hw; 277 278 PMD_INIT_FUNC_TRACE(); 279 280 if (vq) { 281 hw = vq->hw; 282 /* Select and deactivate the queue */ 283 VTPCI_OPS(hw)->del_queue(hw, vq); 284 285 rte_memzone_free(vq->mz); 286 rte_mempool_free(vq->mpool); 287 rte_free(vq); 288 } 289 } 290 291 #define MPOOL_MAX_NAME_SZ 32 292 293 int 294 virtio_crypto_queue_setup(struct rte_cryptodev *dev, 295 int queue_type, 296 uint16_t vtpci_queue_idx, 297 uint16_t nb_desc, 298 int socket_id, 299 struct virtqueue **pvq) 300 { 301 char vq_name[VIRTQUEUE_MAX_NAME_SZ]; 302 char mpool_name[MPOOL_MAX_NAME_SZ]; 303 const struct rte_memzone *mz; 304 unsigned int vq_size, size; 305 struct virtio_crypto_hw *hw = dev->data->dev_private; 306 struct virtqueue *vq = NULL; 307 uint32_t i = 0; 308 uint32_t j; 309 310 PMD_INIT_FUNC_TRACE(); 311 312 VIRTIO_CRYPTO_INIT_LOG_DBG("setting up queue: %u", vtpci_queue_idx); 313 314 /* 315 * Read the virtqueue size from the Queue Size field 316 * Always power of 2 and if 0 virtqueue does not exist 317 */ 318 vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx); 319 if (vq_size == 0) { 320 VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue does not exist"); 321 return -EINVAL; 322 } 323 VIRTIO_CRYPTO_INIT_LOG_DBG("vq_size: %u", vq_size); 324 325 if (!rte_is_power_of_2(vq_size)) { 326 VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue size is not powerof 2"); 327 return -EINVAL; 328 } 329 330 if (queue_type == VTCRYPTO_DATAQ) { 331 snprintf(vq_name, sizeof(vq_name), "dev%d_dataqueue%d", 332 dev->data->dev_id, vtpci_queue_idx); 333 snprintf(mpool_name, sizeof(mpool_name), 334 "dev%d_dataqueue%d_mpool", 335 dev->data->dev_id, vtpci_queue_idx); 336 } else if (queue_type == VTCRYPTO_CTRLQ) { 337 snprintf(vq_name, sizeof(vq_name), "dev%d_controlqueue", 338 dev->data->dev_id); 339 snprintf(mpool_name, sizeof(mpool_name), 340 "dev%d_controlqueue_mpool", 341 dev->data->dev_id); 342 } 343 size = RTE_ALIGN_CEIL(sizeof(*vq) + 344 vq_size * sizeof(struct vq_desc_extra), 345 RTE_CACHE_LINE_SIZE); 346 vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE, 347 socket_id); 348 if (vq == NULL) { 349 VIRTIO_CRYPTO_INIT_LOG_ERR("Can not allocate virtqueue"); 350 return -ENOMEM; 351 } 352 353 if (queue_type == VTCRYPTO_DATAQ) { 354 /* pre-allocate a mempool and use it in the data plane to 355 * improve performance 356 */ 357 vq->mpool = rte_mempool_lookup(mpool_name); 358 if (vq->mpool == NULL) 359 vq->mpool = rte_mempool_create(mpool_name, 360 vq_size, 361 sizeof(struct virtio_crypto_op_cookie), 362 RTE_CACHE_LINE_SIZE, 0, 363 NULL, NULL, NULL, NULL, socket_id, 364 0); 365 if (!vq->mpool) { 366 VIRTIO_CRYPTO_DRV_LOG_ERR("Virtio Crypto PMD " 367 "Cannot create mempool"); 368 goto mpool_create_err; 369 } 370 for (i = 0; i < vq_size; i++) { 371 vq->vq_descx[i].cookie = 372 rte_zmalloc("crypto PMD op cookie pointer", 373 sizeof(struct virtio_crypto_op_cookie), 374 RTE_CACHE_LINE_SIZE); 375 if (vq->vq_descx[i].cookie == NULL) { 376 VIRTIO_CRYPTO_DRV_LOG_ERR("Failed to " 377 "alloc mem for cookie"); 378 goto cookie_alloc_err; 379 } 380 } 381 } 382 383 vq->hw = hw; 384 vq->dev_id = dev->data->dev_id; 385 vq->vq_queue_index = vtpci_queue_idx; 386 vq->vq_nentries = vq_size; 387 388 /* 389 * Using part of the vring entries is permitted, but the maximum 390 * is vq_size 391 */ 392 if (nb_desc == 0 || nb_desc > vq_size) 393 nb_desc = vq_size; 394 vq->vq_free_cnt = nb_desc; 395 396 /* 397 * Reserve a memzone for vring elements 398 */ 399 size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN); 400 vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN); 401 VIRTIO_CRYPTO_INIT_LOG_DBG("%s vring_size: %d, rounded_vring_size: %d", 402 (queue_type == VTCRYPTO_DATAQ) ? "dataq" : "ctrlq", 403 size, vq->vq_ring_size); 404 405 mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, 406 socket_id, 0, VIRTIO_PCI_VRING_ALIGN); 407 if (mz == NULL) { 408 if (rte_errno == EEXIST) 409 mz = rte_memzone_lookup(vq_name); 410 if (mz == NULL) { 411 VIRTIO_CRYPTO_INIT_LOG_ERR("not enough memory"); 412 goto mz_reserve_err; 413 } 414 } 415 416 /* 417 * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit, 418 * and only accepts 32 bit page frame number. 419 * Check if the allocated physical memory exceeds 16TB. 420 */ 421 if ((mz->phys_addr + vq->vq_ring_size - 1) 422 >> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) { 423 VIRTIO_CRYPTO_INIT_LOG_ERR("vring address shouldn't be " 424 "above 16TB!"); 425 goto vring_addr_err; 426 } 427 428 memset(mz->addr, 0, sizeof(mz->len)); 429 vq->mz = mz; 430 vq->vq_ring_mem = mz->phys_addr; 431 vq->vq_ring_virt_mem = mz->addr; 432 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_mem(physical): 0x%"PRIx64, 433 (uint64_t)mz->phys_addr); 434 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_virt_mem: 0x%"PRIx64, 435 (uint64_t)(uintptr_t)mz->addr); 436 437 *pvq = vq; 438 439 return 0; 440 441 vring_addr_err: 442 rte_memzone_free(mz); 443 mz_reserve_err: 444 cookie_alloc_err: 445 rte_mempool_free(vq->mpool); 446 if (i != 0) { 447 for (j = 0; j < i; j++) 448 rte_free(vq->vq_descx[j].cookie); 449 } 450 mpool_create_err: 451 rte_free(vq); 452 return -ENOMEM; 453 } 454 455 static int 456 virtio_crypto_ctrlq_setup(struct rte_cryptodev *dev, uint16_t queue_idx) 457 { 458 int ret; 459 struct virtqueue *vq; 460 struct virtio_crypto_hw *hw = dev->data->dev_private; 461 462 /* if virtio device has started, do not touch the virtqueues */ 463 if (dev->data->dev_started) 464 return 0; 465 466 PMD_INIT_FUNC_TRACE(); 467 468 ret = virtio_crypto_queue_setup(dev, VTCRYPTO_CTRLQ, queue_idx, 469 0, SOCKET_ID_ANY, &vq); 470 if (ret < 0) { 471 VIRTIO_CRYPTO_INIT_LOG_ERR("control vq initialization failed"); 472 return ret; 473 } 474 475 hw->cvq = vq; 476 477 return 0; 478 } 479 480 static void 481 virtio_crypto_free_queues(struct rte_cryptodev *dev) 482 { 483 unsigned int i; 484 struct virtio_crypto_hw *hw = dev->data->dev_private; 485 486 PMD_INIT_FUNC_TRACE(); 487 488 /* control queue release */ 489 virtio_crypto_queue_release(hw->cvq); 490 491 /* data queue release */ 492 for (i = 0; i < hw->max_dataqueues; i++) 493 virtio_crypto_queue_release(dev->data->queue_pairs[i]); 494 } 495 496 static int 497 virtio_crypto_dev_close(struct rte_cryptodev *dev __rte_unused) 498 { 499 return 0; 500 } 501 502 /* 503 * dev_ops for virtio, bare necessities for basic operation 504 */ 505 static struct rte_cryptodev_ops virtio_crypto_dev_ops = { 506 /* Device related operations */ 507 .dev_configure = virtio_crypto_dev_configure, 508 .dev_start = virtio_crypto_dev_start, 509 .dev_stop = virtio_crypto_dev_stop, 510 .dev_close = virtio_crypto_dev_close, 511 .dev_infos_get = virtio_crypto_dev_info_get, 512 513 .stats_get = virtio_crypto_dev_stats_get, 514 .stats_reset = virtio_crypto_dev_stats_reset, 515 516 .queue_pair_setup = virtio_crypto_qp_setup, 517 .queue_pair_release = virtio_crypto_qp_release, 518 .queue_pair_count = NULL, 519 520 /* Crypto related operations */ 521 .sym_session_get_size = virtio_crypto_sym_get_session_private_size, 522 .sym_session_configure = virtio_crypto_sym_configure_session, 523 .sym_session_clear = virtio_crypto_sym_clear_session 524 }; 525 526 static void 527 virtio_crypto_update_stats(struct rte_cryptodev *dev, 528 struct rte_cryptodev_stats *stats) 529 { 530 unsigned int i; 531 struct virtio_crypto_hw *hw = dev->data->dev_private; 532 533 PMD_INIT_FUNC_TRACE(); 534 535 if (stats == NULL) { 536 VIRTIO_CRYPTO_DRV_LOG_ERR("invalid pointer"); 537 return; 538 } 539 540 for (i = 0; i < hw->max_dataqueues; i++) { 541 const struct virtqueue *data_queue 542 = dev->data->queue_pairs[i]; 543 if (data_queue == NULL) 544 continue; 545 546 stats->enqueued_count += data_queue->packets_sent_total; 547 stats->enqueue_err_count += data_queue->packets_sent_failed; 548 549 stats->dequeued_count += data_queue->packets_received_total; 550 stats->dequeue_err_count 551 += data_queue->packets_received_failed; 552 } 553 } 554 555 static void 556 virtio_crypto_dev_stats_get(struct rte_cryptodev *dev, 557 struct rte_cryptodev_stats *stats) 558 { 559 PMD_INIT_FUNC_TRACE(); 560 561 virtio_crypto_update_stats(dev, stats); 562 } 563 564 static void 565 virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev) 566 { 567 unsigned int i; 568 struct virtio_crypto_hw *hw = dev->data->dev_private; 569 570 PMD_INIT_FUNC_TRACE(); 571 572 for (i = 0; i < hw->max_dataqueues; i++) { 573 struct virtqueue *data_queue = dev->data->queue_pairs[i]; 574 if (data_queue == NULL) 575 continue; 576 577 data_queue->packets_sent_total = 0; 578 data_queue->packets_sent_failed = 0; 579 580 data_queue->packets_received_total = 0; 581 data_queue->packets_received_failed = 0; 582 } 583 } 584 585 static int 586 virtio_crypto_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, 587 const struct rte_cryptodev_qp_conf *qp_conf, 588 int socket_id, 589 struct rte_mempool *session_pool __rte_unused) 590 { 591 int ret; 592 struct virtqueue *vq; 593 594 PMD_INIT_FUNC_TRACE(); 595 596 /* if virtio dev is started, do not touch the virtqueues */ 597 if (dev->data->dev_started) 598 return 0; 599 600 ret = virtio_crypto_queue_setup(dev, VTCRYPTO_DATAQ, queue_pair_id, 601 qp_conf->nb_descriptors, socket_id, &vq); 602 if (ret < 0) { 603 VIRTIO_CRYPTO_INIT_LOG_ERR( 604 "virtio crypto data queue initialization failed\n"); 605 return ret; 606 } 607 608 dev->data->queue_pairs[queue_pair_id] = vq; 609 610 return 0; 611 } 612 613 static int 614 virtio_crypto_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) 615 { 616 struct virtqueue *vq 617 = (struct virtqueue *)dev->data->queue_pairs[queue_pair_id]; 618 619 PMD_INIT_FUNC_TRACE(); 620 621 if (vq == NULL) { 622 VIRTIO_CRYPTO_DRV_LOG_DBG("vq already freed"); 623 return 0; 624 } 625 626 virtio_crypto_queue_release(vq); 627 return 0; 628 } 629 630 static int 631 virtio_negotiate_features(struct virtio_crypto_hw *hw, uint64_t req_features) 632 { 633 uint64_t host_features; 634 635 PMD_INIT_FUNC_TRACE(); 636 637 /* Prepare guest_features: feature that driver wants to support */ 638 VIRTIO_CRYPTO_INIT_LOG_DBG("guest_features before negotiate = %" PRIx64, 639 req_features); 640 641 /* Read device(host) feature bits */ 642 host_features = VTPCI_OPS(hw)->get_features(hw); 643 VIRTIO_CRYPTO_INIT_LOG_DBG("host_features before negotiate = %" PRIx64, 644 host_features); 645 646 /* 647 * Negotiate features: Subset of device feature bits are written back 648 * guest feature bits. 649 */ 650 hw->guest_features = req_features; 651 hw->guest_features = vtpci_cryptodev_negotiate_features(hw, 652 host_features); 653 VIRTIO_CRYPTO_INIT_LOG_DBG("features after negotiate = %" PRIx64, 654 hw->guest_features); 655 656 if (hw->modern) { 657 if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) { 658 VIRTIO_CRYPTO_INIT_LOG_ERR( 659 "VIRTIO_F_VERSION_1 features is not enabled."); 660 return -1; 661 } 662 vtpci_cryptodev_set_status(hw, 663 VIRTIO_CONFIG_STATUS_FEATURES_OK); 664 if (!(vtpci_cryptodev_get_status(hw) & 665 VIRTIO_CONFIG_STATUS_FEATURES_OK)) { 666 VIRTIO_CRYPTO_INIT_LOG_ERR("failed to set FEATURES_OK " 667 "status!"); 668 return -1; 669 } 670 } 671 672 hw->req_guest_features = req_features; 673 674 return 0; 675 } 676 677 /* reset device and renegotiate features if needed */ 678 static int 679 virtio_crypto_init_device(struct rte_cryptodev *cryptodev, 680 uint64_t req_features) 681 { 682 struct virtio_crypto_hw *hw = cryptodev->data->dev_private; 683 struct virtio_crypto_config local_config; 684 struct virtio_crypto_config *config = &local_config; 685 686 PMD_INIT_FUNC_TRACE(); 687 688 /* Reset the device although not necessary at startup */ 689 vtpci_cryptodev_reset(hw); 690 691 /* Tell the host we've noticed this device. */ 692 vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); 693 694 /* Tell the host we've known how to drive the device. */ 695 vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); 696 if (virtio_negotiate_features(hw, req_features) < 0) 697 return -1; 698 699 /* Get status of the device */ 700 vtpci_read_cryptodev_config(hw, 701 offsetof(struct virtio_crypto_config, status), 702 &config->status, sizeof(config->status)); 703 if (config->status != VIRTIO_CRYPTO_S_HW_READY) { 704 VIRTIO_CRYPTO_DRV_LOG_ERR("accelerator hardware is " 705 "not ready"); 706 return -1; 707 } 708 709 /* Get number of data queues */ 710 vtpci_read_cryptodev_config(hw, 711 offsetof(struct virtio_crypto_config, max_dataqueues), 712 &config->max_dataqueues, 713 sizeof(config->max_dataqueues)); 714 hw->max_dataqueues = config->max_dataqueues; 715 716 VIRTIO_CRYPTO_INIT_LOG_DBG("hw->max_dataqueues=%d", 717 hw->max_dataqueues); 718 719 return 0; 720 } 721 722 /* 723 * This function is based on probe() function 724 * It returns 0 on success. 725 */ 726 static int 727 crypto_virtio_create(const char *name, struct rte_pci_device *pci_dev, 728 struct rte_cryptodev_pmd_init_params *init_params) 729 { 730 struct rte_cryptodev *cryptodev; 731 struct virtio_crypto_hw *hw; 732 733 PMD_INIT_FUNC_TRACE(); 734 735 cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device, 736 init_params); 737 if (cryptodev == NULL) 738 return -ENODEV; 739 740 cryptodev->driver_id = cryptodev_virtio_driver_id; 741 cryptodev->dev_ops = &virtio_crypto_dev_ops; 742 743 cryptodev->enqueue_burst = virtio_crypto_pkt_tx_burst; 744 cryptodev->dequeue_burst = virtio_crypto_pkt_rx_burst; 745 746 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 747 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING; 748 749 hw = cryptodev->data->dev_private; 750 hw->dev_id = cryptodev->data->dev_id; 751 hw->virtio_dev_capabilities = virtio_capabilities; 752 753 VIRTIO_CRYPTO_INIT_LOG_DBG("dev %d vendorID=0x%x deviceID=0x%x", 754 cryptodev->data->dev_id, pci_dev->id.vendor_id, 755 pci_dev->id.device_id); 756 757 /* pci device init */ 758 if (vtpci_cryptodev_init(pci_dev, hw)) 759 return -1; 760 761 if (virtio_crypto_init_device(cryptodev, 762 VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0) 763 return -1; 764 765 return 0; 766 } 767 768 static int 769 virtio_crypto_dev_uninit(struct rte_cryptodev *cryptodev) 770 { 771 struct virtio_crypto_hw *hw = cryptodev->data->dev_private; 772 773 PMD_INIT_FUNC_TRACE(); 774 775 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 776 return -EPERM; 777 778 if (cryptodev->data->dev_started) { 779 virtio_crypto_dev_stop(cryptodev); 780 virtio_crypto_dev_close(cryptodev); 781 } 782 783 cryptodev->dev_ops = NULL; 784 cryptodev->enqueue_burst = NULL; 785 cryptodev->dequeue_burst = NULL; 786 787 /* release control queue */ 788 virtio_crypto_queue_release(hw->cvq); 789 790 rte_free(cryptodev->data); 791 cryptodev->data = NULL; 792 793 VIRTIO_CRYPTO_DRV_LOG_INFO("dev_uninit completed"); 794 795 return 0; 796 } 797 798 static int 799 virtio_crypto_dev_configure(struct rte_cryptodev *cryptodev, 800 struct rte_cryptodev_config *config __rte_unused) 801 { 802 struct virtio_crypto_hw *hw = cryptodev->data->dev_private; 803 804 PMD_INIT_FUNC_TRACE(); 805 806 if (virtio_crypto_init_device(cryptodev, 807 VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0) 808 return -1; 809 810 /* setup control queue 811 * [0, 1, ... ,(config->max_dataqueues - 1)] are data queues 812 * config->max_dataqueues is the control queue 813 */ 814 if (virtio_crypto_ctrlq_setup(cryptodev, hw->max_dataqueues) < 0) { 815 VIRTIO_CRYPTO_INIT_LOG_ERR("control queue setup error"); 816 return -1; 817 } 818 virtio_crypto_ctrlq_start(cryptodev); 819 820 return 0; 821 } 822 823 static void 824 virtio_crypto_dev_stop(struct rte_cryptodev *dev) 825 { 826 struct virtio_crypto_hw *hw = dev->data->dev_private; 827 828 PMD_INIT_FUNC_TRACE(); 829 VIRTIO_CRYPTO_DRV_LOG_DBG("virtio_dev_stop"); 830 831 vtpci_cryptodev_reset(hw); 832 833 virtio_crypto_dev_free_mbufs(dev); 834 virtio_crypto_free_queues(dev); 835 836 dev->data->dev_started = 0; 837 } 838 839 static int 840 virtio_crypto_dev_start(struct rte_cryptodev *dev) 841 { 842 struct virtio_crypto_hw *hw = dev->data->dev_private; 843 844 if (dev->data->dev_started) 845 return 0; 846 847 /* Do final configuration before queue engine starts */ 848 virtio_crypto_dataq_start(dev); 849 vtpci_cryptodev_reinit_complete(hw); 850 851 dev->data->dev_started = 1; 852 853 return 0; 854 } 855 856 static void 857 virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev) 858 { 859 uint32_t i; 860 struct virtio_crypto_hw *hw = dev->data->dev_private; 861 862 for (i = 0; i < hw->max_dataqueues; i++) { 863 VIRTIO_CRYPTO_INIT_LOG_DBG("Before freeing dataq[%d] used " 864 "and unused buf", i); 865 VIRTQUEUE_DUMP((struct virtqueue *) 866 dev->data->queue_pairs[i]); 867 868 VIRTIO_CRYPTO_INIT_LOG_DBG("queue_pairs[%d]=%p", 869 i, dev->data->queue_pairs[i]); 870 871 virtqueue_detatch_unused(dev->data->queue_pairs[i]); 872 873 VIRTIO_CRYPTO_INIT_LOG_DBG("After freeing dataq[%d] used and " 874 "unused buf", i); 875 VIRTQUEUE_DUMP( 876 (struct virtqueue *)dev->data->queue_pairs[i]); 877 } 878 } 879 880 static unsigned int 881 virtio_crypto_sym_get_session_private_size( 882 struct rte_cryptodev *dev __rte_unused) 883 { 884 PMD_INIT_FUNC_TRACE(); 885 886 return RTE_ALIGN_CEIL(sizeof(struct virtio_crypto_session), 16); 887 } 888 889 static int 890 virtio_crypto_check_sym_session_paras( 891 struct rte_cryptodev *dev) 892 { 893 struct virtio_crypto_hw *hw; 894 895 PMD_INIT_FUNC_TRACE(); 896 897 if (unlikely(dev == NULL)) { 898 VIRTIO_CRYPTO_SESSION_LOG_ERR("dev is NULL"); 899 return -1; 900 } 901 if (unlikely(dev->data == NULL)) { 902 VIRTIO_CRYPTO_SESSION_LOG_ERR("dev->data is NULL"); 903 return -1; 904 } 905 hw = dev->data->dev_private; 906 if (unlikely(hw == NULL)) { 907 VIRTIO_CRYPTO_SESSION_LOG_ERR("hw is NULL"); 908 return -1; 909 } 910 if (unlikely(hw->cvq == NULL)) { 911 VIRTIO_CRYPTO_SESSION_LOG_ERR("vq is NULL"); 912 return -1; 913 } 914 915 return 0; 916 } 917 918 static int 919 virtio_crypto_check_sym_clear_session_paras( 920 struct rte_cryptodev *dev, 921 struct rte_cryptodev_sym_session *sess) 922 { 923 PMD_INIT_FUNC_TRACE(); 924 925 if (sess == NULL) { 926 VIRTIO_CRYPTO_SESSION_LOG_ERR("sym_session is NULL"); 927 return -1; 928 } 929 930 return virtio_crypto_check_sym_session_paras(dev); 931 } 932 933 #define NUM_ENTRY_SYM_CLEAR_SESSION 2 934 935 static void 936 virtio_crypto_sym_clear_session( 937 struct rte_cryptodev *dev, 938 struct rte_cryptodev_sym_session *sess) 939 { 940 struct virtio_crypto_hw *hw; 941 struct virtqueue *vq; 942 struct virtio_crypto_session *session; 943 struct virtio_crypto_op_ctrl_req *ctrl; 944 struct vring_desc *desc; 945 uint8_t *status; 946 uint8_t needed = 1; 947 uint32_t head; 948 uint8_t *malloc_virt_addr; 949 uint64_t malloc_phys_addr; 950 uint8_t len_inhdr = sizeof(struct virtio_crypto_inhdr); 951 uint32_t len_op_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req); 952 uint32_t desc_offset = len_op_ctrl_req + len_inhdr; 953 954 PMD_INIT_FUNC_TRACE(); 955 956 if (virtio_crypto_check_sym_clear_session_paras(dev, sess) < 0) 957 return; 958 959 hw = dev->data->dev_private; 960 vq = hw->cvq; 961 session = (struct virtio_crypto_session *)get_sym_session_private_data( 962 sess, cryptodev_virtio_driver_id); 963 if (session == NULL) { 964 VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid session parameter"); 965 return; 966 } 967 968 VIRTIO_CRYPTO_SESSION_LOG_INFO("vq->vq_desc_head_idx = %d, " 969 "vq = %p", vq->vq_desc_head_idx, vq); 970 971 if (vq->vq_free_cnt < needed) { 972 VIRTIO_CRYPTO_SESSION_LOG_ERR( 973 "vq->vq_free_cnt = %d is less than %d, " 974 "not enough", vq->vq_free_cnt, needed); 975 return; 976 } 977 978 /* 979 * malloc memory to store information of ctrl request op, 980 * returned status and desc vring 981 */ 982 malloc_virt_addr = rte_malloc(NULL, len_op_ctrl_req + len_inhdr 983 + NUM_ENTRY_SYM_CLEAR_SESSION 984 * sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE); 985 if (malloc_virt_addr == NULL) { 986 VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap room"); 987 return; 988 } 989 malloc_phys_addr = rte_malloc_virt2iova(malloc_virt_addr); 990 991 /* assign ctrl request op part */ 992 ctrl = (struct virtio_crypto_op_ctrl_req *)malloc_virt_addr; 993 ctrl->header.opcode = VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION; 994 /* default data virtqueue is 0 */ 995 ctrl->header.queue_id = 0; 996 ctrl->u.destroy_session.session_id = session->session_id; 997 998 /* status part */ 999 status = &(((struct virtio_crypto_inhdr *) 1000 ((uint8_t *)malloc_virt_addr + len_op_ctrl_req))->status); 1001 *status = VIRTIO_CRYPTO_ERR; 1002 1003 /* indirect desc vring part */ 1004 desc = (struct vring_desc *)((uint8_t *)malloc_virt_addr 1005 + desc_offset); 1006 1007 /* ctrl request part */ 1008 desc[0].addr = malloc_phys_addr; 1009 desc[0].len = len_op_ctrl_req; 1010 desc[0].flags = VRING_DESC_F_NEXT; 1011 desc[0].next = 1; 1012 1013 /* status part */ 1014 desc[1].addr = malloc_phys_addr + len_op_ctrl_req; 1015 desc[1].len = len_inhdr; 1016 desc[1].flags = VRING_DESC_F_WRITE; 1017 1018 /* use only a single desc entry */ 1019 head = vq->vq_desc_head_idx; 1020 vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT; 1021 vq->vq_ring.desc[head].addr = malloc_phys_addr + desc_offset; 1022 vq->vq_ring.desc[head].len 1023 = NUM_ENTRY_SYM_CLEAR_SESSION 1024 * sizeof(struct vring_desc); 1025 vq->vq_free_cnt -= needed; 1026 1027 vq->vq_desc_head_idx = vq->vq_ring.desc[head].next; 1028 1029 vq_update_avail_ring(vq, head); 1030 vq_update_avail_idx(vq); 1031 1032 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d", 1033 vq->vq_queue_index); 1034 1035 virtqueue_notify(vq); 1036 1037 rte_rmb(); 1038 while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) { 1039 rte_rmb(); 1040 usleep(100); 1041 } 1042 1043 while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) { 1044 uint32_t idx, desc_idx, used_idx; 1045 struct vring_used_elem *uep; 1046 1047 used_idx = (uint32_t)(vq->vq_used_cons_idx 1048 & (vq->vq_nentries - 1)); 1049 uep = &vq->vq_ring.used->ring[used_idx]; 1050 idx = (uint32_t) uep->id; 1051 desc_idx = idx; 1052 while (vq->vq_ring.desc[desc_idx].flags 1053 & VRING_DESC_F_NEXT) { 1054 desc_idx = vq->vq_ring.desc[desc_idx].next; 1055 vq->vq_free_cnt++; 1056 } 1057 1058 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx; 1059 vq->vq_desc_head_idx = idx; 1060 vq->vq_used_cons_idx++; 1061 vq->vq_free_cnt++; 1062 } 1063 1064 if (*status != VIRTIO_CRYPTO_OK) { 1065 VIRTIO_CRYPTO_SESSION_LOG_ERR("Close session failed " 1066 "status=%"PRIu32", session_id=%"PRIu64"", 1067 *status, session->session_id); 1068 rte_free(malloc_virt_addr); 1069 return; 1070 } 1071 1072 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n" 1073 "vq->vq_desc_head_idx=%d", 1074 vq->vq_free_cnt, vq->vq_desc_head_idx); 1075 1076 VIRTIO_CRYPTO_SESSION_LOG_INFO("Close session %"PRIu64" successfully ", 1077 session->session_id); 1078 1079 memset(sess, 0, sizeof(struct virtio_crypto_session)); 1080 rte_free(malloc_virt_addr); 1081 } 1082 1083 static struct rte_crypto_cipher_xform * 1084 virtio_crypto_get_cipher_xform(struct rte_crypto_sym_xform *xform) 1085 { 1086 do { 1087 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) 1088 return &xform->cipher; 1089 1090 xform = xform->next; 1091 } while (xform); 1092 1093 return NULL; 1094 } 1095 1096 static struct rte_crypto_auth_xform * 1097 virtio_crypto_get_auth_xform(struct rte_crypto_sym_xform *xform) 1098 { 1099 do { 1100 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) 1101 return &xform->auth; 1102 1103 xform = xform->next; 1104 } while (xform); 1105 1106 return NULL; 1107 } 1108 1109 /** Get xform chain order */ 1110 static int 1111 virtio_crypto_get_chain_order(struct rte_crypto_sym_xform *xform) 1112 { 1113 if (xform == NULL) 1114 return -1; 1115 1116 /* Cipher Only */ 1117 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 1118 xform->next == NULL) 1119 return VIRTIO_CRYPTO_CMD_CIPHER; 1120 1121 /* Authentication Only */ 1122 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1123 xform->next == NULL) 1124 return VIRTIO_CRYPTO_CMD_AUTH; 1125 1126 /* Authenticate then Cipher */ 1127 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1128 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) 1129 return VIRTIO_CRYPTO_CMD_HASH_CIPHER; 1130 1131 /* Cipher then Authenticate */ 1132 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 1133 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 1134 return VIRTIO_CRYPTO_CMD_CIPHER_HASH; 1135 1136 return -1; 1137 } 1138 1139 static int 1140 virtio_crypto_sym_pad_cipher_param( 1141 struct virtio_crypto_cipher_session_para *para, 1142 struct rte_crypto_cipher_xform *cipher_xform) 1143 { 1144 switch (cipher_xform->algo) { 1145 case RTE_CRYPTO_CIPHER_AES_CBC: 1146 para->algo = VIRTIO_CRYPTO_CIPHER_AES_CBC; 1147 break; 1148 default: 1149 VIRTIO_CRYPTO_SESSION_LOG_ERR("Crypto: Unsupported " 1150 "Cipher alg %u", cipher_xform->algo); 1151 return -1; 1152 } 1153 1154 para->keylen = cipher_xform->key.length; 1155 switch (cipher_xform->op) { 1156 case RTE_CRYPTO_CIPHER_OP_ENCRYPT: 1157 para->op = VIRTIO_CRYPTO_OP_ENCRYPT; 1158 break; 1159 case RTE_CRYPTO_CIPHER_OP_DECRYPT: 1160 para->op = VIRTIO_CRYPTO_OP_DECRYPT; 1161 break; 1162 default: 1163 VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported cipher operation " 1164 "parameter"); 1165 return -1; 1166 } 1167 1168 return 0; 1169 } 1170 1171 static int 1172 virtio_crypto_sym_pad_auth_param( 1173 struct virtio_crypto_op_ctrl_req *ctrl, 1174 struct rte_crypto_auth_xform *auth_xform) 1175 { 1176 uint32_t *algo; 1177 struct virtio_crypto_alg_chain_session_para *para = 1178 &(ctrl->u.sym_create_session.u.chain.para); 1179 1180 switch (ctrl->u.sym_create_session.u.chain.para.hash_mode) { 1181 case VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN: 1182 algo = &(para->u.hash_param.algo); 1183 break; 1184 case VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH: 1185 algo = &(para->u.mac_param.algo); 1186 break; 1187 default: 1188 VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported hash mode %u " 1189 "specified", 1190 ctrl->u.sym_create_session.u.chain.para.hash_mode); 1191 return -1; 1192 } 1193 1194 switch (auth_xform->algo) { 1195 case RTE_CRYPTO_AUTH_SHA1_HMAC: 1196 *algo = VIRTIO_CRYPTO_MAC_HMAC_SHA1; 1197 break; 1198 default: 1199 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1200 "Crypto: Undefined Hash algo %u specified", 1201 auth_xform->algo); 1202 return -1; 1203 } 1204 1205 return 0; 1206 } 1207 1208 static int 1209 virtio_crypto_sym_pad_op_ctrl_req( 1210 struct virtio_crypto_op_ctrl_req *ctrl, 1211 struct rte_crypto_sym_xform *xform, bool is_chainned, 1212 uint8_t **cipher_key_data, uint8_t **auth_key_data, 1213 struct virtio_crypto_session *session) 1214 { 1215 int ret; 1216 struct rte_crypto_auth_xform *auth_xform = NULL; 1217 struct rte_crypto_cipher_xform *cipher_xform = NULL; 1218 1219 /* Get cipher xform from crypto xform chain */ 1220 cipher_xform = virtio_crypto_get_cipher_xform(xform); 1221 if (cipher_xform) { 1222 if (cipher_xform->iv.length > VIRTIO_CRYPTO_MAX_IV_SIZE) { 1223 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1224 "cipher IV size cannot be longer than %u", 1225 VIRTIO_CRYPTO_MAX_IV_SIZE); 1226 return -1; 1227 } 1228 if (is_chainned) 1229 ret = virtio_crypto_sym_pad_cipher_param( 1230 &ctrl->u.sym_create_session.u.chain.para 1231 .cipher_param, cipher_xform); 1232 else 1233 ret = virtio_crypto_sym_pad_cipher_param( 1234 &ctrl->u.sym_create_session.u.cipher.para, 1235 cipher_xform); 1236 1237 if (ret < 0) { 1238 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1239 "pad cipher parameter failed"); 1240 return -1; 1241 } 1242 1243 *cipher_key_data = cipher_xform->key.data; 1244 1245 session->iv.offset = cipher_xform->iv.offset; 1246 session->iv.length = cipher_xform->iv.length; 1247 } 1248 1249 /* Get auth xform from crypto xform chain */ 1250 auth_xform = virtio_crypto_get_auth_xform(xform); 1251 if (auth_xform) { 1252 /* FIXME: support VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */ 1253 struct virtio_crypto_alg_chain_session_para *para = 1254 &(ctrl->u.sym_create_session.u.chain.para); 1255 if (auth_xform->key.length) { 1256 para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH; 1257 para->u.mac_param.auth_key_len = 1258 (uint32_t)auth_xform->key.length; 1259 para->u.mac_param.hash_result_len = 1260 auth_xform->digest_length; 1261 1262 *auth_key_data = auth_xform->key.data; 1263 } else { 1264 para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN; 1265 para->u.hash_param.hash_result_len = 1266 auth_xform->digest_length; 1267 } 1268 1269 ret = virtio_crypto_sym_pad_auth_param(ctrl, auth_xform); 1270 if (ret < 0) { 1271 VIRTIO_CRYPTO_SESSION_LOG_ERR("pad auth parameter " 1272 "failed"); 1273 return -1; 1274 } 1275 } 1276 1277 return 0; 1278 } 1279 1280 static int 1281 virtio_crypto_check_sym_configure_session_paras( 1282 struct rte_cryptodev *dev, 1283 struct rte_crypto_sym_xform *xform, 1284 struct rte_cryptodev_sym_session *sym_sess, 1285 struct rte_mempool *mempool) 1286 { 1287 if (unlikely(xform == NULL) || unlikely(sym_sess == NULL) || 1288 unlikely(mempool == NULL)) { 1289 VIRTIO_CRYPTO_SESSION_LOG_ERR("NULL pointer"); 1290 return -1; 1291 } 1292 1293 if (virtio_crypto_check_sym_session_paras(dev) < 0) 1294 return -1; 1295 1296 return 0; 1297 } 1298 1299 static int 1300 virtio_crypto_sym_configure_session( 1301 struct rte_cryptodev *dev, 1302 struct rte_crypto_sym_xform *xform, 1303 struct rte_cryptodev_sym_session *sess, 1304 struct rte_mempool *mempool) 1305 { 1306 int ret; 1307 struct virtio_crypto_session crypto_sess; 1308 void *session_private = &crypto_sess; 1309 struct virtio_crypto_session *session; 1310 struct virtio_crypto_op_ctrl_req *ctrl_req; 1311 enum virtio_crypto_cmd_id cmd_id; 1312 uint8_t *cipher_key_data = NULL; 1313 uint8_t *auth_key_data = NULL; 1314 struct virtio_crypto_hw *hw; 1315 struct virtqueue *control_vq; 1316 1317 PMD_INIT_FUNC_TRACE(); 1318 1319 ret = virtio_crypto_check_sym_configure_session_paras(dev, xform, 1320 sess, mempool); 1321 if (ret < 0) { 1322 VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid parameters"); 1323 return ret; 1324 } 1325 1326 if (rte_mempool_get(mempool, &session_private)) { 1327 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1328 "Couldn't get object from session mempool"); 1329 return -ENOMEM; 1330 } 1331 1332 session = (struct virtio_crypto_session *)session_private; 1333 memset(session, 0, sizeof(struct virtio_crypto_session)); 1334 ctrl_req = &session->ctrl; 1335 ctrl_req->header.opcode = VIRTIO_CRYPTO_CIPHER_CREATE_SESSION; 1336 /* FIXME: support multiqueue */ 1337 ctrl_req->header.queue_id = 0; 1338 1339 hw = dev->data->dev_private; 1340 control_vq = hw->cvq; 1341 1342 cmd_id = virtio_crypto_get_chain_order(xform); 1343 if (cmd_id == VIRTIO_CRYPTO_CMD_CIPHER_HASH) 1344 ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order 1345 = VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 1346 if (cmd_id == VIRTIO_CRYPTO_CMD_HASH_CIPHER) 1347 ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order 1348 = VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; 1349 1350 switch (cmd_id) { 1351 case VIRTIO_CRYPTO_CMD_CIPHER_HASH: 1352 case VIRTIO_CRYPTO_CMD_HASH_CIPHER: 1353 ctrl_req->u.sym_create_session.op_type 1354 = VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING; 1355 1356 ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req, 1357 xform, true, &cipher_key_data, &auth_key_data, session); 1358 if (ret < 0) { 1359 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1360 "padding sym op ctrl req failed"); 1361 goto error_out; 1362 } 1363 ret = virtio_crypto_send_command(control_vq, ctrl_req, 1364 cipher_key_data, auth_key_data, session); 1365 if (ret < 0) { 1366 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1367 "create session failed: %d", ret); 1368 goto error_out; 1369 } 1370 break; 1371 case VIRTIO_CRYPTO_CMD_CIPHER: 1372 ctrl_req->u.sym_create_session.op_type 1373 = VIRTIO_CRYPTO_SYM_OP_CIPHER; 1374 ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req, xform, 1375 false, &cipher_key_data, &auth_key_data, session); 1376 if (ret < 0) { 1377 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1378 "padding sym op ctrl req failed"); 1379 goto error_out; 1380 } 1381 ret = virtio_crypto_send_command(control_vq, ctrl_req, 1382 cipher_key_data, NULL, session); 1383 if (ret < 0) { 1384 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1385 "create session failed: %d", ret); 1386 goto error_out; 1387 } 1388 break; 1389 default: 1390 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1391 "Unsupported operation chain order parameter"); 1392 goto error_out; 1393 } 1394 1395 set_sym_session_private_data(sess, dev->driver_id, 1396 session_private); 1397 1398 return 0; 1399 1400 error_out: 1401 return -1; 1402 } 1403 1404 static void 1405 virtio_crypto_dev_info_get(struct rte_cryptodev *dev, 1406 struct rte_cryptodev_info *info) 1407 { 1408 struct virtio_crypto_hw *hw = dev->data->dev_private; 1409 1410 PMD_INIT_FUNC_TRACE(); 1411 1412 if (info != NULL) { 1413 info->driver_id = cryptodev_virtio_driver_id; 1414 info->feature_flags = dev->feature_flags; 1415 info->max_nb_queue_pairs = hw->max_dataqueues; 1416 /* No limit of number of sessions */ 1417 info->sym.max_nb_sessions = 0; 1418 info->capabilities = hw->virtio_dev_capabilities; 1419 } 1420 } 1421 1422 static int 1423 crypto_virtio_pci_probe( 1424 struct rte_pci_driver *pci_drv __rte_unused, 1425 struct rte_pci_device *pci_dev) 1426 { 1427 struct rte_cryptodev_pmd_init_params init_params = { 1428 .name = "", 1429 .socket_id = rte_socket_id(), 1430 .private_data_size = sizeof(struct virtio_crypto_hw) 1431 }; 1432 char name[RTE_CRYPTODEV_NAME_MAX_LEN]; 1433 1434 VIRTIO_CRYPTO_DRV_LOG_DBG("Found Crypto device at %02x:%02x.%x", 1435 pci_dev->addr.bus, 1436 pci_dev->addr.devid, 1437 pci_dev->addr.function); 1438 1439 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 1440 1441 return crypto_virtio_create(name, pci_dev, &init_params); 1442 } 1443 1444 static int 1445 crypto_virtio_pci_remove( 1446 struct rte_pci_device *pci_dev __rte_unused) 1447 { 1448 struct rte_cryptodev *cryptodev; 1449 char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; 1450 1451 if (pci_dev == NULL) 1452 return -EINVAL; 1453 1454 rte_pci_device_name(&pci_dev->addr, cryptodev_name, 1455 sizeof(cryptodev_name)); 1456 1457 cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name); 1458 if (cryptodev == NULL) 1459 return -ENODEV; 1460 1461 return virtio_crypto_dev_uninit(cryptodev); 1462 } 1463 1464 static struct rte_pci_driver rte_virtio_crypto_driver = { 1465 .id_table = pci_id_virtio_crypto_map, 1466 .drv_flags = 0, 1467 .probe = crypto_virtio_pci_probe, 1468 .remove = crypto_virtio_pci_remove 1469 }; 1470 1471 static struct cryptodev_driver virtio_crypto_drv; 1472 1473 RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_VIRTIO_PMD, rte_virtio_crypto_driver); 1474 RTE_PMD_REGISTER_CRYPTO_DRIVER(virtio_crypto_drv, 1475 rte_virtio_crypto_driver.driver, 1476 cryptodev_virtio_driver_id); 1477 1478 RTE_INIT(virtio_crypto_init_log) 1479 { 1480 virtio_crypto_logtype_init = rte_log_register("pmd.crypto.virtio.init"); 1481 if (virtio_crypto_logtype_init >= 0) 1482 rte_log_set_level(virtio_crypto_logtype_init, RTE_LOG_NOTICE); 1483 1484 virtio_crypto_logtype_session = 1485 rte_log_register("pmd.crypto.virtio.session"); 1486 if (virtio_crypto_logtype_session >= 0) 1487 rte_log_set_level(virtio_crypto_logtype_session, 1488 RTE_LOG_NOTICE); 1489 1490 virtio_crypto_logtype_rx = rte_log_register("pmd.crypto.virtio.rx"); 1491 if (virtio_crypto_logtype_rx >= 0) 1492 rte_log_set_level(virtio_crypto_logtype_rx, RTE_LOG_NOTICE); 1493 1494 virtio_crypto_logtype_tx = rte_log_register("pmd.crypto.virtio.tx"); 1495 if (virtio_crypto_logtype_tx >= 0) 1496 rte_log_set_level(virtio_crypto_logtype_tx, RTE_LOG_NOTICE); 1497 1498 virtio_crypto_logtype_driver = 1499 rte_log_register("pmd.crypto.virtio.driver"); 1500 if (virtio_crypto_logtype_driver >= 0) 1501 rte_log_set_level(virtio_crypto_logtype_driver, RTE_LOG_NOTICE); 1502 } 1503