1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD. 3 */ 4 #include <stdbool.h> 5 #include <unistd.h> 6 7 #include <rte_common.h> 8 #include <rte_errno.h> 9 #include <rte_pci.h> 10 #include <rte_bus_pci.h> 11 #include <rte_cryptodev.h> 12 #include <cryptodev_pmd.h> 13 #include <rte_eal.h> 14 15 #include "virtio_cryptodev.h" 16 #include "virtqueue.h" 17 #include "virtio_crypto_algs.h" 18 #include "virtio_crypto_capabilities.h" 19 20 static int virtio_crypto_dev_configure(struct rte_cryptodev *dev, 21 struct rte_cryptodev_config *config); 22 static int virtio_crypto_dev_start(struct rte_cryptodev *dev); 23 static void virtio_crypto_dev_stop(struct rte_cryptodev *dev); 24 static int virtio_crypto_dev_close(struct rte_cryptodev *dev); 25 static void virtio_crypto_dev_info_get(struct rte_cryptodev *dev, 26 struct rte_cryptodev_info *dev_info); 27 static void virtio_crypto_dev_stats_get(struct rte_cryptodev *dev, 28 struct rte_cryptodev_stats *stats); 29 static void virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev); 30 static int virtio_crypto_qp_setup(struct rte_cryptodev *dev, 31 uint16_t queue_pair_id, 32 const struct rte_cryptodev_qp_conf *qp_conf, 33 int socket_id); 34 static int virtio_crypto_qp_release(struct rte_cryptodev *dev, 35 uint16_t queue_pair_id); 36 static void virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev); 37 static unsigned int virtio_crypto_sym_get_session_private_size( 38 struct rte_cryptodev *dev); 39 static void virtio_crypto_sym_clear_session(struct rte_cryptodev *dev, 40 struct rte_cryptodev_sym_session *sess); 41 static int virtio_crypto_sym_configure_session(struct rte_cryptodev *dev, 42 struct rte_crypto_sym_xform *xform, 43 struct rte_cryptodev_sym_session *session, 44 struct rte_mempool *mp); 45 46 /* 47 * The set of PCI devices this driver supports 48 */ 49 static const struct rte_pci_id pci_id_virtio_crypto_map[] = { 50 { RTE_PCI_DEVICE(VIRTIO_CRYPTO_PCI_VENDORID, 51 VIRTIO_CRYPTO_PCI_DEVICEID) }, 52 { .vendor_id = 0, /* sentinel */ }, 53 }; 54 55 static const struct rte_cryptodev_capabilities virtio_capabilities[] = { 56 VIRTIO_SYM_CAPABILITIES, 57 RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() 58 }; 59 60 uint8_t cryptodev_virtio_driver_id; 61 62 #define NUM_ENTRY_SYM_CREATE_SESSION 4 63 64 static int 65 virtio_crypto_send_command(struct virtqueue *vq, 66 struct virtio_crypto_op_ctrl_req *ctrl, uint8_t *cipher_key, 67 uint8_t *auth_key, struct virtio_crypto_session *session) 68 { 69 uint8_t idx = 0; 70 uint8_t needed = 1; 71 uint32_t head = 0; 72 uint32_t len_cipher_key = 0; 73 uint32_t len_auth_key = 0; 74 uint32_t len_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req); 75 uint32_t len_session_input = sizeof(struct virtio_crypto_session_input); 76 uint32_t len_total = 0; 77 uint32_t input_offset = 0; 78 void *virt_addr_started = NULL; 79 phys_addr_t phys_addr_started; 80 struct vring_desc *desc; 81 uint32_t desc_offset; 82 struct virtio_crypto_session_input *input; 83 int ret; 84 85 PMD_INIT_FUNC_TRACE(); 86 87 if (session == NULL) { 88 VIRTIO_CRYPTO_SESSION_LOG_ERR("session is NULL."); 89 return -EINVAL; 90 } 91 /* cipher only is supported, it is available if auth_key is NULL */ 92 if (!cipher_key) { 93 VIRTIO_CRYPTO_SESSION_LOG_ERR("cipher key is NULL."); 94 return -EINVAL; 95 } 96 97 head = vq->vq_desc_head_idx; 98 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_desc_head_idx = %d, vq = %p", 99 head, vq); 100 101 if (vq->vq_free_cnt < needed) { 102 VIRTIO_CRYPTO_SESSION_LOG_ERR("Not enough entry"); 103 return -ENOSPC; 104 } 105 106 /* calculate the length of cipher key */ 107 if (cipher_key) { 108 switch (ctrl->u.sym_create_session.op_type) { 109 case VIRTIO_CRYPTO_SYM_OP_CIPHER: 110 len_cipher_key 111 = ctrl->u.sym_create_session.u.cipher 112 .para.keylen; 113 break; 114 case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING: 115 len_cipher_key 116 = ctrl->u.sym_create_session.u.chain 117 .para.cipher_param.keylen; 118 break; 119 default: 120 VIRTIO_CRYPTO_SESSION_LOG_ERR("invalid op type"); 121 return -EINVAL; 122 } 123 } 124 125 /* calculate the length of auth key */ 126 if (auth_key) { 127 len_auth_key = 128 ctrl->u.sym_create_session.u.chain.para.u.mac_param 129 .auth_key_len; 130 } 131 132 /* 133 * malloc memory to store indirect vring_desc entries, including 134 * ctrl request, cipher key, auth key, session input and desc vring 135 */ 136 desc_offset = len_ctrl_req + len_cipher_key + len_auth_key 137 + len_session_input; 138 virt_addr_started = rte_malloc(NULL, 139 desc_offset + NUM_ENTRY_SYM_CREATE_SESSION 140 * sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE); 141 if (virt_addr_started == NULL) { 142 VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap memory"); 143 return -ENOSPC; 144 } 145 phys_addr_started = rte_malloc_virt2iova(virt_addr_started); 146 147 /* address to store indirect vring desc entries */ 148 desc = (struct vring_desc *) 149 ((uint8_t *)virt_addr_started + desc_offset); 150 151 /* ctrl req part */ 152 memcpy(virt_addr_started, ctrl, len_ctrl_req); 153 desc[idx].addr = phys_addr_started; 154 desc[idx].len = len_ctrl_req; 155 desc[idx].flags = VRING_DESC_F_NEXT; 156 desc[idx].next = idx + 1; 157 idx++; 158 len_total += len_ctrl_req; 159 input_offset += len_ctrl_req; 160 161 /* cipher key part */ 162 if (len_cipher_key > 0) { 163 memcpy((uint8_t *)virt_addr_started + len_total, 164 cipher_key, len_cipher_key); 165 166 desc[idx].addr = phys_addr_started + len_total; 167 desc[idx].len = len_cipher_key; 168 desc[idx].flags = VRING_DESC_F_NEXT; 169 desc[idx].next = idx + 1; 170 idx++; 171 len_total += len_cipher_key; 172 input_offset += len_cipher_key; 173 } 174 175 /* auth key part */ 176 if (len_auth_key > 0) { 177 memcpy((uint8_t *)virt_addr_started + len_total, 178 auth_key, len_auth_key); 179 180 desc[idx].addr = phys_addr_started + len_total; 181 desc[idx].len = len_auth_key; 182 desc[idx].flags = VRING_DESC_F_NEXT; 183 desc[idx].next = idx + 1; 184 idx++; 185 len_total += len_auth_key; 186 input_offset += len_auth_key; 187 } 188 189 /* input part */ 190 input = (struct virtio_crypto_session_input *) 191 ((uint8_t *)virt_addr_started + input_offset); 192 input->status = VIRTIO_CRYPTO_ERR; 193 input->session_id = ~0ULL; 194 desc[idx].addr = phys_addr_started + len_total; 195 desc[idx].len = len_session_input; 196 desc[idx].flags = VRING_DESC_F_WRITE; 197 idx++; 198 199 /* use a single desc entry */ 200 vq->vq_ring.desc[head].addr = phys_addr_started + desc_offset; 201 vq->vq_ring.desc[head].len = idx * sizeof(struct vring_desc); 202 vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT; 203 vq->vq_free_cnt--; 204 205 vq->vq_desc_head_idx = vq->vq_ring.desc[head].next; 206 207 vq_update_avail_ring(vq, head); 208 vq_update_avail_idx(vq); 209 210 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d", 211 vq->vq_queue_index); 212 213 virtqueue_notify(vq); 214 215 rte_rmb(); 216 while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) { 217 rte_rmb(); 218 usleep(100); 219 } 220 221 while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) { 222 uint32_t idx, desc_idx, used_idx; 223 struct vring_used_elem *uep; 224 225 used_idx = (uint32_t)(vq->vq_used_cons_idx 226 & (vq->vq_nentries - 1)); 227 uep = &vq->vq_ring.used->ring[used_idx]; 228 idx = (uint32_t) uep->id; 229 desc_idx = idx; 230 231 while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) { 232 desc_idx = vq->vq_ring.desc[desc_idx].next; 233 vq->vq_free_cnt++; 234 } 235 236 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx; 237 vq->vq_desc_head_idx = idx; 238 239 vq->vq_used_cons_idx++; 240 vq->vq_free_cnt++; 241 } 242 243 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n" 244 "vq->vq_desc_head_idx=%d", 245 vq->vq_free_cnt, vq->vq_desc_head_idx); 246 247 /* get the result */ 248 if (input->status != VIRTIO_CRYPTO_OK) { 249 VIRTIO_CRYPTO_SESSION_LOG_ERR("Something wrong on backend! " 250 "status=%u, session_id=%" PRIu64 "", 251 input->status, input->session_id); 252 rte_free(virt_addr_started); 253 ret = -1; 254 } else { 255 session->session_id = input->session_id; 256 257 VIRTIO_CRYPTO_SESSION_LOG_INFO("Create session successfully, " 258 "session_id=%" PRIu64 "", input->session_id); 259 rte_free(virt_addr_started); 260 ret = 0; 261 } 262 263 return ret; 264 } 265 266 void 267 virtio_crypto_queue_release(struct virtqueue *vq) 268 { 269 struct virtio_crypto_hw *hw; 270 271 PMD_INIT_FUNC_TRACE(); 272 273 if (vq) { 274 hw = vq->hw; 275 /* Select and deactivate the queue */ 276 VTPCI_OPS(hw)->del_queue(hw, vq); 277 278 rte_memzone_free(vq->mz); 279 rte_mempool_free(vq->mpool); 280 rte_free(vq); 281 } 282 } 283 284 #define MPOOL_MAX_NAME_SZ 32 285 286 int 287 virtio_crypto_queue_setup(struct rte_cryptodev *dev, 288 int queue_type, 289 uint16_t vtpci_queue_idx, 290 uint16_t nb_desc, 291 int socket_id, 292 struct virtqueue **pvq) 293 { 294 char vq_name[VIRTQUEUE_MAX_NAME_SZ]; 295 char mpool_name[MPOOL_MAX_NAME_SZ]; 296 const struct rte_memzone *mz; 297 unsigned int vq_size, size; 298 struct virtio_crypto_hw *hw = dev->data->dev_private; 299 struct virtqueue *vq = NULL; 300 uint32_t i = 0; 301 uint32_t j; 302 303 PMD_INIT_FUNC_TRACE(); 304 305 VIRTIO_CRYPTO_INIT_LOG_DBG("setting up queue: %u", vtpci_queue_idx); 306 307 /* 308 * Read the virtqueue size from the Queue Size field 309 * Always power of 2 and if 0 virtqueue does not exist 310 */ 311 vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx); 312 if (vq_size == 0) { 313 VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue does not exist"); 314 return -EINVAL; 315 } 316 VIRTIO_CRYPTO_INIT_LOG_DBG("vq_size: %u", vq_size); 317 318 if (!rte_is_power_of_2(vq_size)) { 319 VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue size is not powerof 2"); 320 return -EINVAL; 321 } 322 323 if (queue_type == VTCRYPTO_DATAQ) { 324 snprintf(vq_name, sizeof(vq_name), "dev%d_dataqueue%d", 325 dev->data->dev_id, vtpci_queue_idx); 326 snprintf(mpool_name, sizeof(mpool_name), 327 "dev%d_dataqueue%d_mpool", 328 dev->data->dev_id, vtpci_queue_idx); 329 } else if (queue_type == VTCRYPTO_CTRLQ) { 330 snprintf(vq_name, sizeof(vq_name), "dev%d_controlqueue", 331 dev->data->dev_id); 332 snprintf(mpool_name, sizeof(mpool_name), 333 "dev%d_controlqueue_mpool", 334 dev->data->dev_id); 335 } 336 size = RTE_ALIGN_CEIL(sizeof(*vq) + 337 vq_size * sizeof(struct vq_desc_extra), 338 RTE_CACHE_LINE_SIZE); 339 vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE, 340 socket_id); 341 if (vq == NULL) { 342 VIRTIO_CRYPTO_INIT_LOG_ERR("Can not allocate virtqueue"); 343 return -ENOMEM; 344 } 345 346 if (queue_type == VTCRYPTO_DATAQ) { 347 /* pre-allocate a mempool and use it in the data plane to 348 * improve performance 349 */ 350 vq->mpool = rte_mempool_lookup(mpool_name); 351 if (vq->mpool == NULL) 352 vq->mpool = rte_mempool_create(mpool_name, 353 vq_size, 354 sizeof(struct virtio_crypto_op_cookie), 355 RTE_CACHE_LINE_SIZE, 0, 356 NULL, NULL, NULL, NULL, socket_id, 357 0); 358 if (!vq->mpool) { 359 VIRTIO_CRYPTO_DRV_LOG_ERR("Virtio Crypto PMD " 360 "Cannot create mempool"); 361 goto mpool_create_err; 362 } 363 for (i = 0; i < vq_size; i++) { 364 vq->vq_descx[i].cookie = 365 rte_zmalloc("crypto PMD op cookie pointer", 366 sizeof(struct virtio_crypto_op_cookie), 367 RTE_CACHE_LINE_SIZE); 368 if (vq->vq_descx[i].cookie == NULL) { 369 VIRTIO_CRYPTO_DRV_LOG_ERR("Failed to " 370 "alloc mem for cookie"); 371 goto cookie_alloc_err; 372 } 373 } 374 } 375 376 vq->hw = hw; 377 vq->dev_id = dev->data->dev_id; 378 vq->vq_queue_index = vtpci_queue_idx; 379 vq->vq_nentries = vq_size; 380 381 /* 382 * Using part of the vring entries is permitted, but the maximum 383 * is vq_size 384 */ 385 if (nb_desc == 0 || nb_desc > vq_size) 386 nb_desc = vq_size; 387 vq->vq_free_cnt = nb_desc; 388 389 /* 390 * Reserve a memzone for vring elements 391 */ 392 size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN); 393 vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN); 394 VIRTIO_CRYPTO_INIT_LOG_DBG("%s vring_size: %d, rounded_vring_size: %d", 395 (queue_type == VTCRYPTO_DATAQ) ? "dataq" : "ctrlq", 396 size, vq->vq_ring_size); 397 398 mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, 399 socket_id, 0, VIRTIO_PCI_VRING_ALIGN); 400 if (mz == NULL) { 401 if (rte_errno == EEXIST) 402 mz = rte_memzone_lookup(vq_name); 403 if (mz == NULL) { 404 VIRTIO_CRYPTO_INIT_LOG_ERR("not enough memory"); 405 goto mz_reserve_err; 406 } 407 } 408 409 /* 410 * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit, 411 * and only accepts 32 bit page frame number. 412 * Check if the allocated physical memory exceeds 16TB. 413 */ 414 if ((mz->iova + vq->vq_ring_size - 1) 415 >> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) { 416 VIRTIO_CRYPTO_INIT_LOG_ERR("vring address shouldn't be " 417 "above 16TB!"); 418 goto vring_addr_err; 419 } 420 421 memset(mz->addr, 0, sizeof(mz->len)); 422 vq->mz = mz; 423 vq->vq_ring_mem = mz->iova; 424 vq->vq_ring_virt_mem = mz->addr; 425 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_mem(physical): 0x%"PRIx64, 426 (uint64_t)mz->iova); 427 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_virt_mem: 0x%"PRIx64, 428 (uint64_t)(uintptr_t)mz->addr); 429 430 *pvq = vq; 431 432 return 0; 433 434 vring_addr_err: 435 rte_memzone_free(mz); 436 mz_reserve_err: 437 cookie_alloc_err: 438 rte_mempool_free(vq->mpool); 439 if (i != 0) { 440 for (j = 0; j < i; j++) 441 rte_free(vq->vq_descx[j].cookie); 442 } 443 mpool_create_err: 444 rte_free(vq); 445 return -ENOMEM; 446 } 447 448 static int 449 virtio_crypto_ctrlq_setup(struct rte_cryptodev *dev, uint16_t queue_idx) 450 { 451 int ret; 452 struct virtqueue *vq; 453 struct virtio_crypto_hw *hw = dev->data->dev_private; 454 455 /* if virtio device has started, do not touch the virtqueues */ 456 if (dev->data->dev_started) 457 return 0; 458 459 PMD_INIT_FUNC_TRACE(); 460 461 ret = virtio_crypto_queue_setup(dev, VTCRYPTO_CTRLQ, queue_idx, 462 0, SOCKET_ID_ANY, &vq); 463 if (ret < 0) { 464 VIRTIO_CRYPTO_INIT_LOG_ERR("control vq initialization failed"); 465 return ret; 466 } 467 468 hw->cvq = vq; 469 470 return 0; 471 } 472 473 static void 474 virtio_crypto_free_queues(struct rte_cryptodev *dev) 475 { 476 unsigned int i; 477 struct virtio_crypto_hw *hw = dev->data->dev_private; 478 479 PMD_INIT_FUNC_TRACE(); 480 481 /* control queue release */ 482 virtio_crypto_queue_release(hw->cvq); 483 484 /* data queue release */ 485 for (i = 0; i < hw->max_dataqueues; i++) 486 virtio_crypto_queue_release(dev->data->queue_pairs[i]); 487 } 488 489 static int 490 virtio_crypto_dev_close(struct rte_cryptodev *dev __rte_unused) 491 { 492 return 0; 493 } 494 495 /* 496 * dev_ops for virtio, bare necessities for basic operation 497 */ 498 static struct rte_cryptodev_ops virtio_crypto_dev_ops = { 499 /* Device related operations */ 500 .dev_configure = virtio_crypto_dev_configure, 501 .dev_start = virtio_crypto_dev_start, 502 .dev_stop = virtio_crypto_dev_stop, 503 .dev_close = virtio_crypto_dev_close, 504 .dev_infos_get = virtio_crypto_dev_info_get, 505 506 .stats_get = virtio_crypto_dev_stats_get, 507 .stats_reset = virtio_crypto_dev_stats_reset, 508 509 .queue_pair_setup = virtio_crypto_qp_setup, 510 .queue_pair_release = virtio_crypto_qp_release, 511 512 /* Crypto related operations */ 513 .sym_session_get_size = virtio_crypto_sym_get_session_private_size, 514 .sym_session_configure = virtio_crypto_sym_configure_session, 515 .sym_session_clear = virtio_crypto_sym_clear_session 516 }; 517 518 static void 519 virtio_crypto_update_stats(struct rte_cryptodev *dev, 520 struct rte_cryptodev_stats *stats) 521 { 522 unsigned int i; 523 struct virtio_crypto_hw *hw = dev->data->dev_private; 524 525 PMD_INIT_FUNC_TRACE(); 526 527 if (stats == NULL) { 528 VIRTIO_CRYPTO_DRV_LOG_ERR("invalid pointer"); 529 return; 530 } 531 532 for (i = 0; i < hw->max_dataqueues; i++) { 533 const struct virtqueue *data_queue 534 = dev->data->queue_pairs[i]; 535 if (data_queue == NULL) 536 continue; 537 538 stats->enqueued_count += data_queue->packets_sent_total; 539 stats->enqueue_err_count += data_queue->packets_sent_failed; 540 541 stats->dequeued_count += data_queue->packets_received_total; 542 stats->dequeue_err_count 543 += data_queue->packets_received_failed; 544 } 545 } 546 547 static void 548 virtio_crypto_dev_stats_get(struct rte_cryptodev *dev, 549 struct rte_cryptodev_stats *stats) 550 { 551 PMD_INIT_FUNC_TRACE(); 552 553 virtio_crypto_update_stats(dev, stats); 554 } 555 556 static void 557 virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev) 558 { 559 unsigned int i; 560 struct virtio_crypto_hw *hw = dev->data->dev_private; 561 562 PMD_INIT_FUNC_TRACE(); 563 564 for (i = 0; i < hw->max_dataqueues; i++) { 565 struct virtqueue *data_queue = dev->data->queue_pairs[i]; 566 if (data_queue == NULL) 567 continue; 568 569 data_queue->packets_sent_total = 0; 570 data_queue->packets_sent_failed = 0; 571 572 data_queue->packets_received_total = 0; 573 data_queue->packets_received_failed = 0; 574 } 575 } 576 577 static int 578 virtio_crypto_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, 579 const struct rte_cryptodev_qp_conf *qp_conf, 580 int socket_id) 581 { 582 int ret; 583 struct virtqueue *vq; 584 585 PMD_INIT_FUNC_TRACE(); 586 587 /* if virtio dev is started, do not touch the virtqueues */ 588 if (dev->data->dev_started) 589 return 0; 590 591 ret = virtio_crypto_queue_setup(dev, VTCRYPTO_DATAQ, queue_pair_id, 592 qp_conf->nb_descriptors, socket_id, &vq); 593 if (ret < 0) { 594 VIRTIO_CRYPTO_INIT_LOG_ERR( 595 "virtio crypto data queue initialization failed\n"); 596 return ret; 597 } 598 599 dev->data->queue_pairs[queue_pair_id] = vq; 600 601 return 0; 602 } 603 604 static int 605 virtio_crypto_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) 606 { 607 struct virtqueue *vq 608 = (struct virtqueue *)dev->data->queue_pairs[queue_pair_id]; 609 610 PMD_INIT_FUNC_TRACE(); 611 612 if (vq == NULL) { 613 VIRTIO_CRYPTO_DRV_LOG_DBG("vq already freed"); 614 return 0; 615 } 616 617 virtio_crypto_queue_release(vq); 618 return 0; 619 } 620 621 static int 622 virtio_negotiate_features(struct virtio_crypto_hw *hw, uint64_t req_features) 623 { 624 uint64_t host_features; 625 626 PMD_INIT_FUNC_TRACE(); 627 628 /* Prepare guest_features: feature that driver wants to support */ 629 VIRTIO_CRYPTO_INIT_LOG_DBG("guest_features before negotiate = %" PRIx64, 630 req_features); 631 632 /* Read device(host) feature bits */ 633 host_features = VTPCI_OPS(hw)->get_features(hw); 634 VIRTIO_CRYPTO_INIT_LOG_DBG("host_features before negotiate = %" PRIx64, 635 host_features); 636 637 /* 638 * Negotiate features: Subset of device feature bits are written back 639 * guest feature bits. 640 */ 641 hw->guest_features = req_features; 642 hw->guest_features = vtpci_cryptodev_negotiate_features(hw, 643 host_features); 644 VIRTIO_CRYPTO_INIT_LOG_DBG("features after negotiate = %" PRIx64, 645 hw->guest_features); 646 647 if (hw->modern) { 648 if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) { 649 VIRTIO_CRYPTO_INIT_LOG_ERR( 650 "VIRTIO_F_VERSION_1 features is not enabled."); 651 return -1; 652 } 653 vtpci_cryptodev_set_status(hw, 654 VIRTIO_CONFIG_STATUS_FEATURES_OK); 655 if (!(vtpci_cryptodev_get_status(hw) & 656 VIRTIO_CONFIG_STATUS_FEATURES_OK)) { 657 VIRTIO_CRYPTO_INIT_LOG_ERR("failed to set FEATURES_OK " 658 "status!"); 659 return -1; 660 } 661 } 662 663 hw->req_guest_features = req_features; 664 665 return 0; 666 } 667 668 /* reset device and renegotiate features if needed */ 669 static int 670 virtio_crypto_init_device(struct rte_cryptodev *cryptodev, 671 uint64_t req_features) 672 { 673 struct virtio_crypto_hw *hw = cryptodev->data->dev_private; 674 struct virtio_crypto_config local_config; 675 struct virtio_crypto_config *config = &local_config; 676 677 PMD_INIT_FUNC_TRACE(); 678 679 /* Reset the device although not necessary at startup */ 680 vtpci_cryptodev_reset(hw); 681 682 /* Tell the host we've noticed this device. */ 683 vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); 684 685 /* Tell the host we've known how to drive the device. */ 686 vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); 687 if (virtio_negotiate_features(hw, req_features) < 0) 688 return -1; 689 690 /* Get status of the device */ 691 vtpci_read_cryptodev_config(hw, 692 offsetof(struct virtio_crypto_config, status), 693 &config->status, sizeof(config->status)); 694 if (config->status != VIRTIO_CRYPTO_S_HW_READY) { 695 VIRTIO_CRYPTO_DRV_LOG_ERR("accelerator hardware is " 696 "not ready"); 697 return -1; 698 } 699 700 /* Get number of data queues */ 701 vtpci_read_cryptodev_config(hw, 702 offsetof(struct virtio_crypto_config, max_dataqueues), 703 &config->max_dataqueues, 704 sizeof(config->max_dataqueues)); 705 hw->max_dataqueues = config->max_dataqueues; 706 707 VIRTIO_CRYPTO_INIT_LOG_DBG("hw->max_dataqueues=%d", 708 hw->max_dataqueues); 709 710 return 0; 711 } 712 713 /* 714 * This function is based on probe() function 715 * It returns 0 on success. 716 */ 717 static int 718 crypto_virtio_create(const char *name, struct rte_pci_device *pci_dev, 719 struct rte_cryptodev_pmd_init_params *init_params) 720 { 721 struct rte_cryptodev *cryptodev; 722 struct virtio_crypto_hw *hw; 723 724 PMD_INIT_FUNC_TRACE(); 725 726 cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device, 727 init_params); 728 if (cryptodev == NULL) 729 return -ENODEV; 730 731 cryptodev->driver_id = cryptodev_virtio_driver_id; 732 cryptodev->dev_ops = &virtio_crypto_dev_ops; 733 734 cryptodev->enqueue_burst = virtio_crypto_pkt_tx_burst; 735 cryptodev->dequeue_burst = virtio_crypto_pkt_rx_burst; 736 737 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 738 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 739 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; 740 741 hw = cryptodev->data->dev_private; 742 hw->dev_id = cryptodev->data->dev_id; 743 hw->virtio_dev_capabilities = virtio_capabilities; 744 745 VIRTIO_CRYPTO_INIT_LOG_DBG("dev %d vendorID=0x%x deviceID=0x%x", 746 cryptodev->data->dev_id, pci_dev->id.vendor_id, 747 pci_dev->id.device_id); 748 749 /* pci device init */ 750 if (vtpci_cryptodev_init(pci_dev, hw)) 751 return -1; 752 753 if (virtio_crypto_init_device(cryptodev, 754 VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0) 755 return -1; 756 757 rte_cryptodev_pmd_probing_finish(cryptodev); 758 759 return 0; 760 } 761 762 static int 763 virtio_crypto_dev_uninit(struct rte_cryptodev *cryptodev) 764 { 765 struct virtio_crypto_hw *hw = cryptodev->data->dev_private; 766 767 PMD_INIT_FUNC_TRACE(); 768 769 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 770 return -EPERM; 771 772 if (cryptodev->data->dev_started) { 773 virtio_crypto_dev_stop(cryptodev); 774 virtio_crypto_dev_close(cryptodev); 775 } 776 777 cryptodev->dev_ops = NULL; 778 cryptodev->enqueue_burst = NULL; 779 cryptodev->dequeue_burst = NULL; 780 781 /* release control queue */ 782 virtio_crypto_queue_release(hw->cvq); 783 784 rte_free(cryptodev->data); 785 cryptodev->data = NULL; 786 787 VIRTIO_CRYPTO_DRV_LOG_INFO("dev_uninit completed"); 788 789 return 0; 790 } 791 792 static int 793 virtio_crypto_dev_configure(struct rte_cryptodev *cryptodev, 794 struct rte_cryptodev_config *config __rte_unused) 795 { 796 struct virtio_crypto_hw *hw = cryptodev->data->dev_private; 797 798 PMD_INIT_FUNC_TRACE(); 799 800 if (virtio_crypto_init_device(cryptodev, 801 VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0) 802 return -1; 803 804 /* setup control queue 805 * [0, 1, ... ,(config->max_dataqueues - 1)] are data queues 806 * config->max_dataqueues is the control queue 807 */ 808 if (virtio_crypto_ctrlq_setup(cryptodev, hw->max_dataqueues) < 0) { 809 VIRTIO_CRYPTO_INIT_LOG_ERR("control queue setup error"); 810 return -1; 811 } 812 virtio_crypto_ctrlq_start(cryptodev); 813 814 return 0; 815 } 816 817 static void 818 virtio_crypto_dev_stop(struct rte_cryptodev *dev) 819 { 820 struct virtio_crypto_hw *hw = dev->data->dev_private; 821 822 PMD_INIT_FUNC_TRACE(); 823 VIRTIO_CRYPTO_DRV_LOG_DBG("virtio_dev_stop"); 824 825 vtpci_cryptodev_reset(hw); 826 827 virtio_crypto_dev_free_mbufs(dev); 828 virtio_crypto_free_queues(dev); 829 830 dev->data->dev_started = 0; 831 } 832 833 static int 834 virtio_crypto_dev_start(struct rte_cryptodev *dev) 835 { 836 struct virtio_crypto_hw *hw = dev->data->dev_private; 837 838 if (dev->data->dev_started) 839 return 0; 840 841 /* Do final configuration before queue engine starts */ 842 virtio_crypto_dataq_start(dev); 843 vtpci_cryptodev_reinit_complete(hw); 844 845 dev->data->dev_started = 1; 846 847 return 0; 848 } 849 850 static void 851 virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev) 852 { 853 uint32_t i; 854 struct virtio_crypto_hw *hw = dev->data->dev_private; 855 856 for (i = 0; i < hw->max_dataqueues; i++) { 857 VIRTIO_CRYPTO_INIT_LOG_DBG("Before freeing dataq[%d] used " 858 "and unused buf", i); 859 VIRTQUEUE_DUMP((struct virtqueue *) 860 dev->data->queue_pairs[i]); 861 862 VIRTIO_CRYPTO_INIT_LOG_DBG("queue_pairs[%d]=%p", 863 i, dev->data->queue_pairs[i]); 864 865 virtqueue_detatch_unused(dev->data->queue_pairs[i]); 866 867 VIRTIO_CRYPTO_INIT_LOG_DBG("After freeing dataq[%d] used and " 868 "unused buf", i); 869 VIRTQUEUE_DUMP( 870 (struct virtqueue *)dev->data->queue_pairs[i]); 871 } 872 } 873 874 static unsigned int 875 virtio_crypto_sym_get_session_private_size( 876 struct rte_cryptodev *dev __rte_unused) 877 { 878 PMD_INIT_FUNC_TRACE(); 879 880 return RTE_ALIGN_CEIL(sizeof(struct virtio_crypto_session), 16); 881 } 882 883 static int 884 virtio_crypto_check_sym_session_paras( 885 struct rte_cryptodev *dev) 886 { 887 struct virtio_crypto_hw *hw; 888 889 PMD_INIT_FUNC_TRACE(); 890 891 if (unlikely(dev == NULL)) { 892 VIRTIO_CRYPTO_SESSION_LOG_ERR("dev is NULL"); 893 return -1; 894 } 895 if (unlikely(dev->data == NULL)) { 896 VIRTIO_CRYPTO_SESSION_LOG_ERR("dev->data is NULL"); 897 return -1; 898 } 899 hw = dev->data->dev_private; 900 if (unlikely(hw == NULL)) { 901 VIRTIO_CRYPTO_SESSION_LOG_ERR("hw is NULL"); 902 return -1; 903 } 904 if (unlikely(hw->cvq == NULL)) { 905 VIRTIO_CRYPTO_SESSION_LOG_ERR("vq is NULL"); 906 return -1; 907 } 908 909 return 0; 910 } 911 912 static int 913 virtio_crypto_check_sym_clear_session_paras( 914 struct rte_cryptodev *dev, 915 struct rte_cryptodev_sym_session *sess) 916 { 917 PMD_INIT_FUNC_TRACE(); 918 919 if (sess == NULL) { 920 VIRTIO_CRYPTO_SESSION_LOG_ERR("sym_session is NULL"); 921 return -1; 922 } 923 924 return virtio_crypto_check_sym_session_paras(dev); 925 } 926 927 #define NUM_ENTRY_SYM_CLEAR_SESSION 2 928 929 static void 930 virtio_crypto_sym_clear_session( 931 struct rte_cryptodev *dev, 932 struct rte_cryptodev_sym_session *sess) 933 { 934 struct virtio_crypto_hw *hw; 935 struct virtqueue *vq; 936 struct virtio_crypto_session *session; 937 struct virtio_crypto_op_ctrl_req *ctrl; 938 struct vring_desc *desc; 939 uint8_t *status; 940 uint8_t needed = 1; 941 uint32_t head; 942 uint8_t *malloc_virt_addr; 943 uint64_t malloc_phys_addr; 944 uint8_t len_inhdr = sizeof(struct virtio_crypto_inhdr); 945 uint32_t len_op_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req); 946 uint32_t desc_offset = len_op_ctrl_req + len_inhdr; 947 948 PMD_INIT_FUNC_TRACE(); 949 950 if (virtio_crypto_check_sym_clear_session_paras(dev, sess) < 0) 951 return; 952 953 hw = dev->data->dev_private; 954 vq = hw->cvq; 955 session = (struct virtio_crypto_session *)get_sym_session_private_data( 956 sess, cryptodev_virtio_driver_id); 957 if (session == NULL) { 958 VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid session parameter"); 959 return; 960 } 961 962 VIRTIO_CRYPTO_SESSION_LOG_INFO("vq->vq_desc_head_idx = %d, " 963 "vq = %p", vq->vq_desc_head_idx, vq); 964 965 if (vq->vq_free_cnt < needed) { 966 VIRTIO_CRYPTO_SESSION_LOG_ERR( 967 "vq->vq_free_cnt = %d is less than %d, " 968 "not enough", vq->vq_free_cnt, needed); 969 return; 970 } 971 972 /* 973 * malloc memory to store information of ctrl request op, 974 * returned status and desc vring 975 */ 976 malloc_virt_addr = rte_malloc(NULL, len_op_ctrl_req + len_inhdr 977 + NUM_ENTRY_SYM_CLEAR_SESSION 978 * sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE); 979 if (malloc_virt_addr == NULL) { 980 VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap room"); 981 return; 982 } 983 malloc_phys_addr = rte_malloc_virt2iova(malloc_virt_addr); 984 985 /* assign ctrl request op part */ 986 ctrl = (struct virtio_crypto_op_ctrl_req *)malloc_virt_addr; 987 ctrl->header.opcode = VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION; 988 /* default data virtqueue is 0 */ 989 ctrl->header.queue_id = 0; 990 ctrl->u.destroy_session.session_id = session->session_id; 991 992 /* status part */ 993 status = &(((struct virtio_crypto_inhdr *) 994 ((uint8_t *)malloc_virt_addr + len_op_ctrl_req))->status); 995 *status = VIRTIO_CRYPTO_ERR; 996 997 /* indirect desc vring part */ 998 desc = (struct vring_desc *)((uint8_t *)malloc_virt_addr 999 + desc_offset); 1000 1001 /* ctrl request part */ 1002 desc[0].addr = malloc_phys_addr; 1003 desc[0].len = len_op_ctrl_req; 1004 desc[0].flags = VRING_DESC_F_NEXT; 1005 desc[0].next = 1; 1006 1007 /* status part */ 1008 desc[1].addr = malloc_phys_addr + len_op_ctrl_req; 1009 desc[1].len = len_inhdr; 1010 desc[1].flags = VRING_DESC_F_WRITE; 1011 1012 /* use only a single desc entry */ 1013 head = vq->vq_desc_head_idx; 1014 vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT; 1015 vq->vq_ring.desc[head].addr = malloc_phys_addr + desc_offset; 1016 vq->vq_ring.desc[head].len 1017 = NUM_ENTRY_SYM_CLEAR_SESSION 1018 * sizeof(struct vring_desc); 1019 vq->vq_free_cnt -= needed; 1020 1021 vq->vq_desc_head_idx = vq->vq_ring.desc[head].next; 1022 1023 vq_update_avail_ring(vq, head); 1024 vq_update_avail_idx(vq); 1025 1026 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d", 1027 vq->vq_queue_index); 1028 1029 virtqueue_notify(vq); 1030 1031 rte_rmb(); 1032 while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) { 1033 rte_rmb(); 1034 usleep(100); 1035 } 1036 1037 while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) { 1038 uint32_t idx, desc_idx, used_idx; 1039 struct vring_used_elem *uep; 1040 1041 used_idx = (uint32_t)(vq->vq_used_cons_idx 1042 & (vq->vq_nentries - 1)); 1043 uep = &vq->vq_ring.used->ring[used_idx]; 1044 idx = (uint32_t) uep->id; 1045 desc_idx = idx; 1046 while (vq->vq_ring.desc[desc_idx].flags 1047 & VRING_DESC_F_NEXT) { 1048 desc_idx = vq->vq_ring.desc[desc_idx].next; 1049 vq->vq_free_cnt++; 1050 } 1051 1052 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx; 1053 vq->vq_desc_head_idx = idx; 1054 vq->vq_used_cons_idx++; 1055 vq->vq_free_cnt++; 1056 } 1057 1058 if (*status != VIRTIO_CRYPTO_OK) { 1059 VIRTIO_CRYPTO_SESSION_LOG_ERR("Close session failed " 1060 "status=%"PRIu32", session_id=%"PRIu64"", 1061 *status, session->session_id); 1062 rte_free(malloc_virt_addr); 1063 return; 1064 } 1065 1066 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n" 1067 "vq->vq_desc_head_idx=%d", 1068 vq->vq_free_cnt, vq->vq_desc_head_idx); 1069 1070 VIRTIO_CRYPTO_SESSION_LOG_INFO("Close session %"PRIu64" successfully ", 1071 session->session_id); 1072 1073 memset(session, 0, sizeof(struct virtio_crypto_session)); 1074 struct rte_mempool *sess_mp = rte_mempool_from_obj(session); 1075 set_sym_session_private_data(sess, cryptodev_virtio_driver_id, NULL); 1076 rte_mempool_put(sess_mp, session); 1077 rte_free(malloc_virt_addr); 1078 } 1079 1080 static struct rte_crypto_cipher_xform * 1081 virtio_crypto_get_cipher_xform(struct rte_crypto_sym_xform *xform) 1082 { 1083 do { 1084 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) 1085 return &xform->cipher; 1086 1087 xform = xform->next; 1088 } while (xform); 1089 1090 return NULL; 1091 } 1092 1093 static struct rte_crypto_auth_xform * 1094 virtio_crypto_get_auth_xform(struct rte_crypto_sym_xform *xform) 1095 { 1096 do { 1097 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) 1098 return &xform->auth; 1099 1100 xform = xform->next; 1101 } while (xform); 1102 1103 return NULL; 1104 } 1105 1106 /** Get xform chain order */ 1107 static int 1108 virtio_crypto_get_chain_order(struct rte_crypto_sym_xform *xform) 1109 { 1110 if (xform == NULL) 1111 return -1; 1112 1113 /* Cipher Only */ 1114 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 1115 xform->next == NULL) 1116 return VIRTIO_CRYPTO_CMD_CIPHER; 1117 1118 /* Authentication Only */ 1119 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1120 xform->next == NULL) 1121 return VIRTIO_CRYPTO_CMD_AUTH; 1122 1123 /* Authenticate then Cipher */ 1124 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1125 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) 1126 return VIRTIO_CRYPTO_CMD_HASH_CIPHER; 1127 1128 /* Cipher then Authenticate */ 1129 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 1130 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 1131 return VIRTIO_CRYPTO_CMD_CIPHER_HASH; 1132 1133 return -1; 1134 } 1135 1136 static int 1137 virtio_crypto_sym_pad_cipher_param( 1138 struct virtio_crypto_cipher_session_para *para, 1139 struct rte_crypto_cipher_xform *cipher_xform) 1140 { 1141 switch (cipher_xform->algo) { 1142 case RTE_CRYPTO_CIPHER_AES_CBC: 1143 para->algo = VIRTIO_CRYPTO_CIPHER_AES_CBC; 1144 break; 1145 default: 1146 VIRTIO_CRYPTO_SESSION_LOG_ERR("Crypto: Unsupported " 1147 "Cipher alg %u", cipher_xform->algo); 1148 return -1; 1149 } 1150 1151 para->keylen = cipher_xform->key.length; 1152 switch (cipher_xform->op) { 1153 case RTE_CRYPTO_CIPHER_OP_ENCRYPT: 1154 para->op = VIRTIO_CRYPTO_OP_ENCRYPT; 1155 break; 1156 case RTE_CRYPTO_CIPHER_OP_DECRYPT: 1157 para->op = VIRTIO_CRYPTO_OP_DECRYPT; 1158 break; 1159 default: 1160 VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported cipher operation " 1161 "parameter"); 1162 return -1; 1163 } 1164 1165 return 0; 1166 } 1167 1168 static int 1169 virtio_crypto_sym_pad_auth_param( 1170 struct virtio_crypto_op_ctrl_req *ctrl, 1171 struct rte_crypto_auth_xform *auth_xform) 1172 { 1173 uint32_t *algo; 1174 struct virtio_crypto_alg_chain_session_para *para = 1175 &(ctrl->u.sym_create_session.u.chain.para); 1176 1177 switch (ctrl->u.sym_create_session.u.chain.para.hash_mode) { 1178 case VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN: 1179 algo = &(para->u.hash_param.algo); 1180 break; 1181 case VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH: 1182 algo = &(para->u.mac_param.algo); 1183 break; 1184 default: 1185 VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported hash mode %u " 1186 "specified", 1187 ctrl->u.sym_create_session.u.chain.para.hash_mode); 1188 return -1; 1189 } 1190 1191 switch (auth_xform->algo) { 1192 case RTE_CRYPTO_AUTH_SHA1_HMAC: 1193 *algo = VIRTIO_CRYPTO_MAC_HMAC_SHA1; 1194 break; 1195 default: 1196 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1197 "Crypto: Undefined Hash algo %u specified", 1198 auth_xform->algo); 1199 return -1; 1200 } 1201 1202 return 0; 1203 } 1204 1205 static int 1206 virtio_crypto_sym_pad_op_ctrl_req( 1207 struct virtio_crypto_op_ctrl_req *ctrl, 1208 struct rte_crypto_sym_xform *xform, bool is_chainned, 1209 uint8_t *cipher_key_data, uint8_t *auth_key_data, 1210 struct virtio_crypto_session *session) 1211 { 1212 int ret; 1213 struct rte_crypto_auth_xform *auth_xform = NULL; 1214 struct rte_crypto_cipher_xform *cipher_xform = NULL; 1215 1216 /* Get cipher xform from crypto xform chain */ 1217 cipher_xform = virtio_crypto_get_cipher_xform(xform); 1218 if (cipher_xform) { 1219 if (cipher_xform->key.length > VIRTIO_CRYPTO_MAX_KEY_SIZE) { 1220 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1221 "cipher key size cannot be longer than %u", 1222 VIRTIO_CRYPTO_MAX_KEY_SIZE); 1223 return -1; 1224 } 1225 if (cipher_xform->iv.length > VIRTIO_CRYPTO_MAX_IV_SIZE) { 1226 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1227 "cipher IV size cannot be longer than %u", 1228 VIRTIO_CRYPTO_MAX_IV_SIZE); 1229 return -1; 1230 } 1231 if (is_chainned) 1232 ret = virtio_crypto_sym_pad_cipher_param( 1233 &ctrl->u.sym_create_session.u.chain.para 1234 .cipher_param, cipher_xform); 1235 else 1236 ret = virtio_crypto_sym_pad_cipher_param( 1237 &ctrl->u.sym_create_session.u.cipher.para, 1238 cipher_xform); 1239 1240 if (ret < 0) { 1241 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1242 "pad cipher parameter failed"); 1243 return -1; 1244 } 1245 1246 memcpy(cipher_key_data, cipher_xform->key.data, 1247 cipher_xform->key.length); 1248 1249 session->iv.offset = cipher_xform->iv.offset; 1250 session->iv.length = cipher_xform->iv.length; 1251 } 1252 1253 /* Get auth xform from crypto xform chain */ 1254 auth_xform = virtio_crypto_get_auth_xform(xform); 1255 if (auth_xform) { 1256 /* FIXME: support VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */ 1257 struct virtio_crypto_alg_chain_session_para *para = 1258 &(ctrl->u.sym_create_session.u.chain.para); 1259 if (auth_xform->key.length) { 1260 if (auth_xform->key.length > 1261 VIRTIO_CRYPTO_MAX_KEY_SIZE) { 1262 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1263 "auth key size cannot be longer than %u", 1264 VIRTIO_CRYPTO_MAX_KEY_SIZE); 1265 return -1; 1266 } 1267 para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH; 1268 para->u.mac_param.auth_key_len = 1269 (uint32_t)auth_xform->key.length; 1270 para->u.mac_param.hash_result_len = 1271 auth_xform->digest_length; 1272 memcpy(auth_key_data, auth_xform->key.data, 1273 auth_xform->key.length); 1274 } else { 1275 para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN; 1276 para->u.hash_param.hash_result_len = 1277 auth_xform->digest_length; 1278 } 1279 1280 ret = virtio_crypto_sym_pad_auth_param(ctrl, auth_xform); 1281 if (ret < 0) { 1282 VIRTIO_CRYPTO_SESSION_LOG_ERR("pad auth parameter " 1283 "failed"); 1284 return -1; 1285 } 1286 } 1287 1288 return 0; 1289 } 1290 1291 static int 1292 virtio_crypto_check_sym_configure_session_paras( 1293 struct rte_cryptodev *dev, 1294 struct rte_crypto_sym_xform *xform, 1295 struct rte_cryptodev_sym_session *sym_sess, 1296 struct rte_mempool *mempool) 1297 { 1298 if (unlikely(xform == NULL) || unlikely(sym_sess == NULL) || 1299 unlikely(mempool == NULL)) { 1300 VIRTIO_CRYPTO_SESSION_LOG_ERR("NULL pointer"); 1301 return -1; 1302 } 1303 1304 if (virtio_crypto_check_sym_session_paras(dev) < 0) 1305 return -1; 1306 1307 return 0; 1308 } 1309 1310 static int 1311 virtio_crypto_sym_configure_session( 1312 struct rte_cryptodev *dev, 1313 struct rte_crypto_sym_xform *xform, 1314 struct rte_cryptodev_sym_session *sess, 1315 struct rte_mempool *mempool) 1316 { 1317 int ret; 1318 struct virtio_crypto_session crypto_sess; 1319 void *session_private = &crypto_sess; 1320 struct virtio_crypto_session *session; 1321 struct virtio_crypto_op_ctrl_req *ctrl_req; 1322 enum virtio_crypto_cmd_id cmd_id; 1323 uint8_t cipher_key_data[VIRTIO_CRYPTO_MAX_KEY_SIZE] = {0}; 1324 uint8_t auth_key_data[VIRTIO_CRYPTO_MAX_KEY_SIZE] = {0}; 1325 struct virtio_crypto_hw *hw; 1326 struct virtqueue *control_vq; 1327 1328 PMD_INIT_FUNC_TRACE(); 1329 1330 ret = virtio_crypto_check_sym_configure_session_paras(dev, xform, 1331 sess, mempool); 1332 if (ret < 0) { 1333 VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid parameters"); 1334 return ret; 1335 } 1336 1337 if (rte_mempool_get(mempool, &session_private)) { 1338 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1339 "Couldn't get object from session mempool"); 1340 return -ENOMEM; 1341 } 1342 1343 session = (struct virtio_crypto_session *)session_private; 1344 memset(session, 0, sizeof(struct virtio_crypto_session)); 1345 ctrl_req = &session->ctrl; 1346 ctrl_req->header.opcode = VIRTIO_CRYPTO_CIPHER_CREATE_SESSION; 1347 /* FIXME: support multiqueue */ 1348 ctrl_req->header.queue_id = 0; 1349 1350 hw = dev->data->dev_private; 1351 control_vq = hw->cvq; 1352 1353 cmd_id = virtio_crypto_get_chain_order(xform); 1354 if (cmd_id == VIRTIO_CRYPTO_CMD_CIPHER_HASH) 1355 ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order 1356 = VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 1357 if (cmd_id == VIRTIO_CRYPTO_CMD_HASH_CIPHER) 1358 ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order 1359 = VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; 1360 1361 switch (cmd_id) { 1362 case VIRTIO_CRYPTO_CMD_CIPHER_HASH: 1363 case VIRTIO_CRYPTO_CMD_HASH_CIPHER: 1364 ctrl_req->u.sym_create_session.op_type 1365 = VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING; 1366 1367 ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req, 1368 xform, true, cipher_key_data, auth_key_data, session); 1369 if (ret < 0) { 1370 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1371 "padding sym op ctrl req failed"); 1372 goto error_out; 1373 } 1374 ret = virtio_crypto_send_command(control_vq, ctrl_req, 1375 cipher_key_data, auth_key_data, session); 1376 if (ret < 0) { 1377 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1378 "create session failed: %d", ret); 1379 goto error_out; 1380 } 1381 break; 1382 case VIRTIO_CRYPTO_CMD_CIPHER: 1383 ctrl_req->u.sym_create_session.op_type 1384 = VIRTIO_CRYPTO_SYM_OP_CIPHER; 1385 ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req, xform, 1386 false, cipher_key_data, auth_key_data, session); 1387 if (ret < 0) { 1388 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1389 "padding sym op ctrl req failed"); 1390 goto error_out; 1391 } 1392 ret = virtio_crypto_send_command(control_vq, ctrl_req, 1393 cipher_key_data, NULL, session); 1394 if (ret < 0) { 1395 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1396 "create session failed: %d", ret); 1397 goto error_out; 1398 } 1399 break; 1400 default: 1401 VIRTIO_CRYPTO_SESSION_LOG_ERR( 1402 "Unsupported operation chain order parameter"); 1403 goto error_out; 1404 } 1405 1406 set_sym_session_private_data(sess, dev->driver_id, 1407 session_private); 1408 1409 return 0; 1410 1411 error_out: 1412 return -1; 1413 } 1414 1415 static void 1416 virtio_crypto_dev_info_get(struct rte_cryptodev *dev, 1417 struct rte_cryptodev_info *info) 1418 { 1419 struct virtio_crypto_hw *hw = dev->data->dev_private; 1420 1421 PMD_INIT_FUNC_TRACE(); 1422 1423 if (info != NULL) { 1424 info->driver_id = cryptodev_virtio_driver_id; 1425 info->feature_flags = dev->feature_flags; 1426 info->max_nb_queue_pairs = hw->max_dataqueues; 1427 /* No limit of number of sessions */ 1428 info->sym.max_nb_sessions = 0; 1429 info->capabilities = hw->virtio_dev_capabilities; 1430 } 1431 } 1432 1433 static int 1434 crypto_virtio_pci_probe( 1435 struct rte_pci_driver *pci_drv __rte_unused, 1436 struct rte_pci_device *pci_dev) 1437 { 1438 struct rte_cryptodev_pmd_init_params init_params = { 1439 .name = "", 1440 .socket_id = pci_dev->device.numa_node, 1441 .private_data_size = sizeof(struct virtio_crypto_hw) 1442 }; 1443 char name[RTE_CRYPTODEV_NAME_MAX_LEN]; 1444 1445 VIRTIO_CRYPTO_DRV_LOG_DBG("Found Crypto device at %02x:%02x.%x", 1446 pci_dev->addr.bus, 1447 pci_dev->addr.devid, 1448 pci_dev->addr.function); 1449 1450 rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); 1451 1452 return crypto_virtio_create(name, pci_dev, &init_params); 1453 } 1454 1455 static int 1456 crypto_virtio_pci_remove( 1457 struct rte_pci_device *pci_dev __rte_unused) 1458 { 1459 struct rte_cryptodev *cryptodev; 1460 char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; 1461 1462 if (pci_dev == NULL) 1463 return -EINVAL; 1464 1465 rte_pci_device_name(&pci_dev->addr, cryptodev_name, 1466 sizeof(cryptodev_name)); 1467 1468 cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name); 1469 if (cryptodev == NULL) 1470 return -ENODEV; 1471 1472 return virtio_crypto_dev_uninit(cryptodev); 1473 } 1474 1475 static struct rte_pci_driver rte_virtio_crypto_driver = { 1476 .id_table = pci_id_virtio_crypto_map, 1477 .drv_flags = 0, 1478 .probe = crypto_virtio_pci_probe, 1479 .remove = crypto_virtio_pci_remove 1480 }; 1481 1482 static struct cryptodev_driver virtio_crypto_drv; 1483 1484 RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_VIRTIO_PMD, rte_virtio_crypto_driver); 1485 RTE_PMD_REGISTER_CRYPTO_DRIVER(virtio_crypto_drv, 1486 rte_virtio_crypto_driver.driver, 1487 cryptodev_virtio_driver_id); 1488 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_init, init, NOTICE); 1489 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_session, session, NOTICE); 1490 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_rx, rx, NOTICE); 1491 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_tx, tx, NOTICE); 1492 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_driver, driver, NOTICE); 1493