1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017-2018 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <unistd.h> 9 #include <stdbool.h> 10 #include <assert.h> 11 #include <getopt.h> 12 13 #include <rte_malloc.h> 14 #include <rte_cycles.h> 15 #include <rte_vhost.h> 16 #include <rte_cryptodev.h> 17 #include <rte_vhost_crypto.h> 18 #include <rte_string_fns.h> 19 20 #include <cmdline_rdline.h> 21 #include <cmdline_parse.h> 22 #include <cmdline_parse_string.h> 23 #include <cmdline.h> 24 25 #define NB_VIRTIO_QUEUES (1) 26 #define MAX_PKT_BURST (64) 27 #define MAX_IV_LEN (32) 28 #define NB_MEMPOOL_OBJS (8192) 29 #define NB_CRYPTO_DESCRIPTORS (4096) 30 #define NB_CACHE_OBJS (128) 31 #define SESSION_MAP_ENTRIES (1024) 32 #define REFRESH_TIME_SEC (3) 33 34 #define MAX_NB_SOCKETS (4) 35 #define MAX_NB_WORKER_CORES (16) 36 37 struct lcore_option { 38 uint32_t lcore_id; 39 char *socket_files[MAX_NB_SOCKETS]; 40 uint32_t nb_sockets; 41 uint8_t cid; 42 uint16_t qid; 43 }; 44 45 struct vhost_crypto_info { 46 int vids[MAX_NB_SOCKETS]; 47 uint32_t nb_vids; 48 struct rte_mempool *sess_pool; 49 struct rte_mempool *sess_priv_pool; 50 struct rte_mempool *cop_pool; 51 uint8_t cid; 52 uint32_t qid; 53 uint32_t nb_inflight_ops; 54 volatile uint32_t initialized[MAX_NB_SOCKETS]; 55 } __rte_cache_aligned; 56 57 struct vhost_crypto_options { 58 struct lcore_option los[MAX_NB_WORKER_CORES]; 59 struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES]; 60 uint32_t nb_los; 61 uint32_t zero_copy; 62 uint32_t guest_polling; 63 } options; 64 65 enum { 66 #define OPT_CONFIG "config" 67 OPT_CONFIG_NUM = 256, 68 #define OPT_SOCKET_FILE "socket-file" 69 OPT_SOCKET_FILE_NUM, 70 #define OPT_ZERO_COPY "zero-copy" 71 OPT_ZERO_COPY_NUM, 72 #define OPT_POLLING "guest-polling" 73 OPT_POLLING_NUM, 74 }; 75 76 #define NB_SOCKET_FIELDS (2) 77 78 static uint32_t 79 find_lo(uint32_t lcore_id) 80 { 81 uint32_t i; 82 83 for (i = 0; i < options.nb_los; i++) 84 if (options.los[i].lcore_id == lcore_id) 85 return i; 86 87 return UINT32_MAX; 88 } 89 90 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */ 91 static int 92 parse_socket_arg(char *arg) 93 { 94 uint32_t nb_sockets; 95 uint32_t lcore_id; 96 char *str_fld[NB_SOCKET_FIELDS]; 97 struct lcore_option *lo; 98 uint32_t idx; 99 char *end; 100 101 if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') != 102 NB_SOCKET_FIELDS) { 103 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg); 104 return -EINVAL; 105 } 106 107 errno = 0; 108 lcore_id = strtoul(str_fld[0], &end, 0); 109 if (errno != 0 || end == str_fld[0] || lcore_id > 255) 110 return -EINVAL; 111 112 idx = find_lo(lcore_id); 113 if (idx == UINT32_MAX) { 114 if (options.nb_los == MAX_NB_WORKER_CORES) 115 return -ENOMEM; 116 lo = &options.los[options.nb_los]; 117 lo->lcore_id = lcore_id; 118 options.nb_los++; 119 } else 120 lo = &options.los[idx]; 121 122 nb_sockets = lo->nb_sockets; 123 124 if (nb_sockets >= MAX_NB_SOCKETS) { 125 RTE_LOG(ERR, USER1, "Too many socket files!\n"); 126 return -ENOMEM; 127 } 128 129 lo->socket_files[nb_sockets] = strdup(str_fld[1]); 130 if (!lo->socket_files[nb_sockets]) { 131 RTE_LOG(ERR, USER1, "Insufficient memory\n"); 132 return -ENOMEM; 133 } 134 135 lo->nb_sockets++; 136 137 return 0; 138 } 139 140 static int 141 parse_config(char *q_arg) 142 { 143 struct lcore_option *lo; 144 char s[256]; 145 const char *p, *p0 = q_arg; 146 char *end; 147 enum fieldnames { 148 FLD_LCORE = 0, 149 FLD_CID, 150 FLD_QID, 151 _NUM_FLD 152 }; 153 uint32_t flds[_NUM_FLD]; 154 char *str_fld[_NUM_FLD]; 155 uint32_t i; 156 uint32_t size; 157 158 while ((p = strchr(p0, '(')) != NULL) { 159 ++p; 160 p0 = strchr(p, ')'); 161 if (p0 == NULL) 162 return -1; 163 164 size = p0 - p; 165 if (size >= sizeof(s)) 166 return -1; 167 168 snprintf(s, sizeof(s), "%.*s", size, p); 169 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 170 _NUM_FLD) 171 return -1; 172 for (i = 0; i < _NUM_FLD; i++) { 173 errno = 0; 174 flds[i] = strtoul(str_fld[i], &end, 0); 175 if (errno != 0 || end == str_fld[i] || flds[i] > 255) 176 return -EINVAL; 177 } 178 179 if (flds[FLD_LCORE] > RTE_MAX_LCORE) 180 return -EINVAL; 181 182 i = find_lo(flds[FLD_LCORE]); 183 if (i == UINT32_MAX) { 184 if (options.nb_los == MAX_NB_WORKER_CORES) 185 return -ENOMEM; 186 lo = &options.los[options.nb_los]; 187 options.nb_los++; 188 } else 189 lo = &options.los[i]; 190 191 lo->lcore_id = flds[FLD_LCORE]; 192 lo->cid = flds[FLD_CID]; 193 lo->qid = flds[FLD_QID]; 194 } 195 196 return 0; 197 } 198 199 static void 200 vhost_crypto_usage(const char *prgname) 201 { 202 printf("%s [EAL options] --\n" 203 " --%s <lcore>,SOCKET-FILE-PATH\n" 204 " --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]\n" 205 " --%s: zero copy\n" 206 " --%s: guest polling\n", 207 prgname, OPT_SOCKET_FILE, OPT_CONFIG, 208 OPT_ZERO_COPY, OPT_POLLING); 209 } 210 211 static int 212 vhost_crypto_parse_args(int argc, char **argv) 213 { 214 int opt, ret; 215 char *prgname = argv[0]; 216 char **argvopt; 217 int option_index; 218 struct option lgopts[] = { 219 {OPT_SOCKET_FILE, required_argument, 220 NULL, OPT_SOCKET_FILE_NUM}, 221 {OPT_CONFIG, required_argument, 222 NULL, OPT_CONFIG_NUM}, 223 {OPT_ZERO_COPY, no_argument, 224 NULL, OPT_ZERO_COPY_NUM}, 225 {OPT_POLLING, no_argument, 226 NULL, OPT_POLLING_NUM}, 227 {NULL, 0, 0, 0} 228 }; 229 230 argvopt = argv; 231 232 while ((opt = getopt_long(argc, argvopt, "", 233 lgopts, &option_index)) != EOF) { 234 235 if (opt == '?') { 236 vhost_crypto_usage(prgname); 237 return -1; 238 } 239 240 switch (opt) { 241 case OPT_SOCKET_FILE_NUM: 242 ret = parse_socket_arg(optarg); 243 if (ret < 0) { 244 vhost_crypto_usage(prgname); 245 return ret; 246 } 247 break; 248 249 case OPT_CONFIG_NUM: 250 ret = parse_config(optarg); 251 if (ret < 0) { 252 vhost_crypto_usage(prgname); 253 return ret; 254 } 255 break; 256 257 case OPT_ZERO_COPY_NUM: 258 options.zero_copy = 259 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE; 260 break; 261 262 case OPT_POLLING_NUM: 263 options.guest_polling = 1; 264 break; 265 266 default: 267 vhost_crypto_usage(prgname); 268 return -EINVAL; 269 } 270 } 271 272 return 0; 273 } 274 275 static int 276 new_device(int vid) 277 { 278 struct vhost_crypto_info *info = NULL; 279 char path[PATH_MAX]; 280 uint32_t i, j; 281 int ret; 282 283 ret = rte_vhost_get_ifname(vid, path, PATH_MAX); 284 if (ret) { 285 RTE_LOG(ERR, USER1, "Cannot find matched socket\n"); 286 return ret; 287 } 288 289 for (i = 0; i < options.nb_los; i++) { 290 for (j = 0; j < options.los[i].nb_sockets; j++) { 291 if (strcmp(path, options.los[i].socket_files[j]) == 0) { 292 info = options.infos[i]; 293 break; 294 } 295 } 296 297 if (info) 298 break; 299 } 300 301 if (!info) { 302 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n"); 303 return -ENOENT; 304 } 305 306 ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool, 307 info->sess_priv_pool, 308 rte_lcore_to_socket_id(options.los[i].lcore_id)); 309 if (ret) { 310 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n"); 311 return ret; 312 } 313 314 ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy); 315 if (ret) { 316 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n", 317 options.zero_copy == 1 ? "enable" : "disable"); 318 return ret; 319 } 320 321 info->vids[j] = vid; 322 info->initialized[j] = 1; 323 324 rte_wmb(); 325 326 RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path, 327 vid); 328 return 0; 329 } 330 331 static void 332 destroy_device(int vid) 333 { 334 struct vhost_crypto_info *info = NULL; 335 uint32_t i, j; 336 337 for (i = 0; i < options.nb_los; i++) { 338 for (j = 0; j < options.los[i].nb_sockets; j++) { 339 if (options.infos[i]->vids[j] == vid) { 340 info = options.infos[i]; 341 break; 342 } 343 } 344 if (info) 345 break; 346 } 347 348 if (!info) { 349 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n"); 350 return; 351 } 352 353 do { 354 355 } while (info->nb_inflight_ops); 356 357 info->initialized[j] = 0; 358 359 rte_wmb(); 360 361 rte_vhost_crypto_free(vid); 362 363 RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid); 364 } 365 366 static const struct vhost_device_ops virtio_crypto_device_ops = { 367 .new_device = new_device, 368 .destroy_device = destroy_device, 369 }; 370 371 static int 372 vhost_crypto_worker(void *arg) 373 { 374 struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 375 struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 376 struct vhost_crypto_info *info = arg; 377 uint16_t nb_callfds; 378 int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS]; 379 uint32_t lcore_id = rte_lcore_id(); 380 uint32_t burst_size = MAX_PKT_BURST; 381 uint32_t i, j, k; 382 uint32_t to_fetch, fetched; 383 384 int ret = 0; 385 386 RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id); 387 388 for (i = 0; i < NB_VIRTIO_QUEUES; i++) { 389 if (rte_crypto_op_bulk_alloc(info->cop_pool, 390 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i], 391 burst_size) < burst_size) { 392 RTE_LOG(ERR, USER1, "Failed to alloc cops\n"); 393 ret = -1; 394 goto exit; 395 } 396 } 397 398 while (1) { 399 for (i = 0; i < info->nb_vids; i++) { 400 if (unlikely(info->initialized[i] == 0)) 401 continue; 402 403 for (j = 0; j < NB_VIRTIO_QUEUES; j++) { 404 to_fetch = RTE_MIN(burst_size, 405 (NB_CRYPTO_DESCRIPTORS - 406 info->nb_inflight_ops)); 407 fetched = rte_vhost_crypto_fetch_requests( 408 info->vids[i], j, ops[j], 409 to_fetch); 410 info->nb_inflight_ops += 411 rte_cryptodev_enqueue_burst( 412 info->cid, info->qid, ops[j], 413 fetched); 414 if (unlikely(rte_crypto_op_bulk_alloc( 415 info->cop_pool, 416 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 417 ops[j], fetched) < fetched)) { 418 RTE_LOG(ERR, USER1, "Failed realloc\n"); 419 return -1; 420 } 421 422 fetched = rte_cryptodev_dequeue_burst( 423 info->cid, info->qid, 424 ops_deq[j], RTE_MIN(burst_size, 425 info->nb_inflight_ops)); 426 fetched = rte_vhost_crypto_finalize_requests( 427 ops_deq[j], fetched, callfds, 428 &nb_callfds); 429 430 info->nb_inflight_ops -= fetched; 431 432 if (!options.guest_polling) { 433 for (k = 0; k < nb_callfds; k++) 434 eventfd_write(callfds[k], 435 (eventfd_t)1); 436 } 437 438 rte_mempool_put_bulk(info->cop_pool, 439 (void **)ops_deq[j], fetched); 440 } 441 } 442 } 443 exit: 444 return ret; 445 } 446 447 static void 448 free_resource(void) 449 { 450 uint32_t i, j; 451 452 for (i = 0; i < options.nb_los; i++) { 453 struct lcore_option *lo = &options.los[i]; 454 struct vhost_crypto_info *info = options.infos[i]; 455 456 if (!info) 457 continue; 458 459 rte_mempool_free(info->cop_pool); 460 rte_mempool_free(info->sess_pool); 461 rte_mempool_free(info->sess_priv_pool); 462 463 for (j = 0; j < lo->nb_sockets; j++) { 464 rte_vhost_driver_unregister(lo->socket_files[i]); 465 free(lo->socket_files[i]); 466 } 467 468 rte_free(info); 469 } 470 471 memset(&options, 0, sizeof(options)); 472 473 /* clean up the EAL */ 474 rte_eal_cleanup(); 475 } 476 477 int 478 main(int argc, char *argv[]) 479 { 480 struct rte_cryptodev_qp_conf qp_conf; 481 struct rte_cryptodev_config config; 482 struct rte_cryptodev_info dev_info; 483 char name[128]; 484 uint32_t i, j, lcore; 485 int ret; 486 487 ret = rte_eal_init(argc, argv); 488 if (ret < 0) 489 return -1; 490 argc -= ret; 491 argv += ret; 492 493 ret = vhost_crypto_parse_args(argc, argv); 494 if (ret < 0) 495 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n"); 496 497 for (i = 0; i < options.nb_los; i++) { 498 struct lcore_option *lo = &options.los[i]; 499 struct vhost_crypto_info *info; 500 501 info = rte_zmalloc_socket(NULL, sizeof(*info), 502 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id( 503 lo->lcore_id)); 504 if (!info) { 505 ret = -ENOMEM; 506 goto error_exit; 507 } 508 509 info->cid = lo->cid; 510 info->qid = lo->qid; 511 info->nb_vids = lo->nb_sockets; 512 513 rte_cryptodev_info_get(info->cid, &dev_info); 514 if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) { 515 #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD crypto_aesni_mb 516 #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm 517 if (strstr(dev_info.driver_name, 518 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) || 519 strstr(dev_info.driver_name, 520 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) { 521 RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n", 522 dev_info.driver_name); 523 ret = -EPERM; 524 goto error_exit; 525 } 526 } 527 528 if (dev_info.max_nb_queue_pairs < info->qid + 1) { 529 RTE_LOG(ERR, USER1, "Number of queues cannot over %u", 530 dev_info.max_nb_queue_pairs); 531 goto error_exit; 532 } 533 534 config.nb_queue_pairs = dev_info.max_nb_queue_pairs; 535 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id); 536 config.ff_disable = RTE_CRYPTODEV_FF_SECURITY; 537 538 ret = rte_cryptodev_configure(info->cid, &config); 539 if (ret < 0) { 540 RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u", 541 info->cid); 542 goto error_exit; 543 } 544 545 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id); 546 info->sess_pool = rte_cryptodev_sym_session_pool_create(name, 547 SESSION_MAP_ENTRIES, 0, 0, 0, 548 rte_lcore_to_socket_id(lo->lcore_id)); 549 550 snprintf(name, 127, "SESS_POOL_PRIV_%u", lo->lcore_id); 551 info->sess_priv_pool = rte_mempool_create(name, 552 SESSION_MAP_ENTRIES, 553 rte_cryptodev_sym_get_private_session_size( 554 info->cid), 64, 0, NULL, NULL, NULL, NULL, 555 rte_lcore_to_socket_id(lo->lcore_id), 0); 556 if (!info->sess_priv_pool || !info->sess_pool) { 557 RTE_LOG(ERR, USER1, "Failed to create mempool"); 558 goto error_exit; 559 } 560 561 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id); 562 info->cop_pool = rte_crypto_op_pool_create(name, 563 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS, 564 NB_CACHE_OBJS, VHOST_CRYPTO_MAX_IV_LEN, 565 rte_lcore_to_socket_id(lo->lcore_id)); 566 567 if (!info->cop_pool) { 568 RTE_LOG(ERR, USER1, "Failed to create crypto pool"); 569 ret = -ENOMEM; 570 goto error_exit; 571 } 572 573 options.infos[i] = info; 574 575 qp_conf.nb_descriptors = NB_CRYPTO_DESCRIPTORS; 576 qp_conf.mp_session = info->sess_pool; 577 qp_conf.mp_session_private = info->sess_priv_pool; 578 579 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) { 580 ret = rte_cryptodev_queue_pair_setup(info->cid, j, 581 &qp_conf, rte_lcore_to_socket_id( 582 lo->lcore_id)); 583 if (ret < 0) { 584 RTE_LOG(ERR, USER1, "Failed to configure qp\n"); 585 goto error_exit; 586 } 587 } 588 } 589 590 for (i = 0; i < options.nb_los; i++) { 591 struct lcore_option *lo = &options.los[i]; 592 struct vhost_crypto_info *info = options.infos[i]; 593 594 ret = rte_cryptodev_start(lo->cid); 595 if (ret < 0) { 596 RTE_LOG(ERR, USER1, "Failed to start cryptodev\n"); 597 goto error_exit; 598 } 599 600 if (rte_eal_remote_launch(vhost_crypto_worker, info, 601 lo->lcore_id) < 0) { 602 RTE_LOG(ERR, USER1, "Failed to start worker lcore"); 603 goto error_exit; 604 } 605 606 for (j = 0; j < lo->nb_sockets; j++) { 607 ret = rte_vhost_driver_register(lo->socket_files[j], 608 RTE_VHOST_USER_ASYNC_COPY); 609 if (ret < 0) { 610 RTE_LOG(ERR, USER1, "socket %s already exists\n", 611 lo->socket_files[j]); 612 goto error_exit; 613 } 614 615 rte_vhost_driver_callback_register(lo->socket_files[j], 616 &virtio_crypto_device_ops); 617 618 ret = rte_vhost_crypto_driver_start( 619 lo->socket_files[j]); 620 if (ret < 0) { 621 RTE_LOG(ERR, USER1, "failed to start vhost.\n"); 622 goto error_exit; 623 } 624 } 625 } 626 627 RTE_LCORE_FOREACH(lcore) 628 rte_eal_wait_lcore(lcore); 629 630 free_resource(); 631 632 return 0; 633 634 error_exit: 635 636 free_resource(); 637 638 return -1; 639 } 640