1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017-2018 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <unistd.h> 9 #include <stdbool.h> 10 #include <assert.h> 11 #include <getopt.h> 12 13 #include <rte_malloc.h> 14 #include <rte_cycles.h> 15 #include <rte_vhost.h> 16 #include <rte_cryptodev.h> 17 #include <rte_vhost_crypto.h> 18 #include <rte_string_fns.h> 19 20 #include <cmdline_rdline.h> 21 #include <cmdline_parse.h> 22 #include <cmdline_parse_string.h> 23 #include <cmdline.h> 24 25 #define NB_VIRTIO_QUEUES (1) 26 #define MAX_PKT_BURST (64) 27 #define MAX_IV_LEN (32) 28 #define NB_MEMPOOL_OBJS (8192) 29 #define NB_CRYPTO_DESCRIPTORS (4096) 30 #define NB_CACHE_OBJS (128) 31 #define SESSION_MAP_ENTRIES (1024) 32 #define REFRESH_TIME_SEC (3) 33 34 #define MAX_NB_SOCKETS (4) 35 #define MAX_NB_WORKER_CORES (16) 36 37 struct lcore_option { 38 uint32_t lcore_id; 39 char *socket_files[MAX_NB_SOCKETS]; 40 uint32_t nb_sockets; 41 uint8_t cid; 42 uint16_t qid; 43 }; 44 45 struct __rte_cache_aligned vhost_crypto_info { 46 int vids[MAX_NB_SOCKETS]; 47 uint32_t nb_vids; 48 struct rte_mempool *sess_pool; 49 struct rte_mempool *cop_pool; 50 uint8_t cid; 51 uint32_t qid; 52 uint32_t nb_inflight_ops; 53 volatile uint32_t initialized[MAX_NB_SOCKETS]; 54 }; 55 56 struct vhost_crypto_options { 57 struct lcore_option los[MAX_NB_WORKER_CORES]; 58 struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES]; 59 uint32_t nb_los; 60 uint32_t zero_copy; 61 uint32_t guest_polling; 62 } options; 63 64 enum { 65 #define OPT_CONFIG "config" 66 OPT_CONFIG_NUM = 256, 67 #define OPT_SOCKET_FILE "socket-file" 68 OPT_SOCKET_FILE_NUM, 69 #define OPT_ZERO_COPY "zero-copy" 70 OPT_ZERO_COPY_NUM, 71 #define OPT_POLLING "guest-polling" 72 OPT_POLLING_NUM, 73 }; 74 75 #define NB_SOCKET_FIELDS (2) 76 77 static uint32_t 78 find_lo(uint32_t lcore_id) 79 { 80 uint32_t i; 81 82 for (i = 0; i < options.nb_los; i++) 83 if (options.los[i].lcore_id == lcore_id) 84 return i; 85 86 return UINT32_MAX; 87 } 88 89 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */ 90 static int 91 parse_socket_arg(char *arg) 92 { 93 uint32_t nb_sockets; 94 uint32_t lcore_id; 95 char *str_fld[NB_SOCKET_FIELDS]; 96 struct lcore_option *lo; 97 uint32_t idx; 98 char *end; 99 100 if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') != 101 NB_SOCKET_FIELDS) { 102 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg); 103 return -EINVAL; 104 } 105 106 errno = 0; 107 lcore_id = strtoul(str_fld[0], &end, 0); 108 if (errno != 0 || end == str_fld[0] || lcore_id > 255) 109 return -EINVAL; 110 111 idx = find_lo(lcore_id); 112 if (idx == UINT32_MAX) { 113 if (options.nb_los == MAX_NB_WORKER_CORES) 114 return -ENOMEM; 115 lo = &options.los[options.nb_los]; 116 lo->lcore_id = lcore_id; 117 options.nb_los++; 118 } else 119 lo = &options.los[idx]; 120 121 nb_sockets = lo->nb_sockets; 122 123 if (nb_sockets >= MAX_NB_SOCKETS) { 124 RTE_LOG(ERR, USER1, "Too many socket files!\n"); 125 return -ENOMEM; 126 } 127 128 lo->socket_files[nb_sockets] = strdup(str_fld[1]); 129 if (!lo->socket_files[nb_sockets]) { 130 RTE_LOG(ERR, USER1, "Insufficient memory\n"); 131 return -ENOMEM; 132 } 133 134 lo->nb_sockets++; 135 136 return 0; 137 } 138 139 static int 140 parse_config(char *q_arg) 141 { 142 struct lcore_option *lo; 143 char s[256]; 144 const char *p, *p0 = q_arg; 145 char *end; 146 enum fieldnames { 147 FLD_LCORE = 0, 148 FLD_CID, 149 FLD_QID, 150 _NUM_FLD 151 }; 152 uint32_t flds[_NUM_FLD]; 153 char *str_fld[_NUM_FLD]; 154 uint32_t i; 155 uint32_t size; 156 157 while ((p = strchr(p0, '(')) != NULL) { 158 ++p; 159 p0 = strchr(p, ')'); 160 if (p0 == NULL) 161 return -1; 162 163 size = p0 - p; 164 if (size >= sizeof(s)) 165 return -1; 166 167 snprintf(s, sizeof(s), "%.*s", size, p); 168 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 169 _NUM_FLD) 170 return -1; 171 for (i = 0; i < _NUM_FLD; i++) { 172 errno = 0; 173 flds[i] = strtoul(str_fld[i], &end, 0); 174 if (errno != 0 || end == str_fld[i] || flds[i] > 255) 175 return -EINVAL; 176 } 177 178 if (flds[FLD_LCORE] > RTE_MAX_LCORE) 179 return -EINVAL; 180 181 i = find_lo(flds[FLD_LCORE]); 182 if (i == UINT32_MAX) { 183 if (options.nb_los == MAX_NB_WORKER_CORES) 184 return -ENOMEM; 185 lo = &options.los[options.nb_los]; 186 options.nb_los++; 187 } else 188 lo = &options.los[i]; 189 190 lo->lcore_id = flds[FLD_LCORE]; 191 lo->cid = flds[FLD_CID]; 192 lo->qid = flds[FLD_QID]; 193 } 194 195 return 0; 196 } 197 198 static void 199 vhost_crypto_usage(const char *prgname) 200 { 201 printf("%s [EAL options] --\n" 202 " --%s <lcore>,SOCKET-FILE-PATH\n" 203 " --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]\n" 204 " --%s: zero copy\n" 205 " --%s: guest polling\n", 206 prgname, OPT_SOCKET_FILE, OPT_CONFIG, 207 OPT_ZERO_COPY, OPT_POLLING); 208 } 209 210 static int 211 vhost_crypto_parse_args(int argc, char **argv) 212 { 213 int opt, ret; 214 char *prgname = argv[0]; 215 char **argvopt; 216 int option_index; 217 struct option lgopts[] = { 218 {OPT_SOCKET_FILE, required_argument, 219 NULL, OPT_SOCKET_FILE_NUM}, 220 {OPT_CONFIG, required_argument, 221 NULL, OPT_CONFIG_NUM}, 222 {OPT_ZERO_COPY, no_argument, 223 NULL, OPT_ZERO_COPY_NUM}, 224 {OPT_POLLING, no_argument, 225 NULL, OPT_POLLING_NUM}, 226 {NULL, 0, 0, 0} 227 }; 228 229 argvopt = argv; 230 231 while ((opt = getopt_long(argc, argvopt, "", 232 lgopts, &option_index)) != EOF) { 233 234 if (opt == '?') { 235 vhost_crypto_usage(prgname); 236 return -1; 237 } 238 239 switch (opt) { 240 case OPT_SOCKET_FILE_NUM: 241 ret = parse_socket_arg(optarg); 242 if (ret < 0) { 243 vhost_crypto_usage(prgname); 244 return ret; 245 } 246 break; 247 248 case OPT_CONFIG_NUM: 249 ret = parse_config(optarg); 250 if (ret < 0) { 251 vhost_crypto_usage(prgname); 252 return ret; 253 } 254 break; 255 256 case OPT_ZERO_COPY_NUM: 257 options.zero_copy = 258 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE; 259 break; 260 261 case OPT_POLLING_NUM: 262 options.guest_polling = 1; 263 break; 264 265 default: 266 vhost_crypto_usage(prgname); 267 return -EINVAL; 268 } 269 } 270 271 return 0; 272 } 273 274 static int 275 new_device(int vid) 276 { 277 struct vhost_crypto_info *info = NULL; 278 char path[PATH_MAX]; 279 uint32_t i, j; 280 int ret; 281 282 ret = rte_vhost_get_ifname(vid, path, PATH_MAX); 283 if (ret) { 284 RTE_LOG(ERR, USER1, "Cannot find matched socket\n"); 285 return ret; 286 } 287 288 for (i = 0; i < options.nb_los; i++) { 289 for (j = 0; j < options.los[i].nb_sockets; j++) { 290 if (strcmp(path, options.los[i].socket_files[j]) == 0) { 291 info = options.infos[i]; 292 break; 293 } 294 } 295 296 if (info) 297 break; 298 } 299 300 if (!info) { 301 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n"); 302 return -ENOENT; 303 } 304 305 ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool, 306 rte_lcore_to_socket_id(options.los[i].lcore_id)); 307 if (ret) { 308 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n"); 309 return ret; 310 } 311 312 ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy); 313 if (ret) { 314 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n", 315 options.zero_copy == 1 ? "enable" : "disable"); 316 return ret; 317 } 318 319 info->vids[j] = vid; 320 info->initialized[j] = 1; 321 322 rte_wmb(); 323 324 RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path, 325 vid); 326 return 0; 327 } 328 329 static void 330 destroy_device(int vid) 331 { 332 struct vhost_crypto_info *info = NULL; 333 uint32_t i, j; 334 335 for (i = 0; i < options.nb_los; i++) { 336 for (j = 0; j < options.los[i].nb_sockets; j++) { 337 if (options.infos[i]->vids[j] == vid) { 338 info = options.infos[i]; 339 break; 340 } 341 } 342 if (info) 343 break; 344 } 345 346 if (!info) { 347 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n"); 348 return; 349 } 350 351 do { 352 353 } while (info->nb_inflight_ops); 354 355 info->initialized[j] = 0; 356 357 rte_wmb(); 358 359 rte_vhost_crypto_free(vid); 360 361 RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid); 362 } 363 364 static const struct rte_vhost_device_ops virtio_crypto_device_ops = { 365 .new_device = new_device, 366 .destroy_device = destroy_device, 367 }; 368 369 static int 370 vhost_crypto_worker(void *arg) 371 { 372 struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 373 struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 374 struct vhost_crypto_info *info = arg; 375 uint16_t nb_callfds; 376 int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS]; 377 uint32_t lcore_id = rte_lcore_id(); 378 uint32_t burst_size = MAX_PKT_BURST; 379 uint32_t i, j, k; 380 uint32_t to_fetch, fetched; 381 382 int ret = 0; 383 384 RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id); 385 386 for (i = 0; i < NB_VIRTIO_QUEUES; i++) { 387 if (rte_crypto_op_bulk_alloc(info->cop_pool, 388 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i], 389 burst_size) < burst_size) { 390 RTE_LOG(ERR, USER1, "Failed to alloc cops\n"); 391 ret = -1; 392 goto exit; 393 } 394 } 395 396 while (1) { 397 for (i = 0; i < info->nb_vids; i++) { 398 if (unlikely(info->initialized[i] == 0)) 399 continue; 400 401 for (j = 0; j < NB_VIRTIO_QUEUES; j++) { 402 to_fetch = RTE_MIN(burst_size, 403 (NB_CRYPTO_DESCRIPTORS - 404 info->nb_inflight_ops)); 405 fetched = rte_vhost_crypto_fetch_requests( 406 info->vids[i], j, ops[j], 407 to_fetch); 408 info->nb_inflight_ops += 409 rte_cryptodev_enqueue_burst( 410 info->cid, info->qid, ops[j], 411 fetched); 412 if (unlikely(rte_crypto_op_bulk_alloc( 413 info->cop_pool, 414 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 415 ops[j], fetched) < fetched)) { 416 RTE_LOG(ERR, USER1, "Failed realloc\n"); 417 return -1; 418 } 419 420 fetched = rte_cryptodev_dequeue_burst( 421 info->cid, info->qid, 422 ops_deq[j], RTE_MIN(burst_size, 423 info->nb_inflight_ops)); 424 fetched = rte_vhost_crypto_finalize_requests( 425 ops_deq[j], fetched, callfds, 426 &nb_callfds); 427 428 info->nb_inflight_ops -= fetched; 429 430 if (!options.guest_polling) { 431 for (k = 0; k < nb_callfds; k++) 432 eventfd_write(callfds[k], 433 (eventfd_t)1); 434 } 435 436 rte_mempool_put_bulk(info->cop_pool, 437 (void **)ops_deq[j], fetched); 438 } 439 } 440 } 441 exit: 442 return ret; 443 } 444 445 static void 446 free_resource(void) 447 { 448 uint32_t i, j; 449 450 for (i = 0; i < options.nb_los; i++) { 451 struct lcore_option *lo = &options.los[i]; 452 struct vhost_crypto_info *info = options.infos[i]; 453 454 if (!info) 455 continue; 456 457 rte_mempool_free(info->cop_pool); 458 rte_mempool_free(info->sess_pool); 459 460 for (j = 0; j < lo->nb_sockets; j++) { 461 rte_vhost_driver_unregister(lo->socket_files[i]); 462 free(lo->socket_files[i]); 463 } 464 465 rte_free(info); 466 } 467 468 memset(&options, 0, sizeof(options)); 469 470 /* clean up the EAL */ 471 rte_eal_cleanup(); 472 } 473 474 int 475 main(int argc, char *argv[]) 476 { 477 struct rte_cryptodev_qp_conf qp_conf; 478 struct rte_cryptodev_config config; 479 struct rte_cryptodev_info dev_info; 480 char name[128]; 481 uint32_t i, j, lcore; 482 int ret; 483 484 ret = rte_eal_init(argc, argv); 485 if (ret < 0) 486 return -1; 487 argc -= ret; 488 argv += ret; 489 490 ret = vhost_crypto_parse_args(argc, argv); 491 if (ret < 0) 492 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n"); 493 494 for (i = 0; i < options.nb_los; i++) { 495 struct lcore_option *lo = &options.los[i]; 496 struct vhost_crypto_info *info; 497 498 info = rte_zmalloc_socket(NULL, sizeof(*info), 499 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id( 500 lo->lcore_id)); 501 if (!info) { 502 ret = -ENOMEM; 503 goto error_exit; 504 } 505 506 info->cid = lo->cid; 507 info->qid = lo->qid; 508 info->nb_vids = lo->nb_sockets; 509 510 rte_cryptodev_info_get(info->cid, &dev_info); 511 if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) { 512 #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD crypto_aesni_mb 513 #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm 514 if (strstr(dev_info.driver_name, 515 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) || 516 strstr(dev_info.driver_name, 517 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) { 518 RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n", 519 dev_info.driver_name); 520 ret = -EPERM; 521 goto error_exit; 522 } 523 } 524 525 if (dev_info.max_nb_queue_pairs < info->qid + 1) { 526 RTE_LOG(ERR, USER1, "Number of queues cannot over %u", 527 dev_info.max_nb_queue_pairs); 528 goto error_exit; 529 } 530 531 config.nb_queue_pairs = dev_info.max_nb_queue_pairs; 532 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id); 533 config.ff_disable = RTE_CRYPTODEV_FF_SECURITY; 534 535 ret = rte_cryptodev_configure(info->cid, &config); 536 if (ret < 0) { 537 RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u", 538 info->cid); 539 goto error_exit; 540 } 541 542 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id); 543 info->sess_pool = rte_cryptodev_sym_session_pool_create(name, 544 SESSION_MAP_ENTRIES, 545 rte_cryptodev_sym_get_private_session_size( 546 info->cid), 0, 0, 547 rte_lcore_to_socket_id(lo->lcore_id)); 548 549 if (!info->sess_pool) { 550 RTE_LOG(ERR, USER1, "Failed to create mempool"); 551 goto error_exit; 552 } 553 554 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id); 555 info->cop_pool = rte_crypto_op_pool_create(name, 556 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS, 557 NB_CACHE_OBJS, VHOST_CRYPTO_MAX_IV_LEN, 558 rte_lcore_to_socket_id(lo->lcore_id)); 559 560 if (!info->cop_pool) { 561 RTE_LOG(ERR, USER1, "Failed to create crypto pool"); 562 ret = -ENOMEM; 563 goto error_exit; 564 } 565 566 options.infos[i] = info; 567 568 qp_conf.nb_descriptors = NB_CRYPTO_DESCRIPTORS; 569 qp_conf.mp_session = info->sess_pool; 570 571 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) { 572 ret = rte_cryptodev_queue_pair_setup(info->cid, j, 573 &qp_conf, rte_lcore_to_socket_id( 574 lo->lcore_id)); 575 if (ret < 0) { 576 RTE_LOG(ERR, USER1, "Failed to configure qp\n"); 577 goto error_exit; 578 } 579 } 580 } 581 582 for (i = 0; i < options.nb_los; i++) { 583 struct lcore_option *lo = &options.los[i]; 584 struct vhost_crypto_info *info = options.infos[i]; 585 586 ret = rte_cryptodev_start(lo->cid); 587 if (ret < 0) { 588 RTE_LOG(ERR, USER1, "Failed to start cryptodev\n"); 589 goto error_exit; 590 } 591 592 if (rte_eal_remote_launch(vhost_crypto_worker, info, 593 lo->lcore_id) < 0) { 594 RTE_LOG(ERR, USER1, "Failed to start worker lcore"); 595 goto error_exit; 596 } 597 598 for (j = 0; j < lo->nb_sockets; j++) { 599 ret = rte_vhost_driver_register(lo->socket_files[j], 600 RTE_VHOST_USER_ASYNC_COPY); 601 if (ret < 0) { 602 RTE_LOG(ERR, USER1, "socket %s already exists\n", 603 lo->socket_files[j]); 604 goto error_exit; 605 } 606 607 rte_vhost_driver_callback_register(lo->socket_files[j], 608 &virtio_crypto_device_ops); 609 610 ret = rte_vhost_crypto_driver_start( 611 lo->socket_files[j]); 612 if (ret < 0) { 613 RTE_LOG(ERR, USER1, "failed to start vhost.\n"); 614 goto error_exit; 615 } 616 } 617 } 618 619 RTE_LCORE_FOREACH(lcore) 620 rte_eal_wait_lcore(lcore); 621 622 free_resource(); 623 624 return 0; 625 626 error_exit: 627 628 free_resource(); 629 630 return -1; 631 } 632