1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017-2018 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <unistd.h> 9 #include <stdbool.h> 10 #include <assert.h> 11 #include <getopt.h> 12 13 #include <rte_malloc.h> 14 #include <rte_cycles.h> 15 #include <rte_vhost.h> 16 #include <rte_cryptodev.h> 17 #include <rte_vhost_crypto.h> 18 #include <rte_string_fns.h> 19 20 #include <cmdline_rdline.h> 21 #include <cmdline_parse.h> 22 #include <cmdline_parse_string.h> 23 #include <cmdline.h> 24 25 #define NB_VIRTIO_QUEUES (1) 26 #define MAX_PKT_BURST (64) 27 #define MAX_IV_LEN (32) 28 #define NB_MEMPOOL_OBJS (8192) 29 #define NB_CRYPTO_DESCRIPTORS (4096) 30 #define NB_CACHE_OBJS (128) 31 #define SESSION_MAP_ENTRIES (1024) 32 #define REFRESH_TIME_SEC (3) 33 34 #define MAX_NB_SOCKETS (4) 35 #define MAX_NB_WORKER_CORES (16) 36 37 struct lcore_option { 38 uint32_t lcore_id; 39 char *socket_files[MAX_NB_SOCKETS]; 40 uint32_t nb_sockets; 41 uint8_t cid; 42 uint16_t qid; 43 }; 44 45 struct vhost_crypto_info { 46 int vids[MAX_NB_SOCKETS]; 47 uint32_t nb_vids; 48 struct rte_mempool *sess_pool; 49 struct rte_mempool *sess_priv_pool; 50 struct rte_mempool *cop_pool; 51 uint8_t cid; 52 uint32_t qid; 53 uint32_t nb_inflight_ops; 54 volatile uint32_t initialized[MAX_NB_SOCKETS]; 55 } __rte_cache_aligned; 56 57 struct vhost_crypto_options { 58 struct lcore_option los[MAX_NB_WORKER_CORES]; 59 struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES]; 60 uint32_t nb_los; 61 uint32_t zero_copy; 62 uint32_t guest_polling; 63 } options; 64 65 #define CONFIG_KEYWORD "config" 66 #define SOCKET_FILE_KEYWORD "socket-file" 67 #define ZERO_COPY_KEYWORD "zero-copy" 68 #define POLLING_KEYWORD "guest-polling" 69 70 #define NB_SOCKET_FIELDS (2) 71 72 static uint32_t 73 find_lo(uint32_t lcore_id) 74 { 75 uint32_t i; 76 77 for (i = 0; i < options.nb_los; i++) 78 if (options.los[i].lcore_id == lcore_id) 79 return i; 80 81 return UINT32_MAX; 82 } 83 84 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */ 85 static int 86 parse_socket_arg(char *arg) 87 { 88 uint32_t nb_sockets; 89 uint32_t lcore_id; 90 char *str_fld[NB_SOCKET_FIELDS]; 91 struct lcore_option *lo; 92 uint32_t idx; 93 char *end; 94 95 if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') != 96 NB_SOCKET_FIELDS) { 97 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg); 98 return -EINVAL; 99 } 100 101 errno = 0; 102 lcore_id = strtoul(str_fld[0], &end, 0); 103 if (errno != 0 || end == str_fld[0] || lcore_id > 255) 104 return -EINVAL; 105 106 idx = find_lo(lcore_id); 107 if (idx == UINT32_MAX) { 108 if (options.nb_los == MAX_NB_WORKER_CORES) 109 return -ENOMEM; 110 lo = &options.los[options.nb_los]; 111 lo->lcore_id = lcore_id; 112 options.nb_los++; 113 } else 114 lo = &options.los[idx]; 115 116 nb_sockets = lo->nb_sockets; 117 118 if (nb_sockets >= MAX_NB_SOCKETS) { 119 RTE_LOG(ERR, USER1, "Too many socket files!\n"); 120 return -ENOMEM; 121 } 122 123 lo->socket_files[nb_sockets] = strdup(str_fld[1]); 124 if (!lo->socket_files[nb_sockets]) { 125 RTE_LOG(ERR, USER1, "Insufficient memory\n"); 126 return -ENOMEM; 127 } 128 129 lo->nb_sockets++; 130 131 return 0; 132 } 133 134 static int 135 parse_config(char *q_arg) 136 { 137 struct lcore_option *lo; 138 char s[256]; 139 const char *p, *p0 = q_arg; 140 char *end; 141 enum fieldnames { 142 FLD_LCORE = 0, 143 FLD_CID, 144 FLD_QID, 145 _NUM_FLD 146 }; 147 uint32_t flds[_NUM_FLD]; 148 char *str_fld[_NUM_FLD]; 149 uint32_t i; 150 uint32_t size; 151 152 while ((p = strchr(p0, '(')) != NULL) { 153 ++p; 154 p0 = strchr(p, ')'); 155 if (p0 == NULL) 156 return -1; 157 158 size = p0 - p; 159 if (size >= sizeof(s)) 160 return -1; 161 162 snprintf(s, sizeof(s), "%.*s", size, p); 163 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 164 _NUM_FLD) 165 return -1; 166 for (i = 0; i < _NUM_FLD; i++) { 167 errno = 0; 168 flds[i] = strtoul(str_fld[i], &end, 0); 169 if (errno != 0 || end == str_fld[i] || flds[i] > 255) 170 return -EINVAL; 171 } 172 173 if (flds[FLD_LCORE] > RTE_MAX_LCORE) 174 return -EINVAL; 175 176 i = find_lo(flds[FLD_LCORE]); 177 if (i == UINT32_MAX) { 178 if (options.nb_los == MAX_NB_WORKER_CORES) 179 return -ENOMEM; 180 lo = &options.los[options.nb_los]; 181 options.nb_los++; 182 } else 183 lo = &options.los[i]; 184 185 lo->lcore_id = flds[FLD_LCORE]; 186 lo->cid = flds[FLD_CID]; 187 lo->qid = flds[FLD_QID]; 188 } 189 190 return 0; 191 } 192 193 static void 194 vhost_crypto_usage(const char *prgname) 195 { 196 printf("%s [EAL options] --\n" 197 " --%s <lcore>,SOCKET-FILE-PATH\n" 198 " --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]" 199 " --%s: zero copy\n" 200 " --%s: guest polling\n", 201 prgname, SOCKET_FILE_KEYWORD, CONFIG_KEYWORD, 202 ZERO_COPY_KEYWORD, POLLING_KEYWORD); 203 } 204 205 static int 206 vhost_crypto_parse_args(int argc, char **argv) 207 { 208 int opt, ret; 209 char *prgname = argv[0]; 210 char **argvopt; 211 int option_index; 212 struct option lgopts[] = { 213 {SOCKET_FILE_KEYWORD, required_argument, 0, 0}, 214 {CONFIG_KEYWORD, required_argument, 0, 0}, 215 {ZERO_COPY_KEYWORD, no_argument, 0, 0}, 216 {POLLING_KEYWORD, no_argument, 0, 0}, 217 {NULL, 0, 0, 0} 218 }; 219 220 argvopt = argv; 221 222 while ((opt = getopt_long(argc, argvopt, "s:", 223 lgopts, &option_index)) != EOF) { 224 225 switch (opt) { 226 case 0: 227 if (strcmp(lgopts[option_index].name, 228 SOCKET_FILE_KEYWORD) == 0) { 229 ret = parse_socket_arg(optarg); 230 if (ret < 0) { 231 vhost_crypto_usage(prgname); 232 return ret; 233 } 234 } else if (strcmp(lgopts[option_index].name, 235 CONFIG_KEYWORD) == 0) { 236 ret = parse_config(optarg); 237 if (ret < 0) { 238 vhost_crypto_usage(prgname); 239 return ret; 240 } 241 } else if (strcmp(lgopts[option_index].name, 242 ZERO_COPY_KEYWORD) == 0) { 243 options.zero_copy = 244 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE; 245 } else if (strcmp(lgopts[option_index].name, 246 POLLING_KEYWORD) == 0) { 247 options.guest_polling = 1; 248 } else { 249 vhost_crypto_usage(prgname); 250 return -EINVAL; 251 } 252 break; 253 default: 254 return -1; 255 } 256 } 257 258 return 0; 259 } 260 261 static int 262 new_device(int vid) 263 { 264 struct vhost_crypto_info *info = NULL; 265 char path[PATH_MAX]; 266 uint32_t i, j; 267 int ret; 268 269 ret = rte_vhost_get_ifname(vid, path, PATH_MAX); 270 if (ret) { 271 RTE_LOG(ERR, USER1, "Cannot find matched socket\n"); 272 return ret; 273 } 274 275 for (i = 0; i < options.nb_los; i++) { 276 for (j = 0; j < options.los[i].nb_sockets; j++) { 277 if (strcmp(path, options.los[i].socket_files[j]) == 0) { 278 info = options.infos[i]; 279 break; 280 } 281 } 282 283 if (info) 284 break; 285 } 286 287 if (!info) { 288 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n"); 289 return -ENOENT; 290 } 291 292 ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool, 293 info->sess_priv_pool, 294 rte_lcore_to_socket_id(options.los[i].lcore_id)); 295 if (ret) { 296 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n"); 297 return ret; 298 } 299 300 ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy); 301 if (ret) { 302 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n", 303 options.zero_copy == 1 ? "enable" : "disable"); 304 return ret; 305 } 306 307 info->vids[j] = vid; 308 info->initialized[j] = 1; 309 310 rte_wmb(); 311 312 RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path, 313 vid); 314 return 0; 315 } 316 317 static void 318 destroy_device(int vid) 319 { 320 struct vhost_crypto_info *info = NULL; 321 uint32_t i, j; 322 323 for (i = 0; i < options.nb_los; i++) { 324 for (j = 0; j < options.los[i].nb_sockets; j++) { 325 if (options.infos[i]->vids[j] == vid) { 326 info = options.infos[i]; 327 break; 328 } 329 } 330 if (info) 331 break; 332 } 333 334 if (!info) { 335 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n"); 336 return; 337 } 338 339 do { 340 341 } while (info->nb_inflight_ops); 342 343 info->initialized[j] = 0; 344 345 rte_wmb(); 346 347 rte_vhost_crypto_free(vid); 348 349 RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid); 350 } 351 352 static const struct vhost_device_ops virtio_crypto_device_ops = { 353 .new_device = new_device, 354 .destroy_device = destroy_device, 355 }; 356 357 __attribute__((unused)) 358 static void clrscr(void) 359 { 360 system("@cls||clear"); 361 } 362 363 static int 364 vhost_crypto_worker(void *arg) 365 { 366 struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 367 struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 368 struct vhost_crypto_info *info = arg; 369 uint16_t nb_callfds; 370 int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS]; 371 uint32_t lcore_id = rte_lcore_id(); 372 uint32_t burst_size = MAX_PKT_BURST; 373 uint32_t i, j, k; 374 uint32_t to_fetch, fetched; 375 376 int ret = 0; 377 378 RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id); 379 380 for (i = 0; i < NB_VIRTIO_QUEUES; i++) { 381 if (rte_crypto_op_bulk_alloc(info->cop_pool, 382 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i], 383 burst_size) < burst_size) { 384 RTE_LOG(ERR, USER1, "Failed to alloc cops\n"); 385 ret = -1; 386 goto exit; 387 } 388 } 389 390 while (1) { 391 for (i = 0; i < info->nb_vids; i++) { 392 if (unlikely(info->initialized[i] == 0)) 393 continue; 394 395 for (j = 0; j < NB_VIRTIO_QUEUES; j++) { 396 to_fetch = RTE_MIN(burst_size, 397 (NB_CRYPTO_DESCRIPTORS - 398 info->nb_inflight_ops)); 399 fetched = rte_vhost_crypto_fetch_requests( 400 info->vids[i], j, ops[j], 401 to_fetch); 402 info->nb_inflight_ops += 403 rte_cryptodev_enqueue_burst( 404 info->cid, info->qid, ops[j], 405 fetched); 406 if (unlikely(rte_crypto_op_bulk_alloc( 407 info->cop_pool, 408 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 409 ops[j], fetched) < fetched)) { 410 RTE_LOG(ERR, USER1, "Failed realloc\n"); 411 return -1; 412 } 413 414 fetched = rte_cryptodev_dequeue_burst( 415 info->cid, info->qid, 416 ops_deq[j], RTE_MIN(burst_size, 417 info->nb_inflight_ops)); 418 fetched = rte_vhost_crypto_finalize_requests( 419 ops_deq[j], fetched, callfds, 420 &nb_callfds); 421 422 info->nb_inflight_ops -= fetched; 423 424 if (!options.guest_polling) { 425 for (k = 0; k < nb_callfds; k++) 426 eventfd_write(callfds[k], 427 (eventfd_t)1); 428 } 429 430 rte_mempool_put_bulk(info->cop_pool, 431 (void **)ops_deq[j], fetched); 432 } 433 } 434 } 435 exit: 436 return ret; 437 } 438 439 static void 440 free_resource(void) 441 { 442 uint32_t i, j; 443 444 for (i = 0; i < options.nb_los; i++) { 445 struct lcore_option *lo = &options.los[i]; 446 struct vhost_crypto_info *info = options.infos[i]; 447 448 if (!info) 449 continue; 450 451 rte_mempool_free(info->cop_pool); 452 rte_mempool_free(info->sess_pool); 453 rte_mempool_free(info->sess_priv_pool); 454 455 for (j = 0; j < lo->nb_sockets; j++) { 456 rte_vhost_driver_unregister(lo->socket_files[i]); 457 free(lo->socket_files[i]); 458 } 459 460 rte_free(info); 461 } 462 463 memset(&options, 0, sizeof(options)); 464 } 465 466 int 467 main(int argc, char *argv[]) 468 { 469 struct rte_cryptodev_qp_conf qp_conf; 470 struct rte_cryptodev_config config; 471 struct rte_cryptodev_info dev_info; 472 char name[128]; 473 uint32_t i, j, lcore; 474 int ret; 475 476 ret = rte_eal_init(argc, argv); 477 if (ret < 0) 478 return -1; 479 argc -= ret; 480 argv += ret; 481 482 ret = vhost_crypto_parse_args(argc, argv); 483 if (ret < 0) 484 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n"); 485 486 for (i = 0; i < options.nb_los; i++) { 487 struct lcore_option *lo = &options.los[i]; 488 struct vhost_crypto_info *info; 489 490 info = rte_zmalloc_socket(NULL, sizeof(*info), 491 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id( 492 lo->lcore_id)); 493 if (!info) { 494 ret = -ENOMEM; 495 goto error_exit; 496 } 497 498 info->cid = lo->cid; 499 info->qid = lo->qid; 500 info->nb_vids = lo->nb_sockets; 501 502 rte_cryptodev_info_get(info->cid, &dev_info); 503 if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) { 504 #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD crypto_aesni_mb 505 #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm 506 if (strstr(dev_info.driver_name, 507 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) || 508 strstr(dev_info.driver_name, 509 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) { 510 RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n", 511 dev_info.driver_name); 512 ret = -EPERM; 513 goto error_exit; 514 } 515 } 516 517 if (dev_info.max_nb_queue_pairs < info->qid + 1) { 518 RTE_LOG(ERR, USER1, "Number of queues cannot over %u", 519 dev_info.max_nb_queue_pairs); 520 goto error_exit; 521 } 522 523 config.nb_queue_pairs = dev_info.max_nb_queue_pairs; 524 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id); 525 526 ret = rte_cryptodev_configure(info->cid, &config); 527 if (ret < 0) { 528 RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u", 529 info->cid); 530 goto error_exit; 531 } 532 533 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id); 534 info->sess_pool = rte_cryptodev_sym_session_pool_create(name, 535 SESSION_MAP_ENTRIES, 0, 0, 0, 536 rte_lcore_to_socket_id(lo->lcore_id)); 537 538 snprintf(name, 127, "SESS_POOL_PRIV_%u", lo->lcore_id); 539 info->sess_priv_pool = rte_mempool_create(name, 540 SESSION_MAP_ENTRIES, 541 rte_cryptodev_sym_get_private_session_size( 542 info->cid), 64, 0, NULL, NULL, NULL, NULL, 543 rte_lcore_to_socket_id(lo->lcore_id), 0); 544 if (!info->sess_priv_pool || !info->sess_pool) { 545 RTE_LOG(ERR, USER1, "Failed to create mempool"); 546 goto error_exit; 547 } 548 549 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id); 550 info->cop_pool = rte_crypto_op_pool_create(name, 551 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS, 552 NB_CACHE_OBJS, 0, 553 rte_lcore_to_socket_id(lo->lcore_id)); 554 555 if (!info->cop_pool) { 556 RTE_LOG(ERR, USER1, "Failed to create crypto pool"); 557 ret = -ENOMEM; 558 goto error_exit; 559 } 560 561 options.infos[i] = info; 562 563 qp_conf.nb_descriptors = NB_CRYPTO_DESCRIPTORS; 564 qp_conf.mp_session = info->sess_pool; 565 qp_conf.mp_session_private = info->sess_priv_pool; 566 567 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) { 568 ret = rte_cryptodev_queue_pair_setup(info->cid, j, 569 &qp_conf, rte_lcore_to_socket_id( 570 lo->lcore_id)); 571 if (ret < 0) { 572 RTE_LOG(ERR, USER1, "Failed to configure qp\n"); 573 goto error_exit; 574 } 575 } 576 } 577 578 for (i = 0; i < options.nb_los; i++) { 579 struct lcore_option *lo = &options.los[i]; 580 struct vhost_crypto_info *info = options.infos[i]; 581 582 ret = rte_cryptodev_start(lo->cid); 583 if (ret < 0) { 584 RTE_LOG(ERR, USER1, "Failed to start cryptodev\n"); 585 goto error_exit; 586 } 587 588 if (rte_eal_remote_launch(vhost_crypto_worker, info, 589 lo->lcore_id) < 0) { 590 RTE_LOG(ERR, USER1, "Failed to start worker lcore"); 591 goto error_exit; 592 } 593 594 for (j = 0; j < lo->nb_sockets; j++) { 595 ret = rte_vhost_driver_register(lo->socket_files[j], 596 RTE_VHOST_USER_DEQUEUE_ZERO_COPY); 597 if (ret < 0) { 598 RTE_LOG(ERR, USER1, "socket %s already exists\n", 599 lo->socket_files[j]); 600 goto error_exit; 601 } 602 603 rte_vhost_driver_callback_register(lo->socket_files[j], 604 &virtio_crypto_device_ops); 605 606 ret = rte_vhost_driver_start(lo->socket_files[j]); 607 if (ret < 0) { 608 RTE_LOG(ERR, USER1, "failed to start vhost.\n"); 609 goto error_exit; 610 } 611 } 612 } 613 614 RTE_LCORE_FOREACH(lcore) 615 rte_eal_wait_lcore(lcore); 616 617 free_resource(); 618 619 return 0; 620 621 error_exit: 622 623 free_resource(); 624 625 return -1; 626 } 627