1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017-2018 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <unistd.h> 9 #include <stdbool.h> 10 #include <assert.h> 11 #include <getopt.h> 12 13 #include <rte_malloc.h> 14 #include <rte_cycles.h> 15 #include <rte_vhost.h> 16 #include <rte_cryptodev.h> 17 #include <rte_vhost_crypto.h> 18 #include <rte_string_fns.h> 19 20 #include <cmdline_rdline.h> 21 #include <cmdline_parse.h> 22 #include <cmdline_parse_string.h> 23 #include <cmdline.h> 24 25 #define NB_VIRTIO_QUEUES (1) 26 #define MAX_PKT_BURST (64) 27 #define MAX_IV_LEN (32) 28 #define NB_MEMPOOL_OBJS (8192) 29 #define NB_CRYPTO_DESCRIPTORS (4096) 30 #define NB_CACHE_OBJS (128) 31 #define SESSION_MAP_ENTRIES (1024) 32 #define REFRESH_TIME_SEC (3) 33 34 #define MAX_NB_SOCKETS (4) 35 #define MAX_NB_WORKER_CORES (16) 36 37 struct lcore_option { 38 uint32_t lcore_id; 39 char *socket_files[MAX_NB_SOCKETS]; 40 uint32_t nb_sockets; 41 uint8_t cid; 42 uint16_t qid; 43 }; 44 45 struct vhost_crypto_info { 46 int vids[MAX_NB_SOCKETS]; 47 uint32_t nb_vids; 48 struct rte_mempool *sess_pool; 49 struct rte_mempool *sess_priv_pool; 50 struct rte_mempool *cop_pool; 51 uint8_t cid; 52 uint32_t qid; 53 uint32_t nb_inflight_ops; 54 volatile uint32_t initialized[MAX_NB_SOCKETS]; 55 } __rte_cache_aligned; 56 57 struct vhost_crypto_options { 58 struct lcore_option los[MAX_NB_WORKER_CORES]; 59 struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES]; 60 uint32_t nb_los; 61 uint32_t zero_copy; 62 uint32_t guest_polling; 63 } options; 64 65 #define CONFIG_KEYWORD "config" 66 #define SOCKET_FILE_KEYWORD "socket-file" 67 #define ZERO_COPY_KEYWORD "zero-copy" 68 #define POLLING_KEYWORD "guest-polling" 69 70 #define NB_SOCKET_FIELDS (2) 71 72 static uint32_t 73 find_lo(uint32_t lcore_id) 74 { 75 uint32_t i; 76 77 for (i = 0; i < options.nb_los; i++) 78 if (options.los[i].lcore_id == lcore_id) 79 return i; 80 81 return UINT32_MAX; 82 } 83 84 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */ 85 static int 86 parse_socket_arg(char *arg) 87 { 88 uint32_t nb_sockets; 89 uint32_t lcore_id; 90 char *str_fld[NB_SOCKET_FIELDS]; 91 struct lcore_option *lo; 92 uint32_t idx; 93 char *end; 94 95 if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') != 96 NB_SOCKET_FIELDS) { 97 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg); 98 return -EINVAL; 99 } 100 101 errno = 0; 102 lcore_id = strtoul(str_fld[0], &end, 0); 103 if (errno != 0 || end == str_fld[0] || lcore_id > 255) 104 return -EINVAL; 105 106 idx = find_lo(lcore_id); 107 if (idx == UINT32_MAX) { 108 if (options.nb_los == MAX_NB_WORKER_CORES) 109 return -ENOMEM; 110 lo = &options.los[options.nb_los]; 111 lo->lcore_id = lcore_id; 112 options.nb_los++; 113 } else 114 lo = &options.los[idx]; 115 116 nb_sockets = lo->nb_sockets; 117 118 if (nb_sockets >= MAX_NB_SOCKETS) { 119 RTE_LOG(ERR, USER1, "Too many socket files!\n"); 120 return -ENOMEM; 121 } 122 123 lo->socket_files[nb_sockets] = strdup(str_fld[1]); 124 if (!lo->socket_files[nb_sockets]) { 125 RTE_LOG(ERR, USER1, "Insufficient memory\n"); 126 return -ENOMEM; 127 } 128 129 lo->nb_sockets++; 130 131 return 0; 132 } 133 134 static int 135 parse_config(char *q_arg) 136 { 137 struct lcore_option *lo; 138 char s[256]; 139 const char *p, *p0 = q_arg; 140 char *end; 141 enum fieldnames { 142 FLD_LCORE = 0, 143 FLD_CID, 144 FLD_QID, 145 _NUM_FLD 146 }; 147 uint32_t flds[_NUM_FLD]; 148 char *str_fld[_NUM_FLD]; 149 uint32_t i; 150 uint32_t size; 151 152 while ((p = strchr(p0, '(')) != NULL) { 153 ++p; 154 p0 = strchr(p, ')'); 155 if (p0 == NULL) 156 return -1; 157 158 size = p0 - p; 159 if (size >= sizeof(s)) 160 return -1; 161 162 snprintf(s, sizeof(s), "%.*s", size, p); 163 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 164 _NUM_FLD) 165 return -1; 166 for (i = 0; i < _NUM_FLD; i++) { 167 errno = 0; 168 flds[i] = strtoul(str_fld[i], &end, 0); 169 if (errno != 0 || end == str_fld[i] || flds[i] > 255) 170 return -EINVAL; 171 } 172 173 if (flds[FLD_LCORE] > RTE_MAX_LCORE) 174 return -EINVAL; 175 176 i = find_lo(flds[FLD_LCORE]); 177 if (i == UINT32_MAX) { 178 if (options.nb_los == MAX_NB_WORKER_CORES) 179 return -ENOMEM; 180 lo = &options.los[options.nb_los]; 181 options.nb_los++; 182 } else 183 lo = &options.los[i]; 184 185 lo->lcore_id = flds[FLD_LCORE]; 186 lo->cid = flds[FLD_CID]; 187 lo->qid = flds[FLD_QID]; 188 } 189 190 return 0; 191 } 192 193 static void 194 vhost_crypto_usage(const char *prgname) 195 { 196 printf("%s [EAL options] --\n" 197 " --%s <lcore>,SOCKET-FILE-PATH\n" 198 " --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]\n" 199 " --%s: zero copy\n" 200 " --%s: guest polling\n", 201 prgname, SOCKET_FILE_KEYWORD, CONFIG_KEYWORD, 202 ZERO_COPY_KEYWORD, POLLING_KEYWORD); 203 } 204 205 static int 206 vhost_crypto_parse_args(int argc, char **argv) 207 { 208 int opt, ret; 209 char *prgname = argv[0]; 210 char **argvopt; 211 int option_index; 212 struct option lgopts[] = { 213 {SOCKET_FILE_KEYWORD, required_argument, 0, 0}, 214 {CONFIG_KEYWORD, required_argument, 0, 0}, 215 {ZERO_COPY_KEYWORD, no_argument, 0, 0}, 216 {POLLING_KEYWORD, no_argument, 0, 0}, 217 {NULL, 0, 0, 0} 218 }; 219 220 argvopt = argv; 221 222 while ((opt = getopt_long(argc, argvopt, "s:", 223 lgopts, &option_index)) != EOF) { 224 225 switch (opt) { 226 case 0: 227 if (strcmp(lgopts[option_index].name, 228 SOCKET_FILE_KEYWORD) == 0) { 229 ret = parse_socket_arg(optarg); 230 if (ret < 0) { 231 vhost_crypto_usage(prgname); 232 return ret; 233 } 234 } else if (strcmp(lgopts[option_index].name, 235 CONFIG_KEYWORD) == 0) { 236 ret = parse_config(optarg); 237 if (ret < 0) { 238 vhost_crypto_usage(prgname); 239 return ret; 240 } 241 } else if (strcmp(lgopts[option_index].name, 242 ZERO_COPY_KEYWORD) == 0) { 243 options.zero_copy = 244 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE; 245 } else if (strcmp(lgopts[option_index].name, 246 POLLING_KEYWORD) == 0) { 247 options.guest_polling = 1; 248 } else { 249 vhost_crypto_usage(prgname); 250 return -EINVAL; 251 } 252 break; 253 default: 254 return -1; 255 } 256 } 257 258 return 0; 259 } 260 261 static int 262 new_device(int vid) 263 { 264 struct vhost_crypto_info *info = NULL; 265 char path[PATH_MAX]; 266 uint32_t i, j; 267 int ret; 268 269 ret = rte_vhost_get_ifname(vid, path, PATH_MAX); 270 if (ret) { 271 RTE_LOG(ERR, USER1, "Cannot find matched socket\n"); 272 return ret; 273 } 274 275 for (i = 0; i < options.nb_los; i++) { 276 for (j = 0; j < options.los[i].nb_sockets; j++) { 277 if (strcmp(path, options.los[i].socket_files[j]) == 0) { 278 info = options.infos[i]; 279 break; 280 } 281 } 282 283 if (info) 284 break; 285 } 286 287 if (!info) { 288 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n"); 289 return -ENOENT; 290 } 291 292 ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool, 293 info->sess_priv_pool, 294 rte_lcore_to_socket_id(options.los[i].lcore_id)); 295 if (ret) { 296 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n"); 297 return ret; 298 } 299 300 ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy); 301 if (ret) { 302 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n", 303 options.zero_copy == 1 ? "enable" : "disable"); 304 return ret; 305 } 306 307 info->vids[j] = vid; 308 info->initialized[j] = 1; 309 310 rte_wmb(); 311 312 RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path, 313 vid); 314 return 0; 315 } 316 317 static void 318 destroy_device(int vid) 319 { 320 struct vhost_crypto_info *info = NULL; 321 uint32_t i, j; 322 323 for (i = 0; i < options.nb_los; i++) { 324 for (j = 0; j < options.los[i].nb_sockets; j++) { 325 if (options.infos[i]->vids[j] == vid) { 326 info = options.infos[i]; 327 break; 328 } 329 } 330 if (info) 331 break; 332 } 333 334 if (!info) { 335 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n"); 336 return; 337 } 338 339 do { 340 341 } while (info->nb_inflight_ops); 342 343 info->initialized[j] = 0; 344 345 rte_wmb(); 346 347 rte_vhost_crypto_free(vid); 348 349 RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid); 350 } 351 352 static const struct vhost_device_ops virtio_crypto_device_ops = { 353 .new_device = new_device, 354 .destroy_device = destroy_device, 355 }; 356 357 static int 358 vhost_crypto_worker(void *arg) 359 { 360 struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 361 struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 362 struct vhost_crypto_info *info = arg; 363 uint16_t nb_callfds; 364 int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS]; 365 uint32_t lcore_id = rte_lcore_id(); 366 uint32_t burst_size = MAX_PKT_BURST; 367 uint32_t i, j, k; 368 uint32_t to_fetch, fetched; 369 370 int ret = 0; 371 372 RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id); 373 374 for (i = 0; i < NB_VIRTIO_QUEUES; i++) { 375 if (rte_crypto_op_bulk_alloc(info->cop_pool, 376 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i], 377 burst_size) < burst_size) { 378 RTE_LOG(ERR, USER1, "Failed to alloc cops\n"); 379 ret = -1; 380 goto exit; 381 } 382 } 383 384 while (1) { 385 for (i = 0; i < info->nb_vids; i++) { 386 if (unlikely(info->initialized[i] == 0)) 387 continue; 388 389 for (j = 0; j < NB_VIRTIO_QUEUES; j++) { 390 to_fetch = RTE_MIN(burst_size, 391 (NB_CRYPTO_DESCRIPTORS - 392 info->nb_inflight_ops)); 393 fetched = rte_vhost_crypto_fetch_requests( 394 info->vids[i], j, ops[j], 395 to_fetch); 396 info->nb_inflight_ops += 397 rte_cryptodev_enqueue_burst( 398 info->cid, info->qid, ops[j], 399 fetched); 400 if (unlikely(rte_crypto_op_bulk_alloc( 401 info->cop_pool, 402 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 403 ops[j], fetched) < fetched)) { 404 RTE_LOG(ERR, USER1, "Failed realloc\n"); 405 return -1; 406 } 407 408 fetched = rte_cryptodev_dequeue_burst( 409 info->cid, info->qid, 410 ops_deq[j], RTE_MIN(burst_size, 411 info->nb_inflight_ops)); 412 fetched = rte_vhost_crypto_finalize_requests( 413 ops_deq[j], fetched, callfds, 414 &nb_callfds); 415 416 info->nb_inflight_ops -= fetched; 417 418 if (!options.guest_polling) { 419 for (k = 0; k < nb_callfds; k++) 420 eventfd_write(callfds[k], 421 (eventfd_t)1); 422 } 423 424 rte_mempool_put_bulk(info->cop_pool, 425 (void **)ops_deq[j], fetched); 426 } 427 } 428 } 429 exit: 430 return ret; 431 } 432 433 static void 434 free_resource(void) 435 { 436 uint32_t i, j; 437 438 for (i = 0; i < options.nb_los; i++) { 439 struct lcore_option *lo = &options.los[i]; 440 struct vhost_crypto_info *info = options.infos[i]; 441 442 if (!info) 443 continue; 444 445 rte_mempool_free(info->cop_pool); 446 rte_mempool_free(info->sess_pool); 447 rte_mempool_free(info->sess_priv_pool); 448 449 for (j = 0; j < lo->nb_sockets; j++) { 450 rte_vhost_driver_unregister(lo->socket_files[i]); 451 free(lo->socket_files[i]); 452 } 453 454 rte_free(info); 455 } 456 457 memset(&options, 0, sizeof(options)); 458 } 459 460 int 461 main(int argc, char *argv[]) 462 { 463 struct rte_cryptodev_qp_conf qp_conf; 464 struct rte_cryptodev_config config; 465 struct rte_cryptodev_info dev_info; 466 char name[128]; 467 uint32_t i, j, lcore; 468 int ret; 469 470 ret = rte_eal_init(argc, argv); 471 if (ret < 0) 472 return -1; 473 argc -= ret; 474 argv += ret; 475 476 ret = vhost_crypto_parse_args(argc, argv); 477 if (ret < 0) 478 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n"); 479 480 for (i = 0; i < options.nb_los; i++) { 481 struct lcore_option *lo = &options.los[i]; 482 struct vhost_crypto_info *info; 483 484 info = rte_zmalloc_socket(NULL, sizeof(*info), 485 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id( 486 lo->lcore_id)); 487 if (!info) { 488 ret = -ENOMEM; 489 goto error_exit; 490 } 491 492 info->cid = lo->cid; 493 info->qid = lo->qid; 494 info->nb_vids = lo->nb_sockets; 495 496 rte_cryptodev_info_get(info->cid, &dev_info); 497 if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) { 498 #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD crypto_aesni_mb 499 #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm 500 if (strstr(dev_info.driver_name, 501 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) || 502 strstr(dev_info.driver_name, 503 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) { 504 RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n", 505 dev_info.driver_name); 506 ret = -EPERM; 507 goto error_exit; 508 } 509 } 510 511 if (dev_info.max_nb_queue_pairs < info->qid + 1) { 512 RTE_LOG(ERR, USER1, "Number of queues cannot over %u", 513 dev_info.max_nb_queue_pairs); 514 goto error_exit; 515 } 516 517 config.nb_queue_pairs = dev_info.max_nb_queue_pairs; 518 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id); 519 config.ff_disable = RTE_CRYPTODEV_FF_SECURITY; 520 521 ret = rte_cryptodev_configure(info->cid, &config); 522 if (ret < 0) { 523 RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u", 524 info->cid); 525 goto error_exit; 526 } 527 528 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id); 529 info->sess_pool = rte_cryptodev_sym_session_pool_create(name, 530 SESSION_MAP_ENTRIES, 0, 0, 0, 531 rte_lcore_to_socket_id(lo->lcore_id)); 532 533 snprintf(name, 127, "SESS_POOL_PRIV_%u", lo->lcore_id); 534 info->sess_priv_pool = rte_mempool_create(name, 535 SESSION_MAP_ENTRIES, 536 rte_cryptodev_sym_get_private_session_size( 537 info->cid), 64, 0, NULL, NULL, NULL, NULL, 538 rte_lcore_to_socket_id(lo->lcore_id), 0); 539 if (!info->sess_priv_pool || !info->sess_pool) { 540 RTE_LOG(ERR, USER1, "Failed to create mempool"); 541 goto error_exit; 542 } 543 544 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id); 545 info->cop_pool = rte_crypto_op_pool_create(name, 546 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS, 547 NB_CACHE_OBJS, VHOST_CRYPTO_MAX_IV_LEN, 548 rte_lcore_to_socket_id(lo->lcore_id)); 549 550 if (!info->cop_pool) { 551 RTE_LOG(ERR, USER1, "Failed to create crypto pool"); 552 ret = -ENOMEM; 553 goto error_exit; 554 } 555 556 options.infos[i] = info; 557 558 qp_conf.nb_descriptors = NB_CRYPTO_DESCRIPTORS; 559 qp_conf.mp_session = info->sess_pool; 560 qp_conf.mp_session_private = info->sess_priv_pool; 561 562 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) { 563 ret = rte_cryptodev_queue_pair_setup(info->cid, j, 564 &qp_conf, rte_lcore_to_socket_id( 565 lo->lcore_id)); 566 if (ret < 0) { 567 RTE_LOG(ERR, USER1, "Failed to configure qp\n"); 568 goto error_exit; 569 } 570 } 571 } 572 573 for (i = 0; i < options.nb_los; i++) { 574 struct lcore_option *lo = &options.los[i]; 575 struct vhost_crypto_info *info = options.infos[i]; 576 577 ret = rte_cryptodev_start(lo->cid); 578 if (ret < 0) { 579 RTE_LOG(ERR, USER1, "Failed to start cryptodev\n"); 580 goto error_exit; 581 } 582 583 if (rte_eal_remote_launch(vhost_crypto_worker, info, 584 lo->lcore_id) < 0) { 585 RTE_LOG(ERR, USER1, "Failed to start worker lcore"); 586 goto error_exit; 587 } 588 589 for (j = 0; j < lo->nb_sockets; j++) { 590 ret = rte_vhost_driver_register(lo->socket_files[j], 591 RTE_VHOST_USER_ASYNC_COPY); 592 if (ret < 0) { 593 RTE_LOG(ERR, USER1, "socket %s already exists\n", 594 lo->socket_files[j]); 595 goto error_exit; 596 } 597 598 rte_vhost_driver_callback_register(lo->socket_files[j], 599 &virtio_crypto_device_ops); 600 601 ret = rte_vhost_crypto_driver_start( 602 lo->socket_files[j]); 603 if (ret < 0) { 604 RTE_LOG(ERR, USER1, "failed to start vhost.\n"); 605 goto error_exit; 606 } 607 } 608 } 609 610 RTE_LCORE_FOREACH(lcore) 611 rte_eal_wait_lcore(lcore); 612 613 free_resource(); 614 615 return 0; 616 617 error_exit: 618 619 free_resource(); 620 621 return -1; 622 } 623