1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017-2018 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <unistd.h> 8 #include <stdbool.h> 9 #include <assert.h> 10 #include <getopt.h> 11 12 #include <rte_malloc.h> 13 #include <rte_cycles.h> 14 #include <rte_vhost.h> 15 #include <rte_cryptodev.h> 16 #include <rte_vhost_crypto.h> 17 #include <rte_string_fns.h> 18 19 #include <cmdline_rdline.h> 20 #include <cmdline_parse.h> 21 #include <cmdline_parse_string.h> 22 #include <cmdline.h> 23 24 #define NB_VIRTIO_QUEUES (1) 25 #define MAX_PKT_BURST (64) 26 #define MAX_IV_LEN (32) 27 #define NB_MEMPOOL_OBJS (8192) 28 #define NB_CRYPTO_DESCRIPTORS (4096) 29 #define NB_CACHE_OBJS (128) 30 #define SESSION_MAP_ENTRIES (1024) 31 #define REFRESH_TIME_SEC (3) 32 33 #define MAX_NB_SOCKETS (4) 34 #define MAX_NB_WORKER_CORES (16) 35 36 struct lcore_option { 37 uint32_t lcore_id; 38 char *socket_files[MAX_NB_SOCKETS]; 39 uint32_t nb_sockets; 40 uint8_t cid; 41 uint16_t qid; 42 }; 43 44 struct vhost_crypto_info { 45 int vids[MAX_NB_SOCKETS]; 46 uint32_t nb_vids; 47 struct rte_mempool *sess_pool; 48 struct rte_mempool *cop_pool; 49 uint8_t cid; 50 uint32_t qid; 51 uint32_t nb_inflight_ops; 52 volatile uint32_t initialized[MAX_NB_SOCKETS]; 53 } __rte_cache_aligned; 54 55 struct vhost_crypto_options { 56 struct lcore_option los[MAX_NB_WORKER_CORES]; 57 struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES]; 58 uint32_t nb_los; 59 uint32_t zero_copy; 60 uint32_t guest_polling; 61 } options; 62 63 #define CONFIG_KEYWORD "config" 64 #define SOCKET_FILE_KEYWORD "socket-file" 65 #define ZERO_COPY_KEYWORD "zero-copy" 66 #define POLLING_KEYWORD "guest-polling" 67 68 #define NB_SOCKET_FIELDS (2) 69 70 static uint32_t 71 find_lo(uint32_t lcore_id) 72 { 73 uint32_t i; 74 75 for (i = 0; i < options.nb_los; i++) 76 if (options.los[i].lcore_id == lcore_id) 77 return i; 78 79 return UINT32_MAX; 80 } 81 82 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */ 83 static int 84 parse_socket_arg(char *arg) 85 { 86 uint32_t nb_sockets; 87 uint32_t lcore_id; 88 char *str_fld[NB_SOCKET_FIELDS]; 89 struct lcore_option *lo; 90 uint32_t idx; 91 char *end; 92 93 if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') != 94 NB_SOCKET_FIELDS) { 95 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg); 96 return -EINVAL; 97 } 98 99 errno = 0; 100 lcore_id = strtoul(str_fld[0], &end, 0); 101 if (errno != 0 || end == str_fld[0] || lcore_id > 255) 102 return -EINVAL; 103 104 idx = find_lo(lcore_id); 105 if (idx == UINT32_MAX) { 106 if (options.nb_los == MAX_NB_WORKER_CORES) 107 return -ENOMEM; 108 lo = &options.los[options.nb_los]; 109 lo->lcore_id = lcore_id; 110 options.nb_los++; 111 } else 112 lo = &options.los[idx]; 113 114 nb_sockets = lo->nb_sockets; 115 116 if (nb_sockets >= MAX_NB_SOCKETS) { 117 RTE_LOG(ERR, USER1, "Too many socket files!\n"); 118 return -ENOMEM; 119 } 120 121 lo->socket_files[nb_sockets] = strdup(str_fld[1]); 122 if (!lo->socket_files[nb_sockets]) { 123 RTE_LOG(ERR, USER1, "Insufficient memory\n"); 124 return -ENOMEM; 125 } 126 127 lo->nb_sockets++; 128 129 return 0; 130 } 131 132 static int 133 parse_config(char *q_arg) 134 { 135 struct lcore_option *lo; 136 char s[256]; 137 const char *p, *p0 = q_arg; 138 char *end; 139 enum fieldnames { 140 FLD_LCORE = 0, 141 FLD_CID, 142 FLD_QID, 143 _NUM_FLD 144 }; 145 uint32_t flds[_NUM_FLD]; 146 char *str_fld[_NUM_FLD]; 147 uint32_t i; 148 uint32_t size; 149 150 while ((p = strchr(p0, '(')) != NULL) { 151 ++p; 152 p0 = strchr(p, ')'); 153 if (p0 == NULL) 154 return -1; 155 156 size = p0 - p; 157 if (size >= sizeof(s)) 158 return -1; 159 160 snprintf(s, sizeof(s), "%.*s", size, p); 161 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 162 _NUM_FLD) 163 return -1; 164 for (i = 0; i < _NUM_FLD; i++) { 165 errno = 0; 166 flds[i] = strtoul(str_fld[i], &end, 0); 167 if (errno != 0 || end == str_fld[i] || flds[i] > 255) 168 return -EINVAL; 169 } 170 171 if (flds[FLD_LCORE] > RTE_MAX_LCORE) 172 return -EINVAL; 173 174 i = find_lo(flds[FLD_LCORE]); 175 if (i == UINT32_MAX) { 176 if (options.nb_los == MAX_NB_WORKER_CORES) 177 return -ENOMEM; 178 lo = &options.los[options.nb_los]; 179 options.nb_los++; 180 } else 181 lo = &options.los[i]; 182 183 lo->lcore_id = flds[FLD_LCORE]; 184 lo->cid = flds[FLD_CID]; 185 lo->qid = flds[FLD_QID]; 186 } 187 188 return 0; 189 } 190 191 static void 192 vhost_crypto_usage(const char *prgname) 193 { 194 printf("%s [EAL options] --\n" 195 " --%s <lcore>,SOCKET-FILE-PATH\n" 196 " --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]" 197 " --%s: zero copy\n" 198 " --%s: guest polling\n", 199 prgname, SOCKET_FILE_KEYWORD, CONFIG_KEYWORD, 200 ZERO_COPY_KEYWORD, POLLING_KEYWORD); 201 } 202 203 static int 204 vhost_crypto_parse_args(int argc, char **argv) 205 { 206 int opt, ret; 207 char *prgname = argv[0]; 208 char **argvopt; 209 int option_index; 210 struct option lgopts[] = { 211 {SOCKET_FILE_KEYWORD, required_argument, 0, 0}, 212 {CONFIG_KEYWORD, required_argument, 0, 0}, 213 {ZERO_COPY_KEYWORD, no_argument, 0, 0}, 214 {POLLING_KEYWORD, no_argument, 0, 0}, 215 {NULL, 0, 0, 0} 216 }; 217 218 argvopt = argv; 219 220 while ((opt = getopt_long(argc, argvopt, "s:", 221 lgopts, &option_index)) != EOF) { 222 223 switch (opt) { 224 case 0: 225 if (strcmp(lgopts[option_index].name, 226 SOCKET_FILE_KEYWORD) == 0) { 227 ret = parse_socket_arg(optarg); 228 if (ret < 0) { 229 vhost_crypto_usage(prgname); 230 return ret; 231 } 232 } else if (strcmp(lgopts[option_index].name, 233 CONFIG_KEYWORD) == 0) { 234 ret = parse_config(optarg); 235 if (ret < 0) { 236 vhost_crypto_usage(prgname); 237 return ret; 238 } 239 } else if (strcmp(lgopts[option_index].name, 240 ZERO_COPY_KEYWORD) == 0) { 241 options.zero_copy = 242 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE; 243 } else if (strcmp(lgopts[option_index].name, 244 POLLING_KEYWORD) == 0) { 245 options.guest_polling = 1; 246 } else { 247 vhost_crypto_usage(prgname); 248 return -EINVAL; 249 } 250 break; 251 default: 252 return -1; 253 } 254 } 255 256 return 0; 257 } 258 259 static int 260 new_device(int vid) 261 { 262 struct vhost_crypto_info *info = NULL; 263 char path[PATH_MAX]; 264 uint32_t i, j; 265 int ret; 266 267 ret = rte_vhost_get_ifname(vid, path, PATH_MAX); 268 if (ret) { 269 RTE_LOG(ERR, USER1, "Cannot find matched socket\n"); 270 return ret; 271 } 272 273 for (i = 0; i < options.nb_los; i++) { 274 for (j = 0; j < options.los[i].nb_sockets; j++) { 275 if (strcmp(path, options.los[i].socket_files[j]) == 0) { 276 info = options.infos[i]; 277 break; 278 } 279 } 280 281 if (info) 282 break; 283 } 284 285 if (!info) { 286 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n"); 287 return -ENOENT; 288 } 289 290 ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool, 291 rte_lcore_to_socket_id(options.los[i].lcore_id)); 292 if (ret) { 293 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n"); 294 return ret; 295 } 296 297 ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy); 298 if (ret) { 299 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n", 300 options.zero_copy == 1 ? "enable" : "disable"); 301 return ret; 302 } 303 304 info->vids[j] = vid; 305 info->initialized[j] = 1; 306 307 rte_wmb(); 308 309 RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path, 310 vid); 311 return 0; 312 } 313 314 static void 315 destroy_device(int vid) 316 { 317 struct vhost_crypto_info *info = NULL; 318 uint32_t i, j; 319 320 for (i = 0; i < options.nb_los; i++) { 321 for (j = 0; j < options.los[i].nb_sockets; j++) { 322 if (options.infos[i]->vids[j] == vid) { 323 info = options.infos[i]; 324 break; 325 } 326 } 327 if (info) 328 break; 329 } 330 331 if (!info) { 332 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n"); 333 return; 334 } 335 336 do { 337 338 } while (info->nb_inflight_ops); 339 340 info->initialized[j] = 0; 341 342 rte_wmb(); 343 344 rte_vhost_crypto_free(vid); 345 346 RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid); 347 } 348 349 static const struct vhost_device_ops virtio_crypto_device_ops = { 350 .new_device = new_device, 351 .destroy_device = destroy_device, 352 }; 353 354 __attribute__((unused)) 355 static void clrscr(void) 356 { 357 system("@cls||clear"); 358 } 359 360 static int 361 vhost_crypto_worker(void *arg) 362 { 363 struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 364 struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 365 struct vhost_crypto_info *info = arg; 366 uint16_t nb_callfds; 367 int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS]; 368 uint32_t lcore_id = rte_lcore_id(); 369 uint32_t burst_size = MAX_PKT_BURST; 370 uint32_t i, j, k; 371 uint32_t to_fetch, fetched; 372 373 int ret = 0; 374 375 RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id); 376 377 for (i = 0; i < NB_VIRTIO_QUEUES; i++) { 378 if (rte_crypto_op_bulk_alloc(info->cop_pool, 379 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i], 380 burst_size) < burst_size) { 381 RTE_LOG(ERR, USER1, "Failed to alloc cops\n"); 382 ret = -1; 383 goto exit; 384 } 385 } 386 387 while (1) { 388 for (i = 0; i < info->nb_vids; i++) { 389 if (unlikely(info->initialized[i] == 0)) 390 continue; 391 392 for (j = 0; j < NB_VIRTIO_QUEUES; j++) { 393 to_fetch = RTE_MIN(burst_size, 394 (NB_CRYPTO_DESCRIPTORS - 395 info->nb_inflight_ops)); 396 fetched = rte_vhost_crypto_fetch_requests( 397 info->vids[i], j, ops[j], 398 to_fetch); 399 info->nb_inflight_ops += 400 rte_cryptodev_enqueue_burst( 401 info->cid, info->qid, ops[j], 402 fetched); 403 if (unlikely(rte_crypto_op_bulk_alloc( 404 info->cop_pool, 405 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 406 ops[j], fetched) < fetched)) { 407 RTE_LOG(ERR, USER1, "Failed realloc\n"); 408 return -1; 409 } 410 411 fetched = rte_cryptodev_dequeue_burst( 412 info->cid, info->qid, 413 ops_deq[j], RTE_MIN(burst_size, 414 info->nb_inflight_ops)); 415 fetched = rte_vhost_crypto_finalize_requests( 416 ops_deq[j], fetched, callfds, 417 &nb_callfds); 418 419 info->nb_inflight_ops -= fetched; 420 421 if (!options.guest_polling) { 422 for (k = 0; k < nb_callfds; k++) 423 eventfd_write(callfds[k], 424 (eventfd_t)1); 425 } 426 427 rte_mempool_put_bulk(info->cop_pool, 428 (void **)ops_deq[j], fetched); 429 } 430 } 431 } 432 exit: 433 return ret; 434 } 435 436 static void 437 free_resource(void) 438 { 439 uint32_t i, j; 440 441 for (i = 0; i < options.nb_los; i++) { 442 struct lcore_option *lo = &options.los[i]; 443 struct vhost_crypto_info *info = options.infos[i]; 444 445 rte_mempool_free(info->cop_pool); 446 rte_mempool_free(info->sess_pool); 447 448 for (j = 0; j < lo->nb_sockets; j++) { 449 rte_vhost_driver_unregister(lo->socket_files[i]); 450 free(lo->socket_files[i]); 451 } 452 453 rte_free(info); 454 } 455 456 memset(&options, 0, sizeof(options)); 457 } 458 459 int 460 main(int argc, char *argv[]) 461 { 462 struct rte_cryptodev_qp_conf qp_conf = {NB_CRYPTO_DESCRIPTORS}; 463 struct rte_cryptodev_config config; 464 struct rte_cryptodev_info dev_info; 465 char name[128]; 466 uint32_t i, j, lcore; 467 int ret; 468 469 ret = rte_eal_init(argc, argv); 470 if (ret < 0) 471 return -1; 472 argc -= ret; 473 argv += ret; 474 475 ret = vhost_crypto_parse_args(argc, argv); 476 if (ret < 0) 477 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n"); 478 479 for (i = 0; i < options.nb_los; i++) { 480 struct lcore_option *lo = &options.los[i]; 481 struct vhost_crypto_info *info; 482 483 info = rte_zmalloc_socket(NULL, sizeof(*info), 484 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id( 485 lo->lcore_id)); 486 if (!info) { 487 ret = -ENOMEM; 488 goto error_exit; 489 } 490 491 info->cid = lo->cid; 492 info->qid = lo->qid; 493 info->nb_vids = lo->nb_sockets; 494 495 rte_cryptodev_info_get(info->cid, &dev_info); 496 if (dev_info.max_nb_queue_pairs < info->qid + 1) { 497 RTE_LOG(ERR, USER1, "Number of queues cannot over %u", 498 dev_info.max_nb_queue_pairs); 499 goto error_exit; 500 } 501 502 config.nb_queue_pairs = dev_info.max_nb_queue_pairs; 503 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id); 504 505 ret = rte_cryptodev_configure(info->cid, &config); 506 if (ret < 0) { 507 RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u", 508 info->cid); 509 goto error_exit; 510 } 511 512 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id); 513 info->sess_pool = rte_mempool_create(name, SESSION_MAP_ENTRIES, 514 rte_cryptodev_sym_get_private_session_size( 515 info->cid), 64, 0, NULL, NULL, NULL, NULL, 516 rte_lcore_to_socket_id(lo->lcore_id), 0); 517 if (!info->sess_pool) { 518 RTE_LOG(ERR, USER1, "Failed to create mempool"); 519 goto error_exit; 520 } 521 522 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id); 523 info->cop_pool = rte_crypto_op_pool_create(name, 524 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS, 525 NB_CACHE_OBJS, 0, 526 rte_lcore_to_socket_id(lo->lcore_id)); 527 528 if (!info->cop_pool) { 529 RTE_LOG(ERR, USER1, "Failed to create crypto pool"); 530 ret = -ENOMEM; 531 goto error_exit; 532 } 533 534 options.infos[i] = info; 535 536 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) { 537 ret = rte_cryptodev_queue_pair_setup(info->cid, j, 538 &qp_conf, rte_lcore_to_socket_id( 539 lo->lcore_id), 540 info->sess_pool); 541 if (ret < 0) { 542 RTE_LOG(ERR, USER1, "Failed to configure qp\n"); 543 goto error_exit; 544 } 545 } 546 } 547 548 for (i = 0; i < options.nb_los; i++) { 549 struct lcore_option *lo = &options.los[i]; 550 struct vhost_crypto_info *info = options.infos[i]; 551 552 ret = rte_cryptodev_start(lo->cid); 553 if (ret < 0) { 554 RTE_LOG(ERR, USER1, "Failed to start cryptodev\n"); 555 goto error_exit; 556 } 557 558 if (rte_eal_remote_launch(vhost_crypto_worker, info, 559 lo->lcore_id) < 0) { 560 RTE_LOG(ERR, USER1, "Failed to start worker lcore"); 561 goto error_exit; 562 } 563 564 for (j = 0; j < lo->nb_sockets; j++) { 565 ret = rte_vhost_driver_register(lo->socket_files[j], 566 RTE_VHOST_USER_DEQUEUE_ZERO_COPY); 567 if (ret < 0) { 568 RTE_LOG(ERR, USER1, "socket %s already exists\n", 569 lo->socket_files[j]); 570 goto error_exit; 571 } 572 573 rte_vhost_driver_callback_register(lo->socket_files[j], 574 &virtio_crypto_device_ops); 575 576 ret = rte_vhost_driver_start(lo->socket_files[j]); 577 if (ret < 0) { 578 RTE_LOG(ERR, USER1, "failed to start vhost.\n"); 579 goto error_exit; 580 } 581 } 582 } 583 584 RTE_LCORE_FOREACH(lcore) 585 rte_eal_wait_lcore(lcore); 586 587 free_resource(); 588 589 return 0; 590 591 error_exit: 592 593 free_resource(); 594 595 return -1; 596 } 597