1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017-2018 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <unistd.h> 9 #include <stdbool.h> 10 #include <assert.h> 11 #include <getopt.h> 12 13 #include <rte_malloc.h> 14 #include <rte_cycles.h> 15 #include <rte_vhost.h> 16 #include <rte_cryptodev.h> 17 #include <rte_vhost_crypto.h> 18 #include <rte_string_fns.h> 19 20 #include <cmdline_rdline.h> 21 #include <cmdline_parse.h> 22 #include <cmdline_parse_string.h> 23 #include <cmdline.h> 24 25 #define NB_VIRTIO_QUEUES (1) 26 #define MAX_PKT_BURST (64) 27 #define MAX_IV_LEN (32) 28 #define NB_MEMPOOL_OBJS (8192) 29 #define NB_CRYPTO_DESCRIPTORS (4096) 30 #define NB_CACHE_OBJS (128) 31 #define SESSION_MAP_ENTRIES (1024) 32 #define REFRESH_TIME_SEC (3) 33 34 #define MAX_NB_SOCKETS (4) 35 #define MAX_NB_WORKER_CORES (16) 36 37 struct lcore_option { 38 uint32_t lcore_id; 39 char *socket_files[MAX_NB_SOCKETS]; 40 uint32_t nb_sockets; 41 uint8_t cid; 42 uint16_t qid; 43 }; 44 45 struct vhost_crypto_info { 46 int vids[MAX_NB_SOCKETS]; 47 uint32_t nb_vids; 48 struct rte_mempool *sess_pool; 49 struct rte_mempool *cop_pool; 50 uint8_t cid; 51 uint32_t qid; 52 uint32_t nb_inflight_ops; 53 volatile uint32_t initialized[MAX_NB_SOCKETS]; 54 } __rte_cache_aligned; 55 56 struct vhost_crypto_options { 57 struct lcore_option los[MAX_NB_WORKER_CORES]; 58 struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES]; 59 uint32_t nb_los; 60 uint32_t zero_copy; 61 uint32_t guest_polling; 62 } options; 63 64 #define CONFIG_KEYWORD "config" 65 #define SOCKET_FILE_KEYWORD "socket-file" 66 #define ZERO_COPY_KEYWORD "zero-copy" 67 #define POLLING_KEYWORD "guest-polling" 68 69 #define NB_SOCKET_FIELDS (2) 70 71 static uint32_t 72 find_lo(uint32_t lcore_id) 73 { 74 uint32_t i; 75 76 for (i = 0; i < options.nb_los; i++) 77 if (options.los[i].lcore_id == lcore_id) 78 return i; 79 80 return UINT32_MAX; 81 } 82 83 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */ 84 static int 85 parse_socket_arg(char *arg) 86 { 87 uint32_t nb_sockets; 88 uint32_t lcore_id; 89 char *str_fld[NB_SOCKET_FIELDS]; 90 struct lcore_option *lo; 91 uint32_t idx; 92 char *end; 93 94 if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') != 95 NB_SOCKET_FIELDS) { 96 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg); 97 return -EINVAL; 98 } 99 100 errno = 0; 101 lcore_id = strtoul(str_fld[0], &end, 0); 102 if (errno != 0 || end == str_fld[0] || lcore_id > 255) 103 return -EINVAL; 104 105 idx = find_lo(lcore_id); 106 if (idx == UINT32_MAX) { 107 if (options.nb_los == MAX_NB_WORKER_CORES) 108 return -ENOMEM; 109 lo = &options.los[options.nb_los]; 110 lo->lcore_id = lcore_id; 111 options.nb_los++; 112 } else 113 lo = &options.los[idx]; 114 115 nb_sockets = lo->nb_sockets; 116 117 if (nb_sockets >= MAX_NB_SOCKETS) { 118 RTE_LOG(ERR, USER1, "Too many socket files!\n"); 119 return -ENOMEM; 120 } 121 122 lo->socket_files[nb_sockets] = strdup(str_fld[1]); 123 if (!lo->socket_files[nb_sockets]) { 124 RTE_LOG(ERR, USER1, "Insufficient memory\n"); 125 return -ENOMEM; 126 } 127 128 lo->nb_sockets++; 129 130 return 0; 131 } 132 133 static int 134 parse_config(char *q_arg) 135 { 136 struct lcore_option *lo; 137 char s[256]; 138 const char *p, *p0 = q_arg; 139 char *end; 140 enum fieldnames { 141 FLD_LCORE = 0, 142 FLD_CID, 143 FLD_QID, 144 _NUM_FLD 145 }; 146 uint32_t flds[_NUM_FLD]; 147 char *str_fld[_NUM_FLD]; 148 uint32_t i; 149 uint32_t size; 150 151 while ((p = strchr(p0, '(')) != NULL) { 152 ++p; 153 p0 = strchr(p, ')'); 154 if (p0 == NULL) 155 return -1; 156 157 size = p0 - p; 158 if (size >= sizeof(s)) 159 return -1; 160 161 snprintf(s, sizeof(s), "%.*s", size, p); 162 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 163 _NUM_FLD) 164 return -1; 165 for (i = 0; i < _NUM_FLD; i++) { 166 errno = 0; 167 flds[i] = strtoul(str_fld[i], &end, 0); 168 if (errno != 0 || end == str_fld[i] || flds[i] > 255) 169 return -EINVAL; 170 } 171 172 if (flds[FLD_LCORE] > RTE_MAX_LCORE) 173 return -EINVAL; 174 175 i = find_lo(flds[FLD_LCORE]); 176 if (i == UINT32_MAX) { 177 if (options.nb_los == MAX_NB_WORKER_CORES) 178 return -ENOMEM; 179 lo = &options.los[options.nb_los]; 180 options.nb_los++; 181 } else 182 lo = &options.los[i]; 183 184 lo->lcore_id = flds[FLD_LCORE]; 185 lo->cid = flds[FLD_CID]; 186 lo->qid = flds[FLD_QID]; 187 } 188 189 return 0; 190 } 191 192 static void 193 vhost_crypto_usage(const char *prgname) 194 { 195 printf("%s [EAL options] --\n" 196 " --%s <lcore>,SOCKET-FILE-PATH\n" 197 " --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]" 198 " --%s: zero copy\n" 199 " --%s: guest polling\n", 200 prgname, SOCKET_FILE_KEYWORD, CONFIG_KEYWORD, 201 ZERO_COPY_KEYWORD, POLLING_KEYWORD); 202 } 203 204 static int 205 vhost_crypto_parse_args(int argc, char **argv) 206 { 207 int opt, ret; 208 char *prgname = argv[0]; 209 char **argvopt; 210 int option_index; 211 struct option lgopts[] = { 212 {SOCKET_FILE_KEYWORD, required_argument, 0, 0}, 213 {CONFIG_KEYWORD, required_argument, 0, 0}, 214 {ZERO_COPY_KEYWORD, no_argument, 0, 0}, 215 {POLLING_KEYWORD, no_argument, 0, 0}, 216 {NULL, 0, 0, 0} 217 }; 218 219 argvopt = argv; 220 221 while ((opt = getopt_long(argc, argvopt, "s:", 222 lgopts, &option_index)) != EOF) { 223 224 switch (opt) { 225 case 0: 226 if (strcmp(lgopts[option_index].name, 227 SOCKET_FILE_KEYWORD) == 0) { 228 ret = parse_socket_arg(optarg); 229 if (ret < 0) { 230 vhost_crypto_usage(prgname); 231 return ret; 232 } 233 } else if (strcmp(lgopts[option_index].name, 234 CONFIG_KEYWORD) == 0) { 235 ret = parse_config(optarg); 236 if (ret < 0) { 237 vhost_crypto_usage(prgname); 238 return ret; 239 } 240 } else if (strcmp(lgopts[option_index].name, 241 ZERO_COPY_KEYWORD) == 0) { 242 options.zero_copy = 243 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE; 244 } else if (strcmp(lgopts[option_index].name, 245 POLLING_KEYWORD) == 0) { 246 options.guest_polling = 1; 247 } else { 248 vhost_crypto_usage(prgname); 249 return -EINVAL; 250 } 251 break; 252 default: 253 return -1; 254 } 255 } 256 257 return 0; 258 } 259 260 static int 261 new_device(int vid) 262 { 263 struct vhost_crypto_info *info = NULL; 264 char path[PATH_MAX]; 265 uint32_t i, j; 266 int ret; 267 268 ret = rte_vhost_get_ifname(vid, path, PATH_MAX); 269 if (ret) { 270 RTE_LOG(ERR, USER1, "Cannot find matched socket\n"); 271 return ret; 272 } 273 274 for (i = 0; i < options.nb_los; i++) { 275 for (j = 0; j < options.los[i].nb_sockets; j++) { 276 if (strcmp(path, options.los[i].socket_files[j]) == 0) { 277 info = options.infos[i]; 278 break; 279 } 280 } 281 282 if (info) 283 break; 284 } 285 286 if (!info) { 287 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n"); 288 return -ENOENT; 289 } 290 291 ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool, 292 rte_lcore_to_socket_id(options.los[i].lcore_id)); 293 if (ret) { 294 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n"); 295 return ret; 296 } 297 298 ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy); 299 if (ret) { 300 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n", 301 options.zero_copy == 1 ? "enable" : "disable"); 302 return ret; 303 } 304 305 info->vids[j] = vid; 306 info->initialized[j] = 1; 307 308 rte_wmb(); 309 310 RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path, 311 vid); 312 return 0; 313 } 314 315 static void 316 destroy_device(int vid) 317 { 318 struct vhost_crypto_info *info = NULL; 319 uint32_t i, j; 320 321 for (i = 0; i < options.nb_los; i++) { 322 for (j = 0; j < options.los[i].nb_sockets; j++) { 323 if (options.infos[i]->vids[j] == vid) { 324 info = options.infos[i]; 325 break; 326 } 327 } 328 if (info) 329 break; 330 } 331 332 if (!info) { 333 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n"); 334 return; 335 } 336 337 do { 338 339 } while (info->nb_inflight_ops); 340 341 info->initialized[j] = 0; 342 343 rte_wmb(); 344 345 rte_vhost_crypto_free(vid); 346 347 RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid); 348 } 349 350 static const struct vhost_device_ops virtio_crypto_device_ops = { 351 .new_device = new_device, 352 .destroy_device = destroy_device, 353 }; 354 355 __attribute__((unused)) 356 static void clrscr(void) 357 { 358 system("@cls||clear"); 359 } 360 361 static int 362 vhost_crypto_worker(void *arg) 363 { 364 struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 365 struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 366 struct vhost_crypto_info *info = arg; 367 uint16_t nb_callfds; 368 int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS]; 369 uint32_t lcore_id = rte_lcore_id(); 370 uint32_t burst_size = MAX_PKT_BURST; 371 uint32_t i, j, k; 372 uint32_t to_fetch, fetched; 373 374 int ret = 0; 375 376 RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id); 377 378 for (i = 0; i < NB_VIRTIO_QUEUES; i++) { 379 if (rte_crypto_op_bulk_alloc(info->cop_pool, 380 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i], 381 burst_size) < burst_size) { 382 RTE_LOG(ERR, USER1, "Failed to alloc cops\n"); 383 ret = -1; 384 goto exit; 385 } 386 } 387 388 while (1) { 389 for (i = 0; i < info->nb_vids; i++) { 390 if (unlikely(info->initialized[i] == 0)) 391 continue; 392 393 for (j = 0; j < NB_VIRTIO_QUEUES; j++) { 394 to_fetch = RTE_MIN(burst_size, 395 (NB_CRYPTO_DESCRIPTORS - 396 info->nb_inflight_ops)); 397 fetched = rte_vhost_crypto_fetch_requests( 398 info->vids[i], j, ops[j], 399 to_fetch); 400 info->nb_inflight_ops += 401 rte_cryptodev_enqueue_burst( 402 info->cid, info->qid, ops[j], 403 fetched); 404 if (unlikely(rte_crypto_op_bulk_alloc( 405 info->cop_pool, 406 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 407 ops[j], fetched) < fetched)) { 408 RTE_LOG(ERR, USER1, "Failed realloc\n"); 409 return -1; 410 } 411 412 fetched = rte_cryptodev_dequeue_burst( 413 info->cid, info->qid, 414 ops_deq[j], RTE_MIN(burst_size, 415 info->nb_inflight_ops)); 416 fetched = rte_vhost_crypto_finalize_requests( 417 ops_deq[j], fetched, callfds, 418 &nb_callfds); 419 420 info->nb_inflight_ops -= fetched; 421 422 if (!options.guest_polling) { 423 for (k = 0; k < nb_callfds; k++) 424 eventfd_write(callfds[k], 425 (eventfd_t)1); 426 } 427 428 rte_mempool_put_bulk(info->cop_pool, 429 (void **)ops_deq[j], fetched); 430 } 431 } 432 } 433 exit: 434 return ret; 435 } 436 437 static void 438 free_resource(void) 439 { 440 uint32_t i, j; 441 442 for (i = 0; i < options.nb_los; i++) { 443 struct lcore_option *lo = &options.los[i]; 444 struct vhost_crypto_info *info = options.infos[i]; 445 446 if (!info) 447 continue; 448 449 rte_mempool_free(info->cop_pool); 450 rte_mempool_free(info->sess_pool); 451 452 for (j = 0; j < lo->nb_sockets; j++) { 453 rte_vhost_driver_unregister(lo->socket_files[i]); 454 free(lo->socket_files[i]); 455 } 456 457 rte_free(info); 458 } 459 460 memset(&options, 0, sizeof(options)); 461 } 462 463 int 464 main(int argc, char *argv[]) 465 { 466 struct rte_cryptodev_qp_conf qp_conf = {NB_CRYPTO_DESCRIPTORS}; 467 struct rte_cryptodev_config config; 468 struct rte_cryptodev_info dev_info; 469 char name[128]; 470 uint32_t i, j, lcore; 471 int ret; 472 473 ret = rte_eal_init(argc, argv); 474 if (ret < 0) 475 return -1; 476 argc -= ret; 477 argv += ret; 478 479 ret = vhost_crypto_parse_args(argc, argv); 480 if (ret < 0) 481 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n"); 482 483 for (i = 0; i < options.nb_los; i++) { 484 struct lcore_option *lo = &options.los[i]; 485 struct vhost_crypto_info *info; 486 487 info = rte_zmalloc_socket(NULL, sizeof(*info), 488 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id( 489 lo->lcore_id)); 490 if (!info) { 491 ret = -ENOMEM; 492 goto error_exit; 493 } 494 495 info->cid = lo->cid; 496 info->qid = lo->qid; 497 info->nb_vids = lo->nb_sockets; 498 499 rte_cryptodev_info_get(info->cid, &dev_info); 500 if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) { 501 #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD crypto_aesni_mb 502 #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm 503 if (strstr(dev_info.driver_name, 504 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) || 505 strstr(dev_info.driver_name, 506 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) { 507 RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n", 508 dev_info.driver_name); 509 ret = -EPERM; 510 goto error_exit; 511 } 512 } 513 514 if (dev_info.max_nb_queue_pairs < info->qid + 1) { 515 RTE_LOG(ERR, USER1, "Number of queues cannot over %u", 516 dev_info.max_nb_queue_pairs); 517 goto error_exit; 518 } 519 520 config.nb_queue_pairs = dev_info.max_nb_queue_pairs; 521 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id); 522 523 ret = rte_cryptodev_configure(info->cid, &config); 524 if (ret < 0) { 525 RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u", 526 info->cid); 527 goto error_exit; 528 } 529 530 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id); 531 info->sess_pool = rte_mempool_create(name, SESSION_MAP_ENTRIES, 532 rte_cryptodev_sym_get_private_session_size( 533 info->cid), 64, 0, NULL, NULL, NULL, NULL, 534 rte_lcore_to_socket_id(lo->lcore_id), 0); 535 if (!info->sess_pool) { 536 RTE_LOG(ERR, USER1, "Failed to create mempool"); 537 goto error_exit; 538 } 539 540 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id); 541 info->cop_pool = rte_crypto_op_pool_create(name, 542 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS, 543 NB_CACHE_OBJS, 0, 544 rte_lcore_to_socket_id(lo->lcore_id)); 545 546 if (!info->cop_pool) { 547 RTE_LOG(ERR, USER1, "Failed to create crypto pool"); 548 ret = -ENOMEM; 549 goto error_exit; 550 } 551 552 options.infos[i] = info; 553 554 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) { 555 ret = rte_cryptodev_queue_pair_setup(info->cid, j, 556 &qp_conf, rte_lcore_to_socket_id( 557 lo->lcore_id), 558 info->sess_pool); 559 if (ret < 0) { 560 RTE_LOG(ERR, USER1, "Failed to configure qp\n"); 561 goto error_exit; 562 } 563 } 564 } 565 566 for (i = 0; i < options.nb_los; i++) { 567 struct lcore_option *lo = &options.los[i]; 568 struct vhost_crypto_info *info = options.infos[i]; 569 570 ret = rte_cryptodev_start(lo->cid); 571 if (ret < 0) { 572 RTE_LOG(ERR, USER1, "Failed to start cryptodev\n"); 573 goto error_exit; 574 } 575 576 if (rte_eal_remote_launch(vhost_crypto_worker, info, 577 lo->lcore_id) < 0) { 578 RTE_LOG(ERR, USER1, "Failed to start worker lcore"); 579 goto error_exit; 580 } 581 582 for (j = 0; j < lo->nb_sockets; j++) { 583 ret = rte_vhost_driver_register(lo->socket_files[j], 584 RTE_VHOST_USER_DEQUEUE_ZERO_COPY); 585 if (ret < 0) { 586 RTE_LOG(ERR, USER1, "socket %s already exists\n", 587 lo->socket_files[j]); 588 goto error_exit; 589 } 590 591 rte_vhost_driver_callback_register(lo->socket_files[j], 592 &virtio_crypto_device_ops); 593 594 ret = rte_vhost_driver_start(lo->socket_files[j]); 595 if (ret < 0) { 596 RTE_LOG(ERR, USER1, "failed to start vhost.\n"); 597 goto error_exit; 598 } 599 } 600 } 601 602 RTE_LCORE_FOREACH(lcore) 603 rte_eal_wait_lcore(lcore); 604 605 free_resource(); 606 607 return 0; 608 609 error_exit: 610 611 free_resource(); 612 613 return -1; 614 } 615