1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017-2018 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <unistd.h> 8 #include <stdbool.h> 9 #include <assert.h> 10 #include <getopt.h> 11 12 #include <rte_malloc.h> 13 #include <rte_cycles.h> 14 #include <rte_vhost.h> 15 #include <rte_cryptodev.h> 16 #include <rte_vhost_crypto.h> 17 18 #include <cmdline_rdline.h> 19 #include <cmdline_parse.h> 20 #include <cmdline_parse_string.h> 21 #include <cmdline.h> 22 23 #define NB_VIRTIO_QUEUES (1) 24 #define MAX_PKT_BURST (64) 25 #define MAX_IV_LEN (32) 26 #define NB_MEMPOOL_OBJS (8192) 27 #define NB_CRYPTO_DESCRIPTORS (4096) 28 #define NB_CACHE_OBJS (128) 29 #define SESSION_MAP_ENTRIES (1024) 30 #define REFRESH_TIME_SEC (3) 31 32 #define MAX_NB_SOCKETS (32) 33 #define DEF_SOCKET_FILE "/tmp/vhost_crypto1.socket" 34 35 struct vhost_crypto_options { 36 char *socket_files[MAX_NB_SOCKETS]; 37 uint32_t nb_sockets; 38 uint8_t cid; 39 uint16_t qid; 40 uint32_t zero_copy; 41 uint32_t guest_polling; 42 } options; 43 44 struct vhost_crypto_info { 45 int vids[MAX_NB_SOCKETS]; 46 struct rte_mempool *sess_pool; 47 struct rte_mempool *cop_pool; 48 uint32_t lcore_id; 49 uint8_t cid; 50 uint32_t qid; 51 uint32_t nb_vids; 52 volatile uint32_t initialized[MAX_NB_SOCKETS]; 53 54 } info; 55 56 #define SOCKET_FILE_KEYWORD "socket-file" 57 #define CRYPTODEV_ID_KEYWORD "cdev-id" 58 #define CRYPTODEV_QUEUE_KEYWORD "cdev-queue-id" 59 #define ZERO_COPY_KEYWORD "zero-copy" 60 #define POLLING_KEYWORD "guest-polling" 61 62 uint64_t vhost_cycles[2], last_v_cycles[2]; 63 uint64_t outpkt_amount; 64 65 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */ 66 static int 67 parse_socket_arg(char *arg) 68 { 69 uint32_t nb_sockets = options.nb_sockets; 70 size_t len = strlen(arg); 71 72 if (nb_sockets >= MAX_NB_SOCKETS) { 73 RTE_LOG(ERR, USER1, "Too many socket files!\n"); 74 return -ENOMEM; 75 } 76 77 options.socket_files[nb_sockets] = rte_malloc(NULL, len, 0); 78 if (!options.socket_files[nb_sockets]) { 79 RTE_LOG(ERR, USER1, "Insufficient memory\n"); 80 return -ENOMEM; 81 } 82 83 rte_memcpy(options.socket_files[nb_sockets], arg, len); 84 85 options.nb_sockets++; 86 87 return 0; 88 } 89 90 static int 91 parse_cryptodev_id(const char *q_arg) 92 { 93 char *end = NULL; 94 uint64_t pm; 95 96 /* parse decimal string */ 97 pm = strtoul(q_arg, &end, 10); 98 if (pm > rte_cryptodev_count()) { 99 RTE_LOG(ERR, USER1, "Invalid Cryptodev ID %s\n", q_arg); 100 return -1; 101 } 102 103 options.cid = (uint8_t)pm; 104 105 return 0; 106 } 107 108 static int 109 parse_cdev_queue_id(const char *q_arg) 110 { 111 char *end = NULL; 112 uint64_t pm; 113 114 /* parse decimal string */ 115 pm = strtoul(q_arg, &end, 10); 116 if (pm == UINT64_MAX) { 117 RTE_LOG(ERR, USER1, "Invalid Cryptodev Queue ID %s\n", q_arg); 118 return -1; 119 } 120 121 options.qid = (uint16_t)pm; 122 123 return 0; 124 } 125 126 static void 127 vhost_crypto_usage(const char *prgname) 128 { 129 printf("%s [EAL options] --\n" 130 " --%s SOCKET-FILE-PATH\n" 131 " --%s CRYPTODEV_ID: crypto device id\n" 132 " --%s CDEV_QUEUE_ID: crypto device queue id\n" 133 " --%s: zero copy\n" 134 " --%s: guest polling\n", 135 prgname, SOCKET_FILE_KEYWORD, CRYPTODEV_ID_KEYWORD, 136 CRYPTODEV_QUEUE_KEYWORD, ZERO_COPY_KEYWORD, POLLING_KEYWORD); 137 } 138 139 static int 140 vhost_crypto_parse_args(int argc, char **argv) 141 { 142 int opt, ret; 143 char *prgname = argv[0]; 144 char **argvopt; 145 int option_index; 146 struct option lgopts[] = { 147 {SOCKET_FILE_KEYWORD, required_argument, 0, 0}, 148 {CRYPTODEV_ID_KEYWORD, required_argument, 0, 0}, 149 {CRYPTODEV_QUEUE_KEYWORD, required_argument, 0, 0}, 150 {ZERO_COPY_KEYWORD, no_argument, 0, 0}, 151 {POLLING_KEYWORD, no_argument, 0, 0}, 152 {NULL, 0, 0, 0} 153 }; 154 155 options.cid = 0; 156 options.qid = 0; 157 options.nb_sockets = 0; 158 options.guest_polling = 0; 159 options.zero_copy = RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE; 160 161 argvopt = argv; 162 163 while ((opt = getopt_long(argc, argvopt, "s:", 164 lgopts, &option_index)) != EOF) { 165 166 switch (opt) { 167 case 0: 168 if (strcmp(lgopts[option_index].name, 169 SOCKET_FILE_KEYWORD) == 0) { 170 ret = parse_socket_arg(optarg); 171 if (ret < 0) { 172 vhost_crypto_usage(prgname); 173 return ret; 174 } 175 } else if (strcmp(lgopts[option_index].name, 176 CRYPTODEV_ID_KEYWORD) == 0) { 177 ret = parse_cryptodev_id(optarg); 178 if (ret < 0) { 179 vhost_crypto_usage(prgname); 180 return ret; 181 } 182 } else if (strcmp(lgopts[option_index].name, 183 CRYPTODEV_QUEUE_KEYWORD) == 0) { 184 ret = parse_cdev_queue_id(optarg); 185 if (ret < 0) { 186 vhost_crypto_usage(prgname); 187 return ret; 188 } 189 } else if (strcmp(lgopts[option_index].name, 190 ZERO_COPY_KEYWORD) == 0) { 191 options.zero_copy = 192 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE; 193 } else if (strcmp(lgopts[option_index].name, 194 POLLING_KEYWORD) == 0) { 195 options.guest_polling = 1; 196 } else { 197 vhost_crypto_usage(prgname); 198 return -EINVAL; 199 } 200 break; 201 default: 202 return -1; 203 } 204 } 205 206 if (options.nb_sockets == 0) { 207 options.socket_files[0] = strdup(DEF_SOCKET_FILE); 208 options.nb_sockets = 1; 209 RTE_LOG(INFO, USER1, 210 "VHOST-CRYPTO: use default socket file %s\n", 211 DEF_SOCKET_FILE); 212 } 213 214 return 0; 215 } 216 217 static int 218 new_device(int vid) 219 { 220 char path[PATH_MAX]; 221 uint32_t idx, i; 222 int ret; 223 224 ret = rte_vhost_get_ifname(vid, path, PATH_MAX); 225 if (ret) { 226 RTE_LOG(ERR, USER1, "Cannot find matched socket\n"); 227 return ret; 228 } 229 230 for (idx = 0; idx < options.nb_sockets; idx++) { 231 if (strcmp(path, options.socket_files[idx]) == 0) 232 break; 233 } 234 235 if (idx == options.nb_sockets) { 236 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n"); 237 return -ENOENT; 238 } 239 240 for (i = 0; i < 2; i++) { 241 vhost_cycles[i] = 0; 242 last_v_cycles[i] = 0; 243 } 244 245 ret = rte_vhost_crypto_create(vid, info.cid, info.sess_pool, 246 rte_lcore_to_socket_id(info.lcore_id)); 247 if (ret) { 248 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n"); 249 return ret; 250 } 251 252 ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy); 253 if (ret) { 254 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n", 255 options.zero_copy == 1 ? "enable" : "disable"); 256 return ret; 257 } 258 259 info.vids[idx] = vid; 260 info.initialized[idx] = 1; 261 262 rte_wmb(); 263 264 RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path, 265 vid); 266 return 0; 267 } 268 269 static void 270 destroy_device(int vid) 271 { 272 uint32_t i; 273 274 for (i = 0; i < info.nb_vids; i++) { 275 if (vid == info.vids[i]) 276 break; 277 } 278 279 if (i == info.nb_vids) { 280 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n"); 281 return; 282 } 283 284 info.initialized[i] = 0; 285 286 rte_wmb(); 287 288 rte_vhost_crypto_free(vid); 289 290 RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid); 291 } 292 293 static const struct vhost_device_ops virtio_crypto_device_ops = { 294 .new_device = new_device, 295 .destroy_device = destroy_device, 296 }; 297 298 __attribute__((unused)) 299 static void clrscr(void) 300 { 301 system("@cls||clear"); 302 } 303 304 static int 305 vhost_crypto_worker(__rte_unused void *arg) 306 { 307 struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 308 struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 309 uint32_t nb_inflight_ops = 0; 310 uint16_t nb_callfds; 311 int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS]; 312 uint32_t lcore_id = rte_lcore_id(); 313 uint32_t burst_size = MAX_PKT_BURST; 314 uint32_t i, j, k; 315 uint32_t to_fetch, fetched; 316 uint64_t t_start, t_end, interval; 317 318 int ret = 0; 319 320 RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id); 321 322 for (i = 0; i < NB_VIRTIO_QUEUES; i++) { 323 if (rte_crypto_op_bulk_alloc(info.cop_pool, 324 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i], 325 burst_size) < burst_size) { 326 RTE_LOG(ERR, USER1, "Failed to alloc cops\n"); 327 ret = -1; 328 goto exit; 329 } 330 } 331 332 while (1) { 333 for (i = 0; i < info.nb_vids; i++) { 334 if (unlikely(info.initialized[i] == 0)) 335 continue; 336 337 for (j = 0; j < NB_VIRTIO_QUEUES; j++) { 338 t_start = rte_rdtsc_precise(); 339 340 to_fetch = RTE_MIN(burst_size, 341 (NB_CRYPTO_DESCRIPTORS - 342 nb_inflight_ops)); 343 fetched = rte_vhost_crypto_fetch_requests( 344 info.vids[i], j, ops[j], 345 to_fetch); 346 nb_inflight_ops += rte_cryptodev_enqueue_burst( 347 info.cid, info.qid, ops[j], 348 fetched); 349 if (unlikely(rte_crypto_op_bulk_alloc( 350 info.cop_pool, 351 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 352 ops[j], fetched) < fetched)) { 353 RTE_LOG(ERR, USER1, "Failed realloc\n"); 354 return -1; 355 } 356 t_end = rte_rdtsc_precise(); 357 interval = t_end - t_start; 358 359 vhost_cycles[fetched > 0] += interval; 360 361 t_start = t_end; 362 fetched = rte_cryptodev_dequeue_burst( 363 info.cid, info.qid, 364 ops_deq[j], RTE_MIN(burst_size, 365 nb_inflight_ops)); 366 fetched = rte_vhost_crypto_finalize_requests( 367 ops_deq[j], fetched, callfds, 368 &nb_callfds); 369 370 nb_inflight_ops -= fetched; 371 outpkt_amount += fetched; 372 373 if (!options.guest_polling) { 374 for (k = 0; k < nb_callfds; k++) 375 eventfd_write(callfds[k], 376 (eventfd_t)1); 377 } 378 379 rte_mempool_put_bulk(info.cop_pool, 380 (void **)ops_deq[j], fetched); 381 interval = rte_rdtsc_precise() - t_start; 382 383 vhost_cycles[fetched > 0] += interval; 384 } 385 } 386 } 387 exit: 388 return ret; 389 } 390 391 392 static void 393 unregister_drivers(int socket_num) 394 { 395 int ret; 396 397 ret = rte_vhost_driver_unregister(options.socket_files[socket_num]); 398 if (ret != 0) 399 RTE_LOG(ERR, USER1, 400 "Fail to unregister vhost driver for %s.\n", 401 options.socket_files[socket_num]); 402 } 403 404 int 405 main(int argc, char *argv[]) 406 { 407 struct rte_cryptodev_qp_conf qp_conf = {NB_CRYPTO_DESCRIPTORS}; 408 struct rte_cryptodev_config config; 409 struct rte_cryptodev_info dev_info; 410 uint32_t cryptodev_id; 411 uint32_t worker_lcore; 412 char name[128]; 413 uint32_t i = 0; 414 int ret; 415 416 ret = rte_eal_init(argc, argv); 417 if (ret < 0) 418 return -1; 419 argc -= ret; 420 argv += ret; 421 422 ret = vhost_crypto_parse_args(argc, argv); 423 if (ret < 0) 424 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n"); 425 426 info.cid = options.cid; 427 info.qid = options.qid; 428 429 worker_lcore = rte_get_next_lcore(0, 1, 0); 430 if (worker_lcore == RTE_MAX_LCORE) 431 rte_exit(EXIT_FAILURE, "Not enough lcore\n"); 432 433 cryptodev_id = info.cid; 434 rte_cryptodev_info_get(cryptodev_id, &dev_info); 435 if (dev_info.max_nb_queue_pairs < info.qid + 1) { 436 RTE_LOG(ERR, USER1, "Number of queues cannot over %u", 437 dev_info.max_nb_queue_pairs); 438 goto error_exit; 439 } 440 441 config.nb_queue_pairs = dev_info.max_nb_queue_pairs; 442 config.socket_id = rte_lcore_to_socket_id(worker_lcore); 443 444 ret = rte_cryptodev_configure(cryptodev_id, &config); 445 if (ret < 0) { 446 RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u", 447 cryptodev_id); 448 goto error_exit; 449 } 450 451 snprintf(name, 127, "SESS_POOL_%u", worker_lcore); 452 info.sess_pool = rte_mempool_create(name, SESSION_MAP_ENTRIES, 453 rte_cryptodev_sym_get_private_session_size( 454 cryptodev_id), 64, 0, NULL, NULL, NULL, NULL, 455 rte_lcore_to_socket_id(worker_lcore), 0); 456 if (!info.sess_pool) { 457 RTE_LOG(ERR, USER1, "Failed to create mempool"); 458 goto error_exit; 459 } 460 461 snprintf(name, 127, "COPPOOL_%u", worker_lcore); 462 info.cop_pool = rte_crypto_op_pool_create(name, 463 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS, 464 NB_CACHE_OBJS, 0, rte_lcore_to_socket_id(worker_lcore)); 465 466 if (!info.cop_pool) { 467 RTE_LOG(ERR, USER1, "Lcore %u failed to create crypto pool", 468 worker_lcore); 469 ret = -1; 470 goto error_exit; 471 } 472 473 info.nb_vids = options.nb_sockets; 474 for (i = 0; i < MAX_NB_SOCKETS; i++) 475 info.vids[i] = -1; 476 477 for (i = 0; i < dev_info.max_nb_queue_pairs; i++) { 478 ret = rte_cryptodev_queue_pair_setup(cryptodev_id, i, 479 &qp_conf, rte_lcore_to_socket_id(worker_lcore), 480 info.sess_pool); 481 if (ret < 0) { 482 RTE_LOG(ERR, USER1, "Failed to configure qp %u\n", 483 info.cid); 484 goto error_exit; 485 } 486 } 487 488 ret = rte_cryptodev_start(cryptodev_id); 489 if (ret < 0) { 490 RTE_LOG(ERR, USER1, "Failed to start cryptodev %u\n", info.cid); 491 goto error_exit; 492 } 493 494 info.cid = cryptodev_id; 495 info.lcore_id = worker_lcore; 496 497 if (rte_eal_remote_launch(vhost_crypto_worker, NULL, worker_lcore) 498 < 0) { 499 RTE_LOG(ERR, USER1, "Failed to start worker lcore"); 500 goto error_exit; 501 } 502 503 for (i = 0; i < options.nb_sockets; i++) { 504 if (rte_vhost_driver_register(options.socket_files[i], 505 RTE_VHOST_USER_DEQUEUE_ZERO_COPY) < 0) { 506 RTE_LOG(ERR, USER1, "socket %s already exists\n", 507 options.socket_files[i]); 508 goto error_exit; 509 } 510 511 rte_vhost_driver_callback_register(options.socket_files[i], 512 &virtio_crypto_device_ops); 513 514 if (rte_vhost_driver_start(options.socket_files[i]) < 0) { 515 RTE_LOG(ERR, USER1, "failed to start vhost driver.\n"); 516 goto error_exit; 517 } 518 } 519 520 RTE_LCORE_FOREACH(worker_lcore) 521 rte_eal_wait_lcore(worker_lcore); 522 523 rte_mempool_free(info.sess_pool); 524 rte_mempool_free(info.cop_pool); 525 526 return 0; 527 528 error_exit: 529 for (i = 0; i < options.nb_sockets; i++) 530 unregister_drivers(i); 531 532 rte_mempool_free(info.cop_pool); 533 rte_mempool_free(info.sess_pool); 534 535 return -1; 536 } 537