1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation
3 */
4
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <string.h>
8 #include <unistd.h>
9 #include <stdbool.h>
10 #include <assert.h>
11 #include <getopt.h>
12
13 #include <rte_malloc.h>
14 #include <rte_cycles.h>
15 #include <rte_vhost.h>
16 #include <rte_cryptodev.h>
17 #include <rte_vhost_crypto.h>
18 #include <rte_string_fns.h>
19
20 #include <cmdline_rdline.h>
21 #include <cmdline_parse.h>
22 #include <cmdline_parse_string.h>
23 #include <cmdline.h>
24
25 #define NB_VIRTIO_QUEUES (1)
26 #define MAX_PKT_BURST (64)
27 #define MAX_IV_LEN (32)
28 #define NB_MEMPOOL_OBJS (8192)
29 #define NB_CRYPTO_DESCRIPTORS (4096)
30 #define NB_CACHE_OBJS (128)
31 #define SESSION_MAP_ENTRIES (1024)
32 #define REFRESH_TIME_SEC (3)
33
34 #define MAX_NB_SOCKETS (4)
35 #define MAX_NB_WORKER_CORES (16)
36
37 struct lcore_option {
38 uint32_t lcore_id;
39 char *socket_files[MAX_NB_SOCKETS];
40 uint32_t nb_sockets;
41 uint8_t cid;
42 uint16_t qid;
43 };
44
45 struct __rte_cache_aligned vhost_crypto_info {
46 int vids[MAX_NB_SOCKETS];
47 uint32_t nb_vids;
48 struct rte_mempool *sess_pool;
49 struct rte_mempool *cop_pool;
50 uint8_t cid;
51 uint32_t qid;
52 uint32_t nb_inflight_ops;
53 volatile uint32_t initialized[MAX_NB_SOCKETS];
54 };
55
56 struct vhost_crypto_options {
57 struct lcore_option los[MAX_NB_WORKER_CORES];
58 struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES];
59 uint32_t nb_los;
60 uint32_t zero_copy;
61 uint32_t guest_polling;
62 } options;
63
64 enum {
65 #define OPT_CONFIG "config"
66 OPT_CONFIG_NUM = 256,
67 #define OPT_SOCKET_FILE "socket-file"
68 OPT_SOCKET_FILE_NUM,
69 #define OPT_ZERO_COPY "zero-copy"
70 OPT_ZERO_COPY_NUM,
71 #define OPT_POLLING "guest-polling"
72 OPT_POLLING_NUM,
73 };
74
75 #define NB_SOCKET_FIELDS (2)
76
77 static uint32_t
find_lo(uint32_t lcore_id)78 find_lo(uint32_t lcore_id)
79 {
80 uint32_t i;
81
82 for (i = 0; i < options.nb_los; i++)
83 if (options.los[i].lcore_id == lcore_id)
84 return i;
85
86 return UINT32_MAX;
87 }
88
89 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */
90 static int
parse_socket_arg(char * arg)91 parse_socket_arg(char *arg)
92 {
93 uint32_t nb_sockets;
94 uint32_t lcore_id;
95 char *str_fld[NB_SOCKET_FIELDS];
96 struct lcore_option *lo;
97 uint32_t idx;
98 char *end;
99
100 if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') !=
101 NB_SOCKET_FIELDS) {
102 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg);
103 return -EINVAL;
104 }
105
106 errno = 0;
107 lcore_id = strtoul(str_fld[0], &end, 0);
108 if (errno != 0 || end == str_fld[0] || lcore_id > 255)
109 return -EINVAL;
110
111 idx = find_lo(lcore_id);
112 if (idx == UINT32_MAX) {
113 if (options.nb_los == MAX_NB_WORKER_CORES)
114 return -ENOMEM;
115 lo = &options.los[options.nb_los];
116 lo->lcore_id = lcore_id;
117 options.nb_los++;
118 } else
119 lo = &options.los[idx];
120
121 nb_sockets = lo->nb_sockets;
122
123 if (nb_sockets >= MAX_NB_SOCKETS) {
124 RTE_LOG(ERR, USER1, "Too many socket files!\n");
125 return -ENOMEM;
126 }
127
128 lo->socket_files[nb_sockets] = strdup(str_fld[1]);
129 if (!lo->socket_files[nb_sockets]) {
130 RTE_LOG(ERR, USER1, "Insufficient memory\n");
131 return -ENOMEM;
132 }
133
134 lo->nb_sockets++;
135
136 return 0;
137 }
138
139 static int
parse_config(char * q_arg)140 parse_config(char *q_arg)
141 {
142 struct lcore_option *lo;
143 char s[256];
144 const char *p, *p0 = q_arg;
145 char *end;
146 enum fieldnames {
147 FLD_LCORE = 0,
148 FLD_CID,
149 FLD_QID,
150 _NUM_FLD
151 };
152 uint32_t flds[_NUM_FLD];
153 char *str_fld[_NUM_FLD];
154 uint32_t i;
155 uint32_t size;
156
157 while ((p = strchr(p0, '(')) != NULL) {
158 ++p;
159 p0 = strchr(p, ')');
160 if (p0 == NULL)
161 return -1;
162
163 size = p0 - p;
164 if (size >= sizeof(s))
165 return -1;
166
167 snprintf(s, sizeof(s), "%.*s", size, p);
168 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
169 _NUM_FLD)
170 return -1;
171 for (i = 0; i < _NUM_FLD; i++) {
172 errno = 0;
173 flds[i] = strtoul(str_fld[i], &end, 0);
174 if (errno != 0 || end == str_fld[i] || flds[i] > 255)
175 return -EINVAL;
176 }
177
178 if (flds[FLD_LCORE] > RTE_MAX_LCORE)
179 return -EINVAL;
180
181 i = find_lo(flds[FLD_LCORE]);
182 if (i == UINT32_MAX) {
183 if (options.nb_los == MAX_NB_WORKER_CORES)
184 return -ENOMEM;
185 lo = &options.los[options.nb_los];
186 options.nb_los++;
187 } else
188 lo = &options.los[i];
189
190 lo->lcore_id = flds[FLD_LCORE];
191 lo->cid = flds[FLD_CID];
192 lo->qid = flds[FLD_QID];
193 }
194
195 return 0;
196 }
197
198 static void
vhost_crypto_usage(const char * prgname)199 vhost_crypto_usage(const char *prgname)
200 {
201 printf("%s [EAL options] --\n"
202 " --%s <lcore>,SOCKET-FILE-PATH\n"
203 " --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]\n"
204 " --%s: zero copy\n"
205 " --%s: guest polling\n",
206 prgname, OPT_SOCKET_FILE, OPT_CONFIG,
207 OPT_ZERO_COPY, OPT_POLLING);
208 }
209
210 static int
vhost_crypto_parse_args(int argc,char ** argv)211 vhost_crypto_parse_args(int argc, char **argv)
212 {
213 int opt, ret;
214 char *prgname = argv[0];
215 char **argvopt;
216 int option_index;
217 struct option lgopts[] = {
218 {OPT_SOCKET_FILE, required_argument,
219 NULL, OPT_SOCKET_FILE_NUM},
220 {OPT_CONFIG, required_argument,
221 NULL, OPT_CONFIG_NUM},
222 {OPT_ZERO_COPY, no_argument,
223 NULL, OPT_ZERO_COPY_NUM},
224 {OPT_POLLING, no_argument,
225 NULL, OPT_POLLING_NUM},
226 {NULL, 0, 0, 0}
227 };
228
229 argvopt = argv;
230
231 while ((opt = getopt_long(argc, argvopt, "",
232 lgopts, &option_index)) != EOF) {
233
234 if (opt == '?') {
235 vhost_crypto_usage(prgname);
236 return -1;
237 }
238
239 switch (opt) {
240 case OPT_SOCKET_FILE_NUM:
241 ret = parse_socket_arg(optarg);
242 if (ret < 0) {
243 vhost_crypto_usage(prgname);
244 return ret;
245 }
246 break;
247
248 case OPT_CONFIG_NUM:
249 ret = parse_config(optarg);
250 if (ret < 0) {
251 vhost_crypto_usage(prgname);
252 return ret;
253 }
254 break;
255
256 case OPT_ZERO_COPY_NUM:
257 options.zero_copy =
258 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE;
259 break;
260
261 case OPT_POLLING_NUM:
262 options.guest_polling = 1;
263 break;
264
265 default:
266 vhost_crypto_usage(prgname);
267 return -EINVAL;
268 }
269 }
270
271 return 0;
272 }
273
274 static int
new_device(int vid)275 new_device(int vid)
276 {
277 struct vhost_crypto_info *info = NULL;
278 char path[PATH_MAX];
279 uint32_t i, j;
280 int ret;
281
282 ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
283 if (ret) {
284 RTE_LOG(ERR, USER1, "Cannot find matched socket\n");
285 return ret;
286 }
287
288 for (i = 0; i < options.nb_los; i++) {
289 for (j = 0; j < options.los[i].nb_sockets; j++) {
290 if (strcmp(path, options.los[i].socket_files[j]) == 0) {
291 info = options.infos[i];
292 break;
293 }
294 }
295
296 if (info)
297 break;
298 }
299
300 if (!info) {
301 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
302 return -ENOENT;
303 }
304
305 ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool,
306 rte_lcore_to_socket_id(options.los[i].lcore_id));
307 if (ret) {
308 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
309 return ret;
310 }
311
312 ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy);
313 if (ret) {
314 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n",
315 options.zero_copy == 1 ? "enable" : "disable");
316 return ret;
317 }
318
319 info->vids[j] = vid;
320 info->initialized[j] = 1;
321
322 rte_wmb();
323
324 RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path,
325 vid);
326 return 0;
327 }
328
329 static void
destroy_device(int vid)330 destroy_device(int vid)
331 {
332 struct vhost_crypto_info *info = NULL;
333 uint32_t i, j;
334
335 for (i = 0; i < options.nb_los; i++) {
336 for (j = 0; j < options.los[i].nb_sockets; j++) {
337 if (options.infos[i]->vids[j] == vid) {
338 info = options.infos[i];
339 break;
340 }
341 }
342 if (info)
343 break;
344 }
345
346 if (!info) {
347 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
348 return;
349 }
350
351 do {
352
353 } while (info->nb_inflight_ops);
354
355 info->initialized[j] = 0;
356
357 rte_wmb();
358
359 rte_vhost_crypto_free(vid);
360
361 RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid);
362 }
363
364 static const struct rte_vhost_device_ops virtio_crypto_device_ops = {
365 .new_device = new_device,
366 .destroy_device = destroy_device,
367 };
368
369 static int
vhost_crypto_worker(void * arg)370 vhost_crypto_worker(void *arg)
371 {
372 struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
373 struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
374 struct vhost_crypto_info *info = arg;
375 uint16_t nb_callfds;
376 int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
377 uint32_t lcore_id = rte_lcore_id();
378 uint32_t burst_size = MAX_PKT_BURST;
379 uint32_t i, j, k;
380 uint32_t to_fetch, fetched;
381
382 int ret = 0;
383
384 RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);
385
386 for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
387 if (rte_crypto_op_bulk_alloc(info->cop_pool,
388 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
389 burst_size) < burst_size) {
390 RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
391 ret = -1;
392 goto exit;
393 }
394 }
395
396 while (1) {
397 for (i = 0; i < info->nb_vids; i++) {
398 if (unlikely(info->initialized[i] == 0))
399 continue;
400
401 for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
402 to_fetch = RTE_MIN(burst_size,
403 (NB_CRYPTO_DESCRIPTORS -
404 info->nb_inflight_ops));
405 fetched = rte_vhost_crypto_fetch_requests(
406 info->vids[i], j, ops[j],
407 to_fetch);
408 info->nb_inflight_ops +=
409 rte_cryptodev_enqueue_burst(
410 info->cid, info->qid, ops[j],
411 fetched);
412 if (unlikely(rte_crypto_op_bulk_alloc(
413 info->cop_pool,
414 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
415 ops[j], fetched) < fetched)) {
416 RTE_LOG(ERR, USER1, "Failed realloc\n");
417 return -1;
418 }
419
420 fetched = rte_cryptodev_dequeue_burst(
421 info->cid, info->qid,
422 ops_deq[j], RTE_MIN(burst_size,
423 info->nb_inflight_ops));
424 fetched = rte_vhost_crypto_finalize_requests(
425 ops_deq[j], fetched, callfds,
426 &nb_callfds);
427
428 info->nb_inflight_ops -= fetched;
429
430 if (!options.guest_polling) {
431 for (k = 0; k < nb_callfds; k++)
432 eventfd_write(callfds[k],
433 (eventfd_t)1);
434 }
435
436 rte_mempool_put_bulk(info->cop_pool,
437 (void **)ops_deq[j], fetched);
438 }
439 }
440 }
441 exit:
442 return ret;
443 }
444
445 static void
free_resource(void)446 free_resource(void)
447 {
448 uint32_t i, j;
449
450 for (i = 0; i < options.nb_los; i++) {
451 struct lcore_option *lo = &options.los[i];
452 struct vhost_crypto_info *info = options.infos[i];
453
454 if (!info)
455 continue;
456
457 rte_mempool_free(info->cop_pool);
458 rte_mempool_free(info->sess_pool);
459
460 for (j = 0; j < lo->nb_sockets; j++) {
461 rte_vhost_driver_unregister(lo->socket_files[i]);
462 free(lo->socket_files[i]);
463 }
464
465 rte_free(info);
466 }
467
468 memset(&options, 0, sizeof(options));
469
470 /* clean up the EAL */
471 rte_eal_cleanup();
472 }
473
474 int
main(int argc,char * argv[])475 main(int argc, char *argv[])
476 {
477 struct rte_cryptodev_qp_conf qp_conf;
478 struct rte_cryptodev_config config;
479 struct rte_cryptodev_info dev_info;
480 char name[128];
481 uint32_t i, j, lcore;
482 int ret;
483
484 ret = rte_eal_init(argc, argv);
485 if (ret < 0)
486 return -1;
487 argc -= ret;
488 argv += ret;
489
490 ret = vhost_crypto_parse_args(argc, argv);
491 if (ret < 0)
492 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
493
494 for (i = 0; i < options.nb_los; i++) {
495 struct lcore_option *lo = &options.los[i];
496 struct vhost_crypto_info *info;
497
498 info = rte_zmalloc_socket(NULL, sizeof(*info),
499 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(
500 lo->lcore_id));
501 if (!info) {
502 ret = -ENOMEM;
503 goto error_exit;
504 }
505
506 info->cid = lo->cid;
507 info->qid = lo->qid;
508 info->nb_vids = lo->nb_sockets;
509
510 rte_cryptodev_info_get(info->cid, &dev_info);
511 if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) {
512 #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD crypto_aesni_mb
513 #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm
514 if (strstr(dev_info.driver_name,
515 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) ||
516 strstr(dev_info.driver_name,
517 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) {
518 RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n",
519 dev_info.driver_name);
520 ret = -EPERM;
521 goto error_exit;
522 }
523 }
524
525 if (dev_info.max_nb_queue_pairs < info->qid + 1) {
526 RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
527 dev_info.max_nb_queue_pairs);
528 goto error_exit;
529 }
530
531 config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
532 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id);
533 config.ff_disable = RTE_CRYPTODEV_FF_SECURITY;
534
535 ret = rte_cryptodev_configure(info->cid, &config);
536 if (ret < 0) {
537 RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
538 info->cid);
539 goto error_exit;
540 }
541
542 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id);
543 info->sess_pool = rte_cryptodev_sym_session_pool_create(name,
544 SESSION_MAP_ENTRIES,
545 rte_cryptodev_sym_get_private_session_size(
546 info->cid), 0, 0,
547 rte_lcore_to_socket_id(lo->lcore_id));
548
549 if (!info->sess_pool) {
550 RTE_LOG(ERR, USER1, "Failed to create mempool");
551 goto error_exit;
552 }
553
554 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id);
555 info->cop_pool = rte_crypto_op_pool_create(name,
556 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
557 NB_CACHE_OBJS, VHOST_CRYPTO_MAX_IV_LEN,
558 rte_lcore_to_socket_id(lo->lcore_id));
559
560 if (!info->cop_pool) {
561 RTE_LOG(ERR, USER1, "Failed to create crypto pool");
562 ret = -ENOMEM;
563 goto error_exit;
564 }
565
566 options.infos[i] = info;
567
568 qp_conf.nb_descriptors = NB_CRYPTO_DESCRIPTORS;
569 qp_conf.mp_session = info->sess_pool;
570
571 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) {
572 ret = rte_cryptodev_queue_pair_setup(info->cid, j,
573 &qp_conf, rte_lcore_to_socket_id(
574 lo->lcore_id));
575 if (ret < 0) {
576 RTE_LOG(ERR, USER1, "Failed to configure qp\n");
577 goto error_exit;
578 }
579 }
580 }
581
582 for (i = 0; i < options.nb_los; i++) {
583 struct lcore_option *lo = &options.los[i];
584 struct vhost_crypto_info *info = options.infos[i];
585
586 ret = rte_cryptodev_start(lo->cid);
587 if (ret < 0) {
588 RTE_LOG(ERR, USER1, "Failed to start cryptodev\n");
589 goto error_exit;
590 }
591
592 if (rte_eal_remote_launch(vhost_crypto_worker, info,
593 lo->lcore_id) < 0) {
594 RTE_LOG(ERR, USER1, "Failed to start worker lcore");
595 goto error_exit;
596 }
597
598 for (j = 0; j < lo->nb_sockets; j++) {
599 ret = rte_vhost_driver_register(lo->socket_files[j],
600 RTE_VHOST_USER_ASYNC_COPY);
601 if (ret < 0) {
602 RTE_LOG(ERR, USER1, "socket %s already exists\n",
603 lo->socket_files[j]);
604 goto error_exit;
605 }
606
607 rte_vhost_driver_callback_register(lo->socket_files[j],
608 &virtio_crypto_device_ops);
609
610 ret = rte_vhost_crypto_driver_start(
611 lo->socket_files[j]);
612 if (ret < 0) {
613 RTE_LOG(ERR, USER1, "failed to start vhost.\n");
614 goto error_exit;
615 }
616 }
617 }
618
619 RTE_LCORE_FOREACH(lcore)
620 rte_eal_wait_lcore(lcore);
621
622 free_resource();
623
624 return 0;
625
626 error_exit:
627
628 free_resource();
629
630 return -1;
631 }
632