1f5188211SFan Zhang /* SPDX-License-Identifier: BSD-3-Clause
2f5188211SFan Zhang * Copyright(c) 2017-2018 Intel Corporation
3f5188211SFan Zhang */
4f5188211SFan Zhang
5f5188211SFan Zhang #include <stdio.h>
6f5188211SFan Zhang #include <stdlib.h>
7e1d90eaaSFan Zhang #include <string.h>
8f5188211SFan Zhang #include <unistd.h>
9f5188211SFan Zhang #include <stdbool.h>
10f5188211SFan Zhang #include <assert.h>
11f5188211SFan Zhang #include <getopt.h>
12f5188211SFan Zhang
13f5188211SFan Zhang #include <rte_malloc.h>
14f5188211SFan Zhang #include <rte_cycles.h>
15f5188211SFan Zhang #include <rte_vhost.h>
16f5188211SFan Zhang #include <rte_cryptodev.h>
17f5188211SFan Zhang #include <rte_vhost_crypto.h>
18709521f4SFan Zhang #include <rte_string_fns.h>
19f5188211SFan Zhang
20f5188211SFan Zhang #include <cmdline_rdline.h>
21f5188211SFan Zhang #include <cmdline_parse.h>
22f5188211SFan Zhang #include <cmdline_parse_string.h>
23f5188211SFan Zhang #include <cmdline.h>
24f5188211SFan Zhang
25f5188211SFan Zhang #define NB_VIRTIO_QUEUES (1)
26f5188211SFan Zhang #define MAX_PKT_BURST (64)
27f5188211SFan Zhang #define MAX_IV_LEN (32)
28f5188211SFan Zhang #define NB_MEMPOOL_OBJS (8192)
29f5188211SFan Zhang #define NB_CRYPTO_DESCRIPTORS (4096)
30f5188211SFan Zhang #define NB_CACHE_OBJS (128)
31f5188211SFan Zhang #define SESSION_MAP_ENTRIES (1024)
32f5188211SFan Zhang #define REFRESH_TIME_SEC (3)
33f5188211SFan Zhang
34709521f4SFan Zhang #define MAX_NB_SOCKETS (4)
35709521f4SFan Zhang #define MAX_NB_WORKER_CORES (16)
36f5188211SFan Zhang
37709521f4SFan Zhang struct lcore_option {
38709521f4SFan Zhang uint32_t lcore_id;
39f5188211SFan Zhang char *socket_files[MAX_NB_SOCKETS];
40f5188211SFan Zhang uint32_t nb_sockets;
41f5188211SFan Zhang uint8_t cid;
42f5188211SFan Zhang uint16_t qid;
43709521f4SFan Zhang };
44709521f4SFan Zhang
45*7e06c0deSTyler Retzlaff struct __rte_cache_aligned vhost_crypto_info {
46709521f4SFan Zhang int vids[MAX_NB_SOCKETS];
47709521f4SFan Zhang uint32_t nb_vids;
48709521f4SFan Zhang struct rte_mempool *sess_pool;
49709521f4SFan Zhang struct rte_mempool *cop_pool;
50709521f4SFan Zhang uint8_t cid;
51709521f4SFan Zhang uint32_t qid;
52709521f4SFan Zhang uint32_t nb_inflight_ops;
53709521f4SFan Zhang volatile uint32_t initialized[MAX_NB_SOCKETS];
54*7e06c0deSTyler Retzlaff };
55709521f4SFan Zhang
56709521f4SFan Zhang struct vhost_crypto_options {
57709521f4SFan Zhang struct lcore_option los[MAX_NB_WORKER_CORES];
58709521f4SFan Zhang struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES];
59709521f4SFan Zhang uint32_t nb_los;
60f5188211SFan Zhang uint32_t zero_copy;
61f5188211SFan Zhang uint32_t guest_polling;
62f5188211SFan Zhang } options;
63f5188211SFan Zhang
64270054edSIbtisam Tariq enum {
65270054edSIbtisam Tariq #define OPT_CONFIG "config"
66270054edSIbtisam Tariq OPT_CONFIG_NUM = 256,
67270054edSIbtisam Tariq #define OPT_SOCKET_FILE "socket-file"
68270054edSIbtisam Tariq OPT_SOCKET_FILE_NUM,
69270054edSIbtisam Tariq #define OPT_ZERO_COPY "zero-copy"
70270054edSIbtisam Tariq OPT_ZERO_COPY_NUM,
71270054edSIbtisam Tariq #define OPT_POLLING "guest-polling"
72270054edSIbtisam Tariq OPT_POLLING_NUM,
73270054edSIbtisam Tariq };
74f5188211SFan Zhang
75709521f4SFan Zhang #define NB_SOCKET_FIELDS (2)
76709521f4SFan Zhang
77709521f4SFan Zhang static uint32_t
find_lo(uint32_t lcore_id)78709521f4SFan Zhang find_lo(uint32_t lcore_id)
79709521f4SFan Zhang {
80709521f4SFan Zhang uint32_t i;
81709521f4SFan Zhang
82709521f4SFan Zhang for (i = 0; i < options.nb_los; i++)
83709521f4SFan Zhang if (options.los[i].lcore_id == lcore_id)
84709521f4SFan Zhang return i;
85709521f4SFan Zhang
86709521f4SFan Zhang return UINT32_MAX;
87709521f4SFan Zhang }
88f5188211SFan Zhang
89f5188211SFan Zhang /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */
90f5188211SFan Zhang static int
parse_socket_arg(char * arg)91f5188211SFan Zhang parse_socket_arg(char *arg)
92f5188211SFan Zhang {
93709521f4SFan Zhang uint32_t nb_sockets;
94709521f4SFan Zhang uint32_t lcore_id;
95709521f4SFan Zhang char *str_fld[NB_SOCKET_FIELDS];
96709521f4SFan Zhang struct lcore_option *lo;
97709521f4SFan Zhang uint32_t idx;
98709521f4SFan Zhang char *end;
99709521f4SFan Zhang
100709521f4SFan Zhang if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') !=
101709521f4SFan Zhang NB_SOCKET_FIELDS) {
102709521f4SFan Zhang RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg);
103709521f4SFan Zhang return -EINVAL;
104709521f4SFan Zhang }
105709521f4SFan Zhang
106709521f4SFan Zhang errno = 0;
107709521f4SFan Zhang lcore_id = strtoul(str_fld[0], &end, 0);
108709521f4SFan Zhang if (errno != 0 || end == str_fld[0] || lcore_id > 255)
109709521f4SFan Zhang return -EINVAL;
110709521f4SFan Zhang
111709521f4SFan Zhang idx = find_lo(lcore_id);
112709521f4SFan Zhang if (idx == UINT32_MAX) {
113709521f4SFan Zhang if (options.nb_los == MAX_NB_WORKER_CORES)
114709521f4SFan Zhang return -ENOMEM;
115709521f4SFan Zhang lo = &options.los[options.nb_los];
116709521f4SFan Zhang lo->lcore_id = lcore_id;
117709521f4SFan Zhang options.nb_los++;
118709521f4SFan Zhang } else
119709521f4SFan Zhang lo = &options.los[idx];
120709521f4SFan Zhang
121709521f4SFan Zhang nb_sockets = lo->nb_sockets;
122f5188211SFan Zhang
123f5188211SFan Zhang if (nb_sockets >= MAX_NB_SOCKETS) {
124f5188211SFan Zhang RTE_LOG(ERR, USER1, "Too many socket files!\n");
125f5188211SFan Zhang return -ENOMEM;
126f5188211SFan Zhang }
127f5188211SFan Zhang
128709521f4SFan Zhang lo->socket_files[nb_sockets] = strdup(str_fld[1]);
129709521f4SFan Zhang if (!lo->socket_files[nb_sockets]) {
130f5188211SFan Zhang RTE_LOG(ERR, USER1, "Insufficient memory\n");
131f5188211SFan Zhang return -ENOMEM;
132f5188211SFan Zhang }
133f5188211SFan Zhang
134709521f4SFan Zhang lo->nb_sockets++;
135f5188211SFan Zhang
136f5188211SFan Zhang return 0;
137f5188211SFan Zhang }
138f5188211SFan Zhang
139f5188211SFan Zhang static int
parse_config(char * q_arg)140709521f4SFan Zhang parse_config(char *q_arg)
141f5188211SFan Zhang {
142709521f4SFan Zhang struct lcore_option *lo;
143709521f4SFan Zhang char s[256];
144709521f4SFan Zhang const char *p, *p0 = q_arg;
145709521f4SFan Zhang char *end;
146709521f4SFan Zhang enum fieldnames {
147709521f4SFan Zhang FLD_LCORE = 0,
148709521f4SFan Zhang FLD_CID,
149709521f4SFan Zhang FLD_QID,
150709521f4SFan Zhang _NUM_FLD
151709521f4SFan Zhang };
152709521f4SFan Zhang uint32_t flds[_NUM_FLD];
153709521f4SFan Zhang char *str_fld[_NUM_FLD];
154709521f4SFan Zhang uint32_t i;
155709521f4SFan Zhang uint32_t size;
156f5188211SFan Zhang
157709521f4SFan Zhang while ((p = strchr(p0, '(')) != NULL) {
158709521f4SFan Zhang ++p;
159709521f4SFan Zhang p0 = strchr(p, ')');
160709521f4SFan Zhang if (p0 == NULL)
161f5188211SFan Zhang return -1;
162f5188211SFan Zhang
163709521f4SFan Zhang size = p0 - p;
164709521f4SFan Zhang if (size >= sizeof(s))
165f5188211SFan Zhang return -1;
166709521f4SFan Zhang
167709521f4SFan Zhang snprintf(s, sizeof(s), "%.*s", size, p);
168709521f4SFan Zhang if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
169709521f4SFan Zhang _NUM_FLD)
170709521f4SFan Zhang return -1;
171709521f4SFan Zhang for (i = 0; i < _NUM_FLD; i++) {
172709521f4SFan Zhang errno = 0;
173709521f4SFan Zhang flds[i] = strtoul(str_fld[i], &end, 0);
174709521f4SFan Zhang if (errno != 0 || end == str_fld[i] || flds[i] > 255)
175709521f4SFan Zhang return -EINVAL;
176f5188211SFan Zhang }
177f5188211SFan Zhang
178709521f4SFan Zhang if (flds[FLD_LCORE] > RTE_MAX_LCORE)
179709521f4SFan Zhang return -EINVAL;
180709521f4SFan Zhang
181709521f4SFan Zhang i = find_lo(flds[FLD_LCORE]);
182709521f4SFan Zhang if (i == UINT32_MAX) {
183709521f4SFan Zhang if (options.nb_los == MAX_NB_WORKER_CORES)
184709521f4SFan Zhang return -ENOMEM;
185709521f4SFan Zhang lo = &options.los[options.nb_los];
186709521f4SFan Zhang options.nb_los++;
187709521f4SFan Zhang } else
188709521f4SFan Zhang lo = &options.los[i];
189709521f4SFan Zhang
190709521f4SFan Zhang lo->lcore_id = flds[FLD_LCORE];
191709521f4SFan Zhang lo->cid = flds[FLD_CID];
192709521f4SFan Zhang lo->qid = flds[FLD_QID];
193709521f4SFan Zhang }
194f5188211SFan Zhang
195f5188211SFan Zhang return 0;
196f5188211SFan Zhang }
197f5188211SFan Zhang
198f5188211SFan Zhang static void
vhost_crypto_usage(const char * prgname)199f5188211SFan Zhang vhost_crypto_usage(const char *prgname)
200f5188211SFan Zhang {
201f5188211SFan Zhang printf("%s [EAL options] --\n"
202709521f4SFan Zhang " --%s <lcore>,SOCKET-FILE-PATH\n"
203854dbee2SIbtisam Tariq " --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]\n"
204f5188211SFan Zhang " --%s: zero copy\n"
205f5188211SFan Zhang " --%s: guest polling\n",
206270054edSIbtisam Tariq prgname, OPT_SOCKET_FILE, OPT_CONFIG,
207270054edSIbtisam Tariq OPT_ZERO_COPY, OPT_POLLING);
208f5188211SFan Zhang }
209f5188211SFan Zhang
210f5188211SFan Zhang static int
vhost_crypto_parse_args(int argc,char ** argv)211f5188211SFan Zhang vhost_crypto_parse_args(int argc, char **argv)
212f5188211SFan Zhang {
213f5188211SFan Zhang int opt, ret;
214f5188211SFan Zhang char *prgname = argv[0];
215f5188211SFan Zhang char **argvopt;
216f5188211SFan Zhang int option_index;
217f5188211SFan Zhang struct option lgopts[] = {
218270054edSIbtisam Tariq {OPT_SOCKET_FILE, required_argument,
219270054edSIbtisam Tariq NULL, OPT_SOCKET_FILE_NUM},
220270054edSIbtisam Tariq {OPT_CONFIG, required_argument,
221270054edSIbtisam Tariq NULL, OPT_CONFIG_NUM},
222270054edSIbtisam Tariq {OPT_ZERO_COPY, no_argument,
223270054edSIbtisam Tariq NULL, OPT_ZERO_COPY_NUM},
224270054edSIbtisam Tariq {OPT_POLLING, no_argument,
225270054edSIbtisam Tariq NULL, OPT_POLLING_NUM},
226f5188211SFan Zhang {NULL, 0, 0, 0}
227f5188211SFan Zhang };
228f5188211SFan Zhang
229f5188211SFan Zhang argvopt = argv;
230f5188211SFan Zhang
231dd0946f9SIbtisam Tariq while ((opt = getopt_long(argc, argvopt, "",
232f5188211SFan Zhang lgopts, &option_index)) != EOF) {
233f5188211SFan Zhang
234270054edSIbtisam Tariq if (opt == '?') {
235270054edSIbtisam Tariq vhost_crypto_usage(prgname);
236270054edSIbtisam Tariq return -1;
237270054edSIbtisam Tariq }
238270054edSIbtisam Tariq
239f5188211SFan Zhang switch (opt) {
240270054edSIbtisam Tariq case OPT_SOCKET_FILE_NUM:
241f5188211SFan Zhang ret = parse_socket_arg(optarg);
242f5188211SFan Zhang if (ret < 0) {
243f5188211SFan Zhang vhost_crypto_usage(prgname);
244f5188211SFan Zhang return ret;
245f5188211SFan Zhang }
246270054edSIbtisam Tariq break;
247270054edSIbtisam Tariq
248270054edSIbtisam Tariq case OPT_CONFIG_NUM:
249709521f4SFan Zhang ret = parse_config(optarg);
250f5188211SFan Zhang if (ret < 0) {
251f5188211SFan Zhang vhost_crypto_usage(prgname);
252f5188211SFan Zhang return ret;
253f5188211SFan Zhang }
254270054edSIbtisam Tariq break;
255270054edSIbtisam Tariq
256270054edSIbtisam Tariq case OPT_ZERO_COPY_NUM:
257f5188211SFan Zhang options.zero_copy =
258f5188211SFan Zhang RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE;
259270054edSIbtisam Tariq break;
260270054edSIbtisam Tariq
261270054edSIbtisam Tariq case OPT_POLLING_NUM:
262f5188211SFan Zhang options.guest_polling = 1;
263270054edSIbtisam Tariq break;
264270054edSIbtisam Tariq
265270054edSIbtisam Tariq default:
266f5188211SFan Zhang vhost_crypto_usage(prgname);
267f5188211SFan Zhang return -EINVAL;
268f5188211SFan Zhang }
269f5188211SFan Zhang }
270f5188211SFan Zhang
271f5188211SFan Zhang return 0;
272f5188211SFan Zhang }
273f5188211SFan Zhang
274f5188211SFan Zhang static int
new_device(int vid)275f5188211SFan Zhang new_device(int vid)
276f5188211SFan Zhang {
277709521f4SFan Zhang struct vhost_crypto_info *info = NULL;
278f5188211SFan Zhang char path[PATH_MAX];
279709521f4SFan Zhang uint32_t i, j;
280f5188211SFan Zhang int ret;
281f5188211SFan Zhang
282f5188211SFan Zhang ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
283f5188211SFan Zhang if (ret) {
284f5188211SFan Zhang RTE_LOG(ERR, USER1, "Cannot find matched socket\n");
285f5188211SFan Zhang return ret;
286f5188211SFan Zhang }
287f5188211SFan Zhang
288709521f4SFan Zhang for (i = 0; i < options.nb_los; i++) {
289709521f4SFan Zhang for (j = 0; j < options.los[i].nb_sockets; j++) {
290709521f4SFan Zhang if (strcmp(path, options.los[i].socket_files[j]) == 0) {
291709521f4SFan Zhang info = options.infos[i];
292709521f4SFan Zhang break;
293709521f4SFan Zhang }
294709521f4SFan Zhang }
295709521f4SFan Zhang
296709521f4SFan Zhang if (info)
297f5188211SFan Zhang break;
298f5188211SFan Zhang }
299f5188211SFan Zhang
300709521f4SFan Zhang if (!info) {
301f5188211SFan Zhang RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
302f5188211SFan Zhang return -ENOENT;
303f5188211SFan Zhang }
304f5188211SFan Zhang
305709521f4SFan Zhang ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool,
306709521f4SFan Zhang rte_lcore_to_socket_id(options.los[i].lcore_id));
307f5188211SFan Zhang if (ret) {
308f5188211SFan Zhang RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
309f5188211SFan Zhang return ret;
310f5188211SFan Zhang }
311f5188211SFan Zhang
312f5188211SFan Zhang ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy);
313f5188211SFan Zhang if (ret) {
314f5188211SFan Zhang RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n",
315f5188211SFan Zhang options.zero_copy == 1 ? "enable" : "disable");
316f5188211SFan Zhang return ret;
317f5188211SFan Zhang }
318f5188211SFan Zhang
319709521f4SFan Zhang info->vids[j] = vid;
320709521f4SFan Zhang info->initialized[j] = 1;
321f5188211SFan Zhang
322f5188211SFan Zhang rte_wmb();
323f5188211SFan Zhang
324f5188211SFan Zhang RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path,
325f5188211SFan Zhang vid);
326f5188211SFan Zhang return 0;
327f5188211SFan Zhang }
328f5188211SFan Zhang
329f5188211SFan Zhang static void
destroy_device(int vid)330f5188211SFan Zhang destroy_device(int vid)
331f5188211SFan Zhang {
332709521f4SFan Zhang struct vhost_crypto_info *info = NULL;
333709521f4SFan Zhang uint32_t i, j;
334f5188211SFan Zhang
335709521f4SFan Zhang for (i = 0; i < options.nb_los; i++) {
336709521f4SFan Zhang for (j = 0; j < options.los[i].nb_sockets; j++) {
337709521f4SFan Zhang if (options.infos[i]->vids[j] == vid) {
338709521f4SFan Zhang info = options.infos[i];
339709521f4SFan Zhang break;
340709521f4SFan Zhang }
341709521f4SFan Zhang }
342709521f4SFan Zhang if (info)
343f5188211SFan Zhang break;
344f5188211SFan Zhang }
345f5188211SFan Zhang
346709521f4SFan Zhang if (!info) {
347f5188211SFan Zhang RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
348f5188211SFan Zhang return;
349f5188211SFan Zhang }
350f5188211SFan Zhang
351709521f4SFan Zhang do {
352709521f4SFan Zhang
353709521f4SFan Zhang } while (info->nb_inflight_ops);
354709521f4SFan Zhang
355709521f4SFan Zhang info->initialized[j] = 0;
356f5188211SFan Zhang
357f5188211SFan Zhang rte_wmb();
358f5188211SFan Zhang
359f5188211SFan Zhang rte_vhost_crypto_free(vid);
360f5188211SFan Zhang
361f5188211SFan Zhang RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid);
362f5188211SFan Zhang }
363f5188211SFan Zhang
364ab4bb424SMaxime Coquelin static const struct rte_vhost_device_ops virtio_crypto_device_ops = {
365f5188211SFan Zhang .new_device = new_device,
366f5188211SFan Zhang .destroy_device = destroy_device,
367f5188211SFan Zhang };
368f5188211SFan Zhang
369f5188211SFan Zhang static int
vhost_crypto_worker(void * arg)370709521f4SFan Zhang vhost_crypto_worker(void *arg)
371f5188211SFan Zhang {
372f5188211SFan Zhang struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
373f5188211SFan Zhang struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
374709521f4SFan Zhang struct vhost_crypto_info *info = arg;
375f5188211SFan Zhang uint16_t nb_callfds;
376f5188211SFan Zhang int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
377f5188211SFan Zhang uint32_t lcore_id = rte_lcore_id();
378f5188211SFan Zhang uint32_t burst_size = MAX_PKT_BURST;
379f5188211SFan Zhang uint32_t i, j, k;
380f5188211SFan Zhang uint32_t to_fetch, fetched;
381f5188211SFan Zhang
382f5188211SFan Zhang int ret = 0;
383f5188211SFan Zhang
384f5188211SFan Zhang RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);
385f5188211SFan Zhang
386f5188211SFan Zhang for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
387709521f4SFan Zhang if (rte_crypto_op_bulk_alloc(info->cop_pool,
388f5188211SFan Zhang RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
389f5188211SFan Zhang burst_size) < burst_size) {
390f5188211SFan Zhang RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
391f5188211SFan Zhang ret = -1;
392f5188211SFan Zhang goto exit;
393f5188211SFan Zhang }
394f5188211SFan Zhang }
395f5188211SFan Zhang
396f5188211SFan Zhang while (1) {
397709521f4SFan Zhang for (i = 0; i < info->nb_vids; i++) {
398709521f4SFan Zhang if (unlikely(info->initialized[i] == 0))
399f5188211SFan Zhang continue;
400f5188211SFan Zhang
401f5188211SFan Zhang for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
402f5188211SFan Zhang to_fetch = RTE_MIN(burst_size,
403f5188211SFan Zhang (NB_CRYPTO_DESCRIPTORS -
404709521f4SFan Zhang info->nb_inflight_ops));
405f5188211SFan Zhang fetched = rte_vhost_crypto_fetch_requests(
406709521f4SFan Zhang info->vids[i], j, ops[j],
407f5188211SFan Zhang to_fetch);
408709521f4SFan Zhang info->nb_inflight_ops +=
409709521f4SFan Zhang rte_cryptodev_enqueue_burst(
410709521f4SFan Zhang info->cid, info->qid, ops[j],
411f5188211SFan Zhang fetched);
412f5188211SFan Zhang if (unlikely(rte_crypto_op_bulk_alloc(
413709521f4SFan Zhang info->cop_pool,
414f5188211SFan Zhang RTE_CRYPTO_OP_TYPE_SYMMETRIC,
415f5188211SFan Zhang ops[j], fetched) < fetched)) {
416f5188211SFan Zhang RTE_LOG(ERR, USER1, "Failed realloc\n");
417f5188211SFan Zhang return -1;
418f5188211SFan Zhang }
419f5188211SFan Zhang
420f5188211SFan Zhang fetched = rte_cryptodev_dequeue_burst(
421709521f4SFan Zhang info->cid, info->qid,
422f5188211SFan Zhang ops_deq[j], RTE_MIN(burst_size,
423709521f4SFan Zhang info->nb_inflight_ops));
424f5188211SFan Zhang fetched = rte_vhost_crypto_finalize_requests(
425f5188211SFan Zhang ops_deq[j], fetched, callfds,
426f5188211SFan Zhang &nb_callfds);
427f5188211SFan Zhang
428709521f4SFan Zhang info->nb_inflight_ops -= fetched;
429f5188211SFan Zhang
430f5188211SFan Zhang if (!options.guest_polling) {
431f5188211SFan Zhang for (k = 0; k < nb_callfds; k++)
432f5188211SFan Zhang eventfd_write(callfds[k],
433f5188211SFan Zhang (eventfd_t)1);
434f5188211SFan Zhang }
435f5188211SFan Zhang
436709521f4SFan Zhang rte_mempool_put_bulk(info->cop_pool,
437f5188211SFan Zhang (void **)ops_deq[j], fetched);
438f5188211SFan Zhang }
439f5188211SFan Zhang }
440f5188211SFan Zhang }
441f5188211SFan Zhang exit:
442f5188211SFan Zhang return ret;
443f5188211SFan Zhang }
444f5188211SFan Zhang
445f5188211SFan Zhang static void
free_resource(void)446709521f4SFan Zhang free_resource(void)
447f5188211SFan Zhang {
448709521f4SFan Zhang uint32_t i, j;
449f5188211SFan Zhang
450709521f4SFan Zhang for (i = 0; i < options.nb_los; i++) {
451709521f4SFan Zhang struct lcore_option *lo = &options.los[i];
452709521f4SFan Zhang struct vhost_crypto_info *info = options.infos[i];
453709521f4SFan Zhang
454e1d90eaaSFan Zhang if (!info)
455e1d90eaaSFan Zhang continue;
456e1d90eaaSFan Zhang
457709521f4SFan Zhang rte_mempool_free(info->cop_pool);
458709521f4SFan Zhang rte_mempool_free(info->sess_pool);
459709521f4SFan Zhang
460709521f4SFan Zhang for (j = 0; j < lo->nb_sockets; j++) {
461709521f4SFan Zhang rte_vhost_driver_unregister(lo->socket_files[i]);
462709521f4SFan Zhang free(lo->socket_files[i]);
463709521f4SFan Zhang }
464709521f4SFan Zhang
465709521f4SFan Zhang rte_free(info);
466709521f4SFan Zhang }
467709521f4SFan Zhang
468709521f4SFan Zhang memset(&options, 0, sizeof(options));
46910aa3757SChengchang Tang
47010aa3757SChengchang Tang /* clean up the EAL */
47110aa3757SChengchang Tang rte_eal_cleanup();
472f5188211SFan Zhang }
473f5188211SFan Zhang
474f5188211SFan Zhang int
main(int argc,char * argv[])475f5188211SFan Zhang main(int argc, char *argv[])
476f5188211SFan Zhang {
477725d2a7fSFan Zhang struct rte_cryptodev_qp_conf qp_conf;
478f5188211SFan Zhang struct rte_cryptodev_config config;
479f5188211SFan Zhang struct rte_cryptodev_info dev_info;
480f5188211SFan Zhang char name[128];
481709521f4SFan Zhang uint32_t i, j, lcore;
482f5188211SFan Zhang int ret;
483f5188211SFan Zhang
484f5188211SFan Zhang ret = rte_eal_init(argc, argv);
485f5188211SFan Zhang if (ret < 0)
486f5188211SFan Zhang return -1;
487f5188211SFan Zhang argc -= ret;
488f5188211SFan Zhang argv += ret;
489f5188211SFan Zhang
490f5188211SFan Zhang ret = vhost_crypto_parse_args(argc, argv);
491f5188211SFan Zhang if (ret < 0)
492f5188211SFan Zhang rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
493f5188211SFan Zhang
494709521f4SFan Zhang for (i = 0; i < options.nb_los; i++) {
495709521f4SFan Zhang struct lcore_option *lo = &options.los[i];
496709521f4SFan Zhang struct vhost_crypto_info *info;
497f5188211SFan Zhang
498709521f4SFan Zhang info = rte_zmalloc_socket(NULL, sizeof(*info),
499709521f4SFan Zhang RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(
500709521f4SFan Zhang lo->lcore_id));
501709521f4SFan Zhang if (!info) {
502709521f4SFan Zhang ret = -ENOMEM;
503709521f4SFan Zhang goto error_exit;
504709521f4SFan Zhang }
505f5188211SFan Zhang
506709521f4SFan Zhang info->cid = lo->cid;
507709521f4SFan Zhang info->qid = lo->qid;
508709521f4SFan Zhang info->nb_vids = lo->nb_sockets;
509709521f4SFan Zhang
510709521f4SFan Zhang rte_cryptodev_info_get(info->cid, &dev_info);
511e1d90eaaSFan Zhang if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) {
512e1d90eaaSFan Zhang #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD crypto_aesni_mb
513e1d90eaaSFan Zhang #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm
514e1d90eaaSFan Zhang if (strstr(dev_info.driver_name,
515e1d90eaaSFan Zhang RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) ||
516e1d90eaaSFan Zhang strstr(dev_info.driver_name,
51757f9a9fcSFan Zhang RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) {
518e1d90eaaSFan Zhang RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n",
519e1d90eaaSFan Zhang dev_info.driver_name);
520e1d90eaaSFan Zhang ret = -EPERM;
521e1d90eaaSFan Zhang goto error_exit;
522e1d90eaaSFan Zhang }
52357f9a9fcSFan Zhang }
524e1d90eaaSFan Zhang
525709521f4SFan Zhang if (dev_info.max_nb_queue_pairs < info->qid + 1) {
526f5188211SFan Zhang RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
527f5188211SFan Zhang dev_info.max_nb_queue_pairs);
528f5188211SFan Zhang goto error_exit;
529f5188211SFan Zhang }
530f5188211SFan Zhang
531f5188211SFan Zhang config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
532709521f4SFan Zhang config.socket_id = rte_lcore_to_socket_id(lo->lcore_id);
533c9030ae3SAnoob Joseph config.ff_disable = RTE_CRYPTODEV_FF_SECURITY;
534f5188211SFan Zhang
535709521f4SFan Zhang ret = rte_cryptodev_configure(info->cid, &config);
536f5188211SFan Zhang if (ret < 0) {
537f5188211SFan Zhang RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
538709521f4SFan Zhang info->cid);
539f5188211SFan Zhang goto error_exit;
540f5188211SFan Zhang }
541f5188211SFan Zhang
542709521f4SFan Zhang snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id);
543ac5e42daSFan Zhang info->sess_pool = rte_cryptodev_sym_session_pool_create(name,
544ac5e42daSFan Zhang SESSION_MAP_ENTRIES,
545a106fcceSPablo de Lara rte_cryptodev_sym_get_private_session_size(
546bdce2564SAkhil Goyal info->cid), 0, 0,
547bdce2564SAkhil Goyal rte_lcore_to_socket_id(lo->lcore_id));
548bdce2564SAkhil Goyal
549bdce2564SAkhil Goyal if (!info->sess_pool) {
550f5188211SFan Zhang RTE_LOG(ERR, USER1, "Failed to create mempool");
551f5188211SFan Zhang goto error_exit;
552f5188211SFan Zhang }
553f5188211SFan Zhang
554709521f4SFan Zhang snprintf(name, 127, "COPPOOL_%u", lo->lcore_id);
555709521f4SFan Zhang info->cop_pool = rte_crypto_op_pool_create(name,
556f5188211SFan Zhang RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
55757680e34SFan Zhang NB_CACHE_OBJS, VHOST_CRYPTO_MAX_IV_LEN,
558709521f4SFan Zhang rte_lcore_to_socket_id(lo->lcore_id));
559f5188211SFan Zhang
560709521f4SFan Zhang if (!info->cop_pool) {
561709521f4SFan Zhang RTE_LOG(ERR, USER1, "Failed to create crypto pool");
562709521f4SFan Zhang ret = -ENOMEM;
563f5188211SFan Zhang goto error_exit;
564f5188211SFan Zhang }
565f5188211SFan Zhang
566709521f4SFan Zhang options.infos[i] = info;
567f5188211SFan Zhang
568725d2a7fSFan Zhang qp_conf.nb_descriptors = NB_CRYPTO_DESCRIPTORS;
569725d2a7fSFan Zhang qp_conf.mp_session = info->sess_pool;
570725d2a7fSFan Zhang
571709521f4SFan Zhang for (j = 0; j < dev_info.max_nb_queue_pairs; j++) {
572709521f4SFan Zhang ret = rte_cryptodev_queue_pair_setup(info->cid, j,
573709521f4SFan Zhang &qp_conf, rte_lcore_to_socket_id(
574725d2a7fSFan Zhang lo->lcore_id));
575f5188211SFan Zhang if (ret < 0) {
576709521f4SFan Zhang RTE_LOG(ERR, USER1, "Failed to configure qp\n");
577f5188211SFan Zhang goto error_exit;
578f5188211SFan Zhang }
579f5188211SFan Zhang }
580709521f4SFan Zhang }
581f5188211SFan Zhang
582709521f4SFan Zhang for (i = 0; i < options.nb_los; i++) {
583709521f4SFan Zhang struct lcore_option *lo = &options.los[i];
584709521f4SFan Zhang struct vhost_crypto_info *info = options.infos[i];
585709521f4SFan Zhang
586709521f4SFan Zhang ret = rte_cryptodev_start(lo->cid);
587f5188211SFan Zhang if (ret < 0) {
588709521f4SFan Zhang RTE_LOG(ERR, USER1, "Failed to start cryptodev\n");
589f5188211SFan Zhang goto error_exit;
590f5188211SFan Zhang }
591f5188211SFan Zhang
592709521f4SFan Zhang if (rte_eal_remote_launch(vhost_crypto_worker, info,
593709521f4SFan Zhang lo->lcore_id) < 0) {
594f5188211SFan Zhang RTE_LOG(ERR, USER1, "Failed to start worker lcore");
595f5188211SFan Zhang goto error_exit;
596f5188211SFan Zhang }
597f5188211SFan Zhang
598709521f4SFan Zhang for (j = 0; j < lo->nb_sockets; j++) {
599709521f4SFan Zhang ret = rte_vhost_driver_register(lo->socket_files[j],
600c08736b3SMaxime Coquelin RTE_VHOST_USER_ASYNC_COPY);
601709521f4SFan Zhang if (ret < 0) {
602f5188211SFan Zhang RTE_LOG(ERR, USER1, "socket %s already exists\n",
603709521f4SFan Zhang lo->socket_files[j]);
604f5188211SFan Zhang goto error_exit;
605f5188211SFan Zhang }
606f5188211SFan Zhang
607709521f4SFan Zhang rte_vhost_driver_callback_register(lo->socket_files[j],
608f5188211SFan Zhang &virtio_crypto_device_ops);
609f5188211SFan Zhang
610ea1b835aSFan Zhang ret = rte_vhost_crypto_driver_start(
611ea1b835aSFan Zhang lo->socket_files[j]);
612709521f4SFan Zhang if (ret < 0) {
613709521f4SFan Zhang RTE_LOG(ERR, USER1, "failed to start vhost.\n");
614f5188211SFan Zhang goto error_exit;
615f5188211SFan Zhang }
616f5188211SFan Zhang }
617709521f4SFan Zhang }
618f5188211SFan Zhang
619709521f4SFan Zhang RTE_LCORE_FOREACH(lcore)
620709521f4SFan Zhang rte_eal_wait_lcore(lcore);
621f5188211SFan Zhang
622709521f4SFan Zhang free_resource();
623f5188211SFan Zhang
624f5188211SFan Zhang return 0;
625f5188211SFan Zhang
626f5188211SFan Zhang error_exit:
627f5188211SFan Zhang
628709521f4SFan Zhang free_resource();
629f5188211SFan Zhang
630f5188211SFan Zhang return -1;
631f5188211SFan Zhang }
632