xref: /dpdk/lib/port/rte_port_sym_crypto.c (revision daa02b5cddbb8e11b31d41e2bf7bb1ae64dcae2f) 
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2018 Intel Corporation
3  */
4 #include <string.h>
5 
6 #include <rte_common.h>
7 #include <rte_malloc.h>
8 
9 #include "rte_port_sym_crypto.h"
10 
11 /*
12  * Port Crypto Reader
13  */
14 #ifdef RTE_PORT_STATS_COLLECT
15 
16 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) \
17 	(port)->stats.n_pkts_in += (val)
18 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val) \
19 	(port)->stats.n_pkts_drop += (val)
20 
21 #else
22 
23 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val)
24 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val)
25 
26 #endif
27 
28 struct rte_port_sym_crypto_reader {
29 	struct rte_port_in_stats stats;
30 
31 	uint8_t cryptodev_id;
32 	uint16_t queue_id;
33 	struct rte_crypto_op *ops[RTE_PORT_IN_BURST_SIZE_MAX];
34 	rte_port_sym_crypto_reader_callback_fn f_callback;
35 	void *arg_callback;
36 };
37 
38 static void *
39 rte_port_sym_crypto_reader_create(void *params, int socket_id)
40 {
41 	struct rte_port_sym_crypto_reader_params *conf =
42 			params;
43 	struct rte_port_sym_crypto_reader *port;
44 
45 	/* Check input parameters */
46 	if (conf == NULL) {
47 		RTE_LOG(ERR, PORT, "%s: params is NULL\n", __func__);
48 		return NULL;
49 	}
50 
51 	/* Memory allocation */
52 	port = rte_zmalloc_socket("PORT", sizeof(*port),
53 		RTE_CACHE_LINE_SIZE, socket_id);
54 	if (port == NULL) {
55 		RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
56 		return NULL;
57 	}
58 
59 	/* Initialization */
60 	port->cryptodev_id = conf->cryptodev_id;
61 	port->queue_id = conf->queue_id;
62 	port->f_callback = conf->f_callback;
63 	port->arg_callback = conf->arg_callback;
64 
65 	return port;
66 }
67 
68 static int
69 rte_port_sym_crypto_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts)
70 {
71 	struct rte_port_sym_crypto_reader *p =
72 			port;
73 	uint16_t rx_ops_cnt, i, n = 0;
74 
75 	rx_ops_cnt = rte_cryptodev_dequeue_burst(p->cryptodev_id, p->queue_id,
76 			p->ops, n_pkts);
77 
78 	for (i = 0; i < rx_ops_cnt; i++) {
79 		struct rte_crypto_op *op = p->ops[i];
80 
81 		/** Drop failed pkts */
82 		if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) {
83 			rte_pktmbuf_free(op->sym->m_src);
84 			continue;
85 		}
86 
87 		pkts[n++] = op->sym->m_src;
88 	}
89 
90 	if (p->f_callback)
91 		(*p->f_callback)(pkts, n, p->arg_callback);
92 
93 	RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(p, n);
94 	RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(p, rx_ops_cnt - n);
95 
96 	return n;
97 }
98 
99 static int
100 rte_port_sym_crypto_reader_free(void *port)
101 {
102 	if (port == NULL) {
103 		RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__);
104 		return -EINVAL;
105 	}
106 
107 	rte_free(port);
108 
109 	return 0;
110 }
111 
112 static int rte_port_sym_crypto_reader_stats_read(void *port,
113 	struct rte_port_in_stats *stats, int clear)
114 {
115 	struct rte_port_sym_crypto_reader *p =
116 			port;
117 
118 	if (stats != NULL)
119 		memcpy(stats, &p->stats, sizeof(p->stats));
120 
121 	if (clear)
122 		memset(&p->stats, 0, sizeof(p->stats));
123 
124 	return 0;
125 }
126 
127 /*
128  * Port crypto Writer
129  */
130 #ifdef RTE_PORT_STATS_COLLECT
131 
132 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) \
133 	(port)->stats.n_pkts_in += (val)
134 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val) \
135 	(port)->stats.n_pkts_drop += (val)
136 
137 #else
138 
139 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val)
140 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val)
141 
142 #endif
143 
144 struct rte_port_sym_crypto_writer {
145 	struct rte_port_out_stats stats;
146 
147 	struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX];
148 
149 	uint32_t tx_burst_sz;
150 	uint32_t tx_buf_count;
151 	uint64_t bsz_mask;
152 
153 	uint8_t cryptodev_id;
154 	uint16_t queue_id;
155 	uint16_t crypto_op_offset;
156 };
157 
158 static void *
159 rte_port_sym_crypto_writer_create(void *params, int socket_id)
160 {
161 	struct rte_port_sym_crypto_writer_params *conf =
162 			params;
163 	struct rte_port_sym_crypto_writer *port;
164 
165 	/* Check input parameters */
166 	if ((conf == NULL) ||
167 		(conf->tx_burst_sz == 0) ||
168 		(conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) ||
169 		(!rte_is_power_of_2(conf->tx_burst_sz))) {
170 		RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
171 		return NULL;
172 	}
173 
174 	/* Memory allocation */
175 	port = rte_zmalloc_socket("PORT", sizeof(*port),
176 		RTE_CACHE_LINE_SIZE, socket_id);
177 	if (port == NULL) {
178 		RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
179 		return NULL;
180 	}
181 
182 	/* Initialization */
183 	port->tx_burst_sz = conf->tx_burst_sz;
184 	port->tx_buf_count = 0;
185 	port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1);
186 
187 	port->cryptodev_id = conf->cryptodev_id;
188 	port->queue_id = conf->queue_id;
189 	port->crypto_op_offset = conf->crypto_op_offset;
190 
191 	return port;
192 }
193 
194 static inline void
195 send_burst(struct rte_port_sym_crypto_writer *p)
196 {
197 	uint32_t nb_tx;
198 
199 	nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id,
200 			p->tx_buf, p->tx_buf_count);
201 
202 	RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count -
203 			nb_tx);
204 	for (; nb_tx < p->tx_buf_count; nb_tx++)
205 		rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src);
206 
207 	p->tx_buf_count = 0;
208 }
209 
210 static int
211 rte_port_sym_crypto_writer_tx(void *port, struct rte_mbuf *pkt)
212 {
213 	struct rte_port_sym_crypto_writer *p =
214 			port;
215 
216 	p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
217 			RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset);
218 	RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1);
219 	if (p->tx_buf_count >= p->tx_burst_sz)
220 		send_burst(p);
221 
222 	return 0;
223 }
224 
225 static int
226 rte_port_sym_crypto_writer_tx_bulk(void *port,
227 	struct rte_mbuf **pkts,
228 	uint64_t pkts_mask)
229 {
230 	struct rte_port_sym_crypto_writer *p =
231 			port;
232 	uint64_t bsz_mask = p->bsz_mask;
233 	uint32_t tx_buf_count = p->tx_buf_count;
234 	uint64_t expr = (pkts_mask & (pkts_mask + 1)) |
235 					((pkts_mask & bsz_mask) ^ bsz_mask);
236 
237 	if (expr == 0) {
238 		uint64_t n_pkts = __builtin_popcountll(pkts_mask);
239 		uint32_t i;
240 
241 		RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, n_pkts);
242 
243 		for (i = 0; i < n_pkts; i++)
244 			p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
245 					RTE_MBUF_METADATA_UINT8_PTR(pkts[i],
246 							p->crypto_op_offset);
247 
248 		if (p->tx_buf_count >= p->tx_burst_sz)
249 			send_burst(p);
250 	} else {
251 		for (; pkts_mask;) {
252 			uint32_t pkt_index = __builtin_ctzll(pkts_mask);
253 			uint64_t pkt_mask = 1LLU << pkt_index;
254 			struct rte_mbuf *pkt = pkts[pkt_index];
255 
256 			p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *)
257 					RTE_MBUF_METADATA_UINT8_PTR(pkt,
258 							p->crypto_op_offset);
259 
260 			RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1);
261 			pkts_mask &= ~pkt_mask;
262 		}
263 
264 		p->tx_buf_count = tx_buf_count;
265 		if (tx_buf_count >= p->tx_burst_sz)
266 			send_burst(p);
267 	}
268 
269 	return 0;
270 }
271 
272 static int
273 rte_port_sym_crypto_writer_flush(void *port)
274 {
275 	struct rte_port_sym_crypto_writer *p =
276 			port;
277 
278 	if (p->tx_buf_count > 0)
279 		send_burst(p);
280 
281 	return 0;
282 }
283 
284 static int
285 rte_port_sym_crypto_writer_free(void *port)
286 {
287 	if (port == NULL) {
288 		RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__);
289 		return -EINVAL;
290 	}
291 
292 	rte_port_sym_crypto_writer_flush(port);
293 	rte_free(port);
294 
295 	return 0;
296 }
297 
298 static int rte_port_sym_crypto_writer_stats_read(void *port,
299 	struct rte_port_out_stats *stats, int clear)
300 {
301 	struct rte_port_sym_crypto_writer *p =
302 			port;
303 
304 	if (stats != NULL)
305 		memcpy(stats, &p->stats, sizeof(p->stats));
306 
307 	if (clear)
308 		memset(&p->stats, 0, sizeof(p->stats));
309 
310 	return 0;
311 }
312 
313 /*
314  * Port crypto Writer Nodrop
315  */
316 #ifdef RTE_PORT_STATS_COLLECT
317 
318 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) \
319 	(port)->stats.n_pkts_in += (val)
320 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) \
321 	(port)->stats.n_pkts_drop += (val)
322 
323 #else
324 
325 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val)
326 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val)
327 
328 #endif
329 
330 struct rte_port_sym_crypto_writer_nodrop {
331 	struct rte_port_out_stats stats;
332 
333 	struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX];
334 	uint32_t tx_burst_sz;
335 	uint32_t tx_buf_count;
336 	uint64_t bsz_mask;
337 	uint64_t n_retries;
338 
339 	uint8_t cryptodev_id;
340 	uint16_t queue_id;
341 	uint16_t crypto_op_offset;
342 };
343 
344 static void *
345 rte_port_sym_crypto_writer_nodrop_create(void *params, int socket_id)
346 {
347 	struct rte_port_sym_crypto_writer_nodrop_params *conf =
348 		params;
349 	struct rte_port_sym_crypto_writer_nodrop *port;
350 
351 	/* Check input parameters */
352 	if ((conf == NULL) ||
353 		(conf->tx_burst_sz == 0) ||
354 		(conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) ||
355 		(!rte_is_power_of_2(conf->tx_burst_sz))) {
356 		RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
357 		return NULL;
358 	}
359 
360 	/* Memory allocation */
361 	port = rte_zmalloc_socket("PORT", sizeof(*port),
362 		RTE_CACHE_LINE_SIZE, socket_id);
363 	if (port == NULL) {
364 		RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
365 		return NULL;
366 	}
367 
368 	/* Initialization */
369 	port->cryptodev_id = conf->cryptodev_id;
370 	port->queue_id = conf->queue_id;
371 	port->crypto_op_offset = conf->crypto_op_offset;
372 	port->tx_burst_sz = conf->tx_burst_sz;
373 	port->tx_buf_count = 0;
374 	port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1);
375 
376 	/*
377 	 * When n_retries is 0 it means that we should wait for every packet to
378 	 * send no matter how many retries should it take. To limit number of
379 	 * branches in fast path, we use UINT64_MAX instead of branching.
380 	 */
381 	port->n_retries = (conf->n_retries == 0) ? UINT64_MAX : conf->n_retries;
382 
383 	return port;
384 }
385 
386 static inline void
387 send_burst_nodrop(struct rte_port_sym_crypto_writer_nodrop *p)
388 {
389 	uint32_t nb_tx = 0, i;
390 
391 	nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id,
392 			p->tx_buf, p->tx_buf_count);
393 
394 	/* We sent all the packets in a first try */
395 	if (nb_tx >= p->tx_buf_count) {
396 		p->tx_buf_count = 0;
397 		return;
398 	}
399 
400 	for (i = 0; i < p->n_retries; i++) {
401 		nb_tx += rte_cryptodev_enqueue_burst(p->cryptodev_id,
402 				p->queue_id, p->tx_buf + nb_tx,
403 				p->tx_buf_count - nb_tx);
404 
405 		/* We sent all the packets in more than one try */
406 		if (nb_tx >= p->tx_buf_count) {
407 			p->tx_buf_count = 0;
408 			return;
409 		}
410 	}
411 
412 	/* We didn't send the packets in maximum allowed attempts */
413 	RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(p,
414 			p->tx_buf_count - nb_tx);
415 	for ( ; nb_tx < p->tx_buf_count; nb_tx++)
416 		rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src);
417 
418 	p->tx_buf_count = 0;
419 }
420 
421 static int
422 rte_port_sym_crypto_writer_nodrop_tx(void *port, struct rte_mbuf *pkt)
423 {
424 	struct rte_port_sym_crypto_writer_nodrop *p =
425 			port;
426 
427 	p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
428 			RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset);
429 	RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1);
430 	if (p->tx_buf_count >= p->tx_burst_sz)
431 		send_burst_nodrop(p);
432 
433 	return 0;
434 }
435 
436 static int
437 rte_port_sym_crypto_writer_nodrop_tx_bulk(void *port,
438 	struct rte_mbuf **pkts,
439 	uint64_t pkts_mask)
440 {
441 	struct rte_port_sym_crypto_writer_nodrop *p =
442 			port;
443 
444 	uint64_t bsz_mask = p->bsz_mask;
445 	uint32_t tx_buf_count = p->tx_buf_count;
446 	uint64_t expr = (pkts_mask & (pkts_mask + 1)) |
447 					((pkts_mask & bsz_mask) ^ bsz_mask);
448 
449 	if (expr == 0) {
450 		uint64_t n_pkts = __builtin_popcountll(pkts_mask);
451 		uint32_t i;
452 
453 		RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts);
454 
455 		for (i = 0; i < n_pkts; i++)
456 			p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
457 					RTE_MBUF_METADATA_UINT8_PTR(pkts[i],
458 							p->crypto_op_offset);
459 
460 		if (p->tx_buf_count >= p->tx_burst_sz)
461 			send_burst_nodrop(p);
462 	} else {
463 		for ( ; pkts_mask; ) {
464 			uint32_t pkt_index = __builtin_ctzll(pkts_mask);
465 			uint64_t pkt_mask = 1LLU << pkt_index;
466 			struct rte_mbuf *pkt = pkts[pkt_index];
467 
468 			p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *)
469 					RTE_MBUF_METADATA_UINT8_PTR(pkt,
470 							p->crypto_op_offset);
471 			RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p,
472 					1);
473 			pkts_mask &= ~pkt_mask;
474 		}
475 
476 		p->tx_buf_count = tx_buf_count;
477 		if (tx_buf_count >= p->tx_burst_sz)
478 			send_burst_nodrop(p);
479 	}
480 
481 	return 0;
482 }
483 
484 static int
485 rte_port_sym_crypto_writer_nodrop_flush(void *port)
486 {
487 	struct rte_port_sym_crypto_writer_nodrop *p =
488 		port;
489 
490 	if (p->tx_buf_count > 0)
491 		send_burst_nodrop(p);
492 
493 	return 0;
494 }
495 
496 static int
497 rte_port_sym_crypto_writer_nodrop_free(void *port)
498 {
499 	if (port == NULL) {
500 		RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__);
501 		return -EINVAL;
502 	}
503 
504 	rte_port_sym_crypto_writer_nodrop_flush(port);
505 	rte_free(port);
506 
507 	return 0;
508 }
509 
510 static int rte_port_sym_crypto_writer_nodrop_stats_read(void *port,
511 	struct rte_port_out_stats *stats, int clear)
512 {
513 	struct rte_port_sym_crypto_writer_nodrop *p =
514 			port;
515 
516 	if (stats != NULL)
517 		memcpy(stats, &p->stats, sizeof(p->stats));
518 
519 	if (clear)
520 		memset(&p->stats, 0, sizeof(p->stats));
521 
522 	return 0;
523 }
524 
525 
526 /*
527  * Summary of port operations
528  */
529 struct rte_port_in_ops rte_port_sym_crypto_reader_ops = {
530 	.f_create = rte_port_sym_crypto_reader_create,
531 	.f_free = rte_port_sym_crypto_reader_free,
532 	.f_rx = rte_port_sym_crypto_reader_rx,
533 	.f_stats = rte_port_sym_crypto_reader_stats_read,
534 };
535 
536 struct rte_port_out_ops rte_port_sym_crypto_writer_ops = {
537 	.f_create = rte_port_sym_crypto_writer_create,
538 	.f_free = rte_port_sym_crypto_writer_free,
539 	.f_tx = rte_port_sym_crypto_writer_tx,
540 	.f_tx_bulk = rte_port_sym_crypto_writer_tx_bulk,
541 	.f_flush = rte_port_sym_crypto_writer_flush,
542 	.f_stats = rte_port_sym_crypto_writer_stats_read,
543 };
544 
545 struct rte_port_out_ops rte_port_sym_crypto_writer_nodrop_ops = {
546 	.f_create = rte_port_sym_crypto_writer_nodrop_create,
547 	.f_free = rte_port_sym_crypto_writer_nodrop_free,
548 	.f_tx = rte_port_sym_crypto_writer_nodrop_tx,
549 	.f_tx_bulk = rte_port_sym_crypto_writer_nodrop_tx_bulk,
550 	.f_flush = rte_port_sym_crypto_writer_nodrop_flush,
551 	.f_stats = rte_port_sym_crypto_writer_nodrop_stats_read,
552 };
553