xref: /dpdk/drivers/crypto/cnxk/cn10k_ipsec.c (revision e9fd1ebf981f361844aea9ec94e17f4bda5e1479)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(C) 2021 Marvell.
3  */
4 
5 #include <cryptodev_pmd.h>
6 #include <rte_esp.h>
7 #include <rte_ip.h>
8 #include <rte_malloc.h>
9 #include <rte_security.h>
10 #include <rte_security_driver.h>
11 #include <rte_udp.h>
12 
13 #include "cn10k_cryptodev_ops.h"
14 #include "cn10k_cryptodev_sec.h"
15 #include "cn10k_ipsec.h"
16 #include "cnxk_cryptodev.h"
17 #include "cnxk_cryptodev_ops.h"
18 #include "cnxk_ipsec.h"
19 #include "cnxk_security.h"
20 
21 #include "roc_api.h"
22 
23 static int
24 cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
25 			   struct rte_security_ipsec_xform *ipsec_xfrm,
26 			   struct rte_crypto_sym_xform *crypto_xfrm,
27 			   struct cn10k_sec_session *sec_sess)
28 {
29 	union roc_ot_ipsec_outb_param1 param1;
30 	struct roc_ot_ipsec_outb_sa *sa_dptr;
31 	struct cnxk_ipsec_outb_rlens rlens;
32 	struct cn10k_ipsec_sa *sa;
33 	union cpt_inst_w4 inst_w4;
34 	void *out_sa;
35 	int ret = 0;
36 
37 	sa = &sec_sess->sa;
38 	out_sa = &sa->out_sa;
39 
40 	/* Allocate memory to be used as dptr for CPT ucode WRITE_SA op */
41 	sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_outb_sa), 8);
42 	if (sa_dptr == NULL) {
43 		plt_err("Couldn't allocate memory for SA dptr");
44 		return -ENOMEM;
45 	}
46 
47 	/* Translate security parameters to SA */
48 	ret = cnxk_ot_ipsec_outb_sa_fill(sa_dptr, ipsec_xfrm, crypto_xfrm);
49 	if (ret) {
50 		plt_err("Could not fill outbound session parameters");
51 		goto sa_dptr_free;
52 	}
53 
54 	sec_sess->inst.w7 = cpt_inst_w7_get(roc_cpt, out_sa);
55 
56 #ifdef LA_IPSEC_DEBUG
57 	/* Use IV from application in debug mode */
58 	if (ipsec_xfrm->options.iv_gen_disable == 1) {
59 		sa_dptr->w2.s.iv_src = ROC_IE_OT_SA_IV_SRC_FROM_SA;
60 		if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
61 			sec_sess->iv_offset = crypto_xfrm->aead.iv.offset;
62 			sec_sess->iv_length = crypto_xfrm->aead.iv.length;
63 		} else if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
64 			sec_sess->iv_offset = crypto_xfrm->cipher.iv.offset;
65 			sec_sess->iv_length = crypto_xfrm->cipher.iv.length;
66 		} else {
67 			sec_sess->iv_offset = crypto_xfrm->auth.iv.offset;
68 			sec_sess->iv_length = crypto_xfrm->auth.iv.length;
69 		}
70 	}
71 #else
72 	if (ipsec_xfrm->options.iv_gen_disable != 0) {
73 		plt_err("Application provided IV not supported");
74 		ret = -ENOTSUP;
75 		goto sa_dptr_free;
76 	}
77 #endif
78 
79 	sec_sess->ipsec.is_outbound = true;
80 
81 	/* Get Rlen calculation data */
82 	ret = cnxk_ipsec_outb_rlens_get(&rlens, ipsec_xfrm, crypto_xfrm);
83 	if (ret)
84 		goto sa_dptr_free;
85 
86 	sec_sess->max_extended_len = rlens.max_extended_len;
87 
88 	/* pre-populate CPT INST word 4 */
89 	inst_w4.u64 = 0;
90 	inst_w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC | ROC_IE_OT_INPLACE_BIT;
91 
92 	param1.u16 = 0;
93 
94 	param1.s.ttl_or_hop_limit = ipsec_xfrm->options.dec_ttl;
95 
96 	/* Disable IP checksum computation by default */
97 	param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_DISABLE;
98 
99 	if (ipsec_xfrm->options.ip_csum_enable) {
100 		param1.s.ip_csum_disable =
101 			ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE;
102 	}
103 
104 	/* Disable L4 checksum computation by default */
105 	param1.s.l4_csum_disable = ROC_IE_OT_SA_INNER_PKT_L4_CSUM_DISABLE;
106 
107 	if (ipsec_xfrm->options.l4_csum_enable) {
108 		param1.s.l4_csum_disable =
109 			ROC_IE_OT_SA_INNER_PKT_L4_CSUM_ENABLE;
110 	}
111 
112 	inst_w4.s.param1 = param1.u16;
113 
114 	sec_sess->inst.w4 = inst_w4.u64;
115 
116 	if (ipsec_xfrm->options.stats == 1) {
117 		/* Enable mib counters */
118 		sa_dptr->w0.s.count_mib_bytes = 1;
119 		sa_dptr->w0.s.count_mib_pkts = 1;
120 	}
121 
122 	memset(out_sa, 0, sizeof(struct roc_ot_ipsec_outb_sa));
123 
124 	/* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */
125 	memcpy(out_sa, sa_dptr, 8);
126 
127 	plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
128 
129 	/* Write session using microcode opcode */
130 	ret = roc_cpt_ctx_write(lf, sa_dptr, out_sa,
131 				sizeof(struct roc_ot_ipsec_outb_sa));
132 	if (ret) {
133 		plt_err("Could not write outbound session to hardware");
134 		goto sa_dptr_free;
135 	}
136 
137 	/* Trigger CTX flush so that data is written back to DRAM */
138 	roc_cpt_lf_ctx_flush(lf, out_sa, false);
139 
140 	sec_sess->proto = RTE_SECURITY_PROTOCOL_IPSEC;
141 	plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
142 
143 sa_dptr_free:
144 	plt_free(sa_dptr);
145 
146 	return ret;
147 }
148 
149 static int
150 cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
151 			  struct rte_security_ipsec_xform *ipsec_xfrm,
152 			  struct rte_crypto_sym_xform *crypto_xfrm,
153 			  struct cn10k_sec_session *sec_sess)
154 {
155 	union roc_ot_ipsec_inb_param1 param1;
156 	struct roc_ot_ipsec_inb_sa *sa_dptr;
157 	struct cn10k_ipsec_sa *sa;
158 	union cpt_inst_w4 inst_w4;
159 	void *in_sa;
160 	int ret = 0;
161 
162 	sa = &sec_sess->sa;
163 	in_sa = &sa->in_sa;
164 
165 	/* Allocate memory to be used as dptr for CPT ucode WRITE_SA op */
166 	sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_inb_sa), 8);
167 	if (sa_dptr == NULL) {
168 		plt_err("Couldn't allocate memory for SA dptr");
169 		return -ENOMEM;
170 	}
171 
172 	/* Translate security parameters to SA */
173 	ret = cnxk_ot_ipsec_inb_sa_fill(sa_dptr, ipsec_xfrm, crypto_xfrm,
174 					false);
175 	if (ret) {
176 		plt_err("Could not fill inbound session parameters");
177 		goto sa_dptr_free;
178 	}
179 
180 	sec_sess->ipsec.is_outbound = false;
181 	sec_sess->inst.w7 = cpt_inst_w7_get(roc_cpt, in_sa);
182 
183 	/* Save index/SPI in cookie, specific required for Rx Inject */
184 	sa_dptr->w1.s.cookie = 0xFFFFFFFF;
185 
186 	/* pre-populate CPT INST word 4 */
187 	inst_w4.u64 = 0;
188 	inst_w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_PROCESS_INBOUND_IPSEC | ROC_IE_OT_INPLACE_BIT;
189 
190 	param1.u16 = 0;
191 
192 	/* Disable IP checksum verification by default */
193 	param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_DISABLE;
194 
195 	/* Set the ip chksum flag in mbuf before enqueue.
196 	 * Reset the flag in post process in case of errors
197 	 */
198 	if (ipsec_xfrm->options.ip_csum_enable) {
199 		param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE;
200 		sec_sess->ipsec.ip_csum = RTE_MBUF_F_RX_IP_CKSUM_GOOD;
201 	}
202 
203 	/* Disable L4 checksum verification by default */
204 	param1.s.l4_csum_disable = ROC_IE_OT_SA_INNER_PKT_L4_CSUM_DISABLE;
205 
206 	if (ipsec_xfrm->options.l4_csum_enable) {
207 		param1.s.l4_csum_disable =
208 			ROC_IE_OT_SA_INNER_PKT_L4_CSUM_ENABLE;
209 	}
210 
211 	param1.s.esp_trailer_disable = 1;
212 
213 	inst_w4.s.param1 = param1.u16;
214 
215 	sec_sess->inst.w4 = inst_w4.u64;
216 
217 	if (ipsec_xfrm->options.stats == 1) {
218 		/* Enable mib counters */
219 		sa_dptr->w0.s.count_mib_bytes = 1;
220 		sa_dptr->w0.s.count_mib_pkts = 1;
221 	}
222 
223 	memset(in_sa, 0, sizeof(struct roc_ot_ipsec_inb_sa));
224 
225 	/* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */
226 	memcpy(in_sa, sa_dptr, 8);
227 
228 	plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
229 
230 	/* Write session using microcode opcode */
231 	ret = roc_cpt_ctx_write(lf, sa_dptr, in_sa,
232 				sizeof(struct roc_ot_ipsec_inb_sa));
233 	if (ret) {
234 		plt_err("Could not write inbound session to hardware");
235 		goto sa_dptr_free;
236 	}
237 
238 	/* Trigger CTX flush so that data is written back to DRAM */
239 	roc_cpt_lf_ctx_flush(lf, in_sa, true);
240 
241 	sec_sess->proto = RTE_SECURITY_PROTOCOL_IPSEC;
242 	plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
243 
244 sa_dptr_free:
245 	plt_free(sa_dptr);
246 
247 	return ret;
248 }
249 
250 int
251 cn10k_ipsec_session_create(struct cnxk_cpt_vf *vf, struct cnxk_cpt_qp *qp,
252 			   struct rte_security_ipsec_xform *ipsec_xfrm,
253 			   struct rte_crypto_sym_xform *crypto_xfrm,
254 			   struct rte_security_session *sess)
255 {
256 	struct roc_cpt *roc_cpt;
257 	int ret;
258 
259 	ret = cnxk_ipsec_xform_verify(ipsec_xfrm, crypto_xfrm);
260 	if (ret)
261 		return ret;
262 
263 	roc_cpt = &vf->cpt;
264 
265 	if (ipsec_xfrm->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
266 		return cn10k_ipsec_inb_sa_create(roc_cpt, &qp->lf, ipsec_xfrm, crypto_xfrm,
267 						 (struct cn10k_sec_session *)sess);
268 	else
269 		return cn10k_ipsec_outb_sa_create(roc_cpt, &qp->lf, ipsec_xfrm, crypto_xfrm,
270 						  (struct cn10k_sec_session *)sess);
271 }
272 
273 int
274 cn10k_sec_ipsec_session_destroy(struct cnxk_cpt_qp *qp, struct cn10k_sec_session *sess)
275 {
276 	union roc_ot_ipsec_sa_word2 *w2;
277 	struct cn10k_ipsec_sa *sa;
278 	struct roc_cpt_lf *lf;
279 	void *sa_dptr = NULL;
280 	int ret;
281 
282 	lf = &qp->lf;
283 
284 	sa = &sess->sa;
285 
286 	/* Trigger CTX flush to write dirty data back to DRAM */
287 	roc_cpt_lf_ctx_flush(lf, &sa->in_sa, false);
288 
289 	ret = -1;
290 
291 	if (sess->ipsec.is_outbound) {
292 		sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_outb_sa), 8);
293 		if (sa_dptr != NULL) {
294 			roc_ot_ipsec_outb_sa_init(sa_dptr);
295 
296 			ret = roc_cpt_ctx_write(
297 				lf, sa_dptr, &sa->out_sa,
298 				sizeof(struct roc_ot_ipsec_outb_sa));
299 		}
300 	} else {
301 		sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_inb_sa), 8);
302 		if (sa_dptr != NULL) {
303 			roc_ot_ipsec_inb_sa_init(sa_dptr, false);
304 
305 			ret = roc_cpt_ctx_write(
306 				lf, sa_dptr, &sa->in_sa,
307 				sizeof(struct roc_ot_ipsec_inb_sa));
308 		}
309 	}
310 
311 	plt_free(sa_dptr);
312 
313 	if (ret) {
314 		/* MC write_ctx failed. Attempt reload of CTX */
315 
316 		/* Wait for 1 ms so that flush is complete */
317 		rte_delay_ms(1);
318 
319 		w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2;
320 		w2->s.valid = 0;
321 
322 		plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
323 
324 		/* Trigger CTX reload to fetch new data from DRAM */
325 		roc_cpt_lf_ctx_reload(lf, &sa->in_sa);
326 	}
327 
328 	return 0;
329 }
330 
331 int
332 cn10k_ipsec_stats_get(struct cnxk_cpt_qp *qp, struct cn10k_sec_session *sess,
333 		      struct rte_security_stats *stats)
334 {
335 	struct roc_ot_ipsec_outb_sa *out_sa;
336 	struct roc_ot_ipsec_inb_sa *in_sa;
337 	struct cn10k_ipsec_sa *sa;
338 
339 	stats->protocol = RTE_SECURITY_PROTOCOL_IPSEC;
340 	sa = &sess->sa;
341 
342 	if (sess->ipsec.is_outbound) {
343 		out_sa = &sa->out_sa;
344 		roc_cpt_lf_ctx_flush(&qp->lf, out_sa, false);
345 		rte_delay_ms(1);
346 		stats->ipsec.opackets = out_sa->ctx.mib_pkts;
347 		stats->ipsec.obytes = out_sa->ctx.mib_octs;
348 	} else {
349 		in_sa = &sa->in_sa;
350 		roc_cpt_lf_ctx_flush(&qp->lf, in_sa, false);
351 		rte_delay_ms(1);
352 		stats->ipsec.ipackets = in_sa->ctx.mib_pkts;
353 		stats->ipsec.ibytes = in_sa->ctx.mib_octs;
354 	}
355 
356 	return 0;
357 }
358 
359 int
360 cn10k_ipsec_session_update(struct cnxk_cpt_vf *vf, struct cnxk_cpt_qp *qp,
361 			   struct cn10k_sec_session *sess, struct rte_security_session_conf *conf)
362 {
363 	struct roc_cpt *roc_cpt;
364 	int ret;
365 
366 	if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
367 		return -ENOTSUP;
368 
369 	ret = cnxk_ipsec_xform_verify(&conf->ipsec, conf->crypto_xform);
370 	if (ret)
371 		return ret;
372 
373 	roc_cpt = &vf->cpt;
374 
375 	return cn10k_ipsec_outb_sa_create(roc_cpt, &qp->lf, &conf->ipsec, conf->crypto_xform,
376 					  (struct cn10k_sec_session *)sess);
377 }
378