1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2024 Marvell.
3 */
4
5 #ifndef __CN10K_TLS_OPS_H__
6 #define __CN10K_TLS_OPS_H__
7
8 #include <rte_crypto_sym.h>
9 #include <rte_security.h>
10
11 #include "roc_ie.h"
12
13 #include "cn10k_cryptodev.h"
14 #include "cn10k_cryptodev_sec.h"
15 #include "cnxk_cryptodev.h"
16 #include "cnxk_cryptodev_ops.h"
17 #include "cnxk_sg.h"
18
19 static __rte_always_inline int
process_tls_write(struct roc_cpt_lf * lf,struct rte_crypto_op * cop,struct cn10k_sec_session * sess,struct cpt_qp_meta_info * m_info,struct cpt_inflight_req * infl_req,struct cpt_inst_s * inst,const bool is_sg_ver2)20 process_tls_write(struct roc_cpt_lf *lf, struct rte_crypto_op *cop, struct cn10k_sec_session *sess,
21 struct cpt_qp_meta_info *m_info, struct cpt_inflight_req *infl_req,
22 struct cpt_inst_s *inst, const bool is_sg_ver2)
23 {
24 struct cn10k_tls_opt tls_opt = sess->tls_opt;
25 struct rte_crypto_sym_op *sym_op = cop->sym;
26 #ifdef LA_IPSEC_DEBUG
27 struct roc_ie_ot_tls_write_sa *write_sa;
28 #endif
29 struct rte_mbuf *m_src = sym_op->m_src;
30 struct rte_mbuf *m_dst = sym_op->m_dst;
31 uint32_t pad_len, pad_bytes;
32 struct rte_mbuf *last_seg;
33 union cpt_inst_w4 w4;
34 void *m_data = NULL;
35 uint8_t *in_buffer;
36
37 pad_bytes = (cop->aux_flags * 8) > 0xff ? 0xff : (cop->aux_flags * 8);
38 pad_len = (pad_bytes >> tls_opt.pad_shift) * tls_opt.enable_padding;
39
40 #ifdef LA_IPSEC_DEBUG
41 write_sa = &sess->tls_rec.write_sa;
42 if (write_sa->w2.s.iv_at_cptr == ROC_IE_OT_TLS_IV_SRC_FROM_SA) {
43
44 uint8_t *iv = PLT_PTR_ADD(write_sa->cipher_key, 32);
45
46 if (write_sa->w2.s.cipher_select == ROC_IE_OT_TLS_CIPHER_AES_GCM) {
47 uint32_t *tmp;
48
49 /* For GCM, the IV and salt format will be like below:
50 * iv[0-3]: lower bytes of IV in BE format.
51 * iv[4-7]: salt / nonce.
52 * iv[12-15]: upper bytes of IV in BE format.
53 */
54 memcpy(iv, rte_crypto_op_ctod_offset(cop, uint8_t *, sess->iv_offset), 4);
55 tmp = (uint32_t *)iv;
56 *tmp = rte_be_to_cpu_32(*tmp);
57
58 memcpy(iv + 12,
59 rte_crypto_op_ctod_offset(cop, uint8_t *, sess->iv_offset + 4), 4);
60 tmp = (uint32_t *)(iv + 12);
61 *tmp = rte_be_to_cpu_32(*tmp);
62 } else if (write_sa->w2.s.cipher_select == ROC_IE_OT_TLS_CIPHER_AES_CBC) {
63 uint64_t *tmp;
64
65 memcpy(iv, rte_crypto_op_ctod_offset(cop, uint8_t *, sess->iv_offset), 16);
66 tmp = (uint64_t *)iv;
67 *tmp = rte_be_to_cpu_64(*tmp);
68 tmp = (uint64_t *)(iv + 8);
69 *tmp = rte_be_to_cpu_64(*tmp);
70 } else if (write_sa->w2.s.cipher_select == ROC_IE_OT_TLS_CIPHER_3DES) {
71 uint64_t *tmp;
72
73 memcpy(iv, rte_crypto_op_ctod_offset(cop, uint8_t *, sess->iv_offset), 8);
74 tmp = (uint64_t *)iv;
75 *tmp = rte_be_to_cpu_64(*tmp);
76 }
77
78 /* Trigger CTX reload to fetch new data from DRAM */
79 roc_cpt_lf_ctx_reload(lf, write_sa);
80 rte_delay_ms(1);
81 }
82 #else
83 RTE_SET_USED(lf);
84 #endif
85 /* Single buffer direct mode */
86 if (likely(m_src->next == NULL)) {
87 void *vaddr;
88
89 if (unlikely(rte_pktmbuf_tailroom(m_src) < sess->max_extended_len)) {
90 plt_dp_err("Not enough tail room");
91 return -ENOMEM;
92 }
93
94 vaddr = rte_pktmbuf_mtod(m_src, void *);
95 inst->dptr = (uint64_t)vaddr;
96 inst->rptr = (uint64_t)vaddr;
97
98 w4.u64 = sess->inst.w4;
99 w4.s.param1 = m_src->data_len;
100 w4.s.dlen = m_src->data_len;
101
102 w4.s.param2 = cop->param1.tls_record.content_type;
103 w4.s.opcode_minor = pad_len;
104
105 inst->w4.u64 = w4.u64;
106 } else if (is_sg_ver2 == false) {
107 struct roc_sglist_comp *scatter_comp, *gather_comp;
108 uint32_t g_size_bytes, s_size_bytes;
109 uint32_t dlen;
110 int i;
111
112 last_seg = rte_pktmbuf_lastseg(m_src);
113
114 if (unlikely(rte_pktmbuf_tailroom(last_seg) < sess->max_extended_len)) {
115 plt_dp_err("Not enough tail room (required: %d, available: %d)",
116 sess->max_extended_len, rte_pktmbuf_tailroom(last_seg));
117 return -ENOMEM;
118 }
119
120 m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
121 if (unlikely(m_data == NULL)) {
122 plt_dp_err("Error allocating meta buffer for request");
123 return -ENOMEM;
124 }
125
126 in_buffer = (uint8_t *)m_data;
127 ((uint16_t *)in_buffer)[0] = 0;
128 ((uint16_t *)in_buffer)[1] = 0;
129
130 /* Input Gather List */
131 i = 0;
132 gather_comp = (struct roc_sglist_comp *)((uint8_t *)in_buffer + 8);
133
134 i = fill_sg_comp_from_pkt(gather_comp, i, m_src);
135 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
136
137 g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
138
139 i = 0;
140 scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
141
142 i = fill_sg_comp_from_pkt(scatter_comp, i, m_src);
143 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
144
145 s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
146
147 dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
148
149 inst->dptr = (uint64_t)in_buffer;
150 inst->rptr = (uint64_t)in_buffer;
151
152 w4.u64 = sess->inst.w4;
153 w4.s.dlen = dlen;
154 w4.s.param1 = rte_pktmbuf_pkt_len(m_src);
155 w4.s.param2 = cop->param1.tls_record.content_type;
156 w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
157 w4.s.opcode_minor = pad_len;
158
159 /* Output Scatter List */
160 last_seg->data_len += sess->max_extended_len + pad_bytes;
161 inst->w4.u64 = w4.u64;
162 } else {
163 struct roc_sg2list_comp *scatter_comp, *gather_comp;
164 union cpt_inst_w5 cpt_inst_w5;
165 union cpt_inst_w6 cpt_inst_w6;
166 uint32_t g_size_bytes;
167 int i;
168
169 last_seg = rte_pktmbuf_lastseg(m_src);
170
171 if (unlikely(rte_pktmbuf_tailroom(last_seg) < sess->max_extended_len)) {
172 plt_dp_err("Not enough tail room (required: %d, available: %d)",
173 sess->max_extended_len, rte_pktmbuf_tailroom(last_seg));
174 return -ENOMEM;
175 }
176
177 m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
178 if (unlikely(m_data == NULL)) {
179 plt_dp_err("Error allocating meta buffer for request");
180 return -ENOMEM;
181 }
182
183 in_buffer = (uint8_t *)m_data;
184 /* Input Gather List */
185 i = 0;
186 gather_comp = (struct roc_sg2list_comp *)((uint8_t *)in_buffer);
187 i = fill_sg2_comp_from_pkt(gather_comp, i, m_src);
188
189 cpt_inst_w5.s.gather_sz = ((i + 2) / 3);
190 g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp);
191
192 i = 0;
193 scatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);
194
195 if (m_dst == NULL)
196 m_dst = m_src;
197 i = fill_sg2_comp_from_pkt(scatter_comp, i, m_dst);
198
199 cpt_inst_w6.s.scatter_sz = ((i + 2) / 3);
200
201 cpt_inst_w5.s.dptr = (uint64_t)gather_comp;
202 cpt_inst_w6.s.rptr = (uint64_t)scatter_comp;
203
204 inst->w5.u64 = cpt_inst_w5.u64;
205 inst->w6.u64 = cpt_inst_w6.u64;
206 w4.u64 = sess->inst.w4;
207 w4.s.dlen = rte_pktmbuf_pkt_len(m_src);
208 w4.s.opcode_major &= (~(ROC_IE_OT_INPLACE_BIT));
209 w4.s.opcode_minor = pad_len;
210 w4.s.param1 = w4.s.dlen;
211 w4.s.param2 = cop->param1.tls_record.content_type;
212 /* Output Scatter List */
213 last_seg->data_len += sess->max_extended_len + pad_bytes;
214 inst->w4.u64 = w4.u64;
215 }
216
217 return 0;
218 }
219
220 static __rte_always_inline int
process_tls_read(struct rte_crypto_op * cop,struct cn10k_sec_session * sess,struct cpt_qp_meta_info * m_info,struct cpt_inflight_req * infl_req,struct cpt_inst_s * inst,const bool is_sg_ver2)221 process_tls_read(struct rte_crypto_op *cop, struct cn10k_sec_session *sess,
222 struct cpt_qp_meta_info *m_info, struct cpt_inflight_req *infl_req,
223 struct cpt_inst_s *inst, const bool is_sg_ver2)
224 {
225 struct rte_crypto_sym_op *sym_op = cop->sym;
226 struct rte_mbuf *m_src = sym_op->m_src;
227 struct rte_mbuf *m_dst = sym_op->m_dst;
228 union cpt_inst_w4 w4;
229 uint8_t *in_buffer;
230 void *m_data;
231
232 if (likely(m_src->next == NULL)) {
233 void *vaddr;
234
235 vaddr = rte_pktmbuf_mtod(m_src, void *);
236
237 inst->dptr = (uint64_t)vaddr;
238 inst->rptr = (uint64_t)vaddr;
239
240 w4.u64 = sess->inst.w4;
241 w4.s.dlen = m_src->data_len;
242 w4.s.param1 = m_src->data_len;
243 inst->w4.u64 = w4.u64;
244 } else if (is_sg_ver2 == false) {
245 struct roc_sglist_comp *scatter_comp, *gather_comp;
246 int tail_len = sess->tls_opt.tail_fetch_len * 16;
247 int pkt_len = rte_pktmbuf_pkt_len(m_src);
248 uint32_t g_size_bytes, s_size_bytes;
249 uint16_t *sg_hdr;
250 uint32_t dlen;
251 int i;
252
253 m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
254 if (unlikely(m_data == NULL)) {
255 plt_dp_err("Error allocating meta buffer for request");
256 return -ENOMEM;
257 }
258
259 /* Input Gather List */
260 in_buffer = (uint8_t *)m_data;
261 sg_hdr = (uint16_t *)(in_buffer + 32);
262 gather_comp = (struct roc_sglist_comp *)((uint8_t *)sg_hdr + 8);
263 i = 0;
264 /* Add the last blocks as first gather component for tail fetch. */
265 if (tail_len) {
266 const uint8_t *output;
267
268 output = rte_pktmbuf_read(m_src, pkt_len - tail_len, tail_len, in_buffer);
269 if (output != in_buffer)
270 rte_memcpy(in_buffer, output, tail_len);
271 i = fill_sg_comp(gather_comp, i, (uint64_t)in_buffer, tail_len);
272 }
273
274 sg_hdr[0] = 0;
275 sg_hdr[1] = 0;
276 i = fill_sg_comp_from_pkt(gather_comp, i, m_src);
277 sg_hdr[2] = rte_cpu_to_be_16(i);
278
279 g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
280
281 i = 0;
282 scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
283
284 i = fill_sg_comp_from_pkt(scatter_comp, i, m_src);
285 sg_hdr[3] = rte_cpu_to_be_16(i);
286
287 s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
288
289 dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
290
291 inst->dptr = (uint64_t)in_buffer;
292 inst->rptr = (uint64_t)in_buffer;
293
294 w4.u64 = sess->inst.w4;
295 w4.s.dlen = dlen;
296 w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
297 w4.s.param1 = pkt_len;
298 inst->w4.u64 = w4.u64;
299 } else {
300 struct roc_sg2list_comp *scatter_comp, *gather_comp;
301 int tail_len = sess->tls_opt.tail_fetch_len * 16;
302 int pkt_len = rte_pktmbuf_pkt_len(m_src);
303 union cpt_inst_w5 cpt_inst_w5;
304 union cpt_inst_w6 cpt_inst_w6;
305 uint32_t g_size_bytes;
306 int i;
307
308 m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
309 if (unlikely(m_data == NULL)) {
310 plt_dp_err("Error allocating meta buffer for request");
311 return -ENOMEM;
312 }
313
314 in_buffer = (uint8_t *)m_data;
315 /* Input Gather List */
316 i = 0;
317
318 /* First 32 bytes in m_data are rsvd for tail fetch.
319 * SG list start from 32 byte onwards.
320 */
321 gather_comp = (struct roc_sg2list_comp *)((uint8_t *)(in_buffer + 32));
322
323 /* Add the last blocks as first gather component for tail fetch. */
324 if (tail_len) {
325 const uint8_t *output;
326
327 output = rte_pktmbuf_read(m_src, pkt_len - tail_len, tail_len, in_buffer);
328 if (output != in_buffer)
329 rte_memcpy(in_buffer, output, tail_len);
330 i = fill_sg2_comp(gather_comp, i, (uint64_t)in_buffer, tail_len);
331 }
332
333 i = fill_sg2_comp_from_pkt(gather_comp, i, m_src);
334
335 cpt_inst_w5.s.gather_sz = ((i + 2) / 3);
336 g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp);
337
338 i = 0;
339 scatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);
340
341 if (m_dst == NULL)
342 m_dst = m_src;
343 i = fill_sg2_comp_from_pkt(scatter_comp, i, m_dst);
344
345 cpt_inst_w6.s.scatter_sz = ((i + 2) / 3);
346
347 cpt_inst_w5.s.dptr = (uint64_t)gather_comp;
348 cpt_inst_w6.s.rptr = (uint64_t)scatter_comp;
349
350 inst->w5.u64 = cpt_inst_w5.u64;
351 inst->w6.u64 = cpt_inst_w6.u64;
352 w4.u64 = sess->inst.w4;
353 w4.s.dlen = pkt_len + tail_len;
354 w4.s.param1 = w4.s.dlen;
355 w4.s.opcode_major &= (~(ROC_IE_OT_INPLACE_BIT));
356 inst->w4.u64 = w4.u64;
357 }
358
359 return 0;
360 }
361 #endif /* __CN10K_TLS_OPS_H__ */
362