1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
3 */
4
5 #ifndef __CN9K_IPSEC_LA_OPS_H__
6 #define __CN9K_IPSEC_LA_OPS_H__
7
8 #include <rte_crypto_sym.h>
9 #include <rte_esp.h>
10 #include <rte_security.h>
11
12 #include "roc_ie.h"
13
14 #include "cn9k_ipsec.h"
15 #include "cnxk_cryptodev_ops.h"
16 #include "cnxk_security_ar.h"
17 #include "cnxk_sg.h"
18
19 static __rte_always_inline int32_t
ipsec_po_out_rlen_get(struct cn9k_sec_session * sess,uint32_t plen,struct rte_mbuf * m_src)20 ipsec_po_out_rlen_get(struct cn9k_sec_session *sess, uint32_t plen, struct rte_mbuf *m_src)
21 {
22 uint32_t enc_payload_len;
23 int adj_len = 0;
24
25 if (sess->sa.out_sa.common_sa.ctl.ipsec_mode == ROC_IE_SA_MODE_TRANSPORT) {
26 adj_len = ROC_CPT_TUNNEL_IPV4_HDR_LEN;
27
28 uintptr_t data = (uintptr_t)m_src->buf_addr + m_src->data_off;
29 struct rte_ipv4_hdr *ip = (struct rte_ipv4_hdr *)data;
30
31 if (unlikely(ip->version != IPVERSION)) {
32 struct rte_ipv6_hdr *ip6 = (struct rte_ipv6_hdr *)ip;
33 uint8_t *nxt_hdr = (uint8_t *)ip6;
34 uint8_t dest_op_cnt = 0;
35 int nh = ip6->proto;
36
37 PLT_ASSERT(ip->version == 6);
38
39 adj_len = ROC_CPT_TUNNEL_IPV6_HDR_LEN;
40 nxt_hdr += ROC_CPT_TUNNEL_IPV6_HDR_LEN;
41 while (nh != -EINVAL) {
42 size_t ext_len = 0;
43
44 nh = rte_ipv6_get_next_ext(nxt_hdr, nh, &ext_len);
45 /* With multiple dest ops headers, the ESP hdr will be before
46 * the 2nd dest ops and after the first dest ops header
47 */
48 if ((nh == IPPROTO_DSTOPTS) && dest_op_cnt)
49 break;
50 else if (nh == IPPROTO_DSTOPTS)
51 dest_op_cnt++;
52 adj_len += ext_len;
53 nxt_hdr += ext_len;
54 }
55 }
56 }
57
58 enc_payload_len =
59 RTE_ALIGN_CEIL(plen + sess->rlens.roundup_len - adj_len, sess->rlens.roundup_byte);
60
61 return sess->custom_hdr_len + sess->rlens.partial_len + enc_payload_len + adj_len;
62 }
63
64 static __rte_always_inline int
process_outb_sa(struct cpt_qp_meta_info * m_info,struct rte_crypto_op * cop,struct cn9k_sec_session * sess,struct cpt_inst_s * inst,struct cpt_inflight_req * infl_req)65 process_outb_sa(struct cpt_qp_meta_info *m_info, struct rte_crypto_op *cop,
66 struct cn9k_sec_session *sess, struct cpt_inst_s *inst,
67 struct cpt_inflight_req *infl_req)
68 {
69 const unsigned int hdr_len = sess->custom_hdr_len;
70 struct rte_crypto_sym_op *sym_op = cop->sym;
71 struct rte_mbuf *m_src = sym_op->m_src;
72 uint32_t dlen, rlen, pkt_len, seq_lo;
73 uint16_t data_off = m_src->data_off;
74 struct roc_ie_on_outb_hdr *hdr;
75 int32_t extend_tail;
76 uint64_t esn;
77
78 pkt_len = rte_pktmbuf_pkt_len(m_src);
79 dlen = pkt_len + hdr_len;
80 rlen = ipsec_po_out_rlen_get(sess, pkt_len, m_src);
81
82 extend_tail = rlen - dlen;
83 pkt_len += extend_tail;
84
85 if (likely((m_src->next == NULL) && (hdr_len <= data_off))) {
86 if (unlikely(extend_tail > rte_pktmbuf_tailroom(m_src))) {
87 plt_dp_err("Not enough tail room (required: %d, available: %d)",
88 extend_tail, rte_pktmbuf_tailroom(m_src));
89 return -ENOMEM;
90 }
91
92 m_src->data_len = pkt_len;
93
94 hdr = PLT_PTR_ADD(m_src->buf_addr, data_off - hdr_len);
95
96 inst->dptr = PLT_U64_CAST(hdr);
97 inst->w4.u64 = sess->inst.w4 | dlen;
98 } else {
99 struct roc_sglist_comp *scatter_comp, *gather_comp;
100 uint32_t g_size_bytes, s_size_bytes;
101 struct rte_mbuf *last_seg;
102 uint8_t *in_buffer;
103 void *m_data;
104 int i;
105
106 last_seg = rte_pktmbuf_lastseg(m_src);
107
108 if (unlikely(extend_tail > rte_pktmbuf_tailroom(last_seg))) {
109 plt_dp_err("Not enough tail room (required: %d, available: %d)",
110 extend_tail, rte_pktmbuf_tailroom(last_seg));
111 return -ENOMEM;
112 }
113
114 m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
115 if (unlikely(m_data == NULL)) {
116 plt_dp_err("Error allocating meta buffer for request");
117 return -ENOMEM;
118 }
119
120 hdr = m_data;
121
122 m_data = (uint8_t *)m_data + hdr_len;
123 in_buffer = m_data;
124
125 ((uint16_t *)in_buffer)[0] = 0;
126 ((uint16_t *)in_buffer)[1] = 0;
127
128 /*
129 * Input Gather List
130 */
131 i = 0;
132 gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_data + 8);
133
134 i = fill_sg_comp(gather_comp, i, (uint64_t)hdr, hdr_len);
135 i = fill_sg_comp_from_pkt(gather_comp, i, m_src);
136 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
137
138 g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
139
140 /*
141 * output Scatter List
142 */
143 last_seg->data_len += extend_tail;
144
145 i = 0;
146 scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
147
148 i = fill_sg_comp(scatter_comp, i, (uint64_t)hdr, hdr_len);
149 i = fill_sg_comp_from_pkt(scatter_comp, i, m_src);
150 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
151
152 s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
153
154 dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
155
156 inst->dptr = (uint64_t)in_buffer;
157
158 inst->w4.u64 = sess->inst.w4 | dlen;
159 inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
160 }
161
162 #ifdef LA_IPSEC_DEBUG
163 if (sess->inst.w4 & ROC_IE_ON_PER_PKT_IV) {
164 memcpy(&hdr->iv[0],
165 rte_crypto_op_ctod_offset(cop, uint8_t *, sess->cipher_iv_off),
166 sess->cipher_iv_len);
167 }
168 #endif
169
170 m_src->pkt_len = pkt_len;
171 esn = ++sess->esn;
172
173 /* Set ESN seq hi */
174 hdr->esn = rte_cpu_to_be_32(esn >> 32);
175
176 /* Set ESN seq lo */
177 seq_lo = rte_cpu_to_be_32(esn & (BIT_ULL(32) - 1));
178 hdr->seq = seq_lo;
179
180 /* Set IPID same as seq_lo */
181 hdr->ip_id = seq_lo;
182
183 /* Prepare CPT instruction */
184 inst->w7.u64 = sess->inst.w7;
185
186 return 0;
187 }
188
189 static __rte_always_inline int
process_inb_sa(struct cpt_qp_meta_info * m_info,struct rte_crypto_op * cop,struct cn9k_sec_session * sess,struct cpt_inst_s * inst,struct cpt_inflight_req * infl_req)190 process_inb_sa(struct cpt_qp_meta_info *m_info, struct rte_crypto_op *cop,
191 struct cn9k_sec_session *sess, struct cpt_inst_s *inst,
192 struct cpt_inflight_req *infl_req)
193 {
194 const unsigned int hdr_len = ROC_IE_ON_INB_RPTR_HDR;
195 struct rte_crypto_sym_op *sym_op = cop->sym;
196 struct rte_mbuf *m_src = sym_op->m_src;
197 struct roc_ie_on_inb_hdr *hdr;
198 uint32_t dlen;
199
200 infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND;
201 if (likely(m_src->next == NULL)) {
202 dlen = rte_pktmbuf_pkt_len(m_src);
203 inst->dptr = rte_pktmbuf_mtod(m_src, uint64_t);
204 inst->w4.u64 = sess->inst.w4 | dlen;
205 } else {
206 struct roc_sglist_comp *scatter_comp, *gather_comp;
207 uint32_t g_size_bytes, s_size_bytes;
208 uint8_t *in_buffer;
209 void *m_data;
210 int i;
211
212 m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
213 if (unlikely(m_data == NULL)) {
214 plt_dp_err("Error allocating meta buffer for request");
215 return -ENOMEM;
216 }
217
218 hdr = m_data;
219
220 m_data = (uint8_t *)m_data + hdr_len;
221 in_buffer = m_data;
222
223 ((uint16_t *)in_buffer)[0] = 0;
224 ((uint16_t *)in_buffer)[1] = 0;
225
226 /*
227 * Input Gather List
228 */
229 i = 0;
230 gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_data + 8);
231 i = fill_sg_comp_from_pkt(gather_comp, i, m_src);
232 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
233
234 g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
235
236 /*
237 * Output Scatter List
238 */
239 i = 0;
240 scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
241 i = fill_sg_comp(scatter_comp, i, (uint64_t)hdr, hdr_len);
242 i = fill_sg_comp_from_pkt(scatter_comp, i, m_src);
243 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
244
245 s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
246
247 dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
248
249 inst->dptr = (uint64_t)in_buffer;
250 inst->w4.u64 = sess->inst.w4 | dlen;
251 inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
252 }
253
254 /* Prepare CPT instruction */
255 inst->w7.u64 = sess->inst.w7;
256
257 if (unlikely(sess->replay_win_sz))
258 infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_INB_REPLAY;
259
260 return 0;
261 }
262 #endif /* __CN9K_IPSEC_LA_OPS_H__ */
263