1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 #include <sys/types.h> 5 #include <netinet/in.h> 6 #include <netinet/ip.h> 7 8 #include <rte_branch_prediction.h> 9 #include <rte_log.h> 10 #include <rte_cryptodev.h> 11 #include <rte_ethdev.h> 12 #include <rte_mbuf.h> 13 14 #include "ipsec.h" 15 16 #define SATP_OUT_IPV4(t) \ 17 ((((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TRANS && \ 18 (((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \ 19 ((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4) 20 21 /* helper routine to free bulk of packets */ 22 static inline void 23 free_pkts(struct rte_mbuf *mb[], uint32_t n) 24 { 25 uint32_t i; 26 27 for (i = 0; i != n; i++) 28 rte_pktmbuf_free(mb[i]); 29 } 30 31 /* helper routine to free bulk of crypto-ops and related packets */ 32 static inline void 33 free_cops(struct rte_crypto_op *cop[], uint32_t n) 34 { 35 uint32_t i; 36 37 for (i = 0; i != n; i++) 38 rte_pktmbuf_free(cop[i]->sym->m_src); 39 } 40 41 /* helper routine to enqueue bulk of crypto ops */ 42 static inline void 43 enqueue_cop_bulk(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num) 44 { 45 uint32_t i, k, len, n; 46 47 len = cqp->len; 48 49 /* 50 * if cqp is empty and we have enough ops, 51 * then queue them to the PMD straightway. 52 */ 53 if (num >= RTE_DIM(cqp->buf) * 3 / 4 && len == 0) { 54 n = rte_cryptodev_enqueue_burst(cqp->id, cqp->qp, cop, num); 55 cqp->in_flight += n; 56 free_cops(cop + n, num - n); 57 return; 58 } 59 60 k = 0; 61 62 do { 63 n = RTE_DIM(cqp->buf) - len; 64 n = RTE_MIN(num - k, n); 65 66 /* put packets into cqp */ 67 for (i = 0; i != n; i++) 68 cqp->buf[len + i] = cop[k + i]; 69 70 len += n; 71 k += n; 72 73 /* if cqp is full then, enqueue crypto-ops to PMD */ 74 if (len == RTE_DIM(cqp->buf)) { 75 n = rte_cryptodev_enqueue_burst(cqp->id, cqp->qp, 76 cqp->buf, len); 77 cqp->in_flight += n; 78 free_cops(cqp->buf + n, len - n); 79 len = 0; 80 } 81 82 83 } while (k != num); 84 85 cqp->len = len; 86 } 87 88 static inline int 89 fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx, 90 struct ipsec_sa *sa) 91 { 92 int32_t rc; 93 94 /* setup crypto section */ 95 if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) { 96 RTE_ASSERT(ss->crypto.ses == NULL); 97 rc = create_lookaside_session(ctx, sa, ss); 98 if (rc != 0) 99 return rc; 100 /* setup session action type */ 101 } else if (ss->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) { 102 RTE_ASSERT(ss->security.ses == NULL); 103 rc = create_lookaside_session(ctx, sa, ss); 104 if (rc != 0) 105 return rc; 106 } else 107 RTE_ASSERT(0); 108 109 rc = rte_ipsec_session_prepare(ss); 110 if (rc != 0) 111 memset(ss, 0, sizeof(*ss)); 112 113 return rc; 114 } 115 116 /* 117 * group input packets byt the SA they belong to. 118 */ 119 static uint32_t 120 sa_group(void *sa_ptr[], struct rte_mbuf *pkts[], 121 struct rte_ipsec_group grp[], uint32_t num) 122 { 123 uint32_t i, n, spi; 124 void *sa; 125 void * const nosa = &spi; 126 127 sa = nosa; 128 for (i = 0, n = 0; i != num; i++) { 129 130 if (sa != sa_ptr[i]) { 131 grp[n].cnt = pkts + i - grp[n].m; 132 n += (sa != nosa); 133 grp[n].id.ptr = sa_ptr[i]; 134 grp[n].m = pkts + i; 135 sa = sa_ptr[i]; 136 } 137 } 138 139 /* terminate last group */ 140 if (sa != nosa) { 141 grp[n].cnt = pkts + i - grp[n].m; 142 n++; 143 } 144 145 return n; 146 } 147 148 /* 149 * helper function, splits processed packets into ipv4/ipv6 traffic. 150 */ 151 static inline void 152 copy_to_trf(struct ipsec_traffic *trf, uint64_t satp, struct rte_mbuf *mb[], 153 uint32_t num) 154 { 155 uint32_t j, ofs, s; 156 struct traffic_type *out; 157 158 /* 159 * determine traffic type(ipv4/ipv6) and offset for ACL classify 160 * based on SA type 161 */ 162 if ((satp & RTE_IPSEC_SATP_DIR_MASK) == RTE_IPSEC_SATP_DIR_IB) { 163 if ((satp & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4) { 164 out = &trf->ip4; 165 ofs = offsetof(struct ip, ip_p); 166 } else { 167 out = &trf->ip6; 168 ofs = offsetof(struct ip6_hdr, ip6_nxt); 169 } 170 } else if (SATP_OUT_IPV4(satp)) { 171 out = &trf->ip4; 172 ofs = offsetof(struct ip, ip_p); 173 } else { 174 out = &trf->ip6; 175 ofs = offsetof(struct ip6_hdr, ip6_nxt); 176 } 177 178 for (j = 0, s = out->num; j != num; j++) { 179 out->data[s + j] = rte_pktmbuf_mtod_offset(mb[j], 180 void *, ofs); 181 out->pkts[s + j] = mb[j]; 182 } 183 184 out->num += num; 185 } 186 187 static uint32_t 188 ipsec_prepare_crypto_group(struct ipsec_ctx *ctx, struct ipsec_sa *sa, 189 struct rte_ipsec_session *ips, struct rte_mbuf **m, 190 unsigned int cnt) 191 { 192 struct cdev_qp *cqp; 193 struct rte_crypto_op *cop[cnt]; 194 uint32_t j, k; 195 struct ipsec_mbuf_metadata *priv; 196 197 cqp = &ctx->tbl[sa->cdev_id_qp]; 198 199 /* for that app each mbuf has it's own crypto op */ 200 for (j = 0; j != cnt; j++) { 201 priv = get_priv(m[j]); 202 cop[j] = &priv->cop; 203 /* 204 * this is just to satisfy inbound_sa_check() 205 * should be removed in future. 206 */ 207 priv->sa = sa; 208 } 209 210 /* prepare and enqueue crypto ops */ 211 k = rte_ipsec_pkt_crypto_prepare(ips, m, cop, cnt); 212 if (k != 0) 213 enqueue_cop_bulk(cqp, cop, k); 214 215 return k; 216 } 217 218 /* 219 * Process ipsec packets. 220 * If packet belong to SA that is subject of inline-crypto, 221 * then process it immediately. 222 * Otherwise do necessary preparations and queue it to related 223 * crypto-dev queue. 224 */ 225 void 226 ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf) 227 { 228 uint64_t satp; 229 uint32_t i, j, k, n; 230 struct ipsec_sa *sa; 231 struct ipsec_mbuf_metadata *priv; 232 struct rte_ipsec_group *pg; 233 struct rte_ipsec_session *ips; 234 struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)]; 235 236 n = sa_group(trf->ipsec.saptr, trf->ipsec.pkts, grp, trf->ipsec.num); 237 238 for (i = 0; i != n; i++) { 239 pg = grp + i; 240 sa = ipsec_mask_saptr(pg->id.ptr); 241 242 ips = ipsec_get_primary_session(sa); 243 244 /* no valid HW session for that SA, try to create one */ 245 if (sa == NULL || (ips->crypto.ses == NULL && 246 fill_ipsec_session(ips, ctx, sa) != 0)) 247 k = 0; 248 249 /* process packets inline */ 250 else if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO || 251 ips->type == 252 RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) { 253 254 /* get SA type */ 255 satp = rte_ipsec_sa_type(ips->sa); 256 257 /* 258 * This is just to satisfy inbound_sa_check() 259 * and get_hop_for_offload_pkt(). 260 * Should be removed in future. 261 */ 262 for (j = 0; j != pg->cnt; j++) { 263 priv = get_priv(pg->m[j]); 264 priv->sa = sa; 265 } 266 267 /* fallback to cryptodev with RX packets which inline 268 * processor was unable to process 269 */ 270 if (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) { 271 /* offload packets to cryptodev */ 272 struct rte_ipsec_session *fallback; 273 274 fallback = ipsec_get_fallback_session(sa); 275 if (fallback->crypto.ses == NULL && 276 fill_ipsec_session(fallback, ctx, sa) 277 != 0) 278 k = 0; 279 else 280 k = ipsec_prepare_crypto_group(ctx, sa, 281 fallback, pg->m, pg->cnt); 282 } else { 283 /* finish processing of packets successfully 284 * decrypted by an inline processor 285 */ 286 k = rte_ipsec_pkt_process(ips, pg->m, pg->cnt); 287 copy_to_trf(trf, satp, pg->m, k); 288 289 } 290 /* enqueue packets to crypto dev */ 291 } else { 292 k = ipsec_prepare_crypto_group(ctx, sa, ips, pg->m, 293 pg->cnt); 294 } 295 296 /* drop packets that cannot be enqueued/processed */ 297 if (k != pg->cnt) 298 free_pkts(pg->m + k, pg->cnt - k); 299 } 300 } 301 302 static inline uint32_t 303 cqp_dequeue(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num) 304 { 305 uint32_t n; 306 307 if (cqp->in_flight == 0) 308 return 0; 309 310 n = rte_cryptodev_dequeue_burst(cqp->id, cqp->qp, cop, num); 311 RTE_ASSERT(cqp->in_flight >= n); 312 cqp->in_flight -= n; 313 314 return n; 315 } 316 317 static inline uint32_t 318 ctx_dequeue(struct ipsec_ctx *ctx, struct rte_crypto_op *cop[], uint32_t num) 319 { 320 uint32_t i, n; 321 322 n = 0; 323 324 for (i = ctx->last_qp; n != num && i != ctx->nb_qps; i++) 325 n += cqp_dequeue(ctx->tbl + i, cop + n, num - n); 326 327 for (i = 0; n != num && i != ctx->last_qp; i++) 328 n += cqp_dequeue(ctx->tbl + i, cop + n, num - n); 329 330 ctx->last_qp = i; 331 return n; 332 } 333 334 /* 335 * dequeue packets from crypto-queues and finalize processing. 336 */ 337 void 338 ipsec_cqp_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf) 339 { 340 uint64_t satp; 341 uint32_t i, k, n, ng; 342 struct rte_ipsec_session *ss; 343 struct traffic_type *out; 344 struct rte_ipsec_group *pg; 345 struct rte_crypto_op *cop[RTE_DIM(trf->ipsec.pkts)]; 346 struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)]; 347 348 trf->ip4.num = 0; 349 trf->ip6.num = 0; 350 351 out = &trf->ipsec; 352 353 /* dequeue completed crypto-ops */ 354 n = ctx_dequeue(ctx, cop, RTE_DIM(cop)); 355 if (n == 0) 356 return; 357 358 /* group them by ipsec session */ 359 ng = rte_ipsec_pkt_crypto_group((const struct rte_crypto_op **) 360 (uintptr_t)cop, out->pkts, grp, n); 361 362 /* process each group of packets */ 363 for (i = 0; i != ng; i++) { 364 365 pg = grp + i; 366 ss = pg->id.ptr; 367 satp = rte_ipsec_sa_type(ss->sa); 368 369 k = rte_ipsec_pkt_process(ss, pg->m, pg->cnt); 370 copy_to_trf(trf, satp, pg->m, k); 371 372 /* free bad packets, if any */ 373 free_pkts(pg->m + k, pg->cnt - k); 374 375 n -= pg->cnt; 376 } 377 378 /* we should never have packet with unknown SA here */ 379 RTE_VERIFY(n == 0); 380 } 381