1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2020 Intel Corporation 3 */ 4 #include <sys/types.h> 5 #include <netinet/in.h> 6 #include <netinet/ip.h> 7 8 #include <rte_branch_prediction.h> 9 #include <rte_log.h> 10 #include <rte_cryptodev.h> 11 #include <rte_ethdev.h> 12 #include <rte_mbuf.h> 13 14 #include "ipsec.h" 15 #include "ipsec-secgw.h" 16 17 #define SATP_OUT_IPV4(t) \ 18 ((((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TRANS && \ 19 (((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \ 20 ((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4) 21 22 /* helper routine to free bulk of crypto-ops and related packets */ 23 static inline void 24 free_cops(struct rte_crypto_op *cop[], uint32_t n) 25 { 26 uint32_t i; 27 28 for (i = 0; i != n; i++) 29 rte_pktmbuf_free(cop[i]->sym->m_src); 30 } 31 32 /* helper routine to enqueue bulk of crypto ops */ 33 static inline void 34 enqueue_cop_bulk(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num) 35 { 36 uint32_t i, k, len, n; 37 38 len = cqp->len; 39 40 /* 41 * if cqp is empty and we have enough ops, 42 * then queue them to the PMD straightway. 43 */ 44 if (num >= RTE_DIM(cqp->buf) * 3 / 4 && len == 0) { 45 n = rte_cryptodev_enqueue_burst(cqp->id, cqp->qp, cop, num); 46 cqp->in_flight += n; 47 free_cops(cop + n, num - n); 48 return; 49 } 50 51 k = 0; 52 53 do { 54 n = RTE_DIM(cqp->buf) - len; 55 n = RTE_MIN(num - k, n); 56 57 /* put packets into cqp */ 58 for (i = 0; i != n; i++) 59 cqp->buf[len + i] = cop[k + i]; 60 61 len += n; 62 k += n; 63 64 /* if cqp is full then, enqueue crypto-ops to PMD */ 65 if (len == RTE_DIM(cqp->buf)) { 66 n = rte_cryptodev_enqueue_burst(cqp->id, cqp->qp, 67 cqp->buf, len); 68 cqp->in_flight += n; 69 free_cops(cqp->buf + n, len - n); 70 len = 0; 71 } 72 73 74 } while (k != num); 75 76 cqp->len = len; 77 } 78 79 static inline int 80 fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx, 81 struct ipsec_sa *sa) 82 { 83 int32_t rc; 84 85 /* setup crypto section */ 86 if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE || 87 ss->type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) { 88 RTE_ASSERT(ss->crypto.ses == NULL); 89 rc = create_lookaside_session(ctx, sa, ss); 90 if (rc != 0) 91 return rc; 92 /* setup session action type */ 93 } else if (ss->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) { 94 RTE_ASSERT(ss->security.ses == NULL); 95 rc = create_lookaside_session(ctx, sa, ss); 96 if (rc != 0) 97 return rc; 98 } else 99 RTE_ASSERT(0); 100 101 rc = rte_ipsec_session_prepare(ss); 102 if (rc != 0) 103 memset(ss, 0, sizeof(*ss)); 104 105 return rc; 106 } 107 108 /* 109 * group input packets byt the SA they belong to. 110 */ 111 static uint32_t 112 sa_group(void *sa_ptr[], struct rte_mbuf *pkts[], 113 struct rte_ipsec_group grp[], uint32_t num) 114 { 115 uint32_t i, n, spi; 116 void *sa; 117 void * const nosa = &spi; 118 119 sa = nosa; 120 grp[0].m = pkts; 121 for (i = 0, n = 0; i != num; i++) { 122 123 if (sa != sa_ptr[i]) { 124 grp[n].cnt = pkts + i - grp[n].m; 125 n += (sa != nosa); 126 grp[n].id.ptr = sa_ptr[i]; 127 grp[n].m = pkts + i; 128 sa = sa_ptr[i]; 129 } 130 } 131 132 /* terminate last group */ 133 if (sa != nosa) { 134 grp[n].cnt = pkts + i - grp[n].m; 135 n++; 136 } 137 138 return n; 139 } 140 141 /* 142 * helper function, splits processed packets into ipv4/ipv6 traffic. 143 */ 144 static inline void 145 copy_to_trf(struct ipsec_traffic *trf, uint64_t satp, struct rte_mbuf *mb[], 146 uint32_t num) 147 { 148 uint32_t j, ofs, s; 149 struct traffic_type *out; 150 151 /* 152 * determine traffic type(ipv4/ipv6) and offset for ACL classify 153 * based on SA type 154 */ 155 if ((satp & RTE_IPSEC_SATP_DIR_MASK) == RTE_IPSEC_SATP_DIR_IB) { 156 if ((satp & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4) { 157 out = &trf->ip4; 158 ofs = offsetof(struct ip, ip_p); 159 } else { 160 out = &trf->ip6; 161 ofs = offsetof(struct ip6_hdr, ip6_nxt); 162 } 163 } else if (SATP_OUT_IPV4(satp)) { 164 out = &trf->ip4; 165 ofs = offsetof(struct ip, ip_p); 166 } else { 167 out = &trf->ip6; 168 ofs = offsetof(struct ip6_hdr, ip6_nxt); 169 } 170 171 for (j = 0, s = out->num; j != num; j++) { 172 out->data[s + j] = rte_pktmbuf_mtod_offset(mb[j], 173 void *, ofs); 174 out->pkts[s + j] = mb[j]; 175 } 176 177 out->num += num; 178 } 179 180 static uint32_t 181 ipsec_prepare_crypto_group(struct ipsec_ctx *ctx, struct ipsec_sa *sa, 182 struct rte_ipsec_session *ips, struct rte_mbuf **m, 183 unsigned int cnt) 184 { 185 struct cdev_qp *cqp; 186 struct rte_crypto_op *cop[cnt]; 187 uint32_t j, k; 188 struct ipsec_mbuf_metadata *priv; 189 190 cqp = &ctx->tbl[sa->cdev_id_qp]; 191 192 /* for that app each mbuf has it's own crypto op */ 193 for (j = 0; j != cnt; j++) { 194 priv = get_priv(m[j]); 195 cop[j] = &priv->cop; 196 /* 197 * this is just to satisfy inbound_sa_check() 198 * should be removed in future. 199 */ 200 priv->sa = sa; 201 } 202 203 /* prepare and enqueue crypto ops */ 204 k = rte_ipsec_pkt_crypto_prepare(ips, m, cop, cnt); 205 if (k != 0) 206 enqueue_cop_bulk(cqp, cop, k); 207 208 return k; 209 } 210 211 /* 212 * helper routine for inline and cpu(synchronous) processing 213 * this is just to satisfy inbound_sa_check() and get_hop_for_offload_pkt(). 214 * Should be removed in future. 215 */ 216 static inline void 217 prep_process_group(void *sa, struct rte_mbuf *mb[], uint32_t cnt) 218 { 219 uint32_t j; 220 struct ipsec_mbuf_metadata *priv; 221 222 for (j = 0; j != cnt; j++) { 223 priv = get_priv(mb[j]); 224 priv->sa = sa; 225 /* setup TSO related fields if TSO enabled*/ 226 if (priv->sa->mss) { 227 uint32_t ptype = mb[j]->packet_type; 228 /* only TCP is supported */ 229 if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) { 230 mb[j]->tso_segsz = priv->sa->mss; 231 if ((IS_TUNNEL(priv->sa->flags))) { 232 mb[j]->outer_l3_len = mb[j]->l3_len; 233 mb[j]->outer_l2_len = mb[j]->l2_len; 234 mb[j]->ol_flags |= 235 RTE_MBUF_F_TX_TUNNEL_ESP; 236 if (RTE_ETH_IS_IPV4_HDR(ptype)) 237 mb[j]->ol_flags |= 238 RTE_MBUF_F_TX_OUTER_IP_CKSUM; 239 } 240 mb[j]->l4_len = sizeof(struct rte_tcp_hdr); 241 mb[j]->ol_flags |= (RTE_MBUF_F_TX_TCP_SEG | 242 RTE_MBUF_F_TX_TCP_CKSUM); 243 if (RTE_ETH_IS_IPV4_HDR(ptype)) 244 mb[j]->ol_flags |= 245 RTE_MBUF_F_TX_OUTER_IPV4; 246 else 247 mb[j]->ol_flags |= 248 RTE_MBUF_F_TX_OUTER_IPV6; 249 } 250 } 251 } 252 } 253 254 /* 255 * finish processing of packets successfully decrypted by an inline processor 256 */ 257 static uint32_t 258 ipsec_process_inline_group(struct rte_ipsec_session *ips, void *sa, 259 struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t cnt) 260 { 261 uint64_t satp; 262 uint32_t k; 263 264 /* get SA type */ 265 satp = rte_ipsec_sa_type(ips->sa); 266 prep_process_group(sa, mb, cnt); 267 268 k = rte_ipsec_pkt_process(ips, mb, cnt); 269 copy_to_trf(trf, satp, mb, k); 270 return k; 271 } 272 273 /* 274 * process packets synchronously 275 */ 276 static uint32_t 277 ipsec_process_cpu_group(struct rte_ipsec_session *ips, void *sa, 278 struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t cnt) 279 { 280 uint64_t satp; 281 uint32_t k; 282 283 /* get SA type */ 284 satp = rte_ipsec_sa_type(ips->sa); 285 prep_process_group(sa, mb, cnt); 286 287 k = rte_ipsec_pkt_cpu_prepare(ips, mb, cnt); 288 k = rte_ipsec_pkt_process(ips, mb, k); 289 copy_to_trf(trf, satp, mb, k); 290 return k; 291 } 292 293 /* 294 * Process ipsec packets. 295 * If packet belong to SA that is subject of inline-crypto, 296 * then process it immediately. 297 * Otherwise do necessary preparations and queue it to related 298 * crypto-dev queue. 299 */ 300 void 301 ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf) 302 { 303 uint32_t i, k, n; 304 struct ipsec_sa *sa; 305 struct rte_ipsec_group *pg; 306 struct rte_ipsec_session *ips; 307 struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)]; 308 309 n = sa_group(trf->ipsec.saptr, trf->ipsec.pkts, grp, trf->ipsec.num); 310 311 for (i = 0; i != n; i++) { 312 313 pg = grp + i; 314 sa = ipsec_mask_saptr(pg->id.ptr); 315 316 /* fallback to cryptodev with RX packets which inline 317 * processor was unable to process 318 */ 319 if (sa != NULL) 320 ips = (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) ? 321 ipsec_get_fallback_session(sa) : 322 ipsec_get_primary_session(sa); 323 324 /* no valid HW session for that SA, try to create one */ 325 if (sa == NULL || (ips->crypto.ses == NULL && 326 fill_ipsec_session(ips, ctx, sa) != 0)) 327 k = 0; 328 329 /* process packets inline */ 330 else { 331 switch (ips->type) { 332 /* enqueue packets to crypto dev */ 333 case RTE_SECURITY_ACTION_TYPE_NONE: 334 case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL: 335 k = ipsec_prepare_crypto_group(ctx, sa, ips, 336 pg->m, pg->cnt); 337 break; 338 case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO: 339 case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL: 340 k = ipsec_process_inline_group(ips, sa, 341 trf, pg->m, pg->cnt); 342 break; 343 case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO: 344 k = ipsec_process_cpu_group(ips, sa, 345 trf, pg->m, pg->cnt); 346 break; 347 default: 348 k = 0; 349 } 350 } 351 352 /* drop packets that cannot be enqueued/processed */ 353 if (k != pg->cnt) 354 free_pkts(pg->m + k, pg->cnt - k); 355 } 356 } 357 358 static inline uint32_t 359 cqp_dequeue(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num) 360 { 361 uint32_t n; 362 363 if (cqp->in_flight == 0) 364 return 0; 365 366 n = rte_cryptodev_dequeue_burst(cqp->id, cqp->qp, cop, num); 367 RTE_ASSERT(cqp->in_flight >= n); 368 cqp->in_flight -= n; 369 370 return n; 371 } 372 373 static inline uint32_t 374 ctx_dequeue(struct ipsec_ctx *ctx, struct rte_crypto_op *cop[], uint32_t num) 375 { 376 uint32_t i, n; 377 378 n = 0; 379 380 for (i = ctx->last_qp; n != num && i != ctx->nb_qps; i++) 381 n += cqp_dequeue(ctx->tbl + i, cop + n, num - n); 382 383 for (i = 0; n != num && i != ctx->last_qp; i++) 384 n += cqp_dequeue(ctx->tbl + i, cop + n, num - n); 385 386 ctx->last_qp = i; 387 return n; 388 } 389 390 /* 391 * dequeue packets from crypto-queues and finalize processing. 392 */ 393 void 394 ipsec_cqp_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf) 395 { 396 uint64_t satp; 397 uint32_t i, k, n, ng; 398 struct rte_ipsec_session *ss; 399 struct traffic_type *out; 400 struct rte_ipsec_group *pg; 401 struct rte_crypto_op *cop[RTE_DIM(trf->ipsec.pkts)]; 402 struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)]; 403 404 trf->ip4.num = 0; 405 trf->ip6.num = 0; 406 407 out = &trf->ipsec; 408 409 /* dequeue completed crypto-ops */ 410 n = ctx_dequeue(ctx, cop, RTE_DIM(cop)); 411 if (n == 0) 412 return; 413 414 /* group them by ipsec session */ 415 ng = rte_ipsec_pkt_crypto_group((const struct rte_crypto_op **) 416 (uintptr_t)cop, out->pkts, grp, n); 417 418 /* process each group of packets */ 419 for (i = 0; i != ng; i++) { 420 421 pg = grp + i; 422 ss = pg->id.ptr; 423 satp = rte_ipsec_sa_type(ss->sa); 424 425 k = rte_ipsec_pkt_process(ss, pg->m, pg->cnt); 426 copy_to_trf(trf, satp, pg->m, k); 427 428 /* free bad packets, if any */ 429 free_pkts(pg->m + k, pg->cnt - k); 430 431 n -= pg->cnt; 432 } 433 434 /* we should never have packet with unknown SA here */ 435 RTE_VERIFY(n == 0); 436 } 437