1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2020 Intel Corporation 3 */ 4 #include <sys/types.h> 5 #include <netinet/in.h> 6 #include <netinet/ip.h> 7 8 #include <rte_branch_prediction.h> 9 #include <rte_log.h> 10 #include <rte_crypto.h> 11 #include <rte_security.h> 12 #include <rte_cryptodev.h> 13 #include <rte_ipsec.h> 14 #include <rte_ethdev.h> 15 #include <rte_mbuf.h> 16 #include <rte_hash.h> 17 18 #include "ipsec.h" 19 #include "esp.h" 20 21 static inline void 22 set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec) 23 { 24 if (ipsec->mode == RTE_SECURITY_IPSEC_SA_MODE_TUNNEL) { 25 struct rte_security_ipsec_tunnel_param *tunnel = 26 &ipsec->tunnel; 27 if (IS_IP4_TUNNEL(sa->flags)) { 28 tunnel->type = 29 RTE_SECURITY_IPSEC_TUNNEL_IPV4; 30 tunnel->ipv4.ttl = IPDEFTTL; 31 32 memcpy((uint8_t *)&tunnel->ipv4.src_ip, 33 (uint8_t *)&sa->src.ip.ip4, 4); 34 35 memcpy((uint8_t *)&tunnel->ipv4.dst_ip, 36 (uint8_t *)&sa->dst.ip.ip4, 4); 37 } else if (IS_IP6_TUNNEL(sa->flags)) { 38 tunnel->type = 39 RTE_SECURITY_IPSEC_TUNNEL_IPV6; 40 tunnel->ipv6.hlimit = IPDEFTTL; 41 tunnel->ipv6.dscp = 0; 42 tunnel->ipv6.flabel = 0; 43 44 memcpy((uint8_t *)&tunnel->ipv6.src_addr, 45 (uint8_t *)&sa->src.ip.ip6.ip6_b, 16); 46 47 memcpy((uint8_t *)&tunnel->ipv6.dst_addr, 48 (uint8_t *)&sa->dst.ip.ip6.ip6_b, 16); 49 } 50 /* TODO support for Transport */ 51 } 52 ipsec->replay_win_sz = app_sa_prm.window_size; 53 ipsec->options.esn = app_sa_prm.enable_esn; 54 ipsec->options.udp_encap = sa->udp_encap; 55 } 56 57 int 58 create_lookaside_session(struct ipsec_ctx *ipsec_ctx_lcore[], 59 struct socket_ctx *skt_ctx, struct ipsec_sa *sa, 60 struct rte_ipsec_session *ips) 61 { 62 uint16_t cdev_id = RTE_CRYPTO_MAX_DEVS; 63 struct rte_cryptodev_info cdev_info; 64 unsigned long cdev_id_qp = 0; 65 struct cdev_key key = { 0 }; 66 struct ipsec_ctx *ipsec_ctx; 67 uint32_t lcore_id; 68 int32_t ret = 0; 69 70 RTE_LCORE_FOREACH(lcore_id) { 71 ipsec_ctx = ipsec_ctx_lcore[lcore_id]; 72 73 /* Core is not bound to any cryptodev, skip it */ 74 if (ipsec_ctx->cdev_map == NULL) 75 continue; 76 77 /* Looking for cryptodev, which can handle this SA */ 78 key.lcore_id = (uint8_t)lcore_id; 79 key.cipher_algo = (uint8_t)sa->cipher_algo; 80 key.auth_algo = (uint8_t)sa->auth_algo; 81 key.aead_algo = (uint8_t)sa->aead_algo; 82 83 ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key, 84 (void **)&cdev_id_qp); 85 if (ret == -ENOENT) 86 continue; 87 if (ret < 0) { 88 RTE_LOG(ERR, IPSEC, 89 "No cryptodev: core %u, cipher_algo %u, " 90 "auth_algo %u, aead_algo %u\n", 91 key.lcore_id, 92 key.cipher_algo, 93 key.auth_algo, 94 key.aead_algo); 95 return ret; 96 } 97 98 /* Verify that all cores are using same cryptodev for current 99 * algorithm combination, required by SA. 100 * Current cryptodev mapping process will map SA to the first 101 * cryptodev that matches requirements, so it's a double check, 102 * not an additional restriction. 103 */ 104 if (cdev_id == RTE_CRYPTO_MAX_DEVS) 105 cdev_id = ipsec_ctx->tbl[cdev_id_qp].id; 106 else if (cdev_id != ipsec_ctx->tbl[cdev_id_qp].id) { 107 RTE_LOG(ERR, IPSEC, 108 "SA mapping to multiple cryptodevs is " 109 "not supported!"); 110 return -EINVAL; 111 } 112 113 /* Store per core queue pair information */ 114 sa->cqp[lcore_id] = &ipsec_ctx->tbl[cdev_id_qp]; 115 } 116 if (cdev_id == RTE_CRYPTO_MAX_DEVS) { 117 RTE_LOG(WARNING, IPSEC, "No cores found to handle SA\n"); 118 return 0; 119 } 120 121 RTE_LOG(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev " 122 "%u\n", sa->spi, cdev_id); 123 124 if (ips->type != RTE_SECURITY_ACTION_TYPE_NONE && 125 ips->type != RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) { 126 struct rte_security_session_conf sess_conf = { 127 .action_type = ips->type, 128 .protocol = RTE_SECURITY_PROTOCOL_IPSEC, 129 {.ipsec = { 130 .spi = sa->spi, 131 .salt = sa->salt, 132 .options = { 0 }, 133 .replay_win_sz = 0, 134 .direction = sa->direction, 135 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, 136 .mode = (IS_TUNNEL(sa->flags)) ? 137 RTE_SECURITY_IPSEC_SA_MODE_TUNNEL : 138 RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT, 139 } }, 140 .crypto_xform = sa->xforms, 141 .userdata = NULL, 142 143 }; 144 145 if (ips->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) { 146 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 147 rte_cryptodev_get_sec_ctx( 148 cdev_id); 149 150 /* Set IPsec parameters in conf */ 151 set_ipsec_conf(sa, &(sess_conf.ipsec)); 152 153 ips->security.ses = rte_security_session_create(ctx, 154 &sess_conf, skt_ctx->session_pool); 155 if (ips->security.ses == NULL) { 156 RTE_LOG(ERR, IPSEC, 157 "SEC Session init failed: err: %d\n", ret); 158 return -1; 159 } 160 ips->security.ctx = ctx; 161 } else { 162 RTE_LOG(ERR, IPSEC, "Inline not supported\n"); 163 return -1; 164 } 165 } else { 166 if (ips->type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) { 167 struct rte_cryptodev_info info; 168 169 rte_cryptodev_info_get(cdev_id, &info); 170 if (!(info.feature_flags & 171 RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO)) 172 return -ENOTSUP; 173 174 } 175 ips->crypto.dev_id = cdev_id; 176 ips->crypto.ses = rte_cryptodev_sym_session_create(cdev_id, 177 sa->xforms, skt_ctx->session_pool); 178 179 rte_cryptodev_info_get(cdev_id, &cdev_info); 180 } 181 182 return 0; 183 } 184 185 int 186 create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa, 187 struct rte_ipsec_session *ips) 188 { 189 int32_t ret = 0; 190 struct rte_security_ctx *sec_ctx; 191 struct rte_security_session_conf sess_conf = { 192 .action_type = ips->type, 193 .protocol = RTE_SECURITY_PROTOCOL_IPSEC, 194 {.ipsec = { 195 .spi = sa->spi, 196 .salt = sa->salt, 197 .options = { 0 }, 198 .replay_win_sz = 0, 199 .direction = sa->direction, 200 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP 201 } }, 202 .crypto_xform = sa->xforms, 203 .userdata = NULL, 204 }; 205 206 if (IS_TRANSPORT(sa->flags)) { 207 sess_conf.ipsec.mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT; 208 if (IS_IP4(sa->flags)) { 209 sess_conf.ipsec.tunnel.type = 210 RTE_SECURITY_IPSEC_TUNNEL_IPV4; 211 212 sess_conf.ipsec.tunnel.ipv4.src_ip.s_addr = 213 sa->src.ip.ip4; 214 sess_conf.ipsec.tunnel.ipv4.dst_ip.s_addr = 215 sa->dst.ip.ip4; 216 } else if (IS_IP6(sa->flags)) { 217 sess_conf.ipsec.tunnel.type = 218 RTE_SECURITY_IPSEC_TUNNEL_IPV6; 219 220 memcpy(sess_conf.ipsec.tunnel.ipv6.src_addr.s6_addr, 221 sa->src.ip.ip6.ip6_b, 16); 222 memcpy(sess_conf.ipsec.tunnel.ipv6.dst_addr.s6_addr, 223 sa->dst.ip.ip6.ip6_b, 16); 224 } 225 } else if (IS_TUNNEL(sa->flags)) { 226 sess_conf.ipsec.mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL; 227 228 if (IS_IP4(sa->flags)) { 229 sess_conf.ipsec.tunnel.type = 230 RTE_SECURITY_IPSEC_TUNNEL_IPV4; 231 232 sess_conf.ipsec.tunnel.ipv4.src_ip.s_addr = 233 sa->src.ip.ip4; 234 sess_conf.ipsec.tunnel.ipv4.dst_ip.s_addr = 235 sa->dst.ip.ip4; 236 } else if (IS_IP6(sa->flags)) { 237 sess_conf.ipsec.tunnel.type = 238 RTE_SECURITY_IPSEC_TUNNEL_IPV6; 239 240 memcpy(sess_conf.ipsec.tunnel.ipv6.src_addr.s6_addr, 241 sa->src.ip.ip6.ip6_b, 16); 242 memcpy(sess_conf.ipsec.tunnel.ipv6.dst_addr.s6_addr, 243 sa->dst.ip.ip6.ip6_b, 16); 244 } else { 245 RTE_LOG(ERR, IPSEC, "invalid tunnel type\n"); 246 return -1; 247 } 248 } 249 250 if (sa->udp_encap) { 251 sess_conf.ipsec.options.udp_encap = 1; 252 sess_conf.ipsec.udp.sport = htons(sa->udp.sport); 253 sess_conf.ipsec.udp.dport = htons(sa->udp.dport); 254 } 255 256 if (sa->esn > 0) { 257 sess_conf.ipsec.options.esn = 1; 258 sess_conf.ipsec.esn.value = sa->esn; 259 } 260 261 262 RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n", 263 sa->spi, sa->portid); 264 265 if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) { 266 struct rte_flow_error err; 267 const struct rte_security_capability *sec_cap; 268 int ret = 0; 269 270 sec_ctx = (struct rte_security_ctx *) 271 rte_eth_dev_get_sec_ctx( 272 sa->portid); 273 if (sec_ctx == NULL) { 274 RTE_LOG(ERR, IPSEC, 275 " rte_eth_dev_get_sec_ctx failed\n"); 276 return -1; 277 } 278 279 ips->security.ses = rte_security_session_create(sec_ctx, 280 &sess_conf, skt_ctx->session_pool); 281 if (ips->security.ses == NULL) { 282 RTE_LOG(ERR, IPSEC, 283 "SEC Session init failed: err: %d\n", ret); 284 return -1; 285 } 286 287 sec_cap = rte_security_capabilities_get(sec_ctx); 288 289 /* iterate until ESP tunnel*/ 290 while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) { 291 if (sec_cap->action == ips->type && 292 sec_cap->protocol == 293 RTE_SECURITY_PROTOCOL_IPSEC && 294 sec_cap->ipsec.mode == 295 RTE_SECURITY_IPSEC_SA_MODE_TUNNEL && 296 sec_cap->ipsec.direction == sa->direction) 297 break; 298 sec_cap++; 299 } 300 301 if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) { 302 RTE_LOG(ERR, IPSEC, 303 "No suitable security capability found\n"); 304 return -1; 305 } 306 307 ips->security.ol_flags = sec_cap->ol_flags; 308 ips->security.ctx = sec_ctx; 309 sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH; 310 311 if (IS_IP6(sa->flags)) { 312 sa->pattern[1].mask = &rte_flow_item_ipv6_mask; 313 sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV6; 314 sa->pattern[1].spec = &sa->ipv6_spec; 315 316 memcpy(sa->ipv6_spec.hdr.dst_addr, 317 sa->dst.ip.ip6.ip6_b, 16); 318 memcpy(sa->ipv6_spec.hdr.src_addr, 319 sa->src.ip.ip6.ip6_b, 16); 320 } else if (IS_IP4(sa->flags)) { 321 sa->pattern[1].mask = &rte_flow_item_ipv4_mask; 322 sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4; 323 sa->pattern[1].spec = &sa->ipv4_spec; 324 325 sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4; 326 sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4; 327 } 328 329 sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi); 330 331 if (sa->udp_encap) { 332 333 sa->udp_spec.hdr.dst_port = 334 rte_cpu_to_be_16(sa->udp.dport); 335 sa->udp_spec.hdr.src_port = 336 rte_cpu_to_be_16(sa->udp.sport); 337 338 sa->pattern[2].mask = &rte_flow_item_udp_mask; 339 sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_UDP; 340 sa->pattern[2].spec = &sa->udp_spec; 341 342 sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_ESP; 343 sa->pattern[3].spec = &sa->esp_spec; 344 sa->pattern[3].mask = &rte_flow_item_esp_mask; 345 346 sa->pattern[4].type = RTE_FLOW_ITEM_TYPE_END; 347 } else { 348 sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP; 349 sa->pattern[2].spec = &sa->esp_spec; 350 sa->pattern[2].mask = &rte_flow_item_esp_mask; 351 352 sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END; 353 } 354 355 sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY; 356 sa->action[0].conf = ips->security.ses; 357 358 sa->action[1].type = RTE_FLOW_ACTION_TYPE_END; 359 360 sa->attr.egress = (sa->direction == 361 RTE_SECURITY_IPSEC_SA_DIR_EGRESS); 362 sa->attr.ingress = (sa->direction == 363 RTE_SECURITY_IPSEC_SA_DIR_INGRESS); 364 if (sa->attr.ingress) { 365 uint8_t rss_key[64]; 366 struct rte_eth_rss_conf rss_conf = { 367 .rss_key = rss_key, 368 .rss_key_len = sizeof(rss_key), 369 }; 370 struct rte_eth_dev_info dev_info; 371 uint16_t queue[RTE_MAX_QUEUES_PER_PORT]; 372 struct rte_flow_action_rss action_rss; 373 unsigned int i; 374 unsigned int j; 375 376 /* Don't create flow if default flow is created */ 377 if (flow_info_tbl[sa->portid].rx_def_flow) 378 return 0; 379 380 ret = rte_eth_dev_info_get(sa->portid, &dev_info); 381 if (ret != 0) { 382 RTE_LOG(ERR, IPSEC, 383 "Error during getting device (port %u) info: %s\n", 384 sa->portid, strerror(-ret)); 385 return ret; 386 } 387 388 sa->action[2].type = RTE_FLOW_ACTION_TYPE_END; 389 /* Try RSS. */ 390 sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS; 391 sa->action[1].conf = &action_rss; 392 ret = rte_eth_dev_rss_hash_conf_get(sa->portid, 393 &rss_conf); 394 if (ret != 0) { 395 RTE_LOG(ERR, IPSEC, 396 "rte_eth_dev_rss_hash_conf_get:ret=%d\n", 397 ret); 398 return -1; 399 } 400 for (i = 0, j = 0; i < dev_info.nb_rx_queues; ++i) 401 queue[j++] = i; 402 403 action_rss = (struct rte_flow_action_rss){ 404 .types = rss_conf.rss_hf, 405 .key_len = rss_conf.rss_key_len, 406 .queue_num = j, 407 .key = rss_key, 408 .queue = queue, 409 }; 410 ret = rte_flow_validate(sa->portid, &sa->attr, 411 sa->pattern, sa->action, 412 &err); 413 if (!ret) 414 goto flow_create; 415 /* Try Queue. */ 416 sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE; 417 sa->action[1].conf = 418 &(struct rte_flow_action_queue){ 419 .index = 0, 420 }; 421 ret = rte_flow_validate(sa->portid, &sa->attr, 422 sa->pattern, sa->action, 423 &err); 424 /* Try End. */ 425 sa->action[1].type = RTE_FLOW_ACTION_TYPE_END; 426 sa->action[1].conf = NULL; 427 ret = rte_flow_validate(sa->portid, &sa->attr, 428 sa->pattern, sa->action, 429 &err); 430 if (ret) 431 goto flow_create_failure; 432 } else if (sa->attr.egress && 433 (ips->security.ol_flags & 434 RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) { 435 sa->action[1].type = 436 RTE_FLOW_ACTION_TYPE_PASSTHRU; 437 sa->action[2].type = 438 RTE_FLOW_ACTION_TYPE_END; 439 } 440 flow_create: 441 sa->flow = rte_flow_create(sa->portid, 442 &sa->attr, sa->pattern, sa->action, &err); 443 if (sa->flow == NULL) { 444 flow_create_failure: 445 RTE_LOG(ERR, IPSEC, 446 "Failed to create ipsec flow msg: %s\n", 447 err.message); 448 return -1; 449 } 450 } else if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) { 451 const struct rte_security_capability *sec_cap; 452 453 sec_ctx = (struct rte_security_ctx *) 454 rte_eth_dev_get_sec_ctx(sa->portid); 455 456 if (sec_ctx == NULL) { 457 RTE_LOG(ERR, IPSEC, 458 "Ethernet device doesn't have security features registered\n"); 459 return -1; 460 } 461 462 /* Set IPsec parameters in conf */ 463 set_ipsec_conf(sa, &(sess_conf.ipsec)); 464 465 /* Save SA as userdata for the security session. When 466 * the packet is received, this userdata will be 467 * retrieved using the metadata from the packet. 468 * 469 * The PMD is expected to set similar metadata for other 470 * operations, like rte_eth_event, which are tied to 471 * security session. In such cases, the userdata could 472 * be obtained to uniquely identify the security 473 * parameters denoted. 474 */ 475 476 sess_conf.userdata = (void *) sa; 477 478 ips->security.ses = rte_security_session_create(sec_ctx, 479 &sess_conf, skt_ctx->session_pool); 480 if (ips->security.ses == NULL) { 481 RTE_LOG(ERR, IPSEC, 482 "SEC Session init failed: err: %d\n", ret); 483 return -1; 484 } 485 486 sec_cap = rte_security_capabilities_get(sec_ctx); 487 if (sec_cap == NULL) { 488 RTE_LOG(ERR, IPSEC, 489 "No capabilities registered\n"); 490 return -1; 491 } 492 493 /* iterate until ESP tunnel*/ 494 while (sec_cap->action != 495 RTE_SECURITY_ACTION_TYPE_NONE) { 496 if (sec_cap->action == ips->type && 497 sec_cap->protocol == 498 RTE_SECURITY_PROTOCOL_IPSEC && 499 sec_cap->ipsec.mode == 500 sess_conf.ipsec.mode && 501 sec_cap->ipsec.direction == sa->direction) 502 break; 503 sec_cap++; 504 } 505 506 if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) { 507 RTE_LOG(ERR, IPSEC, 508 "No suitable security capability found\n"); 509 return -1; 510 } 511 512 ips->security.ol_flags = sec_cap->ol_flags; 513 ips->security.ctx = sec_ctx; 514 } 515 516 return 0; 517 } 518 519 int 520 create_ipsec_esp_flow(struct ipsec_sa *sa) 521 { 522 int ret = 0; 523 struct rte_flow_error err = {}; 524 if (sa->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { 525 RTE_LOG(ERR, IPSEC, 526 "No Flow director rule for Egress traffic\n"); 527 return -1; 528 } 529 if (sa->flags == TRANSPORT) { 530 RTE_LOG(ERR, IPSEC, 531 "No Flow director rule for transport mode\n"); 532 return -1; 533 } 534 sa->action[0].type = RTE_FLOW_ACTION_TYPE_QUEUE; 535 sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH; 536 sa->action[0].conf = &(struct rte_flow_action_queue) { 537 .index = sa->fdir_qid, 538 }; 539 sa->attr.egress = 0; 540 sa->attr.ingress = 1; 541 if (IS_IP6(sa->flags)) { 542 sa->pattern[1].mask = &rte_flow_item_ipv6_mask; 543 sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV6; 544 sa->pattern[1].spec = &sa->ipv6_spec; 545 memcpy(sa->ipv6_spec.hdr.dst_addr, 546 sa->dst.ip.ip6.ip6_b, sizeof(sa->dst.ip.ip6.ip6_b)); 547 memcpy(sa->ipv6_spec.hdr.src_addr, 548 sa->src.ip.ip6.ip6_b, sizeof(sa->src.ip.ip6.ip6_b)); 549 sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP; 550 sa->pattern[2].spec = &sa->esp_spec; 551 sa->pattern[2].mask = &rte_flow_item_esp_mask; 552 sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi); 553 sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END; 554 } else if (IS_IP4(sa->flags)) { 555 sa->pattern[1].mask = &rte_flow_item_ipv4_mask; 556 sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4; 557 sa->pattern[1].spec = &sa->ipv4_spec; 558 sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4; 559 sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4; 560 sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP; 561 sa->pattern[2].spec = &sa->esp_spec; 562 sa->pattern[2].mask = &rte_flow_item_esp_mask; 563 sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi); 564 sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END; 565 } 566 sa->action[1].type = RTE_FLOW_ACTION_TYPE_END; 567 568 ret = rte_flow_validate(sa->portid, &sa->attr, sa->pattern, sa->action, 569 &err); 570 if (ret < 0) { 571 RTE_LOG(ERR, IPSEC, "Flow validation failed %s\n", err.message); 572 return ret; 573 } 574 575 sa->flow = rte_flow_create(sa->portid, &sa->attr, sa->pattern, 576 sa->action, &err); 577 if (!sa->flow) { 578 RTE_LOG(ERR, IPSEC, "Flow creation failed %s\n", err.message); 579 return -1; 580 } 581 582 return 0; 583 } 584 585 /* 586 * queue crypto-ops into PMD queue. 587 */ 588 void 589 enqueue_cop_burst(struct cdev_qp *cqp) 590 { 591 uint32_t i, len, ret; 592 593 len = cqp->len; 594 ret = rte_cryptodev_enqueue_burst(cqp->id, cqp->qp, cqp->buf, len); 595 if (ret < len) { 596 RTE_LOG_DP(DEBUG, IPSEC, "Cryptodev %u queue %u:" 597 " enqueued %u crypto ops out of %u\n", 598 cqp->id, cqp->qp, ret, len); 599 /* drop packets that we fail to enqueue */ 600 for (i = ret; i < len; i++) 601 free_pkts(&cqp->buf[i]->sym->m_src, 1); 602 } 603 cqp->in_flight += ret; 604 cqp->len = 0; 605 } 606 607 static inline void 608 enqueue_cop(struct cdev_qp *cqp, struct rte_crypto_op *cop) 609 { 610 cqp->buf[cqp->len++] = cop; 611 612 if (cqp->len == MAX_PKT_BURST) 613 enqueue_cop_burst(cqp); 614 } 615 616 static inline void 617 ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, 618 struct rte_mbuf *pkts[], void *sas[], 619 uint16_t nb_pkts) 620 { 621 int32_t ret = 0, i; 622 struct ipsec_mbuf_metadata *priv; 623 struct rte_crypto_sym_op *sym_cop; 624 struct ipsec_sa *sa; 625 struct rte_ipsec_session *ips; 626 627 for (i = 0; i < nb_pkts; i++) { 628 if (unlikely(sas[i] == NULL)) { 629 free_pkts(&pkts[i], 1); 630 continue; 631 } 632 633 rte_prefetch0(sas[i]); 634 rte_prefetch0(pkts[i]); 635 636 priv = get_priv(pkts[i]); 637 sa = ipsec_mask_saptr(sas[i]); 638 priv->sa = sa; 639 ips = ipsec_get_primary_session(sa); 640 641 switch (ips->type) { 642 case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL: 643 priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; 644 priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 645 646 rte_prefetch0(&priv->sym_cop); 647 648 if (unlikely(ips->security.ses == NULL)) { 649 free_pkts(&pkts[i], 1); 650 continue; 651 } 652 653 if (unlikely((pkts[i]->packet_type & 654 (RTE_PTYPE_TUNNEL_MASK | 655 RTE_PTYPE_L4_MASK)) == 656 MBUF_PTYPE_TUNNEL_ESP_IN_UDP && 657 sa->udp_encap != 1)) { 658 free_pkts(&pkts[i], 1); 659 continue; 660 } 661 662 sym_cop = get_sym_cop(&priv->cop); 663 sym_cop->m_src = pkts[i]; 664 665 rte_security_attach_session(&priv->cop, 666 ips->security.ses); 667 break; 668 669 case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO: 670 RTE_LOG(ERR, IPSEC, "CPU crypto is not supported by the" 671 " legacy mode."); 672 free_pkts(&pkts[i], 1); 673 continue; 674 675 case RTE_SECURITY_ACTION_TYPE_NONE: 676 677 priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; 678 priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 679 680 rte_prefetch0(&priv->sym_cop); 681 682 if (unlikely(ips->crypto.ses == NULL)) { 683 free_pkts(&pkts[i], 1); 684 continue; 685 } 686 687 rte_crypto_op_attach_sym_session(&priv->cop, 688 ips->crypto.ses); 689 690 ret = xform_func(pkts[i], sa, &priv->cop); 691 if (unlikely(ret)) { 692 free_pkts(&pkts[i], 1); 693 continue; 694 } 695 break; 696 case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL: 697 RTE_ASSERT(ips->security.ses != NULL); 698 ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i]; 699 if (ips->security.ol_flags & 700 RTE_SECURITY_TX_OLOAD_NEED_MDATA) 701 rte_security_set_pkt_metadata( 702 ips->security.ctx, ips->security.ses, 703 pkts[i], NULL); 704 continue; 705 case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO: 706 RTE_ASSERT(ips->security.ses != NULL); 707 priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; 708 priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 709 710 rte_prefetch0(&priv->sym_cop); 711 rte_security_attach_session(&priv->cop, 712 ips->security.ses); 713 714 ret = xform_func(pkts[i], sa, &priv->cop); 715 if (unlikely(ret)) { 716 free_pkts(&pkts[i], 1); 717 continue; 718 } 719 720 ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i]; 721 if (ips->security.ol_flags & 722 RTE_SECURITY_TX_OLOAD_NEED_MDATA) 723 rte_security_set_pkt_metadata( 724 ips->security.ctx, ips->security.ses, 725 pkts[i], NULL); 726 continue; 727 } 728 729 enqueue_cop(sa->cqp[ipsec_ctx->lcore_id], &priv->cop); 730 } 731 } 732 733 static inline int32_t 734 ipsec_inline_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, 735 struct rte_mbuf *pkts[], uint16_t max_pkts) 736 { 737 int32_t nb_pkts, ret; 738 struct ipsec_mbuf_metadata *priv; 739 struct ipsec_sa *sa; 740 struct rte_mbuf *pkt; 741 742 nb_pkts = 0; 743 while (ipsec_ctx->ol_pkts_cnt > 0 && nb_pkts < max_pkts) { 744 pkt = ipsec_ctx->ol_pkts[--ipsec_ctx->ol_pkts_cnt]; 745 rte_prefetch0(pkt); 746 priv = get_priv(pkt); 747 sa = priv->sa; 748 ret = xform_func(pkt, sa, &priv->cop); 749 if (unlikely(ret)) { 750 free_pkts(&pkt, 1); 751 continue; 752 } 753 pkts[nb_pkts++] = pkt; 754 } 755 756 return nb_pkts; 757 } 758 759 static inline int 760 ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, 761 struct rte_mbuf *pkts[], uint16_t max_pkts) 762 { 763 int32_t nb_pkts = 0, ret = 0, i, j, nb_cops; 764 struct ipsec_mbuf_metadata *priv; 765 struct rte_crypto_op *cops[max_pkts]; 766 struct ipsec_sa *sa; 767 struct rte_mbuf *pkt; 768 769 for (i = 0; i < ipsec_ctx->nb_qps && nb_pkts < max_pkts; i++) { 770 struct cdev_qp *cqp; 771 772 cqp = &ipsec_ctx->tbl[ipsec_ctx->last_qp++]; 773 if (ipsec_ctx->last_qp == ipsec_ctx->nb_qps) 774 ipsec_ctx->last_qp %= ipsec_ctx->nb_qps; 775 776 if (cqp->in_flight == 0) 777 continue; 778 779 nb_cops = rte_cryptodev_dequeue_burst(cqp->id, cqp->qp, 780 cops, max_pkts - nb_pkts); 781 782 cqp->in_flight -= nb_cops; 783 784 for (j = 0; j < nb_cops; j++) { 785 pkt = cops[j]->sym->m_src; 786 rte_prefetch0(pkt); 787 788 priv = get_priv(pkt); 789 sa = priv->sa; 790 791 RTE_ASSERT(sa != NULL); 792 793 if (ipsec_get_action_type(sa) == 794 RTE_SECURITY_ACTION_TYPE_NONE) { 795 ret = xform_func(pkt, sa, cops[j]); 796 if (unlikely(ret)) { 797 free_pkts(&pkt, 1); 798 continue; 799 } 800 } else if (ipsec_get_action_type(sa) == 801 RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) { 802 if (cops[j]->status) { 803 free_pkts(&pkt, 1); 804 continue; 805 } 806 } 807 pkts[nb_pkts++] = pkt; 808 } 809 } 810 811 /* return packets */ 812 return nb_pkts; 813 } 814 815 uint16_t 816 ipsec_inbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[], 817 uint16_t nb_pkts, uint16_t len) 818 { 819 void *sas[nb_pkts]; 820 821 inbound_sa_lookup(ctx->sa_ctx, pkts, sas, nb_pkts); 822 823 ipsec_enqueue(esp_inbound, ctx, pkts, sas, nb_pkts); 824 825 return ipsec_inline_dequeue(esp_inbound_post, ctx, pkts, len); 826 } 827 828 uint16_t 829 ipsec_inbound_cqp_dequeue(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[], 830 uint16_t len) 831 { 832 return ipsec_dequeue(esp_inbound_post, ctx, pkts, len); 833 } 834 835 uint16_t 836 ipsec_outbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[], 837 uint32_t sa_idx[], uint16_t nb_pkts, uint16_t len) 838 { 839 void *sas[nb_pkts]; 840 841 outbound_sa_lookup(ctx->sa_ctx, sa_idx, sas, nb_pkts); 842 843 ipsec_enqueue(esp_outbound, ctx, pkts, sas, nb_pkts); 844 845 return ipsec_inline_dequeue(esp_outbound_post, ctx, pkts, len); 846 } 847 848 uint16_t 849 ipsec_outbound_cqp_dequeue(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[], 850 uint16_t len) 851 { 852 return ipsec_dequeue(esp_outbound_post, ctx, pkts, len); 853 } 854