1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 Intel Corporation 3 */ 4 5 #include <stdbool.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <stdint.h> 9 #include <inttypes.h> 10 #include <sys/types.h> 11 #include <netinet/in.h> 12 #include <netinet/ip.h> 13 #include <netinet/ip6.h> 14 #include <string.h> 15 #include <sys/queue.h> 16 #include <stdarg.h> 17 #include <errno.h> 18 #include <signal.h> 19 #include <getopt.h> 20 21 #include <rte_common.h> 22 #include <rte_bitmap.h> 23 #include <rte_byteorder.h> 24 #include <rte_log.h> 25 #include <rte_eal.h> 26 #include <rte_launch.h> 27 #include <rte_atomic.h> 28 #include <rte_cycles.h> 29 #include <rte_prefetch.h> 30 #include <rte_lcore.h> 31 #include <rte_per_lcore.h> 32 #include <rte_branch_prediction.h> 33 #include <rte_interrupts.h> 34 #include <rte_random.h> 35 #include <rte_debug.h> 36 #include <rte_ether.h> 37 #include <rte_ethdev.h> 38 #include <rte_mempool.h> 39 #include <rte_mbuf.h> 40 #include <rte_acl.h> 41 #include <rte_lpm.h> 42 #include <rte_lpm6.h> 43 #include <rte_hash.h> 44 #include <rte_jhash.h> 45 #include <rte_cryptodev.h> 46 #include <rte_security.h> 47 #include <rte_eventdev.h> 48 #include <rte_ip.h> 49 #include <rte_ip_frag.h> 50 51 #include "event_helper.h" 52 #include "ipsec.h" 53 #include "ipsec_worker.h" 54 #include "parser.h" 55 #include "sad.h" 56 57 volatile bool force_quit; 58 59 #define MAX_JUMBO_PKT_LEN 9600 60 61 #define MEMPOOL_CACHE_SIZE 256 62 63 #define CDEV_QUEUE_DESC 2048 64 #define CDEV_MAP_ENTRIES 16384 65 #define CDEV_MP_CACHE_SZ 64 66 #define MAX_QUEUE_PAIRS 1 67 68 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 69 70 /* Configure how many packets ahead to prefetch, when reading packets */ 71 #define PREFETCH_OFFSET 3 72 73 #define MAX_RX_QUEUE_PER_LCORE 16 74 75 #define MAX_LCORE_PARAMS 1024 76 77 /* 78 * Configurable number of RX/TX ring descriptors 79 */ 80 #define IPSEC_SECGW_RX_DESC_DEFAULT 1024 81 #define IPSEC_SECGW_TX_DESC_DEFAULT 1024 82 static uint16_t nb_rxd = IPSEC_SECGW_RX_DESC_DEFAULT; 83 static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT; 84 85 #define ETHADDR_TO_UINT64(addr) __BYTES_TO_UINT64( \ 86 (addr)->addr_bytes[0], (addr)->addr_bytes[1], \ 87 (addr)->addr_bytes[2], (addr)->addr_bytes[3], \ 88 (addr)->addr_bytes[4], (addr)->addr_bytes[5], \ 89 0, 0) 90 91 #define FRAG_TBL_BUCKET_ENTRIES 4 92 #define MAX_FRAG_TTL_NS (10LL * NS_PER_S) 93 94 #define MTU_TO_FRAMELEN(x) ((x) + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN) 95 96 struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = { 97 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a) }, 98 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9) }, 99 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x08, 0x69, 0x26) }, 100 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd) } 101 }; 102 103 struct flow_info flow_info_tbl[RTE_MAX_ETHPORTS]; 104 105 #define CMD_LINE_OPT_CONFIG "config" 106 #define CMD_LINE_OPT_SINGLE_SA "single-sa" 107 #define CMD_LINE_OPT_CRYPTODEV_MASK "cryptodev_mask" 108 #define CMD_LINE_OPT_TRANSFER_MODE "transfer-mode" 109 #define CMD_LINE_OPT_SCHEDULE_TYPE "event-schedule-type" 110 #define CMD_LINE_OPT_RX_OFFLOAD "rxoffload" 111 #define CMD_LINE_OPT_TX_OFFLOAD "txoffload" 112 #define CMD_LINE_OPT_REASSEMBLE "reassemble" 113 #define CMD_LINE_OPT_MTU "mtu" 114 #define CMD_LINE_OPT_FRAG_TTL "frag-ttl" 115 116 #define CMD_LINE_ARG_EVENT "event" 117 #define CMD_LINE_ARG_POLL "poll" 118 #define CMD_LINE_ARG_ORDERED "ordered" 119 #define CMD_LINE_ARG_ATOMIC "atomic" 120 #define CMD_LINE_ARG_PARALLEL "parallel" 121 122 enum { 123 /* long options mapped to a short option */ 124 125 /* first long only option value must be >= 256, so that we won't 126 * conflict with short options 127 */ 128 CMD_LINE_OPT_MIN_NUM = 256, 129 CMD_LINE_OPT_CONFIG_NUM, 130 CMD_LINE_OPT_SINGLE_SA_NUM, 131 CMD_LINE_OPT_CRYPTODEV_MASK_NUM, 132 CMD_LINE_OPT_TRANSFER_MODE_NUM, 133 CMD_LINE_OPT_SCHEDULE_TYPE_NUM, 134 CMD_LINE_OPT_RX_OFFLOAD_NUM, 135 CMD_LINE_OPT_TX_OFFLOAD_NUM, 136 CMD_LINE_OPT_REASSEMBLE_NUM, 137 CMD_LINE_OPT_MTU_NUM, 138 CMD_LINE_OPT_FRAG_TTL_NUM, 139 }; 140 141 static const struct option lgopts[] = { 142 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, 143 {CMD_LINE_OPT_SINGLE_SA, 1, 0, CMD_LINE_OPT_SINGLE_SA_NUM}, 144 {CMD_LINE_OPT_CRYPTODEV_MASK, 1, 0, CMD_LINE_OPT_CRYPTODEV_MASK_NUM}, 145 {CMD_LINE_OPT_TRANSFER_MODE, 1, 0, CMD_LINE_OPT_TRANSFER_MODE_NUM}, 146 {CMD_LINE_OPT_SCHEDULE_TYPE, 1, 0, CMD_LINE_OPT_SCHEDULE_TYPE_NUM}, 147 {CMD_LINE_OPT_RX_OFFLOAD, 1, 0, CMD_LINE_OPT_RX_OFFLOAD_NUM}, 148 {CMD_LINE_OPT_TX_OFFLOAD, 1, 0, CMD_LINE_OPT_TX_OFFLOAD_NUM}, 149 {CMD_LINE_OPT_REASSEMBLE, 1, 0, CMD_LINE_OPT_REASSEMBLE_NUM}, 150 {CMD_LINE_OPT_MTU, 1, 0, CMD_LINE_OPT_MTU_NUM}, 151 {CMD_LINE_OPT_FRAG_TTL, 1, 0, CMD_LINE_OPT_FRAG_TTL_NUM}, 152 {NULL, 0, 0, 0} 153 }; 154 155 uint32_t unprotected_port_mask; 156 uint32_t single_sa_idx; 157 /* mask of enabled ports */ 158 static uint32_t enabled_port_mask; 159 static uint64_t enabled_cryptodev_mask = UINT64_MAX; 160 static int32_t promiscuous_on = 1; 161 static int32_t numa_on = 1; /**< NUMA is enabled by default. */ 162 static uint32_t nb_lcores; 163 static uint32_t single_sa; 164 static uint32_t nb_bufs_in_pool; 165 166 /* 167 * RX/TX HW offload capabilities to enable/use on ethernet ports. 168 * By default all capabilities are enabled. 169 */ 170 static uint64_t dev_rx_offload = UINT64_MAX; 171 static uint64_t dev_tx_offload = UINT64_MAX; 172 173 /* 174 * global values that determine multi-seg policy 175 */ 176 static uint32_t frag_tbl_sz; 177 static uint32_t frame_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE; 178 static uint32_t mtu_size = RTE_ETHER_MTU; 179 static uint64_t frag_ttl_ns = MAX_FRAG_TTL_NS; 180 181 /* application wide librte_ipsec/SA parameters */ 182 struct app_sa_prm app_sa_prm = { 183 .enable = 0, 184 .cache_sz = SA_CACHE_SZ 185 }; 186 static const char *cfgfile; 187 188 struct lcore_rx_queue { 189 uint16_t port_id; 190 uint8_t queue_id; 191 } __rte_cache_aligned; 192 193 struct lcore_params { 194 uint16_t port_id; 195 uint8_t queue_id; 196 uint8_t lcore_id; 197 } __rte_cache_aligned; 198 199 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 200 201 static struct lcore_params *lcore_params; 202 static uint16_t nb_lcore_params; 203 204 static struct rte_hash *cdev_map_in; 205 static struct rte_hash *cdev_map_out; 206 207 struct buffer { 208 uint16_t len; 209 struct rte_mbuf *m_table[MAX_PKT_BURST] __rte_aligned(sizeof(void *)); 210 }; 211 212 struct lcore_conf { 213 uint16_t nb_rx_queue; 214 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 215 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 216 struct buffer tx_mbufs[RTE_MAX_ETHPORTS]; 217 struct ipsec_ctx inbound; 218 struct ipsec_ctx outbound; 219 struct rt_ctx *rt4_ctx; 220 struct rt_ctx *rt6_ctx; 221 struct { 222 struct rte_ip_frag_tbl *tbl; 223 struct rte_mempool *pool_dir; 224 struct rte_mempool *pool_indir; 225 struct rte_ip_frag_death_row dr; 226 } frag; 227 } __rte_cache_aligned; 228 229 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 230 231 static struct rte_eth_conf port_conf = { 232 .rxmode = { 233 .mq_mode = ETH_MQ_RX_RSS, 234 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 235 .split_hdr_size = 0, 236 .offloads = DEV_RX_OFFLOAD_CHECKSUM, 237 }, 238 .rx_adv_conf = { 239 .rss_conf = { 240 .rss_key = NULL, 241 .rss_hf = ETH_RSS_IP | ETH_RSS_UDP | 242 ETH_RSS_TCP | ETH_RSS_SCTP, 243 }, 244 }, 245 .txmode = { 246 .mq_mode = ETH_MQ_TX_NONE, 247 }, 248 }; 249 250 struct socket_ctx socket_ctx[NB_SOCKETS]; 251 252 /* 253 * Determine is multi-segment support required: 254 * - either frame buffer size is smaller then mtu 255 * - or reassmeble support is requested 256 */ 257 static int 258 multi_seg_required(void) 259 { 260 return (MTU_TO_FRAMELEN(mtu_size) + RTE_PKTMBUF_HEADROOM > 261 frame_buf_size || frag_tbl_sz != 0); 262 } 263 264 static inline void 265 adjust_ipv4_pktlen(struct rte_mbuf *m, const struct rte_ipv4_hdr *iph, 266 uint32_t l2_len) 267 { 268 uint32_t plen, trim; 269 270 plen = rte_be_to_cpu_16(iph->total_length) + l2_len; 271 if (plen < m->pkt_len) { 272 trim = m->pkt_len - plen; 273 rte_pktmbuf_trim(m, trim); 274 } 275 } 276 277 static inline void 278 adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, 279 uint32_t l2_len) 280 { 281 uint32_t plen, trim; 282 283 plen = rte_be_to_cpu_16(iph->payload_len) + sizeof(*iph) + l2_len; 284 if (plen < m->pkt_len) { 285 trim = m->pkt_len - plen; 286 rte_pktmbuf_trim(m, trim); 287 } 288 } 289 290 static inline void 291 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t) 292 { 293 const struct rte_ether_hdr *eth; 294 const struct rte_ipv4_hdr *iph4; 295 const struct rte_ipv6_hdr *iph6; 296 297 eth = rte_pktmbuf_mtod(pkt, const struct rte_ether_hdr *); 298 if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 299 300 iph4 = (const struct rte_ipv4_hdr *)rte_pktmbuf_adj(pkt, 301 RTE_ETHER_HDR_LEN); 302 adjust_ipv4_pktlen(pkt, iph4, 0); 303 304 if (iph4->next_proto_id == IPPROTO_ESP) 305 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 306 else { 307 t->ip4.data[t->ip4.num] = &iph4->next_proto_id; 308 t->ip4.pkts[(t->ip4.num)++] = pkt; 309 } 310 pkt->l2_len = 0; 311 pkt->l3_len = sizeof(*iph4); 312 pkt->packet_type |= RTE_PTYPE_L3_IPV4; 313 } else if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 314 int next_proto; 315 size_t l3len, ext_len; 316 uint8_t *p; 317 318 /* get protocol type */ 319 iph6 = (const struct rte_ipv6_hdr *)rte_pktmbuf_adj(pkt, 320 RTE_ETHER_HDR_LEN); 321 adjust_ipv6_pktlen(pkt, iph6, 0); 322 323 next_proto = iph6->proto; 324 325 /* determine l3 header size up to ESP extension */ 326 l3len = sizeof(struct ip6_hdr); 327 p = rte_pktmbuf_mtod(pkt, uint8_t *); 328 while (next_proto != IPPROTO_ESP && l3len < pkt->data_len && 329 (next_proto = rte_ipv6_get_next_ext(p + l3len, 330 next_proto, &ext_len)) >= 0) 331 l3len += ext_len; 332 333 /* drop packet when IPv6 header exceeds first segment length */ 334 if (unlikely(l3len > pkt->data_len)) { 335 rte_pktmbuf_free(pkt); 336 return; 337 } 338 339 if (next_proto == IPPROTO_ESP) 340 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 341 else { 342 t->ip6.data[t->ip6.num] = &iph6->proto; 343 t->ip6.pkts[(t->ip6.num)++] = pkt; 344 } 345 pkt->l2_len = 0; 346 pkt->l3_len = l3len; 347 pkt->packet_type |= RTE_PTYPE_L3_IPV6; 348 } else { 349 /* Unknown/Unsupported type, drop the packet */ 350 RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n", 351 rte_be_to_cpu_16(eth->ether_type)); 352 rte_pktmbuf_free(pkt); 353 return; 354 } 355 356 /* Check if the packet has been processed inline. For inline protocol 357 * processed packets, the metadata in the mbuf can be used to identify 358 * the security processing done on the packet. The metadata will be 359 * used to retrieve the application registered userdata associated 360 * with the security session. 361 */ 362 363 if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) { 364 struct ipsec_sa *sa; 365 struct ipsec_mbuf_metadata *priv; 366 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 367 rte_eth_dev_get_sec_ctx( 368 pkt->port); 369 370 /* Retrieve the userdata registered. Here, the userdata 371 * registered is the SA pointer. 372 */ 373 374 sa = (struct ipsec_sa *) 375 rte_security_get_userdata(ctx, pkt->udata64); 376 377 if (sa == NULL) { 378 /* userdata could not be retrieved */ 379 return; 380 } 381 382 /* Save SA as priv member in mbuf. This will be used in the 383 * IPsec selector(SP-SA) check. 384 */ 385 386 priv = get_priv(pkt); 387 priv->sa = sa; 388 } 389 } 390 391 static inline void 392 prepare_traffic(struct rte_mbuf **pkts, struct ipsec_traffic *t, 393 uint16_t nb_pkts) 394 { 395 int32_t i; 396 397 t->ipsec.num = 0; 398 t->ip4.num = 0; 399 t->ip6.num = 0; 400 401 for (i = 0; i < (nb_pkts - PREFETCH_OFFSET); i++) { 402 rte_prefetch0(rte_pktmbuf_mtod(pkts[i + PREFETCH_OFFSET], 403 void *)); 404 prepare_one_packet(pkts[i], t); 405 } 406 /* Process left packets */ 407 for (; i < nb_pkts; i++) 408 prepare_one_packet(pkts[i], t); 409 } 410 411 static inline void 412 prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port, 413 const struct lcore_conf *qconf) 414 { 415 struct ip *ip; 416 struct rte_ether_hdr *ethhdr; 417 418 ip = rte_pktmbuf_mtod(pkt, struct ip *); 419 420 ethhdr = (struct rte_ether_hdr *) 421 rte_pktmbuf_prepend(pkt, RTE_ETHER_HDR_LEN); 422 423 if (ip->ip_v == IPVERSION) { 424 pkt->ol_flags |= qconf->outbound.ipv4_offloads; 425 pkt->l3_len = sizeof(struct ip); 426 pkt->l2_len = RTE_ETHER_HDR_LEN; 427 428 ip->ip_sum = 0; 429 430 /* calculate IPv4 cksum in SW */ 431 if ((pkt->ol_flags & PKT_TX_IP_CKSUM) == 0) 432 ip->ip_sum = rte_ipv4_cksum((struct rte_ipv4_hdr *)ip); 433 434 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 435 } else { 436 pkt->ol_flags |= qconf->outbound.ipv6_offloads; 437 pkt->l3_len = sizeof(struct ip6_hdr); 438 pkt->l2_len = RTE_ETHER_HDR_LEN; 439 440 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); 441 } 442 443 memcpy(ðhdr->s_addr, ðaddr_tbl[port].src, 444 sizeof(struct rte_ether_addr)); 445 memcpy(ðhdr->d_addr, ðaddr_tbl[port].dst, 446 sizeof(struct rte_ether_addr)); 447 } 448 449 static inline void 450 prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t port, 451 const struct lcore_conf *qconf) 452 { 453 int32_t i; 454 const int32_t prefetch_offset = 2; 455 456 for (i = 0; i < (nb_pkts - prefetch_offset); i++) { 457 rte_mbuf_prefetch_part2(pkts[i + prefetch_offset]); 458 prepare_tx_pkt(pkts[i], port, qconf); 459 } 460 /* Process left packets */ 461 for (; i < nb_pkts; i++) 462 prepare_tx_pkt(pkts[i], port, qconf); 463 } 464 465 /* Send burst of packets on an output interface */ 466 static inline int32_t 467 send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) 468 { 469 struct rte_mbuf **m_table; 470 int32_t ret; 471 uint16_t queueid; 472 473 queueid = qconf->tx_queue_id[port]; 474 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 475 476 prepare_tx_burst(m_table, n, port, qconf); 477 478 ret = rte_eth_tx_burst(port, queueid, m_table, n); 479 if (unlikely(ret < n)) { 480 do { 481 rte_pktmbuf_free(m_table[ret]); 482 } while (++ret < n); 483 } 484 485 return 0; 486 } 487 488 /* 489 * Helper function to fragment and queue for TX one packet. 490 */ 491 static inline uint32_t 492 send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m, 493 uint16_t port, uint8_t proto) 494 { 495 struct buffer *tbl; 496 uint32_t len, n; 497 int32_t rc; 498 499 tbl = qconf->tx_mbufs + port; 500 len = tbl->len; 501 502 /* free space for new fragments */ 503 if (len + RTE_LIBRTE_IP_FRAG_MAX_FRAG >= RTE_DIM(tbl->m_table)) { 504 send_burst(qconf, len, port); 505 len = 0; 506 } 507 508 n = RTE_DIM(tbl->m_table) - len; 509 510 if (proto == IPPROTO_IP) 511 rc = rte_ipv4_fragment_packet(m, tbl->m_table + len, 512 n, mtu_size, qconf->frag.pool_dir, 513 qconf->frag.pool_indir); 514 else 515 rc = rte_ipv6_fragment_packet(m, tbl->m_table + len, 516 n, mtu_size, qconf->frag.pool_dir, 517 qconf->frag.pool_indir); 518 519 if (rc >= 0) 520 len += rc; 521 else 522 RTE_LOG(ERR, IPSEC, 523 "%s: failed to fragment packet with size %u, " 524 "error code: %d\n", 525 __func__, m->pkt_len, rte_errno); 526 527 rte_pktmbuf_free(m); 528 return len; 529 } 530 531 /* Enqueue a single packet, and send burst if queue is filled */ 532 static inline int32_t 533 send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto) 534 { 535 uint32_t lcore_id; 536 uint16_t len; 537 struct lcore_conf *qconf; 538 539 lcore_id = rte_lcore_id(); 540 541 qconf = &lcore_conf[lcore_id]; 542 len = qconf->tx_mbufs[port].len; 543 544 if (m->pkt_len <= mtu_size) { 545 qconf->tx_mbufs[port].m_table[len] = m; 546 len++; 547 548 /* need to fragment the packet */ 549 } else if (frag_tbl_sz > 0) 550 len = send_fragment_packet(qconf, m, port, proto); 551 else 552 rte_pktmbuf_free(m); 553 554 /* enough pkts to be sent */ 555 if (unlikely(len == MAX_PKT_BURST)) { 556 send_burst(qconf, MAX_PKT_BURST, port); 557 len = 0; 558 } 559 560 qconf->tx_mbufs[port].len = len; 561 return 0; 562 } 563 564 static inline void 565 inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip, 566 uint16_t lim) 567 { 568 struct rte_mbuf *m; 569 uint32_t i, j, res, sa_idx; 570 571 if (ip->num == 0 || sp == NULL) 572 return; 573 574 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 575 ip->num, DEFAULT_MAX_CATEGORIES); 576 577 j = 0; 578 for (i = 0; i < ip->num; i++) { 579 m = ip->pkts[i]; 580 res = ip->res[i]; 581 if (res == BYPASS) { 582 ip->pkts[j++] = m; 583 continue; 584 } 585 if (res == DISCARD) { 586 rte_pktmbuf_free(m); 587 continue; 588 } 589 590 /* Only check SPI match for processed IPSec packets */ 591 if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) { 592 rte_pktmbuf_free(m); 593 continue; 594 } 595 596 sa_idx = res - 1; 597 if (!inbound_sa_check(sa, m, sa_idx)) { 598 rte_pktmbuf_free(m); 599 continue; 600 } 601 ip->pkts[j++] = m; 602 } 603 ip->num = j; 604 } 605 606 static void 607 split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) 608 { 609 uint32_t i, n4, n6; 610 struct ip *ip; 611 struct rte_mbuf *m; 612 613 n4 = trf->ip4.num; 614 n6 = trf->ip6.num; 615 616 for (i = 0; i < num; i++) { 617 618 m = mb[i]; 619 ip = rte_pktmbuf_mtod(m, struct ip *); 620 621 if (ip->ip_v == IPVERSION) { 622 trf->ip4.pkts[n4] = m; 623 trf->ip4.data[n4] = rte_pktmbuf_mtod_offset(m, 624 uint8_t *, offsetof(struct ip, ip_p)); 625 n4++; 626 } else if (ip->ip_v == IP6_VERSION) { 627 trf->ip6.pkts[n6] = m; 628 trf->ip6.data[n6] = rte_pktmbuf_mtod_offset(m, 629 uint8_t *, 630 offsetof(struct ip6_hdr, ip6_nxt)); 631 n6++; 632 } else 633 rte_pktmbuf_free(m); 634 } 635 636 trf->ip4.num = n4; 637 trf->ip6.num = n6; 638 } 639 640 641 static inline void 642 process_pkts_inbound(struct ipsec_ctx *ipsec_ctx, 643 struct ipsec_traffic *traffic) 644 { 645 uint16_t nb_pkts_in, n_ip4, n_ip6; 646 647 n_ip4 = traffic->ip4.num; 648 n_ip6 = traffic->ip6.num; 649 650 if (app_sa_prm.enable == 0) { 651 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 652 traffic->ipsec.num, MAX_PKT_BURST); 653 split46_traffic(traffic, traffic->ipsec.pkts, nb_pkts_in); 654 } else { 655 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 656 traffic->ipsec.saptr, traffic->ipsec.num); 657 ipsec_process(ipsec_ctx, traffic); 658 } 659 660 inbound_sp_sa(ipsec_ctx->sp4_ctx, ipsec_ctx->sa_ctx, &traffic->ip4, 661 n_ip4); 662 663 inbound_sp_sa(ipsec_ctx->sp6_ctx, ipsec_ctx->sa_ctx, &traffic->ip6, 664 n_ip6); 665 } 666 667 static inline void 668 outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, 669 struct traffic_type *ipsec) 670 { 671 struct rte_mbuf *m; 672 uint32_t i, j, sa_idx; 673 674 if (ip->num == 0 || sp == NULL) 675 return; 676 677 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 678 ip->num, DEFAULT_MAX_CATEGORIES); 679 680 j = 0; 681 for (i = 0; i < ip->num; i++) { 682 m = ip->pkts[i]; 683 sa_idx = ip->res[i] - 1; 684 if (ip->res[i] == DISCARD) 685 rte_pktmbuf_free(m); 686 else if (ip->res[i] == BYPASS) 687 ip->pkts[j++] = m; 688 else { 689 ipsec->res[ipsec->num] = sa_idx; 690 ipsec->pkts[ipsec->num++] = m; 691 } 692 } 693 ip->num = j; 694 } 695 696 static inline void 697 process_pkts_outbound(struct ipsec_ctx *ipsec_ctx, 698 struct ipsec_traffic *traffic) 699 { 700 struct rte_mbuf *m; 701 uint16_t idx, nb_pkts_out, i; 702 703 /* Drop any IPsec traffic from protected ports */ 704 for (i = 0; i < traffic->ipsec.num; i++) 705 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 706 707 traffic->ipsec.num = 0; 708 709 outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec); 710 711 outbound_sp(ipsec_ctx->sp6_ctx, &traffic->ip6, &traffic->ipsec); 712 713 if (app_sa_prm.enable == 0) { 714 715 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 716 traffic->ipsec.res, traffic->ipsec.num, 717 MAX_PKT_BURST); 718 719 for (i = 0; i < nb_pkts_out; i++) { 720 m = traffic->ipsec.pkts[i]; 721 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 722 if (ip->ip_v == IPVERSION) { 723 idx = traffic->ip4.num++; 724 traffic->ip4.pkts[idx] = m; 725 } else { 726 idx = traffic->ip6.num++; 727 traffic->ip6.pkts[idx] = m; 728 } 729 } 730 } else { 731 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 732 traffic->ipsec.saptr, traffic->ipsec.num); 733 ipsec_process(ipsec_ctx, traffic); 734 } 735 } 736 737 static inline void 738 process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx, 739 struct ipsec_traffic *traffic) 740 { 741 struct rte_mbuf *m; 742 uint32_t nb_pkts_in, i, idx; 743 744 /* Drop any IPv4 traffic from unprotected ports */ 745 for (i = 0; i < traffic->ip4.num; i++) 746 rte_pktmbuf_free(traffic->ip4.pkts[i]); 747 748 traffic->ip4.num = 0; 749 750 /* Drop any IPv6 traffic from unprotected ports */ 751 for (i = 0; i < traffic->ip6.num; i++) 752 rte_pktmbuf_free(traffic->ip6.pkts[i]); 753 754 traffic->ip6.num = 0; 755 756 if (app_sa_prm.enable == 0) { 757 758 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 759 traffic->ipsec.num, MAX_PKT_BURST); 760 761 for (i = 0; i < nb_pkts_in; i++) { 762 m = traffic->ipsec.pkts[i]; 763 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 764 if (ip->ip_v == IPVERSION) { 765 idx = traffic->ip4.num++; 766 traffic->ip4.pkts[idx] = m; 767 } else { 768 idx = traffic->ip6.num++; 769 traffic->ip6.pkts[idx] = m; 770 } 771 } 772 } else { 773 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 774 traffic->ipsec.saptr, traffic->ipsec.num); 775 ipsec_process(ipsec_ctx, traffic); 776 } 777 } 778 779 static inline void 780 process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx, 781 struct ipsec_traffic *traffic) 782 { 783 struct rte_mbuf *m; 784 uint32_t nb_pkts_out, i, n; 785 struct ip *ip; 786 787 /* Drop any IPsec traffic from protected ports */ 788 for (i = 0; i < traffic->ipsec.num; i++) 789 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 790 791 n = 0; 792 793 for (i = 0; i < traffic->ip4.num; i++) { 794 traffic->ipsec.pkts[n] = traffic->ip4.pkts[i]; 795 traffic->ipsec.res[n++] = single_sa_idx; 796 } 797 798 for (i = 0; i < traffic->ip6.num; i++) { 799 traffic->ipsec.pkts[n] = traffic->ip6.pkts[i]; 800 traffic->ipsec.res[n++] = single_sa_idx; 801 } 802 803 traffic->ip4.num = 0; 804 traffic->ip6.num = 0; 805 traffic->ipsec.num = n; 806 807 if (app_sa_prm.enable == 0) { 808 809 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 810 traffic->ipsec.res, traffic->ipsec.num, 811 MAX_PKT_BURST); 812 813 /* They all sue the same SA (ip4 or ip6 tunnel) */ 814 m = traffic->ipsec.pkts[0]; 815 ip = rte_pktmbuf_mtod(m, struct ip *); 816 if (ip->ip_v == IPVERSION) { 817 traffic->ip4.num = nb_pkts_out; 818 for (i = 0; i < nb_pkts_out; i++) 819 traffic->ip4.pkts[i] = traffic->ipsec.pkts[i]; 820 } else { 821 traffic->ip6.num = nb_pkts_out; 822 for (i = 0; i < nb_pkts_out; i++) 823 traffic->ip6.pkts[i] = traffic->ipsec.pkts[i]; 824 } 825 } else { 826 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 827 traffic->ipsec.saptr, traffic->ipsec.num); 828 ipsec_process(ipsec_ctx, traffic); 829 } 830 } 831 832 static inline int32_t 833 get_hop_for_offload_pkt(struct rte_mbuf *pkt, int is_ipv6) 834 { 835 struct ipsec_mbuf_metadata *priv; 836 struct ipsec_sa *sa; 837 838 priv = get_priv(pkt); 839 840 sa = priv->sa; 841 if (unlikely(sa == NULL)) { 842 RTE_LOG(ERR, IPSEC, "SA not saved in private data\n"); 843 goto fail; 844 } 845 846 if (is_ipv6) 847 return sa->portid; 848 849 /* else */ 850 return (sa->portid | RTE_LPM_LOOKUP_SUCCESS); 851 852 fail: 853 if (is_ipv6) 854 return -1; 855 856 /* else */ 857 return 0; 858 } 859 860 static inline void 861 route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 862 { 863 uint32_t hop[MAX_PKT_BURST * 2]; 864 uint32_t dst_ip[MAX_PKT_BURST * 2]; 865 int32_t pkt_hop = 0; 866 uint16_t i, offset; 867 uint16_t lpm_pkts = 0; 868 869 if (nb_pkts == 0) 870 return; 871 872 /* Need to do an LPM lookup for non-inline packets. Inline packets will 873 * have port ID in the SA 874 */ 875 876 for (i = 0; i < nb_pkts; i++) { 877 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 878 /* Security offload not enabled. So an LPM lookup is 879 * required to get the hop 880 */ 881 offset = offsetof(struct ip, ip_dst); 882 dst_ip[lpm_pkts] = *rte_pktmbuf_mtod_offset(pkts[i], 883 uint32_t *, offset); 884 dst_ip[lpm_pkts] = rte_be_to_cpu_32(dst_ip[lpm_pkts]); 885 lpm_pkts++; 886 } 887 } 888 889 rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, lpm_pkts); 890 891 lpm_pkts = 0; 892 893 for (i = 0; i < nb_pkts; i++) { 894 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 895 /* Read hop from the SA */ 896 pkt_hop = get_hop_for_offload_pkt(pkts[i], 0); 897 } else { 898 /* Need to use hop returned by lookup */ 899 pkt_hop = hop[lpm_pkts++]; 900 } 901 902 if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) { 903 rte_pktmbuf_free(pkts[i]); 904 continue; 905 } 906 send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IP); 907 } 908 } 909 910 static inline void 911 route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 912 { 913 int32_t hop[MAX_PKT_BURST * 2]; 914 uint8_t dst_ip[MAX_PKT_BURST * 2][16]; 915 uint8_t *ip6_dst; 916 int32_t pkt_hop = 0; 917 uint16_t i, offset; 918 uint16_t lpm_pkts = 0; 919 920 if (nb_pkts == 0) 921 return; 922 923 /* Need to do an LPM lookup for non-inline packets. Inline packets will 924 * have port ID in the SA 925 */ 926 927 for (i = 0; i < nb_pkts; i++) { 928 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 929 /* Security offload not enabled. So an LPM lookup is 930 * required to get the hop 931 */ 932 offset = offsetof(struct ip6_hdr, ip6_dst); 933 ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *, 934 offset); 935 memcpy(&dst_ip[lpm_pkts][0], ip6_dst, 16); 936 lpm_pkts++; 937 } 938 } 939 940 rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, hop, 941 lpm_pkts); 942 943 lpm_pkts = 0; 944 945 for (i = 0; i < nb_pkts; i++) { 946 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 947 /* Read hop from the SA */ 948 pkt_hop = get_hop_for_offload_pkt(pkts[i], 1); 949 } else { 950 /* Need to use hop returned by lookup */ 951 pkt_hop = hop[lpm_pkts++]; 952 } 953 954 if (pkt_hop == -1) { 955 rte_pktmbuf_free(pkts[i]); 956 continue; 957 } 958 send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IPV6); 959 } 960 } 961 962 static inline void 963 process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts, 964 uint8_t nb_pkts, uint16_t portid) 965 { 966 struct ipsec_traffic traffic; 967 968 prepare_traffic(pkts, &traffic, nb_pkts); 969 970 if (unlikely(single_sa)) { 971 if (is_unprotected_port(portid)) 972 process_pkts_inbound_nosp(&qconf->inbound, &traffic); 973 else 974 process_pkts_outbound_nosp(&qconf->outbound, &traffic); 975 } else { 976 if (is_unprotected_port(portid)) 977 process_pkts_inbound(&qconf->inbound, &traffic); 978 else 979 process_pkts_outbound(&qconf->outbound, &traffic); 980 } 981 982 route4_pkts(qconf->rt4_ctx, traffic.ip4.pkts, traffic.ip4.num); 983 route6_pkts(qconf->rt6_ctx, traffic.ip6.pkts, traffic.ip6.num); 984 } 985 986 static inline void 987 drain_tx_buffers(struct lcore_conf *qconf) 988 { 989 struct buffer *buf; 990 uint32_t portid; 991 992 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 993 buf = &qconf->tx_mbufs[portid]; 994 if (buf->len == 0) 995 continue; 996 send_burst(qconf, buf->len, portid); 997 buf->len = 0; 998 } 999 } 1000 1001 static inline void 1002 drain_crypto_buffers(struct lcore_conf *qconf) 1003 { 1004 uint32_t i; 1005 struct ipsec_ctx *ctx; 1006 1007 /* drain inbound buffers*/ 1008 ctx = &qconf->inbound; 1009 for (i = 0; i != ctx->nb_qps; i++) { 1010 if (ctx->tbl[i].len != 0) 1011 enqueue_cop_burst(ctx->tbl + i); 1012 } 1013 1014 /* drain outbound buffers*/ 1015 ctx = &qconf->outbound; 1016 for (i = 0; i != ctx->nb_qps; i++) { 1017 if (ctx->tbl[i].len != 0) 1018 enqueue_cop_burst(ctx->tbl + i); 1019 } 1020 } 1021 1022 static void 1023 drain_inbound_crypto_queues(const struct lcore_conf *qconf, 1024 struct ipsec_ctx *ctx) 1025 { 1026 uint32_t n; 1027 struct ipsec_traffic trf; 1028 1029 if (app_sa_prm.enable == 0) { 1030 1031 /* dequeue packets from crypto-queue */ 1032 n = ipsec_inbound_cqp_dequeue(ctx, trf.ipsec.pkts, 1033 RTE_DIM(trf.ipsec.pkts)); 1034 1035 trf.ip4.num = 0; 1036 trf.ip6.num = 0; 1037 1038 /* split traffic by ipv4-ipv6 */ 1039 split46_traffic(&trf, trf.ipsec.pkts, n); 1040 } else 1041 ipsec_cqp_process(ctx, &trf); 1042 1043 /* process ipv4 packets */ 1044 if (trf.ip4.num != 0) { 1045 inbound_sp_sa(ctx->sp4_ctx, ctx->sa_ctx, &trf.ip4, 0); 1046 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 1047 } 1048 1049 /* process ipv6 packets */ 1050 if (trf.ip6.num != 0) { 1051 inbound_sp_sa(ctx->sp6_ctx, ctx->sa_ctx, &trf.ip6, 0); 1052 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 1053 } 1054 } 1055 1056 static void 1057 drain_outbound_crypto_queues(const struct lcore_conf *qconf, 1058 struct ipsec_ctx *ctx) 1059 { 1060 uint32_t n; 1061 struct ipsec_traffic trf; 1062 1063 if (app_sa_prm.enable == 0) { 1064 1065 /* dequeue packets from crypto-queue */ 1066 n = ipsec_outbound_cqp_dequeue(ctx, trf.ipsec.pkts, 1067 RTE_DIM(trf.ipsec.pkts)); 1068 1069 trf.ip4.num = 0; 1070 trf.ip6.num = 0; 1071 1072 /* split traffic by ipv4-ipv6 */ 1073 split46_traffic(&trf, trf.ipsec.pkts, n); 1074 } else 1075 ipsec_cqp_process(ctx, &trf); 1076 1077 /* process ipv4 packets */ 1078 if (trf.ip4.num != 0) 1079 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 1080 1081 /* process ipv6 packets */ 1082 if (trf.ip6.num != 0) 1083 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 1084 } 1085 1086 /* main processing loop */ 1087 void 1088 ipsec_poll_mode_worker(void) 1089 { 1090 struct rte_mbuf *pkts[MAX_PKT_BURST]; 1091 uint32_t lcore_id; 1092 uint64_t prev_tsc, diff_tsc, cur_tsc; 1093 int32_t i, nb_rx; 1094 uint16_t portid; 1095 uint8_t queueid; 1096 struct lcore_conf *qconf; 1097 int32_t rc, socket_id; 1098 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) 1099 / US_PER_S * BURST_TX_DRAIN_US; 1100 struct lcore_rx_queue *rxql; 1101 1102 prev_tsc = 0; 1103 lcore_id = rte_lcore_id(); 1104 qconf = &lcore_conf[lcore_id]; 1105 rxql = qconf->rx_queue_list; 1106 socket_id = rte_lcore_to_socket_id(lcore_id); 1107 1108 qconf->rt4_ctx = socket_ctx[socket_id].rt_ip4; 1109 qconf->rt6_ctx = socket_ctx[socket_id].rt_ip6; 1110 qconf->inbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_in; 1111 qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in; 1112 qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in; 1113 qconf->inbound.cdev_map = cdev_map_in; 1114 qconf->inbound.session_pool = socket_ctx[socket_id].session_pool; 1115 qconf->inbound.session_priv_pool = 1116 socket_ctx[socket_id].session_priv_pool; 1117 qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out; 1118 qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out; 1119 qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out; 1120 qconf->outbound.cdev_map = cdev_map_out; 1121 qconf->outbound.session_pool = socket_ctx[socket_id].session_pool; 1122 qconf->outbound.session_priv_pool = 1123 socket_ctx[socket_id].session_priv_pool; 1124 qconf->frag.pool_dir = socket_ctx[socket_id].mbuf_pool; 1125 qconf->frag.pool_indir = socket_ctx[socket_id].mbuf_pool_indir; 1126 1127 rc = ipsec_sad_lcore_cache_init(app_sa_prm.cache_sz); 1128 if (rc != 0) { 1129 RTE_LOG(ERR, IPSEC, 1130 "SAD cache init on lcore %u, failed with code: %d\n", 1131 lcore_id, rc); 1132 return; 1133 } 1134 1135 if (qconf->nb_rx_queue == 0) { 1136 RTE_LOG(DEBUG, IPSEC, "lcore %u has nothing to do\n", 1137 lcore_id); 1138 return; 1139 } 1140 1141 RTE_LOG(INFO, IPSEC, "entering main loop on lcore %u\n", lcore_id); 1142 1143 for (i = 0; i < qconf->nb_rx_queue; i++) { 1144 portid = rxql[i].port_id; 1145 queueid = rxql[i].queue_id; 1146 RTE_LOG(INFO, IPSEC, 1147 " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", 1148 lcore_id, portid, queueid); 1149 } 1150 1151 while (!force_quit) { 1152 cur_tsc = rte_rdtsc(); 1153 1154 /* TX queue buffer drain */ 1155 diff_tsc = cur_tsc - prev_tsc; 1156 1157 if (unlikely(diff_tsc > drain_tsc)) { 1158 drain_tx_buffers(qconf); 1159 drain_crypto_buffers(qconf); 1160 prev_tsc = cur_tsc; 1161 } 1162 1163 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1164 1165 /* Read packets from RX queues */ 1166 portid = rxql[i].port_id; 1167 queueid = rxql[i].queue_id; 1168 nb_rx = rte_eth_rx_burst(portid, queueid, 1169 pkts, MAX_PKT_BURST); 1170 1171 if (nb_rx > 0) 1172 process_pkts(qconf, pkts, nb_rx, portid); 1173 1174 /* dequeue and process completed crypto-ops */ 1175 if (is_unprotected_port(portid)) 1176 drain_inbound_crypto_queues(qconf, 1177 &qconf->inbound); 1178 else 1179 drain_outbound_crypto_queues(qconf, 1180 &qconf->outbound); 1181 } 1182 } 1183 } 1184 1185 int 1186 check_flow_params(uint16_t fdir_portid, uint8_t fdir_qid) 1187 { 1188 uint16_t i; 1189 uint16_t portid; 1190 uint8_t queueid; 1191 1192 for (i = 0; i < nb_lcore_params; ++i) { 1193 portid = lcore_params_array[i].port_id; 1194 if (portid == fdir_portid) { 1195 queueid = lcore_params_array[i].queue_id; 1196 if (queueid == fdir_qid) 1197 break; 1198 } 1199 1200 if (i == nb_lcore_params - 1) 1201 return -1; 1202 } 1203 1204 return 1; 1205 } 1206 1207 static int32_t 1208 check_poll_mode_params(struct eh_conf *eh_conf) 1209 { 1210 uint8_t lcore; 1211 uint16_t portid; 1212 uint16_t i; 1213 int32_t socket_id; 1214 1215 if (!eh_conf) 1216 return -EINVAL; 1217 1218 if (eh_conf->mode != EH_PKT_TRANSFER_MODE_POLL) 1219 return 0; 1220 1221 if (lcore_params == NULL) { 1222 printf("Error: No port/queue/core mappings\n"); 1223 return -1; 1224 } 1225 1226 for (i = 0; i < nb_lcore_params; ++i) { 1227 lcore = lcore_params[i].lcore_id; 1228 if (!rte_lcore_is_enabled(lcore)) { 1229 printf("error: lcore %hhu is not enabled in " 1230 "lcore mask\n", lcore); 1231 return -1; 1232 } 1233 socket_id = rte_lcore_to_socket_id(lcore); 1234 if (socket_id != 0 && numa_on == 0) { 1235 printf("warning: lcore %hhu is on socket %d " 1236 "with numa off\n", 1237 lcore, socket_id); 1238 } 1239 portid = lcore_params[i].port_id; 1240 if ((enabled_port_mask & (1 << portid)) == 0) { 1241 printf("port %u is not enabled in port mask\n", portid); 1242 return -1; 1243 } 1244 if (!rte_eth_dev_is_valid_port(portid)) { 1245 printf("port %u is not present on the board\n", portid); 1246 return -1; 1247 } 1248 } 1249 return 0; 1250 } 1251 1252 static uint8_t 1253 get_port_nb_rx_queues(const uint16_t port) 1254 { 1255 int32_t queue = -1; 1256 uint16_t i; 1257 1258 for (i = 0; i < nb_lcore_params; ++i) { 1259 if (lcore_params[i].port_id == port && 1260 lcore_params[i].queue_id > queue) 1261 queue = lcore_params[i].queue_id; 1262 } 1263 return (uint8_t)(++queue); 1264 } 1265 1266 static int32_t 1267 init_lcore_rx_queues(void) 1268 { 1269 uint16_t i, nb_rx_queue; 1270 uint8_t lcore; 1271 1272 for (i = 0; i < nb_lcore_params; ++i) { 1273 lcore = lcore_params[i].lcore_id; 1274 nb_rx_queue = lcore_conf[lcore].nb_rx_queue; 1275 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 1276 printf("error: too many queues (%u) for lcore: %u\n", 1277 nb_rx_queue + 1, lcore); 1278 return -1; 1279 } 1280 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 1281 lcore_params[i].port_id; 1282 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 1283 lcore_params[i].queue_id; 1284 lcore_conf[lcore].nb_rx_queue++; 1285 } 1286 return 0; 1287 } 1288 1289 /* display usage */ 1290 static void 1291 print_usage(const char *prgname) 1292 { 1293 fprintf(stderr, "%s [EAL options] --" 1294 " -p PORTMASK" 1295 " [-P]" 1296 " [-u PORTMASK]" 1297 " [-j FRAMESIZE]" 1298 " [-l]" 1299 " [-w REPLAY_WINDOW_SIZE]" 1300 " [-e]" 1301 " [-a]" 1302 " [-c]" 1303 " [-s NUMBER_OF_MBUFS_IN_PKT_POOL]" 1304 " -f CONFIG_FILE" 1305 " --config (port,queue,lcore)[,(port,queue,lcore)]" 1306 " [--single-sa SAIDX]" 1307 " [--cryptodev_mask MASK]" 1308 " [--transfer-mode MODE]" 1309 " [--event-schedule-type TYPE]" 1310 " [--" CMD_LINE_OPT_RX_OFFLOAD " RX_OFFLOAD_MASK]" 1311 " [--" CMD_LINE_OPT_TX_OFFLOAD " TX_OFFLOAD_MASK]" 1312 " [--" CMD_LINE_OPT_REASSEMBLE " REASSEMBLE_TABLE_SIZE]" 1313 " [--" CMD_LINE_OPT_MTU " MTU]" 1314 "\n\n" 1315 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n" 1316 " -P : Enable promiscuous mode\n" 1317 " -u PORTMASK: Hexadecimal bitmask of unprotected ports\n" 1318 " -j FRAMESIZE: Data buffer size, minimum (and default)\n" 1319 " value: RTE_MBUF_DEFAULT_BUF_SIZE\n" 1320 " -l enables code-path that uses librte_ipsec\n" 1321 " -w REPLAY_WINDOW_SIZE specifies IPsec SQN replay window\n" 1322 " size for each SA\n" 1323 " -e enables ESN\n" 1324 " -a enables SA SQN atomic behaviour\n" 1325 " -c specifies inbound SAD cache size,\n" 1326 " zero value disables the cache (default value: 128)\n" 1327 " -s number of mbufs in packet pool, if not specified number\n" 1328 " of mbufs will be calculated based on number of cores,\n" 1329 " ports and crypto queues\n" 1330 " -f CONFIG_FILE: Configuration file\n" 1331 " --config (port,queue,lcore): Rx queue configuration. In poll\n" 1332 " mode determines which queues from\n" 1333 " which ports are mapped to which cores.\n" 1334 " In event mode this option is not used\n" 1335 " as packets are dynamically scheduled\n" 1336 " to cores by HW.\n" 1337 " --single-sa SAIDX: In poll mode use single SA index for\n" 1338 " outbound traffic, bypassing the SP\n" 1339 " In event mode selects driver submode,\n" 1340 " SA index value is ignored\n" 1341 " --cryptodev_mask MASK: Hexadecimal bitmask of the crypto\n" 1342 " devices to configure\n" 1343 " --transfer-mode MODE\n" 1344 " \"poll\" : Packet transfer via polling (default)\n" 1345 " \"event\" : Packet transfer via event device\n" 1346 " --event-schedule-type TYPE queue schedule type, used only when\n" 1347 " transfer mode is set to event\n" 1348 " \"ordered\" : Ordered (default)\n" 1349 " \"atomic\" : Atomic\n" 1350 " \"parallel\" : Parallel\n" 1351 " --" CMD_LINE_OPT_RX_OFFLOAD 1352 ": bitmask of the RX HW offload capabilities to enable/use\n" 1353 " (DEV_RX_OFFLOAD_*)\n" 1354 " --" CMD_LINE_OPT_TX_OFFLOAD 1355 ": bitmask of the TX HW offload capabilities to enable/use\n" 1356 " (DEV_TX_OFFLOAD_*)\n" 1357 " --" CMD_LINE_OPT_REASSEMBLE " NUM" 1358 ": max number of entries in reassemble(fragment) table\n" 1359 " (zero (default value) disables reassembly)\n" 1360 " --" CMD_LINE_OPT_MTU " MTU" 1361 ": MTU value on all ports (default value: 1500)\n" 1362 " outgoing packets with bigger size will be fragmented\n" 1363 " incoming packets with bigger size will be discarded\n" 1364 " --" CMD_LINE_OPT_FRAG_TTL " FRAG_TTL_NS" 1365 ": fragments lifetime in nanoseconds, default\n" 1366 " and maximum value is 10.000.000.000 ns (10 s)\n" 1367 "\n", 1368 prgname); 1369 } 1370 1371 static int 1372 parse_mask(const char *str, uint64_t *val) 1373 { 1374 char *end; 1375 unsigned long t; 1376 1377 errno = 0; 1378 t = strtoul(str, &end, 0); 1379 if (errno != 0 || end[0] != 0) 1380 return -EINVAL; 1381 1382 *val = t; 1383 return 0; 1384 } 1385 1386 static int32_t 1387 parse_portmask(const char *portmask) 1388 { 1389 char *end = NULL; 1390 unsigned long pm; 1391 1392 /* parse hexadecimal string */ 1393 pm = strtoul(portmask, &end, 16); 1394 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 1395 return -1; 1396 1397 if ((pm == 0) && errno) 1398 return -1; 1399 1400 return pm; 1401 } 1402 1403 static int64_t 1404 parse_decimal(const char *str) 1405 { 1406 char *end = NULL; 1407 uint64_t num; 1408 1409 num = strtoull(str, &end, 10); 1410 if ((str[0] == '\0') || (end == NULL) || (*end != '\0') 1411 || num > INT64_MAX) 1412 return -1; 1413 1414 return num; 1415 } 1416 1417 static int32_t 1418 parse_config(const char *q_arg) 1419 { 1420 char s[256]; 1421 const char *p, *p0 = q_arg; 1422 char *end; 1423 enum fieldnames { 1424 FLD_PORT = 0, 1425 FLD_QUEUE, 1426 FLD_LCORE, 1427 _NUM_FLD 1428 }; 1429 unsigned long int_fld[_NUM_FLD]; 1430 char *str_fld[_NUM_FLD]; 1431 int32_t i; 1432 uint32_t size; 1433 1434 nb_lcore_params = 0; 1435 1436 while ((p = strchr(p0, '(')) != NULL) { 1437 ++p; 1438 p0 = strchr(p, ')'); 1439 if (p0 == NULL) 1440 return -1; 1441 1442 size = p0 - p; 1443 if (size >= sizeof(s)) 1444 return -1; 1445 1446 snprintf(s, sizeof(s), "%.*s", size, p); 1447 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 1448 _NUM_FLD) 1449 return -1; 1450 for (i = 0; i < _NUM_FLD; i++) { 1451 errno = 0; 1452 int_fld[i] = strtoul(str_fld[i], &end, 0); 1453 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 1454 return -1; 1455 } 1456 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 1457 printf("exceeded max number of lcore params: %hu\n", 1458 nb_lcore_params); 1459 return -1; 1460 } 1461 lcore_params_array[nb_lcore_params].port_id = 1462 (uint8_t)int_fld[FLD_PORT]; 1463 lcore_params_array[nb_lcore_params].queue_id = 1464 (uint8_t)int_fld[FLD_QUEUE]; 1465 lcore_params_array[nb_lcore_params].lcore_id = 1466 (uint8_t)int_fld[FLD_LCORE]; 1467 ++nb_lcore_params; 1468 } 1469 lcore_params = lcore_params_array; 1470 return 0; 1471 } 1472 1473 static void 1474 print_app_sa_prm(const struct app_sa_prm *prm) 1475 { 1476 printf("librte_ipsec usage: %s\n", 1477 (prm->enable == 0) ? "disabled" : "enabled"); 1478 1479 printf("replay window size: %u\n", prm->window_size); 1480 printf("ESN: %s\n", (prm->enable_esn == 0) ? "disabled" : "enabled"); 1481 printf("SA flags: %#" PRIx64 "\n", prm->flags); 1482 printf("Frag TTL: %" PRIu64 " ns\n", frag_ttl_ns); 1483 } 1484 1485 static int 1486 parse_transfer_mode(struct eh_conf *conf, const char *optarg) 1487 { 1488 if (!strcmp(CMD_LINE_ARG_POLL, optarg)) 1489 conf->mode = EH_PKT_TRANSFER_MODE_POLL; 1490 else if (!strcmp(CMD_LINE_ARG_EVENT, optarg)) 1491 conf->mode = EH_PKT_TRANSFER_MODE_EVENT; 1492 else { 1493 printf("Unsupported packet transfer mode\n"); 1494 return -EINVAL; 1495 } 1496 1497 return 0; 1498 } 1499 1500 static int 1501 parse_schedule_type(struct eh_conf *conf, const char *optarg) 1502 { 1503 struct eventmode_conf *em_conf = NULL; 1504 1505 /* Get eventmode conf */ 1506 em_conf = conf->mode_params; 1507 1508 if (!strcmp(CMD_LINE_ARG_ORDERED, optarg)) 1509 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ORDERED; 1510 else if (!strcmp(CMD_LINE_ARG_ATOMIC, optarg)) 1511 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ATOMIC; 1512 else if (!strcmp(CMD_LINE_ARG_PARALLEL, optarg)) 1513 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_PARALLEL; 1514 else { 1515 printf("Unsupported queue schedule type\n"); 1516 return -EINVAL; 1517 } 1518 1519 return 0; 1520 } 1521 1522 static int32_t 1523 parse_args(int32_t argc, char **argv, struct eh_conf *eh_conf) 1524 { 1525 int opt; 1526 int64_t ret; 1527 char **argvopt; 1528 int32_t option_index; 1529 char *prgname = argv[0]; 1530 int32_t f_present = 0; 1531 1532 argvopt = argv; 1533 1534 while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:c:s:", 1535 lgopts, &option_index)) != EOF) { 1536 1537 switch (opt) { 1538 case 'p': 1539 enabled_port_mask = parse_portmask(optarg); 1540 if (enabled_port_mask == 0) { 1541 printf("invalid portmask\n"); 1542 print_usage(prgname); 1543 return -1; 1544 } 1545 break; 1546 case 'P': 1547 printf("Promiscuous mode selected\n"); 1548 promiscuous_on = 1; 1549 break; 1550 case 'u': 1551 unprotected_port_mask = parse_portmask(optarg); 1552 if (unprotected_port_mask == 0) { 1553 printf("invalid unprotected portmask\n"); 1554 print_usage(prgname); 1555 return -1; 1556 } 1557 break; 1558 case 'f': 1559 if (f_present == 1) { 1560 printf("\"-f\" option present more than " 1561 "once!\n"); 1562 print_usage(prgname); 1563 return -1; 1564 } 1565 cfgfile = optarg; 1566 f_present = 1; 1567 break; 1568 1569 case 's': 1570 ret = parse_decimal(optarg); 1571 if (ret < 0) { 1572 printf("Invalid number of buffers in a pool: " 1573 "%s\n", optarg); 1574 print_usage(prgname); 1575 return -1; 1576 } 1577 1578 nb_bufs_in_pool = ret; 1579 break; 1580 1581 case 'j': 1582 ret = parse_decimal(optarg); 1583 if (ret < RTE_MBUF_DEFAULT_BUF_SIZE || 1584 ret > UINT16_MAX) { 1585 printf("Invalid frame buffer size value: %s\n", 1586 optarg); 1587 print_usage(prgname); 1588 return -1; 1589 } 1590 frame_buf_size = ret; 1591 printf("Custom frame buffer size %u\n", frame_buf_size); 1592 break; 1593 case 'l': 1594 app_sa_prm.enable = 1; 1595 break; 1596 case 'w': 1597 app_sa_prm.window_size = parse_decimal(optarg); 1598 break; 1599 case 'e': 1600 app_sa_prm.enable_esn = 1; 1601 break; 1602 case 'a': 1603 app_sa_prm.enable = 1; 1604 app_sa_prm.flags |= RTE_IPSEC_SAFLAG_SQN_ATOM; 1605 break; 1606 case 'c': 1607 ret = parse_decimal(optarg); 1608 if (ret < 0) { 1609 printf("Invalid SA cache size: %s\n", optarg); 1610 print_usage(prgname); 1611 return -1; 1612 } 1613 app_sa_prm.cache_sz = ret; 1614 break; 1615 case CMD_LINE_OPT_CONFIG_NUM: 1616 ret = parse_config(optarg); 1617 if (ret) { 1618 printf("Invalid config\n"); 1619 print_usage(prgname); 1620 return -1; 1621 } 1622 break; 1623 case CMD_LINE_OPT_SINGLE_SA_NUM: 1624 ret = parse_decimal(optarg); 1625 if (ret == -1 || ret > UINT32_MAX) { 1626 printf("Invalid argument[sa_idx]\n"); 1627 print_usage(prgname); 1628 return -1; 1629 } 1630 1631 /* else */ 1632 single_sa = 1; 1633 single_sa_idx = ret; 1634 eh_conf->ipsec_mode = EH_IPSEC_MODE_TYPE_DRIVER; 1635 printf("Configured with single SA index %u\n", 1636 single_sa_idx); 1637 break; 1638 case CMD_LINE_OPT_CRYPTODEV_MASK_NUM: 1639 ret = parse_portmask(optarg); 1640 if (ret == -1) { 1641 printf("Invalid argument[portmask]\n"); 1642 print_usage(prgname); 1643 return -1; 1644 } 1645 1646 /* else */ 1647 enabled_cryptodev_mask = ret; 1648 break; 1649 1650 case CMD_LINE_OPT_TRANSFER_MODE_NUM: 1651 ret = parse_transfer_mode(eh_conf, optarg); 1652 if (ret < 0) { 1653 printf("Invalid packet transfer mode\n"); 1654 print_usage(prgname); 1655 return -1; 1656 } 1657 break; 1658 1659 case CMD_LINE_OPT_SCHEDULE_TYPE_NUM: 1660 ret = parse_schedule_type(eh_conf, optarg); 1661 if (ret < 0) { 1662 printf("Invalid queue schedule type\n"); 1663 print_usage(prgname); 1664 return -1; 1665 } 1666 break; 1667 1668 case CMD_LINE_OPT_RX_OFFLOAD_NUM: 1669 ret = parse_mask(optarg, &dev_rx_offload); 1670 if (ret != 0) { 1671 printf("Invalid argument for \'%s\': %s\n", 1672 CMD_LINE_OPT_RX_OFFLOAD, optarg); 1673 print_usage(prgname); 1674 return -1; 1675 } 1676 break; 1677 case CMD_LINE_OPT_TX_OFFLOAD_NUM: 1678 ret = parse_mask(optarg, &dev_tx_offload); 1679 if (ret != 0) { 1680 printf("Invalid argument for \'%s\': %s\n", 1681 CMD_LINE_OPT_TX_OFFLOAD, optarg); 1682 print_usage(prgname); 1683 return -1; 1684 } 1685 break; 1686 case CMD_LINE_OPT_REASSEMBLE_NUM: 1687 ret = parse_decimal(optarg); 1688 if (ret < 0 || ret > UINT32_MAX) { 1689 printf("Invalid argument for \'%s\': %s\n", 1690 CMD_LINE_OPT_REASSEMBLE, optarg); 1691 print_usage(prgname); 1692 return -1; 1693 } 1694 frag_tbl_sz = ret; 1695 break; 1696 case CMD_LINE_OPT_MTU_NUM: 1697 ret = parse_decimal(optarg); 1698 if (ret < 0 || ret > RTE_IPV4_MAX_PKT_LEN) { 1699 printf("Invalid argument for \'%s\': %s\n", 1700 CMD_LINE_OPT_MTU, optarg); 1701 print_usage(prgname); 1702 return -1; 1703 } 1704 mtu_size = ret; 1705 break; 1706 case CMD_LINE_OPT_FRAG_TTL_NUM: 1707 ret = parse_decimal(optarg); 1708 if (ret < 0 || ret > MAX_FRAG_TTL_NS) { 1709 printf("Invalid argument for \'%s\': %s\n", 1710 CMD_LINE_OPT_MTU, optarg); 1711 print_usage(prgname); 1712 return -1; 1713 } 1714 frag_ttl_ns = ret; 1715 break; 1716 default: 1717 print_usage(prgname); 1718 return -1; 1719 } 1720 } 1721 1722 if (f_present == 0) { 1723 printf("Mandatory option \"-f\" not present\n"); 1724 return -1; 1725 } 1726 1727 /* check do we need to enable multi-seg support */ 1728 if (multi_seg_required()) { 1729 /* legacy mode doesn't support multi-seg */ 1730 app_sa_prm.enable = 1; 1731 printf("frame buf size: %u, mtu: %u, " 1732 "number of reassemble entries: %u\n" 1733 "multi-segment support is required\n", 1734 frame_buf_size, mtu_size, frag_tbl_sz); 1735 } 1736 1737 print_app_sa_prm(&app_sa_prm); 1738 1739 if (optind >= 0) 1740 argv[optind-1] = prgname; 1741 1742 ret = optind-1; 1743 optind = 1; /* reset getopt lib */ 1744 return ret; 1745 } 1746 1747 static void 1748 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr) 1749 { 1750 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 1751 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr); 1752 printf("%s%s", name, buf); 1753 } 1754 1755 /* 1756 * Update destination ethaddr for the port. 1757 */ 1758 int 1759 add_dst_ethaddr(uint16_t port, const struct rte_ether_addr *addr) 1760 { 1761 if (port >= RTE_DIM(ethaddr_tbl)) 1762 return -EINVAL; 1763 1764 ethaddr_tbl[port].dst = ETHADDR_TO_UINT64(addr); 1765 return 0; 1766 } 1767 1768 /* Check the link status of all ports in up to 9s, and print them finally */ 1769 static void 1770 check_all_ports_link_status(uint32_t port_mask) 1771 { 1772 #define CHECK_INTERVAL 100 /* 100ms */ 1773 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1774 uint16_t portid; 1775 uint8_t count, all_ports_up, print_flag = 0; 1776 struct rte_eth_link link; 1777 int ret; 1778 1779 printf("\nChecking link status"); 1780 fflush(stdout); 1781 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1782 all_ports_up = 1; 1783 RTE_ETH_FOREACH_DEV(portid) { 1784 if ((port_mask & (1 << portid)) == 0) 1785 continue; 1786 memset(&link, 0, sizeof(link)); 1787 ret = rte_eth_link_get_nowait(portid, &link); 1788 if (ret < 0) { 1789 all_ports_up = 0; 1790 if (print_flag == 1) 1791 printf("Port %u link get failed: %s\n", 1792 portid, rte_strerror(-ret)); 1793 continue; 1794 } 1795 /* print link status if flag set */ 1796 if (print_flag == 1) { 1797 if (link.link_status) 1798 printf( 1799 "Port%d Link Up - speed %u Mbps -%s\n", 1800 portid, link.link_speed, 1801 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1802 ("full-duplex") : ("half-duplex\n")); 1803 else 1804 printf("Port %d Link Down\n", portid); 1805 continue; 1806 } 1807 /* clear all_ports_up flag if any link down */ 1808 if (link.link_status == ETH_LINK_DOWN) { 1809 all_ports_up = 0; 1810 break; 1811 } 1812 } 1813 /* after finally printing all link status, get out */ 1814 if (print_flag == 1) 1815 break; 1816 1817 if (all_ports_up == 0) { 1818 printf("."); 1819 fflush(stdout); 1820 rte_delay_ms(CHECK_INTERVAL); 1821 } 1822 1823 /* set the print_flag if all ports up or timeout */ 1824 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1825 print_flag = 1; 1826 printf("done\n"); 1827 } 1828 } 1829 } 1830 1831 static int32_t 1832 add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id, 1833 uint16_t qp, struct lcore_params *params, 1834 struct ipsec_ctx *ipsec_ctx, 1835 const struct rte_cryptodev_capabilities *cipher, 1836 const struct rte_cryptodev_capabilities *auth, 1837 const struct rte_cryptodev_capabilities *aead) 1838 { 1839 int32_t ret = 0; 1840 unsigned long i; 1841 struct cdev_key key = { 0 }; 1842 1843 key.lcore_id = params->lcore_id; 1844 if (cipher) 1845 key.cipher_algo = cipher->sym.cipher.algo; 1846 if (auth) 1847 key.auth_algo = auth->sym.auth.algo; 1848 if (aead) 1849 key.aead_algo = aead->sym.aead.algo; 1850 1851 ret = rte_hash_lookup(map, &key); 1852 if (ret != -ENOENT) 1853 return 0; 1854 1855 for (i = 0; i < ipsec_ctx->nb_qps; i++) 1856 if (ipsec_ctx->tbl[i].id == cdev_id) 1857 break; 1858 1859 if (i == ipsec_ctx->nb_qps) { 1860 if (ipsec_ctx->nb_qps == MAX_QP_PER_LCORE) { 1861 printf("Maximum number of crypto devices assigned to " 1862 "a core, increase MAX_QP_PER_LCORE value\n"); 1863 return 0; 1864 } 1865 ipsec_ctx->tbl[i].id = cdev_id; 1866 ipsec_ctx->tbl[i].qp = qp; 1867 ipsec_ctx->nb_qps++; 1868 printf("%s cdev mapping: lcore %u using cdev %u qp %u " 1869 "(cdev_id_qp %lu)\n", str, key.lcore_id, 1870 cdev_id, qp, i); 1871 } 1872 1873 ret = rte_hash_add_key_data(map, &key, (void *)i); 1874 if (ret < 0) { 1875 printf("Faled to insert cdev mapping for (lcore %u, " 1876 "cdev %u, qp %u), errno %d\n", 1877 key.lcore_id, ipsec_ctx->tbl[i].id, 1878 ipsec_ctx->tbl[i].qp, ret); 1879 return 0; 1880 } 1881 1882 return 1; 1883 } 1884 1885 static int32_t 1886 add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id, 1887 uint16_t qp, struct lcore_params *params) 1888 { 1889 int32_t ret = 0; 1890 const struct rte_cryptodev_capabilities *i, *j; 1891 struct rte_hash *map; 1892 struct lcore_conf *qconf; 1893 struct ipsec_ctx *ipsec_ctx; 1894 const char *str; 1895 1896 qconf = &lcore_conf[params->lcore_id]; 1897 1898 if ((unprotected_port_mask & (1 << params->port_id)) == 0) { 1899 map = cdev_map_out; 1900 ipsec_ctx = &qconf->outbound; 1901 str = "Outbound"; 1902 } else { 1903 map = cdev_map_in; 1904 ipsec_ctx = &qconf->inbound; 1905 str = "Inbound"; 1906 } 1907 1908 /* Required cryptodevs with operation chainning */ 1909 if (!(dev_info->feature_flags & 1910 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING)) 1911 return ret; 1912 1913 for (i = dev_info->capabilities; 1914 i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++) { 1915 if (i->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1916 continue; 1917 1918 if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1919 ret |= add_mapping(map, str, cdev_id, qp, params, 1920 ipsec_ctx, NULL, NULL, i); 1921 continue; 1922 } 1923 1924 if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) 1925 continue; 1926 1927 for (j = dev_info->capabilities; 1928 j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; j++) { 1929 if (j->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1930 continue; 1931 1932 if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH) 1933 continue; 1934 1935 ret |= add_mapping(map, str, cdev_id, qp, params, 1936 ipsec_ctx, i, j, NULL); 1937 } 1938 } 1939 1940 return ret; 1941 } 1942 1943 /* Check if the device is enabled by cryptodev_mask */ 1944 static int 1945 check_cryptodev_mask(uint8_t cdev_id) 1946 { 1947 if (enabled_cryptodev_mask & (1 << cdev_id)) 1948 return 0; 1949 1950 return -1; 1951 } 1952 1953 static uint16_t 1954 cryptodevs_init(uint16_t req_queue_num) 1955 { 1956 struct rte_cryptodev_config dev_conf; 1957 struct rte_cryptodev_qp_conf qp_conf; 1958 uint16_t idx, max_nb_qps, qp, total_nb_qps, i; 1959 int16_t cdev_id; 1960 struct rte_hash_parameters params = { 0 }; 1961 1962 const uint64_t mseg_flag = multi_seg_required() ? 1963 RTE_CRYPTODEV_FF_IN_PLACE_SGL : 0; 1964 1965 params.entries = CDEV_MAP_ENTRIES; 1966 params.key_len = sizeof(struct cdev_key); 1967 params.hash_func = rte_jhash; 1968 params.hash_func_init_val = 0; 1969 params.socket_id = rte_socket_id(); 1970 1971 params.name = "cdev_map_in"; 1972 cdev_map_in = rte_hash_create(¶ms); 1973 if (cdev_map_in == NULL) 1974 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1975 rte_errno); 1976 1977 params.name = "cdev_map_out"; 1978 cdev_map_out = rte_hash_create(¶ms); 1979 if (cdev_map_out == NULL) 1980 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1981 rte_errno); 1982 1983 printf("lcore/cryptodev/qp mappings:\n"); 1984 1985 idx = 0; 1986 total_nb_qps = 0; 1987 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 1988 struct rte_cryptodev_info cdev_info; 1989 1990 if (check_cryptodev_mask((uint8_t)cdev_id)) 1991 continue; 1992 1993 rte_cryptodev_info_get(cdev_id, &cdev_info); 1994 1995 if ((mseg_flag & cdev_info.feature_flags) != mseg_flag) 1996 rte_exit(EXIT_FAILURE, 1997 "Device %hd does not support \'%s\' feature\n", 1998 cdev_id, 1999 rte_cryptodev_get_feature_name(mseg_flag)); 2000 2001 if (nb_lcore_params > cdev_info.max_nb_queue_pairs) 2002 max_nb_qps = cdev_info.max_nb_queue_pairs; 2003 else 2004 max_nb_qps = nb_lcore_params; 2005 2006 qp = 0; 2007 i = 0; 2008 while (qp < max_nb_qps && i < nb_lcore_params) { 2009 if (add_cdev_mapping(&cdev_info, cdev_id, qp, 2010 &lcore_params[idx])) 2011 qp++; 2012 idx++; 2013 idx = idx % nb_lcore_params; 2014 i++; 2015 } 2016 2017 qp = RTE_MIN(max_nb_qps, RTE_MAX(req_queue_num, qp)); 2018 if (qp == 0) 2019 continue; 2020 2021 total_nb_qps += qp; 2022 dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id); 2023 dev_conf.nb_queue_pairs = qp; 2024 dev_conf.ff_disable = RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO; 2025 2026 uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions; 2027 if (dev_max_sess != 0 && 2028 dev_max_sess < get_nb_crypto_sessions()) 2029 rte_exit(EXIT_FAILURE, 2030 "Device does not support at least %u " 2031 "sessions", get_nb_crypto_sessions()); 2032 2033 if (rte_cryptodev_configure(cdev_id, &dev_conf)) 2034 rte_panic("Failed to initialize cryptodev %u\n", 2035 cdev_id); 2036 2037 qp_conf.nb_descriptors = CDEV_QUEUE_DESC; 2038 qp_conf.mp_session = 2039 socket_ctx[dev_conf.socket_id].session_pool; 2040 qp_conf.mp_session_private = 2041 socket_ctx[dev_conf.socket_id].session_priv_pool; 2042 for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++) 2043 if (rte_cryptodev_queue_pair_setup(cdev_id, qp, 2044 &qp_conf, dev_conf.socket_id)) 2045 rte_panic("Failed to setup queue %u for " 2046 "cdev_id %u\n", 0, cdev_id); 2047 2048 if (rte_cryptodev_start(cdev_id)) 2049 rte_panic("Failed to start cryptodev %u\n", 2050 cdev_id); 2051 } 2052 2053 printf("\n"); 2054 2055 return total_nb_qps; 2056 } 2057 2058 static void 2059 port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads) 2060 { 2061 uint32_t frame_size; 2062 struct rte_eth_dev_info dev_info; 2063 struct rte_eth_txconf *txconf; 2064 uint16_t nb_tx_queue, nb_rx_queue; 2065 uint16_t tx_queueid, rx_queueid, queue, lcore_id; 2066 int32_t ret, socket_id; 2067 struct lcore_conf *qconf; 2068 struct rte_ether_addr ethaddr; 2069 struct rte_eth_conf local_port_conf = port_conf; 2070 2071 ret = rte_eth_dev_info_get(portid, &dev_info); 2072 if (ret != 0) 2073 rte_exit(EXIT_FAILURE, 2074 "Error during getting device (port %u) info: %s\n", 2075 portid, strerror(-ret)); 2076 2077 /* limit allowed HW offloafs, as user requested */ 2078 dev_info.rx_offload_capa &= dev_rx_offload; 2079 dev_info.tx_offload_capa &= dev_tx_offload; 2080 2081 printf("Configuring device port %u:\n", portid); 2082 2083 ret = rte_eth_macaddr_get(portid, ðaddr); 2084 if (ret != 0) 2085 rte_exit(EXIT_FAILURE, 2086 "Error getting MAC address (port %u): %s\n", 2087 portid, rte_strerror(-ret)); 2088 2089 ethaddr_tbl[portid].src = ETHADDR_TO_UINT64(ðaddr); 2090 print_ethaddr("Address: ", ðaddr); 2091 printf("\n"); 2092 2093 nb_rx_queue = get_port_nb_rx_queues(portid); 2094 nb_tx_queue = nb_lcores; 2095 2096 if (nb_rx_queue > dev_info.max_rx_queues) 2097 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 2098 "(max rx queue is %u)\n", 2099 nb_rx_queue, dev_info.max_rx_queues); 2100 2101 if (nb_tx_queue > dev_info.max_tx_queues) 2102 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 2103 "(max tx queue is %u)\n", 2104 nb_tx_queue, dev_info.max_tx_queues); 2105 2106 printf("Creating queues: nb_rx_queue=%d nb_tx_queue=%u...\n", 2107 nb_rx_queue, nb_tx_queue); 2108 2109 frame_size = MTU_TO_FRAMELEN(mtu_size); 2110 if (frame_size > local_port_conf.rxmode.max_rx_pkt_len) 2111 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 2112 local_port_conf.rxmode.max_rx_pkt_len = frame_size; 2113 2114 if (multi_seg_required()) { 2115 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_SCATTER; 2116 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MULTI_SEGS; 2117 } 2118 2119 local_port_conf.rxmode.offloads |= req_rx_offloads; 2120 local_port_conf.txmode.offloads |= req_tx_offloads; 2121 2122 /* Check that all required capabilities are supported */ 2123 if ((local_port_conf.rxmode.offloads & dev_info.rx_offload_capa) != 2124 local_port_conf.rxmode.offloads) 2125 rte_exit(EXIT_FAILURE, 2126 "Error: port %u required RX offloads: 0x%" PRIx64 2127 ", avaialbe RX offloads: 0x%" PRIx64 "\n", 2128 portid, local_port_conf.rxmode.offloads, 2129 dev_info.rx_offload_capa); 2130 2131 if ((local_port_conf.txmode.offloads & dev_info.tx_offload_capa) != 2132 local_port_conf.txmode.offloads) 2133 rte_exit(EXIT_FAILURE, 2134 "Error: port %u required TX offloads: 0x%" PRIx64 2135 ", avaialbe TX offloads: 0x%" PRIx64 "\n", 2136 portid, local_port_conf.txmode.offloads, 2137 dev_info.tx_offload_capa); 2138 2139 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 2140 local_port_conf.txmode.offloads |= 2141 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 2142 2143 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 2144 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 2145 2146 printf("port %u configurng rx_offloads=0x%" PRIx64 2147 ", tx_offloads=0x%" PRIx64 "\n", 2148 portid, local_port_conf.rxmode.offloads, 2149 local_port_conf.txmode.offloads); 2150 2151 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 2152 dev_info.flow_type_rss_offloads; 2153 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 2154 port_conf.rx_adv_conf.rss_conf.rss_hf) { 2155 printf("Port %u modified RSS hash function based on hardware support," 2156 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 2157 portid, 2158 port_conf.rx_adv_conf.rss_conf.rss_hf, 2159 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 2160 } 2161 2162 ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue, 2163 &local_port_conf); 2164 if (ret < 0) 2165 rte_exit(EXIT_FAILURE, "Cannot configure device: " 2166 "err=%d, port=%d\n", ret, portid); 2167 2168 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd); 2169 if (ret < 0) 2170 rte_exit(EXIT_FAILURE, "Cannot adjust number of descriptors: " 2171 "err=%d, port=%d\n", ret, portid); 2172 2173 /* init one TX queue per lcore */ 2174 tx_queueid = 0; 2175 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2176 if (rte_lcore_is_enabled(lcore_id) == 0) 2177 continue; 2178 2179 if (numa_on) 2180 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2181 else 2182 socket_id = 0; 2183 2184 /* init TX queue */ 2185 printf("Setup txq=%u,%d,%d\n", lcore_id, tx_queueid, socket_id); 2186 2187 txconf = &dev_info.default_txconf; 2188 txconf->offloads = local_port_conf.txmode.offloads; 2189 2190 ret = rte_eth_tx_queue_setup(portid, tx_queueid, nb_txd, 2191 socket_id, txconf); 2192 if (ret < 0) 2193 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: " 2194 "err=%d, port=%d\n", ret, portid); 2195 2196 qconf = &lcore_conf[lcore_id]; 2197 qconf->tx_queue_id[portid] = tx_queueid; 2198 2199 /* Pre-populate pkt offloads based on capabilities */ 2200 qconf->outbound.ipv4_offloads = PKT_TX_IPV4; 2201 qconf->outbound.ipv6_offloads = PKT_TX_IPV6; 2202 if (local_port_conf.txmode.offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) 2203 qconf->outbound.ipv4_offloads |= PKT_TX_IP_CKSUM; 2204 2205 tx_queueid++; 2206 2207 /* init RX queues */ 2208 for (queue = 0; queue < qconf->nb_rx_queue; ++queue) { 2209 struct rte_eth_rxconf rxq_conf; 2210 2211 if (portid != qconf->rx_queue_list[queue].port_id) 2212 continue; 2213 2214 rx_queueid = qconf->rx_queue_list[queue].queue_id; 2215 2216 printf("Setup rxq=%d,%d,%d\n", portid, rx_queueid, 2217 socket_id); 2218 2219 rxq_conf = dev_info.default_rxconf; 2220 rxq_conf.offloads = local_port_conf.rxmode.offloads; 2221 ret = rte_eth_rx_queue_setup(portid, rx_queueid, 2222 nb_rxd, socket_id, &rxq_conf, 2223 socket_ctx[socket_id].mbuf_pool); 2224 if (ret < 0) 2225 rte_exit(EXIT_FAILURE, 2226 "rte_eth_rx_queue_setup: err=%d, " 2227 "port=%d\n", ret, portid); 2228 } 2229 } 2230 printf("\n"); 2231 } 2232 2233 static size_t 2234 max_session_size(void) 2235 { 2236 size_t max_sz, sz; 2237 void *sec_ctx; 2238 int16_t cdev_id, port_id, n; 2239 2240 max_sz = 0; 2241 n = rte_cryptodev_count(); 2242 for (cdev_id = 0; cdev_id != n; cdev_id++) { 2243 sz = rte_cryptodev_sym_get_private_session_size(cdev_id); 2244 if (sz > max_sz) 2245 max_sz = sz; 2246 /* 2247 * If crypto device is security capable, need to check the 2248 * size of security session as well. 2249 */ 2250 2251 /* Get security context of the crypto device */ 2252 sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id); 2253 if (sec_ctx == NULL) 2254 continue; 2255 2256 /* Get size of security session */ 2257 sz = rte_security_session_get_size(sec_ctx); 2258 if (sz > max_sz) 2259 max_sz = sz; 2260 } 2261 2262 RTE_ETH_FOREACH_DEV(port_id) { 2263 if ((enabled_port_mask & (1 << port_id)) == 0) 2264 continue; 2265 2266 sec_ctx = rte_eth_dev_get_sec_ctx(port_id); 2267 if (sec_ctx == NULL) 2268 continue; 2269 2270 sz = rte_security_session_get_size(sec_ctx); 2271 if (sz > max_sz) 2272 max_sz = sz; 2273 } 2274 2275 return max_sz; 2276 } 2277 2278 static void 2279 session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz) 2280 { 2281 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2282 struct rte_mempool *sess_mp; 2283 uint32_t nb_sess; 2284 2285 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2286 "sess_mp_%u", socket_id); 2287 /* 2288 * Doubled due to rte_security_session_create() uses one mempool for 2289 * session and for session private data. 2290 */ 2291 nb_sess = (get_nb_crypto_sessions() + CDEV_MP_CACHE_SZ * 2292 rte_lcore_count()) * 2; 2293 sess_mp = rte_cryptodev_sym_session_pool_create( 2294 mp_name, nb_sess, sess_sz, CDEV_MP_CACHE_SZ, 0, 2295 socket_id); 2296 ctx->session_pool = sess_mp; 2297 2298 if (ctx->session_pool == NULL) 2299 rte_exit(EXIT_FAILURE, 2300 "Cannot init session pool on socket %d\n", socket_id); 2301 else 2302 printf("Allocated session pool on socket %d\n", socket_id); 2303 } 2304 2305 static void 2306 session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id, 2307 size_t sess_sz) 2308 { 2309 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2310 struct rte_mempool *sess_mp; 2311 uint32_t nb_sess; 2312 2313 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2314 "sess_mp_priv_%u", socket_id); 2315 /* 2316 * Doubled due to rte_security_session_create() uses one mempool for 2317 * session and for session private data. 2318 */ 2319 nb_sess = (get_nb_crypto_sessions() + CDEV_MP_CACHE_SZ * 2320 rte_lcore_count()) * 2; 2321 sess_mp = rte_mempool_create(mp_name, 2322 nb_sess, 2323 sess_sz, 2324 CDEV_MP_CACHE_SZ, 2325 0, NULL, NULL, NULL, 2326 NULL, socket_id, 2327 0); 2328 ctx->session_priv_pool = sess_mp; 2329 2330 if (ctx->session_priv_pool == NULL) 2331 rte_exit(EXIT_FAILURE, 2332 "Cannot init session priv pool on socket %d\n", 2333 socket_id); 2334 else 2335 printf("Allocated session priv pool on socket %d\n", 2336 socket_id); 2337 } 2338 2339 static void 2340 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf) 2341 { 2342 char s[64]; 2343 int32_t ms; 2344 2345 snprintf(s, sizeof(s), "mbuf_pool_%d", socket_id); 2346 ctx->mbuf_pool = rte_pktmbuf_pool_create(s, nb_mbuf, 2347 MEMPOOL_CACHE_SIZE, ipsec_metadata_size(), 2348 frame_buf_size, socket_id); 2349 2350 /* 2351 * if multi-segment support is enabled, then create a pool 2352 * for indirect mbufs. 2353 */ 2354 ms = multi_seg_required(); 2355 if (ms != 0) { 2356 snprintf(s, sizeof(s), "mbuf_pool_indir_%d", socket_id); 2357 ctx->mbuf_pool_indir = rte_pktmbuf_pool_create(s, nb_mbuf, 2358 MEMPOOL_CACHE_SIZE, 0, 0, socket_id); 2359 } 2360 2361 if (ctx->mbuf_pool == NULL || (ms != 0 && ctx->mbuf_pool_indir == NULL)) 2362 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n", 2363 socket_id); 2364 else 2365 printf("Allocated mbuf pool on socket %d\n", socket_id); 2366 } 2367 2368 static inline int 2369 inline_ipsec_event_esn_overflow(struct rte_security_ctx *ctx, uint64_t md) 2370 { 2371 struct ipsec_sa *sa; 2372 2373 /* For inline protocol processing, the metadata in the event will 2374 * uniquely identify the security session which raised the event. 2375 * Application would then need the userdata it had registered with the 2376 * security session to process the event. 2377 */ 2378 2379 sa = (struct ipsec_sa *)rte_security_get_userdata(ctx, md); 2380 2381 if (sa == NULL) { 2382 /* userdata could not be retrieved */ 2383 return -1; 2384 } 2385 2386 /* Sequence number over flow. SA need to be re-established */ 2387 RTE_SET_USED(sa); 2388 return 0; 2389 } 2390 2391 static int 2392 inline_ipsec_event_callback(uint16_t port_id, enum rte_eth_event_type type, 2393 void *param, void *ret_param) 2394 { 2395 uint64_t md; 2396 struct rte_eth_event_ipsec_desc *event_desc = NULL; 2397 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 2398 rte_eth_dev_get_sec_ctx(port_id); 2399 2400 RTE_SET_USED(param); 2401 2402 if (type != RTE_ETH_EVENT_IPSEC) 2403 return -1; 2404 2405 event_desc = ret_param; 2406 if (event_desc == NULL) { 2407 printf("Event descriptor not set\n"); 2408 return -1; 2409 } 2410 2411 md = event_desc->metadata; 2412 2413 if (event_desc->subtype == RTE_ETH_EVENT_IPSEC_ESN_OVERFLOW) 2414 return inline_ipsec_event_esn_overflow(ctx, md); 2415 else if (event_desc->subtype >= RTE_ETH_EVENT_IPSEC_MAX) { 2416 printf("Invalid IPsec event reported\n"); 2417 return -1; 2418 } 2419 2420 return -1; 2421 } 2422 2423 static uint16_t 2424 rx_callback(__rte_unused uint16_t port, __rte_unused uint16_t queue, 2425 struct rte_mbuf *pkt[], uint16_t nb_pkts, 2426 __rte_unused uint16_t max_pkts, void *user_param) 2427 { 2428 uint64_t tm; 2429 uint32_t i, k; 2430 struct lcore_conf *lc; 2431 struct rte_mbuf *mb; 2432 struct rte_ether_hdr *eth; 2433 2434 lc = user_param; 2435 k = 0; 2436 tm = 0; 2437 2438 for (i = 0; i != nb_pkts; i++) { 2439 2440 mb = pkt[i]; 2441 eth = rte_pktmbuf_mtod(mb, struct rte_ether_hdr *); 2442 if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 2443 2444 struct rte_ipv4_hdr *iph; 2445 2446 iph = (struct rte_ipv4_hdr *)(eth + 1); 2447 if (rte_ipv4_frag_pkt_is_fragmented(iph)) { 2448 2449 mb->l2_len = sizeof(*eth); 2450 mb->l3_len = sizeof(*iph); 2451 tm = (tm != 0) ? tm : rte_rdtsc(); 2452 mb = rte_ipv4_frag_reassemble_packet( 2453 lc->frag.tbl, &lc->frag.dr, 2454 mb, tm, iph); 2455 2456 if (mb != NULL) { 2457 /* fix ip cksum after reassemble. */ 2458 iph = rte_pktmbuf_mtod_offset(mb, 2459 struct rte_ipv4_hdr *, 2460 mb->l2_len); 2461 iph->hdr_checksum = 0; 2462 iph->hdr_checksum = rte_ipv4_cksum(iph); 2463 } 2464 } 2465 } else if (eth->ether_type == 2466 rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 2467 2468 struct rte_ipv6_hdr *iph; 2469 struct ipv6_extension_fragment *fh; 2470 2471 iph = (struct rte_ipv6_hdr *)(eth + 1); 2472 fh = rte_ipv6_frag_get_ipv6_fragment_header(iph); 2473 if (fh != NULL) { 2474 mb->l2_len = sizeof(*eth); 2475 mb->l3_len = (uintptr_t)fh - (uintptr_t)iph + 2476 sizeof(*fh); 2477 tm = (tm != 0) ? tm : rte_rdtsc(); 2478 mb = rte_ipv6_frag_reassemble_packet( 2479 lc->frag.tbl, &lc->frag.dr, 2480 mb, tm, iph, fh); 2481 if (mb != NULL) 2482 /* fix l3_len after reassemble. */ 2483 mb->l3_len = mb->l3_len - sizeof(*fh); 2484 } 2485 } 2486 2487 pkt[k] = mb; 2488 k += (mb != NULL); 2489 } 2490 2491 /* some fragments were encountered, drain death row */ 2492 if (tm != 0) 2493 rte_ip_frag_free_death_row(&lc->frag.dr, 0); 2494 2495 return k; 2496 } 2497 2498 2499 static int 2500 reassemble_lcore_init(struct lcore_conf *lc, uint32_t cid) 2501 { 2502 int32_t sid; 2503 uint32_t i; 2504 uint64_t frag_cycles; 2505 const struct lcore_rx_queue *rxq; 2506 const struct rte_eth_rxtx_callback *cb; 2507 2508 /* create fragment table */ 2509 sid = rte_lcore_to_socket_id(cid); 2510 frag_cycles = (rte_get_tsc_hz() + NS_PER_S - 1) / 2511 NS_PER_S * frag_ttl_ns; 2512 2513 lc->frag.tbl = rte_ip_frag_table_create(frag_tbl_sz, 2514 FRAG_TBL_BUCKET_ENTRIES, frag_tbl_sz, frag_cycles, sid); 2515 if (lc->frag.tbl == NULL) { 2516 printf("%s(%u): failed to create fragment table of size: %u, " 2517 "error code: %d\n", 2518 __func__, cid, frag_tbl_sz, rte_errno); 2519 return -ENOMEM; 2520 } 2521 2522 /* setup reassemble RX callbacks for all queues */ 2523 for (i = 0; i != lc->nb_rx_queue; i++) { 2524 2525 rxq = lc->rx_queue_list + i; 2526 cb = rte_eth_add_rx_callback(rxq->port_id, rxq->queue_id, 2527 rx_callback, lc); 2528 if (cb == NULL) { 2529 printf("%s(%u): failed to install RX callback for " 2530 "portid=%u, queueid=%u, error code: %d\n", 2531 __func__, cid, 2532 rxq->port_id, rxq->queue_id, rte_errno); 2533 return -ENOMEM; 2534 } 2535 } 2536 2537 return 0; 2538 } 2539 2540 static int 2541 reassemble_init(void) 2542 { 2543 int32_t rc; 2544 uint32_t i, lc; 2545 2546 rc = 0; 2547 for (i = 0; i != nb_lcore_params; i++) { 2548 lc = lcore_params[i].lcore_id; 2549 rc = reassemble_lcore_init(lcore_conf + lc, lc); 2550 if (rc != 0) 2551 break; 2552 } 2553 2554 return rc; 2555 } 2556 2557 static void 2558 create_default_ipsec_flow(uint16_t port_id, uint64_t rx_offloads) 2559 { 2560 struct rte_flow_action action[2]; 2561 struct rte_flow_item pattern[2]; 2562 struct rte_flow_attr attr = {0}; 2563 struct rte_flow_error err; 2564 struct rte_flow *flow; 2565 int ret; 2566 2567 if (!(rx_offloads & DEV_RX_OFFLOAD_SECURITY)) 2568 return; 2569 2570 /* Add the default rte_flow to enable SECURITY for all ESP packets */ 2571 2572 pattern[0].type = RTE_FLOW_ITEM_TYPE_ESP; 2573 pattern[0].spec = NULL; 2574 pattern[0].mask = NULL; 2575 pattern[0].last = NULL; 2576 pattern[1].type = RTE_FLOW_ITEM_TYPE_END; 2577 2578 action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY; 2579 action[0].conf = NULL; 2580 action[1].type = RTE_FLOW_ACTION_TYPE_END; 2581 action[1].conf = NULL; 2582 2583 attr.ingress = 1; 2584 2585 ret = rte_flow_validate(port_id, &attr, pattern, action, &err); 2586 if (ret) 2587 return; 2588 2589 flow = rte_flow_create(port_id, &attr, pattern, action, &err); 2590 if (flow == NULL) 2591 return; 2592 2593 flow_info_tbl[port_id].rx_def_flow = flow; 2594 RTE_LOG(INFO, IPSEC, 2595 "Created default flow enabling SECURITY for all ESP traffic on port %d\n", 2596 port_id); 2597 } 2598 2599 static void 2600 signal_handler(int signum) 2601 { 2602 if (signum == SIGINT || signum == SIGTERM) { 2603 printf("\n\nSignal %d received, preparing to exit...\n", 2604 signum); 2605 force_quit = true; 2606 } 2607 } 2608 2609 static void 2610 ev_mode_sess_verify(struct ipsec_sa *sa, int nb_sa) 2611 { 2612 struct rte_ipsec_session *ips; 2613 int32_t i; 2614 2615 if (!sa || !nb_sa) 2616 return; 2617 2618 for (i = 0; i < nb_sa; i++) { 2619 ips = ipsec_get_primary_session(&sa[i]); 2620 if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) 2621 rte_exit(EXIT_FAILURE, "Event mode supports only " 2622 "inline protocol sessions\n"); 2623 } 2624 2625 } 2626 2627 static int32_t 2628 check_event_mode_params(struct eh_conf *eh_conf) 2629 { 2630 struct eventmode_conf *em_conf = NULL; 2631 struct lcore_params *params; 2632 uint16_t portid; 2633 2634 if (!eh_conf || !eh_conf->mode_params) 2635 return -EINVAL; 2636 2637 /* Get eventmode conf */ 2638 em_conf = eh_conf->mode_params; 2639 2640 if (eh_conf->mode == EH_PKT_TRANSFER_MODE_POLL && 2641 em_conf->ext_params.sched_type != SCHED_TYPE_NOT_SET) { 2642 printf("error: option --event-schedule-type applies only to " 2643 "event mode\n"); 2644 return -EINVAL; 2645 } 2646 2647 if (eh_conf->mode != EH_PKT_TRANSFER_MODE_EVENT) 2648 return 0; 2649 2650 /* Set schedule type to ORDERED if it wasn't explicitly set by user */ 2651 if (em_conf->ext_params.sched_type == SCHED_TYPE_NOT_SET) 2652 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ORDERED; 2653 2654 /* 2655 * Event mode currently supports only inline protocol sessions. 2656 * If there are other types of sessions configured then exit with 2657 * error. 2658 */ 2659 ev_mode_sess_verify(sa_in, nb_sa_in); 2660 ev_mode_sess_verify(sa_out, nb_sa_out); 2661 2662 2663 /* Option --config does not apply to event mode */ 2664 if (nb_lcore_params > 0) { 2665 printf("error: option --config applies only to poll mode\n"); 2666 return -EINVAL; 2667 } 2668 2669 /* 2670 * In order to use the same port_init routine for both poll and event 2671 * modes initialize lcore_params with one queue for each eth port 2672 */ 2673 lcore_params = lcore_params_array; 2674 RTE_ETH_FOREACH_DEV(portid) { 2675 if ((enabled_port_mask & (1 << portid)) == 0) 2676 continue; 2677 2678 params = &lcore_params[nb_lcore_params++]; 2679 params->port_id = portid; 2680 params->queue_id = 0; 2681 params->lcore_id = rte_get_next_lcore(0, 0, 1); 2682 } 2683 2684 return 0; 2685 } 2686 2687 static void 2688 inline_sessions_free(struct sa_ctx *sa_ctx) 2689 { 2690 struct rte_ipsec_session *ips; 2691 struct ipsec_sa *sa; 2692 int32_t ret; 2693 uint32_t i; 2694 2695 if (!sa_ctx) 2696 return; 2697 2698 for (i = 0; i < sa_ctx->nb_sa; i++) { 2699 2700 sa = &sa_ctx->sa[i]; 2701 if (!sa->spi) 2702 continue; 2703 2704 ips = ipsec_get_primary_session(sa); 2705 if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL && 2706 ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) 2707 continue; 2708 2709 if (!rte_eth_dev_is_valid_port(sa->portid)) 2710 continue; 2711 2712 ret = rte_security_session_destroy( 2713 rte_eth_dev_get_sec_ctx(sa->portid), 2714 ips->security.ses); 2715 if (ret) 2716 RTE_LOG(ERR, IPSEC, "Failed to destroy security " 2717 "session type %d, spi %d\n", 2718 ips->type, sa->spi); 2719 } 2720 } 2721 2722 static uint32_t 2723 calculate_nb_mbufs(uint16_t nb_ports, uint16_t nb_crypto_qp, uint32_t nb_rxq, 2724 uint32_t nb_txq) 2725 { 2726 return RTE_MAX((nb_rxq * nb_rxd + 2727 nb_ports * nb_lcores * MAX_PKT_BURST + 2728 nb_ports * nb_txq * nb_txd + 2729 nb_lcores * MEMPOOL_CACHE_SIZE + 2730 nb_crypto_qp * CDEV_QUEUE_DESC + 2731 nb_lcores * frag_tbl_sz * 2732 FRAG_TBL_BUCKET_ENTRIES), 2733 8192U); 2734 } 2735 2736 int32_t 2737 main(int32_t argc, char **argv) 2738 { 2739 int32_t ret; 2740 uint32_t lcore_id, nb_txq, nb_rxq = 0; 2741 uint32_t cdev_id; 2742 uint32_t i; 2743 uint8_t socket_id; 2744 uint16_t portid, nb_crypto_qp, nb_ports = 0; 2745 uint64_t req_rx_offloads[RTE_MAX_ETHPORTS]; 2746 uint64_t req_tx_offloads[RTE_MAX_ETHPORTS]; 2747 struct eh_conf *eh_conf = NULL; 2748 size_t sess_sz; 2749 2750 nb_bufs_in_pool = 0; 2751 2752 /* init EAL */ 2753 ret = rte_eal_init(argc, argv); 2754 if (ret < 0) 2755 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 2756 argc -= ret; 2757 argv += ret; 2758 2759 force_quit = false; 2760 signal(SIGINT, signal_handler); 2761 signal(SIGTERM, signal_handler); 2762 2763 /* initialize event helper configuration */ 2764 eh_conf = eh_conf_init(); 2765 if (eh_conf == NULL) 2766 rte_exit(EXIT_FAILURE, "Failed to init event helper config"); 2767 2768 /* parse application arguments (after the EAL ones) */ 2769 ret = parse_args(argc, argv, eh_conf); 2770 if (ret < 0) 2771 rte_exit(EXIT_FAILURE, "Invalid parameters\n"); 2772 2773 /* parse configuration file */ 2774 if (parse_cfg_file(cfgfile) < 0) { 2775 printf("parsing file \"%s\" failed\n", 2776 optarg); 2777 print_usage(argv[0]); 2778 return -1; 2779 } 2780 2781 if ((unprotected_port_mask & enabled_port_mask) != 2782 unprotected_port_mask) 2783 rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n", 2784 unprotected_port_mask); 2785 2786 if (check_poll_mode_params(eh_conf) < 0) 2787 rte_exit(EXIT_FAILURE, "check_poll_mode_params failed\n"); 2788 2789 if (check_event_mode_params(eh_conf) < 0) 2790 rte_exit(EXIT_FAILURE, "check_event_mode_params failed\n"); 2791 2792 ret = init_lcore_rx_queues(); 2793 if (ret < 0) 2794 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 2795 2796 nb_lcores = rte_lcore_count(); 2797 2798 sess_sz = max_session_size(); 2799 2800 /* 2801 * In event mode request minimum number of crypto queues 2802 * to be reserved equal to number of ports. 2803 */ 2804 if (eh_conf->mode == EH_PKT_TRANSFER_MODE_EVENT) 2805 nb_crypto_qp = rte_eth_dev_count_avail(); 2806 else 2807 nb_crypto_qp = 0; 2808 2809 nb_crypto_qp = cryptodevs_init(nb_crypto_qp); 2810 2811 if (nb_bufs_in_pool == 0) { 2812 RTE_ETH_FOREACH_DEV(portid) { 2813 if ((enabled_port_mask & (1 << portid)) == 0) 2814 continue; 2815 nb_ports++; 2816 nb_rxq += get_port_nb_rx_queues(portid); 2817 } 2818 2819 nb_txq = nb_lcores; 2820 2821 nb_bufs_in_pool = calculate_nb_mbufs(nb_ports, nb_crypto_qp, 2822 nb_rxq, nb_txq); 2823 } 2824 2825 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2826 if (rte_lcore_is_enabled(lcore_id) == 0) 2827 continue; 2828 2829 if (numa_on) 2830 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2831 else 2832 socket_id = 0; 2833 2834 /* mbuf_pool is initialised by the pool_init() function*/ 2835 if (socket_ctx[socket_id].mbuf_pool) 2836 continue; 2837 2838 pool_init(&socket_ctx[socket_id], socket_id, nb_bufs_in_pool); 2839 session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz); 2840 session_priv_pool_init(&socket_ctx[socket_id], socket_id, 2841 sess_sz); 2842 } 2843 printf("Number of mbufs in packet pool %d\n", nb_bufs_in_pool); 2844 2845 RTE_ETH_FOREACH_DEV(portid) { 2846 if ((enabled_port_mask & (1 << portid)) == 0) 2847 continue; 2848 2849 sa_check_offloads(portid, &req_rx_offloads[portid], 2850 &req_tx_offloads[portid]); 2851 port_init(portid, req_rx_offloads[portid], 2852 req_tx_offloads[portid]); 2853 } 2854 2855 /* 2856 * Set the enabled port mask in helper config for use by helper 2857 * sub-system. This will be used while initializing devices using 2858 * helper sub-system. 2859 */ 2860 eh_conf->eth_portmask = enabled_port_mask; 2861 2862 /* Initialize eventmode components */ 2863 ret = eh_devs_init(eh_conf); 2864 if (ret < 0) 2865 rte_exit(EXIT_FAILURE, "eh_devs_init failed, err=%d\n", ret); 2866 2867 /* start ports */ 2868 RTE_ETH_FOREACH_DEV(portid) { 2869 if ((enabled_port_mask & (1 << portid)) == 0) 2870 continue; 2871 2872 /* Create flow before starting the device */ 2873 create_default_ipsec_flow(portid, req_rx_offloads[portid]); 2874 2875 ret = rte_eth_dev_start(portid); 2876 if (ret < 0) 2877 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: " 2878 "err=%d, port=%d\n", ret, portid); 2879 /* 2880 * If enabled, put device in promiscuous mode. 2881 * This allows IO forwarding mode to forward packets 2882 * to itself through 2 cross-connected ports of the 2883 * target machine. 2884 */ 2885 if (promiscuous_on) { 2886 ret = rte_eth_promiscuous_enable(portid); 2887 if (ret != 0) 2888 rte_exit(EXIT_FAILURE, 2889 "rte_eth_promiscuous_enable: err=%s, port=%d\n", 2890 rte_strerror(-ret), portid); 2891 } 2892 2893 rte_eth_dev_callback_register(portid, 2894 RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL); 2895 } 2896 2897 /* fragment reassemble is enabled */ 2898 if (frag_tbl_sz != 0) { 2899 ret = reassemble_init(); 2900 if (ret != 0) 2901 rte_exit(EXIT_FAILURE, "failed at reassemble init"); 2902 } 2903 2904 /* Replicate each context per socket */ 2905 for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) { 2906 socket_id = rte_socket_id_by_idx(i); 2907 if ((socket_ctx[socket_id].mbuf_pool != NULL) && 2908 (socket_ctx[socket_id].sa_in == NULL) && 2909 (socket_ctx[socket_id].sa_out == NULL)) { 2910 sa_init(&socket_ctx[socket_id], socket_id); 2911 sp4_init(&socket_ctx[socket_id], socket_id); 2912 sp6_init(&socket_ctx[socket_id], socket_id); 2913 rt_init(&socket_ctx[socket_id], socket_id); 2914 } 2915 } 2916 2917 check_all_ports_link_status(enabled_port_mask); 2918 2919 /* launch per-lcore init on every lcore */ 2920 rte_eal_mp_remote_launch(ipsec_launch_one_lcore, eh_conf, CALL_MASTER); 2921 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 2922 if (rte_eal_wait_lcore(lcore_id) < 0) 2923 return -1; 2924 } 2925 2926 /* Uninitialize eventmode components */ 2927 ret = eh_devs_uninit(eh_conf); 2928 if (ret < 0) 2929 rte_exit(EXIT_FAILURE, "eh_devs_uninit failed, err=%d\n", ret); 2930 2931 /* Free eventmode configuration memory */ 2932 eh_conf_uninit(eh_conf); 2933 2934 /* Destroy inline inbound and outbound sessions */ 2935 for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) { 2936 socket_id = rte_socket_id_by_idx(i); 2937 inline_sessions_free(socket_ctx[socket_id].sa_in); 2938 inline_sessions_free(socket_ctx[socket_id].sa_out); 2939 } 2940 2941 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 2942 printf("Closing cryptodev %d...", cdev_id); 2943 rte_cryptodev_stop(cdev_id); 2944 rte_cryptodev_close(cdev_id); 2945 printf(" Done\n"); 2946 } 2947 2948 RTE_ETH_FOREACH_DEV(portid) { 2949 if ((enabled_port_mask & (1 << portid)) == 0) 2950 continue; 2951 2952 printf("Closing port %d...", portid); 2953 if (flow_info_tbl[portid].rx_def_flow) { 2954 struct rte_flow_error err; 2955 2956 ret = rte_flow_destroy(portid, 2957 flow_info_tbl[portid].rx_def_flow, &err); 2958 if (ret) 2959 RTE_LOG(ERR, IPSEC, "Failed to destroy flow " 2960 " for port %u, err msg: %s\n", portid, 2961 err.message); 2962 } 2963 rte_eth_dev_stop(portid); 2964 rte_eth_dev_close(portid); 2965 printf(" Done\n"); 2966 } 2967 printf("Bye...\n"); 2968 2969 return 0; 2970 } 2971