1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 Intel Corporation 3 */ 4 5 #include <stdbool.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <stdint.h> 9 #include <inttypes.h> 10 #include <sys/types.h> 11 #include <netinet/in.h> 12 #include <netinet/ip.h> 13 #include <netinet/ip6.h> 14 #include <string.h> 15 #include <sys/queue.h> 16 #include <stdarg.h> 17 #include <errno.h> 18 #include <signal.h> 19 #include <getopt.h> 20 21 #include <rte_common.h> 22 #include <rte_bitmap.h> 23 #include <rte_byteorder.h> 24 #include <rte_log.h> 25 #include <rte_eal.h> 26 #include <rte_launch.h> 27 #include <rte_atomic.h> 28 #include <rte_cycles.h> 29 #include <rte_prefetch.h> 30 #include <rte_lcore.h> 31 #include <rte_per_lcore.h> 32 #include <rte_branch_prediction.h> 33 #include <rte_interrupts.h> 34 #include <rte_random.h> 35 #include <rte_debug.h> 36 #include <rte_ether.h> 37 #include <rte_ethdev.h> 38 #include <rte_mempool.h> 39 #include <rte_mbuf.h> 40 #include <rte_acl.h> 41 #include <rte_lpm.h> 42 #include <rte_lpm6.h> 43 #include <rte_hash.h> 44 #include <rte_jhash.h> 45 #include <rte_cryptodev.h> 46 #include <rte_security.h> 47 #include <rte_eventdev.h> 48 #include <rte_ip.h> 49 #include <rte_ip_frag.h> 50 #include <rte_alarm.h> 51 52 #include "event_helper.h" 53 #include "flow.h" 54 #include "ipsec.h" 55 #include "ipsec_worker.h" 56 #include "parser.h" 57 #include "sad.h" 58 59 volatile bool force_quit; 60 61 #define MAX_JUMBO_PKT_LEN 9600 62 63 #define MEMPOOL_CACHE_SIZE 256 64 65 #define CDEV_QUEUE_DESC 2048 66 #define CDEV_MAP_ENTRIES 16384 67 #define CDEV_MP_CACHE_SZ 64 68 #define CDEV_MP_CACHE_MULTIPLIER 1.5 /* from rte_mempool.c */ 69 #define MAX_QUEUE_PAIRS 1 70 71 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 72 73 /* Configure how many packets ahead to prefetch, when reading packets */ 74 #define PREFETCH_OFFSET 3 75 76 #define MAX_RX_QUEUE_PER_LCORE 16 77 78 #define MAX_LCORE_PARAMS 1024 79 80 /* 81 * Configurable number of RX/TX ring descriptors 82 */ 83 #define IPSEC_SECGW_RX_DESC_DEFAULT 1024 84 #define IPSEC_SECGW_TX_DESC_DEFAULT 1024 85 static uint16_t nb_rxd = IPSEC_SECGW_RX_DESC_DEFAULT; 86 static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT; 87 88 #define ETHADDR_TO_UINT64(addr) __BYTES_TO_UINT64( \ 89 (addr)->addr_bytes[0], (addr)->addr_bytes[1], \ 90 (addr)->addr_bytes[2], (addr)->addr_bytes[3], \ 91 (addr)->addr_bytes[4], (addr)->addr_bytes[5], \ 92 0, 0) 93 94 #define FRAG_TBL_BUCKET_ENTRIES 4 95 #define MAX_FRAG_TTL_NS (10LL * NS_PER_S) 96 97 #define MTU_TO_FRAMELEN(x) ((x) + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN) 98 99 struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = { 100 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a) }, 101 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9) }, 102 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x08, 0x69, 0x26) }, 103 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd) } 104 }; 105 106 struct flow_info flow_info_tbl[RTE_MAX_ETHPORTS]; 107 108 #define CMD_LINE_OPT_CONFIG "config" 109 #define CMD_LINE_OPT_SINGLE_SA "single-sa" 110 #define CMD_LINE_OPT_CRYPTODEV_MASK "cryptodev_mask" 111 #define CMD_LINE_OPT_TRANSFER_MODE "transfer-mode" 112 #define CMD_LINE_OPT_SCHEDULE_TYPE "event-schedule-type" 113 #define CMD_LINE_OPT_RX_OFFLOAD "rxoffload" 114 #define CMD_LINE_OPT_TX_OFFLOAD "txoffload" 115 #define CMD_LINE_OPT_REASSEMBLE "reassemble" 116 #define CMD_LINE_OPT_MTU "mtu" 117 #define CMD_LINE_OPT_FRAG_TTL "frag-ttl" 118 119 #define CMD_LINE_ARG_EVENT "event" 120 #define CMD_LINE_ARG_POLL "poll" 121 #define CMD_LINE_ARG_ORDERED "ordered" 122 #define CMD_LINE_ARG_ATOMIC "atomic" 123 #define CMD_LINE_ARG_PARALLEL "parallel" 124 125 enum { 126 /* long options mapped to a short option */ 127 128 /* first long only option value must be >= 256, so that we won't 129 * conflict with short options 130 */ 131 CMD_LINE_OPT_MIN_NUM = 256, 132 CMD_LINE_OPT_CONFIG_NUM, 133 CMD_LINE_OPT_SINGLE_SA_NUM, 134 CMD_LINE_OPT_CRYPTODEV_MASK_NUM, 135 CMD_LINE_OPT_TRANSFER_MODE_NUM, 136 CMD_LINE_OPT_SCHEDULE_TYPE_NUM, 137 CMD_LINE_OPT_RX_OFFLOAD_NUM, 138 CMD_LINE_OPT_TX_OFFLOAD_NUM, 139 CMD_LINE_OPT_REASSEMBLE_NUM, 140 CMD_LINE_OPT_MTU_NUM, 141 CMD_LINE_OPT_FRAG_TTL_NUM, 142 }; 143 144 static const struct option lgopts[] = { 145 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, 146 {CMD_LINE_OPT_SINGLE_SA, 1, 0, CMD_LINE_OPT_SINGLE_SA_NUM}, 147 {CMD_LINE_OPT_CRYPTODEV_MASK, 1, 0, CMD_LINE_OPT_CRYPTODEV_MASK_NUM}, 148 {CMD_LINE_OPT_TRANSFER_MODE, 1, 0, CMD_LINE_OPT_TRANSFER_MODE_NUM}, 149 {CMD_LINE_OPT_SCHEDULE_TYPE, 1, 0, CMD_LINE_OPT_SCHEDULE_TYPE_NUM}, 150 {CMD_LINE_OPT_RX_OFFLOAD, 1, 0, CMD_LINE_OPT_RX_OFFLOAD_NUM}, 151 {CMD_LINE_OPT_TX_OFFLOAD, 1, 0, CMD_LINE_OPT_TX_OFFLOAD_NUM}, 152 {CMD_LINE_OPT_REASSEMBLE, 1, 0, CMD_LINE_OPT_REASSEMBLE_NUM}, 153 {CMD_LINE_OPT_MTU, 1, 0, CMD_LINE_OPT_MTU_NUM}, 154 {CMD_LINE_OPT_FRAG_TTL, 1, 0, CMD_LINE_OPT_FRAG_TTL_NUM}, 155 {NULL, 0, 0, 0} 156 }; 157 158 uint32_t unprotected_port_mask; 159 uint32_t single_sa_idx; 160 /* mask of enabled ports */ 161 static uint32_t enabled_port_mask; 162 static uint64_t enabled_cryptodev_mask = UINT64_MAX; 163 static int32_t promiscuous_on = 1; 164 static int32_t numa_on = 1; /**< NUMA is enabled by default. */ 165 static uint32_t nb_lcores; 166 static uint32_t single_sa; 167 static uint32_t nb_bufs_in_pool; 168 169 /* 170 * RX/TX HW offload capabilities to enable/use on ethernet ports. 171 * By default all capabilities are enabled. 172 */ 173 static uint64_t dev_rx_offload = UINT64_MAX; 174 static uint64_t dev_tx_offload = UINT64_MAX; 175 176 /* 177 * global values that determine multi-seg policy 178 */ 179 static uint32_t frag_tbl_sz; 180 static uint32_t frame_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE; 181 static uint32_t mtu_size = RTE_ETHER_MTU; 182 static uint64_t frag_ttl_ns = MAX_FRAG_TTL_NS; 183 184 /* application wide librte_ipsec/SA parameters */ 185 struct app_sa_prm app_sa_prm = { 186 .enable = 0, 187 .cache_sz = SA_CACHE_SZ, 188 .udp_encap = 0 189 }; 190 static const char *cfgfile; 191 192 struct lcore_rx_queue { 193 uint16_t port_id; 194 uint8_t queue_id; 195 } __rte_cache_aligned; 196 197 struct lcore_params { 198 uint16_t port_id; 199 uint8_t queue_id; 200 uint8_t lcore_id; 201 } __rte_cache_aligned; 202 203 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 204 205 static struct lcore_params *lcore_params; 206 static uint16_t nb_lcore_params; 207 208 static struct rte_hash *cdev_map_in; 209 static struct rte_hash *cdev_map_out; 210 211 struct buffer { 212 uint16_t len; 213 struct rte_mbuf *m_table[MAX_PKT_BURST] __rte_aligned(sizeof(void *)); 214 }; 215 216 struct lcore_conf { 217 uint16_t nb_rx_queue; 218 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 219 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 220 struct buffer tx_mbufs[RTE_MAX_ETHPORTS]; 221 struct ipsec_ctx inbound; 222 struct ipsec_ctx outbound; 223 struct rt_ctx *rt4_ctx; 224 struct rt_ctx *rt6_ctx; 225 struct { 226 struct rte_ip_frag_tbl *tbl; 227 struct rte_mempool *pool_dir; 228 struct rte_mempool *pool_indir; 229 struct rte_ip_frag_death_row dr; 230 } frag; 231 } __rte_cache_aligned; 232 233 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 234 235 static struct rte_eth_conf port_conf = { 236 .rxmode = { 237 .mq_mode = ETH_MQ_RX_RSS, 238 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 239 .split_hdr_size = 0, 240 .offloads = DEV_RX_OFFLOAD_CHECKSUM, 241 }, 242 .rx_adv_conf = { 243 .rss_conf = { 244 .rss_key = NULL, 245 .rss_hf = ETH_RSS_IP | ETH_RSS_UDP | 246 ETH_RSS_TCP | ETH_RSS_SCTP, 247 }, 248 }, 249 .txmode = { 250 .mq_mode = ETH_MQ_TX_NONE, 251 }, 252 }; 253 254 struct socket_ctx socket_ctx[NB_SOCKETS]; 255 256 /* 257 * Determine is multi-segment support required: 258 * - either frame buffer size is smaller then mtu 259 * - or reassmeble support is requested 260 */ 261 static int 262 multi_seg_required(void) 263 { 264 return (MTU_TO_FRAMELEN(mtu_size) + RTE_PKTMBUF_HEADROOM > 265 frame_buf_size || frag_tbl_sz != 0); 266 } 267 268 static inline void 269 adjust_ipv4_pktlen(struct rte_mbuf *m, const struct rte_ipv4_hdr *iph, 270 uint32_t l2_len) 271 { 272 uint32_t plen, trim; 273 274 plen = rte_be_to_cpu_16(iph->total_length) + l2_len; 275 if (plen < m->pkt_len) { 276 trim = m->pkt_len - plen; 277 rte_pktmbuf_trim(m, trim); 278 } 279 } 280 281 static inline void 282 adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, 283 uint32_t l2_len) 284 { 285 uint32_t plen, trim; 286 287 plen = rte_be_to_cpu_16(iph->payload_len) + sizeof(*iph) + l2_len; 288 if (plen < m->pkt_len) { 289 trim = m->pkt_len - plen; 290 rte_pktmbuf_trim(m, trim); 291 } 292 } 293 294 #if (STATS_INTERVAL > 0) 295 296 /* Print out statistics on packet distribution */ 297 static void 298 print_stats_cb(__rte_unused void *param) 299 { 300 uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; 301 float burst_percent, rx_per_call, tx_per_call; 302 unsigned int coreid; 303 304 total_packets_dropped = 0; 305 total_packets_tx = 0; 306 total_packets_rx = 0; 307 308 const char clr[] = { 27, '[', '2', 'J', '\0' }; 309 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 310 311 /* Clear screen and move to top left */ 312 printf("%s%s", clr, topLeft); 313 314 printf("\nCore statistics ===================================="); 315 316 for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) { 317 /* skip disabled cores */ 318 if (rte_lcore_is_enabled(coreid) == 0) 319 continue; 320 burst_percent = (float)(core_statistics[coreid].burst_rx * 100)/ 321 core_statistics[coreid].rx; 322 rx_per_call = (float)(core_statistics[coreid].rx)/ 323 core_statistics[coreid].rx_call; 324 tx_per_call = (float)(core_statistics[coreid].tx)/ 325 core_statistics[coreid].tx_call; 326 printf("\nStatistics for core %u ------------------------------" 327 "\nPackets received: %20"PRIu64 328 "\nPackets sent: %24"PRIu64 329 "\nPackets dropped: %21"PRIu64 330 "\nBurst percent: %23.2f" 331 "\nPackets per Rx call: %17.2f" 332 "\nPackets per Tx call: %17.2f", 333 coreid, 334 core_statistics[coreid].rx, 335 core_statistics[coreid].tx, 336 core_statistics[coreid].dropped, 337 burst_percent, 338 rx_per_call, 339 tx_per_call); 340 341 total_packets_dropped += core_statistics[coreid].dropped; 342 total_packets_tx += core_statistics[coreid].tx; 343 total_packets_rx += core_statistics[coreid].rx; 344 } 345 printf("\nAggregate statistics ===============================" 346 "\nTotal packets received: %14"PRIu64 347 "\nTotal packets sent: %18"PRIu64 348 "\nTotal packets dropped: %15"PRIu64, 349 total_packets_rx, 350 total_packets_tx, 351 total_packets_dropped); 352 printf("\n====================================================\n"); 353 354 rte_eal_alarm_set(STATS_INTERVAL * US_PER_S, print_stats_cb, NULL); 355 } 356 #endif /* STATS_INTERVAL */ 357 358 static inline void 359 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t) 360 { 361 const struct rte_ether_hdr *eth; 362 const struct rte_ipv4_hdr *iph4; 363 const struct rte_ipv6_hdr *iph6; 364 const struct rte_udp_hdr *udp; 365 uint16_t ip4_hdr_len; 366 uint16_t nat_port; 367 368 eth = rte_pktmbuf_mtod(pkt, const struct rte_ether_hdr *); 369 if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 370 371 iph4 = (const struct rte_ipv4_hdr *)rte_pktmbuf_adj(pkt, 372 RTE_ETHER_HDR_LEN); 373 adjust_ipv4_pktlen(pkt, iph4, 0); 374 375 switch (iph4->next_proto_id) { 376 case IPPROTO_ESP: 377 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 378 break; 379 case IPPROTO_UDP: 380 if (app_sa_prm.udp_encap == 1) { 381 ip4_hdr_len = ((iph4->version_ihl & 382 RTE_IPV4_HDR_IHL_MASK) * 383 RTE_IPV4_IHL_MULTIPLIER); 384 udp = rte_pktmbuf_mtod_offset(pkt, 385 struct rte_udp_hdr *, ip4_hdr_len); 386 nat_port = rte_cpu_to_be_16(IPSEC_NAT_T_PORT); 387 if (udp->src_port == nat_port || 388 udp->dst_port == nat_port){ 389 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 390 pkt->packet_type |= 391 MBUF_PTYPE_TUNNEL_ESP_IN_UDP; 392 break; 393 } 394 } 395 /* Fall through */ 396 default: 397 t->ip4.data[t->ip4.num] = &iph4->next_proto_id; 398 t->ip4.pkts[(t->ip4.num)++] = pkt; 399 } 400 pkt->l2_len = 0; 401 pkt->l3_len = sizeof(*iph4); 402 pkt->packet_type |= RTE_PTYPE_L3_IPV4; 403 } else if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 404 int next_proto; 405 size_t l3len, ext_len; 406 uint8_t *p; 407 408 /* get protocol type */ 409 iph6 = (const struct rte_ipv6_hdr *)rte_pktmbuf_adj(pkt, 410 RTE_ETHER_HDR_LEN); 411 adjust_ipv6_pktlen(pkt, iph6, 0); 412 413 next_proto = iph6->proto; 414 415 /* determine l3 header size up to ESP extension */ 416 l3len = sizeof(struct ip6_hdr); 417 p = rte_pktmbuf_mtod(pkt, uint8_t *); 418 while (next_proto != IPPROTO_ESP && l3len < pkt->data_len && 419 (next_proto = rte_ipv6_get_next_ext(p + l3len, 420 next_proto, &ext_len)) >= 0) 421 l3len += ext_len; 422 423 /* drop packet when IPv6 header exceeds first segment length */ 424 if (unlikely(l3len > pkt->data_len)) { 425 free_pkts(&pkt, 1); 426 return; 427 } 428 429 switch (next_proto) { 430 case IPPROTO_ESP: 431 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 432 break; 433 case IPPROTO_UDP: 434 if (app_sa_prm.udp_encap == 1) { 435 udp = rte_pktmbuf_mtod_offset(pkt, 436 struct rte_udp_hdr *, l3len); 437 nat_port = rte_cpu_to_be_16(IPSEC_NAT_T_PORT); 438 if (udp->src_port == nat_port || 439 udp->dst_port == nat_port){ 440 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 441 pkt->packet_type |= 442 MBUF_PTYPE_TUNNEL_ESP_IN_UDP; 443 break; 444 } 445 } 446 /* Fall through */ 447 default: 448 t->ip6.data[t->ip6.num] = &iph6->proto; 449 t->ip6.pkts[(t->ip6.num)++] = pkt; 450 } 451 pkt->l2_len = 0; 452 pkt->l3_len = l3len; 453 pkt->packet_type |= RTE_PTYPE_L3_IPV6; 454 } else { 455 /* Unknown/Unsupported type, drop the packet */ 456 RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n", 457 rte_be_to_cpu_16(eth->ether_type)); 458 free_pkts(&pkt, 1); 459 return; 460 } 461 462 /* Check if the packet has been processed inline. For inline protocol 463 * processed packets, the metadata in the mbuf can be used to identify 464 * the security processing done on the packet. The metadata will be 465 * used to retrieve the application registered userdata associated 466 * with the security session. 467 */ 468 469 if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD && 470 rte_security_dynfield_is_registered()) { 471 struct ipsec_sa *sa; 472 struct ipsec_mbuf_metadata *priv; 473 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 474 rte_eth_dev_get_sec_ctx( 475 pkt->port); 476 477 /* Retrieve the userdata registered. Here, the userdata 478 * registered is the SA pointer. 479 */ 480 sa = (struct ipsec_sa *)rte_security_get_userdata(ctx, 481 *rte_security_dynfield(pkt)); 482 if (sa == NULL) { 483 /* userdata could not be retrieved */ 484 return; 485 } 486 487 /* Save SA as priv member in mbuf. This will be used in the 488 * IPsec selector(SP-SA) check. 489 */ 490 491 priv = get_priv(pkt); 492 priv->sa = sa; 493 } 494 } 495 496 static inline void 497 prepare_traffic(struct rte_mbuf **pkts, struct ipsec_traffic *t, 498 uint16_t nb_pkts) 499 { 500 int32_t i; 501 502 t->ipsec.num = 0; 503 t->ip4.num = 0; 504 t->ip6.num = 0; 505 506 for (i = 0; i < (nb_pkts - PREFETCH_OFFSET); i++) { 507 rte_prefetch0(rte_pktmbuf_mtod(pkts[i + PREFETCH_OFFSET], 508 void *)); 509 prepare_one_packet(pkts[i], t); 510 } 511 /* Process left packets */ 512 for (; i < nb_pkts; i++) 513 prepare_one_packet(pkts[i], t); 514 } 515 516 static inline void 517 prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port, 518 const struct lcore_conf *qconf) 519 { 520 struct ip *ip; 521 struct rte_ether_hdr *ethhdr; 522 523 ip = rte_pktmbuf_mtod(pkt, struct ip *); 524 525 ethhdr = (struct rte_ether_hdr *) 526 rte_pktmbuf_prepend(pkt, RTE_ETHER_HDR_LEN); 527 528 if (ip->ip_v == IPVERSION) { 529 pkt->ol_flags |= qconf->outbound.ipv4_offloads; 530 pkt->l3_len = sizeof(struct ip); 531 pkt->l2_len = RTE_ETHER_HDR_LEN; 532 533 ip->ip_sum = 0; 534 535 /* calculate IPv4 cksum in SW */ 536 if ((pkt->ol_flags & PKT_TX_IP_CKSUM) == 0) 537 ip->ip_sum = rte_ipv4_cksum((struct rte_ipv4_hdr *)ip); 538 539 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 540 } else { 541 pkt->ol_flags |= qconf->outbound.ipv6_offloads; 542 pkt->l3_len = sizeof(struct ip6_hdr); 543 pkt->l2_len = RTE_ETHER_HDR_LEN; 544 545 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); 546 } 547 548 memcpy(ðhdr->s_addr, ðaddr_tbl[port].src, 549 sizeof(struct rte_ether_addr)); 550 memcpy(ðhdr->d_addr, ðaddr_tbl[port].dst, 551 sizeof(struct rte_ether_addr)); 552 } 553 554 static inline void 555 prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t port, 556 const struct lcore_conf *qconf) 557 { 558 int32_t i; 559 const int32_t prefetch_offset = 2; 560 561 for (i = 0; i < (nb_pkts - prefetch_offset); i++) { 562 rte_mbuf_prefetch_part2(pkts[i + prefetch_offset]); 563 prepare_tx_pkt(pkts[i], port, qconf); 564 } 565 /* Process left packets */ 566 for (; i < nb_pkts; i++) 567 prepare_tx_pkt(pkts[i], port, qconf); 568 } 569 570 /* Send burst of packets on an output interface */ 571 static inline int32_t 572 send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) 573 { 574 struct rte_mbuf **m_table; 575 int32_t ret; 576 uint16_t queueid; 577 578 queueid = qconf->tx_queue_id[port]; 579 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 580 581 prepare_tx_burst(m_table, n, port, qconf); 582 583 ret = rte_eth_tx_burst(port, queueid, m_table, n); 584 585 core_stats_update_tx(ret); 586 587 if (unlikely(ret < n)) { 588 do { 589 free_pkts(&m_table[ret], 1); 590 } while (++ret < n); 591 } 592 593 return 0; 594 } 595 596 /* 597 * Helper function to fragment and queue for TX one packet. 598 */ 599 static inline uint32_t 600 send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m, 601 uint16_t port, uint8_t proto) 602 { 603 struct buffer *tbl; 604 uint32_t len, n; 605 int32_t rc; 606 607 tbl = qconf->tx_mbufs + port; 608 len = tbl->len; 609 610 /* free space for new fragments */ 611 if (len + RTE_LIBRTE_IP_FRAG_MAX_FRAG >= RTE_DIM(tbl->m_table)) { 612 send_burst(qconf, len, port); 613 len = 0; 614 } 615 616 n = RTE_DIM(tbl->m_table) - len; 617 618 if (proto == IPPROTO_IP) 619 rc = rte_ipv4_fragment_packet(m, tbl->m_table + len, 620 n, mtu_size, qconf->frag.pool_dir, 621 qconf->frag.pool_indir); 622 else 623 rc = rte_ipv6_fragment_packet(m, tbl->m_table + len, 624 n, mtu_size, qconf->frag.pool_dir, 625 qconf->frag.pool_indir); 626 627 if (rc >= 0) 628 len += rc; 629 else 630 RTE_LOG(ERR, IPSEC, 631 "%s: failed to fragment packet with size %u, " 632 "error code: %d\n", 633 __func__, m->pkt_len, rte_errno); 634 635 free_pkts(&m, 1); 636 return len; 637 } 638 639 /* Enqueue a single packet, and send burst if queue is filled */ 640 static inline int32_t 641 send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto) 642 { 643 uint32_t lcore_id; 644 uint16_t len; 645 struct lcore_conf *qconf; 646 647 lcore_id = rte_lcore_id(); 648 649 qconf = &lcore_conf[lcore_id]; 650 len = qconf->tx_mbufs[port].len; 651 652 if (m->pkt_len <= mtu_size) { 653 qconf->tx_mbufs[port].m_table[len] = m; 654 len++; 655 656 /* need to fragment the packet */ 657 } else if (frag_tbl_sz > 0) 658 len = send_fragment_packet(qconf, m, port, proto); 659 else 660 free_pkts(&m, 1); 661 662 /* enough pkts to be sent */ 663 if (unlikely(len == MAX_PKT_BURST)) { 664 send_burst(qconf, MAX_PKT_BURST, port); 665 len = 0; 666 } 667 668 qconf->tx_mbufs[port].len = len; 669 return 0; 670 } 671 672 static inline void 673 inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip, 674 uint16_t lim) 675 { 676 struct rte_mbuf *m; 677 uint32_t i, j, res, sa_idx; 678 679 if (ip->num == 0 || sp == NULL) 680 return; 681 682 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 683 ip->num, DEFAULT_MAX_CATEGORIES); 684 685 j = 0; 686 for (i = 0; i < ip->num; i++) { 687 m = ip->pkts[i]; 688 res = ip->res[i]; 689 if (res == BYPASS) { 690 ip->pkts[j++] = m; 691 continue; 692 } 693 if (res == DISCARD) { 694 free_pkts(&m, 1); 695 continue; 696 } 697 698 /* Only check SPI match for processed IPSec packets */ 699 if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) { 700 free_pkts(&m, 1); 701 continue; 702 } 703 704 sa_idx = res - 1; 705 if (!inbound_sa_check(sa, m, sa_idx)) { 706 free_pkts(&m, 1); 707 continue; 708 } 709 ip->pkts[j++] = m; 710 } 711 ip->num = j; 712 } 713 714 static void 715 split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) 716 { 717 uint32_t i, n4, n6; 718 struct ip *ip; 719 struct rte_mbuf *m; 720 721 n4 = trf->ip4.num; 722 n6 = trf->ip6.num; 723 724 for (i = 0; i < num; i++) { 725 726 m = mb[i]; 727 ip = rte_pktmbuf_mtod(m, struct ip *); 728 729 if (ip->ip_v == IPVERSION) { 730 trf->ip4.pkts[n4] = m; 731 trf->ip4.data[n4] = rte_pktmbuf_mtod_offset(m, 732 uint8_t *, offsetof(struct ip, ip_p)); 733 n4++; 734 } else if (ip->ip_v == IP6_VERSION) { 735 trf->ip6.pkts[n6] = m; 736 trf->ip6.data[n6] = rte_pktmbuf_mtod_offset(m, 737 uint8_t *, 738 offsetof(struct ip6_hdr, ip6_nxt)); 739 n6++; 740 } else 741 free_pkts(&m, 1); 742 } 743 744 trf->ip4.num = n4; 745 trf->ip6.num = n6; 746 } 747 748 749 static inline void 750 process_pkts_inbound(struct ipsec_ctx *ipsec_ctx, 751 struct ipsec_traffic *traffic) 752 { 753 uint16_t nb_pkts_in, n_ip4, n_ip6; 754 755 n_ip4 = traffic->ip4.num; 756 n_ip6 = traffic->ip6.num; 757 758 if (app_sa_prm.enable == 0) { 759 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 760 traffic->ipsec.num, MAX_PKT_BURST); 761 split46_traffic(traffic, traffic->ipsec.pkts, nb_pkts_in); 762 } else { 763 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 764 traffic->ipsec.saptr, traffic->ipsec.num); 765 ipsec_process(ipsec_ctx, traffic); 766 } 767 768 inbound_sp_sa(ipsec_ctx->sp4_ctx, ipsec_ctx->sa_ctx, &traffic->ip4, 769 n_ip4); 770 771 inbound_sp_sa(ipsec_ctx->sp6_ctx, ipsec_ctx->sa_ctx, &traffic->ip6, 772 n_ip6); 773 } 774 775 static inline void 776 outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, 777 struct traffic_type *ipsec) 778 { 779 struct rte_mbuf *m; 780 uint32_t i, j, sa_idx; 781 782 if (ip->num == 0 || sp == NULL) 783 return; 784 785 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 786 ip->num, DEFAULT_MAX_CATEGORIES); 787 788 j = 0; 789 for (i = 0; i < ip->num; i++) { 790 m = ip->pkts[i]; 791 sa_idx = ip->res[i] - 1; 792 if (ip->res[i] == DISCARD) 793 free_pkts(&m, 1); 794 else if (ip->res[i] == BYPASS) 795 ip->pkts[j++] = m; 796 else { 797 ipsec->res[ipsec->num] = sa_idx; 798 ipsec->pkts[ipsec->num++] = m; 799 } 800 } 801 ip->num = j; 802 } 803 804 static inline void 805 process_pkts_outbound(struct ipsec_ctx *ipsec_ctx, 806 struct ipsec_traffic *traffic) 807 { 808 struct rte_mbuf *m; 809 uint16_t idx, nb_pkts_out, i; 810 811 /* Drop any IPsec traffic from protected ports */ 812 free_pkts(traffic->ipsec.pkts, traffic->ipsec.num); 813 814 traffic->ipsec.num = 0; 815 816 outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec); 817 818 outbound_sp(ipsec_ctx->sp6_ctx, &traffic->ip6, &traffic->ipsec); 819 820 if (app_sa_prm.enable == 0) { 821 822 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 823 traffic->ipsec.res, traffic->ipsec.num, 824 MAX_PKT_BURST); 825 826 for (i = 0; i < nb_pkts_out; i++) { 827 m = traffic->ipsec.pkts[i]; 828 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 829 if (ip->ip_v == IPVERSION) { 830 idx = traffic->ip4.num++; 831 traffic->ip4.pkts[idx] = m; 832 } else { 833 idx = traffic->ip6.num++; 834 traffic->ip6.pkts[idx] = m; 835 } 836 } 837 } else { 838 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 839 traffic->ipsec.saptr, traffic->ipsec.num); 840 ipsec_process(ipsec_ctx, traffic); 841 } 842 } 843 844 static inline void 845 process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx, 846 struct ipsec_traffic *traffic) 847 { 848 struct rte_mbuf *m; 849 uint32_t nb_pkts_in, i, idx; 850 851 /* Drop any IPv4 traffic from unprotected ports */ 852 free_pkts(traffic->ip4.pkts, traffic->ip4.num); 853 854 traffic->ip4.num = 0; 855 856 /* Drop any IPv6 traffic from unprotected ports */ 857 free_pkts(traffic->ip6.pkts, traffic->ip6.num); 858 859 traffic->ip6.num = 0; 860 861 if (app_sa_prm.enable == 0) { 862 863 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 864 traffic->ipsec.num, MAX_PKT_BURST); 865 866 for (i = 0; i < nb_pkts_in; i++) { 867 m = traffic->ipsec.pkts[i]; 868 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 869 if (ip->ip_v == IPVERSION) { 870 idx = traffic->ip4.num++; 871 traffic->ip4.pkts[idx] = m; 872 } else { 873 idx = traffic->ip6.num++; 874 traffic->ip6.pkts[idx] = m; 875 } 876 } 877 } else { 878 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 879 traffic->ipsec.saptr, traffic->ipsec.num); 880 ipsec_process(ipsec_ctx, traffic); 881 } 882 } 883 884 static inline void 885 process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx, 886 struct ipsec_traffic *traffic) 887 { 888 struct rte_mbuf *m; 889 uint32_t nb_pkts_out, i, n; 890 struct ip *ip; 891 892 /* Drop any IPsec traffic from protected ports */ 893 free_pkts(traffic->ipsec.pkts, traffic->ipsec.num); 894 895 n = 0; 896 897 for (i = 0; i < traffic->ip4.num; i++) { 898 traffic->ipsec.pkts[n] = traffic->ip4.pkts[i]; 899 traffic->ipsec.res[n++] = single_sa_idx; 900 } 901 902 for (i = 0; i < traffic->ip6.num; i++) { 903 traffic->ipsec.pkts[n] = traffic->ip6.pkts[i]; 904 traffic->ipsec.res[n++] = single_sa_idx; 905 } 906 907 traffic->ip4.num = 0; 908 traffic->ip6.num = 0; 909 traffic->ipsec.num = n; 910 911 if (app_sa_prm.enable == 0) { 912 913 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 914 traffic->ipsec.res, traffic->ipsec.num, 915 MAX_PKT_BURST); 916 917 /* They all sue the same SA (ip4 or ip6 tunnel) */ 918 m = traffic->ipsec.pkts[0]; 919 ip = rte_pktmbuf_mtod(m, struct ip *); 920 if (ip->ip_v == IPVERSION) { 921 traffic->ip4.num = nb_pkts_out; 922 for (i = 0; i < nb_pkts_out; i++) 923 traffic->ip4.pkts[i] = traffic->ipsec.pkts[i]; 924 } else { 925 traffic->ip6.num = nb_pkts_out; 926 for (i = 0; i < nb_pkts_out; i++) 927 traffic->ip6.pkts[i] = traffic->ipsec.pkts[i]; 928 } 929 } else { 930 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 931 traffic->ipsec.saptr, traffic->ipsec.num); 932 ipsec_process(ipsec_ctx, traffic); 933 } 934 } 935 936 static inline int32_t 937 get_hop_for_offload_pkt(struct rte_mbuf *pkt, int is_ipv6) 938 { 939 struct ipsec_mbuf_metadata *priv; 940 struct ipsec_sa *sa; 941 942 priv = get_priv(pkt); 943 944 sa = priv->sa; 945 if (unlikely(sa == NULL)) { 946 RTE_LOG(ERR, IPSEC, "SA not saved in private data\n"); 947 goto fail; 948 } 949 950 if (is_ipv6) 951 return sa->portid; 952 953 /* else */ 954 return (sa->portid | RTE_LPM_LOOKUP_SUCCESS); 955 956 fail: 957 if (is_ipv6) 958 return -1; 959 960 /* else */ 961 return 0; 962 } 963 964 static inline void 965 route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 966 { 967 uint32_t hop[MAX_PKT_BURST * 2]; 968 uint32_t dst_ip[MAX_PKT_BURST * 2]; 969 int32_t pkt_hop = 0; 970 uint16_t i, offset; 971 uint16_t lpm_pkts = 0; 972 973 if (nb_pkts == 0) 974 return; 975 976 /* Need to do an LPM lookup for non-inline packets. Inline packets will 977 * have port ID in the SA 978 */ 979 980 for (i = 0; i < nb_pkts; i++) { 981 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 982 /* Security offload not enabled. So an LPM lookup is 983 * required to get the hop 984 */ 985 offset = offsetof(struct ip, ip_dst); 986 dst_ip[lpm_pkts] = *rte_pktmbuf_mtod_offset(pkts[i], 987 uint32_t *, offset); 988 dst_ip[lpm_pkts] = rte_be_to_cpu_32(dst_ip[lpm_pkts]); 989 lpm_pkts++; 990 } 991 } 992 993 rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, lpm_pkts); 994 995 lpm_pkts = 0; 996 997 for (i = 0; i < nb_pkts; i++) { 998 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 999 /* Read hop from the SA */ 1000 pkt_hop = get_hop_for_offload_pkt(pkts[i], 0); 1001 } else { 1002 /* Need to use hop returned by lookup */ 1003 pkt_hop = hop[lpm_pkts++]; 1004 } 1005 1006 if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) { 1007 free_pkts(&pkts[i], 1); 1008 continue; 1009 } 1010 send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IP); 1011 } 1012 } 1013 1014 static inline void 1015 route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 1016 { 1017 int32_t hop[MAX_PKT_BURST * 2]; 1018 uint8_t dst_ip[MAX_PKT_BURST * 2][16]; 1019 uint8_t *ip6_dst; 1020 int32_t pkt_hop = 0; 1021 uint16_t i, offset; 1022 uint16_t lpm_pkts = 0; 1023 1024 if (nb_pkts == 0) 1025 return; 1026 1027 /* Need to do an LPM lookup for non-inline packets. Inline packets will 1028 * have port ID in the SA 1029 */ 1030 1031 for (i = 0; i < nb_pkts; i++) { 1032 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 1033 /* Security offload not enabled. So an LPM lookup is 1034 * required to get the hop 1035 */ 1036 offset = offsetof(struct ip6_hdr, ip6_dst); 1037 ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *, 1038 offset); 1039 memcpy(&dst_ip[lpm_pkts][0], ip6_dst, 16); 1040 lpm_pkts++; 1041 } 1042 } 1043 1044 rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, hop, 1045 lpm_pkts); 1046 1047 lpm_pkts = 0; 1048 1049 for (i = 0; i < nb_pkts; i++) { 1050 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 1051 /* Read hop from the SA */ 1052 pkt_hop = get_hop_for_offload_pkt(pkts[i], 1); 1053 } else { 1054 /* Need to use hop returned by lookup */ 1055 pkt_hop = hop[lpm_pkts++]; 1056 } 1057 1058 if (pkt_hop == -1) { 1059 free_pkts(&pkts[i], 1); 1060 continue; 1061 } 1062 send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IPV6); 1063 } 1064 } 1065 1066 static inline void 1067 process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts, 1068 uint8_t nb_pkts, uint16_t portid) 1069 { 1070 struct ipsec_traffic traffic; 1071 1072 prepare_traffic(pkts, &traffic, nb_pkts); 1073 1074 if (unlikely(single_sa)) { 1075 if (is_unprotected_port(portid)) 1076 process_pkts_inbound_nosp(&qconf->inbound, &traffic); 1077 else 1078 process_pkts_outbound_nosp(&qconf->outbound, &traffic); 1079 } else { 1080 if (is_unprotected_port(portid)) 1081 process_pkts_inbound(&qconf->inbound, &traffic); 1082 else 1083 process_pkts_outbound(&qconf->outbound, &traffic); 1084 } 1085 1086 route4_pkts(qconf->rt4_ctx, traffic.ip4.pkts, traffic.ip4.num); 1087 route6_pkts(qconf->rt6_ctx, traffic.ip6.pkts, traffic.ip6.num); 1088 } 1089 1090 static inline void 1091 drain_tx_buffers(struct lcore_conf *qconf) 1092 { 1093 struct buffer *buf; 1094 uint32_t portid; 1095 1096 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 1097 buf = &qconf->tx_mbufs[portid]; 1098 if (buf->len == 0) 1099 continue; 1100 send_burst(qconf, buf->len, portid); 1101 buf->len = 0; 1102 } 1103 } 1104 1105 static inline void 1106 drain_crypto_buffers(struct lcore_conf *qconf) 1107 { 1108 uint32_t i; 1109 struct ipsec_ctx *ctx; 1110 1111 /* drain inbound buffers*/ 1112 ctx = &qconf->inbound; 1113 for (i = 0; i != ctx->nb_qps; i++) { 1114 if (ctx->tbl[i].len != 0) 1115 enqueue_cop_burst(ctx->tbl + i); 1116 } 1117 1118 /* drain outbound buffers*/ 1119 ctx = &qconf->outbound; 1120 for (i = 0; i != ctx->nb_qps; i++) { 1121 if (ctx->tbl[i].len != 0) 1122 enqueue_cop_burst(ctx->tbl + i); 1123 } 1124 } 1125 1126 static void 1127 drain_inbound_crypto_queues(const struct lcore_conf *qconf, 1128 struct ipsec_ctx *ctx) 1129 { 1130 uint32_t n; 1131 struct ipsec_traffic trf; 1132 1133 if (app_sa_prm.enable == 0) { 1134 1135 /* dequeue packets from crypto-queue */ 1136 n = ipsec_inbound_cqp_dequeue(ctx, trf.ipsec.pkts, 1137 RTE_DIM(trf.ipsec.pkts)); 1138 1139 trf.ip4.num = 0; 1140 trf.ip6.num = 0; 1141 1142 /* split traffic by ipv4-ipv6 */ 1143 split46_traffic(&trf, trf.ipsec.pkts, n); 1144 } else 1145 ipsec_cqp_process(ctx, &trf); 1146 1147 /* process ipv4 packets */ 1148 if (trf.ip4.num != 0) { 1149 inbound_sp_sa(ctx->sp4_ctx, ctx->sa_ctx, &trf.ip4, 0); 1150 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 1151 } 1152 1153 /* process ipv6 packets */ 1154 if (trf.ip6.num != 0) { 1155 inbound_sp_sa(ctx->sp6_ctx, ctx->sa_ctx, &trf.ip6, 0); 1156 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 1157 } 1158 } 1159 1160 static void 1161 drain_outbound_crypto_queues(const struct lcore_conf *qconf, 1162 struct ipsec_ctx *ctx) 1163 { 1164 uint32_t n; 1165 struct ipsec_traffic trf; 1166 1167 if (app_sa_prm.enable == 0) { 1168 1169 /* dequeue packets from crypto-queue */ 1170 n = ipsec_outbound_cqp_dequeue(ctx, trf.ipsec.pkts, 1171 RTE_DIM(trf.ipsec.pkts)); 1172 1173 trf.ip4.num = 0; 1174 trf.ip6.num = 0; 1175 1176 /* split traffic by ipv4-ipv6 */ 1177 split46_traffic(&trf, trf.ipsec.pkts, n); 1178 } else 1179 ipsec_cqp_process(ctx, &trf); 1180 1181 /* process ipv4 packets */ 1182 if (trf.ip4.num != 0) 1183 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 1184 1185 /* process ipv6 packets */ 1186 if (trf.ip6.num != 0) 1187 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 1188 } 1189 1190 /* main processing loop */ 1191 void 1192 ipsec_poll_mode_worker(void) 1193 { 1194 struct rte_mbuf *pkts[MAX_PKT_BURST]; 1195 uint32_t lcore_id; 1196 uint64_t prev_tsc, diff_tsc, cur_tsc; 1197 int32_t i, nb_rx; 1198 uint16_t portid; 1199 uint8_t queueid; 1200 struct lcore_conf *qconf; 1201 int32_t rc, socket_id; 1202 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) 1203 / US_PER_S * BURST_TX_DRAIN_US; 1204 struct lcore_rx_queue *rxql; 1205 1206 prev_tsc = 0; 1207 lcore_id = rte_lcore_id(); 1208 qconf = &lcore_conf[lcore_id]; 1209 rxql = qconf->rx_queue_list; 1210 socket_id = rte_lcore_to_socket_id(lcore_id); 1211 1212 qconf->rt4_ctx = socket_ctx[socket_id].rt_ip4; 1213 qconf->rt6_ctx = socket_ctx[socket_id].rt_ip6; 1214 qconf->inbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_in; 1215 qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in; 1216 qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in; 1217 qconf->inbound.cdev_map = cdev_map_in; 1218 qconf->inbound.session_pool = socket_ctx[socket_id].session_pool; 1219 qconf->inbound.session_priv_pool = 1220 socket_ctx[socket_id].session_priv_pool; 1221 qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out; 1222 qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out; 1223 qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out; 1224 qconf->outbound.cdev_map = cdev_map_out; 1225 qconf->outbound.session_pool = socket_ctx[socket_id].session_pool; 1226 qconf->outbound.session_priv_pool = 1227 socket_ctx[socket_id].session_priv_pool; 1228 qconf->frag.pool_dir = socket_ctx[socket_id].mbuf_pool; 1229 qconf->frag.pool_indir = socket_ctx[socket_id].mbuf_pool_indir; 1230 1231 rc = ipsec_sad_lcore_cache_init(app_sa_prm.cache_sz); 1232 if (rc != 0) { 1233 RTE_LOG(ERR, IPSEC, 1234 "SAD cache init on lcore %u, failed with code: %d\n", 1235 lcore_id, rc); 1236 return; 1237 } 1238 1239 if (qconf->nb_rx_queue == 0) { 1240 RTE_LOG(DEBUG, IPSEC, "lcore %u has nothing to do\n", 1241 lcore_id); 1242 return; 1243 } 1244 1245 RTE_LOG(INFO, IPSEC, "entering main loop on lcore %u\n", lcore_id); 1246 1247 for (i = 0; i < qconf->nb_rx_queue; i++) { 1248 portid = rxql[i].port_id; 1249 queueid = rxql[i].queue_id; 1250 RTE_LOG(INFO, IPSEC, 1251 " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", 1252 lcore_id, portid, queueid); 1253 } 1254 1255 while (!force_quit) { 1256 cur_tsc = rte_rdtsc(); 1257 1258 /* TX queue buffer drain */ 1259 diff_tsc = cur_tsc - prev_tsc; 1260 1261 if (unlikely(diff_tsc > drain_tsc)) { 1262 drain_tx_buffers(qconf); 1263 drain_crypto_buffers(qconf); 1264 prev_tsc = cur_tsc; 1265 } 1266 1267 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1268 1269 /* Read packets from RX queues */ 1270 portid = rxql[i].port_id; 1271 queueid = rxql[i].queue_id; 1272 nb_rx = rte_eth_rx_burst(portid, queueid, 1273 pkts, MAX_PKT_BURST); 1274 1275 if (nb_rx > 0) { 1276 core_stats_update_rx(nb_rx); 1277 process_pkts(qconf, pkts, nb_rx, portid); 1278 } 1279 1280 /* dequeue and process completed crypto-ops */ 1281 if (is_unprotected_port(portid)) 1282 drain_inbound_crypto_queues(qconf, 1283 &qconf->inbound); 1284 else 1285 drain_outbound_crypto_queues(qconf, 1286 &qconf->outbound); 1287 } 1288 } 1289 } 1290 1291 int 1292 check_flow_params(uint16_t fdir_portid, uint8_t fdir_qid) 1293 { 1294 uint16_t i; 1295 uint16_t portid; 1296 uint8_t queueid; 1297 1298 for (i = 0; i < nb_lcore_params; ++i) { 1299 portid = lcore_params_array[i].port_id; 1300 if (portid == fdir_portid) { 1301 queueid = lcore_params_array[i].queue_id; 1302 if (queueid == fdir_qid) 1303 break; 1304 } 1305 1306 if (i == nb_lcore_params - 1) 1307 return -1; 1308 } 1309 1310 return 1; 1311 } 1312 1313 static int32_t 1314 check_poll_mode_params(struct eh_conf *eh_conf) 1315 { 1316 uint8_t lcore; 1317 uint16_t portid; 1318 uint16_t i; 1319 int32_t socket_id; 1320 1321 if (!eh_conf) 1322 return -EINVAL; 1323 1324 if (eh_conf->mode != EH_PKT_TRANSFER_MODE_POLL) 1325 return 0; 1326 1327 if (lcore_params == NULL) { 1328 printf("Error: No port/queue/core mappings\n"); 1329 return -1; 1330 } 1331 1332 for (i = 0; i < nb_lcore_params; ++i) { 1333 lcore = lcore_params[i].lcore_id; 1334 if (!rte_lcore_is_enabled(lcore)) { 1335 printf("error: lcore %hhu is not enabled in " 1336 "lcore mask\n", lcore); 1337 return -1; 1338 } 1339 socket_id = rte_lcore_to_socket_id(lcore); 1340 if (socket_id != 0 && numa_on == 0) { 1341 printf("warning: lcore %hhu is on socket %d " 1342 "with numa off\n", 1343 lcore, socket_id); 1344 } 1345 portid = lcore_params[i].port_id; 1346 if ((enabled_port_mask & (1 << portid)) == 0) { 1347 printf("port %u is not enabled in port mask\n", portid); 1348 return -1; 1349 } 1350 if (!rte_eth_dev_is_valid_port(portid)) { 1351 printf("port %u is not present on the board\n", portid); 1352 return -1; 1353 } 1354 } 1355 return 0; 1356 } 1357 1358 static uint8_t 1359 get_port_nb_rx_queues(const uint16_t port) 1360 { 1361 int32_t queue = -1; 1362 uint16_t i; 1363 1364 for (i = 0; i < nb_lcore_params; ++i) { 1365 if (lcore_params[i].port_id == port && 1366 lcore_params[i].queue_id > queue) 1367 queue = lcore_params[i].queue_id; 1368 } 1369 return (uint8_t)(++queue); 1370 } 1371 1372 static int32_t 1373 init_lcore_rx_queues(void) 1374 { 1375 uint16_t i, nb_rx_queue; 1376 uint8_t lcore; 1377 1378 for (i = 0; i < nb_lcore_params; ++i) { 1379 lcore = lcore_params[i].lcore_id; 1380 nb_rx_queue = lcore_conf[lcore].nb_rx_queue; 1381 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 1382 printf("error: too many queues (%u) for lcore: %u\n", 1383 nb_rx_queue + 1, lcore); 1384 return -1; 1385 } 1386 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 1387 lcore_params[i].port_id; 1388 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 1389 lcore_params[i].queue_id; 1390 lcore_conf[lcore].nb_rx_queue++; 1391 } 1392 return 0; 1393 } 1394 1395 /* display usage */ 1396 static void 1397 print_usage(const char *prgname) 1398 { 1399 fprintf(stderr, "%s [EAL options] --" 1400 " -p PORTMASK" 1401 " [-P]" 1402 " [-u PORTMASK]" 1403 " [-j FRAMESIZE]" 1404 " [-l]" 1405 " [-w REPLAY_WINDOW_SIZE]" 1406 " [-e]" 1407 " [-a]" 1408 " [-c]" 1409 " [-s NUMBER_OF_MBUFS_IN_PKT_POOL]" 1410 " -f CONFIG_FILE" 1411 " --config (port,queue,lcore)[,(port,queue,lcore)]" 1412 " [--single-sa SAIDX]" 1413 " [--cryptodev_mask MASK]" 1414 " [--transfer-mode MODE]" 1415 " [--event-schedule-type TYPE]" 1416 " [--" CMD_LINE_OPT_RX_OFFLOAD " RX_OFFLOAD_MASK]" 1417 " [--" CMD_LINE_OPT_TX_OFFLOAD " TX_OFFLOAD_MASK]" 1418 " [--" CMD_LINE_OPT_REASSEMBLE " REASSEMBLE_TABLE_SIZE]" 1419 " [--" CMD_LINE_OPT_MTU " MTU]" 1420 "\n\n" 1421 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n" 1422 " -P : Enable promiscuous mode\n" 1423 " -u PORTMASK: Hexadecimal bitmask of unprotected ports\n" 1424 " -j FRAMESIZE: Data buffer size, minimum (and default)\n" 1425 " value: RTE_MBUF_DEFAULT_BUF_SIZE\n" 1426 " -l enables code-path that uses librte_ipsec\n" 1427 " -w REPLAY_WINDOW_SIZE specifies IPsec SQN replay window\n" 1428 " size for each SA\n" 1429 " -e enables ESN\n" 1430 " -a enables SA SQN atomic behaviour\n" 1431 " -c specifies inbound SAD cache size,\n" 1432 " zero value disables the cache (default value: 128)\n" 1433 " -s number of mbufs in packet pool, if not specified number\n" 1434 " of mbufs will be calculated based on number of cores,\n" 1435 " ports and crypto queues\n" 1436 " -f CONFIG_FILE: Configuration file\n" 1437 " --config (port,queue,lcore): Rx queue configuration. In poll\n" 1438 " mode determines which queues from\n" 1439 " which ports are mapped to which cores.\n" 1440 " In event mode this option is not used\n" 1441 " as packets are dynamically scheduled\n" 1442 " to cores by HW.\n" 1443 " --single-sa SAIDX: In poll mode use single SA index for\n" 1444 " outbound traffic, bypassing the SP\n" 1445 " In event mode selects driver submode,\n" 1446 " SA index value is ignored\n" 1447 " --cryptodev_mask MASK: Hexadecimal bitmask of the crypto\n" 1448 " devices to configure\n" 1449 " --transfer-mode MODE\n" 1450 " \"poll\" : Packet transfer via polling (default)\n" 1451 " \"event\" : Packet transfer via event device\n" 1452 " --event-schedule-type TYPE queue schedule type, used only when\n" 1453 " transfer mode is set to event\n" 1454 " \"ordered\" : Ordered (default)\n" 1455 " \"atomic\" : Atomic\n" 1456 " \"parallel\" : Parallel\n" 1457 " --" CMD_LINE_OPT_RX_OFFLOAD 1458 ": bitmask of the RX HW offload capabilities to enable/use\n" 1459 " (DEV_RX_OFFLOAD_*)\n" 1460 " --" CMD_LINE_OPT_TX_OFFLOAD 1461 ": bitmask of the TX HW offload capabilities to enable/use\n" 1462 " (DEV_TX_OFFLOAD_*)\n" 1463 " --" CMD_LINE_OPT_REASSEMBLE " NUM" 1464 ": max number of entries in reassemble(fragment) table\n" 1465 " (zero (default value) disables reassembly)\n" 1466 " --" CMD_LINE_OPT_MTU " MTU" 1467 ": MTU value on all ports (default value: 1500)\n" 1468 " outgoing packets with bigger size will be fragmented\n" 1469 " incoming packets with bigger size will be discarded\n" 1470 " --" CMD_LINE_OPT_FRAG_TTL " FRAG_TTL_NS" 1471 ": fragments lifetime in nanoseconds, default\n" 1472 " and maximum value is 10.000.000.000 ns (10 s)\n" 1473 "\n", 1474 prgname); 1475 } 1476 1477 static int 1478 parse_mask(const char *str, uint64_t *val) 1479 { 1480 char *end; 1481 unsigned long t; 1482 1483 errno = 0; 1484 t = strtoul(str, &end, 0); 1485 if (errno != 0 || end[0] != 0) 1486 return -EINVAL; 1487 1488 *val = t; 1489 return 0; 1490 } 1491 1492 static int32_t 1493 parse_portmask(const char *portmask) 1494 { 1495 char *end = NULL; 1496 unsigned long pm; 1497 1498 errno = 0; 1499 1500 /* parse hexadecimal string */ 1501 pm = strtoul(portmask, &end, 16); 1502 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 1503 return -1; 1504 1505 if ((pm == 0) && errno) 1506 return -1; 1507 1508 return pm; 1509 } 1510 1511 static int64_t 1512 parse_decimal(const char *str) 1513 { 1514 char *end = NULL; 1515 uint64_t num; 1516 1517 num = strtoull(str, &end, 10); 1518 if ((str[0] == '\0') || (end == NULL) || (*end != '\0') 1519 || num > INT64_MAX) 1520 return -1; 1521 1522 return num; 1523 } 1524 1525 static int32_t 1526 parse_config(const char *q_arg) 1527 { 1528 char s[256]; 1529 const char *p, *p0 = q_arg; 1530 char *end; 1531 enum fieldnames { 1532 FLD_PORT = 0, 1533 FLD_QUEUE, 1534 FLD_LCORE, 1535 _NUM_FLD 1536 }; 1537 unsigned long int_fld[_NUM_FLD]; 1538 char *str_fld[_NUM_FLD]; 1539 int32_t i; 1540 uint32_t size; 1541 1542 nb_lcore_params = 0; 1543 1544 while ((p = strchr(p0, '(')) != NULL) { 1545 ++p; 1546 p0 = strchr(p, ')'); 1547 if (p0 == NULL) 1548 return -1; 1549 1550 size = p0 - p; 1551 if (size >= sizeof(s)) 1552 return -1; 1553 1554 snprintf(s, sizeof(s), "%.*s", size, p); 1555 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 1556 _NUM_FLD) 1557 return -1; 1558 for (i = 0; i < _NUM_FLD; i++) { 1559 errno = 0; 1560 int_fld[i] = strtoul(str_fld[i], &end, 0); 1561 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 1562 return -1; 1563 } 1564 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 1565 printf("exceeded max number of lcore params: %hu\n", 1566 nb_lcore_params); 1567 return -1; 1568 } 1569 lcore_params_array[nb_lcore_params].port_id = 1570 (uint8_t)int_fld[FLD_PORT]; 1571 lcore_params_array[nb_lcore_params].queue_id = 1572 (uint8_t)int_fld[FLD_QUEUE]; 1573 lcore_params_array[nb_lcore_params].lcore_id = 1574 (uint8_t)int_fld[FLD_LCORE]; 1575 ++nb_lcore_params; 1576 } 1577 lcore_params = lcore_params_array; 1578 return 0; 1579 } 1580 1581 static void 1582 print_app_sa_prm(const struct app_sa_prm *prm) 1583 { 1584 printf("librte_ipsec usage: %s\n", 1585 (prm->enable == 0) ? "disabled" : "enabled"); 1586 1587 printf("replay window size: %u\n", prm->window_size); 1588 printf("ESN: %s\n", (prm->enable_esn == 0) ? "disabled" : "enabled"); 1589 printf("SA flags: %#" PRIx64 "\n", prm->flags); 1590 printf("Frag TTL: %" PRIu64 " ns\n", frag_ttl_ns); 1591 } 1592 1593 static int 1594 parse_transfer_mode(struct eh_conf *conf, const char *optarg) 1595 { 1596 if (!strcmp(CMD_LINE_ARG_POLL, optarg)) 1597 conf->mode = EH_PKT_TRANSFER_MODE_POLL; 1598 else if (!strcmp(CMD_LINE_ARG_EVENT, optarg)) 1599 conf->mode = EH_PKT_TRANSFER_MODE_EVENT; 1600 else { 1601 printf("Unsupported packet transfer mode\n"); 1602 return -EINVAL; 1603 } 1604 1605 return 0; 1606 } 1607 1608 static int 1609 parse_schedule_type(struct eh_conf *conf, const char *optarg) 1610 { 1611 struct eventmode_conf *em_conf = NULL; 1612 1613 /* Get eventmode conf */ 1614 em_conf = conf->mode_params; 1615 1616 if (!strcmp(CMD_LINE_ARG_ORDERED, optarg)) 1617 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ORDERED; 1618 else if (!strcmp(CMD_LINE_ARG_ATOMIC, optarg)) 1619 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ATOMIC; 1620 else if (!strcmp(CMD_LINE_ARG_PARALLEL, optarg)) 1621 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_PARALLEL; 1622 else { 1623 printf("Unsupported queue schedule type\n"); 1624 return -EINVAL; 1625 } 1626 1627 return 0; 1628 } 1629 1630 static int32_t 1631 parse_args(int32_t argc, char **argv, struct eh_conf *eh_conf) 1632 { 1633 int opt; 1634 int64_t ret; 1635 char **argvopt; 1636 int32_t option_index; 1637 char *prgname = argv[0]; 1638 int32_t f_present = 0; 1639 1640 argvopt = argv; 1641 1642 while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:c:s:", 1643 lgopts, &option_index)) != EOF) { 1644 1645 switch (opt) { 1646 case 'p': 1647 enabled_port_mask = parse_portmask(optarg); 1648 if (enabled_port_mask == 0) { 1649 printf("invalid portmask\n"); 1650 print_usage(prgname); 1651 return -1; 1652 } 1653 break; 1654 case 'P': 1655 printf("Promiscuous mode selected\n"); 1656 promiscuous_on = 1; 1657 break; 1658 case 'u': 1659 unprotected_port_mask = parse_portmask(optarg); 1660 if (unprotected_port_mask == 0) { 1661 printf("invalid unprotected portmask\n"); 1662 print_usage(prgname); 1663 return -1; 1664 } 1665 break; 1666 case 'f': 1667 if (f_present == 1) { 1668 printf("\"-f\" option present more than " 1669 "once!\n"); 1670 print_usage(prgname); 1671 return -1; 1672 } 1673 cfgfile = optarg; 1674 f_present = 1; 1675 break; 1676 1677 case 's': 1678 ret = parse_decimal(optarg); 1679 if (ret < 0) { 1680 printf("Invalid number of buffers in a pool: " 1681 "%s\n", optarg); 1682 print_usage(prgname); 1683 return -1; 1684 } 1685 1686 nb_bufs_in_pool = ret; 1687 break; 1688 1689 case 'j': 1690 ret = parse_decimal(optarg); 1691 if (ret < RTE_MBUF_DEFAULT_BUF_SIZE || 1692 ret > UINT16_MAX) { 1693 printf("Invalid frame buffer size value: %s\n", 1694 optarg); 1695 print_usage(prgname); 1696 return -1; 1697 } 1698 frame_buf_size = ret; 1699 printf("Custom frame buffer size %u\n", frame_buf_size); 1700 break; 1701 case 'l': 1702 app_sa_prm.enable = 1; 1703 break; 1704 case 'w': 1705 app_sa_prm.window_size = parse_decimal(optarg); 1706 break; 1707 case 'e': 1708 app_sa_prm.enable_esn = 1; 1709 break; 1710 case 'a': 1711 app_sa_prm.enable = 1; 1712 app_sa_prm.flags |= RTE_IPSEC_SAFLAG_SQN_ATOM; 1713 break; 1714 case 'c': 1715 ret = parse_decimal(optarg); 1716 if (ret < 0) { 1717 printf("Invalid SA cache size: %s\n", optarg); 1718 print_usage(prgname); 1719 return -1; 1720 } 1721 app_sa_prm.cache_sz = ret; 1722 break; 1723 case CMD_LINE_OPT_CONFIG_NUM: 1724 ret = parse_config(optarg); 1725 if (ret) { 1726 printf("Invalid config\n"); 1727 print_usage(prgname); 1728 return -1; 1729 } 1730 break; 1731 case CMD_LINE_OPT_SINGLE_SA_NUM: 1732 ret = parse_decimal(optarg); 1733 if (ret == -1 || ret > UINT32_MAX) { 1734 printf("Invalid argument[sa_idx]\n"); 1735 print_usage(prgname); 1736 return -1; 1737 } 1738 1739 /* else */ 1740 single_sa = 1; 1741 single_sa_idx = ret; 1742 eh_conf->ipsec_mode = EH_IPSEC_MODE_TYPE_DRIVER; 1743 printf("Configured with single SA index %u\n", 1744 single_sa_idx); 1745 break; 1746 case CMD_LINE_OPT_CRYPTODEV_MASK_NUM: 1747 ret = parse_portmask(optarg); 1748 if (ret == -1) { 1749 printf("Invalid argument[portmask]\n"); 1750 print_usage(prgname); 1751 return -1; 1752 } 1753 1754 /* else */ 1755 enabled_cryptodev_mask = ret; 1756 break; 1757 1758 case CMD_LINE_OPT_TRANSFER_MODE_NUM: 1759 ret = parse_transfer_mode(eh_conf, optarg); 1760 if (ret < 0) { 1761 printf("Invalid packet transfer mode\n"); 1762 print_usage(prgname); 1763 return -1; 1764 } 1765 break; 1766 1767 case CMD_LINE_OPT_SCHEDULE_TYPE_NUM: 1768 ret = parse_schedule_type(eh_conf, optarg); 1769 if (ret < 0) { 1770 printf("Invalid queue schedule type\n"); 1771 print_usage(prgname); 1772 return -1; 1773 } 1774 break; 1775 1776 case CMD_LINE_OPT_RX_OFFLOAD_NUM: 1777 ret = parse_mask(optarg, &dev_rx_offload); 1778 if (ret != 0) { 1779 printf("Invalid argument for \'%s\': %s\n", 1780 CMD_LINE_OPT_RX_OFFLOAD, optarg); 1781 print_usage(prgname); 1782 return -1; 1783 } 1784 break; 1785 case CMD_LINE_OPT_TX_OFFLOAD_NUM: 1786 ret = parse_mask(optarg, &dev_tx_offload); 1787 if (ret != 0) { 1788 printf("Invalid argument for \'%s\': %s\n", 1789 CMD_LINE_OPT_TX_OFFLOAD, optarg); 1790 print_usage(prgname); 1791 return -1; 1792 } 1793 break; 1794 case CMD_LINE_OPT_REASSEMBLE_NUM: 1795 ret = parse_decimal(optarg); 1796 if (ret < 0 || ret > UINT32_MAX) { 1797 printf("Invalid argument for \'%s\': %s\n", 1798 CMD_LINE_OPT_REASSEMBLE, optarg); 1799 print_usage(prgname); 1800 return -1; 1801 } 1802 frag_tbl_sz = ret; 1803 break; 1804 case CMD_LINE_OPT_MTU_NUM: 1805 ret = parse_decimal(optarg); 1806 if (ret < 0 || ret > RTE_IPV4_MAX_PKT_LEN) { 1807 printf("Invalid argument for \'%s\': %s\n", 1808 CMD_LINE_OPT_MTU, optarg); 1809 print_usage(prgname); 1810 return -1; 1811 } 1812 mtu_size = ret; 1813 break; 1814 case CMD_LINE_OPT_FRAG_TTL_NUM: 1815 ret = parse_decimal(optarg); 1816 if (ret < 0 || ret > MAX_FRAG_TTL_NS) { 1817 printf("Invalid argument for \'%s\': %s\n", 1818 CMD_LINE_OPT_MTU, optarg); 1819 print_usage(prgname); 1820 return -1; 1821 } 1822 frag_ttl_ns = ret; 1823 break; 1824 default: 1825 print_usage(prgname); 1826 return -1; 1827 } 1828 } 1829 1830 if (f_present == 0) { 1831 printf("Mandatory option \"-f\" not present\n"); 1832 return -1; 1833 } 1834 1835 /* check do we need to enable multi-seg support */ 1836 if (multi_seg_required()) { 1837 /* legacy mode doesn't support multi-seg */ 1838 app_sa_prm.enable = 1; 1839 printf("frame buf size: %u, mtu: %u, " 1840 "number of reassemble entries: %u\n" 1841 "multi-segment support is required\n", 1842 frame_buf_size, mtu_size, frag_tbl_sz); 1843 } 1844 1845 print_app_sa_prm(&app_sa_prm); 1846 1847 if (optind >= 0) 1848 argv[optind-1] = prgname; 1849 1850 ret = optind-1; 1851 optind = 1; /* reset getopt lib */ 1852 return ret; 1853 } 1854 1855 static void 1856 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr) 1857 { 1858 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 1859 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr); 1860 printf("%s%s", name, buf); 1861 } 1862 1863 /* 1864 * Update destination ethaddr for the port. 1865 */ 1866 int 1867 add_dst_ethaddr(uint16_t port, const struct rte_ether_addr *addr) 1868 { 1869 if (port >= RTE_DIM(ethaddr_tbl)) 1870 return -EINVAL; 1871 1872 ethaddr_tbl[port].dst = ETHADDR_TO_UINT64(addr); 1873 return 0; 1874 } 1875 1876 /* Check the link status of all ports in up to 9s, and print them finally */ 1877 static void 1878 check_all_ports_link_status(uint32_t port_mask) 1879 { 1880 #define CHECK_INTERVAL 100 /* 100ms */ 1881 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1882 uint16_t portid; 1883 uint8_t count, all_ports_up, print_flag = 0; 1884 struct rte_eth_link link; 1885 int ret; 1886 char link_status_text[RTE_ETH_LINK_MAX_STR_LEN]; 1887 1888 printf("\nChecking link status"); 1889 fflush(stdout); 1890 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1891 all_ports_up = 1; 1892 RTE_ETH_FOREACH_DEV(portid) { 1893 if ((port_mask & (1 << portid)) == 0) 1894 continue; 1895 memset(&link, 0, sizeof(link)); 1896 ret = rte_eth_link_get_nowait(portid, &link); 1897 if (ret < 0) { 1898 all_ports_up = 0; 1899 if (print_flag == 1) 1900 printf("Port %u link get failed: %s\n", 1901 portid, rte_strerror(-ret)); 1902 continue; 1903 } 1904 /* print link status if flag set */ 1905 if (print_flag == 1) { 1906 rte_eth_link_to_str(link_status_text, 1907 sizeof(link_status_text), &link); 1908 printf("Port %d %s\n", portid, 1909 link_status_text); 1910 continue; 1911 } 1912 /* clear all_ports_up flag if any link down */ 1913 if (link.link_status == ETH_LINK_DOWN) { 1914 all_ports_up = 0; 1915 break; 1916 } 1917 } 1918 /* after finally printing all link status, get out */ 1919 if (print_flag == 1) 1920 break; 1921 1922 if (all_ports_up == 0) { 1923 printf("."); 1924 fflush(stdout); 1925 rte_delay_ms(CHECK_INTERVAL); 1926 } 1927 1928 /* set the print_flag if all ports up or timeout */ 1929 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1930 print_flag = 1; 1931 printf("done\n"); 1932 } 1933 } 1934 } 1935 1936 static int32_t 1937 add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id, 1938 uint16_t qp, struct lcore_params *params, 1939 struct ipsec_ctx *ipsec_ctx, 1940 const struct rte_cryptodev_capabilities *cipher, 1941 const struct rte_cryptodev_capabilities *auth, 1942 const struct rte_cryptodev_capabilities *aead) 1943 { 1944 int32_t ret = 0; 1945 unsigned long i; 1946 struct cdev_key key = { 0 }; 1947 1948 key.lcore_id = params->lcore_id; 1949 if (cipher) 1950 key.cipher_algo = cipher->sym.cipher.algo; 1951 if (auth) 1952 key.auth_algo = auth->sym.auth.algo; 1953 if (aead) 1954 key.aead_algo = aead->sym.aead.algo; 1955 1956 ret = rte_hash_lookup(map, &key); 1957 if (ret != -ENOENT) 1958 return 0; 1959 1960 for (i = 0; i < ipsec_ctx->nb_qps; i++) 1961 if (ipsec_ctx->tbl[i].id == cdev_id) 1962 break; 1963 1964 if (i == ipsec_ctx->nb_qps) { 1965 if (ipsec_ctx->nb_qps == MAX_QP_PER_LCORE) { 1966 printf("Maximum number of crypto devices assigned to " 1967 "a core, increase MAX_QP_PER_LCORE value\n"); 1968 return 0; 1969 } 1970 ipsec_ctx->tbl[i].id = cdev_id; 1971 ipsec_ctx->tbl[i].qp = qp; 1972 ipsec_ctx->nb_qps++; 1973 printf("%s cdev mapping: lcore %u using cdev %u qp %u " 1974 "(cdev_id_qp %lu)\n", str, key.lcore_id, 1975 cdev_id, qp, i); 1976 } 1977 1978 ret = rte_hash_add_key_data(map, &key, (void *)i); 1979 if (ret < 0) { 1980 printf("Faled to insert cdev mapping for (lcore %u, " 1981 "cdev %u, qp %u), errno %d\n", 1982 key.lcore_id, ipsec_ctx->tbl[i].id, 1983 ipsec_ctx->tbl[i].qp, ret); 1984 return 0; 1985 } 1986 1987 return 1; 1988 } 1989 1990 static int32_t 1991 add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id, 1992 uint16_t qp, struct lcore_params *params) 1993 { 1994 int32_t ret = 0; 1995 const struct rte_cryptodev_capabilities *i, *j; 1996 struct rte_hash *map; 1997 struct lcore_conf *qconf; 1998 struct ipsec_ctx *ipsec_ctx; 1999 const char *str; 2000 2001 qconf = &lcore_conf[params->lcore_id]; 2002 2003 if ((unprotected_port_mask & (1 << params->port_id)) == 0) { 2004 map = cdev_map_out; 2005 ipsec_ctx = &qconf->outbound; 2006 str = "Outbound"; 2007 } else { 2008 map = cdev_map_in; 2009 ipsec_ctx = &qconf->inbound; 2010 str = "Inbound"; 2011 } 2012 2013 /* Required cryptodevs with operation chainning */ 2014 if (!(dev_info->feature_flags & 2015 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING)) 2016 return ret; 2017 2018 for (i = dev_info->capabilities; 2019 i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++) { 2020 if (i->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 2021 continue; 2022 2023 if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 2024 ret |= add_mapping(map, str, cdev_id, qp, params, 2025 ipsec_ctx, NULL, NULL, i); 2026 continue; 2027 } 2028 2029 if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) 2030 continue; 2031 2032 for (j = dev_info->capabilities; 2033 j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; j++) { 2034 if (j->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 2035 continue; 2036 2037 if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH) 2038 continue; 2039 2040 ret |= add_mapping(map, str, cdev_id, qp, params, 2041 ipsec_ctx, i, j, NULL); 2042 } 2043 } 2044 2045 return ret; 2046 } 2047 2048 /* Check if the device is enabled by cryptodev_mask */ 2049 static int 2050 check_cryptodev_mask(uint8_t cdev_id) 2051 { 2052 if (enabled_cryptodev_mask & (1 << cdev_id)) 2053 return 0; 2054 2055 return -1; 2056 } 2057 2058 static uint16_t 2059 cryptodevs_init(uint16_t req_queue_num) 2060 { 2061 struct rte_cryptodev_config dev_conf; 2062 struct rte_cryptodev_qp_conf qp_conf; 2063 uint16_t idx, max_nb_qps, qp, total_nb_qps, i; 2064 int16_t cdev_id; 2065 struct rte_hash_parameters params = { 0 }; 2066 2067 const uint64_t mseg_flag = multi_seg_required() ? 2068 RTE_CRYPTODEV_FF_IN_PLACE_SGL : 0; 2069 2070 params.entries = CDEV_MAP_ENTRIES; 2071 params.key_len = sizeof(struct cdev_key); 2072 params.hash_func = rte_jhash; 2073 params.hash_func_init_val = 0; 2074 params.socket_id = rte_socket_id(); 2075 2076 params.name = "cdev_map_in"; 2077 cdev_map_in = rte_hash_create(¶ms); 2078 if (cdev_map_in == NULL) 2079 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 2080 rte_errno); 2081 2082 params.name = "cdev_map_out"; 2083 cdev_map_out = rte_hash_create(¶ms); 2084 if (cdev_map_out == NULL) 2085 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 2086 rte_errno); 2087 2088 printf("lcore/cryptodev/qp mappings:\n"); 2089 2090 idx = 0; 2091 total_nb_qps = 0; 2092 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 2093 struct rte_cryptodev_info cdev_info; 2094 2095 if (check_cryptodev_mask((uint8_t)cdev_id)) 2096 continue; 2097 2098 rte_cryptodev_info_get(cdev_id, &cdev_info); 2099 2100 if ((mseg_flag & cdev_info.feature_flags) != mseg_flag) 2101 rte_exit(EXIT_FAILURE, 2102 "Device %hd does not support \'%s\' feature\n", 2103 cdev_id, 2104 rte_cryptodev_get_feature_name(mseg_flag)); 2105 2106 if (nb_lcore_params > cdev_info.max_nb_queue_pairs) 2107 max_nb_qps = cdev_info.max_nb_queue_pairs; 2108 else 2109 max_nb_qps = nb_lcore_params; 2110 2111 qp = 0; 2112 i = 0; 2113 while (qp < max_nb_qps && i < nb_lcore_params) { 2114 if (add_cdev_mapping(&cdev_info, cdev_id, qp, 2115 &lcore_params[idx])) 2116 qp++; 2117 idx++; 2118 idx = idx % nb_lcore_params; 2119 i++; 2120 } 2121 2122 qp = RTE_MIN(max_nb_qps, RTE_MAX(req_queue_num, qp)); 2123 if (qp == 0) 2124 continue; 2125 2126 total_nb_qps += qp; 2127 dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id); 2128 dev_conf.nb_queue_pairs = qp; 2129 dev_conf.ff_disable = RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO; 2130 2131 uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions; 2132 if (dev_max_sess != 0 && 2133 dev_max_sess < get_nb_crypto_sessions()) 2134 rte_exit(EXIT_FAILURE, 2135 "Device does not support at least %u " 2136 "sessions", get_nb_crypto_sessions()); 2137 2138 if (rte_cryptodev_configure(cdev_id, &dev_conf)) 2139 rte_panic("Failed to initialize cryptodev %u\n", 2140 cdev_id); 2141 2142 qp_conf.nb_descriptors = CDEV_QUEUE_DESC; 2143 qp_conf.mp_session = 2144 socket_ctx[dev_conf.socket_id].session_pool; 2145 qp_conf.mp_session_private = 2146 socket_ctx[dev_conf.socket_id].session_priv_pool; 2147 for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++) 2148 if (rte_cryptodev_queue_pair_setup(cdev_id, qp, 2149 &qp_conf, dev_conf.socket_id)) 2150 rte_panic("Failed to setup queue %u for " 2151 "cdev_id %u\n", 0, cdev_id); 2152 2153 if (rte_cryptodev_start(cdev_id)) 2154 rte_panic("Failed to start cryptodev %u\n", 2155 cdev_id); 2156 } 2157 2158 printf("\n"); 2159 2160 return total_nb_qps; 2161 } 2162 2163 static void 2164 port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads) 2165 { 2166 uint32_t frame_size; 2167 struct rte_eth_dev_info dev_info; 2168 struct rte_eth_txconf *txconf; 2169 uint16_t nb_tx_queue, nb_rx_queue; 2170 uint16_t tx_queueid, rx_queueid, queue, lcore_id; 2171 int32_t ret, socket_id; 2172 struct lcore_conf *qconf; 2173 struct rte_ether_addr ethaddr; 2174 struct rte_eth_conf local_port_conf = port_conf; 2175 2176 ret = rte_eth_dev_info_get(portid, &dev_info); 2177 if (ret != 0) 2178 rte_exit(EXIT_FAILURE, 2179 "Error during getting device (port %u) info: %s\n", 2180 portid, strerror(-ret)); 2181 2182 /* limit allowed HW offloafs, as user requested */ 2183 dev_info.rx_offload_capa &= dev_rx_offload; 2184 dev_info.tx_offload_capa &= dev_tx_offload; 2185 2186 printf("Configuring device port %u:\n", portid); 2187 2188 ret = rte_eth_macaddr_get(portid, ðaddr); 2189 if (ret != 0) 2190 rte_exit(EXIT_FAILURE, 2191 "Error getting MAC address (port %u): %s\n", 2192 portid, rte_strerror(-ret)); 2193 2194 ethaddr_tbl[portid].src = ETHADDR_TO_UINT64(ðaddr); 2195 print_ethaddr("Address: ", ðaddr); 2196 printf("\n"); 2197 2198 nb_rx_queue = get_port_nb_rx_queues(portid); 2199 nb_tx_queue = nb_lcores; 2200 2201 if (nb_rx_queue > dev_info.max_rx_queues) 2202 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 2203 "(max rx queue is %u)\n", 2204 nb_rx_queue, dev_info.max_rx_queues); 2205 2206 if (nb_tx_queue > dev_info.max_tx_queues) 2207 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 2208 "(max tx queue is %u)\n", 2209 nb_tx_queue, dev_info.max_tx_queues); 2210 2211 printf("Creating queues: nb_rx_queue=%d nb_tx_queue=%u...\n", 2212 nb_rx_queue, nb_tx_queue); 2213 2214 frame_size = MTU_TO_FRAMELEN(mtu_size); 2215 if (frame_size > local_port_conf.rxmode.max_rx_pkt_len) 2216 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 2217 local_port_conf.rxmode.max_rx_pkt_len = frame_size; 2218 2219 if (multi_seg_required()) { 2220 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_SCATTER; 2221 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MULTI_SEGS; 2222 } 2223 2224 local_port_conf.rxmode.offloads |= req_rx_offloads; 2225 local_port_conf.txmode.offloads |= req_tx_offloads; 2226 2227 /* Check that all required capabilities are supported */ 2228 if ((local_port_conf.rxmode.offloads & dev_info.rx_offload_capa) != 2229 local_port_conf.rxmode.offloads) 2230 rte_exit(EXIT_FAILURE, 2231 "Error: port %u required RX offloads: 0x%" PRIx64 2232 ", avaialbe RX offloads: 0x%" PRIx64 "\n", 2233 portid, local_port_conf.rxmode.offloads, 2234 dev_info.rx_offload_capa); 2235 2236 if ((local_port_conf.txmode.offloads & dev_info.tx_offload_capa) != 2237 local_port_conf.txmode.offloads) 2238 rte_exit(EXIT_FAILURE, 2239 "Error: port %u required TX offloads: 0x%" PRIx64 2240 ", avaialbe TX offloads: 0x%" PRIx64 "\n", 2241 portid, local_port_conf.txmode.offloads, 2242 dev_info.tx_offload_capa); 2243 2244 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 2245 local_port_conf.txmode.offloads |= 2246 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 2247 2248 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 2249 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 2250 2251 printf("port %u configurng rx_offloads=0x%" PRIx64 2252 ", tx_offloads=0x%" PRIx64 "\n", 2253 portid, local_port_conf.rxmode.offloads, 2254 local_port_conf.txmode.offloads); 2255 2256 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 2257 dev_info.flow_type_rss_offloads; 2258 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 2259 port_conf.rx_adv_conf.rss_conf.rss_hf) { 2260 printf("Port %u modified RSS hash function based on hardware support," 2261 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 2262 portid, 2263 port_conf.rx_adv_conf.rss_conf.rss_hf, 2264 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 2265 } 2266 2267 ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue, 2268 &local_port_conf); 2269 if (ret < 0) 2270 rte_exit(EXIT_FAILURE, "Cannot configure device: " 2271 "err=%d, port=%d\n", ret, portid); 2272 2273 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd); 2274 if (ret < 0) 2275 rte_exit(EXIT_FAILURE, "Cannot adjust number of descriptors: " 2276 "err=%d, port=%d\n", ret, portid); 2277 2278 /* init one TX queue per lcore */ 2279 tx_queueid = 0; 2280 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2281 if (rte_lcore_is_enabled(lcore_id) == 0) 2282 continue; 2283 2284 if (numa_on) 2285 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2286 else 2287 socket_id = 0; 2288 2289 /* init TX queue */ 2290 printf("Setup txq=%u,%d,%d\n", lcore_id, tx_queueid, socket_id); 2291 2292 txconf = &dev_info.default_txconf; 2293 txconf->offloads = local_port_conf.txmode.offloads; 2294 2295 ret = rte_eth_tx_queue_setup(portid, tx_queueid, nb_txd, 2296 socket_id, txconf); 2297 if (ret < 0) 2298 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: " 2299 "err=%d, port=%d\n", ret, portid); 2300 2301 qconf = &lcore_conf[lcore_id]; 2302 qconf->tx_queue_id[portid] = tx_queueid; 2303 2304 /* Pre-populate pkt offloads based on capabilities */ 2305 qconf->outbound.ipv4_offloads = PKT_TX_IPV4; 2306 qconf->outbound.ipv6_offloads = PKT_TX_IPV6; 2307 if (local_port_conf.txmode.offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) 2308 qconf->outbound.ipv4_offloads |= PKT_TX_IP_CKSUM; 2309 2310 tx_queueid++; 2311 2312 /* init RX queues */ 2313 for (queue = 0; queue < qconf->nb_rx_queue; ++queue) { 2314 struct rte_eth_rxconf rxq_conf; 2315 2316 if (portid != qconf->rx_queue_list[queue].port_id) 2317 continue; 2318 2319 rx_queueid = qconf->rx_queue_list[queue].queue_id; 2320 2321 printf("Setup rxq=%d,%d,%d\n", portid, rx_queueid, 2322 socket_id); 2323 2324 rxq_conf = dev_info.default_rxconf; 2325 rxq_conf.offloads = local_port_conf.rxmode.offloads; 2326 ret = rte_eth_rx_queue_setup(portid, rx_queueid, 2327 nb_rxd, socket_id, &rxq_conf, 2328 socket_ctx[socket_id].mbuf_pool); 2329 if (ret < 0) 2330 rte_exit(EXIT_FAILURE, 2331 "rte_eth_rx_queue_setup: err=%d, " 2332 "port=%d\n", ret, portid); 2333 } 2334 } 2335 printf("\n"); 2336 } 2337 2338 static size_t 2339 max_session_size(void) 2340 { 2341 size_t max_sz, sz; 2342 void *sec_ctx; 2343 int16_t cdev_id, port_id, n; 2344 2345 max_sz = 0; 2346 n = rte_cryptodev_count(); 2347 for (cdev_id = 0; cdev_id != n; cdev_id++) { 2348 sz = rte_cryptodev_sym_get_private_session_size(cdev_id); 2349 if (sz > max_sz) 2350 max_sz = sz; 2351 /* 2352 * If crypto device is security capable, need to check the 2353 * size of security session as well. 2354 */ 2355 2356 /* Get security context of the crypto device */ 2357 sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id); 2358 if (sec_ctx == NULL) 2359 continue; 2360 2361 /* Get size of security session */ 2362 sz = rte_security_session_get_size(sec_ctx); 2363 if (sz > max_sz) 2364 max_sz = sz; 2365 } 2366 2367 RTE_ETH_FOREACH_DEV(port_id) { 2368 if ((enabled_port_mask & (1 << port_id)) == 0) 2369 continue; 2370 2371 sec_ctx = rte_eth_dev_get_sec_ctx(port_id); 2372 if (sec_ctx == NULL) 2373 continue; 2374 2375 sz = rte_security_session_get_size(sec_ctx); 2376 if (sz > max_sz) 2377 max_sz = sz; 2378 } 2379 2380 return max_sz; 2381 } 2382 2383 static void 2384 session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz) 2385 { 2386 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2387 struct rte_mempool *sess_mp; 2388 uint32_t nb_sess; 2389 2390 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2391 "sess_mp_%u", socket_id); 2392 nb_sess = (get_nb_crypto_sessions() + CDEV_MP_CACHE_SZ * 2393 rte_lcore_count()); 2394 nb_sess = RTE_MAX(nb_sess, CDEV_MP_CACHE_SZ * 2395 CDEV_MP_CACHE_MULTIPLIER); 2396 sess_mp = rte_cryptodev_sym_session_pool_create( 2397 mp_name, nb_sess, sess_sz, CDEV_MP_CACHE_SZ, 0, 2398 socket_id); 2399 ctx->session_pool = sess_mp; 2400 2401 if (ctx->session_pool == NULL) 2402 rte_exit(EXIT_FAILURE, 2403 "Cannot init session pool on socket %d\n", socket_id); 2404 else 2405 printf("Allocated session pool on socket %d\n", socket_id); 2406 } 2407 2408 static void 2409 session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id, 2410 size_t sess_sz) 2411 { 2412 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2413 struct rte_mempool *sess_mp; 2414 uint32_t nb_sess; 2415 2416 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2417 "sess_mp_priv_%u", socket_id); 2418 nb_sess = (get_nb_crypto_sessions() + CDEV_MP_CACHE_SZ * 2419 rte_lcore_count()); 2420 nb_sess = RTE_MAX(nb_sess, CDEV_MP_CACHE_SZ * 2421 CDEV_MP_CACHE_MULTIPLIER); 2422 sess_mp = rte_mempool_create(mp_name, 2423 nb_sess, 2424 sess_sz, 2425 CDEV_MP_CACHE_SZ, 2426 0, NULL, NULL, NULL, 2427 NULL, socket_id, 2428 0); 2429 ctx->session_priv_pool = sess_mp; 2430 2431 if (ctx->session_priv_pool == NULL) 2432 rte_exit(EXIT_FAILURE, 2433 "Cannot init session priv pool on socket %d\n", 2434 socket_id); 2435 else 2436 printf("Allocated session priv pool on socket %d\n", 2437 socket_id); 2438 } 2439 2440 static void 2441 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf) 2442 { 2443 char s[64]; 2444 int32_t ms; 2445 2446 snprintf(s, sizeof(s), "mbuf_pool_%d", socket_id); 2447 ctx->mbuf_pool = rte_pktmbuf_pool_create(s, nb_mbuf, 2448 MEMPOOL_CACHE_SIZE, ipsec_metadata_size(), 2449 frame_buf_size, socket_id); 2450 2451 /* 2452 * if multi-segment support is enabled, then create a pool 2453 * for indirect mbufs. 2454 */ 2455 ms = multi_seg_required(); 2456 if (ms != 0) { 2457 snprintf(s, sizeof(s), "mbuf_pool_indir_%d", socket_id); 2458 ctx->mbuf_pool_indir = rte_pktmbuf_pool_create(s, nb_mbuf, 2459 MEMPOOL_CACHE_SIZE, 0, 0, socket_id); 2460 } 2461 2462 if (ctx->mbuf_pool == NULL || (ms != 0 && ctx->mbuf_pool_indir == NULL)) 2463 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n", 2464 socket_id); 2465 else 2466 printf("Allocated mbuf pool on socket %d\n", socket_id); 2467 } 2468 2469 static inline int 2470 inline_ipsec_event_esn_overflow(struct rte_security_ctx *ctx, uint64_t md) 2471 { 2472 struct ipsec_sa *sa; 2473 2474 /* For inline protocol processing, the metadata in the event will 2475 * uniquely identify the security session which raised the event. 2476 * Application would then need the userdata it had registered with the 2477 * security session to process the event. 2478 */ 2479 2480 sa = (struct ipsec_sa *)rte_security_get_userdata(ctx, md); 2481 2482 if (sa == NULL) { 2483 /* userdata could not be retrieved */ 2484 return -1; 2485 } 2486 2487 /* Sequence number over flow. SA need to be re-established */ 2488 RTE_SET_USED(sa); 2489 return 0; 2490 } 2491 2492 static int 2493 inline_ipsec_event_callback(uint16_t port_id, enum rte_eth_event_type type, 2494 void *param, void *ret_param) 2495 { 2496 uint64_t md; 2497 struct rte_eth_event_ipsec_desc *event_desc = NULL; 2498 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 2499 rte_eth_dev_get_sec_ctx(port_id); 2500 2501 RTE_SET_USED(param); 2502 2503 if (type != RTE_ETH_EVENT_IPSEC) 2504 return -1; 2505 2506 event_desc = ret_param; 2507 if (event_desc == NULL) { 2508 printf("Event descriptor not set\n"); 2509 return -1; 2510 } 2511 2512 md = event_desc->metadata; 2513 2514 if (event_desc->subtype == RTE_ETH_EVENT_IPSEC_ESN_OVERFLOW) 2515 return inline_ipsec_event_esn_overflow(ctx, md); 2516 else if (event_desc->subtype >= RTE_ETH_EVENT_IPSEC_MAX) { 2517 printf("Invalid IPsec event reported\n"); 2518 return -1; 2519 } 2520 2521 return -1; 2522 } 2523 2524 static uint16_t 2525 rx_callback(__rte_unused uint16_t port, __rte_unused uint16_t queue, 2526 struct rte_mbuf *pkt[], uint16_t nb_pkts, 2527 __rte_unused uint16_t max_pkts, void *user_param) 2528 { 2529 uint64_t tm; 2530 uint32_t i, k; 2531 struct lcore_conf *lc; 2532 struct rte_mbuf *mb; 2533 struct rte_ether_hdr *eth; 2534 2535 lc = user_param; 2536 k = 0; 2537 tm = 0; 2538 2539 for (i = 0; i != nb_pkts; i++) { 2540 2541 mb = pkt[i]; 2542 eth = rte_pktmbuf_mtod(mb, struct rte_ether_hdr *); 2543 if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 2544 2545 struct rte_ipv4_hdr *iph; 2546 2547 iph = (struct rte_ipv4_hdr *)(eth + 1); 2548 if (rte_ipv4_frag_pkt_is_fragmented(iph)) { 2549 2550 mb->l2_len = sizeof(*eth); 2551 mb->l3_len = sizeof(*iph); 2552 tm = (tm != 0) ? tm : rte_rdtsc(); 2553 mb = rte_ipv4_frag_reassemble_packet( 2554 lc->frag.tbl, &lc->frag.dr, 2555 mb, tm, iph); 2556 2557 if (mb != NULL) { 2558 /* fix ip cksum after reassemble. */ 2559 iph = rte_pktmbuf_mtod_offset(mb, 2560 struct rte_ipv4_hdr *, 2561 mb->l2_len); 2562 iph->hdr_checksum = 0; 2563 iph->hdr_checksum = rte_ipv4_cksum(iph); 2564 } 2565 } 2566 } else if (eth->ether_type == 2567 rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 2568 2569 struct rte_ipv6_hdr *iph; 2570 struct ipv6_extension_fragment *fh; 2571 2572 iph = (struct rte_ipv6_hdr *)(eth + 1); 2573 fh = rte_ipv6_frag_get_ipv6_fragment_header(iph); 2574 if (fh != NULL) { 2575 mb->l2_len = sizeof(*eth); 2576 mb->l3_len = (uintptr_t)fh - (uintptr_t)iph + 2577 sizeof(*fh); 2578 tm = (tm != 0) ? tm : rte_rdtsc(); 2579 mb = rte_ipv6_frag_reassemble_packet( 2580 lc->frag.tbl, &lc->frag.dr, 2581 mb, tm, iph, fh); 2582 if (mb != NULL) 2583 /* fix l3_len after reassemble. */ 2584 mb->l3_len = mb->l3_len - sizeof(*fh); 2585 } 2586 } 2587 2588 pkt[k] = mb; 2589 k += (mb != NULL); 2590 } 2591 2592 /* some fragments were encountered, drain death row */ 2593 if (tm != 0) 2594 rte_ip_frag_free_death_row(&lc->frag.dr, 0); 2595 2596 return k; 2597 } 2598 2599 2600 static int 2601 reassemble_lcore_init(struct lcore_conf *lc, uint32_t cid) 2602 { 2603 int32_t sid; 2604 uint32_t i; 2605 uint64_t frag_cycles; 2606 const struct lcore_rx_queue *rxq; 2607 const struct rte_eth_rxtx_callback *cb; 2608 2609 /* create fragment table */ 2610 sid = rte_lcore_to_socket_id(cid); 2611 frag_cycles = (rte_get_tsc_hz() + NS_PER_S - 1) / 2612 NS_PER_S * frag_ttl_ns; 2613 2614 lc->frag.tbl = rte_ip_frag_table_create(frag_tbl_sz, 2615 FRAG_TBL_BUCKET_ENTRIES, frag_tbl_sz, frag_cycles, sid); 2616 if (lc->frag.tbl == NULL) { 2617 printf("%s(%u): failed to create fragment table of size: %u, " 2618 "error code: %d\n", 2619 __func__, cid, frag_tbl_sz, rte_errno); 2620 return -ENOMEM; 2621 } 2622 2623 /* setup reassemble RX callbacks for all queues */ 2624 for (i = 0; i != lc->nb_rx_queue; i++) { 2625 2626 rxq = lc->rx_queue_list + i; 2627 cb = rte_eth_add_rx_callback(rxq->port_id, rxq->queue_id, 2628 rx_callback, lc); 2629 if (cb == NULL) { 2630 printf("%s(%u): failed to install RX callback for " 2631 "portid=%u, queueid=%u, error code: %d\n", 2632 __func__, cid, 2633 rxq->port_id, rxq->queue_id, rte_errno); 2634 return -ENOMEM; 2635 } 2636 } 2637 2638 return 0; 2639 } 2640 2641 static int 2642 reassemble_init(void) 2643 { 2644 int32_t rc; 2645 uint32_t i, lc; 2646 2647 rc = 0; 2648 for (i = 0; i != nb_lcore_params; i++) { 2649 lc = lcore_params[i].lcore_id; 2650 rc = reassemble_lcore_init(lcore_conf + lc, lc); 2651 if (rc != 0) 2652 break; 2653 } 2654 2655 return rc; 2656 } 2657 2658 static void 2659 create_default_ipsec_flow(uint16_t port_id, uint64_t rx_offloads) 2660 { 2661 struct rte_flow_action action[2]; 2662 struct rte_flow_item pattern[2]; 2663 struct rte_flow_attr attr = {0}; 2664 struct rte_flow_error err; 2665 struct rte_flow *flow; 2666 int ret; 2667 2668 if (!(rx_offloads & DEV_RX_OFFLOAD_SECURITY)) 2669 return; 2670 2671 /* Add the default rte_flow to enable SECURITY for all ESP packets */ 2672 2673 pattern[0].type = RTE_FLOW_ITEM_TYPE_ESP; 2674 pattern[0].spec = NULL; 2675 pattern[0].mask = NULL; 2676 pattern[0].last = NULL; 2677 pattern[1].type = RTE_FLOW_ITEM_TYPE_END; 2678 2679 action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY; 2680 action[0].conf = NULL; 2681 action[1].type = RTE_FLOW_ACTION_TYPE_END; 2682 action[1].conf = NULL; 2683 2684 attr.ingress = 1; 2685 2686 ret = rte_flow_validate(port_id, &attr, pattern, action, &err); 2687 if (ret) 2688 return; 2689 2690 flow = rte_flow_create(port_id, &attr, pattern, action, &err); 2691 if (flow == NULL) 2692 return; 2693 2694 flow_info_tbl[port_id].rx_def_flow = flow; 2695 RTE_LOG(INFO, IPSEC, 2696 "Created default flow enabling SECURITY for all ESP traffic on port %d\n", 2697 port_id); 2698 } 2699 2700 static void 2701 signal_handler(int signum) 2702 { 2703 if (signum == SIGINT || signum == SIGTERM) { 2704 printf("\n\nSignal %d received, preparing to exit...\n", 2705 signum); 2706 force_quit = true; 2707 } 2708 } 2709 2710 static void 2711 ev_mode_sess_verify(struct ipsec_sa *sa, int nb_sa) 2712 { 2713 struct rte_ipsec_session *ips; 2714 int32_t i; 2715 2716 if (!sa || !nb_sa) 2717 return; 2718 2719 for (i = 0; i < nb_sa; i++) { 2720 ips = ipsec_get_primary_session(&sa[i]); 2721 if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) 2722 rte_exit(EXIT_FAILURE, "Event mode supports only " 2723 "inline protocol sessions\n"); 2724 } 2725 2726 } 2727 2728 static int32_t 2729 check_event_mode_params(struct eh_conf *eh_conf) 2730 { 2731 struct eventmode_conf *em_conf = NULL; 2732 struct lcore_params *params; 2733 uint16_t portid; 2734 2735 if (!eh_conf || !eh_conf->mode_params) 2736 return -EINVAL; 2737 2738 /* Get eventmode conf */ 2739 em_conf = eh_conf->mode_params; 2740 2741 if (eh_conf->mode == EH_PKT_TRANSFER_MODE_POLL && 2742 em_conf->ext_params.sched_type != SCHED_TYPE_NOT_SET) { 2743 printf("error: option --event-schedule-type applies only to " 2744 "event mode\n"); 2745 return -EINVAL; 2746 } 2747 2748 if (eh_conf->mode != EH_PKT_TRANSFER_MODE_EVENT) 2749 return 0; 2750 2751 /* Set schedule type to ORDERED if it wasn't explicitly set by user */ 2752 if (em_conf->ext_params.sched_type == SCHED_TYPE_NOT_SET) 2753 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ORDERED; 2754 2755 /* 2756 * Event mode currently supports only inline protocol sessions. 2757 * If there are other types of sessions configured then exit with 2758 * error. 2759 */ 2760 ev_mode_sess_verify(sa_in, nb_sa_in); 2761 ev_mode_sess_verify(sa_out, nb_sa_out); 2762 2763 2764 /* Option --config does not apply to event mode */ 2765 if (nb_lcore_params > 0) { 2766 printf("error: option --config applies only to poll mode\n"); 2767 return -EINVAL; 2768 } 2769 2770 /* 2771 * In order to use the same port_init routine for both poll and event 2772 * modes initialize lcore_params with one queue for each eth port 2773 */ 2774 lcore_params = lcore_params_array; 2775 RTE_ETH_FOREACH_DEV(portid) { 2776 if ((enabled_port_mask & (1 << portid)) == 0) 2777 continue; 2778 2779 params = &lcore_params[nb_lcore_params++]; 2780 params->port_id = portid; 2781 params->queue_id = 0; 2782 params->lcore_id = rte_get_next_lcore(0, 0, 1); 2783 } 2784 2785 return 0; 2786 } 2787 2788 static void 2789 inline_sessions_free(struct sa_ctx *sa_ctx) 2790 { 2791 struct rte_ipsec_session *ips; 2792 struct ipsec_sa *sa; 2793 int32_t ret; 2794 uint32_t i; 2795 2796 if (!sa_ctx) 2797 return; 2798 2799 for (i = 0; i < sa_ctx->nb_sa; i++) { 2800 2801 sa = &sa_ctx->sa[i]; 2802 if (!sa->spi) 2803 continue; 2804 2805 ips = ipsec_get_primary_session(sa); 2806 if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL && 2807 ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) 2808 continue; 2809 2810 if (!rte_eth_dev_is_valid_port(sa->portid)) 2811 continue; 2812 2813 ret = rte_security_session_destroy( 2814 rte_eth_dev_get_sec_ctx(sa->portid), 2815 ips->security.ses); 2816 if (ret) 2817 RTE_LOG(ERR, IPSEC, "Failed to destroy security " 2818 "session type %d, spi %d\n", 2819 ips->type, sa->spi); 2820 } 2821 } 2822 2823 static uint32_t 2824 calculate_nb_mbufs(uint16_t nb_ports, uint16_t nb_crypto_qp, uint32_t nb_rxq, 2825 uint32_t nb_txq) 2826 { 2827 return RTE_MAX((nb_rxq * nb_rxd + 2828 nb_ports * nb_lcores * MAX_PKT_BURST + 2829 nb_ports * nb_txq * nb_txd + 2830 nb_lcores * MEMPOOL_CACHE_SIZE + 2831 nb_crypto_qp * CDEV_QUEUE_DESC + 2832 nb_lcores * frag_tbl_sz * 2833 FRAG_TBL_BUCKET_ENTRIES), 2834 8192U); 2835 } 2836 2837 int32_t 2838 main(int32_t argc, char **argv) 2839 { 2840 int32_t ret; 2841 uint32_t lcore_id, nb_txq, nb_rxq = 0; 2842 uint32_t cdev_id; 2843 uint32_t i; 2844 uint8_t socket_id; 2845 uint16_t portid, nb_crypto_qp, nb_ports = 0; 2846 uint64_t req_rx_offloads[RTE_MAX_ETHPORTS]; 2847 uint64_t req_tx_offloads[RTE_MAX_ETHPORTS]; 2848 struct eh_conf *eh_conf = NULL; 2849 size_t sess_sz; 2850 2851 nb_bufs_in_pool = 0; 2852 2853 /* init EAL */ 2854 ret = rte_eal_init(argc, argv); 2855 if (ret < 0) 2856 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 2857 argc -= ret; 2858 argv += ret; 2859 2860 force_quit = false; 2861 signal(SIGINT, signal_handler); 2862 signal(SIGTERM, signal_handler); 2863 2864 /* initialize event helper configuration */ 2865 eh_conf = eh_conf_init(); 2866 if (eh_conf == NULL) 2867 rte_exit(EXIT_FAILURE, "Failed to init event helper config"); 2868 2869 /* parse application arguments (after the EAL ones) */ 2870 ret = parse_args(argc, argv, eh_conf); 2871 if (ret < 0) 2872 rte_exit(EXIT_FAILURE, "Invalid parameters\n"); 2873 2874 /* parse configuration file */ 2875 if (parse_cfg_file(cfgfile) < 0) { 2876 printf("parsing file \"%s\" failed\n", 2877 optarg); 2878 print_usage(argv[0]); 2879 return -1; 2880 } 2881 2882 if ((unprotected_port_mask & enabled_port_mask) != 2883 unprotected_port_mask) 2884 rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n", 2885 unprotected_port_mask); 2886 2887 if (check_poll_mode_params(eh_conf) < 0) 2888 rte_exit(EXIT_FAILURE, "check_poll_mode_params failed\n"); 2889 2890 if (check_event_mode_params(eh_conf) < 0) 2891 rte_exit(EXIT_FAILURE, "check_event_mode_params failed\n"); 2892 2893 ret = init_lcore_rx_queues(); 2894 if (ret < 0) 2895 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 2896 2897 nb_lcores = rte_lcore_count(); 2898 2899 sess_sz = max_session_size(); 2900 2901 /* 2902 * In event mode request minimum number of crypto queues 2903 * to be reserved equal to number of ports. 2904 */ 2905 if (eh_conf->mode == EH_PKT_TRANSFER_MODE_EVENT) 2906 nb_crypto_qp = rte_eth_dev_count_avail(); 2907 else 2908 nb_crypto_qp = 0; 2909 2910 nb_crypto_qp = cryptodevs_init(nb_crypto_qp); 2911 2912 if (nb_bufs_in_pool == 0) { 2913 RTE_ETH_FOREACH_DEV(portid) { 2914 if ((enabled_port_mask & (1 << portid)) == 0) 2915 continue; 2916 nb_ports++; 2917 nb_rxq += get_port_nb_rx_queues(portid); 2918 } 2919 2920 nb_txq = nb_lcores; 2921 2922 nb_bufs_in_pool = calculate_nb_mbufs(nb_ports, nb_crypto_qp, 2923 nb_rxq, nb_txq); 2924 } 2925 2926 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2927 if (rte_lcore_is_enabled(lcore_id) == 0) 2928 continue; 2929 2930 if (numa_on) 2931 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2932 else 2933 socket_id = 0; 2934 2935 /* mbuf_pool is initialised by the pool_init() function*/ 2936 if (socket_ctx[socket_id].mbuf_pool) 2937 continue; 2938 2939 pool_init(&socket_ctx[socket_id], socket_id, nb_bufs_in_pool); 2940 session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz); 2941 session_priv_pool_init(&socket_ctx[socket_id], socket_id, 2942 sess_sz); 2943 } 2944 printf("Number of mbufs in packet pool %d\n", nb_bufs_in_pool); 2945 2946 RTE_ETH_FOREACH_DEV(portid) { 2947 if ((enabled_port_mask & (1 << portid)) == 0) 2948 continue; 2949 2950 sa_check_offloads(portid, &req_rx_offloads[portid], 2951 &req_tx_offloads[portid]); 2952 port_init(portid, req_rx_offloads[portid], 2953 req_tx_offloads[portid]); 2954 } 2955 2956 /* 2957 * Set the enabled port mask in helper config for use by helper 2958 * sub-system. This will be used while initializing devices using 2959 * helper sub-system. 2960 */ 2961 eh_conf->eth_portmask = enabled_port_mask; 2962 2963 /* Initialize eventmode components */ 2964 ret = eh_devs_init(eh_conf); 2965 if (ret < 0) 2966 rte_exit(EXIT_FAILURE, "eh_devs_init failed, err=%d\n", ret); 2967 2968 /* start ports */ 2969 RTE_ETH_FOREACH_DEV(portid) { 2970 if ((enabled_port_mask & (1 << portid)) == 0) 2971 continue; 2972 2973 /* Create flow before starting the device */ 2974 create_default_ipsec_flow(portid, req_rx_offloads[portid]); 2975 2976 ret = rte_eth_dev_start(portid); 2977 if (ret < 0) 2978 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: " 2979 "err=%d, port=%d\n", ret, portid); 2980 /* 2981 * If enabled, put device in promiscuous mode. 2982 * This allows IO forwarding mode to forward packets 2983 * to itself through 2 cross-connected ports of the 2984 * target machine. 2985 */ 2986 if (promiscuous_on) { 2987 ret = rte_eth_promiscuous_enable(portid); 2988 if (ret != 0) 2989 rte_exit(EXIT_FAILURE, 2990 "rte_eth_promiscuous_enable: err=%s, port=%d\n", 2991 rte_strerror(-ret), portid); 2992 } 2993 2994 rte_eth_dev_callback_register(portid, 2995 RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL); 2996 } 2997 2998 /* fragment reassemble is enabled */ 2999 if (frag_tbl_sz != 0) { 3000 ret = reassemble_init(); 3001 if (ret != 0) 3002 rte_exit(EXIT_FAILURE, "failed at reassemble init"); 3003 } 3004 3005 /* Replicate each context per socket */ 3006 for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) { 3007 socket_id = rte_socket_id_by_idx(i); 3008 if ((socket_ctx[socket_id].mbuf_pool != NULL) && 3009 (socket_ctx[socket_id].sa_in == NULL) && 3010 (socket_ctx[socket_id].sa_out == NULL)) { 3011 sa_init(&socket_ctx[socket_id], socket_id); 3012 sp4_init(&socket_ctx[socket_id], socket_id); 3013 sp6_init(&socket_ctx[socket_id], socket_id); 3014 rt_init(&socket_ctx[socket_id], socket_id); 3015 } 3016 } 3017 3018 flow_init(); 3019 3020 check_all_ports_link_status(enabled_port_mask); 3021 3022 #if (STATS_INTERVAL > 0) 3023 rte_eal_alarm_set(STATS_INTERVAL * US_PER_S, print_stats_cb, NULL); 3024 #else 3025 RTE_LOG(INFO, IPSEC, "Stats display disabled\n"); 3026 #endif /* STATS_INTERVAL */ 3027 3028 /* launch per-lcore init on every lcore */ 3029 rte_eal_mp_remote_launch(ipsec_launch_one_lcore, eh_conf, CALL_MAIN); 3030 RTE_LCORE_FOREACH_WORKER(lcore_id) { 3031 if (rte_eal_wait_lcore(lcore_id) < 0) 3032 return -1; 3033 } 3034 3035 /* Uninitialize eventmode components */ 3036 ret = eh_devs_uninit(eh_conf); 3037 if (ret < 0) 3038 rte_exit(EXIT_FAILURE, "eh_devs_uninit failed, err=%d\n", ret); 3039 3040 /* Free eventmode configuration memory */ 3041 eh_conf_uninit(eh_conf); 3042 3043 /* Destroy inline inbound and outbound sessions */ 3044 for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) { 3045 socket_id = rte_socket_id_by_idx(i); 3046 inline_sessions_free(socket_ctx[socket_id].sa_in); 3047 inline_sessions_free(socket_ctx[socket_id].sa_out); 3048 } 3049 3050 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 3051 printf("Closing cryptodev %d...", cdev_id); 3052 rte_cryptodev_stop(cdev_id); 3053 rte_cryptodev_close(cdev_id); 3054 printf(" Done\n"); 3055 } 3056 3057 RTE_ETH_FOREACH_DEV(portid) { 3058 if ((enabled_port_mask & (1 << portid)) == 0) 3059 continue; 3060 3061 printf("Closing port %d...", portid); 3062 if (flow_info_tbl[portid].rx_def_flow) { 3063 struct rte_flow_error err; 3064 3065 ret = rte_flow_destroy(portid, 3066 flow_info_tbl[portid].rx_def_flow, &err); 3067 if (ret) 3068 RTE_LOG(ERR, IPSEC, "Failed to destroy flow " 3069 " for port %u, err msg: %s\n", portid, 3070 err.message); 3071 } 3072 ret = rte_eth_dev_stop(portid); 3073 if (ret != 0) 3074 RTE_LOG(ERR, IPSEC, 3075 "rte_eth_dev_stop: err=%s, port=%u\n", 3076 rte_strerror(-ret), portid); 3077 3078 rte_eth_dev_close(portid); 3079 printf(" Done\n"); 3080 } 3081 3082 /* clean up the EAL */ 3083 rte_eal_cleanup(); 3084 printf("Bye...\n"); 3085 3086 return 0; 3087 } 3088