1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 Intel Corporation 3 */ 4 5 #include <stdbool.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <stdint.h> 9 #include <inttypes.h> 10 #include <sys/types.h> 11 #include <netinet/in.h> 12 #include <netinet/ip.h> 13 #include <netinet/ip6.h> 14 #include <string.h> 15 #include <sys/queue.h> 16 #include <stdarg.h> 17 #include <errno.h> 18 #include <signal.h> 19 #include <getopt.h> 20 21 #include <rte_common.h> 22 #include <rte_bitmap.h> 23 #include <rte_byteorder.h> 24 #include <rte_log.h> 25 #include <rte_eal.h> 26 #include <rte_launch.h> 27 #include <rte_atomic.h> 28 #include <rte_cycles.h> 29 #include <rte_prefetch.h> 30 #include <rte_lcore.h> 31 #include <rte_per_lcore.h> 32 #include <rte_branch_prediction.h> 33 #include <rte_interrupts.h> 34 #include <rte_random.h> 35 #include <rte_debug.h> 36 #include <rte_ether.h> 37 #include <rte_ethdev.h> 38 #include <rte_mempool.h> 39 #include <rte_mbuf.h> 40 #include <rte_acl.h> 41 #include <rte_lpm.h> 42 #include <rte_lpm6.h> 43 #include <rte_hash.h> 44 #include <rte_jhash.h> 45 #include <rte_cryptodev.h> 46 #include <rte_security.h> 47 #include <rte_eventdev.h> 48 #include <rte_ip.h> 49 #include <rte_ip_frag.h> 50 #include <rte_alarm.h> 51 52 #include "event_helper.h" 53 #include "flow.h" 54 #include "ipsec.h" 55 #include "ipsec_worker.h" 56 #include "parser.h" 57 #include "sad.h" 58 59 volatile bool force_quit; 60 61 #define MAX_JUMBO_PKT_LEN 9600 62 63 #define MEMPOOL_CACHE_SIZE 256 64 65 #define CDEV_QUEUE_DESC 2048 66 #define CDEV_MAP_ENTRIES 16384 67 #define CDEV_MP_CACHE_SZ 64 68 #define CDEV_MP_CACHE_MULTIPLIER 1.5 /* from rte_mempool.c */ 69 #define MAX_QUEUE_PAIRS 1 70 71 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 72 73 /* Configure how many packets ahead to prefetch, when reading packets */ 74 #define PREFETCH_OFFSET 3 75 76 #define MAX_RX_QUEUE_PER_LCORE 16 77 78 #define MAX_LCORE_PARAMS 1024 79 80 /* 81 * Configurable number of RX/TX ring descriptors 82 */ 83 #define IPSEC_SECGW_RX_DESC_DEFAULT 1024 84 #define IPSEC_SECGW_TX_DESC_DEFAULT 1024 85 static uint16_t nb_rxd = IPSEC_SECGW_RX_DESC_DEFAULT; 86 static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT; 87 88 #define ETHADDR_TO_UINT64(addr) __BYTES_TO_UINT64( \ 89 (addr)->addr_bytes[0], (addr)->addr_bytes[1], \ 90 (addr)->addr_bytes[2], (addr)->addr_bytes[3], \ 91 (addr)->addr_bytes[4], (addr)->addr_bytes[5], \ 92 0, 0) 93 94 #define FRAG_TBL_BUCKET_ENTRIES 4 95 #define MAX_FRAG_TTL_NS (10LL * NS_PER_S) 96 97 #define MTU_TO_FRAMELEN(x) ((x) + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN) 98 99 struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = { 100 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a) }, 101 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9) }, 102 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x08, 0x69, 0x26) }, 103 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd) } 104 }; 105 106 struct flow_info flow_info_tbl[RTE_MAX_ETHPORTS]; 107 108 #define CMD_LINE_OPT_CONFIG "config" 109 #define CMD_LINE_OPT_SINGLE_SA "single-sa" 110 #define CMD_LINE_OPT_CRYPTODEV_MASK "cryptodev_mask" 111 #define CMD_LINE_OPT_TRANSFER_MODE "transfer-mode" 112 #define CMD_LINE_OPT_SCHEDULE_TYPE "event-schedule-type" 113 #define CMD_LINE_OPT_RX_OFFLOAD "rxoffload" 114 #define CMD_LINE_OPT_TX_OFFLOAD "txoffload" 115 #define CMD_LINE_OPT_REASSEMBLE "reassemble" 116 #define CMD_LINE_OPT_MTU "mtu" 117 #define CMD_LINE_OPT_FRAG_TTL "frag-ttl" 118 119 #define CMD_LINE_ARG_EVENT "event" 120 #define CMD_LINE_ARG_POLL "poll" 121 #define CMD_LINE_ARG_ORDERED "ordered" 122 #define CMD_LINE_ARG_ATOMIC "atomic" 123 #define CMD_LINE_ARG_PARALLEL "parallel" 124 125 enum { 126 /* long options mapped to a short option */ 127 128 /* first long only option value must be >= 256, so that we won't 129 * conflict with short options 130 */ 131 CMD_LINE_OPT_MIN_NUM = 256, 132 CMD_LINE_OPT_CONFIG_NUM, 133 CMD_LINE_OPT_SINGLE_SA_NUM, 134 CMD_LINE_OPT_CRYPTODEV_MASK_NUM, 135 CMD_LINE_OPT_TRANSFER_MODE_NUM, 136 CMD_LINE_OPT_SCHEDULE_TYPE_NUM, 137 CMD_LINE_OPT_RX_OFFLOAD_NUM, 138 CMD_LINE_OPT_TX_OFFLOAD_NUM, 139 CMD_LINE_OPT_REASSEMBLE_NUM, 140 CMD_LINE_OPT_MTU_NUM, 141 CMD_LINE_OPT_FRAG_TTL_NUM, 142 }; 143 144 static const struct option lgopts[] = { 145 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, 146 {CMD_LINE_OPT_SINGLE_SA, 1, 0, CMD_LINE_OPT_SINGLE_SA_NUM}, 147 {CMD_LINE_OPT_CRYPTODEV_MASK, 1, 0, CMD_LINE_OPT_CRYPTODEV_MASK_NUM}, 148 {CMD_LINE_OPT_TRANSFER_MODE, 1, 0, CMD_LINE_OPT_TRANSFER_MODE_NUM}, 149 {CMD_LINE_OPT_SCHEDULE_TYPE, 1, 0, CMD_LINE_OPT_SCHEDULE_TYPE_NUM}, 150 {CMD_LINE_OPT_RX_OFFLOAD, 1, 0, CMD_LINE_OPT_RX_OFFLOAD_NUM}, 151 {CMD_LINE_OPT_TX_OFFLOAD, 1, 0, CMD_LINE_OPT_TX_OFFLOAD_NUM}, 152 {CMD_LINE_OPT_REASSEMBLE, 1, 0, CMD_LINE_OPT_REASSEMBLE_NUM}, 153 {CMD_LINE_OPT_MTU, 1, 0, CMD_LINE_OPT_MTU_NUM}, 154 {CMD_LINE_OPT_FRAG_TTL, 1, 0, CMD_LINE_OPT_FRAG_TTL_NUM}, 155 {NULL, 0, 0, 0} 156 }; 157 158 uint32_t unprotected_port_mask; 159 uint32_t single_sa_idx; 160 /* mask of enabled ports */ 161 static uint32_t enabled_port_mask; 162 static uint64_t enabled_cryptodev_mask = UINT64_MAX; 163 static int32_t promiscuous_on = 1; 164 static int32_t numa_on = 1; /**< NUMA is enabled by default. */ 165 static uint32_t nb_lcores; 166 static uint32_t single_sa; 167 static uint32_t nb_bufs_in_pool; 168 169 /* 170 * RX/TX HW offload capabilities to enable/use on ethernet ports. 171 * By default all capabilities are enabled. 172 */ 173 static uint64_t dev_rx_offload = UINT64_MAX; 174 static uint64_t dev_tx_offload = UINT64_MAX; 175 176 /* 177 * global values that determine multi-seg policy 178 */ 179 static uint32_t frag_tbl_sz; 180 static uint32_t frame_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE; 181 static uint32_t mtu_size = RTE_ETHER_MTU; 182 static uint64_t frag_ttl_ns = MAX_FRAG_TTL_NS; 183 184 /* application wide librte_ipsec/SA parameters */ 185 struct app_sa_prm app_sa_prm = { 186 .enable = 0, 187 .cache_sz = SA_CACHE_SZ 188 }; 189 static const char *cfgfile; 190 191 struct lcore_rx_queue { 192 uint16_t port_id; 193 uint8_t queue_id; 194 } __rte_cache_aligned; 195 196 struct lcore_params { 197 uint16_t port_id; 198 uint8_t queue_id; 199 uint8_t lcore_id; 200 } __rte_cache_aligned; 201 202 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 203 204 static struct lcore_params *lcore_params; 205 static uint16_t nb_lcore_params; 206 207 static struct rte_hash *cdev_map_in; 208 static struct rte_hash *cdev_map_out; 209 210 struct buffer { 211 uint16_t len; 212 struct rte_mbuf *m_table[MAX_PKT_BURST] __rte_aligned(sizeof(void *)); 213 }; 214 215 struct lcore_conf { 216 uint16_t nb_rx_queue; 217 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 218 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 219 struct buffer tx_mbufs[RTE_MAX_ETHPORTS]; 220 struct ipsec_ctx inbound; 221 struct ipsec_ctx outbound; 222 struct rt_ctx *rt4_ctx; 223 struct rt_ctx *rt6_ctx; 224 struct { 225 struct rte_ip_frag_tbl *tbl; 226 struct rte_mempool *pool_dir; 227 struct rte_mempool *pool_indir; 228 struct rte_ip_frag_death_row dr; 229 } frag; 230 } __rte_cache_aligned; 231 232 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 233 234 static struct rte_eth_conf port_conf = { 235 .rxmode = { 236 .mq_mode = ETH_MQ_RX_RSS, 237 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 238 .split_hdr_size = 0, 239 .offloads = DEV_RX_OFFLOAD_CHECKSUM, 240 }, 241 .rx_adv_conf = { 242 .rss_conf = { 243 .rss_key = NULL, 244 .rss_hf = ETH_RSS_IP | ETH_RSS_UDP | 245 ETH_RSS_TCP | ETH_RSS_SCTP, 246 }, 247 }, 248 .txmode = { 249 .mq_mode = ETH_MQ_TX_NONE, 250 }, 251 }; 252 253 struct socket_ctx socket_ctx[NB_SOCKETS]; 254 255 /* 256 * Determine is multi-segment support required: 257 * - either frame buffer size is smaller then mtu 258 * - or reassmeble support is requested 259 */ 260 static int 261 multi_seg_required(void) 262 { 263 return (MTU_TO_FRAMELEN(mtu_size) + RTE_PKTMBUF_HEADROOM > 264 frame_buf_size || frag_tbl_sz != 0); 265 } 266 267 static inline void 268 adjust_ipv4_pktlen(struct rte_mbuf *m, const struct rte_ipv4_hdr *iph, 269 uint32_t l2_len) 270 { 271 uint32_t plen, trim; 272 273 plen = rte_be_to_cpu_16(iph->total_length) + l2_len; 274 if (plen < m->pkt_len) { 275 trim = m->pkt_len - plen; 276 rte_pktmbuf_trim(m, trim); 277 } 278 } 279 280 static inline void 281 adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, 282 uint32_t l2_len) 283 { 284 uint32_t plen, trim; 285 286 plen = rte_be_to_cpu_16(iph->payload_len) + sizeof(*iph) + l2_len; 287 if (plen < m->pkt_len) { 288 trim = m->pkt_len - plen; 289 rte_pktmbuf_trim(m, trim); 290 } 291 } 292 293 #if (STATS_INTERVAL > 0) 294 295 /* Print out statistics on packet distribution */ 296 static void 297 print_stats_cb(__rte_unused void *param) 298 { 299 uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; 300 float burst_percent, rx_per_call, tx_per_call; 301 unsigned int coreid; 302 303 total_packets_dropped = 0; 304 total_packets_tx = 0; 305 total_packets_rx = 0; 306 307 const char clr[] = { 27, '[', '2', 'J', '\0' }; 308 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 309 310 /* Clear screen and move to top left */ 311 printf("%s%s", clr, topLeft); 312 313 printf("\nCore statistics ===================================="); 314 315 for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) { 316 /* skip disabled cores */ 317 if (rte_lcore_is_enabled(coreid) == 0) 318 continue; 319 burst_percent = (float)(core_statistics[coreid].burst_rx * 100)/ 320 core_statistics[coreid].rx; 321 rx_per_call = (float)(core_statistics[coreid].rx)/ 322 core_statistics[coreid].rx_call; 323 tx_per_call = (float)(core_statistics[coreid].tx)/ 324 core_statistics[coreid].tx_call; 325 printf("\nStatistics for core %u ------------------------------" 326 "\nPackets received: %20"PRIu64 327 "\nPackets sent: %24"PRIu64 328 "\nPackets dropped: %21"PRIu64 329 "\nBurst percent: %23.2f" 330 "\nPackets per Rx call: %17.2f" 331 "\nPackets per Tx call: %17.2f", 332 coreid, 333 core_statistics[coreid].rx, 334 core_statistics[coreid].tx, 335 core_statistics[coreid].dropped, 336 burst_percent, 337 rx_per_call, 338 tx_per_call); 339 340 total_packets_dropped += core_statistics[coreid].dropped; 341 total_packets_tx += core_statistics[coreid].tx; 342 total_packets_rx += core_statistics[coreid].rx; 343 } 344 printf("\nAggregate statistics ===============================" 345 "\nTotal packets received: %14"PRIu64 346 "\nTotal packets sent: %18"PRIu64 347 "\nTotal packets dropped: %15"PRIu64, 348 total_packets_rx, 349 total_packets_tx, 350 total_packets_dropped); 351 printf("\n====================================================\n"); 352 353 rte_eal_alarm_set(STATS_INTERVAL * US_PER_S, print_stats_cb, NULL); 354 } 355 #endif /* STATS_INTERVAL */ 356 357 static inline void 358 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t) 359 { 360 const struct rte_ether_hdr *eth; 361 const struct rte_ipv4_hdr *iph4; 362 const struct rte_ipv6_hdr *iph6; 363 364 eth = rte_pktmbuf_mtod(pkt, const struct rte_ether_hdr *); 365 if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 366 367 iph4 = (const struct rte_ipv4_hdr *)rte_pktmbuf_adj(pkt, 368 RTE_ETHER_HDR_LEN); 369 adjust_ipv4_pktlen(pkt, iph4, 0); 370 371 if (iph4->next_proto_id == IPPROTO_ESP) 372 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 373 else { 374 t->ip4.data[t->ip4.num] = &iph4->next_proto_id; 375 t->ip4.pkts[(t->ip4.num)++] = pkt; 376 } 377 pkt->l2_len = 0; 378 pkt->l3_len = sizeof(*iph4); 379 pkt->packet_type |= RTE_PTYPE_L3_IPV4; 380 } else if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 381 int next_proto; 382 size_t l3len, ext_len; 383 uint8_t *p; 384 385 /* get protocol type */ 386 iph6 = (const struct rte_ipv6_hdr *)rte_pktmbuf_adj(pkt, 387 RTE_ETHER_HDR_LEN); 388 adjust_ipv6_pktlen(pkt, iph6, 0); 389 390 next_proto = iph6->proto; 391 392 /* determine l3 header size up to ESP extension */ 393 l3len = sizeof(struct ip6_hdr); 394 p = rte_pktmbuf_mtod(pkt, uint8_t *); 395 while (next_proto != IPPROTO_ESP && l3len < pkt->data_len && 396 (next_proto = rte_ipv6_get_next_ext(p + l3len, 397 next_proto, &ext_len)) >= 0) 398 l3len += ext_len; 399 400 /* drop packet when IPv6 header exceeds first segment length */ 401 if (unlikely(l3len > pkt->data_len)) { 402 free_pkts(&pkt, 1); 403 return; 404 } 405 406 if (next_proto == IPPROTO_ESP) 407 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 408 else { 409 t->ip6.data[t->ip6.num] = &iph6->proto; 410 t->ip6.pkts[(t->ip6.num)++] = pkt; 411 } 412 pkt->l2_len = 0; 413 pkt->l3_len = l3len; 414 pkt->packet_type |= RTE_PTYPE_L3_IPV6; 415 } else { 416 /* Unknown/Unsupported type, drop the packet */ 417 RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n", 418 rte_be_to_cpu_16(eth->ether_type)); 419 free_pkts(&pkt, 1); 420 return; 421 } 422 423 /* Check if the packet has been processed inline. For inline protocol 424 * processed packets, the metadata in the mbuf can be used to identify 425 * the security processing done on the packet. The metadata will be 426 * used to retrieve the application registered userdata associated 427 * with the security session. 428 */ 429 430 if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD && 431 rte_security_dynfield_is_registered()) { 432 struct ipsec_sa *sa; 433 struct ipsec_mbuf_metadata *priv; 434 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 435 rte_eth_dev_get_sec_ctx( 436 pkt->port); 437 438 /* Retrieve the userdata registered. Here, the userdata 439 * registered is the SA pointer. 440 */ 441 sa = (struct ipsec_sa *)rte_security_get_userdata(ctx, 442 *rte_security_dynfield(pkt)); 443 if (sa == NULL) { 444 /* userdata could not be retrieved */ 445 return; 446 } 447 448 /* Save SA as priv member in mbuf. This will be used in the 449 * IPsec selector(SP-SA) check. 450 */ 451 452 priv = get_priv(pkt); 453 priv->sa = sa; 454 } 455 } 456 457 static inline void 458 prepare_traffic(struct rte_mbuf **pkts, struct ipsec_traffic *t, 459 uint16_t nb_pkts) 460 { 461 int32_t i; 462 463 t->ipsec.num = 0; 464 t->ip4.num = 0; 465 t->ip6.num = 0; 466 467 for (i = 0; i < (nb_pkts - PREFETCH_OFFSET); i++) { 468 rte_prefetch0(rte_pktmbuf_mtod(pkts[i + PREFETCH_OFFSET], 469 void *)); 470 prepare_one_packet(pkts[i], t); 471 } 472 /* Process left packets */ 473 for (; i < nb_pkts; i++) 474 prepare_one_packet(pkts[i], t); 475 } 476 477 static inline void 478 prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port, 479 const struct lcore_conf *qconf) 480 { 481 struct ip *ip; 482 struct rte_ether_hdr *ethhdr; 483 484 ip = rte_pktmbuf_mtod(pkt, struct ip *); 485 486 ethhdr = (struct rte_ether_hdr *) 487 rte_pktmbuf_prepend(pkt, RTE_ETHER_HDR_LEN); 488 489 if (ip->ip_v == IPVERSION) { 490 pkt->ol_flags |= qconf->outbound.ipv4_offloads; 491 pkt->l3_len = sizeof(struct ip); 492 pkt->l2_len = RTE_ETHER_HDR_LEN; 493 494 ip->ip_sum = 0; 495 496 /* calculate IPv4 cksum in SW */ 497 if ((pkt->ol_flags & PKT_TX_IP_CKSUM) == 0) 498 ip->ip_sum = rte_ipv4_cksum((struct rte_ipv4_hdr *)ip); 499 500 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 501 } else { 502 pkt->ol_flags |= qconf->outbound.ipv6_offloads; 503 pkt->l3_len = sizeof(struct ip6_hdr); 504 pkt->l2_len = RTE_ETHER_HDR_LEN; 505 506 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); 507 } 508 509 memcpy(ðhdr->s_addr, ðaddr_tbl[port].src, 510 sizeof(struct rte_ether_addr)); 511 memcpy(ðhdr->d_addr, ðaddr_tbl[port].dst, 512 sizeof(struct rte_ether_addr)); 513 } 514 515 static inline void 516 prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t port, 517 const struct lcore_conf *qconf) 518 { 519 int32_t i; 520 const int32_t prefetch_offset = 2; 521 522 for (i = 0; i < (nb_pkts - prefetch_offset); i++) { 523 rte_mbuf_prefetch_part2(pkts[i + prefetch_offset]); 524 prepare_tx_pkt(pkts[i], port, qconf); 525 } 526 /* Process left packets */ 527 for (; i < nb_pkts; i++) 528 prepare_tx_pkt(pkts[i], port, qconf); 529 } 530 531 /* Send burst of packets on an output interface */ 532 static inline int32_t 533 send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) 534 { 535 struct rte_mbuf **m_table; 536 int32_t ret; 537 uint16_t queueid; 538 539 queueid = qconf->tx_queue_id[port]; 540 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 541 542 prepare_tx_burst(m_table, n, port, qconf); 543 544 ret = rte_eth_tx_burst(port, queueid, m_table, n); 545 546 core_stats_update_tx(ret); 547 548 if (unlikely(ret < n)) { 549 do { 550 free_pkts(&m_table[ret], 1); 551 } while (++ret < n); 552 } 553 554 return 0; 555 } 556 557 /* 558 * Helper function to fragment and queue for TX one packet. 559 */ 560 static inline uint32_t 561 send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m, 562 uint16_t port, uint8_t proto) 563 { 564 struct buffer *tbl; 565 uint32_t len, n; 566 int32_t rc; 567 568 tbl = qconf->tx_mbufs + port; 569 len = tbl->len; 570 571 /* free space for new fragments */ 572 if (len + RTE_LIBRTE_IP_FRAG_MAX_FRAG >= RTE_DIM(tbl->m_table)) { 573 send_burst(qconf, len, port); 574 len = 0; 575 } 576 577 n = RTE_DIM(tbl->m_table) - len; 578 579 if (proto == IPPROTO_IP) 580 rc = rte_ipv4_fragment_packet(m, tbl->m_table + len, 581 n, mtu_size, qconf->frag.pool_dir, 582 qconf->frag.pool_indir); 583 else 584 rc = rte_ipv6_fragment_packet(m, tbl->m_table + len, 585 n, mtu_size, qconf->frag.pool_dir, 586 qconf->frag.pool_indir); 587 588 if (rc >= 0) 589 len += rc; 590 else 591 RTE_LOG(ERR, IPSEC, 592 "%s: failed to fragment packet with size %u, " 593 "error code: %d\n", 594 __func__, m->pkt_len, rte_errno); 595 596 free_pkts(&m, 1); 597 return len; 598 } 599 600 /* Enqueue a single packet, and send burst if queue is filled */ 601 static inline int32_t 602 send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto) 603 { 604 uint32_t lcore_id; 605 uint16_t len; 606 struct lcore_conf *qconf; 607 608 lcore_id = rte_lcore_id(); 609 610 qconf = &lcore_conf[lcore_id]; 611 len = qconf->tx_mbufs[port].len; 612 613 if (m->pkt_len <= mtu_size) { 614 qconf->tx_mbufs[port].m_table[len] = m; 615 len++; 616 617 /* need to fragment the packet */ 618 } else if (frag_tbl_sz > 0) 619 len = send_fragment_packet(qconf, m, port, proto); 620 else 621 free_pkts(&m, 1); 622 623 /* enough pkts to be sent */ 624 if (unlikely(len == MAX_PKT_BURST)) { 625 send_burst(qconf, MAX_PKT_BURST, port); 626 len = 0; 627 } 628 629 qconf->tx_mbufs[port].len = len; 630 return 0; 631 } 632 633 static inline void 634 inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip, 635 uint16_t lim) 636 { 637 struct rte_mbuf *m; 638 uint32_t i, j, res, sa_idx; 639 640 if (ip->num == 0 || sp == NULL) 641 return; 642 643 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 644 ip->num, DEFAULT_MAX_CATEGORIES); 645 646 j = 0; 647 for (i = 0; i < ip->num; i++) { 648 m = ip->pkts[i]; 649 res = ip->res[i]; 650 if (res == BYPASS) { 651 ip->pkts[j++] = m; 652 continue; 653 } 654 if (res == DISCARD) { 655 free_pkts(&m, 1); 656 continue; 657 } 658 659 /* Only check SPI match for processed IPSec packets */ 660 if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) { 661 free_pkts(&m, 1); 662 continue; 663 } 664 665 sa_idx = res - 1; 666 if (!inbound_sa_check(sa, m, sa_idx)) { 667 free_pkts(&m, 1); 668 continue; 669 } 670 ip->pkts[j++] = m; 671 } 672 ip->num = j; 673 } 674 675 static void 676 split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) 677 { 678 uint32_t i, n4, n6; 679 struct ip *ip; 680 struct rte_mbuf *m; 681 682 n4 = trf->ip4.num; 683 n6 = trf->ip6.num; 684 685 for (i = 0; i < num; i++) { 686 687 m = mb[i]; 688 ip = rte_pktmbuf_mtod(m, struct ip *); 689 690 if (ip->ip_v == IPVERSION) { 691 trf->ip4.pkts[n4] = m; 692 trf->ip4.data[n4] = rte_pktmbuf_mtod_offset(m, 693 uint8_t *, offsetof(struct ip, ip_p)); 694 n4++; 695 } else if (ip->ip_v == IP6_VERSION) { 696 trf->ip6.pkts[n6] = m; 697 trf->ip6.data[n6] = rte_pktmbuf_mtod_offset(m, 698 uint8_t *, 699 offsetof(struct ip6_hdr, ip6_nxt)); 700 n6++; 701 } else 702 free_pkts(&m, 1); 703 } 704 705 trf->ip4.num = n4; 706 trf->ip6.num = n6; 707 } 708 709 710 static inline void 711 process_pkts_inbound(struct ipsec_ctx *ipsec_ctx, 712 struct ipsec_traffic *traffic) 713 { 714 uint16_t nb_pkts_in, n_ip4, n_ip6; 715 716 n_ip4 = traffic->ip4.num; 717 n_ip6 = traffic->ip6.num; 718 719 if (app_sa_prm.enable == 0) { 720 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 721 traffic->ipsec.num, MAX_PKT_BURST); 722 split46_traffic(traffic, traffic->ipsec.pkts, nb_pkts_in); 723 } else { 724 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 725 traffic->ipsec.saptr, traffic->ipsec.num); 726 ipsec_process(ipsec_ctx, traffic); 727 } 728 729 inbound_sp_sa(ipsec_ctx->sp4_ctx, ipsec_ctx->sa_ctx, &traffic->ip4, 730 n_ip4); 731 732 inbound_sp_sa(ipsec_ctx->sp6_ctx, ipsec_ctx->sa_ctx, &traffic->ip6, 733 n_ip6); 734 } 735 736 static inline void 737 outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, 738 struct traffic_type *ipsec) 739 { 740 struct rte_mbuf *m; 741 uint32_t i, j, sa_idx; 742 743 if (ip->num == 0 || sp == NULL) 744 return; 745 746 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 747 ip->num, DEFAULT_MAX_CATEGORIES); 748 749 j = 0; 750 for (i = 0; i < ip->num; i++) { 751 m = ip->pkts[i]; 752 sa_idx = ip->res[i] - 1; 753 if (ip->res[i] == DISCARD) 754 free_pkts(&m, 1); 755 else if (ip->res[i] == BYPASS) 756 ip->pkts[j++] = m; 757 else { 758 ipsec->res[ipsec->num] = sa_idx; 759 ipsec->pkts[ipsec->num++] = m; 760 } 761 } 762 ip->num = j; 763 } 764 765 static inline void 766 process_pkts_outbound(struct ipsec_ctx *ipsec_ctx, 767 struct ipsec_traffic *traffic) 768 { 769 struct rte_mbuf *m; 770 uint16_t idx, nb_pkts_out, i; 771 772 /* Drop any IPsec traffic from protected ports */ 773 free_pkts(traffic->ipsec.pkts, traffic->ipsec.num); 774 775 traffic->ipsec.num = 0; 776 777 outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec); 778 779 outbound_sp(ipsec_ctx->sp6_ctx, &traffic->ip6, &traffic->ipsec); 780 781 if (app_sa_prm.enable == 0) { 782 783 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 784 traffic->ipsec.res, traffic->ipsec.num, 785 MAX_PKT_BURST); 786 787 for (i = 0; i < nb_pkts_out; i++) { 788 m = traffic->ipsec.pkts[i]; 789 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 790 if (ip->ip_v == IPVERSION) { 791 idx = traffic->ip4.num++; 792 traffic->ip4.pkts[idx] = m; 793 } else { 794 idx = traffic->ip6.num++; 795 traffic->ip6.pkts[idx] = m; 796 } 797 } 798 } else { 799 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 800 traffic->ipsec.saptr, traffic->ipsec.num); 801 ipsec_process(ipsec_ctx, traffic); 802 } 803 } 804 805 static inline void 806 process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx, 807 struct ipsec_traffic *traffic) 808 { 809 struct rte_mbuf *m; 810 uint32_t nb_pkts_in, i, idx; 811 812 /* Drop any IPv4 traffic from unprotected ports */ 813 free_pkts(traffic->ip4.pkts, traffic->ip4.num); 814 815 traffic->ip4.num = 0; 816 817 /* Drop any IPv6 traffic from unprotected ports */ 818 free_pkts(traffic->ip6.pkts, traffic->ip6.num); 819 820 traffic->ip6.num = 0; 821 822 if (app_sa_prm.enable == 0) { 823 824 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 825 traffic->ipsec.num, MAX_PKT_BURST); 826 827 for (i = 0; i < nb_pkts_in; i++) { 828 m = traffic->ipsec.pkts[i]; 829 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 830 if (ip->ip_v == IPVERSION) { 831 idx = traffic->ip4.num++; 832 traffic->ip4.pkts[idx] = m; 833 } else { 834 idx = traffic->ip6.num++; 835 traffic->ip6.pkts[idx] = m; 836 } 837 } 838 } else { 839 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 840 traffic->ipsec.saptr, traffic->ipsec.num); 841 ipsec_process(ipsec_ctx, traffic); 842 } 843 } 844 845 static inline void 846 process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx, 847 struct ipsec_traffic *traffic) 848 { 849 struct rte_mbuf *m; 850 uint32_t nb_pkts_out, i, n; 851 struct ip *ip; 852 853 /* Drop any IPsec traffic from protected ports */ 854 free_pkts(traffic->ipsec.pkts, traffic->ipsec.num); 855 856 n = 0; 857 858 for (i = 0; i < traffic->ip4.num; i++) { 859 traffic->ipsec.pkts[n] = traffic->ip4.pkts[i]; 860 traffic->ipsec.res[n++] = single_sa_idx; 861 } 862 863 for (i = 0; i < traffic->ip6.num; i++) { 864 traffic->ipsec.pkts[n] = traffic->ip6.pkts[i]; 865 traffic->ipsec.res[n++] = single_sa_idx; 866 } 867 868 traffic->ip4.num = 0; 869 traffic->ip6.num = 0; 870 traffic->ipsec.num = n; 871 872 if (app_sa_prm.enable == 0) { 873 874 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 875 traffic->ipsec.res, traffic->ipsec.num, 876 MAX_PKT_BURST); 877 878 /* They all sue the same SA (ip4 or ip6 tunnel) */ 879 m = traffic->ipsec.pkts[0]; 880 ip = rte_pktmbuf_mtod(m, struct ip *); 881 if (ip->ip_v == IPVERSION) { 882 traffic->ip4.num = nb_pkts_out; 883 for (i = 0; i < nb_pkts_out; i++) 884 traffic->ip4.pkts[i] = traffic->ipsec.pkts[i]; 885 } else { 886 traffic->ip6.num = nb_pkts_out; 887 for (i = 0; i < nb_pkts_out; i++) 888 traffic->ip6.pkts[i] = traffic->ipsec.pkts[i]; 889 } 890 } else { 891 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 892 traffic->ipsec.saptr, traffic->ipsec.num); 893 ipsec_process(ipsec_ctx, traffic); 894 } 895 } 896 897 static inline int32_t 898 get_hop_for_offload_pkt(struct rte_mbuf *pkt, int is_ipv6) 899 { 900 struct ipsec_mbuf_metadata *priv; 901 struct ipsec_sa *sa; 902 903 priv = get_priv(pkt); 904 905 sa = priv->sa; 906 if (unlikely(sa == NULL)) { 907 RTE_LOG(ERR, IPSEC, "SA not saved in private data\n"); 908 goto fail; 909 } 910 911 if (is_ipv6) 912 return sa->portid; 913 914 /* else */ 915 return (sa->portid | RTE_LPM_LOOKUP_SUCCESS); 916 917 fail: 918 if (is_ipv6) 919 return -1; 920 921 /* else */ 922 return 0; 923 } 924 925 static inline void 926 route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 927 { 928 uint32_t hop[MAX_PKT_BURST * 2]; 929 uint32_t dst_ip[MAX_PKT_BURST * 2]; 930 int32_t pkt_hop = 0; 931 uint16_t i, offset; 932 uint16_t lpm_pkts = 0; 933 934 if (nb_pkts == 0) 935 return; 936 937 /* Need to do an LPM lookup for non-inline packets. Inline packets will 938 * have port ID in the SA 939 */ 940 941 for (i = 0; i < nb_pkts; i++) { 942 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 943 /* Security offload not enabled. So an LPM lookup is 944 * required to get the hop 945 */ 946 offset = offsetof(struct ip, ip_dst); 947 dst_ip[lpm_pkts] = *rte_pktmbuf_mtod_offset(pkts[i], 948 uint32_t *, offset); 949 dst_ip[lpm_pkts] = rte_be_to_cpu_32(dst_ip[lpm_pkts]); 950 lpm_pkts++; 951 } 952 } 953 954 rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, lpm_pkts); 955 956 lpm_pkts = 0; 957 958 for (i = 0; i < nb_pkts; i++) { 959 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 960 /* Read hop from the SA */ 961 pkt_hop = get_hop_for_offload_pkt(pkts[i], 0); 962 } else { 963 /* Need to use hop returned by lookup */ 964 pkt_hop = hop[lpm_pkts++]; 965 } 966 967 if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) { 968 free_pkts(&pkts[i], 1); 969 continue; 970 } 971 send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IP); 972 } 973 } 974 975 static inline void 976 route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 977 { 978 int32_t hop[MAX_PKT_BURST * 2]; 979 uint8_t dst_ip[MAX_PKT_BURST * 2][16]; 980 uint8_t *ip6_dst; 981 int32_t pkt_hop = 0; 982 uint16_t i, offset; 983 uint16_t lpm_pkts = 0; 984 985 if (nb_pkts == 0) 986 return; 987 988 /* Need to do an LPM lookup for non-inline packets. Inline packets will 989 * have port ID in the SA 990 */ 991 992 for (i = 0; i < nb_pkts; i++) { 993 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 994 /* Security offload not enabled. So an LPM lookup is 995 * required to get the hop 996 */ 997 offset = offsetof(struct ip6_hdr, ip6_dst); 998 ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *, 999 offset); 1000 memcpy(&dst_ip[lpm_pkts][0], ip6_dst, 16); 1001 lpm_pkts++; 1002 } 1003 } 1004 1005 rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, hop, 1006 lpm_pkts); 1007 1008 lpm_pkts = 0; 1009 1010 for (i = 0; i < nb_pkts; i++) { 1011 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 1012 /* Read hop from the SA */ 1013 pkt_hop = get_hop_for_offload_pkt(pkts[i], 1); 1014 } else { 1015 /* Need to use hop returned by lookup */ 1016 pkt_hop = hop[lpm_pkts++]; 1017 } 1018 1019 if (pkt_hop == -1) { 1020 free_pkts(&pkts[i], 1); 1021 continue; 1022 } 1023 send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IPV6); 1024 } 1025 } 1026 1027 static inline void 1028 process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts, 1029 uint8_t nb_pkts, uint16_t portid) 1030 { 1031 struct ipsec_traffic traffic; 1032 1033 prepare_traffic(pkts, &traffic, nb_pkts); 1034 1035 if (unlikely(single_sa)) { 1036 if (is_unprotected_port(portid)) 1037 process_pkts_inbound_nosp(&qconf->inbound, &traffic); 1038 else 1039 process_pkts_outbound_nosp(&qconf->outbound, &traffic); 1040 } else { 1041 if (is_unprotected_port(portid)) 1042 process_pkts_inbound(&qconf->inbound, &traffic); 1043 else 1044 process_pkts_outbound(&qconf->outbound, &traffic); 1045 } 1046 1047 route4_pkts(qconf->rt4_ctx, traffic.ip4.pkts, traffic.ip4.num); 1048 route6_pkts(qconf->rt6_ctx, traffic.ip6.pkts, traffic.ip6.num); 1049 } 1050 1051 static inline void 1052 drain_tx_buffers(struct lcore_conf *qconf) 1053 { 1054 struct buffer *buf; 1055 uint32_t portid; 1056 1057 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 1058 buf = &qconf->tx_mbufs[portid]; 1059 if (buf->len == 0) 1060 continue; 1061 send_burst(qconf, buf->len, portid); 1062 buf->len = 0; 1063 } 1064 } 1065 1066 static inline void 1067 drain_crypto_buffers(struct lcore_conf *qconf) 1068 { 1069 uint32_t i; 1070 struct ipsec_ctx *ctx; 1071 1072 /* drain inbound buffers*/ 1073 ctx = &qconf->inbound; 1074 for (i = 0; i != ctx->nb_qps; i++) { 1075 if (ctx->tbl[i].len != 0) 1076 enqueue_cop_burst(ctx->tbl + i); 1077 } 1078 1079 /* drain outbound buffers*/ 1080 ctx = &qconf->outbound; 1081 for (i = 0; i != ctx->nb_qps; i++) { 1082 if (ctx->tbl[i].len != 0) 1083 enqueue_cop_burst(ctx->tbl + i); 1084 } 1085 } 1086 1087 static void 1088 drain_inbound_crypto_queues(const struct lcore_conf *qconf, 1089 struct ipsec_ctx *ctx) 1090 { 1091 uint32_t n; 1092 struct ipsec_traffic trf; 1093 1094 if (app_sa_prm.enable == 0) { 1095 1096 /* dequeue packets from crypto-queue */ 1097 n = ipsec_inbound_cqp_dequeue(ctx, trf.ipsec.pkts, 1098 RTE_DIM(trf.ipsec.pkts)); 1099 1100 trf.ip4.num = 0; 1101 trf.ip6.num = 0; 1102 1103 /* split traffic by ipv4-ipv6 */ 1104 split46_traffic(&trf, trf.ipsec.pkts, n); 1105 } else 1106 ipsec_cqp_process(ctx, &trf); 1107 1108 /* process ipv4 packets */ 1109 if (trf.ip4.num != 0) { 1110 inbound_sp_sa(ctx->sp4_ctx, ctx->sa_ctx, &trf.ip4, 0); 1111 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 1112 } 1113 1114 /* process ipv6 packets */ 1115 if (trf.ip6.num != 0) { 1116 inbound_sp_sa(ctx->sp6_ctx, ctx->sa_ctx, &trf.ip6, 0); 1117 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 1118 } 1119 } 1120 1121 static void 1122 drain_outbound_crypto_queues(const struct lcore_conf *qconf, 1123 struct ipsec_ctx *ctx) 1124 { 1125 uint32_t n; 1126 struct ipsec_traffic trf; 1127 1128 if (app_sa_prm.enable == 0) { 1129 1130 /* dequeue packets from crypto-queue */ 1131 n = ipsec_outbound_cqp_dequeue(ctx, trf.ipsec.pkts, 1132 RTE_DIM(trf.ipsec.pkts)); 1133 1134 trf.ip4.num = 0; 1135 trf.ip6.num = 0; 1136 1137 /* split traffic by ipv4-ipv6 */ 1138 split46_traffic(&trf, trf.ipsec.pkts, n); 1139 } else 1140 ipsec_cqp_process(ctx, &trf); 1141 1142 /* process ipv4 packets */ 1143 if (trf.ip4.num != 0) 1144 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 1145 1146 /* process ipv6 packets */ 1147 if (trf.ip6.num != 0) 1148 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 1149 } 1150 1151 /* main processing loop */ 1152 void 1153 ipsec_poll_mode_worker(void) 1154 { 1155 struct rte_mbuf *pkts[MAX_PKT_BURST]; 1156 uint32_t lcore_id; 1157 uint64_t prev_tsc, diff_tsc, cur_tsc; 1158 int32_t i, nb_rx; 1159 uint16_t portid; 1160 uint8_t queueid; 1161 struct lcore_conf *qconf; 1162 int32_t rc, socket_id; 1163 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) 1164 / US_PER_S * BURST_TX_DRAIN_US; 1165 struct lcore_rx_queue *rxql; 1166 1167 prev_tsc = 0; 1168 lcore_id = rte_lcore_id(); 1169 qconf = &lcore_conf[lcore_id]; 1170 rxql = qconf->rx_queue_list; 1171 socket_id = rte_lcore_to_socket_id(lcore_id); 1172 1173 qconf->rt4_ctx = socket_ctx[socket_id].rt_ip4; 1174 qconf->rt6_ctx = socket_ctx[socket_id].rt_ip6; 1175 qconf->inbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_in; 1176 qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in; 1177 qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in; 1178 qconf->inbound.cdev_map = cdev_map_in; 1179 qconf->inbound.session_pool = socket_ctx[socket_id].session_pool; 1180 qconf->inbound.session_priv_pool = 1181 socket_ctx[socket_id].session_priv_pool; 1182 qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out; 1183 qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out; 1184 qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out; 1185 qconf->outbound.cdev_map = cdev_map_out; 1186 qconf->outbound.session_pool = socket_ctx[socket_id].session_pool; 1187 qconf->outbound.session_priv_pool = 1188 socket_ctx[socket_id].session_priv_pool; 1189 qconf->frag.pool_dir = socket_ctx[socket_id].mbuf_pool; 1190 qconf->frag.pool_indir = socket_ctx[socket_id].mbuf_pool_indir; 1191 1192 rc = ipsec_sad_lcore_cache_init(app_sa_prm.cache_sz); 1193 if (rc != 0) { 1194 RTE_LOG(ERR, IPSEC, 1195 "SAD cache init on lcore %u, failed with code: %d\n", 1196 lcore_id, rc); 1197 return; 1198 } 1199 1200 if (qconf->nb_rx_queue == 0) { 1201 RTE_LOG(DEBUG, IPSEC, "lcore %u has nothing to do\n", 1202 lcore_id); 1203 return; 1204 } 1205 1206 RTE_LOG(INFO, IPSEC, "entering main loop on lcore %u\n", lcore_id); 1207 1208 for (i = 0; i < qconf->nb_rx_queue; i++) { 1209 portid = rxql[i].port_id; 1210 queueid = rxql[i].queue_id; 1211 RTE_LOG(INFO, IPSEC, 1212 " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", 1213 lcore_id, portid, queueid); 1214 } 1215 1216 while (!force_quit) { 1217 cur_tsc = rte_rdtsc(); 1218 1219 /* TX queue buffer drain */ 1220 diff_tsc = cur_tsc - prev_tsc; 1221 1222 if (unlikely(diff_tsc > drain_tsc)) { 1223 drain_tx_buffers(qconf); 1224 drain_crypto_buffers(qconf); 1225 prev_tsc = cur_tsc; 1226 } 1227 1228 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1229 1230 /* Read packets from RX queues */ 1231 portid = rxql[i].port_id; 1232 queueid = rxql[i].queue_id; 1233 nb_rx = rte_eth_rx_burst(portid, queueid, 1234 pkts, MAX_PKT_BURST); 1235 1236 if (nb_rx > 0) { 1237 core_stats_update_rx(nb_rx); 1238 process_pkts(qconf, pkts, nb_rx, portid); 1239 } 1240 1241 /* dequeue and process completed crypto-ops */ 1242 if (is_unprotected_port(portid)) 1243 drain_inbound_crypto_queues(qconf, 1244 &qconf->inbound); 1245 else 1246 drain_outbound_crypto_queues(qconf, 1247 &qconf->outbound); 1248 } 1249 } 1250 } 1251 1252 int 1253 check_flow_params(uint16_t fdir_portid, uint8_t fdir_qid) 1254 { 1255 uint16_t i; 1256 uint16_t portid; 1257 uint8_t queueid; 1258 1259 for (i = 0; i < nb_lcore_params; ++i) { 1260 portid = lcore_params_array[i].port_id; 1261 if (portid == fdir_portid) { 1262 queueid = lcore_params_array[i].queue_id; 1263 if (queueid == fdir_qid) 1264 break; 1265 } 1266 1267 if (i == nb_lcore_params - 1) 1268 return -1; 1269 } 1270 1271 return 1; 1272 } 1273 1274 static int32_t 1275 check_poll_mode_params(struct eh_conf *eh_conf) 1276 { 1277 uint8_t lcore; 1278 uint16_t portid; 1279 uint16_t i; 1280 int32_t socket_id; 1281 1282 if (!eh_conf) 1283 return -EINVAL; 1284 1285 if (eh_conf->mode != EH_PKT_TRANSFER_MODE_POLL) 1286 return 0; 1287 1288 if (lcore_params == NULL) { 1289 printf("Error: No port/queue/core mappings\n"); 1290 return -1; 1291 } 1292 1293 for (i = 0; i < nb_lcore_params; ++i) { 1294 lcore = lcore_params[i].lcore_id; 1295 if (!rte_lcore_is_enabled(lcore)) { 1296 printf("error: lcore %hhu is not enabled in " 1297 "lcore mask\n", lcore); 1298 return -1; 1299 } 1300 socket_id = rte_lcore_to_socket_id(lcore); 1301 if (socket_id != 0 && numa_on == 0) { 1302 printf("warning: lcore %hhu is on socket %d " 1303 "with numa off\n", 1304 lcore, socket_id); 1305 } 1306 portid = lcore_params[i].port_id; 1307 if ((enabled_port_mask & (1 << portid)) == 0) { 1308 printf("port %u is not enabled in port mask\n", portid); 1309 return -1; 1310 } 1311 if (!rte_eth_dev_is_valid_port(portid)) { 1312 printf("port %u is not present on the board\n", portid); 1313 return -1; 1314 } 1315 } 1316 return 0; 1317 } 1318 1319 static uint8_t 1320 get_port_nb_rx_queues(const uint16_t port) 1321 { 1322 int32_t queue = -1; 1323 uint16_t i; 1324 1325 for (i = 0; i < nb_lcore_params; ++i) { 1326 if (lcore_params[i].port_id == port && 1327 lcore_params[i].queue_id > queue) 1328 queue = lcore_params[i].queue_id; 1329 } 1330 return (uint8_t)(++queue); 1331 } 1332 1333 static int32_t 1334 init_lcore_rx_queues(void) 1335 { 1336 uint16_t i, nb_rx_queue; 1337 uint8_t lcore; 1338 1339 for (i = 0; i < nb_lcore_params; ++i) { 1340 lcore = lcore_params[i].lcore_id; 1341 nb_rx_queue = lcore_conf[lcore].nb_rx_queue; 1342 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 1343 printf("error: too many queues (%u) for lcore: %u\n", 1344 nb_rx_queue + 1, lcore); 1345 return -1; 1346 } 1347 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 1348 lcore_params[i].port_id; 1349 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 1350 lcore_params[i].queue_id; 1351 lcore_conf[lcore].nb_rx_queue++; 1352 } 1353 return 0; 1354 } 1355 1356 /* display usage */ 1357 static void 1358 print_usage(const char *prgname) 1359 { 1360 fprintf(stderr, "%s [EAL options] --" 1361 " -p PORTMASK" 1362 " [-P]" 1363 " [-u PORTMASK]" 1364 " [-j FRAMESIZE]" 1365 " [-l]" 1366 " [-w REPLAY_WINDOW_SIZE]" 1367 " [-e]" 1368 " [-a]" 1369 " [-c]" 1370 " [-s NUMBER_OF_MBUFS_IN_PKT_POOL]" 1371 " -f CONFIG_FILE" 1372 " --config (port,queue,lcore)[,(port,queue,lcore)]" 1373 " [--single-sa SAIDX]" 1374 " [--cryptodev_mask MASK]" 1375 " [--transfer-mode MODE]" 1376 " [--event-schedule-type TYPE]" 1377 " [--" CMD_LINE_OPT_RX_OFFLOAD " RX_OFFLOAD_MASK]" 1378 " [--" CMD_LINE_OPT_TX_OFFLOAD " TX_OFFLOAD_MASK]" 1379 " [--" CMD_LINE_OPT_REASSEMBLE " REASSEMBLE_TABLE_SIZE]" 1380 " [--" CMD_LINE_OPT_MTU " MTU]" 1381 "\n\n" 1382 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n" 1383 " -P : Enable promiscuous mode\n" 1384 " -u PORTMASK: Hexadecimal bitmask of unprotected ports\n" 1385 " -j FRAMESIZE: Data buffer size, minimum (and default)\n" 1386 " value: RTE_MBUF_DEFAULT_BUF_SIZE\n" 1387 " -l enables code-path that uses librte_ipsec\n" 1388 " -w REPLAY_WINDOW_SIZE specifies IPsec SQN replay window\n" 1389 " size for each SA\n" 1390 " -e enables ESN\n" 1391 " -a enables SA SQN atomic behaviour\n" 1392 " -c specifies inbound SAD cache size,\n" 1393 " zero value disables the cache (default value: 128)\n" 1394 " -s number of mbufs in packet pool, if not specified number\n" 1395 " of mbufs will be calculated based on number of cores,\n" 1396 " ports and crypto queues\n" 1397 " -f CONFIG_FILE: Configuration file\n" 1398 " --config (port,queue,lcore): Rx queue configuration. In poll\n" 1399 " mode determines which queues from\n" 1400 " which ports are mapped to which cores.\n" 1401 " In event mode this option is not used\n" 1402 " as packets are dynamically scheduled\n" 1403 " to cores by HW.\n" 1404 " --single-sa SAIDX: In poll mode use single SA index for\n" 1405 " outbound traffic, bypassing the SP\n" 1406 " In event mode selects driver submode,\n" 1407 " SA index value is ignored\n" 1408 " --cryptodev_mask MASK: Hexadecimal bitmask of the crypto\n" 1409 " devices to configure\n" 1410 " --transfer-mode MODE\n" 1411 " \"poll\" : Packet transfer via polling (default)\n" 1412 " \"event\" : Packet transfer via event device\n" 1413 " --event-schedule-type TYPE queue schedule type, used only when\n" 1414 " transfer mode is set to event\n" 1415 " \"ordered\" : Ordered (default)\n" 1416 " \"atomic\" : Atomic\n" 1417 " \"parallel\" : Parallel\n" 1418 " --" CMD_LINE_OPT_RX_OFFLOAD 1419 ": bitmask of the RX HW offload capabilities to enable/use\n" 1420 " (DEV_RX_OFFLOAD_*)\n" 1421 " --" CMD_LINE_OPT_TX_OFFLOAD 1422 ": bitmask of the TX HW offload capabilities to enable/use\n" 1423 " (DEV_TX_OFFLOAD_*)\n" 1424 " --" CMD_LINE_OPT_REASSEMBLE " NUM" 1425 ": max number of entries in reassemble(fragment) table\n" 1426 " (zero (default value) disables reassembly)\n" 1427 " --" CMD_LINE_OPT_MTU " MTU" 1428 ": MTU value on all ports (default value: 1500)\n" 1429 " outgoing packets with bigger size will be fragmented\n" 1430 " incoming packets with bigger size will be discarded\n" 1431 " --" CMD_LINE_OPT_FRAG_TTL " FRAG_TTL_NS" 1432 ": fragments lifetime in nanoseconds, default\n" 1433 " and maximum value is 10.000.000.000 ns (10 s)\n" 1434 "\n", 1435 prgname); 1436 } 1437 1438 static int 1439 parse_mask(const char *str, uint64_t *val) 1440 { 1441 char *end; 1442 unsigned long t; 1443 1444 errno = 0; 1445 t = strtoul(str, &end, 0); 1446 if (errno != 0 || end[0] != 0) 1447 return -EINVAL; 1448 1449 *val = t; 1450 return 0; 1451 } 1452 1453 static int32_t 1454 parse_portmask(const char *portmask) 1455 { 1456 char *end = NULL; 1457 unsigned long pm; 1458 1459 /* parse hexadecimal string */ 1460 pm = strtoul(portmask, &end, 16); 1461 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 1462 return -1; 1463 1464 if ((pm == 0) && errno) 1465 return -1; 1466 1467 return pm; 1468 } 1469 1470 static int64_t 1471 parse_decimal(const char *str) 1472 { 1473 char *end = NULL; 1474 uint64_t num; 1475 1476 num = strtoull(str, &end, 10); 1477 if ((str[0] == '\0') || (end == NULL) || (*end != '\0') 1478 || num > INT64_MAX) 1479 return -1; 1480 1481 return num; 1482 } 1483 1484 static int32_t 1485 parse_config(const char *q_arg) 1486 { 1487 char s[256]; 1488 const char *p, *p0 = q_arg; 1489 char *end; 1490 enum fieldnames { 1491 FLD_PORT = 0, 1492 FLD_QUEUE, 1493 FLD_LCORE, 1494 _NUM_FLD 1495 }; 1496 unsigned long int_fld[_NUM_FLD]; 1497 char *str_fld[_NUM_FLD]; 1498 int32_t i; 1499 uint32_t size; 1500 1501 nb_lcore_params = 0; 1502 1503 while ((p = strchr(p0, '(')) != NULL) { 1504 ++p; 1505 p0 = strchr(p, ')'); 1506 if (p0 == NULL) 1507 return -1; 1508 1509 size = p0 - p; 1510 if (size >= sizeof(s)) 1511 return -1; 1512 1513 snprintf(s, sizeof(s), "%.*s", size, p); 1514 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 1515 _NUM_FLD) 1516 return -1; 1517 for (i = 0; i < _NUM_FLD; i++) { 1518 errno = 0; 1519 int_fld[i] = strtoul(str_fld[i], &end, 0); 1520 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 1521 return -1; 1522 } 1523 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 1524 printf("exceeded max number of lcore params: %hu\n", 1525 nb_lcore_params); 1526 return -1; 1527 } 1528 lcore_params_array[nb_lcore_params].port_id = 1529 (uint8_t)int_fld[FLD_PORT]; 1530 lcore_params_array[nb_lcore_params].queue_id = 1531 (uint8_t)int_fld[FLD_QUEUE]; 1532 lcore_params_array[nb_lcore_params].lcore_id = 1533 (uint8_t)int_fld[FLD_LCORE]; 1534 ++nb_lcore_params; 1535 } 1536 lcore_params = lcore_params_array; 1537 return 0; 1538 } 1539 1540 static void 1541 print_app_sa_prm(const struct app_sa_prm *prm) 1542 { 1543 printf("librte_ipsec usage: %s\n", 1544 (prm->enable == 0) ? "disabled" : "enabled"); 1545 1546 printf("replay window size: %u\n", prm->window_size); 1547 printf("ESN: %s\n", (prm->enable_esn == 0) ? "disabled" : "enabled"); 1548 printf("SA flags: %#" PRIx64 "\n", prm->flags); 1549 printf("Frag TTL: %" PRIu64 " ns\n", frag_ttl_ns); 1550 } 1551 1552 static int 1553 parse_transfer_mode(struct eh_conf *conf, const char *optarg) 1554 { 1555 if (!strcmp(CMD_LINE_ARG_POLL, optarg)) 1556 conf->mode = EH_PKT_TRANSFER_MODE_POLL; 1557 else if (!strcmp(CMD_LINE_ARG_EVENT, optarg)) 1558 conf->mode = EH_PKT_TRANSFER_MODE_EVENT; 1559 else { 1560 printf("Unsupported packet transfer mode\n"); 1561 return -EINVAL; 1562 } 1563 1564 return 0; 1565 } 1566 1567 static int 1568 parse_schedule_type(struct eh_conf *conf, const char *optarg) 1569 { 1570 struct eventmode_conf *em_conf = NULL; 1571 1572 /* Get eventmode conf */ 1573 em_conf = conf->mode_params; 1574 1575 if (!strcmp(CMD_LINE_ARG_ORDERED, optarg)) 1576 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ORDERED; 1577 else if (!strcmp(CMD_LINE_ARG_ATOMIC, optarg)) 1578 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ATOMIC; 1579 else if (!strcmp(CMD_LINE_ARG_PARALLEL, optarg)) 1580 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_PARALLEL; 1581 else { 1582 printf("Unsupported queue schedule type\n"); 1583 return -EINVAL; 1584 } 1585 1586 return 0; 1587 } 1588 1589 static int32_t 1590 parse_args(int32_t argc, char **argv, struct eh_conf *eh_conf) 1591 { 1592 int opt; 1593 int64_t ret; 1594 char **argvopt; 1595 int32_t option_index; 1596 char *prgname = argv[0]; 1597 int32_t f_present = 0; 1598 1599 argvopt = argv; 1600 1601 while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:c:s:", 1602 lgopts, &option_index)) != EOF) { 1603 1604 switch (opt) { 1605 case 'p': 1606 enabled_port_mask = parse_portmask(optarg); 1607 if (enabled_port_mask == 0) { 1608 printf("invalid portmask\n"); 1609 print_usage(prgname); 1610 return -1; 1611 } 1612 break; 1613 case 'P': 1614 printf("Promiscuous mode selected\n"); 1615 promiscuous_on = 1; 1616 break; 1617 case 'u': 1618 unprotected_port_mask = parse_portmask(optarg); 1619 if (unprotected_port_mask == 0) { 1620 printf("invalid unprotected portmask\n"); 1621 print_usage(prgname); 1622 return -1; 1623 } 1624 break; 1625 case 'f': 1626 if (f_present == 1) { 1627 printf("\"-f\" option present more than " 1628 "once!\n"); 1629 print_usage(prgname); 1630 return -1; 1631 } 1632 cfgfile = optarg; 1633 f_present = 1; 1634 break; 1635 1636 case 's': 1637 ret = parse_decimal(optarg); 1638 if (ret < 0) { 1639 printf("Invalid number of buffers in a pool: " 1640 "%s\n", optarg); 1641 print_usage(prgname); 1642 return -1; 1643 } 1644 1645 nb_bufs_in_pool = ret; 1646 break; 1647 1648 case 'j': 1649 ret = parse_decimal(optarg); 1650 if (ret < RTE_MBUF_DEFAULT_BUF_SIZE || 1651 ret > UINT16_MAX) { 1652 printf("Invalid frame buffer size value: %s\n", 1653 optarg); 1654 print_usage(prgname); 1655 return -1; 1656 } 1657 frame_buf_size = ret; 1658 printf("Custom frame buffer size %u\n", frame_buf_size); 1659 break; 1660 case 'l': 1661 app_sa_prm.enable = 1; 1662 break; 1663 case 'w': 1664 app_sa_prm.window_size = parse_decimal(optarg); 1665 break; 1666 case 'e': 1667 app_sa_prm.enable_esn = 1; 1668 break; 1669 case 'a': 1670 app_sa_prm.enable = 1; 1671 app_sa_prm.flags |= RTE_IPSEC_SAFLAG_SQN_ATOM; 1672 break; 1673 case 'c': 1674 ret = parse_decimal(optarg); 1675 if (ret < 0) { 1676 printf("Invalid SA cache size: %s\n", optarg); 1677 print_usage(prgname); 1678 return -1; 1679 } 1680 app_sa_prm.cache_sz = ret; 1681 break; 1682 case CMD_LINE_OPT_CONFIG_NUM: 1683 ret = parse_config(optarg); 1684 if (ret) { 1685 printf("Invalid config\n"); 1686 print_usage(prgname); 1687 return -1; 1688 } 1689 break; 1690 case CMD_LINE_OPT_SINGLE_SA_NUM: 1691 ret = parse_decimal(optarg); 1692 if (ret == -1 || ret > UINT32_MAX) { 1693 printf("Invalid argument[sa_idx]\n"); 1694 print_usage(prgname); 1695 return -1; 1696 } 1697 1698 /* else */ 1699 single_sa = 1; 1700 single_sa_idx = ret; 1701 eh_conf->ipsec_mode = EH_IPSEC_MODE_TYPE_DRIVER; 1702 printf("Configured with single SA index %u\n", 1703 single_sa_idx); 1704 break; 1705 case CMD_LINE_OPT_CRYPTODEV_MASK_NUM: 1706 ret = parse_portmask(optarg); 1707 if (ret == -1) { 1708 printf("Invalid argument[portmask]\n"); 1709 print_usage(prgname); 1710 return -1; 1711 } 1712 1713 /* else */ 1714 enabled_cryptodev_mask = ret; 1715 break; 1716 1717 case CMD_LINE_OPT_TRANSFER_MODE_NUM: 1718 ret = parse_transfer_mode(eh_conf, optarg); 1719 if (ret < 0) { 1720 printf("Invalid packet transfer mode\n"); 1721 print_usage(prgname); 1722 return -1; 1723 } 1724 break; 1725 1726 case CMD_LINE_OPT_SCHEDULE_TYPE_NUM: 1727 ret = parse_schedule_type(eh_conf, optarg); 1728 if (ret < 0) { 1729 printf("Invalid queue schedule type\n"); 1730 print_usage(prgname); 1731 return -1; 1732 } 1733 break; 1734 1735 case CMD_LINE_OPT_RX_OFFLOAD_NUM: 1736 ret = parse_mask(optarg, &dev_rx_offload); 1737 if (ret != 0) { 1738 printf("Invalid argument for \'%s\': %s\n", 1739 CMD_LINE_OPT_RX_OFFLOAD, optarg); 1740 print_usage(prgname); 1741 return -1; 1742 } 1743 break; 1744 case CMD_LINE_OPT_TX_OFFLOAD_NUM: 1745 ret = parse_mask(optarg, &dev_tx_offload); 1746 if (ret != 0) { 1747 printf("Invalid argument for \'%s\': %s\n", 1748 CMD_LINE_OPT_TX_OFFLOAD, optarg); 1749 print_usage(prgname); 1750 return -1; 1751 } 1752 break; 1753 case CMD_LINE_OPT_REASSEMBLE_NUM: 1754 ret = parse_decimal(optarg); 1755 if (ret < 0 || ret > UINT32_MAX) { 1756 printf("Invalid argument for \'%s\': %s\n", 1757 CMD_LINE_OPT_REASSEMBLE, optarg); 1758 print_usage(prgname); 1759 return -1; 1760 } 1761 frag_tbl_sz = ret; 1762 break; 1763 case CMD_LINE_OPT_MTU_NUM: 1764 ret = parse_decimal(optarg); 1765 if (ret < 0 || ret > RTE_IPV4_MAX_PKT_LEN) { 1766 printf("Invalid argument for \'%s\': %s\n", 1767 CMD_LINE_OPT_MTU, optarg); 1768 print_usage(prgname); 1769 return -1; 1770 } 1771 mtu_size = ret; 1772 break; 1773 case CMD_LINE_OPT_FRAG_TTL_NUM: 1774 ret = parse_decimal(optarg); 1775 if (ret < 0 || ret > MAX_FRAG_TTL_NS) { 1776 printf("Invalid argument for \'%s\': %s\n", 1777 CMD_LINE_OPT_MTU, optarg); 1778 print_usage(prgname); 1779 return -1; 1780 } 1781 frag_ttl_ns = ret; 1782 break; 1783 default: 1784 print_usage(prgname); 1785 return -1; 1786 } 1787 } 1788 1789 if (f_present == 0) { 1790 printf("Mandatory option \"-f\" not present\n"); 1791 return -1; 1792 } 1793 1794 /* check do we need to enable multi-seg support */ 1795 if (multi_seg_required()) { 1796 /* legacy mode doesn't support multi-seg */ 1797 app_sa_prm.enable = 1; 1798 printf("frame buf size: %u, mtu: %u, " 1799 "number of reassemble entries: %u\n" 1800 "multi-segment support is required\n", 1801 frame_buf_size, mtu_size, frag_tbl_sz); 1802 } 1803 1804 print_app_sa_prm(&app_sa_prm); 1805 1806 if (optind >= 0) 1807 argv[optind-1] = prgname; 1808 1809 ret = optind-1; 1810 optind = 1; /* reset getopt lib */ 1811 return ret; 1812 } 1813 1814 static void 1815 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr) 1816 { 1817 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 1818 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr); 1819 printf("%s%s", name, buf); 1820 } 1821 1822 /* 1823 * Update destination ethaddr for the port. 1824 */ 1825 int 1826 add_dst_ethaddr(uint16_t port, const struct rte_ether_addr *addr) 1827 { 1828 if (port >= RTE_DIM(ethaddr_tbl)) 1829 return -EINVAL; 1830 1831 ethaddr_tbl[port].dst = ETHADDR_TO_UINT64(addr); 1832 return 0; 1833 } 1834 1835 /* Check the link status of all ports in up to 9s, and print them finally */ 1836 static void 1837 check_all_ports_link_status(uint32_t port_mask) 1838 { 1839 #define CHECK_INTERVAL 100 /* 100ms */ 1840 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1841 uint16_t portid; 1842 uint8_t count, all_ports_up, print_flag = 0; 1843 struct rte_eth_link link; 1844 int ret; 1845 char link_status_text[RTE_ETH_LINK_MAX_STR_LEN]; 1846 1847 printf("\nChecking link status"); 1848 fflush(stdout); 1849 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1850 all_ports_up = 1; 1851 RTE_ETH_FOREACH_DEV(portid) { 1852 if ((port_mask & (1 << portid)) == 0) 1853 continue; 1854 memset(&link, 0, sizeof(link)); 1855 ret = rte_eth_link_get_nowait(portid, &link); 1856 if (ret < 0) { 1857 all_ports_up = 0; 1858 if (print_flag == 1) 1859 printf("Port %u link get failed: %s\n", 1860 portid, rte_strerror(-ret)); 1861 continue; 1862 } 1863 /* print link status if flag set */ 1864 if (print_flag == 1) { 1865 rte_eth_link_to_str(link_status_text, 1866 sizeof(link_status_text), &link); 1867 printf("Port %d %s\n", portid, 1868 link_status_text); 1869 continue; 1870 } 1871 /* clear all_ports_up flag if any link down */ 1872 if (link.link_status == ETH_LINK_DOWN) { 1873 all_ports_up = 0; 1874 break; 1875 } 1876 } 1877 /* after finally printing all link status, get out */ 1878 if (print_flag == 1) 1879 break; 1880 1881 if (all_ports_up == 0) { 1882 printf("."); 1883 fflush(stdout); 1884 rte_delay_ms(CHECK_INTERVAL); 1885 } 1886 1887 /* set the print_flag if all ports up or timeout */ 1888 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1889 print_flag = 1; 1890 printf("done\n"); 1891 } 1892 } 1893 } 1894 1895 static int32_t 1896 add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id, 1897 uint16_t qp, struct lcore_params *params, 1898 struct ipsec_ctx *ipsec_ctx, 1899 const struct rte_cryptodev_capabilities *cipher, 1900 const struct rte_cryptodev_capabilities *auth, 1901 const struct rte_cryptodev_capabilities *aead) 1902 { 1903 int32_t ret = 0; 1904 unsigned long i; 1905 struct cdev_key key = { 0 }; 1906 1907 key.lcore_id = params->lcore_id; 1908 if (cipher) 1909 key.cipher_algo = cipher->sym.cipher.algo; 1910 if (auth) 1911 key.auth_algo = auth->sym.auth.algo; 1912 if (aead) 1913 key.aead_algo = aead->sym.aead.algo; 1914 1915 ret = rte_hash_lookup(map, &key); 1916 if (ret != -ENOENT) 1917 return 0; 1918 1919 for (i = 0; i < ipsec_ctx->nb_qps; i++) 1920 if (ipsec_ctx->tbl[i].id == cdev_id) 1921 break; 1922 1923 if (i == ipsec_ctx->nb_qps) { 1924 if (ipsec_ctx->nb_qps == MAX_QP_PER_LCORE) { 1925 printf("Maximum number of crypto devices assigned to " 1926 "a core, increase MAX_QP_PER_LCORE value\n"); 1927 return 0; 1928 } 1929 ipsec_ctx->tbl[i].id = cdev_id; 1930 ipsec_ctx->tbl[i].qp = qp; 1931 ipsec_ctx->nb_qps++; 1932 printf("%s cdev mapping: lcore %u using cdev %u qp %u " 1933 "(cdev_id_qp %lu)\n", str, key.lcore_id, 1934 cdev_id, qp, i); 1935 } 1936 1937 ret = rte_hash_add_key_data(map, &key, (void *)i); 1938 if (ret < 0) { 1939 printf("Faled to insert cdev mapping for (lcore %u, " 1940 "cdev %u, qp %u), errno %d\n", 1941 key.lcore_id, ipsec_ctx->tbl[i].id, 1942 ipsec_ctx->tbl[i].qp, ret); 1943 return 0; 1944 } 1945 1946 return 1; 1947 } 1948 1949 static int32_t 1950 add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id, 1951 uint16_t qp, struct lcore_params *params) 1952 { 1953 int32_t ret = 0; 1954 const struct rte_cryptodev_capabilities *i, *j; 1955 struct rte_hash *map; 1956 struct lcore_conf *qconf; 1957 struct ipsec_ctx *ipsec_ctx; 1958 const char *str; 1959 1960 qconf = &lcore_conf[params->lcore_id]; 1961 1962 if ((unprotected_port_mask & (1 << params->port_id)) == 0) { 1963 map = cdev_map_out; 1964 ipsec_ctx = &qconf->outbound; 1965 str = "Outbound"; 1966 } else { 1967 map = cdev_map_in; 1968 ipsec_ctx = &qconf->inbound; 1969 str = "Inbound"; 1970 } 1971 1972 /* Required cryptodevs with operation chainning */ 1973 if (!(dev_info->feature_flags & 1974 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING)) 1975 return ret; 1976 1977 for (i = dev_info->capabilities; 1978 i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++) { 1979 if (i->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1980 continue; 1981 1982 if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1983 ret |= add_mapping(map, str, cdev_id, qp, params, 1984 ipsec_ctx, NULL, NULL, i); 1985 continue; 1986 } 1987 1988 if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) 1989 continue; 1990 1991 for (j = dev_info->capabilities; 1992 j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; j++) { 1993 if (j->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1994 continue; 1995 1996 if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH) 1997 continue; 1998 1999 ret |= add_mapping(map, str, cdev_id, qp, params, 2000 ipsec_ctx, i, j, NULL); 2001 } 2002 } 2003 2004 return ret; 2005 } 2006 2007 /* Check if the device is enabled by cryptodev_mask */ 2008 static int 2009 check_cryptodev_mask(uint8_t cdev_id) 2010 { 2011 if (enabled_cryptodev_mask & (1 << cdev_id)) 2012 return 0; 2013 2014 return -1; 2015 } 2016 2017 static uint16_t 2018 cryptodevs_init(uint16_t req_queue_num) 2019 { 2020 struct rte_cryptodev_config dev_conf; 2021 struct rte_cryptodev_qp_conf qp_conf; 2022 uint16_t idx, max_nb_qps, qp, total_nb_qps, i; 2023 int16_t cdev_id; 2024 struct rte_hash_parameters params = { 0 }; 2025 2026 const uint64_t mseg_flag = multi_seg_required() ? 2027 RTE_CRYPTODEV_FF_IN_PLACE_SGL : 0; 2028 2029 params.entries = CDEV_MAP_ENTRIES; 2030 params.key_len = sizeof(struct cdev_key); 2031 params.hash_func = rte_jhash; 2032 params.hash_func_init_val = 0; 2033 params.socket_id = rte_socket_id(); 2034 2035 params.name = "cdev_map_in"; 2036 cdev_map_in = rte_hash_create(¶ms); 2037 if (cdev_map_in == NULL) 2038 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 2039 rte_errno); 2040 2041 params.name = "cdev_map_out"; 2042 cdev_map_out = rte_hash_create(¶ms); 2043 if (cdev_map_out == NULL) 2044 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 2045 rte_errno); 2046 2047 printf("lcore/cryptodev/qp mappings:\n"); 2048 2049 idx = 0; 2050 total_nb_qps = 0; 2051 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 2052 struct rte_cryptodev_info cdev_info; 2053 2054 if (check_cryptodev_mask((uint8_t)cdev_id)) 2055 continue; 2056 2057 rte_cryptodev_info_get(cdev_id, &cdev_info); 2058 2059 if ((mseg_flag & cdev_info.feature_flags) != mseg_flag) 2060 rte_exit(EXIT_FAILURE, 2061 "Device %hd does not support \'%s\' feature\n", 2062 cdev_id, 2063 rte_cryptodev_get_feature_name(mseg_flag)); 2064 2065 if (nb_lcore_params > cdev_info.max_nb_queue_pairs) 2066 max_nb_qps = cdev_info.max_nb_queue_pairs; 2067 else 2068 max_nb_qps = nb_lcore_params; 2069 2070 qp = 0; 2071 i = 0; 2072 while (qp < max_nb_qps && i < nb_lcore_params) { 2073 if (add_cdev_mapping(&cdev_info, cdev_id, qp, 2074 &lcore_params[idx])) 2075 qp++; 2076 idx++; 2077 idx = idx % nb_lcore_params; 2078 i++; 2079 } 2080 2081 qp = RTE_MIN(max_nb_qps, RTE_MAX(req_queue_num, qp)); 2082 if (qp == 0) 2083 continue; 2084 2085 total_nb_qps += qp; 2086 dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id); 2087 dev_conf.nb_queue_pairs = qp; 2088 dev_conf.ff_disable = RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO; 2089 2090 uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions; 2091 if (dev_max_sess != 0 && 2092 dev_max_sess < get_nb_crypto_sessions()) 2093 rte_exit(EXIT_FAILURE, 2094 "Device does not support at least %u " 2095 "sessions", get_nb_crypto_sessions()); 2096 2097 if (rte_cryptodev_configure(cdev_id, &dev_conf)) 2098 rte_panic("Failed to initialize cryptodev %u\n", 2099 cdev_id); 2100 2101 qp_conf.nb_descriptors = CDEV_QUEUE_DESC; 2102 qp_conf.mp_session = 2103 socket_ctx[dev_conf.socket_id].session_pool; 2104 qp_conf.mp_session_private = 2105 socket_ctx[dev_conf.socket_id].session_priv_pool; 2106 for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++) 2107 if (rte_cryptodev_queue_pair_setup(cdev_id, qp, 2108 &qp_conf, dev_conf.socket_id)) 2109 rte_panic("Failed to setup queue %u for " 2110 "cdev_id %u\n", 0, cdev_id); 2111 2112 if (rte_cryptodev_start(cdev_id)) 2113 rte_panic("Failed to start cryptodev %u\n", 2114 cdev_id); 2115 } 2116 2117 printf("\n"); 2118 2119 return total_nb_qps; 2120 } 2121 2122 static void 2123 port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads) 2124 { 2125 uint32_t frame_size; 2126 struct rte_eth_dev_info dev_info; 2127 struct rte_eth_txconf *txconf; 2128 uint16_t nb_tx_queue, nb_rx_queue; 2129 uint16_t tx_queueid, rx_queueid, queue, lcore_id; 2130 int32_t ret, socket_id; 2131 struct lcore_conf *qconf; 2132 struct rte_ether_addr ethaddr; 2133 struct rte_eth_conf local_port_conf = port_conf; 2134 2135 ret = rte_eth_dev_info_get(portid, &dev_info); 2136 if (ret != 0) 2137 rte_exit(EXIT_FAILURE, 2138 "Error during getting device (port %u) info: %s\n", 2139 portid, strerror(-ret)); 2140 2141 /* limit allowed HW offloafs, as user requested */ 2142 dev_info.rx_offload_capa &= dev_rx_offload; 2143 dev_info.tx_offload_capa &= dev_tx_offload; 2144 2145 printf("Configuring device port %u:\n", portid); 2146 2147 ret = rte_eth_macaddr_get(portid, ðaddr); 2148 if (ret != 0) 2149 rte_exit(EXIT_FAILURE, 2150 "Error getting MAC address (port %u): %s\n", 2151 portid, rte_strerror(-ret)); 2152 2153 ethaddr_tbl[portid].src = ETHADDR_TO_UINT64(ðaddr); 2154 print_ethaddr("Address: ", ðaddr); 2155 printf("\n"); 2156 2157 nb_rx_queue = get_port_nb_rx_queues(portid); 2158 nb_tx_queue = nb_lcores; 2159 2160 if (nb_rx_queue > dev_info.max_rx_queues) 2161 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 2162 "(max rx queue is %u)\n", 2163 nb_rx_queue, dev_info.max_rx_queues); 2164 2165 if (nb_tx_queue > dev_info.max_tx_queues) 2166 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 2167 "(max tx queue is %u)\n", 2168 nb_tx_queue, dev_info.max_tx_queues); 2169 2170 printf("Creating queues: nb_rx_queue=%d nb_tx_queue=%u...\n", 2171 nb_rx_queue, nb_tx_queue); 2172 2173 frame_size = MTU_TO_FRAMELEN(mtu_size); 2174 if (frame_size > local_port_conf.rxmode.max_rx_pkt_len) 2175 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 2176 local_port_conf.rxmode.max_rx_pkt_len = frame_size; 2177 2178 if (multi_seg_required()) { 2179 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_SCATTER; 2180 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MULTI_SEGS; 2181 } 2182 2183 local_port_conf.rxmode.offloads |= req_rx_offloads; 2184 local_port_conf.txmode.offloads |= req_tx_offloads; 2185 2186 /* Check that all required capabilities are supported */ 2187 if ((local_port_conf.rxmode.offloads & dev_info.rx_offload_capa) != 2188 local_port_conf.rxmode.offloads) 2189 rte_exit(EXIT_FAILURE, 2190 "Error: port %u required RX offloads: 0x%" PRIx64 2191 ", avaialbe RX offloads: 0x%" PRIx64 "\n", 2192 portid, local_port_conf.rxmode.offloads, 2193 dev_info.rx_offload_capa); 2194 2195 if ((local_port_conf.txmode.offloads & dev_info.tx_offload_capa) != 2196 local_port_conf.txmode.offloads) 2197 rte_exit(EXIT_FAILURE, 2198 "Error: port %u required TX offloads: 0x%" PRIx64 2199 ", avaialbe TX offloads: 0x%" PRIx64 "\n", 2200 portid, local_port_conf.txmode.offloads, 2201 dev_info.tx_offload_capa); 2202 2203 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 2204 local_port_conf.txmode.offloads |= 2205 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 2206 2207 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 2208 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 2209 2210 printf("port %u configurng rx_offloads=0x%" PRIx64 2211 ", tx_offloads=0x%" PRIx64 "\n", 2212 portid, local_port_conf.rxmode.offloads, 2213 local_port_conf.txmode.offloads); 2214 2215 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 2216 dev_info.flow_type_rss_offloads; 2217 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 2218 port_conf.rx_adv_conf.rss_conf.rss_hf) { 2219 printf("Port %u modified RSS hash function based on hardware support," 2220 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 2221 portid, 2222 port_conf.rx_adv_conf.rss_conf.rss_hf, 2223 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 2224 } 2225 2226 ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue, 2227 &local_port_conf); 2228 if (ret < 0) 2229 rte_exit(EXIT_FAILURE, "Cannot configure device: " 2230 "err=%d, port=%d\n", ret, portid); 2231 2232 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd); 2233 if (ret < 0) 2234 rte_exit(EXIT_FAILURE, "Cannot adjust number of descriptors: " 2235 "err=%d, port=%d\n", ret, portid); 2236 2237 /* init one TX queue per lcore */ 2238 tx_queueid = 0; 2239 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2240 if (rte_lcore_is_enabled(lcore_id) == 0) 2241 continue; 2242 2243 if (numa_on) 2244 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2245 else 2246 socket_id = 0; 2247 2248 /* init TX queue */ 2249 printf("Setup txq=%u,%d,%d\n", lcore_id, tx_queueid, socket_id); 2250 2251 txconf = &dev_info.default_txconf; 2252 txconf->offloads = local_port_conf.txmode.offloads; 2253 2254 ret = rte_eth_tx_queue_setup(portid, tx_queueid, nb_txd, 2255 socket_id, txconf); 2256 if (ret < 0) 2257 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: " 2258 "err=%d, port=%d\n", ret, portid); 2259 2260 qconf = &lcore_conf[lcore_id]; 2261 qconf->tx_queue_id[portid] = tx_queueid; 2262 2263 /* Pre-populate pkt offloads based on capabilities */ 2264 qconf->outbound.ipv4_offloads = PKT_TX_IPV4; 2265 qconf->outbound.ipv6_offloads = PKT_TX_IPV6; 2266 if (local_port_conf.txmode.offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) 2267 qconf->outbound.ipv4_offloads |= PKT_TX_IP_CKSUM; 2268 2269 tx_queueid++; 2270 2271 /* init RX queues */ 2272 for (queue = 0; queue < qconf->nb_rx_queue; ++queue) { 2273 struct rte_eth_rxconf rxq_conf; 2274 2275 if (portid != qconf->rx_queue_list[queue].port_id) 2276 continue; 2277 2278 rx_queueid = qconf->rx_queue_list[queue].queue_id; 2279 2280 printf("Setup rxq=%d,%d,%d\n", portid, rx_queueid, 2281 socket_id); 2282 2283 rxq_conf = dev_info.default_rxconf; 2284 rxq_conf.offloads = local_port_conf.rxmode.offloads; 2285 ret = rte_eth_rx_queue_setup(portid, rx_queueid, 2286 nb_rxd, socket_id, &rxq_conf, 2287 socket_ctx[socket_id].mbuf_pool); 2288 if (ret < 0) 2289 rte_exit(EXIT_FAILURE, 2290 "rte_eth_rx_queue_setup: err=%d, " 2291 "port=%d\n", ret, portid); 2292 } 2293 } 2294 printf("\n"); 2295 } 2296 2297 static size_t 2298 max_session_size(void) 2299 { 2300 size_t max_sz, sz; 2301 void *sec_ctx; 2302 int16_t cdev_id, port_id, n; 2303 2304 max_sz = 0; 2305 n = rte_cryptodev_count(); 2306 for (cdev_id = 0; cdev_id != n; cdev_id++) { 2307 sz = rte_cryptodev_sym_get_private_session_size(cdev_id); 2308 if (sz > max_sz) 2309 max_sz = sz; 2310 /* 2311 * If crypto device is security capable, need to check the 2312 * size of security session as well. 2313 */ 2314 2315 /* Get security context of the crypto device */ 2316 sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id); 2317 if (sec_ctx == NULL) 2318 continue; 2319 2320 /* Get size of security session */ 2321 sz = rte_security_session_get_size(sec_ctx); 2322 if (sz > max_sz) 2323 max_sz = sz; 2324 } 2325 2326 RTE_ETH_FOREACH_DEV(port_id) { 2327 if ((enabled_port_mask & (1 << port_id)) == 0) 2328 continue; 2329 2330 sec_ctx = rte_eth_dev_get_sec_ctx(port_id); 2331 if (sec_ctx == NULL) 2332 continue; 2333 2334 sz = rte_security_session_get_size(sec_ctx); 2335 if (sz > max_sz) 2336 max_sz = sz; 2337 } 2338 2339 return max_sz; 2340 } 2341 2342 static void 2343 session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz) 2344 { 2345 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2346 struct rte_mempool *sess_mp; 2347 uint32_t nb_sess; 2348 2349 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2350 "sess_mp_%u", socket_id); 2351 nb_sess = (get_nb_crypto_sessions() + CDEV_MP_CACHE_SZ * 2352 rte_lcore_count()); 2353 nb_sess = RTE_MAX(nb_sess, CDEV_MP_CACHE_SZ * 2354 CDEV_MP_CACHE_MULTIPLIER); 2355 sess_mp = rte_cryptodev_sym_session_pool_create( 2356 mp_name, nb_sess, sess_sz, CDEV_MP_CACHE_SZ, 0, 2357 socket_id); 2358 ctx->session_pool = sess_mp; 2359 2360 if (ctx->session_pool == NULL) 2361 rte_exit(EXIT_FAILURE, 2362 "Cannot init session pool on socket %d\n", socket_id); 2363 else 2364 printf("Allocated session pool on socket %d\n", socket_id); 2365 } 2366 2367 static void 2368 session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id, 2369 size_t sess_sz) 2370 { 2371 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2372 struct rte_mempool *sess_mp; 2373 uint32_t nb_sess; 2374 2375 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2376 "sess_mp_priv_%u", socket_id); 2377 nb_sess = (get_nb_crypto_sessions() + CDEV_MP_CACHE_SZ * 2378 rte_lcore_count()); 2379 nb_sess = RTE_MAX(nb_sess, CDEV_MP_CACHE_SZ * 2380 CDEV_MP_CACHE_MULTIPLIER); 2381 sess_mp = rte_mempool_create(mp_name, 2382 nb_sess, 2383 sess_sz, 2384 CDEV_MP_CACHE_SZ, 2385 0, NULL, NULL, NULL, 2386 NULL, socket_id, 2387 0); 2388 ctx->session_priv_pool = sess_mp; 2389 2390 if (ctx->session_priv_pool == NULL) 2391 rte_exit(EXIT_FAILURE, 2392 "Cannot init session priv pool on socket %d\n", 2393 socket_id); 2394 else 2395 printf("Allocated session priv pool on socket %d\n", 2396 socket_id); 2397 } 2398 2399 static void 2400 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf) 2401 { 2402 char s[64]; 2403 int32_t ms; 2404 2405 snprintf(s, sizeof(s), "mbuf_pool_%d", socket_id); 2406 ctx->mbuf_pool = rte_pktmbuf_pool_create(s, nb_mbuf, 2407 MEMPOOL_CACHE_SIZE, ipsec_metadata_size(), 2408 frame_buf_size, socket_id); 2409 2410 /* 2411 * if multi-segment support is enabled, then create a pool 2412 * for indirect mbufs. 2413 */ 2414 ms = multi_seg_required(); 2415 if (ms != 0) { 2416 snprintf(s, sizeof(s), "mbuf_pool_indir_%d", socket_id); 2417 ctx->mbuf_pool_indir = rte_pktmbuf_pool_create(s, nb_mbuf, 2418 MEMPOOL_CACHE_SIZE, 0, 0, socket_id); 2419 } 2420 2421 if (ctx->mbuf_pool == NULL || (ms != 0 && ctx->mbuf_pool_indir == NULL)) 2422 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n", 2423 socket_id); 2424 else 2425 printf("Allocated mbuf pool on socket %d\n", socket_id); 2426 } 2427 2428 static inline int 2429 inline_ipsec_event_esn_overflow(struct rte_security_ctx *ctx, uint64_t md) 2430 { 2431 struct ipsec_sa *sa; 2432 2433 /* For inline protocol processing, the metadata in the event will 2434 * uniquely identify the security session which raised the event. 2435 * Application would then need the userdata it had registered with the 2436 * security session to process the event. 2437 */ 2438 2439 sa = (struct ipsec_sa *)rte_security_get_userdata(ctx, md); 2440 2441 if (sa == NULL) { 2442 /* userdata could not be retrieved */ 2443 return -1; 2444 } 2445 2446 /* Sequence number over flow. SA need to be re-established */ 2447 RTE_SET_USED(sa); 2448 return 0; 2449 } 2450 2451 static int 2452 inline_ipsec_event_callback(uint16_t port_id, enum rte_eth_event_type type, 2453 void *param, void *ret_param) 2454 { 2455 uint64_t md; 2456 struct rte_eth_event_ipsec_desc *event_desc = NULL; 2457 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 2458 rte_eth_dev_get_sec_ctx(port_id); 2459 2460 RTE_SET_USED(param); 2461 2462 if (type != RTE_ETH_EVENT_IPSEC) 2463 return -1; 2464 2465 event_desc = ret_param; 2466 if (event_desc == NULL) { 2467 printf("Event descriptor not set\n"); 2468 return -1; 2469 } 2470 2471 md = event_desc->metadata; 2472 2473 if (event_desc->subtype == RTE_ETH_EVENT_IPSEC_ESN_OVERFLOW) 2474 return inline_ipsec_event_esn_overflow(ctx, md); 2475 else if (event_desc->subtype >= RTE_ETH_EVENT_IPSEC_MAX) { 2476 printf("Invalid IPsec event reported\n"); 2477 return -1; 2478 } 2479 2480 return -1; 2481 } 2482 2483 static uint16_t 2484 rx_callback(__rte_unused uint16_t port, __rte_unused uint16_t queue, 2485 struct rte_mbuf *pkt[], uint16_t nb_pkts, 2486 __rte_unused uint16_t max_pkts, void *user_param) 2487 { 2488 uint64_t tm; 2489 uint32_t i, k; 2490 struct lcore_conf *lc; 2491 struct rte_mbuf *mb; 2492 struct rte_ether_hdr *eth; 2493 2494 lc = user_param; 2495 k = 0; 2496 tm = 0; 2497 2498 for (i = 0; i != nb_pkts; i++) { 2499 2500 mb = pkt[i]; 2501 eth = rte_pktmbuf_mtod(mb, struct rte_ether_hdr *); 2502 if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 2503 2504 struct rte_ipv4_hdr *iph; 2505 2506 iph = (struct rte_ipv4_hdr *)(eth + 1); 2507 if (rte_ipv4_frag_pkt_is_fragmented(iph)) { 2508 2509 mb->l2_len = sizeof(*eth); 2510 mb->l3_len = sizeof(*iph); 2511 tm = (tm != 0) ? tm : rte_rdtsc(); 2512 mb = rte_ipv4_frag_reassemble_packet( 2513 lc->frag.tbl, &lc->frag.dr, 2514 mb, tm, iph); 2515 2516 if (mb != NULL) { 2517 /* fix ip cksum after reassemble. */ 2518 iph = rte_pktmbuf_mtod_offset(mb, 2519 struct rte_ipv4_hdr *, 2520 mb->l2_len); 2521 iph->hdr_checksum = 0; 2522 iph->hdr_checksum = rte_ipv4_cksum(iph); 2523 } 2524 } 2525 } else if (eth->ether_type == 2526 rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 2527 2528 struct rte_ipv6_hdr *iph; 2529 struct ipv6_extension_fragment *fh; 2530 2531 iph = (struct rte_ipv6_hdr *)(eth + 1); 2532 fh = rte_ipv6_frag_get_ipv6_fragment_header(iph); 2533 if (fh != NULL) { 2534 mb->l2_len = sizeof(*eth); 2535 mb->l3_len = (uintptr_t)fh - (uintptr_t)iph + 2536 sizeof(*fh); 2537 tm = (tm != 0) ? tm : rte_rdtsc(); 2538 mb = rte_ipv6_frag_reassemble_packet( 2539 lc->frag.tbl, &lc->frag.dr, 2540 mb, tm, iph, fh); 2541 if (mb != NULL) 2542 /* fix l3_len after reassemble. */ 2543 mb->l3_len = mb->l3_len - sizeof(*fh); 2544 } 2545 } 2546 2547 pkt[k] = mb; 2548 k += (mb != NULL); 2549 } 2550 2551 /* some fragments were encountered, drain death row */ 2552 if (tm != 0) 2553 rte_ip_frag_free_death_row(&lc->frag.dr, 0); 2554 2555 return k; 2556 } 2557 2558 2559 static int 2560 reassemble_lcore_init(struct lcore_conf *lc, uint32_t cid) 2561 { 2562 int32_t sid; 2563 uint32_t i; 2564 uint64_t frag_cycles; 2565 const struct lcore_rx_queue *rxq; 2566 const struct rte_eth_rxtx_callback *cb; 2567 2568 /* create fragment table */ 2569 sid = rte_lcore_to_socket_id(cid); 2570 frag_cycles = (rte_get_tsc_hz() + NS_PER_S - 1) / 2571 NS_PER_S * frag_ttl_ns; 2572 2573 lc->frag.tbl = rte_ip_frag_table_create(frag_tbl_sz, 2574 FRAG_TBL_BUCKET_ENTRIES, frag_tbl_sz, frag_cycles, sid); 2575 if (lc->frag.tbl == NULL) { 2576 printf("%s(%u): failed to create fragment table of size: %u, " 2577 "error code: %d\n", 2578 __func__, cid, frag_tbl_sz, rte_errno); 2579 return -ENOMEM; 2580 } 2581 2582 /* setup reassemble RX callbacks for all queues */ 2583 for (i = 0; i != lc->nb_rx_queue; i++) { 2584 2585 rxq = lc->rx_queue_list + i; 2586 cb = rte_eth_add_rx_callback(rxq->port_id, rxq->queue_id, 2587 rx_callback, lc); 2588 if (cb == NULL) { 2589 printf("%s(%u): failed to install RX callback for " 2590 "portid=%u, queueid=%u, error code: %d\n", 2591 __func__, cid, 2592 rxq->port_id, rxq->queue_id, rte_errno); 2593 return -ENOMEM; 2594 } 2595 } 2596 2597 return 0; 2598 } 2599 2600 static int 2601 reassemble_init(void) 2602 { 2603 int32_t rc; 2604 uint32_t i, lc; 2605 2606 rc = 0; 2607 for (i = 0; i != nb_lcore_params; i++) { 2608 lc = lcore_params[i].lcore_id; 2609 rc = reassemble_lcore_init(lcore_conf + lc, lc); 2610 if (rc != 0) 2611 break; 2612 } 2613 2614 return rc; 2615 } 2616 2617 static void 2618 create_default_ipsec_flow(uint16_t port_id, uint64_t rx_offloads) 2619 { 2620 struct rte_flow_action action[2]; 2621 struct rte_flow_item pattern[2]; 2622 struct rte_flow_attr attr = {0}; 2623 struct rte_flow_error err; 2624 struct rte_flow *flow; 2625 int ret; 2626 2627 if (!(rx_offloads & DEV_RX_OFFLOAD_SECURITY)) 2628 return; 2629 2630 /* Add the default rte_flow to enable SECURITY for all ESP packets */ 2631 2632 pattern[0].type = RTE_FLOW_ITEM_TYPE_ESP; 2633 pattern[0].spec = NULL; 2634 pattern[0].mask = NULL; 2635 pattern[0].last = NULL; 2636 pattern[1].type = RTE_FLOW_ITEM_TYPE_END; 2637 2638 action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY; 2639 action[0].conf = NULL; 2640 action[1].type = RTE_FLOW_ACTION_TYPE_END; 2641 action[1].conf = NULL; 2642 2643 attr.ingress = 1; 2644 2645 ret = rte_flow_validate(port_id, &attr, pattern, action, &err); 2646 if (ret) 2647 return; 2648 2649 flow = rte_flow_create(port_id, &attr, pattern, action, &err); 2650 if (flow == NULL) 2651 return; 2652 2653 flow_info_tbl[port_id].rx_def_flow = flow; 2654 RTE_LOG(INFO, IPSEC, 2655 "Created default flow enabling SECURITY for all ESP traffic on port %d\n", 2656 port_id); 2657 } 2658 2659 static void 2660 signal_handler(int signum) 2661 { 2662 if (signum == SIGINT || signum == SIGTERM) { 2663 printf("\n\nSignal %d received, preparing to exit...\n", 2664 signum); 2665 force_quit = true; 2666 } 2667 } 2668 2669 static void 2670 ev_mode_sess_verify(struct ipsec_sa *sa, int nb_sa) 2671 { 2672 struct rte_ipsec_session *ips; 2673 int32_t i; 2674 2675 if (!sa || !nb_sa) 2676 return; 2677 2678 for (i = 0; i < nb_sa; i++) { 2679 ips = ipsec_get_primary_session(&sa[i]); 2680 if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) 2681 rte_exit(EXIT_FAILURE, "Event mode supports only " 2682 "inline protocol sessions\n"); 2683 } 2684 2685 } 2686 2687 static int32_t 2688 check_event_mode_params(struct eh_conf *eh_conf) 2689 { 2690 struct eventmode_conf *em_conf = NULL; 2691 struct lcore_params *params; 2692 uint16_t portid; 2693 2694 if (!eh_conf || !eh_conf->mode_params) 2695 return -EINVAL; 2696 2697 /* Get eventmode conf */ 2698 em_conf = eh_conf->mode_params; 2699 2700 if (eh_conf->mode == EH_PKT_TRANSFER_MODE_POLL && 2701 em_conf->ext_params.sched_type != SCHED_TYPE_NOT_SET) { 2702 printf("error: option --event-schedule-type applies only to " 2703 "event mode\n"); 2704 return -EINVAL; 2705 } 2706 2707 if (eh_conf->mode != EH_PKT_TRANSFER_MODE_EVENT) 2708 return 0; 2709 2710 /* Set schedule type to ORDERED if it wasn't explicitly set by user */ 2711 if (em_conf->ext_params.sched_type == SCHED_TYPE_NOT_SET) 2712 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ORDERED; 2713 2714 /* 2715 * Event mode currently supports only inline protocol sessions. 2716 * If there are other types of sessions configured then exit with 2717 * error. 2718 */ 2719 ev_mode_sess_verify(sa_in, nb_sa_in); 2720 ev_mode_sess_verify(sa_out, nb_sa_out); 2721 2722 2723 /* Option --config does not apply to event mode */ 2724 if (nb_lcore_params > 0) { 2725 printf("error: option --config applies only to poll mode\n"); 2726 return -EINVAL; 2727 } 2728 2729 /* 2730 * In order to use the same port_init routine for both poll and event 2731 * modes initialize lcore_params with one queue for each eth port 2732 */ 2733 lcore_params = lcore_params_array; 2734 RTE_ETH_FOREACH_DEV(portid) { 2735 if ((enabled_port_mask & (1 << portid)) == 0) 2736 continue; 2737 2738 params = &lcore_params[nb_lcore_params++]; 2739 params->port_id = portid; 2740 params->queue_id = 0; 2741 params->lcore_id = rte_get_next_lcore(0, 0, 1); 2742 } 2743 2744 return 0; 2745 } 2746 2747 static void 2748 inline_sessions_free(struct sa_ctx *sa_ctx) 2749 { 2750 struct rte_ipsec_session *ips; 2751 struct ipsec_sa *sa; 2752 int32_t ret; 2753 uint32_t i; 2754 2755 if (!sa_ctx) 2756 return; 2757 2758 for (i = 0; i < sa_ctx->nb_sa; i++) { 2759 2760 sa = &sa_ctx->sa[i]; 2761 if (!sa->spi) 2762 continue; 2763 2764 ips = ipsec_get_primary_session(sa); 2765 if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL && 2766 ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) 2767 continue; 2768 2769 if (!rte_eth_dev_is_valid_port(sa->portid)) 2770 continue; 2771 2772 ret = rte_security_session_destroy( 2773 rte_eth_dev_get_sec_ctx(sa->portid), 2774 ips->security.ses); 2775 if (ret) 2776 RTE_LOG(ERR, IPSEC, "Failed to destroy security " 2777 "session type %d, spi %d\n", 2778 ips->type, sa->spi); 2779 } 2780 } 2781 2782 static uint32_t 2783 calculate_nb_mbufs(uint16_t nb_ports, uint16_t nb_crypto_qp, uint32_t nb_rxq, 2784 uint32_t nb_txq) 2785 { 2786 return RTE_MAX((nb_rxq * nb_rxd + 2787 nb_ports * nb_lcores * MAX_PKT_BURST + 2788 nb_ports * nb_txq * nb_txd + 2789 nb_lcores * MEMPOOL_CACHE_SIZE + 2790 nb_crypto_qp * CDEV_QUEUE_DESC + 2791 nb_lcores * frag_tbl_sz * 2792 FRAG_TBL_BUCKET_ENTRIES), 2793 8192U); 2794 } 2795 2796 int32_t 2797 main(int32_t argc, char **argv) 2798 { 2799 int32_t ret; 2800 uint32_t lcore_id, nb_txq, nb_rxq = 0; 2801 uint32_t cdev_id; 2802 uint32_t i; 2803 uint8_t socket_id; 2804 uint16_t portid, nb_crypto_qp, nb_ports = 0; 2805 uint64_t req_rx_offloads[RTE_MAX_ETHPORTS]; 2806 uint64_t req_tx_offloads[RTE_MAX_ETHPORTS]; 2807 struct eh_conf *eh_conf = NULL; 2808 size_t sess_sz; 2809 2810 nb_bufs_in_pool = 0; 2811 2812 /* init EAL */ 2813 ret = rte_eal_init(argc, argv); 2814 if (ret < 0) 2815 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 2816 argc -= ret; 2817 argv += ret; 2818 2819 force_quit = false; 2820 signal(SIGINT, signal_handler); 2821 signal(SIGTERM, signal_handler); 2822 2823 /* initialize event helper configuration */ 2824 eh_conf = eh_conf_init(); 2825 if (eh_conf == NULL) 2826 rte_exit(EXIT_FAILURE, "Failed to init event helper config"); 2827 2828 /* parse application arguments (after the EAL ones) */ 2829 ret = parse_args(argc, argv, eh_conf); 2830 if (ret < 0) 2831 rte_exit(EXIT_FAILURE, "Invalid parameters\n"); 2832 2833 /* parse configuration file */ 2834 if (parse_cfg_file(cfgfile) < 0) { 2835 printf("parsing file \"%s\" failed\n", 2836 optarg); 2837 print_usage(argv[0]); 2838 return -1; 2839 } 2840 2841 if ((unprotected_port_mask & enabled_port_mask) != 2842 unprotected_port_mask) 2843 rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n", 2844 unprotected_port_mask); 2845 2846 if (check_poll_mode_params(eh_conf) < 0) 2847 rte_exit(EXIT_FAILURE, "check_poll_mode_params failed\n"); 2848 2849 if (check_event_mode_params(eh_conf) < 0) 2850 rte_exit(EXIT_FAILURE, "check_event_mode_params failed\n"); 2851 2852 ret = init_lcore_rx_queues(); 2853 if (ret < 0) 2854 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 2855 2856 nb_lcores = rte_lcore_count(); 2857 2858 sess_sz = max_session_size(); 2859 2860 /* 2861 * In event mode request minimum number of crypto queues 2862 * to be reserved equal to number of ports. 2863 */ 2864 if (eh_conf->mode == EH_PKT_TRANSFER_MODE_EVENT) 2865 nb_crypto_qp = rte_eth_dev_count_avail(); 2866 else 2867 nb_crypto_qp = 0; 2868 2869 nb_crypto_qp = cryptodevs_init(nb_crypto_qp); 2870 2871 if (nb_bufs_in_pool == 0) { 2872 RTE_ETH_FOREACH_DEV(portid) { 2873 if ((enabled_port_mask & (1 << portid)) == 0) 2874 continue; 2875 nb_ports++; 2876 nb_rxq += get_port_nb_rx_queues(portid); 2877 } 2878 2879 nb_txq = nb_lcores; 2880 2881 nb_bufs_in_pool = calculate_nb_mbufs(nb_ports, nb_crypto_qp, 2882 nb_rxq, nb_txq); 2883 } 2884 2885 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2886 if (rte_lcore_is_enabled(lcore_id) == 0) 2887 continue; 2888 2889 if (numa_on) 2890 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2891 else 2892 socket_id = 0; 2893 2894 /* mbuf_pool is initialised by the pool_init() function*/ 2895 if (socket_ctx[socket_id].mbuf_pool) 2896 continue; 2897 2898 pool_init(&socket_ctx[socket_id], socket_id, nb_bufs_in_pool); 2899 session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz); 2900 session_priv_pool_init(&socket_ctx[socket_id], socket_id, 2901 sess_sz); 2902 } 2903 printf("Number of mbufs in packet pool %d\n", nb_bufs_in_pool); 2904 2905 RTE_ETH_FOREACH_DEV(portid) { 2906 if ((enabled_port_mask & (1 << portid)) == 0) 2907 continue; 2908 2909 sa_check_offloads(portid, &req_rx_offloads[portid], 2910 &req_tx_offloads[portid]); 2911 port_init(portid, req_rx_offloads[portid], 2912 req_tx_offloads[portid]); 2913 } 2914 2915 /* 2916 * Set the enabled port mask in helper config for use by helper 2917 * sub-system. This will be used while initializing devices using 2918 * helper sub-system. 2919 */ 2920 eh_conf->eth_portmask = enabled_port_mask; 2921 2922 /* Initialize eventmode components */ 2923 ret = eh_devs_init(eh_conf); 2924 if (ret < 0) 2925 rte_exit(EXIT_FAILURE, "eh_devs_init failed, err=%d\n", ret); 2926 2927 /* start ports */ 2928 RTE_ETH_FOREACH_DEV(portid) { 2929 if ((enabled_port_mask & (1 << portid)) == 0) 2930 continue; 2931 2932 /* Create flow before starting the device */ 2933 create_default_ipsec_flow(portid, req_rx_offloads[portid]); 2934 2935 ret = rte_eth_dev_start(portid); 2936 if (ret < 0) 2937 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: " 2938 "err=%d, port=%d\n", ret, portid); 2939 /* 2940 * If enabled, put device in promiscuous mode. 2941 * This allows IO forwarding mode to forward packets 2942 * to itself through 2 cross-connected ports of the 2943 * target machine. 2944 */ 2945 if (promiscuous_on) { 2946 ret = rte_eth_promiscuous_enable(portid); 2947 if (ret != 0) 2948 rte_exit(EXIT_FAILURE, 2949 "rte_eth_promiscuous_enable: err=%s, port=%d\n", 2950 rte_strerror(-ret), portid); 2951 } 2952 2953 rte_eth_dev_callback_register(portid, 2954 RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL); 2955 } 2956 2957 /* fragment reassemble is enabled */ 2958 if (frag_tbl_sz != 0) { 2959 ret = reassemble_init(); 2960 if (ret != 0) 2961 rte_exit(EXIT_FAILURE, "failed at reassemble init"); 2962 } 2963 2964 /* Replicate each context per socket */ 2965 for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) { 2966 socket_id = rte_socket_id_by_idx(i); 2967 if ((socket_ctx[socket_id].mbuf_pool != NULL) && 2968 (socket_ctx[socket_id].sa_in == NULL) && 2969 (socket_ctx[socket_id].sa_out == NULL)) { 2970 sa_init(&socket_ctx[socket_id], socket_id); 2971 sp4_init(&socket_ctx[socket_id], socket_id); 2972 sp6_init(&socket_ctx[socket_id], socket_id); 2973 rt_init(&socket_ctx[socket_id], socket_id); 2974 } 2975 } 2976 2977 flow_init(); 2978 2979 check_all_ports_link_status(enabled_port_mask); 2980 2981 #if (STATS_INTERVAL > 0) 2982 rte_eal_alarm_set(STATS_INTERVAL * US_PER_S, print_stats_cb, NULL); 2983 #else 2984 RTE_LOG(INFO, IPSEC, "Stats display disabled\n"); 2985 #endif /* STATS_INTERVAL */ 2986 2987 /* launch per-lcore init on every lcore */ 2988 rte_eal_mp_remote_launch(ipsec_launch_one_lcore, eh_conf, CALL_MAIN); 2989 RTE_LCORE_FOREACH_WORKER(lcore_id) { 2990 if (rte_eal_wait_lcore(lcore_id) < 0) 2991 return -1; 2992 } 2993 2994 /* Uninitialize eventmode components */ 2995 ret = eh_devs_uninit(eh_conf); 2996 if (ret < 0) 2997 rte_exit(EXIT_FAILURE, "eh_devs_uninit failed, err=%d\n", ret); 2998 2999 /* Free eventmode configuration memory */ 3000 eh_conf_uninit(eh_conf); 3001 3002 /* Destroy inline inbound and outbound sessions */ 3003 for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) { 3004 socket_id = rte_socket_id_by_idx(i); 3005 inline_sessions_free(socket_ctx[socket_id].sa_in); 3006 inline_sessions_free(socket_ctx[socket_id].sa_out); 3007 } 3008 3009 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 3010 printf("Closing cryptodev %d...", cdev_id); 3011 rte_cryptodev_stop(cdev_id); 3012 rte_cryptodev_close(cdev_id); 3013 printf(" Done\n"); 3014 } 3015 3016 RTE_ETH_FOREACH_DEV(portid) { 3017 if ((enabled_port_mask & (1 << portid)) == 0) 3018 continue; 3019 3020 printf("Closing port %d...", portid); 3021 if (flow_info_tbl[portid].rx_def_flow) { 3022 struct rte_flow_error err; 3023 3024 ret = rte_flow_destroy(portid, 3025 flow_info_tbl[portid].rx_def_flow, &err); 3026 if (ret) 3027 RTE_LOG(ERR, IPSEC, "Failed to destroy flow " 3028 " for port %u, err msg: %s\n", portid, 3029 err.message); 3030 } 3031 ret = rte_eth_dev_stop(portid); 3032 if (ret != 0) 3033 RTE_LOG(ERR, IPSEC, 3034 "rte_eth_dev_stop: err=%s, port=%u\n", 3035 rte_strerror(-ret), portid); 3036 3037 rte_eth_dev_close(portid); 3038 printf(" Done\n"); 3039 } 3040 printf("Bye...\n"); 3041 3042 return 0; 3043 } 3044