1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <stdint.h> 8 #include <inttypes.h> 9 #include <sys/types.h> 10 #include <netinet/in.h> 11 #include <netinet/ip.h> 12 #include <netinet/ip6.h> 13 #include <string.h> 14 #include <sys/queue.h> 15 #include <stdarg.h> 16 #include <errno.h> 17 #include <getopt.h> 18 19 #include <rte_common.h> 20 #include <rte_byteorder.h> 21 #include <rte_log.h> 22 #include <rte_eal.h> 23 #include <rte_launch.h> 24 #include <rte_atomic.h> 25 #include <rte_cycles.h> 26 #include <rte_prefetch.h> 27 #include <rte_lcore.h> 28 #include <rte_per_lcore.h> 29 #include <rte_branch_prediction.h> 30 #include <rte_interrupts.h> 31 #include <rte_random.h> 32 #include <rte_debug.h> 33 #include <rte_ether.h> 34 #include <rte_ethdev.h> 35 #include <rte_mempool.h> 36 #include <rte_mbuf.h> 37 #include <rte_acl.h> 38 #include <rte_lpm.h> 39 #include <rte_lpm6.h> 40 #include <rte_hash.h> 41 #include <rte_jhash.h> 42 #include <rte_cryptodev.h> 43 #include <rte_security.h> 44 45 #include "ipsec.h" 46 #include "parser.h" 47 48 #define RTE_LOGTYPE_IPSEC RTE_LOGTYPE_USER1 49 50 #define MAX_JUMBO_PKT_LEN 9600 51 52 #define MEMPOOL_CACHE_SIZE 256 53 54 #define NB_MBUF (32000) 55 56 #define CDEV_QUEUE_DESC 2048 57 #define CDEV_MAP_ENTRIES 16384 58 #define CDEV_MP_NB_OBJS 1024 59 #define CDEV_MP_CACHE_SZ 64 60 #define MAX_QUEUE_PAIRS 1 61 62 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 63 64 #define NB_SOCKETS 4 65 66 /* Configure how many packets ahead to prefetch, when reading packets */ 67 #define PREFETCH_OFFSET 3 68 69 #define MAX_RX_QUEUE_PER_LCORE 16 70 71 #define MAX_LCORE_PARAMS 1024 72 73 #define UNPROTECTED_PORT(port) (unprotected_port_mask & (1 << portid)) 74 75 /* 76 * Configurable number of RX/TX ring descriptors 77 */ 78 #define IPSEC_SECGW_RX_DESC_DEFAULT 1024 79 #define IPSEC_SECGW_TX_DESC_DEFAULT 1024 80 static uint16_t nb_rxd = IPSEC_SECGW_RX_DESC_DEFAULT; 81 static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT; 82 83 #if RTE_BYTE_ORDER != RTE_LITTLE_ENDIAN 84 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \ 85 (((uint64_t)((a) & 0xff) << 56) | \ 86 ((uint64_t)((b) & 0xff) << 48) | \ 87 ((uint64_t)((c) & 0xff) << 40) | \ 88 ((uint64_t)((d) & 0xff) << 32) | \ 89 ((uint64_t)((e) & 0xff) << 24) | \ 90 ((uint64_t)((f) & 0xff) << 16) | \ 91 ((uint64_t)((g) & 0xff) << 8) | \ 92 ((uint64_t)(h) & 0xff)) 93 #else 94 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \ 95 (((uint64_t)((h) & 0xff) << 56) | \ 96 ((uint64_t)((g) & 0xff) << 48) | \ 97 ((uint64_t)((f) & 0xff) << 40) | \ 98 ((uint64_t)((e) & 0xff) << 32) | \ 99 ((uint64_t)((d) & 0xff) << 24) | \ 100 ((uint64_t)((c) & 0xff) << 16) | \ 101 ((uint64_t)((b) & 0xff) << 8) | \ 102 ((uint64_t)(a) & 0xff)) 103 #endif 104 #define ETHADDR(a, b, c, d, e, f) (__BYTES_TO_UINT64(a, b, c, d, e, f, 0, 0)) 105 106 #define ETHADDR_TO_UINT64(addr) __BYTES_TO_UINT64( \ 107 (addr)->addr_bytes[0], (addr)->addr_bytes[1], \ 108 (addr)->addr_bytes[2], (addr)->addr_bytes[3], \ 109 (addr)->addr_bytes[4], (addr)->addr_bytes[5], \ 110 0, 0) 111 112 /* port/source ethernet addr and destination ethernet addr */ 113 struct ethaddr_info { 114 uint64_t src, dst; 115 }; 116 117 struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = { 118 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a) }, 119 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9) }, 120 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x08, 0x69, 0x26) }, 121 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd) } 122 }; 123 124 #define CMD_LINE_OPT_CONFIG "config" 125 #define CMD_LINE_OPT_SINGLE_SA "single-sa" 126 #define CMD_LINE_OPT_CRYPTODEV_MASK "cryptodev_mask" 127 #define CMD_LINE_OPT_RX_OFFLOAD "rxoffload" 128 #define CMD_LINE_OPT_TX_OFFLOAD "txoffload" 129 130 enum { 131 /* long options mapped to a short option */ 132 133 /* first long only option value must be >= 256, so that we won't 134 * conflict with short options 135 */ 136 CMD_LINE_OPT_MIN_NUM = 256, 137 CMD_LINE_OPT_CONFIG_NUM, 138 CMD_LINE_OPT_SINGLE_SA_NUM, 139 CMD_LINE_OPT_CRYPTODEV_MASK_NUM, 140 CMD_LINE_OPT_RX_OFFLOAD_NUM, 141 CMD_LINE_OPT_TX_OFFLOAD_NUM, 142 }; 143 144 static const struct option lgopts[] = { 145 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, 146 {CMD_LINE_OPT_SINGLE_SA, 1, 0, CMD_LINE_OPT_SINGLE_SA_NUM}, 147 {CMD_LINE_OPT_CRYPTODEV_MASK, 1, 0, CMD_LINE_OPT_CRYPTODEV_MASK_NUM}, 148 {CMD_LINE_OPT_RX_OFFLOAD, 1, 0, CMD_LINE_OPT_RX_OFFLOAD_NUM}, 149 {CMD_LINE_OPT_TX_OFFLOAD, 1, 0, CMD_LINE_OPT_TX_OFFLOAD_NUM}, 150 {NULL, 0, 0, 0} 151 }; 152 153 /* mask of enabled ports */ 154 static uint32_t enabled_port_mask; 155 static uint64_t enabled_cryptodev_mask = UINT64_MAX; 156 static uint32_t unprotected_port_mask; 157 static int32_t promiscuous_on = 1; 158 static int32_t numa_on = 1; /**< NUMA is enabled by default. */ 159 static uint32_t nb_lcores; 160 static uint32_t single_sa; 161 static uint32_t single_sa_idx; 162 static uint32_t frame_size; 163 164 /* 165 * RX/TX HW offload capabilities to enable/use on ethernet ports. 166 * By default all capabilities are enabled. 167 */ 168 static uint64_t dev_rx_offload = UINT64_MAX; 169 static uint64_t dev_tx_offload = UINT64_MAX; 170 171 /* application wide librte_ipsec/SA parameters */ 172 struct app_sa_prm app_sa_prm = {.enable = 0}; 173 174 struct lcore_rx_queue { 175 uint16_t port_id; 176 uint8_t queue_id; 177 } __rte_cache_aligned; 178 179 struct lcore_params { 180 uint16_t port_id; 181 uint8_t queue_id; 182 uint8_t lcore_id; 183 } __rte_cache_aligned; 184 185 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 186 187 static struct lcore_params *lcore_params; 188 static uint16_t nb_lcore_params; 189 190 static struct rte_hash *cdev_map_in; 191 static struct rte_hash *cdev_map_out; 192 193 struct buffer { 194 uint16_t len; 195 struct rte_mbuf *m_table[MAX_PKT_BURST] __rte_aligned(sizeof(void *)); 196 }; 197 198 struct lcore_conf { 199 uint16_t nb_rx_queue; 200 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 201 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 202 struct buffer tx_mbufs[RTE_MAX_ETHPORTS]; 203 struct ipsec_ctx inbound; 204 struct ipsec_ctx outbound; 205 struct rt_ctx *rt4_ctx; 206 struct rt_ctx *rt6_ctx; 207 } __rte_cache_aligned; 208 209 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 210 211 static struct rte_eth_conf port_conf = { 212 .rxmode = { 213 .mq_mode = ETH_MQ_RX_RSS, 214 .max_rx_pkt_len = ETHER_MAX_LEN, 215 .split_hdr_size = 0, 216 .offloads = DEV_RX_OFFLOAD_CHECKSUM, 217 }, 218 .rx_adv_conf = { 219 .rss_conf = { 220 .rss_key = NULL, 221 .rss_hf = ETH_RSS_IP | ETH_RSS_UDP | 222 ETH_RSS_TCP | ETH_RSS_SCTP, 223 }, 224 }, 225 .txmode = { 226 .mq_mode = ETH_MQ_TX_NONE, 227 }, 228 }; 229 230 static struct socket_ctx socket_ctx[NB_SOCKETS]; 231 232 static inline void 233 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t) 234 { 235 uint8_t *nlp; 236 struct ether_hdr *eth; 237 238 eth = rte_pktmbuf_mtod(pkt, struct ether_hdr *); 239 if (eth->ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) { 240 nlp = (uint8_t *)rte_pktmbuf_adj(pkt, ETHER_HDR_LEN); 241 nlp = RTE_PTR_ADD(nlp, offsetof(struct ip, ip_p)); 242 if (*nlp == IPPROTO_ESP) 243 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 244 else { 245 t->ip4.data[t->ip4.num] = nlp; 246 t->ip4.pkts[(t->ip4.num)++] = pkt; 247 } 248 pkt->l2_len = 0; 249 pkt->l3_len = sizeof(struct ip); 250 } else if (eth->ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) { 251 nlp = (uint8_t *)rte_pktmbuf_adj(pkt, ETHER_HDR_LEN); 252 nlp = RTE_PTR_ADD(nlp, offsetof(struct ip6_hdr, ip6_nxt)); 253 if (*nlp == IPPROTO_ESP) 254 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 255 else { 256 t->ip6.data[t->ip6.num] = nlp; 257 t->ip6.pkts[(t->ip6.num)++] = pkt; 258 } 259 pkt->l2_len = 0; 260 pkt->l3_len = sizeof(struct ip6_hdr); 261 } else { 262 /* Unknown/Unsupported type, drop the packet */ 263 RTE_LOG(ERR, IPSEC, "Unsupported packet type\n"); 264 rte_pktmbuf_free(pkt); 265 } 266 267 /* Check if the packet has been processed inline. For inline protocol 268 * processed packets, the metadata in the mbuf can be used to identify 269 * the security processing done on the packet. The metadata will be 270 * used to retrieve the application registered userdata associated 271 * with the security session. 272 */ 273 274 if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) { 275 struct ipsec_sa *sa; 276 struct ipsec_mbuf_metadata *priv; 277 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 278 rte_eth_dev_get_sec_ctx( 279 pkt->port); 280 281 /* Retrieve the userdata registered. Here, the userdata 282 * registered is the SA pointer. 283 */ 284 285 sa = (struct ipsec_sa *) 286 rte_security_get_userdata(ctx, pkt->udata64); 287 288 if (sa == NULL) { 289 /* userdata could not be retrieved */ 290 return; 291 } 292 293 /* Save SA as priv member in mbuf. This will be used in the 294 * IPsec selector(SP-SA) check. 295 */ 296 297 priv = get_priv(pkt); 298 priv->sa = sa; 299 } 300 } 301 302 static inline void 303 prepare_traffic(struct rte_mbuf **pkts, struct ipsec_traffic *t, 304 uint16_t nb_pkts) 305 { 306 int32_t i; 307 308 t->ipsec.num = 0; 309 t->ip4.num = 0; 310 t->ip6.num = 0; 311 312 for (i = 0; i < (nb_pkts - PREFETCH_OFFSET); i++) { 313 rte_prefetch0(rte_pktmbuf_mtod(pkts[i + PREFETCH_OFFSET], 314 void *)); 315 prepare_one_packet(pkts[i], t); 316 } 317 /* Process left packets */ 318 for (; i < nb_pkts; i++) 319 prepare_one_packet(pkts[i], t); 320 } 321 322 static inline void 323 prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port, 324 const struct lcore_conf *qconf) 325 { 326 struct ip *ip; 327 struct ether_hdr *ethhdr; 328 329 ip = rte_pktmbuf_mtod(pkt, struct ip *); 330 331 ethhdr = (struct ether_hdr *)rte_pktmbuf_prepend(pkt, ETHER_HDR_LEN); 332 333 if (ip->ip_v == IPVERSION) { 334 pkt->ol_flags |= qconf->outbound.ipv4_offloads; 335 pkt->l3_len = sizeof(struct ip); 336 pkt->l2_len = ETHER_HDR_LEN; 337 338 ip->ip_sum = 0; 339 340 /* calculate IPv4 cksum in SW */ 341 if ((pkt->ol_flags & PKT_TX_IP_CKSUM) == 0) 342 ip->ip_sum = rte_ipv4_cksum((struct ipv4_hdr *)ip); 343 344 ethhdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4); 345 } else { 346 pkt->ol_flags |= qconf->outbound.ipv6_offloads; 347 pkt->l3_len = sizeof(struct ip6_hdr); 348 pkt->l2_len = ETHER_HDR_LEN; 349 350 ethhdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv6); 351 } 352 353 memcpy(ðhdr->s_addr, ðaddr_tbl[port].src, 354 sizeof(struct ether_addr)); 355 memcpy(ðhdr->d_addr, ðaddr_tbl[port].dst, 356 sizeof(struct ether_addr)); 357 } 358 359 static inline void 360 prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t port, 361 const struct lcore_conf *qconf) 362 { 363 int32_t i; 364 const int32_t prefetch_offset = 2; 365 366 for (i = 0; i < (nb_pkts - prefetch_offset); i++) { 367 rte_mbuf_prefetch_part2(pkts[i + prefetch_offset]); 368 prepare_tx_pkt(pkts[i], port, qconf); 369 } 370 /* Process left packets */ 371 for (; i < nb_pkts; i++) 372 prepare_tx_pkt(pkts[i], port, qconf); 373 } 374 375 /* Send burst of packets on an output interface */ 376 static inline int32_t 377 send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) 378 { 379 struct rte_mbuf **m_table; 380 int32_t ret; 381 uint16_t queueid; 382 383 queueid = qconf->tx_queue_id[port]; 384 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 385 386 prepare_tx_burst(m_table, n, port, qconf); 387 388 ret = rte_eth_tx_burst(port, queueid, m_table, n); 389 if (unlikely(ret < n)) { 390 do { 391 rte_pktmbuf_free(m_table[ret]); 392 } while (++ret < n); 393 } 394 395 return 0; 396 } 397 398 /* Enqueue a single packet, and send burst if queue is filled */ 399 static inline int32_t 400 send_single_packet(struct rte_mbuf *m, uint16_t port) 401 { 402 uint32_t lcore_id; 403 uint16_t len; 404 struct lcore_conf *qconf; 405 406 lcore_id = rte_lcore_id(); 407 408 qconf = &lcore_conf[lcore_id]; 409 len = qconf->tx_mbufs[port].len; 410 qconf->tx_mbufs[port].m_table[len] = m; 411 len++; 412 413 /* enough pkts to be sent */ 414 if (unlikely(len == MAX_PKT_BURST)) { 415 send_burst(qconf, MAX_PKT_BURST, port); 416 len = 0; 417 } 418 419 qconf->tx_mbufs[port].len = len; 420 return 0; 421 } 422 423 static inline void 424 inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip, 425 uint16_t lim) 426 { 427 struct rte_mbuf *m; 428 uint32_t i, j, res, sa_idx; 429 430 if (ip->num == 0 || sp == NULL) 431 return; 432 433 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 434 ip->num, DEFAULT_MAX_CATEGORIES); 435 436 j = 0; 437 for (i = 0; i < ip->num; i++) { 438 m = ip->pkts[i]; 439 res = ip->res[i]; 440 if (res & BYPASS) { 441 ip->pkts[j++] = m; 442 continue; 443 } 444 if (res & DISCARD) { 445 rte_pktmbuf_free(m); 446 continue; 447 } 448 449 /* Only check SPI match for processed IPSec packets */ 450 if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) { 451 rte_pktmbuf_free(m); 452 continue; 453 } 454 455 sa_idx = ip->res[i] & PROTECT_MASK; 456 if (sa_idx >= IPSEC_SA_MAX_ENTRIES || 457 !inbound_sa_check(sa, m, sa_idx)) { 458 rte_pktmbuf_free(m); 459 continue; 460 } 461 ip->pkts[j++] = m; 462 } 463 ip->num = j; 464 } 465 466 static void 467 split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) 468 { 469 uint32_t i, n4, n6; 470 struct ip *ip; 471 struct rte_mbuf *m; 472 473 n4 = trf->ip4.num; 474 n6 = trf->ip6.num; 475 476 for (i = 0; i < num; i++) { 477 478 m = mb[i]; 479 ip = rte_pktmbuf_mtod(m, struct ip *); 480 481 if (ip->ip_v == IPVERSION) { 482 trf->ip4.pkts[n4] = m; 483 trf->ip4.data[n4] = rte_pktmbuf_mtod_offset(m, 484 uint8_t *, offsetof(struct ip, ip_p)); 485 n4++; 486 } else if (ip->ip_v == IP6_VERSION) { 487 trf->ip6.pkts[n6] = m; 488 trf->ip6.data[n6] = rte_pktmbuf_mtod_offset(m, 489 uint8_t *, 490 offsetof(struct ip6_hdr, ip6_nxt)); 491 n6++; 492 } else 493 rte_pktmbuf_free(m); 494 } 495 496 trf->ip4.num = n4; 497 trf->ip6.num = n6; 498 } 499 500 501 static inline void 502 process_pkts_inbound(struct ipsec_ctx *ipsec_ctx, 503 struct ipsec_traffic *traffic) 504 { 505 uint16_t nb_pkts_in, n_ip4, n_ip6; 506 507 n_ip4 = traffic->ip4.num; 508 n_ip6 = traffic->ip6.num; 509 510 if (app_sa_prm.enable == 0) { 511 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 512 traffic->ipsec.num, MAX_PKT_BURST); 513 split46_traffic(traffic, traffic->ipsec.pkts, nb_pkts_in); 514 } else { 515 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 516 traffic->ipsec.saptr, traffic->ipsec.num); 517 ipsec_process(ipsec_ctx, traffic); 518 } 519 520 inbound_sp_sa(ipsec_ctx->sp4_ctx, ipsec_ctx->sa_ctx, &traffic->ip4, 521 n_ip4); 522 523 inbound_sp_sa(ipsec_ctx->sp6_ctx, ipsec_ctx->sa_ctx, &traffic->ip6, 524 n_ip6); 525 } 526 527 static inline void 528 outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, 529 struct traffic_type *ipsec) 530 { 531 struct rte_mbuf *m; 532 uint32_t i, j, sa_idx; 533 534 if (ip->num == 0 || sp == NULL) 535 return; 536 537 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 538 ip->num, DEFAULT_MAX_CATEGORIES); 539 540 j = 0; 541 for (i = 0; i < ip->num; i++) { 542 m = ip->pkts[i]; 543 sa_idx = ip->res[i] & PROTECT_MASK; 544 if (ip->res[i] & DISCARD) 545 rte_pktmbuf_free(m); 546 else if (ip->res[i] & BYPASS) 547 ip->pkts[j++] = m; 548 else if (sa_idx < IPSEC_SA_MAX_ENTRIES) { 549 ipsec->res[ipsec->num] = sa_idx; 550 ipsec->pkts[ipsec->num++] = m; 551 } else /* invalid SA idx */ 552 rte_pktmbuf_free(m); 553 } 554 ip->num = j; 555 } 556 557 static inline void 558 process_pkts_outbound(struct ipsec_ctx *ipsec_ctx, 559 struct ipsec_traffic *traffic) 560 { 561 struct rte_mbuf *m; 562 uint16_t idx, nb_pkts_out, i; 563 564 /* Drop any IPsec traffic from protected ports */ 565 for (i = 0; i < traffic->ipsec.num; i++) 566 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 567 568 traffic->ipsec.num = 0; 569 570 outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec); 571 572 outbound_sp(ipsec_ctx->sp6_ctx, &traffic->ip6, &traffic->ipsec); 573 574 if (app_sa_prm.enable == 0) { 575 576 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 577 traffic->ipsec.res, traffic->ipsec.num, 578 MAX_PKT_BURST); 579 580 for (i = 0; i < nb_pkts_out; i++) { 581 m = traffic->ipsec.pkts[i]; 582 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 583 if (ip->ip_v == IPVERSION) { 584 idx = traffic->ip4.num++; 585 traffic->ip4.pkts[idx] = m; 586 } else { 587 idx = traffic->ip6.num++; 588 traffic->ip6.pkts[idx] = m; 589 } 590 } 591 } else { 592 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 593 traffic->ipsec.saptr, traffic->ipsec.num); 594 ipsec_process(ipsec_ctx, traffic); 595 } 596 } 597 598 static inline void 599 process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx, 600 struct ipsec_traffic *traffic) 601 { 602 struct rte_mbuf *m; 603 uint32_t nb_pkts_in, i, idx; 604 605 /* Drop any IPv4 traffic from unprotected ports */ 606 for (i = 0; i < traffic->ip4.num; i++) 607 rte_pktmbuf_free(traffic->ip4.pkts[i]); 608 609 traffic->ip4.num = 0; 610 611 /* Drop any IPv6 traffic from unprotected ports */ 612 for (i = 0; i < traffic->ip6.num; i++) 613 rte_pktmbuf_free(traffic->ip6.pkts[i]); 614 615 traffic->ip6.num = 0; 616 617 if (app_sa_prm.enable == 0) { 618 619 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 620 traffic->ipsec.num, MAX_PKT_BURST); 621 622 for (i = 0; i < nb_pkts_in; i++) { 623 m = traffic->ipsec.pkts[i]; 624 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 625 if (ip->ip_v == IPVERSION) { 626 idx = traffic->ip4.num++; 627 traffic->ip4.pkts[idx] = m; 628 } else { 629 idx = traffic->ip6.num++; 630 traffic->ip6.pkts[idx] = m; 631 } 632 } 633 } else { 634 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 635 traffic->ipsec.saptr, traffic->ipsec.num); 636 ipsec_process(ipsec_ctx, traffic); 637 } 638 } 639 640 static inline void 641 process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx, 642 struct ipsec_traffic *traffic) 643 { 644 struct rte_mbuf *m; 645 uint32_t nb_pkts_out, i, n; 646 struct ip *ip; 647 648 /* Drop any IPsec traffic from protected ports */ 649 for (i = 0; i < traffic->ipsec.num; i++) 650 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 651 652 n = 0; 653 654 for (i = 0; i < traffic->ip4.num; i++) { 655 traffic->ipsec.pkts[n] = traffic->ip4.pkts[i]; 656 traffic->ipsec.res[n++] = single_sa_idx; 657 } 658 659 for (i = 0; i < traffic->ip6.num; i++) { 660 traffic->ipsec.pkts[n] = traffic->ip6.pkts[i]; 661 traffic->ipsec.res[n++] = single_sa_idx; 662 } 663 664 traffic->ip4.num = 0; 665 traffic->ip6.num = 0; 666 traffic->ipsec.num = n; 667 668 if (app_sa_prm.enable == 0) { 669 670 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 671 traffic->ipsec.res, traffic->ipsec.num, 672 MAX_PKT_BURST); 673 674 /* They all sue the same SA (ip4 or ip6 tunnel) */ 675 m = traffic->ipsec.pkts[0]; 676 ip = rte_pktmbuf_mtod(m, struct ip *); 677 if (ip->ip_v == IPVERSION) { 678 traffic->ip4.num = nb_pkts_out; 679 for (i = 0; i < nb_pkts_out; i++) 680 traffic->ip4.pkts[i] = traffic->ipsec.pkts[i]; 681 } else { 682 traffic->ip6.num = nb_pkts_out; 683 for (i = 0; i < nb_pkts_out; i++) 684 traffic->ip6.pkts[i] = traffic->ipsec.pkts[i]; 685 } 686 } else { 687 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 688 traffic->ipsec.saptr, traffic->ipsec.num); 689 ipsec_process(ipsec_ctx, traffic); 690 } 691 } 692 693 static inline int32_t 694 get_hop_for_offload_pkt(struct rte_mbuf *pkt, int is_ipv6) 695 { 696 struct ipsec_mbuf_metadata *priv; 697 struct ipsec_sa *sa; 698 699 priv = get_priv(pkt); 700 701 sa = priv->sa; 702 if (unlikely(sa == NULL)) { 703 RTE_LOG(ERR, IPSEC, "SA not saved in private data\n"); 704 goto fail; 705 } 706 707 if (is_ipv6) 708 return sa->portid; 709 710 /* else */ 711 return (sa->portid | RTE_LPM_LOOKUP_SUCCESS); 712 713 fail: 714 if (is_ipv6) 715 return -1; 716 717 /* else */ 718 return 0; 719 } 720 721 static inline void 722 route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 723 { 724 uint32_t hop[MAX_PKT_BURST * 2]; 725 uint32_t dst_ip[MAX_PKT_BURST * 2]; 726 int32_t pkt_hop = 0; 727 uint16_t i, offset; 728 uint16_t lpm_pkts = 0; 729 730 if (nb_pkts == 0) 731 return; 732 733 /* Need to do an LPM lookup for non-inline packets. Inline packets will 734 * have port ID in the SA 735 */ 736 737 for (i = 0; i < nb_pkts; i++) { 738 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 739 /* Security offload not enabled. So an LPM lookup is 740 * required to get the hop 741 */ 742 offset = offsetof(struct ip, ip_dst); 743 dst_ip[lpm_pkts] = *rte_pktmbuf_mtod_offset(pkts[i], 744 uint32_t *, offset); 745 dst_ip[lpm_pkts] = rte_be_to_cpu_32(dst_ip[lpm_pkts]); 746 lpm_pkts++; 747 } 748 } 749 750 rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, lpm_pkts); 751 752 lpm_pkts = 0; 753 754 for (i = 0; i < nb_pkts; i++) { 755 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 756 /* Read hop from the SA */ 757 pkt_hop = get_hop_for_offload_pkt(pkts[i], 0); 758 } else { 759 /* Need to use hop returned by lookup */ 760 pkt_hop = hop[lpm_pkts++]; 761 } 762 763 if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) { 764 rte_pktmbuf_free(pkts[i]); 765 continue; 766 } 767 send_single_packet(pkts[i], pkt_hop & 0xff); 768 } 769 } 770 771 static inline void 772 route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 773 { 774 int32_t hop[MAX_PKT_BURST * 2]; 775 uint8_t dst_ip[MAX_PKT_BURST * 2][16]; 776 uint8_t *ip6_dst; 777 int32_t pkt_hop = 0; 778 uint16_t i, offset; 779 uint16_t lpm_pkts = 0; 780 781 if (nb_pkts == 0) 782 return; 783 784 /* Need to do an LPM lookup for non-inline packets. Inline packets will 785 * have port ID in the SA 786 */ 787 788 for (i = 0; i < nb_pkts; i++) { 789 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 790 /* Security offload not enabled. So an LPM lookup is 791 * required to get the hop 792 */ 793 offset = offsetof(struct ip6_hdr, ip6_dst); 794 ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *, 795 offset); 796 memcpy(&dst_ip[lpm_pkts][0], ip6_dst, 16); 797 lpm_pkts++; 798 } 799 } 800 801 rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, hop, 802 lpm_pkts); 803 804 lpm_pkts = 0; 805 806 for (i = 0; i < nb_pkts; i++) { 807 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 808 /* Read hop from the SA */ 809 pkt_hop = get_hop_for_offload_pkt(pkts[i], 1); 810 } else { 811 /* Need to use hop returned by lookup */ 812 pkt_hop = hop[lpm_pkts++]; 813 } 814 815 if (pkt_hop == -1) { 816 rte_pktmbuf_free(pkts[i]); 817 continue; 818 } 819 send_single_packet(pkts[i], pkt_hop & 0xff); 820 } 821 } 822 823 static inline void 824 process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts, 825 uint8_t nb_pkts, uint16_t portid) 826 { 827 struct ipsec_traffic traffic; 828 829 prepare_traffic(pkts, &traffic, nb_pkts); 830 831 if (unlikely(single_sa)) { 832 if (UNPROTECTED_PORT(portid)) 833 process_pkts_inbound_nosp(&qconf->inbound, &traffic); 834 else 835 process_pkts_outbound_nosp(&qconf->outbound, &traffic); 836 } else { 837 if (UNPROTECTED_PORT(portid)) 838 process_pkts_inbound(&qconf->inbound, &traffic); 839 else 840 process_pkts_outbound(&qconf->outbound, &traffic); 841 } 842 843 route4_pkts(qconf->rt4_ctx, traffic.ip4.pkts, traffic.ip4.num); 844 route6_pkts(qconf->rt6_ctx, traffic.ip6.pkts, traffic.ip6.num); 845 } 846 847 static inline void 848 drain_tx_buffers(struct lcore_conf *qconf) 849 { 850 struct buffer *buf; 851 uint32_t portid; 852 853 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 854 buf = &qconf->tx_mbufs[portid]; 855 if (buf->len == 0) 856 continue; 857 send_burst(qconf, buf->len, portid); 858 buf->len = 0; 859 } 860 } 861 862 static inline void 863 drain_crypto_buffers(struct lcore_conf *qconf) 864 { 865 uint32_t i; 866 struct ipsec_ctx *ctx; 867 868 /* drain inbound buffers*/ 869 ctx = &qconf->inbound; 870 for (i = 0; i != ctx->nb_qps; i++) { 871 if (ctx->tbl[i].len != 0) 872 enqueue_cop_burst(ctx->tbl + i); 873 } 874 875 /* drain outbound buffers*/ 876 ctx = &qconf->outbound; 877 for (i = 0; i != ctx->nb_qps; i++) { 878 if (ctx->tbl[i].len != 0) 879 enqueue_cop_burst(ctx->tbl + i); 880 } 881 } 882 883 static void 884 drain_inbound_crypto_queues(const struct lcore_conf *qconf, 885 struct ipsec_ctx *ctx) 886 { 887 uint32_t n; 888 struct ipsec_traffic trf; 889 890 if (app_sa_prm.enable == 0) { 891 892 /* dequeue packets from crypto-queue */ 893 n = ipsec_inbound_cqp_dequeue(ctx, trf.ipsec.pkts, 894 RTE_DIM(trf.ipsec.pkts)); 895 896 trf.ip4.num = 0; 897 trf.ip6.num = 0; 898 899 /* split traffic by ipv4-ipv6 */ 900 split46_traffic(&trf, trf.ipsec.pkts, n); 901 } else 902 ipsec_cqp_process(ctx, &trf); 903 904 /* process ipv4 packets */ 905 if (trf.ip4.num != 0) { 906 inbound_sp_sa(ctx->sp4_ctx, ctx->sa_ctx, &trf.ip4, 0); 907 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 908 } 909 910 /* process ipv6 packets */ 911 if (trf.ip6.num != 0) { 912 inbound_sp_sa(ctx->sp6_ctx, ctx->sa_ctx, &trf.ip6, 0); 913 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 914 } 915 } 916 917 static void 918 drain_outbound_crypto_queues(const struct lcore_conf *qconf, 919 struct ipsec_ctx *ctx) 920 { 921 uint32_t n; 922 struct ipsec_traffic trf; 923 924 if (app_sa_prm.enable == 0) { 925 926 /* dequeue packets from crypto-queue */ 927 n = ipsec_outbound_cqp_dequeue(ctx, trf.ipsec.pkts, 928 RTE_DIM(trf.ipsec.pkts)); 929 930 trf.ip4.num = 0; 931 trf.ip6.num = 0; 932 933 /* split traffic by ipv4-ipv6 */ 934 split46_traffic(&trf, trf.ipsec.pkts, n); 935 } else 936 ipsec_cqp_process(ctx, &trf); 937 938 /* process ipv4 packets */ 939 if (trf.ip4.num != 0) 940 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 941 942 /* process ipv6 packets */ 943 if (trf.ip6.num != 0) 944 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 945 } 946 947 /* main processing loop */ 948 static int32_t 949 main_loop(__attribute__((unused)) void *dummy) 950 { 951 struct rte_mbuf *pkts[MAX_PKT_BURST]; 952 uint32_t lcore_id; 953 uint64_t prev_tsc, diff_tsc, cur_tsc; 954 int32_t i, nb_rx; 955 uint16_t portid; 956 uint8_t queueid; 957 struct lcore_conf *qconf; 958 int32_t socket_id; 959 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) 960 / US_PER_S * BURST_TX_DRAIN_US; 961 struct lcore_rx_queue *rxql; 962 963 prev_tsc = 0; 964 lcore_id = rte_lcore_id(); 965 qconf = &lcore_conf[lcore_id]; 966 rxql = qconf->rx_queue_list; 967 socket_id = rte_lcore_to_socket_id(lcore_id); 968 969 qconf->rt4_ctx = socket_ctx[socket_id].rt_ip4; 970 qconf->rt6_ctx = socket_ctx[socket_id].rt_ip6; 971 qconf->inbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_in; 972 qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in; 973 qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in; 974 qconf->inbound.cdev_map = cdev_map_in; 975 qconf->inbound.session_pool = socket_ctx[socket_id].session_pool; 976 qconf->inbound.session_priv_pool = 977 socket_ctx[socket_id].session_priv_pool; 978 qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out; 979 qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out; 980 qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out; 981 qconf->outbound.cdev_map = cdev_map_out; 982 qconf->outbound.session_pool = socket_ctx[socket_id].session_pool; 983 qconf->outbound.session_priv_pool = 984 socket_ctx[socket_id].session_priv_pool; 985 986 if (qconf->nb_rx_queue == 0) { 987 RTE_LOG(INFO, IPSEC, "lcore %u has nothing to do\n", lcore_id); 988 return 0; 989 } 990 991 RTE_LOG(INFO, IPSEC, "entering main loop on lcore %u\n", lcore_id); 992 993 for (i = 0; i < qconf->nb_rx_queue; i++) { 994 portid = rxql[i].port_id; 995 queueid = rxql[i].queue_id; 996 RTE_LOG(INFO, IPSEC, 997 " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", 998 lcore_id, portid, queueid); 999 } 1000 1001 while (1) { 1002 cur_tsc = rte_rdtsc(); 1003 1004 /* TX queue buffer drain */ 1005 diff_tsc = cur_tsc - prev_tsc; 1006 1007 if (unlikely(diff_tsc > drain_tsc)) { 1008 drain_tx_buffers(qconf); 1009 drain_crypto_buffers(qconf); 1010 prev_tsc = cur_tsc; 1011 } 1012 1013 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1014 1015 /* Read packets from RX queues */ 1016 portid = rxql[i].port_id; 1017 queueid = rxql[i].queue_id; 1018 nb_rx = rte_eth_rx_burst(portid, queueid, 1019 pkts, MAX_PKT_BURST); 1020 1021 if (nb_rx > 0) 1022 process_pkts(qconf, pkts, nb_rx, portid); 1023 1024 /* dequeue and process completed crypto-ops */ 1025 if (UNPROTECTED_PORT(portid)) 1026 drain_inbound_crypto_queues(qconf, 1027 &qconf->inbound); 1028 else 1029 drain_outbound_crypto_queues(qconf, 1030 &qconf->outbound); 1031 } 1032 } 1033 } 1034 1035 static int32_t 1036 check_params(void) 1037 { 1038 uint8_t lcore; 1039 uint16_t portid; 1040 uint16_t i; 1041 int32_t socket_id; 1042 1043 if (lcore_params == NULL) { 1044 printf("Error: No port/queue/core mappings\n"); 1045 return -1; 1046 } 1047 1048 for (i = 0; i < nb_lcore_params; ++i) { 1049 lcore = lcore_params[i].lcore_id; 1050 if (!rte_lcore_is_enabled(lcore)) { 1051 printf("error: lcore %hhu is not enabled in " 1052 "lcore mask\n", lcore); 1053 return -1; 1054 } 1055 socket_id = rte_lcore_to_socket_id(lcore); 1056 if (socket_id != 0 && numa_on == 0) { 1057 printf("warning: lcore %hhu is on socket %d " 1058 "with numa off\n", 1059 lcore, socket_id); 1060 } 1061 portid = lcore_params[i].port_id; 1062 if ((enabled_port_mask & (1 << portid)) == 0) { 1063 printf("port %u is not enabled in port mask\n", portid); 1064 return -1; 1065 } 1066 if (!rte_eth_dev_is_valid_port(portid)) { 1067 printf("port %u is not present on the board\n", portid); 1068 return -1; 1069 } 1070 } 1071 return 0; 1072 } 1073 1074 static uint8_t 1075 get_port_nb_rx_queues(const uint16_t port) 1076 { 1077 int32_t queue = -1; 1078 uint16_t i; 1079 1080 for (i = 0; i < nb_lcore_params; ++i) { 1081 if (lcore_params[i].port_id == port && 1082 lcore_params[i].queue_id > queue) 1083 queue = lcore_params[i].queue_id; 1084 } 1085 return (uint8_t)(++queue); 1086 } 1087 1088 static int32_t 1089 init_lcore_rx_queues(void) 1090 { 1091 uint16_t i, nb_rx_queue; 1092 uint8_t lcore; 1093 1094 for (i = 0; i < nb_lcore_params; ++i) { 1095 lcore = lcore_params[i].lcore_id; 1096 nb_rx_queue = lcore_conf[lcore].nb_rx_queue; 1097 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 1098 printf("error: too many queues (%u) for lcore: %u\n", 1099 nb_rx_queue + 1, lcore); 1100 return -1; 1101 } 1102 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 1103 lcore_params[i].port_id; 1104 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 1105 lcore_params[i].queue_id; 1106 lcore_conf[lcore].nb_rx_queue++; 1107 } 1108 return 0; 1109 } 1110 1111 /* display usage */ 1112 static void 1113 print_usage(const char *prgname) 1114 { 1115 fprintf(stderr, "%s [EAL options] --" 1116 " -p PORTMASK" 1117 " [-P]" 1118 " [-u PORTMASK]" 1119 " [-j FRAMESIZE]" 1120 " [-l]" 1121 " [-w REPLAY_WINDOW_SIZE]" 1122 " [-e]" 1123 " [-a]" 1124 " -f CONFIG_FILE" 1125 " --config (port,queue,lcore)[,(port,queue,lcore)]" 1126 " [--single-sa SAIDX]" 1127 " [--cryptodev_mask MASK]" 1128 " [--" CMD_LINE_OPT_RX_OFFLOAD " RX_OFFLOAD_MASK]" 1129 " [--" CMD_LINE_OPT_TX_OFFLOAD " TX_OFFLOAD_MASK]" 1130 "\n\n" 1131 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n" 1132 " -P : Enable promiscuous mode\n" 1133 " -u PORTMASK: Hexadecimal bitmask of unprotected ports\n" 1134 " -j FRAMESIZE: Enable jumbo frame with 'FRAMESIZE' as maximum\n" 1135 " packet size\n" 1136 " -l enables code-path that uses librte_ipsec\n" 1137 " -w REPLAY_WINDOW_SIZE specifies IPsec SQN replay window\n" 1138 " size for each SA\n" 1139 " -e enables ESN\n" 1140 " -a enables SA SQN atomic behaviour\n" 1141 " -f CONFIG_FILE: Configuration file\n" 1142 " --config (port,queue,lcore): Rx queue configuration\n" 1143 " --single-sa SAIDX: Use single SA index for outbound traffic,\n" 1144 " bypassing the SP\n" 1145 " --cryptodev_mask MASK: Hexadecimal bitmask of the crypto\n" 1146 " devices to configure\n" 1147 " --" CMD_LINE_OPT_RX_OFFLOAD 1148 ": bitmask of the RX HW offload capabilities to enable/use\n" 1149 " (DEV_RX_OFFLOAD_*)\n" 1150 " --" CMD_LINE_OPT_TX_OFFLOAD 1151 ": bitmask of the TX HW offload capabilities to enable/use\n" 1152 " (DEV_TX_OFFLOAD_*)\n" 1153 "\n", 1154 prgname); 1155 } 1156 1157 static int 1158 parse_mask(const char *str, uint64_t *val) 1159 { 1160 char *end; 1161 unsigned long t; 1162 1163 errno = 0; 1164 t = strtoul(str, &end, 0); 1165 if (errno != 0 || end[0] != 0) 1166 return -EINVAL; 1167 1168 *val = t; 1169 return 0; 1170 } 1171 1172 static int32_t 1173 parse_portmask(const char *portmask) 1174 { 1175 char *end = NULL; 1176 unsigned long pm; 1177 1178 /* parse hexadecimal string */ 1179 pm = strtoul(portmask, &end, 16); 1180 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 1181 return -1; 1182 1183 if ((pm == 0) && errno) 1184 return -1; 1185 1186 return pm; 1187 } 1188 1189 static int32_t 1190 parse_decimal(const char *str) 1191 { 1192 char *end = NULL; 1193 unsigned long num; 1194 1195 num = strtoul(str, &end, 10); 1196 if ((str[0] == '\0') || (end == NULL) || (*end != '\0')) 1197 return -1; 1198 1199 return num; 1200 } 1201 1202 static int32_t 1203 parse_config(const char *q_arg) 1204 { 1205 char s[256]; 1206 const char *p, *p0 = q_arg; 1207 char *end; 1208 enum fieldnames { 1209 FLD_PORT = 0, 1210 FLD_QUEUE, 1211 FLD_LCORE, 1212 _NUM_FLD 1213 }; 1214 unsigned long int_fld[_NUM_FLD]; 1215 char *str_fld[_NUM_FLD]; 1216 int32_t i; 1217 uint32_t size; 1218 1219 nb_lcore_params = 0; 1220 1221 while ((p = strchr(p0, '(')) != NULL) { 1222 ++p; 1223 p0 = strchr(p, ')'); 1224 if (p0 == NULL) 1225 return -1; 1226 1227 size = p0 - p; 1228 if (size >= sizeof(s)) 1229 return -1; 1230 1231 snprintf(s, sizeof(s), "%.*s", size, p); 1232 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 1233 _NUM_FLD) 1234 return -1; 1235 for (i = 0; i < _NUM_FLD; i++) { 1236 errno = 0; 1237 int_fld[i] = strtoul(str_fld[i], &end, 0); 1238 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 1239 return -1; 1240 } 1241 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 1242 printf("exceeded max number of lcore params: %hu\n", 1243 nb_lcore_params); 1244 return -1; 1245 } 1246 lcore_params_array[nb_lcore_params].port_id = 1247 (uint8_t)int_fld[FLD_PORT]; 1248 lcore_params_array[nb_lcore_params].queue_id = 1249 (uint8_t)int_fld[FLD_QUEUE]; 1250 lcore_params_array[nb_lcore_params].lcore_id = 1251 (uint8_t)int_fld[FLD_LCORE]; 1252 ++nb_lcore_params; 1253 } 1254 lcore_params = lcore_params_array; 1255 return 0; 1256 } 1257 1258 static void 1259 print_app_sa_prm(const struct app_sa_prm *prm) 1260 { 1261 printf("librte_ipsec usage: %s\n", 1262 (prm->enable == 0) ? "disabled" : "enabled"); 1263 1264 if (prm->enable == 0) 1265 return; 1266 1267 printf("replay window size: %u\n", prm->window_size); 1268 printf("ESN: %s\n", (prm->enable_esn == 0) ? "disabled" : "enabled"); 1269 printf("SA flags: %#" PRIx64 "\n", prm->flags); 1270 } 1271 1272 static int32_t 1273 parse_args(int32_t argc, char **argv) 1274 { 1275 int32_t opt, ret; 1276 char **argvopt; 1277 int32_t option_index; 1278 char *prgname = argv[0]; 1279 int32_t f_present = 0; 1280 1281 argvopt = argv; 1282 1283 while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:", 1284 lgopts, &option_index)) != EOF) { 1285 1286 switch (opt) { 1287 case 'p': 1288 enabled_port_mask = parse_portmask(optarg); 1289 if (enabled_port_mask == 0) { 1290 printf("invalid portmask\n"); 1291 print_usage(prgname); 1292 return -1; 1293 } 1294 break; 1295 case 'P': 1296 printf("Promiscuous mode selected\n"); 1297 promiscuous_on = 1; 1298 break; 1299 case 'u': 1300 unprotected_port_mask = parse_portmask(optarg); 1301 if (unprotected_port_mask == 0) { 1302 printf("invalid unprotected portmask\n"); 1303 print_usage(prgname); 1304 return -1; 1305 } 1306 break; 1307 case 'f': 1308 if (f_present == 1) { 1309 printf("\"-f\" option present more than " 1310 "once!\n"); 1311 print_usage(prgname); 1312 return -1; 1313 } 1314 if (parse_cfg_file(optarg) < 0) { 1315 printf("parsing file \"%s\" failed\n", 1316 optarg); 1317 print_usage(prgname); 1318 return -1; 1319 } 1320 f_present = 1; 1321 break; 1322 case 'j': 1323 { 1324 int32_t size = parse_decimal(optarg); 1325 if (size <= 1518) { 1326 printf("Invalid jumbo frame size\n"); 1327 if (size < 0) { 1328 print_usage(prgname); 1329 return -1; 1330 } 1331 printf("Using default value 9000\n"); 1332 frame_size = 9000; 1333 } else { 1334 frame_size = size; 1335 } 1336 } 1337 printf("Enabled jumbo frames size %u\n", frame_size); 1338 break; 1339 case 'l': 1340 app_sa_prm.enable = 1; 1341 break; 1342 case 'w': 1343 app_sa_prm.enable = 1; 1344 app_sa_prm.window_size = parse_decimal(optarg); 1345 break; 1346 case 'e': 1347 app_sa_prm.enable = 1; 1348 app_sa_prm.enable_esn = 1; 1349 break; 1350 case 'a': 1351 app_sa_prm.enable = 1; 1352 app_sa_prm.flags |= RTE_IPSEC_SAFLAG_SQN_ATOM; 1353 break; 1354 case CMD_LINE_OPT_CONFIG_NUM: 1355 ret = parse_config(optarg); 1356 if (ret) { 1357 printf("Invalid config\n"); 1358 print_usage(prgname); 1359 return -1; 1360 } 1361 break; 1362 case CMD_LINE_OPT_SINGLE_SA_NUM: 1363 ret = parse_decimal(optarg); 1364 if (ret == -1) { 1365 printf("Invalid argument[sa_idx]\n"); 1366 print_usage(prgname); 1367 return -1; 1368 } 1369 1370 /* else */ 1371 single_sa = 1; 1372 single_sa_idx = ret; 1373 printf("Configured with single SA index %u\n", 1374 single_sa_idx); 1375 break; 1376 case CMD_LINE_OPT_CRYPTODEV_MASK_NUM: 1377 ret = parse_portmask(optarg); 1378 if (ret == -1) { 1379 printf("Invalid argument[portmask]\n"); 1380 print_usage(prgname); 1381 return -1; 1382 } 1383 1384 /* else */ 1385 enabled_cryptodev_mask = ret; 1386 break; 1387 case CMD_LINE_OPT_RX_OFFLOAD_NUM: 1388 ret = parse_mask(optarg, &dev_rx_offload); 1389 if (ret != 0) { 1390 printf("Invalid argument for \'%s\': %s\n", 1391 CMD_LINE_OPT_RX_OFFLOAD, optarg); 1392 print_usage(prgname); 1393 return -1; 1394 } 1395 break; 1396 case CMD_LINE_OPT_TX_OFFLOAD_NUM: 1397 ret = parse_mask(optarg, &dev_tx_offload); 1398 if (ret != 0) { 1399 printf("Invalid argument for \'%s\': %s\n", 1400 CMD_LINE_OPT_TX_OFFLOAD, optarg); 1401 print_usage(prgname); 1402 return -1; 1403 } 1404 break; 1405 default: 1406 print_usage(prgname); 1407 return -1; 1408 } 1409 } 1410 1411 if (f_present == 0) { 1412 printf("Mandatory option \"-f\" not present\n"); 1413 return -1; 1414 } 1415 1416 print_app_sa_prm(&app_sa_prm); 1417 1418 if (optind >= 0) 1419 argv[optind-1] = prgname; 1420 1421 ret = optind-1; 1422 optind = 1; /* reset getopt lib */ 1423 return ret; 1424 } 1425 1426 static void 1427 print_ethaddr(const char *name, const struct ether_addr *eth_addr) 1428 { 1429 char buf[ETHER_ADDR_FMT_SIZE]; 1430 ether_format_addr(buf, ETHER_ADDR_FMT_SIZE, eth_addr); 1431 printf("%s%s", name, buf); 1432 } 1433 1434 /* 1435 * Update destination ethaddr for the port. 1436 */ 1437 int 1438 add_dst_ethaddr(uint16_t port, const struct ether_addr *addr) 1439 { 1440 if (port > RTE_DIM(ethaddr_tbl)) 1441 return -EINVAL; 1442 1443 ethaddr_tbl[port].dst = ETHADDR_TO_UINT64(addr); 1444 return 0; 1445 } 1446 1447 /* Check the link status of all ports in up to 9s, and print them finally */ 1448 static void 1449 check_all_ports_link_status(uint32_t port_mask) 1450 { 1451 #define CHECK_INTERVAL 100 /* 100ms */ 1452 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1453 uint16_t portid; 1454 uint8_t count, all_ports_up, print_flag = 0; 1455 struct rte_eth_link link; 1456 1457 printf("\nChecking link status"); 1458 fflush(stdout); 1459 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1460 all_ports_up = 1; 1461 RTE_ETH_FOREACH_DEV(portid) { 1462 if ((port_mask & (1 << portid)) == 0) 1463 continue; 1464 memset(&link, 0, sizeof(link)); 1465 rte_eth_link_get_nowait(portid, &link); 1466 /* print link status if flag set */ 1467 if (print_flag == 1) { 1468 if (link.link_status) 1469 printf( 1470 "Port%d Link Up - speed %u Mbps -%s\n", 1471 portid, link.link_speed, 1472 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1473 ("full-duplex") : ("half-duplex\n")); 1474 else 1475 printf("Port %d Link Down\n", portid); 1476 continue; 1477 } 1478 /* clear all_ports_up flag if any link down */ 1479 if (link.link_status == ETH_LINK_DOWN) { 1480 all_ports_up = 0; 1481 break; 1482 } 1483 } 1484 /* after finally printing all link status, get out */ 1485 if (print_flag == 1) 1486 break; 1487 1488 if (all_ports_up == 0) { 1489 printf("."); 1490 fflush(stdout); 1491 rte_delay_ms(CHECK_INTERVAL); 1492 } 1493 1494 /* set the print_flag if all ports up or timeout */ 1495 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1496 print_flag = 1; 1497 printf("done\n"); 1498 } 1499 } 1500 } 1501 1502 static int32_t 1503 add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id, 1504 uint16_t qp, struct lcore_params *params, 1505 struct ipsec_ctx *ipsec_ctx, 1506 const struct rte_cryptodev_capabilities *cipher, 1507 const struct rte_cryptodev_capabilities *auth, 1508 const struct rte_cryptodev_capabilities *aead) 1509 { 1510 int32_t ret = 0; 1511 unsigned long i; 1512 struct cdev_key key = { 0 }; 1513 1514 key.lcore_id = params->lcore_id; 1515 if (cipher) 1516 key.cipher_algo = cipher->sym.cipher.algo; 1517 if (auth) 1518 key.auth_algo = auth->sym.auth.algo; 1519 if (aead) 1520 key.aead_algo = aead->sym.aead.algo; 1521 1522 ret = rte_hash_lookup(map, &key); 1523 if (ret != -ENOENT) 1524 return 0; 1525 1526 for (i = 0; i < ipsec_ctx->nb_qps; i++) 1527 if (ipsec_ctx->tbl[i].id == cdev_id) 1528 break; 1529 1530 if (i == ipsec_ctx->nb_qps) { 1531 if (ipsec_ctx->nb_qps == MAX_QP_PER_LCORE) { 1532 printf("Maximum number of crypto devices assigned to " 1533 "a core, increase MAX_QP_PER_LCORE value\n"); 1534 return 0; 1535 } 1536 ipsec_ctx->tbl[i].id = cdev_id; 1537 ipsec_ctx->tbl[i].qp = qp; 1538 ipsec_ctx->nb_qps++; 1539 printf("%s cdev mapping: lcore %u using cdev %u qp %u " 1540 "(cdev_id_qp %lu)\n", str, key.lcore_id, 1541 cdev_id, qp, i); 1542 } 1543 1544 ret = rte_hash_add_key_data(map, &key, (void *)i); 1545 if (ret < 0) { 1546 printf("Faled to insert cdev mapping for (lcore %u, " 1547 "cdev %u, qp %u), errno %d\n", 1548 key.lcore_id, ipsec_ctx->tbl[i].id, 1549 ipsec_ctx->tbl[i].qp, ret); 1550 return 0; 1551 } 1552 1553 return 1; 1554 } 1555 1556 static int32_t 1557 add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id, 1558 uint16_t qp, struct lcore_params *params) 1559 { 1560 int32_t ret = 0; 1561 const struct rte_cryptodev_capabilities *i, *j; 1562 struct rte_hash *map; 1563 struct lcore_conf *qconf; 1564 struct ipsec_ctx *ipsec_ctx; 1565 const char *str; 1566 1567 qconf = &lcore_conf[params->lcore_id]; 1568 1569 if ((unprotected_port_mask & (1 << params->port_id)) == 0) { 1570 map = cdev_map_out; 1571 ipsec_ctx = &qconf->outbound; 1572 str = "Outbound"; 1573 } else { 1574 map = cdev_map_in; 1575 ipsec_ctx = &qconf->inbound; 1576 str = "Inbound"; 1577 } 1578 1579 /* Required cryptodevs with operation chainning */ 1580 if (!(dev_info->feature_flags & 1581 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING)) 1582 return ret; 1583 1584 for (i = dev_info->capabilities; 1585 i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++) { 1586 if (i->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1587 continue; 1588 1589 if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1590 ret |= add_mapping(map, str, cdev_id, qp, params, 1591 ipsec_ctx, NULL, NULL, i); 1592 continue; 1593 } 1594 1595 if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) 1596 continue; 1597 1598 for (j = dev_info->capabilities; 1599 j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; j++) { 1600 if (j->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1601 continue; 1602 1603 if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH) 1604 continue; 1605 1606 ret |= add_mapping(map, str, cdev_id, qp, params, 1607 ipsec_ctx, i, j, NULL); 1608 } 1609 } 1610 1611 return ret; 1612 } 1613 1614 /* Check if the device is enabled by cryptodev_mask */ 1615 static int 1616 check_cryptodev_mask(uint8_t cdev_id) 1617 { 1618 if (enabled_cryptodev_mask & (1 << cdev_id)) 1619 return 0; 1620 1621 return -1; 1622 } 1623 1624 static int32_t 1625 cryptodevs_init(void) 1626 { 1627 struct rte_cryptodev_config dev_conf; 1628 struct rte_cryptodev_qp_conf qp_conf; 1629 uint16_t idx, max_nb_qps, qp, i; 1630 int16_t cdev_id, port_id; 1631 struct rte_hash_parameters params = { 0 }; 1632 1633 params.entries = CDEV_MAP_ENTRIES; 1634 params.key_len = sizeof(struct cdev_key); 1635 params.hash_func = rte_jhash; 1636 params.hash_func_init_val = 0; 1637 params.socket_id = rte_socket_id(); 1638 1639 params.name = "cdev_map_in"; 1640 cdev_map_in = rte_hash_create(¶ms); 1641 if (cdev_map_in == NULL) 1642 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1643 rte_errno); 1644 1645 params.name = "cdev_map_out"; 1646 cdev_map_out = rte_hash_create(¶ms); 1647 if (cdev_map_out == NULL) 1648 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1649 rte_errno); 1650 1651 printf("lcore/cryptodev/qp mappings:\n"); 1652 1653 uint32_t max_sess_sz = 0, sess_sz; 1654 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 1655 void *sec_ctx; 1656 1657 /* Get crypto priv session size */ 1658 sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id); 1659 if (sess_sz > max_sess_sz) 1660 max_sess_sz = sess_sz; 1661 1662 /* 1663 * If crypto device is security capable, need to check the 1664 * size of security session as well. 1665 */ 1666 1667 /* Get security context of the crypto device */ 1668 sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id); 1669 if (sec_ctx == NULL) 1670 continue; 1671 1672 /* Get size of security session */ 1673 sess_sz = rte_security_session_get_size(sec_ctx); 1674 if (sess_sz > max_sess_sz) 1675 max_sess_sz = sess_sz; 1676 } 1677 RTE_ETH_FOREACH_DEV(port_id) { 1678 void *sec_ctx; 1679 1680 if ((enabled_port_mask & (1 << port_id)) == 0) 1681 continue; 1682 1683 sec_ctx = rte_eth_dev_get_sec_ctx(port_id); 1684 if (sec_ctx == NULL) 1685 continue; 1686 1687 sess_sz = rte_security_session_get_size(sec_ctx); 1688 if (sess_sz > max_sess_sz) 1689 max_sess_sz = sess_sz; 1690 } 1691 1692 idx = 0; 1693 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 1694 struct rte_cryptodev_info cdev_info; 1695 1696 if (check_cryptodev_mask((uint8_t)cdev_id)) 1697 continue; 1698 1699 rte_cryptodev_info_get(cdev_id, &cdev_info); 1700 1701 if (nb_lcore_params > cdev_info.max_nb_queue_pairs) 1702 max_nb_qps = cdev_info.max_nb_queue_pairs; 1703 else 1704 max_nb_qps = nb_lcore_params; 1705 1706 qp = 0; 1707 i = 0; 1708 while (qp < max_nb_qps && i < nb_lcore_params) { 1709 if (add_cdev_mapping(&cdev_info, cdev_id, qp, 1710 &lcore_params[idx])) 1711 qp++; 1712 idx++; 1713 idx = idx % nb_lcore_params; 1714 i++; 1715 } 1716 1717 if (qp == 0) 1718 continue; 1719 1720 dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id); 1721 dev_conf.nb_queue_pairs = qp; 1722 1723 uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions; 1724 if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS) 1725 rte_exit(EXIT_FAILURE, 1726 "Device does not support at least %u " 1727 "sessions", CDEV_MP_NB_OBJS); 1728 1729 if (!socket_ctx[dev_conf.socket_id].session_pool) { 1730 char mp_name[RTE_MEMPOOL_NAMESIZE]; 1731 struct rte_mempool *sess_mp; 1732 1733 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 1734 "sess_mp_%u", dev_conf.socket_id); 1735 sess_mp = rte_cryptodev_sym_session_pool_create( 1736 mp_name, CDEV_MP_NB_OBJS, 1737 0, CDEV_MP_CACHE_SZ, 0, 1738 dev_conf.socket_id); 1739 socket_ctx[dev_conf.socket_id].session_pool = sess_mp; 1740 } 1741 1742 if (!socket_ctx[dev_conf.socket_id].session_priv_pool) { 1743 char mp_name[RTE_MEMPOOL_NAMESIZE]; 1744 struct rte_mempool *sess_mp; 1745 1746 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 1747 "sess_mp_priv_%u", dev_conf.socket_id); 1748 sess_mp = rte_mempool_create(mp_name, 1749 CDEV_MP_NB_OBJS, 1750 max_sess_sz, 1751 CDEV_MP_CACHE_SZ, 1752 0, NULL, NULL, NULL, 1753 NULL, dev_conf.socket_id, 1754 0); 1755 socket_ctx[dev_conf.socket_id].session_priv_pool = 1756 sess_mp; 1757 } 1758 1759 if (!socket_ctx[dev_conf.socket_id].session_priv_pool || 1760 !socket_ctx[dev_conf.socket_id].session_pool) 1761 rte_exit(EXIT_FAILURE, 1762 "Cannot create session pool on socket %d\n", 1763 dev_conf.socket_id); 1764 else 1765 printf("Allocated session pool on socket %d\n", 1766 dev_conf.socket_id); 1767 1768 if (rte_cryptodev_configure(cdev_id, &dev_conf)) 1769 rte_panic("Failed to initialize cryptodev %u\n", 1770 cdev_id); 1771 1772 qp_conf.nb_descriptors = CDEV_QUEUE_DESC; 1773 qp_conf.mp_session = 1774 socket_ctx[dev_conf.socket_id].session_pool; 1775 qp_conf.mp_session_private = 1776 socket_ctx[dev_conf.socket_id].session_priv_pool; 1777 for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++) 1778 if (rte_cryptodev_queue_pair_setup(cdev_id, qp, 1779 &qp_conf, dev_conf.socket_id)) 1780 rte_panic("Failed to setup queue %u for " 1781 "cdev_id %u\n", 0, cdev_id); 1782 1783 if (rte_cryptodev_start(cdev_id)) 1784 rte_panic("Failed to start cryptodev %u\n", 1785 cdev_id); 1786 } 1787 1788 /* create session pools for eth devices that implement security */ 1789 RTE_ETH_FOREACH_DEV(port_id) { 1790 if ((enabled_port_mask & (1 << port_id)) && 1791 rte_eth_dev_get_sec_ctx(port_id)) { 1792 int socket_id = rte_eth_dev_socket_id(port_id); 1793 1794 if (!socket_ctx[socket_id].session_pool) { 1795 char mp_name[RTE_MEMPOOL_NAMESIZE]; 1796 struct rte_mempool *sess_mp; 1797 1798 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 1799 "sess_mp_%u", socket_id); 1800 sess_mp = rte_mempool_create(mp_name, 1801 (CDEV_MP_NB_OBJS * 2), 1802 max_sess_sz, 1803 CDEV_MP_CACHE_SZ, 1804 0, NULL, NULL, NULL, 1805 NULL, socket_id, 1806 0); 1807 if (sess_mp == NULL) 1808 rte_exit(EXIT_FAILURE, 1809 "Cannot create session pool " 1810 "on socket %d\n", socket_id); 1811 else 1812 printf("Allocated session pool " 1813 "on socket %d\n", socket_id); 1814 socket_ctx[socket_id].session_pool = sess_mp; 1815 } 1816 } 1817 } 1818 1819 1820 printf("\n"); 1821 1822 return 0; 1823 } 1824 1825 static void 1826 port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads) 1827 { 1828 struct rte_eth_dev_info dev_info; 1829 struct rte_eth_txconf *txconf; 1830 uint16_t nb_tx_queue, nb_rx_queue; 1831 uint16_t tx_queueid, rx_queueid, queue, lcore_id; 1832 int32_t ret, socket_id; 1833 struct lcore_conf *qconf; 1834 struct ether_addr ethaddr; 1835 struct rte_eth_conf local_port_conf = port_conf; 1836 1837 rte_eth_dev_info_get(portid, &dev_info); 1838 1839 /* limit allowed HW offloafs, as user requested */ 1840 dev_info.rx_offload_capa &= dev_rx_offload; 1841 dev_info.tx_offload_capa &= dev_tx_offload; 1842 1843 printf("Configuring device port %u:\n", portid); 1844 1845 rte_eth_macaddr_get(portid, ðaddr); 1846 ethaddr_tbl[portid].src = ETHADDR_TO_UINT64(ðaddr); 1847 print_ethaddr("Address: ", ðaddr); 1848 printf("\n"); 1849 1850 nb_rx_queue = get_port_nb_rx_queues(portid); 1851 nb_tx_queue = nb_lcores; 1852 1853 if (nb_rx_queue > dev_info.max_rx_queues) 1854 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 1855 "(max rx queue is %u)\n", 1856 nb_rx_queue, dev_info.max_rx_queues); 1857 1858 if (nb_tx_queue > dev_info.max_tx_queues) 1859 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 1860 "(max tx queue is %u)\n", 1861 nb_tx_queue, dev_info.max_tx_queues); 1862 1863 printf("Creating queues: nb_rx_queue=%d nb_tx_queue=%u...\n", 1864 nb_rx_queue, nb_tx_queue); 1865 1866 if (frame_size) { 1867 local_port_conf.rxmode.max_rx_pkt_len = frame_size; 1868 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1869 } 1870 1871 local_port_conf.rxmode.offloads |= req_rx_offloads; 1872 local_port_conf.txmode.offloads |= req_tx_offloads; 1873 1874 /* Check that all required capabilities are supported */ 1875 if ((local_port_conf.rxmode.offloads & dev_info.rx_offload_capa) != 1876 local_port_conf.rxmode.offloads) 1877 rte_exit(EXIT_FAILURE, 1878 "Error: port %u required RX offloads: 0x%" PRIx64 1879 ", avaialbe RX offloads: 0x%" PRIx64 "\n", 1880 portid, local_port_conf.rxmode.offloads, 1881 dev_info.rx_offload_capa); 1882 1883 if ((local_port_conf.txmode.offloads & dev_info.tx_offload_capa) != 1884 local_port_conf.txmode.offloads) 1885 rte_exit(EXIT_FAILURE, 1886 "Error: port %u required TX offloads: 0x%" PRIx64 1887 ", avaialbe TX offloads: 0x%" PRIx64 "\n", 1888 portid, local_port_conf.txmode.offloads, 1889 dev_info.tx_offload_capa); 1890 1891 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 1892 local_port_conf.txmode.offloads |= 1893 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1894 1895 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 1896 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 1897 1898 printf("port %u configurng rx_offloads=0x%" PRIx64 1899 ", tx_offloads=0x%" PRIx64 "\n", 1900 portid, local_port_conf.rxmode.offloads, 1901 local_port_conf.txmode.offloads); 1902 1903 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 1904 dev_info.flow_type_rss_offloads; 1905 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 1906 port_conf.rx_adv_conf.rss_conf.rss_hf) { 1907 printf("Port %u modified RSS hash function based on hardware support," 1908 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 1909 portid, 1910 port_conf.rx_adv_conf.rss_conf.rss_hf, 1911 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 1912 } 1913 1914 ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue, 1915 &local_port_conf); 1916 if (ret < 0) 1917 rte_exit(EXIT_FAILURE, "Cannot configure device: " 1918 "err=%d, port=%d\n", ret, portid); 1919 1920 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd); 1921 if (ret < 0) 1922 rte_exit(EXIT_FAILURE, "Cannot adjust number of descriptors: " 1923 "err=%d, port=%d\n", ret, portid); 1924 1925 /* init one TX queue per lcore */ 1926 tx_queueid = 0; 1927 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1928 if (rte_lcore_is_enabled(lcore_id) == 0) 1929 continue; 1930 1931 if (numa_on) 1932 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 1933 else 1934 socket_id = 0; 1935 1936 /* init TX queue */ 1937 printf("Setup txq=%u,%d,%d\n", lcore_id, tx_queueid, socket_id); 1938 1939 txconf = &dev_info.default_txconf; 1940 txconf->offloads = local_port_conf.txmode.offloads; 1941 1942 ret = rte_eth_tx_queue_setup(portid, tx_queueid, nb_txd, 1943 socket_id, txconf); 1944 if (ret < 0) 1945 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: " 1946 "err=%d, port=%d\n", ret, portid); 1947 1948 qconf = &lcore_conf[lcore_id]; 1949 qconf->tx_queue_id[portid] = tx_queueid; 1950 1951 /* Pre-populate pkt offloads based on capabilities */ 1952 qconf->outbound.ipv4_offloads = PKT_TX_IPV4; 1953 qconf->outbound.ipv6_offloads = PKT_TX_IPV6; 1954 if (local_port_conf.txmode.offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) 1955 qconf->outbound.ipv4_offloads |= PKT_TX_IP_CKSUM; 1956 1957 tx_queueid++; 1958 1959 /* init RX queues */ 1960 for (queue = 0; queue < qconf->nb_rx_queue; ++queue) { 1961 struct rte_eth_rxconf rxq_conf; 1962 1963 if (portid != qconf->rx_queue_list[queue].port_id) 1964 continue; 1965 1966 rx_queueid = qconf->rx_queue_list[queue].queue_id; 1967 1968 printf("Setup rxq=%d,%d,%d\n", portid, rx_queueid, 1969 socket_id); 1970 1971 rxq_conf = dev_info.default_rxconf; 1972 rxq_conf.offloads = local_port_conf.rxmode.offloads; 1973 ret = rte_eth_rx_queue_setup(portid, rx_queueid, 1974 nb_rxd, socket_id, &rxq_conf, 1975 socket_ctx[socket_id].mbuf_pool); 1976 if (ret < 0) 1977 rte_exit(EXIT_FAILURE, 1978 "rte_eth_rx_queue_setup: err=%d, " 1979 "port=%d\n", ret, portid); 1980 } 1981 } 1982 printf("\n"); 1983 } 1984 1985 static void 1986 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf) 1987 { 1988 char s[64]; 1989 uint32_t buff_size = frame_size ? (frame_size + RTE_PKTMBUF_HEADROOM) : 1990 RTE_MBUF_DEFAULT_BUF_SIZE; 1991 1992 1993 snprintf(s, sizeof(s), "mbuf_pool_%d", socket_id); 1994 ctx->mbuf_pool = rte_pktmbuf_pool_create(s, nb_mbuf, 1995 MEMPOOL_CACHE_SIZE, ipsec_metadata_size(), 1996 buff_size, 1997 socket_id); 1998 if (ctx->mbuf_pool == NULL) 1999 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n", 2000 socket_id); 2001 else 2002 printf("Allocated mbuf pool on socket %d\n", socket_id); 2003 } 2004 2005 static inline int 2006 inline_ipsec_event_esn_overflow(struct rte_security_ctx *ctx, uint64_t md) 2007 { 2008 struct ipsec_sa *sa; 2009 2010 /* For inline protocol processing, the metadata in the event will 2011 * uniquely identify the security session which raised the event. 2012 * Application would then need the userdata it had registered with the 2013 * security session to process the event. 2014 */ 2015 2016 sa = (struct ipsec_sa *)rte_security_get_userdata(ctx, md); 2017 2018 if (sa == NULL) { 2019 /* userdata could not be retrieved */ 2020 return -1; 2021 } 2022 2023 /* Sequence number over flow. SA need to be re-established */ 2024 RTE_SET_USED(sa); 2025 return 0; 2026 } 2027 2028 static int 2029 inline_ipsec_event_callback(uint16_t port_id, enum rte_eth_event_type type, 2030 void *param, void *ret_param) 2031 { 2032 uint64_t md; 2033 struct rte_eth_event_ipsec_desc *event_desc = NULL; 2034 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 2035 rte_eth_dev_get_sec_ctx(port_id); 2036 2037 RTE_SET_USED(param); 2038 2039 if (type != RTE_ETH_EVENT_IPSEC) 2040 return -1; 2041 2042 event_desc = ret_param; 2043 if (event_desc == NULL) { 2044 printf("Event descriptor not set\n"); 2045 return -1; 2046 } 2047 2048 md = event_desc->metadata; 2049 2050 if (event_desc->subtype == RTE_ETH_EVENT_IPSEC_ESN_OVERFLOW) 2051 return inline_ipsec_event_esn_overflow(ctx, md); 2052 else if (event_desc->subtype >= RTE_ETH_EVENT_IPSEC_MAX) { 2053 printf("Invalid IPsec event reported\n"); 2054 return -1; 2055 } 2056 2057 return -1; 2058 } 2059 2060 int32_t 2061 main(int32_t argc, char **argv) 2062 { 2063 int32_t ret; 2064 uint32_t lcore_id; 2065 uint8_t socket_id; 2066 uint16_t portid; 2067 uint64_t req_rx_offloads, req_tx_offloads; 2068 2069 /* init EAL */ 2070 ret = rte_eal_init(argc, argv); 2071 if (ret < 0) 2072 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 2073 argc -= ret; 2074 argv += ret; 2075 2076 /* parse application arguments (after the EAL ones) */ 2077 ret = parse_args(argc, argv); 2078 if (ret < 0) 2079 rte_exit(EXIT_FAILURE, "Invalid parameters\n"); 2080 2081 if ((unprotected_port_mask & enabled_port_mask) != 2082 unprotected_port_mask) 2083 rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n", 2084 unprotected_port_mask); 2085 2086 if (check_params() < 0) 2087 rte_exit(EXIT_FAILURE, "check_params failed\n"); 2088 2089 ret = init_lcore_rx_queues(); 2090 if (ret < 0) 2091 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 2092 2093 nb_lcores = rte_lcore_count(); 2094 2095 /* Replicate each context per socket */ 2096 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2097 if (rte_lcore_is_enabled(lcore_id) == 0) 2098 continue; 2099 2100 if (numa_on) 2101 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2102 else 2103 socket_id = 0; 2104 2105 if (socket_ctx[socket_id].mbuf_pool) 2106 continue; 2107 2108 /* initilaze SPD */ 2109 sp4_init(&socket_ctx[socket_id], socket_id); 2110 2111 sp6_init(&socket_ctx[socket_id], socket_id); 2112 2113 /* initilaze SAD */ 2114 sa_init(&socket_ctx[socket_id], socket_id); 2115 2116 rt_init(&socket_ctx[socket_id], socket_id); 2117 2118 pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF); 2119 } 2120 2121 RTE_ETH_FOREACH_DEV(portid) { 2122 if ((enabled_port_mask & (1 << portid)) == 0) 2123 continue; 2124 2125 sa_check_offloads(portid, &req_rx_offloads, &req_tx_offloads); 2126 port_init(portid, req_rx_offloads, req_tx_offloads); 2127 } 2128 2129 cryptodevs_init(); 2130 2131 /* start ports */ 2132 RTE_ETH_FOREACH_DEV(portid) { 2133 if ((enabled_port_mask & (1 << portid)) == 0) 2134 continue; 2135 2136 /* Start device */ 2137 ret = rte_eth_dev_start(portid); 2138 if (ret < 0) 2139 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: " 2140 "err=%d, port=%d\n", ret, portid); 2141 /* 2142 * If enabled, put device in promiscuous mode. 2143 * This allows IO forwarding mode to forward packets 2144 * to itself through 2 cross-connected ports of the 2145 * target machine. 2146 */ 2147 if (promiscuous_on) 2148 rte_eth_promiscuous_enable(portid); 2149 2150 rte_eth_dev_callback_register(portid, 2151 RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL); 2152 } 2153 2154 check_all_ports_link_status(enabled_port_mask); 2155 2156 /* launch per-lcore init on every lcore */ 2157 rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); 2158 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 2159 if (rte_eal_wait_lcore(lcore_id) < 0) 2160 return -1; 2161 } 2162 2163 return 0; 2164 } 2165