1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2015-2016 Intel Corporation 3 */ 4 5 #include <time.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <inttypes.h> 11 #include <sys/types.h> 12 #include <sys/queue.h> 13 #include <netinet/in.h> 14 #include <setjmp.h> 15 #include <stdarg.h> 16 #include <ctype.h> 17 #include <errno.h> 18 #include <getopt.h> 19 #include <fcntl.h> 20 #include <unistd.h> 21 22 #include <rte_string_fns.h> 23 #include <rte_branch_prediction.h> 24 #include <rte_common.h> 25 #include <rte_cryptodev.h> 26 #include <rte_cycles.h> 27 #include <rte_debug.h> 28 #include <rte_eal.h> 29 #include <rte_ether.h> 30 #include <rte_ethdev.h> 31 #include <rte_interrupts.h> 32 #include <rte_ip.h> 33 #include <rte_launch.h> 34 #include <rte_lcore.h> 35 #include <rte_log.h> 36 #include <rte_malloc.h> 37 #include <rte_mbuf.h> 38 #include <rte_memcpy.h> 39 #include <rte_memory.h> 40 #include <rte_mempool.h> 41 #include <rte_per_lcore.h> 42 #include <rte_prefetch.h> 43 #include <rte_random.h> 44 #include <rte_hexdump.h> 45 #ifdef RTE_CRYPTO_SCHEDULER 46 #include <rte_cryptodev_scheduler.h> 47 #endif 48 49 enum cdev_type { 50 CDEV_TYPE_ANY, 51 CDEV_TYPE_HW, 52 CDEV_TYPE_SW 53 }; 54 55 #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1 56 57 #define NB_MBUF 8192 58 59 #define MAX_STR_LEN 32 60 #define MAX_KEY_SIZE 128 61 #define MAX_IV_SIZE 16 62 #define MAX_AAD_SIZE 65535 63 #define MAX_PKT_BURST 32 64 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 65 #define SESSION_POOL_CACHE_SIZE 0 66 67 #define MAXIMUM_IV_LENGTH 16 68 #define IV_OFFSET (sizeof(struct rte_crypto_op) + \ 69 sizeof(struct rte_crypto_sym_op)) 70 71 /* 72 * Configurable number of RX/TX ring descriptors 73 */ 74 #define RTE_TEST_RX_DESC_DEFAULT 1024 75 #define RTE_TEST_TX_DESC_DEFAULT 1024 76 77 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; 78 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; 79 80 /* ethernet addresses of ports */ 81 static struct rte_ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS]; 82 83 /* mask of enabled ports */ 84 static uint64_t l2fwd_enabled_port_mask; 85 static uint64_t l2fwd_enabled_crypto_mask; 86 87 /* list of enabled ports */ 88 static uint16_t l2fwd_dst_ports[RTE_MAX_ETHPORTS]; 89 90 91 struct pkt_buffer { 92 unsigned len; 93 struct rte_mbuf *buffer[MAX_PKT_BURST]; 94 }; 95 96 struct op_buffer { 97 unsigned len; 98 struct rte_crypto_op *buffer[MAX_PKT_BURST]; 99 }; 100 101 #define MAX_RX_QUEUE_PER_LCORE 16 102 #define MAX_TX_QUEUE_PER_PORT 16 103 104 enum l2fwd_crypto_xform_chain { 105 L2FWD_CRYPTO_CIPHER_HASH, 106 L2FWD_CRYPTO_HASH_CIPHER, 107 L2FWD_CRYPTO_CIPHER_ONLY, 108 L2FWD_CRYPTO_HASH_ONLY, 109 L2FWD_CRYPTO_AEAD 110 }; 111 112 struct l2fwd_key { 113 uint8_t *data; 114 uint32_t length; 115 rte_iova_t phys_addr; 116 }; 117 118 struct l2fwd_iv { 119 uint8_t *data; 120 uint16_t length; 121 }; 122 123 /** l2fwd crypto application command line options */ 124 struct l2fwd_crypto_options { 125 unsigned portmask; 126 unsigned nb_ports_per_lcore; 127 unsigned refresh_period; 128 unsigned single_lcore:1; 129 130 enum cdev_type type; 131 unsigned sessionless:1; 132 133 enum l2fwd_crypto_xform_chain xform_chain; 134 135 struct rte_crypto_sym_xform cipher_xform; 136 unsigned ckey_param; 137 int ckey_random_size; 138 uint8_t cipher_key[MAX_KEY_SIZE]; 139 140 struct l2fwd_iv cipher_iv; 141 unsigned int cipher_iv_param; 142 int cipher_iv_random_size; 143 144 struct rte_crypto_sym_xform auth_xform; 145 uint8_t akey_param; 146 int akey_random_size; 147 uint8_t auth_key[MAX_KEY_SIZE]; 148 149 struct l2fwd_iv auth_iv; 150 unsigned int auth_iv_param; 151 int auth_iv_random_size; 152 153 struct rte_crypto_sym_xform aead_xform; 154 unsigned int aead_key_param; 155 int aead_key_random_size; 156 uint8_t aead_key[MAX_KEY_SIZE]; 157 158 struct l2fwd_iv aead_iv; 159 unsigned int aead_iv_param; 160 int aead_iv_random_size; 161 162 struct l2fwd_key aad; 163 unsigned aad_param; 164 int aad_random_size; 165 166 int digest_size; 167 168 uint16_t block_size; 169 char string_type[MAX_STR_LEN]; 170 171 uint64_t cryptodev_mask; 172 173 unsigned int mac_updating; 174 }; 175 176 /** l2fwd crypto lcore params */ 177 struct l2fwd_crypto_params { 178 uint8_t dev_id; 179 uint8_t qp_id; 180 181 unsigned digest_length; 182 unsigned block_size; 183 184 uint32_t cipher_dataunit_len; 185 186 struct l2fwd_iv cipher_iv; 187 struct l2fwd_iv auth_iv; 188 struct l2fwd_iv aead_iv; 189 struct l2fwd_key aad; 190 struct rte_cryptodev_sym_session *session; 191 192 uint8_t do_cipher; 193 uint8_t do_hash; 194 uint8_t do_aead; 195 uint8_t hash_verify; 196 197 enum rte_crypto_cipher_algorithm cipher_algo; 198 enum rte_crypto_auth_algorithm auth_algo; 199 enum rte_crypto_aead_algorithm aead_algo; 200 }; 201 202 /** lcore configuration */ 203 struct lcore_queue_conf { 204 unsigned nb_rx_ports; 205 uint16_t rx_port_list[MAX_RX_QUEUE_PER_LCORE]; 206 207 unsigned nb_crypto_devs; 208 unsigned cryptodev_list[MAX_RX_QUEUE_PER_LCORE]; 209 210 struct op_buffer op_buf[RTE_CRYPTO_MAX_DEVS]; 211 struct pkt_buffer pkt_buf[RTE_MAX_ETHPORTS]; 212 } __rte_cache_aligned; 213 214 struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE]; 215 216 static struct rte_eth_conf port_conf = { 217 .rxmode = { 218 .mq_mode = RTE_ETH_MQ_RX_NONE, 219 .split_hdr_size = 0, 220 }, 221 .txmode = { 222 .mq_mode = RTE_ETH_MQ_TX_NONE, 223 }, 224 }; 225 226 struct rte_mempool *l2fwd_pktmbuf_pool; 227 struct rte_mempool *l2fwd_crypto_op_pool; 228 static struct { 229 struct rte_mempool *sess_mp; 230 struct rte_mempool *priv_mp; 231 } session_pool_socket[RTE_MAX_NUMA_NODES]; 232 233 /* Per-port statistics struct */ 234 struct l2fwd_port_statistics { 235 uint64_t tx; 236 uint64_t rx; 237 238 uint64_t crypto_enqueued; 239 uint64_t crypto_dequeued; 240 241 uint64_t dropped; 242 } __rte_cache_aligned; 243 244 struct l2fwd_crypto_statistics { 245 uint64_t enqueued; 246 uint64_t dequeued; 247 248 uint64_t errors; 249 } __rte_cache_aligned; 250 251 struct l2fwd_port_statistics port_statistics[RTE_MAX_ETHPORTS]; 252 struct l2fwd_crypto_statistics crypto_statistics[RTE_CRYPTO_MAX_DEVS]; 253 254 /* A tsc-based timer responsible for triggering statistics printout */ 255 #define TIMER_MILLISECOND 2000000ULL /* around 1ms at 2 Ghz */ 256 #define MAX_TIMER_PERIOD 86400UL /* 1 day max */ 257 258 /* default period is 10 seconds */ 259 static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000; 260 261 /* Print out statistics on packets dropped */ 262 static void 263 print_stats(void) 264 { 265 uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; 266 uint64_t total_packets_enqueued, total_packets_dequeued, 267 total_packets_errors; 268 uint16_t portid; 269 uint64_t cdevid; 270 271 total_packets_dropped = 0; 272 total_packets_tx = 0; 273 total_packets_rx = 0; 274 total_packets_enqueued = 0; 275 total_packets_dequeued = 0; 276 total_packets_errors = 0; 277 278 const char clr[] = { 27, '[', '2', 'J', '\0' }; 279 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 280 281 /* Clear screen and move to top left */ 282 printf("%s%s", clr, topLeft); 283 284 printf("\nPort statistics ===================================="); 285 286 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 287 /* skip disabled ports */ 288 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) 289 continue; 290 printf("\nStatistics for port %u ------------------------------" 291 "\nPackets sent: %32"PRIu64 292 "\nPackets received: %28"PRIu64 293 "\nPackets dropped: %29"PRIu64, 294 portid, 295 port_statistics[portid].tx, 296 port_statistics[portid].rx, 297 port_statistics[portid].dropped); 298 299 total_packets_dropped += port_statistics[portid].dropped; 300 total_packets_tx += port_statistics[portid].tx; 301 total_packets_rx += port_statistics[portid].rx; 302 } 303 printf("\nCrypto statistics =================================="); 304 305 for (cdevid = 0; cdevid < RTE_CRYPTO_MAX_DEVS; cdevid++) { 306 /* skip disabled ports */ 307 if ((l2fwd_enabled_crypto_mask & (((uint64_t)1) << cdevid)) == 0) 308 continue; 309 printf("\nStatistics for cryptodev %"PRIu64 310 " -------------------------" 311 "\nPackets enqueued: %28"PRIu64 312 "\nPackets dequeued: %28"PRIu64 313 "\nPackets errors: %30"PRIu64, 314 cdevid, 315 crypto_statistics[cdevid].enqueued, 316 crypto_statistics[cdevid].dequeued, 317 crypto_statistics[cdevid].errors); 318 319 total_packets_enqueued += crypto_statistics[cdevid].enqueued; 320 total_packets_dequeued += crypto_statistics[cdevid].dequeued; 321 total_packets_errors += crypto_statistics[cdevid].errors; 322 } 323 printf("\nAggregate statistics ===============================" 324 "\nTotal packets received: %22"PRIu64 325 "\nTotal packets enqueued: %22"PRIu64 326 "\nTotal packets dequeued: %22"PRIu64 327 "\nTotal packets sent: %26"PRIu64 328 "\nTotal packets dropped: %23"PRIu64 329 "\nTotal packets crypto errors: %17"PRIu64, 330 total_packets_rx, 331 total_packets_enqueued, 332 total_packets_dequeued, 333 total_packets_tx, 334 total_packets_dropped, 335 total_packets_errors); 336 printf("\n====================================================\n"); 337 338 fflush(stdout); 339 } 340 341 /* l2fwd_crypto_send_burst 8< */ 342 static int 343 l2fwd_crypto_send_burst(struct lcore_queue_conf *qconf, unsigned n, 344 struct l2fwd_crypto_params *cparams) 345 { 346 struct rte_crypto_op **op_buffer; 347 unsigned ret; 348 349 op_buffer = (struct rte_crypto_op **) 350 qconf->op_buf[cparams->dev_id].buffer; 351 352 ret = rte_cryptodev_enqueue_burst(cparams->dev_id, 353 cparams->qp_id, op_buffer, (uint16_t) n); 354 355 crypto_statistics[cparams->dev_id].enqueued += ret; 356 if (unlikely(ret < n)) { 357 crypto_statistics[cparams->dev_id].errors += (n - ret); 358 do { 359 rte_pktmbuf_free(op_buffer[ret]->sym->m_src); 360 rte_crypto_op_free(op_buffer[ret]); 361 } while (++ret < n); 362 } 363 364 return 0; 365 } 366 /* >8 End of l2fwd_crypto_send_burst. */ 367 368 /* Crypto enqueue. 8< */ 369 static int 370 l2fwd_crypto_enqueue(struct rte_crypto_op *op, 371 struct l2fwd_crypto_params *cparams) 372 { 373 unsigned lcore_id, len; 374 struct lcore_queue_conf *qconf; 375 376 lcore_id = rte_lcore_id(); 377 378 qconf = &lcore_queue_conf[lcore_id]; 379 len = qconf->op_buf[cparams->dev_id].len; 380 qconf->op_buf[cparams->dev_id].buffer[len] = op; 381 len++; 382 383 /* enough ops to be sent */ 384 if (len == MAX_PKT_BURST) { 385 l2fwd_crypto_send_burst(qconf, MAX_PKT_BURST, cparams); 386 len = 0; 387 } 388 389 qconf->op_buf[cparams->dev_id].len = len; 390 return 0; 391 } 392 /* >8 End of crypto enqueue. */ 393 394 static int 395 l2fwd_simple_crypto_enqueue(struct rte_mbuf *m, 396 struct rte_crypto_op *op, 397 struct l2fwd_crypto_params *cparams) 398 { 399 struct rte_ether_hdr *eth_hdr; 400 struct rte_ipv4_hdr *ip_hdr; 401 402 uint32_t ipdata_offset, data_len; 403 uint32_t pad_len = 0; 404 char *padding; 405 406 eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); 407 408 if (eth_hdr->ether_type != rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) 409 return -1; 410 411 ipdata_offset = sizeof(struct rte_ether_hdr); 412 413 ip_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, char *) + 414 ipdata_offset); 415 416 ipdata_offset += (ip_hdr->version_ihl & RTE_IPV4_HDR_IHL_MASK) 417 * RTE_IPV4_IHL_MULTIPLIER; 418 419 420 /* Zero pad data to be crypto'd so it is block aligned */ 421 data_len = rte_pktmbuf_data_len(m) - ipdata_offset; 422 423 if ((cparams->do_hash || cparams->do_aead) && cparams->hash_verify) 424 data_len -= cparams->digest_length; 425 426 if (cparams->do_cipher) { 427 /* 428 * Following algorithms are block cipher algorithms, 429 * and might need padding 430 */ 431 switch (cparams->cipher_algo) { 432 case RTE_CRYPTO_CIPHER_AES_CBC: 433 case RTE_CRYPTO_CIPHER_AES_ECB: 434 case RTE_CRYPTO_CIPHER_DES_CBC: 435 case RTE_CRYPTO_CIPHER_3DES_CBC: 436 case RTE_CRYPTO_CIPHER_3DES_ECB: 437 if (data_len % cparams->block_size) 438 pad_len = cparams->block_size - 439 (data_len % cparams->block_size); 440 break; 441 case RTE_CRYPTO_CIPHER_AES_XTS: 442 if (cparams->cipher_dataunit_len != 0 && 443 (data_len % cparams->cipher_dataunit_len)) 444 pad_len = cparams->cipher_dataunit_len - 445 (data_len % cparams->cipher_dataunit_len); 446 break; 447 default: 448 pad_len = 0; 449 } 450 451 if (pad_len) { 452 padding = rte_pktmbuf_append(m, pad_len); 453 if (unlikely(!padding)) 454 return -1; 455 456 data_len += pad_len; 457 memset(padding, 0, pad_len); 458 } 459 } 460 461 /* Set crypto operation data parameters */ 462 rte_crypto_op_attach_sym_session(op, cparams->session); 463 464 if (cparams->do_hash) { 465 if (cparams->auth_iv.length) { 466 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, 467 uint8_t *, 468 IV_OFFSET + 469 cparams->cipher_iv.length); 470 /* 471 * Copy IV at the end of the crypto operation, 472 * after the cipher IV, if added 473 */ 474 rte_memcpy(iv_ptr, cparams->auth_iv.data, 475 cparams->auth_iv.length); 476 } 477 if (!cparams->hash_verify) { 478 /* Append space for digest to end of packet */ 479 op->sym->auth.digest.data = (uint8_t *)rte_pktmbuf_append(m, 480 cparams->digest_length); 481 } else { 482 op->sym->auth.digest.data = rte_pktmbuf_mtod(m, 483 uint8_t *) + ipdata_offset + data_len; 484 } 485 486 op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m, 487 rte_pktmbuf_pkt_len(m) - cparams->digest_length); 488 489 /* For wireless algorithms, offset/length must be in bits */ 490 if (cparams->auth_algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2 || 491 cparams->auth_algo == RTE_CRYPTO_AUTH_KASUMI_F9 || 492 cparams->auth_algo == RTE_CRYPTO_AUTH_ZUC_EIA3) { 493 op->sym->auth.data.offset = ipdata_offset << 3; 494 op->sym->auth.data.length = data_len << 3; 495 } else { 496 op->sym->auth.data.offset = ipdata_offset; 497 op->sym->auth.data.length = data_len; 498 } 499 } 500 501 if (cparams->do_cipher) { 502 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 503 IV_OFFSET); 504 /* Copy IV at the end of the crypto operation */ 505 rte_memcpy(iv_ptr, cparams->cipher_iv.data, 506 cparams->cipher_iv.length); 507 508 /* For wireless algorithms, offset/length must be in bits */ 509 if (cparams->cipher_algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 || 510 cparams->cipher_algo == RTE_CRYPTO_CIPHER_KASUMI_F8 || 511 cparams->cipher_algo == RTE_CRYPTO_CIPHER_ZUC_EEA3) { 512 op->sym->cipher.data.offset = ipdata_offset << 3; 513 op->sym->cipher.data.length = data_len << 3; 514 } else { 515 op->sym->cipher.data.offset = ipdata_offset; 516 op->sym->cipher.data.length = data_len; 517 } 518 } 519 520 if (cparams->do_aead) { 521 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 522 IV_OFFSET); 523 /* Copy IV at the end of the crypto operation */ 524 /* 525 * If doing AES-CCM, nonce is copied one byte 526 * after the start of IV field 527 */ 528 if (cparams->aead_algo == RTE_CRYPTO_AEAD_AES_CCM) 529 rte_memcpy(iv_ptr + 1, cparams->aead_iv.data, 530 cparams->aead_iv.length); 531 else 532 rte_memcpy(iv_ptr, cparams->aead_iv.data, 533 cparams->aead_iv.length); 534 535 op->sym->aead.data.offset = ipdata_offset; 536 op->sym->aead.data.length = data_len; 537 538 if (!cparams->hash_verify) { 539 /* Append space for digest to end of packet */ 540 op->sym->aead.digest.data = (uint8_t *)rte_pktmbuf_append(m, 541 cparams->digest_length); 542 } else { 543 op->sym->aead.digest.data = rte_pktmbuf_mtod(m, 544 uint8_t *) + ipdata_offset + data_len; 545 } 546 547 op->sym->aead.digest.phys_addr = rte_pktmbuf_iova_offset(m, 548 rte_pktmbuf_pkt_len(m) - cparams->digest_length); 549 550 if (cparams->aad.length) { 551 op->sym->aead.aad.data = cparams->aad.data; 552 op->sym->aead.aad.phys_addr = cparams->aad.phys_addr; 553 } 554 } 555 556 op->sym->m_src = m; 557 558 return l2fwd_crypto_enqueue(op, cparams); 559 } 560 561 562 /* Send the burst of packets on an output interface */ 563 static int 564 l2fwd_send_burst(struct lcore_queue_conf *qconf, unsigned n, 565 uint16_t port) 566 { 567 struct rte_mbuf **pkt_buffer; 568 unsigned ret; 569 570 pkt_buffer = (struct rte_mbuf **)qconf->pkt_buf[port].buffer; 571 572 ret = rte_eth_tx_burst(port, 0, pkt_buffer, (uint16_t)n); 573 port_statistics[port].tx += ret; 574 if (unlikely(ret < n)) { 575 port_statistics[port].dropped += (n - ret); 576 do { 577 rte_pktmbuf_free(pkt_buffer[ret]); 578 } while (++ret < n); 579 } 580 581 return 0; 582 } 583 584 /* Enqueue packets for TX and prepare them to be sent. 8< */ 585 static int 586 l2fwd_send_packet(struct rte_mbuf *m, uint16_t port) 587 { 588 unsigned lcore_id, len; 589 struct lcore_queue_conf *qconf; 590 591 lcore_id = rte_lcore_id(); 592 593 qconf = &lcore_queue_conf[lcore_id]; 594 len = qconf->pkt_buf[port].len; 595 qconf->pkt_buf[port].buffer[len] = m; 596 len++; 597 598 /* enough pkts to be sent */ 599 if (unlikely(len == MAX_PKT_BURST)) { 600 l2fwd_send_burst(qconf, MAX_PKT_BURST, port); 601 len = 0; 602 } 603 604 qconf->pkt_buf[port].len = len; 605 return 0; 606 } 607 /* >8 End of Enqueuing packets for TX. */ 608 609 static void 610 l2fwd_mac_updating(struct rte_mbuf *m, uint16_t dest_portid) 611 { 612 struct rte_ether_hdr *eth; 613 void *tmp; 614 615 eth = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); 616 617 /* 02:00:00:00:00:xx */ 618 tmp = ð->dst_addr.addr_bytes[0]; 619 *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dest_portid << 40); 620 621 /* src addr */ 622 rte_ether_addr_copy(&l2fwd_ports_eth_addr[dest_portid], ð->src_addr); 623 } 624 625 static void 626 l2fwd_simple_forward(struct rte_mbuf *m, uint16_t portid, 627 struct l2fwd_crypto_options *options) 628 { 629 uint16_t dst_port; 630 uint32_t pad_len; 631 struct rte_ipv4_hdr *ip_hdr; 632 uint32_t ipdata_offset = sizeof(struct rte_ether_hdr); 633 634 ip_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, char *) + 635 ipdata_offset); 636 dst_port = l2fwd_dst_ports[portid]; 637 638 if (options->mac_updating) 639 l2fwd_mac_updating(m, dst_port); 640 641 if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_VERIFY) 642 rte_pktmbuf_trim(m, options->auth_xform.auth.digest_length); 643 644 if (options->cipher_xform.cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { 645 pad_len = m->pkt_len - rte_be_to_cpu_16(ip_hdr->total_length) - 646 ipdata_offset; 647 rte_pktmbuf_trim(m, pad_len); 648 } 649 650 l2fwd_send_packet(m, dst_port); 651 } 652 653 /** Generate random key */ 654 static void 655 generate_random_key(uint8_t *key, unsigned length) 656 { 657 int fd; 658 int ret; 659 660 fd = open("/dev/urandom", O_RDONLY); 661 if (fd < 0) 662 rte_exit(EXIT_FAILURE, "Failed to generate random key\n"); 663 664 ret = read(fd, key, length); 665 close(fd); 666 667 if (ret != (signed)length) 668 rte_exit(EXIT_FAILURE, "Failed to generate random key\n"); 669 } 670 671 /* Session is created and is later attached to the crypto operation. 8< */ 672 static struct rte_cryptodev_sym_session * 673 initialize_crypto_session(struct l2fwd_crypto_options *options, uint8_t cdev_id) 674 { 675 struct rte_crypto_sym_xform *first_xform; 676 struct rte_cryptodev_sym_session *session; 677 int retval = rte_cryptodev_socket_id(cdev_id); 678 679 if (retval < 0) 680 return NULL; 681 682 uint8_t socket_id = (uint8_t) retval; 683 684 if (options->xform_chain == L2FWD_CRYPTO_AEAD) { 685 first_xform = &options->aead_xform; 686 } else if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH) { 687 first_xform = &options->cipher_xform; 688 first_xform->next = &options->auth_xform; 689 } else if (options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER) { 690 first_xform = &options->auth_xform; 691 first_xform->next = &options->cipher_xform; 692 } else if (options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { 693 first_xform = &options->cipher_xform; 694 } else { 695 first_xform = &options->auth_xform; 696 } 697 698 session = rte_cryptodev_sym_session_create( 699 session_pool_socket[socket_id].sess_mp); 700 if (session == NULL) 701 return NULL; 702 703 if (rte_cryptodev_sym_session_init(cdev_id, session, 704 first_xform, 705 session_pool_socket[socket_id].priv_mp) < 0) 706 return NULL; 707 708 return session; 709 } 710 /* >8 End of creation of session. */ 711 712 static void 713 l2fwd_crypto_options_print(struct l2fwd_crypto_options *options); 714 715 /* main processing loop */ 716 static void 717 l2fwd_main_loop(struct l2fwd_crypto_options *options) 718 { 719 struct rte_mbuf *m, *pkts_burst[MAX_PKT_BURST]; 720 struct rte_crypto_op *ops_burst[MAX_PKT_BURST]; 721 722 unsigned lcore_id = rte_lcore_id(); 723 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 724 unsigned int i, j, nb_rx, len; 725 uint16_t portid; 726 struct lcore_queue_conf *qconf = &lcore_queue_conf[lcore_id]; 727 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 728 US_PER_S * BURST_TX_DRAIN_US; 729 struct l2fwd_crypto_params *cparams; 730 struct l2fwd_crypto_params port_cparams[qconf->nb_crypto_devs]; 731 struct rte_cryptodev_sym_session *session; 732 733 if (qconf->nb_rx_ports == 0) { 734 RTE_LOG(INFO, L2FWD, "lcore %u has nothing to do\n", lcore_id); 735 return; 736 } 737 738 RTE_LOG(INFO, L2FWD, "entering main loop on lcore %u\n", lcore_id); 739 740 for (i = 0; i < qconf->nb_rx_ports; i++) { 741 742 portid = qconf->rx_port_list[i]; 743 RTE_LOG(INFO, L2FWD, " -- lcoreid=%u portid=%u\n", lcore_id, 744 portid); 745 } 746 747 for (i = 0; i < qconf->nb_crypto_devs; i++) { 748 port_cparams[i].do_cipher = 0; 749 port_cparams[i].do_hash = 0; 750 port_cparams[i].do_aead = 0; 751 752 switch (options->xform_chain) { 753 case L2FWD_CRYPTO_AEAD: 754 port_cparams[i].do_aead = 1; 755 break; 756 case L2FWD_CRYPTO_CIPHER_HASH: 757 case L2FWD_CRYPTO_HASH_CIPHER: 758 port_cparams[i].do_cipher = 1; 759 port_cparams[i].do_hash = 1; 760 break; 761 case L2FWD_CRYPTO_HASH_ONLY: 762 port_cparams[i].do_hash = 1; 763 break; 764 case L2FWD_CRYPTO_CIPHER_ONLY: 765 port_cparams[i].do_cipher = 1; 766 break; 767 } 768 769 port_cparams[i].dev_id = qconf->cryptodev_list[i]; 770 port_cparams[i].qp_id = 0; 771 772 port_cparams[i].block_size = options->block_size; 773 774 if (port_cparams[i].do_hash) { 775 port_cparams[i].auth_iv.data = options->auth_iv.data; 776 port_cparams[i].auth_iv.length = options->auth_iv.length; 777 if (!options->auth_iv_param) 778 generate_random_key(port_cparams[i].auth_iv.data, 779 port_cparams[i].auth_iv.length); 780 if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_VERIFY) 781 port_cparams[i].hash_verify = 1; 782 else 783 port_cparams[i].hash_verify = 0; 784 785 port_cparams[i].auth_algo = options->auth_xform.auth.algo; 786 port_cparams[i].digest_length = 787 options->auth_xform.auth.digest_length; 788 /* Set IV parameters */ 789 if (options->auth_iv.length) { 790 options->auth_xform.auth.iv.offset = 791 IV_OFFSET + options->cipher_iv.length; 792 options->auth_xform.auth.iv.length = 793 options->auth_iv.length; 794 } 795 } 796 797 if (port_cparams[i].do_aead) { 798 port_cparams[i].aead_iv.data = options->aead_iv.data; 799 port_cparams[i].aead_iv.length = options->aead_iv.length; 800 if (!options->aead_iv_param) 801 generate_random_key(port_cparams[i].aead_iv.data, 802 port_cparams[i].aead_iv.length); 803 port_cparams[i].aead_algo = options->aead_xform.aead.algo; 804 port_cparams[i].digest_length = 805 options->aead_xform.aead.digest_length; 806 if (options->aead_xform.aead.aad_length) { 807 port_cparams[i].aad.data = options->aad.data; 808 port_cparams[i].aad.phys_addr = options->aad.phys_addr; 809 port_cparams[i].aad.length = options->aad.length; 810 if (!options->aad_param) 811 generate_random_key(port_cparams[i].aad.data, 812 port_cparams[i].aad.length); 813 /* 814 * If doing AES-CCM, first 18 bytes has to be reserved, 815 * and actual AAD should start from byte 18 816 */ 817 if (port_cparams[i].aead_algo == RTE_CRYPTO_AEAD_AES_CCM) 818 memmove(port_cparams[i].aad.data + 18, 819 port_cparams[i].aad.data, 820 port_cparams[i].aad.length); 821 822 } else 823 port_cparams[i].aad.length = 0; 824 825 if (options->aead_xform.aead.op == RTE_CRYPTO_AEAD_OP_DECRYPT) 826 port_cparams[i].hash_verify = 1; 827 else 828 port_cparams[i].hash_verify = 0; 829 830 /* Set IV parameters */ 831 options->aead_xform.aead.iv.offset = IV_OFFSET; 832 options->aead_xform.aead.iv.length = options->aead_iv.length; 833 } 834 835 if (port_cparams[i].do_cipher) { 836 port_cparams[i].cipher_iv.data = options->cipher_iv.data; 837 port_cparams[i].cipher_iv.length = options->cipher_iv.length; 838 if (!options->cipher_iv_param) 839 generate_random_key(port_cparams[i].cipher_iv.data, 840 port_cparams[i].cipher_iv.length); 841 842 port_cparams[i].cipher_algo = options->cipher_xform.cipher.algo; 843 port_cparams[i].cipher_dataunit_len = 844 options->cipher_xform.cipher.dataunit_len; 845 /* Set IV parameters */ 846 options->cipher_xform.cipher.iv.offset = IV_OFFSET; 847 options->cipher_xform.cipher.iv.length = 848 options->cipher_iv.length; 849 } 850 851 session = initialize_crypto_session(options, 852 port_cparams[i].dev_id); 853 if (session == NULL) 854 rte_exit(EXIT_FAILURE, "Failed to initialize crypto session\n"); 855 856 port_cparams[i].session = session; 857 858 RTE_LOG(INFO, L2FWD, " -- lcoreid=%u cryptoid=%u\n", lcore_id, 859 port_cparams[i].dev_id); 860 } 861 862 l2fwd_crypto_options_print(options); 863 864 /* 865 * Initialize previous tsc timestamp before the loop, 866 * to avoid showing the port statistics immediately, 867 * so user can see the crypto information. 868 */ 869 prev_tsc = rte_rdtsc(); 870 while (1) { 871 872 cur_tsc = rte_rdtsc(); 873 874 /* 875 * Crypto device/TX burst queue drain 876 */ 877 diff_tsc = cur_tsc - prev_tsc; 878 if (unlikely(diff_tsc > drain_tsc)) { 879 /* Enqueue all crypto ops remaining in buffers */ 880 for (i = 0; i < qconf->nb_crypto_devs; i++) { 881 cparams = &port_cparams[i]; 882 len = qconf->op_buf[cparams->dev_id].len; 883 l2fwd_crypto_send_burst(qconf, len, cparams); 884 qconf->op_buf[cparams->dev_id].len = 0; 885 } 886 /* Transmit all packets remaining in buffers */ 887 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 888 if (qconf->pkt_buf[portid].len == 0) 889 continue; 890 l2fwd_send_burst(&lcore_queue_conf[lcore_id], 891 qconf->pkt_buf[portid].len, 892 portid); 893 qconf->pkt_buf[portid].len = 0; 894 } 895 896 /* if timer is enabled */ 897 if (timer_period > 0) { 898 899 /* advance the timer */ 900 timer_tsc += diff_tsc; 901 902 /* if timer has reached its timeout */ 903 if (unlikely(timer_tsc >= 904 (uint64_t)timer_period)) { 905 906 /* do this only on main core */ 907 if (lcore_id == rte_get_main_lcore() 908 && options->refresh_period) { 909 print_stats(); 910 timer_tsc = 0; 911 } 912 } 913 } 914 915 prev_tsc = cur_tsc; 916 } 917 918 /* 919 * Read packet from RX queues 920 */ 921 for (i = 0; i < qconf->nb_rx_ports; i++) { 922 portid = qconf->rx_port_list[i]; 923 924 cparams = &port_cparams[i]; 925 926 nb_rx = rte_eth_rx_burst(portid, 0, 927 pkts_burst, MAX_PKT_BURST); 928 929 port_statistics[portid].rx += nb_rx; 930 931 /* Allocate and fillcrypto operations. 8< */ 932 if (nb_rx) { 933 /* 934 * If we can't allocate a crypto_ops, then drop 935 * the rest of the burst and dequeue and 936 * process the packets to free offload structs 937 */ 938 if (rte_crypto_op_bulk_alloc( 939 l2fwd_crypto_op_pool, 940 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 941 ops_burst, nb_rx) != 942 nb_rx) { 943 for (j = 0; j < nb_rx; j++) 944 rte_pktmbuf_free(pkts_burst[j]); 945 946 nb_rx = 0; 947 } 948 /* >8 End of crypto operation allocated and filled. */ 949 950 /* Enqueue packets from Crypto device*/ 951 for (j = 0; j < nb_rx; j++) { 952 m = pkts_burst[j]; 953 954 l2fwd_simple_crypto_enqueue(m, 955 ops_burst[j], cparams); 956 } 957 } 958 959 /* Dequeue packets from Crypto device. 8< */ 960 do { 961 nb_rx = rte_cryptodev_dequeue_burst( 962 cparams->dev_id, cparams->qp_id, 963 ops_burst, MAX_PKT_BURST); 964 965 crypto_statistics[cparams->dev_id].dequeued += 966 nb_rx; 967 968 /* Forward crypto'd packets */ 969 for (j = 0; j < nb_rx; j++) { 970 m = ops_burst[j]->sym->m_src; 971 972 rte_crypto_op_free(ops_burst[j]); 973 l2fwd_simple_forward(m, portid, 974 options); 975 } 976 } while (nb_rx == MAX_PKT_BURST); 977 /* >8 End of dequeue packets from crypto device. */ 978 } 979 } 980 } 981 982 static int 983 l2fwd_launch_one_lcore(void *arg) 984 { 985 l2fwd_main_loop((struct l2fwd_crypto_options *)arg); 986 return 0; 987 } 988 989 /* Display command line arguments usage */ 990 static void 991 l2fwd_crypto_usage(const char *prgname) 992 { 993 printf("%s [EAL options] --\n" 994 " -p PORTMASK: hexadecimal bitmask of ports to configure\n" 995 " -q NQ: number of queue (=ports) per lcore (default is 1)\n" 996 " -s manage all ports from single lcore\n" 997 " -T PERIOD: statistics will be refreshed each PERIOD seconds" 998 " (0 to disable, 10 default, 86400 maximum)\n" 999 1000 " --cdev_type HW / SW / ANY\n" 1001 " --chain HASH_CIPHER / CIPHER_HASH / CIPHER_ONLY /" 1002 " HASH_ONLY / AEAD\n" 1003 1004 " --cipher_algo ALGO\n" 1005 " --cipher_op ENCRYPT / DECRYPT\n" 1006 " --cipher_key KEY (bytes separated with \":\")\n" 1007 " --cipher_key_random_size SIZE: size of cipher key when generated randomly\n" 1008 " --cipher_iv IV (bytes separated with \":\")\n" 1009 " --cipher_iv_random_size SIZE: size of cipher IV when generated randomly\n" 1010 " --cipher_dataunit_len SIZE: length of the algorithm data-unit\n" 1011 1012 " --auth_algo ALGO\n" 1013 " --auth_op GENERATE / VERIFY\n" 1014 " --auth_key KEY (bytes separated with \":\")\n" 1015 " --auth_key_random_size SIZE: size of auth key when generated randomly\n" 1016 " --auth_iv IV (bytes separated with \":\")\n" 1017 " --auth_iv_random_size SIZE: size of auth IV when generated randomly\n" 1018 1019 " --aead_algo ALGO\n" 1020 " --aead_op ENCRYPT / DECRYPT\n" 1021 " --aead_key KEY (bytes separated with \":\")\n" 1022 " --aead_key_random_size SIZE: size of AEAD key when generated randomly\n" 1023 " --aead_iv IV (bytes separated with \":\")\n" 1024 " --aead_iv_random_size SIZE: size of AEAD IV when generated randomly\n" 1025 " --aad AAD (bytes separated with \":\")\n" 1026 " --aad_random_size SIZE: size of AAD when generated randomly\n" 1027 1028 " --digest_size SIZE: size of digest to be generated/verified\n" 1029 1030 " --sessionless\n" 1031 " --cryptodev_mask MASK: hexadecimal bitmask of crypto devices to configure\n" 1032 1033 " --[no-]mac-updating: Enable or disable MAC addresses updating (enabled by default)\n" 1034 " When enabled:\n" 1035 " - The source MAC address is replaced by the TX port MAC address\n" 1036 " - The destination MAC address is replaced by 02:00:00:00:00:TX_PORT_ID\n", 1037 prgname); 1038 } 1039 1040 /** Parse crypto device type command line argument */ 1041 static int 1042 parse_cryptodev_type(enum cdev_type *type, char *optarg) 1043 { 1044 if (strcmp("HW", optarg) == 0) { 1045 *type = CDEV_TYPE_HW; 1046 return 0; 1047 } else if (strcmp("SW", optarg) == 0) { 1048 *type = CDEV_TYPE_SW; 1049 return 0; 1050 } else if (strcmp("ANY", optarg) == 0) { 1051 *type = CDEV_TYPE_ANY; 1052 return 0; 1053 } 1054 1055 return -1; 1056 } 1057 1058 /** Parse crypto chain xform command line argument */ 1059 static int 1060 parse_crypto_opt_chain(struct l2fwd_crypto_options *options, char *optarg) 1061 { 1062 if (strcmp("CIPHER_HASH", optarg) == 0) { 1063 options->xform_chain = L2FWD_CRYPTO_CIPHER_HASH; 1064 return 0; 1065 } else if (strcmp("HASH_CIPHER", optarg) == 0) { 1066 options->xform_chain = L2FWD_CRYPTO_HASH_CIPHER; 1067 return 0; 1068 } else if (strcmp("CIPHER_ONLY", optarg) == 0) { 1069 options->xform_chain = L2FWD_CRYPTO_CIPHER_ONLY; 1070 return 0; 1071 } else if (strcmp("HASH_ONLY", optarg) == 0) { 1072 options->xform_chain = L2FWD_CRYPTO_HASH_ONLY; 1073 return 0; 1074 } else if (strcmp("AEAD", optarg) == 0) { 1075 options->xform_chain = L2FWD_CRYPTO_AEAD; 1076 return 0; 1077 } 1078 1079 return -1; 1080 } 1081 1082 /** Parse crypto cipher algo option command line argument */ 1083 static int 1084 parse_cipher_algo(enum rte_crypto_cipher_algorithm *algo, char *optarg) 1085 { 1086 1087 if (rte_cryptodev_get_cipher_algo_enum(algo, optarg) < 0) { 1088 RTE_LOG(ERR, USER1, "Cipher algorithm specified " 1089 "not supported!\n"); 1090 return -1; 1091 } 1092 1093 return 0; 1094 } 1095 1096 /** Parse crypto cipher operation command line argument */ 1097 static int 1098 parse_cipher_op(enum rte_crypto_cipher_operation *op, char *optarg) 1099 { 1100 if (strcmp("ENCRYPT", optarg) == 0) { 1101 *op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 1102 return 0; 1103 } else if (strcmp("DECRYPT", optarg) == 0) { 1104 *op = RTE_CRYPTO_CIPHER_OP_DECRYPT; 1105 return 0; 1106 } 1107 1108 printf("Cipher operation not supported!\n"); 1109 return -1; 1110 } 1111 1112 /** Parse bytes from command line argument */ 1113 static int 1114 parse_bytes(uint8_t *data, char *input_arg, uint16_t max_size) 1115 { 1116 unsigned byte_count; 1117 char *token; 1118 1119 errno = 0; 1120 for (byte_count = 0, token = strtok(input_arg, ":"); 1121 (byte_count < max_size) && (token != NULL); 1122 token = strtok(NULL, ":")) { 1123 1124 int number = (int)strtol(token, NULL, 16); 1125 1126 if (errno == EINVAL || errno == ERANGE || number > 0xFF) 1127 return -1; 1128 1129 data[byte_count++] = (uint8_t)number; 1130 } 1131 1132 return byte_count; 1133 } 1134 1135 /** Parse size param*/ 1136 static int 1137 parse_size(int *size, const char *q_arg) 1138 { 1139 char *end = NULL; 1140 unsigned long n; 1141 1142 /* parse hexadecimal string */ 1143 n = strtoul(q_arg, &end, 10); 1144 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) 1145 n = 0; 1146 1147 if (n == 0) { 1148 printf("invalid size\n"); 1149 return -1; 1150 } 1151 1152 *size = n; 1153 return 0; 1154 } 1155 1156 /** Parse crypto cipher operation command line argument */ 1157 static int 1158 parse_auth_algo(enum rte_crypto_auth_algorithm *algo, char *optarg) 1159 { 1160 if (rte_cryptodev_get_auth_algo_enum(algo, optarg) < 0) { 1161 RTE_LOG(ERR, USER1, "Authentication algorithm specified " 1162 "not supported!\n"); 1163 return -1; 1164 } 1165 1166 return 0; 1167 } 1168 1169 static int 1170 parse_auth_op(enum rte_crypto_auth_operation *op, char *optarg) 1171 { 1172 if (strcmp("VERIFY", optarg) == 0) { 1173 *op = RTE_CRYPTO_AUTH_OP_VERIFY; 1174 return 0; 1175 } else if (strcmp("GENERATE", optarg) == 0) { 1176 *op = RTE_CRYPTO_AUTH_OP_GENERATE; 1177 return 0; 1178 } 1179 1180 printf("Authentication operation specified not supported!\n"); 1181 return -1; 1182 } 1183 1184 static int 1185 parse_aead_algo(enum rte_crypto_aead_algorithm *algo, char *optarg) 1186 { 1187 if (rte_cryptodev_get_aead_algo_enum(algo, optarg) < 0) { 1188 RTE_LOG(ERR, USER1, "AEAD algorithm specified " 1189 "not supported!\n"); 1190 return -1; 1191 } 1192 1193 return 0; 1194 } 1195 1196 static int 1197 parse_aead_op(enum rte_crypto_aead_operation *op, char *optarg) 1198 { 1199 if (strcmp("ENCRYPT", optarg) == 0) { 1200 *op = RTE_CRYPTO_AEAD_OP_ENCRYPT; 1201 return 0; 1202 } else if (strcmp("DECRYPT", optarg) == 0) { 1203 *op = RTE_CRYPTO_AEAD_OP_DECRYPT; 1204 return 0; 1205 } 1206 1207 printf("AEAD operation specified not supported!\n"); 1208 return -1; 1209 } 1210 static int 1211 parse_cryptodev_mask(struct l2fwd_crypto_options *options, 1212 const char *q_arg) 1213 { 1214 char *end = NULL; 1215 uint64_t pm; 1216 1217 /* parse hexadecimal string */ 1218 pm = strtoul(q_arg, &end, 16); 1219 if ((pm == '\0') || (end == NULL) || (*end != '\0')) 1220 pm = 0; 1221 1222 options->cryptodev_mask = pm; 1223 if (options->cryptodev_mask == 0) { 1224 printf("invalid cryptodev_mask specified\n"); 1225 return -1; 1226 } 1227 1228 return 0; 1229 } 1230 1231 /** Parse long options */ 1232 static int 1233 l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options, 1234 struct option *lgopts, int option_index) 1235 { 1236 int retval; 1237 int val; 1238 1239 if (strcmp(lgopts[option_index].name, "cdev_type") == 0) { 1240 retval = parse_cryptodev_type(&options->type, optarg); 1241 if (retval == 0) 1242 strlcpy(options->string_type, optarg, MAX_STR_LEN); 1243 return retval; 1244 } 1245 1246 else if (strcmp(lgopts[option_index].name, "chain") == 0) 1247 return parse_crypto_opt_chain(options, optarg); 1248 1249 /* Cipher options */ 1250 else if (strcmp(lgopts[option_index].name, "cipher_algo") == 0) 1251 return parse_cipher_algo(&options->cipher_xform.cipher.algo, 1252 optarg); 1253 1254 else if (strcmp(lgopts[option_index].name, "cipher_op") == 0) 1255 return parse_cipher_op(&options->cipher_xform.cipher.op, 1256 optarg); 1257 1258 else if (strcmp(lgopts[option_index].name, "cipher_key") == 0) { 1259 options->ckey_param = 1; 1260 options->cipher_xform.cipher.key.length = 1261 parse_bytes(options->cipher_key, optarg, MAX_KEY_SIZE); 1262 if (options->cipher_xform.cipher.key.length > 0) 1263 return 0; 1264 else 1265 return -1; 1266 } 1267 1268 else if (strcmp(lgopts[option_index].name, "cipher_dataunit_len") == 0) { 1269 retval = parse_size(&val, optarg); 1270 if (retval == 0 && val >= 0) { 1271 options->cipher_xform.cipher.dataunit_len = 1272 (uint32_t)val; 1273 return 0; 1274 } else 1275 return -1; 1276 } 1277 1278 else if (strcmp(lgopts[option_index].name, "cipher_key_random_size") == 0) 1279 return parse_size(&options->ckey_random_size, optarg); 1280 1281 else if (strcmp(lgopts[option_index].name, "cipher_iv") == 0) { 1282 options->cipher_iv_param = 1; 1283 options->cipher_iv.length = 1284 parse_bytes(options->cipher_iv.data, optarg, MAX_IV_SIZE); 1285 if (options->cipher_iv.length > 0) 1286 return 0; 1287 else 1288 return -1; 1289 } 1290 1291 else if (strcmp(lgopts[option_index].name, "cipher_iv_random_size") == 0) 1292 return parse_size(&options->cipher_iv_random_size, optarg); 1293 1294 /* Authentication options */ 1295 else if (strcmp(lgopts[option_index].name, "auth_algo") == 0) { 1296 return parse_auth_algo(&options->auth_xform.auth.algo, 1297 optarg); 1298 } 1299 1300 else if (strcmp(lgopts[option_index].name, "auth_op") == 0) 1301 return parse_auth_op(&options->auth_xform.auth.op, 1302 optarg); 1303 1304 else if (strcmp(lgopts[option_index].name, "auth_key") == 0) { 1305 options->akey_param = 1; 1306 options->auth_xform.auth.key.length = 1307 parse_bytes(options->auth_key, optarg, MAX_KEY_SIZE); 1308 if (options->auth_xform.auth.key.length > 0) 1309 return 0; 1310 else 1311 return -1; 1312 } 1313 1314 else if (strcmp(lgopts[option_index].name, "auth_key_random_size") == 0) { 1315 return parse_size(&options->akey_random_size, optarg); 1316 } 1317 1318 else if (strcmp(lgopts[option_index].name, "auth_iv") == 0) { 1319 options->auth_iv_param = 1; 1320 options->auth_iv.length = 1321 parse_bytes(options->auth_iv.data, optarg, MAX_IV_SIZE); 1322 if (options->auth_iv.length > 0) 1323 return 0; 1324 else 1325 return -1; 1326 } 1327 1328 else if (strcmp(lgopts[option_index].name, "auth_iv_random_size") == 0) 1329 return parse_size(&options->auth_iv_random_size, optarg); 1330 1331 /* AEAD options */ 1332 else if (strcmp(lgopts[option_index].name, "aead_algo") == 0) { 1333 return parse_aead_algo(&options->aead_xform.aead.algo, 1334 optarg); 1335 } 1336 1337 else if (strcmp(lgopts[option_index].name, "aead_op") == 0) 1338 return parse_aead_op(&options->aead_xform.aead.op, 1339 optarg); 1340 1341 else if (strcmp(lgopts[option_index].name, "aead_key") == 0) { 1342 options->aead_key_param = 1; 1343 options->aead_xform.aead.key.length = 1344 parse_bytes(options->aead_key, optarg, MAX_KEY_SIZE); 1345 if (options->aead_xform.aead.key.length > 0) 1346 return 0; 1347 else 1348 return -1; 1349 } 1350 1351 else if (strcmp(lgopts[option_index].name, "aead_key_random_size") == 0) 1352 return parse_size(&options->aead_key_random_size, optarg); 1353 1354 1355 else if (strcmp(lgopts[option_index].name, "aead_iv") == 0) { 1356 options->aead_iv_param = 1; 1357 options->aead_iv.length = 1358 parse_bytes(options->aead_iv.data, optarg, MAX_IV_SIZE); 1359 if (options->aead_iv.length > 0) 1360 return 0; 1361 else 1362 return -1; 1363 } 1364 1365 else if (strcmp(lgopts[option_index].name, "aead_iv_random_size") == 0) 1366 return parse_size(&options->aead_iv_random_size, optarg); 1367 1368 else if (strcmp(lgopts[option_index].name, "aad") == 0) { 1369 options->aad_param = 1; 1370 options->aad.length = 1371 parse_bytes(options->aad.data, optarg, MAX_AAD_SIZE); 1372 if (options->aad.length > 0) 1373 return 0; 1374 else 1375 return -1; 1376 } 1377 1378 else if (strcmp(lgopts[option_index].name, "aad_random_size") == 0) { 1379 return parse_size(&options->aad_random_size, optarg); 1380 } 1381 1382 else if (strcmp(lgopts[option_index].name, "digest_size") == 0) { 1383 return parse_size(&options->digest_size, optarg); 1384 } 1385 1386 else if (strcmp(lgopts[option_index].name, "sessionless") == 0) { 1387 options->sessionless = 1; 1388 return 0; 1389 } 1390 1391 else if (strcmp(lgopts[option_index].name, "cryptodev_mask") == 0) 1392 return parse_cryptodev_mask(options, optarg); 1393 1394 else if (strcmp(lgopts[option_index].name, "mac-updating") == 0) { 1395 options->mac_updating = 1; 1396 return 0; 1397 } 1398 1399 else if (strcmp(lgopts[option_index].name, "no-mac-updating") == 0) { 1400 options->mac_updating = 0; 1401 return 0; 1402 } 1403 1404 return -1; 1405 } 1406 1407 /** Parse port mask */ 1408 static int 1409 l2fwd_crypto_parse_portmask(struct l2fwd_crypto_options *options, 1410 const char *q_arg) 1411 { 1412 char *end = NULL; 1413 unsigned long pm; 1414 1415 /* parse hexadecimal string */ 1416 pm = strtoul(q_arg, &end, 16); 1417 if ((pm == '\0') || (end == NULL) || (*end != '\0')) 1418 pm = 0; 1419 1420 options->portmask = pm; 1421 if (options->portmask == 0) { 1422 printf("invalid portmask specified\n"); 1423 return -1; 1424 } 1425 1426 return pm; 1427 } 1428 1429 /** Parse number of queues */ 1430 static int 1431 l2fwd_crypto_parse_nqueue(struct l2fwd_crypto_options *options, 1432 const char *q_arg) 1433 { 1434 char *end = NULL; 1435 unsigned long n; 1436 1437 /* parse hexadecimal string */ 1438 n = strtoul(q_arg, &end, 10); 1439 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) 1440 n = 0; 1441 else if (n >= MAX_RX_QUEUE_PER_LCORE) 1442 n = 0; 1443 1444 options->nb_ports_per_lcore = n; 1445 if (options->nb_ports_per_lcore == 0) { 1446 printf("invalid number of ports selected\n"); 1447 return -1; 1448 } 1449 1450 return 0; 1451 } 1452 1453 /** Parse timer period */ 1454 static int 1455 l2fwd_crypto_parse_timer_period(struct l2fwd_crypto_options *options, 1456 const char *q_arg) 1457 { 1458 char *end = NULL; 1459 unsigned long n; 1460 1461 /* parse number string */ 1462 n = (unsigned)strtol(q_arg, &end, 10); 1463 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) 1464 n = 0; 1465 1466 if (n >= MAX_TIMER_PERIOD) { 1467 printf("Warning refresh period specified %lu is greater than " 1468 "max value %lu! using max value", 1469 n, MAX_TIMER_PERIOD); 1470 n = MAX_TIMER_PERIOD; 1471 } 1472 1473 options->refresh_period = n * 1000 * TIMER_MILLISECOND; 1474 1475 return 0; 1476 } 1477 1478 /** Generate default options for application */ 1479 static void 1480 l2fwd_crypto_default_options(struct l2fwd_crypto_options *options) 1481 { 1482 options->portmask = 0xffffffff; 1483 options->nb_ports_per_lcore = 1; 1484 options->refresh_period = 10000; 1485 options->single_lcore = 0; 1486 options->sessionless = 0; 1487 1488 options->xform_chain = L2FWD_CRYPTO_CIPHER_HASH; 1489 1490 /* Cipher Data */ 1491 options->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; 1492 options->cipher_xform.next = NULL; 1493 options->ckey_param = 0; 1494 options->ckey_random_size = -1; 1495 options->cipher_xform.cipher.key.length = 0; 1496 options->cipher_iv_param = 0; 1497 options->cipher_iv_random_size = -1; 1498 options->cipher_iv.length = 0; 1499 1500 options->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; 1501 options->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 1502 options->cipher_xform.cipher.dataunit_len = 0; 1503 1504 /* Authentication Data */ 1505 options->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; 1506 options->auth_xform.next = NULL; 1507 options->akey_param = 0; 1508 options->akey_random_size = -1; 1509 options->auth_xform.auth.key.length = 0; 1510 options->auth_iv_param = 0; 1511 options->auth_iv_random_size = -1; 1512 options->auth_iv.length = 0; 1513 1514 options->auth_xform.auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC; 1515 options->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; 1516 1517 /* AEAD Data */ 1518 options->aead_xform.type = RTE_CRYPTO_SYM_XFORM_AEAD; 1519 options->aead_xform.next = NULL; 1520 options->aead_key_param = 0; 1521 options->aead_key_random_size = -1; 1522 options->aead_xform.aead.key.length = 0; 1523 options->aead_iv_param = 0; 1524 options->aead_iv_random_size = -1; 1525 options->aead_iv.length = 0; 1526 1527 options->aead_xform.aead.algo = RTE_CRYPTO_AEAD_AES_GCM; 1528 options->aead_xform.aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT; 1529 1530 options->aad_param = 0; 1531 options->aad_random_size = -1; 1532 options->aad.length = 0; 1533 1534 options->digest_size = -1; 1535 1536 options->type = CDEV_TYPE_ANY; 1537 options->cryptodev_mask = UINT64_MAX; 1538 1539 options->mac_updating = 1; 1540 } 1541 1542 static void 1543 display_cipher_info(struct l2fwd_crypto_options *options) 1544 { 1545 printf("\n---- Cipher information ---\n"); 1546 printf("Algorithm: %s\n", 1547 rte_crypto_cipher_algorithm_strings[options->cipher_xform.cipher.algo]); 1548 rte_hexdump(stdout, "Cipher key:", 1549 options->cipher_xform.cipher.key.data, 1550 options->cipher_xform.cipher.key.length); 1551 rte_hexdump(stdout, "IV:", options->cipher_iv.data, options->cipher_iv.length); 1552 } 1553 1554 static void 1555 display_auth_info(struct l2fwd_crypto_options *options) 1556 { 1557 printf("\n---- Authentication information ---\n"); 1558 printf("Algorithm: %s\n", 1559 rte_crypto_auth_algorithm_strings[options->auth_xform.auth.algo]); 1560 rte_hexdump(stdout, "Auth key:", 1561 options->auth_xform.auth.key.data, 1562 options->auth_xform.auth.key.length); 1563 rte_hexdump(stdout, "IV:", options->auth_iv.data, options->auth_iv.length); 1564 } 1565 1566 static void 1567 display_aead_info(struct l2fwd_crypto_options *options) 1568 { 1569 printf("\n---- AEAD information ---\n"); 1570 printf("Algorithm: %s\n", 1571 rte_crypto_aead_algorithm_strings[options->aead_xform.aead.algo]); 1572 rte_hexdump(stdout, "AEAD key:", 1573 options->aead_xform.aead.key.data, 1574 options->aead_xform.aead.key.length); 1575 rte_hexdump(stdout, "IV:", options->aead_iv.data, options->aead_iv.length); 1576 rte_hexdump(stdout, "AAD:", options->aad.data, options->aad.length); 1577 } 1578 1579 static void 1580 l2fwd_crypto_options_print(struct l2fwd_crypto_options *options) 1581 { 1582 char string_cipher_op[MAX_STR_LEN]; 1583 char string_auth_op[MAX_STR_LEN]; 1584 char string_aead_op[MAX_STR_LEN]; 1585 1586 if (options->cipher_xform.cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1587 strcpy(string_cipher_op, "Encrypt"); 1588 else 1589 strcpy(string_cipher_op, "Decrypt"); 1590 1591 if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) 1592 strcpy(string_auth_op, "Auth generate"); 1593 else 1594 strcpy(string_auth_op, "Auth verify"); 1595 1596 if (options->aead_xform.aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 1597 strcpy(string_aead_op, "Authenticated encryption"); 1598 else 1599 strcpy(string_aead_op, "Authenticated decryption"); 1600 1601 1602 printf("Options:-\nn"); 1603 printf("portmask: %x\n", options->portmask); 1604 printf("ports per lcore: %u\n", options->nb_ports_per_lcore); 1605 printf("refresh period : %u\n", options->refresh_period); 1606 printf("single lcore mode: %s\n", 1607 options->single_lcore ? "enabled" : "disabled"); 1608 printf("stats_printing: %s\n", 1609 options->refresh_period == 0 ? "disabled" : "enabled"); 1610 1611 printf("sessionless crypto: %s\n", 1612 options->sessionless ? "enabled" : "disabled"); 1613 1614 if (options->ckey_param && (options->ckey_random_size != -1)) 1615 printf("Cipher key already parsed, ignoring size of random key\n"); 1616 1617 if (options->akey_param && (options->akey_random_size != -1)) 1618 printf("Auth key already parsed, ignoring size of random key\n"); 1619 1620 if (options->cipher_iv_param && (options->cipher_iv_random_size != -1)) 1621 printf("Cipher IV already parsed, ignoring size of random IV\n"); 1622 1623 if (options->auth_iv_param && (options->auth_iv_random_size != -1)) 1624 printf("Auth IV already parsed, ignoring size of random IV\n"); 1625 1626 if (options->aad_param && (options->aad_random_size != -1)) 1627 printf("AAD already parsed, ignoring size of random AAD\n"); 1628 1629 printf("\nCrypto chain: "); 1630 switch (options->xform_chain) { 1631 case L2FWD_CRYPTO_AEAD: 1632 printf("Input --> %s --> Output\n", string_aead_op); 1633 display_aead_info(options); 1634 break; 1635 case L2FWD_CRYPTO_CIPHER_HASH: 1636 printf("Input --> %s --> %s --> Output\n", 1637 string_cipher_op, string_auth_op); 1638 display_cipher_info(options); 1639 display_auth_info(options); 1640 break; 1641 case L2FWD_CRYPTO_HASH_CIPHER: 1642 printf("Input --> %s --> %s --> Output\n", 1643 string_auth_op, string_cipher_op); 1644 display_cipher_info(options); 1645 display_auth_info(options); 1646 break; 1647 case L2FWD_CRYPTO_HASH_ONLY: 1648 printf("Input --> %s --> Output\n", string_auth_op); 1649 display_auth_info(options); 1650 break; 1651 case L2FWD_CRYPTO_CIPHER_ONLY: 1652 printf("Input --> %s --> Output\n", string_cipher_op); 1653 display_cipher_info(options); 1654 break; 1655 } 1656 } 1657 1658 /* Parse the argument given in the command line of the application */ 1659 static int 1660 l2fwd_crypto_parse_args(struct l2fwd_crypto_options *options, 1661 int argc, char **argv) 1662 { 1663 int opt, retval, option_index; 1664 char **argvopt = argv, *prgname = argv[0]; 1665 1666 static struct option lgopts[] = { 1667 { "sessionless", no_argument, 0, 0 }, 1668 1669 { "cdev_type", required_argument, 0, 0 }, 1670 { "chain", required_argument, 0, 0 }, 1671 1672 { "cipher_algo", required_argument, 0, 0 }, 1673 { "cipher_op", required_argument, 0, 0 }, 1674 { "cipher_key", required_argument, 0, 0 }, 1675 { "cipher_key_random_size", required_argument, 0, 0 }, 1676 { "cipher_iv", required_argument, 0, 0 }, 1677 { "cipher_iv_random_size", required_argument, 0, 0 }, 1678 { "cipher_dataunit_len", required_argument, 0, 0}, 1679 1680 { "auth_algo", required_argument, 0, 0 }, 1681 { "auth_op", required_argument, 0, 0 }, 1682 { "auth_key", required_argument, 0, 0 }, 1683 { "auth_key_random_size", required_argument, 0, 0 }, 1684 { "auth_iv", required_argument, 0, 0 }, 1685 { "auth_iv_random_size", required_argument, 0, 0 }, 1686 1687 { "aead_algo", required_argument, 0, 0 }, 1688 { "aead_op", required_argument, 0, 0 }, 1689 { "aead_key", required_argument, 0, 0 }, 1690 { "aead_key_random_size", required_argument, 0, 0 }, 1691 { "aead_iv", required_argument, 0, 0 }, 1692 { "aead_iv_random_size", required_argument, 0, 0 }, 1693 1694 { "aad", required_argument, 0, 0 }, 1695 { "aad_random_size", required_argument, 0, 0 }, 1696 1697 { "digest_size", required_argument, 0, 0 }, 1698 1699 { "sessionless", no_argument, 0, 0 }, 1700 { "cryptodev_mask", required_argument, 0, 0}, 1701 1702 { "mac-updating", no_argument, 0, 0}, 1703 { "no-mac-updating", no_argument, 0, 0}, 1704 1705 { NULL, 0, 0, 0 } 1706 }; 1707 1708 l2fwd_crypto_default_options(options); 1709 1710 while ((opt = getopt_long(argc, argvopt, "p:q:sT:", lgopts, 1711 &option_index)) != EOF) { 1712 switch (opt) { 1713 /* long options */ 1714 case 0: 1715 retval = l2fwd_crypto_parse_args_long_options(options, 1716 lgopts, option_index); 1717 if (retval < 0) { 1718 l2fwd_crypto_usage(prgname); 1719 return -1; 1720 } 1721 break; 1722 1723 /* portmask */ 1724 case 'p': 1725 retval = l2fwd_crypto_parse_portmask(options, optarg); 1726 if (retval < 0) { 1727 l2fwd_crypto_usage(prgname); 1728 return -1; 1729 } 1730 break; 1731 1732 /* nqueue */ 1733 case 'q': 1734 retval = l2fwd_crypto_parse_nqueue(options, optarg); 1735 if (retval < 0) { 1736 l2fwd_crypto_usage(prgname); 1737 return -1; 1738 } 1739 break; 1740 1741 /* single */ 1742 case 's': 1743 options->single_lcore = 1; 1744 1745 break; 1746 1747 /* timer period */ 1748 case 'T': 1749 retval = l2fwd_crypto_parse_timer_period(options, 1750 optarg); 1751 if (retval < 0) { 1752 l2fwd_crypto_usage(prgname); 1753 return -1; 1754 } 1755 break; 1756 1757 default: 1758 l2fwd_crypto_usage(prgname); 1759 return -1; 1760 } 1761 } 1762 1763 1764 if (optind >= 0) 1765 argv[optind-1] = prgname; 1766 1767 retval = optind-1; 1768 optind = 1; /* reset getopt lib */ 1769 1770 return retval; 1771 } 1772 1773 /* Check the link status of all ports in up to 9s, and print them finally */ 1774 static void 1775 check_all_ports_link_status(uint32_t port_mask) 1776 { 1777 #define CHECK_INTERVAL 100 /* 100ms */ 1778 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1779 uint16_t portid; 1780 uint8_t count, all_ports_up, print_flag = 0; 1781 struct rte_eth_link link; 1782 int ret; 1783 char link_status_text[RTE_ETH_LINK_MAX_STR_LEN]; 1784 1785 printf("\nChecking link status"); 1786 fflush(stdout); 1787 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1788 all_ports_up = 1; 1789 RTE_ETH_FOREACH_DEV(portid) { 1790 if ((port_mask & (1 << portid)) == 0) 1791 continue; 1792 memset(&link, 0, sizeof(link)); 1793 ret = rte_eth_link_get_nowait(portid, &link); 1794 if (ret < 0) { 1795 all_ports_up = 0; 1796 if (print_flag == 1) 1797 printf("Port %u link get failed: %s\n", 1798 portid, rte_strerror(-ret)); 1799 continue; 1800 } 1801 /* print link status if flag set */ 1802 if (print_flag == 1) { 1803 rte_eth_link_to_str(link_status_text, 1804 sizeof(link_status_text), &link); 1805 printf("Port %d %s\n", portid, 1806 link_status_text); 1807 continue; 1808 } 1809 /* clear all_ports_up flag if any link down */ 1810 if (link.link_status == RTE_ETH_LINK_DOWN) { 1811 all_ports_up = 0; 1812 break; 1813 } 1814 } 1815 /* after finally printing all link status, get out */ 1816 if (print_flag == 1) 1817 break; 1818 1819 if (all_ports_up == 0) { 1820 printf("."); 1821 fflush(stdout); 1822 rte_delay_ms(CHECK_INTERVAL); 1823 } 1824 1825 /* set the print_flag if all ports up or timeout */ 1826 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1827 print_flag = 1; 1828 printf("done\n"); 1829 } 1830 } 1831 } 1832 1833 /* Check if device has to be HW/SW or any */ 1834 static int 1835 check_type(const struct l2fwd_crypto_options *options, 1836 const struct rte_cryptodev_info *dev_info) 1837 { 1838 if (options->type == CDEV_TYPE_HW && 1839 (dev_info->feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED)) 1840 return 0; 1841 if (options->type == CDEV_TYPE_SW && 1842 !(dev_info->feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED)) 1843 return 0; 1844 if (options->type == CDEV_TYPE_ANY) 1845 return 0; 1846 1847 return -1; 1848 } 1849 1850 static const struct rte_cryptodev_capabilities * 1851 check_device_support_cipher_algo(const struct l2fwd_crypto_options *options, 1852 const struct rte_cryptodev_info *dev_info, 1853 uint8_t cdev_id) 1854 { 1855 unsigned int i = 0; 1856 const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; 1857 enum rte_crypto_cipher_algorithm cap_cipher_algo; 1858 enum rte_crypto_cipher_algorithm opt_cipher_algo = 1859 options->cipher_xform.cipher.algo; 1860 1861 while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1862 cap_cipher_algo = cap->sym.cipher.algo; 1863 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 1864 if (cap_cipher_algo == opt_cipher_algo) { 1865 if (check_type(options, dev_info) == 0) 1866 break; 1867 } 1868 } 1869 cap = &dev_info->capabilities[++i]; 1870 } 1871 1872 if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1873 printf("Algorithm %s not supported by cryptodev %u" 1874 " or device not of preferred type (%s)\n", 1875 rte_crypto_cipher_algorithm_strings[opt_cipher_algo], 1876 cdev_id, 1877 options->string_type); 1878 return NULL; 1879 } 1880 1881 return cap; 1882 } 1883 1884 static const struct rte_cryptodev_capabilities * 1885 check_device_support_auth_algo(const struct l2fwd_crypto_options *options, 1886 const struct rte_cryptodev_info *dev_info, 1887 uint8_t cdev_id) 1888 { 1889 unsigned int i = 0; 1890 const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; 1891 enum rte_crypto_auth_algorithm cap_auth_algo; 1892 enum rte_crypto_auth_algorithm opt_auth_algo = 1893 options->auth_xform.auth.algo; 1894 1895 while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1896 cap_auth_algo = cap->sym.auth.algo; 1897 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AUTH) { 1898 if (cap_auth_algo == opt_auth_algo) { 1899 if (check_type(options, dev_info) == 0) 1900 break; 1901 } 1902 } 1903 cap = &dev_info->capabilities[++i]; 1904 } 1905 1906 if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1907 printf("Algorithm %s not supported by cryptodev %u" 1908 " or device not of preferred type (%s)\n", 1909 rte_crypto_auth_algorithm_strings[opt_auth_algo], 1910 cdev_id, 1911 options->string_type); 1912 return NULL; 1913 } 1914 1915 return cap; 1916 } 1917 1918 static const struct rte_cryptodev_capabilities * 1919 check_device_support_aead_algo(const struct l2fwd_crypto_options *options, 1920 const struct rte_cryptodev_info *dev_info, 1921 uint8_t cdev_id) 1922 { 1923 unsigned int i = 0; 1924 const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; 1925 enum rte_crypto_aead_algorithm cap_aead_algo; 1926 enum rte_crypto_aead_algorithm opt_aead_algo = 1927 options->aead_xform.aead.algo; 1928 1929 while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1930 cap_aead_algo = cap->sym.aead.algo; 1931 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1932 if (cap_aead_algo == opt_aead_algo) { 1933 if (check_type(options, dev_info) == 0) 1934 break; 1935 } 1936 } 1937 cap = &dev_info->capabilities[++i]; 1938 } 1939 1940 if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1941 printf("Algorithm %s not supported by cryptodev %u" 1942 " or device not of preferred type (%s)\n", 1943 rte_crypto_aead_algorithm_strings[opt_aead_algo], 1944 cdev_id, 1945 options->string_type); 1946 return NULL; 1947 } 1948 1949 return cap; 1950 } 1951 1952 /* Check if the device is enabled by cryptodev_mask */ 1953 static int 1954 check_cryptodev_mask(struct l2fwd_crypto_options *options, 1955 uint8_t cdev_id) 1956 { 1957 if (options->cryptodev_mask & (1 << cdev_id)) 1958 return 0; 1959 1960 return -1; 1961 } 1962 1963 static inline int 1964 check_supported_size(uint16_t length, uint16_t min, uint16_t max, 1965 uint16_t increment) 1966 { 1967 uint16_t supp_size; 1968 1969 /* Single value */ 1970 if (increment == 0) { 1971 if (length == min) 1972 return 0; 1973 else 1974 return -1; 1975 } 1976 1977 /* Range of values */ 1978 for (supp_size = min; supp_size <= max; supp_size += increment) { 1979 if (length == supp_size) 1980 return 0; 1981 } 1982 1983 return -1; 1984 } 1985 1986 static int 1987 check_iv_param(const struct rte_crypto_param_range *iv_range_size, 1988 unsigned int iv_param, int iv_random_size, 1989 uint16_t iv_length) 1990 { 1991 /* 1992 * Check if length of provided IV is supported 1993 * by the algorithm chosen. 1994 */ 1995 if (iv_param) { 1996 if (check_supported_size(iv_length, 1997 iv_range_size->min, 1998 iv_range_size->max, 1999 iv_range_size->increment) 2000 != 0) 2001 return -1; 2002 /* 2003 * Check if length of IV to be randomly generated 2004 * is supported by the algorithm chosen. 2005 */ 2006 } else if (iv_random_size != -1) { 2007 if (check_supported_size(iv_random_size, 2008 iv_range_size->min, 2009 iv_range_size->max, 2010 iv_range_size->increment) 2011 != 0) 2012 return -1; 2013 } 2014 2015 return 0; 2016 } 2017 2018 static int 2019 check_capabilities(struct l2fwd_crypto_options *options, uint8_t cdev_id) 2020 { 2021 struct rte_cryptodev_info dev_info; 2022 const struct rte_cryptodev_capabilities *cap; 2023 2024 rte_cryptodev_info_get(cdev_id, &dev_info); 2025 2026 /* Set AEAD parameters */ 2027 if (options->xform_chain == L2FWD_CRYPTO_AEAD) { 2028 /* Check if device supports AEAD algo */ 2029 cap = check_device_support_aead_algo(options, &dev_info, 2030 cdev_id); 2031 if (cap == NULL) 2032 return -1; 2033 2034 if (check_iv_param(&cap->sym.aead.iv_size, 2035 options->aead_iv_param, 2036 options->aead_iv_random_size, 2037 options->aead_iv.length) != 0) { 2038 RTE_LOG(DEBUG, USER1, 2039 "Device %u does not support IV length\n", 2040 cdev_id); 2041 return -1; 2042 } 2043 2044 /* 2045 * Check if length of provided AEAD key is supported 2046 * by the algorithm chosen. 2047 */ 2048 if (options->aead_key_param) { 2049 if (check_supported_size( 2050 options->aead_xform.aead.key.length, 2051 cap->sym.aead.key_size.min, 2052 cap->sym.aead.key_size.max, 2053 cap->sym.aead.key_size.increment) 2054 != 0) { 2055 RTE_LOG(DEBUG, USER1, 2056 "Device %u does not support " 2057 "AEAD key length\n", 2058 cdev_id); 2059 return -1; 2060 } 2061 /* 2062 * Check if length of the aead key to be randomly generated 2063 * is supported by the algorithm chosen. 2064 */ 2065 } else if (options->aead_key_random_size != -1) { 2066 if (check_supported_size(options->aead_key_random_size, 2067 cap->sym.aead.key_size.min, 2068 cap->sym.aead.key_size.max, 2069 cap->sym.aead.key_size.increment) 2070 != 0) { 2071 RTE_LOG(DEBUG, USER1, 2072 "Device %u does not support " 2073 "AEAD key length\n", 2074 cdev_id); 2075 return -1; 2076 } 2077 } 2078 2079 2080 /* 2081 * Check if length of provided AAD is supported 2082 * by the algorithm chosen. 2083 */ 2084 if (options->aad_param) { 2085 if (check_supported_size(options->aad.length, 2086 cap->sym.aead.aad_size.min, 2087 cap->sym.aead.aad_size.max, 2088 cap->sym.aead.aad_size.increment) 2089 != 0) { 2090 RTE_LOG(DEBUG, USER1, 2091 "Device %u does not support " 2092 "AAD length\n", 2093 cdev_id); 2094 return -1; 2095 } 2096 /* 2097 * Check if length of AAD to be randomly generated 2098 * is supported by the algorithm chosen. 2099 */ 2100 } else if (options->aad_random_size != -1) { 2101 if (check_supported_size(options->aad_random_size, 2102 cap->sym.aead.aad_size.min, 2103 cap->sym.aead.aad_size.max, 2104 cap->sym.aead.aad_size.increment) 2105 != 0) { 2106 RTE_LOG(DEBUG, USER1, 2107 "Device %u does not support " 2108 "AAD length\n", 2109 cdev_id); 2110 return -1; 2111 } 2112 } 2113 2114 /* Check if digest size is supported by the algorithm. */ 2115 if (options->digest_size != -1) { 2116 if (check_supported_size(options->digest_size, 2117 cap->sym.aead.digest_size.min, 2118 cap->sym.aead.digest_size.max, 2119 cap->sym.aead.digest_size.increment) 2120 != 0) { 2121 RTE_LOG(DEBUG, USER1, 2122 "Device %u does not support " 2123 "digest length\n", 2124 cdev_id); 2125 return -1; 2126 } 2127 } 2128 } 2129 2130 /* Set cipher parameters */ 2131 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2132 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2133 options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { 2134 2135 /* Check if device supports cipher algo. 8< */ 2136 cap = check_device_support_cipher_algo(options, &dev_info, 2137 cdev_id); 2138 if (cap == NULL) 2139 return -1; 2140 2141 if (check_iv_param(&cap->sym.cipher.iv_size, 2142 options->cipher_iv_param, 2143 options->cipher_iv_random_size, 2144 options->cipher_iv.length) != 0) { 2145 RTE_LOG(DEBUG, USER1, 2146 "Device %u does not support IV length\n", 2147 cdev_id); 2148 return -1; 2149 } 2150 /* >8 End of check if device supports cipher algo. */ 2151 2152 /* Check if capable cipher is supported. 8< */ 2153 2154 /* 2155 * Check if length of provided cipher key is supported 2156 * by the algorithm chosen. 2157 */ 2158 if (options->ckey_param) { 2159 if (check_supported_size( 2160 options->cipher_xform.cipher.key.length, 2161 cap->sym.cipher.key_size.min, 2162 cap->sym.cipher.key_size.max, 2163 cap->sym.cipher.key_size.increment) 2164 != 0) { 2165 if (dev_info.feature_flags & 2166 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY) { 2167 RTE_LOG(DEBUG, USER1, 2168 "Key length does not match the device " 2169 "%u capability. Key may be wrapped\n", 2170 cdev_id); 2171 } else { 2172 RTE_LOG(DEBUG, USER1, 2173 "Key length does not match the device " 2174 "%u capability\n", 2175 cdev_id); 2176 return -1; 2177 } 2178 } 2179 2180 /* 2181 * Check if length of the cipher key to be randomly generated 2182 * is supported by the algorithm chosen. 2183 */ 2184 } else if (options->ckey_random_size != -1) { 2185 if (check_supported_size(options->ckey_random_size, 2186 cap->sym.cipher.key_size.min, 2187 cap->sym.cipher.key_size.max, 2188 cap->sym.cipher.key_size.increment) 2189 != 0) { 2190 RTE_LOG(DEBUG, USER1, 2191 "Device %u does not support cipher " 2192 "key length\n", 2193 cdev_id); 2194 return -1; 2195 } 2196 } 2197 2198 if (options->cipher_xform.cipher.dataunit_len > 0) { 2199 if (!(dev_info.feature_flags & 2200 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)) { 2201 RTE_LOG(DEBUG, USER1, 2202 "Device %u does not support " 2203 "cipher multiple data units\n", 2204 cdev_id); 2205 return -1; 2206 } 2207 if (cap->sym.cipher.dataunit_set != 0) { 2208 int ret = 0; 2209 2210 switch (options->cipher_xform.cipher.dataunit_len) { 2211 case 512: 2212 if (!(cap->sym.cipher.dataunit_set & 2213 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES)) 2214 ret = -1; 2215 break; 2216 case 4096: 2217 if (!(cap->sym.cipher.dataunit_set & 2218 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES)) 2219 ret = -1; 2220 break; 2221 default: 2222 ret = -1; 2223 } 2224 if (ret == -1) { 2225 RTE_LOG(DEBUG, USER1, 2226 "Device %u does not support " 2227 "data-unit length %u\n", 2228 cdev_id, 2229 options->cipher_xform.cipher.dataunit_len); 2230 return -1; 2231 } 2232 } 2233 } 2234 /* >8 End of checking if cipher is supported. */ 2235 } 2236 2237 /* Set auth parameters */ 2238 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2239 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2240 options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) { 2241 /* Check if device supports auth algo */ 2242 cap = check_device_support_auth_algo(options, &dev_info, 2243 cdev_id); 2244 if (cap == NULL) 2245 return -1; 2246 2247 if (check_iv_param(&cap->sym.auth.iv_size, 2248 options->auth_iv_param, 2249 options->auth_iv_random_size, 2250 options->auth_iv.length) != 0) { 2251 RTE_LOG(DEBUG, USER1, 2252 "Device %u does not support IV length\n", 2253 cdev_id); 2254 return -1; 2255 } 2256 /* 2257 * Check if length of provided auth key is supported 2258 * by the algorithm chosen. 2259 */ 2260 if (options->akey_param) { 2261 if (check_supported_size( 2262 options->auth_xform.auth.key.length, 2263 cap->sym.auth.key_size.min, 2264 cap->sym.auth.key_size.max, 2265 cap->sym.auth.key_size.increment) 2266 != 0) { 2267 RTE_LOG(DEBUG, USER1, 2268 "Device %u does not support auth " 2269 "key length\n", 2270 cdev_id); 2271 return -1; 2272 } 2273 /* 2274 * Check if length of the auth key to be randomly generated 2275 * is supported by the algorithm chosen. 2276 */ 2277 } else if (options->akey_random_size != -1) { 2278 if (check_supported_size(options->akey_random_size, 2279 cap->sym.auth.key_size.min, 2280 cap->sym.auth.key_size.max, 2281 cap->sym.auth.key_size.increment) 2282 != 0) { 2283 RTE_LOG(DEBUG, USER1, 2284 "Device %u does not support auth " 2285 "key length\n", 2286 cdev_id); 2287 return -1; 2288 } 2289 } 2290 2291 /* Check if digest size is supported by the algorithm. */ 2292 if (options->digest_size != -1) { 2293 if (check_supported_size(options->digest_size, 2294 cap->sym.auth.digest_size.min, 2295 cap->sym.auth.digest_size.max, 2296 cap->sym.auth.digest_size.increment) 2297 != 0) { 2298 RTE_LOG(DEBUG, USER1, 2299 "Device %u does not support " 2300 "digest length\n", 2301 cdev_id); 2302 return -1; 2303 } 2304 } 2305 } 2306 2307 return 0; 2308 } 2309 2310 static int 2311 initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, 2312 uint8_t *enabled_cdevs) 2313 { 2314 uint8_t cdev_id, cdev_count, enabled_cdev_count = 0; 2315 const struct rte_cryptodev_capabilities *cap; 2316 unsigned int sess_sz, max_sess_sz = 0; 2317 uint32_t sessions_needed = 0; 2318 int retval; 2319 2320 cdev_count = rte_cryptodev_count(); 2321 if (cdev_count == 0) { 2322 printf("No crypto devices available\n"); 2323 return -1; 2324 } 2325 2326 for (cdev_id = 0; cdev_id < cdev_count && enabled_cdev_count < nb_ports; 2327 cdev_id++) { 2328 if (check_cryptodev_mask(options, cdev_id) < 0) 2329 continue; 2330 2331 if (check_capabilities(options, cdev_id) < 0) 2332 continue; 2333 2334 sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id); 2335 if (sess_sz > max_sess_sz) 2336 max_sess_sz = sess_sz; 2337 2338 l2fwd_enabled_crypto_mask |= (((uint64_t)1) << cdev_id); 2339 2340 enabled_cdevs[cdev_id] = 1; 2341 enabled_cdev_count++; 2342 } 2343 2344 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { 2345 struct rte_cryptodev_qp_conf qp_conf; 2346 struct rte_cryptodev_info dev_info; 2347 2348 if (enabled_cdevs[cdev_id] == 0) 2349 continue; 2350 2351 if (check_cryptodev_mask(options, cdev_id) < 0) 2352 continue; 2353 2354 if (check_capabilities(options, cdev_id) < 0) 2355 continue; 2356 2357 retval = rte_cryptodev_socket_id(cdev_id); 2358 2359 if (retval < 0) { 2360 printf("Invalid crypto device id used\n"); 2361 return -1; 2362 } 2363 2364 uint8_t socket_id = (uint8_t) retval; 2365 2366 struct rte_cryptodev_config conf = { 2367 .nb_queue_pairs = 1, 2368 .socket_id = socket_id, 2369 .ff_disable = RTE_CRYPTODEV_FF_SECURITY, 2370 }; 2371 2372 rte_cryptodev_info_get(cdev_id, &dev_info); 2373 2374 /* 2375 * Two sessions objects are required for each session 2376 * (one for the header, one for the private data) 2377 */ 2378 if (!strcmp(dev_info.driver_name, "crypto_scheduler")) { 2379 #ifdef RTE_CRYPTO_SCHEDULER 2380 uint32_t nb_workers = 2381 rte_cryptodev_scheduler_workers_get(cdev_id, 2382 NULL); 2383 2384 sessions_needed = enabled_cdev_count * nb_workers; 2385 #endif 2386 } else 2387 sessions_needed = enabled_cdev_count; 2388 2389 if (session_pool_socket[socket_id].priv_mp == NULL) { 2390 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2391 2392 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2393 "priv_sess_mp_%u", socket_id); 2394 2395 session_pool_socket[socket_id].priv_mp = 2396 rte_mempool_create(mp_name, 2397 sessions_needed, 2398 max_sess_sz, 2399 0, 0, NULL, NULL, NULL, 2400 NULL, socket_id, 2401 0); 2402 2403 if (session_pool_socket[socket_id].priv_mp == NULL) { 2404 printf("Cannot create pool on socket %d\n", 2405 socket_id); 2406 return -ENOMEM; 2407 } 2408 2409 printf("Allocated pool \"%s\" on socket %d\n", 2410 mp_name, socket_id); 2411 } 2412 2413 if (session_pool_socket[socket_id].sess_mp == NULL) { 2414 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2415 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2416 "sess_mp_%u", socket_id); 2417 2418 session_pool_socket[socket_id].sess_mp = 2419 rte_cryptodev_sym_session_pool_create( 2420 mp_name, 2421 sessions_needed, 2422 0, 0, 0, socket_id); 2423 2424 if (session_pool_socket[socket_id].sess_mp == NULL) { 2425 printf("Cannot create pool on socket %d\n", 2426 socket_id); 2427 return -ENOMEM; 2428 } 2429 2430 printf("Allocated pool \"%s\" on socket %d\n", 2431 mp_name, socket_id); 2432 } 2433 2434 /* Set AEAD parameters */ 2435 if (options->xform_chain == L2FWD_CRYPTO_AEAD) { 2436 cap = check_device_support_aead_algo(options, &dev_info, 2437 cdev_id); 2438 2439 options->block_size = cap->sym.aead.block_size; 2440 2441 /* Set IV if not provided from command line */ 2442 if (options->aead_iv_param == 0) { 2443 if (options->aead_iv_random_size != -1) 2444 options->aead_iv.length = 2445 options->aead_iv_random_size; 2446 /* No size provided, use minimum size. */ 2447 else 2448 options->aead_iv.length = 2449 cap->sym.aead.iv_size.min; 2450 } 2451 2452 /* Set key if not provided from command line */ 2453 if (options->aead_key_param == 0) { 2454 if (options->aead_key_random_size != -1) 2455 options->aead_xform.aead.key.length = 2456 options->aead_key_random_size; 2457 /* No size provided, use minimum size. */ 2458 else 2459 options->aead_xform.aead.key.length = 2460 cap->sym.aead.key_size.min; 2461 2462 generate_random_key(options->aead_key, 2463 options->aead_xform.aead.key.length); 2464 } 2465 2466 /* Set AAD if not provided from command line */ 2467 if (options->aad_param == 0) { 2468 if (options->aad_random_size != -1) 2469 options->aad.length = 2470 options->aad_random_size; 2471 /* No size provided, use minimum size. */ 2472 else 2473 options->aad.length = 2474 cap->sym.auth.aad_size.min; 2475 } 2476 2477 options->aead_xform.aead.aad_length = 2478 options->aad.length; 2479 2480 /* Set digest size if not provided from command line */ 2481 if (options->digest_size != -1) 2482 options->aead_xform.aead.digest_length = 2483 options->digest_size; 2484 /* No size provided, use minimum size. */ 2485 else 2486 options->aead_xform.aead.digest_length = 2487 cap->sym.aead.digest_size.min; 2488 } 2489 2490 /* Set cipher parameters */ 2491 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2492 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2493 options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { 2494 cap = check_device_support_cipher_algo(options, &dev_info, 2495 cdev_id); 2496 options->block_size = cap->sym.cipher.block_size; 2497 2498 /* Set IV if not provided from command line */ 2499 if (options->cipher_iv_param == 0) { 2500 if (options->cipher_iv_random_size != -1) 2501 options->cipher_iv.length = 2502 options->cipher_iv_random_size; 2503 /* No size provided, use minimum size. */ 2504 else 2505 options->cipher_iv.length = 2506 cap->sym.cipher.iv_size.min; 2507 } 2508 2509 /* Set key if not provided from command line */ 2510 if (options->ckey_param == 0) { 2511 if (options->ckey_random_size != -1) 2512 options->cipher_xform.cipher.key.length = 2513 options->ckey_random_size; 2514 /* No size provided, use minimum size. */ 2515 else 2516 options->cipher_xform.cipher.key.length = 2517 cap->sym.cipher.key_size.min; 2518 2519 generate_random_key(options->cipher_key, 2520 options->cipher_xform.cipher.key.length); 2521 } 2522 } 2523 2524 /* Set auth parameters */ 2525 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2526 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2527 options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) { 2528 cap = check_device_support_auth_algo(options, &dev_info, 2529 cdev_id); 2530 2531 /* Set IV if not provided from command line */ 2532 if (options->auth_iv_param == 0) { 2533 if (options->auth_iv_random_size != -1) 2534 options->auth_iv.length = 2535 options->auth_iv_random_size; 2536 /* No size provided, use minimum size. */ 2537 else 2538 options->auth_iv.length = 2539 cap->sym.auth.iv_size.min; 2540 } 2541 2542 /* Set key if not provided from command line */ 2543 if (options->akey_param == 0) { 2544 if (options->akey_random_size != -1) 2545 options->auth_xform.auth.key.length = 2546 options->akey_random_size; 2547 /* No size provided, use minimum size. */ 2548 else 2549 options->auth_xform.auth.key.length = 2550 cap->sym.auth.key_size.min; 2551 2552 generate_random_key(options->auth_key, 2553 options->auth_xform.auth.key.length); 2554 } 2555 2556 /* Set digest size if not provided from command line */ 2557 if (options->digest_size != -1) 2558 options->auth_xform.auth.digest_length = 2559 options->digest_size; 2560 /* No size provided, use minimum size. */ 2561 else 2562 options->auth_xform.auth.digest_length = 2563 cap->sym.auth.digest_size.min; 2564 } 2565 2566 retval = rte_cryptodev_configure(cdev_id, &conf); 2567 if (retval < 0) { 2568 printf("Failed to configure cryptodev %u", cdev_id); 2569 return -1; 2570 } 2571 2572 qp_conf.nb_descriptors = 2048; 2573 qp_conf.mp_session = session_pool_socket[socket_id].sess_mp; 2574 qp_conf.mp_session_private = 2575 session_pool_socket[socket_id].priv_mp; 2576 2577 retval = rte_cryptodev_queue_pair_setup(cdev_id, 0, &qp_conf, 2578 socket_id); 2579 if (retval < 0) { 2580 printf("Failed to setup queue pair %u on cryptodev %u", 2581 0, cdev_id); 2582 return -1; 2583 } 2584 2585 retval = rte_cryptodev_start(cdev_id); 2586 if (retval < 0) { 2587 printf("Failed to start device %u: error %d\n", 2588 cdev_id, retval); 2589 return -1; 2590 } 2591 } 2592 2593 return enabled_cdev_count; 2594 } 2595 2596 static int 2597 initialize_ports(struct l2fwd_crypto_options *options) 2598 { 2599 uint16_t last_portid = 0, portid; 2600 unsigned enabled_portcount = 0; 2601 unsigned nb_ports = rte_eth_dev_count_avail(); 2602 2603 if (nb_ports == 0) { 2604 printf("No Ethernet ports - bye\n"); 2605 return -1; 2606 } 2607 2608 /* Reset l2fwd_dst_ports */ 2609 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) 2610 l2fwd_dst_ports[portid] = 0; 2611 2612 RTE_ETH_FOREACH_DEV(portid) { 2613 int retval; 2614 struct rte_eth_dev_info dev_info; 2615 struct rte_eth_rxconf rxq_conf; 2616 struct rte_eth_txconf txq_conf; 2617 struct rte_eth_conf local_port_conf = port_conf; 2618 2619 /* Skip ports that are not enabled */ 2620 if ((options->portmask & (1 << portid)) == 0) 2621 continue; 2622 2623 /* init port */ 2624 printf("Initializing port %u... ", portid); 2625 fflush(stdout); 2626 2627 retval = rte_eth_dev_info_get(portid, &dev_info); 2628 if (retval != 0) { 2629 printf("Error during getting device (port %u) info: %s\n", 2630 portid, strerror(-retval)); 2631 return retval; 2632 } 2633 2634 if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 2635 local_port_conf.txmode.offloads |= 2636 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 2637 retval = rte_eth_dev_configure(portid, 1, 1, &local_port_conf); 2638 if (retval < 0) { 2639 printf("Cannot configure device: err=%d, port=%u\n", 2640 retval, portid); 2641 return -1; 2642 } 2643 2644 retval = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, 2645 &nb_txd); 2646 if (retval < 0) { 2647 printf("Cannot adjust number of descriptors: err=%d, port=%u\n", 2648 retval, portid); 2649 return -1; 2650 } 2651 2652 /* init one RX queue */ 2653 fflush(stdout); 2654 rxq_conf = dev_info.default_rxconf; 2655 rxq_conf.offloads = local_port_conf.rxmode.offloads; 2656 retval = rte_eth_rx_queue_setup(portid, 0, nb_rxd, 2657 rte_eth_dev_socket_id(portid), 2658 &rxq_conf, l2fwd_pktmbuf_pool); 2659 if (retval < 0) { 2660 printf("rte_eth_rx_queue_setup:err=%d, port=%u\n", 2661 retval, portid); 2662 return -1; 2663 } 2664 2665 /* init one TX queue on each port */ 2666 fflush(stdout); 2667 txq_conf = dev_info.default_txconf; 2668 txq_conf.offloads = local_port_conf.txmode.offloads; 2669 retval = rte_eth_tx_queue_setup(portid, 0, nb_txd, 2670 rte_eth_dev_socket_id(portid), 2671 &txq_conf); 2672 if (retval < 0) { 2673 printf("rte_eth_tx_queue_setup:err=%d, port=%u\n", 2674 retval, portid); 2675 2676 return -1; 2677 } 2678 2679 /* Start device */ 2680 retval = rte_eth_dev_start(portid); 2681 if (retval < 0) { 2682 printf("rte_eth_dev_start:err=%d, port=%u\n", 2683 retval, portid); 2684 return -1; 2685 } 2686 2687 retval = rte_eth_promiscuous_enable(portid); 2688 if (retval != 0) { 2689 printf("rte_eth_promiscuous_enable:err=%s, port=%u\n", 2690 rte_strerror(-retval), portid); 2691 return -1; 2692 } 2693 2694 retval = rte_eth_macaddr_get(portid, 2695 &l2fwd_ports_eth_addr[portid]); 2696 if (retval < 0) { 2697 printf("rte_eth_macaddr_get :err=%d, port=%u\n", 2698 retval, portid); 2699 return -1; 2700 } 2701 2702 printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n", 2703 portid, 2704 RTE_ETHER_ADDR_BYTES(&l2fwd_ports_eth_addr[portid])); 2705 2706 /* initialize port stats */ 2707 memset(&port_statistics, 0, sizeof(port_statistics)); 2708 2709 /* Setup port forwarding table */ 2710 if (enabled_portcount % 2) { 2711 l2fwd_dst_ports[portid] = last_portid; 2712 l2fwd_dst_ports[last_portid] = portid; 2713 } else { 2714 last_portid = portid; 2715 } 2716 2717 l2fwd_enabled_port_mask |= (1 << portid); 2718 enabled_portcount++; 2719 } 2720 2721 if (enabled_portcount == 1) { 2722 l2fwd_dst_ports[last_portid] = last_portid; 2723 } else if (enabled_portcount % 2) { 2724 printf("odd number of ports in portmask- bye\n"); 2725 return -1; 2726 } 2727 2728 check_all_ports_link_status(l2fwd_enabled_port_mask); 2729 2730 return enabled_portcount; 2731 } 2732 2733 static void 2734 reserve_key_memory(struct l2fwd_crypto_options *options) 2735 { 2736 options->cipher_xform.cipher.key.data = options->cipher_key; 2737 2738 options->auth_xform.auth.key.data = options->auth_key; 2739 2740 options->aead_xform.aead.key.data = options->aead_key; 2741 2742 options->cipher_iv.data = rte_malloc("cipher iv", MAX_KEY_SIZE, 0); 2743 if (options->cipher_iv.data == NULL) 2744 rte_exit(EXIT_FAILURE, "Failed to allocate memory for cipher IV"); 2745 2746 options->auth_iv.data = rte_malloc("auth iv", MAX_KEY_SIZE, 0); 2747 if (options->auth_iv.data == NULL) 2748 rte_exit(EXIT_FAILURE, "Failed to allocate memory for auth IV"); 2749 2750 options->aead_iv.data = rte_malloc("aead_iv", MAX_KEY_SIZE, 0); 2751 if (options->aead_iv.data == NULL) 2752 rte_exit(EXIT_FAILURE, "Failed to allocate memory for AEAD iv"); 2753 2754 options->aad.data = rte_malloc("aad", MAX_KEY_SIZE, 0); 2755 if (options->aad.data == NULL) 2756 rte_exit(EXIT_FAILURE, "Failed to allocate memory for AAD"); 2757 options->aad.phys_addr = rte_malloc_virt2iova(options->aad.data); 2758 } 2759 2760 int 2761 main(int argc, char **argv) 2762 { 2763 struct lcore_queue_conf *qconf = NULL; 2764 struct l2fwd_crypto_options options; 2765 2766 uint8_t nb_cryptodevs, cdev_id; 2767 uint16_t portid; 2768 unsigned lcore_id, rx_lcore_id = 0; 2769 int ret, enabled_cdevcount, enabled_portcount; 2770 uint8_t enabled_cdevs[RTE_CRYPTO_MAX_DEVS] = {0}; 2771 2772 /* init EAL */ 2773 ret = rte_eal_init(argc, argv); 2774 if (ret < 0) 2775 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n"); 2776 argc -= ret; 2777 argv += ret; 2778 2779 /* reserve memory for Cipher/Auth key and IV */ 2780 reserve_key_memory(&options); 2781 2782 /* parse application arguments (after the EAL ones) */ 2783 ret = l2fwd_crypto_parse_args(&options, argc, argv); 2784 if (ret < 0) 2785 rte_exit(EXIT_FAILURE, "Invalid L2FWD-CRYPTO arguments\n"); 2786 2787 printf("MAC updating %s\n", 2788 options.mac_updating ? "enabled" : "disabled"); 2789 2790 /* create the mbuf pool */ 2791 l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF, 512, 2792 RTE_ALIGN(sizeof(struct rte_crypto_op), 2793 RTE_CACHE_LINE_SIZE), 2794 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 2795 if (l2fwd_pktmbuf_pool == NULL) 2796 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 2797 2798 /* create crypto op pool */ 2799 l2fwd_crypto_op_pool = rte_crypto_op_pool_create("crypto_op_pool", 2800 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MBUF, 128, MAXIMUM_IV_LENGTH, 2801 rte_socket_id()); 2802 if (l2fwd_crypto_op_pool == NULL) 2803 rte_exit(EXIT_FAILURE, "Cannot create crypto op pool\n"); 2804 2805 /* Enable Ethernet ports */ 2806 enabled_portcount = initialize_ports(&options); 2807 if (enabled_portcount < 1) 2808 rte_exit(EXIT_FAILURE, "Failed to initial Ethernet ports\n"); 2809 2810 /* Initialize the port/queue configuration of each logical core */ 2811 RTE_ETH_FOREACH_DEV(portid) { 2812 2813 /* skip ports that are not enabled */ 2814 if ((options.portmask & (1 << portid)) == 0) 2815 continue; 2816 2817 if (options.single_lcore && qconf == NULL) { 2818 while (rte_lcore_is_enabled(rx_lcore_id) == 0) { 2819 rx_lcore_id++; 2820 if (rx_lcore_id >= RTE_MAX_LCORE) 2821 rte_exit(EXIT_FAILURE, 2822 "Not enough cores\n"); 2823 } 2824 } else if (!options.single_lcore) { 2825 /* get the lcore_id for this port */ 2826 while (rte_lcore_is_enabled(rx_lcore_id) == 0 || 2827 lcore_queue_conf[rx_lcore_id].nb_rx_ports == 2828 options.nb_ports_per_lcore) { 2829 rx_lcore_id++; 2830 if (rx_lcore_id >= RTE_MAX_LCORE) 2831 rte_exit(EXIT_FAILURE, 2832 "Not enough cores\n"); 2833 } 2834 } 2835 2836 /* Assigned a new logical core in the loop above. */ 2837 if (qconf != &lcore_queue_conf[rx_lcore_id]) 2838 qconf = &lcore_queue_conf[rx_lcore_id]; 2839 2840 qconf->rx_port_list[qconf->nb_rx_ports] = portid; 2841 qconf->nb_rx_ports++; 2842 2843 printf("Lcore %u: RX port %u\n", rx_lcore_id, portid); 2844 } 2845 2846 /* Enable Crypto devices */ 2847 enabled_cdevcount = initialize_cryptodevs(&options, enabled_portcount, 2848 enabled_cdevs); 2849 if (enabled_cdevcount < 0) 2850 rte_exit(EXIT_FAILURE, "Failed to initialize crypto devices\n"); 2851 2852 if (enabled_cdevcount < enabled_portcount) 2853 rte_exit(EXIT_FAILURE, "Number of capable crypto devices (%d) " 2854 "has to be more or equal to number of ports (%d)\n", 2855 enabled_cdevcount, enabled_portcount); 2856 2857 nb_cryptodevs = rte_cryptodev_count(); 2858 2859 /* Initialize the port/cryptodev configuration of each logical core */ 2860 for (rx_lcore_id = 0, qconf = NULL, cdev_id = 0; 2861 cdev_id < nb_cryptodevs && enabled_cdevcount; 2862 cdev_id++) { 2863 /* Crypto op not supported by crypto device */ 2864 if (!enabled_cdevs[cdev_id]) 2865 continue; 2866 2867 if (options.single_lcore && qconf == NULL) { 2868 while (rte_lcore_is_enabled(rx_lcore_id) == 0) { 2869 rx_lcore_id++; 2870 if (rx_lcore_id >= RTE_MAX_LCORE) 2871 rte_exit(EXIT_FAILURE, 2872 "Not enough cores\n"); 2873 } 2874 } else if (!options.single_lcore) { 2875 /* get the lcore_id for this port */ 2876 while (rte_lcore_is_enabled(rx_lcore_id) == 0 || 2877 lcore_queue_conf[rx_lcore_id].nb_crypto_devs == 2878 options.nb_ports_per_lcore) { 2879 rx_lcore_id++; 2880 if (rx_lcore_id >= RTE_MAX_LCORE) 2881 rte_exit(EXIT_FAILURE, 2882 "Not enough cores\n"); 2883 } 2884 } 2885 2886 /* Assigned a new logical core in the loop above. */ 2887 if (qconf != &lcore_queue_conf[rx_lcore_id]) 2888 qconf = &lcore_queue_conf[rx_lcore_id]; 2889 2890 qconf->cryptodev_list[qconf->nb_crypto_devs] = cdev_id; 2891 qconf->nb_crypto_devs++; 2892 2893 enabled_cdevcount--; 2894 2895 printf("Lcore %u: cryptodev %u\n", rx_lcore_id, 2896 (unsigned)cdev_id); 2897 } 2898 2899 /* launch per-lcore init on every lcore */ 2900 rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, (void *)&options, 2901 CALL_MAIN); 2902 RTE_LCORE_FOREACH_WORKER(lcore_id) { 2903 if (rte_eal_wait_lcore(lcore_id) < 0) 2904 return -1; 2905 } 2906 2907 /* clean up the EAL */ 2908 rte_eal_cleanup(); 2909 2910 return 0; 2911 } 2912