1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2015-2016 Intel Corporation 3 */ 4 5 #include <time.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <inttypes.h> 11 #include <sys/types.h> 12 #include <sys/queue.h> 13 #include <netinet/in.h> 14 #include <setjmp.h> 15 #include <stdarg.h> 16 #include <ctype.h> 17 #include <errno.h> 18 #include <getopt.h> 19 #include <fcntl.h> 20 #include <unistd.h> 21 #include <signal.h> 22 23 #include <rte_string_fns.h> 24 #include <rte_branch_prediction.h> 25 #include <rte_common.h> 26 #include <rte_cryptodev.h> 27 #include <rte_cycles.h> 28 #include <rte_debug.h> 29 #include <rte_eal.h> 30 #include <rte_ether.h> 31 #include <rte_ethdev.h> 32 #include <rte_interrupts.h> 33 #include <rte_ip.h> 34 #include <rte_launch.h> 35 #include <rte_lcore.h> 36 #include <rte_log.h> 37 #include <rte_malloc.h> 38 #include <rte_mbuf.h> 39 #include <rte_memcpy.h> 40 #include <rte_memory.h> 41 #include <rte_mempool.h> 42 #include <rte_per_lcore.h> 43 #include <rte_prefetch.h> 44 #include <rte_random.h> 45 #include <rte_hexdump.h> 46 #ifdef RTE_CRYPTO_SCHEDULER 47 #include <rte_cryptodev_scheduler.h> 48 #endif 49 50 enum cdev_type { 51 CDEV_TYPE_ANY, 52 CDEV_TYPE_HW, 53 CDEV_TYPE_SW 54 }; 55 56 #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1 57 58 #define NB_MBUF 8192 59 60 #define MAX_STR_LEN 32 61 #define MAX_KEY_SIZE 128 62 #define MAX_IV_SIZE 16 63 #define MAX_AAD_SIZE 65535 64 #define MAX_PKT_BURST 32 65 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 66 #define SESSION_POOL_CACHE_SIZE 0 67 68 #define MAXIMUM_IV_LENGTH 16 69 #define IV_OFFSET (sizeof(struct rte_crypto_op) + \ 70 sizeof(struct rte_crypto_sym_op)) 71 72 /* 73 * Configurable number of RX/TX ring descriptors 74 */ 75 #define RTE_TEST_RX_DESC_DEFAULT 1024 76 #define RTE_TEST_TX_DESC_DEFAULT 1024 77 78 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; 79 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; 80 81 /* ethernet addresses of ports */ 82 static struct rte_ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS]; 83 84 /* mask of enabled ports */ 85 static uint64_t l2fwd_enabled_port_mask; 86 static uint64_t l2fwd_enabled_crypto_mask; 87 88 /* list of enabled ports */ 89 static uint16_t l2fwd_dst_ports[RTE_MAX_ETHPORTS]; 90 91 92 struct pkt_buffer { 93 unsigned len; 94 struct rte_mbuf *buffer[MAX_PKT_BURST]; 95 }; 96 97 struct op_buffer { 98 unsigned len; 99 struct rte_crypto_op *buffer[MAX_PKT_BURST]; 100 }; 101 102 #define MAX_RX_QUEUE_PER_LCORE 16 103 #define MAX_TX_QUEUE_PER_PORT 16 104 105 enum l2fwd_crypto_xform_chain { 106 L2FWD_CRYPTO_CIPHER_HASH, 107 L2FWD_CRYPTO_HASH_CIPHER, 108 L2FWD_CRYPTO_CIPHER_ONLY, 109 L2FWD_CRYPTO_HASH_ONLY, 110 L2FWD_CRYPTO_AEAD 111 }; 112 113 struct l2fwd_key { 114 uint8_t *data; 115 uint32_t length; 116 rte_iova_t phys_addr; 117 }; 118 119 struct l2fwd_iv { 120 uint8_t *data; 121 uint16_t length; 122 }; 123 124 /** l2fwd crypto application command line options */ 125 struct l2fwd_crypto_options { 126 unsigned portmask; 127 unsigned nb_ports_per_lcore; 128 unsigned refresh_period; 129 unsigned single_lcore:1; 130 131 enum cdev_type type; 132 unsigned sessionless:1; 133 134 enum l2fwd_crypto_xform_chain xform_chain; 135 136 struct rte_crypto_sym_xform cipher_xform; 137 unsigned ckey_param; 138 int ckey_random_size; 139 uint8_t cipher_key[MAX_KEY_SIZE]; 140 141 struct l2fwd_iv cipher_iv; 142 unsigned int cipher_iv_param; 143 int cipher_iv_random_size; 144 145 struct rte_crypto_sym_xform auth_xform; 146 uint8_t akey_param; 147 int akey_random_size; 148 uint8_t auth_key[MAX_KEY_SIZE]; 149 150 struct l2fwd_iv auth_iv; 151 unsigned int auth_iv_param; 152 int auth_iv_random_size; 153 154 struct rte_crypto_sym_xform aead_xform; 155 unsigned int aead_key_param; 156 int aead_key_random_size; 157 uint8_t aead_key[MAX_KEY_SIZE]; 158 159 struct l2fwd_iv aead_iv; 160 unsigned int aead_iv_param; 161 int aead_iv_random_size; 162 163 struct l2fwd_key aad; 164 unsigned aad_param; 165 int aad_random_size; 166 167 int digest_size; 168 169 uint16_t block_size; 170 char string_type[MAX_STR_LEN]; 171 172 uint64_t cryptodev_mask; 173 174 unsigned int mac_updating; 175 }; 176 177 /** l2fwd crypto lcore params */ 178 struct l2fwd_crypto_params { 179 uint8_t dev_id; 180 uint8_t qp_id; 181 182 unsigned digest_length; 183 unsigned block_size; 184 185 uint32_t cipher_dataunit_len; 186 187 struct l2fwd_iv cipher_iv; 188 struct l2fwd_iv auth_iv; 189 struct l2fwd_iv aead_iv; 190 struct l2fwd_key aad; 191 struct rte_cryptodev_sym_session *session; 192 193 uint8_t do_cipher; 194 uint8_t do_hash; 195 uint8_t do_aead; 196 uint8_t hash_verify; 197 198 enum rte_crypto_cipher_algorithm cipher_algo; 199 enum rte_crypto_auth_algorithm auth_algo; 200 enum rte_crypto_aead_algorithm aead_algo; 201 }; 202 203 /** lcore configuration */ 204 struct lcore_queue_conf { 205 unsigned nb_rx_ports; 206 uint16_t rx_port_list[MAX_RX_QUEUE_PER_LCORE]; 207 208 unsigned nb_crypto_devs; 209 unsigned cryptodev_list[MAX_RX_QUEUE_PER_LCORE]; 210 211 struct op_buffer op_buf[RTE_CRYPTO_MAX_DEVS]; 212 struct pkt_buffer pkt_buf[RTE_MAX_ETHPORTS]; 213 } __rte_cache_aligned; 214 215 struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE]; 216 217 static struct rte_eth_conf port_conf = { 218 .rxmode = { 219 .mq_mode = RTE_ETH_MQ_RX_NONE, 220 .split_hdr_size = 0, 221 }, 222 .txmode = { 223 .mq_mode = RTE_ETH_MQ_TX_NONE, 224 }, 225 }; 226 227 struct rte_mempool *l2fwd_pktmbuf_pool; 228 struct rte_mempool *l2fwd_crypto_op_pool; 229 static struct { 230 struct rte_mempool *sess_mp; 231 struct rte_mempool *priv_mp; 232 } session_pool_socket[RTE_MAX_NUMA_NODES]; 233 234 /* Per-port statistics struct */ 235 struct l2fwd_port_statistics { 236 uint64_t tx; 237 uint64_t rx; 238 239 uint64_t crypto_enqueued; 240 uint64_t crypto_dequeued; 241 242 uint64_t dropped; 243 } __rte_cache_aligned; 244 245 struct l2fwd_crypto_statistics { 246 uint64_t enqueued; 247 uint64_t dequeued; 248 249 uint64_t errors; 250 } __rte_cache_aligned; 251 252 struct l2fwd_port_statistics port_statistics[RTE_MAX_ETHPORTS]; 253 struct l2fwd_crypto_statistics crypto_statistics[RTE_CRYPTO_MAX_DEVS]; 254 255 /* A tsc-based timer responsible for triggering statistics printout */ 256 #define TIMER_MILLISECOND (rte_get_tsc_hz() / 1000) 257 #define MAX_TIMER_PERIOD 86400UL /* 1 day max */ 258 #define DEFAULT_TIMER_PERIOD 10UL 259 260 /* Global signal */ 261 static volatile bool signal_received; 262 263 /* Print out statistics on packets dropped */ 264 static void 265 print_stats(void) 266 { 267 uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; 268 uint64_t total_packets_enqueued, total_packets_dequeued, 269 total_packets_errors; 270 uint16_t portid; 271 uint64_t cdevid; 272 273 total_packets_dropped = 0; 274 total_packets_tx = 0; 275 total_packets_rx = 0; 276 total_packets_enqueued = 0; 277 total_packets_dequeued = 0; 278 total_packets_errors = 0; 279 280 const char clr[] = { 27, '[', '2', 'J', '\0' }; 281 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 282 283 /* Clear screen and move to top left */ 284 printf("%s%s", clr, topLeft); 285 286 printf("\nPort statistics ===================================="); 287 288 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 289 /* skip disabled ports */ 290 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) 291 continue; 292 printf("\nStatistics for port %u ------------------------------" 293 "\nPackets sent: %32"PRIu64 294 "\nPackets received: %28"PRIu64 295 "\nPackets dropped: %29"PRIu64, 296 portid, 297 port_statistics[portid].tx, 298 port_statistics[portid].rx, 299 port_statistics[portid].dropped); 300 301 total_packets_dropped += port_statistics[portid].dropped; 302 total_packets_tx += port_statistics[portid].tx; 303 total_packets_rx += port_statistics[portid].rx; 304 } 305 printf("\nCrypto statistics =================================="); 306 307 for (cdevid = 0; cdevid < RTE_CRYPTO_MAX_DEVS; cdevid++) { 308 /* skip disabled ports */ 309 if ((l2fwd_enabled_crypto_mask & (((uint64_t)1) << cdevid)) == 0) 310 continue; 311 printf("\nStatistics for cryptodev %"PRIu64 312 " -------------------------" 313 "\nPackets enqueued: %28"PRIu64 314 "\nPackets dequeued: %28"PRIu64 315 "\nPackets errors: %30"PRIu64, 316 cdevid, 317 crypto_statistics[cdevid].enqueued, 318 crypto_statistics[cdevid].dequeued, 319 crypto_statistics[cdevid].errors); 320 321 total_packets_enqueued += crypto_statistics[cdevid].enqueued; 322 total_packets_dequeued += crypto_statistics[cdevid].dequeued; 323 total_packets_errors += crypto_statistics[cdevid].errors; 324 } 325 printf("\nAggregate statistics ===============================" 326 "\nTotal packets received: %22"PRIu64 327 "\nTotal packets enqueued: %22"PRIu64 328 "\nTotal packets dequeued: %22"PRIu64 329 "\nTotal packets sent: %26"PRIu64 330 "\nTotal packets dropped: %23"PRIu64 331 "\nTotal packets crypto errors: %17"PRIu64, 332 total_packets_rx, 333 total_packets_enqueued, 334 total_packets_dequeued, 335 total_packets_tx, 336 total_packets_dropped, 337 total_packets_errors); 338 printf("\n====================================================\n"); 339 340 fflush(stdout); 341 } 342 343 /* l2fwd_crypto_send_burst 8< */ 344 static int 345 l2fwd_crypto_send_burst(struct lcore_queue_conf *qconf, unsigned n, 346 struct l2fwd_crypto_params *cparams) 347 { 348 struct rte_crypto_op **op_buffer; 349 unsigned ret; 350 351 op_buffer = (struct rte_crypto_op **) 352 qconf->op_buf[cparams->dev_id].buffer; 353 354 ret = rte_cryptodev_enqueue_burst(cparams->dev_id, 355 cparams->qp_id, op_buffer, (uint16_t) n); 356 357 crypto_statistics[cparams->dev_id].enqueued += ret; 358 if (unlikely(ret < n)) { 359 crypto_statistics[cparams->dev_id].errors += (n - ret); 360 do { 361 rte_pktmbuf_free(op_buffer[ret]->sym->m_src); 362 rte_crypto_op_free(op_buffer[ret]); 363 } while (++ret < n); 364 } 365 366 return 0; 367 } 368 /* >8 End of l2fwd_crypto_send_burst. */ 369 370 /* Crypto enqueue. 8< */ 371 static int 372 l2fwd_crypto_enqueue(struct rte_crypto_op *op, 373 struct l2fwd_crypto_params *cparams) 374 { 375 unsigned lcore_id, len; 376 struct lcore_queue_conf *qconf; 377 378 lcore_id = rte_lcore_id(); 379 380 qconf = &lcore_queue_conf[lcore_id]; 381 len = qconf->op_buf[cparams->dev_id].len; 382 qconf->op_buf[cparams->dev_id].buffer[len] = op; 383 len++; 384 385 /* enough ops to be sent */ 386 if (len == MAX_PKT_BURST) { 387 l2fwd_crypto_send_burst(qconf, MAX_PKT_BURST, cparams); 388 len = 0; 389 } 390 391 qconf->op_buf[cparams->dev_id].len = len; 392 return 0; 393 } 394 /* >8 End of crypto enqueue. */ 395 396 static int 397 l2fwd_simple_crypto_enqueue(struct rte_mbuf *m, 398 struct rte_crypto_op *op, 399 struct l2fwd_crypto_params *cparams) 400 { 401 struct rte_ether_hdr *eth_hdr; 402 struct rte_ipv4_hdr *ip_hdr; 403 404 uint32_t ipdata_offset, data_len; 405 uint32_t pad_len = 0; 406 char *padding; 407 408 eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); 409 410 if (eth_hdr->ether_type != rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) 411 return -1; 412 413 ipdata_offset = sizeof(struct rte_ether_hdr); 414 415 ip_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, char *) + 416 ipdata_offset); 417 418 ipdata_offset += (ip_hdr->version_ihl & RTE_IPV4_HDR_IHL_MASK) 419 * RTE_IPV4_IHL_MULTIPLIER; 420 421 422 /* Zero pad data to be crypto'd so it is block aligned */ 423 data_len = rte_pktmbuf_data_len(m) - ipdata_offset; 424 425 if ((cparams->do_hash || cparams->do_aead) && cparams->hash_verify) 426 data_len -= cparams->digest_length; 427 428 if (cparams->do_cipher) { 429 /* 430 * Following algorithms are block cipher algorithms, 431 * and might need padding 432 */ 433 switch (cparams->cipher_algo) { 434 case RTE_CRYPTO_CIPHER_AES_CBC: 435 case RTE_CRYPTO_CIPHER_AES_ECB: 436 case RTE_CRYPTO_CIPHER_DES_CBC: 437 case RTE_CRYPTO_CIPHER_3DES_CBC: 438 case RTE_CRYPTO_CIPHER_3DES_ECB: 439 if (data_len % cparams->block_size) 440 pad_len = cparams->block_size - 441 (data_len % cparams->block_size); 442 break; 443 case RTE_CRYPTO_CIPHER_AES_XTS: 444 if (cparams->cipher_dataunit_len != 0 && 445 (data_len % cparams->cipher_dataunit_len)) 446 pad_len = cparams->cipher_dataunit_len - 447 (data_len % cparams->cipher_dataunit_len); 448 break; 449 default: 450 pad_len = 0; 451 } 452 453 if (pad_len) { 454 padding = rte_pktmbuf_append(m, pad_len); 455 if (unlikely(!padding)) 456 return -1; 457 458 data_len += pad_len; 459 memset(padding, 0, pad_len); 460 } 461 } 462 463 /* Set crypto operation data parameters */ 464 rte_crypto_op_attach_sym_session(op, cparams->session); 465 466 if (cparams->do_hash) { 467 if (cparams->auth_iv.length) { 468 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, 469 uint8_t *, 470 IV_OFFSET + 471 cparams->cipher_iv.length); 472 /* 473 * Copy IV at the end of the crypto operation, 474 * after the cipher IV, if added 475 */ 476 rte_memcpy(iv_ptr, cparams->auth_iv.data, 477 cparams->auth_iv.length); 478 } 479 if (!cparams->hash_verify) { 480 /* Append space for digest to end of packet */ 481 op->sym->auth.digest.data = (uint8_t *)rte_pktmbuf_append(m, 482 cparams->digest_length); 483 } else { 484 op->sym->auth.digest.data = rte_pktmbuf_mtod(m, 485 uint8_t *) + ipdata_offset + data_len; 486 } 487 488 op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m, 489 rte_pktmbuf_pkt_len(m) - cparams->digest_length); 490 491 /* For wireless algorithms, offset/length must be in bits */ 492 if (cparams->auth_algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2 || 493 cparams->auth_algo == RTE_CRYPTO_AUTH_KASUMI_F9 || 494 cparams->auth_algo == RTE_CRYPTO_AUTH_ZUC_EIA3) { 495 op->sym->auth.data.offset = ipdata_offset << 3; 496 op->sym->auth.data.length = data_len << 3; 497 } else { 498 op->sym->auth.data.offset = ipdata_offset; 499 op->sym->auth.data.length = data_len; 500 } 501 } 502 503 if (cparams->do_cipher) { 504 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 505 IV_OFFSET); 506 /* Copy IV at the end of the crypto operation */ 507 rte_memcpy(iv_ptr, cparams->cipher_iv.data, 508 cparams->cipher_iv.length); 509 510 /* For wireless algorithms, offset/length must be in bits */ 511 if (cparams->cipher_algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 || 512 cparams->cipher_algo == RTE_CRYPTO_CIPHER_KASUMI_F8 || 513 cparams->cipher_algo == RTE_CRYPTO_CIPHER_ZUC_EEA3) { 514 op->sym->cipher.data.offset = ipdata_offset << 3; 515 op->sym->cipher.data.length = data_len << 3; 516 } else { 517 op->sym->cipher.data.offset = ipdata_offset; 518 op->sym->cipher.data.length = data_len; 519 } 520 } 521 522 if (cparams->do_aead) { 523 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 524 IV_OFFSET); 525 /* Copy IV at the end of the crypto operation */ 526 /* 527 * If doing AES-CCM, nonce is copied one byte 528 * after the start of IV field 529 */ 530 if (cparams->aead_algo == RTE_CRYPTO_AEAD_AES_CCM) 531 rte_memcpy(iv_ptr + 1, cparams->aead_iv.data, 532 cparams->aead_iv.length); 533 else 534 rte_memcpy(iv_ptr, cparams->aead_iv.data, 535 cparams->aead_iv.length); 536 537 op->sym->aead.data.offset = ipdata_offset; 538 op->sym->aead.data.length = data_len; 539 540 if (!cparams->hash_verify) { 541 /* Append space for digest to end of packet */ 542 op->sym->aead.digest.data = (uint8_t *)rte_pktmbuf_append(m, 543 cparams->digest_length); 544 } else { 545 op->sym->aead.digest.data = rte_pktmbuf_mtod(m, 546 uint8_t *) + ipdata_offset + data_len; 547 } 548 549 op->sym->aead.digest.phys_addr = rte_pktmbuf_iova_offset(m, 550 rte_pktmbuf_pkt_len(m) - cparams->digest_length); 551 552 if (cparams->aad.length) { 553 op->sym->aead.aad.data = cparams->aad.data; 554 op->sym->aead.aad.phys_addr = cparams->aad.phys_addr; 555 } 556 } 557 558 op->sym->m_src = m; 559 560 return l2fwd_crypto_enqueue(op, cparams); 561 } 562 563 564 /* Send the burst of packets on an output interface */ 565 static int 566 l2fwd_send_burst(struct lcore_queue_conf *qconf, unsigned n, 567 uint16_t port) 568 { 569 struct rte_mbuf **pkt_buffer; 570 unsigned ret; 571 572 pkt_buffer = (struct rte_mbuf **)qconf->pkt_buf[port].buffer; 573 574 ret = rte_eth_tx_burst(port, 0, pkt_buffer, (uint16_t)n); 575 port_statistics[port].tx += ret; 576 if (unlikely(ret < n)) { 577 port_statistics[port].dropped += (n - ret); 578 do { 579 rte_pktmbuf_free(pkt_buffer[ret]); 580 } while (++ret < n); 581 } 582 583 return 0; 584 } 585 586 /* Enqueue packets for TX and prepare them to be sent. 8< */ 587 static int 588 l2fwd_send_packet(struct rte_mbuf *m, uint16_t port) 589 { 590 unsigned lcore_id, len; 591 struct lcore_queue_conf *qconf; 592 593 lcore_id = rte_lcore_id(); 594 595 qconf = &lcore_queue_conf[lcore_id]; 596 len = qconf->pkt_buf[port].len; 597 qconf->pkt_buf[port].buffer[len] = m; 598 len++; 599 600 /* enough pkts to be sent */ 601 if (unlikely(len == MAX_PKT_BURST)) { 602 l2fwd_send_burst(qconf, MAX_PKT_BURST, port); 603 len = 0; 604 } 605 606 qconf->pkt_buf[port].len = len; 607 return 0; 608 } 609 /* >8 End of Enqueuing packets for TX. */ 610 611 static void 612 l2fwd_mac_updating(struct rte_mbuf *m, uint16_t dest_portid) 613 { 614 struct rte_ether_hdr *eth; 615 void *tmp; 616 617 eth = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); 618 619 /* 02:00:00:00:00:xx */ 620 tmp = ð->dst_addr.addr_bytes[0]; 621 *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dest_portid << 40); 622 623 /* src addr */ 624 rte_ether_addr_copy(&l2fwd_ports_eth_addr[dest_portid], ð->src_addr); 625 } 626 627 static void 628 l2fwd_simple_forward(struct rte_mbuf *m, uint16_t portid, 629 struct l2fwd_crypto_options *options) 630 { 631 uint16_t dst_port; 632 uint32_t pad_len; 633 struct rte_ipv4_hdr *ip_hdr; 634 uint32_t ipdata_offset = sizeof(struct rte_ether_hdr); 635 636 ip_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, char *) + 637 ipdata_offset); 638 dst_port = l2fwd_dst_ports[portid]; 639 640 if (options->mac_updating) 641 l2fwd_mac_updating(m, dst_port); 642 643 if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_VERIFY) 644 rte_pktmbuf_trim(m, options->auth_xform.auth.digest_length); 645 646 if (options->cipher_xform.cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { 647 pad_len = m->pkt_len - rte_be_to_cpu_16(ip_hdr->total_length) - 648 ipdata_offset; 649 rte_pktmbuf_trim(m, pad_len); 650 } 651 652 l2fwd_send_packet(m, dst_port); 653 } 654 655 /** Generate random key */ 656 static void 657 generate_random_key(uint8_t *key, unsigned length) 658 { 659 int fd; 660 int ret; 661 662 fd = open("/dev/urandom", O_RDONLY); 663 if (fd < 0) 664 rte_exit(EXIT_FAILURE, "Failed to generate random key\n"); 665 666 ret = read(fd, key, length); 667 close(fd); 668 669 if (ret != (signed)length) 670 rte_exit(EXIT_FAILURE, "Failed to generate random key\n"); 671 } 672 673 /* Session is created and is later attached to the crypto operation. 8< */ 674 static struct rte_cryptodev_sym_session * 675 initialize_crypto_session(struct l2fwd_crypto_options *options, uint8_t cdev_id) 676 { 677 struct rte_crypto_sym_xform *first_xform; 678 struct rte_cryptodev_sym_session *session; 679 int retval = rte_cryptodev_socket_id(cdev_id); 680 681 if (retval < 0) 682 return NULL; 683 684 uint8_t socket_id = (uint8_t) retval; 685 686 if (options->xform_chain == L2FWD_CRYPTO_AEAD) { 687 first_xform = &options->aead_xform; 688 } else if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH) { 689 first_xform = &options->cipher_xform; 690 first_xform->next = &options->auth_xform; 691 } else if (options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER) { 692 first_xform = &options->auth_xform; 693 first_xform->next = &options->cipher_xform; 694 } else if (options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { 695 first_xform = &options->cipher_xform; 696 } else { 697 first_xform = &options->auth_xform; 698 } 699 700 session = rte_cryptodev_sym_session_create( 701 session_pool_socket[socket_id].sess_mp); 702 if (session == NULL) 703 return NULL; 704 705 if (rte_cryptodev_sym_session_init(cdev_id, session, 706 first_xform, 707 session_pool_socket[socket_id].priv_mp) < 0) 708 return NULL; 709 710 return session; 711 } 712 /* >8 End of creation of session. */ 713 714 static void 715 l2fwd_crypto_options_print(struct l2fwd_crypto_options *options); 716 717 /* main processing loop */ 718 static void 719 l2fwd_main_loop(struct l2fwd_crypto_options *options) 720 { 721 struct rte_mbuf *m, *pkts_burst[MAX_PKT_BURST]; 722 struct rte_crypto_op *ops_burst[MAX_PKT_BURST]; 723 724 unsigned lcore_id = rte_lcore_id(); 725 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 726 unsigned int i, j, nb_rx, len; 727 uint16_t portid; 728 struct lcore_queue_conf *qconf = &lcore_queue_conf[lcore_id]; 729 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 730 US_PER_S * BURST_TX_DRAIN_US; 731 struct l2fwd_crypto_params *cparams; 732 struct l2fwd_crypto_params port_cparams[qconf->nb_crypto_devs]; 733 struct rte_cryptodev_sym_session *session; 734 735 if (qconf->nb_rx_ports == 0) { 736 RTE_LOG(INFO, L2FWD, "lcore %u has nothing to do\n", lcore_id); 737 return; 738 } 739 740 RTE_LOG(INFO, L2FWD, "entering main loop on lcore %u\n", lcore_id); 741 742 for (i = 0; i < qconf->nb_rx_ports; i++) { 743 744 portid = qconf->rx_port_list[i]; 745 RTE_LOG(INFO, L2FWD, " -- lcoreid=%u portid=%u\n", lcore_id, 746 portid); 747 } 748 749 for (i = 0; i < qconf->nb_crypto_devs; i++) { 750 port_cparams[i].do_cipher = 0; 751 port_cparams[i].do_hash = 0; 752 port_cparams[i].do_aead = 0; 753 754 switch (options->xform_chain) { 755 case L2FWD_CRYPTO_AEAD: 756 port_cparams[i].do_aead = 1; 757 break; 758 case L2FWD_CRYPTO_CIPHER_HASH: 759 case L2FWD_CRYPTO_HASH_CIPHER: 760 port_cparams[i].do_cipher = 1; 761 port_cparams[i].do_hash = 1; 762 break; 763 case L2FWD_CRYPTO_HASH_ONLY: 764 port_cparams[i].do_hash = 1; 765 break; 766 case L2FWD_CRYPTO_CIPHER_ONLY: 767 port_cparams[i].do_cipher = 1; 768 break; 769 } 770 771 port_cparams[i].dev_id = qconf->cryptodev_list[i]; 772 port_cparams[i].qp_id = 0; 773 774 port_cparams[i].block_size = options->block_size; 775 776 if (port_cparams[i].do_hash) { 777 port_cparams[i].auth_iv.data = options->auth_iv.data; 778 port_cparams[i].auth_iv.length = options->auth_iv.length; 779 if (!options->auth_iv_param) 780 generate_random_key(port_cparams[i].auth_iv.data, 781 port_cparams[i].auth_iv.length); 782 if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_VERIFY) 783 port_cparams[i].hash_verify = 1; 784 else 785 port_cparams[i].hash_verify = 0; 786 787 port_cparams[i].auth_algo = options->auth_xform.auth.algo; 788 port_cparams[i].digest_length = 789 options->auth_xform.auth.digest_length; 790 /* Set IV parameters */ 791 if (options->auth_iv.length) { 792 options->auth_xform.auth.iv.offset = 793 IV_OFFSET + options->cipher_iv.length; 794 options->auth_xform.auth.iv.length = 795 options->auth_iv.length; 796 } 797 } 798 799 if (port_cparams[i].do_aead) { 800 port_cparams[i].aead_iv.data = options->aead_iv.data; 801 port_cparams[i].aead_iv.length = options->aead_iv.length; 802 if (!options->aead_iv_param) 803 generate_random_key(port_cparams[i].aead_iv.data, 804 port_cparams[i].aead_iv.length); 805 port_cparams[i].aead_algo = options->aead_xform.aead.algo; 806 port_cparams[i].digest_length = 807 options->aead_xform.aead.digest_length; 808 if (options->aead_xform.aead.aad_length) { 809 port_cparams[i].aad.data = options->aad.data; 810 port_cparams[i].aad.phys_addr = options->aad.phys_addr; 811 port_cparams[i].aad.length = options->aad.length; 812 if (!options->aad_param) 813 generate_random_key(port_cparams[i].aad.data, 814 port_cparams[i].aad.length); 815 /* 816 * If doing AES-CCM, first 18 bytes has to be reserved, 817 * and actual AAD should start from byte 18 818 */ 819 if (port_cparams[i].aead_algo == RTE_CRYPTO_AEAD_AES_CCM) 820 memmove(port_cparams[i].aad.data + 18, 821 port_cparams[i].aad.data, 822 port_cparams[i].aad.length); 823 824 } else 825 port_cparams[i].aad.length = 0; 826 827 if (options->aead_xform.aead.op == RTE_CRYPTO_AEAD_OP_DECRYPT) 828 port_cparams[i].hash_verify = 1; 829 else 830 port_cparams[i].hash_verify = 0; 831 832 /* Set IV parameters */ 833 options->aead_xform.aead.iv.offset = IV_OFFSET; 834 options->aead_xform.aead.iv.length = options->aead_iv.length; 835 } 836 837 if (port_cparams[i].do_cipher) { 838 port_cparams[i].cipher_iv.data = options->cipher_iv.data; 839 port_cparams[i].cipher_iv.length = options->cipher_iv.length; 840 if (!options->cipher_iv_param) 841 generate_random_key(port_cparams[i].cipher_iv.data, 842 port_cparams[i].cipher_iv.length); 843 844 port_cparams[i].cipher_algo = options->cipher_xform.cipher.algo; 845 port_cparams[i].cipher_dataunit_len = 846 options->cipher_xform.cipher.dataunit_len; 847 /* Set IV parameters */ 848 options->cipher_xform.cipher.iv.offset = IV_OFFSET; 849 options->cipher_xform.cipher.iv.length = 850 options->cipher_iv.length; 851 } 852 853 session = initialize_crypto_session(options, 854 port_cparams[i].dev_id); 855 if (session == NULL) 856 rte_exit(EXIT_FAILURE, "Failed to initialize crypto session\n"); 857 858 port_cparams[i].session = session; 859 860 RTE_LOG(INFO, L2FWD, " -- lcoreid=%u cryptoid=%u\n", lcore_id, 861 port_cparams[i].dev_id); 862 } 863 864 l2fwd_crypto_options_print(options); 865 866 /* 867 * Initialize previous tsc timestamp before the loop, 868 * to avoid showing the port statistics immediately, 869 * so user can see the crypto information. 870 */ 871 prev_tsc = rte_rdtsc(); 872 while (1) { 873 874 cur_tsc = rte_rdtsc(); 875 876 /* 877 * Crypto device/TX burst queue drain 878 */ 879 diff_tsc = cur_tsc - prev_tsc; 880 if (unlikely(diff_tsc > drain_tsc)) { 881 /* Enqueue all crypto ops remaining in buffers */ 882 for (i = 0; i < qconf->nb_crypto_devs; i++) { 883 cparams = &port_cparams[i]; 884 len = qconf->op_buf[cparams->dev_id].len; 885 l2fwd_crypto_send_burst(qconf, len, cparams); 886 qconf->op_buf[cparams->dev_id].len = 0; 887 } 888 /* Transmit all packets remaining in buffers */ 889 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 890 if (qconf->pkt_buf[portid].len == 0) 891 continue; 892 l2fwd_send_burst(&lcore_queue_conf[lcore_id], 893 qconf->pkt_buf[portid].len, 894 portid); 895 qconf->pkt_buf[portid].len = 0; 896 } 897 898 /* if timer is enabled */ 899 if (options->refresh_period > 0) { 900 901 /* advance the timer */ 902 timer_tsc += diff_tsc; 903 904 /* if timer has reached its timeout */ 905 if (unlikely(timer_tsc >= 906 options->refresh_period)) { 907 908 /* do this only on main core */ 909 if (lcore_id == rte_get_main_lcore()) { 910 print_stats(); 911 timer_tsc = 0; 912 } 913 } 914 } 915 916 prev_tsc = cur_tsc; 917 } 918 919 /* 920 * Read packet from RX queues 921 */ 922 for (i = 0; i < qconf->nb_rx_ports; i++) { 923 portid = qconf->rx_port_list[i]; 924 925 cparams = &port_cparams[i]; 926 927 nb_rx = rte_eth_rx_burst(portid, 0, 928 pkts_burst, MAX_PKT_BURST); 929 if (unlikely(signal_received)) 930 return; 931 932 port_statistics[portid].rx += nb_rx; 933 934 /* Allocate and fillcrypto operations. 8< */ 935 if (nb_rx) { 936 /* 937 * If we can't allocate a crypto_ops, then drop 938 * the rest of the burst and dequeue and 939 * process the packets to free offload structs 940 */ 941 if (rte_crypto_op_bulk_alloc( 942 l2fwd_crypto_op_pool, 943 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 944 ops_burst, nb_rx) != 945 nb_rx) { 946 for (j = 0; j < nb_rx; j++) 947 rte_pktmbuf_free(pkts_burst[j]); 948 949 nb_rx = 0; 950 } 951 /* >8 End of crypto operation allocated and filled. */ 952 953 /* Enqueue packets from Crypto device*/ 954 for (j = 0; j < nb_rx; j++) { 955 m = pkts_burst[j]; 956 957 l2fwd_simple_crypto_enqueue(m, 958 ops_burst[j], cparams); 959 } 960 } 961 962 /* Dequeue packets from Crypto device. 8< */ 963 do { 964 nb_rx = rte_cryptodev_dequeue_burst( 965 cparams->dev_id, cparams->qp_id, 966 ops_burst, MAX_PKT_BURST); 967 968 crypto_statistics[cparams->dev_id].dequeued += 969 nb_rx; 970 971 /* Forward crypto'd packets */ 972 for (j = 0; j < nb_rx; j++) { 973 m = ops_burst[j]->sym->m_src; 974 975 rte_crypto_op_free(ops_burst[j]); 976 l2fwd_simple_forward(m, portid, 977 options); 978 } 979 } while (nb_rx == MAX_PKT_BURST); 980 /* >8 End of dequeue packets from crypto device. */ 981 } 982 } 983 } 984 985 static int 986 l2fwd_launch_one_lcore(void *arg) 987 { 988 l2fwd_main_loop((struct l2fwd_crypto_options *)arg); 989 return 0; 990 } 991 992 /* Display command line arguments usage */ 993 static void 994 l2fwd_crypto_usage(const char *prgname) 995 { 996 printf("%s [EAL options] --\n" 997 " -p PORTMASK: hexadecimal bitmask of ports to configure\n" 998 " -q NQ: number of queue (=ports) per lcore (default is 1)\n" 999 " -s manage all ports from single lcore\n" 1000 " -T PERIOD: statistics will be refreshed each PERIOD seconds" 1001 " (0 to disable, 10 default, 86400 maximum)\n" 1002 1003 " --cdev_type HW / SW / ANY\n" 1004 " --chain HASH_CIPHER / CIPHER_HASH / CIPHER_ONLY /" 1005 " HASH_ONLY / AEAD\n" 1006 1007 " --cipher_algo ALGO\n" 1008 " --cipher_op ENCRYPT / DECRYPT\n" 1009 " --cipher_key KEY (bytes separated with \":\")\n" 1010 " --cipher_key_random_size SIZE: size of cipher key when generated randomly\n" 1011 " --cipher_iv IV (bytes separated with \":\")\n" 1012 " --cipher_iv_random_size SIZE: size of cipher IV when generated randomly\n" 1013 " --cipher_dataunit_len SIZE: length of the algorithm data-unit\n" 1014 1015 " --auth_algo ALGO\n" 1016 " --auth_op GENERATE / VERIFY\n" 1017 " --auth_key KEY (bytes separated with \":\")\n" 1018 " --auth_key_random_size SIZE: size of auth key when generated randomly\n" 1019 " --auth_iv IV (bytes separated with \":\")\n" 1020 " --auth_iv_random_size SIZE: size of auth IV when generated randomly\n" 1021 1022 " --aead_algo ALGO\n" 1023 " --aead_op ENCRYPT / DECRYPT\n" 1024 " --aead_key KEY (bytes separated with \":\")\n" 1025 " --aead_key_random_size SIZE: size of AEAD key when generated randomly\n" 1026 " --aead_iv IV (bytes separated with \":\")\n" 1027 " --aead_iv_random_size SIZE: size of AEAD IV when generated randomly\n" 1028 " --aad AAD (bytes separated with \":\")\n" 1029 " --aad_random_size SIZE: size of AAD when generated randomly\n" 1030 1031 " --digest_size SIZE: size of digest to be generated/verified\n" 1032 1033 " --sessionless\n" 1034 " --cryptodev_mask MASK: hexadecimal bitmask of crypto devices to configure\n" 1035 1036 " --[no-]mac-updating: Enable or disable MAC addresses updating (enabled by default)\n" 1037 " When enabled:\n" 1038 " - The source MAC address is replaced by the TX port MAC address\n" 1039 " - The destination MAC address is replaced by 02:00:00:00:00:TX_PORT_ID\n", 1040 prgname); 1041 } 1042 1043 /** Parse crypto device type command line argument */ 1044 static int 1045 parse_cryptodev_type(enum cdev_type *type, char *optarg) 1046 { 1047 if (strcmp("HW", optarg) == 0) { 1048 *type = CDEV_TYPE_HW; 1049 return 0; 1050 } else if (strcmp("SW", optarg) == 0) { 1051 *type = CDEV_TYPE_SW; 1052 return 0; 1053 } else if (strcmp("ANY", optarg) == 0) { 1054 *type = CDEV_TYPE_ANY; 1055 return 0; 1056 } 1057 1058 return -1; 1059 } 1060 1061 /** Parse crypto chain xform command line argument */ 1062 static int 1063 parse_crypto_opt_chain(struct l2fwd_crypto_options *options, char *optarg) 1064 { 1065 if (strcmp("CIPHER_HASH", optarg) == 0) { 1066 options->xform_chain = L2FWD_CRYPTO_CIPHER_HASH; 1067 return 0; 1068 } else if (strcmp("HASH_CIPHER", optarg) == 0) { 1069 options->xform_chain = L2FWD_CRYPTO_HASH_CIPHER; 1070 return 0; 1071 } else if (strcmp("CIPHER_ONLY", optarg) == 0) { 1072 options->xform_chain = L2FWD_CRYPTO_CIPHER_ONLY; 1073 return 0; 1074 } else if (strcmp("HASH_ONLY", optarg) == 0) { 1075 options->xform_chain = L2FWD_CRYPTO_HASH_ONLY; 1076 return 0; 1077 } else if (strcmp("AEAD", optarg) == 0) { 1078 options->xform_chain = L2FWD_CRYPTO_AEAD; 1079 return 0; 1080 } 1081 1082 return -1; 1083 } 1084 1085 /** Parse crypto cipher algo option command line argument */ 1086 static int 1087 parse_cipher_algo(enum rte_crypto_cipher_algorithm *algo, char *optarg) 1088 { 1089 1090 if (rte_cryptodev_get_cipher_algo_enum(algo, optarg) < 0) { 1091 RTE_LOG(ERR, USER1, "Cipher algorithm specified " 1092 "not supported!\n"); 1093 return -1; 1094 } 1095 1096 return 0; 1097 } 1098 1099 /** Parse crypto cipher operation command line argument */ 1100 static int 1101 parse_cipher_op(enum rte_crypto_cipher_operation *op, char *optarg) 1102 { 1103 if (strcmp("ENCRYPT", optarg) == 0) { 1104 *op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 1105 return 0; 1106 } else if (strcmp("DECRYPT", optarg) == 0) { 1107 *op = RTE_CRYPTO_CIPHER_OP_DECRYPT; 1108 return 0; 1109 } 1110 1111 printf("Cipher operation not supported!\n"); 1112 return -1; 1113 } 1114 1115 /** Parse bytes from command line argument */ 1116 static int 1117 parse_bytes(uint8_t *data, char *input_arg, uint16_t max_size) 1118 { 1119 unsigned byte_count; 1120 char *token; 1121 1122 errno = 0; 1123 for (byte_count = 0, token = strtok(input_arg, ":"); 1124 (byte_count < max_size) && (token != NULL); 1125 token = strtok(NULL, ":")) { 1126 1127 int number = (int)strtol(token, NULL, 16); 1128 1129 if (errno == EINVAL || errno == ERANGE || number > 0xFF) 1130 return -1; 1131 1132 data[byte_count++] = (uint8_t)number; 1133 } 1134 1135 return byte_count; 1136 } 1137 1138 /** Parse size param*/ 1139 static int 1140 parse_size(int *size, const char *q_arg) 1141 { 1142 char *end = NULL; 1143 unsigned long n; 1144 1145 /* parse hexadecimal string */ 1146 n = strtoul(q_arg, &end, 10); 1147 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) 1148 n = 0; 1149 1150 if (n == 0) { 1151 printf("invalid size\n"); 1152 return -1; 1153 } 1154 1155 *size = n; 1156 return 0; 1157 } 1158 1159 /** Parse crypto cipher operation command line argument */ 1160 static int 1161 parse_auth_algo(enum rte_crypto_auth_algorithm *algo, char *optarg) 1162 { 1163 if (rte_cryptodev_get_auth_algo_enum(algo, optarg) < 0) { 1164 RTE_LOG(ERR, USER1, "Authentication algorithm specified " 1165 "not supported!\n"); 1166 return -1; 1167 } 1168 1169 return 0; 1170 } 1171 1172 static int 1173 parse_auth_op(enum rte_crypto_auth_operation *op, char *optarg) 1174 { 1175 if (strcmp("VERIFY", optarg) == 0) { 1176 *op = RTE_CRYPTO_AUTH_OP_VERIFY; 1177 return 0; 1178 } else if (strcmp("GENERATE", optarg) == 0) { 1179 *op = RTE_CRYPTO_AUTH_OP_GENERATE; 1180 return 0; 1181 } 1182 1183 printf("Authentication operation specified not supported!\n"); 1184 return -1; 1185 } 1186 1187 static int 1188 parse_aead_algo(enum rte_crypto_aead_algorithm *algo, char *optarg) 1189 { 1190 if (rte_cryptodev_get_aead_algo_enum(algo, optarg) < 0) { 1191 RTE_LOG(ERR, USER1, "AEAD algorithm specified " 1192 "not supported!\n"); 1193 return -1; 1194 } 1195 1196 return 0; 1197 } 1198 1199 static int 1200 parse_aead_op(enum rte_crypto_aead_operation *op, char *optarg) 1201 { 1202 if (strcmp("ENCRYPT", optarg) == 0) { 1203 *op = RTE_CRYPTO_AEAD_OP_ENCRYPT; 1204 return 0; 1205 } else if (strcmp("DECRYPT", optarg) == 0) { 1206 *op = RTE_CRYPTO_AEAD_OP_DECRYPT; 1207 return 0; 1208 } 1209 1210 printf("AEAD operation specified not supported!\n"); 1211 return -1; 1212 } 1213 static int 1214 parse_cryptodev_mask(struct l2fwd_crypto_options *options, 1215 const char *q_arg) 1216 { 1217 char *end = NULL; 1218 uint64_t pm; 1219 1220 /* parse hexadecimal string */ 1221 pm = strtoul(q_arg, &end, 16); 1222 if ((pm == '\0') || (end == NULL) || (*end != '\0')) 1223 pm = 0; 1224 1225 options->cryptodev_mask = pm; 1226 if (options->cryptodev_mask == 0) { 1227 printf("invalid cryptodev_mask specified\n"); 1228 return -1; 1229 } 1230 1231 return 0; 1232 } 1233 1234 /** Parse long options */ 1235 static int 1236 l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options, 1237 struct option *lgopts, int option_index) 1238 { 1239 int retval; 1240 int val; 1241 1242 if (strcmp(lgopts[option_index].name, "cdev_type") == 0) { 1243 retval = parse_cryptodev_type(&options->type, optarg); 1244 if (retval == 0) 1245 strlcpy(options->string_type, optarg, MAX_STR_LEN); 1246 return retval; 1247 } 1248 1249 else if (strcmp(lgopts[option_index].name, "chain") == 0) 1250 return parse_crypto_opt_chain(options, optarg); 1251 1252 /* Cipher options */ 1253 else if (strcmp(lgopts[option_index].name, "cipher_algo") == 0) 1254 return parse_cipher_algo(&options->cipher_xform.cipher.algo, 1255 optarg); 1256 1257 else if (strcmp(lgopts[option_index].name, "cipher_op") == 0) 1258 return parse_cipher_op(&options->cipher_xform.cipher.op, 1259 optarg); 1260 1261 else if (strcmp(lgopts[option_index].name, "cipher_key") == 0) { 1262 options->ckey_param = 1; 1263 options->cipher_xform.cipher.key.length = 1264 parse_bytes(options->cipher_key, optarg, MAX_KEY_SIZE); 1265 if (options->cipher_xform.cipher.key.length > 0) 1266 return 0; 1267 else 1268 return -1; 1269 } 1270 1271 else if (strcmp(lgopts[option_index].name, "cipher_dataunit_len") == 0) { 1272 retval = parse_size(&val, optarg); 1273 if (retval == 0 && val >= 0) { 1274 options->cipher_xform.cipher.dataunit_len = 1275 (uint32_t)val; 1276 return 0; 1277 } else 1278 return -1; 1279 } 1280 1281 else if (strcmp(lgopts[option_index].name, "cipher_key_random_size") == 0) 1282 return parse_size(&options->ckey_random_size, optarg); 1283 1284 else if (strcmp(lgopts[option_index].name, "cipher_iv") == 0) { 1285 options->cipher_iv_param = 1; 1286 options->cipher_iv.length = 1287 parse_bytes(options->cipher_iv.data, optarg, MAX_IV_SIZE); 1288 if (options->cipher_iv.length > 0) 1289 return 0; 1290 else 1291 return -1; 1292 } 1293 1294 else if (strcmp(lgopts[option_index].name, "cipher_iv_random_size") == 0) 1295 return parse_size(&options->cipher_iv_random_size, optarg); 1296 1297 /* Authentication options */ 1298 else if (strcmp(lgopts[option_index].name, "auth_algo") == 0) { 1299 return parse_auth_algo(&options->auth_xform.auth.algo, 1300 optarg); 1301 } 1302 1303 else if (strcmp(lgopts[option_index].name, "auth_op") == 0) 1304 return parse_auth_op(&options->auth_xform.auth.op, 1305 optarg); 1306 1307 else if (strcmp(lgopts[option_index].name, "auth_key") == 0) { 1308 options->akey_param = 1; 1309 options->auth_xform.auth.key.length = 1310 parse_bytes(options->auth_key, optarg, MAX_KEY_SIZE); 1311 if (options->auth_xform.auth.key.length > 0) 1312 return 0; 1313 else 1314 return -1; 1315 } 1316 1317 else if (strcmp(lgopts[option_index].name, "auth_key_random_size") == 0) { 1318 return parse_size(&options->akey_random_size, optarg); 1319 } 1320 1321 else if (strcmp(lgopts[option_index].name, "auth_iv") == 0) { 1322 options->auth_iv_param = 1; 1323 options->auth_iv.length = 1324 parse_bytes(options->auth_iv.data, optarg, MAX_IV_SIZE); 1325 if (options->auth_iv.length > 0) 1326 return 0; 1327 else 1328 return -1; 1329 } 1330 1331 else if (strcmp(lgopts[option_index].name, "auth_iv_random_size") == 0) 1332 return parse_size(&options->auth_iv_random_size, optarg); 1333 1334 /* AEAD options */ 1335 else if (strcmp(lgopts[option_index].name, "aead_algo") == 0) { 1336 return parse_aead_algo(&options->aead_xform.aead.algo, 1337 optarg); 1338 } 1339 1340 else if (strcmp(lgopts[option_index].name, "aead_op") == 0) 1341 return parse_aead_op(&options->aead_xform.aead.op, 1342 optarg); 1343 1344 else if (strcmp(lgopts[option_index].name, "aead_key") == 0) { 1345 options->aead_key_param = 1; 1346 options->aead_xform.aead.key.length = 1347 parse_bytes(options->aead_key, optarg, MAX_KEY_SIZE); 1348 if (options->aead_xform.aead.key.length > 0) 1349 return 0; 1350 else 1351 return -1; 1352 } 1353 1354 else if (strcmp(lgopts[option_index].name, "aead_key_random_size") == 0) 1355 return parse_size(&options->aead_key_random_size, optarg); 1356 1357 1358 else if (strcmp(lgopts[option_index].name, "aead_iv") == 0) { 1359 options->aead_iv_param = 1; 1360 options->aead_iv.length = 1361 parse_bytes(options->aead_iv.data, optarg, MAX_IV_SIZE); 1362 if (options->aead_iv.length > 0) 1363 return 0; 1364 else 1365 return -1; 1366 } 1367 1368 else if (strcmp(lgopts[option_index].name, "aead_iv_random_size") == 0) 1369 return parse_size(&options->aead_iv_random_size, optarg); 1370 1371 else if (strcmp(lgopts[option_index].name, "aad") == 0) { 1372 options->aad_param = 1; 1373 options->aad.length = 1374 parse_bytes(options->aad.data, optarg, MAX_AAD_SIZE); 1375 if (options->aad.length > 0) 1376 return 0; 1377 else 1378 return -1; 1379 } 1380 1381 else if (strcmp(lgopts[option_index].name, "aad_random_size") == 0) { 1382 return parse_size(&options->aad_random_size, optarg); 1383 } 1384 1385 else if (strcmp(lgopts[option_index].name, "digest_size") == 0) { 1386 return parse_size(&options->digest_size, optarg); 1387 } 1388 1389 else if (strcmp(lgopts[option_index].name, "sessionless") == 0) { 1390 options->sessionless = 1; 1391 return 0; 1392 } 1393 1394 else if (strcmp(lgopts[option_index].name, "cryptodev_mask") == 0) 1395 return parse_cryptodev_mask(options, optarg); 1396 1397 else if (strcmp(lgopts[option_index].name, "mac-updating") == 0) { 1398 options->mac_updating = 1; 1399 return 0; 1400 } 1401 1402 else if (strcmp(lgopts[option_index].name, "no-mac-updating") == 0) { 1403 options->mac_updating = 0; 1404 return 0; 1405 } 1406 1407 return -1; 1408 } 1409 1410 /** Parse port mask */ 1411 static int 1412 l2fwd_crypto_parse_portmask(struct l2fwd_crypto_options *options, 1413 const char *q_arg) 1414 { 1415 char *end = NULL; 1416 unsigned long pm; 1417 1418 /* parse hexadecimal string */ 1419 pm = strtoul(q_arg, &end, 16); 1420 if ((pm == '\0') || (end == NULL) || (*end != '\0')) 1421 pm = 0; 1422 1423 options->portmask = pm; 1424 if (options->portmask == 0) { 1425 printf("invalid portmask specified\n"); 1426 return -1; 1427 } 1428 1429 return pm; 1430 } 1431 1432 /** Parse number of queues */ 1433 static int 1434 l2fwd_crypto_parse_nqueue(struct l2fwd_crypto_options *options, 1435 const char *q_arg) 1436 { 1437 char *end = NULL; 1438 unsigned long n; 1439 1440 /* parse hexadecimal string */ 1441 n = strtoul(q_arg, &end, 10); 1442 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) 1443 n = 0; 1444 else if (n >= MAX_RX_QUEUE_PER_LCORE) 1445 n = 0; 1446 1447 options->nb_ports_per_lcore = n; 1448 if (options->nb_ports_per_lcore == 0) { 1449 printf("invalid number of ports selected\n"); 1450 return -1; 1451 } 1452 1453 return 0; 1454 } 1455 1456 /** Parse timer period */ 1457 static int 1458 l2fwd_crypto_parse_timer_period(struct l2fwd_crypto_options *options, 1459 const char *q_arg) 1460 { 1461 char *end = NULL; 1462 unsigned long n; 1463 1464 /* parse number string */ 1465 n = (unsigned)strtol(q_arg, &end, 10); 1466 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) 1467 n = 0; 1468 1469 if (n >= MAX_TIMER_PERIOD) { 1470 printf("Warning refresh period specified %lu is greater than " 1471 "max value %lu! using max value", 1472 n, MAX_TIMER_PERIOD); 1473 n = MAX_TIMER_PERIOD; 1474 } 1475 1476 options->refresh_period = n * 1000 * TIMER_MILLISECOND; 1477 1478 return 0; 1479 } 1480 1481 /** Generate default options for application */ 1482 static void 1483 l2fwd_crypto_default_options(struct l2fwd_crypto_options *options) 1484 { 1485 options->portmask = 0xffffffff; 1486 options->nb_ports_per_lcore = 1; 1487 options->refresh_period = DEFAULT_TIMER_PERIOD * 1488 TIMER_MILLISECOND * 1000; 1489 options->single_lcore = 0; 1490 options->sessionless = 0; 1491 1492 options->xform_chain = L2FWD_CRYPTO_CIPHER_HASH; 1493 1494 /* Cipher Data */ 1495 options->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; 1496 options->cipher_xform.next = NULL; 1497 options->ckey_param = 0; 1498 options->ckey_random_size = -1; 1499 options->cipher_xform.cipher.key.length = 0; 1500 options->cipher_iv_param = 0; 1501 options->cipher_iv_random_size = -1; 1502 options->cipher_iv.length = 0; 1503 1504 options->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; 1505 options->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 1506 options->cipher_xform.cipher.dataunit_len = 0; 1507 1508 /* Authentication Data */ 1509 options->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; 1510 options->auth_xform.next = NULL; 1511 options->akey_param = 0; 1512 options->akey_random_size = -1; 1513 options->auth_xform.auth.key.length = 0; 1514 options->auth_iv_param = 0; 1515 options->auth_iv_random_size = -1; 1516 options->auth_iv.length = 0; 1517 1518 options->auth_xform.auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC; 1519 options->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; 1520 1521 /* AEAD Data */ 1522 options->aead_xform.type = RTE_CRYPTO_SYM_XFORM_AEAD; 1523 options->aead_xform.next = NULL; 1524 options->aead_key_param = 0; 1525 options->aead_key_random_size = -1; 1526 options->aead_xform.aead.key.length = 0; 1527 options->aead_iv_param = 0; 1528 options->aead_iv_random_size = -1; 1529 options->aead_iv.length = 0; 1530 1531 options->aead_xform.aead.algo = RTE_CRYPTO_AEAD_AES_GCM; 1532 options->aead_xform.aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT; 1533 1534 options->aad_param = 0; 1535 options->aad_random_size = -1; 1536 options->aad.length = 0; 1537 1538 options->digest_size = -1; 1539 1540 options->type = CDEV_TYPE_ANY; 1541 options->cryptodev_mask = UINT64_MAX; 1542 1543 options->mac_updating = 1; 1544 } 1545 1546 static void 1547 display_cipher_info(struct l2fwd_crypto_options *options) 1548 { 1549 printf("\n---- Cipher information ---\n"); 1550 printf("Algorithm: %s\n", 1551 rte_crypto_cipher_algorithm_strings[options->cipher_xform.cipher.algo]); 1552 rte_hexdump(stdout, "Cipher key:", 1553 options->cipher_xform.cipher.key.data, 1554 options->cipher_xform.cipher.key.length); 1555 rte_hexdump(stdout, "IV:", options->cipher_iv.data, options->cipher_iv.length); 1556 } 1557 1558 static void 1559 display_auth_info(struct l2fwd_crypto_options *options) 1560 { 1561 printf("\n---- Authentication information ---\n"); 1562 printf("Algorithm: %s\n", 1563 rte_crypto_auth_algorithm_strings[options->auth_xform.auth.algo]); 1564 rte_hexdump(stdout, "Auth key:", 1565 options->auth_xform.auth.key.data, 1566 options->auth_xform.auth.key.length); 1567 rte_hexdump(stdout, "IV:", options->auth_iv.data, options->auth_iv.length); 1568 } 1569 1570 static void 1571 display_aead_info(struct l2fwd_crypto_options *options) 1572 { 1573 printf("\n---- AEAD information ---\n"); 1574 printf("Algorithm: %s\n", 1575 rte_crypto_aead_algorithm_strings[options->aead_xform.aead.algo]); 1576 rte_hexdump(stdout, "AEAD key:", 1577 options->aead_xform.aead.key.data, 1578 options->aead_xform.aead.key.length); 1579 rte_hexdump(stdout, "IV:", options->aead_iv.data, options->aead_iv.length); 1580 rte_hexdump(stdout, "AAD:", options->aad.data, options->aad.length); 1581 } 1582 1583 static void 1584 l2fwd_crypto_options_print(struct l2fwd_crypto_options *options) 1585 { 1586 char string_cipher_op[MAX_STR_LEN]; 1587 char string_auth_op[MAX_STR_LEN]; 1588 char string_aead_op[MAX_STR_LEN]; 1589 1590 if (options->cipher_xform.cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1591 strcpy(string_cipher_op, "Encrypt"); 1592 else 1593 strcpy(string_cipher_op, "Decrypt"); 1594 1595 if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) 1596 strcpy(string_auth_op, "Auth generate"); 1597 else 1598 strcpy(string_auth_op, "Auth verify"); 1599 1600 if (options->aead_xform.aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 1601 strcpy(string_aead_op, "Authenticated encryption"); 1602 else 1603 strcpy(string_aead_op, "Authenticated decryption"); 1604 1605 1606 printf("Options:-\nn"); 1607 printf("portmask: %x\n", options->portmask); 1608 printf("ports per lcore: %u\n", options->nb_ports_per_lcore); 1609 printf("refresh period : %u\n", options->refresh_period); 1610 printf("single lcore mode: %s\n", 1611 options->single_lcore ? "enabled" : "disabled"); 1612 printf("stats_printing: %s\n", 1613 options->refresh_period == 0 ? "disabled" : "enabled"); 1614 1615 printf("sessionless crypto: %s\n", 1616 options->sessionless ? "enabled" : "disabled"); 1617 1618 if (options->ckey_param && (options->ckey_random_size != -1)) 1619 printf("Cipher key already parsed, ignoring size of random key\n"); 1620 1621 if (options->akey_param && (options->akey_random_size != -1)) 1622 printf("Auth key already parsed, ignoring size of random key\n"); 1623 1624 if (options->cipher_iv_param && (options->cipher_iv_random_size != -1)) 1625 printf("Cipher IV already parsed, ignoring size of random IV\n"); 1626 1627 if (options->auth_iv_param && (options->auth_iv_random_size != -1)) 1628 printf("Auth IV already parsed, ignoring size of random IV\n"); 1629 1630 if (options->aad_param && (options->aad_random_size != -1)) 1631 printf("AAD already parsed, ignoring size of random AAD\n"); 1632 1633 printf("\nCrypto chain: "); 1634 switch (options->xform_chain) { 1635 case L2FWD_CRYPTO_AEAD: 1636 printf("Input --> %s --> Output\n", string_aead_op); 1637 display_aead_info(options); 1638 break; 1639 case L2FWD_CRYPTO_CIPHER_HASH: 1640 printf("Input --> %s --> %s --> Output\n", 1641 string_cipher_op, string_auth_op); 1642 display_cipher_info(options); 1643 display_auth_info(options); 1644 break; 1645 case L2FWD_CRYPTO_HASH_CIPHER: 1646 printf("Input --> %s --> %s --> Output\n", 1647 string_auth_op, string_cipher_op); 1648 display_cipher_info(options); 1649 display_auth_info(options); 1650 break; 1651 case L2FWD_CRYPTO_HASH_ONLY: 1652 printf("Input --> %s --> Output\n", string_auth_op); 1653 display_auth_info(options); 1654 break; 1655 case L2FWD_CRYPTO_CIPHER_ONLY: 1656 printf("Input --> %s --> Output\n", string_cipher_op); 1657 display_cipher_info(options); 1658 break; 1659 } 1660 } 1661 1662 /* Parse the argument given in the command line of the application */ 1663 static int 1664 l2fwd_crypto_parse_args(struct l2fwd_crypto_options *options, 1665 int argc, char **argv) 1666 { 1667 int opt, retval, option_index; 1668 char **argvopt = argv, *prgname = argv[0]; 1669 1670 static struct option lgopts[] = { 1671 { "sessionless", no_argument, 0, 0 }, 1672 1673 { "cdev_type", required_argument, 0, 0 }, 1674 { "chain", required_argument, 0, 0 }, 1675 1676 { "cipher_algo", required_argument, 0, 0 }, 1677 { "cipher_op", required_argument, 0, 0 }, 1678 { "cipher_key", required_argument, 0, 0 }, 1679 { "cipher_key_random_size", required_argument, 0, 0 }, 1680 { "cipher_iv", required_argument, 0, 0 }, 1681 { "cipher_iv_random_size", required_argument, 0, 0 }, 1682 { "cipher_dataunit_len", required_argument, 0, 0}, 1683 1684 { "auth_algo", required_argument, 0, 0 }, 1685 { "auth_op", required_argument, 0, 0 }, 1686 { "auth_key", required_argument, 0, 0 }, 1687 { "auth_key_random_size", required_argument, 0, 0 }, 1688 { "auth_iv", required_argument, 0, 0 }, 1689 { "auth_iv_random_size", required_argument, 0, 0 }, 1690 1691 { "aead_algo", required_argument, 0, 0 }, 1692 { "aead_op", required_argument, 0, 0 }, 1693 { "aead_key", required_argument, 0, 0 }, 1694 { "aead_key_random_size", required_argument, 0, 0 }, 1695 { "aead_iv", required_argument, 0, 0 }, 1696 { "aead_iv_random_size", required_argument, 0, 0 }, 1697 1698 { "aad", required_argument, 0, 0 }, 1699 { "aad_random_size", required_argument, 0, 0 }, 1700 1701 { "digest_size", required_argument, 0, 0 }, 1702 1703 { "sessionless", no_argument, 0, 0 }, 1704 { "cryptodev_mask", required_argument, 0, 0}, 1705 1706 { "mac-updating", no_argument, 0, 0}, 1707 { "no-mac-updating", no_argument, 0, 0}, 1708 1709 { NULL, 0, 0, 0 } 1710 }; 1711 1712 l2fwd_crypto_default_options(options); 1713 1714 while ((opt = getopt_long(argc, argvopt, "p:q:sT:", lgopts, 1715 &option_index)) != EOF) { 1716 switch (opt) { 1717 /* long options */ 1718 case 0: 1719 retval = l2fwd_crypto_parse_args_long_options(options, 1720 lgopts, option_index); 1721 if (retval < 0) { 1722 l2fwd_crypto_usage(prgname); 1723 return -1; 1724 } 1725 break; 1726 1727 /* portmask */ 1728 case 'p': 1729 retval = l2fwd_crypto_parse_portmask(options, optarg); 1730 if (retval < 0) { 1731 l2fwd_crypto_usage(prgname); 1732 return -1; 1733 } 1734 break; 1735 1736 /* nqueue */ 1737 case 'q': 1738 retval = l2fwd_crypto_parse_nqueue(options, optarg); 1739 if (retval < 0) { 1740 l2fwd_crypto_usage(prgname); 1741 return -1; 1742 } 1743 break; 1744 1745 /* single */ 1746 case 's': 1747 options->single_lcore = 1; 1748 1749 break; 1750 1751 /* timer period */ 1752 case 'T': 1753 retval = l2fwd_crypto_parse_timer_period(options, 1754 optarg); 1755 if (retval < 0) { 1756 l2fwd_crypto_usage(prgname); 1757 return -1; 1758 } 1759 break; 1760 1761 default: 1762 l2fwd_crypto_usage(prgname); 1763 return -1; 1764 } 1765 } 1766 1767 1768 if (optind >= 0) 1769 argv[optind-1] = prgname; 1770 1771 retval = optind-1; 1772 optind = 1; /* reset getopt lib */ 1773 1774 return retval; 1775 } 1776 1777 /* Check the link status of all ports in up to 9s, and print them finally */ 1778 static void 1779 check_all_ports_link_status(uint32_t port_mask) 1780 { 1781 #define CHECK_INTERVAL 100 /* 100ms */ 1782 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1783 uint16_t portid; 1784 uint8_t count, all_ports_up, print_flag = 0; 1785 struct rte_eth_link link; 1786 int ret; 1787 char link_status_text[RTE_ETH_LINK_MAX_STR_LEN]; 1788 1789 printf("\nChecking link status"); 1790 fflush(stdout); 1791 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1792 all_ports_up = 1; 1793 RTE_ETH_FOREACH_DEV(portid) { 1794 if ((port_mask & (1 << portid)) == 0) 1795 continue; 1796 memset(&link, 0, sizeof(link)); 1797 ret = rte_eth_link_get_nowait(portid, &link); 1798 if (ret < 0) { 1799 all_ports_up = 0; 1800 if (print_flag == 1) 1801 printf("Port %u link get failed: %s\n", 1802 portid, rte_strerror(-ret)); 1803 continue; 1804 } 1805 /* print link status if flag set */ 1806 if (print_flag == 1) { 1807 rte_eth_link_to_str(link_status_text, 1808 sizeof(link_status_text), &link); 1809 printf("Port %d %s\n", portid, 1810 link_status_text); 1811 continue; 1812 } 1813 /* clear all_ports_up flag if any link down */ 1814 if (link.link_status == RTE_ETH_LINK_DOWN) { 1815 all_ports_up = 0; 1816 break; 1817 } 1818 } 1819 /* after finally printing all link status, get out */ 1820 if (print_flag == 1) 1821 break; 1822 1823 if (all_ports_up == 0) { 1824 printf("."); 1825 fflush(stdout); 1826 rte_delay_ms(CHECK_INTERVAL); 1827 } 1828 1829 /* set the print_flag if all ports up or timeout */ 1830 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1831 print_flag = 1; 1832 printf("done\n"); 1833 } 1834 } 1835 } 1836 1837 /* Check if device has to be HW/SW or any */ 1838 static int 1839 check_type(const struct l2fwd_crypto_options *options, 1840 const struct rte_cryptodev_info *dev_info) 1841 { 1842 if (options->type == CDEV_TYPE_HW && 1843 (dev_info->feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED)) 1844 return 0; 1845 if (options->type == CDEV_TYPE_SW && 1846 !(dev_info->feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED)) 1847 return 0; 1848 if (options->type == CDEV_TYPE_ANY) 1849 return 0; 1850 1851 return -1; 1852 } 1853 1854 static const struct rte_cryptodev_capabilities * 1855 check_device_support_cipher_algo(const struct l2fwd_crypto_options *options, 1856 const struct rte_cryptodev_info *dev_info, 1857 uint8_t cdev_id) 1858 { 1859 unsigned int i = 0; 1860 const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; 1861 enum rte_crypto_cipher_algorithm cap_cipher_algo; 1862 enum rte_crypto_cipher_algorithm opt_cipher_algo = 1863 options->cipher_xform.cipher.algo; 1864 1865 while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1866 cap_cipher_algo = cap->sym.cipher.algo; 1867 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 1868 if (cap_cipher_algo == opt_cipher_algo) { 1869 if (check_type(options, dev_info) == 0) 1870 break; 1871 } 1872 } 1873 cap = &dev_info->capabilities[++i]; 1874 } 1875 1876 if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1877 printf("Algorithm %s not supported by cryptodev %u" 1878 " or device not of preferred type (%s)\n", 1879 rte_crypto_cipher_algorithm_strings[opt_cipher_algo], 1880 cdev_id, 1881 options->string_type); 1882 return NULL; 1883 } 1884 1885 return cap; 1886 } 1887 1888 static const struct rte_cryptodev_capabilities * 1889 check_device_support_auth_algo(const struct l2fwd_crypto_options *options, 1890 const struct rte_cryptodev_info *dev_info, 1891 uint8_t cdev_id) 1892 { 1893 unsigned int i = 0; 1894 const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; 1895 enum rte_crypto_auth_algorithm cap_auth_algo; 1896 enum rte_crypto_auth_algorithm opt_auth_algo = 1897 options->auth_xform.auth.algo; 1898 1899 while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1900 cap_auth_algo = cap->sym.auth.algo; 1901 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AUTH) { 1902 if (cap_auth_algo == opt_auth_algo) { 1903 if (check_type(options, dev_info) == 0) 1904 break; 1905 } 1906 } 1907 cap = &dev_info->capabilities[++i]; 1908 } 1909 1910 if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1911 printf("Algorithm %s not supported by cryptodev %u" 1912 " or device not of preferred type (%s)\n", 1913 rte_crypto_auth_algorithm_strings[opt_auth_algo], 1914 cdev_id, 1915 options->string_type); 1916 return NULL; 1917 } 1918 1919 return cap; 1920 } 1921 1922 static const struct rte_cryptodev_capabilities * 1923 check_device_support_aead_algo(const struct l2fwd_crypto_options *options, 1924 const struct rte_cryptodev_info *dev_info, 1925 uint8_t cdev_id) 1926 { 1927 unsigned int i = 0; 1928 const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; 1929 enum rte_crypto_aead_algorithm cap_aead_algo; 1930 enum rte_crypto_aead_algorithm opt_aead_algo = 1931 options->aead_xform.aead.algo; 1932 1933 while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1934 cap_aead_algo = cap->sym.aead.algo; 1935 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1936 if (cap_aead_algo == opt_aead_algo) { 1937 if (check_type(options, dev_info) == 0) 1938 break; 1939 } 1940 } 1941 cap = &dev_info->capabilities[++i]; 1942 } 1943 1944 if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1945 printf("Algorithm %s not supported by cryptodev %u" 1946 " or device not of preferred type (%s)\n", 1947 rte_crypto_aead_algorithm_strings[opt_aead_algo], 1948 cdev_id, 1949 options->string_type); 1950 return NULL; 1951 } 1952 1953 return cap; 1954 } 1955 1956 /* Check if the device is enabled by cryptodev_mask */ 1957 static int 1958 check_cryptodev_mask(struct l2fwd_crypto_options *options, 1959 uint8_t cdev_id) 1960 { 1961 if (options->cryptodev_mask & (1 << cdev_id)) 1962 return 0; 1963 1964 return -1; 1965 } 1966 1967 static inline int 1968 check_supported_size(uint16_t length, uint16_t min, uint16_t max, 1969 uint16_t increment) 1970 { 1971 uint16_t supp_size; 1972 1973 /* Single value */ 1974 if (increment == 0) { 1975 if (length == min) 1976 return 0; 1977 else 1978 return -1; 1979 } 1980 1981 /* Range of values */ 1982 for (supp_size = min; supp_size <= max; supp_size += increment) { 1983 if (length == supp_size) 1984 return 0; 1985 } 1986 1987 return -1; 1988 } 1989 1990 static int 1991 check_iv_param(const struct rte_crypto_param_range *iv_range_size, 1992 unsigned int iv_param, int iv_random_size, 1993 uint16_t iv_length) 1994 { 1995 /* 1996 * Check if length of provided IV is supported 1997 * by the algorithm chosen. 1998 */ 1999 if (iv_param) { 2000 if (check_supported_size(iv_length, 2001 iv_range_size->min, 2002 iv_range_size->max, 2003 iv_range_size->increment) 2004 != 0) 2005 return -1; 2006 /* 2007 * Check if length of IV to be randomly generated 2008 * is supported by the algorithm chosen. 2009 */ 2010 } else if (iv_random_size != -1) { 2011 if (check_supported_size(iv_random_size, 2012 iv_range_size->min, 2013 iv_range_size->max, 2014 iv_range_size->increment) 2015 != 0) 2016 return -1; 2017 } 2018 2019 return 0; 2020 } 2021 2022 static int 2023 check_capabilities(struct l2fwd_crypto_options *options, uint8_t cdev_id) 2024 { 2025 struct rte_cryptodev_info dev_info; 2026 const struct rte_cryptodev_capabilities *cap; 2027 2028 rte_cryptodev_info_get(cdev_id, &dev_info); 2029 2030 /* Set AEAD parameters */ 2031 if (options->xform_chain == L2FWD_CRYPTO_AEAD) { 2032 /* Check if device supports AEAD algo */ 2033 cap = check_device_support_aead_algo(options, &dev_info, 2034 cdev_id); 2035 if (cap == NULL) 2036 return -1; 2037 2038 if (check_iv_param(&cap->sym.aead.iv_size, 2039 options->aead_iv_param, 2040 options->aead_iv_random_size, 2041 options->aead_iv.length) != 0) { 2042 RTE_LOG(DEBUG, USER1, 2043 "Device %u does not support IV length\n", 2044 cdev_id); 2045 return -1; 2046 } 2047 2048 /* 2049 * Check if length of provided AEAD key is supported 2050 * by the algorithm chosen. 2051 */ 2052 if (options->aead_key_param) { 2053 if (check_supported_size( 2054 options->aead_xform.aead.key.length, 2055 cap->sym.aead.key_size.min, 2056 cap->sym.aead.key_size.max, 2057 cap->sym.aead.key_size.increment) 2058 != 0) { 2059 RTE_LOG(DEBUG, USER1, 2060 "Device %u does not support " 2061 "AEAD key length\n", 2062 cdev_id); 2063 return -1; 2064 } 2065 /* 2066 * Check if length of the aead key to be randomly generated 2067 * is supported by the algorithm chosen. 2068 */ 2069 } else if (options->aead_key_random_size != -1) { 2070 if (check_supported_size(options->aead_key_random_size, 2071 cap->sym.aead.key_size.min, 2072 cap->sym.aead.key_size.max, 2073 cap->sym.aead.key_size.increment) 2074 != 0) { 2075 RTE_LOG(DEBUG, USER1, 2076 "Device %u does not support " 2077 "AEAD key length\n", 2078 cdev_id); 2079 return -1; 2080 } 2081 } 2082 2083 2084 /* 2085 * Check if length of provided AAD is supported 2086 * by the algorithm chosen. 2087 */ 2088 if (options->aad_param) { 2089 if (check_supported_size(options->aad.length, 2090 cap->sym.aead.aad_size.min, 2091 cap->sym.aead.aad_size.max, 2092 cap->sym.aead.aad_size.increment) 2093 != 0) { 2094 RTE_LOG(DEBUG, USER1, 2095 "Device %u does not support " 2096 "AAD length\n", 2097 cdev_id); 2098 return -1; 2099 } 2100 /* 2101 * Check if length of AAD to be randomly generated 2102 * is supported by the algorithm chosen. 2103 */ 2104 } else if (options->aad_random_size != -1) { 2105 if (check_supported_size(options->aad_random_size, 2106 cap->sym.aead.aad_size.min, 2107 cap->sym.aead.aad_size.max, 2108 cap->sym.aead.aad_size.increment) 2109 != 0) { 2110 RTE_LOG(DEBUG, USER1, 2111 "Device %u does not support " 2112 "AAD length\n", 2113 cdev_id); 2114 return -1; 2115 } 2116 } 2117 2118 /* Check if digest size is supported by the algorithm. */ 2119 if (options->digest_size != -1) { 2120 if (check_supported_size(options->digest_size, 2121 cap->sym.aead.digest_size.min, 2122 cap->sym.aead.digest_size.max, 2123 cap->sym.aead.digest_size.increment) 2124 != 0) { 2125 RTE_LOG(DEBUG, USER1, 2126 "Device %u does not support " 2127 "digest length\n", 2128 cdev_id); 2129 return -1; 2130 } 2131 } 2132 } 2133 2134 /* Set cipher parameters */ 2135 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2136 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2137 options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { 2138 2139 /* Check if device supports cipher algo. 8< */ 2140 cap = check_device_support_cipher_algo(options, &dev_info, 2141 cdev_id); 2142 if (cap == NULL) 2143 return -1; 2144 2145 if (check_iv_param(&cap->sym.cipher.iv_size, 2146 options->cipher_iv_param, 2147 options->cipher_iv_random_size, 2148 options->cipher_iv.length) != 0) { 2149 RTE_LOG(DEBUG, USER1, 2150 "Device %u does not support IV length\n", 2151 cdev_id); 2152 return -1; 2153 } 2154 /* >8 End of check if device supports cipher algo. */ 2155 2156 /* Check if capable cipher is supported. 8< */ 2157 2158 /* 2159 * Check if length of provided cipher key is supported 2160 * by the algorithm chosen. 2161 */ 2162 if (options->ckey_param) { 2163 if (check_supported_size( 2164 options->cipher_xform.cipher.key.length, 2165 cap->sym.cipher.key_size.min, 2166 cap->sym.cipher.key_size.max, 2167 cap->sym.cipher.key_size.increment) 2168 != 0) { 2169 if (dev_info.feature_flags & 2170 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY) { 2171 RTE_LOG(DEBUG, USER1, 2172 "Key length does not match the device " 2173 "%u capability. Key may be wrapped\n", 2174 cdev_id); 2175 } else { 2176 RTE_LOG(DEBUG, USER1, 2177 "Key length does not match the device " 2178 "%u capability\n", 2179 cdev_id); 2180 return -1; 2181 } 2182 } 2183 2184 /* 2185 * Check if length of the cipher key to be randomly generated 2186 * is supported by the algorithm chosen. 2187 */ 2188 } else if (options->ckey_random_size != -1) { 2189 if (check_supported_size(options->ckey_random_size, 2190 cap->sym.cipher.key_size.min, 2191 cap->sym.cipher.key_size.max, 2192 cap->sym.cipher.key_size.increment) 2193 != 0) { 2194 RTE_LOG(DEBUG, USER1, 2195 "Device %u does not support cipher " 2196 "key length\n", 2197 cdev_id); 2198 return -1; 2199 } 2200 } 2201 2202 if (options->cipher_xform.cipher.dataunit_len > 0) { 2203 if (!(dev_info.feature_flags & 2204 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)) { 2205 RTE_LOG(DEBUG, USER1, 2206 "Device %u does not support " 2207 "cipher multiple data units\n", 2208 cdev_id); 2209 return -1; 2210 } 2211 if (cap->sym.cipher.dataunit_set != 0) { 2212 int ret = 0; 2213 2214 switch (options->cipher_xform.cipher.dataunit_len) { 2215 case 512: 2216 if (!(cap->sym.cipher.dataunit_set & 2217 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES)) 2218 ret = -1; 2219 break; 2220 case 4096: 2221 if (!(cap->sym.cipher.dataunit_set & 2222 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES)) 2223 ret = -1; 2224 break; 2225 case 1048576: 2226 if (!(cap->sym.cipher.dataunit_set & 2227 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_1_MEGABYTES)) 2228 ret = -1; 2229 break; 2230 default: 2231 ret = -1; 2232 } 2233 if (ret == -1) { 2234 RTE_LOG(DEBUG, USER1, 2235 "Device %u does not support " 2236 "data-unit length %u\n", 2237 cdev_id, 2238 options->cipher_xform.cipher.dataunit_len); 2239 return -1; 2240 } 2241 } 2242 } 2243 /* >8 End of checking if cipher is supported. */ 2244 } 2245 2246 /* Set auth parameters */ 2247 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2248 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2249 options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) { 2250 /* Check if device supports auth algo */ 2251 cap = check_device_support_auth_algo(options, &dev_info, 2252 cdev_id); 2253 if (cap == NULL) 2254 return -1; 2255 2256 if (check_iv_param(&cap->sym.auth.iv_size, 2257 options->auth_iv_param, 2258 options->auth_iv_random_size, 2259 options->auth_iv.length) != 0) { 2260 RTE_LOG(DEBUG, USER1, 2261 "Device %u does not support IV length\n", 2262 cdev_id); 2263 return -1; 2264 } 2265 /* 2266 * Check if length of provided auth key is supported 2267 * by the algorithm chosen. 2268 */ 2269 if (options->akey_param) { 2270 if (check_supported_size( 2271 options->auth_xform.auth.key.length, 2272 cap->sym.auth.key_size.min, 2273 cap->sym.auth.key_size.max, 2274 cap->sym.auth.key_size.increment) 2275 != 0) { 2276 RTE_LOG(DEBUG, USER1, 2277 "Device %u does not support auth " 2278 "key length\n", 2279 cdev_id); 2280 return -1; 2281 } 2282 /* 2283 * Check if length of the auth key to be randomly generated 2284 * is supported by the algorithm chosen. 2285 */ 2286 } else if (options->akey_random_size != -1) { 2287 if (check_supported_size(options->akey_random_size, 2288 cap->sym.auth.key_size.min, 2289 cap->sym.auth.key_size.max, 2290 cap->sym.auth.key_size.increment) 2291 != 0) { 2292 RTE_LOG(DEBUG, USER1, 2293 "Device %u does not support auth " 2294 "key length\n", 2295 cdev_id); 2296 return -1; 2297 } 2298 } 2299 2300 /* Check if digest size is supported by the algorithm. */ 2301 if (options->digest_size != -1) { 2302 if (check_supported_size(options->digest_size, 2303 cap->sym.auth.digest_size.min, 2304 cap->sym.auth.digest_size.max, 2305 cap->sym.auth.digest_size.increment) 2306 != 0) { 2307 RTE_LOG(DEBUG, USER1, 2308 "Device %u does not support " 2309 "digest length\n", 2310 cdev_id); 2311 return -1; 2312 } 2313 } 2314 } 2315 2316 return 0; 2317 } 2318 2319 static int 2320 initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, 2321 uint8_t *enabled_cdevs) 2322 { 2323 uint8_t cdev_id, cdev_count, enabled_cdev_count = 0; 2324 const struct rte_cryptodev_capabilities *cap; 2325 unsigned int sess_sz, max_sess_sz = 0; 2326 uint32_t sessions_needed = 0; 2327 int retval; 2328 2329 cdev_count = rte_cryptodev_count(); 2330 if (cdev_count == 0) { 2331 printf("No crypto devices available\n"); 2332 return -1; 2333 } 2334 2335 for (cdev_id = 0; cdev_id < cdev_count && enabled_cdev_count < nb_ports; 2336 cdev_id++) { 2337 if (check_cryptodev_mask(options, cdev_id) < 0) 2338 continue; 2339 2340 if (check_capabilities(options, cdev_id) < 0) 2341 continue; 2342 2343 sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id); 2344 if (sess_sz > max_sess_sz) 2345 max_sess_sz = sess_sz; 2346 2347 l2fwd_enabled_crypto_mask |= (((uint64_t)1) << cdev_id); 2348 2349 enabled_cdevs[cdev_id] = 1; 2350 enabled_cdev_count++; 2351 } 2352 2353 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { 2354 struct rte_cryptodev_qp_conf qp_conf; 2355 struct rte_cryptodev_info dev_info; 2356 2357 if (enabled_cdevs[cdev_id] == 0) 2358 continue; 2359 2360 if (check_cryptodev_mask(options, cdev_id) < 0) 2361 continue; 2362 2363 if (check_capabilities(options, cdev_id) < 0) 2364 continue; 2365 2366 retval = rte_cryptodev_socket_id(cdev_id); 2367 2368 if (retval < 0) { 2369 printf("Invalid crypto device id used\n"); 2370 return -1; 2371 } 2372 2373 uint8_t socket_id = (uint8_t) retval; 2374 2375 struct rte_cryptodev_config conf = { 2376 .nb_queue_pairs = 1, 2377 .socket_id = socket_id, 2378 .ff_disable = RTE_CRYPTODEV_FF_SECURITY, 2379 }; 2380 2381 rte_cryptodev_info_get(cdev_id, &dev_info); 2382 2383 /* 2384 * Two sessions objects are required for each session 2385 * (one for the header, one for the private data) 2386 */ 2387 if (!strcmp(dev_info.driver_name, "crypto_scheduler")) { 2388 #ifdef RTE_CRYPTO_SCHEDULER 2389 uint32_t nb_workers = 2390 rte_cryptodev_scheduler_workers_get(cdev_id, 2391 NULL); 2392 2393 sessions_needed = enabled_cdev_count * nb_workers; 2394 #endif 2395 } else 2396 sessions_needed = enabled_cdev_count; 2397 2398 if (session_pool_socket[socket_id].priv_mp == NULL) { 2399 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2400 2401 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2402 "priv_sess_mp_%u", socket_id); 2403 2404 session_pool_socket[socket_id].priv_mp = 2405 rte_mempool_create(mp_name, 2406 sessions_needed, 2407 max_sess_sz, 2408 0, 0, NULL, NULL, NULL, 2409 NULL, socket_id, 2410 0); 2411 2412 if (session_pool_socket[socket_id].priv_mp == NULL) { 2413 printf("Cannot create pool on socket %d\n", 2414 socket_id); 2415 return -ENOMEM; 2416 } 2417 2418 printf("Allocated pool \"%s\" on socket %d\n", 2419 mp_name, socket_id); 2420 } 2421 2422 if (session_pool_socket[socket_id].sess_mp == NULL) { 2423 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2424 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2425 "sess_mp_%u", socket_id); 2426 2427 session_pool_socket[socket_id].sess_mp = 2428 rte_cryptodev_sym_session_pool_create( 2429 mp_name, 2430 sessions_needed, 2431 0, 0, 0, socket_id); 2432 2433 if (session_pool_socket[socket_id].sess_mp == NULL) { 2434 printf("Cannot create pool on socket %d\n", 2435 socket_id); 2436 return -ENOMEM; 2437 } 2438 2439 printf("Allocated pool \"%s\" on socket %d\n", 2440 mp_name, socket_id); 2441 } 2442 2443 /* Set AEAD parameters */ 2444 if (options->xform_chain == L2FWD_CRYPTO_AEAD) { 2445 cap = check_device_support_aead_algo(options, &dev_info, 2446 cdev_id); 2447 2448 options->block_size = cap->sym.aead.block_size; 2449 2450 /* Set IV if not provided from command line */ 2451 if (options->aead_iv_param == 0) { 2452 if (options->aead_iv_random_size != -1) 2453 options->aead_iv.length = 2454 options->aead_iv_random_size; 2455 /* No size provided, use minimum size. */ 2456 else 2457 options->aead_iv.length = 2458 cap->sym.aead.iv_size.min; 2459 } 2460 2461 /* Set key if not provided from command line */ 2462 if (options->aead_key_param == 0) { 2463 if (options->aead_key_random_size != -1) 2464 options->aead_xform.aead.key.length = 2465 options->aead_key_random_size; 2466 /* No size provided, use minimum size. */ 2467 else 2468 options->aead_xform.aead.key.length = 2469 cap->sym.aead.key_size.min; 2470 2471 generate_random_key(options->aead_key, 2472 options->aead_xform.aead.key.length); 2473 } 2474 2475 /* Set AAD if not provided from command line */ 2476 if (options->aad_param == 0) { 2477 if (options->aad_random_size != -1) 2478 options->aad.length = 2479 options->aad_random_size; 2480 /* No size provided, use minimum size. */ 2481 else 2482 options->aad.length = 2483 cap->sym.auth.aad_size.min; 2484 } 2485 2486 options->aead_xform.aead.aad_length = 2487 options->aad.length; 2488 2489 /* Set digest size if not provided from command line */ 2490 if (options->digest_size != -1) 2491 options->aead_xform.aead.digest_length = 2492 options->digest_size; 2493 /* No size provided, use minimum size. */ 2494 else 2495 options->aead_xform.aead.digest_length = 2496 cap->sym.aead.digest_size.min; 2497 } 2498 2499 /* Set cipher parameters */ 2500 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2501 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2502 options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { 2503 cap = check_device_support_cipher_algo(options, &dev_info, 2504 cdev_id); 2505 options->block_size = cap->sym.cipher.block_size; 2506 2507 /* Set IV if not provided from command line */ 2508 if (options->cipher_iv_param == 0) { 2509 if (options->cipher_iv_random_size != -1) 2510 options->cipher_iv.length = 2511 options->cipher_iv_random_size; 2512 /* No size provided, use minimum size. */ 2513 else 2514 options->cipher_iv.length = 2515 cap->sym.cipher.iv_size.min; 2516 } 2517 2518 /* Set key if not provided from command line */ 2519 if (options->ckey_param == 0) { 2520 if (options->ckey_random_size != -1) 2521 options->cipher_xform.cipher.key.length = 2522 options->ckey_random_size; 2523 /* No size provided, use minimum size. */ 2524 else 2525 options->cipher_xform.cipher.key.length = 2526 cap->sym.cipher.key_size.min; 2527 2528 generate_random_key(options->cipher_key, 2529 options->cipher_xform.cipher.key.length); 2530 } 2531 } 2532 2533 /* Set auth parameters */ 2534 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2535 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2536 options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) { 2537 cap = check_device_support_auth_algo(options, &dev_info, 2538 cdev_id); 2539 2540 /* Set IV if not provided from command line */ 2541 if (options->auth_iv_param == 0) { 2542 if (options->auth_iv_random_size != -1) 2543 options->auth_iv.length = 2544 options->auth_iv_random_size; 2545 /* No size provided, use minimum size. */ 2546 else 2547 options->auth_iv.length = 2548 cap->sym.auth.iv_size.min; 2549 } 2550 2551 /* Set key if not provided from command line */ 2552 if (options->akey_param == 0) { 2553 if (options->akey_random_size != -1) 2554 options->auth_xform.auth.key.length = 2555 options->akey_random_size; 2556 /* No size provided, use minimum size. */ 2557 else 2558 options->auth_xform.auth.key.length = 2559 cap->sym.auth.key_size.min; 2560 2561 generate_random_key(options->auth_key, 2562 options->auth_xform.auth.key.length); 2563 } 2564 2565 /* Set digest size if not provided from command line */ 2566 if (options->digest_size != -1) 2567 options->auth_xform.auth.digest_length = 2568 options->digest_size; 2569 /* No size provided, use minimum size. */ 2570 else 2571 options->auth_xform.auth.digest_length = 2572 cap->sym.auth.digest_size.min; 2573 } 2574 2575 retval = rte_cryptodev_configure(cdev_id, &conf); 2576 if (retval < 0) { 2577 printf("Failed to configure cryptodev %u", cdev_id); 2578 return -1; 2579 } 2580 2581 qp_conf.nb_descriptors = 2048; 2582 qp_conf.mp_session = session_pool_socket[socket_id].sess_mp; 2583 qp_conf.mp_session_private = 2584 session_pool_socket[socket_id].priv_mp; 2585 2586 retval = rte_cryptodev_queue_pair_setup(cdev_id, 0, &qp_conf, 2587 socket_id); 2588 if (retval < 0) { 2589 printf("Failed to setup queue pair %u on cryptodev %u", 2590 0, cdev_id); 2591 return -1; 2592 } 2593 2594 retval = rte_cryptodev_start(cdev_id); 2595 if (retval < 0) { 2596 printf("Failed to start device %u: error %d\n", 2597 cdev_id, retval); 2598 return -1; 2599 } 2600 } 2601 2602 return enabled_cdev_count; 2603 } 2604 2605 static int 2606 initialize_ports(struct l2fwd_crypto_options *options) 2607 { 2608 uint16_t last_portid = 0, portid; 2609 unsigned enabled_portcount = 0; 2610 unsigned nb_ports = rte_eth_dev_count_avail(); 2611 2612 if (nb_ports == 0) { 2613 printf("No Ethernet ports - bye\n"); 2614 return -1; 2615 } 2616 2617 /* Reset l2fwd_dst_ports */ 2618 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) 2619 l2fwd_dst_ports[portid] = 0; 2620 2621 RTE_ETH_FOREACH_DEV(portid) { 2622 int retval; 2623 struct rte_eth_dev_info dev_info; 2624 struct rte_eth_rxconf rxq_conf; 2625 struct rte_eth_txconf txq_conf; 2626 struct rte_eth_conf local_port_conf = port_conf; 2627 2628 /* Skip ports that are not enabled */ 2629 if ((options->portmask & (1 << portid)) == 0) 2630 continue; 2631 2632 /* init port */ 2633 printf("Initializing port %u... ", portid); 2634 fflush(stdout); 2635 2636 retval = rte_eth_dev_info_get(portid, &dev_info); 2637 if (retval != 0) { 2638 printf("Error during getting device (port %u) info: %s\n", 2639 portid, strerror(-retval)); 2640 return retval; 2641 } 2642 2643 if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 2644 local_port_conf.txmode.offloads |= 2645 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 2646 retval = rte_eth_dev_configure(portid, 1, 1, &local_port_conf); 2647 if (retval < 0) { 2648 printf("Cannot configure device: err=%d, port=%u\n", 2649 retval, portid); 2650 return -1; 2651 } 2652 2653 retval = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, 2654 &nb_txd); 2655 if (retval < 0) { 2656 printf("Cannot adjust number of descriptors: err=%d, port=%u\n", 2657 retval, portid); 2658 return -1; 2659 } 2660 2661 /* init one RX queue */ 2662 fflush(stdout); 2663 rxq_conf = dev_info.default_rxconf; 2664 rxq_conf.offloads = local_port_conf.rxmode.offloads; 2665 retval = rte_eth_rx_queue_setup(portid, 0, nb_rxd, 2666 rte_eth_dev_socket_id(portid), 2667 &rxq_conf, l2fwd_pktmbuf_pool); 2668 if (retval < 0) { 2669 printf("rte_eth_rx_queue_setup:err=%d, port=%u\n", 2670 retval, portid); 2671 return -1; 2672 } 2673 2674 /* init one TX queue on each port */ 2675 fflush(stdout); 2676 txq_conf = dev_info.default_txconf; 2677 txq_conf.offloads = local_port_conf.txmode.offloads; 2678 retval = rte_eth_tx_queue_setup(portid, 0, nb_txd, 2679 rte_eth_dev_socket_id(portid), 2680 &txq_conf); 2681 if (retval < 0) { 2682 printf("rte_eth_tx_queue_setup:err=%d, port=%u\n", 2683 retval, portid); 2684 2685 return -1; 2686 } 2687 2688 /* Start device */ 2689 retval = rte_eth_dev_start(portid); 2690 if (retval < 0) { 2691 printf("rte_eth_dev_start:err=%d, port=%u\n", 2692 retval, portid); 2693 return -1; 2694 } 2695 2696 retval = rte_eth_promiscuous_enable(portid); 2697 if (retval != 0) { 2698 printf("rte_eth_promiscuous_enable:err=%s, port=%u\n", 2699 rte_strerror(-retval), portid); 2700 return -1; 2701 } 2702 2703 retval = rte_eth_macaddr_get(portid, 2704 &l2fwd_ports_eth_addr[portid]); 2705 if (retval < 0) { 2706 printf("rte_eth_macaddr_get :err=%d, port=%u\n", 2707 retval, portid); 2708 return -1; 2709 } 2710 2711 printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n", 2712 portid, 2713 RTE_ETHER_ADDR_BYTES(&l2fwd_ports_eth_addr[portid])); 2714 2715 /* initialize port stats */ 2716 memset(&port_statistics, 0, sizeof(port_statistics)); 2717 2718 /* Setup port forwarding table */ 2719 if (enabled_portcount % 2) { 2720 l2fwd_dst_ports[portid] = last_portid; 2721 l2fwd_dst_ports[last_portid] = portid; 2722 } else { 2723 last_portid = portid; 2724 } 2725 2726 l2fwd_enabled_port_mask |= (1ULL << portid); 2727 enabled_portcount++; 2728 } 2729 2730 if (enabled_portcount == 1) { 2731 l2fwd_dst_ports[last_portid] = last_portid; 2732 } else if (enabled_portcount % 2) { 2733 printf("odd number of ports in portmask- bye\n"); 2734 return -1; 2735 } 2736 2737 check_all_ports_link_status(l2fwd_enabled_port_mask); 2738 2739 return enabled_portcount; 2740 } 2741 2742 static void 2743 reserve_key_memory(struct l2fwd_crypto_options *options) 2744 { 2745 options->cipher_xform.cipher.key.data = options->cipher_key; 2746 2747 options->auth_xform.auth.key.data = options->auth_key; 2748 2749 options->aead_xform.aead.key.data = options->aead_key; 2750 2751 options->cipher_iv.data = rte_malloc("cipher iv", MAX_KEY_SIZE, 0); 2752 if (options->cipher_iv.data == NULL) 2753 rte_exit(EXIT_FAILURE, "Failed to allocate memory for cipher IV"); 2754 2755 options->auth_iv.data = rte_malloc("auth iv", MAX_KEY_SIZE, 0); 2756 if (options->auth_iv.data == NULL) 2757 rte_exit(EXIT_FAILURE, "Failed to allocate memory for auth IV"); 2758 2759 options->aead_iv.data = rte_malloc("aead_iv", MAX_KEY_SIZE, 0); 2760 if (options->aead_iv.data == NULL) 2761 rte_exit(EXIT_FAILURE, "Failed to allocate memory for AEAD iv"); 2762 2763 options->aad.data = rte_malloc("aad", MAX_KEY_SIZE, 0); 2764 if (options->aad.data == NULL) 2765 rte_exit(EXIT_FAILURE, "Failed to allocate memory for AAD"); 2766 options->aad.phys_addr = rte_malloc_virt2iova(options->aad.data); 2767 } 2768 2769 static void 2770 raise_signal(int signum) 2771 { 2772 if (signum == SIGINT || signum == SIGTERM) 2773 signal_received = true; 2774 } 2775 2776 int 2777 main(int argc, char **argv) 2778 { 2779 struct lcore_queue_conf *qconf = NULL; 2780 struct l2fwd_crypto_options options; 2781 2782 uint8_t nb_cryptodevs, cdev_id; 2783 uint16_t portid; 2784 unsigned lcore_id, rx_lcore_id = 0; 2785 int ret, enabled_cdevcount, enabled_portcount; 2786 uint8_t enabled_cdevs[RTE_CRYPTO_MAX_DEVS] = {0}; 2787 2788 signal(SIGINT, raise_signal); 2789 signal(SIGTERM, raise_signal); 2790 2791 /* init EAL */ 2792 ret = rte_eal_init(argc, argv); 2793 if (ret < 0) 2794 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n"); 2795 argc -= ret; 2796 argv += ret; 2797 2798 /* reserve memory for Cipher/Auth key and IV */ 2799 reserve_key_memory(&options); 2800 2801 /* parse application arguments (after the EAL ones) */ 2802 ret = l2fwd_crypto_parse_args(&options, argc, argv); 2803 if (ret < 0) 2804 rte_exit(EXIT_FAILURE, "Invalid L2FWD-CRYPTO arguments\n"); 2805 2806 printf("MAC updating %s\n", 2807 options.mac_updating ? "enabled" : "disabled"); 2808 2809 /* create the mbuf pool */ 2810 l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF, 512, 2811 RTE_ALIGN(sizeof(struct rte_crypto_op), 2812 RTE_CACHE_LINE_SIZE), 2813 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 2814 if (l2fwd_pktmbuf_pool == NULL) 2815 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 2816 2817 /* create crypto op pool */ 2818 l2fwd_crypto_op_pool = rte_crypto_op_pool_create("crypto_op_pool", 2819 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MBUF, 128, MAXIMUM_IV_LENGTH, 2820 rte_socket_id()); 2821 if (l2fwd_crypto_op_pool == NULL) 2822 rte_exit(EXIT_FAILURE, "Cannot create crypto op pool\n"); 2823 2824 /* Enable Ethernet ports */ 2825 enabled_portcount = initialize_ports(&options); 2826 if (enabled_portcount < 1) 2827 rte_exit(EXIT_FAILURE, "Failed to initial Ethernet ports\n"); 2828 2829 /* Initialize the port/queue configuration of each logical core */ 2830 RTE_ETH_FOREACH_DEV(portid) { 2831 2832 /* skip ports that are not enabled */ 2833 if ((options.portmask & (1 << portid)) == 0) 2834 continue; 2835 2836 if (options.single_lcore && qconf == NULL) { 2837 while (rte_lcore_is_enabled(rx_lcore_id) == 0) { 2838 rx_lcore_id++; 2839 if (rx_lcore_id >= RTE_MAX_LCORE) 2840 rte_exit(EXIT_FAILURE, 2841 "Not enough cores\n"); 2842 } 2843 } else if (!options.single_lcore) { 2844 /* get the lcore_id for this port */ 2845 while (rte_lcore_is_enabled(rx_lcore_id) == 0 || 2846 lcore_queue_conf[rx_lcore_id].nb_rx_ports == 2847 options.nb_ports_per_lcore) { 2848 rx_lcore_id++; 2849 if (rx_lcore_id >= RTE_MAX_LCORE) 2850 rte_exit(EXIT_FAILURE, 2851 "Not enough cores\n"); 2852 } 2853 } 2854 2855 /* Assigned a new logical core in the loop above. */ 2856 if (qconf != &lcore_queue_conf[rx_lcore_id]) 2857 qconf = &lcore_queue_conf[rx_lcore_id]; 2858 2859 qconf->rx_port_list[qconf->nb_rx_ports] = portid; 2860 qconf->nb_rx_ports++; 2861 2862 printf("Lcore %u: RX port %u\n", rx_lcore_id, portid); 2863 } 2864 2865 /* Enable Crypto devices */ 2866 enabled_cdevcount = initialize_cryptodevs(&options, enabled_portcount, 2867 enabled_cdevs); 2868 if (enabled_cdevcount < 0) 2869 rte_exit(EXIT_FAILURE, "Failed to initialize crypto devices\n"); 2870 2871 if (enabled_cdevcount < enabled_portcount) 2872 rte_exit(EXIT_FAILURE, "Number of capable crypto devices (%d) " 2873 "has to be more or equal to number of ports (%d)\n", 2874 enabled_cdevcount, enabled_portcount); 2875 2876 nb_cryptodevs = rte_cryptodev_count(); 2877 2878 /* Initialize the port/cryptodev configuration of each logical core */ 2879 for (rx_lcore_id = 0, qconf = NULL, cdev_id = 0; 2880 cdev_id < nb_cryptodevs && enabled_cdevcount; 2881 cdev_id++) { 2882 /* Crypto op not supported by crypto device */ 2883 if (!enabled_cdevs[cdev_id]) 2884 continue; 2885 2886 if (options.single_lcore && qconf == NULL) { 2887 while (rte_lcore_is_enabled(rx_lcore_id) == 0) { 2888 rx_lcore_id++; 2889 if (rx_lcore_id >= RTE_MAX_LCORE) 2890 rte_exit(EXIT_FAILURE, 2891 "Not enough cores\n"); 2892 } 2893 } else if (!options.single_lcore) { 2894 /* get the lcore_id for this port */ 2895 while (rte_lcore_is_enabled(rx_lcore_id) == 0 || 2896 lcore_queue_conf[rx_lcore_id].nb_crypto_devs == 2897 options.nb_ports_per_lcore) { 2898 rx_lcore_id++; 2899 if (rx_lcore_id >= RTE_MAX_LCORE) 2900 rte_exit(EXIT_FAILURE, 2901 "Not enough cores\n"); 2902 } 2903 } 2904 2905 /* Assigned a new logical core in the loop above. */ 2906 if (qconf != &lcore_queue_conf[rx_lcore_id]) 2907 qconf = &lcore_queue_conf[rx_lcore_id]; 2908 2909 qconf->cryptodev_list[qconf->nb_crypto_devs] = cdev_id; 2910 qconf->nb_crypto_devs++; 2911 2912 enabled_cdevcount--; 2913 2914 printf("Lcore %u: cryptodev %u\n", rx_lcore_id, 2915 (unsigned)cdev_id); 2916 } 2917 2918 /* launch per-lcore init on every lcore */ 2919 rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, (void *)&options, 2920 CALL_MAIN); 2921 RTE_LCORE_FOREACH_WORKER(lcore_id) { 2922 if (rte_eal_wait_lcore(lcore_id) < 0) 2923 return -1; 2924 } 2925 2926 /* clean up the EAL */ 2927 rte_eal_cleanup(); 2928 2929 return 0; 2930 } 2931