1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2015-2016 Intel Corporation 3 */ 4 5 #include <time.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <inttypes.h> 11 #include <sys/types.h> 12 #include <sys/queue.h> 13 #include <netinet/in.h> 14 #include <setjmp.h> 15 #include <stdarg.h> 16 #include <ctype.h> 17 #include <errno.h> 18 #include <getopt.h> 19 #include <fcntl.h> 20 #include <unistd.h> 21 22 #include <rte_string_fns.h> 23 #include <rte_branch_prediction.h> 24 #include <rte_common.h> 25 #include <rte_cryptodev.h> 26 #include <rte_cycles.h> 27 #include <rte_debug.h> 28 #include <rte_eal.h> 29 #include <rte_ether.h> 30 #include <rte_ethdev.h> 31 #include <rte_interrupts.h> 32 #include <rte_ip.h> 33 #include <rte_launch.h> 34 #include <rte_lcore.h> 35 #include <rte_log.h> 36 #include <rte_malloc.h> 37 #include <rte_mbuf.h> 38 #include <rte_memcpy.h> 39 #include <rte_memory.h> 40 #include <rte_mempool.h> 41 #include <rte_per_lcore.h> 42 #include <rte_prefetch.h> 43 #include <rte_random.h> 44 #include <rte_hexdump.h> 45 #ifdef RTE_CRYPTO_SCHEDULER 46 #include <rte_cryptodev_scheduler.h> 47 #endif 48 49 enum cdev_type { 50 CDEV_TYPE_ANY, 51 CDEV_TYPE_HW, 52 CDEV_TYPE_SW 53 }; 54 55 #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1 56 57 #define NB_MBUF 8192 58 59 #define MAX_STR_LEN 32 60 #define MAX_KEY_SIZE 128 61 #define MAX_IV_SIZE 16 62 #define MAX_AAD_SIZE 65535 63 #define MAX_PKT_BURST 32 64 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 65 #define SESSION_POOL_CACHE_SIZE 0 66 67 #define MAXIMUM_IV_LENGTH 16 68 #define IV_OFFSET (sizeof(struct rte_crypto_op) + \ 69 sizeof(struct rte_crypto_sym_op)) 70 71 /* 72 * Configurable number of RX/TX ring descriptors 73 */ 74 #define RTE_TEST_RX_DESC_DEFAULT 1024 75 #define RTE_TEST_TX_DESC_DEFAULT 1024 76 77 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; 78 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; 79 80 /* ethernet addresses of ports */ 81 static struct rte_ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS]; 82 83 /* mask of enabled ports */ 84 static uint64_t l2fwd_enabled_port_mask; 85 static uint64_t l2fwd_enabled_crypto_mask; 86 87 /* list of enabled ports */ 88 static uint16_t l2fwd_dst_ports[RTE_MAX_ETHPORTS]; 89 90 91 struct pkt_buffer { 92 unsigned len; 93 struct rte_mbuf *buffer[MAX_PKT_BURST]; 94 }; 95 96 struct op_buffer { 97 unsigned len; 98 struct rte_crypto_op *buffer[MAX_PKT_BURST]; 99 }; 100 101 #define MAX_RX_QUEUE_PER_LCORE 16 102 #define MAX_TX_QUEUE_PER_PORT 16 103 104 enum l2fwd_crypto_xform_chain { 105 L2FWD_CRYPTO_CIPHER_HASH, 106 L2FWD_CRYPTO_HASH_CIPHER, 107 L2FWD_CRYPTO_CIPHER_ONLY, 108 L2FWD_CRYPTO_HASH_ONLY, 109 L2FWD_CRYPTO_AEAD 110 }; 111 112 struct l2fwd_key { 113 uint8_t *data; 114 uint32_t length; 115 rte_iova_t phys_addr; 116 }; 117 118 struct l2fwd_iv { 119 uint8_t *data; 120 uint16_t length; 121 }; 122 123 /** l2fwd crypto application command line options */ 124 struct l2fwd_crypto_options { 125 unsigned portmask; 126 unsigned nb_ports_per_lcore; 127 unsigned refresh_period; 128 unsigned single_lcore:1; 129 130 enum cdev_type type; 131 unsigned sessionless:1; 132 133 enum l2fwd_crypto_xform_chain xform_chain; 134 135 struct rte_crypto_sym_xform cipher_xform; 136 unsigned ckey_param; 137 int ckey_random_size; 138 uint8_t cipher_key[MAX_KEY_SIZE]; 139 140 struct l2fwd_iv cipher_iv; 141 unsigned int cipher_iv_param; 142 int cipher_iv_random_size; 143 144 struct rte_crypto_sym_xform auth_xform; 145 uint8_t akey_param; 146 int akey_random_size; 147 uint8_t auth_key[MAX_KEY_SIZE]; 148 149 struct l2fwd_iv auth_iv; 150 unsigned int auth_iv_param; 151 int auth_iv_random_size; 152 153 struct rte_crypto_sym_xform aead_xform; 154 unsigned int aead_key_param; 155 int aead_key_random_size; 156 uint8_t aead_key[MAX_KEY_SIZE]; 157 158 struct l2fwd_iv aead_iv; 159 unsigned int aead_iv_param; 160 int aead_iv_random_size; 161 162 struct l2fwd_key aad; 163 unsigned aad_param; 164 int aad_random_size; 165 166 int digest_size; 167 168 uint16_t block_size; 169 char string_type[MAX_STR_LEN]; 170 171 uint64_t cryptodev_mask; 172 173 unsigned int mac_updating; 174 }; 175 176 /** l2fwd crypto lcore params */ 177 struct l2fwd_crypto_params { 178 uint8_t dev_id; 179 uint8_t qp_id; 180 181 unsigned digest_length; 182 unsigned block_size; 183 184 uint32_t cipher_dataunit_len; 185 186 struct l2fwd_iv cipher_iv; 187 struct l2fwd_iv auth_iv; 188 struct l2fwd_iv aead_iv; 189 struct l2fwd_key aad; 190 struct rte_cryptodev_sym_session *session; 191 192 uint8_t do_cipher; 193 uint8_t do_hash; 194 uint8_t do_aead; 195 uint8_t hash_verify; 196 197 enum rte_crypto_cipher_algorithm cipher_algo; 198 enum rte_crypto_auth_algorithm auth_algo; 199 enum rte_crypto_aead_algorithm aead_algo; 200 }; 201 202 /** lcore configuration */ 203 struct lcore_queue_conf { 204 unsigned nb_rx_ports; 205 uint16_t rx_port_list[MAX_RX_QUEUE_PER_LCORE]; 206 207 unsigned nb_crypto_devs; 208 unsigned cryptodev_list[MAX_RX_QUEUE_PER_LCORE]; 209 210 struct op_buffer op_buf[RTE_CRYPTO_MAX_DEVS]; 211 struct pkt_buffer pkt_buf[RTE_MAX_ETHPORTS]; 212 } __rte_cache_aligned; 213 214 struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE]; 215 216 static struct rte_eth_conf port_conf = { 217 .rxmode = { 218 .mq_mode = RTE_ETH_MQ_RX_NONE, 219 .split_hdr_size = 0, 220 }, 221 .txmode = { 222 .mq_mode = RTE_ETH_MQ_TX_NONE, 223 }, 224 }; 225 226 struct rte_mempool *l2fwd_pktmbuf_pool; 227 struct rte_mempool *l2fwd_crypto_op_pool; 228 static struct { 229 struct rte_mempool *sess_mp; 230 struct rte_mempool *priv_mp; 231 } session_pool_socket[RTE_MAX_NUMA_NODES]; 232 233 /* Per-port statistics struct */ 234 struct l2fwd_port_statistics { 235 uint64_t tx; 236 uint64_t rx; 237 238 uint64_t crypto_enqueued; 239 uint64_t crypto_dequeued; 240 241 uint64_t dropped; 242 } __rte_cache_aligned; 243 244 struct l2fwd_crypto_statistics { 245 uint64_t enqueued; 246 uint64_t dequeued; 247 248 uint64_t errors; 249 } __rte_cache_aligned; 250 251 struct l2fwd_port_statistics port_statistics[RTE_MAX_ETHPORTS]; 252 struct l2fwd_crypto_statistics crypto_statistics[RTE_CRYPTO_MAX_DEVS]; 253 254 /* A tsc-based timer responsible for triggering statistics printout */ 255 #define TIMER_MILLISECOND 2000000ULL /* around 1ms at 2 Ghz */ 256 #define MAX_TIMER_PERIOD 86400UL /* 1 day max */ 257 258 /* default period is 10 seconds */ 259 static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000; 260 261 /* Print out statistics on packets dropped */ 262 static void 263 print_stats(void) 264 { 265 uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; 266 uint64_t total_packets_enqueued, total_packets_dequeued, 267 total_packets_errors; 268 uint16_t portid; 269 uint64_t cdevid; 270 271 total_packets_dropped = 0; 272 total_packets_tx = 0; 273 total_packets_rx = 0; 274 total_packets_enqueued = 0; 275 total_packets_dequeued = 0; 276 total_packets_errors = 0; 277 278 const char clr[] = { 27, '[', '2', 'J', '\0' }; 279 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 280 281 /* Clear screen and move to top left */ 282 printf("%s%s", clr, topLeft); 283 284 printf("\nPort statistics ===================================="); 285 286 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 287 /* skip disabled ports */ 288 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) 289 continue; 290 printf("\nStatistics for port %u ------------------------------" 291 "\nPackets sent: %32"PRIu64 292 "\nPackets received: %28"PRIu64 293 "\nPackets dropped: %29"PRIu64, 294 portid, 295 port_statistics[portid].tx, 296 port_statistics[portid].rx, 297 port_statistics[portid].dropped); 298 299 total_packets_dropped += port_statistics[portid].dropped; 300 total_packets_tx += port_statistics[portid].tx; 301 total_packets_rx += port_statistics[portid].rx; 302 } 303 printf("\nCrypto statistics =================================="); 304 305 for (cdevid = 0; cdevid < RTE_CRYPTO_MAX_DEVS; cdevid++) { 306 /* skip disabled ports */ 307 if ((l2fwd_enabled_crypto_mask & (((uint64_t)1) << cdevid)) == 0) 308 continue; 309 printf("\nStatistics for cryptodev %"PRIu64 310 " -------------------------" 311 "\nPackets enqueued: %28"PRIu64 312 "\nPackets dequeued: %28"PRIu64 313 "\nPackets errors: %30"PRIu64, 314 cdevid, 315 crypto_statistics[cdevid].enqueued, 316 crypto_statistics[cdevid].dequeued, 317 crypto_statistics[cdevid].errors); 318 319 total_packets_enqueued += crypto_statistics[cdevid].enqueued; 320 total_packets_dequeued += crypto_statistics[cdevid].dequeued; 321 total_packets_errors += crypto_statistics[cdevid].errors; 322 } 323 printf("\nAggregate statistics ===============================" 324 "\nTotal packets received: %22"PRIu64 325 "\nTotal packets enqueued: %22"PRIu64 326 "\nTotal packets dequeued: %22"PRIu64 327 "\nTotal packets sent: %26"PRIu64 328 "\nTotal packets dropped: %23"PRIu64 329 "\nTotal packets crypto errors: %17"PRIu64, 330 total_packets_rx, 331 total_packets_enqueued, 332 total_packets_dequeued, 333 total_packets_tx, 334 total_packets_dropped, 335 total_packets_errors); 336 printf("\n====================================================\n"); 337 338 fflush(stdout); 339 } 340 341 /* l2fwd_crypto_send_burst 8< */ 342 static int 343 l2fwd_crypto_send_burst(struct lcore_queue_conf *qconf, unsigned n, 344 struct l2fwd_crypto_params *cparams) 345 { 346 struct rte_crypto_op **op_buffer; 347 unsigned ret; 348 349 op_buffer = (struct rte_crypto_op **) 350 qconf->op_buf[cparams->dev_id].buffer; 351 352 ret = rte_cryptodev_enqueue_burst(cparams->dev_id, 353 cparams->qp_id, op_buffer, (uint16_t) n); 354 355 crypto_statistics[cparams->dev_id].enqueued += ret; 356 if (unlikely(ret < n)) { 357 crypto_statistics[cparams->dev_id].errors += (n - ret); 358 do { 359 rte_pktmbuf_free(op_buffer[ret]->sym->m_src); 360 rte_crypto_op_free(op_buffer[ret]); 361 } while (++ret < n); 362 } 363 364 return 0; 365 } 366 /* >8 End of l2fwd_crypto_send_burst. */ 367 368 /* Crypto enqueue. 8< */ 369 static int 370 l2fwd_crypto_enqueue(struct rte_crypto_op *op, 371 struct l2fwd_crypto_params *cparams) 372 { 373 unsigned lcore_id, len; 374 struct lcore_queue_conf *qconf; 375 376 lcore_id = rte_lcore_id(); 377 378 qconf = &lcore_queue_conf[lcore_id]; 379 len = qconf->op_buf[cparams->dev_id].len; 380 qconf->op_buf[cparams->dev_id].buffer[len] = op; 381 len++; 382 383 /* enough ops to be sent */ 384 if (len == MAX_PKT_BURST) { 385 l2fwd_crypto_send_burst(qconf, MAX_PKT_BURST, cparams); 386 len = 0; 387 } 388 389 qconf->op_buf[cparams->dev_id].len = len; 390 return 0; 391 } 392 /* >8 End of crypto enqueue. */ 393 394 static int 395 l2fwd_simple_crypto_enqueue(struct rte_mbuf *m, 396 struct rte_crypto_op *op, 397 struct l2fwd_crypto_params *cparams) 398 { 399 struct rte_ether_hdr *eth_hdr; 400 struct rte_ipv4_hdr *ip_hdr; 401 402 uint32_t ipdata_offset, data_len; 403 uint32_t pad_len = 0; 404 char *padding; 405 406 eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); 407 408 if (eth_hdr->ether_type != rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) 409 return -1; 410 411 ipdata_offset = sizeof(struct rte_ether_hdr); 412 413 ip_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, char *) + 414 ipdata_offset); 415 416 ipdata_offset += (ip_hdr->version_ihl & RTE_IPV4_HDR_IHL_MASK) 417 * RTE_IPV4_IHL_MULTIPLIER; 418 419 420 /* Zero pad data to be crypto'd so it is block aligned */ 421 data_len = rte_pktmbuf_data_len(m) - ipdata_offset; 422 423 if ((cparams->do_hash || cparams->do_aead) && cparams->hash_verify) 424 data_len -= cparams->digest_length; 425 426 if (cparams->do_cipher) { 427 /* 428 * Following algorithms are block cipher algorithms, 429 * and might need padding 430 */ 431 switch (cparams->cipher_algo) { 432 case RTE_CRYPTO_CIPHER_AES_CBC: 433 case RTE_CRYPTO_CIPHER_AES_ECB: 434 case RTE_CRYPTO_CIPHER_DES_CBC: 435 case RTE_CRYPTO_CIPHER_3DES_CBC: 436 case RTE_CRYPTO_CIPHER_3DES_ECB: 437 if (data_len % cparams->block_size) 438 pad_len = cparams->block_size - 439 (data_len % cparams->block_size); 440 break; 441 case RTE_CRYPTO_CIPHER_AES_XTS: 442 if (cparams->cipher_dataunit_len != 0 && 443 (data_len % cparams->cipher_dataunit_len)) 444 pad_len = cparams->cipher_dataunit_len - 445 (data_len % cparams->cipher_dataunit_len); 446 break; 447 default: 448 pad_len = 0; 449 } 450 451 if (pad_len) { 452 padding = rte_pktmbuf_append(m, pad_len); 453 if (unlikely(!padding)) 454 return -1; 455 456 data_len += pad_len; 457 memset(padding, 0, pad_len); 458 } 459 } 460 461 /* Set crypto operation data parameters */ 462 rte_crypto_op_attach_sym_session(op, cparams->session); 463 464 if (cparams->do_hash) { 465 if (cparams->auth_iv.length) { 466 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, 467 uint8_t *, 468 IV_OFFSET + 469 cparams->cipher_iv.length); 470 /* 471 * Copy IV at the end of the crypto operation, 472 * after the cipher IV, if added 473 */ 474 rte_memcpy(iv_ptr, cparams->auth_iv.data, 475 cparams->auth_iv.length); 476 } 477 if (!cparams->hash_verify) { 478 /* Append space for digest to end of packet */ 479 op->sym->auth.digest.data = (uint8_t *)rte_pktmbuf_append(m, 480 cparams->digest_length); 481 } else { 482 op->sym->auth.digest.data = rte_pktmbuf_mtod(m, 483 uint8_t *) + ipdata_offset + data_len; 484 } 485 486 op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m, 487 rte_pktmbuf_pkt_len(m) - cparams->digest_length); 488 489 /* For wireless algorithms, offset/length must be in bits */ 490 if (cparams->auth_algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2 || 491 cparams->auth_algo == RTE_CRYPTO_AUTH_KASUMI_F9 || 492 cparams->auth_algo == RTE_CRYPTO_AUTH_ZUC_EIA3) { 493 op->sym->auth.data.offset = ipdata_offset << 3; 494 op->sym->auth.data.length = data_len << 3; 495 } else { 496 op->sym->auth.data.offset = ipdata_offset; 497 op->sym->auth.data.length = data_len; 498 } 499 } 500 501 if (cparams->do_cipher) { 502 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 503 IV_OFFSET); 504 /* Copy IV at the end of the crypto operation */ 505 rte_memcpy(iv_ptr, cparams->cipher_iv.data, 506 cparams->cipher_iv.length); 507 508 /* For wireless algorithms, offset/length must be in bits */ 509 if (cparams->cipher_algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 || 510 cparams->cipher_algo == RTE_CRYPTO_CIPHER_KASUMI_F8 || 511 cparams->cipher_algo == RTE_CRYPTO_CIPHER_ZUC_EEA3) { 512 op->sym->cipher.data.offset = ipdata_offset << 3; 513 op->sym->cipher.data.length = data_len << 3; 514 } else { 515 op->sym->cipher.data.offset = ipdata_offset; 516 op->sym->cipher.data.length = data_len; 517 } 518 } 519 520 if (cparams->do_aead) { 521 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 522 IV_OFFSET); 523 /* Copy IV at the end of the crypto operation */ 524 /* 525 * If doing AES-CCM, nonce is copied one byte 526 * after the start of IV field 527 */ 528 if (cparams->aead_algo == RTE_CRYPTO_AEAD_AES_CCM) 529 rte_memcpy(iv_ptr + 1, cparams->aead_iv.data, 530 cparams->aead_iv.length); 531 else 532 rte_memcpy(iv_ptr, cparams->aead_iv.data, 533 cparams->aead_iv.length); 534 535 op->sym->aead.data.offset = ipdata_offset; 536 op->sym->aead.data.length = data_len; 537 538 if (!cparams->hash_verify) { 539 /* Append space for digest to end of packet */ 540 op->sym->aead.digest.data = (uint8_t *)rte_pktmbuf_append(m, 541 cparams->digest_length); 542 } else { 543 op->sym->aead.digest.data = rte_pktmbuf_mtod(m, 544 uint8_t *) + ipdata_offset + data_len; 545 } 546 547 op->sym->aead.digest.phys_addr = rte_pktmbuf_iova_offset(m, 548 rte_pktmbuf_pkt_len(m) - cparams->digest_length); 549 550 if (cparams->aad.length) { 551 op->sym->aead.aad.data = cparams->aad.data; 552 op->sym->aead.aad.phys_addr = cparams->aad.phys_addr; 553 } 554 } 555 556 op->sym->m_src = m; 557 558 return l2fwd_crypto_enqueue(op, cparams); 559 } 560 561 562 /* Send the burst of packets on an output interface */ 563 static int 564 l2fwd_send_burst(struct lcore_queue_conf *qconf, unsigned n, 565 uint16_t port) 566 { 567 struct rte_mbuf **pkt_buffer; 568 unsigned ret; 569 570 pkt_buffer = (struct rte_mbuf **)qconf->pkt_buf[port].buffer; 571 572 ret = rte_eth_tx_burst(port, 0, pkt_buffer, (uint16_t)n); 573 port_statistics[port].tx += ret; 574 if (unlikely(ret < n)) { 575 port_statistics[port].dropped += (n - ret); 576 do { 577 rte_pktmbuf_free(pkt_buffer[ret]); 578 } while (++ret < n); 579 } 580 581 return 0; 582 } 583 584 /* Enqueue packets for TX and prepare them to be sent. 8< */ 585 static int 586 l2fwd_send_packet(struct rte_mbuf *m, uint16_t port) 587 { 588 unsigned lcore_id, len; 589 struct lcore_queue_conf *qconf; 590 591 lcore_id = rte_lcore_id(); 592 593 qconf = &lcore_queue_conf[lcore_id]; 594 len = qconf->pkt_buf[port].len; 595 qconf->pkt_buf[port].buffer[len] = m; 596 len++; 597 598 /* enough pkts to be sent */ 599 if (unlikely(len == MAX_PKT_BURST)) { 600 l2fwd_send_burst(qconf, MAX_PKT_BURST, port); 601 len = 0; 602 } 603 604 qconf->pkt_buf[port].len = len; 605 return 0; 606 } 607 /* >8 End of Enqueuing packets for TX. */ 608 609 static void 610 l2fwd_mac_updating(struct rte_mbuf *m, uint16_t dest_portid) 611 { 612 struct rte_ether_hdr *eth; 613 void *tmp; 614 615 eth = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); 616 617 /* 02:00:00:00:00:xx */ 618 tmp = ð->dst_addr.addr_bytes[0]; 619 *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dest_portid << 40); 620 621 /* src addr */ 622 rte_ether_addr_copy(&l2fwd_ports_eth_addr[dest_portid], ð->src_addr); 623 } 624 625 static void 626 l2fwd_simple_forward(struct rte_mbuf *m, uint16_t portid, 627 struct l2fwd_crypto_options *options) 628 { 629 uint16_t dst_port; 630 uint32_t pad_len; 631 struct rte_ipv4_hdr *ip_hdr; 632 uint32_t ipdata_offset = sizeof(struct rte_ether_hdr); 633 634 ip_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, char *) + 635 ipdata_offset); 636 dst_port = l2fwd_dst_ports[portid]; 637 638 if (options->mac_updating) 639 l2fwd_mac_updating(m, dst_port); 640 641 if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_VERIFY) 642 rte_pktmbuf_trim(m, options->auth_xform.auth.digest_length); 643 644 if (options->cipher_xform.cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { 645 pad_len = m->pkt_len - rte_be_to_cpu_16(ip_hdr->total_length) - 646 ipdata_offset; 647 rte_pktmbuf_trim(m, pad_len); 648 } 649 650 l2fwd_send_packet(m, dst_port); 651 } 652 653 /** Generate random key */ 654 static void 655 generate_random_key(uint8_t *key, unsigned length) 656 { 657 int fd; 658 int ret; 659 660 fd = open("/dev/urandom", O_RDONLY); 661 if (fd < 0) 662 rte_exit(EXIT_FAILURE, "Failed to generate random key\n"); 663 664 ret = read(fd, key, length); 665 close(fd); 666 667 if (ret != (signed)length) 668 rte_exit(EXIT_FAILURE, "Failed to generate random key\n"); 669 } 670 671 /* Session is created and is later attached to the crypto operation. 8< */ 672 static struct rte_cryptodev_sym_session * 673 initialize_crypto_session(struct l2fwd_crypto_options *options, uint8_t cdev_id) 674 { 675 struct rte_crypto_sym_xform *first_xform; 676 struct rte_cryptodev_sym_session *session; 677 int retval = rte_cryptodev_socket_id(cdev_id); 678 679 if (retval < 0) 680 return NULL; 681 682 uint8_t socket_id = (uint8_t) retval; 683 684 if (options->xform_chain == L2FWD_CRYPTO_AEAD) { 685 first_xform = &options->aead_xform; 686 } else if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH) { 687 first_xform = &options->cipher_xform; 688 first_xform->next = &options->auth_xform; 689 } else if (options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER) { 690 first_xform = &options->auth_xform; 691 first_xform->next = &options->cipher_xform; 692 } else if (options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { 693 first_xform = &options->cipher_xform; 694 } else { 695 first_xform = &options->auth_xform; 696 } 697 698 session = rte_cryptodev_sym_session_create( 699 session_pool_socket[socket_id].sess_mp); 700 if (session == NULL) 701 return NULL; 702 703 if (rte_cryptodev_sym_session_init(cdev_id, session, 704 first_xform, 705 session_pool_socket[socket_id].priv_mp) < 0) 706 return NULL; 707 708 return session; 709 } 710 /* >8 End of creation of session. */ 711 712 static void 713 l2fwd_crypto_options_print(struct l2fwd_crypto_options *options); 714 715 /* main processing loop */ 716 static void 717 l2fwd_main_loop(struct l2fwd_crypto_options *options) 718 { 719 struct rte_mbuf *m, *pkts_burst[MAX_PKT_BURST]; 720 struct rte_crypto_op *ops_burst[MAX_PKT_BURST]; 721 722 unsigned lcore_id = rte_lcore_id(); 723 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 724 unsigned int i, j, nb_rx, len; 725 uint16_t portid; 726 struct lcore_queue_conf *qconf = &lcore_queue_conf[lcore_id]; 727 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 728 US_PER_S * BURST_TX_DRAIN_US; 729 struct l2fwd_crypto_params *cparams; 730 struct l2fwd_crypto_params port_cparams[qconf->nb_crypto_devs]; 731 struct rte_cryptodev_sym_session *session; 732 733 if (qconf->nb_rx_ports == 0) { 734 RTE_LOG(INFO, L2FWD, "lcore %u has nothing to do\n", lcore_id); 735 return; 736 } 737 738 RTE_LOG(INFO, L2FWD, "entering main loop on lcore %u\n", lcore_id); 739 740 for (i = 0; i < qconf->nb_rx_ports; i++) { 741 742 portid = qconf->rx_port_list[i]; 743 RTE_LOG(INFO, L2FWD, " -- lcoreid=%u portid=%u\n", lcore_id, 744 portid); 745 } 746 747 for (i = 0; i < qconf->nb_crypto_devs; i++) { 748 port_cparams[i].do_cipher = 0; 749 port_cparams[i].do_hash = 0; 750 port_cparams[i].do_aead = 0; 751 752 switch (options->xform_chain) { 753 case L2FWD_CRYPTO_AEAD: 754 port_cparams[i].do_aead = 1; 755 break; 756 case L2FWD_CRYPTO_CIPHER_HASH: 757 case L2FWD_CRYPTO_HASH_CIPHER: 758 port_cparams[i].do_cipher = 1; 759 port_cparams[i].do_hash = 1; 760 break; 761 case L2FWD_CRYPTO_HASH_ONLY: 762 port_cparams[i].do_hash = 1; 763 break; 764 case L2FWD_CRYPTO_CIPHER_ONLY: 765 port_cparams[i].do_cipher = 1; 766 break; 767 } 768 769 port_cparams[i].dev_id = qconf->cryptodev_list[i]; 770 port_cparams[i].qp_id = 0; 771 772 port_cparams[i].block_size = options->block_size; 773 774 if (port_cparams[i].do_hash) { 775 port_cparams[i].auth_iv.data = options->auth_iv.data; 776 port_cparams[i].auth_iv.length = options->auth_iv.length; 777 if (!options->auth_iv_param) 778 generate_random_key(port_cparams[i].auth_iv.data, 779 port_cparams[i].auth_iv.length); 780 if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_VERIFY) 781 port_cparams[i].hash_verify = 1; 782 else 783 port_cparams[i].hash_verify = 0; 784 785 port_cparams[i].auth_algo = options->auth_xform.auth.algo; 786 port_cparams[i].digest_length = 787 options->auth_xform.auth.digest_length; 788 /* Set IV parameters */ 789 if (options->auth_iv.length) { 790 options->auth_xform.auth.iv.offset = 791 IV_OFFSET + options->cipher_iv.length; 792 options->auth_xform.auth.iv.length = 793 options->auth_iv.length; 794 } 795 } 796 797 if (port_cparams[i].do_aead) { 798 port_cparams[i].aead_iv.data = options->aead_iv.data; 799 port_cparams[i].aead_iv.length = options->aead_iv.length; 800 if (!options->aead_iv_param) 801 generate_random_key(port_cparams[i].aead_iv.data, 802 port_cparams[i].aead_iv.length); 803 port_cparams[i].aead_algo = options->aead_xform.aead.algo; 804 port_cparams[i].digest_length = 805 options->aead_xform.aead.digest_length; 806 if (options->aead_xform.aead.aad_length) { 807 port_cparams[i].aad.data = options->aad.data; 808 port_cparams[i].aad.phys_addr = options->aad.phys_addr; 809 port_cparams[i].aad.length = options->aad.length; 810 if (!options->aad_param) 811 generate_random_key(port_cparams[i].aad.data, 812 port_cparams[i].aad.length); 813 /* 814 * If doing AES-CCM, first 18 bytes has to be reserved, 815 * and actual AAD should start from byte 18 816 */ 817 if (port_cparams[i].aead_algo == RTE_CRYPTO_AEAD_AES_CCM) 818 memmove(port_cparams[i].aad.data + 18, 819 port_cparams[i].aad.data, 820 port_cparams[i].aad.length); 821 822 } else 823 port_cparams[i].aad.length = 0; 824 825 if (options->aead_xform.aead.op == RTE_CRYPTO_AEAD_OP_DECRYPT) 826 port_cparams[i].hash_verify = 1; 827 else 828 port_cparams[i].hash_verify = 0; 829 830 /* Set IV parameters */ 831 options->aead_xform.aead.iv.offset = IV_OFFSET; 832 options->aead_xform.aead.iv.length = options->aead_iv.length; 833 } 834 835 if (port_cparams[i].do_cipher) { 836 port_cparams[i].cipher_iv.data = options->cipher_iv.data; 837 port_cparams[i].cipher_iv.length = options->cipher_iv.length; 838 if (!options->cipher_iv_param) 839 generate_random_key(port_cparams[i].cipher_iv.data, 840 port_cparams[i].cipher_iv.length); 841 842 port_cparams[i].cipher_algo = options->cipher_xform.cipher.algo; 843 port_cparams[i].cipher_dataunit_len = 844 options->cipher_xform.cipher.dataunit_len; 845 /* Set IV parameters */ 846 options->cipher_xform.cipher.iv.offset = IV_OFFSET; 847 options->cipher_xform.cipher.iv.length = 848 options->cipher_iv.length; 849 } 850 851 session = initialize_crypto_session(options, 852 port_cparams[i].dev_id); 853 if (session == NULL) 854 rte_exit(EXIT_FAILURE, "Failed to initialize crypto session\n"); 855 856 port_cparams[i].session = session; 857 858 RTE_LOG(INFO, L2FWD, " -- lcoreid=%u cryptoid=%u\n", lcore_id, 859 port_cparams[i].dev_id); 860 } 861 862 l2fwd_crypto_options_print(options); 863 864 /* 865 * Initialize previous tsc timestamp before the loop, 866 * to avoid showing the port statistics immediately, 867 * so user can see the crypto information. 868 */ 869 prev_tsc = rte_rdtsc(); 870 while (1) { 871 872 cur_tsc = rte_rdtsc(); 873 874 /* 875 * Crypto device/TX burst queue drain 876 */ 877 diff_tsc = cur_tsc - prev_tsc; 878 if (unlikely(diff_tsc > drain_tsc)) { 879 /* Enqueue all crypto ops remaining in buffers */ 880 for (i = 0; i < qconf->nb_crypto_devs; i++) { 881 cparams = &port_cparams[i]; 882 len = qconf->op_buf[cparams->dev_id].len; 883 l2fwd_crypto_send_burst(qconf, len, cparams); 884 qconf->op_buf[cparams->dev_id].len = 0; 885 } 886 /* Transmit all packets remaining in buffers */ 887 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 888 if (qconf->pkt_buf[portid].len == 0) 889 continue; 890 l2fwd_send_burst(&lcore_queue_conf[lcore_id], 891 qconf->pkt_buf[portid].len, 892 portid); 893 qconf->pkt_buf[portid].len = 0; 894 } 895 896 /* if timer is enabled */ 897 if (timer_period > 0) { 898 899 /* advance the timer */ 900 timer_tsc += diff_tsc; 901 902 /* if timer has reached its timeout */ 903 if (unlikely(timer_tsc >= 904 (uint64_t)timer_period)) { 905 906 /* do this only on main core */ 907 if (lcore_id == rte_get_main_lcore() 908 && options->refresh_period) { 909 print_stats(); 910 timer_tsc = 0; 911 } 912 } 913 } 914 915 prev_tsc = cur_tsc; 916 } 917 918 /* 919 * Read packet from RX queues 920 */ 921 for (i = 0; i < qconf->nb_rx_ports; i++) { 922 portid = qconf->rx_port_list[i]; 923 924 cparams = &port_cparams[i]; 925 926 nb_rx = rte_eth_rx_burst(portid, 0, 927 pkts_burst, MAX_PKT_BURST); 928 929 port_statistics[portid].rx += nb_rx; 930 931 /* Allocate and fillcrypto operations. 8< */ 932 if (nb_rx) { 933 /* 934 * If we can't allocate a crypto_ops, then drop 935 * the rest of the burst and dequeue and 936 * process the packets to free offload structs 937 */ 938 if (rte_crypto_op_bulk_alloc( 939 l2fwd_crypto_op_pool, 940 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 941 ops_burst, nb_rx) != 942 nb_rx) { 943 for (j = 0; j < nb_rx; j++) 944 rte_pktmbuf_free(pkts_burst[j]); 945 946 nb_rx = 0; 947 } 948 /* >8 End of crypto operation allocated and filled. */ 949 950 /* Enqueue packets from Crypto device*/ 951 for (j = 0; j < nb_rx; j++) { 952 m = pkts_burst[j]; 953 954 l2fwd_simple_crypto_enqueue(m, 955 ops_burst[j], cparams); 956 } 957 } 958 959 /* Dequeue packets from Crypto device. 8< */ 960 do { 961 nb_rx = rte_cryptodev_dequeue_burst( 962 cparams->dev_id, cparams->qp_id, 963 ops_burst, MAX_PKT_BURST); 964 965 crypto_statistics[cparams->dev_id].dequeued += 966 nb_rx; 967 968 /* Forward crypto'd packets */ 969 for (j = 0; j < nb_rx; j++) { 970 m = ops_burst[j]->sym->m_src; 971 972 rte_crypto_op_free(ops_burst[j]); 973 l2fwd_simple_forward(m, portid, 974 options); 975 } 976 } while (nb_rx == MAX_PKT_BURST); 977 /* >8 End of dequeue packets from crypto device. */ 978 } 979 } 980 } 981 982 static int 983 l2fwd_launch_one_lcore(void *arg) 984 { 985 l2fwd_main_loop((struct l2fwd_crypto_options *)arg); 986 return 0; 987 } 988 989 /* Display command line arguments usage */ 990 static void 991 l2fwd_crypto_usage(const char *prgname) 992 { 993 printf("%s [EAL options] --\n" 994 " -p PORTMASK: hexadecimal bitmask of ports to configure\n" 995 " -q NQ: number of queue (=ports) per lcore (default is 1)\n" 996 " -s manage all ports from single lcore\n" 997 " -T PERIOD: statistics will be refreshed each PERIOD seconds" 998 " (0 to disable, 10 default, 86400 maximum)\n" 999 1000 " --cdev_type HW / SW / ANY\n" 1001 " --chain HASH_CIPHER / CIPHER_HASH / CIPHER_ONLY /" 1002 " HASH_ONLY / AEAD\n" 1003 1004 " --cipher_algo ALGO\n" 1005 " --cipher_op ENCRYPT / DECRYPT\n" 1006 " --cipher_key KEY (bytes separated with \":\")\n" 1007 " --cipher_key_random_size SIZE: size of cipher key when generated randomly\n" 1008 " --cipher_iv IV (bytes separated with \":\")\n" 1009 " --cipher_iv_random_size SIZE: size of cipher IV when generated randomly\n" 1010 " --cipher_dataunit_len SIZE: length of the algorithm data-unit\n" 1011 1012 " --auth_algo ALGO\n" 1013 " --auth_op GENERATE / VERIFY\n" 1014 " --auth_key KEY (bytes separated with \":\")\n" 1015 " --auth_key_random_size SIZE: size of auth key when generated randomly\n" 1016 " --auth_iv IV (bytes separated with \":\")\n" 1017 " --auth_iv_random_size SIZE: size of auth IV when generated randomly\n" 1018 1019 " --aead_algo ALGO\n" 1020 " --aead_op ENCRYPT / DECRYPT\n" 1021 " --aead_key KEY (bytes separated with \":\")\n" 1022 " --aead_key_random_size SIZE: size of AEAD key when generated randomly\n" 1023 " --aead_iv IV (bytes separated with \":\")\n" 1024 " --aead_iv_random_size SIZE: size of AEAD IV when generated randomly\n" 1025 " --aad AAD (bytes separated with \":\")\n" 1026 " --aad_random_size SIZE: size of AAD when generated randomly\n" 1027 1028 " --digest_size SIZE: size of digest to be generated/verified\n" 1029 1030 " --sessionless\n" 1031 " --cryptodev_mask MASK: hexadecimal bitmask of crypto devices to configure\n" 1032 1033 " --[no-]mac-updating: Enable or disable MAC addresses updating (enabled by default)\n" 1034 " When enabled:\n" 1035 " - The source MAC address is replaced by the TX port MAC address\n" 1036 " - The destination MAC address is replaced by 02:00:00:00:00:TX_PORT_ID\n", 1037 prgname); 1038 } 1039 1040 /** Parse crypto device type command line argument */ 1041 static int 1042 parse_cryptodev_type(enum cdev_type *type, char *optarg) 1043 { 1044 if (strcmp("HW", optarg) == 0) { 1045 *type = CDEV_TYPE_HW; 1046 return 0; 1047 } else if (strcmp("SW", optarg) == 0) { 1048 *type = CDEV_TYPE_SW; 1049 return 0; 1050 } else if (strcmp("ANY", optarg) == 0) { 1051 *type = CDEV_TYPE_ANY; 1052 return 0; 1053 } 1054 1055 return -1; 1056 } 1057 1058 /** Parse crypto chain xform command line argument */ 1059 static int 1060 parse_crypto_opt_chain(struct l2fwd_crypto_options *options, char *optarg) 1061 { 1062 if (strcmp("CIPHER_HASH", optarg) == 0) { 1063 options->xform_chain = L2FWD_CRYPTO_CIPHER_HASH; 1064 return 0; 1065 } else if (strcmp("HASH_CIPHER", optarg) == 0) { 1066 options->xform_chain = L2FWD_CRYPTO_HASH_CIPHER; 1067 return 0; 1068 } else if (strcmp("CIPHER_ONLY", optarg) == 0) { 1069 options->xform_chain = L2FWD_CRYPTO_CIPHER_ONLY; 1070 return 0; 1071 } else if (strcmp("HASH_ONLY", optarg) == 0) { 1072 options->xform_chain = L2FWD_CRYPTO_HASH_ONLY; 1073 return 0; 1074 } else if (strcmp("AEAD", optarg) == 0) { 1075 options->xform_chain = L2FWD_CRYPTO_AEAD; 1076 return 0; 1077 } 1078 1079 return -1; 1080 } 1081 1082 /** Parse crypto cipher algo option command line argument */ 1083 static int 1084 parse_cipher_algo(enum rte_crypto_cipher_algorithm *algo, char *optarg) 1085 { 1086 1087 if (rte_cryptodev_get_cipher_algo_enum(algo, optarg) < 0) { 1088 RTE_LOG(ERR, USER1, "Cipher algorithm specified " 1089 "not supported!\n"); 1090 return -1; 1091 } 1092 1093 return 0; 1094 } 1095 1096 /** Parse crypto cipher operation command line argument */ 1097 static int 1098 parse_cipher_op(enum rte_crypto_cipher_operation *op, char *optarg) 1099 { 1100 if (strcmp("ENCRYPT", optarg) == 0) { 1101 *op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 1102 return 0; 1103 } else if (strcmp("DECRYPT", optarg) == 0) { 1104 *op = RTE_CRYPTO_CIPHER_OP_DECRYPT; 1105 return 0; 1106 } 1107 1108 printf("Cipher operation not supported!\n"); 1109 return -1; 1110 } 1111 1112 /** Parse bytes from command line argument */ 1113 static int 1114 parse_bytes(uint8_t *data, char *input_arg, uint16_t max_size) 1115 { 1116 unsigned byte_count; 1117 char *token; 1118 1119 errno = 0; 1120 for (byte_count = 0, token = strtok(input_arg, ":"); 1121 (byte_count < max_size) && (token != NULL); 1122 token = strtok(NULL, ":")) { 1123 1124 int number = (int)strtol(token, NULL, 16); 1125 1126 if (errno == EINVAL || errno == ERANGE || number > 0xFF) 1127 return -1; 1128 1129 data[byte_count++] = (uint8_t)number; 1130 } 1131 1132 return byte_count; 1133 } 1134 1135 /** Parse size param*/ 1136 static int 1137 parse_size(int *size, const char *q_arg) 1138 { 1139 char *end = NULL; 1140 unsigned long n; 1141 1142 /* parse hexadecimal string */ 1143 n = strtoul(q_arg, &end, 10); 1144 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) 1145 n = 0; 1146 1147 if (n == 0) { 1148 printf("invalid size\n"); 1149 return -1; 1150 } 1151 1152 *size = n; 1153 return 0; 1154 } 1155 1156 /** Parse crypto cipher operation command line argument */ 1157 static int 1158 parse_auth_algo(enum rte_crypto_auth_algorithm *algo, char *optarg) 1159 { 1160 if (rte_cryptodev_get_auth_algo_enum(algo, optarg) < 0) { 1161 RTE_LOG(ERR, USER1, "Authentication algorithm specified " 1162 "not supported!\n"); 1163 return -1; 1164 } 1165 1166 return 0; 1167 } 1168 1169 static int 1170 parse_auth_op(enum rte_crypto_auth_operation *op, char *optarg) 1171 { 1172 if (strcmp("VERIFY", optarg) == 0) { 1173 *op = RTE_CRYPTO_AUTH_OP_VERIFY; 1174 return 0; 1175 } else if (strcmp("GENERATE", optarg) == 0) { 1176 *op = RTE_CRYPTO_AUTH_OP_GENERATE; 1177 return 0; 1178 } 1179 1180 printf("Authentication operation specified not supported!\n"); 1181 return -1; 1182 } 1183 1184 static int 1185 parse_aead_algo(enum rte_crypto_aead_algorithm *algo, char *optarg) 1186 { 1187 if (rte_cryptodev_get_aead_algo_enum(algo, optarg) < 0) { 1188 RTE_LOG(ERR, USER1, "AEAD algorithm specified " 1189 "not supported!\n"); 1190 return -1; 1191 } 1192 1193 return 0; 1194 } 1195 1196 static int 1197 parse_aead_op(enum rte_crypto_aead_operation *op, char *optarg) 1198 { 1199 if (strcmp("ENCRYPT", optarg) == 0) { 1200 *op = RTE_CRYPTO_AEAD_OP_ENCRYPT; 1201 return 0; 1202 } else if (strcmp("DECRYPT", optarg) == 0) { 1203 *op = RTE_CRYPTO_AEAD_OP_DECRYPT; 1204 return 0; 1205 } 1206 1207 printf("AEAD operation specified not supported!\n"); 1208 return -1; 1209 } 1210 static int 1211 parse_cryptodev_mask(struct l2fwd_crypto_options *options, 1212 const char *q_arg) 1213 { 1214 char *end = NULL; 1215 uint64_t pm; 1216 1217 /* parse hexadecimal string */ 1218 pm = strtoul(q_arg, &end, 16); 1219 if ((pm == '\0') || (end == NULL) || (*end != '\0')) 1220 pm = 0; 1221 1222 options->cryptodev_mask = pm; 1223 if (options->cryptodev_mask == 0) { 1224 printf("invalid cryptodev_mask specified\n"); 1225 return -1; 1226 } 1227 1228 return 0; 1229 } 1230 1231 /** Parse long options */ 1232 static int 1233 l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options, 1234 struct option *lgopts, int option_index) 1235 { 1236 int retval; 1237 int val; 1238 1239 if (strcmp(lgopts[option_index].name, "cdev_type") == 0) { 1240 retval = parse_cryptodev_type(&options->type, optarg); 1241 if (retval == 0) 1242 strlcpy(options->string_type, optarg, MAX_STR_LEN); 1243 return retval; 1244 } 1245 1246 else if (strcmp(lgopts[option_index].name, "chain") == 0) 1247 return parse_crypto_opt_chain(options, optarg); 1248 1249 /* Cipher options */ 1250 else if (strcmp(lgopts[option_index].name, "cipher_algo") == 0) 1251 return parse_cipher_algo(&options->cipher_xform.cipher.algo, 1252 optarg); 1253 1254 else if (strcmp(lgopts[option_index].name, "cipher_op") == 0) 1255 return parse_cipher_op(&options->cipher_xform.cipher.op, 1256 optarg); 1257 1258 else if (strcmp(lgopts[option_index].name, "cipher_key") == 0) { 1259 options->ckey_param = 1; 1260 options->cipher_xform.cipher.key.length = 1261 parse_bytes(options->cipher_key, optarg, MAX_KEY_SIZE); 1262 if (options->cipher_xform.cipher.key.length > 0) 1263 return 0; 1264 else 1265 return -1; 1266 } 1267 1268 else if (strcmp(lgopts[option_index].name, "cipher_dataunit_len") == 0) { 1269 retval = parse_size(&val, optarg); 1270 if (retval == 0 && val >= 0) { 1271 options->cipher_xform.cipher.dataunit_len = 1272 (uint32_t)val; 1273 return 0; 1274 } else 1275 return -1; 1276 } 1277 1278 else if (strcmp(lgopts[option_index].name, "cipher_key_random_size") == 0) 1279 return parse_size(&options->ckey_random_size, optarg); 1280 1281 else if (strcmp(lgopts[option_index].name, "cipher_iv") == 0) { 1282 options->cipher_iv_param = 1; 1283 options->cipher_iv.length = 1284 parse_bytes(options->cipher_iv.data, optarg, MAX_IV_SIZE); 1285 if (options->cipher_iv.length > 0) 1286 return 0; 1287 else 1288 return -1; 1289 } 1290 1291 else if (strcmp(lgopts[option_index].name, "cipher_iv_random_size") == 0) 1292 return parse_size(&options->cipher_iv_random_size, optarg); 1293 1294 /* Authentication options */ 1295 else if (strcmp(lgopts[option_index].name, "auth_algo") == 0) { 1296 return parse_auth_algo(&options->auth_xform.auth.algo, 1297 optarg); 1298 } 1299 1300 else if (strcmp(lgopts[option_index].name, "auth_op") == 0) 1301 return parse_auth_op(&options->auth_xform.auth.op, 1302 optarg); 1303 1304 else if (strcmp(lgopts[option_index].name, "auth_key") == 0) { 1305 options->akey_param = 1; 1306 options->auth_xform.auth.key.length = 1307 parse_bytes(options->auth_key, optarg, MAX_KEY_SIZE); 1308 if (options->auth_xform.auth.key.length > 0) 1309 return 0; 1310 else 1311 return -1; 1312 } 1313 1314 else if (strcmp(lgopts[option_index].name, "auth_key_random_size") == 0) { 1315 return parse_size(&options->akey_random_size, optarg); 1316 } 1317 1318 else if (strcmp(lgopts[option_index].name, "auth_iv") == 0) { 1319 options->auth_iv_param = 1; 1320 options->auth_iv.length = 1321 parse_bytes(options->auth_iv.data, optarg, MAX_IV_SIZE); 1322 if (options->auth_iv.length > 0) 1323 return 0; 1324 else 1325 return -1; 1326 } 1327 1328 else if (strcmp(lgopts[option_index].name, "auth_iv_random_size") == 0) 1329 return parse_size(&options->auth_iv_random_size, optarg); 1330 1331 /* AEAD options */ 1332 else if (strcmp(lgopts[option_index].name, "aead_algo") == 0) { 1333 return parse_aead_algo(&options->aead_xform.aead.algo, 1334 optarg); 1335 } 1336 1337 else if (strcmp(lgopts[option_index].name, "aead_op") == 0) 1338 return parse_aead_op(&options->aead_xform.aead.op, 1339 optarg); 1340 1341 else if (strcmp(lgopts[option_index].name, "aead_key") == 0) { 1342 options->aead_key_param = 1; 1343 options->aead_xform.aead.key.length = 1344 parse_bytes(options->aead_key, optarg, MAX_KEY_SIZE); 1345 if (options->aead_xform.aead.key.length > 0) 1346 return 0; 1347 else 1348 return -1; 1349 } 1350 1351 else if (strcmp(lgopts[option_index].name, "aead_key_random_size") == 0) 1352 return parse_size(&options->aead_key_random_size, optarg); 1353 1354 1355 else if (strcmp(lgopts[option_index].name, "aead_iv") == 0) { 1356 options->aead_iv_param = 1; 1357 options->aead_iv.length = 1358 parse_bytes(options->aead_iv.data, optarg, MAX_IV_SIZE); 1359 if (options->aead_iv.length > 0) 1360 return 0; 1361 else 1362 return -1; 1363 } 1364 1365 else if (strcmp(lgopts[option_index].name, "aead_iv_random_size") == 0) 1366 return parse_size(&options->aead_iv_random_size, optarg); 1367 1368 else if (strcmp(lgopts[option_index].name, "aad") == 0) { 1369 options->aad_param = 1; 1370 options->aad.length = 1371 parse_bytes(options->aad.data, optarg, MAX_AAD_SIZE); 1372 if (options->aad.length > 0) 1373 return 0; 1374 else 1375 return -1; 1376 } 1377 1378 else if (strcmp(lgopts[option_index].name, "aad_random_size") == 0) { 1379 return parse_size(&options->aad_random_size, optarg); 1380 } 1381 1382 else if (strcmp(lgopts[option_index].name, "digest_size") == 0) { 1383 return parse_size(&options->digest_size, optarg); 1384 } 1385 1386 else if (strcmp(lgopts[option_index].name, "sessionless") == 0) { 1387 options->sessionless = 1; 1388 return 0; 1389 } 1390 1391 else if (strcmp(lgopts[option_index].name, "cryptodev_mask") == 0) 1392 return parse_cryptodev_mask(options, optarg); 1393 1394 else if (strcmp(lgopts[option_index].name, "mac-updating") == 0) { 1395 options->mac_updating = 1; 1396 return 0; 1397 } 1398 1399 else if (strcmp(lgopts[option_index].name, "no-mac-updating") == 0) { 1400 options->mac_updating = 0; 1401 return 0; 1402 } 1403 1404 return -1; 1405 } 1406 1407 /** Parse port mask */ 1408 static int 1409 l2fwd_crypto_parse_portmask(struct l2fwd_crypto_options *options, 1410 const char *q_arg) 1411 { 1412 char *end = NULL; 1413 unsigned long pm; 1414 1415 /* parse hexadecimal string */ 1416 pm = strtoul(q_arg, &end, 16); 1417 if ((pm == '\0') || (end == NULL) || (*end != '\0')) 1418 pm = 0; 1419 1420 options->portmask = pm; 1421 if (options->portmask == 0) { 1422 printf("invalid portmask specified\n"); 1423 return -1; 1424 } 1425 1426 return pm; 1427 } 1428 1429 /** Parse number of queues */ 1430 static int 1431 l2fwd_crypto_parse_nqueue(struct l2fwd_crypto_options *options, 1432 const char *q_arg) 1433 { 1434 char *end = NULL; 1435 unsigned long n; 1436 1437 /* parse hexadecimal string */ 1438 n = strtoul(q_arg, &end, 10); 1439 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) 1440 n = 0; 1441 else if (n >= MAX_RX_QUEUE_PER_LCORE) 1442 n = 0; 1443 1444 options->nb_ports_per_lcore = n; 1445 if (options->nb_ports_per_lcore == 0) { 1446 printf("invalid number of ports selected\n"); 1447 return -1; 1448 } 1449 1450 return 0; 1451 } 1452 1453 /** Parse timer period */ 1454 static int 1455 l2fwd_crypto_parse_timer_period(struct l2fwd_crypto_options *options, 1456 const char *q_arg) 1457 { 1458 char *end = NULL; 1459 unsigned long n; 1460 1461 /* parse number string */ 1462 n = (unsigned)strtol(q_arg, &end, 10); 1463 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) 1464 n = 0; 1465 1466 if (n >= MAX_TIMER_PERIOD) { 1467 printf("Warning refresh period specified %lu is greater than " 1468 "max value %lu! using max value", 1469 n, MAX_TIMER_PERIOD); 1470 n = MAX_TIMER_PERIOD; 1471 } 1472 1473 options->refresh_period = n * 1000 * TIMER_MILLISECOND; 1474 1475 return 0; 1476 } 1477 1478 /** Generate default options for application */ 1479 static void 1480 l2fwd_crypto_default_options(struct l2fwd_crypto_options *options) 1481 { 1482 options->portmask = 0xffffffff; 1483 options->nb_ports_per_lcore = 1; 1484 options->refresh_period = 10000; 1485 options->single_lcore = 0; 1486 options->sessionless = 0; 1487 1488 options->xform_chain = L2FWD_CRYPTO_CIPHER_HASH; 1489 1490 /* Cipher Data */ 1491 options->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; 1492 options->cipher_xform.next = NULL; 1493 options->ckey_param = 0; 1494 options->ckey_random_size = -1; 1495 options->cipher_xform.cipher.key.length = 0; 1496 options->cipher_iv_param = 0; 1497 options->cipher_iv_random_size = -1; 1498 options->cipher_iv.length = 0; 1499 1500 options->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; 1501 options->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 1502 options->cipher_xform.cipher.dataunit_len = 0; 1503 1504 /* Authentication Data */ 1505 options->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; 1506 options->auth_xform.next = NULL; 1507 options->akey_param = 0; 1508 options->akey_random_size = -1; 1509 options->auth_xform.auth.key.length = 0; 1510 options->auth_iv_param = 0; 1511 options->auth_iv_random_size = -1; 1512 options->auth_iv.length = 0; 1513 1514 options->auth_xform.auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC; 1515 options->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; 1516 1517 /* AEAD Data */ 1518 options->aead_xform.type = RTE_CRYPTO_SYM_XFORM_AEAD; 1519 options->aead_xform.next = NULL; 1520 options->aead_key_param = 0; 1521 options->aead_key_random_size = -1; 1522 options->aead_xform.aead.key.length = 0; 1523 options->aead_iv_param = 0; 1524 options->aead_iv_random_size = -1; 1525 options->aead_iv.length = 0; 1526 1527 options->aead_xform.aead.algo = RTE_CRYPTO_AEAD_AES_GCM; 1528 options->aead_xform.aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT; 1529 1530 options->aad_param = 0; 1531 options->aad_random_size = -1; 1532 options->aad.length = 0; 1533 1534 options->digest_size = -1; 1535 1536 options->type = CDEV_TYPE_ANY; 1537 options->cryptodev_mask = UINT64_MAX; 1538 1539 options->mac_updating = 1; 1540 } 1541 1542 static void 1543 display_cipher_info(struct l2fwd_crypto_options *options) 1544 { 1545 printf("\n---- Cipher information ---\n"); 1546 printf("Algorithm: %s\n", 1547 rte_crypto_cipher_algorithm_strings[options->cipher_xform.cipher.algo]); 1548 rte_hexdump(stdout, "Cipher key:", 1549 options->cipher_xform.cipher.key.data, 1550 options->cipher_xform.cipher.key.length); 1551 rte_hexdump(stdout, "IV:", options->cipher_iv.data, options->cipher_iv.length); 1552 } 1553 1554 static void 1555 display_auth_info(struct l2fwd_crypto_options *options) 1556 { 1557 printf("\n---- Authentication information ---\n"); 1558 printf("Algorithm: %s\n", 1559 rte_crypto_auth_algorithm_strings[options->auth_xform.auth.algo]); 1560 rte_hexdump(stdout, "Auth key:", 1561 options->auth_xform.auth.key.data, 1562 options->auth_xform.auth.key.length); 1563 rte_hexdump(stdout, "IV:", options->auth_iv.data, options->auth_iv.length); 1564 } 1565 1566 static void 1567 display_aead_info(struct l2fwd_crypto_options *options) 1568 { 1569 printf("\n---- AEAD information ---\n"); 1570 printf("Algorithm: %s\n", 1571 rte_crypto_aead_algorithm_strings[options->aead_xform.aead.algo]); 1572 rte_hexdump(stdout, "AEAD key:", 1573 options->aead_xform.aead.key.data, 1574 options->aead_xform.aead.key.length); 1575 rte_hexdump(stdout, "IV:", options->aead_iv.data, options->aead_iv.length); 1576 rte_hexdump(stdout, "AAD:", options->aad.data, options->aad.length); 1577 } 1578 1579 static void 1580 l2fwd_crypto_options_print(struct l2fwd_crypto_options *options) 1581 { 1582 char string_cipher_op[MAX_STR_LEN]; 1583 char string_auth_op[MAX_STR_LEN]; 1584 char string_aead_op[MAX_STR_LEN]; 1585 1586 if (options->cipher_xform.cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1587 strcpy(string_cipher_op, "Encrypt"); 1588 else 1589 strcpy(string_cipher_op, "Decrypt"); 1590 1591 if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) 1592 strcpy(string_auth_op, "Auth generate"); 1593 else 1594 strcpy(string_auth_op, "Auth verify"); 1595 1596 if (options->aead_xform.aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 1597 strcpy(string_aead_op, "Authenticated encryption"); 1598 else 1599 strcpy(string_aead_op, "Authenticated decryption"); 1600 1601 1602 printf("Options:-\nn"); 1603 printf("portmask: %x\n", options->portmask); 1604 printf("ports per lcore: %u\n", options->nb_ports_per_lcore); 1605 printf("refresh period : %u\n", options->refresh_period); 1606 printf("single lcore mode: %s\n", 1607 options->single_lcore ? "enabled" : "disabled"); 1608 printf("stats_printing: %s\n", 1609 options->refresh_period == 0 ? "disabled" : "enabled"); 1610 1611 printf("sessionless crypto: %s\n", 1612 options->sessionless ? "enabled" : "disabled"); 1613 1614 if (options->ckey_param && (options->ckey_random_size != -1)) 1615 printf("Cipher key already parsed, ignoring size of random key\n"); 1616 1617 if (options->akey_param && (options->akey_random_size != -1)) 1618 printf("Auth key already parsed, ignoring size of random key\n"); 1619 1620 if (options->cipher_iv_param && (options->cipher_iv_random_size != -1)) 1621 printf("Cipher IV already parsed, ignoring size of random IV\n"); 1622 1623 if (options->auth_iv_param && (options->auth_iv_random_size != -1)) 1624 printf("Auth IV already parsed, ignoring size of random IV\n"); 1625 1626 if (options->aad_param && (options->aad_random_size != -1)) 1627 printf("AAD already parsed, ignoring size of random AAD\n"); 1628 1629 printf("\nCrypto chain: "); 1630 switch (options->xform_chain) { 1631 case L2FWD_CRYPTO_AEAD: 1632 printf("Input --> %s --> Output\n", string_aead_op); 1633 display_aead_info(options); 1634 break; 1635 case L2FWD_CRYPTO_CIPHER_HASH: 1636 printf("Input --> %s --> %s --> Output\n", 1637 string_cipher_op, string_auth_op); 1638 display_cipher_info(options); 1639 display_auth_info(options); 1640 break; 1641 case L2FWD_CRYPTO_HASH_CIPHER: 1642 printf("Input --> %s --> %s --> Output\n", 1643 string_auth_op, string_cipher_op); 1644 display_cipher_info(options); 1645 display_auth_info(options); 1646 break; 1647 case L2FWD_CRYPTO_HASH_ONLY: 1648 printf("Input --> %s --> Output\n", string_auth_op); 1649 display_auth_info(options); 1650 break; 1651 case L2FWD_CRYPTO_CIPHER_ONLY: 1652 printf("Input --> %s --> Output\n", string_cipher_op); 1653 display_cipher_info(options); 1654 break; 1655 } 1656 } 1657 1658 /* Parse the argument given in the command line of the application */ 1659 static int 1660 l2fwd_crypto_parse_args(struct l2fwd_crypto_options *options, 1661 int argc, char **argv) 1662 { 1663 int opt, retval, option_index; 1664 char **argvopt = argv, *prgname = argv[0]; 1665 1666 static struct option lgopts[] = { 1667 { "sessionless", no_argument, 0, 0 }, 1668 1669 { "cdev_type", required_argument, 0, 0 }, 1670 { "chain", required_argument, 0, 0 }, 1671 1672 { "cipher_algo", required_argument, 0, 0 }, 1673 { "cipher_op", required_argument, 0, 0 }, 1674 { "cipher_key", required_argument, 0, 0 }, 1675 { "cipher_key_random_size", required_argument, 0, 0 }, 1676 { "cipher_iv", required_argument, 0, 0 }, 1677 { "cipher_iv_random_size", required_argument, 0, 0 }, 1678 { "cipher_dataunit_len", required_argument, 0, 0}, 1679 1680 { "auth_algo", required_argument, 0, 0 }, 1681 { "auth_op", required_argument, 0, 0 }, 1682 { "auth_key", required_argument, 0, 0 }, 1683 { "auth_key_random_size", required_argument, 0, 0 }, 1684 { "auth_iv", required_argument, 0, 0 }, 1685 { "auth_iv_random_size", required_argument, 0, 0 }, 1686 1687 { "aead_algo", required_argument, 0, 0 }, 1688 { "aead_op", required_argument, 0, 0 }, 1689 { "aead_key", required_argument, 0, 0 }, 1690 { "aead_key_random_size", required_argument, 0, 0 }, 1691 { "aead_iv", required_argument, 0, 0 }, 1692 { "aead_iv_random_size", required_argument, 0, 0 }, 1693 1694 { "aad", required_argument, 0, 0 }, 1695 { "aad_random_size", required_argument, 0, 0 }, 1696 1697 { "digest_size", required_argument, 0, 0 }, 1698 1699 { "sessionless", no_argument, 0, 0 }, 1700 { "cryptodev_mask", required_argument, 0, 0}, 1701 1702 { "mac-updating", no_argument, 0, 0}, 1703 { "no-mac-updating", no_argument, 0, 0}, 1704 1705 { NULL, 0, 0, 0 } 1706 }; 1707 1708 l2fwd_crypto_default_options(options); 1709 1710 while ((opt = getopt_long(argc, argvopt, "p:q:sT:", lgopts, 1711 &option_index)) != EOF) { 1712 switch (opt) { 1713 /* long options */ 1714 case 0: 1715 retval = l2fwd_crypto_parse_args_long_options(options, 1716 lgopts, option_index); 1717 if (retval < 0) { 1718 l2fwd_crypto_usage(prgname); 1719 return -1; 1720 } 1721 break; 1722 1723 /* portmask */ 1724 case 'p': 1725 retval = l2fwd_crypto_parse_portmask(options, optarg); 1726 if (retval < 0) { 1727 l2fwd_crypto_usage(prgname); 1728 return -1; 1729 } 1730 break; 1731 1732 /* nqueue */ 1733 case 'q': 1734 retval = l2fwd_crypto_parse_nqueue(options, optarg); 1735 if (retval < 0) { 1736 l2fwd_crypto_usage(prgname); 1737 return -1; 1738 } 1739 break; 1740 1741 /* single */ 1742 case 's': 1743 options->single_lcore = 1; 1744 1745 break; 1746 1747 /* timer period */ 1748 case 'T': 1749 retval = l2fwd_crypto_parse_timer_period(options, 1750 optarg); 1751 if (retval < 0) { 1752 l2fwd_crypto_usage(prgname); 1753 return -1; 1754 } 1755 break; 1756 1757 default: 1758 l2fwd_crypto_usage(prgname); 1759 return -1; 1760 } 1761 } 1762 1763 1764 if (optind >= 0) 1765 argv[optind-1] = prgname; 1766 1767 retval = optind-1; 1768 optind = 1; /* reset getopt lib */ 1769 1770 return retval; 1771 } 1772 1773 /* Check the link status of all ports in up to 9s, and print them finally */ 1774 static void 1775 check_all_ports_link_status(uint32_t port_mask) 1776 { 1777 #define CHECK_INTERVAL 100 /* 100ms */ 1778 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1779 uint16_t portid; 1780 uint8_t count, all_ports_up, print_flag = 0; 1781 struct rte_eth_link link; 1782 int ret; 1783 char link_status_text[RTE_ETH_LINK_MAX_STR_LEN]; 1784 1785 printf("\nChecking link status"); 1786 fflush(stdout); 1787 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1788 all_ports_up = 1; 1789 RTE_ETH_FOREACH_DEV(portid) { 1790 if ((port_mask & (1 << portid)) == 0) 1791 continue; 1792 memset(&link, 0, sizeof(link)); 1793 ret = rte_eth_link_get_nowait(portid, &link); 1794 if (ret < 0) { 1795 all_ports_up = 0; 1796 if (print_flag == 1) 1797 printf("Port %u link get failed: %s\n", 1798 portid, rte_strerror(-ret)); 1799 continue; 1800 } 1801 /* print link status if flag set */ 1802 if (print_flag == 1) { 1803 rte_eth_link_to_str(link_status_text, 1804 sizeof(link_status_text), &link); 1805 printf("Port %d %s\n", portid, 1806 link_status_text); 1807 continue; 1808 } 1809 /* clear all_ports_up flag if any link down */ 1810 if (link.link_status == RTE_ETH_LINK_DOWN) { 1811 all_ports_up = 0; 1812 break; 1813 } 1814 } 1815 /* after finally printing all link status, get out */ 1816 if (print_flag == 1) 1817 break; 1818 1819 if (all_ports_up == 0) { 1820 printf("."); 1821 fflush(stdout); 1822 rte_delay_ms(CHECK_INTERVAL); 1823 } 1824 1825 /* set the print_flag if all ports up or timeout */ 1826 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1827 print_flag = 1; 1828 printf("done\n"); 1829 } 1830 } 1831 } 1832 1833 /* Check if device has to be HW/SW or any */ 1834 static int 1835 check_type(const struct l2fwd_crypto_options *options, 1836 const struct rte_cryptodev_info *dev_info) 1837 { 1838 if (options->type == CDEV_TYPE_HW && 1839 (dev_info->feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED)) 1840 return 0; 1841 if (options->type == CDEV_TYPE_SW && 1842 !(dev_info->feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED)) 1843 return 0; 1844 if (options->type == CDEV_TYPE_ANY) 1845 return 0; 1846 1847 return -1; 1848 } 1849 1850 static const struct rte_cryptodev_capabilities * 1851 check_device_support_cipher_algo(const struct l2fwd_crypto_options *options, 1852 const struct rte_cryptodev_info *dev_info, 1853 uint8_t cdev_id) 1854 { 1855 unsigned int i = 0; 1856 const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; 1857 enum rte_crypto_cipher_algorithm cap_cipher_algo; 1858 enum rte_crypto_cipher_algorithm opt_cipher_algo = 1859 options->cipher_xform.cipher.algo; 1860 1861 while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1862 cap_cipher_algo = cap->sym.cipher.algo; 1863 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 1864 if (cap_cipher_algo == opt_cipher_algo) { 1865 if (check_type(options, dev_info) == 0) 1866 break; 1867 } 1868 } 1869 cap = &dev_info->capabilities[++i]; 1870 } 1871 1872 if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1873 printf("Algorithm %s not supported by cryptodev %u" 1874 " or device not of preferred type (%s)\n", 1875 rte_crypto_cipher_algorithm_strings[opt_cipher_algo], 1876 cdev_id, 1877 options->string_type); 1878 return NULL; 1879 } 1880 1881 return cap; 1882 } 1883 1884 static const struct rte_cryptodev_capabilities * 1885 check_device_support_auth_algo(const struct l2fwd_crypto_options *options, 1886 const struct rte_cryptodev_info *dev_info, 1887 uint8_t cdev_id) 1888 { 1889 unsigned int i = 0; 1890 const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; 1891 enum rte_crypto_auth_algorithm cap_auth_algo; 1892 enum rte_crypto_auth_algorithm opt_auth_algo = 1893 options->auth_xform.auth.algo; 1894 1895 while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1896 cap_auth_algo = cap->sym.auth.algo; 1897 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AUTH) { 1898 if (cap_auth_algo == opt_auth_algo) { 1899 if (check_type(options, dev_info) == 0) 1900 break; 1901 } 1902 } 1903 cap = &dev_info->capabilities[++i]; 1904 } 1905 1906 if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1907 printf("Algorithm %s not supported by cryptodev %u" 1908 " or device not of preferred type (%s)\n", 1909 rte_crypto_auth_algorithm_strings[opt_auth_algo], 1910 cdev_id, 1911 options->string_type); 1912 return NULL; 1913 } 1914 1915 return cap; 1916 } 1917 1918 static const struct rte_cryptodev_capabilities * 1919 check_device_support_aead_algo(const struct l2fwd_crypto_options *options, 1920 const struct rte_cryptodev_info *dev_info, 1921 uint8_t cdev_id) 1922 { 1923 unsigned int i = 0; 1924 const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; 1925 enum rte_crypto_aead_algorithm cap_aead_algo; 1926 enum rte_crypto_aead_algorithm opt_aead_algo = 1927 options->aead_xform.aead.algo; 1928 1929 while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1930 cap_aead_algo = cap->sym.aead.algo; 1931 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1932 if (cap_aead_algo == opt_aead_algo) { 1933 if (check_type(options, dev_info) == 0) 1934 break; 1935 } 1936 } 1937 cap = &dev_info->capabilities[++i]; 1938 } 1939 1940 if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1941 printf("Algorithm %s not supported by cryptodev %u" 1942 " or device not of preferred type (%s)\n", 1943 rte_crypto_aead_algorithm_strings[opt_aead_algo], 1944 cdev_id, 1945 options->string_type); 1946 return NULL; 1947 } 1948 1949 return cap; 1950 } 1951 1952 /* Check if the device is enabled by cryptodev_mask */ 1953 static int 1954 check_cryptodev_mask(struct l2fwd_crypto_options *options, 1955 uint8_t cdev_id) 1956 { 1957 if (options->cryptodev_mask & (1 << cdev_id)) 1958 return 0; 1959 1960 return -1; 1961 } 1962 1963 static inline int 1964 check_supported_size(uint16_t length, uint16_t min, uint16_t max, 1965 uint16_t increment) 1966 { 1967 uint16_t supp_size; 1968 1969 /* Single value */ 1970 if (increment == 0) { 1971 if (length == min) 1972 return 0; 1973 else 1974 return -1; 1975 } 1976 1977 /* Range of values */ 1978 for (supp_size = min; supp_size <= max; supp_size += increment) { 1979 if (length == supp_size) 1980 return 0; 1981 } 1982 1983 return -1; 1984 } 1985 1986 static int 1987 check_iv_param(const struct rte_crypto_param_range *iv_range_size, 1988 unsigned int iv_param, int iv_random_size, 1989 uint16_t iv_length) 1990 { 1991 /* 1992 * Check if length of provided IV is supported 1993 * by the algorithm chosen. 1994 */ 1995 if (iv_param) { 1996 if (check_supported_size(iv_length, 1997 iv_range_size->min, 1998 iv_range_size->max, 1999 iv_range_size->increment) 2000 != 0) 2001 return -1; 2002 /* 2003 * Check if length of IV to be randomly generated 2004 * is supported by the algorithm chosen. 2005 */ 2006 } else if (iv_random_size != -1) { 2007 if (check_supported_size(iv_random_size, 2008 iv_range_size->min, 2009 iv_range_size->max, 2010 iv_range_size->increment) 2011 != 0) 2012 return -1; 2013 } 2014 2015 return 0; 2016 } 2017 2018 static int 2019 check_capabilities(struct l2fwd_crypto_options *options, uint8_t cdev_id) 2020 { 2021 struct rte_cryptodev_info dev_info; 2022 const struct rte_cryptodev_capabilities *cap; 2023 2024 rte_cryptodev_info_get(cdev_id, &dev_info); 2025 2026 /* Set AEAD parameters */ 2027 if (options->xform_chain == L2FWD_CRYPTO_AEAD) { 2028 /* Check if device supports AEAD algo */ 2029 cap = check_device_support_aead_algo(options, &dev_info, 2030 cdev_id); 2031 if (cap == NULL) 2032 return -1; 2033 2034 if (check_iv_param(&cap->sym.aead.iv_size, 2035 options->aead_iv_param, 2036 options->aead_iv_random_size, 2037 options->aead_iv.length) != 0) { 2038 RTE_LOG(DEBUG, USER1, 2039 "Device %u does not support IV length\n", 2040 cdev_id); 2041 return -1; 2042 } 2043 2044 /* 2045 * Check if length of provided AEAD key is supported 2046 * by the algorithm chosen. 2047 */ 2048 if (options->aead_key_param) { 2049 if (check_supported_size( 2050 options->aead_xform.aead.key.length, 2051 cap->sym.aead.key_size.min, 2052 cap->sym.aead.key_size.max, 2053 cap->sym.aead.key_size.increment) 2054 != 0) { 2055 RTE_LOG(DEBUG, USER1, 2056 "Device %u does not support " 2057 "AEAD key length\n", 2058 cdev_id); 2059 return -1; 2060 } 2061 /* 2062 * Check if length of the aead key to be randomly generated 2063 * is supported by the algorithm chosen. 2064 */ 2065 } else if (options->aead_key_random_size != -1) { 2066 if (check_supported_size(options->aead_key_random_size, 2067 cap->sym.aead.key_size.min, 2068 cap->sym.aead.key_size.max, 2069 cap->sym.aead.key_size.increment) 2070 != 0) { 2071 RTE_LOG(DEBUG, USER1, 2072 "Device %u does not support " 2073 "AEAD key length\n", 2074 cdev_id); 2075 return -1; 2076 } 2077 } 2078 2079 2080 /* 2081 * Check if length of provided AAD is supported 2082 * by the algorithm chosen. 2083 */ 2084 if (options->aad_param) { 2085 if (check_supported_size(options->aad.length, 2086 cap->sym.aead.aad_size.min, 2087 cap->sym.aead.aad_size.max, 2088 cap->sym.aead.aad_size.increment) 2089 != 0) { 2090 RTE_LOG(DEBUG, USER1, 2091 "Device %u does not support " 2092 "AAD length\n", 2093 cdev_id); 2094 return -1; 2095 } 2096 /* 2097 * Check if length of AAD to be randomly generated 2098 * is supported by the algorithm chosen. 2099 */ 2100 } else if (options->aad_random_size != -1) { 2101 if (check_supported_size(options->aad_random_size, 2102 cap->sym.aead.aad_size.min, 2103 cap->sym.aead.aad_size.max, 2104 cap->sym.aead.aad_size.increment) 2105 != 0) { 2106 RTE_LOG(DEBUG, USER1, 2107 "Device %u does not support " 2108 "AAD length\n", 2109 cdev_id); 2110 return -1; 2111 } 2112 } 2113 2114 /* Check if digest size is supported by the algorithm. */ 2115 if (options->digest_size != -1) { 2116 if (check_supported_size(options->digest_size, 2117 cap->sym.aead.digest_size.min, 2118 cap->sym.aead.digest_size.max, 2119 cap->sym.aead.digest_size.increment) 2120 != 0) { 2121 RTE_LOG(DEBUG, USER1, 2122 "Device %u does not support " 2123 "digest length\n", 2124 cdev_id); 2125 return -1; 2126 } 2127 } 2128 } 2129 2130 /* Set cipher parameters */ 2131 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2132 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2133 options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { 2134 2135 /* Check if device supports cipher algo. 8< */ 2136 cap = check_device_support_cipher_algo(options, &dev_info, 2137 cdev_id); 2138 if (cap == NULL) 2139 return -1; 2140 2141 if (check_iv_param(&cap->sym.cipher.iv_size, 2142 options->cipher_iv_param, 2143 options->cipher_iv_random_size, 2144 options->cipher_iv.length) != 0) { 2145 RTE_LOG(DEBUG, USER1, 2146 "Device %u does not support IV length\n", 2147 cdev_id); 2148 return -1; 2149 } 2150 /* >8 End of check if device supports cipher algo. */ 2151 2152 /* Check if capable cipher is supported. 8< */ 2153 2154 /* 2155 * Check if length of provided cipher key is supported 2156 * by the algorithm chosen. 2157 */ 2158 if (options->ckey_param) { 2159 if (check_supported_size( 2160 options->cipher_xform.cipher.key.length, 2161 cap->sym.cipher.key_size.min, 2162 cap->sym.cipher.key_size.max, 2163 cap->sym.cipher.key_size.increment) 2164 != 0) { 2165 if (dev_info.feature_flags & 2166 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY) { 2167 RTE_LOG(DEBUG, USER1, 2168 "Key length does not match the device " 2169 "%u capability. Key may be wrapped\n", 2170 cdev_id); 2171 } else { 2172 RTE_LOG(DEBUG, USER1, 2173 "Key length does not match the device " 2174 "%u capability\n", 2175 cdev_id); 2176 return -1; 2177 } 2178 } 2179 2180 /* 2181 * Check if length of the cipher key to be randomly generated 2182 * is supported by the algorithm chosen. 2183 */ 2184 } else if (options->ckey_random_size != -1) { 2185 if (check_supported_size(options->ckey_random_size, 2186 cap->sym.cipher.key_size.min, 2187 cap->sym.cipher.key_size.max, 2188 cap->sym.cipher.key_size.increment) 2189 != 0) { 2190 RTE_LOG(DEBUG, USER1, 2191 "Device %u does not support cipher " 2192 "key length\n", 2193 cdev_id); 2194 return -1; 2195 } 2196 } 2197 2198 if (options->cipher_xform.cipher.dataunit_len > 0) { 2199 if (!(dev_info.feature_flags & 2200 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)) { 2201 RTE_LOG(DEBUG, USER1, 2202 "Device %u does not support " 2203 "cipher multiple data units\n", 2204 cdev_id); 2205 return -1; 2206 } 2207 if (cap->sym.cipher.dataunit_set != 0) { 2208 int ret = 0; 2209 2210 switch (options->cipher_xform.cipher.dataunit_len) { 2211 case 512: 2212 if (!(cap->sym.cipher.dataunit_set & 2213 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES)) 2214 ret = -1; 2215 break; 2216 case 4096: 2217 if (!(cap->sym.cipher.dataunit_set & 2218 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES)) 2219 ret = -1; 2220 break; 2221 case 1048576: 2222 if (!(cap->sym.cipher.dataunit_set & 2223 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_1_MEGABYTES)) 2224 ret = -1; 2225 break; 2226 default: 2227 ret = -1; 2228 } 2229 if (ret == -1) { 2230 RTE_LOG(DEBUG, USER1, 2231 "Device %u does not support " 2232 "data-unit length %u\n", 2233 cdev_id, 2234 options->cipher_xform.cipher.dataunit_len); 2235 return -1; 2236 } 2237 } 2238 } 2239 /* >8 End of checking if cipher is supported. */ 2240 } 2241 2242 /* Set auth parameters */ 2243 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2244 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2245 options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) { 2246 /* Check if device supports auth algo */ 2247 cap = check_device_support_auth_algo(options, &dev_info, 2248 cdev_id); 2249 if (cap == NULL) 2250 return -1; 2251 2252 if (check_iv_param(&cap->sym.auth.iv_size, 2253 options->auth_iv_param, 2254 options->auth_iv_random_size, 2255 options->auth_iv.length) != 0) { 2256 RTE_LOG(DEBUG, USER1, 2257 "Device %u does not support IV length\n", 2258 cdev_id); 2259 return -1; 2260 } 2261 /* 2262 * Check if length of provided auth key is supported 2263 * by the algorithm chosen. 2264 */ 2265 if (options->akey_param) { 2266 if (check_supported_size( 2267 options->auth_xform.auth.key.length, 2268 cap->sym.auth.key_size.min, 2269 cap->sym.auth.key_size.max, 2270 cap->sym.auth.key_size.increment) 2271 != 0) { 2272 RTE_LOG(DEBUG, USER1, 2273 "Device %u does not support auth " 2274 "key length\n", 2275 cdev_id); 2276 return -1; 2277 } 2278 /* 2279 * Check if length of the auth key to be randomly generated 2280 * is supported by the algorithm chosen. 2281 */ 2282 } else if (options->akey_random_size != -1) { 2283 if (check_supported_size(options->akey_random_size, 2284 cap->sym.auth.key_size.min, 2285 cap->sym.auth.key_size.max, 2286 cap->sym.auth.key_size.increment) 2287 != 0) { 2288 RTE_LOG(DEBUG, USER1, 2289 "Device %u does not support auth " 2290 "key length\n", 2291 cdev_id); 2292 return -1; 2293 } 2294 } 2295 2296 /* Check if digest size is supported by the algorithm. */ 2297 if (options->digest_size != -1) { 2298 if (check_supported_size(options->digest_size, 2299 cap->sym.auth.digest_size.min, 2300 cap->sym.auth.digest_size.max, 2301 cap->sym.auth.digest_size.increment) 2302 != 0) { 2303 RTE_LOG(DEBUG, USER1, 2304 "Device %u does not support " 2305 "digest length\n", 2306 cdev_id); 2307 return -1; 2308 } 2309 } 2310 } 2311 2312 return 0; 2313 } 2314 2315 static int 2316 initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, 2317 uint8_t *enabled_cdevs) 2318 { 2319 uint8_t cdev_id, cdev_count, enabled_cdev_count = 0; 2320 const struct rte_cryptodev_capabilities *cap; 2321 unsigned int sess_sz, max_sess_sz = 0; 2322 uint32_t sessions_needed = 0; 2323 int retval; 2324 2325 cdev_count = rte_cryptodev_count(); 2326 if (cdev_count == 0) { 2327 printf("No crypto devices available\n"); 2328 return -1; 2329 } 2330 2331 for (cdev_id = 0; cdev_id < cdev_count && enabled_cdev_count < nb_ports; 2332 cdev_id++) { 2333 if (check_cryptodev_mask(options, cdev_id) < 0) 2334 continue; 2335 2336 if (check_capabilities(options, cdev_id) < 0) 2337 continue; 2338 2339 sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id); 2340 if (sess_sz > max_sess_sz) 2341 max_sess_sz = sess_sz; 2342 2343 l2fwd_enabled_crypto_mask |= (((uint64_t)1) << cdev_id); 2344 2345 enabled_cdevs[cdev_id] = 1; 2346 enabled_cdev_count++; 2347 } 2348 2349 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { 2350 struct rte_cryptodev_qp_conf qp_conf; 2351 struct rte_cryptodev_info dev_info; 2352 2353 if (enabled_cdevs[cdev_id] == 0) 2354 continue; 2355 2356 if (check_cryptodev_mask(options, cdev_id) < 0) 2357 continue; 2358 2359 if (check_capabilities(options, cdev_id) < 0) 2360 continue; 2361 2362 retval = rte_cryptodev_socket_id(cdev_id); 2363 2364 if (retval < 0) { 2365 printf("Invalid crypto device id used\n"); 2366 return -1; 2367 } 2368 2369 uint8_t socket_id = (uint8_t) retval; 2370 2371 struct rte_cryptodev_config conf = { 2372 .nb_queue_pairs = 1, 2373 .socket_id = socket_id, 2374 .ff_disable = RTE_CRYPTODEV_FF_SECURITY, 2375 }; 2376 2377 rte_cryptodev_info_get(cdev_id, &dev_info); 2378 2379 /* 2380 * Two sessions objects are required for each session 2381 * (one for the header, one for the private data) 2382 */ 2383 if (!strcmp(dev_info.driver_name, "crypto_scheduler")) { 2384 #ifdef RTE_CRYPTO_SCHEDULER 2385 uint32_t nb_workers = 2386 rte_cryptodev_scheduler_workers_get(cdev_id, 2387 NULL); 2388 2389 sessions_needed = enabled_cdev_count * nb_workers; 2390 #endif 2391 } else 2392 sessions_needed = enabled_cdev_count; 2393 2394 if (session_pool_socket[socket_id].priv_mp == NULL) { 2395 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2396 2397 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2398 "priv_sess_mp_%u", socket_id); 2399 2400 session_pool_socket[socket_id].priv_mp = 2401 rte_mempool_create(mp_name, 2402 sessions_needed, 2403 max_sess_sz, 2404 0, 0, NULL, NULL, NULL, 2405 NULL, socket_id, 2406 0); 2407 2408 if (session_pool_socket[socket_id].priv_mp == NULL) { 2409 printf("Cannot create pool on socket %d\n", 2410 socket_id); 2411 return -ENOMEM; 2412 } 2413 2414 printf("Allocated pool \"%s\" on socket %d\n", 2415 mp_name, socket_id); 2416 } 2417 2418 if (session_pool_socket[socket_id].sess_mp == NULL) { 2419 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2420 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2421 "sess_mp_%u", socket_id); 2422 2423 session_pool_socket[socket_id].sess_mp = 2424 rte_cryptodev_sym_session_pool_create( 2425 mp_name, 2426 sessions_needed, 2427 0, 0, 0, socket_id); 2428 2429 if (session_pool_socket[socket_id].sess_mp == NULL) { 2430 printf("Cannot create pool on socket %d\n", 2431 socket_id); 2432 return -ENOMEM; 2433 } 2434 2435 printf("Allocated pool \"%s\" on socket %d\n", 2436 mp_name, socket_id); 2437 } 2438 2439 /* Set AEAD parameters */ 2440 if (options->xform_chain == L2FWD_CRYPTO_AEAD) { 2441 cap = check_device_support_aead_algo(options, &dev_info, 2442 cdev_id); 2443 2444 options->block_size = cap->sym.aead.block_size; 2445 2446 /* Set IV if not provided from command line */ 2447 if (options->aead_iv_param == 0) { 2448 if (options->aead_iv_random_size != -1) 2449 options->aead_iv.length = 2450 options->aead_iv_random_size; 2451 /* No size provided, use minimum size. */ 2452 else 2453 options->aead_iv.length = 2454 cap->sym.aead.iv_size.min; 2455 } 2456 2457 /* Set key if not provided from command line */ 2458 if (options->aead_key_param == 0) { 2459 if (options->aead_key_random_size != -1) 2460 options->aead_xform.aead.key.length = 2461 options->aead_key_random_size; 2462 /* No size provided, use minimum size. */ 2463 else 2464 options->aead_xform.aead.key.length = 2465 cap->sym.aead.key_size.min; 2466 2467 generate_random_key(options->aead_key, 2468 options->aead_xform.aead.key.length); 2469 } 2470 2471 /* Set AAD if not provided from command line */ 2472 if (options->aad_param == 0) { 2473 if (options->aad_random_size != -1) 2474 options->aad.length = 2475 options->aad_random_size; 2476 /* No size provided, use minimum size. */ 2477 else 2478 options->aad.length = 2479 cap->sym.auth.aad_size.min; 2480 } 2481 2482 options->aead_xform.aead.aad_length = 2483 options->aad.length; 2484 2485 /* Set digest size if not provided from command line */ 2486 if (options->digest_size != -1) 2487 options->aead_xform.aead.digest_length = 2488 options->digest_size; 2489 /* No size provided, use minimum size. */ 2490 else 2491 options->aead_xform.aead.digest_length = 2492 cap->sym.aead.digest_size.min; 2493 } 2494 2495 /* Set cipher parameters */ 2496 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2497 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2498 options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { 2499 cap = check_device_support_cipher_algo(options, &dev_info, 2500 cdev_id); 2501 options->block_size = cap->sym.cipher.block_size; 2502 2503 /* Set IV if not provided from command line */ 2504 if (options->cipher_iv_param == 0) { 2505 if (options->cipher_iv_random_size != -1) 2506 options->cipher_iv.length = 2507 options->cipher_iv_random_size; 2508 /* No size provided, use minimum size. */ 2509 else 2510 options->cipher_iv.length = 2511 cap->sym.cipher.iv_size.min; 2512 } 2513 2514 /* Set key if not provided from command line */ 2515 if (options->ckey_param == 0) { 2516 if (options->ckey_random_size != -1) 2517 options->cipher_xform.cipher.key.length = 2518 options->ckey_random_size; 2519 /* No size provided, use minimum size. */ 2520 else 2521 options->cipher_xform.cipher.key.length = 2522 cap->sym.cipher.key_size.min; 2523 2524 generate_random_key(options->cipher_key, 2525 options->cipher_xform.cipher.key.length); 2526 } 2527 } 2528 2529 /* Set auth parameters */ 2530 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2531 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2532 options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) { 2533 cap = check_device_support_auth_algo(options, &dev_info, 2534 cdev_id); 2535 2536 /* Set IV if not provided from command line */ 2537 if (options->auth_iv_param == 0) { 2538 if (options->auth_iv_random_size != -1) 2539 options->auth_iv.length = 2540 options->auth_iv_random_size; 2541 /* No size provided, use minimum size. */ 2542 else 2543 options->auth_iv.length = 2544 cap->sym.auth.iv_size.min; 2545 } 2546 2547 /* Set key if not provided from command line */ 2548 if (options->akey_param == 0) { 2549 if (options->akey_random_size != -1) 2550 options->auth_xform.auth.key.length = 2551 options->akey_random_size; 2552 /* No size provided, use minimum size. */ 2553 else 2554 options->auth_xform.auth.key.length = 2555 cap->sym.auth.key_size.min; 2556 2557 generate_random_key(options->auth_key, 2558 options->auth_xform.auth.key.length); 2559 } 2560 2561 /* Set digest size if not provided from command line */ 2562 if (options->digest_size != -1) 2563 options->auth_xform.auth.digest_length = 2564 options->digest_size; 2565 /* No size provided, use minimum size. */ 2566 else 2567 options->auth_xform.auth.digest_length = 2568 cap->sym.auth.digest_size.min; 2569 } 2570 2571 retval = rte_cryptodev_configure(cdev_id, &conf); 2572 if (retval < 0) { 2573 printf("Failed to configure cryptodev %u", cdev_id); 2574 return -1; 2575 } 2576 2577 qp_conf.nb_descriptors = 2048; 2578 qp_conf.mp_session = session_pool_socket[socket_id].sess_mp; 2579 qp_conf.mp_session_private = 2580 session_pool_socket[socket_id].priv_mp; 2581 2582 retval = rte_cryptodev_queue_pair_setup(cdev_id, 0, &qp_conf, 2583 socket_id); 2584 if (retval < 0) { 2585 printf("Failed to setup queue pair %u on cryptodev %u", 2586 0, cdev_id); 2587 return -1; 2588 } 2589 2590 retval = rte_cryptodev_start(cdev_id); 2591 if (retval < 0) { 2592 printf("Failed to start device %u: error %d\n", 2593 cdev_id, retval); 2594 return -1; 2595 } 2596 } 2597 2598 return enabled_cdev_count; 2599 } 2600 2601 static int 2602 initialize_ports(struct l2fwd_crypto_options *options) 2603 { 2604 uint16_t last_portid = 0, portid; 2605 unsigned enabled_portcount = 0; 2606 unsigned nb_ports = rte_eth_dev_count_avail(); 2607 2608 if (nb_ports == 0) { 2609 printf("No Ethernet ports - bye\n"); 2610 return -1; 2611 } 2612 2613 /* Reset l2fwd_dst_ports */ 2614 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) 2615 l2fwd_dst_ports[portid] = 0; 2616 2617 RTE_ETH_FOREACH_DEV(portid) { 2618 int retval; 2619 struct rte_eth_dev_info dev_info; 2620 struct rte_eth_rxconf rxq_conf; 2621 struct rte_eth_txconf txq_conf; 2622 struct rte_eth_conf local_port_conf = port_conf; 2623 2624 /* Skip ports that are not enabled */ 2625 if ((options->portmask & (1 << portid)) == 0) 2626 continue; 2627 2628 /* init port */ 2629 printf("Initializing port %u... ", portid); 2630 fflush(stdout); 2631 2632 retval = rte_eth_dev_info_get(portid, &dev_info); 2633 if (retval != 0) { 2634 printf("Error during getting device (port %u) info: %s\n", 2635 portid, strerror(-retval)); 2636 return retval; 2637 } 2638 2639 if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 2640 local_port_conf.txmode.offloads |= 2641 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 2642 retval = rte_eth_dev_configure(portid, 1, 1, &local_port_conf); 2643 if (retval < 0) { 2644 printf("Cannot configure device: err=%d, port=%u\n", 2645 retval, portid); 2646 return -1; 2647 } 2648 2649 retval = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, 2650 &nb_txd); 2651 if (retval < 0) { 2652 printf("Cannot adjust number of descriptors: err=%d, port=%u\n", 2653 retval, portid); 2654 return -1; 2655 } 2656 2657 /* init one RX queue */ 2658 fflush(stdout); 2659 rxq_conf = dev_info.default_rxconf; 2660 rxq_conf.offloads = local_port_conf.rxmode.offloads; 2661 retval = rte_eth_rx_queue_setup(portid, 0, nb_rxd, 2662 rte_eth_dev_socket_id(portid), 2663 &rxq_conf, l2fwd_pktmbuf_pool); 2664 if (retval < 0) { 2665 printf("rte_eth_rx_queue_setup:err=%d, port=%u\n", 2666 retval, portid); 2667 return -1; 2668 } 2669 2670 /* init one TX queue on each port */ 2671 fflush(stdout); 2672 txq_conf = dev_info.default_txconf; 2673 txq_conf.offloads = local_port_conf.txmode.offloads; 2674 retval = rte_eth_tx_queue_setup(portid, 0, nb_txd, 2675 rte_eth_dev_socket_id(portid), 2676 &txq_conf); 2677 if (retval < 0) { 2678 printf("rte_eth_tx_queue_setup:err=%d, port=%u\n", 2679 retval, portid); 2680 2681 return -1; 2682 } 2683 2684 /* Start device */ 2685 retval = rte_eth_dev_start(portid); 2686 if (retval < 0) { 2687 printf("rte_eth_dev_start:err=%d, port=%u\n", 2688 retval, portid); 2689 return -1; 2690 } 2691 2692 retval = rte_eth_promiscuous_enable(portid); 2693 if (retval != 0) { 2694 printf("rte_eth_promiscuous_enable:err=%s, port=%u\n", 2695 rte_strerror(-retval), portid); 2696 return -1; 2697 } 2698 2699 retval = rte_eth_macaddr_get(portid, 2700 &l2fwd_ports_eth_addr[portid]); 2701 if (retval < 0) { 2702 printf("rte_eth_macaddr_get :err=%d, port=%u\n", 2703 retval, portid); 2704 return -1; 2705 } 2706 2707 printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n", 2708 portid, 2709 RTE_ETHER_ADDR_BYTES(&l2fwd_ports_eth_addr[portid])); 2710 2711 /* initialize port stats */ 2712 memset(&port_statistics, 0, sizeof(port_statistics)); 2713 2714 /* Setup port forwarding table */ 2715 if (enabled_portcount % 2) { 2716 l2fwd_dst_ports[portid] = last_portid; 2717 l2fwd_dst_ports[last_portid] = portid; 2718 } else { 2719 last_portid = portid; 2720 } 2721 2722 l2fwd_enabled_port_mask |= (1 << portid); 2723 enabled_portcount++; 2724 } 2725 2726 if (enabled_portcount == 1) { 2727 l2fwd_dst_ports[last_portid] = last_portid; 2728 } else if (enabled_portcount % 2) { 2729 printf("odd number of ports in portmask- bye\n"); 2730 return -1; 2731 } 2732 2733 check_all_ports_link_status(l2fwd_enabled_port_mask); 2734 2735 return enabled_portcount; 2736 } 2737 2738 static void 2739 reserve_key_memory(struct l2fwd_crypto_options *options) 2740 { 2741 options->cipher_xform.cipher.key.data = options->cipher_key; 2742 2743 options->auth_xform.auth.key.data = options->auth_key; 2744 2745 options->aead_xform.aead.key.data = options->aead_key; 2746 2747 options->cipher_iv.data = rte_malloc("cipher iv", MAX_KEY_SIZE, 0); 2748 if (options->cipher_iv.data == NULL) 2749 rte_exit(EXIT_FAILURE, "Failed to allocate memory for cipher IV"); 2750 2751 options->auth_iv.data = rte_malloc("auth iv", MAX_KEY_SIZE, 0); 2752 if (options->auth_iv.data == NULL) 2753 rte_exit(EXIT_FAILURE, "Failed to allocate memory for auth IV"); 2754 2755 options->aead_iv.data = rte_malloc("aead_iv", MAX_KEY_SIZE, 0); 2756 if (options->aead_iv.data == NULL) 2757 rte_exit(EXIT_FAILURE, "Failed to allocate memory for AEAD iv"); 2758 2759 options->aad.data = rte_malloc("aad", MAX_KEY_SIZE, 0); 2760 if (options->aad.data == NULL) 2761 rte_exit(EXIT_FAILURE, "Failed to allocate memory for AAD"); 2762 options->aad.phys_addr = rte_malloc_virt2iova(options->aad.data); 2763 } 2764 2765 int 2766 main(int argc, char **argv) 2767 { 2768 struct lcore_queue_conf *qconf = NULL; 2769 struct l2fwd_crypto_options options; 2770 2771 uint8_t nb_cryptodevs, cdev_id; 2772 uint16_t portid; 2773 unsigned lcore_id, rx_lcore_id = 0; 2774 int ret, enabled_cdevcount, enabled_portcount; 2775 uint8_t enabled_cdevs[RTE_CRYPTO_MAX_DEVS] = {0}; 2776 2777 /* init EAL */ 2778 ret = rte_eal_init(argc, argv); 2779 if (ret < 0) 2780 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n"); 2781 argc -= ret; 2782 argv += ret; 2783 2784 /* reserve memory for Cipher/Auth key and IV */ 2785 reserve_key_memory(&options); 2786 2787 /* parse application arguments (after the EAL ones) */ 2788 ret = l2fwd_crypto_parse_args(&options, argc, argv); 2789 if (ret < 0) 2790 rte_exit(EXIT_FAILURE, "Invalid L2FWD-CRYPTO arguments\n"); 2791 2792 printf("MAC updating %s\n", 2793 options.mac_updating ? "enabled" : "disabled"); 2794 2795 /* create the mbuf pool */ 2796 l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF, 512, 2797 RTE_ALIGN(sizeof(struct rte_crypto_op), 2798 RTE_CACHE_LINE_SIZE), 2799 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 2800 if (l2fwd_pktmbuf_pool == NULL) 2801 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 2802 2803 /* create crypto op pool */ 2804 l2fwd_crypto_op_pool = rte_crypto_op_pool_create("crypto_op_pool", 2805 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MBUF, 128, MAXIMUM_IV_LENGTH, 2806 rte_socket_id()); 2807 if (l2fwd_crypto_op_pool == NULL) 2808 rte_exit(EXIT_FAILURE, "Cannot create crypto op pool\n"); 2809 2810 /* Enable Ethernet ports */ 2811 enabled_portcount = initialize_ports(&options); 2812 if (enabled_portcount < 1) 2813 rte_exit(EXIT_FAILURE, "Failed to initial Ethernet ports\n"); 2814 2815 /* Initialize the port/queue configuration of each logical core */ 2816 RTE_ETH_FOREACH_DEV(portid) { 2817 2818 /* skip ports that are not enabled */ 2819 if ((options.portmask & (1 << portid)) == 0) 2820 continue; 2821 2822 if (options.single_lcore && qconf == NULL) { 2823 while (rte_lcore_is_enabled(rx_lcore_id) == 0) { 2824 rx_lcore_id++; 2825 if (rx_lcore_id >= RTE_MAX_LCORE) 2826 rte_exit(EXIT_FAILURE, 2827 "Not enough cores\n"); 2828 } 2829 } else if (!options.single_lcore) { 2830 /* get the lcore_id for this port */ 2831 while (rte_lcore_is_enabled(rx_lcore_id) == 0 || 2832 lcore_queue_conf[rx_lcore_id].nb_rx_ports == 2833 options.nb_ports_per_lcore) { 2834 rx_lcore_id++; 2835 if (rx_lcore_id >= RTE_MAX_LCORE) 2836 rte_exit(EXIT_FAILURE, 2837 "Not enough cores\n"); 2838 } 2839 } 2840 2841 /* Assigned a new logical core in the loop above. */ 2842 if (qconf != &lcore_queue_conf[rx_lcore_id]) 2843 qconf = &lcore_queue_conf[rx_lcore_id]; 2844 2845 qconf->rx_port_list[qconf->nb_rx_ports] = portid; 2846 qconf->nb_rx_ports++; 2847 2848 printf("Lcore %u: RX port %u\n", rx_lcore_id, portid); 2849 } 2850 2851 /* Enable Crypto devices */ 2852 enabled_cdevcount = initialize_cryptodevs(&options, enabled_portcount, 2853 enabled_cdevs); 2854 if (enabled_cdevcount < 0) 2855 rte_exit(EXIT_FAILURE, "Failed to initialize crypto devices\n"); 2856 2857 if (enabled_cdevcount < enabled_portcount) 2858 rte_exit(EXIT_FAILURE, "Number of capable crypto devices (%d) " 2859 "has to be more or equal to number of ports (%d)\n", 2860 enabled_cdevcount, enabled_portcount); 2861 2862 nb_cryptodevs = rte_cryptodev_count(); 2863 2864 /* Initialize the port/cryptodev configuration of each logical core */ 2865 for (rx_lcore_id = 0, qconf = NULL, cdev_id = 0; 2866 cdev_id < nb_cryptodevs && enabled_cdevcount; 2867 cdev_id++) { 2868 /* Crypto op not supported by crypto device */ 2869 if (!enabled_cdevs[cdev_id]) 2870 continue; 2871 2872 if (options.single_lcore && qconf == NULL) { 2873 while (rte_lcore_is_enabled(rx_lcore_id) == 0) { 2874 rx_lcore_id++; 2875 if (rx_lcore_id >= RTE_MAX_LCORE) 2876 rte_exit(EXIT_FAILURE, 2877 "Not enough cores\n"); 2878 } 2879 } else if (!options.single_lcore) { 2880 /* get the lcore_id for this port */ 2881 while (rte_lcore_is_enabled(rx_lcore_id) == 0 || 2882 lcore_queue_conf[rx_lcore_id].nb_crypto_devs == 2883 options.nb_ports_per_lcore) { 2884 rx_lcore_id++; 2885 if (rx_lcore_id >= RTE_MAX_LCORE) 2886 rte_exit(EXIT_FAILURE, 2887 "Not enough cores\n"); 2888 } 2889 } 2890 2891 /* Assigned a new logical core in the loop above. */ 2892 if (qconf != &lcore_queue_conf[rx_lcore_id]) 2893 qconf = &lcore_queue_conf[rx_lcore_id]; 2894 2895 qconf->cryptodev_list[qconf->nb_crypto_devs] = cdev_id; 2896 qconf->nb_crypto_devs++; 2897 2898 enabled_cdevcount--; 2899 2900 printf("Lcore %u: cryptodev %u\n", rx_lcore_id, 2901 (unsigned)cdev_id); 2902 } 2903 2904 /* launch per-lcore init on every lcore */ 2905 rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, (void *)&options, 2906 CALL_MAIN); 2907 RTE_LCORE_FOREACH_WORKER(lcore_id) { 2908 if (rte_eal_wait_lcore(lcore_id) < 0) 2909 return -1; 2910 } 2911 2912 /* clean up the EAL */ 2913 rte_eal_cleanup(); 2914 2915 return 0; 2916 } 2917