1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2015-2016 Intel Corporation 3 */ 4 5 #include <time.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <inttypes.h> 11 #include <sys/types.h> 12 #include <sys/queue.h> 13 #include <netinet/in.h> 14 #include <setjmp.h> 15 #include <stdarg.h> 16 #include <ctype.h> 17 #include <errno.h> 18 #include <getopt.h> 19 #include <fcntl.h> 20 #include <unistd.h> 21 22 #include <rte_string_fns.h> 23 #include <rte_atomic.h> 24 #include <rte_branch_prediction.h> 25 #include <rte_common.h> 26 #include <rte_cryptodev.h> 27 #include <rte_cycles.h> 28 #include <rte_debug.h> 29 #include <rte_eal.h> 30 #include <rte_ether.h> 31 #include <rte_ethdev.h> 32 #include <rte_interrupts.h> 33 #include <rte_ip.h> 34 #include <rte_launch.h> 35 #include <rte_lcore.h> 36 #include <rte_log.h> 37 #include <rte_malloc.h> 38 #include <rte_mbuf.h> 39 #include <rte_memcpy.h> 40 #include <rte_memory.h> 41 #include <rte_mempool.h> 42 #include <rte_per_lcore.h> 43 #include <rte_prefetch.h> 44 #include <rte_random.h> 45 #include <rte_hexdump.h> 46 #ifdef RTE_CRYPTO_SCHEDULER 47 #include <rte_cryptodev_scheduler.h> 48 #endif 49 50 enum cdev_type { 51 CDEV_TYPE_ANY, 52 CDEV_TYPE_HW, 53 CDEV_TYPE_SW 54 }; 55 56 #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1 57 58 #define NB_MBUF 8192 59 60 #define MAX_STR_LEN 32 61 #define MAX_KEY_SIZE 128 62 #define MAX_IV_SIZE 16 63 #define MAX_AAD_SIZE 65535 64 #define MAX_PKT_BURST 32 65 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 66 #define SESSION_POOL_CACHE_SIZE 0 67 68 #define MAXIMUM_IV_LENGTH 16 69 #define IV_OFFSET (sizeof(struct rte_crypto_op) + \ 70 sizeof(struct rte_crypto_sym_op)) 71 72 /* 73 * Configurable number of RX/TX ring descriptors 74 */ 75 #define RTE_TEST_RX_DESC_DEFAULT 1024 76 #define RTE_TEST_TX_DESC_DEFAULT 1024 77 78 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; 79 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; 80 81 /* ethernet addresses of ports */ 82 static struct rte_ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS]; 83 84 /* mask of enabled ports */ 85 static uint64_t l2fwd_enabled_port_mask; 86 static uint64_t l2fwd_enabled_crypto_mask; 87 88 /* list of enabled ports */ 89 static uint16_t l2fwd_dst_ports[RTE_MAX_ETHPORTS]; 90 91 92 struct pkt_buffer { 93 unsigned len; 94 struct rte_mbuf *buffer[MAX_PKT_BURST]; 95 }; 96 97 struct op_buffer { 98 unsigned len; 99 struct rte_crypto_op *buffer[MAX_PKT_BURST]; 100 }; 101 102 #define MAX_RX_QUEUE_PER_LCORE 16 103 #define MAX_TX_QUEUE_PER_PORT 16 104 105 enum l2fwd_crypto_xform_chain { 106 L2FWD_CRYPTO_CIPHER_HASH, 107 L2FWD_CRYPTO_HASH_CIPHER, 108 L2FWD_CRYPTO_CIPHER_ONLY, 109 L2FWD_CRYPTO_HASH_ONLY, 110 L2FWD_CRYPTO_AEAD 111 }; 112 113 struct l2fwd_key { 114 uint8_t *data; 115 uint32_t length; 116 rte_iova_t phys_addr; 117 }; 118 119 struct l2fwd_iv { 120 uint8_t *data; 121 uint16_t length; 122 }; 123 124 /** l2fwd crypto application command line options */ 125 struct l2fwd_crypto_options { 126 unsigned portmask; 127 unsigned nb_ports_per_lcore; 128 unsigned refresh_period; 129 unsigned single_lcore:1; 130 131 enum cdev_type type; 132 unsigned sessionless:1; 133 134 enum l2fwd_crypto_xform_chain xform_chain; 135 136 struct rte_crypto_sym_xform cipher_xform; 137 unsigned ckey_param; 138 int ckey_random_size; 139 uint8_t cipher_key[MAX_KEY_SIZE]; 140 141 struct l2fwd_iv cipher_iv; 142 unsigned int cipher_iv_param; 143 int cipher_iv_random_size; 144 145 struct rte_crypto_sym_xform auth_xform; 146 uint8_t akey_param; 147 int akey_random_size; 148 uint8_t auth_key[MAX_KEY_SIZE]; 149 150 struct l2fwd_iv auth_iv; 151 unsigned int auth_iv_param; 152 int auth_iv_random_size; 153 154 struct rte_crypto_sym_xform aead_xform; 155 unsigned int aead_key_param; 156 int aead_key_random_size; 157 uint8_t aead_key[MAX_KEY_SIZE]; 158 159 struct l2fwd_iv aead_iv; 160 unsigned int aead_iv_param; 161 int aead_iv_random_size; 162 163 struct l2fwd_key aad; 164 unsigned aad_param; 165 int aad_random_size; 166 167 int digest_size; 168 169 uint16_t block_size; 170 char string_type[MAX_STR_LEN]; 171 172 uint64_t cryptodev_mask; 173 174 unsigned int mac_updating; 175 }; 176 177 /** l2fwd crypto lcore params */ 178 struct l2fwd_crypto_params { 179 uint8_t dev_id; 180 uint8_t qp_id; 181 182 unsigned digest_length; 183 unsigned block_size; 184 185 uint16_t cipher_dataunit_len; 186 187 struct l2fwd_iv cipher_iv; 188 struct l2fwd_iv auth_iv; 189 struct l2fwd_iv aead_iv; 190 struct l2fwd_key aad; 191 struct rte_cryptodev_sym_session *session; 192 193 uint8_t do_cipher; 194 uint8_t do_hash; 195 uint8_t do_aead; 196 uint8_t hash_verify; 197 198 enum rte_crypto_cipher_algorithm cipher_algo; 199 enum rte_crypto_auth_algorithm auth_algo; 200 enum rte_crypto_aead_algorithm aead_algo; 201 }; 202 203 /** lcore configuration */ 204 struct lcore_queue_conf { 205 unsigned nb_rx_ports; 206 uint16_t rx_port_list[MAX_RX_QUEUE_PER_LCORE]; 207 208 unsigned nb_crypto_devs; 209 unsigned cryptodev_list[MAX_RX_QUEUE_PER_LCORE]; 210 211 struct op_buffer op_buf[RTE_CRYPTO_MAX_DEVS]; 212 struct pkt_buffer pkt_buf[RTE_MAX_ETHPORTS]; 213 } __rte_cache_aligned; 214 215 struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE]; 216 217 static struct rte_eth_conf port_conf = { 218 .rxmode = { 219 .mq_mode = ETH_MQ_RX_NONE, 220 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 221 .split_hdr_size = 0, 222 }, 223 .txmode = { 224 .mq_mode = ETH_MQ_TX_NONE, 225 }, 226 }; 227 228 struct rte_mempool *l2fwd_pktmbuf_pool; 229 struct rte_mempool *l2fwd_crypto_op_pool; 230 static struct { 231 struct rte_mempool *sess_mp; 232 struct rte_mempool *priv_mp; 233 } session_pool_socket[RTE_MAX_NUMA_NODES]; 234 235 /* Per-port statistics struct */ 236 struct l2fwd_port_statistics { 237 uint64_t tx; 238 uint64_t rx; 239 240 uint64_t crypto_enqueued; 241 uint64_t crypto_dequeued; 242 243 uint64_t dropped; 244 } __rte_cache_aligned; 245 246 struct l2fwd_crypto_statistics { 247 uint64_t enqueued; 248 uint64_t dequeued; 249 250 uint64_t errors; 251 } __rte_cache_aligned; 252 253 struct l2fwd_port_statistics port_statistics[RTE_MAX_ETHPORTS]; 254 struct l2fwd_crypto_statistics crypto_statistics[RTE_CRYPTO_MAX_DEVS]; 255 256 /* A tsc-based timer responsible for triggering statistics printout */ 257 #define TIMER_MILLISECOND 2000000ULL /* around 1ms at 2 Ghz */ 258 #define MAX_TIMER_PERIOD 86400UL /* 1 day max */ 259 260 /* default period is 10 seconds */ 261 static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000; 262 263 /* Print out statistics on packets dropped */ 264 static void 265 print_stats(void) 266 { 267 uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; 268 uint64_t total_packets_enqueued, total_packets_dequeued, 269 total_packets_errors; 270 uint16_t portid; 271 uint64_t cdevid; 272 273 total_packets_dropped = 0; 274 total_packets_tx = 0; 275 total_packets_rx = 0; 276 total_packets_enqueued = 0; 277 total_packets_dequeued = 0; 278 total_packets_errors = 0; 279 280 const char clr[] = { 27, '[', '2', 'J', '\0' }; 281 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 282 283 /* Clear screen and move to top left */ 284 printf("%s%s", clr, topLeft); 285 286 printf("\nPort statistics ===================================="); 287 288 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 289 /* skip disabled ports */ 290 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) 291 continue; 292 printf("\nStatistics for port %u ------------------------------" 293 "\nPackets sent: %32"PRIu64 294 "\nPackets received: %28"PRIu64 295 "\nPackets dropped: %29"PRIu64, 296 portid, 297 port_statistics[portid].tx, 298 port_statistics[portid].rx, 299 port_statistics[portid].dropped); 300 301 total_packets_dropped += port_statistics[portid].dropped; 302 total_packets_tx += port_statistics[portid].tx; 303 total_packets_rx += port_statistics[portid].rx; 304 } 305 printf("\nCrypto statistics =================================="); 306 307 for (cdevid = 0; cdevid < RTE_CRYPTO_MAX_DEVS; cdevid++) { 308 /* skip disabled ports */ 309 if ((l2fwd_enabled_crypto_mask & (((uint64_t)1) << cdevid)) == 0) 310 continue; 311 printf("\nStatistics for cryptodev %"PRIu64 312 " -------------------------" 313 "\nPackets enqueued: %28"PRIu64 314 "\nPackets dequeued: %28"PRIu64 315 "\nPackets errors: %30"PRIu64, 316 cdevid, 317 crypto_statistics[cdevid].enqueued, 318 crypto_statistics[cdevid].dequeued, 319 crypto_statistics[cdevid].errors); 320 321 total_packets_enqueued += crypto_statistics[cdevid].enqueued; 322 total_packets_dequeued += crypto_statistics[cdevid].dequeued; 323 total_packets_errors += crypto_statistics[cdevid].errors; 324 } 325 printf("\nAggregate statistics ===============================" 326 "\nTotal packets received: %22"PRIu64 327 "\nTotal packets enqueued: %22"PRIu64 328 "\nTotal packets dequeued: %22"PRIu64 329 "\nTotal packets sent: %26"PRIu64 330 "\nTotal packets dropped: %23"PRIu64 331 "\nTotal packets crypto errors: %17"PRIu64, 332 total_packets_rx, 333 total_packets_enqueued, 334 total_packets_dequeued, 335 total_packets_tx, 336 total_packets_dropped, 337 total_packets_errors); 338 printf("\n====================================================\n"); 339 340 fflush(stdout); 341 } 342 343 /* l2fwd_crypto_send_burst 8< */ 344 static int 345 l2fwd_crypto_send_burst(struct lcore_queue_conf *qconf, unsigned n, 346 struct l2fwd_crypto_params *cparams) 347 { 348 struct rte_crypto_op **op_buffer; 349 unsigned ret; 350 351 op_buffer = (struct rte_crypto_op **) 352 qconf->op_buf[cparams->dev_id].buffer; 353 354 ret = rte_cryptodev_enqueue_burst(cparams->dev_id, 355 cparams->qp_id, op_buffer, (uint16_t) n); 356 357 crypto_statistics[cparams->dev_id].enqueued += ret; 358 if (unlikely(ret < n)) { 359 crypto_statistics[cparams->dev_id].errors += (n - ret); 360 do { 361 rte_pktmbuf_free(op_buffer[ret]->sym->m_src); 362 rte_crypto_op_free(op_buffer[ret]); 363 } while (++ret < n); 364 } 365 366 return 0; 367 } 368 /* >8 End of l2fwd_crypto_send_burst. */ 369 370 /* Crypto enqueue. 8< */ 371 static int 372 l2fwd_crypto_enqueue(struct rte_crypto_op *op, 373 struct l2fwd_crypto_params *cparams) 374 { 375 unsigned lcore_id, len; 376 struct lcore_queue_conf *qconf; 377 378 lcore_id = rte_lcore_id(); 379 380 qconf = &lcore_queue_conf[lcore_id]; 381 len = qconf->op_buf[cparams->dev_id].len; 382 qconf->op_buf[cparams->dev_id].buffer[len] = op; 383 len++; 384 385 /* enough ops to be sent */ 386 if (len == MAX_PKT_BURST) { 387 l2fwd_crypto_send_burst(qconf, MAX_PKT_BURST, cparams); 388 len = 0; 389 } 390 391 qconf->op_buf[cparams->dev_id].len = len; 392 return 0; 393 } 394 /* >8 End of crypto enqueue. */ 395 396 static int 397 l2fwd_simple_crypto_enqueue(struct rte_mbuf *m, 398 struct rte_crypto_op *op, 399 struct l2fwd_crypto_params *cparams) 400 { 401 struct rte_ether_hdr *eth_hdr; 402 struct rte_ipv4_hdr *ip_hdr; 403 404 uint32_t ipdata_offset, data_len; 405 uint32_t pad_len = 0; 406 char *padding; 407 408 eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); 409 410 if (eth_hdr->ether_type != rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) 411 return -1; 412 413 ipdata_offset = sizeof(struct rte_ether_hdr); 414 415 ip_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, char *) + 416 ipdata_offset); 417 418 ipdata_offset += (ip_hdr->version_ihl & RTE_IPV4_HDR_IHL_MASK) 419 * RTE_IPV4_IHL_MULTIPLIER; 420 421 422 /* Zero pad data to be crypto'd so it is block aligned */ 423 data_len = rte_pktmbuf_data_len(m) - ipdata_offset; 424 425 if ((cparams->do_hash || cparams->do_aead) && cparams->hash_verify) 426 data_len -= cparams->digest_length; 427 428 if (cparams->do_cipher) { 429 /* 430 * Following algorithms are block cipher algorithms, 431 * and might need padding 432 */ 433 switch (cparams->cipher_algo) { 434 case RTE_CRYPTO_CIPHER_AES_CBC: 435 case RTE_CRYPTO_CIPHER_AES_ECB: 436 case RTE_CRYPTO_CIPHER_DES_CBC: 437 case RTE_CRYPTO_CIPHER_3DES_CBC: 438 case RTE_CRYPTO_CIPHER_3DES_ECB: 439 if (data_len % cparams->block_size) 440 pad_len = cparams->block_size - 441 (data_len % cparams->block_size); 442 break; 443 case RTE_CRYPTO_CIPHER_AES_XTS: 444 if (cparams->cipher_dataunit_len != 0 && 445 (data_len % cparams->cipher_dataunit_len)) 446 pad_len = cparams->cipher_dataunit_len - 447 (data_len % cparams->cipher_dataunit_len); 448 break; 449 default: 450 pad_len = 0; 451 } 452 453 if (pad_len) { 454 padding = rte_pktmbuf_append(m, pad_len); 455 if (unlikely(!padding)) 456 return -1; 457 458 data_len += pad_len; 459 memset(padding, 0, pad_len); 460 } 461 } 462 463 /* Set crypto operation data parameters */ 464 rte_crypto_op_attach_sym_session(op, cparams->session); 465 466 if (cparams->do_hash) { 467 if (cparams->auth_iv.length) { 468 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, 469 uint8_t *, 470 IV_OFFSET + 471 cparams->cipher_iv.length); 472 /* 473 * Copy IV at the end of the crypto operation, 474 * after the cipher IV, if added 475 */ 476 rte_memcpy(iv_ptr, cparams->auth_iv.data, 477 cparams->auth_iv.length); 478 } 479 if (!cparams->hash_verify) { 480 /* Append space for digest to end of packet */ 481 op->sym->auth.digest.data = (uint8_t *)rte_pktmbuf_append(m, 482 cparams->digest_length); 483 } else { 484 op->sym->auth.digest.data = rte_pktmbuf_mtod(m, 485 uint8_t *) + ipdata_offset + data_len; 486 } 487 488 op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m, 489 rte_pktmbuf_pkt_len(m) - cparams->digest_length); 490 491 /* For wireless algorithms, offset/length must be in bits */ 492 if (cparams->auth_algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2 || 493 cparams->auth_algo == RTE_CRYPTO_AUTH_KASUMI_F9 || 494 cparams->auth_algo == RTE_CRYPTO_AUTH_ZUC_EIA3) { 495 op->sym->auth.data.offset = ipdata_offset << 3; 496 op->sym->auth.data.length = data_len << 3; 497 } else { 498 op->sym->auth.data.offset = ipdata_offset; 499 op->sym->auth.data.length = data_len; 500 } 501 } 502 503 if (cparams->do_cipher) { 504 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 505 IV_OFFSET); 506 /* Copy IV at the end of the crypto operation */ 507 rte_memcpy(iv_ptr, cparams->cipher_iv.data, 508 cparams->cipher_iv.length); 509 510 /* For wireless algorithms, offset/length must be in bits */ 511 if (cparams->cipher_algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 || 512 cparams->cipher_algo == RTE_CRYPTO_CIPHER_KASUMI_F8 || 513 cparams->cipher_algo == RTE_CRYPTO_CIPHER_ZUC_EEA3) { 514 op->sym->cipher.data.offset = ipdata_offset << 3; 515 op->sym->cipher.data.length = data_len << 3; 516 } else { 517 op->sym->cipher.data.offset = ipdata_offset; 518 op->sym->cipher.data.length = data_len; 519 } 520 } 521 522 if (cparams->do_aead) { 523 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 524 IV_OFFSET); 525 /* Copy IV at the end of the crypto operation */ 526 /* 527 * If doing AES-CCM, nonce is copied one byte 528 * after the start of IV field 529 */ 530 if (cparams->aead_algo == RTE_CRYPTO_AEAD_AES_CCM) 531 rte_memcpy(iv_ptr + 1, cparams->aead_iv.data, 532 cparams->aead_iv.length); 533 else 534 rte_memcpy(iv_ptr, cparams->aead_iv.data, 535 cparams->aead_iv.length); 536 537 op->sym->aead.data.offset = ipdata_offset; 538 op->sym->aead.data.length = data_len; 539 540 if (!cparams->hash_verify) { 541 /* Append space for digest to end of packet */ 542 op->sym->aead.digest.data = (uint8_t *)rte_pktmbuf_append(m, 543 cparams->digest_length); 544 } else { 545 op->sym->aead.digest.data = rte_pktmbuf_mtod(m, 546 uint8_t *) + ipdata_offset + data_len; 547 } 548 549 op->sym->aead.digest.phys_addr = rte_pktmbuf_iova_offset(m, 550 rte_pktmbuf_pkt_len(m) - cparams->digest_length); 551 552 if (cparams->aad.length) { 553 op->sym->aead.aad.data = cparams->aad.data; 554 op->sym->aead.aad.phys_addr = cparams->aad.phys_addr; 555 } 556 } 557 558 op->sym->m_src = m; 559 560 return l2fwd_crypto_enqueue(op, cparams); 561 } 562 563 564 /* Send the burst of packets on an output interface */ 565 static int 566 l2fwd_send_burst(struct lcore_queue_conf *qconf, unsigned n, 567 uint16_t port) 568 { 569 struct rte_mbuf **pkt_buffer; 570 unsigned ret; 571 572 pkt_buffer = (struct rte_mbuf **)qconf->pkt_buf[port].buffer; 573 574 ret = rte_eth_tx_burst(port, 0, pkt_buffer, (uint16_t)n); 575 port_statistics[port].tx += ret; 576 if (unlikely(ret < n)) { 577 port_statistics[port].dropped += (n - ret); 578 do { 579 rte_pktmbuf_free(pkt_buffer[ret]); 580 } while (++ret < n); 581 } 582 583 return 0; 584 } 585 586 /* Enqueue packets for TX and prepare them to be sent. 8< */ 587 static int 588 l2fwd_send_packet(struct rte_mbuf *m, uint16_t port) 589 { 590 unsigned lcore_id, len; 591 struct lcore_queue_conf *qconf; 592 593 lcore_id = rte_lcore_id(); 594 595 qconf = &lcore_queue_conf[lcore_id]; 596 len = qconf->pkt_buf[port].len; 597 qconf->pkt_buf[port].buffer[len] = m; 598 len++; 599 600 /* enough pkts to be sent */ 601 if (unlikely(len == MAX_PKT_BURST)) { 602 l2fwd_send_burst(qconf, MAX_PKT_BURST, port); 603 len = 0; 604 } 605 606 qconf->pkt_buf[port].len = len; 607 return 0; 608 } 609 /* >8 End of Enqueuing packets for TX. */ 610 611 static void 612 l2fwd_mac_updating(struct rte_mbuf *m, uint16_t dest_portid) 613 { 614 struct rte_ether_hdr *eth; 615 void *tmp; 616 617 eth = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); 618 619 /* 02:00:00:00:00:xx */ 620 tmp = ð->d_addr.addr_bytes[0]; 621 *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dest_portid << 40); 622 623 /* src addr */ 624 rte_ether_addr_copy(&l2fwd_ports_eth_addr[dest_portid], ð->s_addr); 625 } 626 627 static void 628 l2fwd_simple_forward(struct rte_mbuf *m, uint16_t portid, 629 struct l2fwd_crypto_options *options) 630 { 631 uint16_t dst_port; 632 uint32_t pad_len; 633 struct rte_ipv4_hdr *ip_hdr; 634 uint32_t ipdata_offset = sizeof(struct rte_ether_hdr); 635 636 ip_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, char *) + 637 ipdata_offset); 638 dst_port = l2fwd_dst_ports[portid]; 639 640 if (options->mac_updating) 641 l2fwd_mac_updating(m, dst_port); 642 643 if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_VERIFY) 644 rte_pktmbuf_trim(m, options->auth_xform.auth.digest_length); 645 646 if (options->cipher_xform.cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { 647 pad_len = m->pkt_len - rte_be_to_cpu_16(ip_hdr->total_length) - 648 ipdata_offset; 649 rte_pktmbuf_trim(m, pad_len); 650 } 651 652 l2fwd_send_packet(m, dst_port); 653 } 654 655 /** Generate random key */ 656 static void 657 generate_random_key(uint8_t *key, unsigned length) 658 { 659 int fd; 660 int ret; 661 662 fd = open("/dev/urandom", O_RDONLY); 663 if (fd < 0) 664 rte_exit(EXIT_FAILURE, "Failed to generate random key\n"); 665 666 ret = read(fd, key, length); 667 close(fd); 668 669 if (ret != (signed)length) 670 rte_exit(EXIT_FAILURE, "Failed to generate random key\n"); 671 } 672 673 /* Session is created and is later attached to the crypto operation. 8< */ 674 static struct rte_cryptodev_sym_session * 675 initialize_crypto_session(struct l2fwd_crypto_options *options, uint8_t cdev_id) 676 { 677 struct rte_crypto_sym_xform *first_xform; 678 struct rte_cryptodev_sym_session *session; 679 int retval = rte_cryptodev_socket_id(cdev_id); 680 681 if (retval < 0) 682 return NULL; 683 684 uint8_t socket_id = (uint8_t) retval; 685 686 if (options->xform_chain == L2FWD_CRYPTO_AEAD) { 687 first_xform = &options->aead_xform; 688 } else if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH) { 689 first_xform = &options->cipher_xform; 690 first_xform->next = &options->auth_xform; 691 } else if (options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER) { 692 first_xform = &options->auth_xform; 693 first_xform->next = &options->cipher_xform; 694 } else if (options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { 695 first_xform = &options->cipher_xform; 696 } else { 697 first_xform = &options->auth_xform; 698 } 699 700 session = rte_cryptodev_sym_session_create( 701 session_pool_socket[socket_id].sess_mp); 702 if (session == NULL) 703 return NULL; 704 705 if (rte_cryptodev_sym_session_init(cdev_id, session, 706 first_xform, 707 session_pool_socket[socket_id].priv_mp) < 0) 708 return NULL; 709 710 return session; 711 } 712 /* >8 End of creation of session. */ 713 714 static void 715 l2fwd_crypto_options_print(struct l2fwd_crypto_options *options); 716 717 /* main processing loop */ 718 static void 719 l2fwd_main_loop(struct l2fwd_crypto_options *options) 720 { 721 struct rte_mbuf *m, *pkts_burst[MAX_PKT_BURST]; 722 struct rte_crypto_op *ops_burst[MAX_PKT_BURST]; 723 724 unsigned lcore_id = rte_lcore_id(); 725 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 726 unsigned int i, j, nb_rx, len; 727 uint16_t portid; 728 struct lcore_queue_conf *qconf = &lcore_queue_conf[lcore_id]; 729 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 730 US_PER_S * BURST_TX_DRAIN_US; 731 struct l2fwd_crypto_params *cparams; 732 struct l2fwd_crypto_params port_cparams[qconf->nb_crypto_devs]; 733 struct rte_cryptodev_sym_session *session; 734 735 if (qconf->nb_rx_ports == 0) { 736 RTE_LOG(INFO, L2FWD, "lcore %u has nothing to do\n", lcore_id); 737 return; 738 } 739 740 RTE_LOG(INFO, L2FWD, "entering main loop on lcore %u\n", lcore_id); 741 742 for (i = 0; i < qconf->nb_rx_ports; i++) { 743 744 portid = qconf->rx_port_list[i]; 745 RTE_LOG(INFO, L2FWD, " -- lcoreid=%u portid=%u\n", lcore_id, 746 portid); 747 } 748 749 for (i = 0; i < qconf->nb_crypto_devs; i++) { 750 port_cparams[i].do_cipher = 0; 751 port_cparams[i].do_hash = 0; 752 port_cparams[i].do_aead = 0; 753 754 switch (options->xform_chain) { 755 case L2FWD_CRYPTO_AEAD: 756 port_cparams[i].do_aead = 1; 757 break; 758 case L2FWD_CRYPTO_CIPHER_HASH: 759 case L2FWD_CRYPTO_HASH_CIPHER: 760 port_cparams[i].do_cipher = 1; 761 port_cparams[i].do_hash = 1; 762 break; 763 case L2FWD_CRYPTO_HASH_ONLY: 764 port_cparams[i].do_hash = 1; 765 break; 766 case L2FWD_CRYPTO_CIPHER_ONLY: 767 port_cparams[i].do_cipher = 1; 768 break; 769 } 770 771 port_cparams[i].dev_id = qconf->cryptodev_list[i]; 772 port_cparams[i].qp_id = 0; 773 774 port_cparams[i].block_size = options->block_size; 775 776 if (port_cparams[i].do_hash) { 777 port_cparams[i].auth_iv.data = options->auth_iv.data; 778 port_cparams[i].auth_iv.length = options->auth_iv.length; 779 if (!options->auth_iv_param) 780 generate_random_key(port_cparams[i].auth_iv.data, 781 port_cparams[i].auth_iv.length); 782 if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_VERIFY) 783 port_cparams[i].hash_verify = 1; 784 else 785 port_cparams[i].hash_verify = 0; 786 787 port_cparams[i].auth_algo = options->auth_xform.auth.algo; 788 port_cparams[i].digest_length = 789 options->auth_xform.auth.digest_length; 790 /* Set IV parameters */ 791 if (options->auth_iv.length) { 792 options->auth_xform.auth.iv.offset = 793 IV_OFFSET + options->cipher_iv.length; 794 options->auth_xform.auth.iv.length = 795 options->auth_iv.length; 796 } 797 } 798 799 if (port_cparams[i].do_aead) { 800 port_cparams[i].aead_iv.data = options->aead_iv.data; 801 port_cparams[i].aead_iv.length = options->aead_iv.length; 802 if (!options->aead_iv_param) 803 generate_random_key(port_cparams[i].aead_iv.data, 804 port_cparams[i].aead_iv.length); 805 port_cparams[i].aead_algo = options->aead_xform.aead.algo; 806 port_cparams[i].digest_length = 807 options->aead_xform.aead.digest_length; 808 if (options->aead_xform.aead.aad_length) { 809 port_cparams[i].aad.data = options->aad.data; 810 port_cparams[i].aad.phys_addr = options->aad.phys_addr; 811 port_cparams[i].aad.length = options->aad.length; 812 if (!options->aad_param) 813 generate_random_key(port_cparams[i].aad.data, 814 port_cparams[i].aad.length); 815 /* 816 * If doing AES-CCM, first 18 bytes has to be reserved, 817 * and actual AAD should start from byte 18 818 */ 819 if (port_cparams[i].aead_algo == RTE_CRYPTO_AEAD_AES_CCM) 820 memmove(port_cparams[i].aad.data + 18, 821 port_cparams[i].aad.data, 822 port_cparams[i].aad.length); 823 824 } else 825 port_cparams[i].aad.length = 0; 826 827 if (options->aead_xform.aead.op == RTE_CRYPTO_AEAD_OP_DECRYPT) 828 port_cparams[i].hash_verify = 1; 829 else 830 port_cparams[i].hash_verify = 0; 831 832 /* Set IV parameters */ 833 options->aead_xform.aead.iv.offset = IV_OFFSET; 834 options->aead_xform.aead.iv.length = options->aead_iv.length; 835 } 836 837 if (port_cparams[i].do_cipher) { 838 port_cparams[i].cipher_iv.data = options->cipher_iv.data; 839 port_cparams[i].cipher_iv.length = options->cipher_iv.length; 840 if (!options->cipher_iv_param) 841 generate_random_key(port_cparams[i].cipher_iv.data, 842 port_cparams[i].cipher_iv.length); 843 844 port_cparams[i].cipher_algo = options->cipher_xform.cipher.algo; 845 port_cparams[i].cipher_dataunit_len = 846 options->cipher_xform.cipher.dataunit_len; 847 /* Set IV parameters */ 848 options->cipher_xform.cipher.iv.offset = IV_OFFSET; 849 options->cipher_xform.cipher.iv.length = 850 options->cipher_iv.length; 851 } 852 853 session = initialize_crypto_session(options, 854 port_cparams[i].dev_id); 855 if (session == NULL) 856 rte_exit(EXIT_FAILURE, "Failed to initialize crypto session\n"); 857 858 port_cparams[i].session = session; 859 860 RTE_LOG(INFO, L2FWD, " -- lcoreid=%u cryptoid=%u\n", lcore_id, 861 port_cparams[i].dev_id); 862 } 863 864 l2fwd_crypto_options_print(options); 865 866 /* 867 * Initialize previous tsc timestamp before the loop, 868 * to avoid showing the port statistics immediately, 869 * so user can see the crypto information. 870 */ 871 prev_tsc = rte_rdtsc(); 872 while (1) { 873 874 cur_tsc = rte_rdtsc(); 875 876 /* 877 * Crypto device/TX burst queue drain 878 */ 879 diff_tsc = cur_tsc - prev_tsc; 880 if (unlikely(diff_tsc > drain_tsc)) { 881 /* Enqueue all crypto ops remaining in buffers */ 882 for (i = 0; i < qconf->nb_crypto_devs; i++) { 883 cparams = &port_cparams[i]; 884 len = qconf->op_buf[cparams->dev_id].len; 885 l2fwd_crypto_send_burst(qconf, len, cparams); 886 qconf->op_buf[cparams->dev_id].len = 0; 887 } 888 /* Transmit all packets remaining in buffers */ 889 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 890 if (qconf->pkt_buf[portid].len == 0) 891 continue; 892 l2fwd_send_burst(&lcore_queue_conf[lcore_id], 893 qconf->pkt_buf[portid].len, 894 portid); 895 qconf->pkt_buf[portid].len = 0; 896 } 897 898 /* if timer is enabled */ 899 if (timer_period > 0) { 900 901 /* advance the timer */ 902 timer_tsc += diff_tsc; 903 904 /* if timer has reached its timeout */ 905 if (unlikely(timer_tsc >= 906 (uint64_t)timer_period)) { 907 908 /* do this only on main core */ 909 if (lcore_id == rte_get_main_lcore() 910 && options->refresh_period) { 911 print_stats(); 912 timer_tsc = 0; 913 } 914 } 915 } 916 917 prev_tsc = cur_tsc; 918 } 919 920 /* 921 * Read packet from RX queues 922 */ 923 for (i = 0; i < qconf->nb_rx_ports; i++) { 924 portid = qconf->rx_port_list[i]; 925 926 cparams = &port_cparams[i]; 927 928 nb_rx = rte_eth_rx_burst(portid, 0, 929 pkts_burst, MAX_PKT_BURST); 930 931 port_statistics[portid].rx += nb_rx; 932 933 /* Allocate and fillcrypto operations. 8< */ 934 if (nb_rx) { 935 /* 936 * If we can't allocate a crypto_ops, then drop 937 * the rest of the burst and dequeue and 938 * process the packets to free offload structs 939 */ 940 if (rte_crypto_op_bulk_alloc( 941 l2fwd_crypto_op_pool, 942 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 943 ops_burst, nb_rx) != 944 nb_rx) { 945 for (j = 0; j < nb_rx; j++) 946 rte_pktmbuf_free(pkts_burst[j]); 947 948 nb_rx = 0; 949 } 950 /* >8 End of crypto operation allocated and filled. */ 951 952 /* Enqueue packets from Crypto device*/ 953 for (j = 0; j < nb_rx; j++) { 954 m = pkts_burst[j]; 955 956 l2fwd_simple_crypto_enqueue(m, 957 ops_burst[j], cparams); 958 } 959 } 960 961 /* Dequeue packets from Crypto device. 8< */ 962 do { 963 nb_rx = rte_cryptodev_dequeue_burst( 964 cparams->dev_id, cparams->qp_id, 965 ops_burst, MAX_PKT_BURST); 966 967 crypto_statistics[cparams->dev_id].dequeued += 968 nb_rx; 969 970 /* Forward crypto'd packets */ 971 for (j = 0; j < nb_rx; j++) { 972 m = ops_burst[j]->sym->m_src; 973 974 rte_crypto_op_free(ops_burst[j]); 975 l2fwd_simple_forward(m, portid, 976 options); 977 } 978 } while (nb_rx == MAX_PKT_BURST); 979 /* >8 End of dequeue packets from crypto device. */ 980 } 981 } 982 } 983 984 static int 985 l2fwd_launch_one_lcore(void *arg) 986 { 987 l2fwd_main_loop((struct l2fwd_crypto_options *)arg); 988 return 0; 989 } 990 991 /* Display command line arguments usage */ 992 static void 993 l2fwd_crypto_usage(const char *prgname) 994 { 995 printf("%s [EAL options] --\n" 996 " -p PORTMASK: hexadecimal bitmask of ports to configure\n" 997 " -q NQ: number of queue (=ports) per lcore (default is 1)\n" 998 " -s manage all ports from single lcore\n" 999 " -T PERIOD: statistics will be refreshed each PERIOD seconds" 1000 " (0 to disable, 10 default, 86400 maximum)\n" 1001 1002 " --cdev_type HW / SW / ANY\n" 1003 " --chain HASH_CIPHER / CIPHER_HASH / CIPHER_ONLY /" 1004 " HASH_ONLY / AEAD\n" 1005 1006 " --cipher_algo ALGO\n" 1007 " --cipher_op ENCRYPT / DECRYPT\n" 1008 " --cipher_key KEY (bytes separated with \":\")\n" 1009 " --cipher_key_random_size SIZE: size of cipher key when generated randomly\n" 1010 " --cipher_iv IV (bytes separated with \":\")\n" 1011 " --cipher_iv_random_size SIZE: size of cipher IV when generated randomly\n" 1012 " --cipher_dataunit_len SIZE: length of the algorithm data-unit\n" 1013 1014 " --auth_algo ALGO\n" 1015 " --auth_op GENERATE / VERIFY\n" 1016 " --auth_key KEY (bytes separated with \":\")\n" 1017 " --auth_key_random_size SIZE: size of auth key when generated randomly\n" 1018 " --auth_iv IV (bytes separated with \":\")\n" 1019 " --auth_iv_random_size SIZE: size of auth IV when generated randomly\n" 1020 1021 " --aead_algo ALGO\n" 1022 " --aead_op ENCRYPT / DECRYPT\n" 1023 " --aead_key KEY (bytes separated with \":\")\n" 1024 " --aead_key_random_size SIZE: size of AEAD key when generated randomly\n" 1025 " --aead_iv IV (bytes separated with \":\")\n" 1026 " --aead_iv_random_size SIZE: size of AEAD IV when generated randomly\n" 1027 " --aad AAD (bytes separated with \":\")\n" 1028 " --aad_random_size SIZE: size of AAD when generated randomly\n" 1029 1030 " --digest_size SIZE: size of digest to be generated/verified\n" 1031 1032 " --sessionless\n" 1033 " --cryptodev_mask MASK: hexadecimal bitmask of crypto devices to configure\n" 1034 1035 " --[no-]mac-updating: Enable or disable MAC addresses updating (enabled by default)\n" 1036 " When enabled:\n" 1037 " - The source MAC address is replaced by the TX port MAC address\n" 1038 " - The destination MAC address is replaced by 02:00:00:00:00:TX_PORT_ID\n", 1039 prgname); 1040 } 1041 1042 /** Parse crypto device type command line argument */ 1043 static int 1044 parse_cryptodev_type(enum cdev_type *type, char *optarg) 1045 { 1046 if (strcmp("HW", optarg) == 0) { 1047 *type = CDEV_TYPE_HW; 1048 return 0; 1049 } else if (strcmp("SW", optarg) == 0) { 1050 *type = CDEV_TYPE_SW; 1051 return 0; 1052 } else if (strcmp("ANY", optarg) == 0) { 1053 *type = CDEV_TYPE_ANY; 1054 return 0; 1055 } 1056 1057 return -1; 1058 } 1059 1060 /** Parse crypto chain xform command line argument */ 1061 static int 1062 parse_crypto_opt_chain(struct l2fwd_crypto_options *options, char *optarg) 1063 { 1064 if (strcmp("CIPHER_HASH", optarg) == 0) { 1065 options->xform_chain = L2FWD_CRYPTO_CIPHER_HASH; 1066 return 0; 1067 } else if (strcmp("HASH_CIPHER", optarg) == 0) { 1068 options->xform_chain = L2FWD_CRYPTO_HASH_CIPHER; 1069 return 0; 1070 } else if (strcmp("CIPHER_ONLY", optarg) == 0) { 1071 options->xform_chain = L2FWD_CRYPTO_CIPHER_ONLY; 1072 return 0; 1073 } else if (strcmp("HASH_ONLY", optarg) == 0) { 1074 options->xform_chain = L2FWD_CRYPTO_HASH_ONLY; 1075 return 0; 1076 } else if (strcmp("AEAD", optarg) == 0) { 1077 options->xform_chain = L2FWD_CRYPTO_AEAD; 1078 return 0; 1079 } 1080 1081 return -1; 1082 } 1083 1084 /** Parse crypto cipher algo option command line argument */ 1085 static int 1086 parse_cipher_algo(enum rte_crypto_cipher_algorithm *algo, char *optarg) 1087 { 1088 1089 if (rte_cryptodev_get_cipher_algo_enum(algo, optarg) < 0) { 1090 RTE_LOG(ERR, USER1, "Cipher algorithm specified " 1091 "not supported!\n"); 1092 return -1; 1093 } 1094 1095 return 0; 1096 } 1097 1098 /** Parse crypto cipher operation command line argument */ 1099 static int 1100 parse_cipher_op(enum rte_crypto_cipher_operation *op, char *optarg) 1101 { 1102 if (strcmp("ENCRYPT", optarg) == 0) { 1103 *op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 1104 return 0; 1105 } else if (strcmp("DECRYPT", optarg) == 0) { 1106 *op = RTE_CRYPTO_CIPHER_OP_DECRYPT; 1107 return 0; 1108 } 1109 1110 printf("Cipher operation not supported!\n"); 1111 return -1; 1112 } 1113 1114 /** Parse bytes from command line argument */ 1115 static int 1116 parse_bytes(uint8_t *data, char *input_arg, uint16_t max_size) 1117 { 1118 unsigned byte_count; 1119 char *token; 1120 1121 errno = 0; 1122 for (byte_count = 0, token = strtok(input_arg, ":"); 1123 (byte_count < max_size) && (token != NULL); 1124 token = strtok(NULL, ":")) { 1125 1126 int number = (int)strtol(token, NULL, 16); 1127 1128 if (errno == EINVAL || errno == ERANGE || number > 0xFF) 1129 return -1; 1130 1131 data[byte_count++] = (uint8_t)number; 1132 } 1133 1134 return byte_count; 1135 } 1136 1137 /** Parse size param*/ 1138 static int 1139 parse_size(int *size, const char *q_arg) 1140 { 1141 char *end = NULL; 1142 unsigned long n; 1143 1144 /* parse hexadecimal string */ 1145 n = strtoul(q_arg, &end, 10); 1146 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) 1147 n = 0; 1148 1149 if (n == 0) { 1150 printf("invalid size\n"); 1151 return -1; 1152 } 1153 1154 *size = n; 1155 return 0; 1156 } 1157 1158 /** Parse crypto cipher operation command line argument */ 1159 static int 1160 parse_auth_algo(enum rte_crypto_auth_algorithm *algo, char *optarg) 1161 { 1162 if (rte_cryptodev_get_auth_algo_enum(algo, optarg) < 0) { 1163 RTE_LOG(ERR, USER1, "Authentication algorithm specified " 1164 "not supported!\n"); 1165 return -1; 1166 } 1167 1168 return 0; 1169 } 1170 1171 static int 1172 parse_auth_op(enum rte_crypto_auth_operation *op, char *optarg) 1173 { 1174 if (strcmp("VERIFY", optarg) == 0) { 1175 *op = RTE_CRYPTO_AUTH_OP_VERIFY; 1176 return 0; 1177 } else if (strcmp("GENERATE", optarg) == 0) { 1178 *op = RTE_CRYPTO_AUTH_OP_GENERATE; 1179 return 0; 1180 } 1181 1182 printf("Authentication operation specified not supported!\n"); 1183 return -1; 1184 } 1185 1186 static int 1187 parse_aead_algo(enum rte_crypto_aead_algorithm *algo, char *optarg) 1188 { 1189 if (rte_cryptodev_get_aead_algo_enum(algo, optarg) < 0) { 1190 RTE_LOG(ERR, USER1, "AEAD algorithm specified " 1191 "not supported!\n"); 1192 return -1; 1193 } 1194 1195 return 0; 1196 } 1197 1198 static int 1199 parse_aead_op(enum rte_crypto_aead_operation *op, char *optarg) 1200 { 1201 if (strcmp("ENCRYPT", optarg) == 0) { 1202 *op = RTE_CRYPTO_AEAD_OP_ENCRYPT; 1203 return 0; 1204 } else if (strcmp("DECRYPT", optarg) == 0) { 1205 *op = RTE_CRYPTO_AEAD_OP_DECRYPT; 1206 return 0; 1207 } 1208 1209 printf("AEAD operation specified not supported!\n"); 1210 return -1; 1211 } 1212 static int 1213 parse_cryptodev_mask(struct l2fwd_crypto_options *options, 1214 const char *q_arg) 1215 { 1216 char *end = NULL; 1217 uint64_t pm; 1218 1219 /* parse hexadecimal string */ 1220 pm = strtoul(q_arg, &end, 16); 1221 if ((pm == '\0') || (end == NULL) || (*end != '\0')) 1222 pm = 0; 1223 1224 options->cryptodev_mask = pm; 1225 if (options->cryptodev_mask == 0) { 1226 printf("invalid cryptodev_mask specified\n"); 1227 return -1; 1228 } 1229 1230 return 0; 1231 } 1232 1233 /** Parse long options */ 1234 static int 1235 l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options, 1236 struct option *lgopts, int option_index) 1237 { 1238 int retval; 1239 int val; 1240 1241 if (strcmp(lgopts[option_index].name, "cdev_type") == 0) { 1242 retval = parse_cryptodev_type(&options->type, optarg); 1243 if (retval == 0) 1244 strlcpy(options->string_type, optarg, MAX_STR_LEN); 1245 return retval; 1246 } 1247 1248 else if (strcmp(lgopts[option_index].name, "chain") == 0) 1249 return parse_crypto_opt_chain(options, optarg); 1250 1251 /* Cipher options */ 1252 else if (strcmp(lgopts[option_index].name, "cipher_algo") == 0) 1253 return parse_cipher_algo(&options->cipher_xform.cipher.algo, 1254 optarg); 1255 1256 else if (strcmp(lgopts[option_index].name, "cipher_op") == 0) 1257 return parse_cipher_op(&options->cipher_xform.cipher.op, 1258 optarg); 1259 1260 else if (strcmp(lgopts[option_index].name, "cipher_key") == 0) { 1261 options->ckey_param = 1; 1262 options->cipher_xform.cipher.key.length = 1263 parse_bytes(options->cipher_key, optarg, MAX_KEY_SIZE); 1264 if (options->cipher_xform.cipher.key.length > 0) 1265 return 0; 1266 else 1267 return -1; 1268 } 1269 1270 else if (strcmp(lgopts[option_index].name, "cipher_dataunit_len") == 0) { 1271 retval = parse_size(&val, optarg); 1272 if (retval == 0 && val >= 0 && val <= UINT16_MAX) { 1273 options->cipher_xform.cipher.dataunit_len = 1274 (uint16_t)val; 1275 return 0; 1276 } else 1277 return -1; 1278 } 1279 1280 else if (strcmp(lgopts[option_index].name, "cipher_key_random_size") == 0) 1281 return parse_size(&options->ckey_random_size, optarg); 1282 1283 else if (strcmp(lgopts[option_index].name, "cipher_iv") == 0) { 1284 options->cipher_iv_param = 1; 1285 options->cipher_iv.length = 1286 parse_bytes(options->cipher_iv.data, optarg, MAX_IV_SIZE); 1287 if (options->cipher_iv.length > 0) 1288 return 0; 1289 else 1290 return -1; 1291 } 1292 1293 else if (strcmp(lgopts[option_index].name, "cipher_iv_random_size") == 0) 1294 return parse_size(&options->cipher_iv_random_size, optarg); 1295 1296 /* Authentication options */ 1297 else if (strcmp(lgopts[option_index].name, "auth_algo") == 0) { 1298 return parse_auth_algo(&options->auth_xform.auth.algo, 1299 optarg); 1300 } 1301 1302 else if (strcmp(lgopts[option_index].name, "auth_op") == 0) 1303 return parse_auth_op(&options->auth_xform.auth.op, 1304 optarg); 1305 1306 else if (strcmp(lgopts[option_index].name, "auth_key") == 0) { 1307 options->akey_param = 1; 1308 options->auth_xform.auth.key.length = 1309 parse_bytes(options->auth_key, optarg, MAX_KEY_SIZE); 1310 if (options->auth_xform.auth.key.length > 0) 1311 return 0; 1312 else 1313 return -1; 1314 } 1315 1316 else if (strcmp(lgopts[option_index].name, "auth_key_random_size") == 0) { 1317 return parse_size(&options->akey_random_size, optarg); 1318 } 1319 1320 else if (strcmp(lgopts[option_index].name, "auth_iv") == 0) { 1321 options->auth_iv_param = 1; 1322 options->auth_iv.length = 1323 parse_bytes(options->auth_iv.data, optarg, MAX_IV_SIZE); 1324 if (options->auth_iv.length > 0) 1325 return 0; 1326 else 1327 return -1; 1328 } 1329 1330 else if (strcmp(lgopts[option_index].name, "auth_iv_random_size") == 0) 1331 return parse_size(&options->auth_iv_random_size, optarg); 1332 1333 /* AEAD options */ 1334 else if (strcmp(lgopts[option_index].name, "aead_algo") == 0) { 1335 return parse_aead_algo(&options->aead_xform.aead.algo, 1336 optarg); 1337 } 1338 1339 else if (strcmp(lgopts[option_index].name, "aead_op") == 0) 1340 return parse_aead_op(&options->aead_xform.aead.op, 1341 optarg); 1342 1343 else if (strcmp(lgopts[option_index].name, "aead_key") == 0) { 1344 options->aead_key_param = 1; 1345 options->aead_xform.aead.key.length = 1346 parse_bytes(options->aead_key, optarg, MAX_KEY_SIZE); 1347 if (options->aead_xform.aead.key.length > 0) 1348 return 0; 1349 else 1350 return -1; 1351 } 1352 1353 else if (strcmp(lgopts[option_index].name, "aead_key_random_size") == 0) 1354 return parse_size(&options->aead_key_random_size, optarg); 1355 1356 1357 else if (strcmp(lgopts[option_index].name, "aead_iv") == 0) { 1358 options->aead_iv_param = 1; 1359 options->aead_iv.length = 1360 parse_bytes(options->aead_iv.data, optarg, MAX_IV_SIZE); 1361 if (options->aead_iv.length > 0) 1362 return 0; 1363 else 1364 return -1; 1365 } 1366 1367 else if (strcmp(lgopts[option_index].name, "aead_iv_random_size") == 0) 1368 return parse_size(&options->aead_iv_random_size, optarg); 1369 1370 else if (strcmp(lgopts[option_index].name, "aad") == 0) { 1371 options->aad_param = 1; 1372 options->aad.length = 1373 parse_bytes(options->aad.data, optarg, MAX_AAD_SIZE); 1374 if (options->aad.length > 0) 1375 return 0; 1376 else 1377 return -1; 1378 } 1379 1380 else if (strcmp(lgopts[option_index].name, "aad_random_size") == 0) { 1381 return parse_size(&options->aad_random_size, optarg); 1382 } 1383 1384 else if (strcmp(lgopts[option_index].name, "digest_size") == 0) { 1385 return parse_size(&options->digest_size, optarg); 1386 } 1387 1388 else if (strcmp(lgopts[option_index].name, "sessionless") == 0) { 1389 options->sessionless = 1; 1390 return 0; 1391 } 1392 1393 else if (strcmp(lgopts[option_index].name, "cryptodev_mask") == 0) 1394 return parse_cryptodev_mask(options, optarg); 1395 1396 else if (strcmp(lgopts[option_index].name, "mac-updating") == 0) { 1397 options->mac_updating = 1; 1398 return 0; 1399 } 1400 1401 else if (strcmp(lgopts[option_index].name, "no-mac-updating") == 0) { 1402 options->mac_updating = 0; 1403 return 0; 1404 } 1405 1406 return -1; 1407 } 1408 1409 /** Parse port mask */ 1410 static int 1411 l2fwd_crypto_parse_portmask(struct l2fwd_crypto_options *options, 1412 const char *q_arg) 1413 { 1414 char *end = NULL; 1415 unsigned long pm; 1416 1417 /* parse hexadecimal string */ 1418 pm = strtoul(q_arg, &end, 16); 1419 if ((pm == '\0') || (end == NULL) || (*end != '\0')) 1420 pm = 0; 1421 1422 options->portmask = pm; 1423 if (options->portmask == 0) { 1424 printf("invalid portmask specified\n"); 1425 return -1; 1426 } 1427 1428 return pm; 1429 } 1430 1431 /** Parse number of queues */ 1432 static int 1433 l2fwd_crypto_parse_nqueue(struct l2fwd_crypto_options *options, 1434 const char *q_arg) 1435 { 1436 char *end = NULL; 1437 unsigned long n; 1438 1439 /* parse hexadecimal string */ 1440 n = strtoul(q_arg, &end, 10); 1441 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) 1442 n = 0; 1443 else if (n >= MAX_RX_QUEUE_PER_LCORE) 1444 n = 0; 1445 1446 options->nb_ports_per_lcore = n; 1447 if (options->nb_ports_per_lcore == 0) { 1448 printf("invalid number of ports selected\n"); 1449 return -1; 1450 } 1451 1452 return 0; 1453 } 1454 1455 /** Parse timer period */ 1456 static int 1457 l2fwd_crypto_parse_timer_period(struct l2fwd_crypto_options *options, 1458 const char *q_arg) 1459 { 1460 char *end = NULL; 1461 unsigned long n; 1462 1463 /* parse number string */ 1464 n = (unsigned)strtol(q_arg, &end, 10); 1465 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) 1466 n = 0; 1467 1468 if (n >= MAX_TIMER_PERIOD) { 1469 printf("Warning refresh period specified %lu is greater than " 1470 "max value %lu! using max value", 1471 n, MAX_TIMER_PERIOD); 1472 n = MAX_TIMER_PERIOD; 1473 } 1474 1475 options->refresh_period = n * 1000 * TIMER_MILLISECOND; 1476 1477 return 0; 1478 } 1479 1480 /** Generate default options for application */ 1481 static void 1482 l2fwd_crypto_default_options(struct l2fwd_crypto_options *options) 1483 { 1484 options->portmask = 0xffffffff; 1485 options->nb_ports_per_lcore = 1; 1486 options->refresh_period = 10000; 1487 options->single_lcore = 0; 1488 options->sessionless = 0; 1489 1490 options->xform_chain = L2FWD_CRYPTO_CIPHER_HASH; 1491 1492 /* Cipher Data */ 1493 options->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; 1494 options->cipher_xform.next = NULL; 1495 options->ckey_param = 0; 1496 options->ckey_random_size = -1; 1497 options->cipher_xform.cipher.key.length = 0; 1498 options->cipher_iv_param = 0; 1499 options->cipher_iv_random_size = -1; 1500 options->cipher_iv.length = 0; 1501 1502 options->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; 1503 options->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 1504 options->cipher_xform.cipher.dataunit_len = 0; 1505 1506 /* Authentication Data */ 1507 options->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; 1508 options->auth_xform.next = NULL; 1509 options->akey_param = 0; 1510 options->akey_random_size = -1; 1511 options->auth_xform.auth.key.length = 0; 1512 options->auth_iv_param = 0; 1513 options->auth_iv_random_size = -1; 1514 options->auth_iv.length = 0; 1515 1516 options->auth_xform.auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC; 1517 options->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; 1518 1519 /* AEAD Data */ 1520 options->aead_xform.type = RTE_CRYPTO_SYM_XFORM_AEAD; 1521 options->aead_xform.next = NULL; 1522 options->aead_key_param = 0; 1523 options->aead_key_random_size = -1; 1524 options->aead_xform.aead.key.length = 0; 1525 options->aead_iv_param = 0; 1526 options->aead_iv_random_size = -1; 1527 options->aead_iv.length = 0; 1528 1529 options->aead_xform.aead.algo = RTE_CRYPTO_AEAD_AES_GCM; 1530 options->aead_xform.aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT; 1531 1532 options->aad_param = 0; 1533 options->aad_random_size = -1; 1534 options->aad.length = 0; 1535 1536 options->digest_size = -1; 1537 1538 options->type = CDEV_TYPE_ANY; 1539 options->cryptodev_mask = UINT64_MAX; 1540 1541 options->mac_updating = 1; 1542 } 1543 1544 static void 1545 display_cipher_info(struct l2fwd_crypto_options *options) 1546 { 1547 printf("\n---- Cipher information ---\n"); 1548 printf("Algorithm: %s\n", 1549 rte_crypto_cipher_algorithm_strings[options->cipher_xform.cipher.algo]); 1550 rte_hexdump(stdout, "Cipher key:", 1551 options->cipher_xform.cipher.key.data, 1552 options->cipher_xform.cipher.key.length); 1553 rte_hexdump(stdout, "IV:", options->cipher_iv.data, options->cipher_iv.length); 1554 } 1555 1556 static void 1557 display_auth_info(struct l2fwd_crypto_options *options) 1558 { 1559 printf("\n---- Authentication information ---\n"); 1560 printf("Algorithm: %s\n", 1561 rte_crypto_auth_algorithm_strings[options->auth_xform.auth.algo]); 1562 rte_hexdump(stdout, "Auth key:", 1563 options->auth_xform.auth.key.data, 1564 options->auth_xform.auth.key.length); 1565 rte_hexdump(stdout, "IV:", options->auth_iv.data, options->auth_iv.length); 1566 } 1567 1568 static void 1569 display_aead_info(struct l2fwd_crypto_options *options) 1570 { 1571 printf("\n---- AEAD information ---\n"); 1572 printf("Algorithm: %s\n", 1573 rte_crypto_aead_algorithm_strings[options->aead_xform.aead.algo]); 1574 rte_hexdump(stdout, "AEAD key:", 1575 options->aead_xform.aead.key.data, 1576 options->aead_xform.aead.key.length); 1577 rte_hexdump(stdout, "IV:", options->aead_iv.data, options->aead_iv.length); 1578 rte_hexdump(stdout, "AAD:", options->aad.data, options->aad.length); 1579 } 1580 1581 static void 1582 l2fwd_crypto_options_print(struct l2fwd_crypto_options *options) 1583 { 1584 char string_cipher_op[MAX_STR_LEN]; 1585 char string_auth_op[MAX_STR_LEN]; 1586 char string_aead_op[MAX_STR_LEN]; 1587 1588 if (options->cipher_xform.cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1589 strcpy(string_cipher_op, "Encrypt"); 1590 else 1591 strcpy(string_cipher_op, "Decrypt"); 1592 1593 if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) 1594 strcpy(string_auth_op, "Auth generate"); 1595 else 1596 strcpy(string_auth_op, "Auth verify"); 1597 1598 if (options->aead_xform.aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 1599 strcpy(string_aead_op, "Authenticated encryption"); 1600 else 1601 strcpy(string_aead_op, "Authenticated decryption"); 1602 1603 1604 printf("Options:-\nn"); 1605 printf("portmask: %x\n", options->portmask); 1606 printf("ports per lcore: %u\n", options->nb_ports_per_lcore); 1607 printf("refresh period : %u\n", options->refresh_period); 1608 printf("single lcore mode: %s\n", 1609 options->single_lcore ? "enabled" : "disabled"); 1610 printf("stats_printing: %s\n", 1611 options->refresh_period == 0 ? "disabled" : "enabled"); 1612 1613 printf("sessionless crypto: %s\n", 1614 options->sessionless ? "enabled" : "disabled"); 1615 1616 if (options->ckey_param && (options->ckey_random_size != -1)) 1617 printf("Cipher key already parsed, ignoring size of random key\n"); 1618 1619 if (options->akey_param && (options->akey_random_size != -1)) 1620 printf("Auth key already parsed, ignoring size of random key\n"); 1621 1622 if (options->cipher_iv_param && (options->cipher_iv_random_size != -1)) 1623 printf("Cipher IV already parsed, ignoring size of random IV\n"); 1624 1625 if (options->auth_iv_param && (options->auth_iv_random_size != -1)) 1626 printf("Auth IV already parsed, ignoring size of random IV\n"); 1627 1628 if (options->aad_param && (options->aad_random_size != -1)) 1629 printf("AAD already parsed, ignoring size of random AAD\n"); 1630 1631 printf("\nCrypto chain: "); 1632 switch (options->xform_chain) { 1633 case L2FWD_CRYPTO_AEAD: 1634 printf("Input --> %s --> Output\n", string_aead_op); 1635 display_aead_info(options); 1636 break; 1637 case L2FWD_CRYPTO_CIPHER_HASH: 1638 printf("Input --> %s --> %s --> Output\n", 1639 string_cipher_op, string_auth_op); 1640 display_cipher_info(options); 1641 display_auth_info(options); 1642 break; 1643 case L2FWD_CRYPTO_HASH_CIPHER: 1644 printf("Input --> %s --> %s --> Output\n", 1645 string_auth_op, string_cipher_op); 1646 display_cipher_info(options); 1647 display_auth_info(options); 1648 break; 1649 case L2FWD_CRYPTO_HASH_ONLY: 1650 printf("Input --> %s --> Output\n", string_auth_op); 1651 display_auth_info(options); 1652 break; 1653 case L2FWD_CRYPTO_CIPHER_ONLY: 1654 printf("Input --> %s --> Output\n", string_cipher_op); 1655 display_cipher_info(options); 1656 break; 1657 } 1658 } 1659 1660 /* Parse the argument given in the command line of the application */ 1661 static int 1662 l2fwd_crypto_parse_args(struct l2fwd_crypto_options *options, 1663 int argc, char **argv) 1664 { 1665 int opt, retval, option_index; 1666 char **argvopt = argv, *prgname = argv[0]; 1667 1668 static struct option lgopts[] = { 1669 { "sessionless", no_argument, 0, 0 }, 1670 1671 { "cdev_type", required_argument, 0, 0 }, 1672 { "chain", required_argument, 0, 0 }, 1673 1674 { "cipher_algo", required_argument, 0, 0 }, 1675 { "cipher_op", required_argument, 0, 0 }, 1676 { "cipher_key", required_argument, 0, 0 }, 1677 { "cipher_key_random_size", required_argument, 0, 0 }, 1678 { "cipher_iv", required_argument, 0, 0 }, 1679 { "cipher_iv_random_size", required_argument, 0, 0 }, 1680 { "cipher_dataunit_len", required_argument, 0, 0}, 1681 1682 { "auth_algo", required_argument, 0, 0 }, 1683 { "auth_op", required_argument, 0, 0 }, 1684 { "auth_key", required_argument, 0, 0 }, 1685 { "auth_key_random_size", required_argument, 0, 0 }, 1686 { "auth_iv", required_argument, 0, 0 }, 1687 { "auth_iv_random_size", required_argument, 0, 0 }, 1688 1689 { "aead_algo", required_argument, 0, 0 }, 1690 { "aead_op", required_argument, 0, 0 }, 1691 { "aead_key", required_argument, 0, 0 }, 1692 { "aead_key_random_size", required_argument, 0, 0 }, 1693 { "aead_iv", required_argument, 0, 0 }, 1694 { "aead_iv_random_size", required_argument, 0, 0 }, 1695 1696 { "aad", required_argument, 0, 0 }, 1697 { "aad_random_size", required_argument, 0, 0 }, 1698 1699 { "digest_size", required_argument, 0, 0 }, 1700 1701 { "sessionless", no_argument, 0, 0 }, 1702 { "cryptodev_mask", required_argument, 0, 0}, 1703 1704 { "mac-updating", no_argument, 0, 0}, 1705 { "no-mac-updating", no_argument, 0, 0}, 1706 1707 { NULL, 0, 0, 0 } 1708 }; 1709 1710 l2fwd_crypto_default_options(options); 1711 1712 while ((opt = getopt_long(argc, argvopt, "p:q:sT:", lgopts, 1713 &option_index)) != EOF) { 1714 switch (opt) { 1715 /* long options */ 1716 case 0: 1717 retval = l2fwd_crypto_parse_args_long_options(options, 1718 lgopts, option_index); 1719 if (retval < 0) { 1720 l2fwd_crypto_usage(prgname); 1721 return -1; 1722 } 1723 break; 1724 1725 /* portmask */ 1726 case 'p': 1727 retval = l2fwd_crypto_parse_portmask(options, optarg); 1728 if (retval < 0) { 1729 l2fwd_crypto_usage(prgname); 1730 return -1; 1731 } 1732 break; 1733 1734 /* nqueue */ 1735 case 'q': 1736 retval = l2fwd_crypto_parse_nqueue(options, optarg); 1737 if (retval < 0) { 1738 l2fwd_crypto_usage(prgname); 1739 return -1; 1740 } 1741 break; 1742 1743 /* single */ 1744 case 's': 1745 options->single_lcore = 1; 1746 1747 break; 1748 1749 /* timer period */ 1750 case 'T': 1751 retval = l2fwd_crypto_parse_timer_period(options, 1752 optarg); 1753 if (retval < 0) { 1754 l2fwd_crypto_usage(prgname); 1755 return -1; 1756 } 1757 break; 1758 1759 default: 1760 l2fwd_crypto_usage(prgname); 1761 return -1; 1762 } 1763 } 1764 1765 1766 if (optind >= 0) 1767 argv[optind-1] = prgname; 1768 1769 retval = optind-1; 1770 optind = 1; /* reset getopt lib */ 1771 1772 return retval; 1773 } 1774 1775 /* Check the link status of all ports in up to 9s, and print them finally */ 1776 static void 1777 check_all_ports_link_status(uint32_t port_mask) 1778 { 1779 #define CHECK_INTERVAL 100 /* 100ms */ 1780 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1781 uint16_t portid; 1782 uint8_t count, all_ports_up, print_flag = 0; 1783 struct rte_eth_link link; 1784 int ret; 1785 char link_status_text[RTE_ETH_LINK_MAX_STR_LEN]; 1786 1787 printf("\nChecking link status"); 1788 fflush(stdout); 1789 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1790 all_ports_up = 1; 1791 RTE_ETH_FOREACH_DEV(portid) { 1792 if ((port_mask & (1 << portid)) == 0) 1793 continue; 1794 memset(&link, 0, sizeof(link)); 1795 ret = rte_eth_link_get_nowait(portid, &link); 1796 if (ret < 0) { 1797 all_ports_up = 0; 1798 if (print_flag == 1) 1799 printf("Port %u link get failed: %s\n", 1800 portid, rte_strerror(-ret)); 1801 continue; 1802 } 1803 /* print link status if flag set */ 1804 if (print_flag == 1) { 1805 rte_eth_link_to_str(link_status_text, 1806 sizeof(link_status_text), &link); 1807 printf("Port %d %s\n", portid, 1808 link_status_text); 1809 continue; 1810 } 1811 /* clear all_ports_up flag if any link down */ 1812 if (link.link_status == ETH_LINK_DOWN) { 1813 all_ports_up = 0; 1814 break; 1815 } 1816 } 1817 /* after finally printing all link status, get out */ 1818 if (print_flag == 1) 1819 break; 1820 1821 if (all_ports_up == 0) { 1822 printf("."); 1823 fflush(stdout); 1824 rte_delay_ms(CHECK_INTERVAL); 1825 } 1826 1827 /* set the print_flag if all ports up or timeout */ 1828 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1829 print_flag = 1; 1830 printf("done\n"); 1831 } 1832 } 1833 } 1834 1835 /* Check if device has to be HW/SW or any */ 1836 static int 1837 check_type(const struct l2fwd_crypto_options *options, 1838 const struct rte_cryptodev_info *dev_info) 1839 { 1840 if (options->type == CDEV_TYPE_HW && 1841 (dev_info->feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED)) 1842 return 0; 1843 if (options->type == CDEV_TYPE_SW && 1844 !(dev_info->feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED)) 1845 return 0; 1846 if (options->type == CDEV_TYPE_ANY) 1847 return 0; 1848 1849 return -1; 1850 } 1851 1852 static const struct rte_cryptodev_capabilities * 1853 check_device_support_cipher_algo(const struct l2fwd_crypto_options *options, 1854 const struct rte_cryptodev_info *dev_info, 1855 uint8_t cdev_id) 1856 { 1857 unsigned int i = 0; 1858 const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; 1859 enum rte_crypto_cipher_algorithm cap_cipher_algo; 1860 enum rte_crypto_cipher_algorithm opt_cipher_algo = 1861 options->cipher_xform.cipher.algo; 1862 1863 while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1864 cap_cipher_algo = cap->sym.cipher.algo; 1865 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 1866 if (cap_cipher_algo == opt_cipher_algo) { 1867 if (check_type(options, dev_info) == 0) 1868 break; 1869 } 1870 } 1871 cap = &dev_info->capabilities[++i]; 1872 } 1873 1874 if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1875 printf("Algorithm %s not supported by cryptodev %u" 1876 " or device not of preferred type (%s)\n", 1877 rte_crypto_cipher_algorithm_strings[opt_cipher_algo], 1878 cdev_id, 1879 options->string_type); 1880 return NULL; 1881 } 1882 1883 return cap; 1884 } 1885 1886 static const struct rte_cryptodev_capabilities * 1887 check_device_support_auth_algo(const struct l2fwd_crypto_options *options, 1888 const struct rte_cryptodev_info *dev_info, 1889 uint8_t cdev_id) 1890 { 1891 unsigned int i = 0; 1892 const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; 1893 enum rte_crypto_auth_algorithm cap_auth_algo; 1894 enum rte_crypto_auth_algorithm opt_auth_algo = 1895 options->auth_xform.auth.algo; 1896 1897 while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1898 cap_auth_algo = cap->sym.auth.algo; 1899 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AUTH) { 1900 if (cap_auth_algo == opt_auth_algo) { 1901 if (check_type(options, dev_info) == 0) 1902 break; 1903 } 1904 } 1905 cap = &dev_info->capabilities[++i]; 1906 } 1907 1908 if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1909 printf("Algorithm %s not supported by cryptodev %u" 1910 " or device not of preferred type (%s)\n", 1911 rte_crypto_auth_algorithm_strings[opt_auth_algo], 1912 cdev_id, 1913 options->string_type); 1914 return NULL; 1915 } 1916 1917 return cap; 1918 } 1919 1920 static const struct rte_cryptodev_capabilities * 1921 check_device_support_aead_algo(const struct l2fwd_crypto_options *options, 1922 const struct rte_cryptodev_info *dev_info, 1923 uint8_t cdev_id) 1924 { 1925 unsigned int i = 0; 1926 const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; 1927 enum rte_crypto_aead_algorithm cap_aead_algo; 1928 enum rte_crypto_aead_algorithm opt_aead_algo = 1929 options->aead_xform.aead.algo; 1930 1931 while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1932 cap_aead_algo = cap->sym.aead.algo; 1933 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1934 if (cap_aead_algo == opt_aead_algo) { 1935 if (check_type(options, dev_info) == 0) 1936 break; 1937 } 1938 } 1939 cap = &dev_info->capabilities[++i]; 1940 } 1941 1942 if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { 1943 printf("Algorithm %s not supported by cryptodev %u" 1944 " or device not of preferred type (%s)\n", 1945 rte_crypto_aead_algorithm_strings[opt_aead_algo], 1946 cdev_id, 1947 options->string_type); 1948 return NULL; 1949 } 1950 1951 return cap; 1952 } 1953 1954 /* Check if the device is enabled by cryptodev_mask */ 1955 static int 1956 check_cryptodev_mask(struct l2fwd_crypto_options *options, 1957 uint8_t cdev_id) 1958 { 1959 if (options->cryptodev_mask & (1 << cdev_id)) 1960 return 0; 1961 1962 return -1; 1963 } 1964 1965 static inline int 1966 check_supported_size(uint16_t length, uint16_t min, uint16_t max, 1967 uint16_t increment) 1968 { 1969 uint16_t supp_size; 1970 1971 /* Single value */ 1972 if (increment == 0) { 1973 if (length == min) 1974 return 0; 1975 else 1976 return -1; 1977 } 1978 1979 /* Range of values */ 1980 for (supp_size = min; supp_size <= max; supp_size += increment) { 1981 if (length == supp_size) 1982 return 0; 1983 } 1984 1985 return -1; 1986 } 1987 1988 static int 1989 check_iv_param(const struct rte_crypto_param_range *iv_range_size, 1990 unsigned int iv_param, int iv_random_size, 1991 uint16_t iv_length) 1992 { 1993 /* 1994 * Check if length of provided IV is supported 1995 * by the algorithm chosen. 1996 */ 1997 if (iv_param) { 1998 if (check_supported_size(iv_length, 1999 iv_range_size->min, 2000 iv_range_size->max, 2001 iv_range_size->increment) 2002 != 0) 2003 return -1; 2004 /* 2005 * Check if length of IV to be randomly generated 2006 * is supported by the algorithm chosen. 2007 */ 2008 } else if (iv_random_size != -1) { 2009 if (check_supported_size(iv_random_size, 2010 iv_range_size->min, 2011 iv_range_size->max, 2012 iv_range_size->increment) 2013 != 0) 2014 return -1; 2015 } 2016 2017 return 0; 2018 } 2019 2020 static int 2021 check_capabilities(struct l2fwd_crypto_options *options, uint8_t cdev_id) 2022 { 2023 struct rte_cryptodev_info dev_info; 2024 const struct rte_cryptodev_capabilities *cap; 2025 2026 rte_cryptodev_info_get(cdev_id, &dev_info); 2027 2028 /* Set AEAD parameters */ 2029 if (options->xform_chain == L2FWD_CRYPTO_AEAD) { 2030 /* Check if device supports AEAD algo */ 2031 cap = check_device_support_aead_algo(options, &dev_info, 2032 cdev_id); 2033 if (cap == NULL) 2034 return -1; 2035 2036 if (check_iv_param(&cap->sym.aead.iv_size, 2037 options->aead_iv_param, 2038 options->aead_iv_random_size, 2039 options->aead_iv.length) != 0) { 2040 RTE_LOG(DEBUG, USER1, 2041 "Device %u does not support IV length\n", 2042 cdev_id); 2043 return -1; 2044 } 2045 2046 /* 2047 * Check if length of provided AEAD key is supported 2048 * by the algorithm chosen. 2049 */ 2050 if (options->aead_key_param) { 2051 if (check_supported_size( 2052 options->aead_xform.aead.key.length, 2053 cap->sym.aead.key_size.min, 2054 cap->sym.aead.key_size.max, 2055 cap->sym.aead.key_size.increment) 2056 != 0) { 2057 RTE_LOG(DEBUG, USER1, 2058 "Device %u does not support " 2059 "AEAD key length\n", 2060 cdev_id); 2061 return -1; 2062 } 2063 /* 2064 * Check if length of the aead key to be randomly generated 2065 * is supported by the algorithm chosen. 2066 */ 2067 } else if (options->aead_key_random_size != -1) { 2068 if (check_supported_size(options->aead_key_random_size, 2069 cap->sym.aead.key_size.min, 2070 cap->sym.aead.key_size.max, 2071 cap->sym.aead.key_size.increment) 2072 != 0) { 2073 RTE_LOG(DEBUG, USER1, 2074 "Device %u does not support " 2075 "AEAD key length\n", 2076 cdev_id); 2077 return -1; 2078 } 2079 } 2080 2081 2082 /* 2083 * Check if length of provided AAD is supported 2084 * by the algorithm chosen. 2085 */ 2086 if (options->aad_param) { 2087 if (check_supported_size(options->aad.length, 2088 cap->sym.aead.aad_size.min, 2089 cap->sym.aead.aad_size.max, 2090 cap->sym.aead.aad_size.increment) 2091 != 0) { 2092 RTE_LOG(DEBUG, USER1, 2093 "Device %u does not support " 2094 "AAD length\n", 2095 cdev_id); 2096 return -1; 2097 } 2098 /* 2099 * Check if length of AAD to be randomly generated 2100 * is supported by the algorithm chosen. 2101 */ 2102 } else if (options->aad_random_size != -1) { 2103 if (check_supported_size(options->aad_random_size, 2104 cap->sym.aead.aad_size.min, 2105 cap->sym.aead.aad_size.max, 2106 cap->sym.aead.aad_size.increment) 2107 != 0) { 2108 RTE_LOG(DEBUG, USER1, 2109 "Device %u does not support " 2110 "AAD length\n", 2111 cdev_id); 2112 return -1; 2113 } 2114 } 2115 2116 /* Check if digest size is supported by the algorithm. */ 2117 if (options->digest_size != -1) { 2118 if (check_supported_size(options->digest_size, 2119 cap->sym.aead.digest_size.min, 2120 cap->sym.aead.digest_size.max, 2121 cap->sym.aead.digest_size.increment) 2122 != 0) { 2123 RTE_LOG(DEBUG, USER1, 2124 "Device %u does not support " 2125 "digest length\n", 2126 cdev_id); 2127 return -1; 2128 } 2129 } 2130 } 2131 2132 /* Set cipher parameters */ 2133 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2134 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2135 options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { 2136 2137 /* Check if device supports cipher algo. 8< */ 2138 cap = check_device_support_cipher_algo(options, &dev_info, 2139 cdev_id); 2140 if (cap == NULL) 2141 return -1; 2142 2143 if (check_iv_param(&cap->sym.cipher.iv_size, 2144 options->cipher_iv_param, 2145 options->cipher_iv_random_size, 2146 options->cipher_iv.length) != 0) { 2147 RTE_LOG(DEBUG, USER1, 2148 "Device %u does not support IV length\n", 2149 cdev_id); 2150 return -1; 2151 } 2152 /* >8 End of check if device supports cipher algo. */ 2153 2154 /* Check if capable cipher is supported. 8< */ 2155 2156 /* 2157 * Check if length of provided cipher key is supported 2158 * by the algorithm chosen. 2159 */ 2160 if (options->ckey_param) { 2161 if (check_supported_size( 2162 options->cipher_xform.cipher.key.length, 2163 cap->sym.cipher.key_size.min, 2164 cap->sym.cipher.key_size.max, 2165 cap->sym.cipher.key_size.increment) 2166 != 0) { 2167 if (dev_info.feature_flags & 2168 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY) { 2169 RTE_LOG(DEBUG, USER1, 2170 "Key length does not match the device " 2171 "%u capability. Key may be wrapped\n", 2172 cdev_id); 2173 } else { 2174 RTE_LOG(DEBUG, USER1, 2175 "Key length does not match the device " 2176 "%u capability\n", 2177 cdev_id); 2178 return -1; 2179 } 2180 } 2181 2182 /* 2183 * Check if length of the cipher key to be randomly generated 2184 * is supported by the algorithm chosen. 2185 */ 2186 } else if (options->ckey_random_size != -1) { 2187 if (check_supported_size(options->ckey_random_size, 2188 cap->sym.cipher.key_size.min, 2189 cap->sym.cipher.key_size.max, 2190 cap->sym.cipher.key_size.increment) 2191 != 0) { 2192 RTE_LOG(DEBUG, USER1, 2193 "Device %u does not support cipher " 2194 "key length\n", 2195 cdev_id); 2196 return -1; 2197 } 2198 } 2199 2200 if (options->cipher_xform.cipher.dataunit_len > 0) { 2201 if (!(dev_info.feature_flags & 2202 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)) { 2203 RTE_LOG(DEBUG, USER1, 2204 "Device %u does not support " 2205 "cipher multiple data units\n", 2206 cdev_id); 2207 return -1; 2208 } 2209 if (cap->sym.cipher.dataunit_set != 0) { 2210 int ret = 0; 2211 2212 switch (options->cipher_xform.cipher.dataunit_len) { 2213 case 512: 2214 if (!(cap->sym.cipher.dataunit_set & 2215 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES)) 2216 ret = -1; 2217 break; 2218 case 4096: 2219 if (!(cap->sym.cipher.dataunit_set & 2220 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES)) 2221 ret = -1; 2222 break; 2223 default: 2224 ret = -1; 2225 } 2226 if (ret == -1) { 2227 RTE_LOG(DEBUG, USER1, 2228 "Device %u does not support " 2229 "data-unit length %u\n", 2230 cdev_id, 2231 options->cipher_xform.cipher.dataunit_len); 2232 return -1; 2233 } 2234 } 2235 } 2236 /* >8 End of checking if cipher is supported. */ 2237 } 2238 2239 /* Set auth parameters */ 2240 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2241 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2242 options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) { 2243 /* Check if device supports auth algo */ 2244 cap = check_device_support_auth_algo(options, &dev_info, 2245 cdev_id); 2246 if (cap == NULL) 2247 return -1; 2248 2249 if (check_iv_param(&cap->sym.auth.iv_size, 2250 options->auth_iv_param, 2251 options->auth_iv_random_size, 2252 options->auth_iv.length) != 0) { 2253 RTE_LOG(DEBUG, USER1, 2254 "Device %u does not support IV length\n", 2255 cdev_id); 2256 return -1; 2257 } 2258 /* 2259 * Check if length of provided auth key is supported 2260 * by the algorithm chosen. 2261 */ 2262 if (options->akey_param) { 2263 if (check_supported_size( 2264 options->auth_xform.auth.key.length, 2265 cap->sym.auth.key_size.min, 2266 cap->sym.auth.key_size.max, 2267 cap->sym.auth.key_size.increment) 2268 != 0) { 2269 RTE_LOG(DEBUG, USER1, 2270 "Device %u does not support auth " 2271 "key length\n", 2272 cdev_id); 2273 return -1; 2274 } 2275 /* 2276 * Check if length of the auth key to be randomly generated 2277 * is supported by the algorithm chosen. 2278 */ 2279 } else if (options->akey_random_size != -1) { 2280 if (check_supported_size(options->akey_random_size, 2281 cap->sym.auth.key_size.min, 2282 cap->sym.auth.key_size.max, 2283 cap->sym.auth.key_size.increment) 2284 != 0) { 2285 RTE_LOG(DEBUG, USER1, 2286 "Device %u does not support auth " 2287 "key length\n", 2288 cdev_id); 2289 return -1; 2290 } 2291 } 2292 2293 /* Check if digest size is supported by the algorithm. */ 2294 if (options->digest_size != -1) { 2295 if (check_supported_size(options->digest_size, 2296 cap->sym.auth.digest_size.min, 2297 cap->sym.auth.digest_size.max, 2298 cap->sym.auth.digest_size.increment) 2299 != 0) { 2300 RTE_LOG(DEBUG, USER1, 2301 "Device %u does not support " 2302 "digest length\n", 2303 cdev_id); 2304 return -1; 2305 } 2306 } 2307 } 2308 2309 return 0; 2310 } 2311 2312 static int 2313 initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, 2314 uint8_t *enabled_cdevs) 2315 { 2316 uint8_t cdev_id, cdev_count, enabled_cdev_count = 0; 2317 const struct rte_cryptodev_capabilities *cap; 2318 unsigned int sess_sz, max_sess_sz = 0; 2319 uint32_t sessions_needed = 0; 2320 int retval; 2321 2322 cdev_count = rte_cryptodev_count(); 2323 if (cdev_count == 0) { 2324 printf("No crypto devices available\n"); 2325 return -1; 2326 } 2327 2328 for (cdev_id = 0; cdev_id < cdev_count && enabled_cdev_count < nb_ports; 2329 cdev_id++) { 2330 if (check_cryptodev_mask(options, cdev_id) < 0) 2331 continue; 2332 2333 if (check_capabilities(options, cdev_id) < 0) 2334 continue; 2335 2336 sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id); 2337 if (sess_sz > max_sess_sz) 2338 max_sess_sz = sess_sz; 2339 2340 l2fwd_enabled_crypto_mask |= (((uint64_t)1) << cdev_id); 2341 2342 enabled_cdevs[cdev_id] = 1; 2343 enabled_cdev_count++; 2344 } 2345 2346 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { 2347 struct rte_cryptodev_qp_conf qp_conf; 2348 struct rte_cryptodev_info dev_info; 2349 2350 if (enabled_cdevs[cdev_id] == 0) 2351 continue; 2352 2353 if (check_cryptodev_mask(options, cdev_id) < 0) 2354 continue; 2355 2356 if (check_capabilities(options, cdev_id) < 0) 2357 continue; 2358 2359 retval = rte_cryptodev_socket_id(cdev_id); 2360 2361 if (retval < 0) { 2362 printf("Invalid crypto device id used\n"); 2363 return -1; 2364 } 2365 2366 uint8_t socket_id = (uint8_t) retval; 2367 2368 struct rte_cryptodev_config conf = { 2369 .nb_queue_pairs = 1, 2370 .socket_id = socket_id, 2371 .ff_disable = RTE_CRYPTODEV_FF_SECURITY, 2372 }; 2373 2374 rte_cryptodev_info_get(cdev_id, &dev_info); 2375 2376 /* 2377 * Two sessions objects are required for each session 2378 * (one for the header, one for the private data) 2379 */ 2380 if (!strcmp(dev_info.driver_name, "crypto_scheduler")) { 2381 #ifdef RTE_CRYPTO_SCHEDULER 2382 uint32_t nb_workers = 2383 rte_cryptodev_scheduler_workers_get(cdev_id, 2384 NULL); 2385 2386 sessions_needed = enabled_cdev_count * nb_workers; 2387 #endif 2388 } else 2389 sessions_needed = enabled_cdev_count; 2390 2391 if (session_pool_socket[socket_id].priv_mp == NULL) { 2392 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2393 2394 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2395 "priv_sess_mp_%u", socket_id); 2396 2397 session_pool_socket[socket_id].priv_mp = 2398 rte_mempool_create(mp_name, 2399 sessions_needed, 2400 max_sess_sz, 2401 0, 0, NULL, NULL, NULL, 2402 NULL, socket_id, 2403 0); 2404 2405 if (session_pool_socket[socket_id].priv_mp == NULL) { 2406 printf("Cannot create pool on socket %d\n", 2407 socket_id); 2408 return -ENOMEM; 2409 } 2410 2411 printf("Allocated pool \"%s\" on socket %d\n", 2412 mp_name, socket_id); 2413 } 2414 2415 if (session_pool_socket[socket_id].sess_mp == NULL) { 2416 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2417 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2418 "sess_mp_%u", socket_id); 2419 2420 session_pool_socket[socket_id].sess_mp = 2421 rte_cryptodev_sym_session_pool_create( 2422 mp_name, 2423 sessions_needed, 2424 0, 0, 0, socket_id); 2425 2426 if (session_pool_socket[socket_id].sess_mp == NULL) { 2427 printf("Cannot create pool on socket %d\n", 2428 socket_id); 2429 return -ENOMEM; 2430 } 2431 2432 printf("Allocated pool \"%s\" on socket %d\n", 2433 mp_name, socket_id); 2434 } 2435 2436 /* Set AEAD parameters */ 2437 if (options->xform_chain == L2FWD_CRYPTO_AEAD) { 2438 cap = check_device_support_aead_algo(options, &dev_info, 2439 cdev_id); 2440 2441 options->block_size = cap->sym.aead.block_size; 2442 2443 /* Set IV if not provided from command line */ 2444 if (options->aead_iv_param == 0) { 2445 if (options->aead_iv_random_size != -1) 2446 options->aead_iv.length = 2447 options->aead_iv_random_size; 2448 /* No size provided, use minimum size. */ 2449 else 2450 options->aead_iv.length = 2451 cap->sym.aead.iv_size.min; 2452 } 2453 2454 /* Set key if not provided from command line */ 2455 if (options->aead_key_param == 0) { 2456 if (options->aead_key_random_size != -1) 2457 options->aead_xform.aead.key.length = 2458 options->aead_key_random_size; 2459 /* No size provided, use minimum size. */ 2460 else 2461 options->aead_xform.aead.key.length = 2462 cap->sym.aead.key_size.min; 2463 2464 generate_random_key(options->aead_key, 2465 options->aead_xform.aead.key.length); 2466 } 2467 2468 /* Set AAD if not provided from command line */ 2469 if (options->aad_param == 0) { 2470 if (options->aad_random_size != -1) 2471 options->aad.length = 2472 options->aad_random_size; 2473 /* No size provided, use minimum size. */ 2474 else 2475 options->aad.length = 2476 cap->sym.auth.aad_size.min; 2477 } 2478 2479 options->aead_xform.aead.aad_length = 2480 options->aad.length; 2481 2482 /* Set digest size if not provided from command line */ 2483 if (options->digest_size != -1) 2484 options->aead_xform.aead.digest_length = 2485 options->digest_size; 2486 /* No size provided, use minimum size. */ 2487 else 2488 options->aead_xform.aead.digest_length = 2489 cap->sym.aead.digest_size.min; 2490 } 2491 2492 /* Set cipher parameters */ 2493 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2494 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2495 options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { 2496 cap = check_device_support_cipher_algo(options, &dev_info, 2497 cdev_id); 2498 options->block_size = cap->sym.cipher.block_size; 2499 2500 /* Set IV if not provided from command line */ 2501 if (options->cipher_iv_param == 0) { 2502 if (options->cipher_iv_random_size != -1) 2503 options->cipher_iv.length = 2504 options->cipher_iv_random_size; 2505 /* No size provided, use minimum size. */ 2506 else 2507 options->cipher_iv.length = 2508 cap->sym.cipher.iv_size.min; 2509 } 2510 2511 /* Set key if not provided from command line */ 2512 if (options->ckey_param == 0) { 2513 if (options->ckey_random_size != -1) 2514 options->cipher_xform.cipher.key.length = 2515 options->ckey_random_size; 2516 /* No size provided, use minimum size. */ 2517 else 2518 options->cipher_xform.cipher.key.length = 2519 cap->sym.cipher.key_size.min; 2520 2521 generate_random_key(options->cipher_key, 2522 options->cipher_xform.cipher.key.length); 2523 } 2524 } 2525 2526 /* Set auth parameters */ 2527 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || 2528 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || 2529 options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) { 2530 cap = check_device_support_auth_algo(options, &dev_info, 2531 cdev_id); 2532 2533 /* Set IV if not provided from command line */ 2534 if (options->auth_iv_param == 0) { 2535 if (options->auth_iv_random_size != -1) 2536 options->auth_iv.length = 2537 options->auth_iv_random_size; 2538 /* No size provided, use minimum size. */ 2539 else 2540 options->auth_iv.length = 2541 cap->sym.auth.iv_size.min; 2542 } 2543 2544 /* Set key if not provided from command line */ 2545 if (options->akey_param == 0) { 2546 if (options->akey_random_size != -1) 2547 options->auth_xform.auth.key.length = 2548 options->akey_random_size; 2549 /* No size provided, use minimum size. */ 2550 else 2551 options->auth_xform.auth.key.length = 2552 cap->sym.auth.key_size.min; 2553 2554 generate_random_key(options->auth_key, 2555 options->auth_xform.auth.key.length); 2556 } 2557 2558 /* Set digest size if not provided from command line */ 2559 if (options->digest_size != -1) 2560 options->auth_xform.auth.digest_length = 2561 options->digest_size; 2562 /* No size provided, use minimum size. */ 2563 else 2564 options->auth_xform.auth.digest_length = 2565 cap->sym.auth.digest_size.min; 2566 } 2567 2568 retval = rte_cryptodev_configure(cdev_id, &conf); 2569 if (retval < 0) { 2570 printf("Failed to configure cryptodev %u", cdev_id); 2571 return -1; 2572 } 2573 2574 qp_conf.nb_descriptors = 2048; 2575 qp_conf.mp_session = session_pool_socket[socket_id].sess_mp; 2576 qp_conf.mp_session_private = 2577 session_pool_socket[socket_id].priv_mp; 2578 2579 retval = rte_cryptodev_queue_pair_setup(cdev_id, 0, &qp_conf, 2580 socket_id); 2581 if (retval < 0) { 2582 printf("Failed to setup queue pair %u on cryptodev %u", 2583 0, cdev_id); 2584 return -1; 2585 } 2586 2587 retval = rte_cryptodev_start(cdev_id); 2588 if (retval < 0) { 2589 printf("Failed to start device %u: error %d\n", 2590 cdev_id, retval); 2591 return -1; 2592 } 2593 } 2594 2595 return enabled_cdev_count; 2596 } 2597 2598 static int 2599 initialize_ports(struct l2fwd_crypto_options *options) 2600 { 2601 uint16_t last_portid = 0, portid; 2602 unsigned enabled_portcount = 0; 2603 unsigned nb_ports = rte_eth_dev_count_avail(); 2604 2605 if (nb_ports == 0) { 2606 printf("No Ethernet ports - bye\n"); 2607 return -1; 2608 } 2609 2610 /* Reset l2fwd_dst_ports */ 2611 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) 2612 l2fwd_dst_ports[portid] = 0; 2613 2614 RTE_ETH_FOREACH_DEV(portid) { 2615 int retval; 2616 struct rte_eth_dev_info dev_info; 2617 struct rte_eth_rxconf rxq_conf; 2618 struct rte_eth_txconf txq_conf; 2619 struct rte_eth_conf local_port_conf = port_conf; 2620 2621 /* Skip ports that are not enabled */ 2622 if ((options->portmask & (1 << portid)) == 0) 2623 continue; 2624 2625 /* init port */ 2626 printf("Initializing port %u... ", portid); 2627 fflush(stdout); 2628 2629 retval = rte_eth_dev_info_get(portid, &dev_info); 2630 if (retval != 0) { 2631 printf("Error during getting device (port %u) info: %s\n", 2632 portid, strerror(-retval)); 2633 return retval; 2634 } 2635 2636 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 2637 local_port_conf.txmode.offloads |= 2638 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 2639 retval = rte_eth_dev_configure(portid, 1, 1, &local_port_conf); 2640 if (retval < 0) { 2641 printf("Cannot configure device: err=%d, port=%u\n", 2642 retval, portid); 2643 return -1; 2644 } 2645 2646 retval = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, 2647 &nb_txd); 2648 if (retval < 0) { 2649 printf("Cannot adjust number of descriptors: err=%d, port=%u\n", 2650 retval, portid); 2651 return -1; 2652 } 2653 2654 /* init one RX queue */ 2655 fflush(stdout); 2656 rxq_conf = dev_info.default_rxconf; 2657 rxq_conf.offloads = local_port_conf.rxmode.offloads; 2658 retval = rte_eth_rx_queue_setup(portid, 0, nb_rxd, 2659 rte_eth_dev_socket_id(portid), 2660 &rxq_conf, l2fwd_pktmbuf_pool); 2661 if (retval < 0) { 2662 printf("rte_eth_rx_queue_setup:err=%d, port=%u\n", 2663 retval, portid); 2664 return -1; 2665 } 2666 2667 /* init one TX queue on each port */ 2668 fflush(stdout); 2669 txq_conf = dev_info.default_txconf; 2670 txq_conf.offloads = local_port_conf.txmode.offloads; 2671 retval = rte_eth_tx_queue_setup(portid, 0, nb_txd, 2672 rte_eth_dev_socket_id(portid), 2673 &txq_conf); 2674 if (retval < 0) { 2675 printf("rte_eth_tx_queue_setup:err=%d, port=%u\n", 2676 retval, portid); 2677 2678 return -1; 2679 } 2680 2681 /* Start device */ 2682 retval = rte_eth_dev_start(portid); 2683 if (retval < 0) { 2684 printf("rte_eth_dev_start:err=%d, port=%u\n", 2685 retval, portid); 2686 return -1; 2687 } 2688 2689 retval = rte_eth_promiscuous_enable(portid); 2690 if (retval != 0) { 2691 printf("rte_eth_promiscuous_enable:err=%s, port=%u\n", 2692 rte_strerror(-retval), portid); 2693 return -1; 2694 } 2695 2696 retval = rte_eth_macaddr_get(portid, 2697 &l2fwd_ports_eth_addr[portid]); 2698 if (retval < 0) { 2699 printf("rte_eth_macaddr_get :err=%d, port=%u\n", 2700 retval, portid); 2701 return -1; 2702 } 2703 2704 printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n", 2705 portid, 2706 RTE_ETHER_ADDR_BYTES(&l2fwd_ports_eth_addr[portid])); 2707 2708 /* initialize port stats */ 2709 memset(&port_statistics, 0, sizeof(port_statistics)); 2710 2711 /* Setup port forwarding table */ 2712 if (enabled_portcount % 2) { 2713 l2fwd_dst_ports[portid] = last_portid; 2714 l2fwd_dst_ports[last_portid] = portid; 2715 } else { 2716 last_portid = portid; 2717 } 2718 2719 l2fwd_enabled_port_mask |= (1 << portid); 2720 enabled_portcount++; 2721 } 2722 2723 if (enabled_portcount == 1) { 2724 l2fwd_dst_ports[last_portid] = last_portid; 2725 } else if (enabled_portcount % 2) { 2726 printf("odd number of ports in portmask- bye\n"); 2727 return -1; 2728 } 2729 2730 check_all_ports_link_status(l2fwd_enabled_port_mask); 2731 2732 return enabled_portcount; 2733 } 2734 2735 static void 2736 reserve_key_memory(struct l2fwd_crypto_options *options) 2737 { 2738 options->cipher_xform.cipher.key.data = options->cipher_key; 2739 2740 options->auth_xform.auth.key.data = options->auth_key; 2741 2742 options->aead_xform.aead.key.data = options->aead_key; 2743 2744 options->cipher_iv.data = rte_malloc("cipher iv", MAX_KEY_SIZE, 0); 2745 if (options->cipher_iv.data == NULL) 2746 rte_exit(EXIT_FAILURE, "Failed to allocate memory for cipher IV"); 2747 2748 options->auth_iv.data = rte_malloc("auth iv", MAX_KEY_SIZE, 0); 2749 if (options->auth_iv.data == NULL) 2750 rte_exit(EXIT_FAILURE, "Failed to allocate memory for auth IV"); 2751 2752 options->aead_iv.data = rte_malloc("aead_iv", MAX_KEY_SIZE, 0); 2753 if (options->aead_iv.data == NULL) 2754 rte_exit(EXIT_FAILURE, "Failed to allocate memory for AEAD iv"); 2755 2756 options->aad.data = rte_malloc("aad", MAX_KEY_SIZE, 0); 2757 if (options->aad.data == NULL) 2758 rte_exit(EXIT_FAILURE, "Failed to allocate memory for AAD"); 2759 options->aad.phys_addr = rte_malloc_virt2iova(options->aad.data); 2760 } 2761 2762 int 2763 main(int argc, char **argv) 2764 { 2765 struct lcore_queue_conf *qconf = NULL; 2766 struct l2fwd_crypto_options options; 2767 2768 uint8_t nb_cryptodevs, cdev_id; 2769 uint16_t portid; 2770 unsigned lcore_id, rx_lcore_id = 0; 2771 int ret, enabled_cdevcount, enabled_portcount; 2772 uint8_t enabled_cdevs[RTE_CRYPTO_MAX_DEVS] = {0}; 2773 2774 /* init EAL */ 2775 ret = rte_eal_init(argc, argv); 2776 if (ret < 0) 2777 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n"); 2778 argc -= ret; 2779 argv += ret; 2780 2781 /* reserve memory for Cipher/Auth key and IV */ 2782 reserve_key_memory(&options); 2783 2784 /* parse application arguments (after the EAL ones) */ 2785 ret = l2fwd_crypto_parse_args(&options, argc, argv); 2786 if (ret < 0) 2787 rte_exit(EXIT_FAILURE, "Invalid L2FWD-CRYPTO arguments\n"); 2788 2789 printf("MAC updating %s\n", 2790 options.mac_updating ? "enabled" : "disabled"); 2791 2792 /* create the mbuf pool */ 2793 l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF, 512, 2794 RTE_ALIGN(sizeof(struct rte_crypto_op), 2795 RTE_CACHE_LINE_SIZE), 2796 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 2797 if (l2fwd_pktmbuf_pool == NULL) 2798 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 2799 2800 /* create crypto op pool */ 2801 l2fwd_crypto_op_pool = rte_crypto_op_pool_create("crypto_op_pool", 2802 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MBUF, 128, MAXIMUM_IV_LENGTH, 2803 rte_socket_id()); 2804 if (l2fwd_crypto_op_pool == NULL) 2805 rte_exit(EXIT_FAILURE, "Cannot create crypto op pool\n"); 2806 2807 /* Enable Ethernet ports */ 2808 enabled_portcount = initialize_ports(&options); 2809 if (enabled_portcount < 1) 2810 rte_exit(EXIT_FAILURE, "Failed to initial Ethernet ports\n"); 2811 2812 /* Initialize the port/queue configuration of each logical core */ 2813 RTE_ETH_FOREACH_DEV(portid) { 2814 2815 /* skip ports that are not enabled */ 2816 if ((options.portmask & (1 << portid)) == 0) 2817 continue; 2818 2819 if (options.single_lcore && qconf == NULL) { 2820 while (rte_lcore_is_enabled(rx_lcore_id) == 0) { 2821 rx_lcore_id++; 2822 if (rx_lcore_id >= RTE_MAX_LCORE) 2823 rte_exit(EXIT_FAILURE, 2824 "Not enough cores\n"); 2825 } 2826 } else if (!options.single_lcore) { 2827 /* get the lcore_id for this port */ 2828 while (rte_lcore_is_enabled(rx_lcore_id) == 0 || 2829 lcore_queue_conf[rx_lcore_id].nb_rx_ports == 2830 options.nb_ports_per_lcore) { 2831 rx_lcore_id++; 2832 if (rx_lcore_id >= RTE_MAX_LCORE) 2833 rte_exit(EXIT_FAILURE, 2834 "Not enough cores\n"); 2835 } 2836 } 2837 2838 /* Assigned a new logical core in the loop above. */ 2839 if (qconf != &lcore_queue_conf[rx_lcore_id]) 2840 qconf = &lcore_queue_conf[rx_lcore_id]; 2841 2842 qconf->rx_port_list[qconf->nb_rx_ports] = portid; 2843 qconf->nb_rx_ports++; 2844 2845 printf("Lcore %u: RX port %u\n", rx_lcore_id, portid); 2846 } 2847 2848 /* Enable Crypto devices */ 2849 enabled_cdevcount = initialize_cryptodevs(&options, enabled_portcount, 2850 enabled_cdevs); 2851 if (enabled_cdevcount < 0) 2852 rte_exit(EXIT_FAILURE, "Failed to initialize crypto devices\n"); 2853 2854 if (enabled_cdevcount < enabled_portcount) 2855 rte_exit(EXIT_FAILURE, "Number of capable crypto devices (%d) " 2856 "has to be more or equal to number of ports (%d)\n", 2857 enabled_cdevcount, enabled_portcount); 2858 2859 nb_cryptodevs = rte_cryptodev_count(); 2860 2861 /* Initialize the port/cryptodev configuration of each logical core */ 2862 for (rx_lcore_id = 0, qconf = NULL, cdev_id = 0; 2863 cdev_id < nb_cryptodevs && enabled_cdevcount; 2864 cdev_id++) { 2865 /* Crypto op not supported by crypto device */ 2866 if (!enabled_cdevs[cdev_id]) 2867 continue; 2868 2869 if (options.single_lcore && qconf == NULL) { 2870 while (rte_lcore_is_enabled(rx_lcore_id) == 0) { 2871 rx_lcore_id++; 2872 if (rx_lcore_id >= RTE_MAX_LCORE) 2873 rte_exit(EXIT_FAILURE, 2874 "Not enough cores\n"); 2875 } 2876 } else if (!options.single_lcore) { 2877 /* get the lcore_id for this port */ 2878 while (rte_lcore_is_enabled(rx_lcore_id) == 0 || 2879 lcore_queue_conf[rx_lcore_id].nb_crypto_devs == 2880 options.nb_ports_per_lcore) { 2881 rx_lcore_id++; 2882 if (rx_lcore_id >= RTE_MAX_LCORE) 2883 rte_exit(EXIT_FAILURE, 2884 "Not enough cores\n"); 2885 } 2886 } 2887 2888 /* Assigned a new logical core in the loop above. */ 2889 if (qconf != &lcore_queue_conf[rx_lcore_id]) 2890 qconf = &lcore_queue_conf[rx_lcore_id]; 2891 2892 qconf->cryptodev_list[qconf->nb_crypto_devs] = cdev_id; 2893 qconf->nb_crypto_devs++; 2894 2895 enabled_cdevcount--; 2896 2897 printf("Lcore %u: cryptodev %u\n", rx_lcore_id, 2898 (unsigned)cdev_id); 2899 } 2900 2901 /* launch per-lcore init on every lcore */ 2902 rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, (void *)&options, 2903 CALL_MAIN); 2904 RTE_LCORE_FOREACH_WORKER(lcore_id) { 2905 if (rte_eal_wait_lcore(lcore_id) < 0) 2906 return -1; 2907 } 2908 2909 /* clean up the EAL */ 2910 rte_eal_cleanup(); 2911 2912 return 0; 2913 } 2914