1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <string.h> 6 #include <stdio.h> 7 8 #include <rte_common.h> 9 #include <rte_mbuf.h> 10 #include <rte_malloc.h> 11 #include <rte_string_fns.h> 12 13 #include "rte_pipeline.h" 14 15 RTE_LOG_REGISTER_DEFAULT(pipeline_logtype, INFO); 16 #define RTE_LOGTYPE_PIPELINE pipeline_logtype 17 18 #define PIPELINE_LOG(level, ...) \ 19 RTE_LOG_LINE(level, PIPELINE, "" __VA_ARGS__) 20 21 #define RTE_TABLE_INVALID UINT32_MAX 22 23 #ifdef RTE_PIPELINE_STATS_COLLECT 24 25 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) \ 26 __extension__ ({ (p)->n_pkts_ah_drop = rte_popcount64(mask); }) 27 28 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) \ 29 __extension__ ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; }) 30 31 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) \ 32 __extension__ ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; }) 33 34 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) \ 35 __extension__ ({ \ 36 uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; \ 37 mask ^= (p)->pkts_drop_mask; \ 38 (counter) += rte_popcount64(mask); \ 39 }) 40 41 #else 42 43 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) 44 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) 45 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) 46 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) 47 48 #endif 49 50 struct rte_port_in { 51 /* Input parameters */ 52 struct rte_port_in_ops ops; 53 rte_pipeline_port_in_action_handler f_action; 54 void *arg_ah; 55 uint32_t burst_size; 56 57 /* The table to which this port is connected */ 58 uint32_t table_id; 59 60 /* Handle to low-level port */ 61 void *h_port; 62 63 /* List of enabled ports */ 64 struct rte_port_in *next; 65 66 /* Statistics */ 67 uint64_t n_pkts_dropped_by_ah; 68 }; 69 70 struct rte_port_out { 71 /* Input parameters */ 72 struct rte_port_out_ops ops; 73 rte_pipeline_port_out_action_handler f_action; 74 void *arg_ah; 75 76 /* Handle to low-level port */ 77 void *h_port; 78 79 /* Statistics */ 80 uint64_t n_pkts_dropped_by_ah; 81 }; 82 83 struct rte_table { 84 /* Input parameters */ 85 struct rte_table_ops ops; 86 rte_pipeline_table_action_handler_hit f_action_hit; 87 rte_pipeline_table_action_handler_miss f_action_miss; 88 void *arg_ah; 89 struct rte_pipeline_table_entry *default_entry; 90 uint32_t entry_size; 91 92 uint32_t table_next_id; 93 uint32_t table_next_id_valid; 94 95 /* Handle to the low-level table object */ 96 void *h_table; 97 98 /* Statistics */ 99 uint64_t n_pkts_dropped_by_lkp_hit_ah; 100 uint64_t n_pkts_dropped_by_lkp_miss_ah; 101 uint64_t n_pkts_dropped_lkp_hit; 102 uint64_t n_pkts_dropped_lkp_miss; 103 }; 104 105 #define RTE_PIPELINE_MAX_NAME_SZ 124 106 107 struct rte_pipeline { 108 /* Input parameters */ 109 char name[RTE_PIPELINE_MAX_NAME_SZ]; 110 int socket_id; 111 uint32_t offset_port_id; 112 113 /* Internal tables */ 114 struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX]; 115 struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX]; 116 struct rte_table tables[RTE_PIPELINE_TABLE_MAX]; 117 118 /* Occupancy of internal tables */ 119 uint32_t num_ports_in; 120 uint32_t num_ports_out; 121 uint32_t num_tables; 122 123 /* List of enabled ports */ 124 uint64_t enabled_port_in_mask; 125 struct rte_port_in *port_in_next; 126 127 /* Pipeline run structures */ 128 struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX]; 129 struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX]; 130 uint64_t action_mask0[RTE_PIPELINE_ACTIONS]; 131 uint64_t action_mask1[RTE_PIPELINE_ACTIONS]; 132 uint64_t pkts_mask; 133 uint64_t n_pkts_ah_drop; 134 uint64_t pkts_drop_mask; 135 } __rte_cache_aligned; 136 137 static inline uint32_t 138 rte_mask_get_next(uint64_t mask, uint32_t pos) 139 { 140 uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) | 141 (mask >> ((pos + 1) & 0x3F)); 142 return (rte_ctz64(mask_rot) - (63 - pos)) & 0x3F; 143 } 144 145 static inline uint32_t 146 rte_mask_get_prev(uint64_t mask, uint32_t pos) 147 { 148 uint64_t mask_rot = (mask >> (pos & 0x3F)) | 149 (mask << ((64 - pos) & 0x3F)); 150 return ((63 - rte_clz64(mask_rot)) + pos) & 0x3F; 151 } 152 153 static void 154 rte_pipeline_table_free(struct rte_table *table); 155 156 static void 157 rte_pipeline_port_in_free(struct rte_port_in *port); 158 159 static void 160 rte_pipeline_port_out_free(struct rte_port_out *port); 161 162 /* 163 * Pipeline 164 */ 165 static int 166 rte_pipeline_check_params(struct rte_pipeline_params *params) 167 { 168 if (params == NULL) { 169 PIPELINE_LOG(ERR, 170 "%s: Incorrect value for parameter params", __func__); 171 return -EINVAL; 172 } 173 174 /* name */ 175 if (params->name == NULL) { 176 PIPELINE_LOG(ERR, 177 "%s: Incorrect value for parameter name", __func__); 178 return -EINVAL; 179 } 180 181 /* socket */ 182 if (params->socket_id < 0) { 183 PIPELINE_LOG(ERR, 184 "%s: Incorrect value for parameter socket_id", 185 __func__); 186 return -EINVAL; 187 } 188 189 return 0; 190 } 191 192 struct rte_pipeline * 193 rte_pipeline_create(struct rte_pipeline_params *params) 194 { 195 struct rte_pipeline *p; 196 int status; 197 198 /* Check input parameters */ 199 status = rte_pipeline_check_params(params); 200 if (status != 0) { 201 PIPELINE_LOG(ERR, 202 "%s: Pipeline params check failed (%d)", 203 __func__, status); 204 return NULL; 205 } 206 207 /* Allocate memory for the pipeline on requested socket */ 208 p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline), 209 RTE_CACHE_LINE_SIZE, params->socket_id); 210 211 if (p == NULL) { 212 PIPELINE_LOG(ERR, 213 "%s: Pipeline memory allocation failed", __func__); 214 return NULL; 215 } 216 217 /* Save input parameters */ 218 strlcpy(p->name, params->name, RTE_PIPELINE_MAX_NAME_SZ); 219 p->socket_id = params->socket_id; 220 p->offset_port_id = params->offset_port_id; 221 222 /* Initialize pipeline internal data structure */ 223 p->num_ports_in = 0; 224 p->num_ports_out = 0; 225 p->num_tables = 0; 226 p->enabled_port_in_mask = 0; 227 p->port_in_next = NULL; 228 p->pkts_mask = 0; 229 p->n_pkts_ah_drop = 0; 230 231 return p; 232 } 233 234 int 235 rte_pipeline_free(struct rte_pipeline *p) 236 { 237 uint32_t i; 238 239 /* Check input parameters */ 240 if (p == NULL) { 241 PIPELINE_LOG(ERR, 242 "%s: rte_pipeline parameter is NULL", __func__); 243 return -EINVAL; 244 } 245 246 /* Free input ports */ 247 for (i = 0; i < p->num_ports_in; i++) { 248 struct rte_port_in *port = &p->ports_in[i]; 249 250 rte_pipeline_port_in_free(port); 251 } 252 253 /* Free tables */ 254 for (i = 0; i < p->num_tables; i++) { 255 struct rte_table *table = &p->tables[i]; 256 257 rte_pipeline_table_free(table); 258 } 259 260 /* Free output ports */ 261 for (i = 0; i < p->num_ports_out; i++) { 262 struct rte_port_out *port = &p->ports_out[i]; 263 264 rte_pipeline_port_out_free(port); 265 } 266 267 /* Free pipeline memory */ 268 rte_free(p); 269 270 return 0; 271 } 272 273 /* 274 * Table 275 */ 276 static int 277 rte_table_check_params(struct rte_pipeline *p, 278 struct rte_pipeline_table_params *params, 279 uint32_t *table_id) 280 { 281 if (p == NULL) { 282 PIPELINE_LOG(ERR, "%s: pipeline parameter is NULL", 283 __func__); 284 return -EINVAL; 285 } 286 if (params == NULL) { 287 PIPELINE_LOG(ERR, "%s: params parameter is NULL", 288 __func__); 289 return -EINVAL; 290 } 291 if (table_id == NULL) { 292 PIPELINE_LOG(ERR, "%s: table_id parameter is NULL", 293 __func__); 294 return -EINVAL; 295 } 296 297 /* ops */ 298 if (params->ops == NULL) { 299 PIPELINE_LOG(ERR, "%s: params->ops is NULL", 300 __func__); 301 return -EINVAL; 302 } 303 304 if (params->ops->f_create == NULL) { 305 PIPELINE_LOG(ERR, 306 "%s: f_create function pointer is NULL", __func__); 307 return -EINVAL; 308 } 309 310 if (params->ops->f_lookup == NULL) { 311 PIPELINE_LOG(ERR, 312 "%s: f_lookup function pointer is NULL", __func__); 313 return -EINVAL; 314 } 315 316 /* De we have room for one more table? */ 317 if (p->num_tables == RTE_PIPELINE_TABLE_MAX) { 318 PIPELINE_LOG(ERR, 319 "%s: Incorrect value for num_tables parameter", 320 __func__); 321 return -EINVAL; 322 } 323 324 return 0; 325 } 326 327 int 328 rte_pipeline_table_create(struct rte_pipeline *p, 329 struct rte_pipeline_table_params *params, 330 uint32_t *table_id) 331 { 332 struct rte_table *table; 333 struct rte_pipeline_table_entry *default_entry; 334 void *h_table; 335 uint32_t entry_size, id; 336 int status; 337 338 /* Check input arguments */ 339 status = rte_table_check_params(p, params, table_id); 340 if (status != 0) 341 return status; 342 343 id = p->num_tables; 344 table = &p->tables[id]; 345 346 /* Allocate space for the default table entry */ 347 entry_size = sizeof(struct rte_pipeline_table_entry) + 348 params->action_data_size; 349 default_entry = rte_zmalloc_socket( 350 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id); 351 if (default_entry == NULL) { 352 PIPELINE_LOG(ERR, 353 "%s: Failed to allocate default entry", __func__); 354 return -EINVAL; 355 } 356 357 /* Create the table */ 358 h_table = params->ops->f_create(params->arg_create, p->socket_id, 359 entry_size); 360 if (h_table == NULL) { 361 rte_free(default_entry); 362 PIPELINE_LOG(ERR, "%s: Table creation failed", __func__); 363 return -EINVAL; 364 } 365 366 /* Commit current table to the pipeline */ 367 p->num_tables++; 368 *table_id = id; 369 370 /* Save input parameters */ 371 memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops)); 372 table->f_action_hit = params->f_action_hit; 373 table->f_action_miss = params->f_action_miss; 374 table->arg_ah = params->arg_ah; 375 table->entry_size = entry_size; 376 377 /* Clear the lookup miss actions (to be set later through API) */ 378 table->default_entry = default_entry; 379 table->default_entry->action = RTE_PIPELINE_ACTION_DROP; 380 381 /* Initialize table internal data structure */ 382 table->h_table = h_table; 383 table->table_next_id = 0; 384 table->table_next_id_valid = 0; 385 386 return 0; 387 } 388 389 void 390 rte_pipeline_table_free(struct rte_table *table) 391 { 392 if (table->ops.f_free != NULL) 393 table->ops.f_free(table->h_table); 394 395 rte_free(table->default_entry); 396 } 397 398 int 399 rte_pipeline_table_default_entry_add(struct rte_pipeline *p, 400 uint32_t table_id, 401 struct rte_pipeline_table_entry *default_entry, 402 struct rte_pipeline_table_entry **default_entry_ptr) 403 { 404 struct rte_table *table; 405 406 /* Check input arguments */ 407 if (p == NULL) { 408 PIPELINE_LOG(ERR, "%s: pipeline parameter is NULL", 409 __func__); 410 return -EINVAL; 411 } 412 413 if (default_entry == NULL) { 414 PIPELINE_LOG(ERR, 415 "%s: default_entry parameter is NULL", __func__); 416 return -EINVAL; 417 } 418 419 if (table_id >= p->num_tables) { 420 PIPELINE_LOG(ERR, 421 "%s: table_id %d out of range", __func__, table_id); 422 return -EINVAL; 423 } 424 425 table = &p->tables[table_id]; 426 427 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) && 428 table->table_next_id_valid && 429 (default_entry->table_id != table->table_next_id)) { 430 PIPELINE_LOG(ERR, 431 "%s: Tree-like topologies not allowed", __func__); 432 return -EINVAL; 433 } 434 435 /* Set the lookup miss actions */ 436 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) && 437 (table->table_next_id_valid == 0)) { 438 table->table_next_id = default_entry->table_id; 439 table->table_next_id_valid = 1; 440 } 441 442 memcpy(table->default_entry, default_entry, table->entry_size); 443 444 *default_entry_ptr = table->default_entry; 445 return 0; 446 } 447 448 int 449 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p, 450 uint32_t table_id, 451 struct rte_pipeline_table_entry *entry) 452 { 453 struct rte_table *table; 454 455 /* Check input arguments */ 456 if (p == NULL) { 457 PIPELINE_LOG(ERR, 458 "%s: pipeline parameter is NULL", __func__); 459 return -EINVAL; 460 } 461 462 if (table_id >= p->num_tables) { 463 PIPELINE_LOG(ERR, 464 "%s: table_id %d out of range", __func__, table_id); 465 return -EINVAL; 466 } 467 468 table = &p->tables[table_id]; 469 470 /* Save the current contents of the default entry */ 471 if (entry) 472 memcpy(entry, table->default_entry, table->entry_size); 473 474 /* Clear the lookup miss actions */ 475 memset(table->default_entry, 0, table->entry_size); 476 table->default_entry->action = RTE_PIPELINE_ACTION_DROP; 477 478 return 0; 479 } 480 481 int 482 rte_pipeline_table_entry_add(struct rte_pipeline *p, 483 uint32_t table_id, 484 void *key, 485 struct rte_pipeline_table_entry *entry, 486 int *key_found, 487 struct rte_pipeline_table_entry **entry_ptr) 488 { 489 struct rte_table *table; 490 491 /* Check input arguments */ 492 if (p == NULL) { 493 PIPELINE_LOG(ERR, "%s: pipeline parameter is NULL", 494 __func__); 495 return -EINVAL; 496 } 497 498 if (key == NULL) { 499 PIPELINE_LOG(ERR, "%s: key parameter is NULL", __func__); 500 return -EINVAL; 501 } 502 503 if (entry == NULL) { 504 PIPELINE_LOG(ERR, "%s: entry parameter is NULL", 505 __func__); 506 return -EINVAL; 507 } 508 509 if (table_id >= p->num_tables) { 510 PIPELINE_LOG(ERR, 511 "%s: table_id %d out of range", __func__, table_id); 512 return -EINVAL; 513 } 514 515 table = &p->tables[table_id]; 516 517 if (table->ops.f_add == NULL) { 518 PIPELINE_LOG(ERR, "%s: f_add function pointer NULL", 519 __func__); 520 return -EINVAL; 521 } 522 523 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) && 524 table->table_next_id_valid && 525 (entry->table_id != table->table_next_id)) { 526 PIPELINE_LOG(ERR, 527 "%s: Tree-like topologies not allowed", __func__); 528 return -EINVAL; 529 } 530 531 /* Add entry */ 532 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) && 533 (table->table_next_id_valid == 0)) { 534 table->table_next_id = entry->table_id; 535 table->table_next_id_valid = 1; 536 } 537 538 return (table->ops.f_add)(table->h_table, key, (void *) entry, 539 key_found, (void **) entry_ptr); 540 } 541 542 int 543 rte_pipeline_table_entry_delete(struct rte_pipeline *p, 544 uint32_t table_id, 545 void *key, 546 int *key_found, 547 struct rte_pipeline_table_entry *entry) 548 { 549 struct rte_table *table; 550 551 /* Check input arguments */ 552 if (p == NULL) { 553 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 554 __func__); 555 return -EINVAL; 556 } 557 558 if (key == NULL) { 559 PIPELINE_LOG(ERR, "%s: key parameter is NULL", 560 __func__); 561 return -EINVAL; 562 } 563 564 if (table_id >= p->num_tables) { 565 PIPELINE_LOG(ERR, 566 "%s: table_id %d out of range", __func__, table_id); 567 return -EINVAL; 568 } 569 570 table = &p->tables[table_id]; 571 572 if (table->ops.f_delete == NULL) { 573 PIPELINE_LOG(ERR, 574 "%s: f_delete function pointer NULL", __func__); 575 return -EINVAL; 576 } 577 578 return (table->ops.f_delete)(table->h_table, key, key_found, entry); 579 } 580 581 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p, 582 uint32_t table_id, 583 void **keys, 584 struct rte_pipeline_table_entry **entries, 585 uint32_t n_keys, 586 int *key_found, 587 struct rte_pipeline_table_entry **entries_ptr) 588 { 589 struct rte_table *table; 590 uint32_t i; 591 592 /* Check input arguments */ 593 if (p == NULL) { 594 PIPELINE_LOG(ERR, "%s: pipeline parameter is NULL", 595 __func__); 596 return -EINVAL; 597 } 598 599 if (keys == NULL) { 600 PIPELINE_LOG(ERR, "%s: keys parameter is NULL", __func__); 601 return -EINVAL; 602 } 603 604 if (entries == NULL) { 605 PIPELINE_LOG(ERR, "%s: entries parameter is NULL", 606 __func__); 607 return -EINVAL; 608 } 609 610 if (table_id >= p->num_tables) { 611 PIPELINE_LOG(ERR, 612 "%s: table_id %d out of range", __func__, table_id); 613 return -EINVAL; 614 } 615 616 table = &p->tables[table_id]; 617 618 if (table->ops.f_add_bulk == NULL) { 619 PIPELINE_LOG(ERR, "%s: f_add_bulk function pointer NULL", 620 __func__); 621 return -EINVAL; 622 } 623 624 for (i = 0; i < n_keys; i++) { 625 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) && 626 table->table_next_id_valid && 627 (entries[i]->table_id != table->table_next_id)) { 628 PIPELINE_LOG(ERR, 629 "%s: Tree-like topologies not allowed", __func__); 630 return -EINVAL; 631 } 632 } 633 634 /* Add entry */ 635 for (i = 0; i < n_keys; i++) { 636 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) && 637 (table->table_next_id_valid == 0)) { 638 table->table_next_id = entries[i]->table_id; 639 table->table_next_id_valid = 1; 640 } 641 } 642 643 return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries, 644 n_keys, key_found, (void **) entries_ptr); 645 } 646 647 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p, 648 uint32_t table_id, 649 void **keys, 650 uint32_t n_keys, 651 int *key_found, 652 struct rte_pipeline_table_entry **entries) 653 { 654 struct rte_table *table; 655 656 /* Check input arguments */ 657 if (p == NULL) { 658 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 659 __func__); 660 return -EINVAL; 661 } 662 663 if (keys == NULL) { 664 PIPELINE_LOG(ERR, "%s: key parameter is NULL", 665 __func__); 666 return -EINVAL; 667 } 668 669 if (table_id >= p->num_tables) { 670 PIPELINE_LOG(ERR, 671 "%s: table_id %d out of range", __func__, table_id); 672 return -EINVAL; 673 } 674 675 table = &p->tables[table_id]; 676 677 if (table->ops.f_delete_bulk == NULL) { 678 PIPELINE_LOG(ERR, 679 "%s: f_delete function pointer NULL", __func__); 680 return -EINVAL; 681 } 682 683 return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found, 684 (void **) entries); 685 } 686 687 /* 688 * Port 689 */ 690 static int 691 rte_pipeline_port_in_check_params(struct rte_pipeline *p, 692 struct rte_pipeline_port_in_params *params, 693 uint32_t *port_id) 694 { 695 if (p == NULL) { 696 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 697 __func__); 698 return -EINVAL; 699 } 700 if (params == NULL) { 701 PIPELINE_LOG(ERR, "%s: params parameter NULL", __func__); 702 return -EINVAL; 703 } 704 if (port_id == NULL) { 705 PIPELINE_LOG(ERR, "%s: port_id parameter NULL", 706 __func__); 707 return -EINVAL; 708 } 709 710 /* ops */ 711 if (params->ops == NULL) { 712 PIPELINE_LOG(ERR, "%s: params->ops parameter NULL", 713 __func__); 714 return -EINVAL; 715 } 716 717 if (params->ops->f_create == NULL) { 718 PIPELINE_LOG(ERR, 719 "%s: f_create function pointer NULL", __func__); 720 return -EINVAL; 721 } 722 723 if (params->ops->f_rx == NULL) { 724 PIPELINE_LOG(ERR, "%s: f_rx function pointer NULL", 725 __func__); 726 return -EINVAL; 727 } 728 729 /* burst_size */ 730 if ((params->burst_size == 0) || 731 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) { 732 PIPELINE_LOG(ERR, "%s: invalid value for burst_size", 733 __func__); 734 return -EINVAL; 735 } 736 737 /* Do we have room for one more port? */ 738 if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) { 739 PIPELINE_LOG(ERR, 740 "%s: invalid value for num_ports_in", __func__); 741 return -EINVAL; 742 } 743 744 return 0; 745 } 746 747 static int 748 rte_pipeline_port_out_check_params(struct rte_pipeline *p, 749 struct rte_pipeline_port_out_params *params, 750 uint32_t *port_id) 751 { 752 if (p == NULL) { 753 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 754 __func__); 755 return -EINVAL; 756 } 757 758 if (params == NULL) { 759 PIPELINE_LOG(ERR, "%s: params parameter NULL", __func__); 760 return -EINVAL; 761 } 762 763 if (port_id == NULL) { 764 PIPELINE_LOG(ERR, "%s: port_id parameter NULL", 765 __func__); 766 return -EINVAL; 767 } 768 769 /* ops */ 770 if (params->ops == NULL) { 771 PIPELINE_LOG(ERR, "%s: params->ops parameter NULL", 772 __func__); 773 return -EINVAL; 774 } 775 776 if (params->ops->f_create == NULL) { 777 PIPELINE_LOG(ERR, 778 "%s: f_create function pointer NULL", __func__); 779 return -EINVAL; 780 } 781 782 if (params->ops->f_tx == NULL) { 783 PIPELINE_LOG(ERR, 784 "%s: f_tx function pointer NULL", __func__); 785 return -EINVAL; 786 } 787 788 if (params->ops->f_tx_bulk == NULL) { 789 PIPELINE_LOG(ERR, 790 "%s: f_tx_bulk function pointer NULL", __func__); 791 return -EINVAL; 792 } 793 794 /* Do we have room for one more port? */ 795 if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) { 796 PIPELINE_LOG(ERR, 797 "%s: invalid value for num_ports_out", __func__); 798 return -EINVAL; 799 } 800 801 return 0; 802 } 803 804 int 805 rte_pipeline_port_in_create(struct rte_pipeline *p, 806 struct rte_pipeline_port_in_params *params, 807 uint32_t *port_id) 808 { 809 struct rte_port_in *port; 810 void *h_port; 811 uint32_t id; 812 int status; 813 814 /* Check input arguments */ 815 status = rte_pipeline_port_in_check_params(p, params, port_id); 816 if (status != 0) 817 return status; 818 819 id = p->num_ports_in; 820 port = &p->ports_in[id]; 821 822 /* Create the port */ 823 h_port = params->ops->f_create(params->arg_create, p->socket_id); 824 if (h_port == NULL) { 825 PIPELINE_LOG(ERR, "%s: Port creation failed", __func__); 826 return -EINVAL; 827 } 828 829 /* Commit current table to the pipeline */ 830 p->num_ports_in++; 831 *port_id = id; 832 833 /* Save input parameters */ 834 memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops)); 835 port->f_action = params->f_action; 836 port->arg_ah = params->arg_ah; 837 port->burst_size = params->burst_size; 838 839 /* Initialize port internal data structure */ 840 port->table_id = RTE_TABLE_INVALID; 841 port->h_port = h_port; 842 port->next = NULL; 843 844 return 0; 845 } 846 847 void 848 rte_pipeline_port_in_free(struct rte_port_in *port) 849 { 850 if (port->ops.f_free != NULL) 851 port->ops.f_free(port->h_port); 852 } 853 854 int 855 rte_pipeline_port_out_create(struct rte_pipeline *p, 856 struct rte_pipeline_port_out_params *params, 857 uint32_t *port_id) 858 { 859 struct rte_port_out *port; 860 void *h_port; 861 uint32_t id; 862 int status; 863 864 /* Check input arguments */ 865 status = rte_pipeline_port_out_check_params(p, params, port_id); 866 if (status != 0) 867 return status; 868 869 id = p->num_ports_out; 870 port = &p->ports_out[id]; 871 872 /* Create the port */ 873 h_port = params->ops->f_create(params->arg_create, p->socket_id); 874 if (h_port == NULL) { 875 PIPELINE_LOG(ERR, "%s: Port creation failed", __func__); 876 return -EINVAL; 877 } 878 879 /* Commit current table to the pipeline */ 880 p->num_ports_out++; 881 *port_id = id; 882 883 /* Save input parameters */ 884 memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops)); 885 port->f_action = params->f_action; 886 port->arg_ah = params->arg_ah; 887 888 /* Initialize port internal data structure */ 889 port->h_port = h_port; 890 891 return 0; 892 } 893 894 void 895 rte_pipeline_port_out_free(struct rte_port_out *port) 896 { 897 if (port->ops.f_free != NULL) 898 port->ops.f_free(port->h_port); 899 } 900 901 int 902 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p, 903 uint32_t port_id, 904 uint32_t table_id) 905 { 906 struct rte_port_in *port; 907 908 /* Check input arguments */ 909 if (p == NULL) { 910 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 911 __func__); 912 return -EINVAL; 913 } 914 915 if (port_id >= p->num_ports_in) { 916 PIPELINE_LOG(ERR, 917 "%s: port IN ID %u is out of range", 918 __func__, port_id); 919 return -EINVAL; 920 } 921 922 if (table_id >= p->num_tables) { 923 PIPELINE_LOG(ERR, 924 "%s: Table ID %u is out of range", 925 __func__, table_id); 926 return -EINVAL; 927 } 928 929 port = &p->ports_in[port_id]; 930 port->table_id = table_id; 931 932 return 0; 933 } 934 935 int 936 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id) 937 { 938 struct rte_port_in *port, *port_prev, *port_next; 939 uint64_t port_mask; 940 uint32_t port_prev_id, port_next_id; 941 942 /* Check input arguments */ 943 if (p == NULL) { 944 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 945 __func__); 946 return -EINVAL; 947 } 948 949 if (port_id >= p->num_ports_in) { 950 PIPELINE_LOG(ERR, 951 "%s: port IN ID %u is out of range", 952 __func__, port_id); 953 return -EINVAL; 954 } 955 956 port = &p->ports_in[port_id]; 957 958 /* Return if current input port is already enabled */ 959 port_mask = 1LLU << port_id; 960 if (p->enabled_port_in_mask & port_mask) 961 return 0; 962 963 p->enabled_port_in_mask |= port_mask; 964 965 /* Add current input port to the pipeline chain of enabled ports */ 966 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id); 967 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id); 968 969 port_prev = &p->ports_in[port_prev_id]; 970 port_next = &p->ports_in[port_next_id]; 971 972 port_prev->next = port; 973 port->next = port_next; 974 975 /* Check if list of enabled ports was previously empty */ 976 if (p->enabled_port_in_mask == port_mask) 977 p->port_in_next = port; 978 979 return 0; 980 } 981 982 int 983 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id) 984 { 985 struct rte_port_in *port, *port_prev, *port_next; 986 uint64_t port_mask; 987 uint32_t port_prev_id, port_next_id; 988 989 /* Check input arguments */ 990 if (p == NULL) { 991 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 992 __func__); 993 return -EINVAL; 994 } 995 996 if (port_id >= p->num_ports_in) { 997 PIPELINE_LOG(ERR, "%s: port IN ID %u is out of range", 998 __func__, port_id); 999 return -EINVAL; 1000 } 1001 1002 port = &p->ports_in[port_id]; 1003 1004 /* Return if current input port is already disabled */ 1005 port_mask = 1LLU << port_id; 1006 if ((p->enabled_port_in_mask & port_mask) == 0) 1007 return 0; 1008 1009 p->enabled_port_in_mask &= ~port_mask; 1010 1011 /* Return if no other enabled ports */ 1012 if (p->enabled_port_in_mask == 0) { 1013 p->port_in_next = NULL; 1014 1015 return 0; 1016 } 1017 1018 /* Add current input port to the pipeline chain of enabled ports */ 1019 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id); 1020 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id); 1021 1022 port_prev = &p->ports_in[port_prev_id]; 1023 port_next = &p->ports_in[port_next_id]; 1024 1025 port_prev->next = port_next; 1026 1027 /* Check if the port which has just been disabled is next to serve */ 1028 if (port == p->port_in_next) 1029 p->port_in_next = port_next; 1030 1031 return 0; 1032 } 1033 1034 /* 1035 * Pipeline run-time 1036 */ 1037 int 1038 rte_pipeline_check(struct rte_pipeline *p) 1039 { 1040 uint32_t port_in_id; 1041 1042 /* Check input arguments */ 1043 if (p == NULL) { 1044 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 1045 __func__); 1046 return -EINVAL; 1047 } 1048 1049 /* Check that pipeline has at least one input port, one table and one 1050 output port */ 1051 if (p->num_ports_in == 0) { 1052 PIPELINE_LOG(ERR, "%s: must have at least 1 input port", 1053 __func__); 1054 return -EINVAL; 1055 } 1056 if (p->num_tables == 0) { 1057 PIPELINE_LOG(ERR, "%s: must have at least 1 table", 1058 __func__); 1059 return -EINVAL; 1060 } 1061 if (p->num_ports_out == 0) { 1062 PIPELINE_LOG(ERR, "%s: must have at least 1 output port", 1063 __func__); 1064 return -EINVAL; 1065 } 1066 1067 /* Check that all input ports are connected */ 1068 for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) { 1069 struct rte_port_in *port_in = &p->ports_in[port_in_id]; 1070 1071 if (port_in->table_id == RTE_TABLE_INVALID) { 1072 PIPELINE_LOG(ERR, 1073 "%s: Port IN ID %u is not connected", 1074 __func__, port_in_id); 1075 return -EINVAL; 1076 } 1077 } 1078 1079 return 0; 1080 } 1081 1082 static inline void 1083 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask) 1084 { 1085 p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0; 1086 p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0; 1087 p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0; 1088 p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0; 1089 1090 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1091 uint64_t n_pkts = rte_popcount64(pkts_mask); 1092 uint32_t i; 1093 1094 for (i = 0; i < n_pkts; i++) { 1095 uint64_t pkt_mask = 1LLU << i; 1096 uint32_t pos = p->entries[i]->action; 1097 1098 p->action_mask1[pos] |= pkt_mask; 1099 } 1100 } else { 1101 uint32_t i; 1102 1103 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1104 uint64_t pkt_mask = 1LLU << i; 1105 uint32_t pos; 1106 1107 if ((pkt_mask & pkts_mask) == 0) 1108 continue; 1109 1110 pos = p->entries[i]->action; 1111 p->action_mask1[pos] |= pkt_mask; 1112 } 1113 } 1114 } 1115 1116 static inline void 1117 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p, 1118 uint64_t pkts_mask, uint32_t port_id) 1119 { 1120 struct rte_port_out *port_out = &p->ports_out[port_id]; 1121 1122 p->pkts_mask = pkts_mask; 1123 1124 /* Output port user actions */ 1125 if (port_out->f_action != NULL) { 1126 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah); 1127 1128 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1129 port_out->n_pkts_dropped_by_ah); 1130 } 1131 1132 /* Output port TX */ 1133 if (p->pkts_mask != 0) 1134 port_out->ops.f_tx_bulk(port_out->h_port, 1135 p->pkts, 1136 p->pkts_mask); 1137 } 1138 1139 static inline void 1140 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask) 1141 { 1142 p->pkts_mask = pkts_mask; 1143 1144 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1145 uint64_t n_pkts = rte_popcount64(pkts_mask); 1146 uint32_t i; 1147 1148 for (i = 0; i < n_pkts; i++) { 1149 struct rte_mbuf *pkt = p->pkts[i]; 1150 uint32_t port_out_id = p->entries[i]->port_id; 1151 struct rte_port_out *port_out = 1152 &p->ports_out[port_out_id]; 1153 1154 /* Output port user actions */ 1155 if (port_out->f_action == NULL) /* Output port TX */ 1156 port_out->ops.f_tx(port_out->h_port, pkt); 1157 else { 1158 uint64_t pkt_mask = 1LLU << i; 1159 1160 port_out->f_action(p, 1161 p->pkts, 1162 pkt_mask, 1163 port_out->arg_ah); 1164 1165 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1166 port_out->n_pkts_dropped_by_ah); 1167 1168 /* Output port TX */ 1169 if (pkt_mask & p->pkts_mask) 1170 port_out->ops.f_tx(port_out->h_port, 1171 pkt); 1172 } 1173 } 1174 } else { 1175 uint32_t i; 1176 1177 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1178 uint64_t pkt_mask = 1LLU << i; 1179 struct rte_mbuf *pkt; 1180 struct rte_port_out *port_out; 1181 uint32_t port_out_id; 1182 1183 if ((pkt_mask & pkts_mask) == 0) 1184 continue; 1185 1186 pkt = p->pkts[i]; 1187 port_out_id = p->entries[i]->port_id; 1188 port_out = &p->ports_out[port_out_id]; 1189 1190 /* Output port user actions */ 1191 if (port_out->f_action == NULL) /* Output port TX */ 1192 port_out->ops.f_tx(port_out->h_port, pkt); 1193 else { 1194 port_out->f_action(p, 1195 p->pkts, 1196 pkt_mask, 1197 port_out->arg_ah); 1198 1199 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1200 port_out->n_pkts_dropped_by_ah); 1201 1202 /* Output port TX */ 1203 if (pkt_mask & p->pkts_mask) 1204 port_out->ops.f_tx(port_out->h_port, 1205 pkt); 1206 } 1207 } 1208 } 1209 } 1210 1211 static inline void 1212 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p, 1213 uint64_t pkts_mask) 1214 { 1215 p->pkts_mask = pkts_mask; 1216 1217 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1218 uint64_t n_pkts = rte_popcount64(pkts_mask); 1219 uint32_t i; 1220 1221 for (i = 0; i < n_pkts; i++) { 1222 struct rte_mbuf *pkt = p->pkts[i]; 1223 uint32_t port_out_id = 1224 RTE_MBUF_METADATA_UINT32(pkt, 1225 p->offset_port_id); 1226 struct rte_port_out *port_out = &p->ports_out[ 1227 port_out_id]; 1228 1229 /* Output port user actions */ 1230 if (port_out->f_action == NULL) /* Output port TX */ 1231 port_out->ops.f_tx(port_out->h_port, pkt); 1232 else { 1233 uint64_t pkt_mask = 1LLU << i; 1234 1235 port_out->f_action(p, 1236 p->pkts, 1237 pkt_mask, 1238 port_out->arg_ah); 1239 1240 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1241 port_out->n_pkts_dropped_by_ah); 1242 1243 /* Output port TX */ 1244 if (pkt_mask & p->pkts_mask) 1245 port_out->ops.f_tx(port_out->h_port, 1246 pkt); 1247 } 1248 } 1249 } else { 1250 uint32_t i; 1251 1252 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1253 uint64_t pkt_mask = 1LLU << i; 1254 struct rte_mbuf *pkt; 1255 struct rte_port_out *port_out; 1256 uint32_t port_out_id; 1257 1258 if ((pkt_mask & pkts_mask) == 0) 1259 continue; 1260 1261 pkt = p->pkts[i]; 1262 port_out_id = RTE_MBUF_METADATA_UINT32(pkt, 1263 p->offset_port_id); 1264 port_out = &p->ports_out[port_out_id]; 1265 1266 /* Output port user actions */ 1267 if (port_out->f_action == NULL) /* Output port TX */ 1268 port_out->ops.f_tx(port_out->h_port, pkt); 1269 else { 1270 port_out->f_action(p, 1271 p->pkts, 1272 pkt_mask, 1273 port_out->arg_ah); 1274 1275 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1276 port_out->n_pkts_dropped_by_ah); 1277 1278 /* Output port TX */ 1279 if (pkt_mask & p->pkts_mask) 1280 port_out->ops.f_tx(port_out->h_port, 1281 pkt); 1282 } 1283 } 1284 } 1285 } 1286 1287 static inline void 1288 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask) 1289 { 1290 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1291 uint64_t n_pkts = rte_popcount64(pkts_mask); 1292 uint32_t i; 1293 1294 for (i = 0; i < n_pkts; i++) 1295 rte_pktmbuf_free(p->pkts[i]); 1296 } else { 1297 uint32_t i; 1298 1299 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1300 uint64_t pkt_mask = 1LLU << i; 1301 1302 if ((pkt_mask & pkts_mask) == 0) 1303 continue; 1304 1305 rte_pktmbuf_free(p->pkts[i]); 1306 } 1307 } 1308 } 1309 1310 int 1311 rte_pipeline_run(struct rte_pipeline *p) 1312 { 1313 struct rte_port_in *port_in = p->port_in_next; 1314 uint32_t n_pkts, table_id; 1315 1316 if (port_in == NULL) 1317 return 0; 1318 1319 /* Input port RX */ 1320 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts, 1321 port_in->burst_size); 1322 if (n_pkts == 0) { 1323 p->port_in_next = port_in->next; 1324 return 0; 1325 } 1326 1327 p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t); 1328 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0; 1329 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0; 1330 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0; 1331 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; 1332 1333 /* Input port user actions */ 1334 if (port_in->f_action != NULL) { 1335 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah); 1336 1337 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1338 port_in->n_pkts_dropped_by_ah); 1339 } 1340 1341 /* Table */ 1342 for (table_id = port_in->table_id; p->pkts_mask != 0; ) { 1343 struct rte_table *table; 1344 uint64_t lookup_hit_mask, lookup_miss_mask; 1345 1346 /* Lookup */ 1347 table = &p->tables[table_id]; 1348 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask, 1349 &lookup_hit_mask, (void **) p->entries); 1350 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask); 1351 1352 /* Lookup miss */ 1353 if (lookup_miss_mask != 0) { 1354 struct rte_pipeline_table_entry *default_entry = 1355 table->default_entry; 1356 1357 p->pkts_mask = lookup_miss_mask; 1358 1359 /* Table user actions */ 1360 if (table->f_action_miss != NULL) { 1361 table->f_action_miss(p, 1362 p->pkts, 1363 lookup_miss_mask, 1364 default_entry, 1365 table->arg_ah); 1366 1367 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1368 table->n_pkts_dropped_by_lkp_miss_ah); 1369 } 1370 1371 /* Table reserved actions */ 1372 if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) && 1373 (p->pkts_mask != 0)) 1374 rte_pipeline_action_handler_port_bulk(p, 1375 p->pkts_mask, 1376 default_entry->port_id); 1377 else { 1378 uint32_t pos = default_entry->action; 1379 1380 RTE_PIPELINE_STATS_TABLE_DROP0(p); 1381 1382 p->action_mask0[pos] |= p->pkts_mask; 1383 1384 RTE_PIPELINE_STATS_TABLE_DROP1(p, 1385 table->n_pkts_dropped_lkp_miss); 1386 } 1387 } 1388 1389 /* Lookup hit */ 1390 if (lookup_hit_mask != 0) { 1391 p->pkts_mask = lookup_hit_mask; 1392 1393 /* Table user actions */ 1394 if (table->f_action_hit != NULL) { 1395 table->f_action_hit(p, 1396 p->pkts, 1397 lookup_hit_mask, 1398 p->entries, 1399 table->arg_ah); 1400 1401 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1402 table->n_pkts_dropped_by_lkp_hit_ah); 1403 } 1404 1405 /* Table reserved actions */ 1406 RTE_PIPELINE_STATS_TABLE_DROP0(p); 1407 rte_pipeline_compute_masks(p, p->pkts_mask); 1408 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= 1409 p->action_mask1[ 1410 RTE_PIPELINE_ACTION_DROP]; 1411 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |= 1412 p->action_mask1[ 1413 RTE_PIPELINE_ACTION_PORT]; 1414 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |= 1415 p->action_mask1[ 1416 RTE_PIPELINE_ACTION_PORT_META]; 1417 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |= 1418 p->action_mask1[ 1419 RTE_PIPELINE_ACTION_TABLE]; 1420 1421 RTE_PIPELINE_STATS_TABLE_DROP1(p, 1422 table->n_pkts_dropped_lkp_hit); 1423 } 1424 1425 /* Prepare for next iteration */ 1426 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE]; 1427 table_id = table->table_next_id; 1428 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; 1429 } 1430 1431 /* Table reserved action PORT */ 1432 rte_pipeline_action_handler_port(p, 1433 p->action_mask0[RTE_PIPELINE_ACTION_PORT]); 1434 1435 /* Table reserved action PORT META */ 1436 rte_pipeline_action_handler_port_meta(p, 1437 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]); 1438 1439 /* Table reserved action DROP */ 1440 rte_pipeline_action_handler_drop(p, 1441 p->action_mask0[RTE_PIPELINE_ACTION_DROP]); 1442 1443 /* Pick candidate for next port IN to serve */ 1444 p->port_in_next = port_in->next; 1445 1446 return (int) n_pkts; 1447 } 1448 1449 int 1450 rte_pipeline_flush(struct rte_pipeline *p) 1451 { 1452 uint32_t port_id; 1453 1454 /* Check input arguments */ 1455 if (p == NULL) { 1456 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 1457 __func__); 1458 return -EINVAL; 1459 } 1460 1461 for (port_id = 0; port_id < p->num_ports_out; port_id++) { 1462 struct rte_port_out *port = &p->ports_out[port_id]; 1463 1464 if (port->ops.f_flush != NULL) 1465 port->ops.f_flush(port->h_port); 1466 } 1467 1468 return 0; 1469 } 1470 1471 int 1472 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p, 1473 uint32_t port_id, struct rte_mbuf *pkt) 1474 { 1475 struct rte_port_out *port_out = &p->ports_out[port_id]; 1476 1477 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */ 1478 1479 return 0; 1480 } 1481 1482 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p, 1483 uint64_t pkts_mask) 1484 { 1485 pkts_mask &= p->pkts_mask; 1486 p->pkts_mask &= ~pkts_mask; 1487 1488 return 0; 1489 } 1490 1491 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p, 1492 uint64_t pkts_mask) 1493 { 1494 pkts_mask &= p->pkts_mask; 1495 p->pkts_mask &= ~pkts_mask; 1496 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask; 1497 1498 RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask); 1499 return 0; 1500 } 1501 1502 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id, 1503 struct rte_pipeline_port_in_stats *stats, int clear) 1504 { 1505 struct rte_port_in *port; 1506 int retval; 1507 1508 if (p == NULL) { 1509 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 1510 __func__); 1511 return -EINVAL; 1512 } 1513 1514 if (port_id >= p->num_ports_in) { 1515 PIPELINE_LOG(ERR, 1516 "%s: port IN ID %u is out of range", 1517 __func__, port_id); 1518 return -EINVAL; 1519 } 1520 1521 port = &p->ports_in[port_id]; 1522 1523 if (port->ops.f_stats != NULL) { 1524 retval = port->ops.f_stats(port->h_port, &stats->stats, clear); 1525 if (retval) 1526 return retval; 1527 } else if (stats != NULL) 1528 memset(&stats->stats, 0, sizeof(stats->stats)); 1529 1530 if (stats != NULL) 1531 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah; 1532 1533 if (clear != 0) 1534 port->n_pkts_dropped_by_ah = 0; 1535 1536 return 0; 1537 } 1538 1539 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id, 1540 struct rte_pipeline_port_out_stats *stats, int clear) 1541 { 1542 struct rte_port_out *port; 1543 int retval; 1544 1545 if (p == NULL) { 1546 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", __func__); 1547 return -EINVAL; 1548 } 1549 1550 if (port_id >= p->num_ports_out) { 1551 PIPELINE_LOG(ERR, 1552 "%s: port OUT ID %u is out of range", __func__, port_id); 1553 return -EINVAL; 1554 } 1555 1556 port = &p->ports_out[port_id]; 1557 if (port->ops.f_stats != NULL) { 1558 retval = port->ops.f_stats(port->h_port, &stats->stats, clear); 1559 if (retval != 0) 1560 return retval; 1561 } else if (stats != NULL) 1562 memset(&stats->stats, 0, sizeof(stats->stats)); 1563 1564 if (stats != NULL) 1565 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah; 1566 1567 if (clear != 0) 1568 port->n_pkts_dropped_by_ah = 0; 1569 1570 return 0; 1571 } 1572 1573 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id, 1574 struct rte_pipeline_table_stats *stats, int clear) 1575 { 1576 struct rte_table *table; 1577 int retval; 1578 1579 if (p == NULL) { 1580 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 1581 __func__); 1582 return -EINVAL; 1583 } 1584 1585 if (table_id >= p->num_tables) { 1586 PIPELINE_LOG(ERR, 1587 "%s: table %u is out of range", __func__, table_id); 1588 return -EINVAL; 1589 } 1590 1591 table = &p->tables[table_id]; 1592 if (table->ops.f_stats != NULL) { 1593 retval = table->ops.f_stats(table->h_table, &stats->stats, clear); 1594 if (retval != 0) 1595 return retval; 1596 } else if (stats != NULL) 1597 memset(&stats->stats, 0, sizeof(stats->stats)); 1598 1599 if (stats != NULL) { 1600 stats->n_pkts_dropped_by_lkp_hit_ah = 1601 table->n_pkts_dropped_by_lkp_hit_ah; 1602 stats->n_pkts_dropped_by_lkp_miss_ah = 1603 table->n_pkts_dropped_by_lkp_miss_ah; 1604 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit; 1605 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss; 1606 } 1607 1608 if (clear != 0) { 1609 table->n_pkts_dropped_by_lkp_hit_ah = 0; 1610 table->n_pkts_dropped_by_lkp_miss_ah = 0; 1611 table->n_pkts_dropped_lkp_hit = 0; 1612 table->n_pkts_dropped_lkp_miss = 0; 1613 } 1614 1615 return 0; 1616 } 1617