1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <string.h> 6 #include <stdio.h> 7 8 #include <rte_common.h> 9 #include <rte_mbuf.h> 10 #include <rte_malloc.h> 11 #include <rte_string_fns.h> 12 13 #include "rte_pipeline.h" 14 15 #define PIPELINE_LOG(level, ...) \ 16 RTE_LOG_LINE(level, PIPELINE, "" __VA_ARGS__) 17 18 #define RTE_TABLE_INVALID UINT32_MAX 19 20 #ifdef RTE_PIPELINE_STATS_COLLECT 21 22 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) \ 23 ({ (p)->n_pkts_ah_drop = rte_popcount64(mask); }) 24 25 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) \ 26 ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; }) 27 28 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) \ 29 ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; }) 30 31 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) \ 32 ({ \ 33 uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; \ 34 mask ^= (p)->pkts_drop_mask; \ 35 (counter) += rte_popcount64(mask); \ 36 }) 37 38 #else 39 40 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) 41 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) 42 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) 43 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) 44 45 #endif 46 47 struct rte_port_in { 48 /* Input parameters */ 49 struct rte_port_in_ops ops; 50 rte_pipeline_port_in_action_handler f_action; 51 void *arg_ah; 52 uint32_t burst_size; 53 54 /* The table to which this port is connected */ 55 uint32_t table_id; 56 57 /* Handle to low-level port */ 58 void *h_port; 59 60 /* List of enabled ports */ 61 struct rte_port_in *next; 62 63 /* Statistics */ 64 uint64_t n_pkts_dropped_by_ah; 65 }; 66 67 struct rte_port_out { 68 /* Input parameters */ 69 struct rte_port_out_ops ops; 70 rte_pipeline_port_out_action_handler f_action; 71 void *arg_ah; 72 73 /* Handle to low-level port */ 74 void *h_port; 75 76 /* Statistics */ 77 uint64_t n_pkts_dropped_by_ah; 78 }; 79 80 struct rte_table { 81 /* Input parameters */ 82 struct rte_table_ops ops; 83 rte_pipeline_table_action_handler_hit f_action_hit; 84 rte_pipeline_table_action_handler_miss f_action_miss; 85 void *arg_ah; 86 struct rte_pipeline_table_entry *default_entry; 87 uint32_t entry_size; 88 89 uint32_t table_next_id; 90 uint32_t table_next_id_valid; 91 92 /* Handle to the low-level table object */ 93 void *h_table; 94 95 /* Statistics */ 96 uint64_t n_pkts_dropped_by_lkp_hit_ah; 97 uint64_t n_pkts_dropped_by_lkp_miss_ah; 98 uint64_t n_pkts_dropped_lkp_hit; 99 uint64_t n_pkts_dropped_lkp_miss; 100 }; 101 102 #define RTE_PIPELINE_MAX_NAME_SZ 124 103 104 struct rte_pipeline { 105 /* Input parameters */ 106 char name[RTE_PIPELINE_MAX_NAME_SZ]; 107 int socket_id; 108 uint32_t offset_port_id; 109 110 /* Internal tables */ 111 struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX]; 112 struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX]; 113 struct rte_table tables[RTE_PIPELINE_TABLE_MAX]; 114 115 /* Occupancy of internal tables */ 116 uint32_t num_ports_in; 117 uint32_t num_ports_out; 118 uint32_t num_tables; 119 120 /* List of enabled ports */ 121 uint64_t enabled_port_in_mask; 122 struct rte_port_in *port_in_next; 123 124 /* Pipeline run structures */ 125 struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX]; 126 struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX]; 127 uint64_t action_mask0[RTE_PIPELINE_ACTIONS]; 128 uint64_t action_mask1[RTE_PIPELINE_ACTIONS]; 129 uint64_t pkts_mask; 130 uint64_t n_pkts_ah_drop; 131 uint64_t pkts_drop_mask; 132 } __rte_cache_aligned; 133 134 static inline uint32_t 135 rte_mask_get_next(uint64_t mask, uint32_t pos) 136 { 137 uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) | 138 (mask >> ((pos + 1) & 0x3F)); 139 return (rte_ctz64(mask_rot) - (63 - pos)) & 0x3F; 140 } 141 142 static inline uint32_t 143 rte_mask_get_prev(uint64_t mask, uint32_t pos) 144 { 145 uint64_t mask_rot = (mask >> (pos & 0x3F)) | 146 (mask << ((64 - pos) & 0x3F)); 147 return ((63 - rte_clz64(mask_rot)) + pos) & 0x3F; 148 } 149 150 static void 151 rte_pipeline_table_free(struct rte_table *table); 152 153 static void 154 rte_pipeline_port_in_free(struct rte_port_in *port); 155 156 static void 157 rte_pipeline_port_out_free(struct rte_port_out *port); 158 159 /* 160 * Pipeline 161 */ 162 static int 163 rte_pipeline_check_params(struct rte_pipeline_params *params) 164 { 165 if (params == NULL) { 166 PIPELINE_LOG(ERR, 167 "%s: Incorrect value for parameter params", __func__); 168 return -EINVAL; 169 } 170 171 /* name */ 172 if (params->name == NULL) { 173 PIPELINE_LOG(ERR, 174 "%s: Incorrect value for parameter name", __func__); 175 return -EINVAL; 176 } 177 178 /* socket */ 179 if (params->socket_id < 0) { 180 PIPELINE_LOG(ERR, 181 "%s: Incorrect value for parameter socket_id", 182 __func__); 183 return -EINVAL; 184 } 185 186 return 0; 187 } 188 189 struct rte_pipeline * 190 rte_pipeline_create(struct rte_pipeline_params *params) 191 { 192 struct rte_pipeline *p; 193 int status; 194 195 /* Check input parameters */ 196 status = rte_pipeline_check_params(params); 197 if (status != 0) { 198 PIPELINE_LOG(ERR, 199 "%s: Pipeline params check failed (%d)", 200 __func__, status); 201 return NULL; 202 } 203 204 /* Allocate memory for the pipeline on requested socket */ 205 p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline), 206 RTE_CACHE_LINE_SIZE, params->socket_id); 207 208 if (p == NULL) { 209 PIPELINE_LOG(ERR, 210 "%s: Pipeline memory allocation failed", __func__); 211 return NULL; 212 } 213 214 /* Save input parameters */ 215 strlcpy(p->name, params->name, RTE_PIPELINE_MAX_NAME_SZ); 216 p->socket_id = params->socket_id; 217 p->offset_port_id = params->offset_port_id; 218 219 /* Initialize pipeline internal data structure */ 220 p->num_ports_in = 0; 221 p->num_ports_out = 0; 222 p->num_tables = 0; 223 p->enabled_port_in_mask = 0; 224 p->port_in_next = NULL; 225 p->pkts_mask = 0; 226 p->n_pkts_ah_drop = 0; 227 228 return p; 229 } 230 231 int 232 rte_pipeline_free(struct rte_pipeline *p) 233 { 234 uint32_t i; 235 236 /* Check input parameters */ 237 if (p == NULL) { 238 PIPELINE_LOG(ERR, 239 "%s: rte_pipeline parameter is NULL", __func__); 240 return -EINVAL; 241 } 242 243 /* Free input ports */ 244 for (i = 0; i < p->num_ports_in; i++) { 245 struct rte_port_in *port = &p->ports_in[i]; 246 247 rte_pipeline_port_in_free(port); 248 } 249 250 /* Free tables */ 251 for (i = 0; i < p->num_tables; i++) { 252 struct rte_table *table = &p->tables[i]; 253 254 rte_pipeline_table_free(table); 255 } 256 257 /* Free output ports */ 258 for (i = 0; i < p->num_ports_out; i++) { 259 struct rte_port_out *port = &p->ports_out[i]; 260 261 rte_pipeline_port_out_free(port); 262 } 263 264 /* Free pipeline memory */ 265 rte_free(p); 266 267 return 0; 268 } 269 270 /* 271 * Table 272 */ 273 static int 274 rte_table_check_params(struct rte_pipeline *p, 275 struct rte_pipeline_table_params *params, 276 uint32_t *table_id) 277 { 278 if (p == NULL) { 279 PIPELINE_LOG(ERR, "%s: pipeline parameter is NULL", 280 __func__); 281 return -EINVAL; 282 } 283 if (params == NULL) { 284 PIPELINE_LOG(ERR, "%s: params parameter is NULL", 285 __func__); 286 return -EINVAL; 287 } 288 if (table_id == NULL) { 289 PIPELINE_LOG(ERR, "%s: table_id parameter is NULL", 290 __func__); 291 return -EINVAL; 292 } 293 294 /* ops */ 295 if (params->ops == NULL) { 296 PIPELINE_LOG(ERR, "%s: params->ops is NULL", 297 __func__); 298 return -EINVAL; 299 } 300 301 if (params->ops->f_create == NULL) { 302 PIPELINE_LOG(ERR, 303 "%s: f_create function pointer is NULL", __func__); 304 return -EINVAL; 305 } 306 307 if (params->ops->f_lookup == NULL) { 308 PIPELINE_LOG(ERR, 309 "%s: f_lookup function pointer is NULL", __func__); 310 return -EINVAL; 311 } 312 313 /* De we have room for one more table? */ 314 if (p->num_tables == RTE_PIPELINE_TABLE_MAX) { 315 PIPELINE_LOG(ERR, 316 "%s: Incorrect value for num_tables parameter", 317 __func__); 318 return -EINVAL; 319 } 320 321 return 0; 322 } 323 324 int 325 rte_pipeline_table_create(struct rte_pipeline *p, 326 struct rte_pipeline_table_params *params, 327 uint32_t *table_id) 328 { 329 struct rte_table *table; 330 struct rte_pipeline_table_entry *default_entry; 331 void *h_table; 332 uint32_t entry_size, id; 333 int status; 334 335 /* Check input arguments */ 336 status = rte_table_check_params(p, params, table_id); 337 if (status != 0) 338 return status; 339 340 id = p->num_tables; 341 table = &p->tables[id]; 342 343 /* Allocate space for the default table entry */ 344 entry_size = sizeof(struct rte_pipeline_table_entry) + 345 params->action_data_size; 346 default_entry = rte_zmalloc_socket( 347 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id); 348 if (default_entry == NULL) { 349 PIPELINE_LOG(ERR, 350 "%s: Failed to allocate default entry", __func__); 351 return -EINVAL; 352 } 353 354 /* Create the table */ 355 h_table = params->ops->f_create(params->arg_create, p->socket_id, 356 entry_size); 357 if (h_table == NULL) { 358 rte_free(default_entry); 359 PIPELINE_LOG(ERR, "%s: Table creation failed", __func__); 360 return -EINVAL; 361 } 362 363 /* Commit current table to the pipeline */ 364 p->num_tables++; 365 *table_id = id; 366 367 /* Save input parameters */ 368 memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops)); 369 table->f_action_hit = params->f_action_hit; 370 table->f_action_miss = params->f_action_miss; 371 table->arg_ah = params->arg_ah; 372 table->entry_size = entry_size; 373 374 /* Clear the lookup miss actions (to be set later through API) */ 375 table->default_entry = default_entry; 376 table->default_entry->action = RTE_PIPELINE_ACTION_DROP; 377 378 /* Initialize table internal data structure */ 379 table->h_table = h_table; 380 table->table_next_id = 0; 381 table->table_next_id_valid = 0; 382 383 return 0; 384 } 385 386 void 387 rte_pipeline_table_free(struct rte_table *table) 388 { 389 if (table->ops.f_free != NULL) 390 table->ops.f_free(table->h_table); 391 392 rte_free(table->default_entry); 393 } 394 395 int 396 rte_pipeline_table_default_entry_add(struct rte_pipeline *p, 397 uint32_t table_id, 398 struct rte_pipeline_table_entry *default_entry, 399 struct rte_pipeline_table_entry **default_entry_ptr) 400 { 401 struct rte_table *table; 402 403 /* Check input arguments */ 404 if (p == NULL) { 405 PIPELINE_LOG(ERR, "%s: pipeline parameter is NULL", 406 __func__); 407 return -EINVAL; 408 } 409 410 if (default_entry == NULL) { 411 PIPELINE_LOG(ERR, 412 "%s: default_entry parameter is NULL", __func__); 413 return -EINVAL; 414 } 415 416 if (table_id >= p->num_tables) { 417 PIPELINE_LOG(ERR, 418 "%s: table_id %d out of range", __func__, table_id); 419 return -EINVAL; 420 } 421 422 table = &p->tables[table_id]; 423 424 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) && 425 table->table_next_id_valid && 426 (default_entry->table_id != table->table_next_id)) { 427 PIPELINE_LOG(ERR, 428 "%s: Tree-like topologies not allowed", __func__); 429 return -EINVAL; 430 } 431 432 /* Set the lookup miss actions */ 433 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) && 434 (table->table_next_id_valid == 0)) { 435 table->table_next_id = default_entry->table_id; 436 table->table_next_id_valid = 1; 437 } 438 439 memcpy(table->default_entry, default_entry, table->entry_size); 440 441 *default_entry_ptr = table->default_entry; 442 return 0; 443 } 444 445 int 446 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p, 447 uint32_t table_id, 448 struct rte_pipeline_table_entry *entry) 449 { 450 struct rte_table *table; 451 452 /* Check input arguments */ 453 if (p == NULL) { 454 PIPELINE_LOG(ERR, 455 "%s: pipeline parameter is NULL", __func__); 456 return -EINVAL; 457 } 458 459 if (table_id >= p->num_tables) { 460 PIPELINE_LOG(ERR, 461 "%s: table_id %d out of range", __func__, table_id); 462 return -EINVAL; 463 } 464 465 table = &p->tables[table_id]; 466 467 /* Save the current contents of the default entry */ 468 if (entry) 469 memcpy(entry, table->default_entry, table->entry_size); 470 471 /* Clear the lookup miss actions */ 472 memset(table->default_entry, 0, table->entry_size); 473 table->default_entry->action = RTE_PIPELINE_ACTION_DROP; 474 475 return 0; 476 } 477 478 int 479 rte_pipeline_table_entry_add(struct rte_pipeline *p, 480 uint32_t table_id, 481 void *key, 482 struct rte_pipeline_table_entry *entry, 483 int *key_found, 484 struct rte_pipeline_table_entry **entry_ptr) 485 { 486 struct rte_table *table; 487 488 /* Check input arguments */ 489 if (p == NULL) { 490 PIPELINE_LOG(ERR, "%s: pipeline parameter is NULL", 491 __func__); 492 return -EINVAL; 493 } 494 495 if (key == NULL) { 496 PIPELINE_LOG(ERR, "%s: key parameter is NULL", __func__); 497 return -EINVAL; 498 } 499 500 if (entry == NULL) { 501 PIPELINE_LOG(ERR, "%s: entry parameter is NULL", 502 __func__); 503 return -EINVAL; 504 } 505 506 if (table_id >= p->num_tables) { 507 PIPELINE_LOG(ERR, 508 "%s: table_id %d out of range", __func__, table_id); 509 return -EINVAL; 510 } 511 512 table = &p->tables[table_id]; 513 514 if (table->ops.f_add == NULL) { 515 PIPELINE_LOG(ERR, "%s: f_add function pointer NULL", 516 __func__); 517 return -EINVAL; 518 } 519 520 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) && 521 table->table_next_id_valid && 522 (entry->table_id != table->table_next_id)) { 523 PIPELINE_LOG(ERR, 524 "%s: Tree-like topologies not allowed", __func__); 525 return -EINVAL; 526 } 527 528 /* Add entry */ 529 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) && 530 (table->table_next_id_valid == 0)) { 531 table->table_next_id = entry->table_id; 532 table->table_next_id_valid = 1; 533 } 534 535 return (table->ops.f_add)(table->h_table, key, (void *) entry, 536 key_found, (void **) entry_ptr); 537 } 538 539 int 540 rte_pipeline_table_entry_delete(struct rte_pipeline *p, 541 uint32_t table_id, 542 void *key, 543 int *key_found, 544 struct rte_pipeline_table_entry *entry) 545 { 546 struct rte_table *table; 547 548 /* Check input arguments */ 549 if (p == NULL) { 550 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 551 __func__); 552 return -EINVAL; 553 } 554 555 if (key == NULL) { 556 PIPELINE_LOG(ERR, "%s: key parameter is NULL", 557 __func__); 558 return -EINVAL; 559 } 560 561 if (table_id >= p->num_tables) { 562 PIPELINE_LOG(ERR, 563 "%s: table_id %d out of range", __func__, table_id); 564 return -EINVAL; 565 } 566 567 table = &p->tables[table_id]; 568 569 if (table->ops.f_delete == NULL) { 570 PIPELINE_LOG(ERR, 571 "%s: f_delete function pointer NULL", __func__); 572 return -EINVAL; 573 } 574 575 return (table->ops.f_delete)(table->h_table, key, key_found, entry); 576 } 577 578 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p, 579 uint32_t table_id, 580 void **keys, 581 struct rte_pipeline_table_entry **entries, 582 uint32_t n_keys, 583 int *key_found, 584 struct rte_pipeline_table_entry **entries_ptr) 585 { 586 struct rte_table *table; 587 uint32_t i; 588 589 /* Check input arguments */ 590 if (p == NULL) { 591 PIPELINE_LOG(ERR, "%s: pipeline parameter is NULL", 592 __func__); 593 return -EINVAL; 594 } 595 596 if (keys == NULL) { 597 PIPELINE_LOG(ERR, "%s: keys parameter is NULL", __func__); 598 return -EINVAL; 599 } 600 601 if (entries == NULL) { 602 PIPELINE_LOG(ERR, "%s: entries parameter is NULL", 603 __func__); 604 return -EINVAL; 605 } 606 607 if (table_id >= p->num_tables) { 608 PIPELINE_LOG(ERR, 609 "%s: table_id %d out of range", __func__, table_id); 610 return -EINVAL; 611 } 612 613 table = &p->tables[table_id]; 614 615 if (table->ops.f_add_bulk == NULL) { 616 PIPELINE_LOG(ERR, "%s: f_add_bulk function pointer NULL", 617 __func__); 618 return -EINVAL; 619 } 620 621 for (i = 0; i < n_keys; i++) { 622 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) && 623 table->table_next_id_valid && 624 (entries[i]->table_id != table->table_next_id)) { 625 PIPELINE_LOG(ERR, 626 "%s: Tree-like topologies not allowed", __func__); 627 return -EINVAL; 628 } 629 } 630 631 /* Add entry */ 632 for (i = 0; i < n_keys; i++) { 633 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) && 634 (table->table_next_id_valid == 0)) { 635 table->table_next_id = entries[i]->table_id; 636 table->table_next_id_valid = 1; 637 } 638 } 639 640 return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries, 641 n_keys, key_found, (void **) entries_ptr); 642 } 643 644 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p, 645 uint32_t table_id, 646 void **keys, 647 uint32_t n_keys, 648 int *key_found, 649 struct rte_pipeline_table_entry **entries) 650 { 651 struct rte_table *table; 652 653 /* Check input arguments */ 654 if (p == NULL) { 655 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 656 __func__); 657 return -EINVAL; 658 } 659 660 if (keys == NULL) { 661 PIPELINE_LOG(ERR, "%s: key parameter is NULL", 662 __func__); 663 return -EINVAL; 664 } 665 666 if (table_id >= p->num_tables) { 667 PIPELINE_LOG(ERR, 668 "%s: table_id %d out of range", __func__, table_id); 669 return -EINVAL; 670 } 671 672 table = &p->tables[table_id]; 673 674 if (table->ops.f_delete_bulk == NULL) { 675 PIPELINE_LOG(ERR, 676 "%s: f_delete function pointer NULL", __func__); 677 return -EINVAL; 678 } 679 680 return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found, 681 (void **) entries); 682 } 683 684 /* 685 * Port 686 */ 687 static int 688 rte_pipeline_port_in_check_params(struct rte_pipeline *p, 689 struct rte_pipeline_port_in_params *params, 690 uint32_t *port_id) 691 { 692 if (p == NULL) { 693 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 694 __func__); 695 return -EINVAL; 696 } 697 if (params == NULL) { 698 PIPELINE_LOG(ERR, "%s: params parameter NULL", __func__); 699 return -EINVAL; 700 } 701 if (port_id == NULL) { 702 PIPELINE_LOG(ERR, "%s: port_id parameter NULL", 703 __func__); 704 return -EINVAL; 705 } 706 707 /* ops */ 708 if (params->ops == NULL) { 709 PIPELINE_LOG(ERR, "%s: params->ops parameter NULL", 710 __func__); 711 return -EINVAL; 712 } 713 714 if (params->ops->f_create == NULL) { 715 PIPELINE_LOG(ERR, 716 "%s: f_create function pointer NULL", __func__); 717 return -EINVAL; 718 } 719 720 if (params->ops->f_rx == NULL) { 721 PIPELINE_LOG(ERR, "%s: f_rx function pointer NULL", 722 __func__); 723 return -EINVAL; 724 } 725 726 /* burst_size */ 727 if ((params->burst_size == 0) || 728 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) { 729 PIPELINE_LOG(ERR, "%s: invalid value for burst_size", 730 __func__); 731 return -EINVAL; 732 } 733 734 /* Do we have room for one more port? */ 735 if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) { 736 PIPELINE_LOG(ERR, 737 "%s: invalid value for num_ports_in", __func__); 738 return -EINVAL; 739 } 740 741 return 0; 742 } 743 744 static int 745 rte_pipeline_port_out_check_params(struct rte_pipeline *p, 746 struct rte_pipeline_port_out_params *params, 747 uint32_t *port_id) 748 { 749 if (p == NULL) { 750 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 751 __func__); 752 return -EINVAL; 753 } 754 755 if (params == NULL) { 756 PIPELINE_LOG(ERR, "%s: params parameter NULL", __func__); 757 return -EINVAL; 758 } 759 760 if (port_id == NULL) { 761 PIPELINE_LOG(ERR, "%s: port_id parameter NULL", 762 __func__); 763 return -EINVAL; 764 } 765 766 /* ops */ 767 if (params->ops == NULL) { 768 PIPELINE_LOG(ERR, "%s: params->ops parameter NULL", 769 __func__); 770 return -EINVAL; 771 } 772 773 if (params->ops->f_create == NULL) { 774 PIPELINE_LOG(ERR, 775 "%s: f_create function pointer NULL", __func__); 776 return -EINVAL; 777 } 778 779 if (params->ops->f_tx == NULL) { 780 PIPELINE_LOG(ERR, 781 "%s: f_tx function pointer NULL", __func__); 782 return -EINVAL; 783 } 784 785 if (params->ops->f_tx_bulk == NULL) { 786 PIPELINE_LOG(ERR, 787 "%s: f_tx_bulk function pointer NULL", __func__); 788 return -EINVAL; 789 } 790 791 /* Do we have room for one more port? */ 792 if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) { 793 PIPELINE_LOG(ERR, 794 "%s: invalid value for num_ports_out", __func__); 795 return -EINVAL; 796 } 797 798 return 0; 799 } 800 801 int 802 rte_pipeline_port_in_create(struct rte_pipeline *p, 803 struct rte_pipeline_port_in_params *params, 804 uint32_t *port_id) 805 { 806 struct rte_port_in *port; 807 void *h_port; 808 uint32_t id; 809 int status; 810 811 /* Check input arguments */ 812 status = rte_pipeline_port_in_check_params(p, params, port_id); 813 if (status != 0) 814 return status; 815 816 id = p->num_ports_in; 817 port = &p->ports_in[id]; 818 819 /* Create the port */ 820 h_port = params->ops->f_create(params->arg_create, p->socket_id); 821 if (h_port == NULL) { 822 PIPELINE_LOG(ERR, "%s: Port creation failed", __func__); 823 return -EINVAL; 824 } 825 826 /* Commit current table to the pipeline */ 827 p->num_ports_in++; 828 *port_id = id; 829 830 /* Save input parameters */ 831 memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops)); 832 port->f_action = params->f_action; 833 port->arg_ah = params->arg_ah; 834 port->burst_size = params->burst_size; 835 836 /* Initialize port internal data structure */ 837 port->table_id = RTE_TABLE_INVALID; 838 port->h_port = h_port; 839 port->next = NULL; 840 841 return 0; 842 } 843 844 void 845 rte_pipeline_port_in_free(struct rte_port_in *port) 846 { 847 if (port->ops.f_free != NULL) 848 port->ops.f_free(port->h_port); 849 } 850 851 int 852 rte_pipeline_port_out_create(struct rte_pipeline *p, 853 struct rte_pipeline_port_out_params *params, 854 uint32_t *port_id) 855 { 856 struct rte_port_out *port; 857 void *h_port; 858 uint32_t id; 859 int status; 860 861 /* Check input arguments */ 862 status = rte_pipeline_port_out_check_params(p, params, port_id); 863 if (status != 0) 864 return status; 865 866 id = p->num_ports_out; 867 port = &p->ports_out[id]; 868 869 /* Create the port */ 870 h_port = params->ops->f_create(params->arg_create, p->socket_id); 871 if (h_port == NULL) { 872 PIPELINE_LOG(ERR, "%s: Port creation failed", __func__); 873 return -EINVAL; 874 } 875 876 /* Commit current table to the pipeline */ 877 p->num_ports_out++; 878 *port_id = id; 879 880 /* Save input parameters */ 881 memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops)); 882 port->f_action = params->f_action; 883 port->arg_ah = params->arg_ah; 884 885 /* Initialize port internal data structure */ 886 port->h_port = h_port; 887 888 return 0; 889 } 890 891 void 892 rte_pipeline_port_out_free(struct rte_port_out *port) 893 { 894 if (port->ops.f_free != NULL) 895 port->ops.f_free(port->h_port); 896 } 897 898 int 899 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p, 900 uint32_t port_id, 901 uint32_t table_id) 902 { 903 struct rte_port_in *port; 904 905 /* Check input arguments */ 906 if (p == NULL) { 907 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 908 __func__); 909 return -EINVAL; 910 } 911 912 if (port_id >= p->num_ports_in) { 913 PIPELINE_LOG(ERR, 914 "%s: port IN ID %u is out of range", 915 __func__, port_id); 916 return -EINVAL; 917 } 918 919 if (table_id >= p->num_tables) { 920 PIPELINE_LOG(ERR, 921 "%s: Table ID %u is out of range", 922 __func__, table_id); 923 return -EINVAL; 924 } 925 926 port = &p->ports_in[port_id]; 927 port->table_id = table_id; 928 929 return 0; 930 } 931 932 int 933 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id) 934 { 935 struct rte_port_in *port, *port_prev, *port_next; 936 uint64_t port_mask; 937 uint32_t port_prev_id, port_next_id; 938 939 /* Check input arguments */ 940 if (p == NULL) { 941 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 942 __func__); 943 return -EINVAL; 944 } 945 946 if (port_id >= p->num_ports_in) { 947 PIPELINE_LOG(ERR, 948 "%s: port IN ID %u is out of range", 949 __func__, port_id); 950 return -EINVAL; 951 } 952 953 port = &p->ports_in[port_id]; 954 955 /* Return if current input port is already enabled */ 956 port_mask = 1LLU << port_id; 957 if (p->enabled_port_in_mask & port_mask) 958 return 0; 959 960 p->enabled_port_in_mask |= port_mask; 961 962 /* Add current input port to the pipeline chain of enabled ports */ 963 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id); 964 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id); 965 966 port_prev = &p->ports_in[port_prev_id]; 967 port_next = &p->ports_in[port_next_id]; 968 969 port_prev->next = port; 970 port->next = port_next; 971 972 /* Check if list of enabled ports was previously empty */ 973 if (p->enabled_port_in_mask == port_mask) 974 p->port_in_next = port; 975 976 return 0; 977 } 978 979 int 980 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id) 981 { 982 struct rte_port_in *port, *port_prev, *port_next; 983 uint64_t port_mask; 984 uint32_t port_prev_id, port_next_id; 985 986 /* Check input arguments */ 987 if (p == NULL) { 988 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 989 __func__); 990 return -EINVAL; 991 } 992 993 if (port_id >= p->num_ports_in) { 994 PIPELINE_LOG(ERR, "%s: port IN ID %u is out of range", 995 __func__, port_id); 996 return -EINVAL; 997 } 998 999 port = &p->ports_in[port_id]; 1000 1001 /* Return if current input port is already disabled */ 1002 port_mask = 1LLU << port_id; 1003 if ((p->enabled_port_in_mask & port_mask) == 0) 1004 return 0; 1005 1006 p->enabled_port_in_mask &= ~port_mask; 1007 1008 /* Return if no other enabled ports */ 1009 if (p->enabled_port_in_mask == 0) { 1010 p->port_in_next = NULL; 1011 1012 return 0; 1013 } 1014 1015 /* Add current input port to the pipeline chain of enabled ports */ 1016 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id); 1017 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id); 1018 1019 port_prev = &p->ports_in[port_prev_id]; 1020 port_next = &p->ports_in[port_next_id]; 1021 1022 port_prev->next = port_next; 1023 1024 /* Check if the port which has just been disabled is next to serve */ 1025 if (port == p->port_in_next) 1026 p->port_in_next = port_next; 1027 1028 return 0; 1029 } 1030 1031 /* 1032 * Pipeline run-time 1033 */ 1034 int 1035 rte_pipeline_check(struct rte_pipeline *p) 1036 { 1037 uint32_t port_in_id; 1038 1039 /* Check input arguments */ 1040 if (p == NULL) { 1041 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 1042 __func__); 1043 return -EINVAL; 1044 } 1045 1046 /* Check that pipeline has at least one input port, one table and one 1047 output port */ 1048 if (p->num_ports_in == 0) { 1049 PIPELINE_LOG(ERR, "%s: must have at least 1 input port", 1050 __func__); 1051 return -EINVAL; 1052 } 1053 if (p->num_tables == 0) { 1054 PIPELINE_LOG(ERR, "%s: must have at least 1 table", 1055 __func__); 1056 return -EINVAL; 1057 } 1058 if (p->num_ports_out == 0) { 1059 PIPELINE_LOG(ERR, "%s: must have at least 1 output port", 1060 __func__); 1061 return -EINVAL; 1062 } 1063 1064 /* Check that all input ports are connected */ 1065 for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) { 1066 struct rte_port_in *port_in = &p->ports_in[port_in_id]; 1067 1068 if (port_in->table_id == RTE_TABLE_INVALID) { 1069 PIPELINE_LOG(ERR, 1070 "%s: Port IN ID %u is not connected", 1071 __func__, port_in_id); 1072 return -EINVAL; 1073 } 1074 } 1075 1076 return 0; 1077 } 1078 1079 static inline void 1080 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask) 1081 { 1082 p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0; 1083 p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0; 1084 p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0; 1085 p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0; 1086 1087 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1088 uint64_t n_pkts = rte_popcount64(pkts_mask); 1089 uint32_t i; 1090 1091 for (i = 0; i < n_pkts; i++) { 1092 uint64_t pkt_mask = 1LLU << i; 1093 uint32_t pos = p->entries[i]->action; 1094 1095 p->action_mask1[pos] |= pkt_mask; 1096 } 1097 } else { 1098 uint32_t i; 1099 1100 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1101 uint64_t pkt_mask = 1LLU << i; 1102 uint32_t pos; 1103 1104 if ((pkt_mask & pkts_mask) == 0) 1105 continue; 1106 1107 pos = p->entries[i]->action; 1108 p->action_mask1[pos] |= pkt_mask; 1109 } 1110 } 1111 } 1112 1113 static inline void 1114 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p, 1115 uint64_t pkts_mask, uint32_t port_id) 1116 { 1117 struct rte_port_out *port_out = &p->ports_out[port_id]; 1118 1119 p->pkts_mask = pkts_mask; 1120 1121 /* Output port user actions */ 1122 if (port_out->f_action != NULL) { 1123 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah); 1124 1125 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1126 port_out->n_pkts_dropped_by_ah); 1127 } 1128 1129 /* Output port TX */ 1130 if (p->pkts_mask != 0) 1131 port_out->ops.f_tx_bulk(port_out->h_port, 1132 p->pkts, 1133 p->pkts_mask); 1134 } 1135 1136 static inline void 1137 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask) 1138 { 1139 p->pkts_mask = pkts_mask; 1140 1141 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1142 uint64_t n_pkts = rte_popcount64(pkts_mask); 1143 uint32_t i; 1144 1145 for (i = 0; i < n_pkts; i++) { 1146 struct rte_mbuf *pkt = p->pkts[i]; 1147 uint32_t port_out_id = p->entries[i]->port_id; 1148 struct rte_port_out *port_out = 1149 &p->ports_out[port_out_id]; 1150 1151 /* Output port user actions */ 1152 if (port_out->f_action == NULL) /* Output port TX */ 1153 port_out->ops.f_tx(port_out->h_port, pkt); 1154 else { 1155 uint64_t pkt_mask = 1LLU << i; 1156 1157 port_out->f_action(p, 1158 p->pkts, 1159 pkt_mask, 1160 port_out->arg_ah); 1161 1162 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1163 port_out->n_pkts_dropped_by_ah); 1164 1165 /* Output port TX */ 1166 if (pkt_mask & p->pkts_mask) 1167 port_out->ops.f_tx(port_out->h_port, 1168 pkt); 1169 } 1170 } 1171 } else { 1172 uint32_t i; 1173 1174 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1175 uint64_t pkt_mask = 1LLU << i; 1176 struct rte_mbuf *pkt; 1177 struct rte_port_out *port_out; 1178 uint32_t port_out_id; 1179 1180 if ((pkt_mask & pkts_mask) == 0) 1181 continue; 1182 1183 pkt = p->pkts[i]; 1184 port_out_id = p->entries[i]->port_id; 1185 port_out = &p->ports_out[port_out_id]; 1186 1187 /* Output port user actions */ 1188 if (port_out->f_action == NULL) /* Output port TX */ 1189 port_out->ops.f_tx(port_out->h_port, pkt); 1190 else { 1191 port_out->f_action(p, 1192 p->pkts, 1193 pkt_mask, 1194 port_out->arg_ah); 1195 1196 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1197 port_out->n_pkts_dropped_by_ah); 1198 1199 /* Output port TX */ 1200 if (pkt_mask & p->pkts_mask) 1201 port_out->ops.f_tx(port_out->h_port, 1202 pkt); 1203 } 1204 } 1205 } 1206 } 1207 1208 static inline void 1209 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p, 1210 uint64_t pkts_mask) 1211 { 1212 p->pkts_mask = pkts_mask; 1213 1214 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1215 uint64_t n_pkts = rte_popcount64(pkts_mask); 1216 uint32_t i; 1217 1218 for (i = 0; i < n_pkts; i++) { 1219 struct rte_mbuf *pkt = p->pkts[i]; 1220 uint32_t port_out_id = 1221 RTE_MBUF_METADATA_UINT32(pkt, 1222 p->offset_port_id); 1223 struct rte_port_out *port_out = &p->ports_out[ 1224 port_out_id]; 1225 1226 /* Output port user actions */ 1227 if (port_out->f_action == NULL) /* Output port TX */ 1228 port_out->ops.f_tx(port_out->h_port, pkt); 1229 else { 1230 uint64_t pkt_mask = 1LLU << i; 1231 1232 port_out->f_action(p, 1233 p->pkts, 1234 pkt_mask, 1235 port_out->arg_ah); 1236 1237 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1238 port_out->n_pkts_dropped_by_ah); 1239 1240 /* Output port TX */ 1241 if (pkt_mask & p->pkts_mask) 1242 port_out->ops.f_tx(port_out->h_port, 1243 pkt); 1244 } 1245 } 1246 } else { 1247 uint32_t i; 1248 1249 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1250 uint64_t pkt_mask = 1LLU << i; 1251 struct rte_mbuf *pkt; 1252 struct rte_port_out *port_out; 1253 uint32_t port_out_id; 1254 1255 if ((pkt_mask & pkts_mask) == 0) 1256 continue; 1257 1258 pkt = p->pkts[i]; 1259 port_out_id = RTE_MBUF_METADATA_UINT32(pkt, 1260 p->offset_port_id); 1261 port_out = &p->ports_out[port_out_id]; 1262 1263 /* Output port user actions */ 1264 if (port_out->f_action == NULL) /* Output port TX */ 1265 port_out->ops.f_tx(port_out->h_port, pkt); 1266 else { 1267 port_out->f_action(p, 1268 p->pkts, 1269 pkt_mask, 1270 port_out->arg_ah); 1271 1272 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1273 port_out->n_pkts_dropped_by_ah); 1274 1275 /* Output port TX */ 1276 if (pkt_mask & p->pkts_mask) 1277 port_out->ops.f_tx(port_out->h_port, 1278 pkt); 1279 } 1280 } 1281 } 1282 } 1283 1284 static inline void 1285 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask) 1286 { 1287 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1288 uint64_t n_pkts = rte_popcount64(pkts_mask); 1289 uint32_t i; 1290 1291 for (i = 0; i < n_pkts; i++) 1292 rte_pktmbuf_free(p->pkts[i]); 1293 } else { 1294 uint32_t i; 1295 1296 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1297 uint64_t pkt_mask = 1LLU << i; 1298 1299 if ((pkt_mask & pkts_mask) == 0) 1300 continue; 1301 1302 rte_pktmbuf_free(p->pkts[i]); 1303 } 1304 } 1305 } 1306 1307 int 1308 rte_pipeline_run(struct rte_pipeline *p) 1309 { 1310 struct rte_port_in *port_in = p->port_in_next; 1311 uint32_t n_pkts, table_id; 1312 1313 if (port_in == NULL) 1314 return 0; 1315 1316 /* Input port RX */ 1317 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts, 1318 port_in->burst_size); 1319 if (n_pkts == 0) { 1320 p->port_in_next = port_in->next; 1321 return 0; 1322 } 1323 1324 p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t); 1325 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0; 1326 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0; 1327 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0; 1328 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; 1329 1330 /* Input port user actions */ 1331 if (port_in->f_action != NULL) { 1332 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah); 1333 1334 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1335 port_in->n_pkts_dropped_by_ah); 1336 } 1337 1338 /* Table */ 1339 for (table_id = port_in->table_id; p->pkts_mask != 0; ) { 1340 struct rte_table *table; 1341 uint64_t lookup_hit_mask, lookup_miss_mask; 1342 1343 /* Lookup */ 1344 table = &p->tables[table_id]; 1345 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask, 1346 &lookup_hit_mask, (void **) p->entries); 1347 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask); 1348 1349 /* Lookup miss */ 1350 if (lookup_miss_mask != 0) { 1351 struct rte_pipeline_table_entry *default_entry = 1352 table->default_entry; 1353 1354 p->pkts_mask = lookup_miss_mask; 1355 1356 /* Table user actions */ 1357 if (table->f_action_miss != NULL) { 1358 table->f_action_miss(p, 1359 p->pkts, 1360 lookup_miss_mask, 1361 default_entry, 1362 table->arg_ah); 1363 1364 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1365 table->n_pkts_dropped_by_lkp_miss_ah); 1366 } 1367 1368 /* Table reserved actions */ 1369 if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) && 1370 (p->pkts_mask != 0)) 1371 rte_pipeline_action_handler_port_bulk(p, 1372 p->pkts_mask, 1373 default_entry->port_id); 1374 else { 1375 uint32_t pos = default_entry->action; 1376 1377 RTE_PIPELINE_STATS_TABLE_DROP0(p); 1378 1379 p->action_mask0[pos] |= p->pkts_mask; 1380 1381 RTE_PIPELINE_STATS_TABLE_DROP1(p, 1382 table->n_pkts_dropped_lkp_miss); 1383 } 1384 } 1385 1386 /* Lookup hit */ 1387 if (lookup_hit_mask != 0) { 1388 p->pkts_mask = lookup_hit_mask; 1389 1390 /* Table user actions */ 1391 if (table->f_action_hit != NULL) { 1392 table->f_action_hit(p, 1393 p->pkts, 1394 lookup_hit_mask, 1395 p->entries, 1396 table->arg_ah); 1397 1398 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1399 table->n_pkts_dropped_by_lkp_hit_ah); 1400 } 1401 1402 /* Table reserved actions */ 1403 RTE_PIPELINE_STATS_TABLE_DROP0(p); 1404 rte_pipeline_compute_masks(p, p->pkts_mask); 1405 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= 1406 p->action_mask1[ 1407 RTE_PIPELINE_ACTION_DROP]; 1408 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |= 1409 p->action_mask1[ 1410 RTE_PIPELINE_ACTION_PORT]; 1411 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |= 1412 p->action_mask1[ 1413 RTE_PIPELINE_ACTION_PORT_META]; 1414 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |= 1415 p->action_mask1[ 1416 RTE_PIPELINE_ACTION_TABLE]; 1417 1418 RTE_PIPELINE_STATS_TABLE_DROP1(p, 1419 table->n_pkts_dropped_lkp_hit); 1420 } 1421 1422 /* Prepare for next iteration */ 1423 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE]; 1424 table_id = table->table_next_id; 1425 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; 1426 } 1427 1428 /* Table reserved action PORT */ 1429 rte_pipeline_action_handler_port(p, 1430 p->action_mask0[RTE_PIPELINE_ACTION_PORT]); 1431 1432 /* Table reserved action PORT META */ 1433 rte_pipeline_action_handler_port_meta(p, 1434 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]); 1435 1436 /* Table reserved action DROP */ 1437 rte_pipeline_action_handler_drop(p, 1438 p->action_mask0[RTE_PIPELINE_ACTION_DROP]); 1439 1440 /* Pick candidate for next port IN to serve */ 1441 p->port_in_next = port_in->next; 1442 1443 return (int) n_pkts; 1444 } 1445 1446 int 1447 rte_pipeline_flush(struct rte_pipeline *p) 1448 { 1449 uint32_t port_id; 1450 1451 /* Check input arguments */ 1452 if (p == NULL) { 1453 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 1454 __func__); 1455 return -EINVAL; 1456 } 1457 1458 for (port_id = 0; port_id < p->num_ports_out; port_id++) { 1459 struct rte_port_out *port = &p->ports_out[port_id]; 1460 1461 if (port->ops.f_flush != NULL) 1462 port->ops.f_flush(port->h_port); 1463 } 1464 1465 return 0; 1466 } 1467 1468 int 1469 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p, 1470 uint32_t port_id, struct rte_mbuf *pkt) 1471 { 1472 struct rte_port_out *port_out = &p->ports_out[port_id]; 1473 1474 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */ 1475 1476 return 0; 1477 } 1478 1479 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p, 1480 uint64_t pkts_mask) 1481 { 1482 pkts_mask &= p->pkts_mask; 1483 p->pkts_mask &= ~pkts_mask; 1484 1485 return 0; 1486 } 1487 1488 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p, 1489 uint64_t pkts_mask) 1490 { 1491 pkts_mask &= p->pkts_mask; 1492 p->pkts_mask &= ~pkts_mask; 1493 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask; 1494 1495 RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask); 1496 return 0; 1497 } 1498 1499 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id, 1500 struct rte_pipeline_port_in_stats *stats, int clear) 1501 { 1502 struct rte_port_in *port; 1503 int retval; 1504 1505 if (p == NULL) { 1506 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 1507 __func__); 1508 return -EINVAL; 1509 } 1510 1511 if (port_id >= p->num_ports_in) { 1512 PIPELINE_LOG(ERR, 1513 "%s: port IN ID %u is out of range", 1514 __func__, port_id); 1515 return -EINVAL; 1516 } 1517 1518 port = &p->ports_in[port_id]; 1519 1520 if (port->ops.f_stats != NULL) { 1521 retval = port->ops.f_stats(port->h_port, &stats->stats, clear); 1522 if (retval) 1523 return retval; 1524 } else if (stats != NULL) 1525 memset(&stats->stats, 0, sizeof(stats->stats)); 1526 1527 if (stats != NULL) 1528 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah; 1529 1530 if (clear != 0) 1531 port->n_pkts_dropped_by_ah = 0; 1532 1533 return 0; 1534 } 1535 1536 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id, 1537 struct rte_pipeline_port_out_stats *stats, int clear) 1538 { 1539 struct rte_port_out *port; 1540 int retval; 1541 1542 if (p == NULL) { 1543 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", __func__); 1544 return -EINVAL; 1545 } 1546 1547 if (port_id >= p->num_ports_out) { 1548 PIPELINE_LOG(ERR, 1549 "%s: port OUT ID %u is out of range", __func__, port_id); 1550 return -EINVAL; 1551 } 1552 1553 port = &p->ports_out[port_id]; 1554 if (port->ops.f_stats != NULL) { 1555 retval = port->ops.f_stats(port->h_port, &stats->stats, clear); 1556 if (retval != 0) 1557 return retval; 1558 } else if (stats != NULL) 1559 memset(&stats->stats, 0, sizeof(stats->stats)); 1560 1561 if (stats != NULL) 1562 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah; 1563 1564 if (clear != 0) 1565 port->n_pkts_dropped_by_ah = 0; 1566 1567 return 0; 1568 } 1569 1570 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id, 1571 struct rte_pipeline_table_stats *stats, int clear) 1572 { 1573 struct rte_table *table; 1574 int retval; 1575 1576 if (p == NULL) { 1577 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", 1578 __func__); 1579 return -EINVAL; 1580 } 1581 1582 if (table_id >= p->num_tables) { 1583 PIPELINE_LOG(ERR, 1584 "%s: table %u is out of range", __func__, table_id); 1585 return -EINVAL; 1586 } 1587 1588 table = &p->tables[table_id]; 1589 if (table->ops.f_stats != NULL) { 1590 retval = table->ops.f_stats(table->h_table, &stats->stats, clear); 1591 if (retval != 0) 1592 return retval; 1593 } else if (stats != NULL) 1594 memset(&stats->stats, 0, sizeof(stats->stats)); 1595 1596 if (stats != NULL) { 1597 stats->n_pkts_dropped_by_lkp_hit_ah = 1598 table->n_pkts_dropped_by_lkp_hit_ah; 1599 stats->n_pkts_dropped_by_lkp_miss_ah = 1600 table->n_pkts_dropped_by_lkp_miss_ah; 1601 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit; 1602 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss; 1603 } 1604 1605 if (clear != 0) { 1606 table->n_pkts_dropped_by_lkp_hit_ah = 0; 1607 table->n_pkts_dropped_by_lkp_miss_ah = 0; 1608 table->n_pkts_dropped_lkp_hit = 0; 1609 table->n_pkts_dropped_lkp_miss = 0; 1610 } 1611 1612 return 0; 1613 } 1614