1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <string.h> 6 #include <stdio.h> 7 8 #include <rte_common.h> 9 #include <rte_mbuf.h> 10 #include <rte_malloc.h> 11 #include <rte_string_fns.h> 12 13 #include "rte_pipeline.h" 14 15 #define RTE_TABLE_INVALID UINT32_MAX 16 17 #ifdef RTE_PIPELINE_STATS_COLLECT 18 19 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) \ 20 ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); }) 21 22 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) \ 23 ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; }) 24 25 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) \ 26 ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; }) 27 28 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) \ 29 ({ \ 30 uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; \ 31 mask ^= (p)->pkts_drop_mask; \ 32 (counter) += __builtin_popcountll(mask); \ 33 }) 34 35 #else 36 37 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) 38 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) 39 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) 40 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) 41 42 #endif 43 44 struct rte_port_in { 45 /* Input parameters */ 46 struct rte_port_in_ops ops; 47 rte_pipeline_port_in_action_handler f_action; 48 void *arg_ah; 49 uint32_t burst_size; 50 51 /* The table to which this port is connected */ 52 uint32_t table_id; 53 54 /* Handle to low-level port */ 55 void *h_port; 56 57 /* List of enabled ports */ 58 struct rte_port_in *next; 59 60 /* Statistics */ 61 uint64_t n_pkts_dropped_by_ah; 62 }; 63 64 struct rte_port_out { 65 /* Input parameters */ 66 struct rte_port_out_ops ops; 67 rte_pipeline_port_out_action_handler f_action; 68 void *arg_ah; 69 70 /* Handle to low-level port */ 71 void *h_port; 72 73 /* Statistics */ 74 uint64_t n_pkts_dropped_by_ah; 75 }; 76 77 struct rte_table { 78 /* Input parameters */ 79 struct rte_table_ops ops; 80 rte_pipeline_table_action_handler_hit f_action_hit; 81 rte_pipeline_table_action_handler_miss f_action_miss; 82 void *arg_ah; 83 struct rte_pipeline_table_entry *default_entry; 84 uint32_t entry_size; 85 86 uint32_t table_next_id; 87 uint32_t table_next_id_valid; 88 89 /* Handle to the low-level table object */ 90 void *h_table; 91 92 /* Statistics */ 93 uint64_t n_pkts_dropped_by_lkp_hit_ah; 94 uint64_t n_pkts_dropped_by_lkp_miss_ah; 95 uint64_t n_pkts_dropped_lkp_hit; 96 uint64_t n_pkts_dropped_lkp_miss; 97 }; 98 99 #define RTE_PIPELINE_MAX_NAME_SZ 124 100 101 struct rte_pipeline { 102 /* Input parameters */ 103 char name[RTE_PIPELINE_MAX_NAME_SZ]; 104 int socket_id; 105 uint32_t offset_port_id; 106 107 /* Internal tables */ 108 struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX]; 109 struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX]; 110 struct rte_table tables[RTE_PIPELINE_TABLE_MAX]; 111 112 /* Occupancy of internal tables */ 113 uint32_t num_ports_in; 114 uint32_t num_ports_out; 115 uint32_t num_tables; 116 117 /* List of enabled ports */ 118 uint64_t enabled_port_in_mask; 119 struct rte_port_in *port_in_next; 120 121 /* Pipeline run structures */ 122 struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX]; 123 struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX]; 124 uint64_t action_mask0[RTE_PIPELINE_ACTIONS]; 125 uint64_t action_mask1[RTE_PIPELINE_ACTIONS]; 126 uint64_t pkts_mask; 127 uint64_t n_pkts_ah_drop; 128 uint64_t pkts_drop_mask; 129 } __rte_cache_aligned; 130 131 static inline uint32_t 132 rte_mask_get_next(uint64_t mask, uint32_t pos) 133 { 134 uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) | 135 (mask >> ((pos + 1) & 0x3F)); 136 return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F; 137 } 138 139 static inline uint32_t 140 rte_mask_get_prev(uint64_t mask, uint32_t pos) 141 { 142 uint64_t mask_rot = (mask >> (pos & 0x3F)) | 143 (mask << ((64 - pos) & 0x3F)); 144 return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F; 145 } 146 147 static void 148 rte_pipeline_table_free(struct rte_table *table); 149 150 static void 151 rte_pipeline_port_in_free(struct rte_port_in *port); 152 153 static void 154 rte_pipeline_port_out_free(struct rte_port_out *port); 155 156 /* 157 * Pipeline 158 * 159 */ 160 static int 161 rte_pipeline_check_params(struct rte_pipeline_params *params) 162 { 163 if (params == NULL) { 164 RTE_LOG(ERR, PIPELINE, 165 "%s: Incorrect value for parameter params\n", __func__); 166 return -EINVAL; 167 } 168 169 /* name */ 170 if (params->name == NULL) { 171 RTE_LOG(ERR, PIPELINE, 172 "%s: Incorrect value for parameter name\n", __func__); 173 return -EINVAL; 174 } 175 176 /* socket */ 177 if (params->socket_id < 0) { 178 RTE_LOG(ERR, PIPELINE, 179 "%s: Incorrect value for parameter socket_id\n", 180 __func__); 181 return -EINVAL; 182 } 183 184 return 0; 185 } 186 187 struct rte_pipeline * 188 rte_pipeline_create(struct rte_pipeline_params *params) 189 { 190 struct rte_pipeline *p; 191 int status; 192 193 /* Check input parameters */ 194 status = rte_pipeline_check_params(params); 195 if (status != 0) { 196 RTE_LOG(ERR, PIPELINE, 197 "%s: Pipeline params check failed (%d)\n", 198 __func__, status); 199 return NULL; 200 } 201 202 /* Allocate memory for the pipeline on requested socket */ 203 p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline), 204 RTE_CACHE_LINE_SIZE, params->socket_id); 205 206 if (p == NULL) { 207 RTE_LOG(ERR, PIPELINE, 208 "%s: Pipeline memory allocation failed\n", __func__); 209 return NULL; 210 } 211 212 /* Save input parameters */ 213 strlcpy(p->name, params->name, RTE_PIPELINE_MAX_NAME_SZ); 214 p->socket_id = params->socket_id; 215 p->offset_port_id = params->offset_port_id; 216 217 /* Initialize pipeline internal data structure */ 218 p->num_ports_in = 0; 219 p->num_ports_out = 0; 220 p->num_tables = 0; 221 p->enabled_port_in_mask = 0; 222 p->port_in_next = NULL; 223 p->pkts_mask = 0; 224 p->n_pkts_ah_drop = 0; 225 226 return p; 227 } 228 229 int 230 rte_pipeline_free(struct rte_pipeline *p) 231 { 232 uint32_t i; 233 234 /* Check input parameters */ 235 if (p == NULL) { 236 RTE_LOG(ERR, PIPELINE, 237 "%s: rte_pipeline parameter is NULL\n", __func__); 238 return -EINVAL; 239 } 240 241 /* Free input ports */ 242 for (i = 0; i < p->num_ports_in; i++) { 243 struct rte_port_in *port = &p->ports_in[i]; 244 245 rte_pipeline_port_in_free(port); 246 } 247 248 /* Free tables */ 249 for (i = 0; i < p->num_tables; i++) { 250 struct rte_table *table = &p->tables[i]; 251 252 rte_pipeline_table_free(table); 253 } 254 255 /* Free output ports */ 256 for (i = 0; i < p->num_ports_out; i++) { 257 struct rte_port_out *port = &p->ports_out[i]; 258 259 rte_pipeline_port_out_free(port); 260 } 261 262 /* Free pipeline memory */ 263 rte_free(p); 264 265 return 0; 266 } 267 268 /* 269 * Table 270 * 271 */ 272 static int 273 rte_table_check_params(struct rte_pipeline *p, 274 struct rte_pipeline_table_params *params, 275 uint32_t *table_id) 276 { 277 if (p == NULL) { 278 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", 279 __func__); 280 return -EINVAL; 281 } 282 if (params == NULL) { 283 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n", 284 __func__); 285 return -EINVAL; 286 } 287 if (table_id == NULL) { 288 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n", 289 __func__); 290 return -EINVAL; 291 } 292 293 /* ops */ 294 if (params->ops == NULL) { 295 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n", 296 __func__); 297 return -EINVAL; 298 } 299 300 if (params->ops->f_create == NULL) { 301 RTE_LOG(ERR, PIPELINE, 302 "%s: f_create function pointer is NULL\n", __func__); 303 return -EINVAL; 304 } 305 306 if (params->ops->f_lookup == NULL) { 307 RTE_LOG(ERR, PIPELINE, 308 "%s: f_lookup function pointer is NULL\n", __func__); 309 return -EINVAL; 310 } 311 312 /* De we have room for one more table? */ 313 if (p->num_tables == RTE_PIPELINE_TABLE_MAX) { 314 RTE_LOG(ERR, PIPELINE, 315 "%s: Incorrect value for num_tables parameter\n", 316 __func__); 317 return -EINVAL; 318 } 319 320 return 0; 321 } 322 323 int 324 rte_pipeline_table_create(struct rte_pipeline *p, 325 struct rte_pipeline_table_params *params, 326 uint32_t *table_id) 327 { 328 struct rte_table *table; 329 struct rte_pipeline_table_entry *default_entry; 330 void *h_table; 331 uint32_t entry_size, id; 332 int status; 333 334 /* Check input arguments */ 335 status = rte_table_check_params(p, params, table_id); 336 if (status != 0) 337 return status; 338 339 id = p->num_tables; 340 table = &p->tables[id]; 341 342 /* Allocate space for the default table entry */ 343 entry_size = sizeof(struct rte_pipeline_table_entry) + 344 params->action_data_size; 345 default_entry = rte_zmalloc_socket( 346 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id); 347 if (default_entry == NULL) { 348 RTE_LOG(ERR, PIPELINE, 349 "%s: Failed to allocate default entry\n", __func__); 350 return -EINVAL; 351 } 352 353 /* Create the table */ 354 h_table = params->ops->f_create(params->arg_create, p->socket_id, 355 entry_size); 356 if (h_table == NULL) { 357 rte_free(default_entry); 358 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__); 359 return -EINVAL; 360 } 361 362 /* Commit current table to the pipeline */ 363 p->num_tables++; 364 *table_id = id; 365 366 /* Save input parameters */ 367 memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops)); 368 table->f_action_hit = params->f_action_hit; 369 table->f_action_miss = params->f_action_miss; 370 table->arg_ah = params->arg_ah; 371 table->entry_size = entry_size; 372 373 /* Clear the lookup miss actions (to be set later through API) */ 374 table->default_entry = default_entry; 375 table->default_entry->action = RTE_PIPELINE_ACTION_DROP; 376 377 /* Initialize table internal data structure */ 378 table->h_table = h_table; 379 table->table_next_id = 0; 380 table->table_next_id_valid = 0; 381 382 return 0; 383 } 384 385 void 386 rte_pipeline_table_free(struct rte_table *table) 387 { 388 if (table->ops.f_free != NULL) 389 table->ops.f_free(table->h_table); 390 391 rte_free(table->default_entry); 392 } 393 394 int 395 rte_pipeline_table_default_entry_add(struct rte_pipeline *p, 396 uint32_t table_id, 397 struct rte_pipeline_table_entry *default_entry, 398 struct rte_pipeline_table_entry **default_entry_ptr) 399 { 400 struct rte_table *table; 401 402 /* Check input arguments */ 403 if (p == NULL) { 404 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", 405 __func__); 406 return -EINVAL; 407 } 408 409 if (default_entry == NULL) { 410 RTE_LOG(ERR, PIPELINE, 411 "%s: default_entry parameter is NULL\n", __func__); 412 return -EINVAL; 413 } 414 415 if (table_id >= p->num_tables) { 416 RTE_LOG(ERR, PIPELINE, 417 "%s: table_id %d out of range\n", __func__, table_id); 418 return -EINVAL; 419 } 420 421 table = &p->tables[table_id]; 422 423 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) && 424 table->table_next_id_valid && 425 (default_entry->table_id != table->table_next_id)) { 426 RTE_LOG(ERR, PIPELINE, 427 "%s: Tree-like topologies not allowed\n", __func__); 428 return -EINVAL; 429 } 430 431 /* Set the lookup miss actions */ 432 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) && 433 (table->table_next_id_valid == 0)) { 434 table->table_next_id = default_entry->table_id; 435 table->table_next_id_valid = 1; 436 } 437 438 memcpy(table->default_entry, default_entry, table->entry_size); 439 440 *default_entry_ptr = table->default_entry; 441 return 0; 442 } 443 444 int 445 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p, 446 uint32_t table_id, 447 struct rte_pipeline_table_entry *entry) 448 { 449 struct rte_table *table; 450 451 /* Check input arguments */ 452 if (p == NULL) { 453 RTE_LOG(ERR, PIPELINE, 454 "%s: pipeline parameter is NULL\n", __func__); 455 return -EINVAL; 456 } 457 458 if (table_id >= p->num_tables) { 459 RTE_LOG(ERR, PIPELINE, 460 "%s: table_id %d out of range\n", __func__, table_id); 461 return -EINVAL; 462 } 463 464 table = &p->tables[table_id]; 465 466 /* Save the current contents of the default entry */ 467 if (entry) 468 memcpy(entry, table->default_entry, table->entry_size); 469 470 /* Clear the lookup miss actions */ 471 memset(table->default_entry, 0, table->entry_size); 472 table->default_entry->action = RTE_PIPELINE_ACTION_DROP; 473 474 return 0; 475 } 476 477 int 478 rte_pipeline_table_entry_add(struct rte_pipeline *p, 479 uint32_t table_id, 480 void *key, 481 struct rte_pipeline_table_entry *entry, 482 int *key_found, 483 struct rte_pipeline_table_entry **entry_ptr) 484 { 485 struct rte_table *table; 486 487 /* Check input arguments */ 488 if (p == NULL) { 489 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", 490 __func__); 491 return -EINVAL; 492 } 493 494 if (key == NULL) { 495 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__); 496 return -EINVAL; 497 } 498 499 if (entry == NULL) { 500 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n", 501 __func__); 502 return -EINVAL; 503 } 504 505 if (table_id >= p->num_tables) { 506 RTE_LOG(ERR, PIPELINE, 507 "%s: table_id %d out of range\n", __func__, table_id); 508 return -EINVAL; 509 } 510 511 table = &p->tables[table_id]; 512 513 if (table->ops.f_add == NULL) { 514 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n", 515 __func__); 516 return -EINVAL; 517 } 518 519 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) && 520 table->table_next_id_valid && 521 (entry->table_id != table->table_next_id)) { 522 RTE_LOG(ERR, PIPELINE, 523 "%s: Tree-like topologies not allowed\n", __func__); 524 return -EINVAL; 525 } 526 527 /* Add entry */ 528 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) && 529 (table->table_next_id_valid == 0)) { 530 table->table_next_id = entry->table_id; 531 table->table_next_id_valid = 1; 532 } 533 534 return (table->ops.f_add)(table->h_table, key, (void *) entry, 535 key_found, (void **) entry_ptr); 536 } 537 538 int 539 rte_pipeline_table_entry_delete(struct rte_pipeline *p, 540 uint32_t table_id, 541 void *key, 542 int *key_found, 543 struct rte_pipeline_table_entry *entry) 544 { 545 struct rte_table *table; 546 547 /* Check input arguments */ 548 if (p == NULL) { 549 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 550 __func__); 551 return -EINVAL; 552 } 553 554 if (key == NULL) { 555 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", 556 __func__); 557 return -EINVAL; 558 } 559 560 if (table_id >= p->num_tables) { 561 RTE_LOG(ERR, PIPELINE, 562 "%s: table_id %d out of range\n", __func__, table_id); 563 return -EINVAL; 564 } 565 566 table = &p->tables[table_id]; 567 568 if (table->ops.f_delete == NULL) { 569 RTE_LOG(ERR, PIPELINE, 570 "%s: f_delete function pointer NULL\n", __func__); 571 return -EINVAL; 572 } 573 574 return (table->ops.f_delete)(table->h_table, key, key_found, entry); 575 } 576 577 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p, 578 uint32_t table_id, 579 void **keys, 580 struct rte_pipeline_table_entry **entries, 581 uint32_t n_keys, 582 int *key_found, 583 struct rte_pipeline_table_entry **entries_ptr) 584 { 585 struct rte_table *table; 586 uint32_t i; 587 588 /* Check input arguments */ 589 if (p == NULL) { 590 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", 591 __func__); 592 return -EINVAL; 593 } 594 595 if (keys == NULL) { 596 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__); 597 return -EINVAL; 598 } 599 600 if (entries == NULL) { 601 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n", 602 __func__); 603 return -EINVAL; 604 } 605 606 if (table_id >= p->num_tables) { 607 RTE_LOG(ERR, PIPELINE, 608 "%s: table_id %d out of range\n", __func__, table_id); 609 return -EINVAL; 610 } 611 612 table = &p->tables[table_id]; 613 614 if (table->ops.f_add_bulk == NULL) { 615 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n", 616 __func__); 617 return -EINVAL; 618 } 619 620 for (i = 0; i < n_keys; i++) { 621 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) && 622 table->table_next_id_valid && 623 (entries[i]->table_id != table->table_next_id)) { 624 RTE_LOG(ERR, PIPELINE, 625 "%s: Tree-like topologies not allowed\n", __func__); 626 return -EINVAL; 627 } 628 } 629 630 /* Add entry */ 631 for (i = 0; i < n_keys; i++) { 632 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) && 633 (table->table_next_id_valid == 0)) { 634 table->table_next_id = entries[i]->table_id; 635 table->table_next_id_valid = 1; 636 } 637 } 638 639 return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries, 640 n_keys, key_found, (void **) entries_ptr); 641 } 642 643 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p, 644 uint32_t table_id, 645 void **keys, 646 uint32_t n_keys, 647 int *key_found, 648 struct rte_pipeline_table_entry **entries) 649 { 650 struct rte_table *table; 651 652 /* Check input arguments */ 653 if (p == NULL) { 654 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 655 __func__); 656 return -EINVAL; 657 } 658 659 if (keys == NULL) { 660 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", 661 __func__); 662 return -EINVAL; 663 } 664 665 if (table_id >= p->num_tables) { 666 RTE_LOG(ERR, PIPELINE, 667 "%s: table_id %d out of range\n", __func__, table_id); 668 return -EINVAL; 669 } 670 671 table = &p->tables[table_id]; 672 673 if (table->ops.f_delete_bulk == NULL) { 674 RTE_LOG(ERR, PIPELINE, 675 "%s: f_delete function pointer NULL\n", __func__); 676 return -EINVAL; 677 } 678 679 return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found, 680 (void **) entries); 681 } 682 683 /* 684 * Port 685 * 686 */ 687 static int 688 rte_pipeline_port_in_check_params(struct rte_pipeline *p, 689 struct rte_pipeline_port_in_params *params, 690 uint32_t *port_id) 691 { 692 if (p == NULL) { 693 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 694 __func__); 695 return -EINVAL; 696 } 697 if (params == NULL) { 698 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__); 699 return -EINVAL; 700 } 701 if (port_id == NULL) { 702 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n", 703 __func__); 704 return -EINVAL; 705 } 706 707 /* ops */ 708 if (params->ops == NULL) { 709 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n", 710 __func__); 711 return -EINVAL; 712 } 713 714 if (params->ops->f_create == NULL) { 715 RTE_LOG(ERR, PIPELINE, 716 "%s: f_create function pointer NULL\n", __func__); 717 return -EINVAL; 718 } 719 720 if (params->ops->f_rx == NULL) { 721 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n", 722 __func__); 723 return -EINVAL; 724 } 725 726 /* burst_size */ 727 if ((params->burst_size == 0) || 728 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) { 729 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n", 730 __func__); 731 return -EINVAL; 732 } 733 734 /* Do we have room for one more port? */ 735 if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) { 736 RTE_LOG(ERR, PIPELINE, 737 "%s: invalid value for num_ports_in\n", __func__); 738 return -EINVAL; 739 } 740 741 return 0; 742 } 743 744 static int 745 rte_pipeline_port_out_check_params(struct rte_pipeline *p, 746 struct rte_pipeline_port_out_params *params, 747 uint32_t *port_id) 748 { 749 if (p == NULL) { 750 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 751 __func__); 752 return -EINVAL; 753 } 754 755 if (params == NULL) { 756 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__); 757 return -EINVAL; 758 } 759 760 if (port_id == NULL) { 761 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n", 762 __func__); 763 return -EINVAL; 764 } 765 766 /* ops */ 767 if (params->ops == NULL) { 768 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n", 769 __func__); 770 return -EINVAL; 771 } 772 773 if (params->ops->f_create == NULL) { 774 RTE_LOG(ERR, PIPELINE, 775 "%s: f_create function pointer NULL\n", __func__); 776 return -EINVAL; 777 } 778 779 if (params->ops->f_tx == NULL) { 780 RTE_LOG(ERR, PIPELINE, 781 "%s: f_tx function pointer NULL\n", __func__); 782 return -EINVAL; 783 } 784 785 if (params->ops->f_tx_bulk == NULL) { 786 RTE_LOG(ERR, PIPELINE, 787 "%s: f_tx_bulk function pointer NULL\n", __func__); 788 return -EINVAL; 789 } 790 791 /* Do we have room for one more port? */ 792 if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) { 793 RTE_LOG(ERR, PIPELINE, 794 "%s: invalid value for num_ports_out\n", __func__); 795 return -EINVAL; 796 } 797 798 return 0; 799 } 800 801 int 802 rte_pipeline_port_in_create(struct rte_pipeline *p, 803 struct rte_pipeline_port_in_params *params, 804 uint32_t *port_id) 805 { 806 struct rte_port_in *port; 807 void *h_port; 808 uint32_t id; 809 int status; 810 811 /* Check input arguments */ 812 status = rte_pipeline_port_in_check_params(p, params, port_id); 813 if (status != 0) 814 return status; 815 816 id = p->num_ports_in; 817 port = &p->ports_in[id]; 818 819 /* Create the port */ 820 h_port = params->ops->f_create(params->arg_create, p->socket_id); 821 if (h_port == NULL) { 822 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__); 823 return -EINVAL; 824 } 825 826 /* Commit current table to the pipeline */ 827 p->num_ports_in++; 828 *port_id = id; 829 830 /* Save input parameters */ 831 memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops)); 832 port->f_action = params->f_action; 833 port->arg_ah = params->arg_ah; 834 port->burst_size = params->burst_size; 835 836 /* Initialize port internal data structure */ 837 port->table_id = RTE_TABLE_INVALID; 838 port->h_port = h_port; 839 port->next = NULL; 840 841 return 0; 842 } 843 844 void 845 rte_pipeline_port_in_free(struct rte_port_in *port) 846 { 847 if (port->ops.f_free != NULL) 848 port->ops.f_free(port->h_port); 849 } 850 851 int 852 rte_pipeline_port_out_create(struct rte_pipeline *p, 853 struct rte_pipeline_port_out_params *params, 854 uint32_t *port_id) 855 { 856 struct rte_port_out *port; 857 void *h_port; 858 uint32_t id; 859 int status; 860 861 /* Check input arguments */ 862 status = rte_pipeline_port_out_check_params(p, params, port_id); 863 if (status != 0) 864 return status; 865 866 id = p->num_ports_out; 867 port = &p->ports_out[id]; 868 869 /* Create the port */ 870 h_port = params->ops->f_create(params->arg_create, p->socket_id); 871 if (h_port == NULL) { 872 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__); 873 return -EINVAL; 874 } 875 876 /* Commit current table to the pipeline */ 877 p->num_ports_out++; 878 *port_id = id; 879 880 /* Save input parameters */ 881 memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops)); 882 port->f_action = params->f_action; 883 port->arg_ah = params->arg_ah; 884 885 /* Initialize port internal data structure */ 886 port->h_port = h_port; 887 888 return 0; 889 } 890 891 void 892 rte_pipeline_port_out_free(struct rte_port_out *port) 893 { 894 if (port->ops.f_free != NULL) 895 port->ops.f_free(port->h_port); 896 } 897 898 int 899 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p, 900 uint32_t port_id, 901 uint32_t table_id) 902 { 903 struct rte_port_in *port; 904 905 /* Check input arguments */ 906 if (p == NULL) { 907 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 908 __func__); 909 return -EINVAL; 910 } 911 912 if (port_id >= p->num_ports_in) { 913 RTE_LOG(ERR, PIPELINE, 914 "%s: port IN ID %u is out of range\n", 915 __func__, port_id); 916 return -EINVAL; 917 } 918 919 if (table_id >= p->num_tables) { 920 RTE_LOG(ERR, PIPELINE, 921 "%s: Table ID %u is out of range\n", 922 __func__, table_id); 923 return -EINVAL; 924 } 925 926 port = &p->ports_in[port_id]; 927 port->table_id = table_id; 928 929 return 0; 930 } 931 932 int 933 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id) 934 { 935 struct rte_port_in *port, *port_prev, *port_next; 936 uint64_t port_mask; 937 uint32_t port_prev_id, port_next_id; 938 939 /* Check input arguments */ 940 if (p == NULL) { 941 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 942 __func__); 943 return -EINVAL; 944 } 945 946 if (port_id >= p->num_ports_in) { 947 RTE_LOG(ERR, PIPELINE, 948 "%s: port IN ID %u is out of range\n", 949 __func__, port_id); 950 return -EINVAL; 951 } 952 953 port = &p->ports_in[port_id]; 954 955 /* Return if current input port is already enabled */ 956 port_mask = 1LLU << port_id; 957 if (p->enabled_port_in_mask & port_mask) 958 return 0; 959 960 p->enabled_port_in_mask |= port_mask; 961 962 /* Add current input port to the pipeline chain of enabled ports */ 963 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id); 964 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id); 965 966 port_prev = &p->ports_in[port_prev_id]; 967 port_next = &p->ports_in[port_next_id]; 968 969 port_prev->next = port; 970 port->next = port_next; 971 972 /* Check if list of enabled ports was previously empty */ 973 if (p->enabled_port_in_mask == port_mask) 974 p->port_in_next = port; 975 976 return 0; 977 } 978 979 int 980 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id) 981 { 982 struct rte_port_in *port, *port_prev, *port_next; 983 uint64_t port_mask; 984 uint32_t port_prev_id, port_next_id; 985 986 /* Check input arguments */ 987 if (p == NULL) { 988 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 989 __func__); 990 return -EINVAL; 991 } 992 993 if (port_id >= p->num_ports_in) { 994 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n", 995 __func__, port_id); 996 return -EINVAL; 997 } 998 999 port = &p->ports_in[port_id]; 1000 1001 /* Return if current input port is already disabled */ 1002 port_mask = 1LLU << port_id; 1003 if ((p->enabled_port_in_mask & port_mask) == 0) 1004 return 0; 1005 1006 p->enabled_port_in_mask &= ~port_mask; 1007 1008 /* Return if no other enabled ports */ 1009 if (p->enabled_port_in_mask == 0) { 1010 p->port_in_next = NULL; 1011 1012 return 0; 1013 } 1014 1015 /* Add current input port to the pipeline chain of enabled ports */ 1016 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id); 1017 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id); 1018 1019 port_prev = &p->ports_in[port_prev_id]; 1020 port_next = &p->ports_in[port_next_id]; 1021 1022 port_prev->next = port_next; 1023 1024 /* Check if the port which has just been disabled is next to serve */ 1025 if (port == p->port_in_next) 1026 p->port_in_next = port_next; 1027 1028 return 0; 1029 } 1030 1031 /* 1032 * Pipeline run-time 1033 * 1034 */ 1035 int 1036 rte_pipeline_check(struct rte_pipeline *p) 1037 { 1038 uint32_t port_in_id; 1039 1040 /* Check input arguments */ 1041 if (p == NULL) { 1042 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 1043 __func__); 1044 return -EINVAL; 1045 } 1046 1047 /* Check that pipeline has at least one input port, one table and one 1048 output port */ 1049 if (p->num_ports_in == 0) { 1050 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n", 1051 __func__); 1052 return -EINVAL; 1053 } 1054 if (p->num_tables == 0) { 1055 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n", 1056 __func__); 1057 return -EINVAL; 1058 } 1059 if (p->num_ports_out == 0) { 1060 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n", 1061 __func__); 1062 return -EINVAL; 1063 } 1064 1065 /* Check that all input ports are connected */ 1066 for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) { 1067 struct rte_port_in *port_in = &p->ports_in[port_in_id]; 1068 1069 if (port_in->table_id == RTE_TABLE_INVALID) { 1070 RTE_LOG(ERR, PIPELINE, 1071 "%s: Port IN ID %u is not connected\n", 1072 __func__, port_in_id); 1073 return -EINVAL; 1074 } 1075 } 1076 1077 return 0; 1078 } 1079 1080 static inline void 1081 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask) 1082 { 1083 p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0; 1084 p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0; 1085 p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0; 1086 p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0; 1087 1088 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1089 uint64_t n_pkts = __builtin_popcountll(pkts_mask); 1090 uint32_t i; 1091 1092 for (i = 0; i < n_pkts; i++) { 1093 uint64_t pkt_mask = 1LLU << i; 1094 uint32_t pos = p->entries[i]->action; 1095 1096 p->action_mask1[pos] |= pkt_mask; 1097 } 1098 } else { 1099 uint32_t i; 1100 1101 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1102 uint64_t pkt_mask = 1LLU << i; 1103 uint32_t pos; 1104 1105 if ((pkt_mask & pkts_mask) == 0) 1106 continue; 1107 1108 pos = p->entries[i]->action; 1109 p->action_mask1[pos] |= pkt_mask; 1110 } 1111 } 1112 } 1113 1114 static inline void 1115 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p, 1116 uint64_t pkts_mask, uint32_t port_id) 1117 { 1118 struct rte_port_out *port_out = &p->ports_out[port_id]; 1119 1120 p->pkts_mask = pkts_mask; 1121 1122 /* Output port user actions */ 1123 if (port_out->f_action != NULL) { 1124 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah); 1125 1126 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1127 port_out->n_pkts_dropped_by_ah); 1128 } 1129 1130 /* Output port TX */ 1131 if (p->pkts_mask != 0) 1132 port_out->ops.f_tx_bulk(port_out->h_port, 1133 p->pkts, 1134 p->pkts_mask); 1135 } 1136 1137 static inline void 1138 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask) 1139 { 1140 p->pkts_mask = pkts_mask; 1141 1142 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1143 uint64_t n_pkts = __builtin_popcountll(pkts_mask); 1144 uint32_t i; 1145 1146 for (i = 0; i < n_pkts; i++) { 1147 struct rte_mbuf *pkt = p->pkts[i]; 1148 uint32_t port_out_id = p->entries[i]->port_id; 1149 struct rte_port_out *port_out = 1150 &p->ports_out[port_out_id]; 1151 1152 /* Output port user actions */ 1153 if (port_out->f_action == NULL) /* Output port TX */ 1154 port_out->ops.f_tx(port_out->h_port, pkt); 1155 else { 1156 uint64_t pkt_mask = 1LLU << i; 1157 1158 port_out->f_action(p, 1159 p->pkts, 1160 pkt_mask, 1161 port_out->arg_ah); 1162 1163 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1164 port_out->n_pkts_dropped_by_ah); 1165 1166 /* Output port TX */ 1167 if (pkt_mask & p->pkts_mask) 1168 port_out->ops.f_tx(port_out->h_port, 1169 pkt); 1170 } 1171 } 1172 } else { 1173 uint32_t i; 1174 1175 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1176 uint64_t pkt_mask = 1LLU << i; 1177 struct rte_mbuf *pkt; 1178 struct rte_port_out *port_out; 1179 uint32_t port_out_id; 1180 1181 if ((pkt_mask & pkts_mask) == 0) 1182 continue; 1183 1184 pkt = p->pkts[i]; 1185 port_out_id = p->entries[i]->port_id; 1186 port_out = &p->ports_out[port_out_id]; 1187 1188 /* Output port user actions */ 1189 if (port_out->f_action == NULL) /* Output port TX */ 1190 port_out->ops.f_tx(port_out->h_port, pkt); 1191 else { 1192 port_out->f_action(p, 1193 p->pkts, 1194 pkt_mask, 1195 port_out->arg_ah); 1196 1197 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1198 port_out->n_pkts_dropped_by_ah); 1199 1200 /* Output port TX */ 1201 if (pkt_mask & p->pkts_mask) 1202 port_out->ops.f_tx(port_out->h_port, 1203 pkt); 1204 } 1205 } 1206 } 1207 } 1208 1209 static inline void 1210 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p, 1211 uint64_t pkts_mask) 1212 { 1213 p->pkts_mask = pkts_mask; 1214 1215 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1216 uint64_t n_pkts = __builtin_popcountll(pkts_mask); 1217 uint32_t i; 1218 1219 for (i = 0; i < n_pkts; i++) { 1220 struct rte_mbuf *pkt = p->pkts[i]; 1221 uint32_t port_out_id = 1222 RTE_MBUF_METADATA_UINT32(pkt, 1223 p->offset_port_id); 1224 struct rte_port_out *port_out = &p->ports_out[ 1225 port_out_id]; 1226 1227 /* Output port user actions */ 1228 if (port_out->f_action == NULL) /* Output port TX */ 1229 port_out->ops.f_tx(port_out->h_port, pkt); 1230 else { 1231 uint64_t pkt_mask = 1LLU << i; 1232 1233 port_out->f_action(p, 1234 p->pkts, 1235 pkt_mask, 1236 port_out->arg_ah); 1237 1238 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1239 port_out->n_pkts_dropped_by_ah); 1240 1241 /* Output port TX */ 1242 if (pkt_mask & p->pkts_mask) 1243 port_out->ops.f_tx(port_out->h_port, 1244 pkt); 1245 } 1246 } 1247 } else { 1248 uint32_t i; 1249 1250 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1251 uint64_t pkt_mask = 1LLU << i; 1252 struct rte_mbuf *pkt; 1253 struct rte_port_out *port_out; 1254 uint32_t port_out_id; 1255 1256 if ((pkt_mask & pkts_mask) == 0) 1257 continue; 1258 1259 pkt = p->pkts[i]; 1260 port_out_id = RTE_MBUF_METADATA_UINT32(pkt, 1261 p->offset_port_id); 1262 port_out = &p->ports_out[port_out_id]; 1263 1264 /* Output port user actions */ 1265 if (port_out->f_action == NULL) /* Output port TX */ 1266 port_out->ops.f_tx(port_out->h_port, pkt); 1267 else { 1268 port_out->f_action(p, 1269 p->pkts, 1270 pkt_mask, 1271 port_out->arg_ah); 1272 1273 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1274 port_out->n_pkts_dropped_by_ah); 1275 1276 /* Output port TX */ 1277 if (pkt_mask & p->pkts_mask) 1278 port_out->ops.f_tx(port_out->h_port, 1279 pkt); 1280 } 1281 } 1282 } 1283 } 1284 1285 static inline void 1286 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask) 1287 { 1288 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1289 uint64_t n_pkts = __builtin_popcountll(pkts_mask); 1290 uint32_t i; 1291 1292 for (i = 0; i < n_pkts; i++) 1293 rte_pktmbuf_free(p->pkts[i]); 1294 } else { 1295 uint32_t i; 1296 1297 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1298 uint64_t pkt_mask = 1LLU << i; 1299 1300 if ((pkt_mask & pkts_mask) == 0) 1301 continue; 1302 1303 rte_pktmbuf_free(p->pkts[i]); 1304 } 1305 } 1306 } 1307 1308 int 1309 rte_pipeline_run(struct rte_pipeline *p) 1310 { 1311 struct rte_port_in *port_in = p->port_in_next; 1312 uint32_t n_pkts, table_id; 1313 1314 if (port_in == NULL) 1315 return 0; 1316 1317 /* Input port RX */ 1318 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts, 1319 port_in->burst_size); 1320 if (n_pkts == 0) { 1321 p->port_in_next = port_in->next; 1322 return 0; 1323 } 1324 1325 p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t); 1326 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0; 1327 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0; 1328 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0; 1329 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; 1330 1331 /* Input port user actions */ 1332 if (port_in->f_action != NULL) { 1333 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah); 1334 1335 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1336 port_in->n_pkts_dropped_by_ah); 1337 } 1338 1339 /* Table */ 1340 for (table_id = port_in->table_id; p->pkts_mask != 0; ) { 1341 struct rte_table *table; 1342 uint64_t lookup_hit_mask, lookup_miss_mask; 1343 1344 /* Lookup */ 1345 table = &p->tables[table_id]; 1346 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask, 1347 &lookup_hit_mask, (void **) p->entries); 1348 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask); 1349 1350 /* Lookup miss */ 1351 if (lookup_miss_mask != 0) { 1352 struct rte_pipeline_table_entry *default_entry = 1353 table->default_entry; 1354 1355 p->pkts_mask = lookup_miss_mask; 1356 1357 /* Table user actions */ 1358 if (table->f_action_miss != NULL) { 1359 table->f_action_miss(p, 1360 p->pkts, 1361 lookup_miss_mask, 1362 default_entry, 1363 table->arg_ah); 1364 1365 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1366 table->n_pkts_dropped_by_lkp_miss_ah); 1367 } 1368 1369 /* Table reserved actions */ 1370 if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) && 1371 (p->pkts_mask != 0)) 1372 rte_pipeline_action_handler_port_bulk(p, 1373 p->pkts_mask, 1374 default_entry->port_id); 1375 else { 1376 uint32_t pos = default_entry->action; 1377 1378 RTE_PIPELINE_STATS_TABLE_DROP0(p); 1379 1380 p->action_mask0[pos] |= p->pkts_mask; 1381 1382 RTE_PIPELINE_STATS_TABLE_DROP1(p, 1383 table->n_pkts_dropped_lkp_miss); 1384 } 1385 } 1386 1387 /* Lookup hit */ 1388 if (lookup_hit_mask != 0) { 1389 p->pkts_mask = lookup_hit_mask; 1390 1391 /* Table user actions */ 1392 if (table->f_action_hit != NULL) { 1393 table->f_action_hit(p, 1394 p->pkts, 1395 lookup_hit_mask, 1396 p->entries, 1397 table->arg_ah); 1398 1399 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1400 table->n_pkts_dropped_by_lkp_hit_ah); 1401 } 1402 1403 /* Table reserved actions */ 1404 RTE_PIPELINE_STATS_TABLE_DROP0(p); 1405 rte_pipeline_compute_masks(p, p->pkts_mask); 1406 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= 1407 p->action_mask1[ 1408 RTE_PIPELINE_ACTION_DROP]; 1409 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |= 1410 p->action_mask1[ 1411 RTE_PIPELINE_ACTION_PORT]; 1412 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |= 1413 p->action_mask1[ 1414 RTE_PIPELINE_ACTION_PORT_META]; 1415 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |= 1416 p->action_mask1[ 1417 RTE_PIPELINE_ACTION_TABLE]; 1418 1419 RTE_PIPELINE_STATS_TABLE_DROP1(p, 1420 table->n_pkts_dropped_lkp_hit); 1421 } 1422 1423 /* Prepare for next iteration */ 1424 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE]; 1425 table_id = table->table_next_id; 1426 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; 1427 } 1428 1429 /* Table reserved action PORT */ 1430 rte_pipeline_action_handler_port(p, 1431 p->action_mask0[RTE_PIPELINE_ACTION_PORT]); 1432 1433 /* Table reserved action PORT META */ 1434 rte_pipeline_action_handler_port_meta(p, 1435 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]); 1436 1437 /* Table reserved action DROP */ 1438 rte_pipeline_action_handler_drop(p, 1439 p->action_mask0[RTE_PIPELINE_ACTION_DROP]); 1440 1441 /* Pick candidate for next port IN to serve */ 1442 p->port_in_next = port_in->next; 1443 1444 return (int) n_pkts; 1445 } 1446 1447 int 1448 rte_pipeline_flush(struct rte_pipeline *p) 1449 { 1450 uint32_t port_id; 1451 1452 /* Check input arguments */ 1453 if (p == NULL) { 1454 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 1455 __func__); 1456 return -EINVAL; 1457 } 1458 1459 for (port_id = 0; port_id < p->num_ports_out; port_id++) { 1460 struct rte_port_out *port = &p->ports_out[port_id]; 1461 1462 if (port->ops.f_flush != NULL) 1463 port->ops.f_flush(port->h_port); 1464 } 1465 1466 return 0; 1467 } 1468 1469 int 1470 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p, 1471 uint32_t port_id, struct rte_mbuf *pkt) 1472 { 1473 struct rte_port_out *port_out = &p->ports_out[port_id]; 1474 1475 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */ 1476 1477 return 0; 1478 } 1479 1480 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p, 1481 uint64_t pkts_mask) 1482 { 1483 pkts_mask &= p->pkts_mask; 1484 p->pkts_mask &= ~pkts_mask; 1485 1486 return 0; 1487 } 1488 1489 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p, 1490 uint64_t pkts_mask) 1491 { 1492 pkts_mask &= p->pkts_mask; 1493 p->pkts_mask &= ~pkts_mask; 1494 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask; 1495 1496 RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask); 1497 return 0; 1498 } 1499 1500 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id, 1501 struct rte_pipeline_port_in_stats *stats, int clear) 1502 { 1503 struct rte_port_in *port; 1504 int retval; 1505 1506 if (p == NULL) { 1507 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 1508 __func__); 1509 return -EINVAL; 1510 } 1511 1512 if (port_id >= p->num_ports_in) { 1513 RTE_LOG(ERR, PIPELINE, 1514 "%s: port IN ID %u is out of range\n", 1515 __func__, port_id); 1516 return -EINVAL; 1517 } 1518 1519 port = &p->ports_in[port_id]; 1520 1521 if (port->ops.f_stats != NULL) { 1522 retval = port->ops.f_stats(port->h_port, &stats->stats, clear); 1523 if (retval) 1524 return retval; 1525 } else if (stats != NULL) 1526 memset(&stats->stats, 0, sizeof(stats->stats)); 1527 1528 if (stats != NULL) 1529 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah; 1530 1531 if (clear != 0) 1532 port->n_pkts_dropped_by_ah = 0; 1533 1534 return 0; 1535 } 1536 1537 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id, 1538 struct rte_pipeline_port_out_stats *stats, int clear) 1539 { 1540 struct rte_port_out *port; 1541 int retval; 1542 1543 if (p == NULL) { 1544 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__); 1545 return -EINVAL; 1546 } 1547 1548 if (port_id >= p->num_ports_out) { 1549 RTE_LOG(ERR, PIPELINE, 1550 "%s: port OUT ID %u is out of range\n", __func__, port_id); 1551 return -EINVAL; 1552 } 1553 1554 port = &p->ports_out[port_id]; 1555 if (port->ops.f_stats != NULL) { 1556 retval = port->ops.f_stats(port->h_port, &stats->stats, clear); 1557 if (retval != 0) 1558 return retval; 1559 } else if (stats != NULL) 1560 memset(&stats->stats, 0, sizeof(stats->stats)); 1561 1562 if (stats != NULL) 1563 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah; 1564 1565 if (clear != 0) 1566 port->n_pkts_dropped_by_ah = 0; 1567 1568 return 0; 1569 } 1570 1571 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id, 1572 struct rte_pipeline_table_stats *stats, int clear) 1573 { 1574 struct rte_table *table; 1575 int retval; 1576 1577 if (p == NULL) { 1578 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 1579 __func__); 1580 return -EINVAL; 1581 } 1582 1583 if (table_id >= p->num_tables) { 1584 RTE_LOG(ERR, PIPELINE, 1585 "%s: table %u is out of range\n", __func__, table_id); 1586 return -EINVAL; 1587 } 1588 1589 table = &p->tables[table_id]; 1590 if (table->ops.f_stats != NULL) { 1591 retval = table->ops.f_stats(table->h_table, &stats->stats, clear); 1592 if (retval != 0) 1593 return retval; 1594 } else if (stats != NULL) 1595 memset(&stats->stats, 0, sizeof(stats->stats)); 1596 1597 if (stats != NULL) { 1598 stats->n_pkts_dropped_by_lkp_hit_ah = 1599 table->n_pkts_dropped_by_lkp_hit_ah; 1600 stats->n_pkts_dropped_by_lkp_miss_ah = 1601 table->n_pkts_dropped_by_lkp_miss_ah; 1602 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit; 1603 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss; 1604 } 1605 1606 if (clear != 0) { 1607 table->n_pkts_dropped_by_lkp_hit_ah = 0; 1608 table->n_pkts_dropped_by_lkp_miss_ah = 0; 1609 table->n_pkts_dropped_lkp_hit = 0; 1610 table->n_pkts_dropped_lkp_miss = 0; 1611 } 1612 1613 return 0; 1614 } 1615