1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <string.h> 6 #include <stdio.h> 7 8 #include <rte_common.h> 9 #include <rte_mbuf.h> 10 #include <rte_malloc.h> 11 #include <rte_string_fns.h> 12 13 #include "rte_pipeline.h" 14 15 #define RTE_TABLE_INVALID UINT32_MAX 16 17 #ifdef RTE_PIPELINE_STATS_COLLECT 18 19 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) \ 20 ({ (p)->n_pkts_ah_drop = rte_popcount64(mask); }) 21 22 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) \ 23 ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; }) 24 25 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) \ 26 ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; }) 27 28 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) \ 29 ({ \ 30 uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; \ 31 mask ^= (p)->pkts_drop_mask; \ 32 (counter) += rte_popcount64(mask); \ 33 }) 34 35 #else 36 37 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) 38 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) 39 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) 40 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) 41 42 #endif 43 44 struct rte_port_in { 45 /* Input parameters */ 46 struct rte_port_in_ops ops; 47 rte_pipeline_port_in_action_handler f_action; 48 void *arg_ah; 49 uint32_t burst_size; 50 51 /* The table to which this port is connected */ 52 uint32_t table_id; 53 54 /* Handle to low-level port */ 55 void *h_port; 56 57 /* List of enabled ports */ 58 struct rte_port_in *next; 59 60 /* Statistics */ 61 uint64_t n_pkts_dropped_by_ah; 62 }; 63 64 struct rte_port_out { 65 /* Input parameters */ 66 struct rte_port_out_ops ops; 67 rte_pipeline_port_out_action_handler f_action; 68 void *arg_ah; 69 70 /* Handle to low-level port */ 71 void *h_port; 72 73 /* Statistics */ 74 uint64_t n_pkts_dropped_by_ah; 75 }; 76 77 struct rte_table { 78 /* Input parameters */ 79 struct rte_table_ops ops; 80 rte_pipeline_table_action_handler_hit f_action_hit; 81 rte_pipeline_table_action_handler_miss f_action_miss; 82 void *arg_ah; 83 struct rte_pipeline_table_entry *default_entry; 84 uint32_t entry_size; 85 86 uint32_t table_next_id; 87 uint32_t table_next_id_valid; 88 89 /* Handle to the low-level table object */ 90 void *h_table; 91 92 /* Statistics */ 93 uint64_t n_pkts_dropped_by_lkp_hit_ah; 94 uint64_t n_pkts_dropped_by_lkp_miss_ah; 95 uint64_t n_pkts_dropped_lkp_hit; 96 uint64_t n_pkts_dropped_lkp_miss; 97 }; 98 99 #define RTE_PIPELINE_MAX_NAME_SZ 124 100 101 struct rte_pipeline { 102 /* Input parameters */ 103 char name[RTE_PIPELINE_MAX_NAME_SZ]; 104 int socket_id; 105 uint32_t offset_port_id; 106 107 /* Internal tables */ 108 struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX]; 109 struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX]; 110 struct rte_table tables[RTE_PIPELINE_TABLE_MAX]; 111 112 /* Occupancy of internal tables */ 113 uint32_t num_ports_in; 114 uint32_t num_ports_out; 115 uint32_t num_tables; 116 117 /* List of enabled ports */ 118 uint64_t enabled_port_in_mask; 119 struct rte_port_in *port_in_next; 120 121 /* Pipeline run structures */ 122 struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX]; 123 struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX]; 124 uint64_t action_mask0[RTE_PIPELINE_ACTIONS]; 125 uint64_t action_mask1[RTE_PIPELINE_ACTIONS]; 126 uint64_t pkts_mask; 127 uint64_t n_pkts_ah_drop; 128 uint64_t pkts_drop_mask; 129 } __rte_cache_aligned; 130 131 static inline uint32_t 132 rte_mask_get_next(uint64_t mask, uint32_t pos) 133 { 134 uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) | 135 (mask >> ((pos + 1) & 0x3F)); 136 return (rte_ctz64(mask_rot) - (63 - pos)) & 0x3F; 137 } 138 139 static inline uint32_t 140 rte_mask_get_prev(uint64_t mask, uint32_t pos) 141 { 142 uint64_t mask_rot = (mask >> (pos & 0x3F)) | 143 (mask << ((64 - pos) & 0x3F)); 144 return ((63 - rte_clz64(mask_rot)) + pos) & 0x3F; 145 } 146 147 static void 148 rte_pipeline_table_free(struct rte_table *table); 149 150 static void 151 rte_pipeline_port_in_free(struct rte_port_in *port); 152 153 static void 154 rte_pipeline_port_out_free(struct rte_port_out *port); 155 156 /* 157 * Pipeline 158 */ 159 static int 160 rte_pipeline_check_params(struct rte_pipeline_params *params) 161 { 162 if (params == NULL) { 163 RTE_LOG(ERR, PIPELINE, 164 "%s: Incorrect value for parameter params\n", __func__); 165 return -EINVAL; 166 } 167 168 /* name */ 169 if (params->name == NULL) { 170 RTE_LOG(ERR, PIPELINE, 171 "%s: Incorrect value for parameter name\n", __func__); 172 return -EINVAL; 173 } 174 175 /* socket */ 176 if (params->socket_id < 0) { 177 RTE_LOG(ERR, PIPELINE, 178 "%s: Incorrect value for parameter socket_id\n", 179 __func__); 180 return -EINVAL; 181 } 182 183 return 0; 184 } 185 186 struct rte_pipeline * 187 rte_pipeline_create(struct rte_pipeline_params *params) 188 { 189 struct rte_pipeline *p; 190 int status; 191 192 /* Check input parameters */ 193 status = rte_pipeline_check_params(params); 194 if (status != 0) { 195 RTE_LOG(ERR, PIPELINE, 196 "%s: Pipeline params check failed (%d)\n", 197 __func__, status); 198 return NULL; 199 } 200 201 /* Allocate memory for the pipeline on requested socket */ 202 p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline), 203 RTE_CACHE_LINE_SIZE, params->socket_id); 204 205 if (p == NULL) { 206 RTE_LOG(ERR, PIPELINE, 207 "%s: Pipeline memory allocation failed\n", __func__); 208 return NULL; 209 } 210 211 /* Save input parameters */ 212 strlcpy(p->name, params->name, RTE_PIPELINE_MAX_NAME_SZ); 213 p->socket_id = params->socket_id; 214 p->offset_port_id = params->offset_port_id; 215 216 /* Initialize pipeline internal data structure */ 217 p->num_ports_in = 0; 218 p->num_ports_out = 0; 219 p->num_tables = 0; 220 p->enabled_port_in_mask = 0; 221 p->port_in_next = NULL; 222 p->pkts_mask = 0; 223 p->n_pkts_ah_drop = 0; 224 225 return p; 226 } 227 228 int 229 rte_pipeline_free(struct rte_pipeline *p) 230 { 231 uint32_t i; 232 233 /* Check input parameters */ 234 if (p == NULL) { 235 RTE_LOG(ERR, PIPELINE, 236 "%s: rte_pipeline parameter is NULL\n", __func__); 237 return -EINVAL; 238 } 239 240 /* Free input ports */ 241 for (i = 0; i < p->num_ports_in; i++) { 242 struct rte_port_in *port = &p->ports_in[i]; 243 244 rte_pipeline_port_in_free(port); 245 } 246 247 /* Free tables */ 248 for (i = 0; i < p->num_tables; i++) { 249 struct rte_table *table = &p->tables[i]; 250 251 rte_pipeline_table_free(table); 252 } 253 254 /* Free output ports */ 255 for (i = 0; i < p->num_ports_out; i++) { 256 struct rte_port_out *port = &p->ports_out[i]; 257 258 rte_pipeline_port_out_free(port); 259 } 260 261 /* Free pipeline memory */ 262 rte_free(p); 263 264 return 0; 265 } 266 267 /* 268 * Table 269 */ 270 static int 271 rte_table_check_params(struct rte_pipeline *p, 272 struct rte_pipeline_table_params *params, 273 uint32_t *table_id) 274 { 275 if (p == NULL) { 276 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", 277 __func__); 278 return -EINVAL; 279 } 280 if (params == NULL) { 281 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n", 282 __func__); 283 return -EINVAL; 284 } 285 if (table_id == NULL) { 286 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n", 287 __func__); 288 return -EINVAL; 289 } 290 291 /* ops */ 292 if (params->ops == NULL) { 293 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n", 294 __func__); 295 return -EINVAL; 296 } 297 298 if (params->ops->f_create == NULL) { 299 RTE_LOG(ERR, PIPELINE, 300 "%s: f_create function pointer is NULL\n", __func__); 301 return -EINVAL; 302 } 303 304 if (params->ops->f_lookup == NULL) { 305 RTE_LOG(ERR, PIPELINE, 306 "%s: f_lookup function pointer is NULL\n", __func__); 307 return -EINVAL; 308 } 309 310 /* De we have room for one more table? */ 311 if (p->num_tables == RTE_PIPELINE_TABLE_MAX) { 312 RTE_LOG(ERR, PIPELINE, 313 "%s: Incorrect value for num_tables parameter\n", 314 __func__); 315 return -EINVAL; 316 } 317 318 return 0; 319 } 320 321 int 322 rte_pipeline_table_create(struct rte_pipeline *p, 323 struct rte_pipeline_table_params *params, 324 uint32_t *table_id) 325 { 326 struct rte_table *table; 327 struct rte_pipeline_table_entry *default_entry; 328 void *h_table; 329 uint32_t entry_size, id; 330 int status; 331 332 /* Check input arguments */ 333 status = rte_table_check_params(p, params, table_id); 334 if (status != 0) 335 return status; 336 337 id = p->num_tables; 338 table = &p->tables[id]; 339 340 /* Allocate space for the default table entry */ 341 entry_size = sizeof(struct rte_pipeline_table_entry) + 342 params->action_data_size; 343 default_entry = rte_zmalloc_socket( 344 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id); 345 if (default_entry == NULL) { 346 RTE_LOG(ERR, PIPELINE, 347 "%s: Failed to allocate default entry\n", __func__); 348 return -EINVAL; 349 } 350 351 /* Create the table */ 352 h_table = params->ops->f_create(params->arg_create, p->socket_id, 353 entry_size); 354 if (h_table == NULL) { 355 rte_free(default_entry); 356 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__); 357 return -EINVAL; 358 } 359 360 /* Commit current table to the pipeline */ 361 p->num_tables++; 362 *table_id = id; 363 364 /* Save input parameters */ 365 memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops)); 366 table->f_action_hit = params->f_action_hit; 367 table->f_action_miss = params->f_action_miss; 368 table->arg_ah = params->arg_ah; 369 table->entry_size = entry_size; 370 371 /* Clear the lookup miss actions (to be set later through API) */ 372 table->default_entry = default_entry; 373 table->default_entry->action = RTE_PIPELINE_ACTION_DROP; 374 375 /* Initialize table internal data structure */ 376 table->h_table = h_table; 377 table->table_next_id = 0; 378 table->table_next_id_valid = 0; 379 380 return 0; 381 } 382 383 void 384 rte_pipeline_table_free(struct rte_table *table) 385 { 386 if (table->ops.f_free != NULL) 387 table->ops.f_free(table->h_table); 388 389 rte_free(table->default_entry); 390 } 391 392 int 393 rte_pipeline_table_default_entry_add(struct rte_pipeline *p, 394 uint32_t table_id, 395 struct rte_pipeline_table_entry *default_entry, 396 struct rte_pipeline_table_entry **default_entry_ptr) 397 { 398 struct rte_table *table; 399 400 /* Check input arguments */ 401 if (p == NULL) { 402 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", 403 __func__); 404 return -EINVAL; 405 } 406 407 if (default_entry == NULL) { 408 RTE_LOG(ERR, PIPELINE, 409 "%s: default_entry parameter is NULL\n", __func__); 410 return -EINVAL; 411 } 412 413 if (table_id >= p->num_tables) { 414 RTE_LOG(ERR, PIPELINE, 415 "%s: table_id %d out of range\n", __func__, table_id); 416 return -EINVAL; 417 } 418 419 table = &p->tables[table_id]; 420 421 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) && 422 table->table_next_id_valid && 423 (default_entry->table_id != table->table_next_id)) { 424 RTE_LOG(ERR, PIPELINE, 425 "%s: Tree-like topologies not allowed\n", __func__); 426 return -EINVAL; 427 } 428 429 /* Set the lookup miss actions */ 430 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) && 431 (table->table_next_id_valid == 0)) { 432 table->table_next_id = default_entry->table_id; 433 table->table_next_id_valid = 1; 434 } 435 436 memcpy(table->default_entry, default_entry, table->entry_size); 437 438 *default_entry_ptr = table->default_entry; 439 return 0; 440 } 441 442 int 443 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p, 444 uint32_t table_id, 445 struct rte_pipeline_table_entry *entry) 446 { 447 struct rte_table *table; 448 449 /* Check input arguments */ 450 if (p == NULL) { 451 RTE_LOG(ERR, PIPELINE, 452 "%s: pipeline parameter is NULL\n", __func__); 453 return -EINVAL; 454 } 455 456 if (table_id >= p->num_tables) { 457 RTE_LOG(ERR, PIPELINE, 458 "%s: table_id %d out of range\n", __func__, table_id); 459 return -EINVAL; 460 } 461 462 table = &p->tables[table_id]; 463 464 /* Save the current contents of the default entry */ 465 if (entry) 466 memcpy(entry, table->default_entry, table->entry_size); 467 468 /* Clear the lookup miss actions */ 469 memset(table->default_entry, 0, table->entry_size); 470 table->default_entry->action = RTE_PIPELINE_ACTION_DROP; 471 472 return 0; 473 } 474 475 int 476 rte_pipeline_table_entry_add(struct rte_pipeline *p, 477 uint32_t table_id, 478 void *key, 479 struct rte_pipeline_table_entry *entry, 480 int *key_found, 481 struct rte_pipeline_table_entry **entry_ptr) 482 { 483 struct rte_table *table; 484 485 /* Check input arguments */ 486 if (p == NULL) { 487 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", 488 __func__); 489 return -EINVAL; 490 } 491 492 if (key == NULL) { 493 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__); 494 return -EINVAL; 495 } 496 497 if (entry == NULL) { 498 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n", 499 __func__); 500 return -EINVAL; 501 } 502 503 if (table_id >= p->num_tables) { 504 RTE_LOG(ERR, PIPELINE, 505 "%s: table_id %d out of range\n", __func__, table_id); 506 return -EINVAL; 507 } 508 509 table = &p->tables[table_id]; 510 511 if (table->ops.f_add == NULL) { 512 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n", 513 __func__); 514 return -EINVAL; 515 } 516 517 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) && 518 table->table_next_id_valid && 519 (entry->table_id != table->table_next_id)) { 520 RTE_LOG(ERR, PIPELINE, 521 "%s: Tree-like topologies not allowed\n", __func__); 522 return -EINVAL; 523 } 524 525 /* Add entry */ 526 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) && 527 (table->table_next_id_valid == 0)) { 528 table->table_next_id = entry->table_id; 529 table->table_next_id_valid = 1; 530 } 531 532 return (table->ops.f_add)(table->h_table, key, (void *) entry, 533 key_found, (void **) entry_ptr); 534 } 535 536 int 537 rte_pipeline_table_entry_delete(struct rte_pipeline *p, 538 uint32_t table_id, 539 void *key, 540 int *key_found, 541 struct rte_pipeline_table_entry *entry) 542 { 543 struct rte_table *table; 544 545 /* Check input arguments */ 546 if (p == NULL) { 547 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 548 __func__); 549 return -EINVAL; 550 } 551 552 if (key == NULL) { 553 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", 554 __func__); 555 return -EINVAL; 556 } 557 558 if (table_id >= p->num_tables) { 559 RTE_LOG(ERR, PIPELINE, 560 "%s: table_id %d out of range\n", __func__, table_id); 561 return -EINVAL; 562 } 563 564 table = &p->tables[table_id]; 565 566 if (table->ops.f_delete == NULL) { 567 RTE_LOG(ERR, PIPELINE, 568 "%s: f_delete function pointer NULL\n", __func__); 569 return -EINVAL; 570 } 571 572 return (table->ops.f_delete)(table->h_table, key, key_found, entry); 573 } 574 575 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p, 576 uint32_t table_id, 577 void **keys, 578 struct rte_pipeline_table_entry **entries, 579 uint32_t n_keys, 580 int *key_found, 581 struct rte_pipeline_table_entry **entries_ptr) 582 { 583 struct rte_table *table; 584 uint32_t i; 585 586 /* Check input arguments */ 587 if (p == NULL) { 588 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", 589 __func__); 590 return -EINVAL; 591 } 592 593 if (keys == NULL) { 594 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__); 595 return -EINVAL; 596 } 597 598 if (entries == NULL) { 599 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n", 600 __func__); 601 return -EINVAL; 602 } 603 604 if (table_id >= p->num_tables) { 605 RTE_LOG(ERR, PIPELINE, 606 "%s: table_id %d out of range\n", __func__, table_id); 607 return -EINVAL; 608 } 609 610 table = &p->tables[table_id]; 611 612 if (table->ops.f_add_bulk == NULL) { 613 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n", 614 __func__); 615 return -EINVAL; 616 } 617 618 for (i = 0; i < n_keys; i++) { 619 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) && 620 table->table_next_id_valid && 621 (entries[i]->table_id != table->table_next_id)) { 622 RTE_LOG(ERR, PIPELINE, 623 "%s: Tree-like topologies not allowed\n", __func__); 624 return -EINVAL; 625 } 626 } 627 628 /* Add entry */ 629 for (i = 0; i < n_keys; i++) { 630 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) && 631 (table->table_next_id_valid == 0)) { 632 table->table_next_id = entries[i]->table_id; 633 table->table_next_id_valid = 1; 634 } 635 } 636 637 return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries, 638 n_keys, key_found, (void **) entries_ptr); 639 } 640 641 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p, 642 uint32_t table_id, 643 void **keys, 644 uint32_t n_keys, 645 int *key_found, 646 struct rte_pipeline_table_entry **entries) 647 { 648 struct rte_table *table; 649 650 /* Check input arguments */ 651 if (p == NULL) { 652 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 653 __func__); 654 return -EINVAL; 655 } 656 657 if (keys == NULL) { 658 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", 659 __func__); 660 return -EINVAL; 661 } 662 663 if (table_id >= p->num_tables) { 664 RTE_LOG(ERR, PIPELINE, 665 "%s: table_id %d out of range\n", __func__, table_id); 666 return -EINVAL; 667 } 668 669 table = &p->tables[table_id]; 670 671 if (table->ops.f_delete_bulk == NULL) { 672 RTE_LOG(ERR, PIPELINE, 673 "%s: f_delete function pointer NULL\n", __func__); 674 return -EINVAL; 675 } 676 677 return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found, 678 (void **) entries); 679 } 680 681 /* 682 * Port 683 */ 684 static int 685 rte_pipeline_port_in_check_params(struct rte_pipeline *p, 686 struct rte_pipeline_port_in_params *params, 687 uint32_t *port_id) 688 { 689 if (p == NULL) { 690 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 691 __func__); 692 return -EINVAL; 693 } 694 if (params == NULL) { 695 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__); 696 return -EINVAL; 697 } 698 if (port_id == NULL) { 699 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n", 700 __func__); 701 return -EINVAL; 702 } 703 704 /* ops */ 705 if (params->ops == NULL) { 706 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n", 707 __func__); 708 return -EINVAL; 709 } 710 711 if (params->ops->f_create == NULL) { 712 RTE_LOG(ERR, PIPELINE, 713 "%s: f_create function pointer NULL\n", __func__); 714 return -EINVAL; 715 } 716 717 if (params->ops->f_rx == NULL) { 718 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n", 719 __func__); 720 return -EINVAL; 721 } 722 723 /* burst_size */ 724 if ((params->burst_size == 0) || 725 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) { 726 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n", 727 __func__); 728 return -EINVAL; 729 } 730 731 /* Do we have room for one more port? */ 732 if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) { 733 RTE_LOG(ERR, PIPELINE, 734 "%s: invalid value for num_ports_in\n", __func__); 735 return -EINVAL; 736 } 737 738 return 0; 739 } 740 741 static int 742 rte_pipeline_port_out_check_params(struct rte_pipeline *p, 743 struct rte_pipeline_port_out_params *params, 744 uint32_t *port_id) 745 { 746 if (p == NULL) { 747 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 748 __func__); 749 return -EINVAL; 750 } 751 752 if (params == NULL) { 753 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__); 754 return -EINVAL; 755 } 756 757 if (port_id == NULL) { 758 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n", 759 __func__); 760 return -EINVAL; 761 } 762 763 /* ops */ 764 if (params->ops == NULL) { 765 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n", 766 __func__); 767 return -EINVAL; 768 } 769 770 if (params->ops->f_create == NULL) { 771 RTE_LOG(ERR, PIPELINE, 772 "%s: f_create function pointer NULL\n", __func__); 773 return -EINVAL; 774 } 775 776 if (params->ops->f_tx == NULL) { 777 RTE_LOG(ERR, PIPELINE, 778 "%s: f_tx function pointer NULL\n", __func__); 779 return -EINVAL; 780 } 781 782 if (params->ops->f_tx_bulk == NULL) { 783 RTE_LOG(ERR, PIPELINE, 784 "%s: f_tx_bulk function pointer NULL\n", __func__); 785 return -EINVAL; 786 } 787 788 /* Do we have room for one more port? */ 789 if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) { 790 RTE_LOG(ERR, PIPELINE, 791 "%s: invalid value for num_ports_out\n", __func__); 792 return -EINVAL; 793 } 794 795 return 0; 796 } 797 798 int 799 rte_pipeline_port_in_create(struct rte_pipeline *p, 800 struct rte_pipeline_port_in_params *params, 801 uint32_t *port_id) 802 { 803 struct rte_port_in *port; 804 void *h_port; 805 uint32_t id; 806 int status; 807 808 /* Check input arguments */ 809 status = rte_pipeline_port_in_check_params(p, params, port_id); 810 if (status != 0) 811 return status; 812 813 id = p->num_ports_in; 814 port = &p->ports_in[id]; 815 816 /* Create the port */ 817 h_port = params->ops->f_create(params->arg_create, p->socket_id); 818 if (h_port == NULL) { 819 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__); 820 return -EINVAL; 821 } 822 823 /* Commit current table to the pipeline */ 824 p->num_ports_in++; 825 *port_id = id; 826 827 /* Save input parameters */ 828 memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops)); 829 port->f_action = params->f_action; 830 port->arg_ah = params->arg_ah; 831 port->burst_size = params->burst_size; 832 833 /* Initialize port internal data structure */ 834 port->table_id = RTE_TABLE_INVALID; 835 port->h_port = h_port; 836 port->next = NULL; 837 838 return 0; 839 } 840 841 void 842 rte_pipeline_port_in_free(struct rte_port_in *port) 843 { 844 if (port->ops.f_free != NULL) 845 port->ops.f_free(port->h_port); 846 } 847 848 int 849 rte_pipeline_port_out_create(struct rte_pipeline *p, 850 struct rte_pipeline_port_out_params *params, 851 uint32_t *port_id) 852 { 853 struct rte_port_out *port; 854 void *h_port; 855 uint32_t id; 856 int status; 857 858 /* Check input arguments */ 859 status = rte_pipeline_port_out_check_params(p, params, port_id); 860 if (status != 0) 861 return status; 862 863 id = p->num_ports_out; 864 port = &p->ports_out[id]; 865 866 /* Create the port */ 867 h_port = params->ops->f_create(params->arg_create, p->socket_id); 868 if (h_port == NULL) { 869 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__); 870 return -EINVAL; 871 } 872 873 /* Commit current table to the pipeline */ 874 p->num_ports_out++; 875 *port_id = id; 876 877 /* Save input parameters */ 878 memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops)); 879 port->f_action = params->f_action; 880 port->arg_ah = params->arg_ah; 881 882 /* Initialize port internal data structure */ 883 port->h_port = h_port; 884 885 return 0; 886 } 887 888 void 889 rte_pipeline_port_out_free(struct rte_port_out *port) 890 { 891 if (port->ops.f_free != NULL) 892 port->ops.f_free(port->h_port); 893 } 894 895 int 896 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p, 897 uint32_t port_id, 898 uint32_t table_id) 899 { 900 struct rte_port_in *port; 901 902 /* Check input arguments */ 903 if (p == NULL) { 904 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 905 __func__); 906 return -EINVAL; 907 } 908 909 if (port_id >= p->num_ports_in) { 910 RTE_LOG(ERR, PIPELINE, 911 "%s: port IN ID %u is out of range\n", 912 __func__, port_id); 913 return -EINVAL; 914 } 915 916 if (table_id >= p->num_tables) { 917 RTE_LOG(ERR, PIPELINE, 918 "%s: Table ID %u is out of range\n", 919 __func__, table_id); 920 return -EINVAL; 921 } 922 923 port = &p->ports_in[port_id]; 924 port->table_id = table_id; 925 926 return 0; 927 } 928 929 int 930 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id) 931 { 932 struct rte_port_in *port, *port_prev, *port_next; 933 uint64_t port_mask; 934 uint32_t port_prev_id, port_next_id; 935 936 /* Check input arguments */ 937 if (p == NULL) { 938 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 939 __func__); 940 return -EINVAL; 941 } 942 943 if (port_id >= p->num_ports_in) { 944 RTE_LOG(ERR, PIPELINE, 945 "%s: port IN ID %u is out of range\n", 946 __func__, port_id); 947 return -EINVAL; 948 } 949 950 port = &p->ports_in[port_id]; 951 952 /* Return if current input port is already enabled */ 953 port_mask = 1LLU << port_id; 954 if (p->enabled_port_in_mask & port_mask) 955 return 0; 956 957 p->enabled_port_in_mask |= port_mask; 958 959 /* Add current input port to the pipeline chain of enabled ports */ 960 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id); 961 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id); 962 963 port_prev = &p->ports_in[port_prev_id]; 964 port_next = &p->ports_in[port_next_id]; 965 966 port_prev->next = port; 967 port->next = port_next; 968 969 /* Check if list of enabled ports was previously empty */ 970 if (p->enabled_port_in_mask == port_mask) 971 p->port_in_next = port; 972 973 return 0; 974 } 975 976 int 977 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id) 978 { 979 struct rte_port_in *port, *port_prev, *port_next; 980 uint64_t port_mask; 981 uint32_t port_prev_id, port_next_id; 982 983 /* Check input arguments */ 984 if (p == NULL) { 985 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 986 __func__); 987 return -EINVAL; 988 } 989 990 if (port_id >= p->num_ports_in) { 991 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n", 992 __func__, port_id); 993 return -EINVAL; 994 } 995 996 port = &p->ports_in[port_id]; 997 998 /* Return if current input port is already disabled */ 999 port_mask = 1LLU << port_id; 1000 if ((p->enabled_port_in_mask & port_mask) == 0) 1001 return 0; 1002 1003 p->enabled_port_in_mask &= ~port_mask; 1004 1005 /* Return if no other enabled ports */ 1006 if (p->enabled_port_in_mask == 0) { 1007 p->port_in_next = NULL; 1008 1009 return 0; 1010 } 1011 1012 /* Add current input port to the pipeline chain of enabled ports */ 1013 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id); 1014 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id); 1015 1016 port_prev = &p->ports_in[port_prev_id]; 1017 port_next = &p->ports_in[port_next_id]; 1018 1019 port_prev->next = port_next; 1020 1021 /* Check if the port which has just been disabled is next to serve */ 1022 if (port == p->port_in_next) 1023 p->port_in_next = port_next; 1024 1025 return 0; 1026 } 1027 1028 /* 1029 * Pipeline run-time 1030 */ 1031 int 1032 rte_pipeline_check(struct rte_pipeline *p) 1033 { 1034 uint32_t port_in_id; 1035 1036 /* Check input arguments */ 1037 if (p == NULL) { 1038 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 1039 __func__); 1040 return -EINVAL; 1041 } 1042 1043 /* Check that pipeline has at least one input port, one table and one 1044 output port */ 1045 if (p->num_ports_in == 0) { 1046 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n", 1047 __func__); 1048 return -EINVAL; 1049 } 1050 if (p->num_tables == 0) { 1051 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n", 1052 __func__); 1053 return -EINVAL; 1054 } 1055 if (p->num_ports_out == 0) { 1056 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n", 1057 __func__); 1058 return -EINVAL; 1059 } 1060 1061 /* Check that all input ports are connected */ 1062 for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) { 1063 struct rte_port_in *port_in = &p->ports_in[port_in_id]; 1064 1065 if (port_in->table_id == RTE_TABLE_INVALID) { 1066 RTE_LOG(ERR, PIPELINE, 1067 "%s: Port IN ID %u is not connected\n", 1068 __func__, port_in_id); 1069 return -EINVAL; 1070 } 1071 } 1072 1073 return 0; 1074 } 1075 1076 static inline void 1077 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask) 1078 { 1079 p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0; 1080 p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0; 1081 p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0; 1082 p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0; 1083 1084 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1085 uint64_t n_pkts = rte_popcount64(pkts_mask); 1086 uint32_t i; 1087 1088 for (i = 0; i < n_pkts; i++) { 1089 uint64_t pkt_mask = 1LLU << i; 1090 uint32_t pos = p->entries[i]->action; 1091 1092 p->action_mask1[pos] |= pkt_mask; 1093 } 1094 } else { 1095 uint32_t i; 1096 1097 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1098 uint64_t pkt_mask = 1LLU << i; 1099 uint32_t pos; 1100 1101 if ((pkt_mask & pkts_mask) == 0) 1102 continue; 1103 1104 pos = p->entries[i]->action; 1105 p->action_mask1[pos] |= pkt_mask; 1106 } 1107 } 1108 } 1109 1110 static inline void 1111 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p, 1112 uint64_t pkts_mask, uint32_t port_id) 1113 { 1114 struct rte_port_out *port_out = &p->ports_out[port_id]; 1115 1116 p->pkts_mask = pkts_mask; 1117 1118 /* Output port user actions */ 1119 if (port_out->f_action != NULL) { 1120 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah); 1121 1122 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1123 port_out->n_pkts_dropped_by_ah); 1124 } 1125 1126 /* Output port TX */ 1127 if (p->pkts_mask != 0) 1128 port_out->ops.f_tx_bulk(port_out->h_port, 1129 p->pkts, 1130 p->pkts_mask); 1131 } 1132 1133 static inline void 1134 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask) 1135 { 1136 p->pkts_mask = pkts_mask; 1137 1138 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1139 uint64_t n_pkts = rte_popcount64(pkts_mask); 1140 uint32_t i; 1141 1142 for (i = 0; i < n_pkts; i++) { 1143 struct rte_mbuf *pkt = p->pkts[i]; 1144 uint32_t port_out_id = p->entries[i]->port_id; 1145 struct rte_port_out *port_out = 1146 &p->ports_out[port_out_id]; 1147 1148 /* Output port user actions */ 1149 if (port_out->f_action == NULL) /* Output port TX */ 1150 port_out->ops.f_tx(port_out->h_port, pkt); 1151 else { 1152 uint64_t pkt_mask = 1LLU << i; 1153 1154 port_out->f_action(p, 1155 p->pkts, 1156 pkt_mask, 1157 port_out->arg_ah); 1158 1159 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1160 port_out->n_pkts_dropped_by_ah); 1161 1162 /* Output port TX */ 1163 if (pkt_mask & p->pkts_mask) 1164 port_out->ops.f_tx(port_out->h_port, 1165 pkt); 1166 } 1167 } 1168 } else { 1169 uint32_t i; 1170 1171 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1172 uint64_t pkt_mask = 1LLU << i; 1173 struct rte_mbuf *pkt; 1174 struct rte_port_out *port_out; 1175 uint32_t port_out_id; 1176 1177 if ((pkt_mask & pkts_mask) == 0) 1178 continue; 1179 1180 pkt = p->pkts[i]; 1181 port_out_id = p->entries[i]->port_id; 1182 port_out = &p->ports_out[port_out_id]; 1183 1184 /* Output port user actions */ 1185 if (port_out->f_action == NULL) /* Output port TX */ 1186 port_out->ops.f_tx(port_out->h_port, pkt); 1187 else { 1188 port_out->f_action(p, 1189 p->pkts, 1190 pkt_mask, 1191 port_out->arg_ah); 1192 1193 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1194 port_out->n_pkts_dropped_by_ah); 1195 1196 /* Output port TX */ 1197 if (pkt_mask & p->pkts_mask) 1198 port_out->ops.f_tx(port_out->h_port, 1199 pkt); 1200 } 1201 } 1202 } 1203 } 1204 1205 static inline void 1206 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p, 1207 uint64_t pkts_mask) 1208 { 1209 p->pkts_mask = pkts_mask; 1210 1211 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1212 uint64_t n_pkts = rte_popcount64(pkts_mask); 1213 uint32_t i; 1214 1215 for (i = 0; i < n_pkts; i++) { 1216 struct rte_mbuf *pkt = p->pkts[i]; 1217 uint32_t port_out_id = 1218 RTE_MBUF_METADATA_UINT32(pkt, 1219 p->offset_port_id); 1220 struct rte_port_out *port_out = &p->ports_out[ 1221 port_out_id]; 1222 1223 /* Output port user actions */ 1224 if (port_out->f_action == NULL) /* Output port TX */ 1225 port_out->ops.f_tx(port_out->h_port, pkt); 1226 else { 1227 uint64_t pkt_mask = 1LLU << i; 1228 1229 port_out->f_action(p, 1230 p->pkts, 1231 pkt_mask, 1232 port_out->arg_ah); 1233 1234 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1235 port_out->n_pkts_dropped_by_ah); 1236 1237 /* Output port TX */ 1238 if (pkt_mask & p->pkts_mask) 1239 port_out->ops.f_tx(port_out->h_port, 1240 pkt); 1241 } 1242 } 1243 } else { 1244 uint32_t i; 1245 1246 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1247 uint64_t pkt_mask = 1LLU << i; 1248 struct rte_mbuf *pkt; 1249 struct rte_port_out *port_out; 1250 uint32_t port_out_id; 1251 1252 if ((pkt_mask & pkts_mask) == 0) 1253 continue; 1254 1255 pkt = p->pkts[i]; 1256 port_out_id = RTE_MBUF_METADATA_UINT32(pkt, 1257 p->offset_port_id); 1258 port_out = &p->ports_out[port_out_id]; 1259 1260 /* Output port user actions */ 1261 if (port_out->f_action == NULL) /* Output port TX */ 1262 port_out->ops.f_tx(port_out->h_port, pkt); 1263 else { 1264 port_out->f_action(p, 1265 p->pkts, 1266 pkt_mask, 1267 port_out->arg_ah); 1268 1269 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1270 port_out->n_pkts_dropped_by_ah); 1271 1272 /* Output port TX */ 1273 if (pkt_mask & p->pkts_mask) 1274 port_out->ops.f_tx(port_out->h_port, 1275 pkt); 1276 } 1277 } 1278 } 1279 } 1280 1281 static inline void 1282 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask) 1283 { 1284 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1285 uint64_t n_pkts = rte_popcount64(pkts_mask); 1286 uint32_t i; 1287 1288 for (i = 0; i < n_pkts; i++) 1289 rte_pktmbuf_free(p->pkts[i]); 1290 } else { 1291 uint32_t i; 1292 1293 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1294 uint64_t pkt_mask = 1LLU << i; 1295 1296 if ((pkt_mask & pkts_mask) == 0) 1297 continue; 1298 1299 rte_pktmbuf_free(p->pkts[i]); 1300 } 1301 } 1302 } 1303 1304 int 1305 rte_pipeline_run(struct rte_pipeline *p) 1306 { 1307 struct rte_port_in *port_in = p->port_in_next; 1308 uint32_t n_pkts, table_id; 1309 1310 if (port_in == NULL) 1311 return 0; 1312 1313 /* Input port RX */ 1314 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts, 1315 port_in->burst_size); 1316 if (n_pkts == 0) { 1317 p->port_in_next = port_in->next; 1318 return 0; 1319 } 1320 1321 p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t); 1322 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0; 1323 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0; 1324 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0; 1325 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; 1326 1327 /* Input port user actions */ 1328 if (port_in->f_action != NULL) { 1329 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah); 1330 1331 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1332 port_in->n_pkts_dropped_by_ah); 1333 } 1334 1335 /* Table */ 1336 for (table_id = port_in->table_id; p->pkts_mask != 0; ) { 1337 struct rte_table *table; 1338 uint64_t lookup_hit_mask, lookup_miss_mask; 1339 1340 /* Lookup */ 1341 table = &p->tables[table_id]; 1342 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask, 1343 &lookup_hit_mask, (void **) p->entries); 1344 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask); 1345 1346 /* Lookup miss */ 1347 if (lookup_miss_mask != 0) { 1348 struct rte_pipeline_table_entry *default_entry = 1349 table->default_entry; 1350 1351 p->pkts_mask = lookup_miss_mask; 1352 1353 /* Table user actions */ 1354 if (table->f_action_miss != NULL) { 1355 table->f_action_miss(p, 1356 p->pkts, 1357 lookup_miss_mask, 1358 default_entry, 1359 table->arg_ah); 1360 1361 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1362 table->n_pkts_dropped_by_lkp_miss_ah); 1363 } 1364 1365 /* Table reserved actions */ 1366 if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) && 1367 (p->pkts_mask != 0)) 1368 rte_pipeline_action_handler_port_bulk(p, 1369 p->pkts_mask, 1370 default_entry->port_id); 1371 else { 1372 uint32_t pos = default_entry->action; 1373 1374 RTE_PIPELINE_STATS_TABLE_DROP0(p); 1375 1376 p->action_mask0[pos] |= p->pkts_mask; 1377 1378 RTE_PIPELINE_STATS_TABLE_DROP1(p, 1379 table->n_pkts_dropped_lkp_miss); 1380 } 1381 } 1382 1383 /* Lookup hit */ 1384 if (lookup_hit_mask != 0) { 1385 p->pkts_mask = lookup_hit_mask; 1386 1387 /* Table user actions */ 1388 if (table->f_action_hit != NULL) { 1389 table->f_action_hit(p, 1390 p->pkts, 1391 lookup_hit_mask, 1392 p->entries, 1393 table->arg_ah); 1394 1395 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1396 table->n_pkts_dropped_by_lkp_hit_ah); 1397 } 1398 1399 /* Table reserved actions */ 1400 RTE_PIPELINE_STATS_TABLE_DROP0(p); 1401 rte_pipeline_compute_masks(p, p->pkts_mask); 1402 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= 1403 p->action_mask1[ 1404 RTE_PIPELINE_ACTION_DROP]; 1405 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |= 1406 p->action_mask1[ 1407 RTE_PIPELINE_ACTION_PORT]; 1408 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |= 1409 p->action_mask1[ 1410 RTE_PIPELINE_ACTION_PORT_META]; 1411 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |= 1412 p->action_mask1[ 1413 RTE_PIPELINE_ACTION_TABLE]; 1414 1415 RTE_PIPELINE_STATS_TABLE_DROP1(p, 1416 table->n_pkts_dropped_lkp_hit); 1417 } 1418 1419 /* Prepare for next iteration */ 1420 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE]; 1421 table_id = table->table_next_id; 1422 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; 1423 } 1424 1425 /* Table reserved action PORT */ 1426 rte_pipeline_action_handler_port(p, 1427 p->action_mask0[RTE_PIPELINE_ACTION_PORT]); 1428 1429 /* Table reserved action PORT META */ 1430 rte_pipeline_action_handler_port_meta(p, 1431 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]); 1432 1433 /* Table reserved action DROP */ 1434 rte_pipeline_action_handler_drop(p, 1435 p->action_mask0[RTE_PIPELINE_ACTION_DROP]); 1436 1437 /* Pick candidate for next port IN to serve */ 1438 p->port_in_next = port_in->next; 1439 1440 return (int) n_pkts; 1441 } 1442 1443 int 1444 rte_pipeline_flush(struct rte_pipeline *p) 1445 { 1446 uint32_t port_id; 1447 1448 /* Check input arguments */ 1449 if (p == NULL) { 1450 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 1451 __func__); 1452 return -EINVAL; 1453 } 1454 1455 for (port_id = 0; port_id < p->num_ports_out; port_id++) { 1456 struct rte_port_out *port = &p->ports_out[port_id]; 1457 1458 if (port->ops.f_flush != NULL) 1459 port->ops.f_flush(port->h_port); 1460 } 1461 1462 return 0; 1463 } 1464 1465 int 1466 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p, 1467 uint32_t port_id, struct rte_mbuf *pkt) 1468 { 1469 struct rte_port_out *port_out = &p->ports_out[port_id]; 1470 1471 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */ 1472 1473 return 0; 1474 } 1475 1476 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p, 1477 uint64_t pkts_mask) 1478 { 1479 pkts_mask &= p->pkts_mask; 1480 p->pkts_mask &= ~pkts_mask; 1481 1482 return 0; 1483 } 1484 1485 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p, 1486 uint64_t pkts_mask) 1487 { 1488 pkts_mask &= p->pkts_mask; 1489 p->pkts_mask &= ~pkts_mask; 1490 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask; 1491 1492 RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask); 1493 return 0; 1494 } 1495 1496 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id, 1497 struct rte_pipeline_port_in_stats *stats, int clear) 1498 { 1499 struct rte_port_in *port; 1500 int retval; 1501 1502 if (p == NULL) { 1503 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 1504 __func__); 1505 return -EINVAL; 1506 } 1507 1508 if (port_id >= p->num_ports_in) { 1509 RTE_LOG(ERR, PIPELINE, 1510 "%s: port IN ID %u is out of range\n", 1511 __func__, port_id); 1512 return -EINVAL; 1513 } 1514 1515 port = &p->ports_in[port_id]; 1516 1517 if (port->ops.f_stats != NULL) { 1518 retval = port->ops.f_stats(port->h_port, &stats->stats, clear); 1519 if (retval) 1520 return retval; 1521 } else if (stats != NULL) 1522 memset(&stats->stats, 0, sizeof(stats->stats)); 1523 1524 if (stats != NULL) 1525 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah; 1526 1527 if (clear != 0) 1528 port->n_pkts_dropped_by_ah = 0; 1529 1530 return 0; 1531 } 1532 1533 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id, 1534 struct rte_pipeline_port_out_stats *stats, int clear) 1535 { 1536 struct rte_port_out *port; 1537 int retval; 1538 1539 if (p == NULL) { 1540 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__); 1541 return -EINVAL; 1542 } 1543 1544 if (port_id >= p->num_ports_out) { 1545 RTE_LOG(ERR, PIPELINE, 1546 "%s: port OUT ID %u is out of range\n", __func__, port_id); 1547 return -EINVAL; 1548 } 1549 1550 port = &p->ports_out[port_id]; 1551 if (port->ops.f_stats != NULL) { 1552 retval = port->ops.f_stats(port->h_port, &stats->stats, clear); 1553 if (retval != 0) 1554 return retval; 1555 } else if (stats != NULL) 1556 memset(&stats->stats, 0, sizeof(stats->stats)); 1557 1558 if (stats != NULL) 1559 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah; 1560 1561 if (clear != 0) 1562 port->n_pkts_dropped_by_ah = 0; 1563 1564 return 0; 1565 } 1566 1567 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id, 1568 struct rte_pipeline_table_stats *stats, int clear) 1569 { 1570 struct rte_table *table; 1571 int retval; 1572 1573 if (p == NULL) { 1574 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 1575 __func__); 1576 return -EINVAL; 1577 } 1578 1579 if (table_id >= p->num_tables) { 1580 RTE_LOG(ERR, PIPELINE, 1581 "%s: table %u is out of range\n", __func__, table_id); 1582 return -EINVAL; 1583 } 1584 1585 table = &p->tables[table_id]; 1586 if (table->ops.f_stats != NULL) { 1587 retval = table->ops.f_stats(table->h_table, &stats->stats, clear); 1588 if (retval != 0) 1589 return retval; 1590 } else if (stats != NULL) 1591 memset(&stats->stats, 0, sizeof(stats->stats)); 1592 1593 if (stats != NULL) { 1594 stats->n_pkts_dropped_by_lkp_hit_ah = 1595 table->n_pkts_dropped_by_lkp_hit_ah; 1596 stats->n_pkts_dropped_by_lkp_miss_ah = 1597 table->n_pkts_dropped_by_lkp_miss_ah; 1598 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit; 1599 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss; 1600 } 1601 1602 if (clear != 0) { 1603 table->n_pkts_dropped_by_lkp_hit_ah = 0; 1604 table->n_pkts_dropped_by_lkp_miss_ah = 0; 1605 table->n_pkts_dropped_lkp_hit = 0; 1606 table->n_pkts_dropped_lkp_miss = 0; 1607 } 1608 1609 return 0; 1610 } 1611