1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <string.h> 6 #include <stdio.h> 7 8 #include <rte_common.h> 9 #include <rte_memory.h> 10 #include <rte_cycles.h> 11 #include <rte_prefetch.h> 12 #include <rte_branch_prediction.h> 13 #include <rte_mbuf.h> 14 #include <rte_malloc.h> 15 #include <rte_string_fns.h> 16 17 #include "rte_pipeline.h" 18 19 #define RTE_TABLE_INVALID UINT32_MAX 20 21 #ifdef RTE_PIPELINE_STATS_COLLECT 22 23 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) \ 24 ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); }) 25 26 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) \ 27 ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; }) 28 29 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) \ 30 ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; }) 31 32 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) \ 33 ({ \ 34 uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; \ 35 mask ^= (p)->pkts_drop_mask; \ 36 (counter) += __builtin_popcountll(mask); \ 37 }) 38 39 #else 40 41 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) 42 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) 43 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) 44 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) 45 46 #endif 47 48 struct rte_port_in { 49 /* Input parameters */ 50 struct rte_port_in_ops ops; 51 rte_pipeline_port_in_action_handler f_action; 52 void *arg_ah; 53 uint32_t burst_size; 54 55 /* The table to which this port is connected */ 56 uint32_t table_id; 57 58 /* Handle to low-level port */ 59 void *h_port; 60 61 /* List of enabled ports */ 62 struct rte_port_in *next; 63 64 /* Statistics */ 65 uint64_t n_pkts_dropped_by_ah; 66 }; 67 68 struct rte_port_out { 69 /* Input parameters */ 70 struct rte_port_out_ops ops; 71 rte_pipeline_port_out_action_handler f_action; 72 void *arg_ah; 73 74 /* Handle to low-level port */ 75 void *h_port; 76 77 /* Statistics */ 78 uint64_t n_pkts_dropped_by_ah; 79 }; 80 81 struct rte_table { 82 /* Input parameters */ 83 struct rte_table_ops ops; 84 rte_pipeline_table_action_handler_hit f_action_hit; 85 rte_pipeline_table_action_handler_miss f_action_miss; 86 void *arg_ah; 87 struct rte_pipeline_table_entry *default_entry; 88 uint32_t entry_size; 89 90 uint32_t table_next_id; 91 uint32_t table_next_id_valid; 92 93 /* Handle to the low-level table object */ 94 void *h_table; 95 96 /* Statistics */ 97 uint64_t n_pkts_dropped_by_lkp_hit_ah; 98 uint64_t n_pkts_dropped_by_lkp_miss_ah; 99 uint64_t n_pkts_dropped_lkp_hit; 100 uint64_t n_pkts_dropped_lkp_miss; 101 }; 102 103 #define RTE_PIPELINE_MAX_NAME_SZ 124 104 105 struct rte_pipeline { 106 /* Input parameters */ 107 char name[RTE_PIPELINE_MAX_NAME_SZ]; 108 int socket_id; 109 uint32_t offset_port_id; 110 111 /* Internal tables */ 112 struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX]; 113 struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX]; 114 struct rte_table tables[RTE_PIPELINE_TABLE_MAX]; 115 116 /* Occupancy of internal tables */ 117 uint32_t num_ports_in; 118 uint32_t num_ports_out; 119 uint32_t num_tables; 120 121 /* List of enabled ports */ 122 uint64_t enabled_port_in_mask; 123 struct rte_port_in *port_in_next; 124 125 /* Pipeline run structures */ 126 struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX]; 127 struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX]; 128 uint64_t action_mask0[RTE_PIPELINE_ACTIONS]; 129 uint64_t action_mask1[RTE_PIPELINE_ACTIONS]; 130 uint64_t pkts_mask; 131 uint64_t n_pkts_ah_drop; 132 uint64_t pkts_drop_mask; 133 } __rte_cache_aligned; 134 135 static inline uint32_t 136 rte_mask_get_next(uint64_t mask, uint32_t pos) 137 { 138 uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) | 139 (mask >> ((pos + 1) & 0x3F)); 140 return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F; 141 } 142 143 static inline uint32_t 144 rte_mask_get_prev(uint64_t mask, uint32_t pos) 145 { 146 uint64_t mask_rot = (mask >> (pos & 0x3F)) | 147 (mask << ((64 - pos) & 0x3F)); 148 return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F; 149 } 150 151 static void 152 rte_pipeline_table_free(struct rte_table *table); 153 154 static void 155 rte_pipeline_port_in_free(struct rte_port_in *port); 156 157 static void 158 rte_pipeline_port_out_free(struct rte_port_out *port); 159 160 /* 161 * Pipeline 162 * 163 */ 164 static int 165 rte_pipeline_check_params(struct rte_pipeline_params *params) 166 { 167 if (params == NULL) { 168 RTE_LOG(ERR, PIPELINE, 169 "%s: Incorrect value for parameter params\n", __func__); 170 return -EINVAL; 171 } 172 173 /* name */ 174 if (params->name == NULL) { 175 RTE_LOG(ERR, PIPELINE, 176 "%s: Incorrect value for parameter name\n", __func__); 177 return -EINVAL; 178 } 179 180 /* socket */ 181 if (params->socket_id < 0) { 182 RTE_LOG(ERR, PIPELINE, 183 "%s: Incorrect value for parameter socket_id\n", 184 __func__); 185 return -EINVAL; 186 } 187 188 return 0; 189 } 190 191 struct rte_pipeline * 192 rte_pipeline_create(struct rte_pipeline_params *params) 193 { 194 struct rte_pipeline *p; 195 int status; 196 197 /* Check input parameters */ 198 status = rte_pipeline_check_params(params); 199 if (status != 0) { 200 RTE_LOG(ERR, PIPELINE, 201 "%s: Pipeline params check failed (%d)\n", 202 __func__, status); 203 return NULL; 204 } 205 206 /* Allocate memory for the pipeline on requested socket */ 207 p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline), 208 RTE_CACHE_LINE_SIZE, params->socket_id); 209 210 if (p == NULL) { 211 RTE_LOG(ERR, PIPELINE, 212 "%s: Pipeline memory allocation failed\n", __func__); 213 return NULL; 214 } 215 216 /* Save input parameters */ 217 strlcpy(p->name, params->name, RTE_PIPELINE_MAX_NAME_SZ); 218 p->socket_id = params->socket_id; 219 p->offset_port_id = params->offset_port_id; 220 221 /* Initialize pipeline internal data structure */ 222 p->num_ports_in = 0; 223 p->num_ports_out = 0; 224 p->num_tables = 0; 225 p->enabled_port_in_mask = 0; 226 p->port_in_next = NULL; 227 p->pkts_mask = 0; 228 p->n_pkts_ah_drop = 0; 229 230 return p; 231 } 232 233 int 234 rte_pipeline_free(struct rte_pipeline *p) 235 { 236 uint32_t i; 237 238 /* Check input parameters */ 239 if (p == NULL) { 240 RTE_LOG(ERR, PIPELINE, 241 "%s: rte_pipeline parameter is NULL\n", __func__); 242 return -EINVAL; 243 } 244 245 /* Free input ports */ 246 for (i = 0; i < p->num_ports_in; i++) { 247 struct rte_port_in *port = &p->ports_in[i]; 248 249 rte_pipeline_port_in_free(port); 250 } 251 252 /* Free tables */ 253 for (i = 0; i < p->num_tables; i++) { 254 struct rte_table *table = &p->tables[i]; 255 256 rte_pipeline_table_free(table); 257 } 258 259 /* Free output ports */ 260 for (i = 0; i < p->num_ports_out; i++) { 261 struct rte_port_out *port = &p->ports_out[i]; 262 263 rte_pipeline_port_out_free(port); 264 } 265 266 /* Free pipeline memory */ 267 rte_free(p); 268 269 return 0; 270 } 271 272 /* 273 * Table 274 * 275 */ 276 static int 277 rte_table_check_params(struct rte_pipeline *p, 278 struct rte_pipeline_table_params *params, 279 uint32_t *table_id) 280 { 281 if (p == NULL) { 282 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", 283 __func__); 284 return -EINVAL; 285 } 286 if (params == NULL) { 287 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n", 288 __func__); 289 return -EINVAL; 290 } 291 if (table_id == NULL) { 292 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n", 293 __func__); 294 return -EINVAL; 295 } 296 297 /* ops */ 298 if (params->ops == NULL) { 299 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n", 300 __func__); 301 return -EINVAL; 302 } 303 304 if (params->ops->f_create == NULL) { 305 RTE_LOG(ERR, PIPELINE, 306 "%s: f_create function pointer is NULL\n", __func__); 307 return -EINVAL; 308 } 309 310 if (params->ops->f_lookup == NULL) { 311 RTE_LOG(ERR, PIPELINE, 312 "%s: f_lookup function pointer is NULL\n", __func__); 313 return -EINVAL; 314 } 315 316 /* De we have room for one more table? */ 317 if (p->num_tables == RTE_PIPELINE_TABLE_MAX) { 318 RTE_LOG(ERR, PIPELINE, 319 "%s: Incorrect value for num_tables parameter\n", 320 __func__); 321 return -EINVAL; 322 } 323 324 return 0; 325 } 326 327 int 328 rte_pipeline_table_create(struct rte_pipeline *p, 329 struct rte_pipeline_table_params *params, 330 uint32_t *table_id) 331 { 332 struct rte_table *table; 333 struct rte_pipeline_table_entry *default_entry; 334 void *h_table; 335 uint32_t entry_size, id; 336 int status; 337 338 /* Check input arguments */ 339 status = rte_table_check_params(p, params, table_id); 340 if (status != 0) 341 return status; 342 343 id = p->num_tables; 344 table = &p->tables[id]; 345 346 /* Allocate space for the default table entry */ 347 entry_size = sizeof(struct rte_pipeline_table_entry) + 348 params->action_data_size; 349 default_entry = rte_zmalloc_socket( 350 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id); 351 if (default_entry == NULL) { 352 RTE_LOG(ERR, PIPELINE, 353 "%s: Failed to allocate default entry\n", __func__); 354 return -EINVAL; 355 } 356 357 /* Create the table */ 358 h_table = params->ops->f_create(params->arg_create, p->socket_id, 359 entry_size); 360 if (h_table == NULL) { 361 rte_free(default_entry); 362 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__); 363 return -EINVAL; 364 } 365 366 /* Commit current table to the pipeline */ 367 p->num_tables++; 368 *table_id = id; 369 370 /* Save input parameters */ 371 memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops)); 372 table->f_action_hit = params->f_action_hit; 373 table->f_action_miss = params->f_action_miss; 374 table->arg_ah = params->arg_ah; 375 table->entry_size = entry_size; 376 377 /* Clear the lookup miss actions (to be set later through API) */ 378 table->default_entry = default_entry; 379 table->default_entry->action = RTE_PIPELINE_ACTION_DROP; 380 381 /* Initialize table internal data structure */ 382 table->h_table = h_table; 383 table->table_next_id = 0; 384 table->table_next_id_valid = 0; 385 386 return 0; 387 } 388 389 void 390 rte_pipeline_table_free(struct rte_table *table) 391 { 392 if (table->ops.f_free != NULL) 393 table->ops.f_free(table->h_table); 394 395 rte_free(table->default_entry); 396 } 397 398 int 399 rte_pipeline_table_default_entry_add(struct rte_pipeline *p, 400 uint32_t table_id, 401 struct rte_pipeline_table_entry *default_entry, 402 struct rte_pipeline_table_entry **default_entry_ptr) 403 { 404 struct rte_table *table; 405 406 /* Check input arguments */ 407 if (p == NULL) { 408 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", 409 __func__); 410 return -EINVAL; 411 } 412 413 if (default_entry == NULL) { 414 RTE_LOG(ERR, PIPELINE, 415 "%s: default_entry parameter is NULL\n", __func__); 416 return -EINVAL; 417 } 418 419 if (table_id >= p->num_tables) { 420 RTE_LOG(ERR, PIPELINE, 421 "%s: table_id %d out of range\n", __func__, table_id); 422 return -EINVAL; 423 } 424 425 table = &p->tables[table_id]; 426 427 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) && 428 table->table_next_id_valid && 429 (default_entry->table_id != table->table_next_id)) { 430 RTE_LOG(ERR, PIPELINE, 431 "%s: Tree-like topologies not allowed\n", __func__); 432 return -EINVAL; 433 } 434 435 /* Set the lookup miss actions */ 436 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) && 437 (table->table_next_id_valid == 0)) { 438 table->table_next_id = default_entry->table_id; 439 table->table_next_id_valid = 1; 440 } 441 442 memcpy(table->default_entry, default_entry, table->entry_size); 443 444 *default_entry_ptr = table->default_entry; 445 return 0; 446 } 447 448 int 449 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p, 450 uint32_t table_id, 451 struct rte_pipeline_table_entry *entry) 452 { 453 struct rte_table *table; 454 455 /* Check input arguments */ 456 if (p == NULL) { 457 RTE_LOG(ERR, PIPELINE, 458 "%s: pipeline parameter is NULL\n", __func__); 459 return -EINVAL; 460 } 461 462 if (table_id >= p->num_tables) { 463 RTE_LOG(ERR, PIPELINE, 464 "%s: table_id %d out of range\n", __func__, table_id); 465 return -EINVAL; 466 } 467 468 table = &p->tables[table_id]; 469 470 /* Save the current contents of the default entry */ 471 if (entry) 472 memcpy(entry, table->default_entry, table->entry_size); 473 474 /* Clear the lookup miss actions */ 475 memset(table->default_entry, 0, table->entry_size); 476 table->default_entry->action = RTE_PIPELINE_ACTION_DROP; 477 478 return 0; 479 } 480 481 int 482 rte_pipeline_table_entry_add(struct rte_pipeline *p, 483 uint32_t table_id, 484 void *key, 485 struct rte_pipeline_table_entry *entry, 486 int *key_found, 487 struct rte_pipeline_table_entry **entry_ptr) 488 { 489 struct rte_table *table; 490 491 /* Check input arguments */ 492 if (p == NULL) { 493 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", 494 __func__); 495 return -EINVAL; 496 } 497 498 if (key == NULL) { 499 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__); 500 return -EINVAL; 501 } 502 503 if (entry == NULL) { 504 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n", 505 __func__); 506 return -EINVAL; 507 } 508 509 if (table_id >= p->num_tables) { 510 RTE_LOG(ERR, PIPELINE, 511 "%s: table_id %d out of range\n", __func__, table_id); 512 return -EINVAL; 513 } 514 515 table = &p->tables[table_id]; 516 517 if (table->ops.f_add == NULL) { 518 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n", 519 __func__); 520 return -EINVAL; 521 } 522 523 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) && 524 table->table_next_id_valid && 525 (entry->table_id != table->table_next_id)) { 526 RTE_LOG(ERR, PIPELINE, 527 "%s: Tree-like topologies not allowed\n", __func__); 528 return -EINVAL; 529 } 530 531 /* Add entry */ 532 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) && 533 (table->table_next_id_valid == 0)) { 534 table->table_next_id = entry->table_id; 535 table->table_next_id_valid = 1; 536 } 537 538 return (table->ops.f_add)(table->h_table, key, (void *) entry, 539 key_found, (void **) entry_ptr); 540 } 541 542 int 543 rte_pipeline_table_entry_delete(struct rte_pipeline *p, 544 uint32_t table_id, 545 void *key, 546 int *key_found, 547 struct rte_pipeline_table_entry *entry) 548 { 549 struct rte_table *table; 550 551 /* Check input arguments */ 552 if (p == NULL) { 553 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 554 __func__); 555 return -EINVAL; 556 } 557 558 if (key == NULL) { 559 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", 560 __func__); 561 return -EINVAL; 562 } 563 564 if (table_id >= p->num_tables) { 565 RTE_LOG(ERR, PIPELINE, 566 "%s: table_id %d out of range\n", __func__, table_id); 567 return -EINVAL; 568 } 569 570 table = &p->tables[table_id]; 571 572 if (table->ops.f_delete == NULL) { 573 RTE_LOG(ERR, PIPELINE, 574 "%s: f_delete function pointer NULL\n", __func__); 575 return -EINVAL; 576 } 577 578 return (table->ops.f_delete)(table->h_table, key, key_found, entry); 579 } 580 581 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p, 582 uint32_t table_id, 583 void **keys, 584 struct rte_pipeline_table_entry **entries, 585 uint32_t n_keys, 586 int *key_found, 587 struct rte_pipeline_table_entry **entries_ptr) 588 { 589 struct rte_table *table; 590 uint32_t i; 591 592 /* Check input arguments */ 593 if (p == NULL) { 594 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", 595 __func__); 596 return -EINVAL; 597 } 598 599 if (keys == NULL) { 600 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__); 601 return -EINVAL; 602 } 603 604 if (entries == NULL) { 605 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n", 606 __func__); 607 return -EINVAL; 608 } 609 610 if (table_id >= p->num_tables) { 611 RTE_LOG(ERR, PIPELINE, 612 "%s: table_id %d out of range\n", __func__, table_id); 613 return -EINVAL; 614 } 615 616 table = &p->tables[table_id]; 617 618 if (table->ops.f_add_bulk == NULL) { 619 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n", 620 __func__); 621 return -EINVAL; 622 } 623 624 for (i = 0; i < n_keys; i++) { 625 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) && 626 table->table_next_id_valid && 627 (entries[i]->table_id != table->table_next_id)) { 628 RTE_LOG(ERR, PIPELINE, 629 "%s: Tree-like topologies not allowed\n", __func__); 630 return -EINVAL; 631 } 632 } 633 634 /* Add entry */ 635 for (i = 0; i < n_keys; i++) { 636 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) && 637 (table->table_next_id_valid == 0)) { 638 table->table_next_id = entries[i]->table_id; 639 table->table_next_id_valid = 1; 640 } 641 } 642 643 return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries, 644 n_keys, key_found, (void **) entries_ptr); 645 } 646 647 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p, 648 uint32_t table_id, 649 void **keys, 650 uint32_t n_keys, 651 int *key_found, 652 struct rte_pipeline_table_entry **entries) 653 { 654 struct rte_table *table; 655 656 /* Check input arguments */ 657 if (p == NULL) { 658 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 659 __func__); 660 return -EINVAL; 661 } 662 663 if (keys == NULL) { 664 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", 665 __func__); 666 return -EINVAL; 667 } 668 669 if (table_id >= p->num_tables) { 670 RTE_LOG(ERR, PIPELINE, 671 "%s: table_id %d out of range\n", __func__, table_id); 672 return -EINVAL; 673 } 674 675 table = &p->tables[table_id]; 676 677 if (table->ops.f_delete_bulk == NULL) { 678 RTE_LOG(ERR, PIPELINE, 679 "%s: f_delete function pointer NULL\n", __func__); 680 return -EINVAL; 681 } 682 683 return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found, 684 (void **) entries); 685 } 686 687 /* 688 * Port 689 * 690 */ 691 static int 692 rte_pipeline_port_in_check_params(struct rte_pipeline *p, 693 struct rte_pipeline_port_in_params *params, 694 uint32_t *port_id) 695 { 696 if (p == NULL) { 697 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 698 __func__); 699 return -EINVAL; 700 } 701 if (params == NULL) { 702 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__); 703 return -EINVAL; 704 } 705 if (port_id == NULL) { 706 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n", 707 __func__); 708 return -EINVAL; 709 } 710 711 /* ops */ 712 if (params->ops == NULL) { 713 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n", 714 __func__); 715 return -EINVAL; 716 } 717 718 if (params->ops->f_create == NULL) { 719 RTE_LOG(ERR, PIPELINE, 720 "%s: f_create function pointer NULL\n", __func__); 721 return -EINVAL; 722 } 723 724 if (params->ops->f_rx == NULL) { 725 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n", 726 __func__); 727 return -EINVAL; 728 } 729 730 /* burst_size */ 731 if ((params->burst_size == 0) || 732 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) { 733 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n", 734 __func__); 735 return -EINVAL; 736 } 737 738 /* Do we have room for one more port? */ 739 if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) { 740 RTE_LOG(ERR, PIPELINE, 741 "%s: invalid value for num_ports_in\n", __func__); 742 return -EINVAL; 743 } 744 745 return 0; 746 } 747 748 static int 749 rte_pipeline_port_out_check_params(struct rte_pipeline *p, 750 struct rte_pipeline_port_out_params *params, 751 uint32_t *port_id) 752 { 753 if (p == NULL) { 754 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 755 __func__); 756 return -EINVAL; 757 } 758 759 if (params == NULL) { 760 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__); 761 return -EINVAL; 762 } 763 764 if (port_id == NULL) { 765 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n", 766 __func__); 767 return -EINVAL; 768 } 769 770 /* ops */ 771 if (params->ops == NULL) { 772 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n", 773 __func__); 774 return -EINVAL; 775 } 776 777 if (params->ops->f_create == NULL) { 778 RTE_LOG(ERR, PIPELINE, 779 "%s: f_create function pointer NULL\n", __func__); 780 return -EINVAL; 781 } 782 783 if (params->ops->f_tx == NULL) { 784 RTE_LOG(ERR, PIPELINE, 785 "%s: f_tx function pointer NULL\n", __func__); 786 return -EINVAL; 787 } 788 789 if (params->ops->f_tx_bulk == NULL) { 790 RTE_LOG(ERR, PIPELINE, 791 "%s: f_tx_bulk function pointer NULL\n", __func__); 792 return -EINVAL; 793 } 794 795 /* Do we have room for one more port? */ 796 if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) { 797 RTE_LOG(ERR, PIPELINE, 798 "%s: invalid value for num_ports_out\n", __func__); 799 return -EINVAL; 800 } 801 802 return 0; 803 } 804 805 int 806 rte_pipeline_port_in_create(struct rte_pipeline *p, 807 struct rte_pipeline_port_in_params *params, 808 uint32_t *port_id) 809 { 810 struct rte_port_in *port; 811 void *h_port; 812 uint32_t id; 813 int status; 814 815 /* Check input arguments */ 816 status = rte_pipeline_port_in_check_params(p, params, port_id); 817 if (status != 0) 818 return status; 819 820 id = p->num_ports_in; 821 port = &p->ports_in[id]; 822 823 /* Create the port */ 824 h_port = params->ops->f_create(params->arg_create, p->socket_id); 825 if (h_port == NULL) { 826 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__); 827 return -EINVAL; 828 } 829 830 /* Commit current table to the pipeline */ 831 p->num_ports_in++; 832 *port_id = id; 833 834 /* Save input parameters */ 835 memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops)); 836 port->f_action = params->f_action; 837 port->arg_ah = params->arg_ah; 838 port->burst_size = params->burst_size; 839 840 /* Initialize port internal data structure */ 841 port->table_id = RTE_TABLE_INVALID; 842 port->h_port = h_port; 843 port->next = NULL; 844 845 return 0; 846 } 847 848 void 849 rte_pipeline_port_in_free(struct rte_port_in *port) 850 { 851 if (port->ops.f_free != NULL) 852 port->ops.f_free(port->h_port); 853 } 854 855 int 856 rte_pipeline_port_out_create(struct rte_pipeline *p, 857 struct rte_pipeline_port_out_params *params, 858 uint32_t *port_id) 859 { 860 struct rte_port_out *port; 861 void *h_port; 862 uint32_t id; 863 int status; 864 865 /* Check input arguments */ 866 status = rte_pipeline_port_out_check_params(p, params, port_id); 867 if (status != 0) 868 return status; 869 870 id = p->num_ports_out; 871 port = &p->ports_out[id]; 872 873 /* Create the port */ 874 h_port = params->ops->f_create(params->arg_create, p->socket_id); 875 if (h_port == NULL) { 876 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__); 877 return -EINVAL; 878 } 879 880 /* Commit current table to the pipeline */ 881 p->num_ports_out++; 882 *port_id = id; 883 884 /* Save input parameters */ 885 memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops)); 886 port->f_action = params->f_action; 887 port->arg_ah = params->arg_ah; 888 889 /* Initialize port internal data structure */ 890 port->h_port = h_port; 891 892 return 0; 893 } 894 895 void 896 rte_pipeline_port_out_free(struct rte_port_out *port) 897 { 898 if (port->ops.f_free != NULL) 899 port->ops.f_free(port->h_port); 900 } 901 902 int 903 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p, 904 uint32_t port_id, 905 uint32_t table_id) 906 { 907 struct rte_port_in *port; 908 909 /* Check input arguments */ 910 if (p == NULL) { 911 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 912 __func__); 913 return -EINVAL; 914 } 915 916 if (port_id >= p->num_ports_in) { 917 RTE_LOG(ERR, PIPELINE, 918 "%s: port IN ID %u is out of range\n", 919 __func__, port_id); 920 return -EINVAL; 921 } 922 923 if (table_id >= p->num_tables) { 924 RTE_LOG(ERR, PIPELINE, 925 "%s: Table ID %u is out of range\n", 926 __func__, table_id); 927 return -EINVAL; 928 } 929 930 port = &p->ports_in[port_id]; 931 port->table_id = table_id; 932 933 return 0; 934 } 935 936 int 937 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id) 938 { 939 struct rte_port_in *port, *port_prev, *port_next; 940 uint64_t port_mask; 941 uint32_t port_prev_id, port_next_id; 942 943 /* Check input arguments */ 944 if (p == NULL) { 945 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 946 __func__); 947 return -EINVAL; 948 } 949 950 if (port_id >= p->num_ports_in) { 951 RTE_LOG(ERR, PIPELINE, 952 "%s: port IN ID %u is out of range\n", 953 __func__, port_id); 954 return -EINVAL; 955 } 956 957 port = &p->ports_in[port_id]; 958 959 /* Return if current input port is already enabled */ 960 port_mask = 1LLU << port_id; 961 if (p->enabled_port_in_mask & port_mask) 962 return 0; 963 964 p->enabled_port_in_mask |= port_mask; 965 966 /* Add current input port to the pipeline chain of enabled ports */ 967 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id); 968 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id); 969 970 port_prev = &p->ports_in[port_prev_id]; 971 port_next = &p->ports_in[port_next_id]; 972 973 port_prev->next = port; 974 port->next = port_next; 975 976 /* Check if list of enabled ports was previously empty */ 977 if (p->enabled_port_in_mask == port_mask) 978 p->port_in_next = port; 979 980 return 0; 981 } 982 983 int 984 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id) 985 { 986 struct rte_port_in *port, *port_prev, *port_next; 987 uint64_t port_mask; 988 uint32_t port_prev_id, port_next_id; 989 990 /* Check input arguments */ 991 if (p == NULL) { 992 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 993 __func__); 994 return -EINVAL; 995 } 996 997 if (port_id >= p->num_ports_in) { 998 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n", 999 __func__, port_id); 1000 return -EINVAL; 1001 } 1002 1003 port = &p->ports_in[port_id]; 1004 1005 /* Return if current input port is already disabled */ 1006 port_mask = 1LLU << port_id; 1007 if ((p->enabled_port_in_mask & port_mask) == 0) 1008 return 0; 1009 1010 p->enabled_port_in_mask &= ~port_mask; 1011 1012 /* Return if no other enabled ports */ 1013 if (p->enabled_port_in_mask == 0) { 1014 p->port_in_next = NULL; 1015 1016 return 0; 1017 } 1018 1019 /* Add current input port to the pipeline chain of enabled ports */ 1020 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id); 1021 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id); 1022 1023 port_prev = &p->ports_in[port_prev_id]; 1024 port_next = &p->ports_in[port_next_id]; 1025 1026 port_prev->next = port_next; 1027 1028 /* Check if the port which has just been disabled is next to serve */ 1029 if (port == p->port_in_next) 1030 p->port_in_next = port_next; 1031 1032 return 0; 1033 } 1034 1035 /* 1036 * Pipeline run-time 1037 * 1038 */ 1039 int 1040 rte_pipeline_check(struct rte_pipeline *p) 1041 { 1042 uint32_t port_in_id; 1043 1044 /* Check input arguments */ 1045 if (p == NULL) { 1046 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 1047 __func__); 1048 return -EINVAL; 1049 } 1050 1051 /* Check that pipeline has at least one input port, one table and one 1052 output port */ 1053 if (p->num_ports_in == 0) { 1054 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n", 1055 __func__); 1056 return -EINVAL; 1057 } 1058 if (p->num_tables == 0) { 1059 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n", 1060 __func__); 1061 return -EINVAL; 1062 } 1063 if (p->num_ports_out == 0) { 1064 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n", 1065 __func__); 1066 return -EINVAL; 1067 } 1068 1069 /* Check that all input ports are connected */ 1070 for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) { 1071 struct rte_port_in *port_in = &p->ports_in[port_in_id]; 1072 1073 if (port_in->table_id == RTE_TABLE_INVALID) { 1074 RTE_LOG(ERR, PIPELINE, 1075 "%s: Port IN ID %u is not connected\n", 1076 __func__, port_in_id); 1077 return -EINVAL; 1078 } 1079 } 1080 1081 return 0; 1082 } 1083 1084 static inline void 1085 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask) 1086 { 1087 p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0; 1088 p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0; 1089 p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0; 1090 p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0; 1091 1092 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1093 uint64_t n_pkts = __builtin_popcountll(pkts_mask); 1094 uint32_t i; 1095 1096 for (i = 0; i < n_pkts; i++) { 1097 uint64_t pkt_mask = 1LLU << i; 1098 uint32_t pos = p->entries[i]->action; 1099 1100 p->action_mask1[pos] |= pkt_mask; 1101 } 1102 } else { 1103 uint32_t i; 1104 1105 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1106 uint64_t pkt_mask = 1LLU << i; 1107 uint32_t pos; 1108 1109 if ((pkt_mask & pkts_mask) == 0) 1110 continue; 1111 1112 pos = p->entries[i]->action; 1113 p->action_mask1[pos] |= pkt_mask; 1114 } 1115 } 1116 } 1117 1118 static inline void 1119 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p, 1120 uint64_t pkts_mask, uint32_t port_id) 1121 { 1122 struct rte_port_out *port_out = &p->ports_out[port_id]; 1123 1124 p->pkts_mask = pkts_mask; 1125 1126 /* Output port user actions */ 1127 if (port_out->f_action != NULL) { 1128 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah); 1129 1130 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1131 port_out->n_pkts_dropped_by_ah); 1132 } 1133 1134 /* Output port TX */ 1135 if (p->pkts_mask != 0) 1136 port_out->ops.f_tx_bulk(port_out->h_port, 1137 p->pkts, 1138 p->pkts_mask); 1139 } 1140 1141 static inline void 1142 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask) 1143 { 1144 p->pkts_mask = pkts_mask; 1145 1146 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1147 uint64_t n_pkts = __builtin_popcountll(pkts_mask); 1148 uint32_t i; 1149 1150 for (i = 0; i < n_pkts; i++) { 1151 struct rte_mbuf *pkt = p->pkts[i]; 1152 uint32_t port_out_id = p->entries[i]->port_id; 1153 struct rte_port_out *port_out = 1154 &p->ports_out[port_out_id]; 1155 1156 /* Output port user actions */ 1157 if (port_out->f_action == NULL) /* Output port TX */ 1158 port_out->ops.f_tx(port_out->h_port, pkt); 1159 else { 1160 uint64_t pkt_mask = 1LLU << i; 1161 1162 port_out->f_action(p, 1163 p->pkts, 1164 pkt_mask, 1165 port_out->arg_ah); 1166 1167 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1168 port_out->n_pkts_dropped_by_ah); 1169 1170 /* Output port TX */ 1171 if (pkt_mask & p->pkts_mask) 1172 port_out->ops.f_tx(port_out->h_port, 1173 pkt); 1174 } 1175 } 1176 } else { 1177 uint32_t i; 1178 1179 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1180 uint64_t pkt_mask = 1LLU << i; 1181 struct rte_mbuf *pkt; 1182 struct rte_port_out *port_out; 1183 uint32_t port_out_id; 1184 1185 if ((pkt_mask & pkts_mask) == 0) 1186 continue; 1187 1188 pkt = p->pkts[i]; 1189 port_out_id = p->entries[i]->port_id; 1190 port_out = &p->ports_out[port_out_id]; 1191 1192 /* Output port user actions */ 1193 if (port_out->f_action == NULL) /* Output port TX */ 1194 port_out->ops.f_tx(port_out->h_port, pkt); 1195 else { 1196 port_out->f_action(p, 1197 p->pkts, 1198 pkt_mask, 1199 port_out->arg_ah); 1200 1201 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1202 port_out->n_pkts_dropped_by_ah); 1203 1204 /* Output port TX */ 1205 if (pkt_mask & p->pkts_mask) 1206 port_out->ops.f_tx(port_out->h_port, 1207 pkt); 1208 } 1209 } 1210 } 1211 } 1212 1213 static inline void 1214 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p, 1215 uint64_t pkts_mask) 1216 { 1217 p->pkts_mask = pkts_mask; 1218 1219 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1220 uint64_t n_pkts = __builtin_popcountll(pkts_mask); 1221 uint32_t i; 1222 1223 for (i = 0; i < n_pkts; i++) { 1224 struct rte_mbuf *pkt = p->pkts[i]; 1225 uint32_t port_out_id = 1226 RTE_MBUF_METADATA_UINT32(pkt, 1227 p->offset_port_id); 1228 struct rte_port_out *port_out = &p->ports_out[ 1229 port_out_id]; 1230 1231 /* Output port user actions */ 1232 if (port_out->f_action == NULL) /* Output port TX */ 1233 port_out->ops.f_tx(port_out->h_port, pkt); 1234 else { 1235 uint64_t pkt_mask = 1LLU << i; 1236 1237 port_out->f_action(p, 1238 p->pkts, 1239 pkt_mask, 1240 port_out->arg_ah); 1241 1242 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1243 port_out->n_pkts_dropped_by_ah); 1244 1245 /* Output port TX */ 1246 if (pkt_mask & p->pkts_mask) 1247 port_out->ops.f_tx(port_out->h_port, 1248 pkt); 1249 } 1250 } 1251 } else { 1252 uint32_t i; 1253 1254 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1255 uint64_t pkt_mask = 1LLU << i; 1256 struct rte_mbuf *pkt; 1257 struct rte_port_out *port_out; 1258 uint32_t port_out_id; 1259 1260 if ((pkt_mask & pkts_mask) == 0) 1261 continue; 1262 1263 pkt = p->pkts[i]; 1264 port_out_id = RTE_MBUF_METADATA_UINT32(pkt, 1265 p->offset_port_id); 1266 port_out = &p->ports_out[port_out_id]; 1267 1268 /* Output port user actions */ 1269 if (port_out->f_action == NULL) /* Output port TX */ 1270 port_out->ops.f_tx(port_out->h_port, pkt); 1271 else { 1272 port_out->f_action(p, 1273 p->pkts, 1274 pkt_mask, 1275 port_out->arg_ah); 1276 1277 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1278 port_out->n_pkts_dropped_by_ah); 1279 1280 /* Output port TX */ 1281 if (pkt_mask & p->pkts_mask) 1282 port_out->ops.f_tx(port_out->h_port, 1283 pkt); 1284 } 1285 } 1286 } 1287 } 1288 1289 static inline void 1290 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask) 1291 { 1292 if ((pkts_mask & (pkts_mask + 1)) == 0) { 1293 uint64_t n_pkts = __builtin_popcountll(pkts_mask); 1294 uint32_t i; 1295 1296 for (i = 0; i < n_pkts; i++) 1297 rte_pktmbuf_free(p->pkts[i]); 1298 } else { 1299 uint32_t i; 1300 1301 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { 1302 uint64_t pkt_mask = 1LLU << i; 1303 1304 if ((pkt_mask & pkts_mask) == 0) 1305 continue; 1306 1307 rte_pktmbuf_free(p->pkts[i]); 1308 } 1309 } 1310 } 1311 1312 int 1313 rte_pipeline_run(struct rte_pipeline *p) 1314 { 1315 struct rte_port_in *port_in = p->port_in_next; 1316 uint32_t n_pkts, table_id; 1317 1318 if (port_in == NULL) 1319 return 0; 1320 1321 /* Input port RX */ 1322 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts, 1323 port_in->burst_size); 1324 if (n_pkts == 0) { 1325 p->port_in_next = port_in->next; 1326 return 0; 1327 } 1328 1329 p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t); 1330 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0; 1331 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0; 1332 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0; 1333 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; 1334 1335 /* Input port user actions */ 1336 if (port_in->f_action != NULL) { 1337 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah); 1338 1339 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1340 port_in->n_pkts_dropped_by_ah); 1341 } 1342 1343 /* Table */ 1344 for (table_id = port_in->table_id; p->pkts_mask != 0; ) { 1345 struct rte_table *table; 1346 uint64_t lookup_hit_mask, lookup_miss_mask; 1347 1348 /* Lookup */ 1349 table = &p->tables[table_id]; 1350 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask, 1351 &lookup_hit_mask, (void **) p->entries); 1352 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask); 1353 1354 /* Lookup miss */ 1355 if (lookup_miss_mask != 0) { 1356 struct rte_pipeline_table_entry *default_entry = 1357 table->default_entry; 1358 1359 p->pkts_mask = lookup_miss_mask; 1360 1361 /* Table user actions */ 1362 if (table->f_action_miss != NULL) { 1363 table->f_action_miss(p, 1364 p->pkts, 1365 lookup_miss_mask, 1366 default_entry, 1367 table->arg_ah); 1368 1369 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1370 table->n_pkts_dropped_by_lkp_miss_ah); 1371 } 1372 1373 /* Table reserved actions */ 1374 if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) && 1375 (p->pkts_mask != 0)) 1376 rte_pipeline_action_handler_port_bulk(p, 1377 p->pkts_mask, 1378 default_entry->port_id); 1379 else { 1380 uint32_t pos = default_entry->action; 1381 1382 RTE_PIPELINE_STATS_TABLE_DROP0(p); 1383 1384 p->action_mask0[pos] |= p->pkts_mask; 1385 1386 RTE_PIPELINE_STATS_TABLE_DROP1(p, 1387 table->n_pkts_dropped_lkp_miss); 1388 } 1389 } 1390 1391 /* Lookup hit */ 1392 if (lookup_hit_mask != 0) { 1393 p->pkts_mask = lookup_hit_mask; 1394 1395 /* Table user actions */ 1396 if (table->f_action_hit != NULL) { 1397 table->f_action_hit(p, 1398 p->pkts, 1399 lookup_hit_mask, 1400 p->entries, 1401 table->arg_ah); 1402 1403 RTE_PIPELINE_STATS_AH_DROP_READ(p, 1404 table->n_pkts_dropped_by_lkp_hit_ah); 1405 } 1406 1407 /* Table reserved actions */ 1408 RTE_PIPELINE_STATS_TABLE_DROP0(p); 1409 rte_pipeline_compute_masks(p, p->pkts_mask); 1410 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= 1411 p->action_mask1[ 1412 RTE_PIPELINE_ACTION_DROP]; 1413 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |= 1414 p->action_mask1[ 1415 RTE_PIPELINE_ACTION_PORT]; 1416 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |= 1417 p->action_mask1[ 1418 RTE_PIPELINE_ACTION_PORT_META]; 1419 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |= 1420 p->action_mask1[ 1421 RTE_PIPELINE_ACTION_TABLE]; 1422 1423 RTE_PIPELINE_STATS_TABLE_DROP1(p, 1424 table->n_pkts_dropped_lkp_hit); 1425 } 1426 1427 /* Prepare for next iteration */ 1428 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE]; 1429 table_id = table->table_next_id; 1430 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; 1431 } 1432 1433 /* Table reserved action PORT */ 1434 rte_pipeline_action_handler_port(p, 1435 p->action_mask0[RTE_PIPELINE_ACTION_PORT]); 1436 1437 /* Table reserved action PORT META */ 1438 rte_pipeline_action_handler_port_meta(p, 1439 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]); 1440 1441 /* Table reserved action DROP */ 1442 rte_pipeline_action_handler_drop(p, 1443 p->action_mask0[RTE_PIPELINE_ACTION_DROP]); 1444 1445 /* Pick candidate for next port IN to serve */ 1446 p->port_in_next = port_in->next; 1447 1448 return (int) n_pkts; 1449 } 1450 1451 int 1452 rte_pipeline_flush(struct rte_pipeline *p) 1453 { 1454 uint32_t port_id; 1455 1456 /* Check input arguments */ 1457 if (p == NULL) { 1458 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 1459 __func__); 1460 return -EINVAL; 1461 } 1462 1463 for (port_id = 0; port_id < p->num_ports_out; port_id++) { 1464 struct rte_port_out *port = &p->ports_out[port_id]; 1465 1466 if (port->ops.f_flush != NULL) 1467 port->ops.f_flush(port->h_port); 1468 } 1469 1470 return 0; 1471 } 1472 1473 int 1474 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p, 1475 uint32_t port_id, struct rte_mbuf *pkt) 1476 { 1477 struct rte_port_out *port_out = &p->ports_out[port_id]; 1478 1479 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */ 1480 1481 return 0; 1482 } 1483 1484 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p, 1485 uint64_t pkts_mask) 1486 { 1487 pkts_mask &= p->pkts_mask; 1488 p->pkts_mask &= ~pkts_mask; 1489 1490 return 0; 1491 } 1492 1493 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p, 1494 uint64_t pkts_mask) 1495 { 1496 pkts_mask &= p->pkts_mask; 1497 p->pkts_mask &= ~pkts_mask; 1498 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask; 1499 1500 RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask); 1501 return 0; 1502 } 1503 1504 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id, 1505 struct rte_pipeline_port_in_stats *stats, int clear) 1506 { 1507 struct rte_port_in *port; 1508 int retval; 1509 1510 if (p == NULL) { 1511 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 1512 __func__); 1513 return -EINVAL; 1514 } 1515 1516 if (port_id >= p->num_ports_in) { 1517 RTE_LOG(ERR, PIPELINE, 1518 "%s: port IN ID %u is out of range\n", 1519 __func__, port_id); 1520 return -EINVAL; 1521 } 1522 1523 port = &p->ports_in[port_id]; 1524 1525 if (port->ops.f_stats != NULL) { 1526 retval = port->ops.f_stats(port->h_port, &stats->stats, clear); 1527 if (retval) 1528 return retval; 1529 } else if (stats != NULL) 1530 memset(&stats->stats, 0, sizeof(stats->stats)); 1531 1532 if (stats != NULL) 1533 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah; 1534 1535 if (clear != 0) 1536 port->n_pkts_dropped_by_ah = 0; 1537 1538 return 0; 1539 } 1540 1541 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id, 1542 struct rte_pipeline_port_out_stats *stats, int clear) 1543 { 1544 struct rte_port_out *port; 1545 int retval; 1546 1547 if (p == NULL) { 1548 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__); 1549 return -EINVAL; 1550 } 1551 1552 if (port_id >= p->num_ports_out) { 1553 RTE_LOG(ERR, PIPELINE, 1554 "%s: port OUT ID %u is out of range\n", __func__, port_id); 1555 return -EINVAL; 1556 } 1557 1558 port = &p->ports_out[port_id]; 1559 if (port->ops.f_stats != NULL) { 1560 retval = port->ops.f_stats(port->h_port, &stats->stats, clear); 1561 if (retval != 0) 1562 return retval; 1563 } else if (stats != NULL) 1564 memset(&stats->stats, 0, sizeof(stats->stats)); 1565 1566 if (stats != NULL) 1567 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah; 1568 1569 if (clear != 0) 1570 port->n_pkts_dropped_by_ah = 0; 1571 1572 return 0; 1573 } 1574 1575 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id, 1576 struct rte_pipeline_table_stats *stats, int clear) 1577 { 1578 struct rte_table *table; 1579 int retval; 1580 1581 if (p == NULL) { 1582 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", 1583 __func__); 1584 return -EINVAL; 1585 } 1586 1587 if (table_id >= p->num_tables) { 1588 RTE_LOG(ERR, PIPELINE, 1589 "%s: table %u is out of range\n", __func__, table_id); 1590 return -EINVAL; 1591 } 1592 1593 table = &p->tables[table_id]; 1594 if (table->ops.f_stats != NULL) { 1595 retval = table->ops.f_stats(table->h_table, &stats->stats, clear); 1596 if (retval != 0) 1597 return retval; 1598 } else if (stats != NULL) 1599 memset(&stats->stats, 0, sizeof(stats->stats)); 1600 1601 if (stats != NULL) { 1602 stats->n_pkts_dropped_by_lkp_hit_ah = 1603 table->n_pkts_dropped_by_lkp_hit_ah; 1604 stats->n_pkts_dropped_by_lkp_miss_ah = 1605 table->n_pkts_dropped_by_lkp_miss_ah; 1606 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit; 1607 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss; 1608 } 1609 1610 if (clear != 0) { 1611 table->n_pkts_dropped_by_lkp_hit_ah = 0; 1612 table->n_pkts_dropped_by_lkp_miss_ah = 0; 1613 table->n_pkts_dropped_lkp_hit = 0; 1614 table->n_pkts_dropped_lkp_miss = 0; 1615 } 1616 1617 return 0; 1618 } 1619