1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation
3 */
4
5 #include <string.h>
6 #include <stdio.h>
7
8 #include <rte_common.h>
9 #include <rte_mbuf.h>
10 #include <rte_malloc.h>
11 #include <rte_string_fns.h>
12
13 #include "rte_pipeline.h"
14
15 RTE_LOG_REGISTER_DEFAULT(pipeline_logtype, INFO);
16 #define RTE_LOGTYPE_PIPELINE pipeline_logtype
17
18 #define PIPELINE_LOG(level, ...) \
19 RTE_LOG_LINE(level, PIPELINE, "" __VA_ARGS__)
20
21 #define RTE_TABLE_INVALID UINT32_MAX
22
23 #ifdef RTE_PIPELINE_STATS_COLLECT
24
25 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) \
26 __extension__ ({ (p)->n_pkts_ah_drop = rte_popcount64(mask); })
27
28 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) \
29 __extension__ ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; })
30
31 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) \
32 __extension__ ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; })
33
34 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) \
35 __extension__ ({ \
36 uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; \
37 mask ^= (p)->pkts_drop_mask; \
38 (counter) += rte_popcount64(mask); \
39 })
40
41 #else
42
43 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)
44 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)
45 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)
46 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)
47
48 #endif
49
50 struct rte_port_in {
51 /* Input parameters */
52 struct rte_port_in_ops ops;
53 rte_pipeline_port_in_action_handler f_action;
54 void *arg_ah;
55 uint32_t burst_size;
56
57 /* The table to which this port is connected */
58 uint32_t table_id;
59
60 /* Handle to low-level port */
61 void *h_port;
62
63 /* List of enabled ports */
64 struct rte_port_in *next;
65
66 /* Statistics */
67 uint64_t n_pkts_dropped_by_ah;
68 };
69
70 struct rte_port_out {
71 /* Input parameters */
72 struct rte_port_out_ops ops;
73 rte_pipeline_port_out_action_handler f_action;
74 void *arg_ah;
75
76 /* Handle to low-level port */
77 void *h_port;
78
79 /* Statistics */
80 uint64_t n_pkts_dropped_by_ah;
81 };
82
83 struct rte_table {
84 /* Input parameters */
85 struct rte_table_ops ops;
86 rte_pipeline_table_action_handler_hit f_action_hit;
87 rte_pipeline_table_action_handler_miss f_action_miss;
88 void *arg_ah;
89 struct rte_pipeline_table_entry *default_entry;
90 uint32_t entry_size;
91
92 uint32_t table_next_id;
93 uint32_t table_next_id_valid;
94
95 /* Handle to the low-level table object */
96 void *h_table;
97
98 /* Statistics */
99 uint64_t n_pkts_dropped_by_lkp_hit_ah;
100 uint64_t n_pkts_dropped_by_lkp_miss_ah;
101 uint64_t n_pkts_dropped_lkp_hit;
102 uint64_t n_pkts_dropped_lkp_miss;
103 };
104
105 #define RTE_PIPELINE_MAX_NAME_SZ 124
106
107 struct __rte_cache_aligned rte_pipeline {
108 /* Input parameters */
109 char name[RTE_PIPELINE_MAX_NAME_SZ];
110 int socket_id;
111 uint32_t offset_port_id;
112
113 /* Internal tables */
114 struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
115 struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
116 struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
117
118 /* Occupancy of internal tables */
119 uint32_t num_ports_in;
120 uint32_t num_ports_out;
121 uint32_t num_tables;
122
123 /* List of enabled ports */
124 uint64_t enabled_port_in_mask;
125 struct rte_port_in *port_in_next;
126
127 /* Pipeline run structures */
128 struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
129 struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
130 uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
131 uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
132 uint64_t pkts_mask;
133 uint64_t n_pkts_ah_drop;
134 uint64_t pkts_drop_mask;
135 };
136
137 static inline uint32_t
rte_mask_get_next(uint64_t mask,uint32_t pos)138 rte_mask_get_next(uint64_t mask, uint32_t pos)
139 {
140 uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
141 (mask >> ((pos + 1) & 0x3F));
142 return (rte_ctz64(mask_rot) - (63 - pos)) & 0x3F;
143 }
144
145 static inline uint32_t
rte_mask_get_prev(uint64_t mask,uint32_t pos)146 rte_mask_get_prev(uint64_t mask, uint32_t pos)
147 {
148 uint64_t mask_rot = (mask >> (pos & 0x3F)) |
149 (mask << ((64 - pos) & 0x3F));
150 return ((63 - rte_clz64(mask_rot)) + pos) & 0x3F;
151 }
152
153 static void
154 rte_pipeline_table_free(struct rte_table *table);
155
156 static void
157 rte_pipeline_port_in_free(struct rte_port_in *port);
158
159 static void
160 rte_pipeline_port_out_free(struct rte_port_out *port);
161
162 /*
163 * Pipeline
164 */
165 static int
rte_pipeline_check_params(struct rte_pipeline_params * params)166 rte_pipeline_check_params(struct rte_pipeline_params *params)
167 {
168 if (params == NULL) {
169 PIPELINE_LOG(ERR,
170 "%s: Incorrect value for parameter params", __func__);
171 return -EINVAL;
172 }
173
174 /* name */
175 if (params->name == NULL) {
176 PIPELINE_LOG(ERR,
177 "%s: Incorrect value for parameter name", __func__);
178 return -EINVAL;
179 }
180
181 /* socket */
182 if (params->socket_id < 0) {
183 PIPELINE_LOG(ERR,
184 "%s: Incorrect value for parameter socket_id",
185 __func__);
186 return -EINVAL;
187 }
188
189 return 0;
190 }
191
192 struct rte_pipeline *
rte_pipeline_create(struct rte_pipeline_params * params)193 rte_pipeline_create(struct rte_pipeline_params *params)
194 {
195 struct rte_pipeline *p;
196 int status;
197
198 /* Check input parameters */
199 status = rte_pipeline_check_params(params);
200 if (status != 0) {
201 PIPELINE_LOG(ERR,
202 "%s: Pipeline params check failed (%d)",
203 __func__, status);
204 return NULL;
205 }
206
207 /* Allocate memory for the pipeline on requested socket */
208 p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
209 RTE_CACHE_LINE_SIZE, params->socket_id);
210
211 if (p == NULL) {
212 PIPELINE_LOG(ERR,
213 "%s: Pipeline memory allocation failed", __func__);
214 return NULL;
215 }
216
217 /* Save input parameters */
218 strlcpy(p->name, params->name, RTE_PIPELINE_MAX_NAME_SZ);
219 p->socket_id = params->socket_id;
220 p->offset_port_id = params->offset_port_id;
221
222 /* Initialize pipeline internal data structure */
223 p->num_ports_in = 0;
224 p->num_ports_out = 0;
225 p->num_tables = 0;
226 p->enabled_port_in_mask = 0;
227 p->port_in_next = NULL;
228 p->pkts_mask = 0;
229 p->n_pkts_ah_drop = 0;
230
231 return p;
232 }
233
234 int
rte_pipeline_free(struct rte_pipeline * p)235 rte_pipeline_free(struct rte_pipeline *p)
236 {
237 uint32_t i;
238
239 /* Check input parameters */
240 if (p == NULL) {
241 PIPELINE_LOG(ERR,
242 "%s: rte_pipeline parameter is NULL", __func__);
243 return -EINVAL;
244 }
245
246 /* Free input ports */
247 for (i = 0; i < p->num_ports_in; i++) {
248 struct rte_port_in *port = &p->ports_in[i];
249
250 rte_pipeline_port_in_free(port);
251 }
252
253 /* Free tables */
254 for (i = 0; i < p->num_tables; i++) {
255 struct rte_table *table = &p->tables[i];
256
257 rte_pipeline_table_free(table);
258 }
259
260 /* Free output ports */
261 for (i = 0; i < p->num_ports_out; i++) {
262 struct rte_port_out *port = &p->ports_out[i];
263
264 rte_pipeline_port_out_free(port);
265 }
266
267 /* Free pipeline memory */
268 rte_free(p);
269
270 return 0;
271 }
272
273 /*
274 * Table
275 */
276 static int
rte_table_check_params(struct rte_pipeline * p,struct rte_pipeline_table_params * params,uint32_t * table_id)277 rte_table_check_params(struct rte_pipeline *p,
278 struct rte_pipeline_table_params *params,
279 uint32_t *table_id)
280 {
281 if (p == NULL) {
282 PIPELINE_LOG(ERR, "%s: pipeline parameter is NULL",
283 __func__);
284 return -EINVAL;
285 }
286 if (params == NULL) {
287 PIPELINE_LOG(ERR, "%s: params parameter is NULL",
288 __func__);
289 return -EINVAL;
290 }
291 if (table_id == NULL) {
292 PIPELINE_LOG(ERR, "%s: table_id parameter is NULL",
293 __func__);
294 return -EINVAL;
295 }
296
297 /* ops */
298 if (params->ops == NULL) {
299 PIPELINE_LOG(ERR, "%s: params->ops is NULL",
300 __func__);
301 return -EINVAL;
302 }
303
304 if (params->ops->f_create == NULL) {
305 PIPELINE_LOG(ERR,
306 "%s: f_create function pointer is NULL", __func__);
307 return -EINVAL;
308 }
309
310 if (params->ops->f_lookup == NULL) {
311 PIPELINE_LOG(ERR,
312 "%s: f_lookup function pointer is NULL", __func__);
313 return -EINVAL;
314 }
315
316 /* De we have room for one more table? */
317 if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
318 PIPELINE_LOG(ERR,
319 "%s: Incorrect value for num_tables parameter",
320 __func__);
321 return -EINVAL;
322 }
323
324 return 0;
325 }
326
327 int
rte_pipeline_table_create(struct rte_pipeline * p,struct rte_pipeline_table_params * params,uint32_t * table_id)328 rte_pipeline_table_create(struct rte_pipeline *p,
329 struct rte_pipeline_table_params *params,
330 uint32_t *table_id)
331 {
332 struct rte_table *table;
333 struct rte_pipeline_table_entry *default_entry;
334 void *h_table;
335 uint32_t entry_size, id;
336 int status;
337
338 /* Check input arguments */
339 status = rte_table_check_params(p, params, table_id);
340 if (status != 0)
341 return status;
342
343 id = p->num_tables;
344 table = &p->tables[id];
345
346 /* Allocate space for the default table entry */
347 entry_size = sizeof(struct rte_pipeline_table_entry) +
348 params->action_data_size;
349 default_entry = rte_zmalloc_socket(
350 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
351 if (default_entry == NULL) {
352 PIPELINE_LOG(ERR,
353 "%s: Failed to allocate default entry", __func__);
354 return -EINVAL;
355 }
356
357 /* Create the table */
358 h_table = params->ops->f_create(params->arg_create, p->socket_id,
359 entry_size);
360 if (h_table == NULL) {
361 rte_free(default_entry);
362 PIPELINE_LOG(ERR, "%s: Table creation failed", __func__);
363 return -EINVAL;
364 }
365
366 /* Commit current table to the pipeline */
367 p->num_tables++;
368 *table_id = id;
369
370 /* Save input parameters */
371 memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
372 table->f_action_hit = params->f_action_hit;
373 table->f_action_miss = params->f_action_miss;
374 table->arg_ah = params->arg_ah;
375 table->entry_size = entry_size;
376
377 /* Clear the lookup miss actions (to be set later through API) */
378 table->default_entry = default_entry;
379 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
380
381 /* Initialize table internal data structure */
382 table->h_table = h_table;
383 table->table_next_id = 0;
384 table->table_next_id_valid = 0;
385
386 return 0;
387 }
388
389 void
rte_pipeline_table_free(struct rte_table * table)390 rte_pipeline_table_free(struct rte_table *table)
391 {
392 if (table->ops.f_free != NULL)
393 table->ops.f_free(table->h_table);
394
395 rte_free(table->default_entry);
396 }
397
398 int
rte_pipeline_table_default_entry_add(struct rte_pipeline * p,uint32_t table_id,struct rte_pipeline_table_entry * default_entry,struct rte_pipeline_table_entry ** default_entry_ptr)399 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
400 uint32_t table_id,
401 struct rte_pipeline_table_entry *default_entry,
402 struct rte_pipeline_table_entry **default_entry_ptr)
403 {
404 struct rte_table *table;
405
406 /* Check input arguments */
407 if (p == NULL) {
408 PIPELINE_LOG(ERR, "%s: pipeline parameter is NULL",
409 __func__);
410 return -EINVAL;
411 }
412
413 if (default_entry == NULL) {
414 PIPELINE_LOG(ERR,
415 "%s: default_entry parameter is NULL", __func__);
416 return -EINVAL;
417 }
418
419 if (table_id >= p->num_tables) {
420 PIPELINE_LOG(ERR,
421 "%s: table_id %d out of range", __func__, table_id);
422 return -EINVAL;
423 }
424
425 table = &p->tables[table_id];
426
427 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
428 table->table_next_id_valid &&
429 (default_entry->table_id != table->table_next_id)) {
430 PIPELINE_LOG(ERR,
431 "%s: Tree-like topologies not allowed", __func__);
432 return -EINVAL;
433 }
434
435 /* Set the lookup miss actions */
436 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
437 (table->table_next_id_valid == 0)) {
438 table->table_next_id = default_entry->table_id;
439 table->table_next_id_valid = 1;
440 }
441
442 memcpy(table->default_entry, default_entry, table->entry_size);
443
444 *default_entry_ptr = table->default_entry;
445 return 0;
446 }
447
448 int
rte_pipeline_table_default_entry_delete(struct rte_pipeline * p,uint32_t table_id,struct rte_pipeline_table_entry * entry)449 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
450 uint32_t table_id,
451 struct rte_pipeline_table_entry *entry)
452 {
453 struct rte_table *table;
454
455 /* Check input arguments */
456 if (p == NULL) {
457 PIPELINE_LOG(ERR,
458 "%s: pipeline parameter is NULL", __func__);
459 return -EINVAL;
460 }
461
462 if (table_id >= p->num_tables) {
463 PIPELINE_LOG(ERR,
464 "%s: table_id %d out of range", __func__, table_id);
465 return -EINVAL;
466 }
467
468 table = &p->tables[table_id];
469
470 /* Save the current contents of the default entry */
471 if (entry)
472 memcpy(entry, table->default_entry, table->entry_size);
473
474 /* Clear the lookup miss actions */
475 memset(table->default_entry, 0, table->entry_size);
476 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
477
478 return 0;
479 }
480
481 int
rte_pipeline_table_entry_add(struct rte_pipeline * p,uint32_t table_id,void * key,struct rte_pipeline_table_entry * entry,int * key_found,struct rte_pipeline_table_entry ** entry_ptr)482 rte_pipeline_table_entry_add(struct rte_pipeline *p,
483 uint32_t table_id,
484 void *key,
485 struct rte_pipeline_table_entry *entry,
486 int *key_found,
487 struct rte_pipeline_table_entry **entry_ptr)
488 {
489 struct rte_table *table;
490
491 /* Check input arguments */
492 if (p == NULL) {
493 PIPELINE_LOG(ERR, "%s: pipeline parameter is NULL",
494 __func__);
495 return -EINVAL;
496 }
497
498 if (key == NULL) {
499 PIPELINE_LOG(ERR, "%s: key parameter is NULL", __func__);
500 return -EINVAL;
501 }
502
503 if (entry == NULL) {
504 PIPELINE_LOG(ERR, "%s: entry parameter is NULL",
505 __func__);
506 return -EINVAL;
507 }
508
509 if (table_id >= p->num_tables) {
510 PIPELINE_LOG(ERR,
511 "%s: table_id %d out of range", __func__, table_id);
512 return -EINVAL;
513 }
514
515 table = &p->tables[table_id];
516
517 if (table->ops.f_add == NULL) {
518 PIPELINE_LOG(ERR, "%s: f_add function pointer NULL",
519 __func__);
520 return -EINVAL;
521 }
522
523 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
524 table->table_next_id_valid &&
525 (entry->table_id != table->table_next_id)) {
526 PIPELINE_LOG(ERR,
527 "%s: Tree-like topologies not allowed", __func__);
528 return -EINVAL;
529 }
530
531 /* Add entry */
532 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
533 (table->table_next_id_valid == 0)) {
534 table->table_next_id = entry->table_id;
535 table->table_next_id_valid = 1;
536 }
537
538 return (table->ops.f_add)(table->h_table, key, (void *) entry,
539 key_found, (void **) entry_ptr);
540 }
541
542 int
rte_pipeline_table_entry_delete(struct rte_pipeline * p,uint32_t table_id,void * key,int * key_found,struct rte_pipeline_table_entry * entry)543 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
544 uint32_t table_id,
545 void *key,
546 int *key_found,
547 struct rte_pipeline_table_entry *entry)
548 {
549 struct rte_table *table;
550
551 /* Check input arguments */
552 if (p == NULL) {
553 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL",
554 __func__);
555 return -EINVAL;
556 }
557
558 if (key == NULL) {
559 PIPELINE_LOG(ERR, "%s: key parameter is NULL",
560 __func__);
561 return -EINVAL;
562 }
563
564 if (table_id >= p->num_tables) {
565 PIPELINE_LOG(ERR,
566 "%s: table_id %d out of range", __func__, table_id);
567 return -EINVAL;
568 }
569
570 table = &p->tables[table_id];
571
572 if (table->ops.f_delete == NULL) {
573 PIPELINE_LOG(ERR,
574 "%s: f_delete function pointer NULL", __func__);
575 return -EINVAL;
576 }
577
578 return (table->ops.f_delete)(table->h_table, key, key_found, entry);
579 }
580
rte_pipeline_table_entry_add_bulk(struct rte_pipeline * p,uint32_t table_id,void ** keys,struct rte_pipeline_table_entry ** entries,uint32_t n_keys,int * key_found,struct rte_pipeline_table_entry ** entries_ptr)581 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p,
582 uint32_t table_id,
583 void **keys,
584 struct rte_pipeline_table_entry **entries,
585 uint32_t n_keys,
586 int *key_found,
587 struct rte_pipeline_table_entry **entries_ptr)
588 {
589 struct rte_table *table;
590 uint32_t i;
591
592 /* Check input arguments */
593 if (p == NULL) {
594 PIPELINE_LOG(ERR, "%s: pipeline parameter is NULL",
595 __func__);
596 return -EINVAL;
597 }
598
599 if (keys == NULL) {
600 PIPELINE_LOG(ERR, "%s: keys parameter is NULL", __func__);
601 return -EINVAL;
602 }
603
604 if (entries == NULL) {
605 PIPELINE_LOG(ERR, "%s: entries parameter is NULL",
606 __func__);
607 return -EINVAL;
608 }
609
610 if (table_id >= p->num_tables) {
611 PIPELINE_LOG(ERR,
612 "%s: table_id %d out of range", __func__, table_id);
613 return -EINVAL;
614 }
615
616 table = &p->tables[table_id];
617
618 if (table->ops.f_add_bulk == NULL) {
619 PIPELINE_LOG(ERR, "%s: f_add_bulk function pointer NULL",
620 __func__);
621 return -EINVAL;
622 }
623
624 for (i = 0; i < n_keys; i++) {
625 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
626 table->table_next_id_valid &&
627 (entries[i]->table_id != table->table_next_id)) {
628 PIPELINE_LOG(ERR,
629 "%s: Tree-like topologies not allowed", __func__);
630 return -EINVAL;
631 }
632 }
633
634 /* Add entry */
635 for (i = 0; i < n_keys; i++) {
636 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
637 (table->table_next_id_valid == 0)) {
638 table->table_next_id = entries[i]->table_id;
639 table->table_next_id_valid = 1;
640 }
641 }
642
643 return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries,
644 n_keys, key_found, (void **) entries_ptr);
645 }
646
rte_pipeline_table_entry_delete_bulk(struct rte_pipeline * p,uint32_t table_id,void ** keys,uint32_t n_keys,int * key_found,struct rte_pipeline_table_entry ** entries)647 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p,
648 uint32_t table_id,
649 void **keys,
650 uint32_t n_keys,
651 int *key_found,
652 struct rte_pipeline_table_entry **entries)
653 {
654 struct rte_table *table;
655
656 /* Check input arguments */
657 if (p == NULL) {
658 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL",
659 __func__);
660 return -EINVAL;
661 }
662
663 if (keys == NULL) {
664 PIPELINE_LOG(ERR, "%s: key parameter is NULL",
665 __func__);
666 return -EINVAL;
667 }
668
669 if (table_id >= p->num_tables) {
670 PIPELINE_LOG(ERR,
671 "%s: table_id %d out of range", __func__, table_id);
672 return -EINVAL;
673 }
674
675 table = &p->tables[table_id];
676
677 if (table->ops.f_delete_bulk == NULL) {
678 PIPELINE_LOG(ERR,
679 "%s: f_delete function pointer NULL", __func__);
680 return -EINVAL;
681 }
682
683 return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found,
684 (void **) entries);
685 }
686
687 /*
688 * Port
689 */
690 static int
rte_pipeline_port_in_check_params(struct rte_pipeline * p,struct rte_pipeline_port_in_params * params,uint32_t * port_id)691 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
692 struct rte_pipeline_port_in_params *params,
693 uint32_t *port_id)
694 {
695 if (p == NULL) {
696 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL",
697 __func__);
698 return -EINVAL;
699 }
700 if (params == NULL) {
701 PIPELINE_LOG(ERR, "%s: params parameter NULL", __func__);
702 return -EINVAL;
703 }
704 if (port_id == NULL) {
705 PIPELINE_LOG(ERR, "%s: port_id parameter NULL",
706 __func__);
707 return -EINVAL;
708 }
709
710 /* ops */
711 if (params->ops == NULL) {
712 PIPELINE_LOG(ERR, "%s: params->ops parameter NULL",
713 __func__);
714 return -EINVAL;
715 }
716
717 if (params->ops->f_create == NULL) {
718 PIPELINE_LOG(ERR,
719 "%s: f_create function pointer NULL", __func__);
720 return -EINVAL;
721 }
722
723 if (params->ops->f_rx == NULL) {
724 PIPELINE_LOG(ERR, "%s: f_rx function pointer NULL",
725 __func__);
726 return -EINVAL;
727 }
728
729 /* burst_size */
730 if ((params->burst_size == 0) ||
731 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
732 PIPELINE_LOG(ERR, "%s: invalid value for burst_size",
733 __func__);
734 return -EINVAL;
735 }
736
737 /* Do we have room for one more port? */
738 if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
739 PIPELINE_LOG(ERR,
740 "%s: invalid value for num_ports_in", __func__);
741 return -EINVAL;
742 }
743
744 return 0;
745 }
746
747 static int
rte_pipeline_port_out_check_params(struct rte_pipeline * p,struct rte_pipeline_port_out_params * params,uint32_t * port_id)748 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
749 struct rte_pipeline_port_out_params *params,
750 uint32_t *port_id)
751 {
752 if (p == NULL) {
753 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL",
754 __func__);
755 return -EINVAL;
756 }
757
758 if (params == NULL) {
759 PIPELINE_LOG(ERR, "%s: params parameter NULL", __func__);
760 return -EINVAL;
761 }
762
763 if (port_id == NULL) {
764 PIPELINE_LOG(ERR, "%s: port_id parameter NULL",
765 __func__);
766 return -EINVAL;
767 }
768
769 /* ops */
770 if (params->ops == NULL) {
771 PIPELINE_LOG(ERR, "%s: params->ops parameter NULL",
772 __func__);
773 return -EINVAL;
774 }
775
776 if (params->ops->f_create == NULL) {
777 PIPELINE_LOG(ERR,
778 "%s: f_create function pointer NULL", __func__);
779 return -EINVAL;
780 }
781
782 if (params->ops->f_tx == NULL) {
783 PIPELINE_LOG(ERR,
784 "%s: f_tx function pointer NULL", __func__);
785 return -EINVAL;
786 }
787
788 if (params->ops->f_tx_bulk == NULL) {
789 PIPELINE_LOG(ERR,
790 "%s: f_tx_bulk function pointer NULL", __func__);
791 return -EINVAL;
792 }
793
794 /* Do we have room for one more port? */
795 if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
796 PIPELINE_LOG(ERR,
797 "%s: invalid value for num_ports_out", __func__);
798 return -EINVAL;
799 }
800
801 return 0;
802 }
803
804 int
rte_pipeline_port_in_create(struct rte_pipeline * p,struct rte_pipeline_port_in_params * params,uint32_t * port_id)805 rte_pipeline_port_in_create(struct rte_pipeline *p,
806 struct rte_pipeline_port_in_params *params,
807 uint32_t *port_id)
808 {
809 struct rte_port_in *port;
810 void *h_port;
811 uint32_t id;
812 int status;
813
814 /* Check input arguments */
815 status = rte_pipeline_port_in_check_params(p, params, port_id);
816 if (status != 0)
817 return status;
818
819 id = p->num_ports_in;
820 port = &p->ports_in[id];
821
822 /* Create the port */
823 h_port = params->ops->f_create(params->arg_create, p->socket_id);
824 if (h_port == NULL) {
825 PIPELINE_LOG(ERR, "%s: Port creation failed", __func__);
826 return -EINVAL;
827 }
828
829 /* Commit current table to the pipeline */
830 p->num_ports_in++;
831 *port_id = id;
832
833 /* Save input parameters */
834 memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
835 port->f_action = params->f_action;
836 port->arg_ah = params->arg_ah;
837 port->burst_size = params->burst_size;
838
839 /* Initialize port internal data structure */
840 port->table_id = RTE_TABLE_INVALID;
841 port->h_port = h_port;
842 port->next = NULL;
843
844 return 0;
845 }
846
847 void
rte_pipeline_port_in_free(struct rte_port_in * port)848 rte_pipeline_port_in_free(struct rte_port_in *port)
849 {
850 if (port->ops.f_free != NULL)
851 port->ops.f_free(port->h_port);
852 }
853
854 int
rte_pipeline_port_out_create(struct rte_pipeline * p,struct rte_pipeline_port_out_params * params,uint32_t * port_id)855 rte_pipeline_port_out_create(struct rte_pipeline *p,
856 struct rte_pipeline_port_out_params *params,
857 uint32_t *port_id)
858 {
859 struct rte_port_out *port;
860 void *h_port;
861 uint32_t id;
862 int status;
863
864 /* Check input arguments */
865 status = rte_pipeline_port_out_check_params(p, params, port_id);
866 if (status != 0)
867 return status;
868
869 id = p->num_ports_out;
870 port = &p->ports_out[id];
871
872 /* Create the port */
873 h_port = params->ops->f_create(params->arg_create, p->socket_id);
874 if (h_port == NULL) {
875 PIPELINE_LOG(ERR, "%s: Port creation failed", __func__);
876 return -EINVAL;
877 }
878
879 /* Commit current table to the pipeline */
880 p->num_ports_out++;
881 *port_id = id;
882
883 /* Save input parameters */
884 memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
885 port->f_action = params->f_action;
886 port->arg_ah = params->arg_ah;
887
888 /* Initialize port internal data structure */
889 port->h_port = h_port;
890
891 return 0;
892 }
893
894 void
rte_pipeline_port_out_free(struct rte_port_out * port)895 rte_pipeline_port_out_free(struct rte_port_out *port)
896 {
897 if (port->ops.f_free != NULL)
898 port->ops.f_free(port->h_port);
899 }
900
901 int
rte_pipeline_port_in_connect_to_table(struct rte_pipeline * p,uint32_t port_id,uint32_t table_id)902 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
903 uint32_t port_id,
904 uint32_t table_id)
905 {
906 struct rte_port_in *port;
907
908 /* Check input arguments */
909 if (p == NULL) {
910 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL",
911 __func__);
912 return -EINVAL;
913 }
914
915 if (port_id >= p->num_ports_in) {
916 PIPELINE_LOG(ERR,
917 "%s: port IN ID %u is out of range",
918 __func__, port_id);
919 return -EINVAL;
920 }
921
922 if (table_id >= p->num_tables) {
923 PIPELINE_LOG(ERR,
924 "%s: Table ID %u is out of range",
925 __func__, table_id);
926 return -EINVAL;
927 }
928
929 port = &p->ports_in[port_id];
930 port->table_id = table_id;
931
932 return 0;
933 }
934
935 int
rte_pipeline_port_in_enable(struct rte_pipeline * p,uint32_t port_id)936 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
937 {
938 struct rte_port_in *port, *port_prev, *port_next;
939 uint64_t port_mask;
940 uint32_t port_prev_id, port_next_id;
941
942 /* Check input arguments */
943 if (p == NULL) {
944 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL",
945 __func__);
946 return -EINVAL;
947 }
948
949 if (port_id >= p->num_ports_in) {
950 PIPELINE_LOG(ERR,
951 "%s: port IN ID %u is out of range",
952 __func__, port_id);
953 return -EINVAL;
954 }
955
956 port = &p->ports_in[port_id];
957
958 /* Return if current input port is already enabled */
959 port_mask = 1LLU << port_id;
960 if (p->enabled_port_in_mask & port_mask)
961 return 0;
962
963 p->enabled_port_in_mask |= port_mask;
964
965 /* Add current input port to the pipeline chain of enabled ports */
966 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
967 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
968
969 port_prev = &p->ports_in[port_prev_id];
970 port_next = &p->ports_in[port_next_id];
971
972 port_prev->next = port;
973 port->next = port_next;
974
975 /* Check if list of enabled ports was previously empty */
976 if (p->enabled_port_in_mask == port_mask)
977 p->port_in_next = port;
978
979 return 0;
980 }
981
982 int
rte_pipeline_port_in_disable(struct rte_pipeline * p,uint32_t port_id)983 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
984 {
985 struct rte_port_in *port, *port_prev, *port_next;
986 uint64_t port_mask;
987 uint32_t port_prev_id, port_next_id;
988
989 /* Check input arguments */
990 if (p == NULL) {
991 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL",
992 __func__);
993 return -EINVAL;
994 }
995
996 if (port_id >= p->num_ports_in) {
997 PIPELINE_LOG(ERR, "%s: port IN ID %u is out of range",
998 __func__, port_id);
999 return -EINVAL;
1000 }
1001
1002 port = &p->ports_in[port_id];
1003
1004 /* Return if current input port is already disabled */
1005 port_mask = 1LLU << port_id;
1006 if ((p->enabled_port_in_mask & port_mask) == 0)
1007 return 0;
1008
1009 p->enabled_port_in_mask &= ~port_mask;
1010
1011 /* Return if no other enabled ports */
1012 if (p->enabled_port_in_mask == 0) {
1013 p->port_in_next = NULL;
1014
1015 return 0;
1016 }
1017
1018 /* Add current input port to the pipeline chain of enabled ports */
1019 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
1020 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1021
1022 port_prev = &p->ports_in[port_prev_id];
1023 port_next = &p->ports_in[port_next_id];
1024
1025 port_prev->next = port_next;
1026
1027 /* Check if the port which has just been disabled is next to serve */
1028 if (port == p->port_in_next)
1029 p->port_in_next = port_next;
1030
1031 return 0;
1032 }
1033
1034 /*
1035 * Pipeline run-time
1036 */
1037 int
rte_pipeline_check(struct rte_pipeline * p)1038 rte_pipeline_check(struct rte_pipeline *p)
1039 {
1040 uint32_t port_in_id;
1041
1042 /* Check input arguments */
1043 if (p == NULL) {
1044 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL",
1045 __func__);
1046 return -EINVAL;
1047 }
1048
1049 /* Check that pipeline has at least one input port, one table and one
1050 output port */
1051 if (p->num_ports_in == 0) {
1052 PIPELINE_LOG(ERR, "%s: must have at least 1 input port",
1053 __func__);
1054 return -EINVAL;
1055 }
1056 if (p->num_tables == 0) {
1057 PIPELINE_LOG(ERR, "%s: must have at least 1 table",
1058 __func__);
1059 return -EINVAL;
1060 }
1061 if (p->num_ports_out == 0) {
1062 PIPELINE_LOG(ERR, "%s: must have at least 1 output port",
1063 __func__);
1064 return -EINVAL;
1065 }
1066
1067 /* Check that all input ports are connected */
1068 for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
1069 struct rte_port_in *port_in = &p->ports_in[port_in_id];
1070
1071 if (port_in->table_id == RTE_TABLE_INVALID) {
1072 PIPELINE_LOG(ERR,
1073 "%s: Port IN ID %u is not connected",
1074 __func__, port_in_id);
1075 return -EINVAL;
1076 }
1077 }
1078
1079 return 0;
1080 }
1081
1082 static inline void
rte_pipeline_compute_masks(struct rte_pipeline * p,uint64_t pkts_mask)1083 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1084 {
1085 p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1086 p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1087 p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1088 p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1089
1090 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1091 uint64_t n_pkts = rte_popcount64(pkts_mask);
1092 uint32_t i;
1093
1094 for (i = 0; i < n_pkts; i++) {
1095 uint64_t pkt_mask = 1LLU << i;
1096 uint32_t pos = p->entries[i]->action;
1097
1098 p->action_mask1[pos] |= pkt_mask;
1099 }
1100 } else {
1101 uint32_t i;
1102
1103 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1104 uint64_t pkt_mask = 1LLU << i;
1105 uint32_t pos;
1106
1107 if ((pkt_mask & pkts_mask) == 0)
1108 continue;
1109
1110 pos = p->entries[i]->action;
1111 p->action_mask1[pos] |= pkt_mask;
1112 }
1113 }
1114 }
1115
1116 static inline void
rte_pipeline_action_handler_port_bulk(struct rte_pipeline * p,uint64_t pkts_mask,uint32_t port_id)1117 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1118 uint64_t pkts_mask, uint32_t port_id)
1119 {
1120 struct rte_port_out *port_out = &p->ports_out[port_id];
1121
1122 p->pkts_mask = pkts_mask;
1123
1124 /* Output port user actions */
1125 if (port_out->f_action != NULL) {
1126 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah);
1127
1128 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1129 port_out->n_pkts_dropped_by_ah);
1130 }
1131
1132 /* Output port TX */
1133 if (p->pkts_mask != 0)
1134 port_out->ops.f_tx_bulk(port_out->h_port,
1135 p->pkts,
1136 p->pkts_mask);
1137 }
1138
1139 static inline void
rte_pipeline_action_handler_port(struct rte_pipeline * p,uint64_t pkts_mask)1140 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1141 {
1142 p->pkts_mask = pkts_mask;
1143
1144 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1145 uint64_t n_pkts = rte_popcount64(pkts_mask);
1146 uint32_t i;
1147
1148 for (i = 0; i < n_pkts; i++) {
1149 struct rte_mbuf *pkt = p->pkts[i];
1150 uint32_t port_out_id = p->entries[i]->port_id;
1151 struct rte_port_out *port_out =
1152 &p->ports_out[port_out_id];
1153
1154 /* Output port user actions */
1155 if (port_out->f_action == NULL) /* Output port TX */
1156 port_out->ops.f_tx(port_out->h_port, pkt);
1157 else {
1158 uint64_t pkt_mask = 1LLU << i;
1159
1160 port_out->f_action(p,
1161 p->pkts,
1162 pkt_mask,
1163 port_out->arg_ah);
1164
1165 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1166 port_out->n_pkts_dropped_by_ah);
1167
1168 /* Output port TX */
1169 if (pkt_mask & p->pkts_mask)
1170 port_out->ops.f_tx(port_out->h_port,
1171 pkt);
1172 }
1173 }
1174 } else {
1175 uint32_t i;
1176
1177 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1178 uint64_t pkt_mask = 1LLU << i;
1179 struct rte_mbuf *pkt;
1180 struct rte_port_out *port_out;
1181 uint32_t port_out_id;
1182
1183 if ((pkt_mask & pkts_mask) == 0)
1184 continue;
1185
1186 pkt = p->pkts[i];
1187 port_out_id = p->entries[i]->port_id;
1188 port_out = &p->ports_out[port_out_id];
1189
1190 /* Output port user actions */
1191 if (port_out->f_action == NULL) /* Output port TX */
1192 port_out->ops.f_tx(port_out->h_port, pkt);
1193 else {
1194 port_out->f_action(p,
1195 p->pkts,
1196 pkt_mask,
1197 port_out->arg_ah);
1198
1199 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1200 port_out->n_pkts_dropped_by_ah);
1201
1202 /* Output port TX */
1203 if (pkt_mask & p->pkts_mask)
1204 port_out->ops.f_tx(port_out->h_port,
1205 pkt);
1206 }
1207 }
1208 }
1209 }
1210
1211 static inline void
rte_pipeline_action_handler_port_meta(struct rte_pipeline * p,uint64_t pkts_mask)1212 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1213 uint64_t pkts_mask)
1214 {
1215 p->pkts_mask = pkts_mask;
1216
1217 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1218 uint64_t n_pkts = rte_popcount64(pkts_mask);
1219 uint32_t i;
1220
1221 for (i = 0; i < n_pkts; i++) {
1222 struct rte_mbuf *pkt = p->pkts[i];
1223 uint32_t port_out_id =
1224 RTE_MBUF_METADATA_UINT32(pkt,
1225 p->offset_port_id);
1226 struct rte_port_out *port_out = &p->ports_out[
1227 port_out_id];
1228
1229 /* Output port user actions */
1230 if (port_out->f_action == NULL) /* Output port TX */
1231 port_out->ops.f_tx(port_out->h_port, pkt);
1232 else {
1233 uint64_t pkt_mask = 1LLU << i;
1234
1235 port_out->f_action(p,
1236 p->pkts,
1237 pkt_mask,
1238 port_out->arg_ah);
1239
1240 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1241 port_out->n_pkts_dropped_by_ah);
1242
1243 /* Output port TX */
1244 if (pkt_mask & p->pkts_mask)
1245 port_out->ops.f_tx(port_out->h_port,
1246 pkt);
1247 }
1248 }
1249 } else {
1250 uint32_t i;
1251
1252 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1253 uint64_t pkt_mask = 1LLU << i;
1254 struct rte_mbuf *pkt;
1255 struct rte_port_out *port_out;
1256 uint32_t port_out_id;
1257
1258 if ((pkt_mask & pkts_mask) == 0)
1259 continue;
1260
1261 pkt = p->pkts[i];
1262 port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1263 p->offset_port_id);
1264 port_out = &p->ports_out[port_out_id];
1265
1266 /* Output port user actions */
1267 if (port_out->f_action == NULL) /* Output port TX */
1268 port_out->ops.f_tx(port_out->h_port, pkt);
1269 else {
1270 port_out->f_action(p,
1271 p->pkts,
1272 pkt_mask,
1273 port_out->arg_ah);
1274
1275 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1276 port_out->n_pkts_dropped_by_ah);
1277
1278 /* Output port TX */
1279 if (pkt_mask & p->pkts_mask)
1280 port_out->ops.f_tx(port_out->h_port,
1281 pkt);
1282 }
1283 }
1284 }
1285 }
1286
1287 static inline void
rte_pipeline_action_handler_drop(struct rte_pipeline * p,uint64_t pkts_mask)1288 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1289 {
1290 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1291 uint64_t n_pkts = rte_popcount64(pkts_mask);
1292 uint32_t i;
1293
1294 for (i = 0; i < n_pkts; i++)
1295 rte_pktmbuf_free(p->pkts[i]);
1296 } else {
1297 uint32_t i;
1298
1299 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1300 uint64_t pkt_mask = 1LLU << i;
1301
1302 if ((pkt_mask & pkts_mask) == 0)
1303 continue;
1304
1305 rte_pktmbuf_free(p->pkts[i]);
1306 }
1307 }
1308 }
1309
1310 int
rte_pipeline_run(struct rte_pipeline * p)1311 rte_pipeline_run(struct rte_pipeline *p)
1312 {
1313 struct rte_port_in *port_in = p->port_in_next;
1314 uint32_t n_pkts, table_id;
1315
1316 if (port_in == NULL)
1317 return 0;
1318
1319 /* Input port RX */
1320 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1321 port_in->burst_size);
1322 if (n_pkts == 0) {
1323 p->port_in_next = port_in->next;
1324 return 0;
1325 }
1326
1327 p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1328 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1329 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1330 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1331 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1332
1333 /* Input port user actions */
1334 if (port_in->f_action != NULL) {
1335 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah);
1336
1337 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1338 port_in->n_pkts_dropped_by_ah);
1339 }
1340
1341 /* Table */
1342 for (table_id = port_in->table_id; p->pkts_mask != 0; ) {
1343 struct rte_table *table;
1344 uint64_t lookup_hit_mask, lookup_miss_mask;
1345
1346 /* Lookup */
1347 table = &p->tables[table_id];
1348 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask,
1349 &lookup_hit_mask, (void **) p->entries);
1350 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask);
1351
1352 /* Lookup miss */
1353 if (lookup_miss_mask != 0) {
1354 struct rte_pipeline_table_entry *default_entry =
1355 table->default_entry;
1356
1357 p->pkts_mask = lookup_miss_mask;
1358
1359 /* Table user actions */
1360 if (table->f_action_miss != NULL) {
1361 table->f_action_miss(p,
1362 p->pkts,
1363 lookup_miss_mask,
1364 default_entry,
1365 table->arg_ah);
1366
1367 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1368 table->n_pkts_dropped_by_lkp_miss_ah);
1369 }
1370
1371 /* Table reserved actions */
1372 if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) &&
1373 (p->pkts_mask != 0))
1374 rte_pipeline_action_handler_port_bulk(p,
1375 p->pkts_mask,
1376 default_entry->port_id);
1377 else {
1378 uint32_t pos = default_entry->action;
1379
1380 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1381
1382 p->action_mask0[pos] |= p->pkts_mask;
1383
1384 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1385 table->n_pkts_dropped_lkp_miss);
1386 }
1387 }
1388
1389 /* Lookup hit */
1390 if (lookup_hit_mask != 0) {
1391 p->pkts_mask = lookup_hit_mask;
1392
1393 /* Table user actions */
1394 if (table->f_action_hit != NULL) {
1395 table->f_action_hit(p,
1396 p->pkts,
1397 lookup_hit_mask,
1398 p->entries,
1399 table->arg_ah);
1400
1401 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1402 table->n_pkts_dropped_by_lkp_hit_ah);
1403 }
1404
1405 /* Table reserved actions */
1406 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1407 rte_pipeline_compute_masks(p, p->pkts_mask);
1408 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1409 p->action_mask1[
1410 RTE_PIPELINE_ACTION_DROP];
1411 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1412 p->action_mask1[
1413 RTE_PIPELINE_ACTION_PORT];
1414 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1415 p->action_mask1[
1416 RTE_PIPELINE_ACTION_PORT_META];
1417 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1418 p->action_mask1[
1419 RTE_PIPELINE_ACTION_TABLE];
1420
1421 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1422 table->n_pkts_dropped_lkp_hit);
1423 }
1424
1425 /* Prepare for next iteration */
1426 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1427 table_id = table->table_next_id;
1428 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1429 }
1430
1431 /* Table reserved action PORT */
1432 rte_pipeline_action_handler_port(p,
1433 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1434
1435 /* Table reserved action PORT META */
1436 rte_pipeline_action_handler_port_meta(p,
1437 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1438
1439 /* Table reserved action DROP */
1440 rte_pipeline_action_handler_drop(p,
1441 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1442
1443 /* Pick candidate for next port IN to serve */
1444 p->port_in_next = port_in->next;
1445
1446 return (int) n_pkts;
1447 }
1448
1449 int
rte_pipeline_flush(struct rte_pipeline * p)1450 rte_pipeline_flush(struct rte_pipeline *p)
1451 {
1452 uint32_t port_id;
1453
1454 /* Check input arguments */
1455 if (p == NULL) {
1456 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL",
1457 __func__);
1458 return -EINVAL;
1459 }
1460
1461 for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1462 struct rte_port_out *port = &p->ports_out[port_id];
1463
1464 if (port->ops.f_flush != NULL)
1465 port->ops.f_flush(port->h_port);
1466 }
1467
1468 return 0;
1469 }
1470
1471 int
rte_pipeline_port_out_packet_insert(struct rte_pipeline * p,uint32_t port_id,struct rte_mbuf * pkt)1472 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1473 uint32_t port_id, struct rte_mbuf *pkt)
1474 {
1475 struct rte_port_out *port_out = &p->ports_out[port_id];
1476
1477 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1478
1479 return 0;
1480 }
1481
rte_pipeline_ah_packet_hijack(struct rte_pipeline * p,uint64_t pkts_mask)1482 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p,
1483 uint64_t pkts_mask)
1484 {
1485 pkts_mask &= p->pkts_mask;
1486 p->pkts_mask &= ~pkts_mask;
1487
1488 return 0;
1489 }
1490
rte_pipeline_ah_packet_drop(struct rte_pipeline * p,uint64_t pkts_mask)1491 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p,
1492 uint64_t pkts_mask)
1493 {
1494 pkts_mask &= p->pkts_mask;
1495 p->pkts_mask &= ~pkts_mask;
1496 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask;
1497
1498 RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask);
1499 return 0;
1500 }
1501
rte_pipeline_port_in_stats_read(struct rte_pipeline * p,uint32_t port_id,struct rte_pipeline_port_in_stats * stats,int clear)1502 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1503 struct rte_pipeline_port_in_stats *stats, int clear)
1504 {
1505 struct rte_port_in *port;
1506 int retval;
1507
1508 if (p == NULL) {
1509 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL",
1510 __func__);
1511 return -EINVAL;
1512 }
1513
1514 if (port_id >= p->num_ports_in) {
1515 PIPELINE_LOG(ERR,
1516 "%s: port IN ID %u is out of range",
1517 __func__, port_id);
1518 return -EINVAL;
1519 }
1520
1521 port = &p->ports_in[port_id];
1522
1523 if (port->ops.f_stats != NULL) {
1524 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1525 if (retval)
1526 return retval;
1527 } else if (stats != NULL)
1528 memset(&stats->stats, 0, sizeof(stats->stats));
1529
1530 if (stats != NULL)
1531 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1532
1533 if (clear != 0)
1534 port->n_pkts_dropped_by_ah = 0;
1535
1536 return 0;
1537 }
1538
rte_pipeline_port_out_stats_read(struct rte_pipeline * p,uint32_t port_id,struct rte_pipeline_port_out_stats * stats,int clear)1539 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1540 struct rte_pipeline_port_out_stats *stats, int clear)
1541 {
1542 struct rte_port_out *port;
1543 int retval;
1544
1545 if (p == NULL) {
1546 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL", __func__);
1547 return -EINVAL;
1548 }
1549
1550 if (port_id >= p->num_ports_out) {
1551 PIPELINE_LOG(ERR,
1552 "%s: port OUT ID %u is out of range", __func__, port_id);
1553 return -EINVAL;
1554 }
1555
1556 port = &p->ports_out[port_id];
1557 if (port->ops.f_stats != NULL) {
1558 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1559 if (retval != 0)
1560 return retval;
1561 } else if (stats != NULL)
1562 memset(&stats->stats, 0, sizeof(stats->stats));
1563
1564 if (stats != NULL)
1565 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1566
1567 if (clear != 0)
1568 port->n_pkts_dropped_by_ah = 0;
1569
1570 return 0;
1571 }
1572
rte_pipeline_table_stats_read(struct rte_pipeline * p,uint32_t table_id,struct rte_pipeline_table_stats * stats,int clear)1573 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1574 struct rte_pipeline_table_stats *stats, int clear)
1575 {
1576 struct rte_table *table;
1577 int retval;
1578
1579 if (p == NULL) {
1580 PIPELINE_LOG(ERR, "%s: pipeline parameter NULL",
1581 __func__);
1582 return -EINVAL;
1583 }
1584
1585 if (table_id >= p->num_tables) {
1586 PIPELINE_LOG(ERR,
1587 "%s: table %u is out of range", __func__, table_id);
1588 return -EINVAL;
1589 }
1590
1591 table = &p->tables[table_id];
1592 if (table->ops.f_stats != NULL) {
1593 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1594 if (retval != 0)
1595 return retval;
1596 } else if (stats != NULL)
1597 memset(&stats->stats, 0, sizeof(stats->stats));
1598
1599 if (stats != NULL) {
1600 stats->n_pkts_dropped_by_lkp_hit_ah =
1601 table->n_pkts_dropped_by_lkp_hit_ah;
1602 stats->n_pkts_dropped_by_lkp_miss_ah =
1603 table->n_pkts_dropped_by_lkp_miss_ah;
1604 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1605 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1606 }
1607
1608 if (clear != 0) {
1609 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1610 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1611 table->n_pkts_dropped_lkp_hit = 0;
1612 table->n_pkts_dropped_lkp_miss = 0;
1613 }
1614
1615 return 0;
1616 }
1617