1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 Intel Corporation
3 */
4 #ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
5 #define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
6
7 #include <inttypes.h>
8 #include <string.h>
9 #include <sys/queue.h>
10
11 #include <rte_bitops.h>
12 #include <rte_byteorder.h>
13 #include <rte_common.h>
14 #include <rte_cycles.h>
15 #include <rte_prefetch.h>
16 #include <rte_meter.h>
17
18 #include <rte_swx_table_selector.h>
19 #include <rte_swx_table_learner.h>
20 #include <rte_swx_pipeline.h>
21 #include <rte_swx_ctl.h>
22
23 #ifndef TRACE_LEVEL
24 #define TRACE_LEVEL 0
25 #endif
26
27 #if TRACE_LEVEL
28 #define TRACE(...) printf(__VA_ARGS__)
29 #else
30 #define TRACE(...)
31 #endif
32
33 /*
34 * Environment.
35 */
36 #define ntoh64(x) rte_be_to_cpu_64(x)
37 #define hton64(x) rte_cpu_to_be_64(x)
38
39 /*
40 * Struct.
41 */
42 struct field {
43 char name[RTE_SWX_NAME_SIZE];
44 uint32_t n_bits;
45 uint32_t offset;
46 int var_size;
47 };
48
49 struct struct_type {
50 TAILQ_ENTRY(struct_type) node;
51 char name[RTE_SWX_NAME_SIZE];
52 struct field *fields;
53 uint32_t n_fields;
54 uint32_t n_bits;
55 uint32_t n_bits_min;
56 int var_size;
57 };
58
59 TAILQ_HEAD(struct_type_tailq, struct_type);
60
61 /*
62 * Input port.
63 */
64 struct port_in_type {
65 TAILQ_ENTRY(port_in_type) node;
66 char name[RTE_SWX_NAME_SIZE];
67 struct rte_swx_port_in_ops ops;
68 };
69
70 TAILQ_HEAD(port_in_type_tailq, port_in_type);
71
72 struct port_in {
73 TAILQ_ENTRY(port_in) node;
74 struct port_in_type *type;
75 void *obj;
76 uint32_t id;
77 };
78
79 TAILQ_HEAD(port_in_tailq, port_in);
80
81 struct port_in_runtime {
82 rte_swx_port_in_pkt_rx_t pkt_rx;
83 void *obj;
84 };
85
86 /*
87 * Output port.
88 */
89 struct port_out_type {
90 TAILQ_ENTRY(port_out_type) node;
91 char name[RTE_SWX_NAME_SIZE];
92 struct rte_swx_port_out_ops ops;
93 };
94
95 TAILQ_HEAD(port_out_type_tailq, port_out_type);
96
97 struct port_out {
98 TAILQ_ENTRY(port_out) node;
99 struct port_out_type *type;
100 void *obj;
101 uint32_t id;
102 };
103
104 TAILQ_HEAD(port_out_tailq, port_out);
105
106 struct port_out_runtime {
107 rte_swx_port_out_pkt_tx_t pkt_tx;
108 rte_swx_port_out_pkt_fast_clone_tx_t pkt_fast_clone_tx;
109 rte_swx_port_out_pkt_clone_tx_t pkt_clone_tx;
110 rte_swx_port_out_flush_t flush;
111 void *obj;
112 };
113
114 /*
115 * Packet mirroring.
116 */
117 struct mirroring_session {
118 uint32_t port_id;
119 int fast_clone;
120 uint32_t truncation_length;
121 };
122
123 /*
124 * Extern object.
125 */
126 struct extern_type_member_func {
127 TAILQ_ENTRY(extern_type_member_func) node;
128 char name[RTE_SWX_NAME_SIZE];
129 rte_swx_extern_type_member_func_t func;
130 uint32_t id;
131 };
132
133 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
134
135 struct extern_type {
136 TAILQ_ENTRY(extern_type) node;
137 char name[RTE_SWX_NAME_SIZE];
138 struct struct_type *mailbox_struct_type;
139 rte_swx_extern_type_constructor_t constructor;
140 rte_swx_extern_type_destructor_t destructor;
141 struct extern_type_member_func_tailq funcs;
142 uint32_t n_funcs;
143 };
144
145 TAILQ_HEAD(extern_type_tailq, extern_type);
146
147 struct extern_obj {
148 TAILQ_ENTRY(extern_obj) node;
149 char name[RTE_SWX_NAME_SIZE];
150 struct extern_type *type;
151 void *obj;
152 uint32_t struct_id;
153 uint32_t id;
154 };
155
156 TAILQ_HEAD(extern_obj_tailq, extern_obj);
157
158 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
159 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
160 #endif
161
162 struct extern_obj_runtime {
163 void *obj;
164 uint8_t *mailbox;
165 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
166 };
167
168 /*
169 * Extern function.
170 */
171 struct extern_func {
172 TAILQ_ENTRY(extern_func) node;
173 char name[RTE_SWX_NAME_SIZE];
174 struct struct_type *mailbox_struct_type;
175 rte_swx_extern_func_t func;
176 uint32_t struct_id;
177 uint32_t id;
178 };
179
180 TAILQ_HEAD(extern_func_tailq, extern_func);
181
182 struct extern_func_runtime {
183 uint8_t *mailbox;
184 rte_swx_extern_func_t func;
185 };
186
187 /*
188 * Hash function.
189 */
190 struct hash_func {
191 TAILQ_ENTRY(hash_func) node;
192 char name[RTE_SWX_NAME_SIZE];
193 rte_swx_hash_func_t func;
194 uint32_t id;
195 };
196
197 TAILQ_HEAD(hash_func_tailq, hash_func);
198
199 struct hash_func_runtime {
200 rte_swx_hash_func_t func;
201 };
202
203 /*
204 * RSS.
205 */
206 struct rss {
207 TAILQ_ENTRY(rss) node;
208 char name[RTE_SWX_NAME_SIZE];
209 uint32_t id;
210 };
211
212 TAILQ_HEAD(rss_tailq, rss);
213
214 struct rss_runtime {
215 uint32_t key_size; /* key size in bytes. */
216 uint8_t key[]; /* key. */
217 };
218
219 /*
220 * Header.
221 */
222 struct header {
223 TAILQ_ENTRY(header) node;
224 char name[RTE_SWX_NAME_SIZE];
225 struct struct_type *st;
226 uint32_t struct_id;
227 uint32_t id;
228 };
229
230 TAILQ_HEAD(header_tailq, header);
231
232 struct header_runtime {
233 uint8_t *ptr0;
234 uint32_t n_bytes;
235 };
236
237 struct header_out_runtime {
238 uint8_t *ptr0;
239 uint8_t *ptr;
240 uint32_t n_bytes;
241 };
242
243 /*
244 * Instruction.
245 */
246
247 /* Operand endianness conventions:
248 *
249 * Case 1: Small fields (i.e. fields with size <= 64 bits)
250 *
251 * Packet headers are always in Network Byte Order (NBO), i.e. big endian.
252 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
253 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
254 * when transferred to packet meta-data and in NBO when transferred to packet
255 * headers.
256 *
257 * Notation conventions:
258 * -Header field: H = h.header.field (dst/src)
259 * -Meta-data field: M = m.field (dst/src)
260 * -Extern object mailbox field: E = e.field (dst/src)
261 * -Extern function mailbox field: F = f.field (dst/src)
262 * -Table action data field: T = t.field (src only)
263 * -Immediate value: I = 32-bit unsigned value (src only)
264 *
265 * Case 2: Big fields (i.e. fields with size > 64 bits)
266 *
267 * The big fields are allowed in both headers and meta-data, but they are always
268 * stored in NBO. This is why the few instructions that accept a big field
269 * operand require that the other operand, in case it is a small operand, be
270 * stored in NBO as well, i.e. the small operand must be a header field
271 * (i.e. meta-data field not allowed in this case).
272 *
273 * Notation conventions:
274 * -Header or meta-data big field: HM-NBO.
275 */
276
277 enum instruction_type {
278 /* rx m.port_in */
279 INSTR_RX,
280
281 /* tx port_out
282 * port_out = MI
283 */
284 INSTR_TX, /* port_out = M */
285 INSTR_TX_I, /* port_out = I */
286 INSTR_DROP,
287
288 /*
289 * mirror slot_id session_id
290 * slot_id = MEFT
291 * session_id = MEFT
292 */
293 INSTR_MIRROR,
294
295 /* recirculate
296 */
297 INSTR_RECIRCULATE,
298
299 /* recircid m.recirc_pass_id
300 * Read the internal recirculation pass ID into the specified meta-data field.
301 */
302 INSTR_RECIRCID,
303
304 /* extract h.header */
305 INSTR_HDR_EXTRACT,
306 INSTR_HDR_EXTRACT2,
307 INSTR_HDR_EXTRACT3,
308 INSTR_HDR_EXTRACT4,
309 INSTR_HDR_EXTRACT5,
310 INSTR_HDR_EXTRACT6,
311 INSTR_HDR_EXTRACT7,
312 INSTR_HDR_EXTRACT8,
313
314 /* extract h.header m.last_field_size */
315 INSTR_HDR_EXTRACT_M,
316
317 /* lookahead h.header */
318 INSTR_HDR_LOOKAHEAD,
319
320 /* emit h.header */
321 INSTR_HDR_EMIT,
322 INSTR_HDR_EMIT_TX,
323 INSTR_HDR_EMIT2_TX,
324 INSTR_HDR_EMIT3_TX,
325 INSTR_HDR_EMIT4_TX,
326 INSTR_HDR_EMIT5_TX,
327 INSTR_HDR_EMIT6_TX,
328 INSTR_HDR_EMIT7_TX,
329 INSTR_HDR_EMIT8_TX,
330
331 /* validate h.header */
332 INSTR_HDR_VALIDATE,
333
334 /* invalidate h.header */
335 INSTR_HDR_INVALIDATE,
336
337 /* mov dst src
338 * dst = src
339 * dst = HMEF, src = HMEFTI
340 */
341 INSTR_MOV, /* dst = MEF, src = MEFT; size(dst) <= 64 bits, size(src) <= 64 bits. */
342 INSTR_MOV_MH, /* dst = MEF, src = H; size(dst) <= 64 bits, size(src) <= 64 bits. */
343 INSTR_MOV_HM, /* dst = H, src = MEFT; size(dst) <= 64 bits, size(src) <= 64 bits. */
344 INSTR_MOV_HH, /* dst = H, src = H; size(dst) <= 64 bits, size(src) <= 64 bits. */
345 INSTR_MOV_DMA, /* dst and src in NBO format. */
346 INSTR_MOV_128, /* dst and src in NBO format, size(dst) = size(src) = 128 bits. */
347 INSTR_MOV_128_64, /* dst and src in NBO format, size(dst) = 128 bits, size(src) = 64 b. */
348 INSTR_MOV_64_128, /* dst and src in NBO format, size(dst) = 64 bits, size(src) = 128 b. */
349 INSTR_MOV_128_32, /* dst and src in NBO format, size(dst) = 128 bits, size(src) = 32 b. */
350 INSTR_MOV_32_128, /* dst and src in NBO format, size(dst) = 32 bits, size(src) = 128 b. */
351 INSTR_MOV_I, /* dst = HMEF, src = I; size(dst) <= 64 bits. */
352
353 /* movh dst src
354 * Read/write the upper half (i.e. bits 127 .. 64) of a 128-bit field into/from a 64-bit
355 * header field:
356 *
357 * dst64 = src128[127:64], where: dst64 = H, src128 = HM-NBO.
358 * dst128[127:64] = src64, where: dst128 = HM-NBO, src64 = H.
359 *
360 * Typically required for operations involving IPv6 addresses.
361 */
362 INSTR_MOVH,
363
364 /* dma h.header t.field
365 * memcpy(h.header, t.field, sizeof(h.header))
366 */
367 INSTR_DMA_HT,
368 INSTR_DMA_HT2,
369 INSTR_DMA_HT3,
370 INSTR_DMA_HT4,
371 INSTR_DMA_HT5,
372 INSTR_DMA_HT6,
373 INSTR_DMA_HT7,
374 INSTR_DMA_HT8,
375
376 /* add dst src
377 * dst += src
378 * dst = HMEF, src = HMEFTI
379 */
380 INSTR_ALU_ADD, /* dst = MEF, src = MEF */
381 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
382 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
383 INSTR_ALU_ADD_HH, /* dst = H, src = H */
384 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
385 INSTR_ALU_ADD_HI, /* dst = H, src = I */
386
387 /* sub dst src
388 * dst -= src
389 * dst = HMEF, src = HMEFTI
390 */
391 INSTR_ALU_SUB, /* dst = MEF, src = MEF */
392 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
393 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
394 INSTR_ALU_SUB_HH, /* dst = H, src = H */
395 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
396 INSTR_ALU_SUB_HI, /* dst = H, src = I */
397
398 /* ckadd dst src
399 * dst = dst '+ src[0:1] '+ src[2:3] '+ ...
400 * dst = H, src = {H, h.header}, '+ = 1's complement addition operator
401 */
402 INSTR_ALU_CKADD_FIELD, /* src = H */
403 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 bytes. */
404 INSTR_ALU_CKADD_STRUCT, /* src = h.header, with sizeof(header) any 4-byte multiple. */
405
406 /* cksub dst src
407 * dst = dst '- src
408 * dst = H, src = H, '- = 1's complement subtraction operator
409 */
410 INSTR_ALU_CKSUB_FIELD,
411
412 /* and dst src
413 * dst &= src
414 * dst = HMEF, src = HMEFTI
415 */
416 INSTR_ALU_AND, /* dst = MEF, src = MEFT */
417 INSTR_ALU_AND_MH, /* dst = MEF, src = H */
418 INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
419 INSTR_ALU_AND_HH, /* dst = H, src = H */
420 INSTR_ALU_AND_I, /* dst = HMEF, src = I */
421
422 /* or dst src
423 * dst |= src
424 * dst = HMEF, src = HMEFTI
425 */
426 INSTR_ALU_OR, /* dst = MEF, src = MEFT */
427 INSTR_ALU_OR_MH, /* dst = MEF, src = H */
428 INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
429 INSTR_ALU_OR_HH, /* dst = H, src = H */
430 INSTR_ALU_OR_I, /* dst = HMEF, src = I */
431
432 /* xor dst src
433 * dst ^= src
434 * dst = HMEF, src = HMEFTI
435 */
436 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
437 INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
438 INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
439 INSTR_ALU_XOR_HH, /* dst = H, src = H */
440 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
441
442 /* shl dst src
443 * dst <<= src
444 * dst = HMEF, src = HMEFTI
445 */
446 INSTR_ALU_SHL, /* dst = MEF, src = MEF */
447 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
448 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
449 INSTR_ALU_SHL_HH, /* dst = H, src = H */
450 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
451 INSTR_ALU_SHL_HI, /* dst = H, src = I */
452
453 /* shr dst src
454 * dst >>= src
455 * dst = HMEF, src = HMEFTI
456 */
457 INSTR_ALU_SHR, /* dst = MEF, src = MEF */
458 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
459 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
460 INSTR_ALU_SHR_HH, /* dst = H, src = H */
461 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
462 INSTR_ALU_SHR_HI, /* dst = H, src = I */
463
464 /* regprefetch REGARRAY index
465 * prefetch REGARRAY[index]
466 * index = HMEFTI
467 */
468 INSTR_REGPREFETCH_RH, /* index = H */
469 INSTR_REGPREFETCH_RM, /* index = MEFT */
470 INSTR_REGPREFETCH_RI, /* index = I */
471
472 /* regrd dst REGARRAY index
473 * dst = REGARRAY[index]
474 * dst = HMEF, index = HMEFTI
475 */
476 INSTR_REGRD_HRH, /* dst = H, index = H */
477 INSTR_REGRD_HRM, /* dst = H, index = MEFT */
478 INSTR_REGRD_HRI, /* dst = H, index = I */
479 INSTR_REGRD_MRH, /* dst = MEF, index = H */
480 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
481 INSTR_REGRD_MRI, /* dst = MEF, index = I */
482
483 /* regwr REGARRAY index src
484 * REGARRAY[index] = src
485 * index = HMEFTI, src = HMEFTI
486 */
487 INSTR_REGWR_RHH, /* index = H, src = H */
488 INSTR_REGWR_RHM, /* index = H, src = MEFT */
489 INSTR_REGWR_RHI, /* index = H, src = I */
490 INSTR_REGWR_RMH, /* index = MEFT, src = H */
491 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
492 INSTR_REGWR_RMI, /* index = MEFT, src = I */
493 INSTR_REGWR_RIH, /* index = I, src = H */
494 INSTR_REGWR_RIM, /* index = I, src = MEFT */
495 INSTR_REGWR_RII, /* index = I, src = I */
496
497 /* regadd REGARRAY index src
498 * REGARRAY[index] += src
499 * index = HMEFTI, src = HMEFTI
500 */
501 INSTR_REGADD_RHH, /* index = H, src = H */
502 INSTR_REGADD_RHM, /* index = H, src = MEFT */
503 INSTR_REGADD_RHI, /* index = H, src = I */
504 INSTR_REGADD_RMH, /* index = MEFT, src = H */
505 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
506 INSTR_REGADD_RMI, /* index = MEFT, src = I */
507 INSTR_REGADD_RIH, /* index = I, src = H */
508 INSTR_REGADD_RIM, /* index = I, src = MEFT */
509 INSTR_REGADD_RII, /* index = I, src = I */
510
511 /* metprefetch METARRAY index
512 * prefetch METARRAY[index]
513 * index = HMEFTI
514 */
515 INSTR_METPREFETCH_H, /* index = H */
516 INSTR_METPREFETCH_M, /* index = MEFT */
517 INSTR_METPREFETCH_I, /* index = I */
518
519 /* meter METARRAY index length color_in color_out
520 * color_out = meter(METARRAY[index], length, color_in)
521 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
522 */
523 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
524 INSTR_METER_HHI, /* index = H, length = H, color_in = I */
525 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
526 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
527 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
528 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
529 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
530 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
531 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
532 INSTR_METER_IHI, /* index = I, length = H, color_in = I */
533 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
534 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
535
536 /* table TABLE */
537 INSTR_TABLE,
538 INSTR_TABLE_AF,
539 INSTR_SELECTOR,
540 INSTR_LEARNER,
541 INSTR_LEARNER_AF,
542
543 /* learn ACTION_NAME [ m.action_first_arg ] m.timeout_id */
544 INSTR_LEARNER_LEARN,
545
546 /* rearm [ m.timeout_id ] */
547 INSTR_LEARNER_REARM,
548 INSTR_LEARNER_REARM_NEW,
549
550 /* forget */
551 INSTR_LEARNER_FORGET,
552
553 /* entryid m.table_entry_id
554 * Read the internal table entry ID into the specified meta-data field.
555 */
556 INSTR_ENTRYID,
557
558 /* extern e.obj.func */
559 INSTR_EXTERN_OBJ,
560
561 /* extern f.func */
562 INSTR_EXTERN_FUNC,
563
564 /* hash HASH_FUNC_NAME dst src_first src_last
565 * Compute hash value over range of struct fields.
566 * dst = M
567 * src_first = HMEFT
568 * src_last = HMEFT
569 * src_first and src_last must be fields within the same struct
570 */
571 INSTR_HASH_FUNC,
572
573 /* rss RSS_OBJ_NAME dst src_first src_last
574 * Compute the RSS hash value over range of struct fields.
575 * dst = M
576 * src_first = HMEFT
577 * src_last = HMEFT
578 * src_first and src_last must be fields within the same struct
579 */
580 INSTR_RSS,
581
582 /* jmp LABEL
583 * Unconditional jump
584 */
585 INSTR_JMP,
586
587 /* jmpv LABEL h.header
588 * Jump if header is valid
589 */
590 INSTR_JMP_VALID,
591
592 /* jmpnv LABEL h.header
593 * Jump if header is invalid
594 */
595 INSTR_JMP_INVALID,
596
597 /* jmph LABEL
598 * Jump if table lookup hit
599 */
600 INSTR_JMP_HIT,
601
602 /* jmpnh LABEL
603 * Jump if table lookup miss
604 */
605 INSTR_JMP_MISS,
606
607 /* jmpa LABEL ACTION
608 * Jump if action run
609 */
610 INSTR_JMP_ACTION_HIT,
611
612 /* jmpna LABEL ACTION
613 * Jump if action not run
614 */
615 INSTR_JMP_ACTION_MISS,
616
617 /* jmpeq LABEL a b
618 * Jump if a is equal to b
619 * a = HMEFT, b = HMEFTI
620 */
621 INSTR_JMP_EQ, /* a = MEFT, b = MEFT */
622 INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
623 INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
624 INSTR_JMP_EQ_HH, /* a = H, b = H */
625 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
626
627 /* jmpneq LABEL a b
628 * Jump if a is not equal to b
629 * a = HMEFT, b = HMEFTI
630 */
631 INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */
632 INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
633 INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
634 INSTR_JMP_NEQ_HH, /* a = H, b = H */
635 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
636
637 /* jmplt LABEL a b
638 * Jump if a is less than b
639 * a = HMEFT, b = HMEFTI
640 */
641 INSTR_JMP_LT, /* a = MEFT, b = MEFT */
642 INSTR_JMP_LT_MH, /* a = MEFT, b = H */
643 INSTR_JMP_LT_HM, /* a = H, b = MEFT */
644 INSTR_JMP_LT_HH, /* a = H, b = H */
645 INSTR_JMP_LT_MI, /* a = MEFT, b = I */
646 INSTR_JMP_LT_HI, /* a = H, b = I */
647
648 /* jmpgt LABEL a b
649 * Jump if a is greater than b
650 * a = HMEFT, b = HMEFTI
651 */
652 INSTR_JMP_GT, /* a = MEFT, b = MEFT */
653 INSTR_JMP_GT_MH, /* a = MEFT, b = H */
654 INSTR_JMP_GT_HM, /* a = H, b = MEFT */
655 INSTR_JMP_GT_HH, /* a = H, b = H */
656 INSTR_JMP_GT_MI, /* a = MEFT, b = I */
657 INSTR_JMP_GT_HI, /* a = H, b = I */
658
659 /* return
660 * Return from action
661 */
662 INSTR_RETURN,
663
664 /* Start of custom instructions. */
665 INSTR_CUSTOM_0,
666 };
667
668 struct instr_operand {
669 uint8_t struct_id;
670 uint8_t n_bits;
671 uint8_t offset;
672 uint8_t pad;
673 };
674
675 struct instr_io {
676 struct {
677 union {
678 struct {
679 uint8_t offset;
680 uint8_t n_bits;
681 uint8_t pad[2];
682 };
683
684 uint32_t val;
685 };
686 } io;
687
688 struct {
689 uint8_t header_id[8];
690 uint8_t struct_id[8];
691 uint8_t n_bytes[8];
692 } hdr;
693 };
694
695 struct instr_hdr_validity {
696 uint8_t header_id;
697 uint8_t struct_id;
698 };
699
700 struct instr_table {
701 uint8_t table_id;
702 };
703
704 struct instr_learn {
705 uint8_t action_id;
706 uint8_t mf_first_arg_offset;
707 uint8_t mf_timeout_id_offset;
708 uint8_t mf_timeout_id_n_bits;
709 };
710
711 struct instr_extern_obj {
712 uint8_t ext_obj_id;
713 uint8_t func_id;
714 };
715
716 struct instr_extern_func {
717 uint8_t ext_func_id;
718 };
719
720 struct instr_hash_func {
721 uint8_t hash_func_id;
722
723 struct {
724 uint8_t offset;
725 uint8_t n_bits;
726 } dst;
727
728 struct {
729 uint8_t struct_id;
730 uint16_t offset;
731 uint16_t n_bytes;
732 } src;
733 };
734
735 struct instr_rss {
736 uint8_t rss_obj_id;
737
738 struct {
739 uint8_t offset;
740 uint8_t n_bits;
741 } dst;
742
743 struct {
744 uint8_t struct_id;
745 uint16_t offset;
746 uint16_t n_bytes;
747 } src;
748 };
749
750 struct instr_dst_src {
751 struct instr_operand dst;
752 union {
753 struct instr_operand src;
754 uint64_t src_val;
755 };
756 };
757
758 struct instr_regarray {
759 uint8_t regarray_id;
760 uint8_t pad[3];
761
762 union {
763 struct instr_operand idx;
764 uint32_t idx_val;
765 };
766
767 union {
768 struct instr_operand dstsrc;
769 uint64_t dstsrc_val;
770 };
771 };
772
773 struct instr_meter {
774 uint8_t metarray_id;
775 uint8_t pad[3];
776
777 union {
778 struct instr_operand idx;
779 uint32_t idx_val;
780 };
781
782 struct instr_operand length;
783
784 union {
785 struct instr_operand color_in;
786 uint32_t color_in_val;
787 };
788
789 struct instr_operand color_out;
790 };
791
792 struct instr_dma {
793 struct {
794 uint8_t header_id[8];
795 uint8_t struct_id[8];
796 } dst;
797
798 struct {
799 uint8_t offset[8];
800 } src;
801
802 uint16_t n_bytes[8];
803 };
804
805 struct instr_jmp {
806 struct instruction *ip;
807
808 union {
809 struct instr_operand a;
810 uint8_t header_id;
811 uint8_t action_id;
812 };
813
814 union {
815 struct instr_operand b;
816 uint64_t b_val;
817 };
818 };
819
820 struct instruction {
821 enum instruction_type type;
822 union {
823 struct instr_io io;
824 struct instr_dst_src mirror;
825 struct instr_hdr_validity valid;
826 struct instr_dst_src mov;
827 struct instr_regarray regarray;
828 struct instr_meter meter;
829 struct instr_dma dma;
830 struct instr_dst_src alu;
831 struct instr_table table;
832 struct instr_learn learn;
833 struct instr_extern_obj ext_obj;
834 struct instr_extern_func ext_func;
835 struct instr_hash_func hash_func;
836 struct instr_rss rss;
837 struct instr_jmp jmp;
838 };
839 };
840
841 struct instruction_data {
842 char label[RTE_SWX_NAME_SIZE];
843 char jmp_label[RTE_SWX_NAME_SIZE];
844 uint32_t n_users; /* user = jmp instruction to this instruction. */
845 int invalid;
846 };
847
848 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
849
850 /*
851 * Action.
852 */
853 typedef void
854 (*action_func_t)(struct rte_swx_pipeline *p);
855
856 struct action {
857 TAILQ_ENTRY(action) node;
858 char name[RTE_SWX_NAME_SIZE];
859 struct struct_type *st;
860 int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */
861 struct instruction *instructions;
862 struct instruction_data *instruction_data;
863 uint32_t n_instructions;
864 uint32_t id;
865 };
866
867 TAILQ_HEAD(action_tailq, action);
868
869 /*
870 * Table.
871 */
872 struct table_type {
873 TAILQ_ENTRY(table_type) node;
874 char name[RTE_SWX_NAME_SIZE];
875 enum rte_swx_table_match_type match_type;
876 struct rte_swx_table_ops ops;
877 };
878
879 TAILQ_HEAD(table_type_tailq, table_type);
880
881 struct match_field {
882 enum rte_swx_table_match_type match_type;
883 struct field *field;
884 };
885
886 struct table {
887 TAILQ_ENTRY(table) node;
888 char name[RTE_SWX_NAME_SIZE];
889 char args[RTE_SWX_NAME_SIZE];
890 struct table_type *type; /* NULL when n_fields == 0. */
891
892 /* Match. */
893 struct match_field *fields;
894 uint32_t n_fields;
895 struct header *header; /* Only valid when n_fields > 0. */
896
897 /* Action. */
898 struct action **actions;
899 struct action *default_action;
900 uint8_t *default_action_data;
901 uint32_t n_actions;
902 int default_action_is_const;
903 uint32_t action_data_size_max;
904 int *action_is_for_table_entries;
905 int *action_is_for_default_entry;
906
907 struct hash_func *hf;
908 uint32_t size;
909 uint32_t id;
910 };
911
912 TAILQ_HEAD(table_tailq, table);
913
914 struct table_runtime {
915 rte_swx_table_lookup_t func;
916 void *mailbox;
917 uint8_t **key;
918 };
919
920 struct table_statistics {
921 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
922 uint64_t *n_pkts_action;
923 };
924
925 /*
926 * Selector.
927 */
928 struct selector {
929 TAILQ_ENTRY(selector) node;
930 char name[RTE_SWX_NAME_SIZE];
931
932 struct field *group_id_field;
933 struct field **selector_fields;
934 uint32_t n_selector_fields;
935 struct header *selector_header;
936 struct field *member_id_field;
937
938 uint32_t n_groups_max;
939 uint32_t n_members_per_group_max;
940
941 uint32_t id;
942 };
943
944 TAILQ_HEAD(selector_tailq, selector);
945
946 struct selector_runtime {
947 void *mailbox;
948 uint8_t **group_id_buffer;
949 uint8_t **selector_buffer;
950 uint8_t **member_id_buffer;
951 };
952
953 struct selector_statistics {
954 uint64_t n_pkts;
955 };
956
957 /*
958 * Learner table.
959 */
960 struct learner {
961 TAILQ_ENTRY(learner) node;
962 char name[RTE_SWX_NAME_SIZE];
963
964 /* Match. */
965 struct field **fields;
966 uint32_t n_fields;
967 struct header *header;
968
969 /* Action. */
970 struct action **actions;
971 struct action *default_action;
972 uint8_t *default_action_data;
973 uint32_t n_actions;
974 int default_action_is_const;
975 uint32_t action_data_size_max;
976 int *action_is_for_table_entries;
977 int *action_is_for_default_entry;
978
979 struct hash_func *hf;
980 uint32_t size;
981 uint32_t timeout[RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX];
982 uint32_t n_timeouts;
983 uint32_t id;
984 };
985
986 TAILQ_HEAD(learner_tailq, learner);
987
988 struct learner_runtime {
989 void *mailbox;
990 uint8_t **key;
991 };
992
993 struct learner_statistics {
994 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
995 uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */
996 uint64_t n_pkts_rearm;
997 uint64_t n_pkts_forget;
998 uint64_t *n_pkts_action;
999 };
1000
1001 /*
1002 * Register array.
1003 */
1004 struct regarray {
1005 TAILQ_ENTRY(regarray) node;
1006 char name[RTE_SWX_NAME_SIZE];
1007 uint64_t init_val;
1008 uint32_t size;
1009 uint32_t id;
1010 };
1011
1012 TAILQ_HEAD(regarray_tailq, regarray);
1013
1014 struct regarray_runtime {
1015 uint64_t *regarray;
1016 uint32_t size_mask;
1017 };
1018
1019 /*
1020 * Meter array.
1021 */
1022 struct meter_profile {
1023 TAILQ_ENTRY(meter_profile) node;
1024 char name[RTE_SWX_NAME_SIZE];
1025 struct rte_meter_trtcm_params params;
1026 struct rte_meter_trtcm_profile profile;
1027 uint32_t n_users;
1028 };
1029
1030 TAILQ_HEAD(meter_profile_tailq, meter_profile);
1031
1032 struct metarray {
1033 TAILQ_ENTRY(metarray) node;
1034 char name[RTE_SWX_NAME_SIZE];
1035 uint32_t size;
1036 uint32_t id;
1037 };
1038
1039 TAILQ_HEAD(metarray_tailq, metarray);
1040
1041 struct meter {
1042 struct rte_meter_trtcm m;
1043 struct meter_profile *profile;
1044 enum rte_color color_mask;
1045 uint8_t pad[20];
1046
1047 uint64_t n_pkts[RTE_COLORS];
1048 uint64_t n_bytes[RTE_COLORS];
1049 };
1050
1051 struct metarray_runtime {
1052 struct meter *metarray;
1053 uint32_t size_mask;
1054 };
1055
1056 /*
1057 * Pipeline.
1058 */
1059 struct thread {
1060 /* Packet. */
1061 struct rte_swx_pkt pkt;
1062 uint8_t *ptr;
1063 uint32_t *mirroring_slots;
1064 uint64_t mirroring_slots_mask;
1065 int recirculate;
1066 uint32_t recirc_pass_id;
1067
1068 /* Structures. */
1069 uint8_t **structs;
1070
1071 /* Packet headers. */
1072 struct header_runtime *headers; /* Extracted or generated headers. */
1073 struct header_out_runtime *headers_out; /* Emitted headers. */
1074 uint8_t *header_storage;
1075 uint8_t *header_out_storage;
1076 uint64_t valid_headers;
1077 uint32_t n_headers_out;
1078
1079 /* Packet meta-data. */
1080 uint8_t *metadata;
1081
1082 /* Tables. */
1083 struct table_runtime *tables;
1084 struct selector_runtime *selectors;
1085 struct learner_runtime *learners;
1086 struct rte_swx_table_state *table_state;
1087 uint64_t action_id;
1088 size_t entry_id;
1089 int hit; /* 0 = Miss, 1 = Hit. */
1090 uint32_t learner_id;
1091 uint64_t time;
1092
1093 /* Extern objects and functions. */
1094 struct extern_obj_runtime *extern_objs;
1095 struct extern_func_runtime *extern_funcs;
1096
1097 /* Instructions. */
1098 struct instruction *ip;
1099 struct instruction *ret;
1100 };
1101
1102 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
1103 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
1104 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
1105
1106 #define HEADER_VALID(thread, header_id) \
1107 MASK64_BIT_GET((thread)->valid_headers, header_id)
1108
1109 static inline uint64_t
instr_operand_hbo(struct thread * t,const struct instr_operand * x)1110 instr_operand_hbo(struct thread *t, const struct instr_operand *x)
1111 {
1112 uint8_t *x_struct = t->structs[x->struct_id];
1113 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
1114 uint64_t x64 = *x64_ptr;
1115 uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits);
1116
1117 return x64 & x64_mask;
1118 }
1119
1120 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1121
1122 static inline uint64_t
instr_operand_nbo(struct thread * t,const struct instr_operand * x)1123 instr_operand_nbo(struct thread *t, const struct instr_operand *x)
1124 {
1125 uint8_t *x_struct = t->structs[x->struct_id];
1126 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
1127 uint64_t x64 = *x64_ptr;
1128
1129 return ntoh64(x64) >> (64 - x->n_bits);
1130 }
1131
1132 #else
1133
1134 #define instr_operand_nbo instr_operand_hbo
1135
1136 #endif
1137
1138 #define ALU(thread, ip, operator) \
1139 { \
1140 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1141 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1142 uint64_t dst64 = *dst64_ptr; \
1143 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1144 uint64_t dst = dst64 & dst64_mask; \
1145 \
1146 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1147 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1148 uint64_t src64 = *src64_ptr; \
1149 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1150 uint64_t src = src64 & src64_mask; \
1151 \
1152 uint64_t result = dst operator src; \
1153 \
1154 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1155 }
1156
1157 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1158
1159 #define ALU_MH(thread, ip, operator) \
1160 { \
1161 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1162 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1163 uint64_t dst64 = *dst64_ptr; \
1164 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1165 uint64_t dst = dst64 & dst64_mask; \
1166 \
1167 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1168 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1169 uint64_t src64 = *src64_ptr; \
1170 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1171 \
1172 uint64_t result = dst operator src; \
1173 \
1174 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1175 }
1176
1177 #define ALU_HM(thread, ip, operator) \
1178 { \
1179 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1180 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1181 uint64_t dst64 = *dst64_ptr; \
1182 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1183 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1184 \
1185 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1186 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1187 uint64_t src64 = *src64_ptr; \
1188 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1189 uint64_t src = src64 & src64_mask; \
1190 \
1191 uint64_t result = dst operator src; \
1192 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1193 \
1194 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1195 }
1196
1197 #define ALU_HM_FAST(thread, ip, operator) \
1198 { \
1199 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1200 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1201 uint64_t dst64 = *dst64_ptr; \
1202 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1203 uint64_t dst = dst64 & dst64_mask; \
1204 \
1205 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1206 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1207 uint64_t src64 = *src64_ptr; \
1208 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1209 uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1210 \
1211 uint64_t result = dst operator src; \
1212 \
1213 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1214 }
1215
1216 #define ALU_HH(thread, ip, operator) \
1217 { \
1218 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1219 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1220 uint64_t dst64 = *dst64_ptr; \
1221 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1222 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1223 \
1224 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1225 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1226 uint64_t src64 = *src64_ptr; \
1227 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1228 \
1229 uint64_t result = dst operator src; \
1230 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1231 \
1232 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1233 }
1234
1235 #define ALU_HH_FAST(thread, ip, operator) \
1236 { \
1237 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1238 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1239 uint64_t dst64 = *dst64_ptr; \
1240 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1241 uint64_t dst = dst64 & dst64_mask; \
1242 \
1243 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1244 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1245 uint64_t src64 = *src64_ptr; \
1246 uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1247 \
1248 uint64_t result = dst operator src; \
1249 \
1250 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1251 }
1252
1253 #else
1254
1255 #define ALU_MH ALU
1256 #define ALU_HM ALU
1257 #define ALU_HM_FAST ALU
1258 #define ALU_HH ALU
1259 #define ALU_HH_FAST ALU
1260
1261 #endif
1262
1263 #define ALU_I(thread, ip, operator) \
1264 { \
1265 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1266 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1267 uint64_t dst64 = *dst64_ptr; \
1268 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1269 uint64_t dst = dst64 & dst64_mask; \
1270 \
1271 uint64_t src = (ip)->alu.src_val; \
1272 \
1273 uint64_t result = dst operator src; \
1274 \
1275 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1276 }
1277
1278 #define ALU_MI ALU_I
1279
1280 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1281
1282 #define ALU_HI(thread, ip, operator) \
1283 { \
1284 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1285 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1286 uint64_t dst64 = *dst64_ptr; \
1287 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1288 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1289 \
1290 uint64_t src = (ip)->alu.src_val; \
1291 \
1292 uint64_t result = dst operator src; \
1293 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1294 \
1295 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1296 }
1297
1298 #else
1299
1300 #define ALU_HI ALU_I
1301
1302 #endif
1303
1304 #define MOV(thread, ip) \
1305 { \
1306 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1307 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1308 uint64_t dst64 = *dst64_ptr; \
1309 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1310 \
1311 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1312 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1313 uint64_t src64 = *src64_ptr; \
1314 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1315 uint64_t src = src64 & src64_mask; \
1316 \
1317 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1318 }
1319
1320 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1321
1322 #define MOV_MH(thread, ip) \
1323 { \
1324 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1325 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1326 uint64_t dst64 = *dst64_ptr; \
1327 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1328 \
1329 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1330 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1331 uint64_t src64 = *src64_ptr; \
1332 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
1333 \
1334 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1335 }
1336
1337 #define MOV_HM(thread, ip) \
1338 { \
1339 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1340 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1341 uint64_t dst64 = *dst64_ptr; \
1342 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1343 \
1344 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1345 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1346 uint64_t src64 = *src64_ptr; \
1347 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1348 uint64_t src = src64 & src64_mask; \
1349 \
1350 src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \
1351 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1352 }
1353
1354 #define MOV_HH(thread, ip) \
1355 { \
1356 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1357 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1358 uint64_t dst64 = *dst64_ptr; \
1359 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1360 \
1361 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1362 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1363 uint64_t src64 = *src64_ptr; \
1364 \
1365 uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \
1366 src = src >> (64 - (ip)->mov.dst.n_bits); \
1367 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1368 }
1369
1370 #else
1371
1372 #define MOV_MH MOV
1373 #define MOV_HM MOV
1374 #define MOV_HH MOV
1375
1376 #endif
1377
1378 #define MOV_I(thread, ip) \
1379 { \
1380 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1381 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1382 uint64_t dst64 = *dst64_ptr; \
1383 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1384 \
1385 uint64_t src = (ip)->mov.src_val; \
1386 \
1387 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1388 }
1389
1390 #define JMP_CMP(thread, ip, operator) \
1391 { \
1392 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1393 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1394 uint64_t a64 = *a64_ptr; \
1395 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1396 uint64_t a = a64 & a64_mask; \
1397 \
1398 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1399 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1400 uint64_t b64 = *b64_ptr; \
1401 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1402 uint64_t b = b64 & b64_mask; \
1403 \
1404 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1405 }
1406
1407 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1408
1409 #define JMP_CMP_MH(thread, ip, operator) \
1410 { \
1411 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1412 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1413 uint64_t a64 = *a64_ptr; \
1414 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1415 uint64_t a = a64 & a64_mask; \
1416 \
1417 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1418 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1419 uint64_t b64 = *b64_ptr; \
1420 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1421 \
1422 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1423 }
1424
1425 #define JMP_CMP_HM(thread, ip, operator) \
1426 { \
1427 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1428 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1429 uint64_t a64 = *a64_ptr; \
1430 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1431 \
1432 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1433 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1434 uint64_t b64 = *b64_ptr; \
1435 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1436 uint64_t b = b64 & b64_mask; \
1437 \
1438 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1439 }
1440
1441 #define JMP_CMP_HH(thread, ip, operator) \
1442 { \
1443 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1444 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1445 uint64_t a64 = *a64_ptr; \
1446 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1447 \
1448 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1449 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1450 uint64_t b64 = *b64_ptr; \
1451 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1452 \
1453 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1454 }
1455
1456 #define JMP_CMP_HH_FAST(thread, ip, operator) \
1457 { \
1458 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1459 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1460 uint64_t a64 = *a64_ptr; \
1461 uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \
1462 \
1463 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1464 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1465 uint64_t b64 = *b64_ptr; \
1466 uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \
1467 \
1468 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1469 }
1470
1471 #else
1472
1473 #define JMP_CMP_MH JMP_CMP
1474 #define JMP_CMP_HM JMP_CMP
1475 #define JMP_CMP_HH JMP_CMP
1476 #define JMP_CMP_HH_FAST JMP_CMP
1477
1478 #endif
1479
1480 #define JMP_CMP_I(thread, ip, operator) \
1481 { \
1482 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1483 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1484 uint64_t a64 = *a64_ptr; \
1485 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1486 uint64_t a = a64 & a64_mask; \
1487 \
1488 uint64_t b = (ip)->jmp.b_val; \
1489 \
1490 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1491 }
1492
1493 #define JMP_CMP_MI JMP_CMP_I
1494
1495 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1496
1497 #define JMP_CMP_HI(thread, ip, operator) \
1498 { \
1499 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1500 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1501 uint64_t a64 = *a64_ptr; \
1502 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1503 \
1504 uint64_t b = (ip)->jmp.b_val; \
1505 \
1506 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1507 }
1508
1509 #else
1510
1511 #define JMP_CMP_HI JMP_CMP_I
1512
1513 #endif
1514
1515 #define METADATA_READ(thread, offset, n_bits) \
1516 __extension__ ({ \
1517 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1518 uint64_t m64 = *m64_ptr; \
1519 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1520 (m64 & m64_mask); \
1521 })
1522
1523 #define METADATA_WRITE(thread, offset, n_bits, value) \
1524 { \
1525 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1526 uint64_t m64 = *m64_ptr; \
1527 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1528 \
1529 uint64_t m_new = value; \
1530 \
1531 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
1532 }
1533
1534 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1535 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1536 #endif
1537
1538 #ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX
1539 #define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 1024
1540 #endif
1541
1542 struct rte_swx_pipeline {
1543 char name[RTE_SWX_NAME_SIZE];
1544
1545 struct struct_type_tailq struct_types;
1546 struct port_in_type_tailq port_in_types;
1547 struct port_in_tailq ports_in;
1548 struct port_out_type_tailq port_out_types;
1549 struct port_out_tailq ports_out;
1550 struct extern_type_tailq extern_types;
1551 struct extern_obj_tailq extern_objs;
1552 struct extern_func_tailq extern_funcs;
1553 struct hash_func_tailq hash_funcs;
1554 struct rss_tailq rss;
1555 struct header_tailq headers;
1556 struct struct_type *metadata_st;
1557 uint32_t metadata_struct_id;
1558 struct action_tailq actions;
1559 struct table_type_tailq table_types;
1560 struct table_tailq tables;
1561 struct selector_tailq selectors;
1562 struct learner_tailq learners;
1563 struct regarray_tailq regarrays;
1564 struct meter_profile_tailq meter_profiles;
1565 struct metarray_tailq metarrays;
1566
1567 struct port_in_runtime *in;
1568 struct port_out_runtime *out;
1569 struct mirroring_session *mirroring_sessions;
1570 struct instruction **action_instructions;
1571 action_func_t *action_funcs;
1572 struct rte_swx_table_state *table_state;
1573 struct table_statistics *table_stats;
1574 struct selector_statistics *selector_stats;
1575 struct learner_statistics *learner_stats;
1576 struct hash_func_runtime *hash_func_runtime;
1577 struct rss_runtime **rss_runtime;
1578 struct regarray_runtime *regarray_runtime;
1579 struct metarray_runtime *metarray_runtime;
1580 struct instruction *instructions;
1581 struct instruction_data *instruction_data;
1582 instr_exec_t *instruction_table;
1583 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1584 void *lib;
1585
1586 uint32_t n_structs;
1587 uint32_t n_ports_in;
1588 uint32_t n_ports_out;
1589 uint32_t n_mirroring_slots;
1590 uint32_t n_mirroring_sessions;
1591 uint32_t n_extern_objs;
1592 uint32_t n_extern_funcs;
1593 uint32_t n_hash_funcs;
1594 uint32_t n_rss;
1595 uint32_t n_actions;
1596 uint32_t n_tables;
1597 uint32_t n_selectors;
1598 uint32_t n_learners;
1599 uint32_t n_regarrays;
1600 uint32_t n_metarrays;
1601 uint32_t n_headers;
1602 uint32_t thread_id;
1603 uint32_t port_id;
1604 uint32_t n_instructions;
1605 int build_done;
1606 int numa_node;
1607 };
1608
1609 /*
1610 * Instruction.
1611 */
1612 static inline void
pipeline_port_inc(struct rte_swx_pipeline * p)1613 pipeline_port_inc(struct rte_swx_pipeline *p)
1614 {
1615 uint32_t port_id = p->port_id;
1616
1617 port_id++;
1618 if (port_id == p->n_ports_in)
1619 port_id = 0;
1620
1621 p->port_id = port_id;
1622 }
1623
1624 static inline void
thread_ip_reset(struct rte_swx_pipeline * p,struct thread * t)1625 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1626 {
1627 t->ip = p->instructions;
1628 }
1629
1630 static inline void
thread_ip_set(struct thread * t,struct instruction * ip)1631 thread_ip_set(struct thread *t, struct instruction *ip)
1632 {
1633 t->ip = ip;
1634 }
1635
1636 static inline void
thread_ip_action_call(struct rte_swx_pipeline * p,struct thread * t,uint32_t action_id)1637 thread_ip_action_call(struct rte_swx_pipeline *p,
1638 struct thread *t,
1639 uint32_t action_id)
1640 {
1641 t->ret = t->ip + 1;
1642 t->ip = p->action_instructions[action_id];
1643 }
1644
1645 static inline void
1646 thread_ip_inc(struct rte_swx_pipeline *p);
1647
1648 static inline void
thread_ip_inc(struct rte_swx_pipeline * p)1649 thread_ip_inc(struct rte_swx_pipeline *p)
1650 {
1651 struct thread *t = &p->threads[p->thread_id];
1652
1653 t->ip++;
1654 }
1655
1656 static inline void
thread_ip_inc_cond(struct thread * t,int cond)1657 thread_ip_inc_cond(struct thread *t, int cond)
1658 {
1659 t->ip += cond;
1660 }
1661
1662 static inline void
thread_yield(struct rte_swx_pipeline * p)1663 thread_yield(struct rte_swx_pipeline *p)
1664 {
1665 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1666 }
1667
1668 static inline void
thread_yield_cond(struct rte_swx_pipeline * p,int cond)1669 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
1670 {
1671 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1672 }
1673
1674 /*
1675 * rx.
1676 */
1677 static inline int
__instr_rx_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)1678 __instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1679 {
1680 struct port_in_runtime *port = &p->in[p->port_id];
1681 struct rte_swx_pkt *pkt = &t->pkt;
1682 int pkt_received;
1683
1684 /* Recirculation: keep the current packet. */
1685 if (t->recirculate) {
1686 TRACE("[Thread %2u] rx - recirculate (pass %u)\n",
1687 p->thread_id,
1688 t->recirc_pass_id + 1);
1689
1690 /* Packet. */
1691 t->ptr = &pkt->pkt[pkt->offset];
1692 t->mirroring_slots_mask = 0;
1693 t->recirculate = 0;
1694 t->recirc_pass_id++;
1695
1696 /* Headers. */
1697 t->valid_headers = 0;
1698 t->n_headers_out = 0;
1699
1700 /* Tables. */
1701 t->table_state = p->table_state;
1702
1703 return 1;
1704 }
1705
1706 /* Packet. */
1707 pkt_received = port->pkt_rx(port->obj, pkt);
1708 t->ptr = &pkt->pkt[pkt->offset];
1709 rte_prefetch0(t->ptr);
1710
1711 TRACE("[Thread %2u] rx %s from port %u\n",
1712 p->thread_id,
1713 pkt_received ? "1 pkt" : "0 pkts",
1714 p->port_id);
1715
1716 t->mirroring_slots_mask = 0;
1717 t->recirc_pass_id = 0;
1718
1719 /* Headers. */
1720 t->valid_headers = 0;
1721 t->n_headers_out = 0;
1722
1723 /* Meta-data. */
1724 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1725
1726 /* Tables. */
1727 t->table_state = p->table_state;
1728
1729 /* Thread. */
1730 pipeline_port_inc(p);
1731
1732 return pkt_received;
1733 }
1734
1735 static inline void
instr_rx_exec(struct rte_swx_pipeline * p)1736 instr_rx_exec(struct rte_swx_pipeline *p)
1737 {
1738 struct thread *t = &p->threads[p->thread_id];
1739 struct instruction *ip = t->ip;
1740 int pkt_received;
1741
1742 /* Packet. */
1743 pkt_received = __instr_rx_exec(p, t, ip);
1744
1745 /* Thread. */
1746 thread_ip_inc_cond(t, pkt_received);
1747 thread_yield(p);
1748 }
1749
1750 /*
1751 * tx.
1752 */
1753 static inline void
emit_handler(struct thread * t)1754 emit_handler(struct thread *t)
1755 {
1756 struct header_out_runtime *h0 = &t->headers_out[0];
1757 struct header_out_runtime *h1 = &t->headers_out[1];
1758 uint32_t offset = 0, i;
1759
1760 /* No header change or header decapsulation. */
1761 if ((t->n_headers_out == 1) &&
1762 (h0->ptr + h0->n_bytes == t->ptr)) {
1763 TRACE("Emit handler: no header change or header decap.\n");
1764
1765 t->pkt.offset -= h0->n_bytes;
1766 t->pkt.length += h0->n_bytes;
1767
1768 return;
1769 }
1770
1771 /* Header encapsulation (optionally, with prior header decapsulation). */
1772 if ((t->n_headers_out == 2) &&
1773 (h1->ptr + h1->n_bytes == t->ptr) &&
1774 (h0->ptr == h0->ptr0)) {
1775 uint32_t offset;
1776
1777 TRACE("Emit handler: header encapsulation.\n");
1778
1779 offset = h0->n_bytes + h1->n_bytes;
1780 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1781 t->pkt.offset -= offset;
1782 t->pkt.length += offset;
1783
1784 return;
1785 }
1786
1787 /* For any other case. */
1788 TRACE("Emit handler: complex case.\n");
1789
1790 for (i = 0; i < t->n_headers_out; i++) {
1791 struct header_out_runtime *h = &t->headers_out[i];
1792
1793 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1794 offset += h->n_bytes;
1795 }
1796
1797 if (offset) {
1798 memcpy(t->ptr - offset, t->header_out_storage, offset);
1799 t->pkt.offset -= offset;
1800 t->pkt.length += offset;
1801 }
1802 }
1803
1804 static inline void
mirroring_handler(struct rte_swx_pipeline * p,struct thread * t,struct rte_swx_pkt * pkt)1805 mirroring_handler(struct rte_swx_pipeline *p, struct thread *t, struct rte_swx_pkt *pkt)
1806 {
1807 uint64_t slots_mask = t->mirroring_slots_mask, slot_mask;
1808 uint32_t slot_id;
1809
1810 for (slot_id = 0, slot_mask = 1LLU ; slots_mask; slot_id++, slot_mask <<= 1)
1811 if (slot_mask & slots_mask) {
1812 struct port_out_runtime *port;
1813 struct mirroring_session *session;
1814 uint32_t port_id, session_id;
1815
1816 session_id = t->mirroring_slots[slot_id];
1817 session = &p->mirroring_sessions[session_id];
1818
1819 port_id = session->port_id;
1820 port = &p->out[port_id];
1821
1822 if (session->fast_clone)
1823 port->pkt_fast_clone_tx(port->obj, pkt);
1824 else
1825 port->pkt_clone_tx(port->obj, pkt, session->truncation_length);
1826
1827 slots_mask &= ~slot_mask;
1828 }
1829 }
1830
1831 static inline void
__instr_tx_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)1832 __instr_tx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1833 {
1834 struct rte_swx_pkt *pkt = &t->pkt;
1835 struct port_out_runtime *port;
1836 uint64_t port_id;
1837
1838 /* Recirculation: keep the current packet. */
1839 if (t->recirculate) {
1840 TRACE("[Thread %2u]: tx 1 pkt - recirculate\n",
1841 p->thread_id);
1842
1843 /* Headers. */
1844 emit_handler(t);
1845
1846 /* Packet. */
1847 mirroring_handler(p, t, pkt);
1848
1849 return;
1850 }
1851
1852 /* If the output port ID is invalid, then set it to the drop output port that has been set
1853 * up internally by the pipeline for this purpose.
1854 */
1855 port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1856 if (port_id >= p->n_ports_out)
1857 port_id = p->n_ports_out - 1;
1858
1859 port = &p->out[port_id];
1860
1861 TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
1862 p->thread_id,
1863 (uint32_t)port_id);
1864
1865 /* Headers. */
1866 emit_handler(t);
1867
1868 /* Packet. */
1869 mirroring_handler(p, t, pkt);
1870 port->pkt_tx(port->obj, pkt);
1871 }
1872
1873 static inline void
__instr_tx_i_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)1874 __instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1875 {
1876 struct rte_swx_pkt *pkt = &t->pkt;
1877 struct port_out_runtime *port;
1878 uint64_t port_id;
1879
1880 /* Recirculation: keep the current packet. */
1881 if (t->recirculate) {
1882 TRACE("[Thread %2u]: tx (i) 1 pkt - recirculate\n",
1883 p->thread_id);
1884
1885 /* Headers. */
1886 emit_handler(t);
1887
1888 /* Packet. */
1889 mirroring_handler(p, t, pkt);
1890
1891 return;
1892 }
1893
1894 /* If the output port ID is invalid, then set it to the drop output port that has been set
1895 * up internally by the pipeline for this purpose.
1896 *
1897 * This test cannot be done earlier at instruction translation time, even though the output
1898 * port ID is an immediate value, as the number of output ports is only known later at the
1899 * pipeline build time.
1900 */
1901 port_id = ip->io.io.val;
1902 if (port_id >= p->n_ports_out)
1903 port_id = p->n_ports_out - 1;
1904
1905 port = &p->out[port_id];
1906
1907 TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
1908 p->thread_id,
1909 (uint32_t)port_id);
1910
1911 /* Headers. */
1912 emit_handler(t);
1913
1914 /* Packet. */
1915 mirroring_handler(p, t, pkt);
1916 port->pkt_tx(port->obj, pkt);
1917 }
1918
1919 static inline void
__instr_drop_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip __rte_unused)1920 __instr_drop_exec(struct rte_swx_pipeline *p,
1921 struct thread *t,
1922 const struct instruction *ip __rte_unused)
1923 {
1924 uint64_t port_id = p->n_ports_out - 1;
1925 struct port_out_runtime *port = &p->out[port_id];
1926 struct rte_swx_pkt *pkt = &t->pkt;
1927
1928 TRACE("[Thread %2u]: drop 1 pkt\n",
1929 p->thread_id);
1930
1931 /* Headers. */
1932 emit_handler(t);
1933
1934 /* Packet. */
1935 mirroring_handler(p, t, pkt);
1936 port->pkt_tx(port->obj, pkt);
1937 }
1938
1939 static inline void
__instr_mirror_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)1940 __instr_mirror_exec(struct rte_swx_pipeline *p,
1941 struct thread *t,
1942 const struct instruction *ip)
1943 {
1944 uint64_t slot_id = instr_operand_hbo(t, &ip->mirror.dst);
1945 uint64_t session_id = instr_operand_hbo(t, &ip->mirror.src);
1946
1947 slot_id &= p->n_mirroring_slots - 1;
1948 session_id &= p->n_mirroring_sessions - 1;
1949
1950 TRACE("[Thread %2u]: mirror pkt (slot = %u, session = %u)\n",
1951 p->thread_id,
1952 (uint32_t)slot_id,
1953 (uint32_t)session_id);
1954
1955 t->mirroring_slots[slot_id] = session_id;
1956 t->mirroring_slots_mask |= 1LLU << slot_id;
1957 }
1958
1959 static inline void
__instr_recirculate_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip __rte_unused)1960 __instr_recirculate_exec(struct rte_swx_pipeline *p __rte_unused,
1961 struct thread *t,
1962 const struct instruction *ip __rte_unused)
1963 {
1964 TRACE("[Thread %2u]: recirculate\n",
1965 p->thread_id);
1966
1967 t->recirculate = 1;
1968 }
1969
1970 static inline void
__instr_recircid_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)1971 __instr_recircid_exec(struct rte_swx_pipeline *p __rte_unused,
1972 struct thread *t,
1973 const struct instruction *ip)
1974 {
1975 TRACE("[Thread %2u]: recircid (pass %u)\n",
1976 p->thread_id,
1977 t->recirc_pass_id);
1978
1979 /* Meta-data. */
1980 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, t->recirc_pass_id);
1981 }
1982
1983 /*
1984 * extract.
1985 */
1986 static inline void
__instr_hdr_extract_many_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip,uint32_t n_extract)1987 __instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused,
1988 struct thread *t,
1989 const struct instruction *ip,
1990 uint32_t n_extract)
1991 {
1992 uint64_t valid_headers = t->valid_headers;
1993 uint8_t *ptr = t->ptr;
1994 uint32_t offset = t->pkt.offset;
1995 uint32_t length = t->pkt.length;
1996 uint32_t i;
1997
1998 for (i = 0; i < n_extract; i++) {
1999 uint32_t header_id = ip->io.hdr.header_id[i];
2000 uint32_t struct_id = ip->io.hdr.struct_id[i];
2001 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2002
2003 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2004 p->thread_id,
2005 header_id,
2006 n_bytes);
2007
2008 /* Headers. */
2009 t->structs[struct_id] = ptr;
2010 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2011
2012 /* Packet. */
2013 offset += n_bytes;
2014 length -= n_bytes;
2015 ptr += n_bytes;
2016 }
2017
2018 /* Headers. */
2019 t->valid_headers = valid_headers;
2020
2021 /* Packet. */
2022 t->pkt.offset = offset;
2023 t->pkt.length = length;
2024 t->ptr = ptr;
2025 }
2026
2027 static inline void
__instr_hdr_extract_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2028 __instr_hdr_extract_exec(struct rte_swx_pipeline *p,
2029 struct thread *t,
2030 const struct instruction *ip)
2031 {
2032 __instr_hdr_extract_many_exec(p, t, ip, 1);
2033 }
2034
2035 static inline void
__instr_hdr_extract2_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2036 __instr_hdr_extract2_exec(struct rte_swx_pipeline *p,
2037 struct thread *t,
2038 const struct instruction *ip)
2039 {
2040 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2041
2042 __instr_hdr_extract_many_exec(p, t, ip, 2);
2043 }
2044
2045 static inline void
__instr_hdr_extract3_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2046 __instr_hdr_extract3_exec(struct rte_swx_pipeline *p,
2047 struct thread *t,
2048 const struct instruction *ip)
2049 {
2050 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2051
2052 __instr_hdr_extract_many_exec(p, t, ip, 3);
2053 }
2054
2055 static inline void
__instr_hdr_extract4_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2056 __instr_hdr_extract4_exec(struct rte_swx_pipeline *p,
2057 struct thread *t,
2058 const struct instruction *ip)
2059 {
2060 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2061
2062 __instr_hdr_extract_many_exec(p, t, ip, 4);
2063 }
2064
2065 static inline void
__instr_hdr_extract5_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2066 __instr_hdr_extract5_exec(struct rte_swx_pipeline *p,
2067 struct thread *t,
2068 const struct instruction *ip)
2069 {
2070 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2071
2072 __instr_hdr_extract_many_exec(p, t, ip, 5);
2073 }
2074
2075 static inline void
__instr_hdr_extract6_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2076 __instr_hdr_extract6_exec(struct rte_swx_pipeline *p,
2077 struct thread *t,
2078 const struct instruction *ip)
2079 {
2080 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2081
2082 __instr_hdr_extract_many_exec(p, t, ip, 6);
2083 }
2084
2085 static inline void
__instr_hdr_extract7_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2086 __instr_hdr_extract7_exec(struct rte_swx_pipeline *p,
2087 struct thread *t,
2088 const struct instruction *ip)
2089 {
2090 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2091
2092 __instr_hdr_extract_many_exec(p, t, ip, 7);
2093 }
2094
2095 static inline void
__instr_hdr_extract8_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2096 __instr_hdr_extract8_exec(struct rte_swx_pipeline *p,
2097 struct thread *t,
2098 const struct instruction *ip)
2099 {
2100 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2101
2102 __instr_hdr_extract_many_exec(p, t, ip, 8);
2103 }
2104
2105 static inline void
__instr_hdr_extract_m_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2106 __instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused,
2107 struct thread *t,
2108 const struct instruction *ip)
2109 {
2110 uint64_t valid_headers = t->valid_headers;
2111 uint8_t *ptr = t->ptr;
2112 uint32_t offset = t->pkt.offset;
2113 uint32_t length = t->pkt.length;
2114
2115 uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2116 uint32_t header_id = ip->io.hdr.header_id[0];
2117 uint32_t struct_id = ip->io.hdr.struct_id[0];
2118 uint32_t n_bytes = ip->io.hdr.n_bytes[0];
2119
2120 struct header_runtime *h = &t->headers[header_id];
2121
2122 TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
2123 p->thread_id,
2124 header_id,
2125 n_bytes,
2126 n_bytes_last);
2127
2128 n_bytes += n_bytes_last;
2129
2130 /* Headers. */
2131 t->structs[struct_id] = ptr;
2132 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2133 h->n_bytes = n_bytes;
2134
2135 /* Packet. */
2136 t->pkt.offset = offset + n_bytes;
2137 t->pkt.length = length - n_bytes;
2138 t->ptr = ptr + n_bytes;
2139 }
2140
2141 static inline void
__instr_hdr_lookahead_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2142 __instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused,
2143 struct thread *t,
2144 const struct instruction *ip)
2145 {
2146 uint64_t valid_headers = t->valid_headers;
2147 uint8_t *ptr = t->ptr;
2148
2149 uint32_t header_id = ip->io.hdr.header_id[0];
2150 uint32_t struct_id = ip->io.hdr.struct_id[0];
2151
2152 TRACE("[Thread %2u]: lookahead header %u\n",
2153 p->thread_id,
2154 header_id);
2155
2156 /* Headers. */
2157 t->structs[struct_id] = ptr;
2158 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2159 }
2160
2161 /*
2162 * emit.
2163 */
2164 static inline void
__instr_hdr_emit_many_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip,uint32_t n_emit)2165 __instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused,
2166 struct thread *t,
2167 const struct instruction *ip,
2168 uint32_t n_emit)
2169 {
2170 uint64_t valid_headers = t->valid_headers;
2171 uint32_t n_headers_out = t->n_headers_out;
2172 struct header_out_runtime *ho = NULL;
2173 uint8_t *ho_ptr = NULL;
2174 uint32_t ho_nbytes = 0, i;
2175
2176 for (i = 0; i < n_emit; i++) {
2177 uint32_t header_id = ip->io.hdr.header_id[i];
2178 uint32_t struct_id = ip->io.hdr.struct_id[i];
2179
2180 struct header_runtime *hi = &t->headers[header_id];
2181 uint8_t *hi_ptr0 = hi->ptr0;
2182 uint32_t n_bytes = hi->n_bytes;
2183
2184 uint8_t *hi_ptr = t->structs[struct_id];
2185
2186 if (!MASK64_BIT_GET(valid_headers, header_id)) {
2187 TRACE("[Thread %2u]: emit header %u (invalid)\n",
2188 p->thread_id,
2189 header_id);
2190
2191 continue;
2192 }
2193
2194 TRACE("[Thread %2u]: emit header %u (valid)\n",
2195 p->thread_id,
2196 header_id);
2197
2198 /* Headers. */
2199 if (!ho) {
2200 if (!n_headers_out) {
2201 ho = &t->headers_out[0];
2202
2203 ho->ptr0 = hi_ptr0;
2204 ho->ptr = hi_ptr;
2205
2206 ho_ptr = hi_ptr;
2207 ho_nbytes = n_bytes;
2208
2209 n_headers_out = 1;
2210
2211 continue;
2212 } else {
2213 ho = &t->headers_out[n_headers_out - 1];
2214
2215 ho_ptr = ho->ptr;
2216 ho_nbytes = ho->n_bytes;
2217 }
2218 }
2219
2220 if (ho_ptr + ho_nbytes == hi_ptr) {
2221 ho_nbytes += n_bytes;
2222 } else {
2223 ho->n_bytes = ho_nbytes;
2224
2225 ho++;
2226 ho->ptr0 = hi_ptr0;
2227 ho->ptr = hi_ptr;
2228
2229 ho_ptr = hi_ptr;
2230 ho_nbytes = n_bytes;
2231
2232 n_headers_out++;
2233 }
2234 }
2235
2236 if (ho)
2237 ho->n_bytes = ho_nbytes;
2238 t->n_headers_out = n_headers_out;
2239 }
2240
2241 static inline void
__instr_hdr_emit_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2242 __instr_hdr_emit_exec(struct rte_swx_pipeline *p,
2243 struct thread *t,
2244 const struct instruction *ip)
2245 {
2246 __instr_hdr_emit_many_exec(p, t, ip, 1);
2247 }
2248
2249 static inline void
__instr_hdr_emit_tx_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2250 __instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p,
2251 struct thread *t,
2252 const struct instruction *ip)
2253 {
2254 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2255
2256 __instr_hdr_emit_many_exec(p, t, ip, 1);
2257 __instr_tx_exec(p, t, ip);
2258 }
2259
2260 static inline void
__instr_hdr_emit2_tx_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2261 __instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p,
2262 struct thread *t,
2263 const struct instruction *ip)
2264 {
2265 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2266
2267 __instr_hdr_emit_many_exec(p, t, ip, 2);
2268 __instr_tx_exec(p, t, ip);
2269 }
2270
2271 static inline void
__instr_hdr_emit3_tx_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2272 __instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p,
2273 struct thread *t,
2274 const struct instruction *ip)
2275 {
2276 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2277
2278 __instr_hdr_emit_many_exec(p, t, ip, 3);
2279 __instr_tx_exec(p, t, ip);
2280 }
2281
2282 static inline void
__instr_hdr_emit4_tx_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2283 __instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p,
2284 struct thread *t,
2285 const struct instruction *ip)
2286 {
2287 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2288
2289 __instr_hdr_emit_many_exec(p, t, ip, 4);
2290 __instr_tx_exec(p, t, ip);
2291 }
2292
2293 static inline void
__instr_hdr_emit5_tx_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2294 __instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p,
2295 struct thread *t,
2296 const struct instruction *ip)
2297 {
2298 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2299
2300 __instr_hdr_emit_many_exec(p, t, ip, 5);
2301 __instr_tx_exec(p, t, ip);
2302 }
2303
2304 static inline void
__instr_hdr_emit6_tx_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2305 __instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p,
2306 struct thread *t,
2307 const struct instruction *ip)
2308 {
2309 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2310
2311 __instr_hdr_emit_many_exec(p, t, ip, 6);
2312 __instr_tx_exec(p, t, ip);
2313 }
2314
2315 static inline void
__instr_hdr_emit7_tx_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2316 __instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p,
2317 struct thread *t,
2318 const struct instruction *ip)
2319 {
2320 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2321
2322 __instr_hdr_emit_many_exec(p, t, ip, 7);
2323 __instr_tx_exec(p, t, ip);
2324 }
2325
2326 static inline void
__instr_hdr_emit8_tx_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2327 __instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p,
2328 struct thread *t,
2329 const struct instruction *ip)
2330 {
2331 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id);
2332
2333 __instr_hdr_emit_many_exec(p, t, ip, 8);
2334 __instr_tx_exec(p, t, ip);
2335 }
2336
2337 /*
2338 * validate.
2339 */
2340 static inline void
__instr_hdr_validate_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2341 __instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused,
2342 struct thread *t,
2343 const struct instruction *ip)
2344 {
2345 uint32_t header_id = ip->valid.header_id;
2346 uint32_t struct_id = ip->valid.struct_id;
2347 uint64_t valid_headers = t->valid_headers;
2348 struct header_runtime *h = &t->headers[header_id];
2349
2350 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2351
2352 /* If this header is already valid, then its associated t->structs[] element is also valid
2353 * and therefore it should not be modified. It could point to the packet buffer (in case of
2354 * extracted header) and setting it to the default location (h->ptr0) would be incorrect.
2355 */
2356 if (MASK64_BIT_GET(valid_headers, header_id))
2357 return;
2358
2359 /* Headers. */
2360 t->structs[struct_id] = h->ptr0;
2361 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2362 }
2363
2364 /*
2365 * invalidate.
2366 */
2367 static inline void
__instr_hdr_invalidate_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2368 __instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused,
2369 struct thread *t,
2370 const struct instruction *ip)
2371 {
2372 uint32_t header_id = ip->valid.header_id;
2373
2374 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2375
2376 /* Headers. */
2377 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2378 }
2379
2380 /*
2381 * learn.
2382 */
2383 static inline void
__instr_learn_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2384 __instr_learn_exec(struct rte_swx_pipeline *p,
2385 struct thread *t,
2386 const struct instruction *ip)
2387 {
2388 uint64_t action_id = ip->learn.action_id;
2389 uint32_t mf_first_arg_offset = ip->learn.mf_first_arg_offset;
2390 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset,
2391 ip->learn.mf_timeout_id_n_bits);
2392 uint32_t learner_id = t->learner_id;
2393 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2394 p->n_selectors + learner_id];
2395 struct learner_runtime *l = &t->learners[learner_id];
2396 struct learner_statistics *stats = &p->learner_stats[learner_id];
2397 uint32_t status;
2398
2399 /* Table. */
2400 status = rte_swx_table_learner_add(ts->obj,
2401 l->mailbox,
2402 t->time,
2403 action_id,
2404 &t->metadata[mf_first_arg_offset],
2405 timeout_id);
2406
2407 TRACE("[Thread %2u] learner %u learn %s\n",
2408 p->thread_id,
2409 learner_id,
2410 status ? "ok" : "error");
2411
2412 stats->n_pkts_learn[status] += 1;
2413 }
2414
2415 /*
2416 * rearm.
2417 */
2418 static inline void
__instr_rearm_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip __rte_unused)2419 __instr_rearm_exec(struct rte_swx_pipeline *p,
2420 struct thread *t,
2421 const struct instruction *ip __rte_unused)
2422 {
2423 uint32_t learner_id = t->learner_id;
2424 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2425 p->n_selectors + learner_id];
2426 struct learner_runtime *l = &t->learners[learner_id];
2427 struct learner_statistics *stats = &p->learner_stats[learner_id];
2428
2429 /* Table. */
2430 rte_swx_table_learner_rearm(ts->obj, l->mailbox, t->time);
2431
2432 TRACE("[Thread %2u] learner %u rearm\n",
2433 p->thread_id,
2434 learner_id);
2435
2436 stats->n_pkts_rearm += 1;
2437 }
2438
2439 static inline void
__instr_rearm_new_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2440 __instr_rearm_new_exec(struct rte_swx_pipeline *p,
2441 struct thread *t,
2442 const struct instruction *ip)
2443 {
2444 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset,
2445 ip->learn.mf_timeout_id_n_bits);
2446 uint32_t learner_id = t->learner_id;
2447 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2448 p->n_selectors + learner_id];
2449 struct learner_runtime *l = &t->learners[learner_id];
2450 struct learner_statistics *stats = &p->learner_stats[learner_id];
2451
2452 /* Table. */
2453 rte_swx_table_learner_rearm_new(ts->obj, l->mailbox, t->time, timeout_id);
2454
2455 TRACE("[Thread %2u] learner %u rearm with timeout ID %u\n",
2456 p->thread_id,
2457 learner_id,
2458 timeout_id);
2459
2460 stats->n_pkts_rearm += 1;
2461 }
2462
2463 /*
2464 * forget.
2465 */
2466 static inline void
__instr_forget_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip __rte_unused)2467 __instr_forget_exec(struct rte_swx_pipeline *p,
2468 struct thread *t,
2469 const struct instruction *ip __rte_unused)
2470 {
2471 uint32_t learner_id = t->learner_id;
2472 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2473 p->n_selectors + learner_id];
2474 struct learner_runtime *l = &t->learners[learner_id];
2475 struct learner_statistics *stats = &p->learner_stats[learner_id];
2476
2477 /* Table. */
2478 rte_swx_table_learner_delete(ts->obj, l->mailbox);
2479
2480 TRACE("[Thread %2u] learner %u forget\n",
2481 p->thread_id,
2482 learner_id);
2483
2484 stats->n_pkts_forget += 1;
2485 }
2486
2487 /*
2488 * entryid.
2489 */
2490 static inline void
__instr_entryid_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2491 __instr_entryid_exec(struct rte_swx_pipeline *p __rte_unused,
2492 struct thread *t,
2493 const struct instruction *ip)
2494 {
2495 TRACE("[Thread %2u]: entryid\n",
2496 p->thread_id);
2497
2498 /* Meta-data. */
2499 METADATA_WRITE(t, ip->mov.dst.offset, ip->mov.dst.n_bits, t->entry_id);
2500 }
2501
2502 /*
2503 * extern.
2504 */
2505 static inline uint32_t
__instr_extern_obj_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2506 __instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused,
2507 struct thread *t,
2508 const struct instruction *ip)
2509 {
2510 uint32_t obj_id = ip->ext_obj.ext_obj_id;
2511 uint32_t func_id = ip->ext_obj.func_id;
2512 struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2513 rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2514 uint32_t done;
2515
2516 TRACE("[Thread %2u] extern obj %u member func %u\n",
2517 p->thread_id,
2518 obj_id,
2519 func_id);
2520
2521 done = func(obj->obj, obj->mailbox);
2522
2523 return done;
2524 }
2525
2526 static inline uint32_t
__instr_extern_func_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2527 __instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused,
2528 struct thread *t,
2529 const struct instruction *ip)
2530 {
2531 uint32_t ext_func_id = ip->ext_func.ext_func_id;
2532 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2533 rte_swx_extern_func_t func = ext_func->func;
2534 uint32_t done;
2535
2536 TRACE("[Thread %2u] extern func %u\n",
2537 p->thread_id,
2538 ext_func_id);
2539
2540 done = func(ext_func->mailbox);
2541
2542 return done;
2543 }
2544
2545 /*
2546 * hash.
2547 */
2548 static inline void
__instr_hash_func_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2549 __instr_hash_func_exec(struct rte_swx_pipeline *p,
2550 struct thread *t,
2551 const struct instruction *ip)
2552 {
2553 uint32_t hash_func_id = ip->hash_func.hash_func_id;
2554 uint32_t dst_offset = ip->hash_func.dst.offset;
2555 uint32_t n_dst_bits = ip->hash_func.dst.n_bits;
2556 uint32_t src_struct_id = ip->hash_func.src.struct_id;
2557 uint32_t src_offset = ip->hash_func.src.offset;
2558 uint32_t n_src_bytes = ip->hash_func.src.n_bytes;
2559
2560 struct hash_func_runtime *func = &p->hash_func_runtime[hash_func_id];
2561 uint8_t *src_ptr = t->structs[src_struct_id];
2562 uint32_t result;
2563
2564 TRACE("[Thread %2u] hash %u\n",
2565 p->thread_id,
2566 hash_func_id);
2567
2568 result = func->func(&src_ptr[src_offset], n_src_bytes, 0);
2569 METADATA_WRITE(t, dst_offset, n_dst_bits, result);
2570 }
2571
2572 /*
2573 * rss.
2574 */
2575 static inline uint32_t
rss_func(void * rss_key,uint32_t rss_key_size,void * input_data,uint32_t input_data_size)2576 rss_func(void *rss_key, uint32_t rss_key_size, void *input_data, uint32_t input_data_size)
2577 {
2578 uint32_t *key = (uint32_t *)rss_key;
2579 uint32_t *data = (uint32_t *)input_data;
2580 uint32_t key_size = rss_key_size >> 2;
2581 uint32_t data_size = input_data_size >> 2;
2582 uint32_t hash_val = 0, i;
2583
2584 for (i = 0; i < data_size; i++) {
2585 uint32_t d;
2586
2587 for (d = data[i]; d; d &= (d - 1)) {
2588 uint32_t key0, key1, pos;
2589
2590 pos = rte_bsf32(d);
2591 key0 = key[i % key_size] << (31 - pos);
2592 key1 = key[(i + 1) % key_size] >> (pos + 1);
2593 hash_val ^= key0 | key1;
2594 }
2595 }
2596
2597 return hash_val;
2598 }
2599
2600 static inline void
__instr_rss_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2601 __instr_rss_exec(struct rte_swx_pipeline *p,
2602 struct thread *t,
2603 const struct instruction *ip)
2604 {
2605 uint32_t rss_obj_id = ip->rss.rss_obj_id;
2606 uint32_t dst_offset = ip->rss.dst.offset;
2607 uint32_t n_dst_bits = ip->rss.dst.n_bits;
2608 uint32_t src_struct_id = ip->rss.src.struct_id;
2609 uint32_t src_offset = ip->rss.src.offset;
2610 uint32_t n_src_bytes = ip->rss.src.n_bytes;
2611
2612 struct rss_runtime *r = p->rss_runtime[rss_obj_id];
2613 uint8_t *src_ptr = t->structs[src_struct_id];
2614 uint32_t result;
2615
2616 TRACE("[Thread %2u] rss %u\n",
2617 p->thread_id,
2618 rss_obj_id);
2619
2620 result = rss_func(r->key, r->key_size, &src_ptr[src_offset], n_src_bytes);
2621 METADATA_WRITE(t, dst_offset, n_dst_bits, result);
2622 }
2623
2624 /*
2625 * mov.
2626 */
2627 static inline void
__instr_mov_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2628 __instr_mov_exec(struct rte_swx_pipeline *p __rte_unused,
2629 struct thread *t,
2630 const struct instruction *ip)
2631 {
2632 TRACE("[Thread %2u] mov\n", p->thread_id);
2633
2634 MOV(t, ip);
2635 }
2636
2637 static inline void
__instr_mov_mh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2638 __instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2639 struct thread *t,
2640 const struct instruction *ip)
2641 {
2642 TRACE("[Thread %2u] mov (mh)\n", p->thread_id);
2643
2644 MOV_MH(t, ip);
2645 }
2646
2647 static inline void
__instr_mov_hm_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2648 __instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2649 struct thread *t,
2650 const struct instruction *ip)
2651 {
2652 TRACE("[Thread %2u] mov (hm)\n", p->thread_id);
2653
2654 MOV_HM(t, ip);
2655 }
2656
2657 static inline void
__instr_mov_hh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2658 __instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2659 struct thread *t,
2660 const struct instruction *ip)
2661 {
2662 TRACE("[Thread %2u] mov (hh)\n", p->thread_id);
2663
2664 MOV_HH(t, ip);
2665 }
2666
2667 static inline void
__instr_mov_dma_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2668 __instr_mov_dma_exec(struct rte_swx_pipeline *p __rte_unused,
2669 struct thread *t,
2670 const struct instruction *ip)
2671 {
2672 uint8_t *dst = t->structs[ip->mov.dst.struct_id] + ip->mov.dst.offset;
2673 uint8_t *src = t->structs[ip->mov.src.struct_id] + ip->mov.src.offset;
2674
2675 uint32_t n_dst = ip->mov.dst.n_bits >> 3;
2676 uint32_t n_src = ip->mov.src.n_bits >> 3;
2677
2678 TRACE("[Thread %2u] mov (dma) %u bytes\n", p->thread_id, n);
2679
2680 /* Both dst and src are in NBO format. */
2681 if (n_dst > n_src) {
2682 uint32_t n_dst_zero = n_dst - n_src;
2683
2684 /* Zero padding the most significant bytes in dst. */
2685 memset(dst, 0, n_dst_zero);
2686 dst += n_dst_zero;
2687
2688 /* Copy src to dst. */
2689 memcpy(dst, src, n_src);
2690 } else {
2691 uint32_t n_src_skipped = n_src - n_dst;
2692
2693 /* Copy src to dst. */
2694 src += n_src_skipped;
2695 memcpy(dst, src, n_dst);
2696 }
2697 }
2698
2699 static inline void
__instr_mov_128_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2700 __instr_mov_128_exec(struct rte_swx_pipeline *p __rte_unused,
2701 struct thread *t,
2702 const struct instruction *ip)
2703 {
2704 uint8_t *dst_struct = t->structs[ip->mov.dst.struct_id];
2705 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->mov.dst.offset];
2706
2707 uint8_t *src_struct = t->structs[ip->mov.src.struct_id];
2708 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->mov.src.offset];
2709
2710 TRACE("[Thread %2u] mov (128)\n", p->thread_id);
2711
2712 dst64_ptr[0] = src64_ptr[0];
2713 dst64_ptr[1] = src64_ptr[1];
2714 }
2715
2716 static inline void
__instr_mov_128_64_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2717 __instr_mov_128_64_exec(struct rte_swx_pipeline *p __rte_unused,
2718 struct thread *t,
2719 const struct instruction *ip)
2720 {
2721 uint8_t *dst = t->structs[ip->mov.dst.struct_id] + ip->mov.dst.offset;
2722 uint8_t *src = t->structs[ip->mov.src.struct_id] + ip->mov.src.offset;
2723
2724 uint64_t *dst64 = (uint64_t *)dst;
2725 uint64_t *src64 = (uint64_t *)src;
2726
2727 TRACE("[Thread %2u] mov (128 <- 64)\n", p->thread_id);
2728
2729 dst64[0] = 0;
2730 dst64[1] = src64[0];
2731 }
2732
2733 static inline void
__instr_mov_64_128_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2734 __instr_mov_64_128_exec(struct rte_swx_pipeline *p __rte_unused,
2735 struct thread *t,
2736 const struct instruction *ip)
2737 {
2738 uint8_t *dst = t->structs[ip->mov.dst.struct_id] + ip->mov.dst.offset;
2739 uint8_t *src = t->structs[ip->mov.src.struct_id] + ip->mov.src.offset;
2740
2741 uint64_t *dst64 = (uint64_t *)dst;
2742 uint64_t *src64 = (uint64_t *)src;
2743
2744 TRACE("[Thread %2u] mov (64 <- 128)\n", p->thread_id);
2745
2746 dst64[0] = src64[1];
2747 }
2748
2749 static inline void
__instr_mov_128_32_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2750 __instr_mov_128_32_exec(struct rte_swx_pipeline *p __rte_unused,
2751 struct thread *t,
2752 const struct instruction *ip)
2753 {
2754 uint8_t *dst = t->structs[ip->mov.dst.struct_id] + ip->mov.dst.offset;
2755 uint8_t *src = t->structs[ip->mov.src.struct_id] + ip->mov.src.offset;
2756
2757 uint32_t *dst32 = (uint32_t *)dst;
2758 uint32_t *src32 = (uint32_t *)src;
2759
2760 TRACE("[Thread %2u] mov (128 <- 32)\n", p->thread_id);
2761
2762 dst32[0] = 0;
2763 dst32[1] = 0;
2764 dst32[2] = 0;
2765 dst32[3] = src32[0];
2766 }
2767
2768 static inline void
__instr_mov_32_128_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2769 __instr_mov_32_128_exec(struct rte_swx_pipeline *p __rte_unused,
2770 struct thread *t,
2771 const struct instruction *ip)
2772 {
2773 uint8_t *dst = t->structs[ip->mov.dst.struct_id] + ip->mov.dst.offset;
2774 uint8_t *src = t->structs[ip->mov.src.struct_id] + ip->mov.src.offset;
2775
2776 uint32_t *dst32 = (uint32_t *)dst;
2777 uint32_t *src32 = (uint32_t *)src;
2778
2779 TRACE("[Thread %2u] mov (32 <- 128)\n", p->thread_id);
2780
2781 dst32[0] = src32[3];
2782 }
2783
2784 static inline void
__instr_mov_i_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2785 __instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused,
2786 struct thread *t,
2787 const struct instruction *ip)
2788 {
2789 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val);
2790
2791 MOV_I(t, ip);
2792 }
2793
2794 /*
2795 * movh.
2796 */
2797 static inline void
__instr_movh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2798 __instr_movh_exec(struct rte_swx_pipeline *p __rte_unused,
2799 struct thread *t,
2800 const struct instruction *ip)
2801 {
2802 uint8_t *dst = t->structs[ip->mov.dst.struct_id] + ip->mov.dst.offset;
2803 uint8_t *src = t->structs[ip->mov.src.struct_id] + ip->mov.src.offset;
2804
2805 uint64_t *dst64 = (uint64_t *)dst;
2806 uint64_t *src64 = (uint64_t *)src;
2807
2808 TRACE("[Thread %2u] movh\n", p->thread_id);
2809
2810 dst64[0] = src64[0];
2811 }
2812
2813 /*
2814 * dma.
2815 */
2816 static inline void
__instr_dma_ht_many_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip,uint32_t n_dma)2817 __instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused,
2818 struct thread *t,
2819 const struct instruction *ip,
2820 uint32_t n_dma)
2821 {
2822 uint8_t *action_data = t->structs[0];
2823 uint64_t valid_headers = t->valid_headers;
2824 uint32_t i;
2825
2826 for (i = 0; i < n_dma; i++) {
2827 uint32_t header_id = ip->dma.dst.header_id[i];
2828 uint32_t struct_id = ip->dma.dst.struct_id[i];
2829 uint32_t offset = ip->dma.src.offset[i];
2830 uint32_t n_bytes = ip->dma.n_bytes[i];
2831
2832 struct header_runtime *h = &t->headers[header_id];
2833 uint8_t *h_ptr0 = h->ptr0;
2834 uint8_t *h_ptr = t->structs[struct_id];
2835
2836 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2837 h_ptr : h_ptr0;
2838 void *src = &action_data[offset];
2839
2840 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2841
2842 /* Headers. */
2843 memcpy(dst, src, n_bytes);
2844 t->structs[struct_id] = dst;
2845 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2846 }
2847
2848 t->valid_headers = valid_headers;
2849 }
2850
2851 static inline void
__instr_dma_ht_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2852 __instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2853 {
2854 __instr_dma_ht_many_exec(p, t, ip, 1);
2855 }
2856
2857 static inline void
__instr_dma_ht2_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2858 __instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2859 {
2860 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2861
2862 __instr_dma_ht_many_exec(p, t, ip, 2);
2863 }
2864
2865 static inline void
__instr_dma_ht3_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2866 __instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2867 {
2868 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2869
2870 __instr_dma_ht_many_exec(p, t, ip, 3);
2871 }
2872
2873 static inline void
__instr_dma_ht4_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2874 __instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2875 {
2876 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2877
2878 __instr_dma_ht_many_exec(p, t, ip, 4);
2879 }
2880
2881 static inline void
__instr_dma_ht5_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2882 __instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2883 {
2884 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2885
2886 __instr_dma_ht_many_exec(p, t, ip, 5);
2887 }
2888
2889 static inline void
__instr_dma_ht6_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2890 __instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2891 {
2892 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2893
2894 __instr_dma_ht_many_exec(p, t, ip, 6);
2895 }
2896
2897 static inline void
__instr_dma_ht7_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2898 __instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2899 {
2900 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2901
2902 __instr_dma_ht_many_exec(p, t, ip, 7);
2903 }
2904
2905 static inline void
__instr_dma_ht8_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)2906 __instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2907 {
2908 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2909
2910 __instr_dma_ht_many_exec(p, t, ip, 8);
2911 }
2912
2913 /*
2914 * alu.
2915 */
2916 static inline void
__instr_alu_add_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2917 __instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused,
2918 struct thread *t,
2919 const struct instruction *ip)
2920 {
2921 TRACE("[Thread %2u] add\n", p->thread_id);
2922
2923 ALU(t, ip, +);
2924 }
2925
2926 static inline void
__instr_alu_add_mh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2927 __instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2928 struct thread *t,
2929 const struct instruction *ip)
2930 {
2931 TRACE("[Thread %2u] add (mh)\n", p->thread_id);
2932
2933 ALU_MH(t, ip, +);
2934 }
2935
2936 static inline void
__instr_alu_add_hm_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2937 __instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2938 struct thread *t,
2939 const struct instruction *ip)
2940 {
2941 TRACE("[Thread %2u] add (hm)\n", p->thread_id);
2942
2943 ALU_HM(t, ip, +);
2944 }
2945
2946 static inline void
__instr_alu_add_hh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2947 __instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2948 struct thread *t,
2949 const struct instruction *ip)
2950 {
2951 TRACE("[Thread %2u] add (hh)\n", p->thread_id);
2952
2953 ALU_HH(t, ip, +);
2954 }
2955
2956 static inline void
__instr_alu_add_mi_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2957 __instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2958 struct thread *t,
2959 const struct instruction *ip)
2960 {
2961 TRACE("[Thread %2u] add (mi)\n", p->thread_id);
2962
2963 ALU_MI(t, ip, +);
2964 }
2965
2966 static inline void
__instr_alu_add_hi_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2967 __instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2968 struct thread *t,
2969 const struct instruction *ip)
2970 {
2971 TRACE("[Thread %2u] add (hi)\n", p->thread_id);
2972
2973 ALU_HI(t, ip, +);
2974 }
2975
2976 static inline void
__instr_alu_sub_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2977 __instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused,
2978 struct thread *t,
2979 const struct instruction *ip)
2980 {
2981 TRACE("[Thread %2u] sub\n", p->thread_id);
2982
2983 ALU(t, ip, -);
2984 }
2985
2986 static inline void
__instr_alu_sub_mh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2987 __instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2988 struct thread *t,
2989 const struct instruction *ip)
2990 {
2991 TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
2992
2993 ALU_MH(t, ip, -);
2994 }
2995
2996 static inline void
__instr_alu_sub_hm_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)2997 __instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2998 struct thread *t,
2999 const struct instruction *ip)
3000 {
3001 TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3002
3003 ALU_HM(t, ip, -);
3004 }
3005
3006 static inline void
__instr_alu_sub_hh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3007 __instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3008 struct thread *t,
3009 const struct instruction *ip)
3010 {
3011 TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3012
3013 ALU_HH(t, ip, -);
3014 }
3015
3016 static inline void
__instr_alu_sub_mi_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3017 __instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused,
3018 struct thread *t,
3019 const struct instruction *ip)
3020 {
3021 TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3022
3023 ALU_MI(t, ip, -);
3024 }
3025
3026 static inline void
__instr_alu_sub_hi_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3027 __instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused,
3028 struct thread *t,
3029 const struct instruction *ip)
3030 {
3031 TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3032
3033 ALU_HI(t, ip, -);
3034 }
3035
3036 static inline void
__instr_alu_shl_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3037 __instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused,
3038 struct thread *t,
3039 const struct instruction *ip)
3040 {
3041 TRACE("[Thread %2u] shl\n", p->thread_id);
3042
3043 ALU(t, ip, <<);
3044 }
3045
3046 static inline void
__instr_alu_shl_mh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3047 __instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3048 struct thread *t,
3049 const struct instruction *ip)
3050 {
3051 TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3052
3053 ALU_MH(t, ip, <<);
3054 }
3055
3056 static inline void
__instr_alu_shl_hm_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3057 __instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3058 struct thread *t,
3059 const struct instruction *ip)
3060 {
3061 TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3062
3063 ALU_HM(t, ip, <<);
3064 }
3065
3066 static inline void
__instr_alu_shl_hh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3067 __instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3068 struct thread *t,
3069 const struct instruction *ip)
3070 {
3071 TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3072
3073 ALU_HH(t, ip, <<);
3074 }
3075
3076 static inline void
__instr_alu_shl_mi_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3077 __instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused,
3078 struct thread *t,
3079 const struct instruction *ip)
3080 {
3081 TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3082
3083 ALU_MI(t, ip, <<);
3084 }
3085
3086 static inline void
__instr_alu_shl_hi_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3087 __instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused,
3088 struct thread *t,
3089 const struct instruction *ip)
3090 {
3091 TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3092
3093 ALU_HI(t, ip, <<);
3094 }
3095
3096 static inline void
__instr_alu_shr_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3097 __instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused,
3098 struct thread *t,
3099 const struct instruction *ip)
3100 {
3101 TRACE("[Thread %2u] shr\n", p->thread_id);
3102
3103 ALU(t, ip, >>);
3104 }
3105
3106 static inline void
__instr_alu_shr_mh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3107 __instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3108 struct thread *t,
3109 const struct instruction *ip)
3110 {
3111 TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
3112
3113 ALU_MH(t, ip, >>);
3114 }
3115
3116 static inline void
__instr_alu_shr_hm_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3117 __instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3118 struct thread *t,
3119 const struct instruction *ip)
3120 {
3121 TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
3122
3123 ALU_HM(t, ip, >>);
3124 }
3125
3126 static inline void
__instr_alu_shr_hh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3127 __instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3128 struct thread *t,
3129 const struct instruction *ip)
3130 {
3131 TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3132
3133 ALU_HH(t, ip, >>);
3134 }
3135
3136 static inline void
__instr_alu_shr_mi_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3137 __instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused,
3138 struct thread *t,
3139 const struct instruction *ip)
3140 {
3141 TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3142
3143 /* Structs. */
3144 ALU_MI(t, ip, >>);
3145 }
3146
3147 static inline void
__instr_alu_shr_hi_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3148 __instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused,
3149 struct thread *t,
3150 const struct instruction *ip)
3151 {
3152 TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3153
3154 ALU_HI(t, ip, >>);
3155 }
3156
3157 static inline void
__instr_alu_and_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3158 __instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused,
3159 struct thread *t,
3160 const struct instruction *ip)
3161 {
3162 TRACE("[Thread %2u] and\n", p->thread_id);
3163
3164 ALU(t, ip, &);
3165 }
3166
3167 static inline void
__instr_alu_and_mh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3168 __instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3169 struct thread *t,
3170 const struct instruction *ip)
3171 {
3172 TRACE("[Thread %2u] and (mh)\n", p->thread_id);
3173
3174 ALU_MH(t, ip, &);
3175 }
3176
3177 static inline void
__instr_alu_and_hm_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3178 __instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3179 struct thread *t,
3180 const struct instruction *ip)
3181 {
3182 TRACE("[Thread %2u] and (hm)\n", p->thread_id);
3183
3184 ALU_HM_FAST(t, ip, &);
3185 }
3186
3187 static inline void
__instr_alu_and_hh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3188 __instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3189 struct thread *t,
3190 const struct instruction *ip)
3191 {
3192 TRACE("[Thread %2u] and (hh)\n", p->thread_id);
3193
3194 ALU_HH_FAST(t, ip, &);
3195 }
3196
3197 static inline void
__instr_alu_and_i_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3198 __instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused,
3199 struct thread *t,
3200 const struct instruction *ip)
3201 {
3202 TRACE("[Thread %2u] and (i)\n", p->thread_id);
3203
3204 ALU_I(t, ip, &);
3205 }
3206
3207 static inline void
__instr_alu_or_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3208 __instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused,
3209 struct thread *t,
3210 const struct instruction *ip)
3211 {
3212 TRACE("[Thread %2u] or\n", p->thread_id);
3213
3214 ALU(t, ip, |);
3215 }
3216
3217 static inline void
__instr_alu_or_mh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3218 __instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3219 struct thread *t,
3220 const struct instruction *ip)
3221 {
3222 TRACE("[Thread %2u] or (mh)\n", p->thread_id);
3223
3224 ALU_MH(t, ip, |);
3225 }
3226
3227 static inline void
__instr_alu_or_hm_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3228 __instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3229 struct thread *t,
3230 const struct instruction *ip)
3231 {
3232 TRACE("[Thread %2u] or (hm)\n", p->thread_id);
3233
3234 ALU_HM_FAST(t, ip, |);
3235 }
3236
3237 static inline void
__instr_alu_or_hh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3238 __instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3239 struct thread *t,
3240 const struct instruction *ip)
3241 {
3242 TRACE("[Thread %2u] or (hh)\n", p->thread_id);
3243
3244 ALU_HH_FAST(t, ip, |);
3245 }
3246
3247 static inline void
__instr_alu_or_i_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3248 __instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused,
3249 struct thread *t,
3250 const struct instruction *ip)
3251 {
3252 TRACE("[Thread %2u] or (i)\n", p->thread_id);
3253
3254 ALU_I(t, ip, |);
3255 }
3256
3257 static inline void
__instr_alu_xor_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3258 __instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused,
3259 struct thread *t,
3260 const struct instruction *ip)
3261 {
3262 TRACE("[Thread %2u] xor\n", p->thread_id);
3263
3264 ALU(t, ip, ^);
3265 }
3266
3267 static inline void
__instr_alu_xor_mh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3268 __instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3269 struct thread *t,
3270 const struct instruction *ip)
3271 {
3272 TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
3273
3274 ALU_MH(t, ip, ^);
3275 }
3276
3277 static inline void
__instr_alu_xor_hm_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3278 __instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3279 struct thread *t,
3280 const struct instruction *ip)
3281 {
3282 TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
3283
3284 ALU_HM_FAST(t, ip, ^);
3285 }
3286
3287 static inline void
__instr_alu_xor_hh_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3288 __instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3289 struct thread *t,
3290 const struct instruction *ip)
3291 {
3292 TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
3293
3294 ALU_HH_FAST(t, ip, ^);
3295 }
3296
3297 static inline void
__instr_alu_xor_i_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3298 __instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused,
3299 struct thread *t,
3300 const struct instruction *ip)
3301 {
3302 TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3303
3304 ALU_I(t, ip, ^);
3305 }
3306
3307 static inline void
__instr_alu_ckadd_field_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3308 __instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused,
3309 struct thread *t,
3310 const struct instruction *ip)
3311 {
3312 uint8_t *dst_struct, *src_struct;
3313 uint16_t *dst16_ptr, dst;
3314 uint64_t *src64_ptr, src64, src64_mask, src;
3315 uint64_t r;
3316
3317 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3318
3319 /* Structs. */
3320 dst_struct = t->structs[ip->alu.dst.struct_id];
3321 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3322 dst = *dst16_ptr;
3323
3324 src_struct = t->structs[ip->alu.src.struct_id];
3325 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3326 src64 = *src64_ptr;
3327 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3328 src = src64 & src64_mask;
3329
3330 /* Initialize the result with destination 1's complement. */
3331 r = dst;
3332 r = ~r & 0xFFFF;
3333
3334 /* The first input (r) is a 16-bit number. The second and the third
3335 * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3336 * three numbers (output r) is a 34-bit number.
3337 */
3338 r += (src >> 32) + (src & 0xFFFFFFFF);
3339
3340 /* The first input is a 16-bit number. The second input is an 18-bit
3341 * number. In the worst case scenario, the sum of the two numbers is a
3342 * 19-bit number.
3343 */
3344 r = (r & 0xFFFF) + (r >> 16);
3345
3346 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3347 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3348 */
3349 r = (r & 0xFFFF) + (r >> 16);
3350
3351 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3352 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3353 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3354 * therefore the output r is always a 16-bit number.
3355 */
3356 r = (r & 0xFFFF) + (r >> 16);
3357
3358 /* Apply 1's complement to the result. */
3359 r = ~r & 0xFFFF;
3360 r = r ? r : 0xFFFF;
3361
3362 *dst16_ptr = (uint16_t)r;
3363 }
3364
3365 static inline void
__instr_alu_cksub_field_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3366 __instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused,
3367 struct thread *t,
3368 const struct instruction *ip)
3369 {
3370 uint8_t *dst_struct, *src_struct;
3371 uint16_t *dst16_ptr, dst;
3372 uint64_t *src64_ptr, src64, src64_mask, src;
3373 uint64_t r;
3374
3375 TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3376
3377 /* Structs. */
3378 dst_struct = t->structs[ip->alu.dst.struct_id];
3379 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3380 dst = *dst16_ptr;
3381
3382 src_struct = t->structs[ip->alu.src.struct_id];
3383 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3384 src64 = *src64_ptr;
3385 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3386 src = src64 & src64_mask;
3387
3388 /* Initialize the result with destination 1's complement. */
3389 r = dst;
3390 r = ~r & 0xFFFF;
3391
3392 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3393 * the following sequence of operations in 2's complement arithmetic:
3394 * a '- b = (a - b) % 0xFFFF.
3395 *
3396 * In order to prevent an underflow for the below subtraction, in which
3397 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3398 * minuend), we first add a multiple of the 0xFFFF modulus to the
3399 * minuend. The number we add to the minuend needs to be a 34-bit number
3400 * or higher, so for readability reasons we picked the 36-bit multiple.
3401 * We are effectively turning the 16-bit minuend into a 36-bit number:
3402 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3403 */
3404 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3405
3406 /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3407 * result (the output r) is a 36-bit number.
3408 */
3409 r -= (src >> 32) + (src & 0xFFFFFFFF);
3410
3411 /* The first input is a 16-bit number. The second input is a 20-bit
3412 * number. Their sum is a 21-bit number.
3413 */
3414 r = (r & 0xFFFF) + (r >> 16);
3415
3416 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3417 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3418 */
3419 r = (r & 0xFFFF) + (r >> 16);
3420
3421 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3422 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3423 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3424 * generated, therefore the output r is always a 16-bit number.
3425 */
3426 r = (r & 0xFFFF) + (r >> 16);
3427
3428 /* Apply 1's complement to the result. */
3429 r = ~r & 0xFFFF;
3430 r = r ? r : 0xFFFF;
3431
3432 *dst16_ptr = (uint16_t)r;
3433 }
3434
3435 static inline void
__instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3436 __instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused,
3437 struct thread *t,
3438 const struct instruction *ip)
3439 {
3440 uint8_t *dst_struct, *src_struct;
3441 uint16_t *dst16_ptr, dst;
3442 uint32_t *src32_ptr;
3443 uint64_t r0, r1;
3444
3445 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3446
3447 /* Structs. */
3448 dst_struct = t->structs[ip->alu.dst.struct_id];
3449 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3450 dst = *dst16_ptr;
3451
3452 src_struct = t->structs[ip->alu.src.struct_id];
3453 src32_ptr = (uint32_t *)&src_struct[0];
3454
3455 /* Initialize the result with destination 1's complement. */
3456 r0 = dst;
3457 r0 = ~r0 & 0xFFFF;
3458
3459 r0 += src32_ptr[0]; /* The output r0 is a 33-bit number. */
3460 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3461 r0 += src32_ptr[2]; /* The output r0 is a 34-bit number. */
3462 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3463 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3464
3465 /* The first input is a 16-bit number. The second input is a 19-bit
3466 * number. Their sum is a 20-bit number.
3467 */
3468 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3469
3470 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3471 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3472 */
3473 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3474
3475 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3476 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3477 * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3478 * generated, therefore the output r is always a 16-bit number.
3479 */
3480 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3481
3482 /* Apply 1's complement to the result. */
3483 r0 = ~r0 & 0xFFFF;
3484 r0 = r0 ? r0 : 0xFFFF;
3485
3486 *dst16_ptr = (uint16_t)r0;
3487 }
3488
3489 static inline void
__instr_alu_ckadd_struct_exec(struct rte_swx_pipeline * p __rte_unused,struct thread * t,const struct instruction * ip)3490 __instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused,
3491 struct thread *t,
3492 const struct instruction *ip)
3493 {
3494 uint32_t src_header_id = ip->alu.src.n_bits; /* The src header ID is stored here. */
3495 uint32_t n_src_header_bytes = t->headers[src_header_id].n_bytes;
3496 uint8_t *dst_struct, *src_struct;
3497 uint16_t *dst16_ptr, dst;
3498 uint32_t *src32_ptr;
3499 uint64_t r;
3500 uint32_t i;
3501
3502 if (n_src_header_bytes == 20) {
3503 __instr_alu_ckadd_struct20_exec(p, t, ip);
3504 return;
3505 }
3506
3507 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3508
3509 /* Structs. */
3510 dst_struct = t->structs[ip->alu.dst.struct_id];
3511 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3512 dst = *dst16_ptr;
3513
3514 src_struct = t->structs[ip->alu.src.struct_id];
3515 src32_ptr = (uint32_t *)&src_struct[0];
3516
3517 /* Initialize the result with destination 1's complement. */
3518 r = dst;
3519 r = ~r & 0xFFFF;
3520
3521 /* The max number of 32-bit words in a 32K-byte header is 2^13.
3522 * Therefore, in the worst case scenario, a 45-bit number is added to a
3523 * 16-bit number (the input r), so the output r is 46-bit number.
3524 */
3525 for (i = 0; i < n_src_header_bytes / 4; i++, src32_ptr++)
3526 r += *src32_ptr;
3527
3528 /* The first input is a 16-bit number. The second input is a 30-bit
3529 * number. Their sum is a 31-bit number.
3530 */
3531 r = (r & 0xFFFF) + (r >> 16);
3532
3533 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3534 * a 15-bit number (0 .. 0x7FFF). The sum is a 17-bit number (0 .. 0x17FFE).
3535 */
3536 r = (r & 0xFFFF) + (r >> 16);
3537
3538 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3539 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3540 * 0x17FFE), the output r is (0 .. 0x7FFF). So no carry bit can be
3541 * generated, therefore the output r is always a 16-bit number.
3542 */
3543 r = (r & 0xFFFF) + (r >> 16);
3544
3545 /* Apply 1's complement to the result. */
3546 r = ~r & 0xFFFF;
3547 r = r ? r : 0xFFFF;
3548
3549 *dst16_ptr = (uint16_t)r;
3550 }
3551
3552 /*
3553 * Register array.
3554 */
3555 static inline uint64_t *
instr_regarray_regarray(struct rte_swx_pipeline * p,const struct instruction * ip)3556 instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip)
3557 {
3558 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3559 return r->regarray;
3560 }
3561
3562 static inline uint64_t
instr_regarray_idx_hbo(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3563 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3564 {
3565 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3566
3567 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3568 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3569 uint64_t idx64 = *idx64_ptr;
3570 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
3571 uint64_t idx = idx64 & idx64_mask & r->size_mask;
3572
3573 return idx;
3574 }
3575
3576 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3577
3578 static inline uint64_t
instr_regarray_idx_nbo(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3579 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3580 {
3581 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3582
3583 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3584 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3585 uint64_t idx64 = *idx64_ptr;
3586 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
3587
3588 return idx;
3589 }
3590
3591 #else
3592
3593 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
3594
3595 #endif
3596
3597 static inline uint64_t
instr_regarray_idx_imm(struct rte_swx_pipeline * p,const struct instruction * ip)3598 instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3599 {
3600 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3601
3602 uint64_t idx = ip->regarray.idx_val & r->size_mask;
3603
3604 return idx;
3605 }
3606
3607 static inline uint64_t
instr_regarray_src_hbo(struct thread * t,const struct instruction * ip)3608 instr_regarray_src_hbo(struct thread *t, const struct instruction *ip)
3609 {
3610 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3611 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3612 uint64_t src64 = *src64_ptr;
3613 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3614 uint64_t src = src64 & src64_mask;
3615
3616 return src;
3617 }
3618
3619 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3620
3621 static inline uint64_t
instr_regarray_src_nbo(struct thread * t,const struct instruction * ip)3622 instr_regarray_src_nbo(struct thread *t, const struct instruction *ip)
3623 {
3624 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3625 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3626 uint64_t src64 = *src64_ptr;
3627 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
3628
3629 return src;
3630 }
3631
3632 #else
3633
3634 #define instr_regarray_src_nbo instr_regarray_src_hbo
3635
3636 #endif
3637
3638 static inline void
instr_regarray_dst_hbo_src_hbo_set(struct thread * t,const struct instruction * ip,uint64_t src)3639 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
3640 {
3641 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3642 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3643 uint64_t dst64 = *dst64_ptr;
3644 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3645
3646 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3647
3648 }
3649
3650 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3651
3652 static inline void
instr_regarray_dst_nbo_src_hbo_set(struct thread * t,const struct instruction * ip,uint64_t src)3653 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
3654 {
3655 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3656 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3657 uint64_t dst64 = *dst64_ptr;
3658 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3659
3660 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
3661 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3662 }
3663
3664 #else
3665
3666 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
3667
3668 #endif
3669
3670 static inline void
__instr_regprefetch_rh_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3671 __instr_regprefetch_rh_exec(struct rte_swx_pipeline *p,
3672 struct thread *t,
3673 const struct instruction *ip)
3674 {
3675 uint64_t *regarray, idx;
3676
3677 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
3678
3679 regarray = instr_regarray_regarray(p, ip);
3680 idx = instr_regarray_idx_nbo(p, t, ip);
3681 rte_prefetch0(®array[idx]);
3682 }
3683
3684 static inline void
__instr_regprefetch_rm_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3685 __instr_regprefetch_rm_exec(struct rte_swx_pipeline *p,
3686 struct thread *t,
3687 const struct instruction *ip)
3688 {
3689 uint64_t *regarray, idx;
3690
3691 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
3692
3693 regarray = instr_regarray_regarray(p, ip);
3694 idx = instr_regarray_idx_hbo(p, t, ip);
3695 rte_prefetch0(®array[idx]);
3696 }
3697
3698 static inline void
__instr_regprefetch_ri_exec(struct rte_swx_pipeline * p,struct thread * t __rte_unused,const struct instruction * ip)3699 __instr_regprefetch_ri_exec(struct rte_swx_pipeline *p,
3700 struct thread *t __rte_unused,
3701 const struct instruction *ip)
3702 {
3703 uint64_t *regarray, idx;
3704
3705 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
3706
3707 regarray = instr_regarray_regarray(p, ip);
3708 idx = instr_regarray_idx_imm(p, ip);
3709 rte_prefetch0(®array[idx]);
3710 }
3711
3712 static inline void
__instr_regrd_hrh_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3713 __instr_regrd_hrh_exec(struct rte_swx_pipeline *p,
3714 struct thread *t,
3715 const struct instruction *ip)
3716 {
3717 uint64_t *regarray, idx;
3718
3719 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
3720
3721 regarray = instr_regarray_regarray(p, ip);
3722 idx = instr_regarray_idx_nbo(p, t, ip);
3723 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3724 }
3725
3726 static inline void
__instr_regrd_hrm_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3727 __instr_regrd_hrm_exec(struct rte_swx_pipeline *p,
3728 struct thread *t,
3729 const struct instruction *ip)
3730 {
3731 uint64_t *regarray, idx;
3732
3733 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
3734
3735 /* Structs. */
3736 regarray = instr_regarray_regarray(p, ip);
3737 idx = instr_regarray_idx_hbo(p, t, ip);
3738 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3739 }
3740
3741 static inline void
__instr_regrd_mrh_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3742 __instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3743 {
3744 uint64_t *regarray, idx;
3745
3746 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
3747
3748 regarray = instr_regarray_regarray(p, ip);
3749 idx = instr_regarray_idx_nbo(p, t, ip);
3750 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3751 }
3752
3753 static inline void
__instr_regrd_mrm_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3754 __instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3755 {
3756 uint64_t *regarray, idx;
3757
3758 TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id);
3759
3760 regarray = instr_regarray_regarray(p, ip);
3761 idx = instr_regarray_idx_hbo(p, t, ip);
3762 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3763 }
3764
3765 static inline void
__instr_regrd_hri_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3766 __instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3767 {
3768 uint64_t *regarray, idx;
3769
3770 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
3771
3772 regarray = instr_regarray_regarray(p, ip);
3773 idx = instr_regarray_idx_imm(p, ip);
3774 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3775 }
3776
3777 static inline void
__instr_regrd_mri_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3778 __instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3779 {
3780 uint64_t *regarray, idx;
3781
3782 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
3783
3784 regarray = instr_regarray_regarray(p, ip);
3785 idx = instr_regarray_idx_imm(p, ip);
3786 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3787 }
3788
3789 static inline void
__instr_regwr_rhh_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3790 __instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3791 {
3792 uint64_t *regarray, idx, src;
3793
3794 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
3795
3796 regarray = instr_regarray_regarray(p, ip);
3797 idx = instr_regarray_idx_nbo(p, t, ip);
3798 src = instr_regarray_src_nbo(t, ip);
3799 regarray[idx] = src;
3800 }
3801
3802 static inline void
__instr_regwr_rhm_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3803 __instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3804 {
3805 uint64_t *regarray, idx, src;
3806
3807 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
3808
3809 regarray = instr_regarray_regarray(p, ip);
3810 idx = instr_regarray_idx_nbo(p, t, ip);
3811 src = instr_regarray_src_hbo(t, ip);
3812 regarray[idx] = src;
3813 }
3814
3815 static inline void
__instr_regwr_rmh_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3816 __instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3817 {
3818 uint64_t *regarray, idx, src;
3819
3820 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
3821
3822 regarray = instr_regarray_regarray(p, ip);
3823 idx = instr_regarray_idx_hbo(p, t, ip);
3824 src = instr_regarray_src_nbo(t, ip);
3825 regarray[idx] = src;
3826 }
3827
3828 static inline void
__instr_regwr_rmm_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3829 __instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3830 {
3831 uint64_t *regarray, idx, src;
3832
3833 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
3834
3835 regarray = instr_regarray_regarray(p, ip);
3836 idx = instr_regarray_idx_hbo(p, t, ip);
3837 src = instr_regarray_src_hbo(t, ip);
3838 regarray[idx] = src;
3839 }
3840
3841 static inline void
__instr_regwr_rhi_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3842 __instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3843 {
3844 uint64_t *regarray, idx, src;
3845
3846 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
3847
3848 regarray = instr_regarray_regarray(p, ip);
3849 idx = instr_regarray_idx_nbo(p, t, ip);
3850 src = ip->regarray.dstsrc_val;
3851 regarray[idx] = src;
3852 }
3853
3854 static inline void
__instr_regwr_rmi_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3855 __instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3856 {
3857 uint64_t *regarray, idx, src;
3858
3859 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
3860
3861 regarray = instr_regarray_regarray(p, ip);
3862 idx = instr_regarray_idx_hbo(p, t, ip);
3863 src = ip->regarray.dstsrc_val;
3864 regarray[idx] = src;
3865 }
3866
3867 static inline void
__instr_regwr_rih_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3868 __instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3869 {
3870 uint64_t *regarray, idx, src;
3871
3872 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
3873
3874 regarray = instr_regarray_regarray(p, ip);
3875 idx = instr_regarray_idx_imm(p, ip);
3876 src = instr_regarray_src_nbo(t, ip);
3877 regarray[idx] = src;
3878 }
3879
3880 static inline void
__instr_regwr_rim_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3881 __instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3882 {
3883 uint64_t *regarray, idx, src;
3884
3885 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
3886
3887 regarray = instr_regarray_regarray(p, ip);
3888 idx = instr_regarray_idx_imm(p, ip);
3889 src = instr_regarray_src_hbo(t, ip);
3890 regarray[idx] = src;
3891 }
3892
3893 static inline void
__instr_regwr_rii_exec(struct rte_swx_pipeline * p,struct thread * t __rte_unused,const struct instruction * ip)3894 __instr_regwr_rii_exec(struct rte_swx_pipeline *p,
3895 struct thread *t __rte_unused,
3896 const struct instruction *ip)
3897 {
3898 uint64_t *regarray, idx, src;
3899
3900 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
3901
3902 regarray = instr_regarray_regarray(p, ip);
3903 idx = instr_regarray_idx_imm(p, ip);
3904 src = ip->regarray.dstsrc_val;
3905 regarray[idx] = src;
3906 }
3907
3908 static inline void
__instr_regadd_rhh_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3909 __instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3910 {
3911 uint64_t *regarray, idx, src;
3912
3913 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
3914
3915 regarray = instr_regarray_regarray(p, ip);
3916 idx = instr_regarray_idx_nbo(p, t, ip);
3917 src = instr_regarray_src_nbo(t, ip);
3918 regarray[idx] += src;
3919 }
3920
3921 static inline void
__instr_regadd_rhm_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3922 __instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3923 {
3924 uint64_t *regarray, idx, src;
3925
3926 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
3927
3928 regarray = instr_regarray_regarray(p, ip);
3929 idx = instr_regarray_idx_nbo(p, t, ip);
3930 src = instr_regarray_src_hbo(t, ip);
3931 regarray[idx] += src;
3932 }
3933
3934 static inline void
__instr_regadd_rmh_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3935 __instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3936 {
3937 uint64_t *regarray, idx, src;
3938
3939 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
3940
3941 regarray = instr_regarray_regarray(p, ip);
3942 idx = instr_regarray_idx_hbo(p, t, ip);
3943 src = instr_regarray_src_nbo(t, ip);
3944 regarray[idx] += src;
3945 }
3946
3947 static inline void
__instr_regadd_rmm_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3948 __instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3949 {
3950 uint64_t *regarray, idx, src;
3951
3952 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
3953
3954 regarray = instr_regarray_regarray(p, ip);
3955 idx = instr_regarray_idx_hbo(p, t, ip);
3956 src = instr_regarray_src_hbo(t, ip);
3957 regarray[idx] += src;
3958 }
3959
3960 static inline void
__instr_regadd_rhi_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3961 __instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3962 {
3963 uint64_t *regarray, idx, src;
3964
3965 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
3966
3967 regarray = instr_regarray_regarray(p, ip);
3968 idx = instr_regarray_idx_nbo(p, t, ip);
3969 src = ip->regarray.dstsrc_val;
3970 regarray[idx] += src;
3971 }
3972
3973 static inline void
__instr_regadd_rmi_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3974 __instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3975 {
3976 uint64_t *regarray, idx, src;
3977
3978 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
3979
3980 regarray = instr_regarray_regarray(p, ip);
3981 idx = instr_regarray_idx_hbo(p, t, ip);
3982 src = ip->regarray.dstsrc_val;
3983 regarray[idx] += src;
3984 }
3985
3986 static inline void
__instr_regadd_rih_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)3987 __instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3988 {
3989 uint64_t *regarray, idx, src;
3990
3991 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
3992
3993 regarray = instr_regarray_regarray(p, ip);
3994 idx = instr_regarray_idx_imm(p, ip);
3995 src = instr_regarray_src_nbo(t, ip);
3996 regarray[idx] += src;
3997 }
3998
3999 static inline void
__instr_regadd_rim_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4000 __instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4001 {
4002 uint64_t *regarray, idx, src;
4003
4004 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
4005
4006 regarray = instr_regarray_regarray(p, ip);
4007 idx = instr_regarray_idx_imm(p, ip);
4008 src = instr_regarray_src_hbo(t, ip);
4009 regarray[idx] += src;
4010 }
4011
4012 static inline void
__instr_regadd_rii_exec(struct rte_swx_pipeline * p,struct thread * t __rte_unused,const struct instruction * ip)4013 __instr_regadd_rii_exec(struct rte_swx_pipeline *p,
4014 struct thread *t __rte_unused,
4015 const struct instruction *ip)
4016 {
4017 uint64_t *regarray, idx, src;
4018
4019 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
4020
4021 regarray = instr_regarray_regarray(p, ip);
4022 idx = instr_regarray_idx_imm(p, ip);
4023 src = ip->regarray.dstsrc_val;
4024 regarray[idx] += src;
4025 }
4026
4027 /*
4028 * metarray.
4029 */
4030 static inline struct meter *
instr_meter_idx_hbo(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4031 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4032 {
4033 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
4034
4035 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
4036 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
4037 uint64_t idx64 = *idx64_ptr;
4038 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
4039 uint64_t idx = idx64 & idx64_mask & r->size_mask;
4040
4041 return &r->metarray[idx];
4042 }
4043
4044 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4045
4046 static inline struct meter *
instr_meter_idx_nbo(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4047 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4048 {
4049 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
4050
4051 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
4052 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
4053 uint64_t idx64 = *idx64_ptr;
4054 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
4055
4056 return &r->metarray[idx];
4057 }
4058
4059 #else
4060
4061 #define instr_meter_idx_nbo instr_meter_idx_hbo
4062
4063 #endif
4064
4065 static inline struct meter *
instr_meter_idx_imm(struct rte_swx_pipeline * p,const struct instruction * ip)4066 instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
4067 {
4068 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
4069
4070 uint64_t idx = ip->meter.idx_val & r->size_mask;
4071
4072 return &r->metarray[idx];
4073 }
4074
4075 static inline uint32_t
instr_meter_length_hbo(struct thread * t,const struct instruction * ip)4076 instr_meter_length_hbo(struct thread *t, const struct instruction *ip)
4077 {
4078 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
4079 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
4080 uint64_t src64 = *src64_ptr;
4081 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
4082 uint64_t src = src64 & src64_mask;
4083
4084 return (uint32_t)src;
4085 }
4086
4087 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
4088
4089 static inline uint32_t
instr_meter_length_nbo(struct thread * t,const struct instruction * ip)4090 instr_meter_length_nbo(struct thread *t, const struct instruction *ip)
4091 {
4092 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
4093 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
4094 uint64_t src64 = *src64_ptr;
4095 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
4096
4097 return (uint32_t)src;
4098 }
4099
4100 #else
4101
4102 #define instr_meter_length_nbo instr_meter_length_hbo
4103
4104 #endif
4105
4106 static inline enum rte_color
instr_meter_color_in_hbo(struct thread * t,const struct instruction * ip)4107 instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip)
4108 {
4109 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
4110 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
4111 uint64_t src64 = *src64_ptr;
4112 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
4113 uint64_t src = src64 & src64_mask;
4114
4115 return (enum rte_color)src;
4116 }
4117
4118 static inline void
instr_meter_color_out_hbo_set(struct thread * t,const struct instruction * ip,enum rte_color color_out)4119 instr_meter_color_out_hbo_set(struct thread *t,
4120 const struct instruction *ip,
4121 enum rte_color color_out)
4122 {
4123 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
4124 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
4125 uint64_t dst64 = *dst64_ptr;
4126 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
4127
4128 uint64_t src = (uint64_t)color_out;
4129
4130 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4131 }
4132
4133 static inline void
__instr_metprefetch_h_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4134 __instr_metprefetch_h_exec(struct rte_swx_pipeline *p,
4135 struct thread *t,
4136 const struct instruction *ip)
4137 {
4138 struct meter *m;
4139
4140 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
4141
4142 m = instr_meter_idx_nbo(p, t, ip);
4143 rte_prefetch0(m);
4144 }
4145
4146 static inline void
__instr_metprefetch_m_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4147 __instr_metprefetch_m_exec(struct rte_swx_pipeline *p,
4148 struct thread *t,
4149 const struct instruction *ip)
4150 {
4151 struct meter *m;
4152
4153 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
4154
4155 m = instr_meter_idx_hbo(p, t, ip);
4156 rte_prefetch0(m);
4157 }
4158
4159 static inline void
__instr_metprefetch_i_exec(struct rte_swx_pipeline * p,struct thread * t __rte_unused,const struct instruction * ip)4160 __instr_metprefetch_i_exec(struct rte_swx_pipeline *p,
4161 struct thread *t __rte_unused,
4162 const struct instruction *ip)
4163 {
4164 struct meter *m;
4165
4166 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
4167
4168 m = instr_meter_idx_imm(p, ip);
4169 rte_prefetch0(m);
4170 }
4171
4172 static inline void
__instr_meter_hhm_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4173 __instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4174 {
4175 struct meter *m;
4176 uint64_t time, n_pkts, n_bytes;
4177 uint32_t length;
4178 enum rte_color color_in, color_out;
4179
4180 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
4181
4182 m = instr_meter_idx_nbo(p, t, ip);
4183 rte_prefetch0(m->n_pkts);
4184 time = rte_get_tsc_cycles();
4185 length = instr_meter_length_nbo(t, ip);
4186 color_in = instr_meter_color_in_hbo(t, ip);
4187
4188 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4189 &m->profile->profile,
4190 time,
4191 length,
4192 color_in);
4193
4194 color_out &= m->color_mask;
4195
4196 n_pkts = m->n_pkts[color_out];
4197 n_bytes = m->n_bytes[color_out];
4198
4199 instr_meter_color_out_hbo_set(t, ip, color_out);
4200
4201 m->n_pkts[color_out] = n_pkts + 1;
4202 m->n_bytes[color_out] = n_bytes + length;
4203 }
4204
4205 static inline void
__instr_meter_hhi_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4206 __instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4207 {
4208 struct meter *m;
4209 uint64_t time, n_pkts, n_bytes;
4210 uint32_t length;
4211 enum rte_color color_in, color_out;
4212
4213 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
4214
4215 m = instr_meter_idx_nbo(p, t, ip);
4216 rte_prefetch0(m->n_pkts);
4217 time = rte_get_tsc_cycles();
4218 length = instr_meter_length_nbo(t, ip);
4219 color_in = (enum rte_color)ip->meter.color_in_val;
4220
4221 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4222 &m->profile->profile,
4223 time,
4224 length,
4225 color_in);
4226
4227 color_out &= m->color_mask;
4228
4229 n_pkts = m->n_pkts[color_out];
4230 n_bytes = m->n_bytes[color_out];
4231
4232 instr_meter_color_out_hbo_set(t, ip, color_out);
4233
4234 m->n_pkts[color_out] = n_pkts + 1;
4235 m->n_bytes[color_out] = n_bytes + length;
4236 }
4237
4238 static inline void
__instr_meter_hmm_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4239 __instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4240 {
4241 struct meter *m;
4242 uint64_t time, n_pkts, n_bytes;
4243 uint32_t length;
4244 enum rte_color color_in, color_out;
4245
4246 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
4247
4248 m = instr_meter_idx_nbo(p, t, ip);
4249 rte_prefetch0(m->n_pkts);
4250 time = rte_get_tsc_cycles();
4251 length = instr_meter_length_hbo(t, ip);
4252 color_in = instr_meter_color_in_hbo(t, ip);
4253
4254 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4255 &m->profile->profile,
4256 time,
4257 length,
4258 color_in);
4259
4260 color_out &= m->color_mask;
4261
4262 n_pkts = m->n_pkts[color_out];
4263 n_bytes = m->n_bytes[color_out];
4264
4265 instr_meter_color_out_hbo_set(t, ip, color_out);
4266
4267 m->n_pkts[color_out] = n_pkts + 1;
4268 m->n_bytes[color_out] = n_bytes + length;
4269 }
4270
4271 static inline void
__instr_meter_hmi_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4272 __instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4273 {
4274 struct meter *m;
4275 uint64_t time, n_pkts, n_bytes;
4276 uint32_t length;
4277 enum rte_color color_in, color_out;
4278
4279 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
4280
4281 m = instr_meter_idx_nbo(p, t, ip);
4282 rte_prefetch0(m->n_pkts);
4283 time = rte_get_tsc_cycles();
4284 length = instr_meter_length_hbo(t, ip);
4285 color_in = (enum rte_color)ip->meter.color_in_val;
4286
4287 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4288 &m->profile->profile,
4289 time,
4290 length,
4291 color_in);
4292
4293 color_out &= m->color_mask;
4294
4295 n_pkts = m->n_pkts[color_out];
4296 n_bytes = m->n_bytes[color_out];
4297
4298 instr_meter_color_out_hbo_set(t, ip, color_out);
4299
4300 m->n_pkts[color_out] = n_pkts + 1;
4301 m->n_bytes[color_out] = n_bytes + length;
4302 }
4303
4304 static inline void
__instr_meter_mhm_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4305 __instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4306 {
4307 struct meter *m;
4308 uint64_t time, n_pkts, n_bytes;
4309 uint32_t length;
4310 enum rte_color color_in, color_out;
4311
4312 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
4313
4314 m = instr_meter_idx_hbo(p, t, ip);
4315 rte_prefetch0(m->n_pkts);
4316 time = rte_get_tsc_cycles();
4317 length = instr_meter_length_nbo(t, ip);
4318 color_in = instr_meter_color_in_hbo(t, ip);
4319
4320 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4321 &m->profile->profile,
4322 time,
4323 length,
4324 color_in);
4325
4326 color_out &= m->color_mask;
4327
4328 n_pkts = m->n_pkts[color_out];
4329 n_bytes = m->n_bytes[color_out];
4330
4331 instr_meter_color_out_hbo_set(t, ip, color_out);
4332
4333 m->n_pkts[color_out] = n_pkts + 1;
4334 m->n_bytes[color_out] = n_bytes + length;
4335 }
4336
4337 static inline void
__instr_meter_mhi_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4338 __instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4339 {
4340 struct meter *m;
4341 uint64_t time, n_pkts, n_bytes;
4342 uint32_t length;
4343 enum rte_color color_in, color_out;
4344
4345 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
4346
4347 m = instr_meter_idx_hbo(p, t, ip);
4348 rte_prefetch0(m->n_pkts);
4349 time = rte_get_tsc_cycles();
4350 length = instr_meter_length_nbo(t, ip);
4351 color_in = (enum rte_color)ip->meter.color_in_val;
4352
4353 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4354 &m->profile->profile,
4355 time,
4356 length,
4357 color_in);
4358
4359 color_out &= m->color_mask;
4360
4361 n_pkts = m->n_pkts[color_out];
4362 n_bytes = m->n_bytes[color_out];
4363
4364 instr_meter_color_out_hbo_set(t, ip, color_out);
4365
4366 m->n_pkts[color_out] = n_pkts + 1;
4367 m->n_bytes[color_out] = n_bytes + length;
4368 }
4369
4370 static inline void
__instr_meter_mmm_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4371 __instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4372 {
4373 struct meter *m;
4374 uint64_t time, n_pkts, n_bytes;
4375 uint32_t length;
4376 enum rte_color color_in, color_out;
4377
4378 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
4379
4380 m = instr_meter_idx_hbo(p, t, ip);
4381 rte_prefetch0(m->n_pkts);
4382 time = rte_get_tsc_cycles();
4383 length = instr_meter_length_hbo(t, ip);
4384 color_in = instr_meter_color_in_hbo(t, ip);
4385
4386 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4387 &m->profile->profile,
4388 time,
4389 length,
4390 color_in);
4391
4392 color_out &= m->color_mask;
4393
4394 n_pkts = m->n_pkts[color_out];
4395 n_bytes = m->n_bytes[color_out];
4396
4397 instr_meter_color_out_hbo_set(t, ip, color_out);
4398
4399 m->n_pkts[color_out] = n_pkts + 1;
4400 m->n_bytes[color_out] = n_bytes + length;
4401 }
4402
4403 static inline void
__instr_meter_mmi_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4404 __instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4405 {
4406 struct meter *m;
4407 uint64_t time, n_pkts, n_bytes;
4408 uint32_t length;
4409 enum rte_color color_in, color_out;
4410
4411 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
4412
4413 m = instr_meter_idx_hbo(p, t, ip);
4414 rte_prefetch0(m->n_pkts);
4415 time = rte_get_tsc_cycles();
4416 length = instr_meter_length_hbo(t, ip);
4417 color_in = (enum rte_color)ip->meter.color_in_val;
4418
4419 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4420 &m->profile->profile,
4421 time,
4422 length,
4423 color_in);
4424
4425 color_out &= m->color_mask;
4426
4427 n_pkts = m->n_pkts[color_out];
4428 n_bytes = m->n_bytes[color_out];
4429
4430 instr_meter_color_out_hbo_set(t, ip, color_out);
4431
4432 m->n_pkts[color_out] = n_pkts + 1;
4433 m->n_bytes[color_out] = n_bytes + length;
4434 }
4435
4436 static inline void
__instr_meter_ihm_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4437 __instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4438 {
4439 struct meter *m;
4440 uint64_t time, n_pkts, n_bytes;
4441 uint32_t length;
4442 enum rte_color color_in, color_out;
4443
4444 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
4445
4446 m = instr_meter_idx_imm(p, ip);
4447 rte_prefetch0(m->n_pkts);
4448 time = rte_get_tsc_cycles();
4449 length = instr_meter_length_nbo(t, ip);
4450 color_in = instr_meter_color_in_hbo(t, ip);
4451
4452 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4453 &m->profile->profile,
4454 time,
4455 length,
4456 color_in);
4457
4458 color_out &= m->color_mask;
4459
4460 n_pkts = m->n_pkts[color_out];
4461 n_bytes = m->n_bytes[color_out];
4462
4463 instr_meter_color_out_hbo_set(t, ip, color_out);
4464
4465 m->n_pkts[color_out] = n_pkts + 1;
4466 m->n_bytes[color_out] = n_bytes + length;
4467 }
4468
4469 static inline void
__instr_meter_ihi_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4470 __instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4471 {
4472 struct meter *m;
4473 uint64_t time, n_pkts, n_bytes;
4474 uint32_t length;
4475 enum rte_color color_in, color_out;
4476
4477 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
4478
4479 m = instr_meter_idx_imm(p, ip);
4480 rte_prefetch0(m->n_pkts);
4481 time = rte_get_tsc_cycles();
4482 length = instr_meter_length_nbo(t, ip);
4483 color_in = (enum rte_color)ip->meter.color_in_val;
4484
4485 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4486 &m->profile->profile,
4487 time,
4488 length,
4489 color_in);
4490
4491 color_out &= m->color_mask;
4492
4493 n_pkts = m->n_pkts[color_out];
4494 n_bytes = m->n_bytes[color_out];
4495
4496 instr_meter_color_out_hbo_set(t, ip, color_out);
4497
4498 m->n_pkts[color_out] = n_pkts + 1;
4499 m->n_bytes[color_out] = n_bytes + length;
4500 }
4501
4502 static inline void
__instr_meter_imm_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4503 __instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4504 {
4505 struct meter *m;
4506 uint64_t time, n_pkts, n_bytes;
4507 uint32_t length;
4508 enum rte_color color_in, color_out;
4509
4510 TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
4511
4512 m = instr_meter_idx_imm(p, ip);
4513 rte_prefetch0(m->n_pkts);
4514 time = rte_get_tsc_cycles();
4515 length = instr_meter_length_hbo(t, ip);
4516 color_in = instr_meter_color_in_hbo(t, ip);
4517
4518 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4519 &m->profile->profile,
4520 time,
4521 length,
4522 color_in);
4523
4524 color_out &= m->color_mask;
4525
4526 n_pkts = m->n_pkts[color_out];
4527 n_bytes = m->n_bytes[color_out];
4528
4529 instr_meter_color_out_hbo_set(t, ip, color_out);
4530
4531 m->n_pkts[color_out] = n_pkts + 1;
4532 m->n_bytes[color_out] = n_bytes + length;
4533 }
4534
4535 static inline void
__instr_meter_imi_exec(struct rte_swx_pipeline * p,struct thread * t,const struct instruction * ip)4536 __instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4537 {
4538 struct meter *m;
4539 uint64_t time, n_pkts, n_bytes;
4540 uint32_t length;
4541 enum rte_color color_in, color_out;
4542
4543 TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
4544
4545 m = instr_meter_idx_imm(p, ip);
4546 rte_prefetch0(m->n_pkts);
4547 time = rte_get_tsc_cycles();
4548 length = instr_meter_length_hbo(t, ip);
4549 color_in = (enum rte_color)ip->meter.color_in_val;
4550
4551 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4552 &m->profile->profile,
4553 time,
4554 length,
4555 color_in);
4556
4557 color_out &= m->color_mask;
4558
4559 n_pkts = m->n_pkts[color_out];
4560 n_bytes = m->n_bytes[color_out];
4561
4562 instr_meter_color_out_hbo_set(t, ip, color_out);
4563
4564 m->n_pkts[color_out] = n_pkts + 1;
4565 m->n_bytes[color_out] = n_bytes + length;
4566 }
4567
4568 #endif
4569