1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2021 Intel Corporation 3 */ 4 #ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__ 5 #define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__ 6 7 #include <inttypes.h> 8 #include <string.h> 9 #include <sys/queue.h> 10 11 #include <rte_byteorder.h> 12 #include <rte_common.h> 13 #include <rte_cycles.h> 14 #include <rte_prefetch.h> 15 #include <rte_meter.h> 16 17 #include <rte_swx_table_selector.h> 18 #include <rte_swx_table_learner.h> 19 #include <rte_swx_pipeline.h> 20 #include <rte_swx_ctl.h> 21 22 #ifndef TRACE_LEVEL 23 #define TRACE_LEVEL 0 24 #endif 25 26 #if TRACE_LEVEL 27 #define TRACE(...) printf(__VA_ARGS__) 28 #else 29 #define TRACE(...) 30 #endif 31 32 /* 33 * Environment. 34 */ 35 #define ntoh64(x) rte_be_to_cpu_64(x) 36 #define hton64(x) rte_cpu_to_be_64(x) 37 38 /* 39 * Struct. 40 */ 41 struct field { 42 char name[RTE_SWX_NAME_SIZE]; 43 uint32_t n_bits; 44 uint32_t offset; 45 int var_size; 46 }; 47 48 struct struct_type { 49 TAILQ_ENTRY(struct_type) node; 50 char name[RTE_SWX_NAME_SIZE]; 51 struct field *fields; 52 uint32_t n_fields; 53 uint32_t n_bits; 54 uint32_t n_bits_min; 55 int var_size; 56 }; 57 58 TAILQ_HEAD(struct_type_tailq, struct_type); 59 60 /* 61 * Input port. 62 */ 63 struct port_in_type { 64 TAILQ_ENTRY(port_in_type) node; 65 char name[RTE_SWX_NAME_SIZE]; 66 struct rte_swx_port_in_ops ops; 67 }; 68 69 TAILQ_HEAD(port_in_type_tailq, port_in_type); 70 71 struct port_in { 72 TAILQ_ENTRY(port_in) node; 73 struct port_in_type *type; 74 void *obj; 75 uint32_t id; 76 }; 77 78 TAILQ_HEAD(port_in_tailq, port_in); 79 80 struct port_in_runtime { 81 rte_swx_port_in_pkt_rx_t pkt_rx; 82 void *obj; 83 }; 84 85 /* 86 * Output port. 87 */ 88 struct port_out_type { 89 TAILQ_ENTRY(port_out_type) node; 90 char name[RTE_SWX_NAME_SIZE]; 91 struct rte_swx_port_out_ops ops; 92 }; 93 94 TAILQ_HEAD(port_out_type_tailq, port_out_type); 95 96 struct port_out { 97 TAILQ_ENTRY(port_out) node; 98 struct port_out_type *type; 99 void *obj; 100 uint32_t id; 101 }; 102 103 TAILQ_HEAD(port_out_tailq, port_out); 104 105 struct port_out_runtime { 106 rte_swx_port_out_pkt_tx_t pkt_tx; 107 rte_swx_port_out_pkt_fast_clone_tx_t pkt_fast_clone_tx; 108 rte_swx_port_out_pkt_clone_tx_t pkt_clone_tx; 109 rte_swx_port_out_flush_t flush; 110 void *obj; 111 }; 112 113 /* 114 * Packet mirroring. 115 */ 116 struct mirroring_session { 117 uint32_t port_id; 118 int fast_clone; 119 uint32_t truncation_length; 120 }; 121 122 /* 123 * Extern object. 124 */ 125 struct extern_type_member_func { 126 TAILQ_ENTRY(extern_type_member_func) node; 127 char name[RTE_SWX_NAME_SIZE]; 128 rte_swx_extern_type_member_func_t func; 129 uint32_t id; 130 }; 131 132 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func); 133 134 struct extern_type { 135 TAILQ_ENTRY(extern_type) node; 136 char name[RTE_SWX_NAME_SIZE]; 137 struct struct_type *mailbox_struct_type; 138 rte_swx_extern_type_constructor_t constructor; 139 rte_swx_extern_type_destructor_t destructor; 140 struct extern_type_member_func_tailq funcs; 141 uint32_t n_funcs; 142 }; 143 144 TAILQ_HEAD(extern_type_tailq, extern_type); 145 146 struct extern_obj { 147 TAILQ_ENTRY(extern_obj) node; 148 char name[RTE_SWX_NAME_SIZE]; 149 struct extern_type *type; 150 void *obj; 151 uint32_t struct_id; 152 uint32_t id; 153 }; 154 155 TAILQ_HEAD(extern_obj_tailq, extern_obj); 156 157 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 158 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8 159 #endif 160 161 struct extern_obj_runtime { 162 void *obj; 163 uint8_t *mailbox; 164 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX]; 165 }; 166 167 /* 168 * Extern function. 169 */ 170 struct extern_func { 171 TAILQ_ENTRY(extern_func) node; 172 char name[RTE_SWX_NAME_SIZE]; 173 struct struct_type *mailbox_struct_type; 174 rte_swx_extern_func_t func; 175 uint32_t struct_id; 176 uint32_t id; 177 }; 178 179 TAILQ_HEAD(extern_func_tailq, extern_func); 180 181 struct extern_func_runtime { 182 uint8_t *mailbox; 183 rte_swx_extern_func_t func; 184 }; 185 186 /* 187 * Hash function. 188 */ 189 struct hash_func { 190 TAILQ_ENTRY(hash_func) node; 191 char name[RTE_SWX_NAME_SIZE]; 192 rte_swx_hash_func_t func; 193 uint32_t id; 194 }; 195 196 TAILQ_HEAD(hash_func_tailq, hash_func); 197 198 struct hash_func_runtime { 199 rte_swx_hash_func_t func; 200 }; 201 202 /* 203 * Header. 204 */ 205 struct header { 206 TAILQ_ENTRY(header) node; 207 char name[RTE_SWX_NAME_SIZE]; 208 struct struct_type *st; 209 uint32_t struct_id; 210 uint32_t id; 211 }; 212 213 TAILQ_HEAD(header_tailq, header); 214 215 struct header_runtime { 216 uint8_t *ptr0; 217 uint32_t n_bytes; 218 }; 219 220 struct header_out_runtime { 221 uint8_t *ptr0; 222 uint8_t *ptr; 223 uint32_t n_bytes; 224 }; 225 226 /* 227 * Instruction. 228 */ 229 230 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian. 231 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO). 232 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO 233 * when transferred to packet meta-data and in NBO when transferred to packet 234 * headers. 235 */ 236 237 /* Notation conventions: 238 * -Header field: H = h.header.field (dst/src) 239 * -Meta-data field: M = m.field (dst/src) 240 * -Extern object mailbox field: E = e.field (dst/src) 241 * -Extern function mailbox field: F = f.field (dst/src) 242 * -Table action data field: T = t.field (src only) 243 * -Immediate value: I = 32-bit unsigned value (src only) 244 */ 245 246 enum instruction_type { 247 /* rx m.port_in */ 248 INSTR_RX, 249 250 /* tx port_out 251 * port_out = MI 252 */ 253 INSTR_TX, /* port_out = M */ 254 INSTR_TX_I, /* port_out = I */ 255 INSTR_DROP, 256 257 /* 258 * mirror slot_id session_id 259 * slot_id = MEFT 260 * session_id = MEFT 261 */ 262 INSTR_MIRROR, 263 264 /* recirculate 265 */ 266 INSTR_RECIRCULATE, 267 268 /* recircid m.recirc_pass_id 269 * Read the internal recirculation pass ID into the specified meta-data field. 270 */ 271 INSTR_RECIRCID, 272 273 /* extract h.header */ 274 INSTR_HDR_EXTRACT, 275 INSTR_HDR_EXTRACT2, 276 INSTR_HDR_EXTRACT3, 277 INSTR_HDR_EXTRACT4, 278 INSTR_HDR_EXTRACT5, 279 INSTR_HDR_EXTRACT6, 280 INSTR_HDR_EXTRACT7, 281 INSTR_HDR_EXTRACT8, 282 283 /* extract h.header m.last_field_size */ 284 INSTR_HDR_EXTRACT_M, 285 286 /* lookahead h.header */ 287 INSTR_HDR_LOOKAHEAD, 288 289 /* emit h.header */ 290 INSTR_HDR_EMIT, 291 INSTR_HDR_EMIT_TX, 292 INSTR_HDR_EMIT2_TX, 293 INSTR_HDR_EMIT3_TX, 294 INSTR_HDR_EMIT4_TX, 295 INSTR_HDR_EMIT5_TX, 296 INSTR_HDR_EMIT6_TX, 297 INSTR_HDR_EMIT7_TX, 298 INSTR_HDR_EMIT8_TX, 299 300 /* validate h.header */ 301 INSTR_HDR_VALIDATE, 302 303 /* invalidate h.header */ 304 INSTR_HDR_INVALIDATE, 305 306 /* mov dst src 307 * dst = src 308 * dst = HMEF, src = HMEFTI 309 */ 310 INSTR_MOV, /* dst = MEF, src = MEFT */ 311 INSTR_MOV_MH, /* dst = MEF, src = H */ 312 INSTR_MOV_HM, /* dst = H, src = MEFT */ 313 INSTR_MOV_HH, /* dst = H, src = H */ 314 INSTR_MOV_I, /* dst = HMEF, src = I */ 315 316 /* dma h.header t.field 317 * memcpy(h.header, t.field, sizeof(h.header)) 318 */ 319 INSTR_DMA_HT, 320 INSTR_DMA_HT2, 321 INSTR_DMA_HT3, 322 INSTR_DMA_HT4, 323 INSTR_DMA_HT5, 324 INSTR_DMA_HT6, 325 INSTR_DMA_HT7, 326 INSTR_DMA_HT8, 327 328 /* add dst src 329 * dst += src 330 * dst = HMEF, src = HMEFTI 331 */ 332 INSTR_ALU_ADD, /* dst = MEF, src = MEF */ 333 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */ 334 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */ 335 INSTR_ALU_ADD_HH, /* dst = H, src = H */ 336 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */ 337 INSTR_ALU_ADD_HI, /* dst = H, src = I */ 338 339 /* sub dst src 340 * dst -= src 341 * dst = HMEF, src = HMEFTI 342 */ 343 INSTR_ALU_SUB, /* dst = MEF, src = MEF */ 344 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */ 345 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */ 346 INSTR_ALU_SUB_HH, /* dst = H, src = H */ 347 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */ 348 INSTR_ALU_SUB_HI, /* dst = H, src = I */ 349 350 /* ckadd dst src 351 * dst = dst '+ src[0:1] '+ src[2:3] '+ ... 352 * dst = H, src = {H, h.header}, '+ = 1's complement addition operator 353 */ 354 INSTR_ALU_CKADD_FIELD, /* src = H */ 355 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 bytes. */ 356 INSTR_ALU_CKADD_STRUCT, /* src = h.header, with sizeof(header) any 4-byte multiple. */ 357 358 /* cksub dst src 359 * dst = dst '- src 360 * dst = H, src = H, '- = 1's complement subtraction operator 361 */ 362 INSTR_ALU_CKSUB_FIELD, 363 364 /* and dst src 365 * dst &= src 366 * dst = HMEF, src = HMEFTI 367 */ 368 INSTR_ALU_AND, /* dst = MEF, src = MEFT */ 369 INSTR_ALU_AND_MH, /* dst = MEF, src = H */ 370 INSTR_ALU_AND_HM, /* dst = H, src = MEFT */ 371 INSTR_ALU_AND_HH, /* dst = H, src = H */ 372 INSTR_ALU_AND_I, /* dst = HMEF, src = I */ 373 374 /* or dst src 375 * dst |= src 376 * dst = HMEF, src = HMEFTI 377 */ 378 INSTR_ALU_OR, /* dst = MEF, src = MEFT */ 379 INSTR_ALU_OR_MH, /* dst = MEF, src = H */ 380 INSTR_ALU_OR_HM, /* dst = H, src = MEFT */ 381 INSTR_ALU_OR_HH, /* dst = H, src = H */ 382 INSTR_ALU_OR_I, /* dst = HMEF, src = I */ 383 384 /* xor dst src 385 * dst ^= src 386 * dst = HMEF, src = HMEFTI 387 */ 388 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */ 389 INSTR_ALU_XOR_MH, /* dst = MEF, src = H */ 390 INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */ 391 INSTR_ALU_XOR_HH, /* dst = H, src = H */ 392 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */ 393 394 /* shl dst src 395 * dst <<= src 396 * dst = HMEF, src = HMEFTI 397 */ 398 INSTR_ALU_SHL, /* dst = MEF, src = MEF */ 399 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */ 400 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */ 401 INSTR_ALU_SHL_HH, /* dst = H, src = H */ 402 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */ 403 INSTR_ALU_SHL_HI, /* dst = H, src = I */ 404 405 /* shr dst src 406 * dst >>= src 407 * dst = HMEF, src = HMEFTI 408 */ 409 INSTR_ALU_SHR, /* dst = MEF, src = MEF */ 410 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */ 411 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */ 412 INSTR_ALU_SHR_HH, /* dst = H, src = H */ 413 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */ 414 INSTR_ALU_SHR_HI, /* dst = H, src = I */ 415 416 /* regprefetch REGARRAY index 417 * prefetch REGARRAY[index] 418 * index = HMEFTI 419 */ 420 INSTR_REGPREFETCH_RH, /* index = H */ 421 INSTR_REGPREFETCH_RM, /* index = MEFT */ 422 INSTR_REGPREFETCH_RI, /* index = I */ 423 424 /* regrd dst REGARRAY index 425 * dst = REGARRAY[index] 426 * dst = HMEF, index = HMEFTI 427 */ 428 INSTR_REGRD_HRH, /* dst = H, index = H */ 429 INSTR_REGRD_HRM, /* dst = H, index = MEFT */ 430 INSTR_REGRD_HRI, /* dst = H, index = I */ 431 INSTR_REGRD_MRH, /* dst = MEF, index = H */ 432 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */ 433 INSTR_REGRD_MRI, /* dst = MEF, index = I */ 434 435 /* regwr REGARRAY index src 436 * REGARRAY[index] = src 437 * index = HMEFTI, src = HMEFTI 438 */ 439 INSTR_REGWR_RHH, /* index = H, src = H */ 440 INSTR_REGWR_RHM, /* index = H, src = MEFT */ 441 INSTR_REGWR_RHI, /* index = H, src = I */ 442 INSTR_REGWR_RMH, /* index = MEFT, src = H */ 443 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */ 444 INSTR_REGWR_RMI, /* index = MEFT, src = I */ 445 INSTR_REGWR_RIH, /* index = I, src = H */ 446 INSTR_REGWR_RIM, /* index = I, src = MEFT */ 447 INSTR_REGWR_RII, /* index = I, src = I */ 448 449 /* regadd REGARRAY index src 450 * REGARRAY[index] += src 451 * index = HMEFTI, src = HMEFTI 452 */ 453 INSTR_REGADD_RHH, /* index = H, src = H */ 454 INSTR_REGADD_RHM, /* index = H, src = MEFT */ 455 INSTR_REGADD_RHI, /* index = H, src = I */ 456 INSTR_REGADD_RMH, /* index = MEFT, src = H */ 457 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */ 458 INSTR_REGADD_RMI, /* index = MEFT, src = I */ 459 INSTR_REGADD_RIH, /* index = I, src = H */ 460 INSTR_REGADD_RIM, /* index = I, src = MEFT */ 461 INSTR_REGADD_RII, /* index = I, src = I */ 462 463 /* metprefetch METARRAY index 464 * prefetch METARRAY[index] 465 * index = HMEFTI 466 */ 467 INSTR_METPREFETCH_H, /* index = H */ 468 INSTR_METPREFETCH_M, /* index = MEFT */ 469 INSTR_METPREFETCH_I, /* index = I */ 470 471 /* meter METARRAY index length color_in color_out 472 * color_out = meter(METARRAY[index], length, color_in) 473 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF 474 */ 475 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */ 476 INSTR_METER_HHI, /* index = H, length = H, color_in = I */ 477 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */ 478 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */ 479 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */ 480 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */ 481 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */ 482 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */ 483 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */ 484 INSTR_METER_IHI, /* index = I, length = H, color_in = I */ 485 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */ 486 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */ 487 488 /* table TABLE */ 489 INSTR_TABLE, 490 INSTR_TABLE_AF, 491 INSTR_SELECTOR, 492 INSTR_LEARNER, 493 INSTR_LEARNER_AF, 494 495 /* learn ACTION_NAME [ m.action_first_arg ] m.timeout_id */ 496 INSTR_LEARNER_LEARN, 497 498 /* rearm [ m.timeout_id ] */ 499 INSTR_LEARNER_REARM, 500 INSTR_LEARNER_REARM_NEW, 501 502 /* forget */ 503 INSTR_LEARNER_FORGET, 504 505 /* extern e.obj.func */ 506 INSTR_EXTERN_OBJ, 507 508 /* extern f.func */ 509 INSTR_EXTERN_FUNC, 510 511 /* hash HASH_FUNC_NAME dst src_first src_last 512 * Compute hash value over range of struct fields. 513 * dst = M 514 * src_first = HMEFT 515 * src_last = HMEFT 516 * src_first and src_last must be fields within the same struct 517 */ 518 INSTR_HASH_FUNC, 519 520 /* jmp LABEL 521 * Unconditional jump 522 */ 523 INSTR_JMP, 524 525 /* jmpv LABEL h.header 526 * Jump if header is valid 527 */ 528 INSTR_JMP_VALID, 529 530 /* jmpnv LABEL h.header 531 * Jump if header is invalid 532 */ 533 INSTR_JMP_INVALID, 534 535 /* jmph LABEL 536 * Jump if table lookup hit 537 */ 538 INSTR_JMP_HIT, 539 540 /* jmpnh LABEL 541 * Jump if table lookup miss 542 */ 543 INSTR_JMP_MISS, 544 545 /* jmpa LABEL ACTION 546 * Jump if action run 547 */ 548 INSTR_JMP_ACTION_HIT, 549 550 /* jmpna LABEL ACTION 551 * Jump if action not run 552 */ 553 INSTR_JMP_ACTION_MISS, 554 555 /* jmpeq LABEL a b 556 * Jump if a is equal to b 557 * a = HMEFT, b = HMEFTI 558 */ 559 INSTR_JMP_EQ, /* a = MEFT, b = MEFT */ 560 INSTR_JMP_EQ_MH, /* a = MEFT, b = H */ 561 INSTR_JMP_EQ_HM, /* a = H, b = MEFT */ 562 INSTR_JMP_EQ_HH, /* a = H, b = H */ 563 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */ 564 565 /* jmpneq LABEL a b 566 * Jump if a is not equal to b 567 * a = HMEFT, b = HMEFTI 568 */ 569 INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */ 570 INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */ 571 INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */ 572 INSTR_JMP_NEQ_HH, /* a = H, b = H */ 573 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */ 574 575 /* jmplt LABEL a b 576 * Jump if a is less than b 577 * a = HMEFT, b = HMEFTI 578 */ 579 INSTR_JMP_LT, /* a = MEFT, b = MEFT */ 580 INSTR_JMP_LT_MH, /* a = MEFT, b = H */ 581 INSTR_JMP_LT_HM, /* a = H, b = MEFT */ 582 INSTR_JMP_LT_HH, /* a = H, b = H */ 583 INSTR_JMP_LT_MI, /* a = MEFT, b = I */ 584 INSTR_JMP_LT_HI, /* a = H, b = I */ 585 586 /* jmpgt LABEL a b 587 * Jump if a is greater than b 588 * a = HMEFT, b = HMEFTI 589 */ 590 INSTR_JMP_GT, /* a = MEFT, b = MEFT */ 591 INSTR_JMP_GT_MH, /* a = MEFT, b = H */ 592 INSTR_JMP_GT_HM, /* a = H, b = MEFT */ 593 INSTR_JMP_GT_HH, /* a = H, b = H */ 594 INSTR_JMP_GT_MI, /* a = MEFT, b = I */ 595 INSTR_JMP_GT_HI, /* a = H, b = I */ 596 597 /* return 598 * Return from action 599 */ 600 INSTR_RETURN, 601 602 /* Start of custom instructions. */ 603 INSTR_CUSTOM_0, 604 }; 605 606 struct instr_operand { 607 uint8_t struct_id; 608 uint8_t n_bits; 609 uint8_t offset; 610 uint8_t pad; 611 }; 612 613 struct instr_io { 614 struct { 615 union { 616 struct { 617 uint8_t offset; 618 uint8_t n_bits; 619 uint8_t pad[2]; 620 }; 621 622 uint32_t val; 623 }; 624 } io; 625 626 struct { 627 uint8_t header_id[8]; 628 uint8_t struct_id[8]; 629 uint8_t n_bytes[8]; 630 } hdr; 631 }; 632 633 struct instr_hdr_validity { 634 uint8_t header_id; 635 uint8_t struct_id; 636 }; 637 638 struct instr_table { 639 uint8_t table_id; 640 }; 641 642 struct instr_learn { 643 uint8_t action_id; 644 uint8_t mf_first_arg_offset; 645 uint8_t mf_timeout_id_offset; 646 uint8_t mf_timeout_id_n_bits; 647 }; 648 649 struct instr_extern_obj { 650 uint8_t ext_obj_id; 651 uint8_t func_id; 652 }; 653 654 struct instr_extern_func { 655 uint8_t ext_func_id; 656 }; 657 658 struct instr_hash_func { 659 uint8_t hash_func_id; 660 661 struct { 662 uint8_t offset; 663 uint8_t n_bits; 664 } dst; 665 666 struct { 667 uint8_t struct_id; 668 uint16_t offset; 669 uint16_t n_bytes; 670 } src; 671 }; 672 673 struct instr_dst_src { 674 struct instr_operand dst; 675 union { 676 struct instr_operand src; 677 uint64_t src_val; 678 }; 679 }; 680 681 struct instr_regarray { 682 uint8_t regarray_id; 683 uint8_t pad[3]; 684 685 union { 686 struct instr_operand idx; 687 uint32_t idx_val; 688 }; 689 690 union { 691 struct instr_operand dstsrc; 692 uint64_t dstsrc_val; 693 }; 694 }; 695 696 struct instr_meter { 697 uint8_t metarray_id; 698 uint8_t pad[3]; 699 700 union { 701 struct instr_operand idx; 702 uint32_t idx_val; 703 }; 704 705 struct instr_operand length; 706 707 union { 708 struct instr_operand color_in; 709 uint32_t color_in_val; 710 }; 711 712 struct instr_operand color_out; 713 }; 714 715 struct instr_dma { 716 struct { 717 uint8_t header_id[8]; 718 uint8_t struct_id[8]; 719 } dst; 720 721 struct { 722 uint8_t offset[8]; 723 } src; 724 725 uint16_t n_bytes[8]; 726 }; 727 728 struct instr_jmp { 729 struct instruction *ip; 730 731 union { 732 struct instr_operand a; 733 uint8_t header_id; 734 uint8_t action_id; 735 }; 736 737 union { 738 struct instr_operand b; 739 uint64_t b_val; 740 }; 741 }; 742 743 struct instruction { 744 enum instruction_type type; 745 union { 746 struct instr_io io; 747 struct instr_dst_src mirror; 748 struct instr_hdr_validity valid; 749 struct instr_dst_src mov; 750 struct instr_regarray regarray; 751 struct instr_meter meter; 752 struct instr_dma dma; 753 struct instr_dst_src alu; 754 struct instr_table table; 755 struct instr_learn learn; 756 struct instr_extern_obj ext_obj; 757 struct instr_extern_func ext_func; 758 struct instr_hash_func hash_func; 759 struct instr_jmp jmp; 760 }; 761 }; 762 763 struct instruction_data { 764 char label[RTE_SWX_NAME_SIZE]; 765 char jmp_label[RTE_SWX_NAME_SIZE]; 766 uint32_t n_users; /* user = jmp instruction to this instruction. */ 767 int invalid; 768 }; 769 770 typedef void (*instr_exec_t)(struct rte_swx_pipeline *); 771 772 /* 773 * Action. 774 */ 775 typedef void 776 (*action_func_t)(struct rte_swx_pipeline *p); 777 778 struct action { 779 TAILQ_ENTRY(action) node; 780 char name[RTE_SWX_NAME_SIZE]; 781 struct struct_type *st; 782 int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */ 783 struct instruction *instructions; 784 struct instruction_data *instruction_data; 785 uint32_t n_instructions; 786 uint32_t id; 787 }; 788 789 TAILQ_HEAD(action_tailq, action); 790 791 /* 792 * Table. 793 */ 794 struct table_type { 795 TAILQ_ENTRY(table_type) node; 796 char name[RTE_SWX_NAME_SIZE]; 797 enum rte_swx_table_match_type match_type; 798 struct rte_swx_table_ops ops; 799 }; 800 801 TAILQ_HEAD(table_type_tailq, table_type); 802 803 struct match_field { 804 enum rte_swx_table_match_type match_type; 805 struct field *field; 806 }; 807 808 struct table { 809 TAILQ_ENTRY(table) node; 810 char name[RTE_SWX_NAME_SIZE]; 811 char args[RTE_SWX_NAME_SIZE]; 812 struct table_type *type; /* NULL when n_fields == 0. */ 813 814 /* Match. */ 815 struct match_field *fields; 816 uint32_t n_fields; 817 struct header *header; /* Only valid when n_fields > 0. */ 818 819 /* Action. */ 820 struct action **actions; 821 struct action *default_action; 822 uint8_t *default_action_data; 823 uint32_t n_actions; 824 int default_action_is_const; 825 uint32_t action_data_size_max; 826 int *action_is_for_table_entries; 827 int *action_is_for_default_entry; 828 829 uint32_t size; 830 uint32_t id; 831 }; 832 833 TAILQ_HEAD(table_tailq, table); 834 835 struct table_runtime { 836 rte_swx_table_lookup_t func; 837 void *mailbox; 838 uint8_t **key; 839 }; 840 841 struct table_statistics { 842 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */ 843 uint64_t *n_pkts_action; 844 }; 845 846 /* 847 * Selector. 848 */ 849 struct selector { 850 TAILQ_ENTRY(selector) node; 851 char name[RTE_SWX_NAME_SIZE]; 852 853 struct field *group_id_field; 854 struct field **selector_fields; 855 uint32_t n_selector_fields; 856 struct header *selector_header; 857 struct field *member_id_field; 858 859 uint32_t n_groups_max; 860 uint32_t n_members_per_group_max; 861 862 uint32_t id; 863 }; 864 865 TAILQ_HEAD(selector_tailq, selector); 866 867 struct selector_runtime { 868 void *mailbox; 869 uint8_t **group_id_buffer; 870 uint8_t **selector_buffer; 871 uint8_t **member_id_buffer; 872 }; 873 874 struct selector_statistics { 875 uint64_t n_pkts; 876 }; 877 878 /* 879 * Learner table. 880 */ 881 struct learner { 882 TAILQ_ENTRY(learner) node; 883 char name[RTE_SWX_NAME_SIZE]; 884 885 /* Match. */ 886 struct field **fields; 887 uint32_t n_fields; 888 struct header *header; 889 890 /* Action. */ 891 struct action **actions; 892 struct action *default_action; 893 uint8_t *default_action_data; 894 uint32_t n_actions; 895 int default_action_is_const; 896 uint32_t action_data_size_max; 897 int *action_is_for_table_entries; 898 int *action_is_for_default_entry; 899 900 uint32_t size; 901 uint32_t timeout[RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX]; 902 uint32_t n_timeouts; 903 uint32_t id; 904 }; 905 906 TAILQ_HEAD(learner_tailq, learner); 907 908 struct learner_runtime { 909 void *mailbox; 910 uint8_t **key; 911 }; 912 913 struct learner_statistics { 914 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */ 915 uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */ 916 uint64_t n_pkts_rearm; 917 uint64_t n_pkts_forget; 918 uint64_t *n_pkts_action; 919 }; 920 921 /* 922 * Register array. 923 */ 924 struct regarray { 925 TAILQ_ENTRY(regarray) node; 926 char name[RTE_SWX_NAME_SIZE]; 927 uint64_t init_val; 928 uint32_t size; 929 uint32_t id; 930 }; 931 932 TAILQ_HEAD(regarray_tailq, regarray); 933 934 struct regarray_runtime { 935 uint64_t *regarray; 936 uint32_t size_mask; 937 }; 938 939 /* 940 * Meter array. 941 */ 942 struct meter_profile { 943 TAILQ_ENTRY(meter_profile) node; 944 char name[RTE_SWX_NAME_SIZE]; 945 struct rte_meter_trtcm_params params; 946 struct rte_meter_trtcm_profile profile; 947 uint32_t n_users; 948 }; 949 950 TAILQ_HEAD(meter_profile_tailq, meter_profile); 951 952 struct metarray { 953 TAILQ_ENTRY(metarray) node; 954 char name[RTE_SWX_NAME_SIZE]; 955 uint32_t size; 956 uint32_t id; 957 }; 958 959 TAILQ_HEAD(metarray_tailq, metarray); 960 961 struct meter { 962 struct rte_meter_trtcm m; 963 struct meter_profile *profile; 964 enum rte_color color_mask; 965 uint8_t pad[20]; 966 967 uint64_t n_pkts[RTE_COLORS]; 968 uint64_t n_bytes[RTE_COLORS]; 969 }; 970 971 struct metarray_runtime { 972 struct meter *metarray; 973 uint32_t size_mask; 974 }; 975 976 /* 977 * Pipeline. 978 */ 979 struct thread { 980 /* Packet. */ 981 struct rte_swx_pkt pkt; 982 uint8_t *ptr; 983 uint32_t *mirroring_slots; 984 uint64_t mirroring_slots_mask; 985 int recirculate; 986 uint32_t recirc_pass_id; 987 988 /* Structures. */ 989 uint8_t **structs; 990 991 /* Packet headers. */ 992 struct header_runtime *headers; /* Extracted or generated headers. */ 993 struct header_out_runtime *headers_out; /* Emitted headers. */ 994 uint8_t *header_storage; 995 uint8_t *header_out_storage; 996 uint64_t valid_headers; 997 uint32_t n_headers_out; 998 999 /* Packet meta-data. */ 1000 uint8_t *metadata; 1001 1002 /* Tables. */ 1003 struct table_runtime *tables; 1004 struct selector_runtime *selectors; 1005 struct learner_runtime *learners; 1006 struct rte_swx_table_state *table_state; 1007 uint64_t action_id; 1008 int hit; /* 0 = Miss, 1 = Hit. */ 1009 uint32_t learner_id; 1010 uint64_t time; 1011 1012 /* Extern objects and functions. */ 1013 struct extern_obj_runtime *extern_objs; 1014 struct extern_func_runtime *extern_funcs; 1015 1016 /* Instructions. */ 1017 struct instruction *ip; 1018 struct instruction *ret; 1019 }; 1020 1021 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos))) 1022 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos))) 1023 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos))) 1024 1025 #define HEADER_VALID(thread, header_id) \ 1026 MASK64_BIT_GET((thread)->valid_headers, header_id) 1027 1028 static inline uint64_t 1029 instr_operand_hbo(struct thread *t, const struct instr_operand *x) 1030 { 1031 uint8_t *x_struct = t->structs[x->struct_id]; 1032 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset]; 1033 uint64_t x64 = *x64_ptr; 1034 uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits); 1035 1036 return x64 & x64_mask; 1037 } 1038 1039 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1040 1041 static inline uint64_t 1042 instr_operand_nbo(struct thread *t, const struct instr_operand *x) 1043 { 1044 uint8_t *x_struct = t->structs[x->struct_id]; 1045 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset]; 1046 uint64_t x64 = *x64_ptr; 1047 1048 return ntoh64(x64) >> (64 - x->n_bits); 1049 } 1050 1051 #else 1052 1053 #define instr_operand_nbo instr_operand_hbo 1054 1055 #endif 1056 1057 #define ALU(thread, ip, operator) \ 1058 { \ 1059 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1060 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1061 uint64_t dst64 = *dst64_ptr; \ 1062 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1063 uint64_t dst = dst64 & dst64_mask; \ 1064 \ 1065 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1066 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1067 uint64_t src64 = *src64_ptr; \ 1068 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \ 1069 uint64_t src = src64 & src64_mask; \ 1070 \ 1071 uint64_t result = dst operator src; \ 1072 \ 1073 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \ 1074 } 1075 1076 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1077 1078 #define ALU_MH(thread, ip, operator) \ 1079 { \ 1080 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1081 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1082 uint64_t dst64 = *dst64_ptr; \ 1083 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1084 uint64_t dst = dst64 & dst64_mask; \ 1085 \ 1086 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1087 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1088 uint64_t src64 = *src64_ptr; \ 1089 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \ 1090 \ 1091 uint64_t result = dst operator src; \ 1092 \ 1093 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \ 1094 } 1095 1096 #define ALU_HM(thread, ip, operator) \ 1097 { \ 1098 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1099 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1100 uint64_t dst64 = *dst64_ptr; \ 1101 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1102 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \ 1103 \ 1104 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1105 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1106 uint64_t src64 = *src64_ptr; \ 1107 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \ 1108 uint64_t src = src64 & src64_mask; \ 1109 \ 1110 uint64_t result = dst operator src; \ 1111 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \ 1112 \ 1113 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1114 } 1115 1116 #define ALU_HM_FAST(thread, ip, operator) \ 1117 { \ 1118 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1119 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1120 uint64_t dst64 = *dst64_ptr; \ 1121 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1122 uint64_t dst = dst64 & dst64_mask; \ 1123 \ 1124 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1125 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1126 uint64_t src64 = *src64_ptr; \ 1127 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \ 1128 uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \ 1129 \ 1130 uint64_t result = dst operator src; \ 1131 \ 1132 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1133 } 1134 1135 #define ALU_HH(thread, ip, operator) \ 1136 { \ 1137 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1138 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1139 uint64_t dst64 = *dst64_ptr; \ 1140 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1141 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \ 1142 \ 1143 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1144 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1145 uint64_t src64 = *src64_ptr; \ 1146 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \ 1147 \ 1148 uint64_t result = dst operator src; \ 1149 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \ 1150 \ 1151 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1152 } 1153 1154 #define ALU_HH_FAST(thread, ip, operator) \ 1155 { \ 1156 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1157 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1158 uint64_t dst64 = *dst64_ptr; \ 1159 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1160 uint64_t dst = dst64 & dst64_mask; \ 1161 \ 1162 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1163 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1164 uint64_t src64 = *src64_ptr; \ 1165 uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \ 1166 \ 1167 uint64_t result = dst operator src; \ 1168 \ 1169 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1170 } 1171 1172 #else 1173 1174 #define ALU_MH ALU 1175 #define ALU_HM ALU 1176 #define ALU_HM_FAST ALU 1177 #define ALU_HH ALU 1178 #define ALU_HH_FAST ALU 1179 1180 #endif 1181 1182 #define ALU_I(thread, ip, operator) \ 1183 { \ 1184 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1185 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1186 uint64_t dst64 = *dst64_ptr; \ 1187 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1188 uint64_t dst = dst64 & dst64_mask; \ 1189 \ 1190 uint64_t src = (ip)->alu.src_val; \ 1191 \ 1192 uint64_t result = dst operator src; \ 1193 \ 1194 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \ 1195 } 1196 1197 #define ALU_MI ALU_I 1198 1199 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1200 1201 #define ALU_HI(thread, ip, operator) \ 1202 { \ 1203 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1204 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1205 uint64_t dst64 = *dst64_ptr; \ 1206 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1207 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \ 1208 \ 1209 uint64_t src = (ip)->alu.src_val; \ 1210 \ 1211 uint64_t result = dst operator src; \ 1212 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \ 1213 \ 1214 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1215 } 1216 1217 #else 1218 1219 #define ALU_HI ALU_I 1220 1221 #endif 1222 1223 #define MOV(thread, ip) \ 1224 { \ 1225 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1226 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1227 uint64_t dst64 = *dst64_ptr; \ 1228 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1229 \ 1230 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \ 1231 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \ 1232 uint64_t src64 = *src64_ptr; \ 1233 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \ 1234 uint64_t src = src64 & src64_mask; \ 1235 \ 1236 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \ 1237 } 1238 1239 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1240 1241 #define MOV_MH(thread, ip) \ 1242 { \ 1243 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1244 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1245 uint64_t dst64 = *dst64_ptr; \ 1246 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1247 \ 1248 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \ 1249 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \ 1250 uint64_t src64 = *src64_ptr; \ 1251 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \ 1252 \ 1253 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \ 1254 } 1255 1256 #define MOV_HM(thread, ip) \ 1257 { \ 1258 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1259 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1260 uint64_t dst64 = *dst64_ptr; \ 1261 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1262 \ 1263 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \ 1264 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \ 1265 uint64_t src64 = *src64_ptr; \ 1266 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \ 1267 uint64_t src = src64 & src64_mask; \ 1268 \ 1269 src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \ 1270 *dst64_ptr = (dst64 & ~dst64_mask) | src; \ 1271 } 1272 1273 #define MOV_HH(thread, ip) \ 1274 { \ 1275 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1276 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1277 uint64_t dst64 = *dst64_ptr; \ 1278 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1279 \ 1280 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \ 1281 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \ 1282 uint64_t src64 = *src64_ptr; \ 1283 \ 1284 uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \ 1285 src = src >> (64 - (ip)->mov.dst.n_bits); \ 1286 *dst64_ptr = (dst64 & ~dst64_mask) | src; \ 1287 } 1288 1289 #else 1290 1291 #define MOV_MH MOV 1292 #define MOV_HM MOV 1293 #define MOV_HH MOV 1294 1295 #endif 1296 1297 #define MOV_I(thread, ip) \ 1298 { \ 1299 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1300 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1301 uint64_t dst64 = *dst64_ptr; \ 1302 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1303 \ 1304 uint64_t src = (ip)->mov.src_val; \ 1305 \ 1306 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \ 1307 } 1308 1309 #define JMP_CMP(thread, ip, operator) \ 1310 { \ 1311 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1312 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1313 uint64_t a64 = *a64_ptr; \ 1314 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \ 1315 uint64_t a = a64 & a64_mask; \ 1316 \ 1317 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1318 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1319 uint64_t b64 = *b64_ptr; \ 1320 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \ 1321 uint64_t b = b64 & b64_mask; \ 1322 \ 1323 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1324 } 1325 1326 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1327 1328 #define JMP_CMP_MH(thread, ip, operator) \ 1329 { \ 1330 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1331 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1332 uint64_t a64 = *a64_ptr; \ 1333 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \ 1334 uint64_t a = a64 & a64_mask; \ 1335 \ 1336 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1337 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1338 uint64_t b64 = *b64_ptr; \ 1339 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \ 1340 \ 1341 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1342 } 1343 1344 #define JMP_CMP_HM(thread, ip, operator) \ 1345 { \ 1346 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1347 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1348 uint64_t a64 = *a64_ptr; \ 1349 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \ 1350 \ 1351 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1352 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1353 uint64_t b64 = *b64_ptr; \ 1354 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \ 1355 uint64_t b = b64 & b64_mask; \ 1356 \ 1357 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1358 } 1359 1360 #define JMP_CMP_HH(thread, ip, operator) \ 1361 { \ 1362 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1363 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1364 uint64_t a64 = *a64_ptr; \ 1365 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \ 1366 \ 1367 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1368 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1369 uint64_t b64 = *b64_ptr; \ 1370 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \ 1371 \ 1372 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1373 } 1374 1375 #define JMP_CMP_HH_FAST(thread, ip, operator) \ 1376 { \ 1377 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1378 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1379 uint64_t a64 = *a64_ptr; \ 1380 uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \ 1381 \ 1382 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1383 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1384 uint64_t b64 = *b64_ptr; \ 1385 uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \ 1386 \ 1387 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1388 } 1389 1390 #else 1391 1392 #define JMP_CMP_MH JMP_CMP 1393 #define JMP_CMP_HM JMP_CMP 1394 #define JMP_CMP_HH JMP_CMP 1395 #define JMP_CMP_HH_FAST JMP_CMP 1396 1397 #endif 1398 1399 #define JMP_CMP_I(thread, ip, operator) \ 1400 { \ 1401 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1402 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1403 uint64_t a64 = *a64_ptr; \ 1404 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \ 1405 uint64_t a = a64 & a64_mask; \ 1406 \ 1407 uint64_t b = (ip)->jmp.b_val; \ 1408 \ 1409 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1410 } 1411 1412 #define JMP_CMP_MI JMP_CMP_I 1413 1414 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1415 1416 #define JMP_CMP_HI(thread, ip, operator) \ 1417 { \ 1418 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1419 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1420 uint64_t a64 = *a64_ptr; \ 1421 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \ 1422 \ 1423 uint64_t b = (ip)->jmp.b_val; \ 1424 \ 1425 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1426 } 1427 1428 #else 1429 1430 #define JMP_CMP_HI JMP_CMP_I 1431 1432 #endif 1433 1434 #define METADATA_READ(thread, offset, n_bits) \ 1435 ({ \ 1436 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \ 1437 uint64_t m64 = *m64_ptr; \ 1438 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \ 1439 (m64 & m64_mask); \ 1440 }) 1441 1442 #define METADATA_WRITE(thread, offset, n_bits, value) \ 1443 { \ 1444 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \ 1445 uint64_t m64 = *m64_ptr; \ 1446 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \ 1447 \ 1448 uint64_t m_new = value; \ 1449 \ 1450 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \ 1451 } 1452 1453 #ifndef RTE_SWX_PIPELINE_THREADS_MAX 1454 #define RTE_SWX_PIPELINE_THREADS_MAX 16 1455 #endif 1456 1457 #ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 1458 #define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 256 1459 #endif 1460 1461 struct rte_swx_pipeline { 1462 struct struct_type_tailq struct_types; 1463 struct port_in_type_tailq port_in_types; 1464 struct port_in_tailq ports_in; 1465 struct port_out_type_tailq port_out_types; 1466 struct port_out_tailq ports_out; 1467 struct extern_type_tailq extern_types; 1468 struct extern_obj_tailq extern_objs; 1469 struct extern_func_tailq extern_funcs; 1470 struct hash_func_tailq hash_funcs; 1471 struct header_tailq headers; 1472 struct struct_type *metadata_st; 1473 uint32_t metadata_struct_id; 1474 struct action_tailq actions; 1475 struct table_type_tailq table_types; 1476 struct table_tailq tables; 1477 struct selector_tailq selectors; 1478 struct learner_tailq learners; 1479 struct regarray_tailq regarrays; 1480 struct meter_profile_tailq meter_profiles; 1481 struct metarray_tailq metarrays; 1482 1483 struct port_in_runtime *in; 1484 struct port_out_runtime *out; 1485 struct mirroring_session *mirroring_sessions; 1486 struct instruction **action_instructions; 1487 action_func_t *action_funcs; 1488 struct rte_swx_table_state *table_state; 1489 struct table_statistics *table_stats; 1490 struct selector_statistics *selector_stats; 1491 struct learner_statistics *learner_stats; 1492 struct hash_func_runtime *hash_func_runtime; 1493 struct regarray_runtime *regarray_runtime; 1494 struct metarray_runtime *metarray_runtime; 1495 struct instruction *instructions; 1496 struct instruction_data *instruction_data; 1497 instr_exec_t *instruction_table; 1498 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX]; 1499 void *lib; 1500 1501 uint32_t n_structs; 1502 uint32_t n_ports_in; 1503 uint32_t n_ports_out; 1504 uint32_t n_mirroring_slots; 1505 uint32_t n_mirroring_sessions; 1506 uint32_t n_extern_objs; 1507 uint32_t n_extern_funcs; 1508 uint32_t n_hash_funcs; 1509 uint32_t n_actions; 1510 uint32_t n_tables; 1511 uint32_t n_selectors; 1512 uint32_t n_learners; 1513 uint32_t n_regarrays; 1514 uint32_t n_metarrays; 1515 uint32_t n_headers; 1516 uint32_t thread_id; 1517 uint32_t port_id; 1518 uint32_t n_instructions; 1519 int build_done; 1520 int numa_node; 1521 }; 1522 1523 /* 1524 * Instruction. 1525 */ 1526 static inline void 1527 pipeline_port_inc(struct rte_swx_pipeline *p) 1528 { 1529 p->port_id = (p->port_id + 1) & (p->n_ports_in - 1); 1530 } 1531 1532 static inline void 1533 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t) 1534 { 1535 t->ip = p->instructions; 1536 } 1537 1538 static inline void 1539 thread_ip_set(struct thread *t, struct instruction *ip) 1540 { 1541 t->ip = ip; 1542 } 1543 1544 static inline void 1545 thread_ip_action_call(struct rte_swx_pipeline *p, 1546 struct thread *t, 1547 uint32_t action_id) 1548 { 1549 t->ret = t->ip + 1; 1550 t->ip = p->action_instructions[action_id]; 1551 } 1552 1553 static inline void 1554 thread_ip_inc(struct rte_swx_pipeline *p); 1555 1556 static inline void 1557 thread_ip_inc(struct rte_swx_pipeline *p) 1558 { 1559 struct thread *t = &p->threads[p->thread_id]; 1560 1561 t->ip++; 1562 } 1563 1564 static inline void 1565 thread_ip_inc_cond(struct thread *t, int cond) 1566 { 1567 t->ip += cond; 1568 } 1569 1570 static inline void 1571 thread_yield(struct rte_swx_pipeline *p) 1572 { 1573 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1); 1574 } 1575 1576 static inline void 1577 thread_yield_cond(struct rte_swx_pipeline *p, int cond) 1578 { 1579 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1); 1580 } 1581 1582 /* 1583 * rx. 1584 */ 1585 static inline int 1586 __instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 1587 { 1588 struct port_in_runtime *port = &p->in[p->port_id]; 1589 struct rte_swx_pkt *pkt = &t->pkt; 1590 int pkt_received; 1591 1592 /* Recirculation: keep the current packet. */ 1593 if (t->recirculate) { 1594 TRACE("[Thread %2u] rx - recirculate (pass %u)\n", 1595 p->thread_id, 1596 t->recirc_pass_id + 1); 1597 1598 /* Packet. */ 1599 t->ptr = &pkt->pkt[pkt->offset]; 1600 t->mirroring_slots_mask = 0; 1601 t->recirculate = 0; 1602 t->recirc_pass_id++; 1603 1604 /* Headers. */ 1605 t->valid_headers = 0; 1606 t->n_headers_out = 0; 1607 1608 /* Tables. */ 1609 t->table_state = p->table_state; 1610 1611 return 1; 1612 } 1613 1614 /* Packet. */ 1615 pkt_received = port->pkt_rx(port->obj, pkt); 1616 t->ptr = &pkt->pkt[pkt->offset]; 1617 rte_prefetch0(t->ptr); 1618 1619 TRACE("[Thread %2u] rx %s from port %u\n", 1620 p->thread_id, 1621 pkt_received ? "1 pkt" : "0 pkts", 1622 p->port_id); 1623 1624 t->mirroring_slots_mask = 0; 1625 t->recirc_pass_id = 0; 1626 1627 /* Headers. */ 1628 t->valid_headers = 0; 1629 t->n_headers_out = 0; 1630 1631 /* Meta-data. */ 1632 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id); 1633 1634 /* Tables. */ 1635 t->table_state = p->table_state; 1636 1637 /* Thread. */ 1638 pipeline_port_inc(p); 1639 1640 return pkt_received; 1641 } 1642 1643 static inline void 1644 instr_rx_exec(struct rte_swx_pipeline *p) 1645 { 1646 struct thread *t = &p->threads[p->thread_id]; 1647 struct instruction *ip = t->ip; 1648 int pkt_received; 1649 1650 /* Packet. */ 1651 pkt_received = __instr_rx_exec(p, t, ip); 1652 1653 /* Thread. */ 1654 thread_ip_inc_cond(t, pkt_received); 1655 thread_yield(p); 1656 } 1657 1658 /* 1659 * tx. 1660 */ 1661 static inline void 1662 emit_handler(struct thread *t) 1663 { 1664 struct header_out_runtime *h0 = &t->headers_out[0]; 1665 struct header_out_runtime *h1 = &t->headers_out[1]; 1666 uint32_t offset = 0, i; 1667 1668 /* No header change or header decapsulation. */ 1669 if ((t->n_headers_out == 1) && 1670 (h0->ptr + h0->n_bytes == t->ptr)) { 1671 TRACE("Emit handler: no header change or header decap.\n"); 1672 1673 t->pkt.offset -= h0->n_bytes; 1674 t->pkt.length += h0->n_bytes; 1675 1676 return; 1677 } 1678 1679 /* Header encapsulation (optionally, with prior header decapsulation). */ 1680 if ((t->n_headers_out == 2) && 1681 (h1->ptr + h1->n_bytes == t->ptr) && 1682 (h0->ptr == h0->ptr0)) { 1683 uint32_t offset; 1684 1685 TRACE("Emit handler: header encapsulation.\n"); 1686 1687 offset = h0->n_bytes + h1->n_bytes; 1688 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes); 1689 t->pkt.offset -= offset; 1690 t->pkt.length += offset; 1691 1692 return; 1693 } 1694 1695 /* For any other case. */ 1696 TRACE("Emit handler: complex case.\n"); 1697 1698 for (i = 0; i < t->n_headers_out; i++) { 1699 struct header_out_runtime *h = &t->headers_out[i]; 1700 1701 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes); 1702 offset += h->n_bytes; 1703 } 1704 1705 if (offset) { 1706 memcpy(t->ptr - offset, t->header_out_storage, offset); 1707 t->pkt.offset -= offset; 1708 t->pkt.length += offset; 1709 } 1710 } 1711 1712 static inline void 1713 mirroring_handler(struct rte_swx_pipeline *p, struct thread *t, struct rte_swx_pkt *pkt) 1714 { 1715 uint64_t slots_mask = t->mirroring_slots_mask, slot_mask; 1716 uint32_t slot_id; 1717 1718 for (slot_id = 0, slot_mask = 1LLU ; slots_mask; slot_id++, slot_mask <<= 1) 1719 if (slot_mask & slots_mask) { 1720 struct port_out_runtime *port; 1721 struct mirroring_session *session; 1722 uint32_t port_id, session_id; 1723 1724 session_id = t->mirroring_slots[slot_id]; 1725 session = &p->mirroring_sessions[session_id]; 1726 1727 port_id = session->port_id; 1728 port = &p->out[port_id]; 1729 1730 if (session->fast_clone) 1731 port->pkt_fast_clone_tx(port->obj, pkt); 1732 else 1733 port->pkt_clone_tx(port->obj, pkt, session->truncation_length); 1734 1735 slots_mask &= ~slot_mask; 1736 } 1737 } 1738 1739 static inline void 1740 __instr_tx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 1741 { 1742 uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits); 1743 struct port_out_runtime *port = &p->out[port_id]; 1744 struct rte_swx_pkt *pkt = &t->pkt; 1745 1746 /* Recirculation: keep the current packet. */ 1747 if (t->recirculate) { 1748 TRACE("[Thread %2u]: tx 1 pkt - recirculate\n", 1749 p->thread_id); 1750 1751 /* Headers. */ 1752 emit_handler(t); 1753 1754 /* Packet. */ 1755 mirroring_handler(p, t, pkt); 1756 1757 return; 1758 } 1759 1760 TRACE("[Thread %2u]: tx 1 pkt to port %u\n", 1761 p->thread_id, 1762 (uint32_t)port_id); 1763 1764 /* Headers. */ 1765 emit_handler(t); 1766 1767 /* Packet. */ 1768 mirroring_handler(p, t, pkt); 1769 port->pkt_tx(port->obj, pkt); 1770 } 1771 1772 static inline void 1773 __instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 1774 { 1775 uint64_t port_id = ip->io.io.val; 1776 struct port_out_runtime *port = &p->out[port_id]; 1777 struct rte_swx_pkt *pkt = &t->pkt; 1778 1779 /* Recirculation: keep the current packet. */ 1780 if (t->recirculate) { 1781 TRACE("[Thread %2u]: tx (i) 1 pkt - recirculate\n", 1782 p->thread_id); 1783 1784 /* Headers. */ 1785 emit_handler(t); 1786 1787 /* Packet. */ 1788 mirroring_handler(p, t, pkt); 1789 1790 return; 1791 } 1792 1793 TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n", 1794 p->thread_id, 1795 (uint32_t)port_id); 1796 1797 /* Headers. */ 1798 emit_handler(t); 1799 1800 /* Packet. */ 1801 mirroring_handler(p, t, pkt); 1802 port->pkt_tx(port->obj, pkt); 1803 } 1804 1805 static inline void 1806 __instr_drop_exec(struct rte_swx_pipeline *p, 1807 struct thread *t, 1808 const struct instruction *ip __rte_unused) 1809 { 1810 uint64_t port_id = p->n_ports_out - 1; 1811 struct port_out_runtime *port = &p->out[port_id]; 1812 struct rte_swx_pkt *pkt = &t->pkt; 1813 1814 TRACE("[Thread %2u]: drop 1 pkt\n", 1815 p->thread_id); 1816 1817 /* Headers. */ 1818 emit_handler(t); 1819 1820 /* Packet. */ 1821 mirroring_handler(p, t, pkt); 1822 port->pkt_tx(port->obj, pkt); 1823 } 1824 1825 static inline void 1826 __instr_mirror_exec(struct rte_swx_pipeline *p, 1827 struct thread *t, 1828 const struct instruction *ip) 1829 { 1830 uint64_t slot_id = instr_operand_hbo(t, &ip->mirror.dst); 1831 uint64_t session_id = instr_operand_hbo(t, &ip->mirror.src); 1832 1833 slot_id &= p->n_mirroring_slots - 1; 1834 session_id &= p->n_mirroring_sessions - 1; 1835 1836 TRACE("[Thread %2u]: mirror pkt (slot = %u, session = %u)\n", 1837 p->thread_id, 1838 (uint32_t)slot_id, 1839 (uint32_t)session_id); 1840 1841 t->mirroring_slots[slot_id] = session_id; 1842 t->mirroring_slots_mask |= 1LLU << slot_id; 1843 } 1844 1845 static inline void 1846 __instr_recirculate_exec(struct rte_swx_pipeline *p __rte_unused, 1847 struct thread *t, 1848 const struct instruction *ip __rte_unused) 1849 { 1850 TRACE("[Thread %2u]: recirculate\n", 1851 p->thread_id); 1852 1853 t->recirculate = 1; 1854 } 1855 1856 static inline void 1857 __instr_recircid_exec(struct rte_swx_pipeline *p __rte_unused, 1858 struct thread *t, 1859 const struct instruction *ip) 1860 { 1861 TRACE("[Thread %2u]: recircid (pass %u)\n", 1862 p->thread_id, 1863 t->recirc_pass_id); 1864 1865 /* Meta-data. */ 1866 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, t->recirc_pass_id); 1867 } 1868 1869 /* 1870 * extract. 1871 */ 1872 static inline void 1873 __instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused, 1874 struct thread *t, 1875 const struct instruction *ip, 1876 uint32_t n_extract) 1877 { 1878 uint64_t valid_headers = t->valid_headers; 1879 uint8_t *ptr = t->ptr; 1880 uint32_t offset = t->pkt.offset; 1881 uint32_t length = t->pkt.length; 1882 uint32_t i; 1883 1884 for (i = 0; i < n_extract; i++) { 1885 uint32_t header_id = ip->io.hdr.header_id[i]; 1886 uint32_t struct_id = ip->io.hdr.struct_id[i]; 1887 uint32_t n_bytes = ip->io.hdr.n_bytes[i]; 1888 1889 TRACE("[Thread %2u]: extract header %u (%u bytes)\n", 1890 p->thread_id, 1891 header_id, 1892 n_bytes); 1893 1894 /* Headers. */ 1895 t->structs[struct_id] = ptr; 1896 valid_headers = MASK64_BIT_SET(valid_headers, header_id); 1897 1898 /* Packet. */ 1899 offset += n_bytes; 1900 length -= n_bytes; 1901 ptr += n_bytes; 1902 } 1903 1904 /* Headers. */ 1905 t->valid_headers = valid_headers; 1906 1907 /* Packet. */ 1908 t->pkt.offset = offset; 1909 t->pkt.length = length; 1910 t->ptr = ptr; 1911 } 1912 1913 static inline void 1914 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, 1915 struct thread *t, 1916 const struct instruction *ip) 1917 { 1918 __instr_hdr_extract_many_exec(p, t, ip, 1); 1919 } 1920 1921 static inline void 1922 __instr_hdr_extract2_exec(struct rte_swx_pipeline *p, 1923 struct thread *t, 1924 const struct instruction *ip) 1925 { 1926 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id); 1927 1928 __instr_hdr_extract_many_exec(p, t, ip, 2); 1929 } 1930 1931 static inline void 1932 __instr_hdr_extract3_exec(struct rte_swx_pipeline *p, 1933 struct thread *t, 1934 const struct instruction *ip) 1935 { 1936 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id); 1937 1938 __instr_hdr_extract_many_exec(p, t, ip, 3); 1939 } 1940 1941 static inline void 1942 __instr_hdr_extract4_exec(struct rte_swx_pipeline *p, 1943 struct thread *t, 1944 const struct instruction *ip) 1945 { 1946 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id); 1947 1948 __instr_hdr_extract_many_exec(p, t, ip, 4); 1949 } 1950 1951 static inline void 1952 __instr_hdr_extract5_exec(struct rte_swx_pipeline *p, 1953 struct thread *t, 1954 const struct instruction *ip) 1955 { 1956 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id); 1957 1958 __instr_hdr_extract_many_exec(p, t, ip, 5); 1959 } 1960 1961 static inline void 1962 __instr_hdr_extract6_exec(struct rte_swx_pipeline *p, 1963 struct thread *t, 1964 const struct instruction *ip) 1965 { 1966 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id); 1967 1968 __instr_hdr_extract_many_exec(p, t, ip, 6); 1969 } 1970 1971 static inline void 1972 __instr_hdr_extract7_exec(struct rte_swx_pipeline *p, 1973 struct thread *t, 1974 const struct instruction *ip) 1975 { 1976 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id); 1977 1978 __instr_hdr_extract_many_exec(p, t, ip, 7); 1979 } 1980 1981 static inline void 1982 __instr_hdr_extract8_exec(struct rte_swx_pipeline *p, 1983 struct thread *t, 1984 const struct instruction *ip) 1985 { 1986 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id); 1987 1988 __instr_hdr_extract_many_exec(p, t, ip, 8); 1989 } 1990 1991 static inline void 1992 __instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused, 1993 struct thread *t, 1994 const struct instruction *ip) 1995 { 1996 uint64_t valid_headers = t->valid_headers; 1997 uint8_t *ptr = t->ptr; 1998 uint32_t offset = t->pkt.offset; 1999 uint32_t length = t->pkt.length; 2000 2001 uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits); 2002 uint32_t header_id = ip->io.hdr.header_id[0]; 2003 uint32_t struct_id = ip->io.hdr.struct_id[0]; 2004 uint32_t n_bytes = ip->io.hdr.n_bytes[0]; 2005 2006 struct header_runtime *h = &t->headers[header_id]; 2007 2008 TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n", 2009 p->thread_id, 2010 header_id, 2011 n_bytes, 2012 n_bytes_last); 2013 2014 n_bytes += n_bytes_last; 2015 2016 /* Headers. */ 2017 t->structs[struct_id] = ptr; 2018 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id); 2019 h->n_bytes = n_bytes; 2020 2021 /* Packet. */ 2022 t->pkt.offset = offset + n_bytes; 2023 t->pkt.length = length - n_bytes; 2024 t->ptr = ptr + n_bytes; 2025 } 2026 2027 static inline void 2028 __instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused, 2029 struct thread *t, 2030 const struct instruction *ip) 2031 { 2032 uint64_t valid_headers = t->valid_headers; 2033 uint8_t *ptr = t->ptr; 2034 2035 uint32_t header_id = ip->io.hdr.header_id[0]; 2036 uint32_t struct_id = ip->io.hdr.struct_id[0]; 2037 2038 TRACE("[Thread %2u]: lookahead header %u\n", 2039 p->thread_id, 2040 header_id); 2041 2042 /* Headers. */ 2043 t->structs[struct_id] = ptr; 2044 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id); 2045 } 2046 2047 /* 2048 * emit. 2049 */ 2050 static inline void 2051 __instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused, 2052 struct thread *t, 2053 const struct instruction *ip, 2054 uint32_t n_emit) 2055 { 2056 uint64_t valid_headers = t->valid_headers; 2057 uint32_t n_headers_out = t->n_headers_out; 2058 struct header_out_runtime *ho = NULL; 2059 uint8_t *ho_ptr = NULL; 2060 uint32_t ho_nbytes = 0, i; 2061 2062 for (i = 0; i < n_emit; i++) { 2063 uint32_t header_id = ip->io.hdr.header_id[i]; 2064 uint32_t struct_id = ip->io.hdr.struct_id[i]; 2065 2066 struct header_runtime *hi = &t->headers[header_id]; 2067 uint8_t *hi_ptr0 = hi->ptr0; 2068 uint32_t n_bytes = hi->n_bytes; 2069 2070 uint8_t *hi_ptr = t->structs[struct_id]; 2071 2072 if (!MASK64_BIT_GET(valid_headers, header_id)) { 2073 TRACE("[Thread %2u]: emit header %u (invalid)\n", 2074 p->thread_id, 2075 header_id); 2076 2077 continue; 2078 } 2079 2080 TRACE("[Thread %2u]: emit header %u (valid)\n", 2081 p->thread_id, 2082 header_id); 2083 2084 /* Headers. */ 2085 if (!ho) { 2086 if (!n_headers_out) { 2087 ho = &t->headers_out[0]; 2088 2089 ho->ptr0 = hi_ptr0; 2090 ho->ptr = hi_ptr; 2091 2092 ho_ptr = hi_ptr; 2093 ho_nbytes = n_bytes; 2094 2095 n_headers_out = 1; 2096 2097 continue; 2098 } else { 2099 ho = &t->headers_out[n_headers_out - 1]; 2100 2101 ho_ptr = ho->ptr; 2102 ho_nbytes = ho->n_bytes; 2103 } 2104 } 2105 2106 if (ho_ptr + ho_nbytes == hi_ptr) { 2107 ho_nbytes += n_bytes; 2108 } else { 2109 ho->n_bytes = ho_nbytes; 2110 2111 ho++; 2112 ho->ptr0 = hi_ptr0; 2113 ho->ptr = hi_ptr; 2114 2115 ho_ptr = hi_ptr; 2116 ho_nbytes = n_bytes; 2117 2118 n_headers_out++; 2119 } 2120 } 2121 2122 if (ho) 2123 ho->n_bytes = ho_nbytes; 2124 t->n_headers_out = n_headers_out; 2125 } 2126 2127 static inline void 2128 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, 2129 struct thread *t, 2130 const struct instruction *ip) 2131 { 2132 __instr_hdr_emit_many_exec(p, t, ip, 1); 2133 } 2134 2135 static inline void 2136 __instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p, 2137 struct thread *t, 2138 const struct instruction *ip) 2139 { 2140 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id); 2141 2142 __instr_hdr_emit_many_exec(p, t, ip, 1); 2143 __instr_tx_exec(p, t, ip); 2144 } 2145 2146 static inline void 2147 __instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p, 2148 struct thread *t, 2149 const struct instruction *ip) 2150 { 2151 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id); 2152 2153 __instr_hdr_emit_many_exec(p, t, ip, 2); 2154 __instr_tx_exec(p, t, ip); 2155 } 2156 2157 static inline void 2158 __instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p, 2159 struct thread *t, 2160 const struct instruction *ip) 2161 { 2162 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id); 2163 2164 __instr_hdr_emit_many_exec(p, t, ip, 3); 2165 __instr_tx_exec(p, t, ip); 2166 } 2167 2168 static inline void 2169 __instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p, 2170 struct thread *t, 2171 const struct instruction *ip) 2172 { 2173 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id); 2174 2175 __instr_hdr_emit_many_exec(p, t, ip, 4); 2176 __instr_tx_exec(p, t, ip); 2177 } 2178 2179 static inline void 2180 __instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p, 2181 struct thread *t, 2182 const struct instruction *ip) 2183 { 2184 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id); 2185 2186 __instr_hdr_emit_many_exec(p, t, ip, 5); 2187 __instr_tx_exec(p, t, ip); 2188 } 2189 2190 static inline void 2191 __instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p, 2192 struct thread *t, 2193 const struct instruction *ip) 2194 { 2195 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id); 2196 2197 __instr_hdr_emit_many_exec(p, t, ip, 6); 2198 __instr_tx_exec(p, t, ip); 2199 } 2200 2201 static inline void 2202 __instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p, 2203 struct thread *t, 2204 const struct instruction *ip) 2205 { 2206 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id); 2207 2208 __instr_hdr_emit_many_exec(p, t, ip, 7); 2209 __instr_tx_exec(p, t, ip); 2210 } 2211 2212 static inline void 2213 __instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p, 2214 struct thread *t, 2215 const struct instruction *ip) 2216 { 2217 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id); 2218 2219 __instr_hdr_emit_many_exec(p, t, ip, 8); 2220 __instr_tx_exec(p, t, ip); 2221 } 2222 2223 /* 2224 * validate. 2225 */ 2226 static inline void 2227 __instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused, 2228 struct thread *t, 2229 const struct instruction *ip) 2230 { 2231 uint32_t header_id = ip->valid.header_id; 2232 uint32_t struct_id = ip->valid.struct_id; 2233 uint64_t valid_headers = t->valid_headers; 2234 struct header_runtime *h = &t->headers[header_id]; 2235 2236 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id); 2237 2238 /* If this header is already valid, then its associated t->structs[] element is also valid 2239 * and therefore it should not be modified. It could point to the packet buffer (in case of 2240 * extracted header) and setting it to the default location (h->ptr0) would be incorrect. 2241 */ 2242 if (MASK64_BIT_GET(valid_headers, header_id)) 2243 return; 2244 2245 /* Headers. */ 2246 t->structs[struct_id] = h->ptr0; 2247 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id); 2248 } 2249 2250 /* 2251 * invalidate. 2252 */ 2253 static inline void 2254 __instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused, 2255 struct thread *t, 2256 const struct instruction *ip) 2257 { 2258 uint32_t header_id = ip->valid.header_id; 2259 2260 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id); 2261 2262 /* Headers. */ 2263 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id); 2264 } 2265 2266 /* 2267 * learn. 2268 */ 2269 static inline void 2270 __instr_learn_exec(struct rte_swx_pipeline *p, 2271 struct thread *t, 2272 const struct instruction *ip) 2273 { 2274 uint64_t action_id = ip->learn.action_id; 2275 uint32_t mf_first_arg_offset = ip->learn.mf_first_arg_offset; 2276 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset, 2277 ip->learn.mf_timeout_id_n_bits); 2278 uint32_t learner_id = t->learner_id; 2279 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + 2280 p->n_selectors + learner_id]; 2281 struct learner_runtime *l = &t->learners[learner_id]; 2282 struct learner_statistics *stats = &p->learner_stats[learner_id]; 2283 uint32_t status; 2284 2285 /* Table. */ 2286 status = rte_swx_table_learner_add(ts->obj, 2287 l->mailbox, 2288 t->time, 2289 action_id, 2290 &t->metadata[mf_first_arg_offset], 2291 timeout_id); 2292 2293 TRACE("[Thread %2u] learner %u learn %s\n", 2294 p->thread_id, 2295 learner_id, 2296 status ? "ok" : "error"); 2297 2298 stats->n_pkts_learn[status] += 1; 2299 } 2300 2301 /* 2302 * rearm. 2303 */ 2304 static inline void 2305 __instr_rearm_exec(struct rte_swx_pipeline *p, 2306 struct thread *t, 2307 const struct instruction *ip __rte_unused) 2308 { 2309 uint32_t learner_id = t->learner_id; 2310 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + 2311 p->n_selectors + learner_id]; 2312 struct learner_runtime *l = &t->learners[learner_id]; 2313 struct learner_statistics *stats = &p->learner_stats[learner_id]; 2314 2315 /* Table. */ 2316 rte_swx_table_learner_rearm(ts->obj, l->mailbox, t->time); 2317 2318 TRACE("[Thread %2u] learner %u rearm\n", 2319 p->thread_id, 2320 learner_id); 2321 2322 stats->n_pkts_rearm += 1; 2323 } 2324 2325 static inline void 2326 __instr_rearm_new_exec(struct rte_swx_pipeline *p, 2327 struct thread *t, 2328 const struct instruction *ip) 2329 { 2330 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset, 2331 ip->learn.mf_timeout_id_n_bits); 2332 uint32_t learner_id = t->learner_id; 2333 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + 2334 p->n_selectors + learner_id]; 2335 struct learner_runtime *l = &t->learners[learner_id]; 2336 struct learner_statistics *stats = &p->learner_stats[learner_id]; 2337 2338 /* Table. */ 2339 rte_swx_table_learner_rearm_new(ts->obj, l->mailbox, t->time, timeout_id); 2340 2341 TRACE("[Thread %2u] learner %u rearm with timeout ID %u\n", 2342 p->thread_id, 2343 learner_id, 2344 timeout_id); 2345 2346 stats->n_pkts_rearm += 1; 2347 } 2348 2349 /* 2350 * forget. 2351 */ 2352 static inline void 2353 __instr_forget_exec(struct rte_swx_pipeline *p, 2354 struct thread *t, 2355 const struct instruction *ip __rte_unused) 2356 { 2357 uint32_t learner_id = t->learner_id; 2358 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + 2359 p->n_selectors + learner_id]; 2360 struct learner_runtime *l = &t->learners[learner_id]; 2361 struct learner_statistics *stats = &p->learner_stats[learner_id]; 2362 2363 /* Table. */ 2364 rte_swx_table_learner_delete(ts->obj, l->mailbox); 2365 2366 TRACE("[Thread %2u] learner %u forget\n", 2367 p->thread_id, 2368 learner_id); 2369 2370 stats->n_pkts_forget += 1; 2371 } 2372 2373 /* 2374 * extern. 2375 */ 2376 static inline uint32_t 2377 __instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused, 2378 struct thread *t, 2379 const struct instruction *ip) 2380 { 2381 uint32_t obj_id = ip->ext_obj.ext_obj_id; 2382 uint32_t func_id = ip->ext_obj.func_id; 2383 struct extern_obj_runtime *obj = &t->extern_objs[obj_id]; 2384 rte_swx_extern_type_member_func_t func = obj->funcs[func_id]; 2385 uint32_t done; 2386 2387 TRACE("[Thread %2u] extern obj %u member func %u\n", 2388 p->thread_id, 2389 obj_id, 2390 func_id); 2391 2392 done = func(obj->obj, obj->mailbox); 2393 2394 return done; 2395 } 2396 2397 static inline uint32_t 2398 __instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused, 2399 struct thread *t, 2400 const struct instruction *ip) 2401 { 2402 uint32_t ext_func_id = ip->ext_func.ext_func_id; 2403 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id]; 2404 rte_swx_extern_func_t func = ext_func->func; 2405 uint32_t done; 2406 2407 TRACE("[Thread %2u] extern func %u\n", 2408 p->thread_id, 2409 ext_func_id); 2410 2411 done = func(ext_func->mailbox); 2412 2413 return done; 2414 } 2415 2416 /* 2417 * hash. 2418 */ 2419 static inline void 2420 __instr_hash_func_exec(struct rte_swx_pipeline *p, 2421 struct thread *t, 2422 const struct instruction *ip) 2423 { 2424 uint32_t hash_func_id = ip->hash_func.hash_func_id; 2425 uint32_t dst_offset = ip->hash_func.dst.offset; 2426 uint32_t n_dst_bits = ip->hash_func.dst.n_bits; 2427 uint32_t src_struct_id = ip->hash_func.src.struct_id; 2428 uint32_t src_offset = ip->hash_func.src.offset; 2429 uint32_t n_src_bytes = ip->hash_func.src.n_bytes; 2430 2431 struct hash_func_runtime *func = &p->hash_func_runtime[hash_func_id]; 2432 uint8_t *src_ptr = t->structs[src_struct_id]; 2433 uint32_t result; 2434 2435 TRACE("[Thread %2u] hash %u\n", 2436 p->thread_id, 2437 hash_func_id); 2438 2439 result = func->func(&src_ptr[src_offset], n_src_bytes, 0); 2440 METADATA_WRITE(t, dst_offset, n_dst_bits, result); 2441 } 2442 2443 /* 2444 * mov. 2445 */ 2446 static inline void 2447 __instr_mov_exec(struct rte_swx_pipeline *p __rte_unused, 2448 struct thread *t, 2449 const struct instruction *ip) 2450 { 2451 TRACE("[Thread %2u] mov\n", p->thread_id); 2452 2453 MOV(t, ip); 2454 } 2455 2456 static inline void 2457 __instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2458 struct thread *t, 2459 const struct instruction *ip) 2460 { 2461 TRACE("[Thread %2u] mov (mh)\n", p->thread_id); 2462 2463 MOV_MH(t, ip); 2464 } 2465 2466 static inline void 2467 __instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2468 struct thread *t, 2469 const struct instruction *ip) 2470 { 2471 TRACE("[Thread %2u] mov (hm)\n", p->thread_id); 2472 2473 MOV_HM(t, ip); 2474 } 2475 2476 static inline void 2477 __instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2478 struct thread *t, 2479 const struct instruction *ip) 2480 { 2481 TRACE("[Thread %2u] mov (hh)\n", p->thread_id); 2482 2483 MOV_HH(t, ip); 2484 } 2485 2486 static inline void 2487 __instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused, 2488 struct thread *t, 2489 const struct instruction *ip) 2490 { 2491 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val); 2492 2493 MOV_I(t, ip); 2494 } 2495 2496 /* 2497 * dma. 2498 */ 2499 static inline void 2500 __instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused, 2501 struct thread *t, 2502 const struct instruction *ip, 2503 uint32_t n_dma) 2504 { 2505 uint8_t *action_data = t->structs[0]; 2506 uint64_t valid_headers = t->valid_headers; 2507 uint32_t i; 2508 2509 for (i = 0; i < n_dma; i++) { 2510 uint32_t header_id = ip->dma.dst.header_id[i]; 2511 uint32_t struct_id = ip->dma.dst.struct_id[i]; 2512 uint32_t offset = ip->dma.src.offset[i]; 2513 uint32_t n_bytes = ip->dma.n_bytes[i]; 2514 2515 struct header_runtime *h = &t->headers[header_id]; 2516 uint8_t *h_ptr0 = h->ptr0; 2517 uint8_t *h_ptr = t->structs[struct_id]; 2518 2519 void *dst = MASK64_BIT_GET(valid_headers, header_id) ? 2520 h_ptr : h_ptr0; 2521 void *src = &action_data[offset]; 2522 2523 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id); 2524 2525 /* Headers. */ 2526 memcpy(dst, src, n_bytes); 2527 t->structs[struct_id] = dst; 2528 valid_headers = MASK64_BIT_SET(valid_headers, header_id); 2529 } 2530 2531 t->valid_headers = valid_headers; 2532 } 2533 2534 static inline void 2535 __instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2536 { 2537 __instr_dma_ht_many_exec(p, t, ip, 1); 2538 } 2539 2540 static inline void 2541 __instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2542 { 2543 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id); 2544 2545 __instr_dma_ht_many_exec(p, t, ip, 2); 2546 } 2547 2548 static inline void 2549 __instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2550 { 2551 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id); 2552 2553 __instr_dma_ht_many_exec(p, t, ip, 3); 2554 } 2555 2556 static inline void 2557 __instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2558 { 2559 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id); 2560 2561 __instr_dma_ht_many_exec(p, t, ip, 4); 2562 } 2563 2564 static inline void 2565 __instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2566 { 2567 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id); 2568 2569 __instr_dma_ht_many_exec(p, t, ip, 5); 2570 } 2571 2572 static inline void 2573 __instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2574 { 2575 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id); 2576 2577 __instr_dma_ht_many_exec(p, t, ip, 6); 2578 } 2579 2580 static inline void 2581 __instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2582 { 2583 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id); 2584 2585 __instr_dma_ht_many_exec(p, t, ip, 7); 2586 } 2587 2588 static inline void 2589 __instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2590 { 2591 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id); 2592 2593 __instr_dma_ht_many_exec(p, t, ip, 8); 2594 } 2595 2596 /* 2597 * alu. 2598 */ 2599 static inline void 2600 __instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused, 2601 struct thread *t, 2602 const struct instruction *ip) 2603 { 2604 TRACE("[Thread %2u] add\n", p->thread_id); 2605 2606 ALU(t, ip, +); 2607 } 2608 2609 static inline void 2610 __instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2611 struct thread *t, 2612 const struct instruction *ip) 2613 { 2614 TRACE("[Thread %2u] add (mh)\n", p->thread_id); 2615 2616 ALU_MH(t, ip, +); 2617 } 2618 2619 static inline void 2620 __instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2621 struct thread *t, 2622 const struct instruction *ip) 2623 { 2624 TRACE("[Thread %2u] add (hm)\n", p->thread_id); 2625 2626 ALU_HM(t, ip, +); 2627 } 2628 2629 static inline void 2630 __instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2631 struct thread *t, 2632 const struct instruction *ip) 2633 { 2634 TRACE("[Thread %2u] add (hh)\n", p->thread_id); 2635 2636 ALU_HH(t, ip, +); 2637 } 2638 2639 static inline void 2640 __instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused, 2641 struct thread *t, 2642 const struct instruction *ip) 2643 { 2644 TRACE("[Thread %2u] add (mi)\n", p->thread_id); 2645 2646 ALU_MI(t, ip, +); 2647 } 2648 2649 static inline void 2650 __instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused, 2651 struct thread *t, 2652 const struct instruction *ip) 2653 { 2654 TRACE("[Thread %2u] add (hi)\n", p->thread_id); 2655 2656 ALU_HI(t, ip, +); 2657 } 2658 2659 static inline void 2660 __instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused, 2661 struct thread *t, 2662 const struct instruction *ip) 2663 { 2664 TRACE("[Thread %2u] sub\n", p->thread_id); 2665 2666 ALU(t, ip, -); 2667 } 2668 2669 static inline void 2670 __instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2671 struct thread *t, 2672 const struct instruction *ip) 2673 { 2674 TRACE("[Thread %2u] sub (mh)\n", p->thread_id); 2675 2676 ALU_MH(t, ip, -); 2677 } 2678 2679 static inline void 2680 __instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2681 struct thread *t, 2682 const struct instruction *ip) 2683 { 2684 TRACE("[Thread %2u] sub (hm)\n", p->thread_id); 2685 2686 ALU_HM(t, ip, -); 2687 } 2688 2689 static inline void 2690 __instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2691 struct thread *t, 2692 const struct instruction *ip) 2693 { 2694 TRACE("[Thread %2u] sub (hh)\n", p->thread_id); 2695 2696 ALU_HH(t, ip, -); 2697 } 2698 2699 static inline void 2700 __instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused, 2701 struct thread *t, 2702 const struct instruction *ip) 2703 { 2704 TRACE("[Thread %2u] sub (mi)\n", p->thread_id); 2705 2706 ALU_MI(t, ip, -); 2707 } 2708 2709 static inline void 2710 __instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused, 2711 struct thread *t, 2712 const struct instruction *ip) 2713 { 2714 TRACE("[Thread %2u] sub (hi)\n", p->thread_id); 2715 2716 ALU_HI(t, ip, -); 2717 } 2718 2719 static inline void 2720 __instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused, 2721 struct thread *t, 2722 const struct instruction *ip) 2723 { 2724 TRACE("[Thread %2u] shl\n", p->thread_id); 2725 2726 ALU(t, ip, <<); 2727 } 2728 2729 static inline void 2730 __instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2731 struct thread *t, 2732 const struct instruction *ip) 2733 { 2734 TRACE("[Thread %2u] shl (mh)\n", p->thread_id); 2735 2736 ALU_MH(t, ip, <<); 2737 } 2738 2739 static inline void 2740 __instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2741 struct thread *t, 2742 const struct instruction *ip) 2743 { 2744 TRACE("[Thread %2u] shl (hm)\n", p->thread_id); 2745 2746 ALU_HM(t, ip, <<); 2747 } 2748 2749 static inline void 2750 __instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2751 struct thread *t, 2752 const struct instruction *ip) 2753 { 2754 TRACE("[Thread %2u] shl (hh)\n", p->thread_id); 2755 2756 ALU_HH(t, ip, <<); 2757 } 2758 2759 static inline void 2760 __instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused, 2761 struct thread *t, 2762 const struct instruction *ip) 2763 { 2764 TRACE("[Thread %2u] shl (mi)\n", p->thread_id); 2765 2766 ALU_MI(t, ip, <<); 2767 } 2768 2769 static inline void 2770 __instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused, 2771 struct thread *t, 2772 const struct instruction *ip) 2773 { 2774 TRACE("[Thread %2u] shl (hi)\n", p->thread_id); 2775 2776 ALU_HI(t, ip, <<); 2777 } 2778 2779 static inline void 2780 __instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused, 2781 struct thread *t, 2782 const struct instruction *ip) 2783 { 2784 TRACE("[Thread %2u] shr\n", p->thread_id); 2785 2786 ALU(t, ip, >>); 2787 } 2788 2789 static inline void 2790 __instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2791 struct thread *t, 2792 const struct instruction *ip) 2793 { 2794 TRACE("[Thread %2u] shr (mh)\n", p->thread_id); 2795 2796 ALU_MH(t, ip, >>); 2797 } 2798 2799 static inline void 2800 __instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2801 struct thread *t, 2802 const struct instruction *ip) 2803 { 2804 TRACE("[Thread %2u] shr (hm)\n", p->thread_id); 2805 2806 ALU_HM(t, ip, >>); 2807 } 2808 2809 static inline void 2810 __instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2811 struct thread *t, 2812 const struct instruction *ip) 2813 { 2814 TRACE("[Thread %2u] shr (hh)\n", p->thread_id); 2815 2816 ALU_HH(t, ip, >>); 2817 } 2818 2819 static inline void 2820 __instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused, 2821 struct thread *t, 2822 const struct instruction *ip) 2823 { 2824 TRACE("[Thread %2u] shr (mi)\n", p->thread_id); 2825 2826 /* Structs. */ 2827 ALU_MI(t, ip, >>); 2828 } 2829 2830 static inline void 2831 __instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused, 2832 struct thread *t, 2833 const struct instruction *ip) 2834 { 2835 TRACE("[Thread %2u] shr (hi)\n", p->thread_id); 2836 2837 ALU_HI(t, ip, >>); 2838 } 2839 2840 static inline void 2841 __instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused, 2842 struct thread *t, 2843 const struct instruction *ip) 2844 { 2845 TRACE("[Thread %2u] and\n", p->thread_id); 2846 2847 ALU(t, ip, &); 2848 } 2849 2850 static inline void 2851 __instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2852 struct thread *t, 2853 const struct instruction *ip) 2854 { 2855 TRACE("[Thread %2u] and (mh)\n", p->thread_id); 2856 2857 ALU_MH(t, ip, &); 2858 } 2859 2860 static inline void 2861 __instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2862 struct thread *t, 2863 const struct instruction *ip) 2864 { 2865 TRACE("[Thread %2u] and (hm)\n", p->thread_id); 2866 2867 ALU_HM_FAST(t, ip, &); 2868 } 2869 2870 static inline void 2871 __instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2872 struct thread *t, 2873 const struct instruction *ip) 2874 { 2875 TRACE("[Thread %2u] and (hh)\n", p->thread_id); 2876 2877 ALU_HH_FAST(t, ip, &); 2878 } 2879 2880 static inline void 2881 __instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused, 2882 struct thread *t, 2883 const struct instruction *ip) 2884 { 2885 TRACE("[Thread %2u] and (i)\n", p->thread_id); 2886 2887 ALU_I(t, ip, &); 2888 } 2889 2890 static inline void 2891 __instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused, 2892 struct thread *t, 2893 const struct instruction *ip) 2894 { 2895 TRACE("[Thread %2u] or\n", p->thread_id); 2896 2897 ALU(t, ip, |); 2898 } 2899 2900 static inline void 2901 __instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2902 struct thread *t, 2903 const struct instruction *ip) 2904 { 2905 TRACE("[Thread %2u] or (mh)\n", p->thread_id); 2906 2907 ALU_MH(t, ip, |); 2908 } 2909 2910 static inline void 2911 __instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2912 struct thread *t, 2913 const struct instruction *ip) 2914 { 2915 TRACE("[Thread %2u] or (hm)\n", p->thread_id); 2916 2917 ALU_HM_FAST(t, ip, |); 2918 } 2919 2920 static inline void 2921 __instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2922 struct thread *t, 2923 const struct instruction *ip) 2924 { 2925 TRACE("[Thread %2u] or (hh)\n", p->thread_id); 2926 2927 ALU_HH_FAST(t, ip, |); 2928 } 2929 2930 static inline void 2931 __instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused, 2932 struct thread *t, 2933 const struct instruction *ip) 2934 { 2935 TRACE("[Thread %2u] or (i)\n", p->thread_id); 2936 2937 ALU_I(t, ip, |); 2938 } 2939 2940 static inline void 2941 __instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused, 2942 struct thread *t, 2943 const struct instruction *ip) 2944 { 2945 TRACE("[Thread %2u] xor\n", p->thread_id); 2946 2947 ALU(t, ip, ^); 2948 } 2949 2950 static inline void 2951 __instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2952 struct thread *t, 2953 const struct instruction *ip) 2954 { 2955 TRACE("[Thread %2u] xor (mh)\n", p->thread_id); 2956 2957 ALU_MH(t, ip, ^); 2958 } 2959 2960 static inline void 2961 __instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2962 struct thread *t, 2963 const struct instruction *ip) 2964 { 2965 TRACE("[Thread %2u] xor (hm)\n", p->thread_id); 2966 2967 ALU_HM_FAST(t, ip, ^); 2968 } 2969 2970 static inline void 2971 __instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2972 struct thread *t, 2973 const struct instruction *ip) 2974 { 2975 TRACE("[Thread %2u] xor (hh)\n", p->thread_id); 2976 2977 ALU_HH_FAST(t, ip, ^); 2978 } 2979 2980 static inline void 2981 __instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused, 2982 struct thread *t, 2983 const struct instruction *ip) 2984 { 2985 TRACE("[Thread %2u] xor (i)\n", p->thread_id); 2986 2987 ALU_I(t, ip, ^); 2988 } 2989 2990 static inline void 2991 __instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused, 2992 struct thread *t, 2993 const struct instruction *ip) 2994 { 2995 uint8_t *dst_struct, *src_struct; 2996 uint16_t *dst16_ptr, dst; 2997 uint64_t *src64_ptr, src64, src64_mask, src; 2998 uint64_t r; 2999 3000 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id); 3001 3002 /* Structs. */ 3003 dst_struct = t->structs[ip->alu.dst.struct_id]; 3004 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; 3005 dst = *dst16_ptr; 3006 3007 src_struct = t->structs[ip->alu.src.struct_id]; 3008 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset]; 3009 src64 = *src64_ptr; 3010 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits); 3011 src = src64 & src64_mask; 3012 3013 /* Initialize the result with destination 1's complement. */ 3014 r = dst; 3015 r = ~r & 0xFFFF; 3016 3017 /* The first input (r) is a 16-bit number. The second and the third 3018 * inputs are 32-bit numbers. In the worst case scenario, the sum of the 3019 * three numbers (output r) is a 34-bit number. 3020 */ 3021 r += (src >> 32) + (src & 0xFFFFFFFF); 3022 3023 /* The first input is a 16-bit number. The second input is an 18-bit 3024 * number. In the worst case scenario, the sum of the two numbers is a 3025 * 19-bit number. 3026 */ 3027 r = (r & 0xFFFF) + (r >> 16); 3028 3029 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is 3030 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006). 3031 */ 3032 r = (r & 0xFFFF) + (r >> 16); 3033 3034 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input 3035 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. 3036 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated, 3037 * therefore the output r is always a 16-bit number. 3038 */ 3039 r = (r & 0xFFFF) + (r >> 16); 3040 3041 /* Apply 1's complement to the result. */ 3042 r = ~r & 0xFFFF; 3043 r = r ? r : 0xFFFF; 3044 3045 *dst16_ptr = (uint16_t)r; 3046 } 3047 3048 static inline void 3049 __instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused, 3050 struct thread *t, 3051 const struct instruction *ip) 3052 { 3053 uint8_t *dst_struct, *src_struct; 3054 uint16_t *dst16_ptr, dst; 3055 uint64_t *src64_ptr, src64, src64_mask, src; 3056 uint64_t r; 3057 3058 TRACE("[Thread %2u] cksub (field)\n", p->thread_id); 3059 3060 /* Structs. */ 3061 dst_struct = t->structs[ip->alu.dst.struct_id]; 3062 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; 3063 dst = *dst16_ptr; 3064 3065 src_struct = t->structs[ip->alu.src.struct_id]; 3066 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset]; 3067 src64 = *src64_ptr; 3068 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits); 3069 src = src64 & src64_mask; 3070 3071 /* Initialize the result with destination 1's complement. */ 3072 r = dst; 3073 r = ~r & 0xFFFF; 3074 3075 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as 3076 * the following sequence of operations in 2's complement arithmetic: 3077 * a '- b = (a - b) % 0xFFFF. 3078 * 3079 * In order to prevent an underflow for the below subtraction, in which 3080 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the 3081 * minuend), we first add a multiple of the 0xFFFF modulus to the 3082 * minuend. The number we add to the minuend needs to be a 34-bit number 3083 * or higher, so for readability reasons we picked the 36-bit multiple. 3084 * We are effectively turning the 16-bit minuend into a 36-bit number: 3085 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF. 3086 */ 3087 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */ 3088 3089 /* A 33-bit number is subtracted from a 36-bit number (the input r). The 3090 * result (the output r) is a 36-bit number. 3091 */ 3092 r -= (src >> 32) + (src & 0xFFFFFFFF); 3093 3094 /* The first input is a 16-bit number. The second input is a 20-bit 3095 * number. Their sum is a 21-bit number. 3096 */ 3097 r = (r & 0xFFFF) + (r >> 16); 3098 3099 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is 3100 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E). 3101 */ 3102 r = (r & 0xFFFF) + (r >> 16); 3103 3104 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input 3105 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. 3106 * 0x1001E), the output r is (0 .. 31). So no carry bit can be 3107 * generated, therefore the output r is always a 16-bit number. 3108 */ 3109 r = (r & 0xFFFF) + (r >> 16); 3110 3111 /* Apply 1's complement to the result. */ 3112 r = ~r & 0xFFFF; 3113 r = r ? r : 0xFFFF; 3114 3115 *dst16_ptr = (uint16_t)r; 3116 } 3117 3118 static inline void 3119 __instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused, 3120 struct thread *t, 3121 const struct instruction *ip) 3122 { 3123 uint8_t *dst_struct, *src_struct; 3124 uint16_t *dst16_ptr, dst; 3125 uint32_t *src32_ptr; 3126 uint64_t r0, r1; 3127 3128 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id); 3129 3130 /* Structs. */ 3131 dst_struct = t->structs[ip->alu.dst.struct_id]; 3132 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; 3133 dst = *dst16_ptr; 3134 3135 src_struct = t->structs[ip->alu.src.struct_id]; 3136 src32_ptr = (uint32_t *)&src_struct[0]; 3137 3138 /* Initialize the result with destination 1's complement. */ 3139 r0 = dst; 3140 r0 = ~r0 & 0xFFFF; 3141 3142 r0 += src32_ptr[0]; /* The output r0 is a 33-bit number. */ 3143 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */ 3144 r0 += src32_ptr[2]; /* The output r0 is a 34-bit number. */ 3145 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */ 3146 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */ 3147 3148 /* The first input is a 16-bit number. The second input is a 19-bit 3149 * number. Their sum is a 20-bit number. 3150 */ 3151 r0 = (r0 & 0xFFFF) + (r0 >> 16); 3152 3153 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is 3154 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E). 3155 */ 3156 r0 = (r0 & 0xFFFF) + (r0 >> 16); 3157 3158 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input 3159 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. 3160 * 0x1000E), the output r is (0 .. 15). So no carry bit can be 3161 * generated, therefore the output r is always a 16-bit number. 3162 */ 3163 r0 = (r0 & 0xFFFF) + (r0 >> 16); 3164 3165 /* Apply 1's complement to the result. */ 3166 r0 = ~r0 & 0xFFFF; 3167 r0 = r0 ? r0 : 0xFFFF; 3168 3169 *dst16_ptr = (uint16_t)r0; 3170 } 3171 3172 static inline void 3173 __instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused, 3174 struct thread *t, 3175 const struct instruction *ip) 3176 { 3177 uint32_t src_header_id = ip->alu.src.n_bits; /* The src header ID is stored here. */ 3178 uint32_t n_src_header_bytes = t->headers[src_header_id].n_bytes; 3179 uint8_t *dst_struct, *src_struct; 3180 uint16_t *dst16_ptr, dst; 3181 uint32_t *src32_ptr; 3182 uint64_t r; 3183 uint32_t i; 3184 3185 if (n_src_header_bytes == 20) { 3186 __instr_alu_ckadd_struct20_exec(p, t, ip); 3187 return; 3188 } 3189 3190 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id); 3191 3192 /* Structs. */ 3193 dst_struct = t->structs[ip->alu.dst.struct_id]; 3194 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; 3195 dst = *dst16_ptr; 3196 3197 src_struct = t->structs[ip->alu.src.struct_id]; 3198 src32_ptr = (uint32_t *)&src_struct[0]; 3199 3200 /* Initialize the result with destination 1's complement. */ 3201 r = dst; 3202 r = ~r & 0xFFFF; 3203 3204 /* The max number of 32-bit words in a 32K-byte header is 2^13. 3205 * Therefore, in the worst case scenario, a 45-bit number is added to a 3206 * 16-bit number (the input r), so the output r is 46-bit number. 3207 */ 3208 for (i = 0; i < n_src_header_bytes / 4; i++, src32_ptr++) 3209 r += *src32_ptr; 3210 3211 /* The first input is a 16-bit number. The second input is a 30-bit 3212 * number. Their sum is a 31-bit number. 3213 */ 3214 r = (r & 0xFFFF) + (r >> 16); 3215 3216 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is 3217 * a 15-bit number (0 .. 0x7FFF). The sum is a 17-bit number (0 .. 0x17FFE). 3218 */ 3219 r = (r & 0xFFFF) + (r >> 16); 3220 3221 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input 3222 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. 3223 * 0x17FFE), the output r is (0 .. 0x7FFF). So no carry bit can be 3224 * generated, therefore the output r is always a 16-bit number. 3225 */ 3226 r = (r & 0xFFFF) + (r >> 16); 3227 3228 /* Apply 1's complement to the result. */ 3229 r = ~r & 0xFFFF; 3230 r = r ? r : 0xFFFF; 3231 3232 *dst16_ptr = (uint16_t)r; 3233 } 3234 3235 /* 3236 * Register array. 3237 */ 3238 static inline uint64_t * 3239 instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip) 3240 { 3241 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; 3242 return r->regarray; 3243 } 3244 3245 static inline uint64_t 3246 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3247 { 3248 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; 3249 3250 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; 3251 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; 3252 uint64_t idx64 = *idx64_ptr; 3253 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits); 3254 uint64_t idx = idx64 & idx64_mask & r->size_mask; 3255 3256 return idx; 3257 } 3258 3259 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3260 3261 static inline uint64_t 3262 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3263 { 3264 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; 3265 3266 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; 3267 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; 3268 uint64_t idx64 = *idx64_ptr; 3269 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask; 3270 3271 return idx; 3272 } 3273 3274 #else 3275 3276 #define instr_regarray_idx_nbo instr_regarray_idx_hbo 3277 3278 #endif 3279 3280 static inline uint64_t 3281 instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip) 3282 { 3283 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; 3284 3285 uint64_t idx = ip->regarray.idx_val & r->size_mask; 3286 3287 return idx; 3288 } 3289 3290 static inline uint64_t 3291 instr_regarray_src_hbo(struct thread *t, const struct instruction *ip) 3292 { 3293 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; 3294 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; 3295 uint64_t src64 = *src64_ptr; 3296 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); 3297 uint64_t src = src64 & src64_mask; 3298 3299 return src; 3300 } 3301 3302 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3303 3304 static inline uint64_t 3305 instr_regarray_src_nbo(struct thread *t, const struct instruction *ip) 3306 { 3307 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; 3308 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; 3309 uint64_t src64 = *src64_ptr; 3310 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits); 3311 3312 return src; 3313 } 3314 3315 #else 3316 3317 #define instr_regarray_src_nbo instr_regarray_src_hbo 3318 3319 #endif 3320 3321 static inline void 3322 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src) 3323 { 3324 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; 3325 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; 3326 uint64_t dst64 = *dst64_ptr; 3327 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); 3328 3329 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); 3330 3331 } 3332 3333 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3334 3335 static inline void 3336 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src) 3337 { 3338 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; 3339 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; 3340 uint64_t dst64 = *dst64_ptr; 3341 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); 3342 3343 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits); 3344 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); 3345 } 3346 3347 #else 3348 3349 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set 3350 3351 #endif 3352 3353 static inline void 3354 __instr_regprefetch_rh_exec(struct rte_swx_pipeline *p, 3355 struct thread *t, 3356 const struct instruction *ip) 3357 { 3358 uint64_t *regarray, idx; 3359 3360 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id); 3361 3362 regarray = instr_regarray_regarray(p, ip); 3363 idx = instr_regarray_idx_nbo(p, t, ip); 3364 rte_prefetch0(®array[idx]); 3365 } 3366 3367 static inline void 3368 __instr_regprefetch_rm_exec(struct rte_swx_pipeline *p, 3369 struct thread *t, 3370 const struct instruction *ip) 3371 { 3372 uint64_t *regarray, idx; 3373 3374 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id); 3375 3376 regarray = instr_regarray_regarray(p, ip); 3377 idx = instr_regarray_idx_hbo(p, t, ip); 3378 rte_prefetch0(®array[idx]); 3379 } 3380 3381 static inline void 3382 __instr_regprefetch_ri_exec(struct rte_swx_pipeline *p, 3383 struct thread *t __rte_unused, 3384 const struct instruction *ip) 3385 { 3386 uint64_t *regarray, idx; 3387 3388 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id); 3389 3390 regarray = instr_regarray_regarray(p, ip); 3391 idx = instr_regarray_idx_imm(p, ip); 3392 rte_prefetch0(®array[idx]); 3393 } 3394 3395 static inline void 3396 __instr_regrd_hrh_exec(struct rte_swx_pipeline *p, 3397 struct thread *t, 3398 const struct instruction *ip) 3399 { 3400 uint64_t *regarray, idx; 3401 3402 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id); 3403 3404 regarray = instr_regarray_regarray(p, ip); 3405 idx = instr_regarray_idx_nbo(p, t, ip); 3406 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); 3407 } 3408 3409 static inline void 3410 __instr_regrd_hrm_exec(struct rte_swx_pipeline *p, 3411 struct thread *t, 3412 const struct instruction *ip) 3413 { 3414 uint64_t *regarray, idx; 3415 3416 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id); 3417 3418 /* Structs. */ 3419 regarray = instr_regarray_regarray(p, ip); 3420 idx = instr_regarray_idx_hbo(p, t, ip); 3421 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); 3422 } 3423 3424 static inline void 3425 __instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3426 { 3427 uint64_t *regarray, idx; 3428 3429 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id); 3430 3431 regarray = instr_regarray_regarray(p, ip); 3432 idx = instr_regarray_idx_nbo(p, t, ip); 3433 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); 3434 } 3435 3436 static inline void 3437 __instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3438 { 3439 uint64_t *regarray, idx; 3440 3441 TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id); 3442 3443 regarray = instr_regarray_regarray(p, ip); 3444 idx = instr_regarray_idx_hbo(p, t, ip); 3445 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); 3446 } 3447 3448 static inline void 3449 __instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3450 { 3451 uint64_t *regarray, idx; 3452 3453 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id); 3454 3455 regarray = instr_regarray_regarray(p, ip); 3456 idx = instr_regarray_idx_imm(p, ip); 3457 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); 3458 } 3459 3460 static inline void 3461 __instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3462 { 3463 uint64_t *regarray, idx; 3464 3465 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id); 3466 3467 regarray = instr_regarray_regarray(p, ip); 3468 idx = instr_regarray_idx_imm(p, ip); 3469 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); 3470 } 3471 3472 static inline void 3473 __instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3474 { 3475 uint64_t *regarray, idx, src; 3476 3477 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id); 3478 3479 regarray = instr_regarray_regarray(p, ip); 3480 idx = instr_regarray_idx_nbo(p, t, ip); 3481 src = instr_regarray_src_nbo(t, ip); 3482 regarray[idx] = src; 3483 } 3484 3485 static inline void 3486 __instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3487 { 3488 uint64_t *regarray, idx, src; 3489 3490 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id); 3491 3492 regarray = instr_regarray_regarray(p, ip); 3493 idx = instr_regarray_idx_nbo(p, t, ip); 3494 src = instr_regarray_src_hbo(t, ip); 3495 regarray[idx] = src; 3496 } 3497 3498 static inline void 3499 __instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3500 { 3501 uint64_t *regarray, idx, src; 3502 3503 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id); 3504 3505 regarray = instr_regarray_regarray(p, ip); 3506 idx = instr_regarray_idx_hbo(p, t, ip); 3507 src = instr_regarray_src_nbo(t, ip); 3508 regarray[idx] = src; 3509 } 3510 3511 static inline void 3512 __instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3513 { 3514 uint64_t *regarray, idx, src; 3515 3516 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id); 3517 3518 regarray = instr_regarray_regarray(p, ip); 3519 idx = instr_regarray_idx_hbo(p, t, ip); 3520 src = instr_regarray_src_hbo(t, ip); 3521 regarray[idx] = src; 3522 } 3523 3524 static inline void 3525 __instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3526 { 3527 uint64_t *regarray, idx, src; 3528 3529 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id); 3530 3531 regarray = instr_regarray_regarray(p, ip); 3532 idx = instr_regarray_idx_nbo(p, t, ip); 3533 src = ip->regarray.dstsrc_val; 3534 regarray[idx] = src; 3535 } 3536 3537 static inline void 3538 __instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3539 { 3540 uint64_t *regarray, idx, src; 3541 3542 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id); 3543 3544 regarray = instr_regarray_regarray(p, ip); 3545 idx = instr_regarray_idx_hbo(p, t, ip); 3546 src = ip->regarray.dstsrc_val; 3547 regarray[idx] = src; 3548 } 3549 3550 static inline void 3551 __instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3552 { 3553 uint64_t *regarray, idx, src; 3554 3555 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id); 3556 3557 regarray = instr_regarray_regarray(p, ip); 3558 idx = instr_regarray_idx_imm(p, ip); 3559 src = instr_regarray_src_nbo(t, ip); 3560 regarray[idx] = src; 3561 } 3562 3563 static inline void 3564 __instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3565 { 3566 uint64_t *regarray, idx, src; 3567 3568 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id); 3569 3570 regarray = instr_regarray_regarray(p, ip); 3571 idx = instr_regarray_idx_imm(p, ip); 3572 src = instr_regarray_src_hbo(t, ip); 3573 regarray[idx] = src; 3574 } 3575 3576 static inline void 3577 __instr_regwr_rii_exec(struct rte_swx_pipeline *p, 3578 struct thread *t __rte_unused, 3579 const struct instruction *ip) 3580 { 3581 uint64_t *regarray, idx, src; 3582 3583 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id); 3584 3585 regarray = instr_regarray_regarray(p, ip); 3586 idx = instr_regarray_idx_imm(p, ip); 3587 src = ip->regarray.dstsrc_val; 3588 regarray[idx] = src; 3589 } 3590 3591 static inline void 3592 __instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3593 { 3594 uint64_t *regarray, idx, src; 3595 3596 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id); 3597 3598 regarray = instr_regarray_regarray(p, ip); 3599 idx = instr_regarray_idx_nbo(p, t, ip); 3600 src = instr_regarray_src_nbo(t, ip); 3601 regarray[idx] += src; 3602 } 3603 3604 static inline void 3605 __instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3606 { 3607 uint64_t *regarray, idx, src; 3608 3609 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id); 3610 3611 regarray = instr_regarray_regarray(p, ip); 3612 idx = instr_regarray_idx_nbo(p, t, ip); 3613 src = instr_regarray_src_hbo(t, ip); 3614 regarray[idx] += src; 3615 } 3616 3617 static inline void 3618 __instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3619 { 3620 uint64_t *regarray, idx, src; 3621 3622 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id); 3623 3624 regarray = instr_regarray_regarray(p, ip); 3625 idx = instr_regarray_idx_hbo(p, t, ip); 3626 src = instr_regarray_src_nbo(t, ip); 3627 regarray[idx] += src; 3628 } 3629 3630 static inline void 3631 __instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3632 { 3633 uint64_t *regarray, idx, src; 3634 3635 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id); 3636 3637 regarray = instr_regarray_regarray(p, ip); 3638 idx = instr_regarray_idx_hbo(p, t, ip); 3639 src = instr_regarray_src_hbo(t, ip); 3640 regarray[idx] += src; 3641 } 3642 3643 static inline void 3644 __instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3645 { 3646 uint64_t *regarray, idx, src; 3647 3648 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id); 3649 3650 regarray = instr_regarray_regarray(p, ip); 3651 idx = instr_regarray_idx_nbo(p, t, ip); 3652 src = ip->regarray.dstsrc_val; 3653 regarray[idx] += src; 3654 } 3655 3656 static inline void 3657 __instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3658 { 3659 uint64_t *regarray, idx, src; 3660 3661 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id); 3662 3663 regarray = instr_regarray_regarray(p, ip); 3664 idx = instr_regarray_idx_hbo(p, t, ip); 3665 src = ip->regarray.dstsrc_val; 3666 regarray[idx] += src; 3667 } 3668 3669 static inline void 3670 __instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3671 { 3672 uint64_t *regarray, idx, src; 3673 3674 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id); 3675 3676 regarray = instr_regarray_regarray(p, ip); 3677 idx = instr_regarray_idx_imm(p, ip); 3678 src = instr_regarray_src_nbo(t, ip); 3679 regarray[idx] += src; 3680 } 3681 3682 static inline void 3683 __instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3684 { 3685 uint64_t *regarray, idx, src; 3686 3687 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id); 3688 3689 regarray = instr_regarray_regarray(p, ip); 3690 idx = instr_regarray_idx_imm(p, ip); 3691 src = instr_regarray_src_hbo(t, ip); 3692 regarray[idx] += src; 3693 } 3694 3695 static inline void 3696 __instr_regadd_rii_exec(struct rte_swx_pipeline *p, 3697 struct thread *t __rte_unused, 3698 const struct instruction *ip) 3699 { 3700 uint64_t *regarray, idx, src; 3701 3702 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id); 3703 3704 regarray = instr_regarray_regarray(p, ip); 3705 idx = instr_regarray_idx_imm(p, ip); 3706 src = ip->regarray.dstsrc_val; 3707 regarray[idx] += src; 3708 } 3709 3710 /* 3711 * metarray. 3712 */ 3713 static inline struct meter * 3714 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3715 { 3716 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; 3717 3718 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; 3719 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; 3720 uint64_t idx64 = *idx64_ptr; 3721 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits); 3722 uint64_t idx = idx64 & idx64_mask & r->size_mask; 3723 3724 return &r->metarray[idx]; 3725 } 3726 3727 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3728 3729 static inline struct meter * 3730 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3731 { 3732 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; 3733 3734 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; 3735 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; 3736 uint64_t idx64 = *idx64_ptr; 3737 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask; 3738 3739 return &r->metarray[idx]; 3740 } 3741 3742 #else 3743 3744 #define instr_meter_idx_nbo instr_meter_idx_hbo 3745 3746 #endif 3747 3748 static inline struct meter * 3749 instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip) 3750 { 3751 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; 3752 3753 uint64_t idx = ip->meter.idx_val & r->size_mask; 3754 3755 return &r->metarray[idx]; 3756 } 3757 3758 static inline uint32_t 3759 instr_meter_length_hbo(struct thread *t, const struct instruction *ip) 3760 { 3761 uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; 3762 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; 3763 uint64_t src64 = *src64_ptr; 3764 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits); 3765 uint64_t src = src64 & src64_mask; 3766 3767 return (uint32_t)src; 3768 } 3769 3770 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3771 3772 static inline uint32_t 3773 instr_meter_length_nbo(struct thread *t, const struct instruction *ip) 3774 { 3775 uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; 3776 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; 3777 uint64_t src64 = *src64_ptr; 3778 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits); 3779 3780 return (uint32_t)src; 3781 } 3782 3783 #else 3784 3785 #define instr_meter_length_nbo instr_meter_length_hbo 3786 3787 #endif 3788 3789 static inline enum rte_color 3790 instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip) 3791 { 3792 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id]; 3793 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset]; 3794 uint64_t src64 = *src64_ptr; 3795 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits); 3796 uint64_t src = src64 & src64_mask; 3797 3798 return (enum rte_color)src; 3799 } 3800 3801 static inline void 3802 instr_meter_color_out_hbo_set(struct thread *t, 3803 const struct instruction *ip, 3804 enum rte_color color_out) 3805 { 3806 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id]; 3807 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset]; 3808 uint64_t dst64 = *dst64_ptr; 3809 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits); 3810 3811 uint64_t src = (uint64_t)color_out; 3812 3813 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); 3814 } 3815 3816 static inline void 3817 __instr_metprefetch_h_exec(struct rte_swx_pipeline *p, 3818 struct thread *t, 3819 const struct instruction *ip) 3820 { 3821 struct meter *m; 3822 3823 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id); 3824 3825 m = instr_meter_idx_nbo(p, t, ip); 3826 rte_prefetch0(m); 3827 } 3828 3829 static inline void 3830 __instr_metprefetch_m_exec(struct rte_swx_pipeline *p, 3831 struct thread *t, 3832 const struct instruction *ip) 3833 { 3834 struct meter *m; 3835 3836 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id); 3837 3838 m = instr_meter_idx_hbo(p, t, ip); 3839 rte_prefetch0(m); 3840 } 3841 3842 static inline void 3843 __instr_metprefetch_i_exec(struct rte_swx_pipeline *p, 3844 struct thread *t __rte_unused, 3845 const struct instruction *ip) 3846 { 3847 struct meter *m; 3848 3849 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id); 3850 3851 m = instr_meter_idx_imm(p, ip); 3852 rte_prefetch0(m); 3853 } 3854 3855 static inline void 3856 __instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3857 { 3858 struct meter *m; 3859 uint64_t time, n_pkts, n_bytes; 3860 uint32_t length; 3861 enum rte_color color_in, color_out; 3862 3863 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id); 3864 3865 m = instr_meter_idx_nbo(p, t, ip); 3866 rte_prefetch0(m->n_pkts); 3867 time = rte_get_tsc_cycles(); 3868 length = instr_meter_length_nbo(t, ip); 3869 color_in = instr_meter_color_in_hbo(t, ip); 3870 3871 color_out = rte_meter_trtcm_color_aware_check(&m->m, 3872 &m->profile->profile, 3873 time, 3874 length, 3875 color_in); 3876 3877 color_out &= m->color_mask; 3878 3879 n_pkts = m->n_pkts[color_out]; 3880 n_bytes = m->n_bytes[color_out]; 3881 3882 instr_meter_color_out_hbo_set(t, ip, color_out); 3883 3884 m->n_pkts[color_out] = n_pkts + 1; 3885 m->n_bytes[color_out] = n_bytes + length; 3886 } 3887 3888 static inline void 3889 __instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3890 { 3891 struct meter *m; 3892 uint64_t time, n_pkts, n_bytes; 3893 uint32_t length; 3894 enum rte_color color_in, color_out; 3895 3896 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id); 3897 3898 m = instr_meter_idx_nbo(p, t, ip); 3899 rte_prefetch0(m->n_pkts); 3900 time = rte_get_tsc_cycles(); 3901 length = instr_meter_length_nbo(t, ip); 3902 color_in = (enum rte_color)ip->meter.color_in_val; 3903 3904 color_out = rte_meter_trtcm_color_aware_check(&m->m, 3905 &m->profile->profile, 3906 time, 3907 length, 3908 color_in); 3909 3910 color_out &= m->color_mask; 3911 3912 n_pkts = m->n_pkts[color_out]; 3913 n_bytes = m->n_bytes[color_out]; 3914 3915 instr_meter_color_out_hbo_set(t, ip, color_out); 3916 3917 m->n_pkts[color_out] = n_pkts + 1; 3918 m->n_bytes[color_out] = n_bytes + length; 3919 } 3920 3921 static inline void 3922 __instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3923 { 3924 struct meter *m; 3925 uint64_t time, n_pkts, n_bytes; 3926 uint32_t length; 3927 enum rte_color color_in, color_out; 3928 3929 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id); 3930 3931 m = instr_meter_idx_nbo(p, t, ip); 3932 rte_prefetch0(m->n_pkts); 3933 time = rte_get_tsc_cycles(); 3934 length = instr_meter_length_hbo(t, ip); 3935 color_in = instr_meter_color_in_hbo(t, ip); 3936 3937 color_out = rte_meter_trtcm_color_aware_check(&m->m, 3938 &m->profile->profile, 3939 time, 3940 length, 3941 color_in); 3942 3943 color_out &= m->color_mask; 3944 3945 n_pkts = m->n_pkts[color_out]; 3946 n_bytes = m->n_bytes[color_out]; 3947 3948 instr_meter_color_out_hbo_set(t, ip, color_out); 3949 3950 m->n_pkts[color_out] = n_pkts + 1; 3951 m->n_bytes[color_out] = n_bytes + length; 3952 } 3953 3954 static inline void 3955 __instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3956 { 3957 struct meter *m; 3958 uint64_t time, n_pkts, n_bytes; 3959 uint32_t length; 3960 enum rte_color color_in, color_out; 3961 3962 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id); 3963 3964 m = instr_meter_idx_nbo(p, t, ip); 3965 rte_prefetch0(m->n_pkts); 3966 time = rte_get_tsc_cycles(); 3967 length = instr_meter_length_hbo(t, ip); 3968 color_in = (enum rte_color)ip->meter.color_in_val; 3969 3970 color_out = rte_meter_trtcm_color_aware_check(&m->m, 3971 &m->profile->profile, 3972 time, 3973 length, 3974 color_in); 3975 3976 color_out &= m->color_mask; 3977 3978 n_pkts = m->n_pkts[color_out]; 3979 n_bytes = m->n_bytes[color_out]; 3980 3981 instr_meter_color_out_hbo_set(t, ip, color_out); 3982 3983 m->n_pkts[color_out] = n_pkts + 1; 3984 m->n_bytes[color_out] = n_bytes + length; 3985 } 3986 3987 static inline void 3988 __instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3989 { 3990 struct meter *m; 3991 uint64_t time, n_pkts, n_bytes; 3992 uint32_t length; 3993 enum rte_color color_in, color_out; 3994 3995 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id); 3996 3997 m = instr_meter_idx_hbo(p, t, ip); 3998 rte_prefetch0(m->n_pkts); 3999 time = rte_get_tsc_cycles(); 4000 length = instr_meter_length_nbo(t, ip); 4001 color_in = instr_meter_color_in_hbo(t, ip); 4002 4003 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4004 &m->profile->profile, 4005 time, 4006 length, 4007 color_in); 4008 4009 color_out &= m->color_mask; 4010 4011 n_pkts = m->n_pkts[color_out]; 4012 n_bytes = m->n_bytes[color_out]; 4013 4014 instr_meter_color_out_hbo_set(t, ip, color_out); 4015 4016 m->n_pkts[color_out] = n_pkts + 1; 4017 m->n_bytes[color_out] = n_bytes + length; 4018 } 4019 4020 static inline void 4021 __instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4022 { 4023 struct meter *m; 4024 uint64_t time, n_pkts, n_bytes; 4025 uint32_t length; 4026 enum rte_color color_in, color_out; 4027 4028 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id); 4029 4030 m = instr_meter_idx_hbo(p, t, ip); 4031 rte_prefetch0(m->n_pkts); 4032 time = rte_get_tsc_cycles(); 4033 length = instr_meter_length_nbo(t, ip); 4034 color_in = (enum rte_color)ip->meter.color_in_val; 4035 4036 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4037 &m->profile->profile, 4038 time, 4039 length, 4040 color_in); 4041 4042 color_out &= m->color_mask; 4043 4044 n_pkts = m->n_pkts[color_out]; 4045 n_bytes = m->n_bytes[color_out]; 4046 4047 instr_meter_color_out_hbo_set(t, ip, color_out); 4048 4049 m->n_pkts[color_out] = n_pkts + 1; 4050 m->n_bytes[color_out] = n_bytes + length; 4051 } 4052 4053 static inline void 4054 __instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4055 { 4056 struct meter *m; 4057 uint64_t time, n_pkts, n_bytes; 4058 uint32_t length; 4059 enum rte_color color_in, color_out; 4060 4061 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id); 4062 4063 m = instr_meter_idx_hbo(p, t, ip); 4064 rte_prefetch0(m->n_pkts); 4065 time = rte_get_tsc_cycles(); 4066 length = instr_meter_length_hbo(t, ip); 4067 color_in = instr_meter_color_in_hbo(t, ip); 4068 4069 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4070 &m->profile->profile, 4071 time, 4072 length, 4073 color_in); 4074 4075 color_out &= m->color_mask; 4076 4077 n_pkts = m->n_pkts[color_out]; 4078 n_bytes = m->n_bytes[color_out]; 4079 4080 instr_meter_color_out_hbo_set(t, ip, color_out); 4081 4082 m->n_pkts[color_out] = n_pkts + 1; 4083 m->n_bytes[color_out] = n_bytes + length; 4084 } 4085 4086 static inline void 4087 __instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4088 { 4089 struct meter *m; 4090 uint64_t time, n_pkts, n_bytes; 4091 uint32_t length; 4092 enum rte_color color_in, color_out; 4093 4094 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id); 4095 4096 m = instr_meter_idx_hbo(p, t, ip); 4097 rte_prefetch0(m->n_pkts); 4098 time = rte_get_tsc_cycles(); 4099 length = instr_meter_length_hbo(t, ip); 4100 color_in = (enum rte_color)ip->meter.color_in_val; 4101 4102 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4103 &m->profile->profile, 4104 time, 4105 length, 4106 color_in); 4107 4108 color_out &= m->color_mask; 4109 4110 n_pkts = m->n_pkts[color_out]; 4111 n_bytes = m->n_bytes[color_out]; 4112 4113 instr_meter_color_out_hbo_set(t, ip, color_out); 4114 4115 m->n_pkts[color_out] = n_pkts + 1; 4116 m->n_bytes[color_out] = n_bytes + length; 4117 } 4118 4119 static inline void 4120 __instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4121 { 4122 struct meter *m; 4123 uint64_t time, n_pkts, n_bytes; 4124 uint32_t length; 4125 enum rte_color color_in, color_out; 4126 4127 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id); 4128 4129 m = instr_meter_idx_imm(p, ip); 4130 rte_prefetch0(m->n_pkts); 4131 time = rte_get_tsc_cycles(); 4132 length = instr_meter_length_nbo(t, ip); 4133 color_in = instr_meter_color_in_hbo(t, ip); 4134 4135 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4136 &m->profile->profile, 4137 time, 4138 length, 4139 color_in); 4140 4141 color_out &= m->color_mask; 4142 4143 n_pkts = m->n_pkts[color_out]; 4144 n_bytes = m->n_bytes[color_out]; 4145 4146 instr_meter_color_out_hbo_set(t, ip, color_out); 4147 4148 m->n_pkts[color_out] = n_pkts + 1; 4149 m->n_bytes[color_out] = n_bytes + length; 4150 } 4151 4152 static inline void 4153 __instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4154 { 4155 struct meter *m; 4156 uint64_t time, n_pkts, n_bytes; 4157 uint32_t length; 4158 enum rte_color color_in, color_out; 4159 4160 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id); 4161 4162 m = instr_meter_idx_imm(p, ip); 4163 rte_prefetch0(m->n_pkts); 4164 time = rte_get_tsc_cycles(); 4165 length = instr_meter_length_nbo(t, ip); 4166 color_in = (enum rte_color)ip->meter.color_in_val; 4167 4168 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4169 &m->profile->profile, 4170 time, 4171 length, 4172 color_in); 4173 4174 color_out &= m->color_mask; 4175 4176 n_pkts = m->n_pkts[color_out]; 4177 n_bytes = m->n_bytes[color_out]; 4178 4179 instr_meter_color_out_hbo_set(t, ip, color_out); 4180 4181 m->n_pkts[color_out] = n_pkts + 1; 4182 m->n_bytes[color_out] = n_bytes + length; 4183 } 4184 4185 static inline void 4186 __instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4187 { 4188 struct meter *m; 4189 uint64_t time, n_pkts, n_bytes; 4190 uint32_t length; 4191 enum rte_color color_in, color_out; 4192 4193 TRACE("[Thread %2u] meter (imm)\n", p->thread_id); 4194 4195 m = instr_meter_idx_imm(p, ip); 4196 rte_prefetch0(m->n_pkts); 4197 time = rte_get_tsc_cycles(); 4198 length = instr_meter_length_hbo(t, ip); 4199 color_in = instr_meter_color_in_hbo(t, ip); 4200 4201 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4202 &m->profile->profile, 4203 time, 4204 length, 4205 color_in); 4206 4207 color_out &= m->color_mask; 4208 4209 n_pkts = m->n_pkts[color_out]; 4210 n_bytes = m->n_bytes[color_out]; 4211 4212 instr_meter_color_out_hbo_set(t, ip, color_out); 4213 4214 m->n_pkts[color_out] = n_pkts + 1; 4215 m->n_bytes[color_out] = n_bytes + length; 4216 } 4217 4218 static inline void 4219 __instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4220 { 4221 struct meter *m; 4222 uint64_t time, n_pkts, n_bytes; 4223 uint32_t length; 4224 enum rte_color color_in, color_out; 4225 4226 TRACE("[Thread %2u] meter (imi)\n", p->thread_id); 4227 4228 m = instr_meter_idx_imm(p, ip); 4229 rte_prefetch0(m->n_pkts); 4230 time = rte_get_tsc_cycles(); 4231 length = instr_meter_length_hbo(t, ip); 4232 color_in = (enum rte_color)ip->meter.color_in_val; 4233 4234 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4235 &m->profile->profile, 4236 time, 4237 length, 4238 color_in); 4239 4240 color_out &= m->color_mask; 4241 4242 n_pkts = m->n_pkts[color_out]; 4243 n_bytes = m->n_bytes[color_out]; 4244 4245 instr_meter_color_out_hbo_set(t, ip, color_out); 4246 4247 m->n_pkts[color_out] = n_pkts + 1; 4248 m->n_bytes[color_out] = n_bytes + length; 4249 } 4250 4251 #endif 4252