1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2021 Intel Corporation 3 */ 4 #ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__ 5 #define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__ 6 7 #include <inttypes.h> 8 #include <string.h> 9 #include <sys/queue.h> 10 11 #include <rte_byteorder.h> 12 #include <rte_common.h> 13 #include <rte_cycles.h> 14 #include <rte_prefetch.h> 15 #include <rte_meter.h> 16 17 #include <rte_swx_table_selector.h> 18 #include <rte_swx_table_learner.h> 19 #include <rte_swx_pipeline.h> 20 #include <rte_swx_ctl.h> 21 22 #ifndef TRACE_LEVEL 23 #define TRACE_LEVEL 0 24 #endif 25 26 #if TRACE_LEVEL 27 #define TRACE(...) printf(__VA_ARGS__) 28 #else 29 #define TRACE(...) 30 #endif 31 32 /* 33 * Environment. 34 */ 35 #define ntoh64(x) rte_be_to_cpu_64(x) 36 #define hton64(x) rte_cpu_to_be_64(x) 37 38 /* 39 * Struct. 40 */ 41 struct field { 42 char name[RTE_SWX_NAME_SIZE]; 43 uint32_t n_bits; 44 uint32_t offset; 45 int var_size; 46 }; 47 48 struct struct_type { 49 TAILQ_ENTRY(struct_type) node; 50 char name[RTE_SWX_NAME_SIZE]; 51 struct field *fields; 52 uint32_t n_fields; 53 uint32_t n_bits; 54 uint32_t n_bits_min; 55 int var_size; 56 }; 57 58 TAILQ_HEAD(struct_type_tailq, struct_type); 59 60 /* 61 * Input port. 62 */ 63 struct port_in_type { 64 TAILQ_ENTRY(port_in_type) node; 65 char name[RTE_SWX_NAME_SIZE]; 66 struct rte_swx_port_in_ops ops; 67 }; 68 69 TAILQ_HEAD(port_in_type_tailq, port_in_type); 70 71 struct port_in { 72 TAILQ_ENTRY(port_in) node; 73 struct port_in_type *type; 74 void *obj; 75 uint32_t id; 76 }; 77 78 TAILQ_HEAD(port_in_tailq, port_in); 79 80 struct port_in_runtime { 81 rte_swx_port_in_pkt_rx_t pkt_rx; 82 void *obj; 83 }; 84 85 /* 86 * Output port. 87 */ 88 struct port_out_type { 89 TAILQ_ENTRY(port_out_type) node; 90 char name[RTE_SWX_NAME_SIZE]; 91 struct rte_swx_port_out_ops ops; 92 }; 93 94 TAILQ_HEAD(port_out_type_tailq, port_out_type); 95 96 struct port_out { 97 TAILQ_ENTRY(port_out) node; 98 struct port_out_type *type; 99 void *obj; 100 uint32_t id; 101 }; 102 103 TAILQ_HEAD(port_out_tailq, port_out); 104 105 struct port_out_runtime { 106 rte_swx_port_out_pkt_tx_t pkt_tx; 107 rte_swx_port_out_pkt_fast_clone_tx_t pkt_fast_clone_tx; 108 rte_swx_port_out_pkt_clone_tx_t pkt_clone_tx; 109 rte_swx_port_out_flush_t flush; 110 void *obj; 111 }; 112 113 /* 114 * Packet mirroring. 115 */ 116 struct mirroring_session { 117 uint32_t port_id; 118 int fast_clone; 119 uint32_t truncation_length; 120 }; 121 122 /* 123 * Extern object. 124 */ 125 struct extern_type_member_func { 126 TAILQ_ENTRY(extern_type_member_func) node; 127 char name[RTE_SWX_NAME_SIZE]; 128 rte_swx_extern_type_member_func_t func; 129 uint32_t id; 130 }; 131 132 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func); 133 134 struct extern_type { 135 TAILQ_ENTRY(extern_type) node; 136 char name[RTE_SWX_NAME_SIZE]; 137 struct struct_type *mailbox_struct_type; 138 rte_swx_extern_type_constructor_t constructor; 139 rte_swx_extern_type_destructor_t destructor; 140 struct extern_type_member_func_tailq funcs; 141 uint32_t n_funcs; 142 }; 143 144 TAILQ_HEAD(extern_type_tailq, extern_type); 145 146 struct extern_obj { 147 TAILQ_ENTRY(extern_obj) node; 148 char name[RTE_SWX_NAME_SIZE]; 149 struct extern_type *type; 150 void *obj; 151 uint32_t struct_id; 152 uint32_t id; 153 }; 154 155 TAILQ_HEAD(extern_obj_tailq, extern_obj); 156 157 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 158 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8 159 #endif 160 161 struct extern_obj_runtime { 162 void *obj; 163 uint8_t *mailbox; 164 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX]; 165 }; 166 167 /* 168 * Extern function. 169 */ 170 struct extern_func { 171 TAILQ_ENTRY(extern_func) node; 172 char name[RTE_SWX_NAME_SIZE]; 173 struct struct_type *mailbox_struct_type; 174 rte_swx_extern_func_t func; 175 uint32_t struct_id; 176 uint32_t id; 177 }; 178 179 TAILQ_HEAD(extern_func_tailq, extern_func); 180 181 struct extern_func_runtime { 182 uint8_t *mailbox; 183 rte_swx_extern_func_t func; 184 }; 185 186 /* 187 * Hash function. 188 */ 189 struct hash_func { 190 TAILQ_ENTRY(hash_func) node; 191 char name[RTE_SWX_NAME_SIZE]; 192 rte_swx_hash_func_t func; 193 uint32_t id; 194 }; 195 196 TAILQ_HEAD(hash_func_tailq, hash_func); 197 198 struct hash_func_runtime { 199 rte_swx_hash_func_t func; 200 }; 201 202 /* 203 * Header. 204 */ 205 struct header { 206 TAILQ_ENTRY(header) node; 207 char name[RTE_SWX_NAME_SIZE]; 208 struct struct_type *st; 209 uint32_t struct_id; 210 uint32_t id; 211 }; 212 213 TAILQ_HEAD(header_tailq, header); 214 215 struct header_runtime { 216 uint8_t *ptr0; 217 uint32_t n_bytes; 218 }; 219 220 struct header_out_runtime { 221 uint8_t *ptr0; 222 uint8_t *ptr; 223 uint32_t n_bytes; 224 }; 225 226 /* 227 * Instruction. 228 */ 229 230 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian. 231 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO). 232 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO 233 * when transferred to packet meta-data and in NBO when transferred to packet 234 * headers. 235 */ 236 237 /* Notation conventions: 238 * -Header field: H = h.header.field (dst/src) 239 * -Meta-data field: M = m.field (dst/src) 240 * -Extern object mailbox field: E = e.field (dst/src) 241 * -Extern function mailbox field: F = f.field (dst/src) 242 * -Table action data field: T = t.field (src only) 243 * -Immediate value: I = 32-bit unsigned value (src only) 244 */ 245 246 enum instruction_type { 247 /* rx m.port_in */ 248 INSTR_RX, 249 250 /* tx port_out 251 * port_out = MI 252 */ 253 INSTR_TX, /* port_out = M */ 254 INSTR_TX_I, /* port_out = I */ 255 INSTR_DROP, 256 257 /* 258 * mirror slot_id session_id 259 * slot_id = MEFT 260 * session_id = MEFT 261 */ 262 INSTR_MIRROR, 263 264 /* recirculate 265 */ 266 INSTR_RECIRCULATE, 267 268 /* recircid m.recirc_pass_id 269 * Read the internal recirculation pass ID into the specified meta-data field. 270 */ 271 INSTR_RECIRCID, 272 273 /* extract h.header */ 274 INSTR_HDR_EXTRACT, 275 INSTR_HDR_EXTRACT2, 276 INSTR_HDR_EXTRACT3, 277 INSTR_HDR_EXTRACT4, 278 INSTR_HDR_EXTRACT5, 279 INSTR_HDR_EXTRACT6, 280 INSTR_HDR_EXTRACT7, 281 INSTR_HDR_EXTRACT8, 282 283 /* extract h.header m.last_field_size */ 284 INSTR_HDR_EXTRACT_M, 285 286 /* lookahead h.header */ 287 INSTR_HDR_LOOKAHEAD, 288 289 /* emit h.header */ 290 INSTR_HDR_EMIT, 291 INSTR_HDR_EMIT_TX, 292 INSTR_HDR_EMIT2_TX, 293 INSTR_HDR_EMIT3_TX, 294 INSTR_HDR_EMIT4_TX, 295 INSTR_HDR_EMIT5_TX, 296 INSTR_HDR_EMIT6_TX, 297 INSTR_HDR_EMIT7_TX, 298 INSTR_HDR_EMIT8_TX, 299 300 /* validate h.header */ 301 INSTR_HDR_VALIDATE, 302 303 /* invalidate h.header */ 304 INSTR_HDR_INVALIDATE, 305 306 /* mov dst src 307 * dst = src 308 * dst = HMEF, src = HMEFTI 309 */ 310 INSTR_MOV, /* dst = MEF, src = MEFT; size(dst) <= 64 bits, size(src) <= 64 bits. */ 311 INSTR_MOV_MH, /* dst = MEF, src = H; size(dst) <= 64 bits, size(src) <= 64 bits. */ 312 INSTR_MOV_HM, /* dst = H, src = MEFT; size(dst) <= 64 bits, size(src) <= 64 bits. */ 313 INSTR_MOV_HH, /* dst = H, src = H; size(dst) <= 64 bits, size(src) <= 64 bits. */ 314 INSTR_MOV_DMA, /* dst = HMEF, src = HMEF; size(dst) = size(src) > 64 bits, NBO format. */ 315 INSTR_MOV_128, /* dst = HMEF, src = HMEF; size(dst) = size(src) = 128 bits, NBO format. */ 316 INSTR_MOV_I, /* dst = HMEF, src = I; size(dst) <= 64 bits. */ 317 318 /* dma h.header t.field 319 * memcpy(h.header, t.field, sizeof(h.header)) 320 */ 321 INSTR_DMA_HT, 322 INSTR_DMA_HT2, 323 INSTR_DMA_HT3, 324 INSTR_DMA_HT4, 325 INSTR_DMA_HT5, 326 INSTR_DMA_HT6, 327 INSTR_DMA_HT7, 328 INSTR_DMA_HT8, 329 330 /* add dst src 331 * dst += src 332 * dst = HMEF, src = HMEFTI 333 */ 334 INSTR_ALU_ADD, /* dst = MEF, src = MEF */ 335 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */ 336 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */ 337 INSTR_ALU_ADD_HH, /* dst = H, src = H */ 338 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */ 339 INSTR_ALU_ADD_HI, /* dst = H, src = I */ 340 341 /* sub dst src 342 * dst -= src 343 * dst = HMEF, src = HMEFTI 344 */ 345 INSTR_ALU_SUB, /* dst = MEF, src = MEF */ 346 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */ 347 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */ 348 INSTR_ALU_SUB_HH, /* dst = H, src = H */ 349 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */ 350 INSTR_ALU_SUB_HI, /* dst = H, src = I */ 351 352 /* ckadd dst src 353 * dst = dst '+ src[0:1] '+ src[2:3] '+ ... 354 * dst = H, src = {H, h.header}, '+ = 1's complement addition operator 355 */ 356 INSTR_ALU_CKADD_FIELD, /* src = H */ 357 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 bytes. */ 358 INSTR_ALU_CKADD_STRUCT, /* src = h.header, with sizeof(header) any 4-byte multiple. */ 359 360 /* cksub dst src 361 * dst = dst '- src 362 * dst = H, src = H, '- = 1's complement subtraction operator 363 */ 364 INSTR_ALU_CKSUB_FIELD, 365 366 /* and dst src 367 * dst &= src 368 * dst = HMEF, src = HMEFTI 369 */ 370 INSTR_ALU_AND, /* dst = MEF, src = MEFT */ 371 INSTR_ALU_AND_MH, /* dst = MEF, src = H */ 372 INSTR_ALU_AND_HM, /* dst = H, src = MEFT */ 373 INSTR_ALU_AND_HH, /* dst = H, src = H */ 374 INSTR_ALU_AND_I, /* dst = HMEF, src = I */ 375 376 /* or dst src 377 * dst |= src 378 * dst = HMEF, src = HMEFTI 379 */ 380 INSTR_ALU_OR, /* dst = MEF, src = MEFT */ 381 INSTR_ALU_OR_MH, /* dst = MEF, src = H */ 382 INSTR_ALU_OR_HM, /* dst = H, src = MEFT */ 383 INSTR_ALU_OR_HH, /* dst = H, src = H */ 384 INSTR_ALU_OR_I, /* dst = HMEF, src = I */ 385 386 /* xor dst src 387 * dst ^= src 388 * dst = HMEF, src = HMEFTI 389 */ 390 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */ 391 INSTR_ALU_XOR_MH, /* dst = MEF, src = H */ 392 INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */ 393 INSTR_ALU_XOR_HH, /* dst = H, src = H */ 394 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */ 395 396 /* shl dst src 397 * dst <<= src 398 * dst = HMEF, src = HMEFTI 399 */ 400 INSTR_ALU_SHL, /* dst = MEF, src = MEF */ 401 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */ 402 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */ 403 INSTR_ALU_SHL_HH, /* dst = H, src = H */ 404 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */ 405 INSTR_ALU_SHL_HI, /* dst = H, src = I */ 406 407 /* shr dst src 408 * dst >>= src 409 * dst = HMEF, src = HMEFTI 410 */ 411 INSTR_ALU_SHR, /* dst = MEF, src = MEF */ 412 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */ 413 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */ 414 INSTR_ALU_SHR_HH, /* dst = H, src = H */ 415 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */ 416 INSTR_ALU_SHR_HI, /* dst = H, src = I */ 417 418 /* regprefetch REGARRAY index 419 * prefetch REGARRAY[index] 420 * index = HMEFTI 421 */ 422 INSTR_REGPREFETCH_RH, /* index = H */ 423 INSTR_REGPREFETCH_RM, /* index = MEFT */ 424 INSTR_REGPREFETCH_RI, /* index = I */ 425 426 /* regrd dst REGARRAY index 427 * dst = REGARRAY[index] 428 * dst = HMEF, index = HMEFTI 429 */ 430 INSTR_REGRD_HRH, /* dst = H, index = H */ 431 INSTR_REGRD_HRM, /* dst = H, index = MEFT */ 432 INSTR_REGRD_HRI, /* dst = H, index = I */ 433 INSTR_REGRD_MRH, /* dst = MEF, index = H */ 434 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */ 435 INSTR_REGRD_MRI, /* dst = MEF, index = I */ 436 437 /* regwr REGARRAY index src 438 * REGARRAY[index] = src 439 * index = HMEFTI, src = HMEFTI 440 */ 441 INSTR_REGWR_RHH, /* index = H, src = H */ 442 INSTR_REGWR_RHM, /* index = H, src = MEFT */ 443 INSTR_REGWR_RHI, /* index = H, src = I */ 444 INSTR_REGWR_RMH, /* index = MEFT, src = H */ 445 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */ 446 INSTR_REGWR_RMI, /* index = MEFT, src = I */ 447 INSTR_REGWR_RIH, /* index = I, src = H */ 448 INSTR_REGWR_RIM, /* index = I, src = MEFT */ 449 INSTR_REGWR_RII, /* index = I, src = I */ 450 451 /* regadd REGARRAY index src 452 * REGARRAY[index] += src 453 * index = HMEFTI, src = HMEFTI 454 */ 455 INSTR_REGADD_RHH, /* index = H, src = H */ 456 INSTR_REGADD_RHM, /* index = H, src = MEFT */ 457 INSTR_REGADD_RHI, /* index = H, src = I */ 458 INSTR_REGADD_RMH, /* index = MEFT, src = H */ 459 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */ 460 INSTR_REGADD_RMI, /* index = MEFT, src = I */ 461 INSTR_REGADD_RIH, /* index = I, src = H */ 462 INSTR_REGADD_RIM, /* index = I, src = MEFT */ 463 INSTR_REGADD_RII, /* index = I, src = I */ 464 465 /* metprefetch METARRAY index 466 * prefetch METARRAY[index] 467 * index = HMEFTI 468 */ 469 INSTR_METPREFETCH_H, /* index = H */ 470 INSTR_METPREFETCH_M, /* index = MEFT */ 471 INSTR_METPREFETCH_I, /* index = I */ 472 473 /* meter METARRAY index length color_in color_out 474 * color_out = meter(METARRAY[index], length, color_in) 475 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF 476 */ 477 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */ 478 INSTR_METER_HHI, /* index = H, length = H, color_in = I */ 479 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */ 480 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */ 481 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */ 482 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */ 483 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */ 484 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */ 485 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */ 486 INSTR_METER_IHI, /* index = I, length = H, color_in = I */ 487 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */ 488 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */ 489 490 /* table TABLE */ 491 INSTR_TABLE, 492 INSTR_TABLE_AF, 493 INSTR_SELECTOR, 494 INSTR_LEARNER, 495 INSTR_LEARNER_AF, 496 497 /* learn ACTION_NAME [ m.action_first_arg ] m.timeout_id */ 498 INSTR_LEARNER_LEARN, 499 500 /* rearm [ m.timeout_id ] */ 501 INSTR_LEARNER_REARM, 502 INSTR_LEARNER_REARM_NEW, 503 504 /* forget */ 505 INSTR_LEARNER_FORGET, 506 507 /* entryid m.table_entry_id 508 * Read the internal table entry ID into the specified meta-data field. 509 */ 510 INSTR_ENTRYID, 511 512 /* extern e.obj.func */ 513 INSTR_EXTERN_OBJ, 514 515 /* extern f.func */ 516 INSTR_EXTERN_FUNC, 517 518 /* hash HASH_FUNC_NAME dst src_first src_last 519 * Compute hash value over range of struct fields. 520 * dst = M 521 * src_first = HMEFT 522 * src_last = HMEFT 523 * src_first and src_last must be fields within the same struct 524 */ 525 INSTR_HASH_FUNC, 526 527 /* jmp LABEL 528 * Unconditional jump 529 */ 530 INSTR_JMP, 531 532 /* jmpv LABEL h.header 533 * Jump if header is valid 534 */ 535 INSTR_JMP_VALID, 536 537 /* jmpnv LABEL h.header 538 * Jump if header is invalid 539 */ 540 INSTR_JMP_INVALID, 541 542 /* jmph LABEL 543 * Jump if table lookup hit 544 */ 545 INSTR_JMP_HIT, 546 547 /* jmpnh LABEL 548 * Jump if table lookup miss 549 */ 550 INSTR_JMP_MISS, 551 552 /* jmpa LABEL ACTION 553 * Jump if action run 554 */ 555 INSTR_JMP_ACTION_HIT, 556 557 /* jmpna LABEL ACTION 558 * Jump if action not run 559 */ 560 INSTR_JMP_ACTION_MISS, 561 562 /* jmpeq LABEL a b 563 * Jump if a is equal to b 564 * a = HMEFT, b = HMEFTI 565 */ 566 INSTR_JMP_EQ, /* a = MEFT, b = MEFT */ 567 INSTR_JMP_EQ_MH, /* a = MEFT, b = H */ 568 INSTR_JMP_EQ_HM, /* a = H, b = MEFT */ 569 INSTR_JMP_EQ_HH, /* a = H, b = H */ 570 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */ 571 572 /* jmpneq LABEL a b 573 * Jump if a is not equal to b 574 * a = HMEFT, b = HMEFTI 575 */ 576 INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */ 577 INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */ 578 INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */ 579 INSTR_JMP_NEQ_HH, /* a = H, b = H */ 580 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */ 581 582 /* jmplt LABEL a b 583 * Jump if a is less than b 584 * a = HMEFT, b = HMEFTI 585 */ 586 INSTR_JMP_LT, /* a = MEFT, b = MEFT */ 587 INSTR_JMP_LT_MH, /* a = MEFT, b = H */ 588 INSTR_JMP_LT_HM, /* a = H, b = MEFT */ 589 INSTR_JMP_LT_HH, /* a = H, b = H */ 590 INSTR_JMP_LT_MI, /* a = MEFT, b = I */ 591 INSTR_JMP_LT_HI, /* a = H, b = I */ 592 593 /* jmpgt LABEL a b 594 * Jump if a is greater than b 595 * a = HMEFT, b = HMEFTI 596 */ 597 INSTR_JMP_GT, /* a = MEFT, b = MEFT */ 598 INSTR_JMP_GT_MH, /* a = MEFT, b = H */ 599 INSTR_JMP_GT_HM, /* a = H, b = MEFT */ 600 INSTR_JMP_GT_HH, /* a = H, b = H */ 601 INSTR_JMP_GT_MI, /* a = MEFT, b = I */ 602 INSTR_JMP_GT_HI, /* a = H, b = I */ 603 604 /* return 605 * Return from action 606 */ 607 INSTR_RETURN, 608 609 /* Start of custom instructions. */ 610 INSTR_CUSTOM_0, 611 }; 612 613 struct instr_operand { 614 uint8_t struct_id; 615 uint8_t n_bits; 616 uint8_t offset; 617 uint8_t pad; 618 }; 619 620 struct instr_io { 621 struct { 622 union { 623 struct { 624 uint8_t offset; 625 uint8_t n_bits; 626 uint8_t pad[2]; 627 }; 628 629 uint32_t val; 630 }; 631 } io; 632 633 struct { 634 uint8_t header_id[8]; 635 uint8_t struct_id[8]; 636 uint8_t n_bytes[8]; 637 } hdr; 638 }; 639 640 struct instr_hdr_validity { 641 uint8_t header_id; 642 uint8_t struct_id; 643 }; 644 645 struct instr_table { 646 uint8_t table_id; 647 }; 648 649 struct instr_learn { 650 uint8_t action_id; 651 uint8_t mf_first_arg_offset; 652 uint8_t mf_timeout_id_offset; 653 uint8_t mf_timeout_id_n_bits; 654 }; 655 656 struct instr_extern_obj { 657 uint8_t ext_obj_id; 658 uint8_t func_id; 659 }; 660 661 struct instr_extern_func { 662 uint8_t ext_func_id; 663 }; 664 665 struct instr_hash_func { 666 uint8_t hash_func_id; 667 668 struct { 669 uint8_t offset; 670 uint8_t n_bits; 671 } dst; 672 673 struct { 674 uint8_t struct_id; 675 uint16_t offset; 676 uint16_t n_bytes; 677 } src; 678 }; 679 680 struct instr_dst_src { 681 struct instr_operand dst; 682 union { 683 struct instr_operand src; 684 uint64_t src_val; 685 }; 686 }; 687 688 struct instr_regarray { 689 uint8_t regarray_id; 690 uint8_t pad[3]; 691 692 union { 693 struct instr_operand idx; 694 uint32_t idx_val; 695 }; 696 697 union { 698 struct instr_operand dstsrc; 699 uint64_t dstsrc_val; 700 }; 701 }; 702 703 struct instr_meter { 704 uint8_t metarray_id; 705 uint8_t pad[3]; 706 707 union { 708 struct instr_operand idx; 709 uint32_t idx_val; 710 }; 711 712 struct instr_operand length; 713 714 union { 715 struct instr_operand color_in; 716 uint32_t color_in_val; 717 }; 718 719 struct instr_operand color_out; 720 }; 721 722 struct instr_dma { 723 struct { 724 uint8_t header_id[8]; 725 uint8_t struct_id[8]; 726 } dst; 727 728 struct { 729 uint8_t offset[8]; 730 } src; 731 732 uint16_t n_bytes[8]; 733 }; 734 735 struct instr_jmp { 736 struct instruction *ip; 737 738 union { 739 struct instr_operand a; 740 uint8_t header_id; 741 uint8_t action_id; 742 }; 743 744 union { 745 struct instr_operand b; 746 uint64_t b_val; 747 }; 748 }; 749 750 struct instruction { 751 enum instruction_type type; 752 union { 753 struct instr_io io; 754 struct instr_dst_src mirror; 755 struct instr_hdr_validity valid; 756 struct instr_dst_src mov; 757 struct instr_regarray regarray; 758 struct instr_meter meter; 759 struct instr_dma dma; 760 struct instr_dst_src alu; 761 struct instr_table table; 762 struct instr_learn learn; 763 struct instr_extern_obj ext_obj; 764 struct instr_extern_func ext_func; 765 struct instr_hash_func hash_func; 766 struct instr_jmp jmp; 767 }; 768 }; 769 770 struct instruction_data { 771 char label[RTE_SWX_NAME_SIZE]; 772 char jmp_label[RTE_SWX_NAME_SIZE]; 773 uint32_t n_users; /* user = jmp instruction to this instruction. */ 774 int invalid; 775 }; 776 777 typedef void (*instr_exec_t)(struct rte_swx_pipeline *); 778 779 /* 780 * Action. 781 */ 782 typedef void 783 (*action_func_t)(struct rte_swx_pipeline *p); 784 785 struct action { 786 TAILQ_ENTRY(action) node; 787 char name[RTE_SWX_NAME_SIZE]; 788 struct struct_type *st; 789 int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */ 790 struct instruction *instructions; 791 struct instruction_data *instruction_data; 792 uint32_t n_instructions; 793 uint32_t id; 794 }; 795 796 TAILQ_HEAD(action_tailq, action); 797 798 /* 799 * Table. 800 */ 801 struct table_type { 802 TAILQ_ENTRY(table_type) node; 803 char name[RTE_SWX_NAME_SIZE]; 804 enum rte_swx_table_match_type match_type; 805 struct rte_swx_table_ops ops; 806 }; 807 808 TAILQ_HEAD(table_type_tailq, table_type); 809 810 struct match_field { 811 enum rte_swx_table_match_type match_type; 812 struct field *field; 813 }; 814 815 struct table { 816 TAILQ_ENTRY(table) node; 817 char name[RTE_SWX_NAME_SIZE]; 818 char args[RTE_SWX_NAME_SIZE]; 819 struct table_type *type; /* NULL when n_fields == 0. */ 820 821 /* Match. */ 822 struct match_field *fields; 823 uint32_t n_fields; 824 struct header *header; /* Only valid when n_fields > 0. */ 825 826 /* Action. */ 827 struct action **actions; 828 struct action *default_action; 829 uint8_t *default_action_data; 830 uint32_t n_actions; 831 int default_action_is_const; 832 uint32_t action_data_size_max; 833 int *action_is_for_table_entries; 834 int *action_is_for_default_entry; 835 836 struct hash_func *hf; 837 uint32_t size; 838 uint32_t id; 839 }; 840 841 TAILQ_HEAD(table_tailq, table); 842 843 struct table_runtime { 844 rte_swx_table_lookup_t func; 845 void *mailbox; 846 uint8_t **key; 847 }; 848 849 struct table_statistics { 850 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */ 851 uint64_t *n_pkts_action; 852 }; 853 854 /* 855 * Selector. 856 */ 857 struct selector { 858 TAILQ_ENTRY(selector) node; 859 char name[RTE_SWX_NAME_SIZE]; 860 861 struct field *group_id_field; 862 struct field **selector_fields; 863 uint32_t n_selector_fields; 864 struct header *selector_header; 865 struct field *member_id_field; 866 867 uint32_t n_groups_max; 868 uint32_t n_members_per_group_max; 869 870 uint32_t id; 871 }; 872 873 TAILQ_HEAD(selector_tailq, selector); 874 875 struct selector_runtime { 876 void *mailbox; 877 uint8_t **group_id_buffer; 878 uint8_t **selector_buffer; 879 uint8_t **member_id_buffer; 880 }; 881 882 struct selector_statistics { 883 uint64_t n_pkts; 884 }; 885 886 /* 887 * Learner table. 888 */ 889 struct learner { 890 TAILQ_ENTRY(learner) node; 891 char name[RTE_SWX_NAME_SIZE]; 892 893 /* Match. */ 894 struct field **fields; 895 uint32_t n_fields; 896 struct header *header; 897 898 /* Action. */ 899 struct action **actions; 900 struct action *default_action; 901 uint8_t *default_action_data; 902 uint32_t n_actions; 903 int default_action_is_const; 904 uint32_t action_data_size_max; 905 int *action_is_for_table_entries; 906 int *action_is_for_default_entry; 907 908 struct hash_func *hf; 909 uint32_t size; 910 uint32_t timeout[RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX]; 911 uint32_t n_timeouts; 912 uint32_t id; 913 }; 914 915 TAILQ_HEAD(learner_tailq, learner); 916 917 struct learner_runtime { 918 void *mailbox; 919 uint8_t **key; 920 }; 921 922 struct learner_statistics { 923 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */ 924 uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */ 925 uint64_t n_pkts_rearm; 926 uint64_t n_pkts_forget; 927 uint64_t *n_pkts_action; 928 }; 929 930 /* 931 * Register array. 932 */ 933 struct regarray { 934 TAILQ_ENTRY(regarray) node; 935 char name[RTE_SWX_NAME_SIZE]; 936 uint64_t init_val; 937 uint32_t size; 938 uint32_t id; 939 }; 940 941 TAILQ_HEAD(regarray_tailq, regarray); 942 943 struct regarray_runtime { 944 uint64_t *regarray; 945 uint32_t size_mask; 946 }; 947 948 /* 949 * Meter array. 950 */ 951 struct meter_profile { 952 TAILQ_ENTRY(meter_profile) node; 953 char name[RTE_SWX_NAME_SIZE]; 954 struct rte_meter_trtcm_params params; 955 struct rte_meter_trtcm_profile profile; 956 uint32_t n_users; 957 }; 958 959 TAILQ_HEAD(meter_profile_tailq, meter_profile); 960 961 struct metarray { 962 TAILQ_ENTRY(metarray) node; 963 char name[RTE_SWX_NAME_SIZE]; 964 uint32_t size; 965 uint32_t id; 966 }; 967 968 TAILQ_HEAD(metarray_tailq, metarray); 969 970 struct meter { 971 struct rte_meter_trtcm m; 972 struct meter_profile *profile; 973 enum rte_color color_mask; 974 uint8_t pad[20]; 975 976 uint64_t n_pkts[RTE_COLORS]; 977 uint64_t n_bytes[RTE_COLORS]; 978 }; 979 980 struct metarray_runtime { 981 struct meter *metarray; 982 uint32_t size_mask; 983 }; 984 985 /* 986 * Pipeline. 987 */ 988 struct thread { 989 /* Packet. */ 990 struct rte_swx_pkt pkt; 991 uint8_t *ptr; 992 uint32_t *mirroring_slots; 993 uint64_t mirroring_slots_mask; 994 int recirculate; 995 uint32_t recirc_pass_id; 996 997 /* Structures. */ 998 uint8_t **structs; 999 1000 /* Packet headers. */ 1001 struct header_runtime *headers; /* Extracted or generated headers. */ 1002 struct header_out_runtime *headers_out; /* Emitted headers. */ 1003 uint8_t *header_storage; 1004 uint8_t *header_out_storage; 1005 uint64_t valid_headers; 1006 uint32_t n_headers_out; 1007 1008 /* Packet meta-data. */ 1009 uint8_t *metadata; 1010 1011 /* Tables. */ 1012 struct table_runtime *tables; 1013 struct selector_runtime *selectors; 1014 struct learner_runtime *learners; 1015 struct rte_swx_table_state *table_state; 1016 uint64_t action_id; 1017 size_t entry_id; 1018 int hit; /* 0 = Miss, 1 = Hit. */ 1019 uint32_t learner_id; 1020 uint64_t time; 1021 1022 /* Extern objects and functions. */ 1023 struct extern_obj_runtime *extern_objs; 1024 struct extern_func_runtime *extern_funcs; 1025 1026 /* Instructions. */ 1027 struct instruction *ip; 1028 struct instruction *ret; 1029 }; 1030 1031 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos))) 1032 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos))) 1033 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos))) 1034 1035 #define HEADER_VALID(thread, header_id) \ 1036 MASK64_BIT_GET((thread)->valid_headers, header_id) 1037 1038 static inline uint64_t 1039 instr_operand_hbo(struct thread *t, const struct instr_operand *x) 1040 { 1041 uint8_t *x_struct = t->structs[x->struct_id]; 1042 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset]; 1043 uint64_t x64 = *x64_ptr; 1044 uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits); 1045 1046 return x64 & x64_mask; 1047 } 1048 1049 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1050 1051 static inline uint64_t 1052 instr_operand_nbo(struct thread *t, const struct instr_operand *x) 1053 { 1054 uint8_t *x_struct = t->structs[x->struct_id]; 1055 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset]; 1056 uint64_t x64 = *x64_ptr; 1057 1058 return ntoh64(x64) >> (64 - x->n_bits); 1059 } 1060 1061 #else 1062 1063 #define instr_operand_nbo instr_operand_hbo 1064 1065 #endif 1066 1067 #define ALU(thread, ip, operator) \ 1068 { \ 1069 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1070 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1071 uint64_t dst64 = *dst64_ptr; \ 1072 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1073 uint64_t dst = dst64 & dst64_mask; \ 1074 \ 1075 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1076 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1077 uint64_t src64 = *src64_ptr; \ 1078 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \ 1079 uint64_t src = src64 & src64_mask; \ 1080 \ 1081 uint64_t result = dst operator src; \ 1082 \ 1083 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \ 1084 } 1085 1086 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1087 1088 #define ALU_MH(thread, ip, operator) \ 1089 { \ 1090 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1091 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1092 uint64_t dst64 = *dst64_ptr; \ 1093 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1094 uint64_t dst = dst64 & dst64_mask; \ 1095 \ 1096 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1097 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1098 uint64_t src64 = *src64_ptr; \ 1099 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \ 1100 \ 1101 uint64_t result = dst operator src; \ 1102 \ 1103 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \ 1104 } 1105 1106 #define ALU_HM(thread, ip, operator) \ 1107 { \ 1108 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1109 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1110 uint64_t dst64 = *dst64_ptr; \ 1111 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1112 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \ 1113 \ 1114 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1115 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1116 uint64_t src64 = *src64_ptr; \ 1117 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \ 1118 uint64_t src = src64 & src64_mask; \ 1119 \ 1120 uint64_t result = dst operator src; \ 1121 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \ 1122 \ 1123 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1124 } 1125 1126 #define ALU_HM_FAST(thread, ip, operator) \ 1127 { \ 1128 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1129 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1130 uint64_t dst64 = *dst64_ptr; \ 1131 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1132 uint64_t dst = dst64 & dst64_mask; \ 1133 \ 1134 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1135 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1136 uint64_t src64 = *src64_ptr; \ 1137 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \ 1138 uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \ 1139 \ 1140 uint64_t result = dst operator src; \ 1141 \ 1142 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1143 } 1144 1145 #define ALU_HH(thread, ip, operator) \ 1146 { \ 1147 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1148 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1149 uint64_t dst64 = *dst64_ptr; \ 1150 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1151 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \ 1152 \ 1153 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1154 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1155 uint64_t src64 = *src64_ptr; \ 1156 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \ 1157 \ 1158 uint64_t result = dst operator src; \ 1159 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \ 1160 \ 1161 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1162 } 1163 1164 #define ALU_HH_FAST(thread, ip, operator) \ 1165 { \ 1166 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1167 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1168 uint64_t dst64 = *dst64_ptr; \ 1169 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1170 uint64_t dst = dst64 & dst64_mask; \ 1171 \ 1172 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1173 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1174 uint64_t src64 = *src64_ptr; \ 1175 uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \ 1176 \ 1177 uint64_t result = dst operator src; \ 1178 \ 1179 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1180 } 1181 1182 #else 1183 1184 #define ALU_MH ALU 1185 #define ALU_HM ALU 1186 #define ALU_HM_FAST ALU 1187 #define ALU_HH ALU 1188 #define ALU_HH_FAST ALU 1189 1190 #endif 1191 1192 #define ALU_I(thread, ip, operator) \ 1193 { \ 1194 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1195 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1196 uint64_t dst64 = *dst64_ptr; \ 1197 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1198 uint64_t dst = dst64 & dst64_mask; \ 1199 \ 1200 uint64_t src = (ip)->alu.src_val; \ 1201 \ 1202 uint64_t result = dst operator src; \ 1203 \ 1204 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \ 1205 } 1206 1207 #define ALU_MI ALU_I 1208 1209 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1210 1211 #define ALU_HI(thread, ip, operator) \ 1212 { \ 1213 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1214 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1215 uint64_t dst64 = *dst64_ptr; \ 1216 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1217 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \ 1218 \ 1219 uint64_t src = (ip)->alu.src_val; \ 1220 \ 1221 uint64_t result = dst operator src; \ 1222 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \ 1223 \ 1224 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1225 } 1226 1227 #else 1228 1229 #define ALU_HI ALU_I 1230 1231 #endif 1232 1233 #define MOV(thread, ip) \ 1234 { \ 1235 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1236 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1237 uint64_t dst64 = *dst64_ptr; \ 1238 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1239 \ 1240 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \ 1241 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \ 1242 uint64_t src64 = *src64_ptr; \ 1243 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \ 1244 uint64_t src = src64 & src64_mask; \ 1245 \ 1246 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \ 1247 } 1248 1249 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1250 1251 #define MOV_MH(thread, ip) \ 1252 { \ 1253 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1254 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1255 uint64_t dst64 = *dst64_ptr; \ 1256 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1257 \ 1258 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \ 1259 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \ 1260 uint64_t src64 = *src64_ptr; \ 1261 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \ 1262 \ 1263 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \ 1264 } 1265 1266 #define MOV_HM(thread, ip) \ 1267 { \ 1268 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1269 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1270 uint64_t dst64 = *dst64_ptr; \ 1271 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1272 \ 1273 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \ 1274 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \ 1275 uint64_t src64 = *src64_ptr; \ 1276 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \ 1277 uint64_t src = src64 & src64_mask; \ 1278 \ 1279 src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \ 1280 *dst64_ptr = (dst64 & ~dst64_mask) | src; \ 1281 } 1282 1283 #define MOV_HH(thread, ip) \ 1284 { \ 1285 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1286 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1287 uint64_t dst64 = *dst64_ptr; \ 1288 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1289 \ 1290 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \ 1291 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \ 1292 uint64_t src64 = *src64_ptr; \ 1293 \ 1294 uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \ 1295 src = src >> (64 - (ip)->mov.dst.n_bits); \ 1296 *dst64_ptr = (dst64 & ~dst64_mask) | src; \ 1297 } 1298 1299 #else 1300 1301 #define MOV_MH MOV 1302 #define MOV_HM MOV 1303 #define MOV_HH MOV 1304 1305 #endif 1306 1307 #define MOV_I(thread, ip) \ 1308 { \ 1309 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1310 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1311 uint64_t dst64 = *dst64_ptr; \ 1312 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1313 \ 1314 uint64_t src = (ip)->mov.src_val; \ 1315 \ 1316 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \ 1317 } 1318 1319 #define JMP_CMP(thread, ip, operator) \ 1320 { \ 1321 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1322 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1323 uint64_t a64 = *a64_ptr; \ 1324 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \ 1325 uint64_t a = a64 & a64_mask; \ 1326 \ 1327 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1328 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1329 uint64_t b64 = *b64_ptr; \ 1330 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \ 1331 uint64_t b = b64 & b64_mask; \ 1332 \ 1333 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1334 } 1335 1336 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1337 1338 #define JMP_CMP_MH(thread, ip, operator) \ 1339 { \ 1340 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1341 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1342 uint64_t a64 = *a64_ptr; \ 1343 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \ 1344 uint64_t a = a64 & a64_mask; \ 1345 \ 1346 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1347 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1348 uint64_t b64 = *b64_ptr; \ 1349 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \ 1350 \ 1351 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1352 } 1353 1354 #define JMP_CMP_HM(thread, ip, operator) \ 1355 { \ 1356 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1357 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1358 uint64_t a64 = *a64_ptr; \ 1359 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \ 1360 \ 1361 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1362 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1363 uint64_t b64 = *b64_ptr; \ 1364 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \ 1365 uint64_t b = b64 & b64_mask; \ 1366 \ 1367 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1368 } 1369 1370 #define JMP_CMP_HH(thread, ip, operator) \ 1371 { \ 1372 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1373 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1374 uint64_t a64 = *a64_ptr; \ 1375 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \ 1376 \ 1377 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1378 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1379 uint64_t b64 = *b64_ptr; \ 1380 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \ 1381 \ 1382 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1383 } 1384 1385 #define JMP_CMP_HH_FAST(thread, ip, operator) \ 1386 { \ 1387 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1388 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1389 uint64_t a64 = *a64_ptr; \ 1390 uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \ 1391 \ 1392 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1393 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1394 uint64_t b64 = *b64_ptr; \ 1395 uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \ 1396 \ 1397 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1398 } 1399 1400 #else 1401 1402 #define JMP_CMP_MH JMP_CMP 1403 #define JMP_CMP_HM JMP_CMP 1404 #define JMP_CMP_HH JMP_CMP 1405 #define JMP_CMP_HH_FAST JMP_CMP 1406 1407 #endif 1408 1409 #define JMP_CMP_I(thread, ip, operator) \ 1410 { \ 1411 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1412 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1413 uint64_t a64 = *a64_ptr; \ 1414 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \ 1415 uint64_t a = a64 & a64_mask; \ 1416 \ 1417 uint64_t b = (ip)->jmp.b_val; \ 1418 \ 1419 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1420 } 1421 1422 #define JMP_CMP_MI JMP_CMP_I 1423 1424 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1425 1426 #define JMP_CMP_HI(thread, ip, operator) \ 1427 { \ 1428 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1429 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1430 uint64_t a64 = *a64_ptr; \ 1431 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \ 1432 \ 1433 uint64_t b = (ip)->jmp.b_val; \ 1434 \ 1435 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1436 } 1437 1438 #else 1439 1440 #define JMP_CMP_HI JMP_CMP_I 1441 1442 #endif 1443 1444 #define METADATA_READ(thread, offset, n_bits) \ 1445 ({ \ 1446 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \ 1447 uint64_t m64 = *m64_ptr; \ 1448 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \ 1449 (m64 & m64_mask); \ 1450 }) 1451 1452 #define METADATA_WRITE(thread, offset, n_bits, value) \ 1453 { \ 1454 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \ 1455 uint64_t m64 = *m64_ptr; \ 1456 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \ 1457 \ 1458 uint64_t m_new = value; \ 1459 \ 1460 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \ 1461 } 1462 1463 #ifndef RTE_SWX_PIPELINE_THREADS_MAX 1464 #define RTE_SWX_PIPELINE_THREADS_MAX 16 1465 #endif 1466 1467 #ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 1468 #define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 1024 1469 #endif 1470 1471 struct rte_swx_pipeline { 1472 char name[RTE_SWX_NAME_SIZE]; 1473 1474 struct struct_type_tailq struct_types; 1475 struct port_in_type_tailq port_in_types; 1476 struct port_in_tailq ports_in; 1477 struct port_out_type_tailq port_out_types; 1478 struct port_out_tailq ports_out; 1479 struct extern_type_tailq extern_types; 1480 struct extern_obj_tailq extern_objs; 1481 struct extern_func_tailq extern_funcs; 1482 struct hash_func_tailq hash_funcs; 1483 struct header_tailq headers; 1484 struct struct_type *metadata_st; 1485 uint32_t metadata_struct_id; 1486 struct action_tailq actions; 1487 struct table_type_tailq table_types; 1488 struct table_tailq tables; 1489 struct selector_tailq selectors; 1490 struct learner_tailq learners; 1491 struct regarray_tailq regarrays; 1492 struct meter_profile_tailq meter_profiles; 1493 struct metarray_tailq metarrays; 1494 1495 struct port_in_runtime *in; 1496 struct port_out_runtime *out; 1497 struct mirroring_session *mirroring_sessions; 1498 struct instruction **action_instructions; 1499 action_func_t *action_funcs; 1500 struct rte_swx_table_state *table_state; 1501 struct table_statistics *table_stats; 1502 struct selector_statistics *selector_stats; 1503 struct learner_statistics *learner_stats; 1504 struct hash_func_runtime *hash_func_runtime; 1505 struct regarray_runtime *regarray_runtime; 1506 struct metarray_runtime *metarray_runtime; 1507 struct instruction *instructions; 1508 struct instruction_data *instruction_data; 1509 instr_exec_t *instruction_table; 1510 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX]; 1511 void *lib; 1512 1513 uint32_t n_structs; 1514 uint32_t n_ports_in; 1515 uint32_t n_ports_out; 1516 uint32_t n_mirroring_slots; 1517 uint32_t n_mirroring_sessions; 1518 uint32_t n_extern_objs; 1519 uint32_t n_extern_funcs; 1520 uint32_t n_hash_funcs; 1521 uint32_t n_actions; 1522 uint32_t n_tables; 1523 uint32_t n_selectors; 1524 uint32_t n_learners; 1525 uint32_t n_regarrays; 1526 uint32_t n_metarrays; 1527 uint32_t n_headers; 1528 uint32_t thread_id; 1529 uint32_t port_id; 1530 uint32_t n_instructions; 1531 int build_done; 1532 int numa_node; 1533 }; 1534 1535 /* 1536 * Instruction. 1537 */ 1538 static inline void 1539 pipeline_port_inc(struct rte_swx_pipeline *p) 1540 { 1541 p->port_id = (p->port_id + 1) & (p->n_ports_in - 1); 1542 } 1543 1544 static inline void 1545 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t) 1546 { 1547 t->ip = p->instructions; 1548 } 1549 1550 static inline void 1551 thread_ip_set(struct thread *t, struct instruction *ip) 1552 { 1553 t->ip = ip; 1554 } 1555 1556 static inline void 1557 thread_ip_action_call(struct rte_swx_pipeline *p, 1558 struct thread *t, 1559 uint32_t action_id) 1560 { 1561 t->ret = t->ip + 1; 1562 t->ip = p->action_instructions[action_id]; 1563 } 1564 1565 static inline void 1566 thread_ip_inc(struct rte_swx_pipeline *p); 1567 1568 static inline void 1569 thread_ip_inc(struct rte_swx_pipeline *p) 1570 { 1571 struct thread *t = &p->threads[p->thread_id]; 1572 1573 t->ip++; 1574 } 1575 1576 static inline void 1577 thread_ip_inc_cond(struct thread *t, int cond) 1578 { 1579 t->ip += cond; 1580 } 1581 1582 static inline void 1583 thread_yield(struct rte_swx_pipeline *p) 1584 { 1585 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1); 1586 } 1587 1588 static inline void 1589 thread_yield_cond(struct rte_swx_pipeline *p, int cond) 1590 { 1591 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1); 1592 } 1593 1594 /* 1595 * rx. 1596 */ 1597 static inline int 1598 __instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 1599 { 1600 struct port_in_runtime *port = &p->in[p->port_id]; 1601 struct rte_swx_pkt *pkt = &t->pkt; 1602 int pkt_received; 1603 1604 /* Recirculation: keep the current packet. */ 1605 if (t->recirculate) { 1606 TRACE("[Thread %2u] rx - recirculate (pass %u)\n", 1607 p->thread_id, 1608 t->recirc_pass_id + 1); 1609 1610 /* Packet. */ 1611 t->ptr = &pkt->pkt[pkt->offset]; 1612 t->mirroring_slots_mask = 0; 1613 t->recirculate = 0; 1614 t->recirc_pass_id++; 1615 1616 /* Headers. */ 1617 t->valid_headers = 0; 1618 t->n_headers_out = 0; 1619 1620 /* Tables. */ 1621 t->table_state = p->table_state; 1622 1623 return 1; 1624 } 1625 1626 /* Packet. */ 1627 pkt_received = port->pkt_rx(port->obj, pkt); 1628 t->ptr = &pkt->pkt[pkt->offset]; 1629 rte_prefetch0(t->ptr); 1630 1631 TRACE("[Thread %2u] rx %s from port %u\n", 1632 p->thread_id, 1633 pkt_received ? "1 pkt" : "0 pkts", 1634 p->port_id); 1635 1636 t->mirroring_slots_mask = 0; 1637 t->recirc_pass_id = 0; 1638 1639 /* Headers. */ 1640 t->valid_headers = 0; 1641 t->n_headers_out = 0; 1642 1643 /* Meta-data. */ 1644 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id); 1645 1646 /* Tables. */ 1647 t->table_state = p->table_state; 1648 1649 /* Thread. */ 1650 pipeline_port_inc(p); 1651 1652 return pkt_received; 1653 } 1654 1655 static inline void 1656 instr_rx_exec(struct rte_swx_pipeline *p) 1657 { 1658 struct thread *t = &p->threads[p->thread_id]; 1659 struct instruction *ip = t->ip; 1660 int pkt_received; 1661 1662 /* Packet. */ 1663 pkt_received = __instr_rx_exec(p, t, ip); 1664 1665 /* Thread. */ 1666 thread_ip_inc_cond(t, pkt_received); 1667 thread_yield(p); 1668 } 1669 1670 /* 1671 * tx. 1672 */ 1673 static inline void 1674 emit_handler(struct thread *t) 1675 { 1676 struct header_out_runtime *h0 = &t->headers_out[0]; 1677 struct header_out_runtime *h1 = &t->headers_out[1]; 1678 uint32_t offset = 0, i; 1679 1680 /* No header change or header decapsulation. */ 1681 if ((t->n_headers_out == 1) && 1682 (h0->ptr + h0->n_bytes == t->ptr)) { 1683 TRACE("Emit handler: no header change or header decap.\n"); 1684 1685 t->pkt.offset -= h0->n_bytes; 1686 t->pkt.length += h0->n_bytes; 1687 1688 return; 1689 } 1690 1691 /* Header encapsulation (optionally, with prior header decapsulation). */ 1692 if ((t->n_headers_out == 2) && 1693 (h1->ptr + h1->n_bytes == t->ptr) && 1694 (h0->ptr == h0->ptr0)) { 1695 uint32_t offset; 1696 1697 TRACE("Emit handler: header encapsulation.\n"); 1698 1699 offset = h0->n_bytes + h1->n_bytes; 1700 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes); 1701 t->pkt.offset -= offset; 1702 t->pkt.length += offset; 1703 1704 return; 1705 } 1706 1707 /* For any other case. */ 1708 TRACE("Emit handler: complex case.\n"); 1709 1710 for (i = 0; i < t->n_headers_out; i++) { 1711 struct header_out_runtime *h = &t->headers_out[i]; 1712 1713 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes); 1714 offset += h->n_bytes; 1715 } 1716 1717 if (offset) { 1718 memcpy(t->ptr - offset, t->header_out_storage, offset); 1719 t->pkt.offset -= offset; 1720 t->pkt.length += offset; 1721 } 1722 } 1723 1724 static inline void 1725 mirroring_handler(struct rte_swx_pipeline *p, struct thread *t, struct rte_swx_pkt *pkt) 1726 { 1727 uint64_t slots_mask = t->mirroring_slots_mask, slot_mask; 1728 uint32_t slot_id; 1729 1730 for (slot_id = 0, slot_mask = 1LLU ; slots_mask; slot_id++, slot_mask <<= 1) 1731 if (slot_mask & slots_mask) { 1732 struct port_out_runtime *port; 1733 struct mirroring_session *session; 1734 uint32_t port_id, session_id; 1735 1736 session_id = t->mirroring_slots[slot_id]; 1737 session = &p->mirroring_sessions[session_id]; 1738 1739 port_id = session->port_id; 1740 port = &p->out[port_id]; 1741 1742 if (session->fast_clone) 1743 port->pkt_fast_clone_tx(port->obj, pkt); 1744 else 1745 port->pkt_clone_tx(port->obj, pkt, session->truncation_length); 1746 1747 slots_mask &= ~slot_mask; 1748 } 1749 } 1750 1751 static inline void 1752 __instr_tx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 1753 { 1754 uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits); 1755 struct port_out_runtime *port = &p->out[port_id]; 1756 struct rte_swx_pkt *pkt = &t->pkt; 1757 1758 /* Recirculation: keep the current packet. */ 1759 if (t->recirculate) { 1760 TRACE("[Thread %2u]: tx 1 pkt - recirculate\n", 1761 p->thread_id); 1762 1763 /* Headers. */ 1764 emit_handler(t); 1765 1766 /* Packet. */ 1767 mirroring_handler(p, t, pkt); 1768 1769 return; 1770 } 1771 1772 TRACE("[Thread %2u]: tx 1 pkt to port %u\n", 1773 p->thread_id, 1774 (uint32_t)port_id); 1775 1776 /* Headers. */ 1777 emit_handler(t); 1778 1779 /* Packet. */ 1780 mirroring_handler(p, t, pkt); 1781 port->pkt_tx(port->obj, pkt); 1782 } 1783 1784 static inline void 1785 __instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 1786 { 1787 uint64_t port_id = ip->io.io.val; 1788 struct port_out_runtime *port = &p->out[port_id]; 1789 struct rte_swx_pkt *pkt = &t->pkt; 1790 1791 /* Recirculation: keep the current packet. */ 1792 if (t->recirculate) { 1793 TRACE("[Thread %2u]: tx (i) 1 pkt - recirculate\n", 1794 p->thread_id); 1795 1796 /* Headers. */ 1797 emit_handler(t); 1798 1799 /* Packet. */ 1800 mirroring_handler(p, t, pkt); 1801 1802 return; 1803 } 1804 1805 TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n", 1806 p->thread_id, 1807 (uint32_t)port_id); 1808 1809 /* Headers. */ 1810 emit_handler(t); 1811 1812 /* Packet. */ 1813 mirroring_handler(p, t, pkt); 1814 port->pkt_tx(port->obj, pkt); 1815 } 1816 1817 static inline void 1818 __instr_drop_exec(struct rte_swx_pipeline *p, 1819 struct thread *t, 1820 const struct instruction *ip __rte_unused) 1821 { 1822 uint64_t port_id = p->n_ports_out - 1; 1823 struct port_out_runtime *port = &p->out[port_id]; 1824 struct rte_swx_pkt *pkt = &t->pkt; 1825 1826 TRACE("[Thread %2u]: drop 1 pkt\n", 1827 p->thread_id); 1828 1829 /* Headers. */ 1830 emit_handler(t); 1831 1832 /* Packet. */ 1833 mirroring_handler(p, t, pkt); 1834 port->pkt_tx(port->obj, pkt); 1835 } 1836 1837 static inline void 1838 __instr_mirror_exec(struct rte_swx_pipeline *p, 1839 struct thread *t, 1840 const struct instruction *ip) 1841 { 1842 uint64_t slot_id = instr_operand_hbo(t, &ip->mirror.dst); 1843 uint64_t session_id = instr_operand_hbo(t, &ip->mirror.src); 1844 1845 slot_id &= p->n_mirroring_slots - 1; 1846 session_id &= p->n_mirroring_sessions - 1; 1847 1848 TRACE("[Thread %2u]: mirror pkt (slot = %u, session = %u)\n", 1849 p->thread_id, 1850 (uint32_t)slot_id, 1851 (uint32_t)session_id); 1852 1853 t->mirroring_slots[slot_id] = session_id; 1854 t->mirroring_slots_mask |= 1LLU << slot_id; 1855 } 1856 1857 static inline void 1858 __instr_recirculate_exec(struct rte_swx_pipeline *p __rte_unused, 1859 struct thread *t, 1860 const struct instruction *ip __rte_unused) 1861 { 1862 TRACE("[Thread %2u]: recirculate\n", 1863 p->thread_id); 1864 1865 t->recirculate = 1; 1866 } 1867 1868 static inline void 1869 __instr_recircid_exec(struct rte_swx_pipeline *p __rte_unused, 1870 struct thread *t, 1871 const struct instruction *ip) 1872 { 1873 TRACE("[Thread %2u]: recircid (pass %u)\n", 1874 p->thread_id, 1875 t->recirc_pass_id); 1876 1877 /* Meta-data. */ 1878 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, t->recirc_pass_id); 1879 } 1880 1881 /* 1882 * extract. 1883 */ 1884 static inline void 1885 __instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused, 1886 struct thread *t, 1887 const struct instruction *ip, 1888 uint32_t n_extract) 1889 { 1890 uint64_t valid_headers = t->valid_headers; 1891 uint8_t *ptr = t->ptr; 1892 uint32_t offset = t->pkt.offset; 1893 uint32_t length = t->pkt.length; 1894 uint32_t i; 1895 1896 for (i = 0; i < n_extract; i++) { 1897 uint32_t header_id = ip->io.hdr.header_id[i]; 1898 uint32_t struct_id = ip->io.hdr.struct_id[i]; 1899 uint32_t n_bytes = ip->io.hdr.n_bytes[i]; 1900 1901 TRACE("[Thread %2u]: extract header %u (%u bytes)\n", 1902 p->thread_id, 1903 header_id, 1904 n_bytes); 1905 1906 /* Headers. */ 1907 t->structs[struct_id] = ptr; 1908 valid_headers = MASK64_BIT_SET(valid_headers, header_id); 1909 1910 /* Packet. */ 1911 offset += n_bytes; 1912 length -= n_bytes; 1913 ptr += n_bytes; 1914 } 1915 1916 /* Headers. */ 1917 t->valid_headers = valid_headers; 1918 1919 /* Packet. */ 1920 t->pkt.offset = offset; 1921 t->pkt.length = length; 1922 t->ptr = ptr; 1923 } 1924 1925 static inline void 1926 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, 1927 struct thread *t, 1928 const struct instruction *ip) 1929 { 1930 __instr_hdr_extract_many_exec(p, t, ip, 1); 1931 } 1932 1933 static inline void 1934 __instr_hdr_extract2_exec(struct rte_swx_pipeline *p, 1935 struct thread *t, 1936 const struct instruction *ip) 1937 { 1938 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id); 1939 1940 __instr_hdr_extract_many_exec(p, t, ip, 2); 1941 } 1942 1943 static inline void 1944 __instr_hdr_extract3_exec(struct rte_swx_pipeline *p, 1945 struct thread *t, 1946 const struct instruction *ip) 1947 { 1948 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id); 1949 1950 __instr_hdr_extract_many_exec(p, t, ip, 3); 1951 } 1952 1953 static inline void 1954 __instr_hdr_extract4_exec(struct rte_swx_pipeline *p, 1955 struct thread *t, 1956 const struct instruction *ip) 1957 { 1958 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id); 1959 1960 __instr_hdr_extract_many_exec(p, t, ip, 4); 1961 } 1962 1963 static inline void 1964 __instr_hdr_extract5_exec(struct rte_swx_pipeline *p, 1965 struct thread *t, 1966 const struct instruction *ip) 1967 { 1968 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id); 1969 1970 __instr_hdr_extract_many_exec(p, t, ip, 5); 1971 } 1972 1973 static inline void 1974 __instr_hdr_extract6_exec(struct rte_swx_pipeline *p, 1975 struct thread *t, 1976 const struct instruction *ip) 1977 { 1978 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id); 1979 1980 __instr_hdr_extract_many_exec(p, t, ip, 6); 1981 } 1982 1983 static inline void 1984 __instr_hdr_extract7_exec(struct rte_swx_pipeline *p, 1985 struct thread *t, 1986 const struct instruction *ip) 1987 { 1988 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id); 1989 1990 __instr_hdr_extract_many_exec(p, t, ip, 7); 1991 } 1992 1993 static inline void 1994 __instr_hdr_extract8_exec(struct rte_swx_pipeline *p, 1995 struct thread *t, 1996 const struct instruction *ip) 1997 { 1998 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id); 1999 2000 __instr_hdr_extract_many_exec(p, t, ip, 8); 2001 } 2002 2003 static inline void 2004 __instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused, 2005 struct thread *t, 2006 const struct instruction *ip) 2007 { 2008 uint64_t valid_headers = t->valid_headers; 2009 uint8_t *ptr = t->ptr; 2010 uint32_t offset = t->pkt.offset; 2011 uint32_t length = t->pkt.length; 2012 2013 uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits); 2014 uint32_t header_id = ip->io.hdr.header_id[0]; 2015 uint32_t struct_id = ip->io.hdr.struct_id[0]; 2016 uint32_t n_bytes = ip->io.hdr.n_bytes[0]; 2017 2018 struct header_runtime *h = &t->headers[header_id]; 2019 2020 TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n", 2021 p->thread_id, 2022 header_id, 2023 n_bytes, 2024 n_bytes_last); 2025 2026 n_bytes += n_bytes_last; 2027 2028 /* Headers. */ 2029 t->structs[struct_id] = ptr; 2030 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id); 2031 h->n_bytes = n_bytes; 2032 2033 /* Packet. */ 2034 t->pkt.offset = offset + n_bytes; 2035 t->pkt.length = length - n_bytes; 2036 t->ptr = ptr + n_bytes; 2037 } 2038 2039 static inline void 2040 __instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused, 2041 struct thread *t, 2042 const struct instruction *ip) 2043 { 2044 uint64_t valid_headers = t->valid_headers; 2045 uint8_t *ptr = t->ptr; 2046 2047 uint32_t header_id = ip->io.hdr.header_id[0]; 2048 uint32_t struct_id = ip->io.hdr.struct_id[0]; 2049 2050 TRACE("[Thread %2u]: lookahead header %u\n", 2051 p->thread_id, 2052 header_id); 2053 2054 /* Headers. */ 2055 t->structs[struct_id] = ptr; 2056 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id); 2057 } 2058 2059 /* 2060 * emit. 2061 */ 2062 static inline void 2063 __instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused, 2064 struct thread *t, 2065 const struct instruction *ip, 2066 uint32_t n_emit) 2067 { 2068 uint64_t valid_headers = t->valid_headers; 2069 uint32_t n_headers_out = t->n_headers_out; 2070 struct header_out_runtime *ho = NULL; 2071 uint8_t *ho_ptr = NULL; 2072 uint32_t ho_nbytes = 0, i; 2073 2074 for (i = 0; i < n_emit; i++) { 2075 uint32_t header_id = ip->io.hdr.header_id[i]; 2076 uint32_t struct_id = ip->io.hdr.struct_id[i]; 2077 2078 struct header_runtime *hi = &t->headers[header_id]; 2079 uint8_t *hi_ptr0 = hi->ptr0; 2080 uint32_t n_bytes = hi->n_bytes; 2081 2082 uint8_t *hi_ptr = t->structs[struct_id]; 2083 2084 if (!MASK64_BIT_GET(valid_headers, header_id)) { 2085 TRACE("[Thread %2u]: emit header %u (invalid)\n", 2086 p->thread_id, 2087 header_id); 2088 2089 continue; 2090 } 2091 2092 TRACE("[Thread %2u]: emit header %u (valid)\n", 2093 p->thread_id, 2094 header_id); 2095 2096 /* Headers. */ 2097 if (!ho) { 2098 if (!n_headers_out) { 2099 ho = &t->headers_out[0]; 2100 2101 ho->ptr0 = hi_ptr0; 2102 ho->ptr = hi_ptr; 2103 2104 ho_ptr = hi_ptr; 2105 ho_nbytes = n_bytes; 2106 2107 n_headers_out = 1; 2108 2109 continue; 2110 } else { 2111 ho = &t->headers_out[n_headers_out - 1]; 2112 2113 ho_ptr = ho->ptr; 2114 ho_nbytes = ho->n_bytes; 2115 } 2116 } 2117 2118 if (ho_ptr + ho_nbytes == hi_ptr) { 2119 ho_nbytes += n_bytes; 2120 } else { 2121 ho->n_bytes = ho_nbytes; 2122 2123 ho++; 2124 ho->ptr0 = hi_ptr0; 2125 ho->ptr = hi_ptr; 2126 2127 ho_ptr = hi_ptr; 2128 ho_nbytes = n_bytes; 2129 2130 n_headers_out++; 2131 } 2132 } 2133 2134 if (ho) 2135 ho->n_bytes = ho_nbytes; 2136 t->n_headers_out = n_headers_out; 2137 } 2138 2139 static inline void 2140 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, 2141 struct thread *t, 2142 const struct instruction *ip) 2143 { 2144 __instr_hdr_emit_many_exec(p, t, ip, 1); 2145 } 2146 2147 static inline void 2148 __instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p, 2149 struct thread *t, 2150 const struct instruction *ip) 2151 { 2152 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id); 2153 2154 __instr_hdr_emit_many_exec(p, t, ip, 1); 2155 __instr_tx_exec(p, t, ip); 2156 } 2157 2158 static inline void 2159 __instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p, 2160 struct thread *t, 2161 const struct instruction *ip) 2162 { 2163 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id); 2164 2165 __instr_hdr_emit_many_exec(p, t, ip, 2); 2166 __instr_tx_exec(p, t, ip); 2167 } 2168 2169 static inline void 2170 __instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p, 2171 struct thread *t, 2172 const struct instruction *ip) 2173 { 2174 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id); 2175 2176 __instr_hdr_emit_many_exec(p, t, ip, 3); 2177 __instr_tx_exec(p, t, ip); 2178 } 2179 2180 static inline void 2181 __instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p, 2182 struct thread *t, 2183 const struct instruction *ip) 2184 { 2185 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id); 2186 2187 __instr_hdr_emit_many_exec(p, t, ip, 4); 2188 __instr_tx_exec(p, t, ip); 2189 } 2190 2191 static inline void 2192 __instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p, 2193 struct thread *t, 2194 const struct instruction *ip) 2195 { 2196 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id); 2197 2198 __instr_hdr_emit_many_exec(p, t, ip, 5); 2199 __instr_tx_exec(p, t, ip); 2200 } 2201 2202 static inline void 2203 __instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p, 2204 struct thread *t, 2205 const struct instruction *ip) 2206 { 2207 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id); 2208 2209 __instr_hdr_emit_many_exec(p, t, ip, 6); 2210 __instr_tx_exec(p, t, ip); 2211 } 2212 2213 static inline void 2214 __instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p, 2215 struct thread *t, 2216 const struct instruction *ip) 2217 { 2218 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id); 2219 2220 __instr_hdr_emit_many_exec(p, t, ip, 7); 2221 __instr_tx_exec(p, t, ip); 2222 } 2223 2224 static inline void 2225 __instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p, 2226 struct thread *t, 2227 const struct instruction *ip) 2228 { 2229 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id); 2230 2231 __instr_hdr_emit_many_exec(p, t, ip, 8); 2232 __instr_tx_exec(p, t, ip); 2233 } 2234 2235 /* 2236 * validate. 2237 */ 2238 static inline void 2239 __instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused, 2240 struct thread *t, 2241 const struct instruction *ip) 2242 { 2243 uint32_t header_id = ip->valid.header_id; 2244 uint32_t struct_id = ip->valid.struct_id; 2245 uint64_t valid_headers = t->valid_headers; 2246 struct header_runtime *h = &t->headers[header_id]; 2247 2248 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id); 2249 2250 /* If this header is already valid, then its associated t->structs[] element is also valid 2251 * and therefore it should not be modified. It could point to the packet buffer (in case of 2252 * extracted header) and setting it to the default location (h->ptr0) would be incorrect. 2253 */ 2254 if (MASK64_BIT_GET(valid_headers, header_id)) 2255 return; 2256 2257 /* Headers. */ 2258 t->structs[struct_id] = h->ptr0; 2259 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id); 2260 } 2261 2262 /* 2263 * invalidate. 2264 */ 2265 static inline void 2266 __instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused, 2267 struct thread *t, 2268 const struct instruction *ip) 2269 { 2270 uint32_t header_id = ip->valid.header_id; 2271 2272 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id); 2273 2274 /* Headers. */ 2275 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id); 2276 } 2277 2278 /* 2279 * learn. 2280 */ 2281 static inline void 2282 __instr_learn_exec(struct rte_swx_pipeline *p, 2283 struct thread *t, 2284 const struct instruction *ip) 2285 { 2286 uint64_t action_id = ip->learn.action_id; 2287 uint32_t mf_first_arg_offset = ip->learn.mf_first_arg_offset; 2288 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset, 2289 ip->learn.mf_timeout_id_n_bits); 2290 uint32_t learner_id = t->learner_id; 2291 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + 2292 p->n_selectors + learner_id]; 2293 struct learner_runtime *l = &t->learners[learner_id]; 2294 struct learner_statistics *stats = &p->learner_stats[learner_id]; 2295 uint32_t status; 2296 2297 /* Table. */ 2298 status = rte_swx_table_learner_add(ts->obj, 2299 l->mailbox, 2300 t->time, 2301 action_id, 2302 &t->metadata[mf_first_arg_offset], 2303 timeout_id); 2304 2305 TRACE("[Thread %2u] learner %u learn %s\n", 2306 p->thread_id, 2307 learner_id, 2308 status ? "ok" : "error"); 2309 2310 stats->n_pkts_learn[status] += 1; 2311 } 2312 2313 /* 2314 * rearm. 2315 */ 2316 static inline void 2317 __instr_rearm_exec(struct rte_swx_pipeline *p, 2318 struct thread *t, 2319 const struct instruction *ip __rte_unused) 2320 { 2321 uint32_t learner_id = t->learner_id; 2322 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + 2323 p->n_selectors + learner_id]; 2324 struct learner_runtime *l = &t->learners[learner_id]; 2325 struct learner_statistics *stats = &p->learner_stats[learner_id]; 2326 2327 /* Table. */ 2328 rte_swx_table_learner_rearm(ts->obj, l->mailbox, t->time); 2329 2330 TRACE("[Thread %2u] learner %u rearm\n", 2331 p->thread_id, 2332 learner_id); 2333 2334 stats->n_pkts_rearm += 1; 2335 } 2336 2337 static inline void 2338 __instr_rearm_new_exec(struct rte_swx_pipeline *p, 2339 struct thread *t, 2340 const struct instruction *ip) 2341 { 2342 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset, 2343 ip->learn.mf_timeout_id_n_bits); 2344 uint32_t learner_id = t->learner_id; 2345 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + 2346 p->n_selectors + learner_id]; 2347 struct learner_runtime *l = &t->learners[learner_id]; 2348 struct learner_statistics *stats = &p->learner_stats[learner_id]; 2349 2350 /* Table. */ 2351 rte_swx_table_learner_rearm_new(ts->obj, l->mailbox, t->time, timeout_id); 2352 2353 TRACE("[Thread %2u] learner %u rearm with timeout ID %u\n", 2354 p->thread_id, 2355 learner_id, 2356 timeout_id); 2357 2358 stats->n_pkts_rearm += 1; 2359 } 2360 2361 /* 2362 * forget. 2363 */ 2364 static inline void 2365 __instr_forget_exec(struct rte_swx_pipeline *p, 2366 struct thread *t, 2367 const struct instruction *ip __rte_unused) 2368 { 2369 uint32_t learner_id = t->learner_id; 2370 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + 2371 p->n_selectors + learner_id]; 2372 struct learner_runtime *l = &t->learners[learner_id]; 2373 struct learner_statistics *stats = &p->learner_stats[learner_id]; 2374 2375 /* Table. */ 2376 rte_swx_table_learner_delete(ts->obj, l->mailbox); 2377 2378 TRACE("[Thread %2u] learner %u forget\n", 2379 p->thread_id, 2380 learner_id); 2381 2382 stats->n_pkts_forget += 1; 2383 } 2384 2385 /* 2386 * entryid. 2387 */ 2388 static inline void 2389 __instr_entryid_exec(struct rte_swx_pipeline *p __rte_unused, 2390 struct thread *t, 2391 const struct instruction *ip) 2392 { 2393 TRACE("[Thread %2u]: entryid\n", 2394 p->thread_id); 2395 2396 /* Meta-data. */ 2397 METADATA_WRITE(t, ip->mov.dst.offset, ip->mov.dst.n_bits, t->entry_id); 2398 } 2399 2400 /* 2401 * extern. 2402 */ 2403 static inline uint32_t 2404 __instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused, 2405 struct thread *t, 2406 const struct instruction *ip) 2407 { 2408 uint32_t obj_id = ip->ext_obj.ext_obj_id; 2409 uint32_t func_id = ip->ext_obj.func_id; 2410 struct extern_obj_runtime *obj = &t->extern_objs[obj_id]; 2411 rte_swx_extern_type_member_func_t func = obj->funcs[func_id]; 2412 uint32_t done; 2413 2414 TRACE("[Thread %2u] extern obj %u member func %u\n", 2415 p->thread_id, 2416 obj_id, 2417 func_id); 2418 2419 done = func(obj->obj, obj->mailbox); 2420 2421 return done; 2422 } 2423 2424 static inline uint32_t 2425 __instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused, 2426 struct thread *t, 2427 const struct instruction *ip) 2428 { 2429 uint32_t ext_func_id = ip->ext_func.ext_func_id; 2430 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id]; 2431 rte_swx_extern_func_t func = ext_func->func; 2432 uint32_t done; 2433 2434 TRACE("[Thread %2u] extern func %u\n", 2435 p->thread_id, 2436 ext_func_id); 2437 2438 done = func(ext_func->mailbox); 2439 2440 return done; 2441 } 2442 2443 /* 2444 * hash. 2445 */ 2446 static inline void 2447 __instr_hash_func_exec(struct rte_swx_pipeline *p, 2448 struct thread *t, 2449 const struct instruction *ip) 2450 { 2451 uint32_t hash_func_id = ip->hash_func.hash_func_id; 2452 uint32_t dst_offset = ip->hash_func.dst.offset; 2453 uint32_t n_dst_bits = ip->hash_func.dst.n_bits; 2454 uint32_t src_struct_id = ip->hash_func.src.struct_id; 2455 uint32_t src_offset = ip->hash_func.src.offset; 2456 uint32_t n_src_bytes = ip->hash_func.src.n_bytes; 2457 2458 struct hash_func_runtime *func = &p->hash_func_runtime[hash_func_id]; 2459 uint8_t *src_ptr = t->structs[src_struct_id]; 2460 uint32_t result; 2461 2462 TRACE("[Thread %2u] hash %u\n", 2463 p->thread_id, 2464 hash_func_id); 2465 2466 result = func->func(&src_ptr[src_offset], n_src_bytes, 0); 2467 METADATA_WRITE(t, dst_offset, n_dst_bits, result); 2468 } 2469 2470 /* 2471 * mov. 2472 */ 2473 static inline void 2474 __instr_mov_exec(struct rte_swx_pipeline *p __rte_unused, 2475 struct thread *t, 2476 const struct instruction *ip) 2477 { 2478 TRACE("[Thread %2u] mov\n", p->thread_id); 2479 2480 MOV(t, ip); 2481 } 2482 2483 static inline void 2484 __instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2485 struct thread *t, 2486 const struct instruction *ip) 2487 { 2488 TRACE("[Thread %2u] mov (mh)\n", p->thread_id); 2489 2490 MOV_MH(t, ip); 2491 } 2492 2493 static inline void 2494 __instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2495 struct thread *t, 2496 const struct instruction *ip) 2497 { 2498 TRACE("[Thread %2u] mov (hm)\n", p->thread_id); 2499 2500 MOV_HM(t, ip); 2501 } 2502 2503 static inline void 2504 __instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2505 struct thread *t, 2506 const struct instruction *ip) 2507 { 2508 TRACE("[Thread %2u] mov (hh)\n", p->thread_id); 2509 2510 MOV_HH(t, ip); 2511 } 2512 2513 static inline void 2514 __instr_mov_dma_exec(struct rte_swx_pipeline *p __rte_unused, 2515 struct thread *t, 2516 const struct instruction *ip) 2517 { 2518 uint8_t *dst_struct = t->structs[ip->mov.dst.struct_id]; 2519 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->mov.dst.offset]; 2520 uint32_t *dst32_ptr; 2521 uint16_t *dst16_ptr; 2522 uint8_t *dst8_ptr; 2523 2524 uint8_t *src_struct = t->structs[ip->mov.src.struct_id]; 2525 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->mov.src.offset]; 2526 uint32_t *src32_ptr; 2527 uint16_t *src16_ptr; 2528 uint8_t *src8_ptr; 2529 2530 uint32_t n = ip->mov.dst.n_bits >> 3, i; 2531 2532 TRACE("[Thread %2u] mov (dma) %u bytes\n", p->thread_id, n); 2533 2534 /* 8-byte transfers. */ 2535 for (i = 0; i < n >> 3; i++) 2536 *dst64_ptr++ = *src64_ptr++; 2537 2538 /* 4-byte transfers. */ 2539 n &= 7; 2540 dst32_ptr = (uint32_t *)dst64_ptr; 2541 src32_ptr = (uint32_t *)src64_ptr; 2542 2543 for (i = 0; i < n >> 2; i++) 2544 *dst32_ptr++ = *src32_ptr++; 2545 2546 /* 2-byte transfers. */ 2547 n &= 3; 2548 dst16_ptr = (uint16_t *)dst32_ptr; 2549 src16_ptr = (uint16_t *)src32_ptr; 2550 2551 for (i = 0; i < n >> 1; i++) 2552 *dst16_ptr++ = *src16_ptr++; 2553 2554 /* 1-byte transfer. */ 2555 n &= 1; 2556 dst8_ptr = (uint8_t *)dst16_ptr; 2557 src8_ptr = (uint8_t *)src16_ptr; 2558 if (n) 2559 *dst8_ptr = *src8_ptr; 2560 } 2561 2562 static inline void 2563 __instr_mov_128_exec(struct rte_swx_pipeline *p __rte_unused, 2564 struct thread *t, 2565 const struct instruction *ip) 2566 { 2567 uint8_t *dst_struct = t->structs[ip->mov.dst.struct_id]; 2568 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->mov.dst.offset]; 2569 2570 uint8_t *src_struct = t->structs[ip->mov.src.struct_id]; 2571 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->mov.src.offset]; 2572 2573 TRACE("[Thread %2u] mov (128)\n", p->thread_id); 2574 2575 dst64_ptr[0] = src64_ptr[0]; 2576 dst64_ptr[1] = src64_ptr[1]; 2577 } 2578 2579 static inline void 2580 __instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused, 2581 struct thread *t, 2582 const struct instruction *ip) 2583 { 2584 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val); 2585 2586 MOV_I(t, ip); 2587 } 2588 2589 /* 2590 * dma. 2591 */ 2592 static inline void 2593 __instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused, 2594 struct thread *t, 2595 const struct instruction *ip, 2596 uint32_t n_dma) 2597 { 2598 uint8_t *action_data = t->structs[0]; 2599 uint64_t valid_headers = t->valid_headers; 2600 uint32_t i; 2601 2602 for (i = 0; i < n_dma; i++) { 2603 uint32_t header_id = ip->dma.dst.header_id[i]; 2604 uint32_t struct_id = ip->dma.dst.struct_id[i]; 2605 uint32_t offset = ip->dma.src.offset[i]; 2606 uint32_t n_bytes = ip->dma.n_bytes[i]; 2607 2608 struct header_runtime *h = &t->headers[header_id]; 2609 uint8_t *h_ptr0 = h->ptr0; 2610 uint8_t *h_ptr = t->structs[struct_id]; 2611 2612 void *dst = MASK64_BIT_GET(valid_headers, header_id) ? 2613 h_ptr : h_ptr0; 2614 void *src = &action_data[offset]; 2615 2616 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id); 2617 2618 /* Headers. */ 2619 memcpy(dst, src, n_bytes); 2620 t->structs[struct_id] = dst; 2621 valid_headers = MASK64_BIT_SET(valid_headers, header_id); 2622 } 2623 2624 t->valid_headers = valid_headers; 2625 } 2626 2627 static inline void 2628 __instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2629 { 2630 __instr_dma_ht_many_exec(p, t, ip, 1); 2631 } 2632 2633 static inline void 2634 __instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2635 { 2636 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id); 2637 2638 __instr_dma_ht_many_exec(p, t, ip, 2); 2639 } 2640 2641 static inline void 2642 __instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2643 { 2644 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id); 2645 2646 __instr_dma_ht_many_exec(p, t, ip, 3); 2647 } 2648 2649 static inline void 2650 __instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2651 { 2652 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id); 2653 2654 __instr_dma_ht_many_exec(p, t, ip, 4); 2655 } 2656 2657 static inline void 2658 __instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2659 { 2660 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id); 2661 2662 __instr_dma_ht_many_exec(p, t, ip, 5); 2663 } 2664 2665 static inline void 2666 __instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2667 { 2668 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id); 2669 2670 __instr_dma_ht_many_exec(p, t, ip, 6); 2671 } 2672 2673 static inline void 2674 __instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2675 { 2676 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id); 2677 2678 __instr_dma_ht_many_exec(p, t, ip, 7); 2679 } 2680 2681 static inline void 2682 __instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2683 { 2684 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id); 2685 2686 __instr_dma_ht_many_exec(p, t, ip, 8); 2687 } 2688 2689 /* 2690 * alu. 2691 */ 2692 static inline void 2693 __instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused, 2694 struct thread *t, 2695 const struct instruction *ip) 2696 { 2697 TRACE("[Thread %2u] add\n", p->thread_id); 2698 2699 ALU(t, ip, +); 2700 } 2701 2702 static inline void 2703 __instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2704 struct thread *t, 2705 const struct instruction *ip) 2706 { 2707 TRACE("[Thread %2u] add (mh)\n", p->thread_id); 2708 2709 ALU_MH(t, ip, +); 2710 } 2711 2712 static inline void 2713 __instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2714 struct thread *t, 2715 const struct instruction *ip) 2716 { 2717 TRACE("[Thread %2u] add (hm)\n", p->thread_id); 2718 2719 ALU_HM(t, ip, +); 2720 } 2721 2722 static inline void 2723 __instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2724 struct thread *t, 2725 const struct instruction *ip) 2726 { 2727 TRACE("[Thread %2u] add (hh)\n", p->thread_id); 2728 2729 ALU_HH(t, ip, +); 2730 } 2731 2732 static inline void 2733 __instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused, 2734 struct thread *t, 2735 const struct instruction *ip) 2736 { 2737 TRACE("[Thread %2u] add (mi)\n", p->thread_id); 2738 2739 ALU_MI(t, ip, +); 2740 } 2741 2742 static inline void 2743 __instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused, 2744 struct thread *t, 2745 const struct instruction *ip) 2746 { 2747 TRACE("[Thread %2u] add (hi)\n", p->thread_id); 2748 2749 ALU_HI(t, ip, +); 2750 } 2751 2752 static inline void 2753 __instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused, 2754 struct thread *t, 2755 const struct instruction *ip) 2756 { 2757 TRACE("[Thread %2u] sub\n", p->thread_id); 2758 2759 ALU(t, ip, -); 2760 } 2761 2762 static inline void 2763 __instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2764 struct thread *t, 2765 const struct instruction *ip) 2766 { 2767 TRACE("[Thread %2u] sub (mh)\n", p->thread_id); 2768 2769 ALU_MH(t, ip, -); 2770 } 2771 2772 static inline void 2773 __instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2774 struct thread *t, 2775 const struct instruction *ip) 2776 { 2777 TRACE("[Thread %2u] sub (hm)\n", p->thread_id); 2778 2779 ALU_HM(t, ip, -); 2780 } 2781 2782 static inline void 2783 __instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2784 struct thread *t, 2785 const struct instruction *ip) 2786 { 2787 TRACE("[Thread %2u] sub (hh)\n", p->thread_id); 2788 2789 ALU_HH(t, ip, -); 2790 } 2791 2792 static inline void 2793 __instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused, 2794 struct thread *t, 2795 const struct instruction *ip) 2796 { 2797 TRACE("[Thread %2u] sub (mi)\n", p->thread_id); 2798 2799 ALU_MI(t, ip, -); 2800 } 2801 2802 static inline void 2803 __instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused, 2804 struct thread *t, 2805 const struct instruction *ip) 2806 { 2807 TRACE("[Thread %2u] sub (hi)\n", p->thread_id); 2808 2809 ALU_HI(t, ip, -); 2810 } 2811 2812 static inline void 2813 __instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused, 2814 struct thread *t, 2815 const struct instruction *ip) 2816 { 2817 TRACE("[Thread %2u] shl\n", p->thread_id); 2818 2819 ALU(t, ip, <<); 2820 } 2821 2822 static inline void 2823 __instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2824 struct thread *t, 2825 const struct instruction *ip) 2826 { 2827 TRACE("[Thread %2u] shl (mh)\n", p->thread_id); 2828 2829 ALU_MH(t, ip, <<); 2830 } 2831 2832 static inline void 2833 __instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2834 struct thread *t, 2835 const struct instruction *ip) 2836 { 2837 TRACE("[Thread %2u] shl (hm)\n", p->thread_id); 2838 2839 ALU_HM(t, ip, <<); 2840 } 2841 2842 static inline void 2843 __instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2844 struct thread *t, 2845 const struct instruction *ip) 2846 { 2847 TRACE("[Thread %2u] shl (hh)\n", p->thread_id); 2848 2849 ALU_HH(t, ip, <<); 2850 } 2851 2852 static inline void 2853 __instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused, 2854 struct thread *t, 2855 const struct instruction *ip) 2856 { 2857 TRACE("[Thread %2u] shl (mi)\n", p->thread_id); 2858 2859 ALU_MI(t, ip, <<); 2860 } 2861 2862 static inline void 2863 __instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused, 2864 struct thread *t, 2865 const struct instruction *ip) 2866 { 2867 TRACE("[Thread %2u] shl (hi)\n", p->thread_id); 2868 2869 ALU_HI(t, ip, <<); 2870 } 2871 2872 static inline void 2873 __instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused, 2874 struct thread *t, 2875 const struct instruction *ip) 2876 { 2877 TRACE("[Thread %2u] shr\n", p->thread_id); 2878 2879 ALU(t, ip, >>); 2880 } 2881 2882 static inline void 2883 __instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2884 struct thread *t, 2885 const struct instruction *ip) 2886 { 2887 TRACE("[Thread %2u] shr (mh)\n", p->thread_id); 2888 2889 ALU_MH(t, ip, >>); 2890 } 2891 2892 static inline void 2893 __instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2894 struct thread *t, 2895 const struct instruction *ip) 2896 { 2897 TRACE("[Thread %2u] shr (hm)\n", p->thread_id); 2898 2899 ALU_HM(t, ip, >>); 2900 } 2901 2902 static inline void 2903 __instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2904 struct thread *t, 2905 const struct instruction *ip) 2906 { 2907 TRACE("[Thread %2u] shr (hh)\n", p->thread_id); 2908 2909 ALU_HH(t, ip, >>); 2910 } 2911 2912 static inline void 2913 __instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused, 2914 struct thread *t, 2915 const struct instruction *ip) 2916 { 2917 TRACE("[Thread %2u] shr (mi)\n", p->thread_id); 2918 2919 /* Structs. */ 2920 ALU_MI(t, ip, >>); 2921 } 2922 2923 static inline void 2924 __instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused, 2925 struct thread *t, 2926 const struct instruction *ip) 2927 { 2928 TRACE("[Thread %2u] shr (hi)\n", p->thread_id); 2929 2930 ALU_HI(t, ip, >>); 2931 } 2932 2933 static inline void 2934 __instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused, 2935 struct thread *t, 2936 const struct instruction *ip) 2937 { 2938 TRACE("[Thread %2u] and\n", p->thread_id); 2939 2940 ALU(t, ip, &); 2941 } 2942 2943 static inline void 2944 __instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2945 struct thread *t, 2946 const struct instruction *ip) 2947 { 2948 TRACE("[Thread %2u] and (mh)\n", p->thread_id); 2949 2950 ALU_MH(t, ip, &); 2951 } 2952 2953 static inline void 2954 __instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2955 struct thread *t, 2956 const struct instruction *ip) 2957 { 2958 TRACE("[Thread %2u] and (hm)\n", p->thread_id); 2959 2960 ALU_HM_FAST(t, ip, &); 2961 } 2962 2963 static inline void 2964 __instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2965 struct thread *t, 2966 const struct instruction *ip) 2967 { 2968 TRACE("[Thread %2u] and (hh)\n", p->thread_id); 2969 2970 ALU_HH_FAST(t, ip, &); 2971 } 2972 2973 static inline void 2974 __instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused, 2975 struct thread *t, 2976 const struct instruction *ip) 2977 { 2978 TRACE("[Thread %2u] and (i)\n", p->thread_id); 2979 2980 ALU_I(t, ip, &); 2981 } 2982 2983 static inline void 2984 __instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused, 2985 struct thread *t, 2986 const struct instruction *ip) 2987 { 2988 TRACE("[Thread %2u] or\n", p->thread_id); 2989 2990 ALU(t, ip, |); 2991 } 2992 2993 static inline void 2994 __instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2995 struct thread *t, 2996 const struct instruction *ip) 2997 { 2998 TRACE("[Thread %2u] or (mh)\n", p->thread_id); 2999 3000 ALU_MH(t, ip, |); 3001 } 3002 3003 static inline void 3004 __instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused, 3005 struct thread *t, 3006 const struct instruction *ip) 3007 { 3008 TRACE("[Thread %2u] or (hm)\n", p->thread_id); 3009 3010 ALU_HM_FAST(t, ip, |); 3011 } 3012 3013 static inline void 3014 __instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused, 3015 struct thread *t, 3016 const struct instruction *ip) 3017 { 3018 TRACE("[Thread %2u] or (hh)\n", p->thread_id); 3019 3020 ALU_HH_FAST(t, ip, |); 3021 } 3022 3023 static inline void 3024 __instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused, 3025 struct thread *t, 3026 const struct instruction *ip) 3027 { 3028 TRACE("[Thread %2u] or (i)\n", p->thread_id); 3029 3030 ALU_I(t, ip, |); 3031 } 3032 3033 static inline void 3034 __instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused, 3035 struct thread *t, 3036 const struct instruction *ip) 3037 { 3038 TRACE("[Thread %2u] xor\n", p->thread_id); 3039 3040 ALU(t, ip, ^); 3041 } 3042 3043 static inline void 3044 __instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused, 3045 struct thread *t, 3046 const struct instruction *ip) 3047 { 3048 TRACE("[Thread %2u] xor (mh)\n", p->thread_id); 3049 3050 ALU_MH(t, ip, ^); 3051 } 3052 3053 static inline void 3054 __instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused, 3055 struct thread *t, 3056 const struct instruction *ip) 3057 { 3058 TRACE("[Thread %2u] xor (hm)\n", p->thread_id); 3059 3060 ALU_HM_FAST(t, ip, ^); 3061 } 3062 3063 static inline void 3064 __instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused, 3065 struct thread *t, 3066 const struct instruction *ip) 3067 { 3068 TRACE("[Thread %2u] xor (hh)\n", p->thread_id); 3069 3070 ALU_HH_FAST(t, ip, ^); 3071 } 3072 3073 static inline void 3074 __instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused, 3075 struct thread *t, 3076 const struct instruction *ip) 3077 { 3078 TRACE("[Thread %2u] xor (i)\n", p->thread_id); 3079 3080 ALU_I(t, ip, ^); 3081 } 3082 3083 static inline void 3084 __instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused, 3085 struct thread *t, 3086 const struct instruction *ip) 3087 { 3088 uint8_t *dst_struct, *src_struct; 3089 uint16_t *dst16_ptr, dst; 3090 uint64_t *src64_ptr, src64, src64_mask, src; 3091 uint64_t r; 3092 3093 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id); 3094 3095 /* Structs. */ 3096 dst_struct = t->structs[ip->alu.dst.struct_id]; 3097 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; 3098 dst = *dst16_ptr; 3099 3100 src_struct = t->structs[ip->alu.src.struct_id]; 3101 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset]; 3102 src64 = *src64_ptr; 3103 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits); 3104 src = src64 & src64_mask; 3105 3106 /* Initialize the result with destination 1's complement. */ 3107 r = dst; 3108 r = ~r & 0xFFFF; 3109 3110 /* The first input (r) is a 16-bit number. The second and the third 3111 * inputs are 32-bit numbers. In the worst case scenario, the sum of the 3112 * three numbers (output r) is a 34-bit number. 3113 */ 3114 r += (src >> 32) + (src & 0xFFFFFFFF); 3115 3116 /* The first input is a 16-bit number. The second input is an 18-bit 3117 * number. In the worst case scenario, the sum of the two numbers is a 3118 * 19-bit number. 3119 */ 3120 r = (r & 0xFFFF) + (r >> 16); 3121 3122 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is 3123 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006). 3124 */ 3125 r = (r & 0xFFFF) + (r >> 16); 3126 3127 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input 3128 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. 3129 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated, 3130 * therefore the output r is always a 16-bit number. 3131 */ 3132 r = (r & 0xFFFF) + (r >> 16); 3133 3134 /* Apply 1's complement to the result. */ 3135 r = ~r & 0xFFFF; 3136 r = r ? r : 0xFFFF; 3137 3138 *dst16_ptr = (uint16_t)r; 3139 } 3140 3141 static inline void 3142 __instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused, 3143 struct thread *t, 3144 const struct instruction *ip) 3145 { 3146 uint8_t *dst_struct, *src_struct; 3147 uint16_t *dst16_ptr, dst; 3148 uint64_t *src64_ptr, src64, src64_mask, src; 3149 uint64_t r; 3150 3151 TRACE("[Thread %2u] cksub (field)\n", p->thread_id); 3152 3153 /* Structs. */ 3154 dst_struct = t->structs[ip->alu.dst.struct_id]; 3155 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; 3156 dst = *dst16_ptr; 3157 3158 src_struct = t->structs[ip->alu.src.struct_id]; 3159 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset]; 3160 src64 = *src64_ptr; 3161 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits); 3162 src = src64 & src64_mask; 3163 3164 /* Initialize the result with destination 1's complement. */ 3165 r = dst; 3166 r = ~r & 0xFFFF; 3167 3168 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as 3169 * the following sequence of operations in 2's complement arithmetic: 3170 * a '- b = (a - b) % 0xFFFF. 3171 * 3172 * In order to prevent an underflow for the below subtraction, in which 3173 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the 3174 * minuend), we first add a multiple of the 0xFFFF modulus to the 3175 * minuend. The number we add to the minuend needs to be a 34-bit number 3176 * or higher, so for readability reasons we picked the 36-bit multiple. 3177 * We are effectively turning the 16-bit minuend into a 36-bit number: 3178 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF. 3179 */ 3180 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */ 3181 3182 /* A 33-bit number is subtracted from a 36-bit number (the input r). The 3183 * result (the output r) is a 36-bit number. 3184 */ 3185 r -= (src >> 32) + (src & 0xFFFFFFFF); 3186 3187 /* The first input is a 16-bit number. The second input is a 20-bit 3188 * number. Their sum is a 21-bit number. 3189 */ 3190 r = (r & 0xFFFF) + (r >> 16); 3191 3192 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is 3193 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E). 3194 */ 3195 r = (r & 0xFFFF) + (r >> 16); 3196 3197 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input 3198 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. 3199 * 0x1001E), the output r is (0 .. 31). So no carry bit can be 3200 * generated, therefore the output r is always a 16-bit number. 3201 */ 3202 r = (r & 0xFFFF) + (r >> 16); 3203 3204 /* Apply 1's complement to the result. */ 3205 r = ~r & 0xFFFF; 3206 r = r ? r : 0xFFFF; 3207 3208 *dst16_ptr = (uint16_t)r; 3209 } 3210 3211 static inline void 3212 __instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused, 3213 struct thread *t, 3214 const struct instruction *ip) 3215 { 3216 uint8_t *dst_struct, *src_struct; 3217 uint16_t *dst16_ptr, dst; 3218 uint32_t *src32_ptr; 3219 uint64_t r0, r1; 3220 3221 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id); 3222 3223 /* Structs. */ 3224 dst_struct = t->structs[ip->alu.dst.struct_id]; 3225 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; 3226 dst = *dst16_ptr; 3227 3228 src_struct = t->structs[ip->alu.src.struct_id]; 3229 src32_ptr = (uint32_t *)&src_struct[0]; 3230 3231 /* Initialize the result with destination 1's complement. */ 3232 r0 = dst; 3233 r0 = ~r0 & 0xFFFF; 3234 3235 r0 += src32_ptr[0]; /* The output r0 is a 33-bit number. */ 3236 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */ 3237 r0 += src32_ptr[2]; /* The output r0 is a 34-bit number. */ 3238 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */ 3239 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */ 3240 3241 /* The first input is a 16-bit number. The second input is a 19-bit 3242 * number. Their sum is a 20-bit number. 3243 */ 3244 r0 = (r0 & 0xFFFF) + (r0 >> 16); 3245 3246 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is 3247 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E). 3248 */ 3249 r0 = (r0 & 0xFFFF) + (r0 >> 16); 3250 3251 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input 3252 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. 3253 * 0x1000E), the output r is (0 .. 15). So no carry bit can be 3254 * generated, therefore the output r is always a 16-bit number. 3255 */ 3256 r0 = (r0 & 0xFFFF) + (r0 >> 16); 3257 3258 /* Apply 1's complement to the result. */ 3259 r0 = ~r0 & 0xFFFF; 3260 r0 = r0 ? r0 : 0xFFFF; 3261 3262 *dst16_ptr = (uint16_t)r0; 3263 } 3264 3265 static inline void 3266 __instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused, 3267 struct thread *t, 3268 const struct instruction *ip) 3269 { 3270 uint32_t src_header_id = ip->alu.src.n_bits; /* The src header ID is stored here. */ 3271 uint32_t n_src_header_bytes = t->headers[src_header_id].n_bytes; 3272 uint8_t *dst_struct, *src_struct; 3273 uint16_t *dst16_ptr, dst; 3274 uint32_t *src32_ptr; 3275 uint64_t r; 3276 uint32_t i; 3277 3278 if (n_src_header_bytes == 20) { 3279 __instr_alu_ckadd_struct20_exec(p, t, ip); 3280 return; 3281 } 3282 3283 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id); 3284 3285 /* Structs. */ 3286 dst_struct = t->structs[ip->alu.dst.struct_id]; 3287 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; 3288 dst = *dst16_ptr; 3289 3290 src_struct = t->structs[ip->alu.src.struct_id]; 3291 src32_ptr = (uint32_t *)&src_struct[0]; 3292 3293 /* Initialize the result with destination 1's complement. */ 3294 r = dst; 3295 r = ~r & 0xFFFF; 3296 3297 /* The max number of 32-bit words in a 32K-byte header is 2^13. 3298 * Therefore, in the worst case scenario, a 45-bit number is added to a 3299 * 16-bit number (the input r), so the output r is 46-bit number. 3300 */ 3301 for (i = 0; i < n_src_header_bytes / 4; i++, src32_ptr++) 3302 r += *src32_ptr; 3303 3304 /* The first input is a 16-bit number. The second input is a 30-bit 3305 * number. Their sum is a 31-bit number. 3306 */ 3307 r = (r & 0xFFFF) + (r >> 16); 3308 3309 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is 3310 * a 15-bit number (0 .. 0x7FFF). The sum is a 17-bit number (0 .. 0x17FFE). 3311 */ 3312 r = (r & 0xFFFF) + (r >> 16); 3313 3314 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input 3315 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. 3316 * 0x17FFE), the output r is (0 .. 0x7FFF). So no carry bit can be 3317 * generated, therefore the output r is always a 16-bit number. 3318 */ 3319 r = (r & 0xFFFF) + (r >> 16); 3320 3321 /* Apply 1's complement to the result. */ 3322 r = ~r & 0xFFFF; 3323 r = r ? r : 0xFFFF; 3324 3325 *dst16_ptr = (uint16_t)r; 3326 } 3327 3328 /* 3329 * Register array. 3330 */ 3331 static inline uint64_t * 3332 instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip) 3333 { 3334 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; 3335 return r->regarray; 3336 } 3337 3338 static inline uint64_t 3339 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3340 { 3341 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; 3342 3343 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; 3344 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; 3345 uint64_t idx64 = *idx64_ptr; 3346 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits); 3347 uint64_t idx = idx64 & idx64_mask & r->size_mask; 3348 3349 return idx; 3350 } 3351 3352 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3353 3354 static inline uint64_t 3355 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3356 { 3357 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; 3358 3359 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; 3360 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; 3361 uint64_t idx64 = *idx64_ptr; 3362 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask; 3363 3364 return idx; 3365 } 3366 3367 #else 3368 3369 #define instr_regarray_idx_nbo instr_regarray_idx_hbo 3370 3371 #endif 3372 3373 static inline uint64_t 3374 instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip) 3375 { 3376 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; 3377 3378 uint64_t idx = ip->regarray.idx_val & r->size_mask; 3379 3380 return idx; 3381 } 3382 3383 static inline uint64_t 3384 instr_regarray_src_hbo(struct thread *t, const struct instruction *ip) 3385 { 3386 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; 3387 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; 3388 uint64_t src64 = *src64_ptr; 3389 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); 3390 uint64_t src = src64 & src64_mask; 3391 3392 return src; 3393 } 3394 3395 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3396 3397 static inline uint64_t 3398 instr_regarray_src_nbo(struct thread *t, const struct instruction *ip) 3399 { 3400 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; 3401 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; 3402 uint64_t src64 = *src64_ptr; 3403 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits); 3404 3405 return src; 3406 } 3407 3408 #else 3409 3410 #define instr_regarray_src_nbo instr_regarray_src_hbo 3411 3412 #endif 3413 3414 static inline void 3415 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src) 3416 { 3417 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; 3418 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; 3419 uint64_t dst64 = *dst64_ptr; 3420 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); 3421 3422 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); 3423 3424 } 3425 3426 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3427 3428 static inline void 3429 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src) 3430 { 3431 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; 3432 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; 3433 uint64_t dst64 = *dst64_ptr; 3434 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); 3435 3436 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits); 3437 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); 3438 } 3439 3440 #else 3441 3442 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set 3443 3444 #endif 3445 3446 static inline void 3447 __instr_regprefetch_rh_exec(struct rte_swx_pipeline *p, 3448 struct thread *t, 3449 const struct instruction *ip) 3450 { 3451 uint64_t *regarray, idx; 3452 3453 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id); 3454 3455 regarray = instr_regarray_regarray(p, ip); 3456 idx = instr_regarray_idx_nbo(p, t, ip); 3457 rte_prefetch0(®array[idx]); 3458 } 3459 3460 static inline void 3461 __instr_regprefetch_rm_exec(struct rte_swx_pipeline *p, 3462 struct thread *t, 3463 const struct instruction *ip) 3464 { 3465 uint64_t *regarray, idx; 3466 3467 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id); 3468 3469 regarray = instr_regarray_regarray(p, ip); 3470 idx = instr_regarray_idx_hbo(p, t, ip); 3471 rte_prefetch0(®array[idx]); 3472 } 3473 3474 static inline void 3475 __instr_regprefetch_ri_exec(struct rte_swx_pipeline *p, 3476 struct thread *t __rte_unused, 3477 const struct instruction *ip) 3478 { 3479 uint64_t *regarray, idx; 3480 3481 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id); 3482 3483 regarray = instr_regarray_regarray(p, ip); 3484 idx = instr_regarray_idx_imm(p, ip); 3485 rte_prefetch0(®array[idx]); 3486 } 3487 3488 static inline void 3489 __instr_regrd_hrh_exec(struct rte_swx_pipeline *p, 3490 struct thread *t, 3491 const struct instruction *ip) 3492 { 3493 uint64_t *regarray, idx; 3494 3495 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id); 3496 3497 regarray = instr_regarray_regarray(p, ip); 3498 idx = instr_regarray_idx_nbo(p, t, ip); 3499 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); 3500 } 3501 3502 static inline void 3503 __instr_regrd_hrm_exec(struct rte_swx_pipeline *p, 3504 struct thread *t, 3505 const struct instruction *ip) 3506 { 3507 uint64_t *regarray, idx; 3508 3509 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id); 3510 3511 /* Structs. */ 3512 regarray = instr_regarray_regarray(p, ip); 3513 idx = instr_regarray_idx_hbo(p, t, ip); 3514 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); 3515 } 3516 3517 static inline void 3518 __instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3519 { 3520 uint64_t *regarray, idx; 3521 3522 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id); 3523 3524 regarray = instr_regarray_regarray(p, ip); 3525 idx = instr_regarray_idx_nbo(p, t, ip); 3526 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); 3527 } 3528 3529 static inline void 3530 __instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3531 { 3532 uint64_t *regarray, idx; 3533 3534 TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id); 3535 3536 regarray = instr_regarray_regarray(p, ip); 3537 idx = instr_regarray_idx_hbo(p, t, ip); 3538 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); 3539 } 3540 3541 static inline void 3542 __instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3543 { 3544 uint64_t *regarray, idx; 3545 3546 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id); 3547 3548 regarray = instr_regarray_regarray(p, ip); 3549 idx = instr_regarray_idx_imm(p, ip); 3550 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); 3551 } 3552 3553 static inline void 3554 __instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3555 { 3556 uint64_t *regarray, idx; 3557 3558 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id); 3559 3560 regarray = instr_regarray_regarray(p, ip); 3561 idx = instr_regarray_idx_imm(p, ip); 3562 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); 3563 } 3564 3565 static inline void 3566 __instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3567 { 3568 uint64_t *regarray, idx, src; 3569 3570 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id); 3571 3572 regarray = instr_regarray_regarray(p, ip); 3573 idx = instr_regarray_idx_nbo(p, t, ip); 3574 src = instr_regarray_src_nbo(t, ip); 3575 regarray[idx] = src; 3576 } 3577 3578 static inline void 3579 __instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3580 { 3581 uint64_t *regarray, idx, src; 3582 3583 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id); 3584 3585 regarray = instr_regarray_regarray(p, ip); 3586 idx = instr_regarray_idx_nbo(p, t, ip); 3587 src = instr_regarray_src_hbo(t, ip); 3588 regarray[idx] = src; 3589 } 3590 3591 static inline void 3592 __instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3593 { 3594 uint64_t *regarray, idx, src; 3595 3596 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id); 3597 3598 regarray = instr_regarray_regarray(p, ip); 3599 idx = instr_regarray_idx_hbo(p, t, ip); 3600 src = instr_regarray_src_nbo(t, ip); 3601 regarray[idx] = src; 3602 } 3603 3604 static inline void 3605 __instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3606 { 3607 uint64_t *regarray, idx, src; 3608 3609 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id); 3610 3611 regarray = instr_regarray_regarray(p, ip); 3612 idx = instr_regarray_idx_hbo(p, t, ip); 3613 src = instr_regarray_src_hbo(t, ip); 3614 regarray[idx] = src; 3615 } 3616 3617 static inline void 3618 __instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3619 { 3620 uint64_t *regarray, idx, src; 3621 3622 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id); 3623 3624 regarray = instr_regarray_regarray(p, ip); 3625 idx = instr_regarray_idx_nbo(p, t, ip); 3626 src = ip->regarray.dstsrc_val; 3627 regarray[idx] = src; 3628 } 3629 3630 static inline void 3631 __instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3632 { 3633 uint64_t *regarray, idx, src; 3634 3635 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id); 3636 3637 regarray = instr_regarray_regarray(p, ip); 3638 idx = instr_regarray_idx_hbo(p, t, ip); 3639 src = ip->regarray.dstsrc_val; 3640 regarray[idx] = src; 3641 } 3642 3643 static inline void 3644 __instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3645 { 3646 uint64_t *regarray, idx, src; 3647 3648 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id); 3649 3650 regarray = instr_regarray_regarray(p, ip); 3651 idx = instr_regarray_idx_imm(p, ip); 3652 src = instr_regarray_src_nbo(t, ip); 3653 regarray[idx] = src; 3654 } 3655 3656 static inline void 3657 __instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3658 { 3659 uint64_t *regarray, idx, src; 3660 3661 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id); 3662 3663 regarray = instr_regarray_regarray(p, ip); 3664 idx = instr_regarray_idx_imm(p, ip); 3665 src = instr_regarray_src_hbo(t, ip); 3666 regarray[idx] = src; 3667 } 3668 3669 static inline void 3670 __instr_regwr_rii_exec(struct rte_swx_pipeline *p, 3671 struct thread *t __rte_unused, 3672 const struct instruction *ip) 3673 { 3674 uint64_t *regarray, idx, src; 3675 3676 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id); 3677 3678 regarray = instr_regarray_regarray(p, ip); 3679 idx = instr_regarray_idx_imm(p, ip); 3680 src = ip->regarray.dstsrc_val; 3681 regarray[idx] = src; 3682 } 3683 3684 static inline void 3685 __instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3686 { 3687 uint64_t *regarray, idx, src; 3688 3689 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id); 3690 3691 regarray = instr_regarray_regarray(p, ip); 3692 idx = instr_regarray_idx_nbo(p, t, ip); 3693 src = instr_regarray_src_nbo(t, ip); 3694 regarray[idx] += src; 3695 } 3696 3697 static inline void 3698 __instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3699 { 3700 uint64_t *regarray, idx, src; 3701 3702 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id); 3703 3704 regarray = instr_regarray_regarray(p, ip); 3705 idx = instr_regarray_idx_nbo(p, t, ip); 3706 src = instr_regarray_src_hbo(t, ip); 3707 regarray[idx] += src; 3708 } 3709 3710 static inline void 3711 __instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3712 { 3713 uint64_t *regarray, idx, src; 3714 3715 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id); 3716 3717 regarray = instr_regarray_regarray(p, ip); 3718 idx = instr_regarray_idx_hbo(p, t, ip); 3719 src = instr_regarray_src_nbo(t, ip); 3720 regarray[idx] += src; 3721 } 3722 3723 static inline void 3724 __instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3725 { 3726 uint64_t *regarray, idx, src; 3727 3728 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id); 3729 3730 regarray = instr_regarray_regarray(p, ip); 3731 idx = instr_regarray_idx_hbo(p, t, ip); 3732 src = instr_regarray_src_hbo(t, ip); 3733 regarray[idx] += src; 3734 } 3735 3736 static inline void 3737 __instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3738 { 3739 uint64_t *regarray, idx, src; 3740 3741 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id); 3742 3743 regarray = instr_regarray_regarray(p, ip); 3744 idx = instr_regarray_idx_nbo(p, t, ip); 3745 src = ip->regarray.dstsrc_val; 3746 regarray[idx] += src; 3747 } 3748 3749 static inline void 3750 __instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3751 { 3752 uint64_t *regarray, idx, src; 3753 3754 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id); 3755 3756 regarray = instr_regarray_regarray(p, ip); 3757 idx = instr_regarray_idx_hbo(p, t, ip); 3758 src = ip->regarray.dstsrc_val; 3759 regarray[idx] += src; 3760 } 3761 3762 static inline void 3763 __instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3764 { 3765 uint64_t *regarray, idx, src; 3766 3767 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id); 3768 3769 regarray = instr_regarray_regarray(p, ip); 3770 idx = instr_regarray_idx_imm(p, ip); 3771 src = instr_regarray_src_nbo(t, ip); 3772 regarray[idx] += src; 3773 } 3774 3775 static inline void 3776 __instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3777 { 3778 uint64_t *regarray, idx, src; 3779 3780 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id); 3781 3782 regarray = instr_regarray_regarray(p, ip); 3783 idx = instr_regarray_idx_imm(p, ip); 3784 src = instr_regarray_src_hbo(t, ip); 3785 regarray[idx] += src; 3786 } 3787 3788 static inline void 3789 __instr_regadd_rii_exec(struct rte_swx_pipeline *p, 3790 struct thread *t __rte_unused, 3791 const struct instruction *ip) 3792 { 3793 uint64_t *regarray, idx, src; 3794 3795 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id); 3796 3797 regarray = instr_regarray_regarray(p, ip); 3798 idx = instr_regarray_idx_imm(p, ip); 3799 src = ip->regarray.dstsrc_val; 3800 regarray[idx] += src; 3801 } 3802 3803 /* 3804 * metarray. 3805 */ 3806 static inline struct meter * 3807 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3808 { 3809 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; 3810 3811 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; 3812 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; 3813 uint64_t idx64 = *idx64_ptr; 3814 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits); 3815 uint64_t idx = idx64 & idx64_mask & r->size_mask; 3816 3817 return &r->metarray[idx]; 3818 } 3819 3820 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3821 3822 static inline struct meter * 3823 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3824 { 3825 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; 3826 3827 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; 3828 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; 3829 uint64_t idx64 = *idx64_ptr; 3830 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask; 3831 3832 return &r->metarray[idx]; 3833 } 3834 3835 #else 3836 3837 #define instr_meter_idx_nbo instr_meter_idx_hbo 3838 3839 #endif 3840 3841 static inline struct meter * 3842 instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip) 3843 { 3844 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; 3845 3846 uint64_t idx = ip->meter.idx_val & r->size_mask; 3847 3848 return &r->metarray[idx]; 3849 } 3850 3851 static inline uint32_t 3852 instr_meter_length_hbo(struct thread *t, const struct instruction *ip) 3853 { 3854 uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; 3855 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; 3856 uint64_t src64 = *src64_ptr; 3857 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits); 3858 uint64_t src = src64 & src64_mask; 3859 3860 return (uint32_t)src; 3861 } 3862 3863 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3864 3865 static inline uint32_t 3866 instr_meter_length_nbo(struct thread *t, const struct instruction *ip) 3867 { 3868 uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; 3869 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; 3870 uint64_t src64 = *src64_ptr; 3871 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits); 3872 3873 return (uint32_t)src; 3874 } 3875 3876 #else 3877 3878 #define instr_meter_length_nbo instr_meter_length_hbo 3879 3880 #endif 3881 3882 static inline enum rte_color 3883 instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip) 3884 { 3885 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id]; 3886 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset]; 3887 uint64_t src64 = *src64_ptr; 3888 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits); 3889 uint64_t src = src64 & src64_mask; 3890 3891 return (enum rte_color)src; 3892 } 3893 3894 static inline void 3895 instr_meter_color_out_hbo_set(struct thread *t, 3896 const struct instruction *ip, 3897 enum rte_color color_out) 3898 { 3899 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id]; 3900 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset]; 3901 uint64_t dst64 = *dst64_ptr; 3902 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits); 3903 3904 uint64_t src = (uint64_t)color_out; 3905 3906 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); 3907 } 3908 3909 static inline void 3910 __instr_metprefetch_h_exec(struct rte_swx_pipeline *p, 3911 struct thread *t, 3912 const struct instruction *ip) 3913 { 3914 struct meter *m; 3915 3916 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id); 3917 3918 m = instr_meter_idx_nbo(p, t, ip); 3919 rte_prefetch0(m); 3920 } 3921 3922 static inline void 3923 __instr_metprefetch_m_exec(struct rte_swx_pipeline *p, 3924 struct thread *t, 3925 const struct instruction *ip) 3926 { 3927 struct meter *m; 3928 3929 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id); 3930 3931 m = instr_meter_idx_hbo(p, t, ip); 3932 rte_prefetch0(m); 3933 } 3934 3935 static inline void 3936 __instr_metprefetch_i_exec(struct rte_swx_pipeline *p, 3937 struct thread *t __rte_unused, 3938 const struct instruction *ip) 3939 { 3940 struct meter *m; 3941 3942 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id); 3943 3944 m = instr_meter_idx_imm(p, ip); 3945 rte_prefetch0(m); 3946 } 3947 3948 static inline void 3949 __instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3950 { 3951 struct meter *m; 3952 uint64_t time, n_pkts, n_bytes; 3953 uint32_t length; 3954 enum rte_color color_in, color_out; 3955 3956 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id); 3957 3958 m = instr_meter_idx_nbo(p, t, ip); 3959 rte_prefetch0(m->n_pkts); 3960 time = rte_get_tsc_cycles(); 3961 length = instr_meter_length_nbo(t, ip); 3962 color_in = instr_meter_color_in_hbo(t, ip); 3963 3964 color_out = rte_meter_trtcm_color_aware_check(&m->m, 3965 &m->profile->profile, 3966 time, 3967 length, 3968 color_in); 3969 3970 color_out &= m->color_mask; 3971 3972 n_pkts = m->n_pkts[color_out]; 3973 n_bytes = m->n_bytes[color_out]; 3974 3975 instr_meter_color_out_hbo_set(t, ip, color_out); 3976 3977 m->n_pkts[color_out] = n_pkts + 1; 3978 m->n_bytes[color_out] = n_bytes + length; 3979 } 3980 3981 static inline void 3982 __instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3983 { 3984 struct meter *m; 3985 uint64_t time, n_pkts, n_bytes; 3986 uint32_t length; 3987 enum rte_color color_in, color_out; 3988 3989 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id); 3990 3991 m = instr_meter_idx_nbo(p, t, ip); 3992 rte_prefetch0(m->n_pkts); 3993 time = rte_get_tsc_cycles(); 3994 length = instr_meter_length_nbo(t, ip); 3995 color_in = (enum rte_color)ip->meter.color_in_val; 3996 3997 color_out = rte_meter_trtcm_color_aware_check(&m->m, 3998 &m->profile->profile, 3999 time, 4000 length, 4001 color_in); 4002 4003 color_out &= m->color_mask; 4004 4005 n_pkts = m->n_pkts[color_out]; 4006 n_bytes = m->n_bytes[color_out]; 4007 4008 instr_meter_color_out_hbo_set(t, ip, color_out); 4009 4010 m->n_pkts[color_out] = n_pkts + 1; 4011 m->n_bytes[color_out] = n_bytes + length; 4012 } 4013 4014 static inline void 4015 __instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4016 { 4017 struct meter *m; 4018 uint64_t time, n_pkts, n_bytes; 4019 uint32_t length; 4020 enum rte_color color_in, color_out; 4021 4022 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id); 4023 4024 m = instr_meter_idx_nbo(p, t, ip); 4025 rte_prefetch0(m->n_pkts); 4026 time = rte_get_tsc_cycles(); 4027 length = instr_meter_length_hbo(t, ip); 4028 color_in = instr_meter_color_in_hbo(t, ip); 4029 4030 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4031 &m->profile->profile, 4032 time, 4033 length, 4034 color_in); 4035 4036 color_out &= m->color_mask; 4037 4038 n_pkts = m->n_pkts[color_out]; 4039 n_bytes = m->n_bytes[color_out]; 4040 4041 instr_meter_color_out_hbo_set(t, ip, color_out); 4042 4043 m->n_pkts[color_out] = n_pkts + 1; 4044 m->n_bytes[color_out] = n_bytes + length; 4045 } 4046 4047 static inline void 4048 __instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4049 { 4050 struct meter *m; 4051 uint64_t time, n_pkts, n_bytes; 4052 uint32_t length; 4053 enum rte_color color_in, color_out; 4054 4055 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id); 4056 4057 m = instr_meter_idx_nbo(p, t, ip); 4058 rte_prefetch0(m->n_pkts); 4059 time = rte_get_tsc_cycles(); 4060 length = instr_meter_length_hbo(t, ip); 4061 color_in = (enum rte_color)ip->meter.color_in_val; 4062 4063 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4064 &m->profile->profile, 4065 time, 4066 length, 4067 color_in); 4068 4069 color_out &= m->color_mask; 4070 4071 n_pkts = m->n_pkts[color_out]; 4072 n_bytes = m->n_bytes[color_out]; 4073 4074 instr_meter_color_out_hbo_set(t, ip, color_out); 4075 4076 m->n_pkts[color_out] = n_pkts + 1; 4077 m->n_bytes[color_out] = n_bytes + length; 4078 } 4079 4080 static inline void 4081 __instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4082 { 4083 struct meter *m; 4084 uint64_t time, n_pkts, n_bytes; 4085 uint32_t length; 4086 enum rte_color color_in, color_out; 4087 4088 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id); 4089 4090 m = instr_meter_idx_hbo(p, t, ip); 4091 rte_prefetch0(m->n_pkts); 4092 time = rte_get_tsc_cycles(); 4093 length = instr_meter_length_nbo(t, ip); 4094 color_in = instr_meter_color_in_hbo(t, ip); 4095 4096 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4097 &m->profile->profile, 4098 time, 4099 length, 4100 color_in); 4101 4102 color_out &= m->color_mask; 4103 4104 n_pkts = m->n_pkts[color_out]; 4105 n_bytes = m->n_bytes[color_out]; 4106 4107 instr_meter_color_out_hbo_set(t, ip, color_out); 4108 4109 m->n_pkts[color_out] = n_pkts + 1; 4110 m->n_bytes[color_out] = n_bytes + length; 4111 } 4112 4113 static inline void 4114 __instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4115 { 4116 struct meter *m; 4117 uint64_t time, n_pkts, n_bytes; 4118 uint32_t length; 4119 enum rte_color color_in, color_out; 4120 4121 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id); 4122 4123 m = instr_meter_idx_hbo(p, t, ip); 4124 rte_prefetch0(m->n_pkts); 4125 time = rte_get_tsc_cycles(); 4126 length = instr_meter_length_nbo(t, ip); 4127 color_in = (enum rte_color)ip->meter.color_in_val; 4128 4129 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4130 &m->profile->profile, 4131 time, 4132 length, 4133 color_in); 4134 4135 color_out &= m->color_mask; 4136 4137 n_pkts = m->n_pkts[color_out]; 4138 n_bytes = m->n_bytes[color_out]; 4139 4140 instr_meter_color_out_hbo_set(t, ip, color_out); 4141 4142 m->n_pkts[color_out] = n_pkts + 1; 4143 m->n_bytes[color_out] = n_bytes + length; 4144 } 4145 4146 static inline void 4147 __instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4148 { 4149 struct meter *m; 4150 uint64_t time, n_pkts, n_bytes; 4151 uint32_t length; 4152 enum rte_color color_in, color_out; 4153 4154 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id); 4155 4156 m = instr_meter_idx_hbo(p, t, ip); 4157 rte_prefetch0(m->n_pkts); 4158 time = rte_get_tsc_cycles(); 4159 length = instr_meter_length_hbo(t, ip); 4160 color_in = instr_meter_color_in_hbo(t, ip); 4161 4162 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4163 &m->profile->profile, 4164 time, 4165 length, 4166 color_in); 4167 4168 color_out &= m->color_mask; 4169 4170 n_pkts = m->n_pkts[color_out]; 4171 n_bytes = m->n_bytes[color_out]; 4172 4173 instr_meter_color_out_hbo_set(t, ip, color_out); 4174 4175 m->n_pkts[color_out] = n_pkts + 1; 4176 m->n_bytes[color_out] = n_bytes + length; 4177 } 4178 4179 static inline void 4180 __instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4181 { 4182 struct meter *m; 4183 uint64_t time, n_pkts, n_bytes; 4184 uint32_t length; 4185 enum rte_color color_in, color_out; 4186 4187 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id); 4188 4189 m = instr_meter_idx_hbo(p, t, ip); 4190 rte_prefetch0(m->n_pkts); 4191 time = rte_get_tsc_cycles(); 4192 length = instr_meter_length_hbo(t, ip); 4193 color_in = (enum rte_color)ip->meter.color_in_val; 4194 4195 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4196 &m->profile->profile, 4197 time, 4198 length, 4199 color_in); 4200 4201 color_out &= m->color_mask; 4202 4203 n_pkts = m->n_pkts[color_out]; 4204 n_bytes = m->n_bytes[color_out]; 4205 4206 instr_meter_color_out_hbo_set(t, ip, color_out); 4207 4208 m->n_pkts[color_out] = n_pkts + 1; 4209 m->n_bytes[color_out] = n_bytes + length; 4210 } 4211 4212 static inline void 4213 __instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4214 { 4215 struct meter *m; 4216 uint64_t time, n_pkts, n_bytes; 4217 uint32_t length; 4218 enum rte_color color_in, color_out; 4219 4220 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id); 4221 4222 m = instr_meter_idx_imm(p, ip); 4223 rte_prefetch0(m->n_pkts); 4224 time = rte_get_tsc_cycles(); 4225 length = instr_meter_length_nbo(t, ip); 4226 color_in = instr_meter_color_in_hbo(t, ip); 4227 4228 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4229 &m->profile->profile, 4230 time, 4231 length, 4232 color_in); 4233 4234 color_out &= m->color_mask; 4235 4236 n_pkts = m->n_pkts[color_out]; 4237 n_bytes = m->n_bytes[color_out]; 4238 4239 instr_meter_color_out_hbo_set(t, ip, color_out); 4240 4241 m->n_pkts[color_out] = n_pkts + 1; 4242 m->n_bytes[color_out] = n_bytes + length; 4243 } 4244 4245 static inline void 4246 __instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4247 { 4248 struct meter *m; 4249 uint64_t time, n_pkts, n_bytes; 4250 uint32_t length; 4251 enum rte_color color_in, color_out; 4252 4253 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id); 4254 4255 m = instr_meter_idx_imm(p, ip); 4256 rte_prefetch0(m->n_pkts); 4257 time = rte_get_tsc_cycles(); 4258 length = instr_meter_length_nbo(t, ip); 4259 color_in = (enum rte_color)ip->meter.color_in_val; 4260 4261 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4262 &m->profile->profile, 4263 time, 4264 length, 4265 color_in); 4266 4267 color_out &= m->color_mask; 4268 4269 n_pkts = m->n_pkts[color_out]; 4270 n_bytes = m->n_bytes[color_out]; 4271 4272 instr_meter_color_out_hbo_set(t, ip, color_out); 4273 4274 m->n_pkts[color_out] = n_pkts + 1; 4275 m->n_bytes[color_out] = n_bytes + length; 4276 } 4277 4278 static inline void 4279 __instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4280 { 4281 struct meter *m; 4282 uint64_t time, n_pkts, n_bytes; 4283 uint32_t length; 4284 enum rte_color color_in, color_out; 4285 4286 TRACE("[Thread %2u] meter (imm)\n", p->thread_id); 4287 4288 m = instr_meter_idx_imm(p, ip); 4289 rte_prefetch0(m->n_pkts); 4290 time = rte_get_tsc_cycles(); 4291 length = instr_meter_length_hbo(t, ip); 4292 color_in = instr_meter_color_in_hbo(t, ip); 4293 4294 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4295 &m->profile->profile, 4296 time, 4297 length, 4298 color_in); 4299 4300 color_out &= m->color_mask; 4301 4302 n_pkts = m->n_pkts[color_out]; 4303 n_bytes = m->n_bytes[color_out]; 4304 4305 instr_meter_color_out_hbo_set(t, ip, color_out); 4306 4307 m->n_pkts[color_out] = n_pkts + 1; 4308 m->n_bytes[color_out] = n_bytes + length; 4309 } 4310 4311 static inline void 4312 __instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4313 { 4314 struct meter *m; 4315 uint64_t time, n_pkts, n_bytes; 4316 uint32_t length; 4317 enum rte_color color_in, color_out; 4318 4319 TRACE("[Thread %2u] meter (imi)\n", p->thread_id); 4320 4321 m = instr_meter_idx_imm(p, ip); 4322 rte_prefetch0(m->n_pkts); 4323 time = rte_get_tsc_cycles(); 4324 length = instr_meter_length_hbo(t, ip); 4325 color_in = (enum rte_color)ip->meter.color_in_val; 4326 4327 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4328 &m->profile->profile, 4329 time, 4330 length, 4331 color_in); 4332 4333 color_out &= m->color_mask; 4334 4335 n_pkts = m->n_pkts[color_out]; 4336 n_bytes = m->n_bytes[color_out]; 4337 4338 instr_meter_color_out_hbo_set(t, ip, color_out); 4339 4340 m->n_pkts[color_out] = n_pkts + 1; 4341 m->n_bytes[color_out] = n_bytes + length; 4342 } 4343 4344 #endif 4345