1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2021 Intel Corporation 3 */ 4 #ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__ 5 #define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__ 6 7 #include <inttypes.h> 8 #include <string.h> 9 #include <sys/queue.h> 10 11 #include <rte_byteorder.h> 12 #include <rte_common.h> 13 #include <rte_cycles.h> 14 #include <rte_prefetch.h> 15 #include <rte_meter.h> 16 17 #include <rte_swx_table_selector.h> 18 #include <rte_swx_table_learner.h> 19 #include <rte_swx_pipeline.h> 20 #include <rte_swx_ctl.h> 21 22 #ifndef TRACE_LEVEL 23 #define TRACE_LEVEL 0 24 #endif 25 26 #if TRACE_LEVEL 27 #define TRACE(...) printf(__VA_ARGS__) 28 #else 29 #define TRACE(...) 30 #endif 31 32 /* 33 * Environment. 34 */ 35 #define ntoh64(x) rte_be_to_cpu_64(x) 36 #define hton64(x) rte_cpu_to_be_64(x) 37 38 /* 39 * Struct. 40 */ 41 struct field { 42 char name[RTE_SWX_NAME_SIZE]; 43 uint32_t n_bits; 44 uint32_t offset; 45 int var_size; 46 }; 47 48 struct struct_type { 49 TAILQ_ENTRY(struct_type) node; 50 char name[RTE_SWX_NAME_SIZE]; 51 struct field *fields; 52 uint32_t n_fields; 53 uint32_t n_bits; 54 uint32_t n_bits_min; 55 int var_size; 56 }; 57 58 TAILQ_HEAD(struct_type_tailq, struct_type); 59 60 /* 61 * Input port. 62 */ 63 struct port_in_type { 64 TAILQ_ENTRY(port_in_type) node; 65 char name[RTE_SWX_NAME_SIZE]; 66 struct rte_swx_port_in_ops ops; 67 }; 68 69 TAILQ_HEAD(port_in_type_tailq, port_in_type); 70 71 struct port_in { 72 TAILQ_ENTRY(port_in) node; 73 struct port_in_type *type; 74 void *obj; 75 uint32_t id; 76 }; 77 78 TAILQ_HEAD(port_in_tailq, port_in); 79 80 struct port_in_runtime { 81 rte_swx_port_in_pkt_rx_t pkt_rx; 82 void *obj; 83 }; 84 85 /* 86 * Output port. 87 */ 88 struct port_out_type { 89 TAILQ_ENTRY(port_out_type) node; 90 char name[RTE_SWX_NAME_SIZE]; 91 struct rte_swx_port_out_ops ops; 92 }; 93 94 TAILQ_HEAD(port_out_type_tailq, port_out_type); 95 96 struct port_out { 97 TAILQ_ENTRY(port_out) node; 98 struct port_out_type *type; 99 void *obj; 100 uint32_t id; 101 }; 102 103 TAILQ_HEAD(port_out_tailq, port_out); 104 105 struct port_out_runtime { 106 rte_swx_port_out_pkt_tx_t pkt_tx; 107 rte_swx_port_out_pkt_fast_clone_tx_t pkt_fast_clone_tx; 108 rte_swx_port_out_pkt_clone_tx_t pkt_clone_tx; 109 rte_swx_port_out_flush_t flush; 110 void *obj; 111 }; 112 113 /* 114 * Packet mirroring. 115 */ 116 struct mirroring_session { 117 uint32_t port_id; 118 int fast_clone; 119 uint32_t truncation_length; 120 }; 121 122 /* 123 * Extern object. 124 */ 125 struct extern_type_member_func { 126 TAILQ_ENTRY(extern_type_member_func) node; 127 char name[RTE_SWX_NAME_SIZE]; 128 rte_swx_extern_type_member_func_t func; 129 uint32_t id; 130 }; 131 132 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func); 133 134 struct extern_type { 135 TAILQ_ENTRY(extern_type) node; 136 char name[RTE_SWX_NAME_SIZE]; 137 struct struct_type *mailbox_struct_type; 138 rte_swx_extern_type_constructor_t constructor; 139 rte_swx_extern_type_destructor_t destructor; 140 struct extern_type_member_func_tailq funcs; 141 uint32_t n_funcs; 142 }; 143 144 TAILQ_HEAD(extern_type_tailq, extern_type); 145 146 struct extern_obj { 147 TAILQ_ENTRY(extern_obj) node; 148 char name[RTE_SWX_NAME_SIZE]; 149 struct extern_type *type; 150 void *obj; 151 uint32_t struct_id; 152 uint32_t id; 153 }; 154 155 TAILQ_HEAD(extern_obj_tailq, extern_obj); 156 157 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 158 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8 159 #endif 160 161 struct extern_obj_runtime { 162 void *obj; 163 uint8_t *mailbox; 164 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX]; 165 }; 166 167 /* 168 * Extern function. 169 */ 170 struct extern_func { 171 TAILQ_ENTRY(extern_func) node; 172 char name[RTE_SWX_NAME_SIZE]; 173 struct struct_type *mailbox_struct_type; 174 rte_swx_extern_func_t func; 175 uint32_t struct_id; 176 uint32_t id; 177 }; 178 179 TAILQ_HEAD(extern_func_tailq, extern_func); 180 181 struct extern_func_runtime { 182 uint8_t *mailbox; 183 rte_swx_extern_func_t func; 184 }; 185 186 /* 187 * Hash function. 188 */ 189 struct hash_func { 190 TAILQ_ENTRY(hash_func) node; 191 char name[RTE_SWX_NAME_SIZE]; 192 rte_swx_hash_func_t func; 193 uint32_t id; 194 }; 195 196 TAILQ_HEAD(hash_func_tailq, hash_func); 197 198 struct hash_func_runtime { 199 rte_swx_hash_func_t func; 200 }; 201 202 /* 203 * RSS. 204 */ 205 struct rss { 206 TAILQ_ENTRY(rss) node; 207 char name[RTE_SWX_NAME_SIZE]; 208 uint32_t id; 209 }; 210 211 TAILQ_HEAD(rss_tailq, rss); 212 213 struct rss_runtime { 214 uint32_t key_size; /* key size in bytes. */ 215 uint8_t key[0]; /* key. */ 216 }; 217 218 /* 219 * Header. 220 */ 221 struct header { 222 TAILQ_ENTRY(header) node; 223 char name[RTE_SWX_NAME_SIZE]; 224 struct struct_type *st; 225 uint32_t struct_id; 226 uint32_t id; 227 }; 228 229 TAILQ_HEAD(header_tailq, header); 230 231 struct header_runtime { 232 uint8_t *ptr0; 233 uint32_t n_bytes; 234 }; 235 236 struct header_out_runtime { 237 uint8_t *ptr0; 238 uint8_t *ptr; 239 uint32_t n_bytes; 240 }; 241 242 /* 243 * Instruction. 244 */ 245 246 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian. 247 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO). 248 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO 249 * when transferred to packet meta-data and in NBO when transferred to packet 250 * headers. 251 */ 252 253 /* Notation conventions: 254 * -Header field: H = h.header.field (dst/src) 255 * -Meta-data field: M = m.field (dst/src) 256 * -Extern object mailbox field: E = e.field (dst/src) 257 * -Extern function mailbox field: F = f.field (dst/src) 258 * -Table action data field: T = t.field (src only) 259 * -Immediate value: I = 32-bit unsigned value (src only) 260 */ 261 262 enum instruction_type { 263 /* rx m.port_in */ 264 INSTR_RX, 265 266 /* tx port_out 267 * port_out = MI 268 */ 269 INSTR_TX, /* port_out = M */ 270 INSTR_TX_I, /* port_out = I */ 271 INSTR_DROP, 272 273 /* 274 * mirror slot_id session_id 275 * slot_id = MEFT 276 * session_id = MEFT 277 */ 278 INSTR_MIRROR, 279 280 /* recirculate 281 */ 282 INSTR_RECIRCULATE, 283 284 /* recircid m.recirc_pass_id 285 * Read the internal recirculation pass ID into the specified meta-data field. 286 */ 287 INSTR_RECIRCID, 288 289 /* extract h.header */ 290 INSTR_HDR_EXTRACT, 291 INSTR_HDR_EXTRACT2, 292 INSTR_HDR_EXTRACT3, 293 INSTR_HDR_EXTRACT4, 294 INSTR_HDR_EXTRACT5, 295 INSTR_HDR_EXTRACT6, 296 INSTR_HDR_EXTRACT7, 297 INSTR_HDR_EXTRACT8, 298 299 /* extract h.header m.last_field_size */ 300 INSTR_HDR_EXTRACT_M, 301 302 /* lookahead h.header */ 303 INSTR_HDR_LOOKAHEAD, 304 305 /* emit h.header */ 306 INSTR_HDR_EMIT, 307 INSTR_HDR_EMIT_TX, 308 INSTR_HDR_EMIT2_TX, 309 INSTR_HDR_EMIT3_TX, 310 INSTR_HDR_EMIT4_TX, 311 INSTR_HDR_EMIT5_TX, 312 INSTR_HDR_EMIT6_TX, 313 INSTR_HDR_EMIT7_TX, 314 INSTR_HDR_EMIT8_TX, 315 316 /* validate h.header */ 317 INSTR_HDR_VALIDATE, 318 319 /* invalidate h.header */ 320 INSTR_HDR_INVALIDATE, 321 322 /* mov dst src 323 * dst = src 324 * dst = HMEF, src = HMEFTI 325 */ 326 INSTR_MOV, /* dst = MEF, src = MEFT; size(dst) <= 64 bits, size(src) <= 64 bits. */ 327 INSTR_MOV_MH, /* dst = MEF, src = H; size(dst) <= 64 bits, size(src) <= 64 bits. */ 328 INSTR_MOV_HM, /* dst = H, src = MEFT; size(dst) <= 64 bits, size(src) <= 64 bits. */ 329 INSTR_MOV_HH, /* dst = H, src = H; size(dst) <= 64 bits, size(src) <= 64 bits. */ 330 INSTR_MOV_DMA, /* dst = HMEF, src = HMEF; size(dst) = size(src) > 64 bits, NBO format. */ 331 INSTR_MOV_128, /* dst = HMEF, src = HMEF; size(dst) = size(src) = 128 bits, NBO format. */ 332 INSTR_MOV_I, /* dst = HMEF, src = I; size(dst) <= 64 bits. */ 333 334 /* dma h.header t.field 335 * memcpy(h.header, t.field, sizeof(h.header)) 336 */ 337 INSTR_DMA_HT, 338 INSTR_DMA_HT2, 339 INSTR_DMA_HT3, 340 INSTR_DMA_HT4, 341 INSTR_DMA_HT5, 342 INSTR_DMA_HT6, 343 INSTR_DMA_HT7, 344 INSTR_DMA_HT8, 345 346 /* add dst src 347 * dst += src 348 * dst = HMEF, src = HMEFTI 349 */ 350 INSTR_ALU_ADD, /* dst = MEF, src = MEF */ 351 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */ 352 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */ 353 INSTR_ALU_ADD_HH, /* dst = H, src = H */ 354 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */ 355 INSTR_ALU_ADD_HI, /* dst = H, src = I */ 356 357 /* sub dst src 358 * dst -= src 359 * dst = HMEF, src = HMEFTI 360 */ 361 INSTR_ALU_SUB, /* dst = MEF, src = MEF */ 362 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */ 363 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */ 364 INSTR_ALU_SUB_HH, /* dst = H, src = H */ 365 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */ 366 INSTR_ALU_SUB_HI, /* dst = H, src = I */ 367 368 /* ckadd dst src 369 * dst = dst '+ src[0:1] '+ src[2:3] '+ ... 370 * dst = H, src = {H, h.header}, '+ = 1's complement addition operator 371 */ 372 INSTR_ALU_CKADD_FIELD, /* src = H */ 373 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 bytes. */ 374 INSTR_ALU_CKADD_STRUCT, /* src = h.header, with sizeof(header) any 4-byte multiple. */ 375 376 /* cksub dst src 377 * dst = dst '- src 378 * dst = H, src = H, '- = 1's complement subtraction operator 379 */ 380 INSTR_ALU_CKSUB_FIELD, 381 382 /* and dst src 383 * dst &= src 384 * dst = HMEF, src = HMEFTI 385 */ 386 INSTR_ALU_AND, /* dst = MEF, src = MEFT */ 387 INSTR_ALU_AND_MH, /* dst = MEF, src = H */ 388 INSTR_ALU_AND_HM, /* dst = H, src = MEFT */ 389 INSTR_ALU_AND_HH, /* dst = H, src = H */ 390 INSTR_ALU_AND_I, /* dst = HMEF, src = I */ 391 392 /* or dst src 393 * dst |= src 394 * dst = HMEF, src = HMEFTI 395 */ 396 INSTR_ALU_OR, /* dst = MEF, src = MEFT */ 397 INSTR_ALU_OR_MH, /* dst = MEF, src = H */ 398 INSTR_ALU_OR_HM, /* dst = H, src = MEFT */ 399 INSTR_ALU_OR_HH, /* dst = H, src = H */ 400 INSTR_ALU_OR_I, /* dst = HMEF, src = I */ 401 402 /* xor dst src 403 * dst ^= src 404 * dst = HMEF, src = HMEFTI 405 */ 406 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */ 407 INSTR_ALU_XOR_MH, /* dst = MEF, src = H */ 408 INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */ 409 INSTR_ALU_XOR_HH, /* dst = H, src = H */ 410 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */ 411 412 /* shl dst src 413 * dst <<= src 414 * dst = HMEF, src = HMEFTI 415 */ 416 INSTR_ALU_SHL, /* dst = MEF, src = MEF */ 417 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */ 418 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */ 419 INSTR_ALU_SHL_HH, /* dst = H, src = H */ 420 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */ 421 INSTR_ALU_SHL_HI, /* dst = H, src = I */ 422 423 /* shr dst src 424 * dst >>= src 425 * dst = HMEF, src = HMEFTI 426 */ 427 INSTR_ALU_SHR, /* dst = MEF, src = MEF */ 428 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */ 429 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */ 430 INSTR_ALU_SHR_HH, /* dst = H, src = H */ 431 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */ 432 INSTR_ALU_SHR_HI, /* dst = H, src = I */ 433 434 /* regprefetch REGARRAY index 435 * prefetch REGARRAY[index] 436 * index = HMEFTI 437 */ 438 INSTR_REGPREFETCH_RH, /* index = H */ 439 INSTR_REGPREFETCH_RM, /* index = MEFT */ 440 INSTR_REGPREFETCH_RI, /* index = I */ 441 442 /* regrd dst REGARRAY index 443 * dst = REGARRAY[index] 444 * dst = HMEF, index = HMEFTI 445 */ 446 INSTR_REGRD_HRH, /* dst = H, index = H */ 447 INSTR_REGRD_HRM, /* dst = H, index = MEFT */ 448 INSTR_REGRD_HRI, /* dst = H, index = I */ 449 INSTR_REGRD_MRH, /* dst = MEF, index = H */ 450 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */ 451 INSTR_REGRD_MRI, /* dst = MEF, index = I */ 452 453 /* regwr REGARRAY index src 454 * REGARRAY[index] = src 455 * index = HMEFTI, src = HMEFTI 456 */ 457 INSTR_REGWR_RHH, /* index = H, src = H */ 458 INSTR_REGWR_RHM, /* index = H, src = MEFT */ 459 INSTR_REGWR_RHI, /* index = H, src = I */ 460 INSTR_REGWR_RMH, /* index = MEFT, src = H */ 461 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */ 462 INSTR_REGWR_RMI, /* index = MEFT, src = I */ 463 INSTR_REGWR_RIH, /* index = I, src = H */ 464 INSTR_REGWR_RIM, /* index = I, src = MEFT */ 465 INSTR_REGWR_RII, /* index = I, src = I */ 466 467 /* regadd REGARRAY index src 468 * REGARRAY[index] += src 469 * index = HMEFTI, src = HMEFTI 470 */ 471 INSTR_REGADD_RHH, /* index = H, src = H */ 472 INSTR_REGADD_RHM, /* index = H, src = MEFT */ 473 INSTR_REGADD_RHI, /* index = H, src = I */ 474 INSTR_REGADD_RMH, /* index = MEFT, src = H */ 475 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */ 476 INSTR_REGADD_RMI, /* index = MEFT, src = I */ 477 INSTR_REGADD_RIH, /* index = I, src = H */ 478 INSTR_REGADD_RIM, /* index = I, src = MEFT */ 479 INSTR_REGADD_RII, /* index = I, src = I */ 480 481 /* metprefetch METARRAY index 482 * prefetch METARRAY[index] 483 * index = HMEFTI 484 */ 485 INSTR_METPREFETCH_H, /* index = H */ 486 INSTR_METPREFETCH_M, /* index = MEFT */ 487 INSTR_METPREFETCH_I, /* index = I */ 488 489 /* meter METARRAY index length color_in color_out 490 * color_out = meter(METARRAY[index], length, color_in) 491 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF 492 */ 493 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */ 494 INSTR_METER_HHI, /* index = H, length = H, color_in = I */ 495 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */ 496 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */ 497 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */ 498 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */ 499 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */ 500 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */ 501 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */ 502 INSTR_METER_IHI, /* index = I, length = H, color_in = I */ 503 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */ 504 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */ 505 506 /* table TABLE */ 507 INSTR_TABLE, 508 INSTR_TABLE_AF, 509 INSTR_SELECTOR, 510 INSTR_LEARNER, 511 INSTR_LEARNER_AF, 512 513 /* learn ACTION_NAME [ m.action_first_arg ] m.timeout_id */ 514 INSTR_LEARNER_LEARN, 515 516 /* rearm [ m.timeout_id ] */ 517 INSTR_LEARNER_REARM, 518 INSTR_LEARNER_REARM_NEW, 519 520 /* forget */ 521 INSTR_LEARNER_FORGET, 522 523 /* entryid m.table_entry_id 524 * Read the internal table entry ID into the specified meta-data field. 525 */ 526 INSTR_ENTRYID, 527 528 /* extern e.obj.func */ 529 INSTR_EXTERN_OBJ, 530 531 /* extern f.func */ 532 INSTR_EXTERN_FUNC, 533 534 /* hash HASH_FUNC_NAME dst src_first src_last 535 * Compute hash value over range of struct fields. 536 * dst = M 537 * src_first = HMEFT 538 * src_last = HMEFT 539 * src_first and src_last must be fields within the same struct 540 */ 541 INSTR_HASH_FUNC, 542 543 /* rss RSS_OBJ_NAME dst src_first src_last 544 * Compute the RSS hash value over range of struct fields. 545 * dst = M 546 * src_first = HMEFT 547 * src_last = HMEFT 548 * src_first and src_last must be fields within the same struct 549 */ 550 INSTR_RSS, 551 552 /* jmp LABEL 553 * Unconditional jump 554 */ 555 INSTR_JMP, 556 557 /* jmpv LABEL h.header 558 * Jump if header is valid 559 */ 560 INSTR_JMP_VALID, 561 562 /* jmpnv LABEL h.header 563 * Jump if header is invalid 564 */ 565 INSTR_JMP_INVALID, 566 567 /* jmph LABEL 568 * Jump if table lookup hit 569 */ 570 INSTR_JMP_HIT, 571 572 /* jmpnh LABEL 573 * Jump if table lookup miss 574 */ 575 INSTR_JMP_MISS, 576 577 /* jmpa LABEL ACTION 578 * Jump if action run 579 */ 580 INSTR_JMP_ACTION_HIT, 581 582 /* jmpna LABEL ACTION 583 * Jump if action not run 584 */ 585 INSTR_JMP_ACTION_MISS, 586 587 /* jmpeq LABEL a b 588 * Jump if a is equal to b 589 * a = HMEFT, b = HMEFTI 590 */ 591 INSTR_JMP_EQ, /* a = MEFT, b = MEFT */ 592 INSTR_JMP_EQ_MH, /* a = MEFT, b = H */ 593 INSTR_JMP_EQ_HM, /* a = H, b = MEFT */ 594 INSTR_JMP_EQ_HH, /* a = H, b = H */ 595 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */ 596 597 /* jmpneq LABEL a b 598 * Jump if a is not equal to b 599 * a = HMEFT, b = HMEFTI 600 */ 601 INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */ 602 INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */ 603 INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */ 604 INSTR_JMP_NEQ_HH, /* a = H, b = H */ 605 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */ 606 607 /* jmplt LABEL a b 608 * Jump if a is less than b 609 * a = HMEFT, b = HMEFTI 610 */ 611 INSTR_JMP_LT, /* a = MEFT, b = MEFT */ 612 INSTR_JMP_LT_MH, /* a = MEFT, b = H */ 613 INSTR_JMP_LT_HM, /* a = H, b = MEFT */ 614 INSTR_JMP_LT_HH, /* a = H, b = H */ 615 INSTR_JMP_LT_MI, /* a = MEFT, b = I */ 616 INSTR_JMP_LT_HI, /* a = H, b = I */ 617 618 /* jmpgt LABEL a b 619 * Jump if a is greater than b 620 * a = HMEFT, b = HMEFTI 621 */ 622 INSTR_JMP_GT, /* a = MEFT, b = MEFT */ 623 INSTR_JMP_GT_MH, /* a = MEFT, b = H */ 624 INSTR_JMP_GT_HM, /* a = H, b = MEFT */ 625 INSTR_JMP_GT_HH, /* a = H, b = H */ 626 INSTR_JMP_GT_MI, /* a = MEFT, b = I */ 627 INSTR_JMP_GT_HI, /* a = H, b = I */ 628 629 /* return 630 * Return from action 631 */ 632 INSTR_RETURN, 633 634 /* Start of custom instructions. */ 635 INSTR_CUSTOM_0, 636 }; 637 638 struct instr_operand { 639 uint8_t struct_id; 640 uint8_t n_bits; 641 uint8_t offset; 642 uint8_t pad; 643 }; 644 645 struct instr_io { 646 struct { 647 union { 648 struct { 649 uint8_t offset; 650 uint8_t n_bits; 651 uint8_t pad[2]; 652 }; 653 654 uint32_t val; 655 }; 656 } io; 657 658 struct { 659 uint8_t header_id[8]; 660 uint8_t struct_id[8]; 661 uint8_t n_bytes[8]; 662 } hdr; 663 }; 664 665 struct instr_hdr_validity { 666 uint8_t header_id; 667 uint8_t struct_id; 668 }; 669 670 struct instr_table { 671 uint8_t table_id; 672 }; 673 674 struct instr_learn { 675 uint8_t action_id; 676 uint8_t mf_first_arg_offset; 677 uint8_t mf_timeout_id_offset; 678 uint8_t mf_timeout_id_n_bits; 679 }; 680 681 struct instr_extern_obj { 682 uint8_t ext_obj_id; 683 uint8_t func_id; 684 }; 685 686 struct instr_extern_func { 687 uint8_t ext_func_id; 688 }; 689 690 struct instr_hash_func { 691 uint8_t hash_func_id; 692 693 struct { 694 uint8_t offset; 695 uint8_t n_bits; 696 } dst; 697 698 struct { 699 uint8_t struct_id; 700 uint16_t offset; 701 uint16_t n_bytes; 702 } src; 703 }; 704 705 struct instr_rss { 706 uint8_t rss_obj_id; 707 708 struct { 709 uint8_t offset; 710 uint8_t n_bits; 711 } dst; 712 713 struct { 714 uint8_t struct_id; 715 uint16_t offset; 716 uint16_t n_bytes; 717 } src; 718 }; 719 720 struct instr_dst_src { 721 struct instr_operand dst; 722 union { 723 struct instr_operand src; 724 uint64_t src_val; 725 }; 726 }; 727 728 struct instr_regarray { 729 uint8_t regarray_id; 730 uint8_t pad[3]; 731 732 union { 733 struct instr_operand idx; 734 uint32_t idx_val; 735 }; 736 737 union { 738 struct instr_operand dstsrc; 739 uint64_t dstsrc_val; 740 }; 741 }; 742 743 struct instr_meter { 744 uint8_t metarray_id; 745 uint8_t pad[3]; 746 747 union { 748 struct instr_operand idx; 749 uint32_t idx_val; 750 }; 751 752 struct instr_operand length; 753 754 union { 755 struct instr_operand color_in; 756 uint32_t color_in_val; 757 }; 758 759 struct instr_operand color_out; 760 }; 761 762 struct instr_dma { 763 struct { 764 uint8_t header_id[8]; 765 uint8_t struct_id[8]; 766 } dst; 767 768 struct { 769 uint8_t offset[8]; 770 } src; 771 772 uint16_t n_bytes[8]; 773 }; 774 775 struct instr_jmp { 776 struct instruction *ip; 777 778 union { 779 struct instr_operand a; 780 uint8_t header_id; 781 uint8_t action_id; 782 }; 783 784 union { 785 struct instr_operand b; 786 uint64_t b_val; 787 }; 788 }; 789 790 struct instruction { 791 enum instruction_type type; 792 union { 793 struct instr_io io; 794 struct instr_dst_src mirror; 795 struct instr_hdr_validity valid; 796 struct instr_dst_src mov; 797 struct instr_regarray regarray; 798 struct instr_meter meter; 799 struct instr_dma dma; 800 struct instr_dst_src alu; 801 struct instr_table table; 802 struct instr_learn learn; 803 struct instr_extern_obj ext_obj; 804 struct instr_extern_func ext_func; 805 struct instr_hash_func hash_func; 806 struct instr_rss rss; 807 struct instr_jmp jmp; 808 }; 809 }; 810 811 struct instruction_data { 812 char label[RTE_SWX_NAME_SIZE]; 813 char jmp_label[RTE_SWX_NAME_SIZE]; 814 uint32_t n_users; /* user = jmp instruction to this instruction. */ 815 int invalid; 816 }; 817 818 typedef void (*instr_exec_t)(struct rte_swx_pipeline *); 819 820 /* 821 * Action. 822 */ 823 typedef void 824 (*action_func_t)(struct rte_swx_pipeline *p); 825 826 struct action { 827 TAILQ_ENTRY(action) node; 828 char name[RTE_SWX_NAME_SIZE]; 829 struct struct_type *st; 830 int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */ 831 struct instruction *instructions; 832 struct instruction_data *instruction_data; 833 uint32_t n_instructions; 834 uint32_t id; 835 }; 836 837 TAILQ_HEAD(action_tailq, action); 838 839 /* 840 * Table. 841 */ 842 struct table_type { 843 TAILQ_ENTRY(table_type) node; 844 char name[RTE_SWX_NAME_SIZE]; 845 enum rte_swx_table_match_type match_type; 846 struct rte_swx_table_ops ops; 847 }; 848 849 TAILQ_HEAD(table_type_tailq, table_type); 850 851 struct match_field { 852 enum rte_swx_table_match_type match_type; 853 struct field *field; 854 }; 855 856 struct table { 857 TAILQ_ENTRY(table) node; 858 char name[RTE_SWX_NAME_SIZE]; 859 char args[RTE_SWX_NAME_SIZE]; 860 struct table_type *type; /* NULL when n_fields == 0. */ 861 862 /* Match. */ 863 struct match_field *fields; 864 uint32_t n_fields; 865 struct header *header; /* Only valid when n_fields > 0. */ 866 867 /* Action. */ 868 struct action **actions; 869 struct action *default_action; 870 uint8_t *default_action_data; 871 uint32_t n_actions; 872 int default_action_is_const; 873 uint32_t action_data_size_max; 874 int *action_is_for_table_entries; 875 int *action_is_for_default_entry; 876 877 struct hash_func *hf; 878 uint32_t size; 879 uint32_t id; 880 }; 881 882 TAILQ_HEAD(table_tailq, table); 883 884 struct table_runtime { 885 rte_swx_table_lookup_t func; 886 void *mailbox; 887 uint8_t **key; 888 }; 889 890 struct table_statistics { 891 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */ 892 uint64_t *n_pkts_action; 893 }; 894 895 /* 896 * Selector. 897 */ 898 struct selector { 899 TAILQ_ENTRY(selector) node; 900 char name[RTE_SWX_NAME_SIZE]; 901 902 struct field *group_id_field; 903 struct field **selector_fields; 904 uint32_t n_selector_fields; 905 struct header *selector_header; 906 struct field *member_id_field; 907 908 uint32_t n_groups_max; 909 uint32_t n_members_per_group_max; 910 911 uint32_t id; 912 }; 913 914 TAILQ_HEAD(selector_tailq, selector); 915 916 struct selector_runtime { 917 void *mailbox; 918 uint8_t **group_id_buffer; 919 uint8_t **selector_buffer; 920 uint8_t **member_id_buffer; 921 }; 922 923 struct selector_statistics { 924 uint64_t n_pkts; 925 }; 926 927 /* 928 * Learner table. 929 */ 930 struct learner { 931 TAILQ_ENTRY(learner) node; 932 char name[RTE_SWX_NAME_SIZE]; 933 934 /* Match. */ 935 struct field **fields; 936 uint32_t n_fields; 937 struct header *header; 938 939 /* Action. */ 940 struct action **actions; 941 struct action *default_action; 942 uint8_t *default_action_data; 943 uint32_t n_actions; 944 int default_action_is_const; 945 uint32_t action_data_size_max; 946 int *action_is_for_table_entries; 947 int *action_is_for_default_entry; 948 949 struct hash_func *hf; 950 uint32_t size; 951 uint32_t timeout[RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX]; 952 uint32_t n_timeouts; 953 uint32_t id; 954 }; 955 956 TAILQ_HEAD(learner_tailq, learner); 957 958 struct learner_runtime { 959 void *mailbox; 960 uint8_t **key; 961 }; 962 963 struct learner_statistics { 964 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */ 965 uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */ 966 uint64_t n_pkts_rearm; 967 uint64_t n_pkts_forget; 968 uint64_t *n_pkts_action; 969 }; 970 971 /* 972 * Register array. 973 */ 974 struct regarray { 975 TAILQ_ENTRY(regarray) node; 976 char name[RTE_SWX_NAME_SIZE]; 977 uint64_t init_val; 978 uint32_t size; 979 uint32_t id; 980 }; 981 982 TAILQ_HEAD(regarray_tailq, regarray); 983 984 struct regarray_runtime { 985 uint64_t *regarray; 986 uint32_t size_mask; 987 }; 988 989 /* 990 * Meter array. 991 */ 992 struct meter_profile { 993 TAILQ_ENTRY(meter_profile) node; 994 char name[RTE_SWX_NAME_SIZE]; 995 struct rte_meter_trtcm_params params; 996 struct rte_meter_trtcm_profile profile; 997 uint32_t n_users; 998 }; 999 1000 TAILQ_HEAD(meter_profile_tailq, meter_profile); 1001 1002 struct metarray { 1003 TAILQ_ENTRY(metarray) node; 1004 char name[RTE_SWX_NAME_SIZE]; 1005 uint32_t size; 1006 uint32_t id; 1007 }; 1008 1009 TAILQ_HEAD(metarray_tailq, metarray); 1010 1011 struct meter { 1012 struct rte_meter_trtcm m; 1013 struct meter_profile *profile; 1014 enum rte_color color_mask; 1015 uint8_t pad[20]; 1016 1017 uint64_t n_pkts[RTE_COLORS]; 1018 uint64_t n_bytes[RTE_COLORS]; 1019 }; 1020 1021 struct metarray_runtime { 1022 struct meter *metarray; 1023 uint32_t size_mask; 1024 }; 1025 1026 /* 1027 * Pipeline. 1028 */ 1029 struct thread { 1030 /* Packet. */ 1031 struct rte_swx_pkt pkt; 1032 uint8_t *ptr; 1033 uint32_t *mirroring_slots; 1034 uint64_t mirroring_slots_mask; 1035 int recirculate; 1036 uint32_t recirc_pass_id; 1037 1038 /* Structures. */ 1039 uint8_t **structs; 1040 1041 /* Packet headers. */ 1042 struct header_runtime *headers; /* Extracted or generated headers. */ 1043 struct header_out_runtime *headers_out; /* Emitted headers. */ 1044 uint8_t *header_storage; 1045 uint8_t *header_out_storage; 1046 uint64_t valid_headers; 1047 uint32_t n_headers_out; 1048 1049 /* Packet meta-data. */ 1050 uint8_t *metadata; 1051 1052 /* Tables. */ 1053 struct table_runtime *tables; 1054 struct selector_runtime *selectors; 1055 struct learner_runtime *learners; 1056 struct rte_swx_table_state *table_state; 1057 uint64_t action_id; 1058 size_t entry_id; 1059 int hit; /* 0 = Miss, 1 = Hit. */ 1060 uint32_t learner_id; 1061 uint64_t time; 1062 1063 /* Extern objects and functions. */ 1064 struct extern_obj_runtime *extern_objs; 1065 struct extern_func_runtime *extern_funcs; 1066 1067 /* Instructions. */ 1068 struct instruction *ip; 1069 struct instruction *ret; 1070 }; 1071 1072 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos))) 1073 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos))) 1074 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos))) 1075 1076 #define HEADER_VALID(thread, header_id) \ 1077 MASK64_BIT_GET((thread)->valid_headers, header_id) 1078 1079 static inline uint64_t 1080 instr_operand_hbo(struct thread *t, const struct instr_operand *x) 1081 { 1082 uint8_t *x_struct = t->structs[x->struct_id]; 1083 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset]; 1084 uint64_t x64 = *x64_ptr; 1085 uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits); 1086 1087 return x64 & x64_mask; 1088 } 1089 1090 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1091 1092 static inline uint64_t 1093 instr_operand_nbo(struct thread *t, const struct instr_operand *x) 1094 { 1095 uint8_t *x_struct = t->structs[x->struct_id]; 1096 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset]; 1097 uint64_t x64 = *x64_ptr; 1098 1099 return ntoh64(x64) >> (64 - x->n_bits); 1100 } 1101 1102 #else 1103 1104 #define instr_operand_nbo instr_operand_hbo 1105 1106 #endif 1107 1108 #define ALU(thread, ip, operator) \ 1109 { \ 1110 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1111 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1112 uint64_t dst64 = *dst64_ptr; \ 1113 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1114 uint64_t dst = dst64 & dst64_mask; \ 1115 \ 1116 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1117 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1118 uint64_t src64 = *src64_ptr; \ 1119 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \ 1120 uint64_t src = src64 & src64_mask; \ 1121 \ 1122 uint64_t result = dst operator src; \ 1123 \ 1124 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \ 1125 } 1126 1127 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1128 1129 #define ALU_MH(thread, ip, operator) \ 1130 { \ 1131 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1132 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1133 uint64_t dst64 = *dst64_ptr; \ 1134 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1135 uint64_t dst = dst64 & dst64_mask; \ 1136 \ 1137 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1138 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1139 uint64_t src64 = *src64_ptr; \ 1140 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \ 1141 \ 1142 uint64_t result = dst operator src; \ 1143 \ 1144 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \ 1145 } 1146 1147 #define ALU_HM(thread, ip, operator) \ 1148 { \ 1149 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1150 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1151 uint64_t dst64 = *dst64_ptr; \ 1152 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1153 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \ 1154 \ 1155 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1156 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1157 uint64_t src64 = *src64_ptr; \ 1158 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \ 1159 uint64_t src = src64 & src64_mask; \ 1160 \ 1161 uint64_t result = dst operator src; \ 1162 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \ 1163 \ 1164 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1165 } 1166 1167 #define ALU_HM_FAST(thread, ip, operator) \ 1168 { \ 1169 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1170 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1171 uint64_t dst64 = *dst64_ptr; \ 1172 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1173 uint64_t dst = dst64 & dst64_mask; \ 1174 \ 1175 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1176 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1177 uint64_t src64 = *src64_ptr; \ 1178 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \ 1179 uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \ 1180 \ 1181 uint64_t result = dst operator src; \ 1182 \ 1183 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1184 } 1185 1186 #define ALU_HH(thread, ip, operator) \ 1187 { \ 1188 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1189 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1190 uint64_t dst64 = *dst64_ptr; \ 1191 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1192 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \ 1193 \ 1194 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1195 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1196 uint64_t src64 = *src64_ptr; \ 1197 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \ 1198 \ 1199 uint64_t result = dst operator src; \ 1200 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \ 1201 \ 1202 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1203 } 1204 1205 #define ALU_HH_FAST(thread, ip, operator) \ 1206 { \ 1207 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1208 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1209 uint64_t dst64 = *dst64_ptr; \ 1210 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1211 uint64_t dst = dst64 & dst64_mask; \ 1212 \ 1213 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \ 1214 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \ 1215 uint64_t src64 = *src64_ptr; \ 1216 uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \ 1217 \ 1218 uint64_t result = dst operator src; \ 1219 \ 1220 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1221 } 1222 1223 #else 1224 1225 #define ALU_MH ALU 1226 #define ALU_HM ALU 1227 #define ALU_HM_FAST ALU 1228 #define ALU_HH ALU 1229 #define ALU_HH_FAST ALU 1230 1231 #endif 1232 1233 #define ALU_I(thread, ip, operator) \ 1234 { \ 1235 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1236 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1237 uint64_t dst64 = *dst64_ptr; \ 1238 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1239 uint64_t dst = dst64 & dst64_mask; \ 1240 \ 1241 uint64_t src = (ip)->alu.src_val; \ 1242 \ 1243 uint64_t result = dst operator src; \ 1244 \ 1245 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \ 1246 } 1247 1248 #define ALU_MI ALU_I 1249 1250 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1251 1252 #define ALU_HI(thread, ip, operator) \ 1253 { \ 1254 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ 1255 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \ 1256 uint64_t dst64 = *dst64_ptr; \ 1257 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \ 1258 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \ 1259 \ 1260 uint64_t src = (ip)->alu.src_val; \ 1261 \ 1262 uint64_t result = dst operator src; \ 1263 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \ 1264 \ 1265 *dst64_ptr = (dst64 & ~dst64_mask) | result; \ 1266 } 1267 1268 #else 1269 1270 #define ALU_HI ALU_I 1271 1272 #endif 1273 1274 #define MOV(thread, ip) \ 1275 { \ 1276 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1277 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1278 uint64_t dst64 = *dst64_ptr; \ 1279 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1280 \ 1281 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \ 1282 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \ 1283 uint64_t src64 = *src64_ptr; \ 1284 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \ 1285 uint64_t src = src64 & src64_mask; \ 1286 \ 1287 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \ 1288 } 1289 1290 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1291 1292 #define MOV_MH(thread, ip) \ 1293 { \ 1294 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1295 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1296 uint64_t dst64 = *dst64_ptr; \ 1297 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1298 \ 1299 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \ 1300 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \ 1301 uint64_t src64 = *src64_ptr; \ 1302 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \ 1303 \ 1304 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \ 1305 } 1306 1307 #define MOV_HM(thread, ip) \ 1308 { \ 1309 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1310 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1311 uint64_t dst64 = *dst64_ptr; \ 1312 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1313 \ 1314 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \ 1315 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \ 1316 uint64_t src64 = *src64_ptr; \ 1317 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \ 1318 uint64_t src = src64 & src64_mask; \ 1319 \ 1320 src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \ 1321 *dst64_ptr = (dst64 & ~dst64_mask) | src; \ 1322 } 1323 1324 #define MOV_HH(thread, ip) \ 1325 { \ 1326 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1327 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1328 uint64_t dst64 = *dst64_ptr; \ 1329 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1330 \ 1331 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \ 1332 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \ 1333 uint64_t src64 = *src64_ptr; \ 1334 \ 1335 uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \ 1336 src = src >> (64 - (ip)->mov.dst.n_bits); \ 1337 *dst64_ptr = (dst64 & ~dst64_mask) | src; \ 1338 } 1339 1340 #else 1341 1342 #define MOV_MH MOV 1343 #define MOV_HM MOV 1344 #define MOV_HH MOV 1345 1346 #endif 1347 1348 #define MOV_I(thread, ip) \ 1349 { \ 1350 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \ 1351 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \ 1352 uint64_t dst64 = *dst64_ptr; \ 1353 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \ 1354 \ 1355 uint64_t src = (ip)->mov.src_val; \ 1356 \ 1357 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \ 1358 } 1359 1360 #define JMP_CMP(thread, ip, operator) \ 1361 { \ 1362 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1363 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1364 uint64_t a64 = *a64_ptr; \ 1365 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \ 1366 uint64_t a = a64 & a64_mask; \ 1367 \ 1368 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1369 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1370 uint64_t b64 = *b64_ptr; \ 1371 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \ 1372 uint64_t b = b64 & b64_mask; \ 1373 \ 1374 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1375 } 1376 1377 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1378 1379 #define JMP_CMP_MH(thread, ip, operator) \ 1380 { \ 1381 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1382 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1383 uint64_t a64 = *a64_ptr; \ 1384 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \ 1385 uint64_t a = a64 & a64_mask; \ 1386 \ 1387 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1388 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1389 uint64_t b64 = *b64_ptr; \ 1390 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \ 1391 \ 1392 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1393 } 1394 1395 #define JMP_CMP_HM(thread, ip, operator) \ 1396 { \ 1397 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1398 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1399 uint64_t a64 = *a64_ptr; \ 1400 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \ 1401 \ 1402 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1403 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1404 uint64_t b64 = *b64_ptr; \ 1405 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \ 1406 uint64_t b = b64 & b64_mask; \ 1407 \ 1408 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1409 } 1410 1411 #define JMP_CMP_HH(thread, ip, operator) \ 1412 { \ 1413 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1414 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1415 uint64_t a64 = *a64_ptr; \ 1416 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \ 1417 \ 1418 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1419 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1420 uint64_t b64 = *b64_ptr; \ 1421 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \ 1422 \ 1423 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1424 } 1425 1426 #define JMP_CMP_HH_FAST(thread, ip, operator) \ 1427 { \ 1428 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1429 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1430 uint64_t a64 = *a64_ptr; \ 1431 uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \ 1432 \ 1433 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ 1434 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ 1435 uint64_t b64 = *b64_ptr; \ 1436 uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \ 1437 \ 1438 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1439 } 1440 1441 #else 1442 1443 #define JMP_CMP_MH JMP_CMP 1444 #define JMP_CMP_HM JMP_CMP 1445 #define JMP_CMP_HH JMP_CMP 1446 #define JMP_CMP_HH_FAST JMP_CMP 1447 1448 #endif 1449 1450 #define JMP_CMP_I(thread, ip, operator) \ 1451 { \ 1452 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1453 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1454 uint64_t a64 = *a64_ptr; \ 1455 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \ 1456 uint64_t a = a64 & a64_mask; \ 1457 \ 1458 uint64_t b = (ip)->jmp.b_val; \ 1459 \ 1460 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1461 } 1462 1463 #define JMP_CMP_MI JMP_CMP_I 1464 1465 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1466 1467 #define JMP_CMP_HI(thread, ip, operator) \ 1468 { \ 1469 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ 1470 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ 1471 uint64_t a64 = *a64_ptr; \ 1472 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \ 1473 \ 1474 uint64_t b = (ip)->jmp.b_val; \ 1475 \ 1476 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ 1477 } 1478 1479 #else 1480 1481 #define JMP_CMP_HI JMP_CMP_I 1482 1483 #endif 1484 1485 #define METADATA_READ(thread, offset, n_bits) \ 1486 ({ \ 1487 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \ 1488 uint64_t m64 = *m64_ptr; \ 1489 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \ 1490 (m64 & m64_mask); \ 1491 }) 1492 1493 #define METADATA_WRITE(thread, offset, n_bits, value) \ 1494 { \ 1495 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \ 1496 uint64_t m64 = *m64_ptr; \ 1497 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \ 1498 \ 1499 uint64_t m_new = value; \ 1500 \ 1501 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \ 1502 } 1503 1504 #ifndef RTE_SWX_PIPELINE_THREADS_MAX 1505 #define RTE_SWX_PIPELINE_THREADS_MAX 16 1506 #endif 1507 1508 #ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 1509 #define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 1024 1510 #endif 1511 1512 struct rte_swx_pipeline { 1513 char name[RTE_SWX_NAME_SIZE]; 1514 1515 struct struct_type_tailq struct_types; 1516 struct port_in_type_tailq port_in_types; 1517 struct port_in_tailq ports_in; 1518 struct port_out_type_tailq port_out_types; 1519 struct port_out_tailq ports_out; 1520 struct extern_type_tailq extern_types; 1521 struct extern_obj_tailq extern_objs; 1522 struct extern_func_tailq extern_funcs; 1523 struct hash_func_tailq hash_funcs; 1524 struct rss_tailq rss; 1525 struct header_tailq headers; 1526 struct struct_type *metadata_st; 1527 uint32_t metadata_struct_id; 1528 struct action_tailq actions; 1529 struct table_type_tailq table_types; 1530 struct table_tailq tables; 1531 struct selector_tailq selectors; 1532 struct learner_tailq learners; 1533 struct regarray_tailq regarrays; 1534 struct meter_profile_tailq meter_profiles; 1535 struct metarray_tailq metarrays; 1536 1537 struct port_in_runtime *in; 1538 struct port_out_runtime *out; 1539 struct mirroring_session *mirroring_sessions; 1540 struct instruction **action_instructions; 1541 action_func_t *action_funcs; 1542 struct rte_swx_table_state *table_state; 1543 struct table_statistics *table_stats; 1544 struct selector_statistics *selector_stats; 1545 struct learner_statistics *learner_stats; 1546 struct hash_func_runtime *hash_func_runtime; 1547 struct rss_runtime **rss_runtime; 1548 struct regarray_runtime *regarray_runtime; 1549 struct metarray_runtime *metarray_runtime; 1550 struct instruction *instructions; 1551 struct instruction_data *instruction_data; 1552 instr_exec_t *instruction_table; 1553 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX]; 1554 void *lib; 1555 1556 uint32_t n_structs; 1557 uint32_t n_ports_in; 1558 uint32_t n_ports_out; 1559 uint32_t n_mirroring_slots; 1560 uint32_t n_mirroring_sessions; 1561 uint32_t n_extern_objs; 1562 uint32_t n_extern_funcs; 1563 uint32_t n_hash_funcs; 1564 uint32_t n_rss; 1565 uint32_t n_actions; 1566 uint32_t n_tables; 1567 uint32_t n_selectors; 1568 uint32_t n_learners; 1569 uint32_t n_regarrays; 1570 uint32_t n_metarrays; 1571 uint32_t n_headers; 1572 uint32_t thread_id; 1573 uint32_t port_id; 1574 uint32_t n_instructions; 1575 int build_done; 1576 int numa_node; 1577 }; 1578 1579 /* 1580 * Instruction. 1581 */ 1582 static inline void 1583 pipeline_port_inc(struct rte_swx_pipeline *p) 1584 { 1585 p->port_id = (p->port_id + 1) & (p->n_ports_in - 1); 1586 } 1587 1588 static inline void 1589 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t) 1590 { 1591 t->ip = p->instructions; 1592 } 1593 1594 static inline void 1595 thread_ip_set(struct thread *t, struct instruction *ip) 1596 { 1597 t->ip = ip; 1598 } 1599 1600 static inline void 1601 thread_ip_action_call(struct rte_swx_pipeline *p, 1602 struct thread *t, 1603 uint32_t action_id) 1604 { 1605 t->ret = t->ip + 1; 1606 t->ip = p->action_instructions[action_id]; 1607 } 1608 1609 static inline void 1610 thread_ip_inc(struct rte_swx_pipeline *p); 1611 1612 static inline void 1613 thread_ip_inc(struct rte_swx_pipeline *p) 1614 { 1615 struct thread *t = &p->threads[p->thread_id]; 1616 1617 t->ip++; 1618 } 1619 1620 static inline void 1621 thread_ip_inc_cond(struct thread *t, int cond) 1622 { 1623 t->ip += cond; 1624 } 1625 1626 static inline void 1627 thread_yield(struct rte_swx_pipeline *p) 1628 { 1629 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1); 1630 } 1631 1632 static inline void 1633 thread_yield_cond(struct rte_swx_pipeline *p, int cond) 1634 { 1635 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1); 1636 } 1637 1638 /* 1639 * rx. 1640 */ 1641 static inline int 1642 __instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 1643 { 1644 struct port_in_runtime *port = &p->in[p->port_id]; 1645 struct rte_swx_pkt *pkt = &t->pkt; 1646 int pkt_received; 1647 1648 /* Recirculation: keep the current packet. */ 1649 if (t->recirculate) { 1650 TRACE("[Thread %2u] rx - recirculate (pass %u)\n", 1651 p->thread_id, 1652 t->recirc_pass_id + 1); 1653 1654 /* Packet. */ 1655 t->ptr = &pkt->pkt[pkt->offset]; 1656 t->mirroring_slots_mask = 0; 1657 t->recirculate = 0; 1658 t->recirc_pass_id++; 1659 1660 /* Headers. */ 1661 t->valid_headers = 0; 1662 t->n_headers_out = 0; 1663 1664 /* Tables. */ 1665 t->table_state = p->table_state; 1666 1667 return 1; 1668 } 1669 1670 /* Packet. */ 1671 pkt_received = port->pkt_rx(port->obj, pkt); 1672 t->ptr = &pkt->pkt[pkt->offset]; 1673 rte_prefetch0(t->ptr); 1674 1675 TRACE("[Thread %2u] rx %s from port %u\n", 1676 p->thread_id, 1677 pkt_received ? "1 pkt" : "0 pkts", 1678 p->port_id); 1679 1680 t->mirroring_slots_mask = 0; 1681 t->recirc_pass_id = 0; 1682 1683 /* Headers. */ 1684 t->valid_headers = 0; 1685 t->n_headers_out = 0; 1686 1687 /* Meta-data. */ 1688 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id); 1689 1690 /* Tables. */ 1691 t->table_state = p->table_state; 1692 1693 /* Thread. */ 1694 pipeline_port_inc(p); 1695 1696 return pkt_received; 1697 } 1698 1699 static inline void 1700 instr_rx_exec(struct rte_swx_pipeline *p) 1701 { 1702 struct thread *t = &p->threads[p->thread_id]; 1703 struct instruction *ip = t->ip; 1704 int pkt_received; 1705 1706 /* Packet. */ 1707 pkt_received = __instr_rx_exec(p, t, ip); 1708 1709 /* Thread. */ 1710 thread_ip_inc_cond(t, pkt_received); 1711 thread_yield(p); 1712 } 1713 1714 /* 1715 * tx. 1716 */ 1717 static inline void 1718 emit_handler(struct thread *t) 1719 { 1720 struct header_out_runtime *h0 = &t->headers_out[0]; 1721 struct header_out_runtime *h1 = &t->headers_out[1]; 1722 uint32_t offset = 0, i; 1723 1724 /* No header change or header decapsulation. */ 1725 if ((t->n_headers_out == 1) && 1726 (h0->ptr + h0->n_bytes == t->ptr)) { 1727 TRACE("Emit handler: no header change or header decap.\n"); 1728 1729 t->pkt.offset -= h0->n_bytes; 1730 t->pkt.length += h0->n_bytes; 1731 1732 return; 1733 } 1734 1735 /* Header encapsulation (optionally, with prior header decapsulation). */ 1736 if ((t->n_headers_out == 2) && 1737 (h1->ptr + h1->n_bytes == t->ptr) && 1738 (h0->ptr == h0->ptr0)) { 1739 uint32_t offset; 1740 1741 TRACE("Emit handler: header encapsulation.\n"); 1742 1743 offset = h0->n_bytes + h1->n_bytes; 1744 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes); 1745 t->pkt.offset -= offset; 1746 t->pkt.length += offset; 1747 1748 return; 1749 } 1750 1751 /* For any other case. */ 1752 TRACE("Emit handler: complex case.\n"); 1753 1754 for (i = 0; i < t->n_headers_out; i++) { 1755 struct header_out_runtime *h = &t->headers_out[i]; 1756 1757 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes); 1758 offset += h->n_bytes; 1759 } 1760 1761 if (offset) { 1762 memcpy(t->ptr - offset, t->header_out_storage, offset); 1763 t->pkt.offset -= offset; 1764 t->pkt.length += offset; 1765 } 1766 } 1767 1768 static inline void 1769 mirroring_handler(struct rte_swx_pipeline *p, struct thread *t, struct rte_swx_pkt *pkt) 1770 { 1771 uint64_t slots_mask = t->mirroring_slots_mask, slot_mask; 1772 uint32_t slot_id; 1773 1774 for (slot_id = 0, slot_mask = 1LLU ; slots_mask; slot_id++, slot_mask <<= 1) 1775 if (slot_mask & slots_mask) { 1776 struct port_out_runtime *port; 1777 struct mirroring_session *session; 1778 uint32_t port_id, session_id; 1779 1780 session_id = t->mirroring_slots[slot_id]; 1781 session = &p->mirroring_sessions[session_id]; 1782 1783 port_id = session->port_id; 1784 port = &p->out[port_id]; 1785 1786 if (session->fast_clone) 1787 port->pkt_fast_clone_tx(port->obj, pkt); 1788 else 1789 port->pkt_clone_tx(port->obj, pkt, session->truncation_length); 1790 1791 slots_mask &= ~slot_mask; 1792 } 1793 } 1794 1795 static inline void 1796 __instr_tx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 1797 { 1798 uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits); 1799 struct port_out_runtime *port = &p->out[port_id]; 1800 struct rte_swx_pkt *pkt = &t->pkt; 1801 1802 /* Recirculation: keep the current packet. */ 1803 if (t->recirculate) { 1804 TRACE("[Thread %2u]: tx 1 pkt - recirculate\n", 1805 p->thread_id); 1806 1807 /* Headers. */ 1808 emit_handler(t); 1809 1810 /* Packet. */ 1811 mirroring_handler(p, t, pkt); 1812 1813 return; 1814 } 1815 1816 TRACE("[Thread %2u]: tx 1 pkt to port %u\n", 1817 p->thread_id, 1818 (uint32_t)port_id); 1819 1820 /* Headers. */ 1821 emit_handler(t); 1822 1823 /* Packet. */ 1824 mirroring_handler(p, t, pkt); 1825 port->pkt_tx(port->obj, pkt); 1826 } 1827 1828 static inline void 1829 __instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 1830 { 1831 uint64_t port_id = ip->io.io.val; 1832 struct port_out_runtime *port = &p->out[port_id]; 1833 struct rte_swx_pkt *pkt = &t->pkt; 1834 1835 /* Recirculation: keep the current packet. */ 1836 if (t->recirculate) { 1837 TRACE("[Thread %2u]: tx (i) 1 pkt - recirculate\n", 1838 p->thread_id); 1839 1840 /* Headers. */ 1841 emit_handler(t); 1842 1843 /* Packet. */ 1844 mirroring_handler(p, t, pkt); 1845 1846 return; 1847 } 1848 1849 TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n", 1850 p->thread_id, 1851 (uint32_t)port_id); 1852 1853 /* Headers. */ 1854 emit_handler(t); 1855 1856 /* Packet. */ 1857 mirroring_handler(p, t, pkt); 1858 port->pkt_tx(port->obj, pkt); 1859 } 1860 1861 static inline void 1862 __instr_drop_exec(struct rte_swx_pipeline *p, 1863 struct thread *t, 1864 const struct instruction *ip __rte_unused) 1865 { 1866 uint64_t port_id = p->n_ports_out - 1; 1867 struct port_out_runtime *port = &p->out[port_id]; 1868 struct rte_swx_pkt *pkt = &t->pkt; 1869 1870 TRACE("[Thread %2u]: drop 1 pkt\n", 1871 p->thread_id); 1872 1873 /* Headers. */ 1874 emit_handler(t); 1875 1876 /* Packet. */ 1877 mirroring_handler(p, t, pkt); 1878 port->pkt_tx(port->obj, pkt); 1879 } 1880 1881 static inline void 1882 __instr_mirror_exec(struct rte_swx_pipeline *p, 1883 struct thread *t, 1884 const struct instruction *ip) 1885 { 1886 uint64_t slot_id = instr_operand_hbo(t, &ip->mirror.dst); 1887 uint64_t session_id = instr_operand_hbo(t, &ip->mirror.src); 1888 1889 slot_id &= p->n_mirroring_slots - 1; 1890 session_id &= p->n_mirroring_sessions - 1; 1891 1892 TRACE("[Thread %2u]: mirror pkt (slot = %u, session = %u)\n", 1893 p->thread_id, 1894 (uint32_t)slot_id, 1895 (uint32_t)session_id); 1896 1897 t->mirroring_slots[slot_id] = session_id; 1898 t->mirroring_slots_mask |= 1LLU << slot_id; 1899 } 1900 1901 static inline void 1902 __instr_recirculate_exec(struct rte_swx_pipeline *p __rte_unused, 1903 struct thread *t, 1904 const struct instruction *ip __rte_unused) 1905 { 1906 TRACE("[Thread %2u]: recirculate\n", 1907 p->thread_id); 1908 1909 t->recirculate = 1; 1910 } 1911 1912 static inline void 1913 __instr_recircid_exec(struct rte_swx_pipeline *p __rte_unused, 1914 struct thread *t, 1915 const struct instruction *ip) 1916 { 1917 TRACE("[Thread %2u]: recircid (pass %u)\n", 1918 p->thread_id, 1919 t->recirc_pass_id); 1920 1921 /* Meta-data. */ 1922 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, t->recirc_pass_id); 1923 } 1924 1925 /* 1926 * extract. 1927 */ 1928 static inline void 1929 __instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused, 1930 struct thread *t, 1931 const struct instruction *ip, 1932 uint32_t n_extract) 1933 { 1934 uint64_t valid_headers = t->valid_headers; 1935 uint8_t *ptr = t->ptr; 1936 uint32_t offset = t->pkt.offset; 1937 uint32_t length = t->pkt.length; 1938 uint32_t i; 1939 1940 for (i = 0; i < n_extract; i++) { 1941 uint32_t header_id = ip->io.hdr.header_id[i]; 1942 uint32_t struct_id = ip->io.hdr.struct_id[i]; 1943 uint32_t n_bytes = ip->io.hdr.n_bytes[i]; 1944 1945 TRACE("[Thread %2u]: extract header %u (%u bytes)\n", 1946 p->thread_id, 1947 header_id, 1948 n_bytes); 1949 1950 /* Headers. */ 1951 t->structs[struct_id] = ptr; 1952 valid_headers = MASK64_BIT_SET(valid_headers, header_id); 1953 1954 /* Packet. */ 1955 offset += n_bytes; 1956 length -= n_bytes; 1957 ptr += n_bytes; 1958 } 1959 1960 /* Headers. */ 1961 t->valid_headers = valid_headers; 1962 1963 /* Packet. */ 1964 t->pkt.offset = offset; 1965 t->pkt.length = length; 1966 t->ptr = ptr; 1967 } 1968 1969 static inline void 1970 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, 1971 struct thread *t, 1972 const struct instruction *ip) 1973 { 1974 __instr_hdr_extract_many_exec(p, t, ip, 1); 1975 } 1976 1977 static inline void 1978 __instr_hdr_extract2_exec(struct rte_swx_pipeline *p, 1979 struct thread *t, 1980 const struct instruction *ip) 1981 { 1982 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id); 1983 1984 __instr_hdr_extract_many_exec(p, t, ip, 2); 1985 } 1986 1987 static inline void 1988 __instr_hdr_extract3_exec(struct rte_swx_pipeline *p, 1989 struct thread *t, 1990 const struct instruction *ip) 1991 { 1992 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id); 1993 1994 __instr_hdr_extract_many_exec(p, t, ip, 3); 1995 } 1996 1997 static inline void 1998 __instr_hdr_extract4_exec(struct rte_swx_pipeline *p, 1999 struct thread *t, 2000 const struct instruction *ip) 2001 { 2002 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id); 2003 2004 __instr_hdr_extract_many_exec(p, t, ip, 4); 2005 } 2006 2007 static inline void 2008 __instr_hdr_extract5_exec(struct rte_swx_pipeline *p, 2009 struct thread *t, 2010 const struct instruction *ip) 2011 { 2012 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id); 2013 2014 __instr_hdr_extract_many_exec(p, t, ip, 5); 2015 } 2016 2017 static inline void 2018 __instr_hdr_extract6_exec(struct rte_swx_pipeline *p, 2019 struct thread *t, 2020 const struct instruction *ip) 2021 { 2022 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id); 2023 2024 __instr_hdr_extract_many_exec(p, t, ip, 6); 2025 } 2026 2027 static inline void 2028 __instr_hdr_extract7_exec(struct rte_swx_pipeline *p, 2029 struct thread *t, 2030 const struct instruction *ip) 2031 { 2032 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id); 2033 2034 __instr_hdr_extract_many_exec(p, t, ip, 7); 2035 } 2036 2037 static inline void 2038 __instr_hdr_extract8_exec(struct rte_swx_pipeline *p, 2039 struct thread *t, 2040 const struct instruction *ip) 2041 { 2042 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id); 2043 2044 __instr_hdr_extract_many_exec(p, t, ip, 8); 2045 } 2046 2047 static inline void 2048 __instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused, 2049 struct thread *t, 2050 const struct instruction *ip) 2051 { 2052 uint64_t valid_headers = t->valid_headers; 2053 uint8_t *ptr = t->ptr; 2054 uint32_t offset = t->pkt.offset; 2055 uint32_t length = t->pkt.length; 2056 2057 uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits); 2058 uint32_t header_id = ip->io.hdr.header_id[0]; 2059 uint32_t struct_id = ip->io.hdr.struct_id[0]; 2060 uint32_t n_bytes = ip->io.hdr.n_bytes[0]; 2061 2062 struct header_runtime *h = &t->headers[header_id]; 2063 2064 TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n", 2065 p->thread_id, 2066 header_id, 2067 n_bytes, 2068 n_bytes_last); 2069 2070 n_bytes += n_bytes_last; 2071 2072 /* Headers. */ 2073 t->structs[struct_id] = ptr; 2074 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id); 2075 h->n_bytes = n_bytes; 2076 2077 /* Packet. */ 2078 t->pkt.offset = offset + n_bytes; 2079 t->pkt.length = length - n_bytes; 2080 t->ptr = ptr + n_bytes; 2081 } 2082 2083 static inline void 2084 __instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused, 2085 struct thread *t, 2086 const struct instruction *ip) 2087 { 2088 uint64_t valid_headers = t->valid_headers; 2089 uint8_t *ptr = t->ptr; 2090 2091 uint32_t header_id = ip->io.hdr.header_id[0]; 2092 uint32_t struct_id = ip->io.hdr.struct_id[0]; 2093 2094 TRACE("[Thread %2u]: lookahead header %u\n", 2095 p->thread_id, 2096 header_id); 2097 2098 /* Headers. */ 2099 t->structs[struct_id] = ptr; 2100 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id); 2101 } 2102 2103 /* 2104 * emit. 2105 */ 2106 static inline void 2107 __instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused, 2108 struct thread *t, 2109 const struct instruction *ip, 2110 uint32_t n_emit) 2111 { 2112 uint64_t valid_headers = t->valid_headers; 2113 uint32_t n_headers_out = t->n_headers_out; 2114 struct header_out_runtime *ho = NULL; 2115 uint8_t *ho_ptr = NULL; 2116 uint32_t ho_nbytes = 0, i; 2117 2118 for (i = 0; i < n_emit; i++) { 2119 uint32_t header_id = ip->io.hdr.header_id[i]; 2120 uint32_t struct_id = ip->io.hdr.struct_id[i]; 2121 2122 struct header_runtime *hi = &t->headers[header_id]; 2123 uint8_t *hi_ptr0 = hi->ptr0; 2124 uint32_t n_bytes = hi->n_bytes; 2125 2126 uint8_t *hi_ptr = t->structs[struct_id]; 2127 2128 if (!MASK64_BIT_GET(valid_headers, header_id)) { 2129 TRACE("[Thread %2u]: emit header %u (invalid)\n", 2130 p->thread_id, 2131 header_id); 2132 2133 continue; 2134 } 2135 2136 TRACE("[Thread %2u]: emit header %u (valid)\n", 2137 p->thread_id, 2138 header_id); 2139 2140 /* Headers. */ 2141 if (!ho) { 2142 if (!n_headers_out) { 2143 ho = &t->headers_out[0]; 2144 2145 ho->ptr0 = hi_ptr0; 2146 ho->ptr = hi_ptr; 2147 2148 ho_ptr = hi_ptr; 2149 ho_nbytes = n_bytes; 2150 2151 n_headers_out = 1; 2152 2153 continue; 2154 } else { 2155 ho = &t->headers_out[n_headers_out - 1]; 2156 2157 ho_ptr = ho->ptr; 2158 ho_nbytes = ho->n_bytes; 2159 } 2160 } 2161 2162 if (ho_ptr + ho_nbytes == hi_ptr) { 2163 ho_nbytes += n_bytes; 2164 } else { 2165 ho->n_bytes = ho_nbytes; 2166 2167 ho++; 2168 ho->ptr0 = hi_ptr0; 2169 ho->ptr = hi_ptr; 2170 2171 ho_ptr = hi_ptr; 2172 ho_nbytes = n_bytes; 2173 2174 n_headers_out++; 2175 } 2176 } 2177 2178 if (ho) 2179 ho->n_bytes = ho_nbytes; 2180 t->n_headers_out = n_headers_out; 2181 } 2182 2183 static inline void 2184 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, 2185 struct thread *t, 2186 const struct instruction *ip) 2187 { 2188 __instr_hdr_emit_many_exec(p, t, ip, 1); 2189 } 2190 2191 static inline void 2192 __instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p, 2193 struct thread *t, 2194 const struct instruction *ip) 2195 { 2196 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id); 2197 2198 __instr_hdr_emit_many_exec(p, t, ip, 1); 2199 __instr_tx_exec(p, t, ip); 2200 } 2201 2202 static inline void 2203 __instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p, 2204 struct thread *t, 2205 const struct instruction *ip) 2206 { 2207 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id); 2208 2209 __instr_hdr_emit_many_exec(p, t, ip, 2); 2210 __instr_tx_exec(p, t, ip); 2211 } 2212 2213 static inline void 2214 __instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p, 2215 struct thread *t, 2216 const struct instruction *ip) 2217 { 2218 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id); 2219 2220 __instr_hdr_emit_many_exec(p, t, ip, 3); 2221 __instr_tx_exec(p, t, ip); 2222 } 2223 2224 static inline void 2225 __instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p, 2226 struct thread *t, 2227 const struct instruction *ip) 2228 { 2229 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id); 2230 2231 __instr_hdr_emit_many_exec(p, t, ip, 4); 2232 __instr_tx_exec(p, t, ip); 2233 } 2234 2235 static inline void 2236 __instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p, 2237 struct thread *t, 2238 const struct instruction *ip) 2239 { 2240 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id); 2241 2242 __instr_hdr_emit_many_exec(p, t, ip, 5); 2243 __instr_tx_exec(p, t, ip); 2244 } 2245 2246 static inline void 2247 __instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p, 2248 struct thread *t, 2249 const struct instruction *ip) 2250 { 2251 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id); 2252 2253 __instr_hdr_emit_many_exec(p, t, ip, 6); 2254 __instr_tx_exec(p, t, ip); 2255 } 2256 2257 static inline void 2258 __instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p, 2259 struct thread *t, 2260 const struct instruction *ip) 2261 { 2262 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id); 2263 2264 __instr_hdr_emit_many_exec(p, t, ip, 7); 2265 __instr_tx_exec(p, t, ip); 2266 } 2267 2268 static inline void 2269 __instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p, 2270 struct thread *t, 2271 const struct instruction *ip) 2272 { 2273 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id); 2274 2275 __instr_hdr_emit_many_exec(p, t, ip, 8); 2276 __instr_tx_exec(p, t, ip); 2277 } 2278 2279 /* 2280 * validate. 2281 */ 2282 static inline void 2283 __instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused, 2284 struct thread *t, 2285 const struct instruction *ip) 2286 { 2287 uint32_t header_id = ip->valid.header_id; 2288 uint32_t struct_id = ip->valid.struct_id; 2289 uint64_t valid_headers = t->valid_headers; 2290 struct header_runtime *h = &t->headers[header_id]; 2291 2292 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id); 2293 2294 /* If this header is already valid, then its associated t->structs[] element is also valid 2295 * and therefore it should not be modified. It could point to the packet buffer (in case of 2296 * extracted header) and setting it to the default location (h->ptr0) would be incorrect. 2297 */ 2298 if (MASK64_BIT_GET(valid_headers, header_id)) 2299 return; 2300 2301 /* Headers. */ 2302 t->structs[struct_id] = h->ptr0; 2303 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id); 2304 } 2305 2306 /* 2307 * invalidate. 2308 */ 2309 static inline void 2310 __instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused, 2311 struct thread *t, 2312 const struct instruction *ip) 2313 { 2314 uint32_t header_id = ip->valid.header_id; 2315 2316 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id); 2317 2318 /* Headers. */ 2319 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id); 2320 } 2321 2322 /* 2323 * learn. 2324 */ 2325 static inline void 2326 __instr_learn_exec(struct rte_swx_pipeline *p, 2327 struct thread *t, 2328 const struct instruction *ip) 2329 { 2330 uint64_t action_id = ip->learn.action_id; 2331 uint32_t mf_first_arg_offset = ip->learn.mf_first_arg_offset; 2332 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset, 2333 ip->learn.mf_timeout_id_n_bits); 2334 uint32_t learner_id = t->learner_id; 2335 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + 2336 p->n_selectors + learner_id]; 2337 struct learner_runtime *l = &t->learners[learner_id]; 2338 struct learner_statistics *stats = &p->learner_stats[learner_id]; 2339 uint32_t status; 2340 2341 /* Table. */ 2342 status = rte_swx_table_learner_add(ts->obj, 2343 l->mailbox, 2344 t->time, 2345 action_id, 2346 &t->metadata[mf_first_arg_offset], 2347 timeout_id); 2348 2349 TRACE("[Thread %2u] learner %u learn %s\n", 2350 p->thread_id, 2351 learner_id, 2352 status ? "ok" : "error"); 2353 2354 stats->n_pkts_learn[status] += 1; 2355 } 2356 2357 /* 2358 * rearm. 2359 */ 2360 static inline void 2361 __instr_rearm_exec(struct rte_swx_pipeline *p, 2362 struct thread *t, 2363 const struct instruction *ip __rte_unused) 2364 { 2365 uint32_t learner_id = t->learner_id; 2366 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + 2367 p->n_selectors + learner_id]; 2368 struct learner_runtime *l = &t->learners[learner_id]; 2369 struct learner_statistics *stats = &p->learner_stats[learner_id]; 2370 2371 /* Table. */ 2372 rte_swx_table_learner_rearm(ts->obj, l->mailbox, t->time); 2373 2374 TRACE("[Thread %2u] learner %u rearm\n", 2375 p->thread_id, 2376 learner_id); 2377 2378 stats->n_pkts_rearm += 1; 2379 } 2380 2381 static inline void 2382 __instr_rearm_new_exec(struct rte_swx_pipeline *p, 2383 struct thread *t, 2384 const struct instruction *ip) 2385 { 2386 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset, 2387 ip->learn.mf_timeout_id_n_bits); 2388 uint32_t learner_id = t->learner_id; 2389 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + 2390 p->n_selectors + learner_id]; 2391 struct learner_runtime *l = &t->learners[learner_id]; 2392 struct learner_statistics *stats = &p->learner_stats[learner_id]; 2393 2394 /* Table. */ 2395 rte_swx_table_learner_rearm_new(ts->obj, l->mailbox, t->time, timeout_id); 2396 2397 TRACE("[Thread %2u] learner %u rearm with timeout ID %u\n", 2398 p->thread_id, 2399 learner_id, 2400 timeout_id); 2401 2402 stats->n_pkts_rearm += 1; 2403 } 2404 2405 /* 2406 * forget. 2407 */ 2408 static inline void 2409 __instr_forget_exec(struct rte_swx_pipeline *p, 2410 struct thread *t, 2411 const struct instruction *ip __rte_unused) 2412 { 2413 uint32_t learner_id = t->learner_id; 2414 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + 2415 p->n_selectors + learner_id]; 2416 struct learner_runtime *l = &t->learners[learner_id]; 2417 struct learner_statistics *stats = &p->learner_stats[learner_id]; 2418 2419 /* Table. */ 2420 rte_swx_table_learner_delete(ts->obj, l->mailbox); 2421 2422 TRACE("[Thread %2u] learner %u forget\n", 2423 p->thread_id, 2424 learner_id); 2425 2426 stats->n_pkts_forget += 1; 2427 } 2428 2429 /* 2430 * entryid. 2431 */ 2432 static inline void 2433 __instr_entryid_exec(struct rte_swx_pipeline *p __rte_unused, 2434 struct thread *t, 2435 const struct instruction *ip) 2436 { 2437 TRACE("[Thread %2u]: entryid\n", 2438 p->thread_id); 2439 2440 /* Meta-data. */ 2441 METADATA_WRITE(t, ip->mov.dst.offset, ip->mov.dst.n_bits, t->entry_id); 2442 } 2443 2444 /* 2445 * extern. 2446 */ 2447 static inline uint32_t 2448 __instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused, 2449 struct thread *t, 2450 const struct instruction *ip) 2451 { 2452 uint32_t obj_id = ip->ext_obj.ext_obj_id; 2453 uint32_t func_id = ip->ext_obj.func_id; 2454 struct extern_obj_runtime *obj = &t->extern_objs[obj_id]; 2455 rte_swx_extern_type_member_func_t func = obj->funcs[func_id]; 2456 uint32_t done; 2457 2458 TRACE("[Thread %2u] extern obj %u member func %u\n", 2459 p->thread_id, 2460 obj_id, 2461 func_id); 2462 2463 done = func(obj->obj, obj->mailbox); 2464 2465 return done; 2466 } 2467 2468 static inline uint32_t 2469 __instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused, 2470 struct thread *t, 2471 const struct instruction *ip) 2472 { 2473 uint32_t ext_func_id = ip->ext_func.ext_func_id; 2474 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id]; 2475 rte_swx_extern_func_t func = ext_func->func; 2476 uint32_t done; 2477 2478 TRACE("[Thread %2u] extern func %u\n", 2479 p->thread_id, 2480 ext_func_id); 2481 2482 done = func(ext_func->mailbox); 2483 2484 return done; 2485 } 2486 2487 /* 2488 * hash. 2489 */ 2490 static inline void 2491 __instr_hash_func_exec(struct rte_swx_pipeline *p, 2492 struct thread *t, 2493 const struct instruction *ip) 2494 { 2495 uint32_t hash_func_id = ip->hash_func.hash_func_id; 2496 uint32_t dst_offset = ip->hash_func.dst.offset; 2497 uint32_t n_dst_bits = ip->hash_func.dst.n_bits; 2498 uint32_t src_struct_id = ip->hash_func.src.struct_id; 2499 uint32_t src_offset = ip->hash_func.src.offset; 2500 uint32_t n_src_bytes = ip->hash_func.src.n_bytes; 2501 2502 struct hash_func_runtime *func = &p->hash_func_runtime[hash_func_id]; 2503 uint8_t *src_ptr = t->structs[src_struct_id]; 2504 uint32_t result; 2505 2506 TRACE("[Thread %2u] hash %u\n", 2507 p->thread_id, 2508 hash_func_id); 2509 2510 result = func->func(&src_ptr[src_offset], n_src_bytes, 0); 2511 METADATA_WRITE(t, dst_offset, n_dst_bits, result); 2512 } 2513 2514 /* 2515 * rss. 2516 */ 2517 static inline uint32_t 2518 rss_func(void *rss_key, uint32_t rss_key_size, void *input_data, uint32_t input_data_size) 2519 { 2520 uint32_t *key = (uint32_t *)rss_key; 2521 uint32_t *data = (uint32_t *)input_data; 2522 uint32_t key_size = rss_key_size >> 2; 2523 uint32_t data_size = input_data_size >> 2; 2524 uint32_t hash_val = 0, i; 2525 2526 for (i = 0; i < data_size; i++) { 2527 uint32_t d; 2528 2529 for (d = data[i]; d; d &= (d - 1)) { 2530 uint32_t key0, key1, pos; 2531 2532 pos = rte_bsf32(d); 2533 key0 = key[i % key_size] << (31 - pos); 2534 key1 = key[(i + 1) % key_size] >> (pos + 1); 2535 hash_val ^= key0 | key1; 2536 } 2537 } 2538 2539 return hash_val; 2540 } 2541 2542 static inline void 2543 __instr_rss_exec(struct rte_swx_pipeline *p, 2544 struct thread *t, 2545 const struct instruction *ip) 2546 { 2547 uint32_t rss_obj_id = ip->rss.rss_obj_id; 2548 uint32_t dst_offset = ip->rss.dst.offset; 2549 uint32_t n_dst_bits = ip->rss.dst.n_bits; 2550 uint32_t src_struct_id = ip->rss.src.struct_id; 2551 uint32_t src_offset = ip->rss.src.offset; 2552 uint32_t n_src_bytes = ip->rss.src.n_bytes; 2553 2554 struct rss_runtime *r = p->rss_runtime[rss_obj_id]; 2555 uint8_t *src_ptr = t->structs[src_struct_id]; 2556 uint32_t result; 2557 2558 TRACE("[Thread %2u] rss %u\n", 2559 p->thread_id, 2560 rss_obj_id); 2561 2562 result = rss_func(r->key, r->key_size, &src_ptr[src_offset], n_src_bytes); 2563 METADATA_WRITE(t, dst_offset, n_dst_bits, result); 2564 } 2565 2566 /* 2567 * mov. 2568 */ 2569 static inline void 2570 __instr_mov_exec(struct rte_swx_pipeline *p __rte_unused, 2571 struct thread *t, 2572 const struct instruction *ip) 2573 { 2574 TRACE("[Thread %2u] mov\n", p->thread_id); 2575 2576 MOV(t, ip); 2577 } 2578 2579 static inline void 2580 __instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2581 struct thread *t, 2582 const struct instruction *ip) 2583 { 2584 TRACE("[Thread %2u] mov (mh)\n", p->thread_id); 2585 2586 MOV_MH(t, ip); 2587 } 2588 2589 static inline void 2590 __instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2591 struct thread *t, 2592 const struct instruction *ip) 2593 { 2594 TRACE("[Thread %2u] mov (hm)\n", p->thread_id); 2595 2596 MOV_HM(t, ip); 2597 } 2598 2599 static inline void 2600 __instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2601 struct thread *t, 2602 const struct instruction *ip) 2603 { 2604 TRACE("[Thread %2u] mov (hh)\n", p->thread_id); 2605 2606 MOV_HH(t, ip); 2607 } 2608 2609 static inline void 2610 __instr_mov_dma_exec(struct rte_swx_pipeline *p __rte_unused, 2611 struct thread *t, 2612 const struct instruction *ip) 2613 { 2614 uint8_t *dst_struct = t->structs[ip->mov.dst.struct_id]; 2615 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->mov.dst.offset]; 2616 uint32_t *dst32_ptr; 2617 uint16_t *dst16_ptr; 2618 uint8_t *dst8_ptr; 2619 2620 uint8_t *src_struct = t->structs[ip->mov.src.struct_id]; 2621 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->mov.src.offset]; 2622 uint32_t *src32_ptr; 2623 uint16_t *src16_ptr; 2624 uint8_t *src8_ptr; 2625 2626 uint32_t n = ip->mov.dst.n_bits >> 3, i; 2627 2628 TRACE("[Thread %2u] mov (dma) %u bytes\n", p->thread_id, n); 2629 2630 /* 8-byte transfers. */ 2631 for (i = 0; i < n >> 3; i++) 2632 *dst64_ptr++ = *src64_ptr++; 2633 2634 /* 4-byte transfers. */ 2635 n &= 7; 2636 dst32_ptr = (uint32_t *)dst64_ptr; 2637 src32_ptr = (uint32_t *)src64_ptr; 2638 2639 for (i = 0; i < n >> 2; i++) 2640 *dst32_ptr++ = *src32_ptr++; 2641 2642 /* 2-byte transfers. */ 2643 n &= 3; 2644 dst16_ptr = (uint16_t *)dst32_ptr; 2645 src16_ptr = (uint16_t *)src32_ptr; 2646 2647 for (i = 0; i < n >> 1; i++) 2648 *dst16_ptr++ = *src16_ptr++; 2649 2650 /* 1-byte transfer. */ 2651 n &= 1; 2652 dst8_ptr = (uint8_t *)dst16_ptr; 2653 src8_ptr = (uint8_t *)src16_ptr; 2654 if (n) 2655 *dst8_ptr = *src8_ptr; 2656 } 2657 2658 static inline void 2659 __instr_mov_128_exec(struct rte_swx_pipeline *p __rte_unused, 2660 struct thread *t, 2661 const struct instruction *ip) 2662 { 2663 uint8_t *dst_struct = t->structs[ip->mov.dst.struct_id]; 2664 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->mov.dst.offset]; 2665 2666 uint8_t *src_struct = t->structs[ip->mov.src.struct_id]; 2667 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->mov.src.offset]; 2668 2669 TRACE("[Thread %2u] mov (128)\n", p->thread_id); 2670 2671 dst64_ptr[0] = src64_ptr[0]; 2672 dst64_ptr[1] = src64_ptr[1]; 2673 } 2674 2675 static inline void 2676 __instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused, 2677 struct thread *t, 2678 const struct instruction *ip) 2679 { 2680 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val); 2681 2682 MOV_I(t, ip); 2683 } 2684 2685 /* 2686 * dma. 2687 */ 2688 static inline void 2689 __instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused, 2690 struct thread *t, 2691 const struct instruction *ip, 2692 uint32_t n_dma) 2693 { 2694 uint8_t *action_data = t->structs[0]; 2695 uint64_t valid_headers = t->valid_headers; 2696 uint32_t i; 2697 2698 for (i = 0; i < n_dma; i++) { 2699 uint32_t header_id = ip->dma.dst.header_id[i]; 2700 uint32_t struct_id = ip->dma.dst.struct_id[i]; 2701 uint32_t offset = ip->dma.src.offset[i]; 2702 uint32_t n_bytes = ip->dma.n_bytes[i]; 2703 2704 struct header_runtime *h = &t->headers[header_id]; 2705 uint8_t *h_ptr0 = h->ptr0; 2706 uint8_t *h_ptr = t->structs[struct_id]; 2707 2708 void *dst = MASK64_BIT_GET(valid_headers, header_id) ? 2709 h_ptr : h_ptr0; 2710 void *src = &action_data[offset]; 2711 2712 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id); 2713 2714 /* Headers. */ 2715 memcpy(dst, src, n_bytes); 2716 t->structs[struct_id] = dst; 2717 valid_headers = MASK64_BIT_SET(valid_headers, header_id); 2718 } 2719 2720 t->valid_headers = valid_headers; 2721 } 2722 2723 static inline void 2724 __instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2725 { 2726 __instr_dma_ht_many_exec(p, t, ip, 1); 2727 } 2728 2729 static inline void 2730 __instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2731 { 2732 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id); 2733 2734 __instr_dma_ht_many_exec(p, t, ip, 2); 2735 } 2736 2737 static inline void 2738 __instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2739 { 2740 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id); 2741 2742 __instr_dma_ht_many_exec(p, t, ip, 3); 2743 } 2744 2745 static inline void 2746 __instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2747 { 2748 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id); 2749 2750 __instr_dma_ht_many_exec(p, t, ip, 4); 2751 } 2752 2753 static inline void 2754 __instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2755 { 2756 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id); 2757 2758 __instr_dma_ht_many_exec(p, t, ip, 5); 2759 } 2760 2761 static inline void 2762 __instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2763 { 2764 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id); 2765 2766 __instr_dma_ht_many_exec(p, t, ip, 6); 2767 } 2768 2769 static inline void 2770 __instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2771 { 2772 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id); 2773 2774 __instr_dma_ht_many_exec(p, t, ip, 7); 2775 } 2776 2777 static inline void 2778 __instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 2779 { 2780 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id); 2781 2782 __instr_dma_ht_many_exec(p, t, ip, 8); 2783 } 2784 2785 /* 2786 * alu. 2787 */ 2788 static inline void 2789 __instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused, 2790 struct thread *t, 2791 const struct instruction *ip) 2792 { 2793 TRACE("[Thread %2u] add\n", p->thread_id); 2794 2795 ALU(t, ip, +); 2796 } 2797 2798 static inline void 2799 __instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2800 struct thread *t, 2801 const struct instruction *ip) 2802 { 2803 TRACE("[Thread %2u] add (mh)\n", p->thread_id); 2804 2805 ALU_MH(t, ip, +); 2806 } 2807 2808 static inline void 2809 __instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2810 struct thread *t, 2811 const struct instruction *ip) 2812 { 2813 TRACE("[Thread %2u] add (hm)\n", p->thread_id); 2814 2815 ALU_HM(t, ip, +); 2816 } 2817 2818 static inline void 2819 __instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2820 struct thread *t, 2821 const struct instruction *ip) 2822 { 2823 TRACE("[Thread %2u] add (hh)\n", p->thread_id); 2824 2825 ALU_HH(t, ip, +); 2826 } 2827 2828 static inline void 2829 __instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused, 2830 struct thread *t, 2831 const struct instruction *ip) 2832 { 2833 TRACE("[Thread %2u] add (mi)\n", p->thread_id); 2834 2835 ALU_MI(t, ip, +); 2836 } 2837 2838 static inline void 2839 __instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused, 2840 struct thread *t, 2841 const struct instruction *ip) 2842 { 2843 TRACE("[Thread %2u] add (hi)\n", p->thread_id); 2844 2845 ALU_HI(t, ip, +); 2846 } 2847 2848 static inline void 2849 __instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused, 2850 struct thread *t, 2851 const struct instruction *ip) 2852 { 2853 TRACE("[Thread %2u] sub\n", p->thread_id); 2854 2855 ALU(t, ip, -); 2856 } 2857 2858 static inline void 2859 __instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2860 struct thread *t, 2861 const struct instruction *ip) 2862 { 2863 TRACE("[Thread %2u] sub (mh)\n", p->thread_id); 2864 2865 ALU_MH(t, ip, -); 2866 } 2867 2868 static inline void 2869 __instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2870 struct thread *t, 2871 const struct instruction *ip) 2872 { 2873 TRACE("[Thread %2u] sub (hm)\n", p->thread_id); 2874 2875 ALU_HM(t, ip, -); 2876 } 2877 2878 static inline void 2879 __instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2880 struct thread *t, 2881 const struct instruction *ip) 2882 { 2883 TRACE("[Thread %2u] sub (hh)\n", p->thread_id); 2884 2885 ALU_HH(t, ip, -); 2886 } 2887 2888 static inline void 2889 __instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused, 2890 struct thread *t, 2891 const struct instruction *ip) 2892 { 2893 TRACE("[Thread %2u] sub (mi)\n", p->thread_id); 2894 2895 ALU_MI(t, ip, -); 2896 } 2897 2898 static inline void 2899 __instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused, 2900 struct thread *t, 2901 const struct instruction *ip) 2902 { 2903 TRACE("[Thread %2u] sub (hi)\n", p->thread_id); 2904 2905 ALU_HI(t, ip, -); 2906 } 2907 2908 static inline void 2909 __instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused, 2910 struct thread *t, 2911 const struct instruction *ip) 2912 { 2913 TRACE("[Thread %2u] shl\n", p->thread_id); 2914 2915 ALU(t, ip, <<); 2916 } 2917 2918 static inline void 2919 __instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2920 struct thread *t, 2921 const struct instruction *ip) 2922 { 2923 TRACE("[Thread %2u] shl (mh)\n", p->thread_id); 2924 2925 ALU_MH(t, ip, <<); 2926 } 2927 2928 static inline void 2929 __instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2930 struct thread *t, 2931 const struct instruction *ip) 2932 { 2933 TRACE("[Thread %2u] shl (hm)\n", p->thread_id); 2934 2935 ALU_HM(t, ip, <<); 2936 } 2937 2938 static inline void 2939 __instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused, 2940 struct thread *t, 2941 const struct instruction *ip) 2942 { 2943 TRACE("[Thread %2u] shl (hh)\n", p->thread_id); 2944 2945 ALU_HH(t, ip, <<); 2946 } 2947 2948 static inline void 2949 __instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused, 2950 struct thread *t, 2951 const struct instruction *ip) 2952 { 2953 TRACE("[Thread %2u] shl (mi)\n", p->thread_id); 2954 2955 ALU_MI(t, ip, <<); 2956 } 2957 2958 static inline void 2959 __instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused, 2960 struct thread *t, 2961 const struct instruction *ip) 2962 { 2963 TRACE("[Thread %2u] shl (hi)\n", p->thread_id); 2964 2965 ALU_HI(t, ip, <<); 2966 } 2967 2968 static inline void 2969 __instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused, 2970 struct thread *t, 2971 const struct instruction *ip) 2972 { 2973 TRACE("[Thread %2u] shr\n", p->thread_id); 2974 2975 ALU(t, ip, >>); 2976 } 2977 2978 static inline void 2979 __instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused, 2980 struct thread *t, 2981 const struct instruction *ip) 2982 { 2983 TRACE("[Thread %2u] shr (mh)\n", p->thread_id); 2984 2985 ALU_MH(t, ip, >>); 2986 } 2987 2988 static inline void 2989 __instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused, 2990 struct thread *t, 2991 const struct instruction *ip) 2992 { 2993 TRACE("[Thread %2u] shr (hm)\n", p->thread_id); 2994 2995 ALU_HM(t, ip, >>); 2996 } 2997 2998 static inline void 2999 __instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused, 3000 struct thread *t, 3001 const struct instruction *ip) 3002 { 3003 TRACE("[Thread %2u] shr (hh)\n", p->thread_id); 3004 3005 ALU_HH(t, ip, >>); 3006 } 3007 3008 static inline void 3009 __instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused, 3010 struct thread *t, 3011 const struct instruction *ip) 3012 { 3013 TRACE("[Thread %2u] shr (mi)\n", p->thread_id); 3014 3015 /* Structs. */ 3016 ALU_MI(t, ip, >>); 3017 } 3018 3019 static inline void 3020 __instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused, 3021 struct thread *t, 3022 const struct instruction *ip) 3023 { 3024 TRACE("[Thread %2u] shr (hi)\n", p->thread_id); 3025 3026 ALU_HI(t, ip, >>); 3027 } 3028 3029 static inline void 3030 __instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused, 3031 struct thread *t, 3032 const struct instruction *ip) 3033 { 3034 TRACE("[Thread %2u] and\n", p->thread_id); 3035 3036 ALU(t, ip, &); 3037 } 3038 3039 static inline void 3040 __instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused, 3041 struct thread *t, 3042 const struct instruction *ip) 3043 { 3044 TRACE("[Thread %2u] and (mh)\n", p->thread_id); 3045 3046 ALU_MH(t, ip, &); 3047 } 3048 3049 static inline void 3050 __instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused, 3051 struct thread *t, 3052 const struct instruction *ip) 3053 { 3054 TRACE("[Thread %2u] and (hm)\n", p->thread_id); 3055 3056 ALU_HM_FAST(t, ip, &); 3057 } 3058 3059 static inline void 3060 __instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused, 3061 struct thread *t, 3062 const struct instruction *ip) 3063 { 3064 TRACE("[Thread %2u] and (hh)\n", p->thread_id); 3065 3066 ALU_HH_FAST(t, ip, &); 3067 } 3068 3069 static inline void 3070 __instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused, 3071 struct thread *t, 3072 const struct instruction *ip) 3073 { 3074 TRACE("[Thread %2u] and (i)\n", p->thread_id); 3075 3076 ALU_I(t, ip, &); 3077 } 3078 3079 static inline void 3080 __instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused, 3081 struct thread *t, 3082 const struct instruction *ip) 3083 { 3084 TRACE("[Thread %2u] or\n", p->thread_id); 3085 3086 ALU(t, ip, |); 3087 } 3088 3089 static inline void 3090 __instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused, 3091 struct thread *t, 3092 const struct instruction *ip) 3093 { 3094 TRACE("[Thread %2u] or (mh)\n", p->thread_id); 3095 3096 ALU_MH(t, ip, |); 3097 } 3098 3099 static inline void 3100 __instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused, 3101 struct thread *t, 3102 const struct instruction *ip) 3103 { 3104 TRACE("[Thread %2u] or (hm)\n", p->thread_id); 3105 3106 ALU_HM_FAST(t, ip, |); 3107 } 3108 3109 static inline void 3110 __instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused, 3111 struct thread *t, 3112 const struct instruction *ip) 3113 { 3114 TRACE("[Thread %2u] or (hh)\n", p->thread_id); 3115 3116 ALU_HH_FAST(t, ip, |); 3117 } 3118 3119 static inline void 3120 __instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused, 3121 struct thread *t, 3122 const struct instruction *ip) 3123 { 3124 TRACE("[Thread %2u] or (i)\n", p->thread_id); 3125 3126 ALU_I(t, ip, |); 3127 } 3128 3129 static inline void 3130 __instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused, 3131 struct thread *t, 3132 const struct instruction *ip) 3133 { 3134 TRACE("[Thread %2u] xor\n", p->thread_id); 3135 3136 ALU(t, ip, ^); 3137 } 3138 3139 static inline void 3140 __instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused, 3141 struct thread *t, 3142 const struct instruction *ip) 3143 { 3144 TRACE("[Thread %2u] xor (mh)\n", p->thread_id); 3145 3146 ALU_MH(t, ip, ^); 3147 } 3148 3149 static inline void 3150 __instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused, 3151 struct thread *t, 3152 const struct instruction *ip) 3153 { 3154 TRACE("[Thread %2u] xor (hm)\n", p->thread_id); 3155 3156 ALU_HM_FAST(t, ip, ^); 3157 } 3158 3159 static inline void 3160 __instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused, 3161 struct thread *t, 3162 const struct instruction *ip) 3163 { 3164 TRACE("[Thread %2u] xor (hh)\n", p->thread_id); 3165 3166 ALU_HH_FAST(t, ip, ^); 3167 } 3168 3169 static inline void 3170 __instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused, 3171 struct thread *t, 3172 const struct instruction *ip) 3173 { 3174 TRACE("[Thread %2u] xor (i)\n", p->thread_id); 3175 3176 ALU_I(t, ip, ^); 3177 } 3178 3179 static inline void 3180 __instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused, 3181 struct thread *t, 3182 const struct instruction *ip) 3183 { 3184 uint8_t *dst_struct, *src_struct; 3185 uint16_t *dst16_ptr, dst; 3186 uint64_t *src64_ptr, src64, src64_mask, src; 3187 uint64_t r; 3188 3189 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id); 3190 3191 /* Structs. */ 3192 dst_struct = t->structs[ip->alu.dst.struct_id]; 3193 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; 3194 dst = *dst16_ptr; 3195 3196 src_struct = t->structs[ip->alu.src.struct_id]; 3197 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset]; 3198 src64 = *src64_ptr; 3199 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits); 3200 src = src64 & src64_mask; 3201 3202 /* Initialize the result with destination 1's complement. */ 3203 r = dst; 3204 r = ~r & 0xFFFF; 3205 3206 /* The first input (r) is a 16-bit number. The second and the third 3207 * inputs are 32-bit numbers. In the worst case scenario, the sum of the 3208 * three numbers (output r) is a 34-bit number. 3209 */ 3210 r += (src >> 32) + (src & 0xFFFFFFFF); 3211 3212 /* The first input is a 16-bit number. The second input is an 18-bit 3213 * number. In the worst case scenario, the sum of the two numbers is a 3214 * 19-bit number. 3215 */ 3216 r = (r & 0xFFFF) + (r >> 16); 3217 3218 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is 3219 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006). 3220 */ 3221 r = (r & 0xFFFF) + (r >> 16); 3222 3223 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input 3224 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. 3225 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated, 3226 * therefore the output r is always a 16-bit number. 3227 */ 3228 r = (r & 0xFFFF) + (r >> 16); 3229 3230 /* Apply 1's complement to the result. */ 3231 r = ~r & 0xFFFF; 3232 r = r ? r : 0xFFFF; 3233 3234 *dst16_ptr = (uint16_t)r; 3235 } 3236 3237 static inline void 3238 __instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused, 3239 struct thread *t, 3240 const struct instruction *ip) 3241 { 3242 uint8_t *dst_struct, *src_struct; 3243 uint16_t *dst16_ptr, dst; 3244 uint64_t *src64_ptr, src64, src64_mask, src; 3245 uint64_t r; 3246 3247 TRACE("[Thread %2u] cksub (field)\n", p->thread_id); 3248 3249 /* Structs. */ 3250 dst_struct = t->structs[ip->alu.dst.struct_id]; 3251 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; 3252 dst = *dst16_ptr; 3253 3254 src_struct = t->structs[ip->alu.src.struct_id]; 3255 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset]; 3256 src64 = *src64_ptr; 3257 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits); 3258 src = src64 & src64_mask; 3259 3260 /* Initialize the result with destination 1's complement. */ 3261 r = dst; 3262 r = ~r & 0xFFFF; 3263 3264 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as 3265 * the following sequence of operations in 2's complement arithmetic: 3266 * a '- b = (a - b) % 0xFFFF. 3267 * 3268 * In order to prevent an underflow for the below subtraction, in which 3269 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the 3270 * minuend), we first add a multiple of the 0xFFFF modulus to the 3271 * minuend. The number we add to the minuend needs to be a 34-bit number 3272 * or higher, so for readability reasons we picked the 36-bit multiple. 3273 * We are effectively turning the 16-bit minuend into a 36-bit number: 3274 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF. 3275 */ 3276 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */ 3277 3278 /* A 33-bit number is subtracted from a 36-bit number (the input r). The 3279 * result (the output r) is a 36-bit number. 3280 */ 3281 r -= (src >> 32) + (src & 0xFFFFFFFF); 3282 3283 /* The first input is a 16-bit number. The second input is a 20-bit 3284 * number. Their sum is a 21-bit number. 3285 */ 3286 r = (r & 0xFFFF) + (r >> 16); 3287 3288 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is 3289 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E). 3290 */ 3291 r = (r & 0xFFFF) + (r >> 16); 3292 3293 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input 3294 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. 3295 * 0x1001E), the output r is (0 .. 31). So no carry bit can be 3296 * generated, therefore the output r is always a 16-bit number. 3297 */ 3298 r = (r & 0xFFFF) + (r >> 16); 3299 3300 /* Apply 1's complement to the result. */ 3301 r = ~r & 0xFFFF; 3302 r = r ? r : 0xFFFF; 3303 3304 *dst16_ptr = (uint16_t)r; 3305 } 3306 3307 static inline void 3308 __instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused, 3309 struct thread *t, 3310 const struct instruction *ip) 3311 { 3312 uint8_t *dst_struct, *src_struct; 3313 uint16_t *dst16_ptr, dst; 3314 uint32_t *src32_ptr; 3315 uint64_t r0, r1; 3316 3317 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id); 3318 3319 /* Structs. */ 3320 dst_struct = t->structs[ip->alu.dst.struct_id]; 3321 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; 3322 dst = *dst16_ptr; 3323 3324 src_struct = t->structs[ip->alu.src.struct_id]; 3325 src32_ptr = (uint32_t *)&src_struct[0]; 3326 3327 /* Initialize the result with destination 1's complement. */ 3328 r0 = dst; 3329 r0 = ~r0 & 0xFFFF; 3330 3331 r0 += src32_ptr[0]; /* The output r0 is a 33-bit number. */ 3332 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */ 3333 r0 += src32_ptr[2]; /* The output r0 is a 34-bit number. */ 3334 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */ 3335 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */ 3336 3337 /* The first input is a 16-bit number. The second input is a 19-bit 3338 * number. Their sum is a 20-bit number. 3339 */ 3340 r0 = (r0 & 0xFFFF) + (r0 >> 16); 3341 3342 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is 3343 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E). 3344 */ 3345 r0 = (r0 & 0xFFFF) + (r0 >> 16); 3346 3347 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input 3348 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. 3349 * 0x1000E), the output r is (0 .. 15). So no carry bit can be 3350 * generated, therefore the output r is always a 16-bit number. 3351 */ 3352 r0 = (r0 & 0xFFFF) + (r0 >> 16); 3353 3354 /* Apply 1's complement to the result. */ 3355 r0 = ~r0 & 0xFFFF; 3356 r0 = r0 ? r0 : 0xFFFF; 3357 3358 *dst16_ptr = (uint16_t)r0; 3359 } 3360 3361 static inline void 3362 __instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused, 3363 struct thread *t, 3364 const struct instruction *ip) 3365 { 3366 uint32_t src_header_id = ip->alu.src.n_bits; /* The src header ID is stored here. */ 3367 uint32_t n_src_header_bytes = t->headers[src_header_id].n_bytes; 3368 uint8_t *dst_struct, *src_struct; 3369 uint16_t *dst16_ptr, dst; 3370 uint32_t *src32_ptr; 3371 uint64_t r; 3372 uint32_t i; 3373 3374 if (n_src_header_bytes == 20) { 3375 __instr_alu_ckadd_struct20_exec(p, t, ip); 3376 return; 3377 } 3378 3379 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id); 3380 3381 /* Structs. */ 3382 dst_struct = t->structs[ip->alu.dst.struct_id]; 3383 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; 3384 dst = *dst16_ptr; 3385 3386 src_struct = t->structs[ip->alu.src.struct_id]; 3387 src32_ptr = (uint32_t *)&src_struct[0]; 3388 3389 /* Initialize the result with destination 1's complement. */ 3390 r = dst; 3391 r = ~r & 0xFFFF; 3392 3393 /* The max number of 32-bit words in a 32K-byte header is 2^13. 3394 * Therefore, in the worst case scenario, a 45-bit number is added to a 3395 * 16-bit number (the input r), so the output r is 46-bit number. 3396 */ 3397 for (i = 0; i < n_src_header_bytes / 4; i++, src32_ptr++) 3398 r += *src32_ptr; 3399 3400 /* The first input is a 16-bit number. The second input is a 30-bit 3401 * number. Their sum is a 31-bit number. 3402 */ 3403 r = (r & 0xFFFF) + (r >> 16); 3404 3405 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is 3406 * a 15-bit number (0 .. 0x7FFF). The sum is a 17-bit number (0 .. 0x17FFE). 3407 */ 3408 r = (r & 0xFFFF) + (r >> 16); 3409 3410 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input 3411 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. 3412 * 0x17FFE), the output r is (0 .. 0x7FFF). So no carry bit can be 3413 * generated, therefore the output r is always a 16-bit number. 3414 */ 3415 r = (r & 0xFFFF) + (r >> 16); 3416 3417 /* Apply 1's complement to the result. */ 3418 r = ~r & 0xFFFF; 3419 r = r ? r : 0xFFFF; 3420 3421 *dst16_ptr = (uint16_t)r; 3422 } 3423 3424 /* 3425 * Register array. 3426 */ 3427 static inline uint64_t * 3428 instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip) 3429 { 3430 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; 3431 return r->regarray; 3432 } 3433 3434 static inline uint64_t 3435 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3436 { 3437 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; 3438 3439 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; 3440 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; 3441 uint64_t idx64 = *idx64_ptr; 3442 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits); 3443 uint64_t idx = idx64 & idx64_mask & r->size_mask; 3444 3445 return idx; 3446 } 3447 3448 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3449 3450 static inline uint64_t 3451 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3452 { 3453 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; 3454 3455 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; 3456 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; 3457 uint64_t idx64 = *idx64_ptr; 3458 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask; 3459 3460 return idx; 3461 } 3462 3463 #else 3464 3465 #define instr_regarray_idx_nbo instr_regarray_idx_hbo 3466 3467 #endif 3468 3469 static inline uint64_t 3470 instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip) 3471 { 3472 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; 3473 3474 uint64_t idx = ip->regarray.idx_val & r->size_mask; 3475 3476 return idx; 3477 } 3478 3479 static inline uint64_t 3480 instr_regarray_src_hbo(struct thread *t, const struct instruction *ip) 3481 { 3482 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; 3483 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; 3484 uint64_t src64 = *src64_ptr; 3485 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); 3486 uint64_t src = src64 & src64_mask; 3487 3488 return src; 3489 } 3490 3491 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3492 3493 static inline uint64_t 3494 instr_regarray_src_nbo(struct thread *t, const struct instruction *ip) 3495 { 3496 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; 3497 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; 3498 uint64_t src64 = *src64_ptr; 3499 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits); 3500 3501 return src; 3502 } 3503 3504 #else 3505 3506 #define instr_regarray_src_nbo instr_regarray_src_hbo 3507 3508 #endif 3509 3510 static inline void 3511 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src) 3512 { 3513 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; 3514 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; 3515 uint64_t dst64 = *dst64_ptr; 3516 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); 3517 3518 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); 3519 3520 } 3521 3522 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3523 3524 static inline void 3525 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src) 3526 { 3527 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; 3528 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; 3529 uint64_t dst64 = *dst64_ptr; 3530 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); 3531 3532 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits); 3533 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); 3534 } 3535 3536 #else 3537 3538 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set 3539 3540 #endif 3541 3542 static inline void 3543 __instr_regprefetch_rh_exec(struct rte_swx_pipeline *p, 3544 struct thread *t, 3545 const struct instruction *ip) 3546 { 3547 uint64_t *regarray, idx; 3548 3549 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id); 3550 3551 regarray = instr_regarray_regarray(p, ip); 3552 idx = instr_regarray_idx_nbo(p, t, ip); 3553 rte_prefetch0(®array[idx]); 3554 } 3555 3556 static inline void 3557 __instr_regprefetch_rm_exec(struct rte_swx_pipeline *p, 3558 struct thread *t, 3559 const struct instruction *ip) 3560 { 3561 uint64_t *regarray, idx; 3562 3563 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id); 3564 3565 regarray = instr_regarray_regarray(p, ip); 3566 idx = instr_regarray_idx_hbo(p, t, ip); 3567 rte_prefetch0(®array[idx]); 3568 } 3569 3570 static inline void 3571 __instr_regprefetch_ri_exec(struct rte_swx_pipeline *p, 3572 struct thread *t __rte_unused, 3573 const struct instruction *ip) 3574 { 3575 uint64_t *regarray, idx; 3576 3577 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id); 3578 3579 regarray = instr_regarray_regarray(p, ip); 3580 idx = instr_regarray_idx_imm(p, ip); 3581 rte_prefetch0(®array[idx]); 3582 } 3583 3584 static inline void 3585 __instr_regrd_hrh_exec(struct rte_swx_pipeline *p, 3586 struct thread *t, 3587 const struct instruction *ip) 3588 { 3589 uint64_t *regarray, idx; 3590 3591 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id); 3592 3593 regarray = instr_regarray_regarray(p, ip); 3594 idx = instr_regarray_idx_nbo(p, t, ip); 3595 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); 3596 } 3597 3598 static inline void 3599 __instr_regrd_hrm_exec(struct rte_swx_pipeline *p, 3600 struct thread *t, 3601 const struct instruction *ip) 3602 { 3603 uint64_t *regarray, idx; 3604 3605 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id); 3606 3607 /* Structs. */ 3608 regarray = instr_regarray_regarray(p, ip); 3609 idx = instr_regarray_idx_hbo(p, t, ip); 3610 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); 3611 } 3612 3613 static inline void 3614 __instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3615 { 3616 uint64_t *regarray, idx; 3617 3618 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id); 3619 3620 regarray = instr_regarray_regarray(p, ip); 3621 idx = instr_regarray_idx_nbo(p, t, ip); 3622 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); 3623 } 3624 3625 static inline void 3626 __instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3627 { 3628 uint64_t *regarray, idx; 3629 3630 TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id); 3631 3632 regarray = instr_regarray_regarray(p, ip); 3633 idx = instr_regarray_idx_hbo(p, t, ip); 3634 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); 3635 } 3636 3637 static inline void 3638 __instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3639 { 3640 uint64_t *regarray, idx; 3641 3642 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id); 3643 3644 regarray = instr_regarray_regarray(p, ip); 3645 idx = instr_regarray_idx_imm(p, ip); 3646 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); 3647 } 3648 3649 static inline void 3650 __instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3651 { 3652 uint64_t *regarray, idx; 3653 3654 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id); 3655 3656 regarray = instr_regarray_regarray(p, ip); 3657 idx = instr_regarray_idx_imm(p, ip); 3658 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); 3659 } 3660 3661 static inline void 3662 __instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3663 { 3664 uint64_t *regarray, idx, src; 3665 3666 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id); 3667 3668 regarray = instr_regarray_regarray(p, ip); 3669 idx = instr_regarray_idx_nbo(p, t, ip); 3670 src = instr_regarray_src_nbo(t, ip); 3671 regarray[idx] = src; 3672 } 3673 3674 static inline void 3675 __instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3676 { 3677 uint64_t *regarray, idx, src; 3678 3679 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id); 3680 3681 regarray = instr_regarray_regarray(p, ip); 3682 idx = instr_regarray_idx_nbo(p, t, ip); 3683 src = instr_regarray_src_hbo(t, ip); 3684 regarray[idx] = src; 3685 } 3686 3687 static inline void 3688 __instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3689 { 3690 uint64_t *regarray, idx, src; 3691 3692 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id); 3693 3694 regarray = instr_regarray_regarray(p, ip); 3695 idx = instr_regarray_idx_hbo(p, t, ip); 3696 src = instr_regarray_src_nbo(t, ip); 3697 regarray[idx] = src; 3698 } 3699 3700 static inline void 3701 __instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3702 { 3703 uint64_t *regarray, idx, src; 3704 3705 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id); 3706 3707 regarray = instr_regarray_regarray(p, ip); 3708 idx = instr_regarray_idx_hbo(p, t, ip); 3709 src = instr_regarray_src_hbo(t, ip); 3710 regarray[idx] = src; 3711 } 3712 3713 static inline void 3714 __instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3715 { 3716 uint64_t *regarray, idx, src; 3717 3718 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id); 3719 3720 regarray = instr_regarray_regarray(p, ip); 3721 idx = instr_regarray_idx_nbo(p, t, ip); 3722 src = ip->regarray.dstsrc_val; 3723 regarray[idx] = src; 3724 } 3725 3726 static inline void 3727 __instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3728 { 3729 uint64_t *regarray, idx, src; 3730 3731 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id); 3732 3733 regarray = instr_regarray_regarray(p, ip); 3734 idx = instr_regarray_idx_hbo(p, t, ip); 3735 src = ip->regarray.dstsrc_val; 3736 regarray[idx] = src; 3737 } 3738 3739 static inline void 3740 __instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3741 { 3742 uint64_t *regarray, idx, src; 3743 3744 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id); 3745 3746 regarray = instr_regarray_regarray(p, ip); 3747 idx = instr_regarray_idx_imm(p, ip); 3748 src = instr_regarray_src_nbo(t, ip); 3749 regarray[idx] = src; 3750 } 3751 3752 static inline void 3753 __instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3754 { 3755 uint64_t *regarray, idx, src; 3756 3757 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id); 3758 3759 regarray = instr_regarray_regarray(p, ip); 3760 idx = instr_regarray_idx_imm(p, ip); 3761 src = instr_regarray_src_hbo(t, ip); 3762 regarray[idx] = src; 3763 } 3764 3765 static inline void 3766 __instr_regwr_rii_exec(struct rte_swx_pipeline *p, 3767 struct thread *t __rte_unused, 3768 const struct instruction *ip) 3769 { 3770 uint64_t *regarray, idx, src; 3771 3772 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id); 3773 3774 regarray = instr_regarray_regarray(p, ip); 3775 idx = instr_regarray_idx_imm(p, ip); 3776 src = ip->regarray.dstsrc_val; 3777 regarray[idx] = src; 3778 } 3779 3780 static inline void 3781 __instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3782 { 3783 uint64_t *regarray, idx, src; 3784 3785 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id); 3786 3787 regarray = instr_regarray_regarray(p, ip); 3788 idx = instr_regarray_idx_nbo(p, t, ip); 3789 src = instr_regarray_src_nbo(t, ip); 3790 regarray[idx] += src; 3791 } 3792 3793 static inline void 3794 __instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3795 { 3796 uint64_t *regarray, idx, src; 3797 3798 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id); 3799 3800 regarray = instr_regarray_regarray(p, ip); 3801 idx = instr_regarray_idx_nbo(p, t, ip); 3802 src = instr_regarray_src_hbo(t, ip); 3803 regarray[idx] += src; 3804 } 3805 3806 static inline void 3807 __instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3808 { 3809 uint64_t *regarray, idx, src; 3810 3811 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id); 3812 3813 regarray = instr_regarray_regarray(p, ip); 3814 idx = instr_regarray_idx_hbo(p, t, ip); 3815 src = instr_regarray_src_nbo(t, ip); 3816 regarray[idx] += src; 3817 } 3818 3819 static inline void 3820 __instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3821 { 3822 uint64_t *regarray, idx, src; 3823 3824 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id); 3825 3826 regarray = instr_regarray_regarray(p, ip); 3827 idx = instr_regarray_idx_hbo(p, t, ip); 3828 src = instr_regarray_src_hbo(t, ip); 3829 regarray[idx] += src; 3830 } 3831 3832 static inline void 3833 __instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3834 { 3835 uint64_t *regarray, idx, src; 3836 3837 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id); 3838 3839 regarray = instr_regarray_regarray(p, ip); 3840 idx = instr_regarray_idx_nbo(p, t, ip); 3841 src = ip->regarray.dstsrc_val; 3842 regarray[idx] += src; 3843 } 3844 3845 static inline void 3846 __instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3847 { 3848 uint64_t *regarray, idx, src; 3849 3850 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id); 3851 3852 regarray = instr_regarray_regarray(p, ip); 3853 idx = instr_regarray_idx_hbo(p, t, ip); 3854 src = ip->regarray.dstsrc_val; 3855 regarray[idx] += src; 3856 } 3857 3858 static inline void 3859 __instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3860 { 3861 uint64_t *regarray, idx, src; 3862 3863 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id); 3864 3865 regarray = instr_regarray_regarray(p, ip); 3866 idx = instr_regarray_idx_imm(p, ip); 3867 src = instr_regarray_src_nbo(t, ip); 3868 regarray[idx] += src; 3869 } 3870 3871 static inline void 3872 __instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3873 { 3874 uint64_t *regarray, idx, src; 3875 3876 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id); 3877 3878 regarray = instr_regarray_regarray(p, ip); 3879 idx = instr_regarray_idx_imm(p, ip); 3880 src = instr_regarray_src_hbo(t, ip); 3881 regarray[idx] += src; 3882 } 3883 3884 static inline void 3885 __instr_regadd_rii_exec(struct rte_swx_pipeline *p, 3886 struct thread *t __rte_unused, 3887 const struct instruction *ip) 3888 { 3889 uint64_t *regarray, idx, src; 3890 3891 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id); 3892 3893 regarray = instr_regarray_regarray(p, ip); 3894 idx = instr_regarray_idx_imm(p, ip); 3895 src = ip->regarray.dstsrc_val; 3896 regarray[idx] += src; 3897 } 3898 3899 /* 3900 * metarray. 3901 */ 3902 static inline struct meter * 3903 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3904 { 3905 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; 3906 3907 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; 3908 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; 3909 uint64_t idx64 = *idx64_ptr; 3910 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits); 3911 uint64_t idx = idx64 & idx64_mask & r->size_mask; 3912 3913 return &r->metarray[idx]; 3914 } 3915 3916 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3917 3918 static inline struct meter * 3919 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 3920 { 3921 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; 3922 3923 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; 3924 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; 3925 uint64_t idx64 = *idx64_ptr; 3926 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask; 3927 3928 return &r->metarray[idx]; 3929 } 3930 3931 #else 3932 3933 #define instr_meter_idx_nbo instr_meter_idx_hbo 3934 3935 #endif 3936 3937 static inline struct meter * 3938 instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip) 3939 { 3940 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; 3941 3942 uint64_t idx = ip->meter.idx_val & r->size_mask; 3943 3944 return &r->metarray[idx]; 3945 } 3946 3947 static inline uint32_t 3948 instr_meter_length_hbo(struct thread *t, const struct instruction *ip) 3949 { 3950 uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; 3951 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; 3952 uint64_t src64 = *src64_ptr; 3953 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits); 3954 uint64_t src = src64 & src64_mask; 3955 3956 return (uint32_t)src; 3957 } 3958 3959 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 3960 3961 static inline uint32_t 3962 instr_meter_length_nbo(struct thread *t, const struct instruction *ip) 3963 { 3964 uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; 3965 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; 3966 uint64_t src64 = *src64_ptr; 3967 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits); 3968 3969 return (uint32_t)src; 3970 } 3971 3972 #else 3973 3974 #define instr_meter_length_nbo instr_meter_length_hbo 3975 3976 #endif 3977 3978 static inline enum rte_color 3979 instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip) 3980 { 3981 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id]; 3982 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset]; 3983 uint64_t src64 = *src64_ptr; 3984 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits); 3985 uint64_t src = src64 & src64_mask; 3986 3987 return (enum rte_color)src; 3988 } 3989 3990 static inline void 3991 instr_meter_color_out_hbo_set(struct thread *t, 3992 const struct instruction *ip, 3993 enum rte_color color_out) 3994 { 3995 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id]; 3996 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset]; 3997 uint64_t dst64 = *dst64_ptr; 3998 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits); 3999 4000 uint64_t src = (uint64_t)color_out; 4001 4002 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); 4003 } 4004 4005 static inline void 4006 __instr_metprefetch_h_exec(struct rte_swx_pipeline *p, 4007 struct thread *t, 4008 const struct instruction *ip) 4009 { 4010 struct meter *m; 4011 4012 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id); 4013 4014 m = instr_meter_idx_nbo(p, t, ip); 4015 rte_prefetch0(m); 4016 } 4017 4018 static inline void 4019 __instr_metprefetch_m_exec(struct rte_swx_pipeline *p, 4020 struct thread *t, 4021 const struct instruction *ip) 4022 { 4023 struct meter *m; 4024 4025 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id); 4026 4027 m = instr_meter_idx_hbo(p, t, ip); 4028 rte_prefetch0(m); 4029 } 4030 4031 static inline void 4032 __instr_metprefetch_i_exec(struct rte_swx_pipeline *p, 4033 struct thread *t __rte_unused, 4034 const struct instruction *ip) 4035 { 4036 struct meter *m; 4037 4038 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id); 4039 4040 m = instr_meter_idx_imm(p, ip); 4041 rte_prefetch0(m); 4042 } 4043 4044 static inline void 4045 __instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4046 { 4047 struct meter *m; 4048 uint64_t time, n_pkts, n_bytes; 4049 uint32_t length; 4050 enum rte_color color_in, color_out; 4051 4052 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id); 4053 4054 m = instr_meter_idx_nbo(p, t, ip); 4055 rte_prefetch0(m->n_pkts); 4056 time = rte_get_tsc_cycles(); 4057 length = instr_meter_length_nbo(t, ip); 4058 color_in = instr_meter_color_in_hbo(t, ip); 4059 4060 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4061 &m->profile->profile, 4062 time, 4063 length, 4064 color_in); 4065 4066 color_out &= m->color_mask; 4067 4068 n_pkts = m->n_pkts[color_out]; 4069 n_bytes = m->n_bytes[color_out]; 4070 4071 instr_meter_color_out_hbo_set(t, ip, color_out); 4072 4073 m->n_pkts[color_out] = n_pkts + 1; 4074 m->n_bytes[color_out] = n_bytes + length; 4075 } 4076 4077 static inline void 4078 __instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4079 { 4080 struct meter *m; 4081 uint64_t time, n_pkts, n_bytes; 4082 uint32_t length; 4083 enum rte_color color_in, color_out; 4084 4085 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id); 4086 4087 m = instr_meter_idx_nbo(p, t, ip); 4088 rte_prefetch0(m->n_pkts); 4089 time = rte_get_tsc_cycles(); 4090 length = instr_meter_length_nbo(t, ip); 4091 color_in = (enum rte_color)ip->meter.color_in_val; 4092 4093 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4094 &m->profile->profile, 4095 time, 4096 length, 4097 color_in); 4098 4099 color_out &= m->color_mask; 4100 4101 n_pkts = m->n_pkts[color_out]; 4102 n_bytes = m->n_bytes[color_out]; 4103 4104 instr_meter_color_out_hbo_set(t, ip, color_out); 4105 4106 m->n_pkts[color_out] = n_pkts + 1; 4107 m->n_bytes[color_out] = n_bytes + length; 4108 } 4109 4110 static inline void 4111 __instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4112 { 4113 struct meter *m; 4114 uint64_t time, n_pkts, n_bytes; 4115 uint32_t length; 4116 enum rte_color color_in, color_out; 4117 4118 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id); 4119 4120 m = instr_meter_idx_nbo(p, t, ip); 4121 rte_prefetch0(m->n_pkts); 4122 time = rte_get_tsc_cycles(); 4123 length = instr_meter_length_hbo(t, ip); 4124 color_in = instr_meter_color_in_hbo(t, ip); 4125 4126 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4127 &m->profile->profile, 4128 time, 4129 length, 4130 color_in); 4131 4132 color_out &= m->color_mask; 4133 4134 n_pkts = m->n_pkts[color_out]; 4135 n_bytes = m->n_bytes[color_out]; 4136 4137 instr_meter_color_out_hbo_set(t, ip, color_out); 4138 4139 m->n_pkts[color_out] = n_pkts + 1; 4140 m->n_bytes[color_out] = n_bytes + length; 4141 } 4142 4143 static inline void 4144 __instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4145 { 4146 struct meter *m; 4147 uint64_t time, n_pkts, n_bytes; 4148 uint32_t length; 4149 enum rte_color color_in, color_out; 4150 4151 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id); 4152 4153 m = instr_meter_idx_nbo(p, t, ip); 4154 rte_prefetch0(m->n_pkts); 4155 time = rte_get_tsc_cycles(); 4156 length = instr_meter_length_hbo(t, ip); 4157 color_in = (enum rte_color)ip->meter.color_in_val; 4158 4159 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4160 &m->profile->profile, 4161 time, 4162 length, 4163 color_in); 4164 4165 color_out &= m->color_mask; 4166 4167 n_pkts = m->n_pkts[color_out]; 4168 n_bytes = m->n_bytes[color_out]; 4169 4170 instr_meter_color_out_hbo_set(t, ip, color_out); 4171 4172 m->n_pkts[color_out] = n_pkts + 1; 4173 m->n_bytes[color_out] = n_bytes + length; 4174 } 4175 4176 static inline void 4177 __instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4178 { 4179 struct meter *m; 4180 uint64_t time, n_pkts, n_bytes; 4181 uint32_t length; 4182 enum rte_color color_in, color_out; 4183 4184 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id); 4185 4186 m = instr_meter_idx_hbo(p, t, ip); 4187 rte_prefetch0(m->n_pkts); 4188 time = rte_get_tsc_cycles(); 4189 length = instr_meter_length_nbo(t, ip); 4190 color_in = instr_meter_color_in_hbo(t, ip); 4191 4192 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4193 &m->profile->profile, 4194 time, 4195 length, 4196 color_in); 4197 4198 color_out &= m->color_mask; 4199 4200 n_pkts = m->n_pkts[color_out]; 4201 n_bytes = m->n_bytes[color_out]; 4202 4203 instr_meter_color_out_hbo_set(t, ip, color_out); 4204 4205 m->n_pkts[color_out] = n_pkts + 1; 4206 m->n_bytes[color_out] = n_bytes + length; 4207 } 4208 4209 static inline void 4210 __instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4211 { 4212 struct meter *m; 4213 uint64_t time, n_pkts, n_bytes; 4214 uint32_t length; 4215 enum rte_color color_in, color_out; 4216 4217 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id); 4218 4219 m = instr_meter_idx_hbo(p, t, ip); 4220 rte_prefetch0(m->n_pkts); 4221 time = rte_get_tsc_cycles(); 4222 length = instr_meter_length_nbo(t, ip); 4223 color_in = (enum rte_color)ip->meter.color_in_val; 4224 4225 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4226 &m->profile->profile, 4227 time, 4228 length, 4229 color_in); 4230 4231 color_out &= m->color_mask; 4232 4233 n_pkts = m->n_pkts[color_out]; 4234 n_bytes = m->n_bytes[color_out]; 4235 4236 instr_meter_color_out_hbo_set(t, ip, color_out); 4237 4238 m->n_pkts[color_out] = n_pkts + 1; 4239 m->n_bytes[color_out] = n_bytes + length; 4240 } 4241 4242 static inline void 4243 __instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4244 { 4245 struct meter *m; 4246 uint64_t time, n_pkts, n_bytes; 4247 uint32_t length; 4248 enum rte_color color_in, color_out; 4249 4250 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id); 4251 4252 m = instr_meter_idx_hbo(p, t, ip); 4253 rte_prefetch0(m->n_pkts); 4254 time = rte_get_tsc_cycles(); 4255 length = instr_meter_length_hbo(t, ip); 4256 color_in = instr_meter_color_in_hbo(t, ip); 4257 4258 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4259 &m->profile->profile, 4260 time, 4261 length, 4262 color_in); 4263 4264 color_out &= m->color_mask; 4265 4266 n_pkts = m->n_pkts[color_out]; 4267 n_bytes = m->n_bytes[color_out]; 4268 4269 instr_meter_color_out_hbo_set(t, ip, color_out); 4270 4271 m->n_pkts[color_out] = n_pkts + 1; 4272 m->n_bytes[color_out] = n_bytes + length; 4273 } 4274 4275 static inline void 4276 __instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4277 { 4278 struct meter *m; 4279 uint64_t time, n_pkts, n_bytes; 4280 uint32_t length; 4281 enum rte_color color_in, color_out; 4282 4283 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id); 4284 4285 m = instr_meter_idx_hbo(p, t, ip); 4286 rte_prefetch0(m->n_pkts); 4287 time = rte_get_tsc_cycles(); 4288 length = instr_meter_length_hbo(t, ip); 4289 color_in = (enum rte_color)ip->meter.color_in_val; 4290 4291 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4292 &m->profile->profile, 4293 time, 4294 length, 4295 color_in); 4296 4297 color_out &= m->color_mask; 4298 4299 n_pkts = m->n_pkts[color_out]; 4300 n_bytes = m->n_bytes[color_out]; 4301 4302 instr_meter_color_out_hbo_set(t, ip, color_out); 4303 4304 m->n_pkts[color_out] = n_pkts + 1; 4305 m->n_bytes[color_out] = n_bytes + length; 4306 } 4307 4308 static inline void 4309 __instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4310 { 4311 struct meter *m; 4312 uint64_t time, n_pkts, n_bytes; 4313 uint32_t length; 4314 enum rte_color color_in, color_out; 4315 4316 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id); 4317 4318 m = instr_meter_idx_imm(p, ip); 4319 rte_prefetch0(m->n_pkts); 4320 time = rte_get_tsc_cycles(); 4321 length = instr_meter_length_nbo(t, ip); 4322 color_in = instr_meter_color_in_hbo(t, ip); 4323 4324 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4325 &m->profile->profile, 4326 time, 4327 length, 4328 color_in); 4329 4330 color_out &= m->color_mask; 4331 4332 n_pkts = m->n_pkts[color_out]; 4333 n_bytes = m->n_bytes[color_out]; 4334 4335 instr_meter_color_out_hbo_set(t, ip, color_out); 4336 4337 m->n_pkts[color_out] = n_pkts + 1; 4338 m->n_bytes[color_out] = n_bytes + length; 4339 } 4340 4341 static inline void 4342 __instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4343 { 4344 struct meter *m; 4345 uint64_t time, n_pkts, n_bytes; 4346 uint32_t length; 4347 enum rte_color color_in, color_out; 4348 4349 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id); 4350 4351 m = instr_meter_idx_imm(p, ip); 4352 rte_prefetch0(m->n_pkts); 4353 time = rte_get_tsc_cycles(); 4354 length = instr_meter_length_nbo(t, ip); 4355 color_in = (enum rte_color)ip->meter.color_in_val; 4356 4357 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4358 &m->profile->profile, 4359 time, 4360 length, 4361 color_in); 4362 4363 color_out &= m->color_mask; 4364 4365 n_pkts = m->n_pkts[color_out]; 4366 n_bytes = m->n_bytes[color_out]; 4367 4368 instr_meter_color_out_hbo_set(t, ip, color_out); 4369 4370 m->n_pkts[color_out] = n_pkts + 1; 4371 m->n_bytes[color_out] = n_bytes + length; 4372 } 4373 4374 static inline void 4375 __instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4376 { 4377 struct meter *m; 4378 uint64_t time, n_pkts, n_bytes; 4379 uint32_t length; 4380 enum rte_color color_in, color_out; 4381 4382 TRACE("[Thread %2u] meter (imm)\n", p->thread_id); 4383 4384 m = instr_meter_idx_imm(p, ip); 4385 rte_prefetch0(m->n_pkts); 4386 time = rte_get_tsc_cycles(); 4387 length = instr_meter_length_hbo(t, ip); 4388 color_in = instr_meter_color_in_hbo(t, ip); 4389 4390 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4391 &m->profile->profile, 4392 time, 4393 length, 4394 color_in); 4395 4396 color_out &= m->color_mask; 4397 4398 n_pkts = m->n_pkts[color_out]; 4399 n_bytes = m->n_bytes[color_out]; 4400 4401 instr_meter_color_out_hbo_set(t, ip, color_out); 4402 4403 m->n_pkts[color_out] = n_pkts + 1; 4404 m->n_bytes[color_out] = n_bytes + length; 4405 } 4406 4407 static inline void 4408 __instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) 4409 { 4410 struct meter *m; 4411 uint64_t time, n_pkts, n_bytes; 4412 uint32_t length; 4413 enum rte_color color_in, color_out; 4414 4415 TRACE("[Thread %2u] meter (imi)\n", p->thread_id); 4416 4417 m = instr_meter_idx_imm(p, ip); 4418 rte_prefetch0(m->n_pkts); 4419 time = rte_get_tsc_cycles(); 4420 length = instr_meter_length_hbo(t, ip); 4421 color_in = (enum rte_color)ip->meter.color_in_val; 4422 4423 color_out = rte_meter_trtcm_color_aware_check(&m->m, 4424 &m->profile->profile, 4425 time, 4426 length, 4427 color_in); 4428 4429 color_out &= m->color_mask; 4430 4431 n_pkts = m->n_pkts[color_out]; 4432 n_bytes = m->n_bytes[color_out]; 4433 4434 instr_meter_color_out_hbo_set(t, ip, color_out); 4435 4436 m->n_pkts[color_out] = n_pkts + 1; 4437 m->n_bytes[color_out] = n_bytes + length; 4438 } 4439 4440 #endif 4441