1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2022 Intel Corporation 3 */ 4 5 #ifndef _DLB2_PRIV_H_ 6 #define _DLB2_PRIV_H_ 7 8 #include <emmintrin.h> 9 #include <stdbool.h> 10 11 #include <rte_eventdev.h> 12 #include <rte_config.h> 13 #include "dlb2_user.h" 14 #include "dlb2_log.h" 15 #include "rte_pmd_dlb2.h" 16 17 #ifndef RTE_LIBRTE_PMD_DLB2_QUELL_STATS 18 #define DLB2_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val) 19 #else 20 #define DLB2_INC_STAT(_stat, _incr_val) 21 #endif 22 23 #define EVDEV_DLB2_NAME_PMD dlb2_event 24 25 /* Default values for command line devargs */ 26 #define DLB2_POLL_INTERVAL_DEFAULT 1000 27 #define DLB2_SW_CREDIT_QUANTA_DEFAULT 32 /* Default = Worker */ 28 #define DLB2_SW_CREDIT_P_QUANTA_DEFAULT 256 /* Producer */ 29 #define DLB2_SW_CREDIT_C_QUANTA_DEFAULT 256 /* Consumer */ 30 #define DLB2_DEPTH_THRESH_DEFAULT 256 31 #define DLB2_MIN_CQ_DEPTH_OVERRIDE 32 32 #define DLB2_MAX_CQ_DEPTH_OVERRIDE 1024 33 #define DLB2_MAX_CQ_DEPTH_REORDER 128 34 #define DLB2_MIN_ENQ_DEPTH_OVERRIDE 32 35 #define DLB2_MAX_ENQ_DEPTH_OVERRIDE 1024 36 37 38 /* command line arg strings */ 39 #define NUMA_NODE_ARG "numa_node" 40 #define DLB2_MAX_NUM_EVENTS "max_num_events" 41 #define DLB2_NUM_DIR_CREDITS "num_dir_credits" 42 #define DEV_ID_ARG "dev_id" 43 #define DLB2_QID_DEPTH_THRESH_ARG "qid_depth_thresh" 44 #define DLB2_POLL_INTERVAL_ARG "poll_interval" 45 #define DLB2_SW_CREDIT_QUANTA_ARG "sw_credit_quanta" 46 #define DLB2_HW_CREDIT_QUANTA_ARG "hw_credit_quanta" 47 #define DLB2_DEPTH_THRESH_ARG "default_depth_thresh" 48 #define DLB2_VECTOR_OPTS_ENAB_ARG "vector_opts_enable" 49 #define DLB2_MAX_CQ_DEPTH "max_cq_depth" 50 #define DLB2_MAX_ENQ_DEPTH "max_enqueue_depth" 51 #define DLB2_PORT_COS "port_cos" 52 #define DLB2_COS_BW "cos_bw" 53 #define DLB2_PRODUCER_COREMASK "producer_coremask" 54 #define DLB2_DEFAULT_LDB_PORT_ALLOCATION_ARG "default_port_allocation" 55 #define DLB2_ENABLE_CQ_WEIGHT_ARG "enable_cq_weight" 56 57 /* Begin HW related defines and structs */ 58 59 #define DLB2_HW_V2 0 60 #define DLB2_HW_V2_5 1 61 #define DLB2_MAX_NUM_DOMAINS 32 62 #define DLB2_MAX_NUM_VFS 16 63 #define DLB2_MAX_NUM_LDB_QUEUES 32 64 #define DLB2_MAX_NUM_LDB_PORTS 64 65 #define DLB2_MAX_NUM_DIR_PORTS_V2 DLB2_MAX_NUM_DIR_QUEUES_V2 66 #define DLB2_MAX_NUM_DIR_PORTS_V2_5 DLB2_MAX_NUM_DIR_QUEUES_V2_5 67 #define DLB2_MAX_NUM_DIR_PORTS(ver) (ver == DLB2_HW_V2 ? \ 68 DLB2_MAX_NUM_DIR_PORTS_V2 : \ 69 DLB2_MAX_NUM_DIR_PORTS_V2_5) 70 #define DLB2_MAX_NUM_DIR_QUEUES_V2 64 /* DIR == directed */ 71 #define DLB2_MAX_NUM_DIR_QUEUES_V2_5 96 72 /* When needed for array sizing, the DLB 2.5 macro is used */ 73 #define DLB2_MAX_NUM_DIR_QUEUES(ver) (ver == DLB2_HW_V2 ? \ 74 DLB2_MAX_NUM_DIR_QUEUES_V2 : \ 75 DLB2_MAX_NUM_DIR_QUEUES_V2_5) 76 #define DLB2_MAX_NUM_FLOWS (64 * 1024) 77 #define DLB2_MAX_NUM_LDB_CREDITS (8 * 1024) 78 #define DLB2_MAX_NUM_DIR_CREDITS(ver) (ver == DLB2_HW_V2 ? 4096 : 0) 79 #define DLB2_MAX_NUM_CREDITS(ver) (ver == DLB2_HW_V2 ? \ 80 0 : DLB2_MAX_NUM_LDB_CREDITS) 81 #define DLB2_MAX_NUM_LDB_CREDIT_POOLS 64 82 #define DLB2_MAX_NUM_DIR_CREDIT_POOLS 64 83 #define DLB2_MAX_NUM_HIST_LIST_ENTRIES 2048 84 #define DLB2_MAX_NUM_QIDS_PER_LDB_CQ 8 85 #define DLB2_QID_PRIORITIES 8 86 #define DLB2_MAX_DEVICE_PATH 32 87 #define DLB2_MIN_DEQUEUE_TIMEOUT_NS 1 88 /* Note: "- 1" here to support the timeout range check in eventdev_autotest */ 89 #define DLB2_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1) 90 #define DLB2_SW_CREDIT_BATCH_SZ 32 /* Default - Worker */ 91 #define DLB2_SW_CREDIT_P_BATCH_SZ 256 /* Producer */ 92 #define DLB2_SW_CREDIT_C_BATCH_SZ 256 /* Consumer */ 93 #define DLB2_NUM_SN_GROUPS 2 94 #define DLB2_MAX_LDB_SN_ALLOC 1024 95 #define DLB2_MAX_QUEUE_DEPTH_THRESHOLD 8191 96 #define DLB2_MAX_NUM_LDB_PORTS_PER_COS (DLB2_MAX_NUM_LDB_PORTS/DLB2_COS_NUM_VALS) 97 98 /* 2048 total hist list entries and 64 total ldb ports, which 99 * makes for 2048/64 == 32 hist list entries per port. However, CQ 100 * depth must be a power of 2 and must also be >= HIST LIST entries. 101 * As a result we just limit the maximum dequeue depth to 32. 102 */ 103 #define DLB2_MAX_HL_ENTRIES 2048 104 #define DLB2_MIN_CQ_DEPTH 1 105 #define DLB2_DEFAULT_CQ_DEPTH 32 106 #define DLB2_MIN_HARDWARE_CQ_DEPTH 8 107 #define DLB2_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \ 108 DLB2_DEFAULT_CQ_DEPTH 109 110 #define DLB2_HW_DEVICE_FROM_PCI_ID(_pdev) \ 111 (((_pdev->id.device_id == PCI_DEVICE_ID_INTEL_DLB2_5_PF) || \ 112 (_pdev->id.device_id == PCI_DEVICE_ID_INTEL_DLB2_5_VF)) ? \ 113 DLB2_HW_V2_5 : DLB2_HW_V2) 114 115 /* 116 * Static per queue/port provisioning values 117 */ 118 #define DLB2_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 64 119 120 #define CQ_BASE(is_dir) ((is_dir) ? DLB2_DIR_CQ_BASE : DLB2_LDB_CQ_BASE) 121 #define CQ_SIZE(is_dir) ((is_dir) ? DLB2_DIR_CQ_MAX_SIZE : \ 122 DLB2_LDB_CQ_MAX_SIZE) 123 #define PP_BASE(is_dir) ((is_dir) ? DLB2_DIR_PP_BASE : DLB2_LDB_PP_BASE) 124 125 #define DLB2_NUM_QES_PER_CACHE_LINE 4 126 127 #define DLB2_MAX_ENQUEUE_DEPTH 32 128 #define DLB2_MIN_ENQUEUE_DEPTH 4 129 130 #define DLB2_NAME_SIZE 64 131 132 #define DLB2_1K 1024 133 #define DLB2_2K (2 * DLB2_1K) 134 #define DLB2_4K (4 * DLB2_1K) 135 #define DLB2_16K (16 * DLB2_1K) 136 #define DLB2_32K (32 * DLB2_1K) 137 #define DLB2_1MB (DLB2_1K * DLB2_1K) 138 #define DLB2_16MB (16 * DLB2_1MB) 139 140 /* Use the upper 3 bits of the event priority to select the DLB2 priority */ 141 #define EV_TO_DLB2_PRIO(x) ((x) >> 5) 142 #define DLB2_TO_EV_PRIO(x) ((x) << 5) 143 144 enum dlb2_hw_ver { 145 DLB2_HW_VER_2, 146 DLB2_HW_VER_2_5, 147 }; 148 149 enum dlb2_hw_port_types { 150 DLB2_LDB_PORT, 151 DLB2_DIR_PORT, 152 DLB2_NUM_PORT_TYPES /* Must be last */ 153 }; 154 155 enum dlb2_hw_queue_types { 156 DLB2_LDB_QUEUE, 157 DLB2_DIR_QUEUE, 158 DLB2_NUM_QUEUE_TYPES /* Must be last */ 159 }; 160 161 #define DLB2_COMBINED_POOL DLB2_LDB_QUEUE 162 163 #define PORT_TYPE(p) ((p)->is_directed ? DLB2_DIR_PORT : DLB2_LDB_PORT) 164 165 /* Do not change - must match hardware! */ 166 enum dlb2_hw_sched_type { 167 DLB2_SCHED_ATOMIC = 0, 168 DLB2_SCHED_UNORDERED, 169 DLB2_SCHED_ORDERED, 170 DLB2_SCHED_DIRECTED, 171 /* DLB2_NUM_HW_SCHED_TYPES must be last */ 172 DLB2_NUM_HW_SCHED_TYPES 173 }; 174 175 struct dlb2_hw_rsrcs { 176 int32_t nb_events_limit; 177 uint32_t num_queues; /* Total queues (lb + dir) */ 178 uint32_t num_ldb_queues; /* Number of available ldb queues */ 179 uint32_t num_ldb_ports; /* Number of load balanced ports */ 180 uint32_t num_dir_ports; /* Number of directed ports */ 181 union { 182 struct { 183 uint32_t num_ldb_credits; /* Number of ldb credits */ 184 uint32_t num_dir_credits; /* Number of dir credits */ 185 }; 186 struct { 187 uint32_t num_credits; /* Number of combined credits */ 188 }; 189 }; 190 uint32_t reorder_window_size; /* Size of reorder window */ 191 }; 192 193 struct dlb2_hw_resource_info { 194 /**> Max resources that can be provided */ 195 struct dlb2_hw_rsrcs hw_rsrc_max; 196 int num_sched_domains; 197 uint32_t socket_id; 198 }; 199 200 enum dlb2_enqueue_type { 201 /**> 202 * New : Used to inject a new packet into the QM. 203 */ 204 DLB2_ENQ_NEW, 205 /**> 206 * Forward : Enqueues a packet, and 207 * - if atomic: release any lock it holds in the QM 208 * - if ordered: release the packet for egress re-ordering 209 */ 210 DLB2_ENQ_FWD, 211 /**> 212 * Enqueue Drop : Release an inflight packet. Must be called with 213 * event == NULL. Used to drop a packet. 214 * 215 * Note that all packets dequeued from a load-balanced port must be 216 * released, either with DLB2_ENQ_DROP or DLB2_ENQ_FWD. 217 */ 218 DLB2_ENQ_DROP, 219 220 /* marker for array sizing etc. */ 221 _DLB2_NB_ENQ_TYPES 222 }; 223 224 /* hw-specific format - do not change */ 225 226 struct dlb2_event_type { 227 uint16_t major:4; 228 uint16_t unused:4; 229 uint16_t sub:8; 230 }; 231 232 union dlb2_opaque_data { 233 uint16_t opaque_data; 234 struct dlb2_event_type event_type; 235 }; 236 237 struct dlb2_msg_info { 238 uint8_t qid; 239 uint8_t sched_type:2; 240 uint8_t priority:3; 241 uint8_t msg_type:3; 242 }; 243 244 #define DLB2_NEW_CMD_BYTE 0x08 245 #define DLB2_FWD_CMD_BYTE 0x0A 246 #define DLB2_COMP_CMD_BYTE 0x02 247 #define DLB2_POP_CMD_BYTE 0x01 248 #define DLB2_NOOP_CMD_BYTE 0x00 249 250 /* hw-specific format - do not change */ 251 struct dlb2_enqueue_qe { 252 uint64_t data; 253 /* Word 3 */ 254 union dlb2_opaque_data u; 255 uint8_t qid; 256 uint8_t sched_type:2; 257 uint8_t priority:3; 258 uint8_t msg_type:3; 259 /* Word 4 */ 260 uint16_t lock_id; 261 uint8_t meas_lat:1; 262 uint8_t weight:2; /* DLB 2.5 and above */ 263 uint8_t no_dec:1; 264 uint8_t cmp_id:4; 265 union { 266 uint8_t cmd_byte; 267 struct { 268 uint8_t cq_token:1; 269 uint8_t qe_comp:1; 270 uint8_t qe_frag:1; 271 uint8_t qe_valid:1; 272 uint8_t rsvd3:1; 273 uint8_t error:1; 274 uint8_t rsvd:2; 275 }; 276 }; 277 }; 278 279 /* hw-specific format - do not change */ 280 struct dlb2_cq_pop_qe { 281 uint64_t data; 282 union dlb2_opaque_data u; 283 uint8_t qid; 284 uint8_t sched_type:2; 285 uint8_t priority:3; 286 uint8_t msg_type:3; 287 uint16_t tokens:10; 288 uint16_t rsvd2:6; 289 uint8_t meas_lat:1; 290 uint8_t rsvd1:2; 291 uint8_t no_dec:1; 292 uint8_t cmp_id:4; 293 union { 294 uint8_t cmd_byte; 295 struct { 296 uint8_t cq_token:1; 297 uint8_t qe_comp:1; 298 uint8_t qe_frag:1; 299 uint8_t qe_valid:1; 300 uint8_t rsvd3:1; 301 uint8_t error:1; 302 uint8_t rsvd:2; 303 }; 304 }; 305 }; 306 307 /* hw-specific format - do not change */ 308 struct dlb2_dequeue_qe { 309 uint64_t data; 310 union dlb2_opaque_data u; 311 uint8_t qid; 312 uint8_t sched_type:2; 313 uint8_t priority:3; 314 uint8_t msg_type:3; 315 uint16_t flow_id:16; /* was pp_id in v1 */ 316 uint8_t debug; 317 uint8_t cq_gen:1; 318 uint8_t qid_depth:2; /* 2 bits in v2 */ 319 uint8_t rsvd1:2; 320 uint8_t error:1; 321 uint8_t rsvd2:2; 322 }; 323 324 union dlb2_port_config { 325 struct dlb2_create_ldb_port_args ldb; 326 struct dlb2_create_dir_port_args dir; 327 }; 328 329 enum dlb2_port_state { 330 PORT_CLOSED, 331 PORT_STARTED, 332 PORT_STOPPED 333 }; 334 335 enum dlb2_configuration_state { 336 /* The resource has not been configured */ 337 DLB2_NOT_CONFIGURED, 338 /* The resource was configured, but the device was stopped */ 339 DLB2_PREV_CONFIGURED, 340 /* The resource is currently configured */ 341 DLB2_CONFIGURED 342 }; 343 344 struct dlb2_port { 345 uint32_t id; 346 bool is_directed; 347 bool gen_bit; 348 uint16_t dir_credits; 349 uint32_t dequeue_depth; 350 enum dlb2_token_pop_mode token_pop_mode; 351 union dlb2_port_config cfg; 352 RTE_ATOMIC(uint32_t) *credit_pool[DLB2_NUM_QUEUE_TYPES]; 353 union { 354 struct { 355 uint16_t cached_ldb_credits; 356 uint16_t ldb_credits; 357 uint16_t cached_dir_credits; 358 }; 359 struct { 360 uint16_t cached_credits; 361 uint16_t credits; 362 }; 363 }; 364 bool int_armed; 365 uint16_t owed_tokens; 366 int16_t issued_releases; 367 int16_t token_pop_thresh; 368 int cq_depth; 369 uint16_t cq_idx; 370 uint16_t cq_idx_unmasked; 371 uint16_t cq_depth_mask; 372 uint16_t gen_bit_shift; 373 uint64_t cq_rolling_mask; /* 374 * rotate to always have right expected 375 * gen bits 376 */ 377 uint64_t cq_rolling_mask_2; 378 void *cq_addr_cached; /* avoid multiple refs */ 379 enum dlb2_port_state state; 380 enum dlb2_configuration_state config_state; 381 int num_mapped_qids; 382 uint8_t *qid_mappings; 383 struct dlb2_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */ 384 struct dlb2_enqueue_qe *int_arm_qe; 385 struct dlb2_cq_pop_qe *consume_qe; 386 struct dlb2_eventdev *dlb2; /* back ptr */ 387 struct dlb2_eventdev_port *ev_port; /* back ptr */ 388 bool use_scalar; /* force usage of scalar code */ 389 uint16_t hw_credit_quanta; 390 bool use_avx512; 391 bool is_producer; /* True if port is of type producer */ 392 uint8_t reorder_id; /* id used for reordering events coming back into the scheduler */ 393 bool reorder_en; 394 struct dlb2_reorder *order; /* For ordering enqueues */ 395 }; 396 397 struct dlb2_reorder { 398 /* a reorder buffer for events coming back in different order from dequeue 399 * We use UINT8_MAX + 1 elements, but add on three no-ops to make movdirs easier at the end 400 */ 401 union { 402 __m128i m128; 403 struct dlb2_enqueue_qe qe; 404 uint64_t u64[2]; 405 } enq_reorder[UINT8_MAX + 4]; 406 /* id of the next entry in the reorder enqueue ring to send in */ 407 uint8_t next_to_enqueue; 408 }; 409 410 /* Per-process per-port mmio and memory pointers */ 411 struct process_local_port_data { 412 uint64_t *pp_addr; 413 struct dlb2_dequeue_qe *cq_base; 414 const struct rte_memzone *mz; 415 bool mmaped; 416 }; 417 418 struct dlb2_eventdev; 419 420 struct dlb2_port_low_level_io_functions { 421 void (*pp_enqueue_four)(void *qe4, void *pp_addr); 422 }; 423 424 struct dlb2_config { 425 int configured; 426 int reserved; 427 union { 428 struct { 429 uint32_t num_ldb_credits; 430 uint32_t num_dir_credits; 431 }; 432 struct { 433 uint32_t num_credits; 434 }; 435 }; 436 struct dlb2_create_sched_domain_args resources; 437 }; 438 439 enum dlb2_cos { 440 DLB2_COS_DEFAULT = 255, 441 DLB2_COS_0 = 0, 442 DLB2_COS_1, 443 DLB2_COS_2, 444 DLB2_COS_3, 445 DLB2_COS_NUM_VALS 446 }; 447 448 struct __rte_cache_aligned dlb2_hw_dev { 449 struct dlb2_config cfg; 450 struct dlb2_hw_resource_info info; 451 void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb2_dev) */ 452 uint32_t domain_id; 453 rte_spinlock_t resource_lock; /* for MP support */ 454 }; 455 456 /* End HW related defines and structs */ 457 458 /* Begin DLB2 PMD Eventdev related defines and structs */ 459 460 #define DLB2_MAX_NUM_QUEUES(ver) \ 461 (DLB2_MAX_NUM_DIR_QUEUES(ver) + DLB2_MAX_NUM_LDB_QUEUES) 462 463 #define DLB2_MAX_NUM_PORTS(ver) \ 464 (DLB2_MAX_NUM_DIR_PORTS(ver) + DLB2_MAX_NUM_LDB_PORTS) 465 466 #define DLB2_MAX_NUM_DIR_QUEUES_V2_5 96 467 #define DLB2_MAX_NUM_DIR_PORTS_V2_5 DLB2_MAX_NUM_DIR_QUEUES_V2_5 468 #define DLB2_MAX_NUM_QUEUES_ALL \ 469 (DLB2_MAX_NUM_DIR_QUEUES_V2_5 + DLB2_MAX_NUM_LDB_QUEUES) 470 #define DLB2_MAX_NUM_PORTS_ALL \ 471 (DLB2_MAX_NUM_DIR_PORTS_V2_5 + DLB2_MAX_NUM_LDB_PORTS) 472 #define DLB2_MAX_INPUT_QUEUE_DEPTH 256 473 474 /** Structure to hold the queue to port link establishment attributes */ 475 476 struct dlb2_event_queue_link { 477 uint8_t queue_id; 478 uint8_t priority; 479 bool mapped; 480 bool valid; 481 }; 482 483 struct dlb2_traffic_stats { 484 uint64_t rx_ok; 485 uint64_t rx_drop; 486 uint64_t rx_interrupt_wait; 487 uint64_t rx_umonitor_umwait; 488 uint64_t tx_ok; 489 uint64_t total_polls; 490 uint64_t zero_polls; 491 union { 492 struct { 493 uint64_t tx_nospc_ldb_hw_credits; 494 uint64_t tx_nospc_dir_hw_credits; 495 }; 496 struct { 497 uint64_t tx_nospc_hw_credits; 498 }; 499 }; 500 uint64_t tx_nospc_inflight_max; 501 uint64_t tx_nospc_new_event_limit; 502 uint64_t tx_nospc_inflight_credits; 503 }; 504 505 /* DLB2 HW sets the 2bit qid_depth in rx QEs based on the programmable depth 506 * threshold. The global default value in config/common_base (or rte_config.h) 507 * can be overridden on a per-qid basis using a vdev command line parameter. 508 * 3: depth > threshold 509 * 2: threshold >= depth > 3/4 threshold 510 * 1: 3/4 threshold >= depth > 1/2 threshold 511 * 0: depth <= 1/2 threshold. 512 */ 513 #define DLB2_QID_DEPTH_LE50 0 514 #define DLB2_QID_DEPTH_GT50_LE75 1 515 #define DLB2_QID_DEPTH_GT75_LE100 2 516 #define DLB2_QID_DEPTH_GT100 3 517 #define DLB2_NUM_QID_DEPTH_STAT_VALS 4 /* 2 bits */ 518 519 struct dlb2_queue_stats { 520 uint64_t enq_ok; 521 uint64_t qid_depth[DLB2_NUM_QID_DEPTH_STAT_VALS]; 522 }; 523 524 struct dlb2_port_stats { 525 struct dlb2_traffic_stats traffic; 526 uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */ 527 uint64_t tx_implicit_rel; 528 uint64_t tx_sched_cnt[DLB2_NUM_HW_SCHED_TYPES]; 529 uint64_t tx_invalid; 530 uint64_t rx_sched_cnt[DLB2_NUM_HW_SCHED_TYPES]; 531 uint64_t rx_sched_invalid; 532 struct dlb2_queue_stats queue[DLB2_MAX_NUM_QUEUES_ALL]; 533 }; 534 535 struct __rte_cache_aligned dlb2_eventdev_port { 536 struct dlb2_port qm_port; /* hw specific data structure */ 537 struct rte_event_port_conf conf; /* user-supplied configuration */ 538 uint16_t inflight_credits; /* num credits this port has right now */ 539 uint16_t credit_update_quanta; 540 struct dlb2_eventdev *dlb2; /* backlink optimization */ 541 alignas(RTE_CACHE_LINE_SIZE) struct dlb2_port_stats stats; 542 struct dlb2_event_queue_link link[DLB2_MAX_NUM_QIDS_PER_LDB_CQ]; 543 int num_links; 544 uint32_t id; /* port id */ 545 /* num releases yet to be completed on this port. 546 * Only applies to load-balanced ports. 547 */ 548 uint16_t outstanding_releases; 549 uint16_t inflight_max; /* app requested max inflights for this port */ 550 int enq_retries; /* Number of attempts before ret ENOSPC */ 551 /* setup_done is set when the event port is setup */ 552 bool setup_done; 553 /* enq_configured is set when the qm port is created */ 554 bool enq_configured; 555 uint8_t implicit_release; /* release events before dequeuing */ 556 uint32_t cq_weight; /* DLB2.5 and above ldb ports only */ 557 int cos_id; /*ldb port class of service */ 558 }; 559 560 struct dlb2_queue { 561 uint32_t num_qid_inflights; /* User config */ 562 uint32_t num_atm_inflights; /* User config */ 563 enum dlb2_configuration_state config_state; 564 int sched_type; /* LB queue only */ 565 uint8_t id; 566 bool is_directed; 567 }; 568 569 struct dlb2_eventdev_queue { 570 struct dlb2_queue qm_queue; 571 struct rte_event_queue_conf conf; /* User config */ 572 int depth_threshold; /* use default if 0 */ 573 uint32_t id; 574 bool setup_done; 575 uint8_t num_links; 576 }; 577 578 enum dlb2_run_state { 579 DLB2_RUN_STATE_STOPPED = 0, 580 DLB2_RUN_STATE_STOPPING, 581 DLB2_RUN_STATE_STARTING, 582 DLB2_RUN_STATE_STARTED 583 }; 584 585 struct dlb2_eventdev { 586 struct dlb2_eventdev_port ev_ports[DLB2_MAX_NUM_PORTS_ALL]; 587 struct dlb2_eventdev_queue ev_queues[DLB2_MAX_NUM_QUEUES_ALL]; 588 uint8_t qm_ldb_to_ev_queue_id[DLB2_MAX_NUM_QUEUES_ALL]; 589 uint8_t qm_dir_to_ev_queue_id[DLB2_MAX_NUM_QUEUES_ALL]; 590 /* store num stats and offset of the stats for each queue */ 591 uint16_t xstats_count_per_qid[DLB2_MAX_NUM_QUEUES_ALL]; 592 uint16_t xstats_offset_for_qid[DLB2_MAX_NUM_QUEUES_ALL]; 593 /* store num stats and offset of the stats for each port */ 594 uint16_t xstats_count_per_port[DLB2_MAX_NUM_PORTS_ALL]; 595 uint16_t xstats_offset_for_port[DLB2_MAX_NUM_PORTS_ALL]; 596 struct dlb2_get_num_resources_args hw_rsrc_query_results; 597 uint32_t xstats_count_mode_queue; 598 struct dlb2_hw_dev qm_instance; /* strictly hw related */ 599 uint64_t global_dequeue_wait_ticks; 600 struct dlb2_xstats_entry *xstats; 601 struct rte_eventdev *event_dev; /* backlink to dev */ 602 uint32_t xstats_count_mode_dev; 603 uint32_t xstats_count_mode_port; 604 uint32_t xstats_count; 605 RTE_ATOMIC(uint32_t) inflights; 606 uint32_t new_event_limit; 607 int max_num_events_override; 608 int num_dir_credits_override; 609 bool vector_opts_enabled; 610 int max_cq_depth; 611 int max_enq_depth; 612 volatile enum dlb2_run_state run_state; 613 uint16_t num_dir_queues; /* total num of evdev dir queues requested */ 614 union { 615 struct { 616 uint16_t num_dir_credits; 617 uint16_t num_ldb_credits; 618 }; 619 struct { 620 uint16_t num_credits; 621 }; 622 }; 623 uint16_t num_queues; /* total queues */ 624 uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */ 625 uint16_t num_ports; /* total num of evdev ports requested */ 626 uint16_t num_ldb_ports; /* total num of ldb ports requested */ 627 uint16_t num_dir_ports; /* total num of dir ports requested */ 628 bool umwait_allowed; 629 bool global_dequeue_wait; /* Not using per dequeue wait if true */ 630 enum dlb2_cq_poll_modes poll_mode; 631 int poll_interval; 632 int sw_credit_quanta; 633 int hw_credit_quanta; 634 int default_depth_thresh; 635 uint8_t revision; 636 uint8_t version; 637 bool configured; 638 union { 639 struct { 640 uint16_t max_ldb_credits; 641 uint16_t max_dir_credits; 642 alignas(RTE_CACHE_LINE_SIZE) RTE_ATOMIC(uint32_t) ldb_credit_pool; 643 alignas(RTE_CACHE_LINE_SIZE) RTE_ATOMIC(uint32_t) dir_credit_pool; 644 }; 645 struct { 646 uint16_t max_credits; 647 alignas(RTE_CACHE_LINE_SIZE) RTE_ATOMIC(uint32_t) credit_pool; 648 }; 649 }; 650 uint32_t cos_ports[DLB2_COS_NUM_VALS]; /* total ldb ports in each class */ 651 uint32_t cos_bw[DLB2_COS_NUM_VALS]; /* bandwidth per cos domain */ 652 uint8_t max_cos_port; /* Max LDB port from any cos */ 653 bool enable_cq_weight; 654 }; 655 656 /* used for collecting and passing around the dev args */ 657 struct dlb2_qid_depth_thresholds { 658 int val[DLB2_MAX_NUM_QUEUES_ALL]; 659 }; 660 661 struct dlb2_port_cos { 662 int cos_id[DLB2_MAX_NUM_PORTS_ALL]; 663 }; 664 665 struct dlb2_cos_bw { 666 int val[DLB2_COS_NUM_VALS]; 667 }; 668 669 struct dlb2_devargs { 670 int socket_id; 671 int max_num_events; 672 int num_dir_credits_override; 673 int dev_id; 674 struct dlb2_qid_depth_thresholds qid_depth_thresholds; 675 int poll_interval; 676 int sw_credit_quanta; 677 int hw_credit_quanta; 678 int default_depth_thresh; 679 bool vector_opts_enabled; 680 int max_cq_depth; 681 int max_enq_depth; 682 struct dlb2_port_cos port_cos; 683 struct dlb2_cos_bw cos_bw; 684 const char *producer_coremask; 685 bool default_ldb_port_allocation; 686 bool enable_cq_weight; 687 }; 688 689 /* End Eventdev related defines and structs */ 690 691 /* Forwards for non-inlined functions */ 692 693 void dlb2_eventdev_dump(struct rte_eventdev *dev, FILE *f); 694 695 int dlb2_xstats_init(struct dlb2_eventdev *dlb2); 696 697 void dlb2_xstats_uninit(struct dlb2_eventdev *dlb2); 698 699 int dlb2_eventdev_xstats_get(const struct rte_eventdev *dev, 700 enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id, 701 const uint64_t ids[], uint64_t values[], unsigned int n); 702 703 int dlb2_eventdev_xstats_get_names(const struct rte_eventdev *dev, 704 enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id, 705 struct rte_event_dev_xstats_name *xstat_names, 706 uint64_t *ids, unsigned int size); 707 708 uint64_t dlb2_eventdev_xstats_get_by_name(const struct rte_eventdev *dev, 709 const char *name, uint64_t *id); 710 711 int dlb2_eventdev_xstats_reset(struct rte_eventdev *dev, 712 enum rte_event_dev_xstats_mode mode, 713 int16_t queue_port_id, 714 const uint64_t ids[], 715 uint32_t nb_ids); 716 717 int test_dlb2_eventdev(void); 718 719 int dlb2_primary_eventdev_probe(struct rte_eventdev *dev, 720 const char *name, 721 struct dlb2_devargs *dlb2_args); 722 723 int dlb2_secondary_eventdev_probe(struct rte_eventdev *dev, 724 const char *name); 725 726 uint32_t dlb2_get_queue_depth(struct dlb2_eventdev *dlb2, 727 struct dlb2_eventdev_queue *queue); 728 729 int dlb2_parse_params(const char *params, 730 const char *name, 731 struct dlb2_devargs *dlb2_args, 732 uint8_t version); 733 734 void dlb2_event_build_hcws(struct dlb2_port *qm_port, 735 const struct rte_event ev[], 736 int num, 737 uint8_t *sched_type, 738 uint8_t *queue_id); 739 740 /* Extern functions */ 741 extern int rte_eal_parse_coremask(const char *coremask, int *cores); 742 743 /* Extern globals */ 744 extern struct process_local_port_data dlb2_port[][DLB2_NUM_PORT_TYPES]; 745 746 #endif /* _DLB2_PRIV_H_ */ 747