xref: /dpdk/drivers/event/dlb2/dlb2_priv.h (revision 6e2e98d6775b9d39ef4d5ef75d86416085154c88)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2016-2022 Intel Corporation
3  */
4 
5 #ifndef _DLB2_PRIV_H_
6 #define _DLB2_PRIV_H_
7 
8 #include <emmintrin.h>
9 #include <stdbool.h>
10 
11 #include <rte_eventdev.h>
12 #include <rte_config.h>
13 #include "dlb2_user.h"
14 #include "dlb2_log.h"
15 #include "rte_pmd_dlb2.h"
16 
17 #ifndef RTE_LIBRTE_PMD_DLB2_QUELL_STATS
18 #define DLB2_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
19 #else
20 #define DLB2_INC_STAT(_stat, _incr_val)
21 #endif
22 
23 #define EVDEV_DLB2_NAME_PMD dlb2_event
24 
25 /* Default values for command line devargs */
26 #define DLB2_POLL_INTERVAL_DEFAULT 1000
27 #define DLB2_SW_CREDIT_QUANTA_DEFAULT 32 /* Default = Worker */
28 #define DLB2_SW_CREDIT_P_QUANTA_DEFAULT 256 /* Producer */
29 #define DLB2_SW_CREDIT_C_QUANTA_DEFAULT 256 /* Consumer */
30 #define DLB2_DEPTH_THRESH_DEFAULT 256
31 #define DLB2_MIN_CQ_DEPTH_OVERRIDE 32
32 #define DLB2_MAX_CQ_DEPTH_OVERRIDE 1024
33 #define DLB2_MAX_CQ_DEPTH_REORDER 128
34 #define DLB2_MIN_ENQ_DEPTH_OVERRIDE 32
35 #define DLB2_MAX_ENQ_DEPTH_OVERRIDE 1024
36 
37 
38 /*  command line arg strings */
39 #define NUMA_NODE_ARG "numa_node"
40 #define DLB2_MAX_NUM_EVENTS "max_num_events"
41 #define DLB2_NUM_DIR_CREDITS "num_dir_credits"
42 #define DEV_ID_ARG "dev_id"
43 #define DLB2_QID_DEPTH_THRESH_ARG "qid_depth_thresh"
44 #define DLB2_POLL_INTERVAL_ARG "poll_interval"
45 #define DLB2_SW_CREDIT_QUANTA_ARG "sw_credit_quanta"
46 #define DLB2_HW_CREDIT_QUANTA_ARG "hw_credit_quanta"
47 #define DLB2_DEPTH_THRESH_ARG "default_depth_thresh"
48 #define DLB2_VECTOR_OPTS_ENAB_ARG "vector_opts_enable"
49 #define DLB2_MAX_CQ_DEPTH "max_cq_depth"
50 #define DLB2_MAX_ENQ_DEPTH "max_enqueue_depth"
51 #define DLB2_PORT_COS "port_cos"
52 #define DLB2_COS_BW "cos_bw"
53 #define DLB2_PRODUCER_COREMASK "producer_coremask"
54 #define DLB2_DEFAULT_LDB_PORT_ALLOCATION_ARG "default_port_allocation"
55 #define DLB2_ENABLE_CQ_WEIGHT_ARG "enable_cq_weight"
56 
57 /* Begin HW related defines and structs */
58 
59 #define DLB2_HW_V2 0
60 #define DLB2_HW_V2_5 1
61 #define DLB2_MAX_NUM_DOMAINS 32
62 #define DLB2_MAX_NUM_VFS 16
63 #define DLB2_MAX_NUM_LDB_QUEUES 32
64 #define DLB2_MAX_NUM_LDB_PORTS 64
65 #define DLB2_MAX_NUM_DIR_PORTS_V2		DLB2_MAX_NUM_DIR_QUEUES_V2
66 #define DLB2_MAX_NUM_DIR_PORTS_V2_5		DLB2_MAX_NUM_DIR_QUEUES_V2_5
67 #define DLB2_MAX_NUM_DIR_PORTS(ver)		(ver == DLB2_HW_V2 ? \
68 						 DLB2_MAX_NUM_DIR_PORTS_V2 : \
69 						 DLB2_MAX_NUM_DIR_PORTS_V2_5)
70 #define DLB2_MAX_NUM_DIR_QUEUES_V2		64 /* DIR == directed */
71 #define DLB2_MAX_NUM_DIR_QUEUES_V2_5		96
72 /* When needed for array sizing, the DLB 2.5 macro is used */
73 #define DLB2_MAX_NUM_DIR_QUEUES(ver)		(ver == DLB2_HW_V2 ? \
74 						 DLB2_MAX_NUM_DIR_QUEUES_V2 : \
75 						 DLB2_MAX_NUM_DIR_QUEUES_V2_5)
76 #define DLB2_MAX_NUM_FLOWS (64 * 1024)
77 #define DLB2_MAX_NUM_LDB_CREDITS (8 * 1024)
78 #define DLB2_MAX_NUM_DIR_CREDITS(ver)		(ver == DLB2_HW_V2 ? 4096 : 0)
79 #define DLB2_MAX_NUM_CREDITS(ver)		(ver == DLB2_HW_V2 ? \
80 						 0 : DLB2_MAX_NUM_LDB_CREDITS)
81 #define DLB2_MAX_NUM_LDB_CREDIT_POOLS 64
82 #define DLB2_MAX_NUM_DIR_CREDIT_POOLS 64
83 #define DLB2_MAX_NUM_HIST_LIST_ENTRIES 2048
84 #define DLB2_MAX_NUM_QIDS_PER_LDB_CQ 8
85 #define DLB2_QID_PRIORITIES 8
86 #define DLB2_MAX_DEVICE_PATH 32
87 #define DLB2_MIN_DEQUEUE_TIMEOUT_NS 1
88 /* Note: "- 1" here to support the timeout range check in eventdev_autotest */
89 #define DLB2_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
90 #define DLB2_SW_CREDIT_BATCH_SZ 32 /* Default - Worker */
91 #define DLB2_SW_CREDIT_P_BATCH_SZ 256 /* Producer */
92 #define DLB2_SW_CREDIT_C_BATCH_SZ 256 /* Consumer */
93 #define DLB2_NUM_SN_GROUPS 2
94 #define DLB2_MAX_LDB_SN_ALLOC 1024
95 #define DLB2_MAX_QUEUE_DEPTH_THRESHOLD 8191
96 #define DLB2_MAX_NUM_LDB_PORTS_PER_COS (DLB2_MAX_NUM_LDB_PORTS/DLB2_COS_NUM_VALS)
97 
98 /* 2048 total hist list entries and 64 total ldb ports, which
99  * makes for 2048/64 == 32 hist list entries per port. However, CQ
100  * depth must be a power of 2 and must also be >= HIST LIST entries.
101  * As a result we just limit the maximum dequeue depth to 32.
102  */
103 #define DLB2_MAX_HL_ENTRIES 2048
104 #define DLB2_MIN_CQ_DEPTH 1
105 #define DLB2_DEFAULT_CQ_DEPTH 32
106 #define DLB2_MIN_HARDWARE_CQ_DEPTH 8
107 #define DLB2_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
108 	DLB2_DEFAULT_CQ_DEPTH
109 
110 #define DLB2_HW_DEVICE_FROM_PCI_ID(_pdev) \
111 	(((_pdev->id.device_id == PCI_DEVICE_ID_INTEL_DLB2_5_PF) ||        \
112 	  (_pdev->id.device_id == PCI_DEVICE_ID_INTEL_DLB2_5_VF))   ?   \
113 		DLB2_HW_V2_5 : DLB2_HW_V2)
114 
115 /*
116  * Static per queue/port provisioning values
117  */
118 #define DLB2_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 64
119 
120 #define CQ_BASE(is_dir) ((is_dir) ? DLB2_DIR_CQ_BASE : DLB2_LDB_CQ_BASE)
121 #define CQ_SIZE(is_dir) ((is_dir) ? DLB2_DIR_CQ_MAX_SIZE : \
122 				    DLB2_LDB_CQ_MAX_SIZE)
123 #define PP_BASE(is_dir) ((is_dir) ? DLB2_DIR_PP_BASE : DLB2_LDB_PP_BASE)
124 
125 #define DLB2_NUM_QES_PER_CACHE_LINE 4
126 
127 #define DLB2_MAX_ENQUEUE_DEPTH 32
128 #define DLB2_MIN_ENQUEUE_DEPTH 4
129 
130 #define DLB2_NAME_SIZE 64
131 
132 #define DLB2_1K 1024
133 #define DLB2_2K (2 * DLB2_1K)
134 #define DLB2_4K (4 * DLB2_1K)
135 #define DLB2_16K (16 * DLB2_1K)
136 #define DLB2_32K (32 * DLB2_1K)
137 #define DLB2_1MB (DLB2_1K * DLB2_1K)
138 #define DLB2_16MB (16 * DLB2_1MB)
139 
140 /* Use the upper 3 bits of the event priority to select the DLB2 priority */
141 #define EV_TO_DLB2_PRIO(x) ((x) >> 5)
142 #define DLB2_TO_EV_PRIO(x) ((x) << 5)
143 
144 enum dlb2_hw_ver {
145 	DLB2_HW_VER_2,
146 	DLB2_HW_VER_2_5,
147 };
148 
149 enum dlb2_hw_port_types {
150 	DLB2_LDB_PORT,
151 	DLB2_DIR_PORT,
152 	DLB2_NUM_PORT_TYPES /* Must be last */
153 };
154 
155 enum dlb2_hw_queue_types {
156 	DLB2_LDB_QUEUE,
157 	DLB2_DIR_QUEUE,
158 	DLB2_NUM_QUEUE_TYPES /* Must be last */
159 };
160 
161 #define DLB2_COMBINED_POOL DLB2_LDB_QUEUE
162 
163 #define PORT_TYPE(p) ((p)->is_directed ? DLB2_DIR_PORT : DLB2_LDB_PORT)
164 
165 /* Do not change - must match hardware! */
166 enum dlb2_hw_sched_type {
167 	DLB2_SCHED_ATOMIC = 0,
168 	DLB2_SCHED_UNORDERED,
169 	DLB2_SCHED_ORDERED,
170 	DLB2_SCHED_DIRECTED,
171 	/* DLB2_NUM_HW_SCHED_TYPES must be last */
172 	DLB2_NUM_HW_SCHED_TYPES
173 };
174 
175 struct dlb2_hw_rsrcs {
176 	int32_t nb_events_limit;
177 	uint32_t num_queues;		/* Total queues (lb + dir) */
178 	uint32_t num_ldb_queues;	/* Number of available ldb queues */
179 	uint32_t num_ldb_ports;         /* Number of load balanced ports */
180 	uint32_t num_dir_ports;         /* Number of directed ports */
181 	union {
182 		struct {
183 			uint32_t num_ldb_credits; /* Number of ldb credits */
184 			uint32_t num_dir_credits; /* Number of dir credits */
185 		};
186 		struct {
187 			uint32_t num_credits; /* Number of combined credits */
188 		};
189 	};
190 	uint32_t reorder_window_size;   /* Size of reorder window */
191 };
192 
193 struct dlb2_hw_resource_info {
194 	/**> Max resources that can be provided */
195 	struct dlb2_hw_rsrcs hw_rsrc_max;
196 	int num_sched_domains;
197 	uint32_t socket_id;
198 };
199 
200 enum dlb2_enqueue_type {
201 	/**>
202 	 * New : Used to inject a new packet into the QM.
203 	 */
204 	DLB2_ENQ_NEW,
205 	/**>
206 	 * Forward : Enqueues a packet, and
207 	 *  - if atomic: release any lock it holds in the QM
208 	 *  - if ordered: release the packet for egress re-ordering
209 	 */
210 	DLB2_ENQ_FWD,
211 	/**>
212 	 * Enqueue Drop : Release an inflight packet. Must be called with
213 	 * event == NULL. Used to drop a packet.
214 	 *
215 	 * Note that all packets dequeued from a load-balanced port must be
216 	 * released, either with DLB2_ENQ_DROP or DLB2_ENQ_FWD.
217 	 */
218 	DLB2_ENQ_DROP,
219 
220 	/* marker for array sizing etc. */
221 	_DLB2_NB_ENQ_TYPES
222 };
223 
224 /* hw-specific format - do not change */
225 
226 struct dlb2_event_type {
227 	uint16_t major:4;
228 	uint16_t unused:4;
229 	uint16_t sub:8;
230 };
231 
232 union dlb2_opaque_data {
233 	uint16_t opaque_data;
234 	struct dlb2_event_type event_type;
235 };
236 
237 struct dlb2_msg_info {
238 	uint8_t qid;
239 	uint8_t sched_type:2;
240 	uint8_t priority:3;
241 	uint8_t msg_type:3;
242 };
243 
244 #define DLB2_NEW_CMD_BYTE 0x08
245 #define DLB2_FWD_CMD_BYTE 0x0A
246 #define DLB2_COMP_CMD_BYTE 0x02
247 #define DLB2_POP_CMD_BYTE 0x01
248 #define DLB2_NOOP_CMD_BYTE 0x00
249 
250 /* hw-specific format - do not change */
251 struct dlb2_enqueue_qe {
252 	uint64_t data;
253 	/* Word 3 */
254 	union dlb2_opaque_data u;
255 	uint8_t qid;
256 	uint8_t sched_type:2;
257 	uint8_t priority:3;
258 	uint8_t msg_type:3;
259 	/* Word 4 */
260 	uint16_t lock_id;
261 	uint8_t meas_lat:1;
262 	uint8_t weight:2; /* DLB 2.5 and above */
263 	uint8_t no_dec:1;
264 	uint8_t cmp_id:4;
265 	union {
266 		uint8_t cmd_byte;
267 		struct {
268 			uint8_t cq_token:1;
269 			uint8_t qe_comp:1;
270 			uint8_t qe_frag:1;
271 			uint8_t qe_valid:1;
272 			uint8_t rsvd3:1;
273 			uint8_t error:1;
274 			uint8_t rsvd:2;
275 		};
276 	};
277 };
278 
279 /* hw-specific format - do not change */
280 struct dlb2_cq_pop_qe {
281 	uint64_t data;
282 	union dlb2_opaque_data u;
283 	uint8_t qid;
284 	uint8_t sched_type:2;
285 	uint8_t priority:3;
286 	uint8_t msg_type:3;
287 	uint16_t tokens:10;
288 	uint16_t rsvd2:6;
289 	uint8_t meas_lat:1;
290 	uint8_t rsvd1:2;
291 	uint8_t no_dec:1;
292 	uint8_t cmp_id:4;
293 	union {
294 		uint8_t cmd_byte;
295 		struct {
296 			uint8_t cq_token:1;
297 			uint8_t qe_comp:1;
298 			uint8_t qe_frag:1;
299 			uint8_t qe_valid:1;
300 			uint8_t rsvd3:1;
301 			uint8_t error:1;
302 			uint8_t rsvd:2;
303 		};
304 	};
305 };
306 
307 /* hw-specific format - do not change */
308 struct dlb2_dequeue_qe {
309 	uint64_t data;
310 	union dlb2_opaque_data u;
311 	uint8_t qid;
312 	uint8_t sched_type:2;
313 	uint8_t priority:3;
314 	uint8_t msg_type:3;
315 	uint16_t flow_id:16; /* was pp_id in v1 */
316 	uint8_t debug;
317 	uint8_t cq_gen:1;
318 	uint8_t qid_depth:2; /* 2 bits in v2 */
319 	uint8_t rsvd1:2;
320 	uint8_t error:1;
321 	uint8_t rsvd2:2;
322 };
323 
324 union dlb2_port_config {
325 	struct dlb2_create_ldb_port_args ldb;
326 	struct dlb2_create_dir_port_args dir;
327 };
328 
329 enum dlb2_port_state {
330 	PORT_CLOSED,
331 	PORT_STARTED,
332 	PORT_STOPPED
333 };
334 
335 enum dlb2_configuration_state {
336 	/* The resource has not been configured */
337 	DLB2_NOT_CONFIGURED,
338 	/* The resource was configured, but the device was stopped */
339 	DLB2_PREV_CONFIGURED,
340 	/* The resource is currently configured */
341 	DLB2_CONFIGURED
342 };
343 
344 struct dlb2_port {
345 	uint32_t id;
346 	bool is_directed;
347 	bool gen_bit;
348 	uint16_t dir_credits;
349 	uint32_t dequeue_depth;
350 	enum dlb2_token_pop_mode token_pop_mode;
351 	union dlb2_port_config cfg;
352 	RTE_ATOMIC(uint32_t) *credit_pool[DLB2_NUM_QUEUE_TYPES];
353 	union {
354 		struct {
355 			uint16_t cached_ldb_credits;
356 			uint16_t ldb_credits;
357 			uint16_t cached_dir_credits;
358 		};
359 		struct {
360 			uint16_t cached_credits;
361 			uint16_t credits;
362 		};
363 	};
364 	bool int_armed;
365 	uint16_t owed_tokens;
366 	int16_t issued_releases;
367 	int16_t token_pop_thresh;
368 	int cq_depth;
369 	uint16_t cq_idx;
370 	uint16_t cq_idx_unmasked;
371 	uint16_t cq_depth_mask;
372 	uint16_t gen_bit_shift;
373 	uint64_t cq_rolling_mask; /*
374 				   * rotate to always have right expected
375 				   * gen bits
376 				   */
377 	uint64_t cq_rolling_mask_2;
378 	void *cq_addr_cached; /* avoid multiple refs */
379 	enum dlb2_port_state state;
380 	enum dlb2_configuration_state config_state;
381 	int num_mapped_qids;
382 	uint8_t *qid_mappings;
383 	struct dlb2_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
384 	struct dlb2_enqueue_qe *int_arm_qe;
385 	struct dlb2_cq_pop_qe *consume_qe;
386 	struct dlb2_eventdev *dlb2; /* back ptr */
387 	struct dlb2_eventdev_port *ev_port; /* back ptr */
388 	bool use_scalar; /* force usage of scalar code */
389 	uint16_t hw_credit_quanta;
390 	bool use_avx512;
391 	bool is_producer; /* True if port is of type producer */
392 	uint8_t reorder_id; /* id used for reordering events coming back into the scheduler */
393 	bool reorder_en;
394 	struct dlb2_reorder *order; /* For ordering enqueues */
395 };
396 
397 struct dlb2_reorder {
398 	/* a reorder buffer for events coming back in different order from dequeue
399 	 * We use UINT8_MAX + 1 elements, but add on three no-ops to make movdirs easier at the end
400 	 */
401 	union {
402 		__m128i m128;
403 		struct dlb2_enqueue_qe qe;
404 		uint64_t u64[2];
405 	} enq_reorder[UINT8_MAX + 4];
406 	/* id of the next entry in the reorder enqueue ring to send in */
407 	uint8_t next_to_enqueue;
408 };
409 
410 /* Per-process per-port mmio and memory pointers */
411 struct process_local_port_data {
412 	uint64_t *pp_addr;
413 	struct dlb2_dequeue_qe *cq_base;
414 	const struct rte_memzone *mz;
415 	bool mmaped;
416 };
417 
418 struct dlb2_eventdev;
419 
420 struct dlb2_port_low_level_io_functions {
421 	void (*pp_enqueue_four)(void *qe4, void *pp_addr);
422 };
423 
424 struct dlb2_config {
425 	int configured;
426 	int reserved;
427 	union {
428 		struct {
429 			uint32_t num_ldb_credits;
430 			uint32_t num_dir_credits;
431 		};
432 		struct {
433 			uint32_t num_credits;
434 		};
435 	};
436 	struct dlb2_create_sched_domain_args resources;
437 };
438 
439 enum dlb2_cos {
440 	DLB2_COS_DEFAULT = 255,
441 	DLB2_COS_0 = 0,
442 	DLB2_COS_1,
443 	DLB2_COS_2,
444 	DLB2_COS_3,
445 	DLB2_COS_NUM_VALS
446 };
447 
448 struct __rte_cache_aligned dlb2_hw_dev {
449 	struct dlb2_config cfg;
450 	struct dlb2_hw_resource_info info;
451 	void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb2_dev) */
452 	uint32_t domain_id;
453 	rte_spinlock_t resource_lock; /* for MP support */
454 };
455 
456 /* End HW related defines and structs */
457 
458 /* Begin DLB2 PMD Eventdev related defines and structs */
459 
460 #define DLB2_MAX_NUM_QUEUES(ver)                                \
461 	(DLB2_MAX_NUM_DIR_QUEUES(ver) + DLB2_MAX_NUM_LDB_QUEUES)
462 
463 #define DLB2_MAX_NUM_PORTS(ver) \
464 	(DLB2_MAX_NUM_DIR_PORTS(ver) + DLB2_MAX_NUM_LDB_PORTS)
465 
466 #define DLB2_MAX_NUM_DIR_QUEUES_V2_5 96
467 #define DLB2_MAX_NUM_DIR_PORTS_V2_5 DLB2_MAX_NUM_DIR_QUEUES_V2_5
468 #define DLB2_MAX_NUM_QUEUES_ALL \
469 	(DLB2_MAX_NUM_DIR_QUEUES_V2_5 + DLB2_MAX_NUM_LDB_QUEUES)
470 #define DLB2_MAX_NUM_PORTS_ALL \
471 	(DLB2_MAX_NUM_DIR_PORTS_V2_5 + DLB2_MAX_NUM_LDB_PORTS)
472 #define DLB2_MAX_INPUT_QUEUE_DEPTH 256
473 
474 /** Structure to hold the queue to port link establishment attributes */
475 
476 struct dlb2_event_queue_link {
477 	uint8_t queue_id;
478 	uint8_t priority;
479 	bool mapped;
480 	bool valid;
481 };
482 
483 struct dlb2_traffic_stats {
484 	uint64_t rx_ok;
485 	uint64_t rx_drop;
486 	uint64_t rx_interrupt_wait;
487 	uint64_t rx_umonitor_umwait;
488 	uint64_t tx_ok;
489 	uint64_t total_polls;
490 	uint64_t zero_polls;
491 	union {
492 		struct {
493 			uint64_t tx_nospc_ldb_hw_credits;
494 			uint64_t tx_nospc_dir_hw_credits;
495 		};
496 		struct {
497 			uint64_t tx_nospc_hw_credits;
498 		};
499 	};
500 	uint64_t tx_nospc_inflight_max;
501 	uint64_t tx_nospc_new_event_limit;
502 	uint64_t tx_nospc_inflight_credits;
503 };
504 
505 /* DLB2 HW sets the 2bit qid_depth in rx QEs based on the programmable depth
506  * threshold. The global default value in config/common_base (or rte_config.h)
507  * can be overridden on a per-qid basis using a vdev command line parameter.
508  * 3: depth > threshold
509  * 2: threshold >= depth > 3/4 threshold
510  * 1: 3/4 threshold >= depth > 1/2 threshold
511  * 0: depth <= 1/2 threshold.
512  */
513 #define DLB2_QID_DEPTH_LE50 0
514 #define DLB2_QID_DEPTH_GT50_LE75 1
515 #define DLB2_QID_DEPTH_GT75_LE100 2
516 #define DLB2_QID_DEPTH_GT100 3
517 #define DLB2_NUM_QID_DEPTH_STAT_VALS 4 /* 2 bits */
518 
519 struct dlb2_queue_stats {
520 	uint64_t enq_ok;
521 	uint64_t qid_depth[DLB2_NUM_QID_DEPTH_STAT_VALS];
522 };
523 
524 struct dlb2_port_stats {
525 	struct dlb2_traffic_stats traffic;
526 	uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
527 	uint64_t tx_implicit_rel;
528 	uint64_t tx_sched_cnt[DLB2_NUM_HW_SCHED_TYPES];
529 	uint64_t tx_invalid;
530 	uint64_t rx_sched_cnt[DLB2_NUM_HW_SCHED_TYPES];
531 	uint64_t rx_sched_invalid;
532 	struct dlb2_queue_stats queue[DLB2_MAX_NUM_QUEUES_ALL];
533 };
534 
535 struct __rte_cache_aligned dlb2_eventdev_port {
536 	struct dlb2_port qm_port; /* hw specific data structure */
537 	struct rte_event_port_conf conf; /* user-supplied configuration */
538 	uint16_t inflight_credits; /* num credits this port has right now */
539 	uint16_t credit_update_quanta;
540 	struct dlb2_eventdev *dlb2; /* backlink optimization */
541 	alignas(RTE_CACHE_LINE_SIZE) struct dlb2_port_stats stats;
542 	struct dlb2_event_queue_link link[DLB2_MAX_NUM_QIDS_PER_LDB_CQ];
543 	int num_links;
544 	uint32_t id; /* port id */
545 	/* num releases yet to be completed on this port.
546 	 * Only applies to load-balanced ports.
547 	 */
548 	uint16_t outstanding_releases;
549 	uint16_t inflight_max; /* app requested max inflights for this port */
550 	int enq_retries; /* Number of attempts before ret ENOSPC */
551 	/* setup_done is set when the event port is setup */
552 	bool setup_done;
553 	/* enq_configured is set when the qm port is created */
554 	bool enq_configured;
555 	uint8_t implicit_release; /* release events before dequeuing */
556 	uint32_t cq_weight; /* DLB2.5 and above ldb ports only */
557 	int cos_id; /*ldb port class of service */
558 };
559 
560 struct dlb2_queue {
561 	uint32_t num_qid_inflights; /* User config */
562 	uint32_t num_atm_inflights; /* User config */
563 	enum dlb2_configuration_state config_state;
564 	int  sched_type; /* LB queue only */
565 	uint8_t id;
566 	bool	 is_directed;
567 };
568 
569 struct dlb2_eventdev_queue {
570 	struct dlb2_queue qm_queue;
571 	struct rte_event_queue_conf conf; /* User config */
572 	int depth_threshold; /* use default if 0 */
573 	uint32_t id;
574 	bool setup_done;
575 	uint8_t num_links;
576 };
577 
578 enum dlb2_run_state {
579 	DLB2_RUN_STATE_STOPPED = 0,
580 	DLB2_RUN_STATE_STOPPING,
581 	DLB2_RUN_STATE_STARTING,
582 	DLB2_RUN_STATE_STARTED
583 };
584 
585 struct dlb2_eventdev {
586 	struct dlb2_eventdev_port ev_ports[DLB2_MAX_NUM_PORTS_ALL];
587 	struct dlb2_eventdev_queue ev_queues[DLB2_MAX_NUM_QUEUES_ALL];
588 	uint8_t qm_ldb_to_ev_queue_id[DLB2_MAX_NUM_QUEUES_ALL];
589 	uint8_t qm_dir_to_ev_queue_id[DLB2_MAX_NUM_QUEUES_ALL];
590 	/* store num stats and offset of the stats for each queue */
591 	uint16_t xstats_count_per_qid[DLB2_MAX_NUM_QUEUES_ALL];
592 	uint16_t xstats_offset_for_qid[DLB2_MAX_NUM_QUEUES_ALL];
593 	/* store num stats and offset of the stats for each port */
594 	uint16_t xstats_count_per_port[DLB2_MAX_NUM_PORTS_ALL];
595 	uint16_t xstats_offset_for_port[DLB2_MAX_NUM_PORTS_ALL];
596 	struct dlb2_get_num_resources_args hw_rsrc_query_results;
597 	uint32_t xstats_count_mode_queue;
598 	struct dlb2_hw_dev qm_instance; /* strictly hw related */
599 	uint64_t global_dequeue_wait_ticks;
600 	struct dlb2_xstats_entry *xstats;
601 	struct rte_eventdev *event_dev; /* backlink to dev */
602 	uint32_t xstats_count_mode_dev;
603 	uint32_t xstats_count_mode_port;
604 	uint32_t xstats_count;
605 	RTE_ATOMIC(uint32_t) inflights;
606 	uint32_t new_event_limit;
607 	int max_num_events_override;
608 	int num_dir_credits_override;
609 	bool vector_opts_enabled;
610 	int max_cq_depth;
611 	int max_enq_depth;
612 	volatile enum dlb2_run_state run_state;
613 	uint16_t num_dir_queues; /* total num of evdev dir queues requested */
614 	union {
615 		struct {
616 			uint16_t num_dir_credits;
617 			uint16_t num_ldb_credits;
618 		};
619 		struct {
620 			uint16_t num_credits;
621 		};
622 	};
623 	uint16_t num_queues; /* total queues */
624 	uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
625 	uint16_t num_ports; /* total num of evdev ports requested */
626 	uint16_t num_ldb_ports; /* total num of ldb ports requested */
627 	uint16_t num_dir_ports; /* total num of dir ports requested */
628 	bool umwait_allowed;
629 	bool global_dequeue_wait; /* Not using per dequeue wait if true */
630 	enum dlb2_cq_poll_modes poll_mode;
631 	int poll_interval;
632 	int sw_credit_quanta;
633 	int hw_credit_quanta;
634 	int default_depth_thresh;
635 	uint8_t revision;
636 	uint8_t version;
637 	bool configured;
638 	union {
639 		struct {
640 			uint16_t max_ldb_credits;
641 			uint16_t max_dir_credits;
642 			alignas(RTE_CACHE_LINE_SIZE) RTE_ATOMIC(uint32_t) ldb_credit_pool;
643 			alignas(RTE_CACHE_LINE_SIZE) RTE_ATOMIC(uint32_t) dir_credit_pool;
644 		};
645 		struct {
646 			uint16_t max_credits;
647 			alignas(RTE_CACHE_LINE_SIZE) RTE_ATOMIC(uint32_t) credit_pool;
648 		};
649 	};
650 	uint32_t cos_ports[DLB2_COS_NUM_VALS]; /* total ldb ports in each class */
651 	uint32_t cos_bw[DLB2_COS_NUM_VALS]; /* bandwidth per cos domain */
652 	uint8_t max_cos_port; /* Max LDB port from any cos */
653 	bool enable_cq_weight;
654 };
655 
656 /* used for collecting and passing around the dev args */
657 struct dlb2_qid_depth_thresholds {
658 	int val[DLB2_MAX_NUM_QUEUES_ALL];
659 };
660 
661 struct dlb2_port_cos {
662 	int cos_id[DLB2_MAX_NUM_PORTS_ALL];
663 };
664 
665 struct dlb2_cos_bw {
666 	int val[DLB2_COS_NUM_VALS];
667 };
668 
669 struct dlb2_devargs {
670 	int socket_id;
671 	int max_num_events;
672 	int num_dir_credits_override;
673 	int dev_id;
674 	struct dlb2_qid_depth_thresholds qid_depth_thresholds;
675 	int poll_interval;
676 	int sw_credit_quanta;
677 	int hw_credit_quanta;
678 	int default_depth_thresh;
679 	bool vector_opts_enabled;
680 	int max_cq_depth;
681 	int max_enq_depth;
682 	struct dlb2_port_cos port_cos;
683 	struct dlb2_cos_bw cos_bw;
684 	const char *producer_coremask;
685 	bool default_ldb_port_allocation;
686 	bool enable_cq_weight;
687 };
688 
689 /* End Eventdev related defines and structs */
690 
691 /* Forwards for non-inlined functions */
692 
693 void dlb2_eventdev_dump(struct rte_eventdev *dev, FILE *f);
694 
695 int dlb2_xstats_init(struct dlb2_eventdev *dlb2);
696 
697 void dlb2_xstats_uninit(struct dlb2_eventdev *dlb2);
698 
699 int dlb2_eventdev_xstats_get(const struct rte_eventdev *dev,
700 		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
701 		const uint64_t ids[], uint64_t values[], unsigned int n);
702 
703 int dlb2_eventdev_xstats_get_names(const struct rte_eventdev *dev,
704 		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
705 		struct rte_event_dev_xstats_name *xstat_names,
706 		uint64_t *ids, unsigned int size);
707 
708 uint64_t dlb2_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
709 					  const char *name, uint64_t *id);
710 
711 int dlb2_eventdev_xstats_reset(struct rte_eventdev *dev,
712 		enum rte_event_dev_xstats_mode mode,
713 		int16_t queue_port_id,
714 		const uint64_t ids[],
715 		uint32_t nb_ids);
716 
717 int test_dlb2_eventdev(void);
718 
719 int dlb2_primary_eventdev_probe(struct rte_eventdev *dev,
720 				const char *name,
721 				struct dlb2_devargs *dlb2_args);
722 
723 int dlb2_secondary_eventdev_probe(struct rte_eventdev *dev,
724 				  const char *name);
725 
726 uint32_t dlb2_get_queue_depth(struct dlb2_eventdev *dlb2,
727 			      struct dlb2_eventdev_queue *queue);
728 
729 int dlb2_parse_params(const char *params,
730 		      const char *name,
731 		      struct dlb2_devargs *dlb2_args,
732 		      uint8_t version);
733 
734 void dlb2_event_build_hcws(struct dlb2_port *qm_port,
735 			   const struct rte_event ev[],
736 			   int num,
737 			   uint8_t *sched_type,
738 			   uint8_t *queue_id);
739 
740 /* Extern functions */
741 extern int rte_eal_parse_coremask(const char *coremask, int *cores);
742 
743 /* Extern globals */
744 extern struct process_local_port_data dlb2_port[][DLB2_NUM_PORT_TYPES];
745 
746 #endif	/* _DLB2_PRIV_H_ */
747