1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
3 */
4
5 #ifndef _OPDL_EVDEV_H_
6 #define _OPDL_EVDEV_H_
7
8 #include <rte_eventdev.h>
9 #include <eventdev_pmd_vdev.h>
10 #include <rte_atomic.h>
11 #include "opdl_ring.h"
12
13 #define OPDL_QID_NUM_FIDS 1024
14 #define OPDL_IQS_MAX 1
15 #define OPDL_Q_PRIORITY_MAX 1
16 #define OPDL_PORTS_MAX 64
17 #define MAX_OPDL_CONS_Q_DEPTH 128
18 /* OPDL size */
19 #define OPDL_INFLIGHT_EVENTS_TOTAL 4096
20 /* allow for lots of over-provisioning */
21 #define OPDL_FRAGMENTS_MAX 1
22
23 /* report dequeue burst sizes in buckets */
24 #define OPDL_DEQ_STAT_BUCKET_SHIFT 2
25 /* how many packets pulled from port by sched */
26 #define SCHED_DEQUEUE_BURST_SIZE 32
27
28 /* size of our history list */
29 #define OPDL_PORT_HIST_LIST (MAX_OPDL_PROD_Q_DEPTH)
30
31 /* how many data points use for average stats */
32 #define NUM_SAMPLES 64
33
34 #define EVENTDEV_NAME_OPDL_PMD event_opdl
35 #define OPDL_PMD_NAME RTE_STR(event_opdl)
36 #define OPDL_PMD_NAME_MAX 64
37
38 #define OPDL_INVALID_QID 255
39
40 #define OPDL_SCHED_TYPE_DIRECT (RTE_SCHED_TYPE_PARALLEL + 1)
41
42 #define OPDL_NUM_POLL_BUCKETS \
43 (MAX_OPDL_CONS_Q_DEPTH >> OPDL_DEQ_STAT_BUCKET_SHIFT)
44
45 enum {
46 QE_FLAG_VALID_SHIFT = 0,
47 QE_FLAG_COMPLETE_SHIFT,
48 QE_FLAG_NOT_EOP_SHIFT,
49 _QE_FLAG_COUNT
50 };
51
52 enum port_type {
53 OPDL_INVALID_PORT = 0,
54 OPDL_REGULAR_PORT = 1,
55 OPDL_PURE_RX_PORT,
56 OPDL_PURE_TX_PORT,
57 OPDL_ASYNC_PORT
58 };
59
60 enum queue_type {
61 OPDL_Q_TYPE_INVALID = 0,
62 OPDL_Q_TYPE_SINGLE_LINK = 1,
63 OPDL_Q_TYPE_ATOMIC,
64 OPDL_Q_TYPE_ORDERED
65 };
66
67 enum queue_pos {
68 OPDL_Q_POS_START = 0,
69 OPDL_Q_POS_MIDDLE,
70 OPDL_Q_POS_END
71 };
72
73 #define QE_FLAG_VALID (1 << QE_FLAG_VALID_SHIFT) /* for NEW FWD, FRAG */
74 #define QE_FLAG_COMPLETE (1 << QE_FLAG_COMPLETE_SHIFT) /* set for FWD, DROP */
75 #define QE_FLAG_NOT_EOP (1 << QE_FLAG_NOT_EOP_SHIFT) /* set for FRAG only */
76
77 static const uint8_t opdl_qe_flag_map[] = {
78 QE_FLAG_VALID /* NEW Event */,
79 QE_FLAG_VALID | QE_FLAG_COMPLETE /* FWD Event */,
80 QE_FLAG_COMPLETE /* RELEASE Event */,
81
82 /* Values which can be used for future support for partial
83 * events, i.e. where one event comes back to the scheduler
84 * as multiple which need to be tracked together
85 */
86 QE_FLAG_VALID | QE_FLAG_COMPLETE | QE_FLAG_NOT_EOP,
87 };
88
89
90 enum port_xstat_name {
91 claim_pkts_requested = 0,
92 claim_pkts_granted,
93 claim_non_empty,
94 claim_empty,
95 total_cycles,
96 max_num_port_xstat
97 };
98
99 #define OPDL_MAX_PORT_XSTAT_NUM (OPDL_PORTS_MAX * max_num_port_xstat)
100
101 struct opdl_port;
102
103 typedef uint16_t (*opdl_enq_operation)(struct opdl_port *port,
104 const struct rte_event ev[],
105 uint16_t num);
106
107 typedef uint16_t (*opdl_deq_operation)(struct opdl_port *port,
108 struct rte_event ev[],
109 uint16_t num);
110
111 struct opdl_evdev;
112
113 struct opdl_stage_meta_data {
114 uint32_t num_claimed; /* number of entries claimed by this stage */
115 uint32_t burst_sz; /* Port claim burst size */
116 };
117
118 struct opdl_port {
119
120 /* back pointer */
121 struct opdl_evdev *opdl;
122
123 /* enq handler & stage instance */
124 opdl_enq_operation enq;
125 struct opdl_stage *enq_stage_inst;
126
127 /* deq handler & stage instance */
128 opdl_deq_operation deq;
129 struct opdl_stage *deq_stage_inst;
130
131 /* port id has correctly been set */
132 uint8_t configured;
133
134 /* set when the port is initialized */
135 uint8_t initialized;
136
137 /* A numeric ID for the port */
138 uint8_t id;
139
140 /* Space for claimed entries */
141 struct rte_event *entries[MAX_OPDL_CONS_Q_DEPTH];
142
143 /* RX/REGULAR/TX/ASYNC - determined on position in queue */
144 enum port_type p_type;
145
146 /* if the claim is static atomic type */
147 bool atomic_claim;
148
149 /* Queue linked to this port - internal queue id*/
150 uint8_t queue_id;
151
152 /* Queue linked to this port - external queue id*/
153 uint8_t external_qid;
154
155 /* Next queue linked to this port - external queue id*/
156 uint8_t next_external_qid;
157
158 /* number of instances of this stage */
159 uint32_t num_instance;
160
161 /* instance ID of this stage*/
162 uint32_t instance_id;
163
164 /* track packets in and out of this port */
165 uint64_t port_stat[max_num_port_xstat];
166 uint64_t start_cycles;
167 };
168
169 struct opdl_queue_meta_data {
170 uint8_t ext_id;
171 enum queue_type type;
172 int8_t setup;
173 };
174
175 struct opdl_xstats_entry {
176 struct rte_event_dev_xstats_name stat;
177 unsigned int id;
178 uint64_t *value;
179 };
180
181 struct opdl_queue {
182
183 /* Opdl ring this queue is associated with */
184 uint32_t opdl_id;
185
186 /* type and position have correctly been set */
187 uint8_t configured;
188
189 /* port number and associated ports have been associated */
190 uint8_t initialized;
191
192 /* type of this queue (Atomic, Ordered, Parallel, Direct)*/
193 enum queue_type q_type;
194
195 /* position of queue (START, MIDDLE, END) */
196 enum queue_pos q_pos;
197
198 /* external queue id. It is mapped to the queue position */
199 uint8_t external_qid;
200
201 struct opdl_port *ports[OPDL_PORTS_MAX];
202 uint32_t nb_ports;
203
204 /* priority, reserved for future */
205 uint8_t priority;
206 };
207
208
209 #define OPDL_TUR_PER_DEV 12
210
211 /* PMD needs an extra queue per Opdl */
212 #define OPDL_MAX_QUEUES (RTE_EVENT_MAX_QUEUES_PER_DEV - OPDL_TUR_PER_DEV)
213
214
215 struct opdl_evdev {
216 struct rte_eventdev_data *data;
217
218 uint8_t started;
219
220 /* Max number of ports and queues*/
221 uint32_t max_port_nb;
222 uint32_t max_queue_nb;
223
224 /* slots in the opdl ring */
225 uint32_t nb_events_limit;
226
227 /*
228 * Array holding all opdl for this device
229 */
230 struct opdl_ring *opdl[OPDL_TUR_PER_DEV];
231 uint32_t nb_opdls;
232
233 struct opdl_queue_meta_data q_md[OPDL_MAX_QUEUES];
234 uint32_t nb_q_md;
235
236 /* Internal queues - one per logical queue */
237 alignas(RTE_CACHE_LINE_SIZE) struct opdl_queue
238 queue[RTE_EVENT_MAX_QUEUES_PER_DEV];
239
240 uint32_t nb_queues;
241
242 struct opdl_stage_meta_data s_md[OPDL_PORTS_MAX];
243
244 /* Contains all ports - load balanced and directed */
245 alignas(RTE_CACHE_LINE_SIZE) struct opdl_port ports[OPDL_PORTS_MAX];
246 uint32_t nb_ports;
247
248 uint8_t q_map_ex_to_in[OPDL_INVALID_QID];
249
250 /* Stats */
251 struct opdl_xstats_entry port_xstat[OPDL_MAX_PORT_XSTAT_NUM];
252
253 char service_name[OPDL_PMD_NAME_MAX];
254 int socket;
255 int do_validation;
256 int do_test;
257 };
258
259
260 static inline struct opdl_evdev *
opdl_pmd_priv(const struct rte_eventdev * eventdev)261 opdl_pmd_priv(const struct rte_eventdev *eventdev)
262 {
263 return eventdev->data->dev_private;
264 }
265
266 static inline uint8_t
opdl_pmd_dev_id(const struct opdl_evdev * opdl)267 opdl_pmd_dev_id(const struct opdl_evdev *opdl)
268 {
269 return opdl->data->dev_id;
270 }
271
272 static inline const struct opdl_evdev *
opdl_pmd_priv_const(const struct rte_eventdev * eventdev)273 opdl_pmd_priv_const(const struct rte_eventdev *eventdev)
274 {
275 return eventdev->data->dev_private;
276 }
277
278 uint16_t opdl_event_enqueue(void *port, const struct rte_event *ev);
279 uint16_t opdl_event_enqueue_burst(void *port, const struct rte_event ev[],
280 uint16_t num);
281
282 uint16_t opdl_event_dequeue(void *port, struct rte_event *ev, uint64_t wait);
283 uint16_t opdl_event_dequeue_burst(void *port, struct rte_event *ev,
284 uint16_t num, uint64_t wait);
285 void opdl_event_schedule(struct rte_eventdev *dev);
286
287 void opdl_xstats_init(struct rte_eventdev *dev);
288 int opdl_xstats_uninit(struct rte_eventdev *dev);
289 int opdl_xstats_get_names(const struct rte_eventdev *dev,
290 enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
291 struct rte_event_dev_xstats_name *xstats_names,
292 uint64_t *ids, unsigned int size);
293 int opdl_xstats_get(const struct rte_eventdev *dev,
294 enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
295 const uint64_t ids[], uint64_t values[], unsigned int n);
296 uint64_t opdl_xstats_get_by_name(const struct rte_eventdev *dev,
297 const char *name, uint64_t *id);
298 int opdl_xstats_reset(struct rte_eventdev *dev,
299 enum rte_event_dev_xstats_mode mode,
300 int16_t queue_port_id,
301 const uint64_t ids[],
302 uint32_t nb_ids);
303
304 int opdl_add_event_handlers(struct rte_eventdev *dev);
305 int build_all_dependencies(struct rte_eventdev *dev);
306 int check_queues_linked(struct rte_eventdev *dev);
307 int create_queues_and_rings(struct rte_eventdev *dev);
308 int initialise_all_other_ports(struct rte_eventdev *dev);
309 int initialise_queue_zero_ports(struct rte_eventdev *dev);
310 int assign_internal_queue_ids(struct rte_eventdev *dev);
311 void destroy_queues_and_rings(struct rte_eventdev *dev);
312 int opdl_selftest(void);
313
314 #endif /* _OPDL_EVDEV_H_ */
315