1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Arm Limited
3 */
4
5 #include <rte_malloc.h>
6 #include <rte_memcpy.h>
7 #include <rte_ptr_compress.h>
8 #include <rte_ring.h>
9 #include <rte_ring_elem.h>
10
11 /* API type to call
12 * rte_ring_<sp/mp or sc/mc>_enqueue_<bulk/burst>
13 * TEST_RING_THREAD_DEF - Uses configured SPSC/MPMC calls
14 * TEST_RING_THREAD_SPSC - Calls SP or SC API
15 * TEST_RING_THREAD_MPMC - Calls MP or MC API
16 */
17 #define TEST_RING_THREAD_DEF 1
18 #define TEST_RING_THREAD_SPSC 2
19 #define TEST_RING_THREAD_MPMC 4
20
21 /* API type to call
22 * TEST_RING_ELEM_SINGLE - Calls single element APIs
23 * TEST_RING_ELEM_BULK - Calls bulk APIs
24 * TEST_RING_ELEM_BURST - Calls burst APIs
25 */
26 #define TEST_RING_ELEM_SINGLE 8
27 #define TEST_RING_ELEM_BULK 16
28 #define TEST_RING_ELEM_BURST 32
29
30 #define TEST_RING_ELEM_BURST_ZC 64
31 #define TEST_RING_ELEM_BURST_ZC_COMPRESS_PTR_16 128
32 #define TEST_RING_ELEM_BURST_ZC_COMPRESS_PTR_32 256
33
34 #define TEST_RING_IGNORE_API_TYPE ~0U
35
36 /* This function is placed here as it is required for both
37 * performance and functional tests.
38 */
39 static inline struct rte_ring*
test_ring_create(const char * name,int esize,unsigned int count,int socket_id,unsigned int flags)40 test_ring_create(const char *name, int esize, unsigned int count,
41 int socket_id, unsigned int flags)
42 {
43 /* Legacy queue APIs? */
44 if (esize == -1)
45 return rte_ring_create(name, count, socket_id, flags);
46 else
47 return rte_ring_create_elem(name, esize, count,
48 socket_id, flags);
49 }
50
51 static inline void*
test_ring_inc_ptr(void * obj,int esize,unsigned int n)52 test_ring_inc_ptr(void *obj, int esize, unsigned int n)
53 {
54 size_t sz;
55
56 sz = sizeof(void *);
57 /* Legacy queue APIs? */
58 if (esize != -1)
59 sz = esize;
60
61 return (void *)((uint32_t *)obj + (n * sz / sizeof(uint32_t)));
62 }
63
64 static inline void
test_ring_mem_copy(void * dst,void * const * src,int esize,unsigned int num)65 test_ring_mem_copy(void *dst, void * const *src, int esize, unsigned int num)
66 {
67 size_t sz;
68
69 sz = num * sizeof(void *);
70 if (esize != -1)
71 sz = esize * num;
72
73 memcpy(dst, src, sz);
74 }
75
76 /* Copy to the ring memory */
77 static inline void
test_ring_copy_to(struct rte_ring_zc_data * zcd,void * const * src,int esize,unsigned int num)78 test_ring_copy_to(struct rte_ring_zc_data *zcd, void * const *src, int esize,
79 unsigned int num)
80 {
81 test_ring_mem_copy(zcd->ptr1, src, esize, zcd->n1);
82 if (zcd->n1 != num) {
83 if (esize == -1)
84 src = src + zcd->n1;
85 else
86 src = (void * const *)((const uint32_t *)src +
87 (zcd->n1 * esize / sizeof(uint32_t)));
88 test_ring_mem_copy(zcd->ptr2, src,
89 esize, num - zcd->n1);
90 }
91 }
92
93 /* Copy from the ring memory */
94 static inline void
test_ring_copy_from(struct rte_ring_zc_data * zcd,void * dst,int esize,unsigned int num)95 test_ring_copy_from(struct rte_ring_zc_data *zcd, void *dst, int esize,
96 unsigned int num)
97 {
98 test_ring_mem_copy(dst, zcd->ptr1, esize, zcd->n1);
99
100 if (zcd->n1 != num) {
101 dst = test_ring_inc_ptr(dst, esize, zcd->n1);
102 test_ring_mem_copy(dst, zcd->ptr2, esize, num - zcd->n1);
103 }
104 }
105
106 static inline unsigned int
test_ring_enqueue(struct rte_ring * r,void ** obj,int esize,unsigned int n,unsigned int api_type)107 test_ring_enqueue(struct rte_ring *r, void **obj, int esize, unsigned int n,
108 unsigned int api_type)
109 {
110 unsigned int ret;
111 struct rte_ring_zc_data zcd = {0};
112
113 /* Legacy queue APIs? */
114 if (esize == -1)
115 switch (api_type) {
116 case (TEST_RING_THREAD_DEF | TEST_RING_ELEM_SINGLE):
117 return rte_ring_enqueue(r, *obj);
118 case (TEST_RING_THREAD_SPSC | TEST_RING_ELEM_SINGLE):
119 return rte_ring_sp_enqueue(r, *obj);
120 case (TEST_RING_THREAD_MPMC | TEST_RING_ELEM_SINGLE):
121 return rte_ring_mp_enqueue(r, *obj);
122 case (TEST_RING_THREAD_DEF | TEST_RING_ELEM_BULK):
123 return rte_ring_enqueue_bulk(r, obj, n, NULL);
124 case (TEST_RING_THREAD_SPSC | TEST_RING_ELEM_BULK):
125 return rte_ring_sp_enqueue_bulk(r, obj, n, NULL);
126 case (TEST_RING_THREAD_MPMC | TEST_RING_ELEM_BULK):
127 return rte_ring_mp_enqueue_bulk(r, obj, n, NULL);
128 case (TEST_RING_THREAD_DEF | TEST_RING_ELEM_BURST):
129 return rte_ring_enqueue_burst(r, obj, n, NULL);
130 case (TEST_RING_THREAD_SPSC | TEST_RING_ELEM_BURST):
131 return rte_ring_sp_enqueue_burst(r, obj, n, NULL);
132 case (TEST_RING_THREAD_MPMC | TEST_RING_ELEM_BURST):
133 return rte_ring_mp_enqueue_burst(r, obj, n, NULL);
134 default:
135 printf("Invalid API type\n");
136 return 0;
137 }
138 else
139 switch (api_type) {
140 case (TEST_RING_THREAD_DEF | TEST_RING_ELEM_SINGLE):
141 return rte_ring_enqueue_elem(r, obj, esize);
142 case (TEST_RING_THREAD_SPSC | TEST_RING_ELEM_SINGLE):
143 return rte_ring_sp_enqueue_elem(r, obj, esize);
144 case (TEST_RING_THREAD_MPMC | TEST_RING_ELEM_SINGLE):
145 return rte_ring_mp_enqueue_elem(r, obj, esize);
146 case (TEST_RING_THREAD_DEF | TEST_RING_ELEM_BULK):
147 return rte_ring_enqueue_bulk_elem(r, obj, esize, n,
148 NULL);
149 case (TEST_RING_THREAD_SPSC | TEST_RING_ELEM_BULK):
150 return rte_ring_sp_enqueue_bulk_elem(r, obj, esize, n,
151 NULL);
152 case (TEST_RING_THREAD_MPMC | TEST_RING_ELEM_BULK):
153 return rte_ring_mp_enqueue_bulk_elem(r, obj, esize, n,
154 NULL);
155 case (TEST_RING_THREAD_DEF | TEST_RING_ELEM_BURST):
156 return rte_ring_enqueue_burst_elem(r, obj, esize, n,
157 NULL);
158 case (TEST_RING_THREAD_SPSC | TEST_RING_ELEM_BURST):
159 return rte_ring_sp_enqueue_burst_elem(r, obj, esize, n,
160 NULL);
161 case (TEST_RING_THREAD_MPMC | TEST_RING_ELEM_BURST):
162 return rte_ring_mp_enqueue_burst_elem(r, obj, esize, n,
163 NULL);
164 case (TEST_RING_ELEM_BURST_ZC):
165 ret = rte_ring_enqueue_zc_burst_elem_start(
166 r, esize, n, &zcd, NULL);
167 if (unlikely(ret == 0))
168 return 0;
169 rte_memcpy(zcd.ptr1, (char *)obj, zcd.n1 * esize);
170 if (unlikely(zcd.ptr2 != NULL))
171 rte_memcpy(zcd.ptr2,
172 (char *)obj + zcd.n1 * esize,
173 (ret - zcd.n1) * esize);
174 rte_ring_enqueue_zc_finish(r, ret);
175 return ret;
176 case (TEST_RING_ELEM_BURST_ZC_COMPRESS_PTR_16):
177 /* rings cannot store uint16_t so we use a uint32_t
178 * and half the requested number of elements
179 * and compensate by doubling the returned numbers
180 */
181 ret = rte_ring_enqueue_zc_burst_elem_start(
182 r, sizeof(uint32_t), n / 2, &zcd, NULL);
183 if (unlikely(ret == 0))
184 return 0;
185 rte_ptr_compress_16_shift(
186 0, obj, zcd.ptr1, zcd.n1 * 2, 3);
187 if (unlikely(zcd.ptr2 != NULL))
188 rte_ptr_compress_16_shift(0,
189 obj + (zcd.n1 * 2),
190 zcd.ptr2,
191 (ret - zcd.n1) * 2, 3);
192 rte_ring_enqueue_zc_finish(r, ret);
193 return ret * 2;
194 case (TEST_RING_ELEM_BURST_ZC_COMPRESS_PTR_32):
195 ret = rte_ring_enqueue_zc_burst_elem_start(
196 r, sizeof(uint32_t), n, &zcd, NULL);
197 if (unlikely(ret == 0))
198 return 0;
199 rte_ptr_compress_32_shift(0, obj, zcd.ptr1, zcd.n1, 3);
200 if (unlikely(zcd.ptr2 != NULL))
201 rte_ptr_compress_32_shift(0, obj + zcd.n1,
202 zcd.ptr2, ret - zcd.n1, 3);
203 rte_ring_enqueue_zc_finish(r, ret);
204 return ret;
205 default:
206 printf("Invalid API type\n");
207 return 0;
208 }
209 }
210
211 static inline unsigned int
test_ring_dequeue(struct rte_ring * r,void ** obj,int esize,unsigned int n,unsigned int api_type)212 test_ring_dequeue(struct rte_ring *r, void **obj, int esize, unsigned int n,
213 unsigned int api_type)
214 {
215 unsigned int ret;
216 struct rte_ring_zc_data zcd = {0};
217
218 /* Legacy queue APIs? */
219 if (esize == -1)
220 switch (api_type) {
221 case (TEST_RING_THREAD_DEF | TEST_RING_ELEM_SINGLE):
222 return rte_ring_dequeue(r, obj);
223 case (TEST_RING_THREAD_SPSC | TEST_RING_ELEM_SINGLE):
224 return rte_ring_sc_dequeue(r, obj);
225 case (TEST_RING_THREAD_MPMC | TEST_RING_ELEM_SINGLE):
226 return rte_ring_mc_dequeue(r, obj);
227 case (TEST_RING_THREAD_DEF | TEST_RING_ELEM_BULK):
228 return rte_ring_dequeue_bulk(r, obj, n, NULL);
229 case (TEST_RING_THREAD_SPSC | TEST_RING_ELEM_BULK):
230 return rte_ring_sc_dequeue_bulk(r, obj, n, NULL);
231 case (TEST_RING_THREAD_MPMC | TEST_RING_ELEM_BULK):
232 return rte_ring_mc_dequeue_bulk(r, obj, n, NULL);
233 case (TEST_RING_THREAD_DEF | TEST_RING_ELEM_BURST):
234 return rte_ring_dequeue_burst(r, obj, n, NULL);
235 case (TEST_RING_THREAD_SPSC | TEST_RING_ELEM_BURST):
236 return rte_ring_sc_dequeue_burst(r, obj, n, NULL);
237 case (TEST_RING_THREAD_MPMC | TEST_RING_ELEM_BURST):
238 return rte_ring_mc_dequeue_burst(r, obj, n, NULL);
239 default:
240 printf("Invalid API type\n");
241 return 0;
242 }
243 else
244 switch (api_type) {
245 case (TEST_RING_THREAD_DEF | TEST_RING_ELEM_SINGLE):
246 return rte_ring_dequeue_elem(r, obj, esize);
247 case (TEST_RING_THREAD_SPSC | TEST_RING_ELEM_SINGLE):
248 return rte_ring_sc_dequeue_elem(r, obj, esize);
249 case (TEST_RING_THREAD_MPMC | TEST_RING_ELEM_SINGLE):
250 return rte_ring_mc_dequeue_elem(r, obj, esize);
251 case (TEST_RING_THREAD_DEF | TEST_RING_ELEM_BULK):
252 return rte_ring_dequeue_bulk_elem(r, obj, esize,
253 n, NULL);
254 case (TEST_RING_THREAD_SPSC | TEST_RING_ELEM_BULK):
255 return rte_ring_sc_dequeue_bulk_elem(r, obj, esize,
256 n, NULL);
257 case (TEST_RING_THREAD_MPMC | TEST_RING_ELEM_BULK):
258 return rte_ring_mc_dequeue_bulk_elem(r, obj, esize,
259 n, NULL);
260 case (TEST_RING_THREAD_DEF | TEST_RING_ELEM_BURST):
261 return rte_ring_dequeue_burst_elem(r, obj, esize,
262 n, NULL);
263 case (TEST_RING_THREAD_SPSC | TEST_RING_ELEM_BURST):
264 return rte_ring_sc_dequeue_burst_elem(r, obj, esize,
265 n, NULL);
266 case (TEST_RING_THREAD_MPMC | TEST_RING_ELEM_BURST):
267 return rte_ring_mc_dequeue_burst_elem(r, obj, esize,
268 n, NULL);
269 case (TEST_RING_ELEM_BURST_ZC):
270 ret = rte_ring_dequeue_zc_burst_elem_start(
271 r, esize, n, &zcd, NULL);
272 if (unlikely(ret == 0))
273 return 0;
274 rte_memcpy((char *)obj, zcd.ptr1, zcd.n1 * esize);
275 if (unlikely(zcd.ptr2 != NULL))
276 rte_memcpy((char *)obj + zcd.n1 * esize,
277 zcd.ptr2,
278 (ret - zcd.n1) * esize);
279 rte_ring_dequeue_zc_finish(r, ret);
280 return ret;
281 case (TEST_RING_ELEM_BURST_ZC_COMPRESS_PTR_16):
282 /* rings cannot store uint16_t so we use a uint32_t
283 * and half the requested number of elements
284 * and compensate by doubling the returned numbers
285 */
286 ret = rte_ring_dequeue_zc_burst_elem_start(
287 r, sizeof(uint32_t), n / 2, &zcd, NULL);
288 if (unlikely(ret == 0))
289 return 0;
290 rte_ptr_decompress_16_shift(
291 0, zcd.ptr1, obj, zcd.n1 * 2, 3);
292 if (unlikely(zcd.ptr2 != NULL))
293 rte_ptr_decompress_16_shift(0, zcd.ptr2,
294 obj + zcd.n1,
295 (ret - zcd.n1) * 2,
296 3);
297 rte_ring_dequeue_zc_finish(r, ret);
298 return ret * 2;
299 case (TEST_RING_ELEM_BURST_ZC_COMPRESS_PTR_32):
300 ret = rte_ring_dequeue_zc_burst_elem_start(
301 r, sizeof(uint32_t), n, &zcd, NULL);
302 if (unlikely(ret == 0))
303 return 0;
304 rte_ptr_decompress_32_shift(0, zcd.ptr1, obj, zcd.n1, 3);
305 if (unlikely(zcd.ptr2 != NULL))
306 rte_ptr_decompress_32_shift(0, zcd.ptr2,
307 obj + zcd.n1, ret - zcd.n1, 3);
308 rte_ring_dequeue_zc_finish(r, ret);
309 return ret;
310 default:
311 printf("Invalid API type\n");
312 return 0;
313 }
314 }
315
316 /* This function is placed here as it is required for both
317 * performance and functional tests.
318 */
319 static inline void *
test_ring_calloc(unsigned int rsize,int esize)320 test_ring_calloc(unsigned int rsize, int esize)
321 {
322 unsigned int sz;
323 void *p;
324
325 /* Legacy queue APIs? */
326 if (esize == -1)
327 sz = sizeof(void *);
328 else
329 sz = esize;
330
331 p = rte_zmalloc(NULL, rsize * sz, RTE_CACHE_LINE_SIZE);
332 if (p == NULL)
333 printf("Failed to allocate memory\n");
334
335 return p;
336 }
337