xref: /spdk/lib/env_dpdk/env.c (revision d987d777d6b8ce05f11cb1d90f1241bfecfc9af4) 
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright (C) 2016 Intel Corporation.
3  *   Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.
4  *   All rights reserved.
5  */
6 
7 #include "spdk/stdinc.h"
8 #include "spdk/util.h"
9 #include "spdk/env_dpdk.h"
10 #include "spdk/log.h"
11 #include "spdk/assert.h"
12 
13 #include "env_internal.h"
14 
15 #include <rte_config.h>
16 #include <rte_cycles.h>
17 #include <rte_malloc.h>
18 #include <rte_mempool.h>
19 #include <rte_memzone.h>
20 #include <rte_version.h>
21 #include <rte_eal.h>
22 
23 static __thread bool g_is_thread_unaffinitized;
24 
25 SPDK_STATIC_ASSERT(SOCKET_ID_ANY == SPDK_ENV_NUMA_ID_ANY, "SOCKET_ID_ANY mismatch");
26 
27 void *
28 spdk_malloc(size_t size, size_t align, uint64_t *unused, int numa_id, uint32_t flags)
29 {
30 	void *buf;
31 
32 	if (flags == 0 || unused != NULL) {
33 		return NULL;
34 	}
35 
36 	align = spdk_max(align, RTE_CACHE_LINE_SIZE);
37 	buf = rte_malloc_socket(NULL, size, align, numa_id);
38 	if (buf == NULL && numa_id != SOCKET_ID_ANY) {
39 		buf = rte_malloc_socket(NULL, size, align, SOCKET_ID_ANY);
40 	}
41 	return buf;
42 }
43 
44 void *
45 spdk_zmalloc(size_t size, size_t align, uint64_t *unused, int numa_id, uint32_t flags)
46 {
47 	void *buf;
48 
49 	if (flags == 0 || unused != NULL) {
50 		return NULL;
51 	}
52 
53 	align = spdk_max(align, RTE_CACHE_LINE_SIZE);
54 	buf = rte_zmalloc_socket(NULL, size, align, numa_id);
55 	if (buf == NULL && numa_id != SOCKET_ID_ANY) {
56 		buf = rte_zmalloc_socket(NULL, size, align, SOCKET_ID_ANY);
57 	}
58 	return buf;
59 }
60 
61 void *
62 spdk_realloc(void *buf, size_t size, size_t align)
63 {
64 	align = spdk_max(align, RTE_CACHE_LINE_SIZE);
65 	return rte_realloc(buf, size, align);
66 }
67 
68 void
69 spdk_free(void *buf)
70 {
71 	rte_free(buf);
72 }
73 
74 void *
75 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *unused, int numa_id)
76 {
77 	return spdk_malloc(size, align, unused, numa_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE));
78 }
79 
80 void *
81 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *unused, int numa_id)
82 {
83 	return spdk_zmalloc(size, align, unused, numa_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE));
84 }
85 
86 void *
87 spdk_dma_malloc(size_t size, size_t align, uint64_t *unused)
88 {
89 	return spdk_dma_malloc_socket(size, align, unused, SPDK_ENV_NUMA_ID_ANY);
90 }
91 
92 void *
93 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *unused)
94 {
95 	return spdk_dma_zmalloc_socket(size, align, unused, SPDK_ENV_NUMA_ID_ANY);
96 }
97 
98 void *
99 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *unused)
100 {
101 	if (unused != NULL) {
102 		return NULL;
103 	}
104 	align = spdk_max(align, RTE_CACHE_LINE_SIZE);
105 	return rte_realloc(buf, size, align);
106 }
107 
108 void
109 spdk_dma_free(void *buf)
110 {
111 	spdk_free(buf);
112 }
113 
114 void *
115 spdk_memzone_reserve_aligned(const char *name, size_t len, int numa_id,
116 			     unsigned flags, unsigned align)
117 {
118 	const struct rte_memzone *mz;
119 	unsigned dpdk_flags = 0;
120 
121 	if ((flags & SPDK_MEMZONE_NO_IOVA_CONTIG) == 0) {
122 		dpdk_flags |= RTE_MEMZONE_IOVA_CONTIG;
123 	}
124 
125 	if (numa_id == SPDK_ENV_NUMA_ID_ANY) {
126 		numa_id = SOCKET_ID_ANY;
127 	}
128 
129 	mz = rte_memzone_reserve_aligned(name, len, numa_id, dpdk_flags, align);
130 	if (mz == NULL && numa_id != SOCKET_ID_ANY) {
131 		mz = rte_memzone_reserve_aligned(name, len, SOCKET_ID_ANY, dpdk_flags, align);
132 	}
133 
134 	if (mz != NULL) {
135 		memset(mz->addr, 0, len);
136 		return mz->addr;
137 	} else {
138 		return NULL;
139 	}
140 }
141 
142 void *
143 spdk_memzone_reserve(const char *name, size_t len, int numa_id, unsigned flags)
144 {
145 	return spdk_memzone_reserve_aligned(name, len, numa_id, flags,
146 					    RTE_CACHE_LINE_SIZE);
147 }
148 
149 void *
150 spdk_memzone_lookup(const char *name)
151 {
152 	const struct rte_memzone *mz = rte_memzone_lookup(name);
153 
154 	if (mz != NULL) {
155 		return mz->addr;
156 	} else {
157 		return NULL;
158 	}
159 }
160 
161 int
162 spdk_memzone_free(const char *name)
163 {
164 	const struct rte_memzone *mz = rte_memzone_lookup(name);
165 
166 	if (mz != NULL) {
167 		return rte_memzone_free(mz);
168 	}
169 
170 	return -1;
171 }
172 
173 void
174 spdk_memzone_dump(FILE *f)
175 {
176 	rte_memzone_dump(f);
177 }
178 
179 struct spdk_mempool *
180 spdk_mempool_create_ctor(const char *name, size_t count,
181 			 size_t ele_size, size_t cache_size, int numa_id,
182 			 spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg)
183 {
184 	struct rte_mempool *mp;
185 	size_t tmp;
186 
187 	if (numa_id == SPDK_ENV_NUMA_ID_ANY) {
188 		numa_id = SOCKET_ID_ANY;
189 	}
190 
191 	/* No more than half of all elements can be in cache */
192 	tmp = (count / 2) / rte_lcore_count();
193 	if (cache_size > tmp) {
194 		cache_size = tmp;
195 	}
196 
197 	if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) {
198 		cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE;
199 	}
200 
201 	mp = rte_mempool_create(name, count, ele_size, cache_size,
202 				0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg,
203 				numa_id, 0);
204 	if (mp == NULL && numa_id != SOCKET_ID_ANY) {
205 		mp = rte_mempool_create(name, count, ele_size, cache_size,
206 					0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg,
207 					SOCKET_ID_ANY, 0);
208 	}
209 
210 	return (struct spdk_mempool *)mp;
211 }
212 
213 
214 struct spdk_mempool *
215 spdk_mempool_create(const char *name, size_t count,
216 		    size_t ele_size, size_t cache_size, int numa_id)
217 {
218 	return spdk_mempool_create_ctor(name, count, ele_size, cache_size, numa_id,
219 					NULL, NULL);
220 }
221 
222 char *
223 spdk_mempool_get_name(struct spdk_mempool *mp)
224 {
225 	return ((struct rte_mempool *)mp)->name;
226 }
227 
228 void
229 spdk_mempool_free(struct spdk_mempool *mp)
230 {
231 	rte_mempool_free((struct rte_mempool *)mp);
232 }
233 
234 void *
235 spdk_mempool_get(struct spdk_mempool *mp)
236 {
237 	void *ele = NULL;
238 	int rc;
239 
240 	rc = rte_mempool_get((struct rte_mempool *)mp, &ele);
241 	if (rc != 0) {
242 		return NULL;
243 	}
244 	return ele;
245 }
246 
247 int
248 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count)
249 {
250 	return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count);
251 }
252 
253 void
254 spdk_mempool_put(struct spdk_mempool *mp, void *ele)
255 {
256 	rte_mempool_put((struct rte_mempool *)mp, ele);
257 }
258 
259 void
260 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count)
261 {
262 	rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count);
263 }
264 
265 size_t
266 spdk_mempool_count(const struct spdk_mempool *pool)
267 {
268 	return rte_mempool_avail_count((struct rte_mempool *)pool);
269 }
270 
271 uint32_t
272 spdk_mempool_obj_iter(struct spdk_mempool *mp, spdk_mempool_obj_cb_t obj_cb,
273 		      void *obj_cb_arg)
274 {
275 	return rte_mempool_obj_iter((struct rte_mempool *)mp, (rte_mempool_obj_cb_t *)obj_cb,
276 				    obj_cb_arg);
277 }
278 
279 struct env_mempool_mem_iter_ctx {
280 	spdk_mempool_mem_cb_t *user_cb;
281 	void *user_arg;
282 };
283 
284 static void
285 mempool_mem_iter_remap(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr,
286 		       unsigned mem_idx)
287 {
288 	struct env_mempool_mem_iter_ctx *ctx = opaque;
289 
290 	ctx->user_cb((struct spdk_mempool *)mp, ctx->user_arg, memhdr->addr, memhdr->iova, memhdr->len,
291 		     mem_idx);
292 }
293 
294 uint32_t
295 spdk_mempool_mem_iter(struct spdk_mempool *mp, spdk_mempool_mem_cb_t mem_cb,
296 		      void *mem_cb_arg)
297 {
298 	struct env_mempool_mem_iter_ctx ctx = {
299 		.user_cb = mem_cb,
300 		.user_arg = mem_cb_arg
301 	};
302 
303 	return rte_mempool_mem_iter((struct rte_mempool *)mp, mempool_mem_iter_remap, &ctx);
304 }
305 
306 struct spdk_mempool *
307 spdk_mempool_lookup(const char *name)
308 {
309 	return (struct spdk_mempool *)rte_mempool_lookup(name);
310 }
311 
312 bool
313 spdk_process_is_primary(void)
314 {
315 	return (rte_eal_process_type() == RTE_PROC_PRIMARY);
316 }
317 
318 uint64_t
319 spdk_get_ticks(void)
320 {
321 	return rte_get_timer_cycles();
322 }
323 
324 uint64_t
325 spdk_get_ticks_hz(void)
326 {
327 	return rte_get_timer_hz();
328 }
329 
330 void
331 spdk_delay_us(unsigned int us)
332 {
333 	rte_delay_us(us);
334 }
335 
336 void
337 spdk_pause(void)
338 {
339 	rte_pause();
340 }
341 
342 void
343 spdk_unaffinitize_thread(void)
344 {
345 	rte_cpuset_t new_cpuset;
346 	long num_cores, i;
347 
348 	if (g_is_thread_unaffinitized) {
349 		return;
350 	}
351 
352 	CPU_ZERO(&new_cpuset);
353 
354 	num_cores = sysconf(_SC_NPROCESSORS_CONF);
355 
356 	/* Create a mask containing all CPUs */
357 	for (i = 0; i < num_cores; i++) {
358 		CPU_SET(i, &new_cpuset);
359 	}
360 
361 	rte_thread_set_affinity(&new_cpuset);
362 	g_is_thread_unaffinitized = true;
363 }
364 
365 void *
366 spdk_call_unaffinitized(void *cb(void *arg), void *arg)
367 {
368 	rte_cpuset_t orig_cpuset;
369 	void *ret;
370 
371 	if (cb == NULL) {
372 		return NULL;
373 	}
374 
375 	if (g_is_thread_unaffinitized) {
376 		ret = cb(arg);
377 	} else {
378 		rte_thread_get_affinity(&orig_cpuset);
379 		spdk_unaffinitize_thread();
380 
381 		ret = cb(arg);
382 
383 		rte_thread_set_affinity(&orig_cpuset);
384 		g_is_thread_unaffinitized = false;
385 	}
386 
387 	return ret;
388 }
389 
390 struct spdk_ring *
391 spdk_ring_create(enum spdk_ring_type type, size_t count, int numa_id)
392 {
393 	char ring_name[64];
394 	static uint32_t ring_num = 0;
395 	unsigned flags = RING_F_EXACT_SZ;
396 	struct rte_ring *ring;
397 
398 	switch (type) {
399 	case SPDK_RING_TYPE_SP_SC:
400 		flags |= RING_F_SP_ENQ | RING_F_SC_DEQ;
401 		break;
402 	case SPDK_RING_TYPE_MP_SC:
403 		flags |= RING_F_SC_DEQ;
404 		break;
405 	case SPDK_RING_TYPE_MP_MC:
406 		flags |= 0;
407 		break;
408 	default:
409 		return NULL;
410 	}
411 
412 	snprintf(ring_name, sizeof(ring_name), "ring_%u_%d",
413 		 __atomic_fetch_add(&ring_num, 1, __ATOMIC_RELAXED), getpid());
414 
415 	ring = rte_ring_create(ring_name, count, numa_id, flags);
416 	if (ring == NULL && numa_id != SOCKET_ID_ANY) {
417 		ring = rte_ring_create(ring_name, count, SOCKET_ID_ANY, flags);
418 	}
419 	return (struct spdk_ring *)ring;
420 }
421 
422 void
423 spdk_ring_free(struct spdk_ring *ring)
424 {
425 	rte_ring_free((struct rte_ring *)ring);
426 }
427 
428 size_t
429 spdk_ring_count(struct spdk_ring *ring)
430 {
431 	return rte_ring_count((struct rte_ring *)ring);
432 }
433 
434 size_t
435 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count,
436 		  size_t *free_space)
437 {
438 	return rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count,
439 				     (unsigned int *)free_space);
440 }
441 
442 size_t
443 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count)
444 {
445 	return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count, NULL);
446 }
447 
448 void
449 spdk_env_dpdk_dump_mem_stats(FILE *file)
450 {
451 	fprintf(file, "DPDK memory size %" PRIu64 "\n", rte_eal_get_physmem_size());
452 	fprintf(file, "DPDK memory layout\n");
453 	rte_dump_physmem_layout(file);
454 	fprintf(file, "DPDK memzones.\n");
455 	rte_memzone_dump(file);
456 	fprintf(file, "DPDK mempools.\n");
457 	rte_mempool_list_dump(file);
458 	fprintf(file, "DPDK malloc stats.\n");
459 	rte_malloc_dump_stats(file, NULL);
460 	fprintf(file, "DPDK malloc heaps.\n");
461 	rte_malloc_dump_heaps(file);
462 }
463 
464 int
465 spdk_get_tid(void)
466 {
467 	return rte_sys_gettid();
468 }
469