xref: /spdk/lib/env_dpdk/env.c (revision 83ba9086796471697a4975a58f60e2392bccd08c) 
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright (C) 2016 Intel Corporation.
3  *   Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.
4  *   All rights reserved.
5  */
6 
7 #include "spdk/stdinc.h"
8 #include "spdk/util.h"
9 #include "spdk/env_dpdk.h"
10 #include "spdk/log.h"
11 #include "spdk/assert.h"
12 
13 #include "env_internal.h"
14 
15 #include <rte_config.h>
16 #include <rte_cycles.h>
17 #include <rte_malloc.h>
18 #include <rte_mempool.h>
19 #include <rte_memzone.h>
20 #include <rte_version.h>
21 #include <rte_eal.h>
22 
23 static __thread bool g_is_thread_unaffinitized;
24 static bool g_enforce_numa;
25 
26 SPDK_STATIC_ASSERT(SOCKET_ID_ANY == SPDK_ENV_NUMA_ID_ANY, "SOCKET_ID_ANY mismatch");
27 
28 void *
29 spdk_malloc(size_t size, size_t align, uint64_t *unused, int numa_id, uint32_t flags)
30 {
31 	void *buf;
32 
33 	if (flags == 0 || unused != NULL) {
34 		return NULL;
35 	}
36 
37 	align = spdk_max(align, RTE_CACHE_LINE_SIZE);
38 	buf = rte_malloc_socket(NULL, size, align, numa_id);
39 	if (buf == NULL && !g_enforce_numa && numa_id != SOCKET_ID_ANY) {
40 		buf = rte_malloc_socket(NULL, size, align, SOCKET_ID_ANY);
41 	}
42 	return buf;
43 }
44 
45 void *
46 spdk_zmalloc(size_t size, size_t align, uint64_t *unused, int numa_id, uint32_t flags)
47 {
48 	void *buf;
49 
50 	if (flags == 0 || unused != NULL) {
51 		return NULL;
52 	}
53 
54 	align = spdk_max(align, RTE_CACHE_LINE_SIZE);
55 	buf = rte_zmalloc_socket(NULL, size, align, numa_id);
56 	if (buf == NULL && !g_enforce_numa && numa_id != SOCKET_ID_ANY) {
57 		buf = rte_zmalloc_socket(NULL, size, align, SOCKET_ID_ANY);
58 	}
59 	return buf;
60 }
61 
62 void *
63 spdk_realloc(void *buf, size_t size, size_t align)
64 {
65 	align = spdk_max(align, RTE_CACHE_LINE_SIZE);
66 	return rte_realloc(buf, size, align);
67 }
68 
69 void
70 spdk_free(void *buf)
71 {
72 	rte_free(buf);
73 }
74 
75 void *
76 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *unused, int numa_id)
77 {
78 	return spdk_malloc(size, align, unused, numa_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE));
79 }
80 
81 void *
82 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *unused, int numa_id)
83 {
84 	return spdk_zmalloc(size, align, unused, numa_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE));
85 }
86 
87 void *
88 spdk_dma_malloc(size_t size, size_t align, uint64_t *unused)
89 {
90 	return spdk_dma_malloc_socket(size, align, unused, SPDK_ENV_NUMA_ID_ANY);
91 }
92 
93 void *
94 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *unused)
95 {
96 	return spdk_dma_zmalloc_socket(size, align, unused, SPDK_ENV_NUMA_ID_ANY);
97 }
98 
99 void *
100 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *unused)
101 {
102 	if (unused != NULL) {
103 		return NULL;
104 	}
105 	align = spdk_max(align, RTE_CACHE_LINE_SIZE);
106 	return rte_realloc(buf, size, align);
107 }
108 
109 void
110 spdk_dma_free(void *buf)
111 {
112 	spdk_free(buf);
113 }
114 
115 void *
116 spdk_memzone_reserve_aligned(const char *name, size_t len, int numa_id,
117 			     unsigned flags, unsigned align)
118 {
119 	const struct rte_memzone *mz;
120 	unsigned dpdk_flags = 0;
121 
122 	if ((flags & SPDK_MEMZONE_NO_IOVA_CONTIG) == 0) {
123 		dpdk_flags |= RTE_MEMZONE_IOVA_CONTIG;
124 	}
125 
126 	if (numa_id == SPDK_ENV_NUMA_ID_ANY) {
127 		numa_id = SOCKET_ID_ANY;
128 	}
129 
130 	mz = rte_memzone_reserve_aligned(name, len, numa_id, dpdk_flags, align);
131 	if (mz == NULL && !g_enforce_numa && numa_id != SOCKET_ID_ANY) {
132 		mz = rte_memzone_reserve_aligned(name, len, SOCKET_ID_ANY, dpdk_flags, align);
133 	}
134 
135 	if (mz != NULL) {
136 		memset(mz->addr, 0, len);
137 		return mz->addr;
138 	} else {
139 		return NULL;
140 	}
141 }
142 
143 void *
144 spdk_memzone_reserve(const char *name, size_t len, int numa_id, unsigned flags)
145 {
146 	return spdk_memzone_reserve_aligned(name, len, numa_id, flags,
147 					    RTE_CACHE_LINE_SIZE);
148 }
149 
150 void *
151 spdk_memzone_lookup(const char *name)
152 {
153 	const struct rte_memzone *mz = rte_memzone_lookup(name);
154 
155 	if (mz != NULL) {
156 		return mz->addr;
157 	} else {
158 		return NULL;
159 	}
160 }
161 
162 int
163 spdk_memzone_free(const char *name)
164 {
165 	const struct rte_memzone *mz = rte_memzone_lookup(name);
166 
167 	if (mz != NULL) {
168 		return rte_memzone_free(mz);
169 	}
170 
171 	return -1;
172 }
173 
174 void
175 spdk_memzone_dump(FILE *f)
176 {
177 	rte_memzone_dump(f);
178 }
179 
180 struct spdk_mempool *
181 spdk_mempool_create_ctor(const char *name, size_t count,
182 			 size_t ele_size, size_t cache_size, int numa_id,
183 			 spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg)
184 {
185 	struct rte_mempool *mp;
186 	size_t tmp;
187 
188 	if (numa_id == SPDK_ENV_NUMA_ID_ANY) {
189 		numa_id = SOCKET_ID_ANY;
190 	}
191 
192 	/* No more than half of all elements can be in cache */
193 	tmp = (count / 2) / rte_lcore_count();
194 	if (cache_size > tmp) {
195 		cache_size = tmp;
196 	}
197 
198 	if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) {
199 		cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE;
200 	}
201 
202 	mp = rte_mempool_create(name, count, ele_size, cache_size,
203 				0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg,
204 				numa_id, 0);
205 	if (mp == NULL && !g_enforce_numa && numa_id != SOCKET_ID_ANY) {
206 		mp = rte_mempool_create(name, count, ele_size, cache_size,
207 					0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg,
208 					SOCKET_ID_ANY, 0);
209 	}
210 
211 	return (struct spdk_mempool *)mp;
212 }
213 
214 
215 struct spdk_mempool *
216 spdk_mempool_create(const char *name, size_t count,
217 		    size_t ele_size, size_t cache_size, int numa_id)
218 {
219 	return spdk_mempool_create_ctor(name, count, ele_size, cache_size, numa_id,
220 					NULL, NULL);
221 }
222 
223 char *
224 spdk_mempool_get_name(struct spdk_mempool *mp)
225 {
226 	return ((struct rte_mempool *)mp)->name;
227 }
228 
229 void
230 spdk_mempool_free(struct spdk_mempool *mp)
231 {
232 	rte_mempool_free((struct rte_mempool *)mp);
233 }
234 
235 void *
236 spdk_mempool_get(struct spdk_mempool *mp)
237 {
238 	void *ele = NULL;
239 	int rc;
240 
241 	rc = rte_mempool_get((struct rte_mempool *)mp, &ele);
242 	if (rc != 0) {
243 		return NULL;
244 	}
245 	return ele;
246 }
247 
248 int
249 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count)
250 {
251 	return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count);
252 }
253 
254 void
255 spdk_mempool_put(struct spdk_mempool *mp, void *ele)
256 {
257 	rte_mempool_put((struct rte_mempool *)mp, ele);
258 }
259 
260 void
261 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count)
262 {
263 	rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count);
264 }
265 
266 size_t
267 spdk_mempool_count(const struct spdk_mempool *pool)
268 {
269 	return rte_mempool_avail_count((struct rte_mempool *)pool);
270 }
271 
272 uint32_t
273 spdk_mempool_obj_iter(struct spdk_mempool *mp, spdk_mempool_obj_cb_t obj_cb,
274 		      void *obj_cb_arg)
275 {
276 	return rte_mempool_obj_iter((struct rte_mempool *)mp, (rte_mempool_obj_cb_t *)obj_cb,
277 				    obj_cb_arg);
278 }
279 
280 struct env_mempool_mem_iter_ctx {
281 	spdk_mempool_mem_cb_t *user_cb;
282 	void *user_arg;
283 };
284 
285 static void
286 mempool_mem_iter_remap(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr,
287 		       unsigned mem_idx)
288 {
289 	struct env_mempool_mem_iter_ctx *ctx = opaque;
290 
291 	ctx->user_cb((struct spdk_mempool *)mp, ctx->user_arg, memhdr->addr, memhdr->iova, memhdr->len,
292 		     mem_idx);
293 }
294 
295 uint32_t
296 spdk_mempool_mem_iter(struct spdk_mempool *mp, spdk_mempool_mem_cb_t mem_cb,
297 		      void *mem_cb_arg)
298 {
299 	struct env_mempool_mem_iter_ctx ctx = {
300 		.user_cb = mem_cb,
301 		.user_arg = mem_cb_arg
302 	};
303 
304 	return rte_mempool_mem_iter((struct rte_mempool *)mp, mempool_mem_iter_remap, &ctx);
305 }
306 
307 struct spdk_mempool *
308 spdk_mempool_lookup(const char *name)
309 {
310 	return (struct spdk_mempool *)rte_mempool_lookup(name);
311 }
312 
313 bool
314 spdk_process_is_primary(void)
315 {
316 	return (rte_eal_process_type() == RTE_PROC_PRIMARY);
317 }
318 
319 uint64_t
320 spdk_get_ticks(void)
321 {
322 	return rte_get_timer_cycles();
323 }
324 
325 uint64_t
326 spdk_get_ticks_hz(void)
327 {
328 	return rte_get_timer_hz();
329 }
330 
331 void
332 spdk_delay_us(unsigned int us)
333 {
334 	rte_delay_us(us);
335 }
336 
337 void
338 spdk_pause(void)
339 {
340 	rte_pause();
341 }
342 
343 void
344 spdk_unaffinitize_thread(void)
345 {
346 	rte_cpuset_t new_cpuset;
347 	long num_cores, i;
348 
349 	if (g_is_thread_unaffinitized) {
350 		return;
351 	}
352 
353 	CPU_ZERO(&new_cpuset);
354 
355 	num_cores = sysconf(_SC_NPROCESSORS_CONF);
356 
357 	/* Create a mask containing all CPUs */
358 	for (i = 0; i < num_cores; i++) {
359 		CPU_SET(i, &new_cpuset);
360 	}
361 
362 	rte_thread_set_affinity(&new_cpuset);
363 	g_is_thread_unaffinitized = true;
364 }
365 
366 void *
367 spdk_call_unaffinitized(void *cb(void *arg), void *arg)
368 {
369 	rte_cpuset_t orig_cpuset;
370 	void *ret;
371 
372 	if (cb == NULL) {
373 		return NULL;
374 	}
375 
376 	if (g_is_thread_unaffinitized) {
377 		ret = cb(arg);
378 	} else {
379 		rte_thread_get_affinity(&orig_cpuset);
380 		spdk_unaffinitize_thread();
381 
382 		ret = cb(arg);
383 
384 		rte_thread_set_affinity(&orig_cpuset);
385 		g_is_thread_unaffinitized = false;
386 	}
387 
388 	return ret;
389 }
390 
391 struct spdk_ring *
392 spdk_ring_create(enum spdk_ring_type type, size_t count, int numa_id)
393 {
394 	char ring_name[64];
395 	static uint32_t ring_num = 0;
396 	unsigned flags = RING_F_EXACT_SZ;
397 	struct rte_ring *ring;
398 
399 	switch (type) {
400 	case SPDK_RING_TYPE_SP_SC:
401 		flags |= RING_F_SP_ENQ | RING_F_SC_DEQ;
402 		break;
403 	case SPDK_RING_TYPE_MP_SC:
404 		flags |= RING_F_SC_DEQ;
405 		break;
406 	case SPDK_RING_TYPE_MP_MC:
407 		flags |= 0;
408 		break;
409 	default:
410 		return NULL;
411 	}
412 
413 	snprintf(ring_name, sizeof(ring_name), "ring_%u_%d",
414 		 __atomic_fetch_add(&ring_num, 1, __ATOMIC_RELAXED), getpid());
415 
416 	ring = rte_ring_create(ring_name, count, numa_id, flags);
417 	if (ring == NULL && !g_enforce_numa && numa_id != SOCKET_ID_ANY) {
418 		ring = rte_ring_create(ring_name, count, SOCKET_ID_ANY, flags);
419 	}
420 	return (struct spdk_ring *)ring;
421 }
422 
423 void
424 spdk_ring_free(struct spdk_ring *ring)
425 {
426 	rte_ring_free((struct rte_ring *)ring);
427 }
428 
429 size_t
430 spdk_ring_count(struct spdk_ring *ring)
431 {
432 	return rte_ring_count((struct rte_ring *)ring);
433 }
434 
435 size_t
436 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count,
437 		  size_t *free_space)
438 {
439 	return rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count,
440 				     (unsigned int *)free_space);
441 }
442 
443 size_t
444 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count)
445 {
446 	return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count, NULL);
447 }
448 
449 void
450 spdk_env_dpdk_dump_mem_stats(FILE *file)
451 {
452 	fprintf(file, "DPDK memory size %" PRIu64 "\n", rte_eal_get_physmem_size());
453 	fprintf(file, "DPDK memory layout\n");
454 	rte_dump_physmem_layout(file);
455 	fprintf(file, "DPDK memzones.\n");
456 	rte_memzone_dump(file);
457 	fprintf(file, "DPDK mempools.\n");
458 	rte_mempool_list_dump(file);
459 	fprintf(file, "DPDK malloc stats.\n");
460 	rte_malloc_dump_stats(file, NULL);
461 	fprintf(file, "DPDK malloc heaps.\n");
462 	rte_malloc_dump_heaps(file);
463 }
464 
465 int
466 spdk_get_tid(void)
467 {
468 	return rte_sys_gettid();
469 }
470 
471 void
472 mem_enforce_numa(void)
473 {
474 	g_enforce_numa = true;
475 }
476