1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
3 */
4
5 #include <errno.h>
6 #include <stdio.h>
7 #include <stdlib.h>
8 #include <pthread.h>
9 #include <sched.h>
10 #include <assert.h>
11 #include <string.h>
12
13 #include <eal_trace_internal.h>
14 #include <rte_errno.h>
15 #include <rte_lcore.h>
16 #include <rte_log.h>
17 #include <rte_memory.h>
18 #include <rte_trace_point.h>
19
20 #include "eal_internal_cfg.h"
21 #include "eal_private.h"
22 #include "eal_thread.h"
23 #include "eal_trace.h"
24
25 RTE_DEFINE_PER_LCORE(unsigned int, _lcore_id) = LCORE_ID_ANY;
26 RTE_DEFINE_PER_LCORE(int, _thread_id) = -1;
27 static RTE_DEFINE_PER_LCORE(unsigned int, _socket_id) =
28 (unsigned int)SOCKET_ID_ANY;
29 static RTE_DEFINE_PER_LCORE(rte_cpuset_t, _cpuset);
30
rte_socket_id(void)31 unsigned rte_socket_id(void)
32 {
33 return RTE_PER_LCORE(_socket_id);
34 }
35
36 static int
eal_cpuset_socket_id(rte_cpuset_t * cpusetp)37 eal_cpuset_socket_id(rte_cpuset_t *cpusetp)
38 {
39 unsigned cpu = 0;
40 int socket_id = SOCKET_ID_ANY;
41 int sid;
42
43 if (cpusetp == NULL)
44 return SOCKET_ID_ANY;
45
46 do {
47 if (!CPU_ISSET(cpu, cpusetp))
48 continue;
49
50 if (socket_id == SOCKET_ID_ANY)
51 socket_id = eal_cpu_socket_id(cpu);
52
53 sid = eal_cpu_socket_id(cpu);
54 if (socket_id != sid) {
55 socket_id = SOCKET_ID_ANY;
56 break;
57 }
58
59 } while (++cpu < CPU_SETSIZE);
60
61 return socket_id;
62 }
63
64 static void
thread_update_affinity(rte_cpuset_t * cpusetp)65 thread_update_affinity(rte_cpuset_t *cpusetp)
66 {
67 unsigned int lcore_id = rte_lcore_id();
68
69 /* store socket_id in TLS for quick access */
70 RTE_PER_LCORE(_socket_id) =
71 eal_cpuset_socket_id(cpusetp);
72
73 /* store cpuset in TLS for quick access */
74 memmove(&RTE_PER_LCORE(_cpuset), cpusetp,
75 sizeof(rte_cpuset_t));
76
77 if (lcore_id != (unsigned)LCORE_ID_ANY) {
78 /* EAL thread will update lcore_config */
79 lcore_config[lcore_id].socket_id = RTE_PER_LCORE(_socket_id);
80 memmove(&lcore_config[lcore_id].cpuset, cpusetp,
81 sizeof(rte_cpuset_t));
82 }
83 }
84
85 int
rte_thread_set_affinity(rte_cpuset_t * cpusetp)86 rte_thread_set_affinity(rte_cpuset_t *cpusetp)
87 {
88 if (rte_thread_set_affinity_by_id(rte_thread_self(), cpusetp) != 0) {
89 EAL_LOG(ERR, "rte_thread_set_affinity_by_id failed");
90 return -1;
91 }
92
93 thread_update_affinity(cpusetp);
94 return 0;
95 }
96
97 void
rte_thread_get_affinity(rte_cpuset_t * cpusetp)98 rte_thread_get_affinity(rte_cpuset_t *cpusetp)
99 {
100 assert(cpusetp);
101 memmove(cpusetp, &RTE_PER_LCORE(_cpuset),
102 sizeof(rte_cpuset_t));
103 }
104
105 int
eal_thread_dump_affinity(rte_cpuset_t * cpuset,char * str,unsigned int size)106 eal_thread_dump_affinity(rte_cpuset_t *cpuset, char *str, unsigned int size)
107 {
108 unsigned cpu;
109 int ret;
110 unsigned int out = 0;
111
112 for (cpu = 0; cpu < CPU_SETSIZE; cpu++) {
113 if (!CPU_ISSET(cpu, cpuset))
114 continue;
115
116 ret = snprintf(str + out,
117 size - out, "%u,", cpu);
118 if (ret < 0 || (unsigned)ret >= size - out) {
119 /* string will be truncated */
120 ret = -1;
121 goto exit;
122 }
123
124 out += ret;
125 }
126
127 ret = 0;
128 exit:
129 /* remove the last separator */
130 if (out > 0)
131 str[out - 1] = '\0';
132
133 return ret;
134 }
135
136 int
eal_thread_dump_current_affinity(char * str,unsigned int size)137 eal_thread_dump_current_affinity(char *str, unsigned int size)
138 {
139 rte_cpuset_t cpuset;
140
141 rte_thread_get_affinity(&cpuset);
142 return eal_thread_dump_affinity(&cpuset, str, size);
143 }
144
145 void
__rte_thread_init(unsigned int lcore_id,rte_cpuset_t * cpuset)146 __rte_thread_init(unsigned int lcore_id, rte_cpuset_t *cpuset)
147 {
148 /* set the lcore ID in per-lcore memory area */
149 RTE_PER_LCORE(_lcore_id) = lcore_id;
150
151 /* acquire system unique id */
152 rte_gettid();
153
154 thread_update_affinity(cpuset);
155
156 __rte_trace_mem_per_thread_alloc();
157 }
158
159 void
__rte_thread_uninit(void)160 __rte_thread_uninit(void)
161 {
162 trace_mem_per_thread_free();
163
164 RTE_PER_LCORE(_lcore_id) = LCORE_ID_ANY;
165 }
166
167 /* main loop of threads */
168 __rte_noreturn uint32_t
eal_thread_loop(void * arg)169 eal_thread_loop(void *arg)
170 {
171 unsigned int lcore_id = (uintptr_t)arg;
172 char cpuset[RTE_CPU_AFFINITY_STR_LEN];
173 int ret;
174
175 __rte_thread_init(lcore_id, &lcore_config[lcore_id].cpuset);
176
177 ret = eal_thread_dump_current_affinity(cpuset, sizeof(cpuset));
178 EAL_LOG(DEBUG, "lcore %u is ready (tid=%zx;cpuset=[%s%s])",
179 lcore_id, rte_thread_self().opaque_id, cpuset,
180 ret == 0 ? "" : "...");
181
182 rte_eal_trace_thread_lcore_ready(lcore_id, cpuset);
183
184 /* read on our pipe to get commands */
185 while (1) {
186 lcore_function_t *f;
187 void *fct_arg;
188
189 eal_thread_wait_command();
190
191 /* Set the state to 'RUNNING'. Use release order
192 * since 'state' variable is used as the guard variable.
193 */
194 rte_atomic_store_explicit(&lcore_config[lcore_id].state, RUNNING,
195 rte_memory_order_release);
196
197 eal_thread_ack_command();
198
199 /* Load 'f' with acquire order to ensure that
200 * the memory operations from the main thread
201 * are accessed only after update to 'f' is visible.
202 * Wait till the update to 'f' is visible to the worker.
203 */
204 while ((f = rte_atomic_load_explicit(&lcore_config[lcore_id].f,
205 rte_memory_order_acquire)) == NULL)
206 rte_pause();
207
208 rte_eal_trace_thread_lcore_running(lcore_id, f);
209
210 /* call the function and store the return value */
211 fct_arg = lcore_config[lcore_id].arg;
212 ret = f(fct_arg);
213 lcore_config[lcore_id].ret = ret;
214 lcore_config[lcore_id].f = NULL;
215 lcore_config[lcore_id].arg = NULL;
216
217 /* Store the state with release order to ensure that
218 * the memory operations from the worker thread
219 * are completed before the state is updated.
220 * Use 'state' as the guard variable.
221 */
222 rte_atomic_store_explicit(&lcore_config[lcore_id].state, WAIT,
223 rte_memory_order_release);
224
225 rte_eal_trace_thread_lcore_stopped(lcore_id);
226 }
227
228 /* never reached */
229 /* return 0; */
230 }
231
232 enum __rte_ctrl_thread_status {
233 CTRL_THREAD_LAUNCHING, /* Yet to call pthread_create function */
234 CTRL_THREAD_RUNNING, /* Control thread is running successfully */
235 CTRL_THREAD_ERROR /* Control thread encountered an error */
236 };
237
238 struct control_thread_params {
239 rte_thread_func start_routine;
240 void *arg;
241 int ret;
242 /* Control thread status.
243 * If the status is CTRL_THREAD_ERROR, 'ret' has the error code.
244 */
245 RTE_ATOMIC(enum __rte_ctrl_thread_status) status;
246 };
247
control_thread_init(void * arg)248 static int control_thread_init(void *arg)
249 {
250 struct internal_config *internal_conf =
251 eal_get_internal_configuration();
252 rte_cpuset_t *cpuset = &internal_conf->ctrl_cpuset;
253 struct control_thread_params *params = arg;
254
255 __rte_thread_init(rte_lcore_id(), cpuset);
256 /* Set control thread socket ID to SOCKET_ID_ANY
257 * as control threads may be scheduled on any NUMA node.
258 */
259 RTE_PER_LCORE(_socket_id) = SOCKET_ID_ANY;
260 params->ret = rte_thread_set_affinity_by_id(rte_thread_self(), cpuset);
261 if (params->ret != 0) {
262 rte_atomic_store_explicit(¶ms->status,
263 CTRL_THREAD_ERROR, rte_memory_order_release);
264 return 1;
265 }
266
267 rte_atomic_store_explicit(¶ms->status,
268 CTRL_THREAD_RUNNING, rte_memory_order_release);
269
270 return 0;
271 }
272
control_thread_start(void * arg)273 static uint32_t control_thread_start(void *arg)
274 {
275 struct control_thread_params *params = arg;
276 void *start_arg = params->arg;
277 rte_thread_func start_routine = params->start_routine;
278
279 if (control_thread_init(arg) != 0)
280 return 0;
281
282 return start_routine(start_arg);
283 }
284
285 int
rte_thread_create_control(rte_thread_t * thread,const char * name,rte_thread_func start_routine,void * arg)286 rte_thread_create_control(rte_thread_t *thread, const char *name,
287 rte_thread_func start_routine, void *arg)
288 {
289 struct control_thread_params *params;
290 enum __rte_ctrl_thread_status ctrl_thread_status;
291 int ret;
292
293 params = malloc(sizeof(*params));
294 if (params == NULL)
295 return -ENOMEM;
296
297 params->start_routine = start_routine;
298 params->arg = arg;
299 params->ret = 0;
300 params->status = CTRL_THREAD_LAUNCHING;
301
302 ret = rte_thread_create(thread, NULL, control_thread_start, params);
303 if (ret != 0) {
304 free(params);
305 return -ret;
306 }
307
308 if (name != NULL)
309 rte_thread_set_name(*thread, name);
310
311 /* Wait for the control thread to initialize successfully */
312 while ((ctrl_thread_status =
313 rte_atomic_load_explicit(¶ms->status,
314 rte_memory_order_acquire)) == CTRL_THREAD_LAUNCHING) {
315 rte_delay_us_sleep(1);
316 }
317
318 /* Check if the control thread encountered an error */
319 if (ctrl_thread_status == CTRL_THREAD_ERROR) {
320 /* ctrl thread is exiting */
321 rte_thread_join(*thread, NULL);
322 }
323
324 ret = params->ret;
325 free(params);
326
327 return ret;
328 }
329
330 static void
add_internal_prefix(char * prefixed_name,const char * name,size_t size)331 add_internal_prefix(char *prefixed_name, const char *name, size_t size)
332 {
333 size_t prefixlen;
334
335 /* Check RTE_THREAD_INTERNAL_NAME_SIZE definition. */
336 RTE_BUILD_BUG_ON(RTE_THREAD_INTERNAL_NAME_SIZE !=
337 RTE_THREAD_NAME_SIZE - sizeof(RTE_THREAD_INTERNAL_PREFIX) + 1);
338
339 prefixlen = strlen(RTE_THREAD_INTERNAL_PREFIX);
340 strlcpy(prefixed_name, RTE_THREAD_INTERNAL_PREFIX, size);
341 strlcpy(prefixed_name + prefixlen, name, size - prefixlen);
342 }
343
344 int
rte_thread_create_internal_control(rte_thread_t * id,const char * name,rte_thread_func func,void * arg)345 rte_thread_create_internal_control(rte_thread_t *id, const char *name,
346 rte_thread_func func, void *arg)
347 {
348 char prefixed_name[RTE_THREAD_NAME_SIZE];
349
350 add_internal_prefix(prefixed_name, name, sizeof(prefixed_name));
351 return rte_thread_create_control(id, prefixed_name, func, arg);
352 }
353
354 void
rte_thread_set_prefixed_name(rte_thread_t id,const char * name)355 rte_thread_set_prefixed_name(rte_thread_t id, const char *name)
356 {
357 char prefixed_name[RTE_THREAD_NAME_SIZE];
358
359 add_internal_prefix(prefixed_name, name, sizeof(prefixed_name));
360 rte_thread_set_name(id, prefixed_name);
361 }
362
363 int
rte_thread_register(void)364 rte_thread_register(void)
365 {
366 unsigned int lcore_id;
367 rte_cpuset_t cpuset;
368
369 /* EAL init flushes all lcores, we can't register before. */
370 if (eal_get_internal_configuration()->init_complete != 1) {
371 EAL_LOG(DEBUG, "Called %s before EAL init.", __func__);
372 rte_errno = EINVAL;
373 return -1;
374 }
375 if (!rte_mp_disable()) {
376 EAL_LOG(ERR, "Multiprocess in use, registering non-EAL threads is not supported.");
377 rte_errno = EINVAL;
378 return -1;
379 }
380 if (rte_thread_get_affinity_by_id(rte_thread_self(), &cpuset) != 0)
381 CPU_ZERO(&cpuset);
382 lcore_id = eal_lcore_non_eal_allocate();
383 if (lcore_id >= RTE_MAX_LCORE)
384 lcore_id = LCORE_ID_ANY;
385 __rte_thread_init(lcore_id, &cpuset);
386 if (lcore_id == LCORE_ID_ANY) {
387 rte_errno = ENOMEM;
388 return -1;
389 }
390 EAL_LOG(DEBUG, "Registered non-EAL thread as lcore %u.",
391 lcore_id);
392 return 0;
393 }
394
395 void
rte_thread_unregister(void)396 rte_thread_unregister(void)
397 {
398 unsigned int lcore_id = rte_lcore_id();
399
400 if (lcore_id != LCORE_ID_ANY)
401 eal_lcore_non_eal_release(lcore_id);
402 __rte_thread_uninit();
403 if (lcore_id != LCORE_ID_ANY)
404 EAL_LOG(DEBUG, "Unregistered non-EAL thread (was lcore %u).",
405 lcore_id);
406 }
407
408 int
rte_thread_attr_init(rte_thread_attr_t * attr)409 rte_thread_attr_init(rte_thread_attr_t *attr)
410 {
411 if (attr == NULL)
412 return EINVAL;
413
414 CPU_ZERO(&attr->cpuset);
415 attr->priority = RTE_THREAD_PRIORITY_NORMAL;
416
417 return 0;
418 }
419
420 int
rte_thread_attr_set_priority(rte_thread_attr_t * thread_attr,enum rte_thread_priority priority)421 rte_thread_attr_set_priority(rte_thread_attr_t *thread_attr,
422 enum rte_thread_priority priority)
423 {
424 if (thread_attr == NULL)
425 return EINVAL;
426
427 thread_attr->priority = priority;
428
429 return 0;
430 }
431
432 int
rte_thread_attr_set_affinity(rte_thread_attr_t * thread_attr,rte_cpuset_t * cpuset)433 rte_thread_attr_set_affinity(rte_thread_attr_t *thread_attr,
434 rte_cpuset_t *cpuset)
435 {
436 if (thread_attr == NULL)
437 return EINVAL;
438
439 if (cpuset == NULL)
440 return EINVAL;
441
442 thread_attr->cpuset = *cpuset;
443
444 return 0;
445 }
446
447 int
rte_thread_attr_get_affinity(rte_thread_attr_t * thread_attr,rte_cpuset_t * cpuset)448 rte_thread_attr_get_affinity(rte_thread_attr_t *thread_attr,
449 rte_cpuset_t *cpuset)
450 {
451 if (thread_attr == NULL)
452 return EINVAL;
453
454 if (cpuset == NULL)
455 return EINVAL;
456
457 *cpuset = thread_attr->cpuset;
458
459 return 0;
460 }
461