1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <stdint.h> 8 #include <unistd.h> 9 #include <pthread.h> 10 #include <signal.h> 11 #include <sched.h> 12 #include <assert.h> 13 #include <string.h> 14 15 #include <rte_errno.h> 16 #include <rte_lcore.h> 17 #include <rte_log.h> 18 #include <rte_memory.h> 19 #include <rte_trace_point.h> 20 21 #include "eal_internal_cfg.h" 22 #include "eal_private.h" 23 #include "eal_thread.h" 24 #include "eal_trace.h" 25 26 RTE_DEFINE_PER_LCORE(unsigned int, _lcore_id) = LCORE_ID_ANY; 27 RTE_DEFINE_PER_LCORE(int, _thread_id) = -1; 28 static RTE_DEFINE_PER_LCORE(unsigned int, _socket_id) = 29 (unsigned int)SOCKET_ID_ANY; 30 static RTE_DEFINE_PER_LCORE(rte_cpuset_t, _cpuset); 31 32 unsigned rte_socket_id(void) 33 { 34 return RTE_PER_LCORE(_socket_id); 35 } 36 37 static int 38 eal_cpuset_socket_id(rte_cpuset_t *cpusetp) 39 { 40 unsigned cpu = 0; 41 int socket_id = SOCKET_ID_ANY; 42 int sid; 43 44 if (cpusetp == NULL) 45 return SOCKET_ID_ANY; 46 47 do { 48 if (!CPU_ISSET(cpu, cpusetp)) 49 continue; 50 51 if (socket_id == SOCKET_ID_ANY) 52 socket_id = eal_cpu_socket_id(cpu); 53 54 sid = eal_cpu_socket_id(cpu); 55 if (socket_id != sid) { 56 socket_id = SOCKET_ID_ANY; 57 break; 58 } 59 60 } while (++cpu < CPU_SETSIZE); 61 62 return socket_id; 63 } 64 65 static void 66 thread_update_affinity(rte_cpuset_t *cpusetp) 67 { 68 unsigned int lcore_id = rte_lcore_id(); 69 70 /* store socket_id in TLS for quick access */ 71 RTE_PER_LCORE(_socket_id) = 72 eal_cpuset_socket_id(cpusetp); 73 74 /* store cpuset in TLS for quick access */ 75 memmove(&RTE_PER_LCORE(_cpuset), cpusetp, 76 sizeof(rte_cpuset_t)); 77 78 if (lcore_id != (unsigned)LCORE_ID_ANY) { 79 /* EAL thread will update lcore_config */ 80 lcore_config[lcore_id].socket_id = RTE_PER_LCORE(_socket_id); 81 memmove(&lcore_config[lcore_id].cpuset, cpusetp, 82 sizeof(rte_cpuset_t)); 83 } 84 } 85 86 int 87 rte_thread_set_affinity(rte_cpuset_t *cpusetp) 88 { 89 if (pthread_setaffinity_np(pthread_self(), sizeof(rte_cpuset_t), 90 cpusetp) != 0) { 91 RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n"); 92 return -1; 93 } 94 95 thread_update_affinity(cpusetp); 96 return 0; 97 } 98 99 void 100 rte_thread_get_affinity(rte_cpuset_t *cpusetp) 101 { 102 assert(cpusetp); 103 memmove(cpusetp, &RTE_PER_LCORE(_cpuset), 104 sizeof(rte_cpuset_t)); 105 } 106 107 int 108 eal_thread_dump_affinity(rte_cpuset_t *cpuset, char *str, unsigned int size) 109 { 110 unsigned cpu; 111 int ret; 112 unsigned int out = 0; 113 114 for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { 115 if (!CPU_ISSET(cpu, cpuset)) 116 continue; 117 118 ret = snprintf(str + out, 119 size - out, "%u,", cpu); 120 if (ret < 0 || (unsigned)ret >= size - out) { 121 /* string will be truncated */ 122 ret = -1; 123 goto exit; 124 } 125 126 out += ret; 127 } 128 129 ret = 0; 130 exit: 131 /* remove the last separator */ 132 if (out > 0) 133 str[out - 1] = '\0'; 134 135 return ret; 136 } 137 138 int 139 eal_thread_dump_current_affinity(char *str, unsigned int size) 140 { 141 rte_cpuset_t cpuset; 142 143 rte_thread_get_affinity(&cpuset); 144 return eal_thread_dump_affinity(&cpuset, str, size); 145 } 146 147 void 148 __rte_thread_init(unsigned int lcore_id, rte_cpuset_t *cpuset) 149 { 150 /* set the lcore ID in per-lcore memory area */ 151 RTE_PER_LCORE(_lcore_id) = lcore_id; 152 153 /* acquire system unique id */ 154 rte_gettid(); 155 156 thread_update_affinity(cpuset); 157 158 __rte_trace_mem_per_thread_alloc(); 159 } 160 161 void 162 __rte_thread_uninit(void) 163 { 164 trace_mem_per_thread_free(); 165 166 RTE_PER_LCORE(_lcore_id) = LCORE_ID_ANY; 167 } 168 169 struct rte_thread_ctrl_params { 170 void *(*start_routine)(void *); 171 void *arg; 172 pthread_barrier_t configured; 173 unsigned int refcnt; 174 }; 175 176 static void ctrl_params_free(struct rte_thread_ctrl_params *params) 177 { 178 if (__atomic_sub_fetch(¶ms->refcnt, 1, __ATOMIC_ACQ_REL) == 0) { 179 (void)pthread_barrier_destroy(¶ms->configured); 180 free(params); 181 } 182 } 183 184 static void *ctrl_thread_init(void *arg) 185 { 186 struct internal_config *internal_conf = 187 eal_get_internal_configuration(); 188 rte_cpuset_t *cpuset = &internal_conf->ctrl_cpuset; 189 struct rte_thread_ctrl_params *params = arg; 190 void *(*start_routine)(void *); 191 void *routine_arg = params->arg; 192 193 __rte_thread_init(rte_lcore_id(), cpuset); 194 195 pthread_barrier_wait(¶ms->configured); 196 start_routine = params->start_routine; 197 ctrl_params_free(params); 198 199 if (start_routine == NULL) 200 return NULL; 201 202 return start_routine(routine_arg); 203 } 204 205 int 206 rte_ctrl_thread_create(pthread_t *thread, const char *name, 207 const pthread_attr_t *attr, 208 void *(*start_routine)(void *), void *arg) 209 { 210 struct internal_config *internal_conf = 211 eal_get_internal_configuration(); 212 rte_cpuset_t *cpuset = &internal_conf->ctrl_cpuset; 213 struct rte_thread_ctrl_params *params; 214 int ret; 215 216 params = malloc(sizeof(*params)); 217 if (!params) 218 return -ENOMEM; 219 220 params->start_routine = start_routine; 221 params->arg = arg; 222 params->refcnt = 2; 223 224 ret = pthread_barrier_init(¶ms->configured, NULL, 2); 225 if (ret != 0) 226 goto fail_no_barrier; 227 228 ret = pthread_create(thread, attr, ctrl_thread_init, (void *)params); 229 if (ret != 0) 230 goto fail_with_barrier; 231 232 if (name != NULL) { 233 ret = rte_thread_setname(*thread, name); 234 if (ret < 0) 235 RTE_LOG(DEBUG, EAL, 236 "Cannot set name for ctrl thread\n"); 237 } 238 239 ret = pthread_setaffinity_np(*thread, sizeof(*cpuset), cpuset); 240 if (ret != 0) 241 params->start_routine = NULL; 242 243 pthread_barrier_wait(¶ms->configured); 244 ctrl_params_free(params); 245 246 if (ret != 0) 247 /* start_routine has been set to NULL above; */ 248 /* ctrl thread will exit immediately */ 249 pthread_join(*thread, NULL); 250 251 return -ret; 252 253 fail_with_barrier: 254 (void)pthread_barrier_destroy(¶ms->configured); 255 256 fail_no_barrier: 257 free(params); 258 259 return -ret; 260 } 261 262 int 263 rte_thread_register(void) 264 { 265 unsigned int lcore_id; 266 rte_cpuset_t cpuset; 267 268 /* EAL init flushes all lcores, we can't register before. */ 269 if (eal_get_internal_configuration()->init_complete != 1) { 270 RTE_LOG(DEBUG, EAL, "Called %s before EAL init.\n", __func__); 271 rte_errno = EINVAL; 272 return -1; 273 } 274 if (!rte_mp_disable()) { 275 RTE_LOG(ERR, EAL, "Multiprocess in use, registering non-EAL threads is not supported.\n"); 276 rte_errno = EINVAL; 277 return -1; 278 } 279 if (pthread_getaffinity_np(pthread_self(), sizeof(cpuset), 280 &cpuset) != 0) 281 CPU_ZERO(&cpuset); 282 lcore_id = eal_lcore_non_eal_allocate(); 283 if (lcore_id >= RTE_MAX_LCORE) 284 lcore_id = LCORE_ID_ANY; 285 __rte_thread_init(lcore_id, &cpuset); 286 if (lcore_id == LCORE_ID_ANY) { 287 rte_errno = ENOMEM; 288 return -1; 289 } 290 RTE_LOG(DEBUG, EAL, "Registered non-EAL thread as lcore %u.\n", 291 lcore_id); 292 return 0; 293 } 294 295 void 296 rte_thread_unregister(void) 297 { 298 unsigned int lcore_id = rte_lcore_id(); 299 300 if (lcore_id != LCORE_ID_ANY) 301 eal_lcore_non_eal_release(lcore_id); 302 __rte_thread_uninit(); 303 if (lcore_id != LCORE_ID_ANY) 304 RTE_LOG(DEBUG, EAL, "Unregistered non-EAL thread (was lcore %u).\n", 305 lcore_id); 306 } 307