1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <stdint.h> 8 #include <unistd.h> 9 #include <pthread.h> 10 #include <signal.h> 11 #include <sched.h> 12 #include <assert.h> 13 #include <string.h> 14 15 #include <rte_errno.h> 16 #include <rte_lcore.h> 17 #include <rte_log.h> 18 #include <rte_memory.h> 19 #include <rte_trace_point.h> 20 21 #include "eal_internal_cfg.h" 22 #include "eal_private.h" 23 #include "eal_thread.h" 24 #include "eal_trace.h" 25 26 RTE_DEFINE_PER_LCORE(unsigned int, _lcore_id) = LCORE_ID_ANY; 27 RTE_DEFINE_PER_LCORE(int, _thread_id) = -1; 28 static RTE_DEFINE_PER_LCORE(unsigned int, _socket_id) = 29 (unsigned int)SOCKET_ID_ANY; 30 static RTE_DEFINE_PER_LCORE(rte_cpuset_t, _cpuset); 31 32 unsigned rte_socket_id(void) 33 { 34 return RTE_PER_LCORE(_socket_id); 35 } 36 37 static int 38 eal_cpuset_socket_id(rte_cpuset_t *cpusetp) 39 { 40 unsigned cpu = 0; 41 int socket_id = SOCKET_ID_ANY; 42 int sid; 43 44 if (cpusetp == NULL) 45 return SOCKET_ID_ANY; 46 47 do { 48 if (!CPU_ISSET(cpu, cpusetp)) 49 continue; 50 51 if (socket_id == SOCKET_ID_ANY) 52 socket_id = eal_cpu_socket_id(cpu); 53 54 sid = eal_cpu_socket_id(cpu); 55 if (socket_id != sid) { 56 socket_id = SOCKET_ID_ANY; 57 break; 58 } 59 60 } while (++cpu < CPU_SETSIZE); 61 62 return socket_id; 63 } 64 65 static void 66 thread_update_affinity(rte_cpuset_t *cpusetp) 67 { 68 unsigned int lcore_id = rte_lcore_id(); 69 70 /* store socket_id in TLS for quick access */ 71 RTE_PER_LCORE(_socket_id) = 72 eal_cpuset_socket_id(cpusetp); 73 74 /* store cpuset in TLS for quick access */ 75 memmove(&RTE_PER_LCORE(_cpuset), cpusetp, 76 sizeof(rte_cpuset_t)); 77 78 if (lcore_id != (unsigned)LCORE_ID_ANY) { 79 /* EAL thread will update lcore_config */ 80 lcore_config[lcore_id].socket_id = RTE_PER_LCORE(_socket_id); 81 memmove(&lcore_config[lcore_id].cpuset, cpusetp, 82 sizeof(rte_cpuset_t)); 83 } 84 } 85 86 int 87 rte_thread_set_affinity(rte_cpuset_t *cpusetp) 88 { 89 if (pthread_setaffinity_np(pthread_self(), sizeof(rte_cpuset_t), 90 cpusetp) != 0) { 91 RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n"); 92 return -1; 93 } 94 95 thread_update_affinity(cpusetp); 96 return 0; 97 } 98 99 void 100 rte_thread_get_affinity(rte_cpuset_t *cpusetp) 101 { 102 assert(cpusetp); 103 memmove(cpusetp, &RTE_PER_LCORE(_cpuset), 104 sizeof(rte_cpuset_t)); 105 } 106 107 int 108 eal_thread_dump_affinity(rte_cpuset_t *cpuset, char *str, unsigned int size) 109 { 110 unsigned cpu; 111 int ret; 112 unsigned int out = 0; 113 114 for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { 115 if (!CPU_ISSET(cpu, cpuset)) 116 continue; 117 118 ret = snprintf(str + out, 119 size - out, "%u,", cpu); 120 if (ret < 0 || (unsigned)ret >= size - out) { 121 /* string will be truncated */ 122 ret = -1; 123 goto exit; 124 } 125 126 out += ret; 127 } 128 129 ret = 0; 130 exit: 131 /* remove the last separator */ 132 if (out > 0) 133 str[out - 1] = '\0'; 134 135 return ret; 136 } 137 138 int 139 eal_thread_dump_current_affinity(char *str, unsigned int size) 140 { 141 rte_cpuset_t cpuset; 142 143 rte_thread_get_affinity(&cpuset); 144 return eal_thread_dump_affinity(&cpuset, str, size); 145 } 146 147 void 148 __rte_thread_init(unsigned int lcore_id, rte_cpuset_t *cpuset) 149 { 150 /* set the lcore ID in per-lcore memory area */ 151 RTE_PER_LCORE(_lcore_id) = lcore_id; 152 153 /* acquire system unique id */ 154 rte_gettid(); 155 156 thread_update_affinity(cpuset); 157 158 __rte_trace_mem_per_thread_alloc(); 159 } 160 161 void 162 __rte_thread_uninit(void) 163 { 164 trace_mem_per_thread_free(); 165 166 RTE_PER_LCORE(_lcore_id) = LCORE_ID_ANY; 167 } 168 169 enum __rte_ctrl_thread_status { 170 CTRL_THREAD_LAUNCHING, /* Yet to call pthread_create function */ 171 CTRL_THREAD_RUNNING, /* Control thread is running successfully */ 172 CTRL_THREAD_ERROR /* Control thread encountered an error */ 173 }; 174 175 struct rte_thread_ctrl_params { 176 void *(*start_routine)(void *); 177 void *arg; 178 int ret; 179 /* Control thread status. 180 * If the status is CTRL_THREAD_ERROR, 'ret' has the error code. 181 */ 182 enum __rte_ctrl_thread_status ctrl_thread_status; 183 }; 184 185 static void *ctrl_thread_init(void *arg) 186 { 187 struct internal_config *internal_conf = 188 eal_get_internal_configuration(); 189 rte_cpuset_t *cpuset = &internal_conf->ctrl_cpuset; 190 struct rte_thread_ctrl_params *params = arg; 191 void *(*start_routine)(void *) = params->start_routine; 192 void *routine_arg = params->arg; 193 194 __rte_thread_init(rte_lcore_id(), cpuset); 195 params->ret = pthread_setaffinity_np(pthread_self(), sizeof(*cpuset), 196 cpuset); 197 if (params->ret != 0) { 198 __atomic_store_n(¶ms->ctrl_thread_status, 199 CTRL_THREAD_ERROR, __ATOMIC_RELEASE); 200 return NULL; 201 } 202 203 __atomic_store_n(¶ms->ctrl_thread_status, 204 CTRL_THREAD_RUNNING, __ATOMIC_RELEASE); 205 206 return start_routine(routine_arg); 207 } 208 209 int 210 rte_ctrl_thread_create(pthread_t *thread, const char *name, 211 const pthread_attr_t *attr, 212 void *(*start_routine)(void *), void *arg) 213 { 214 struct rte_thread_ctrl_params *params; 215 enum __rte_ctrl_thread_status ctrl_thread_status; 216 int ret; 217 218 params = malloc(sizeof(*params)); 219 if (!params) 220 return -ENOMEM; 221 222 params->start_routine = start_routine; 223 params->arg = arg; 224 params->ret = 0; 225 params->ctrl_thread_status = CTRL_THREAD_LAUNCHING; 226 227 ret = pthread_create(thread, attr, ctrl_thread_init, (void *)params); 228 if (ret != 0) { 229 free(params); 230 return -ret; 231 } 232 233 if (name != NULL) { 234 ret = rte_thread_setname(*thread, name); 235 if (ret < 0) 236 RTE_LOG(DEBUG, EAL, 237 "Cannot set name for ctrl thread\n"); 238 } 239 240 /* Wait for the control thread to initialize successfully */ 241 while ((ctrl_thread_status = 242 __atomic_load_n(¶ms->ctrl_thread_status, 243 __ATOMIC_ACQUIRE)) == CTRL_THREAD_LAUNCHING) { 244 /* Yield the CPU. Using sched_yield call requires maintaining 245 * another implementation for Windows as sched_yield is not 246 * supported on Windows. 247 */ 248 rte_delay_us_sleep(1); 249 } 250 251 /* Check if the control thread encountered an error */ 252 if (ctrl_thread_status == CTRL_THREAD_ERROR) { 253 /* ctrl thread is exiting */ 254 pthread_join(*thread, NULL); 255 } 256 257 ret = params->ret; 258 free(params); 259 260 return -ret; 261 } 262 263 int 264 rte_thread_register(void) 265 { 266 unsigned int lcore_id; 267 rte_cpuset_t cpuset; 268 269 /* EAL init flushes all lcores, we can't register before. */ 270 if (eal_get_internal_configuration()->init_complete != 1) { 271 RTE_LOG(DEBUG, EAL, "Called %s before EAL init.\n", __func__); 272 rte_errno = EINVAL; 273 return -1; 274 } 275 if (!rte_mp_disable()) { 276 RTE_LOG(ERR, EAL, "Multiprocess in use, registering non-EAL threads is not supported.\n"); 277 rte_errno = EINVAL; 278 return -1; 279 } 280 if (pthread_getaffinity_np(pthread_self(), sizeof(cpuset), 281 &cpuset) != 0) 282 CPU_ZERO(&cpuset); 283 lcore_id = eal_lcore_non_eal_allocate(); 284 if (lcore_id >= RTE_MAX_LCORE) 285 lcore_id = LCORE_ID_ANY; 286 __rte_thread_init(lcore_id, &cpuset); 287 if (lcore_id == LCORE_ID_ANY) { 288 rte_errno = ENOMEM; 289 return -1; 290 } 291 RTE_LOG(DEBUG, EAL, "Registered non-EAL thread as lcore %u.\n", 292 lcore_id); 293 return 0; 294 } 295 296 void 297 rte_thread_unregister(void) 298 { 299 unsigned int lcore_id = rte_lcore_id(); 300 301 if (lcore_id != LCORE_ID_ANY) 302 eal_lcore_non_eal_release(lcore_id); 303 __rte_thread_uninit(); 304 if (lcore_id != LCORE_ID_ANY) 305 RTE_LOG(DEBUG, EAL, "Unregistered non-EAL thread (was lcore %u).\n", 306 lcore_id); 307 } 308