1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <pthread.h> 8 #include <sched.h> 9 #include <assert.h> 10 #include <string.h> 11 12 #include <rte_errno.h> 13 #include <rte_lcore.h> 14 #include <rte_log.h> 15 #include <rte_memory.h> 16 #include <rte_trace_point.h> 17 18 #include "eal_internal_cfg.h" 19 #include "eal_private.h" 20 #include "eal_thread.h" 21 #include "eal_trace.h" 22 23 RTE_DEFINE_PER_LCORE(unsigned int, _lcore_id) = LCORE_ID_ANY; 24 RTE_DEFINE_PER_LCORE(int, _thread_id) = -1; 25 static RTE_DEFINE_PER_LCORE(unsigned int, _socket_id) = 26 (unsigned int)SOCKET_ID_ANY; 27 static RTE_DEFINE_PER_LCORE(rte_cpuset_t, _cpuset); 28 29 unsigned rte_socket_id(void) 30 { 31 return RTE_PER_LCORE(_socket_id); 32 } 33 34 static int 35 eal_cpuset_socket_id(rte_cpuset_t *cpusetp) 36 { 37 unsigned cpu = 0; 38 int socket_id = SOCKET_ID_ANY; 39 int sid; 40 41 if (cpusetp == NULL) 42 return SOCKET_ID_ANY; 43 44 do { 45 if (!CPU_ISSET(cpu, cpusetp)) 46 continue; 47 48 if (socket_id == SOCKET_ID_ANY) 49 socket_id = eal_cpu_socket_id(cpu); 50 51 sid = eal_cpu_socket_id(cpu); 52 if (socket_id != sid) { 53 socket_id = SOCKET_ID_ANY; 54 break; 55 } 56 57 } while (++cpu < CPU_SETSIZE); 58 59 return socket_id; 60 } 61 62 static void 63 thread_update_affinity(rte_cpuset_t *cpusetp) 64 { 65 unsigned int lcore_id = rte_lcore_id(); 66 67 /* store socket_id in TLS for quick access */ 68 RTE_PER_LCORE(_socket_id) = 69 eal_cpuset_socket_id(cpusetp); 70 71 /* store cpuset in TLS for quick access */ 72 memmove(&RTE_PER_LCORE(_cpuset), cpusetp, 73 sizeof(rte_cpuset_t)); 74 75 if (lcore_id != (unsigned)LCORE_ID_ANY) { 76 /* EAL thread will update lcore_config */ 77 lcore_config[lcore_id].socket_id = RTE_PER_LCORE(_socket_id); 78 memmove(&lcore_config[lcore_id].cpuset, cpusetp, 79 sizeof(rte_cpuset_t)); 80 } 81 } 82 83 int 84 rte_thread_set_affinity(rte_cpuset_t *cpusetp) 85 { 86 if (pthread_setaffinity_np(pthread_self(), sizeof(rte_cpuset_t), 87 cpusetp) != 0) { 88 RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n"); 89 return -1; 90 } 91 92 thread_update_affinity(cpusetp); 93 return 0; 94 } 95 96 void 97 rte_thread_get_affinity(rte_cpuset_t *cpusetp) 98 { 99 assert(cpusetp); 100 memmove(cpusetp, &RTE_PER_LCORE(_cpuset), 101 sizeof(rte_cpuset_t)); 102 } 103 104 int 105 eal_thread_dump_affinity(rte_cpuset_t *cpuset, char *str, unsigned int size) 106 { 107 unsigned cpu; 108 int ret; 109 unsigned int out = 0; 110 111 for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { 112 if (!CPU_ISSET(cpu, cpuset)) 113 continue; 114 115 ret = snprintf(str + out, 116 size - out, "%u,", cpu); 117 if (ret < 0 || (unsigned)ret >= size - out) { 118 /* string will be truncated */ 119 ret = -1; 120 goto exit; 121 } 122 123 out += ret; 124 } 125 126 ret = 0; 127 exit: 128 /* remove the last separator */ 129 if (out > 0) 130 str[out - 1] = '\0'; 131 132 return ret; 133 } 134 135 int 136 eal_thread_dump_current_affinity(char *str, unsigned int size) 137 { 138 rte_cpuset_t cpuset; 139 140 rte_thread_get_affinity(&cpuset); 141 return eal_thread_dump_affinity(&cpuset, str, size); 142 } 143 144 void 145 __rte_thread_init(unsigned int lcore_id, rte_cpuset_t *cpuset) 146 { 147 /* set the lcore ID in per-lcore memory area */ 148 RTE_PER_LCORE(_lcore_id) = lcore_id; 149 150 /* acquire system unique id */ 151 rte_gettid(); 152 153 thread_update_affinity(cpuset); 154 155 __rte_trace_mem_per_thread_alloc(); 156 } 157 158 void 159 __rte_thread_uninit(void) 160 { 161 trace_mem_per_thread_free(); 162 163 RTE_PER_LCORE(_lcore_id) = LCORE_ID_ANY; 164 } 165 166 enum __rte_ctrl_thread_status { 167 CTRL_THREAD_LAUNCHING, /* Yet to call pthread_create function */ 168 CTRL_THREAD_RUNNING, /* Control thread is running successfully */ 169 CTRL_THREAD_ERROR /* Control thread encountered an error */ 170 }; 171 172 struct rte_thread_ctrl_params { 173 void *(*start_routine)(void *); 174 void *arg; 175 int ret; 176 /* Control thread status. 177 * If the status is CTRL_THREAD_ERROR, 'ret' has the error code. 178 */ 179 enum __rte_ctrl_thread_status ctrl_thread_status; 180 }; 181 182 static void *ctrl_thread_init(void *arg) 183 { 184 struct internal_config *internal_conf = 185 eal_get_internal_configuration(); 186 rte_cpuset_t *cpuset = &internal_conf->ctrl_cpuset; 187 struct rte_thread_ctrl_params *params = arg; 188 void *(*start_routine)(void *) = params->start_routine; 189 void *routine_arg = params->arg; 190 191 __rte_thread_init(rte_lcore_id(), cpuset); 192 params->ret = pthread_setaffinity_np(pthread_self(), sizeof(*cpuset), 193 cpuset); 194 if (params->ret != 0) { 195 __atomic_store_n(¶ms->ctrl_thread_status, 196 CTRL_THREAD_ERROR, __ATOMIC_RELEASE); 197 return NULL; 198 } 199 200 __atomic_store_n(¶ms->ctrl_thread_status, 201 CTRL_THREAD_RUNNING, __ATOMIC_RELEASE); 202 203 return start_routine(routine_arg); 204 } 205 206 int 207 rte_ctrl_thread_create(pthread_t *thread, const char *name, 208 const pthread_attr_t *attr, 209 void *(*start_routine)(void *), void *arg) 210 { 211 struct rte_thread_ctrl_params *params; 212 enum __rte_ctrl_thread_status ctrl_thread_status; 213 int ret; 214 215 params = malloc(sizeof(*params)); 216 if (!params) 217 return -ENOMEM; 218 219 params->start_routine = start_routine; 220 params->arg = arg; 221 params->ret = 0; 222 params->ctrl_thread_status = CTRL_THREAD_LAUNCHING; 223 224 ret = pthread_create(thread, attr, ctrl_thread_init, (void *)params); 225 if (ret != 0) { 226 free(params); 227 return -ret; 228 } 229 230 if (name != NULL) { 231 ret = rte_thread_setname(*thread, name); 232 if (ret < 0) 233 RTE_LOG(DEBUG, EAL, 234 "Cannot set name for ctrl thread\n"); 235 } 236 237 /* Wait for the control thread to initialize successfully */ 238 while ((ctrl_thread_status = 239 __atomic_load_n(¶ms->ctrl_thread_status, 240 __ATOMIC_ACQUIRE)) == CTRL_THREAD_LAUNCHING) { 241 /* Yield the CPU. Using sched_yield call requires maintaining 242 * another implementation for Windows as sched_yield is not 243 * supported on Windows. 244 */ 245 rte_delay_us_sleep(1); 246 } 247 248 /* Check if the control thread encountered an error */ 249 if (ctrl_thread_status == CTRL_THREAD_ERROR) { 250 /* ctrl thread is exiting */ 251 pthread_join(*thread, NULL); 252 } 253 254 ret = params->ret; 255 free(params); 256 257 return -ret; 258 } 259 260 int 261 rte_thread_register(void) 262 { 263 unsigned int lcore_id; 264 rte_cpuset_t cpuset; 265 266 /* EAL init flushes all lcores, we can't register before. */ 267 if (eal_get_internal_configuration()->init_complete != 1) { 268 RTE_LOG(DEBUG, EAL, "Called %s before EAL init.\n", __func__); 269 rte_errno = EINVAL; 270 return -1; 271 } 272 if (!rte_mp_disable()) { 273 RTE_LOG(ERR, EAL, "Multiprocess in use, registering non-EAL threads is not supported.\n"); 274 rte_errno = EINVAL; 275 return -1; 276 } 277 if (pthread_getaffinity_np(pthread_self(), sizeof(cpuset), 278 &cpuset) != 0) 279 CPU_ZERO(&cpuset); 280 lcore_id = eal_lcore_non_eal_allocate(); 281 if (lcore_id >= RTE_MAX_LCORE) 282 lcore_id = LCORE_ID_ANY; 283 __rte_thread_init(lcore_id, &cpuset); 284 if (lcore_id == LCORE_ID_ANY) { 285 rte_errno = ENOMEM; 286 return -1; 287 } 288 RTE_LOG(DEBUG, EAL, "Registered non-EAL thread as lcore %u.\n", 289 lcore_id); 290 return 0; 291 } 292 293 void 294 rte_thread_unregister(void) 295 { 296 unsigned int lcore_id = rte_lcore_id(); 297 298 if (lcore_id != LCORE_ID_ANY) 299 eal_lcore_non_eal_release(lcore_id); 300 __rte_thread_uninit(); 301 if (lcore_id != LCORE_ID_ANY) 302 RTE_LOG(DEBUG, EAL, "Unregistered non-EAL thread (was lcore %u).\n", 303 lcore_id); 304 } 305