1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2021 Mellanox Technologies, Ltd 3 * Copyright (C) 2022 Microsoft Corporation 4 */ 5 6 #include <errno.h> 7 #include <pthread.h> 8 #include <stdbool.h> 9 #include <stdlib.h> 10 #include <string.h> 11 12 #include <rte_errno.h> 13 #include <rte_log.h> 14 #include <rte_thread.h> 15 16 #include "eal_private.h" 17 18 struct eal_tls_key { 19 pthread_key_t thread_index; 20 }; 21 22 #ifndef RTE_EAL_PTHREAD_ATTR_SETAFFINITY_NP 23 struct thread_start_context { 24 rte_thread_func thread_func; 25 void *thread_args; 26 const rte_thread_attr_t *thread_attr; 27 pthread_mutex_t wrapper_mutex; 28 pthread_cond_t wrapper_cond; 29 int wrapper_ret; 30 bool wrapper_done; 31 }; 32 #endif 33 34 static int 35 thread_map_priority_to_os_value(enum rte_thread_priority eal_pri, int *os_pri, 36 int *pol) 37 { 38 /* Clear the output parameters. */ 39 *os_pri = sched_get_priority_min(SCHED_OTHER) - 1; 40 *pol = -1; 41 42 switch (eal_pri) { 43 case RTE_THREAD_PRIORITY_NORMAL: 44 *pol = SCHED_OTHER; 45 46 /* 47 * Choose the middle of the range to represent the priority 48 * 'normal'. 49 * On Linux, this should be 0, since both 50 * sched_get_priority_min/_max return 0 for SCHED_OTHER. 51 */ 52 *os_pri = (sched_get_priority_min(SCHED_OTHER) + 53 sched_get_priority_max(SCHED_OTHER)) / 2; 54 break; 55 case RTE_THREAD_PRIORITY_REALTIME_CRITICAL: 56 *pol = SCHED_RR; 57 *os_pri = sched_get_priority_max(SCHED_RR); 58 break; 59 default: 60 EAL_LOG(DEBUG, "The requested priority value is invalid."); 61 return EINVAL; 62 } 63 64 return 0; 65 } 66 67 static int 68 thread_map_os_priority_to_eal_priority(int policy, int os_pri, 69 enum rte_thread_priority *eal_pri) 70 { 71 switch (policy) { 72 case SCHED_OTHER: 73 if (os_pri == (sched_get_priority_min(SCHED_OTHER) + 74 sched_get_priority_max(SCHED_OTHER)) / 2) { 75 *eal_pri = RTE_THREAD_PRIORITY_NORMAL; 76 return 0; 77 } 78 break; 79 case SCHED_RR: 80 if (os_pri == sched_get_priority_max(SCHED_RR)) { 81 *eal_pri = RTE_THREAD_PRIORITY_REALTIME_CRITICAL; 82 return 0; 83 } 84 break; 85 default: 86 EAL_LOG(DEBUG, "The OS priority value does not map to an EAL-defined priority."); 87 return EINVAL; 88 } 89 90 return 0; 91 } 92 93 #ifndef RTE_EAL_PTHREAD_ATTR_SETAFFINITY_NP 94 static void * 95 thread_start_wrapper(void *arg) 96 { 97 struct thread_start_context *ctx = (struct thread_start_context *)arg; 98 rte_thread_func thread_func = ctx->thread_func; 99 void *thread_args = ctx->thread_args; 100 int ret = 0; 101 102 if (ctx->thread_attr != NULL && CPU_COUNT(&ctx->thread_attr->cpuset) > 0) { 103 ret = rte_thread_set_affinity_by_id(rte_thread_self(), &ctx->thread_attr->cpuset); 104 if (ret != 0) 105 EAL_LOG(DEBUG, "rte_thread_set_affinity_by_id failed"); 106 } 107 108 pthread_mutex_lock(&ctx->wrapper_mutex); 109 ctx->wrapper_ret = ret; 110 ctx->wrapper_done = true; 111 pthread_cond_signal(&ctx->wrapper_cond); 112 pthread_mutex_unlock(&ctx->wrapper_mutex); 113 114 if (ret != 0) 115 return NULL; 116 117 return (void *)(uintptr_t)thread_func(thread_args); 118 } 119 #endif 120 121 int 122 rte_thread_create(rte_thread_t *thread_id, 123 const rte_thread_attr_t *thread_attr, 124 rte_thread_func thread_func, void *args) 125 { 126 int ret = 0; 127 pthread_attr_t attr; 128 pthread_attr_t *attrp = NULL; 129 struct sched_param param = { 130 .sched_priority = 0, 131 }; 132 int policy = SCHED_OTHER; 133 #ifndef RTE_EAL_PTHREAD_ATTR_SETAFFINITY_NP 134 struct thread_start_context ctx = { 135 .thread_func = thread_func, 136 .thread_args = args, 137 .thread_attr = thread_attr, 138 .wrapper_done = false, 139 .wrapper_mutex = PTHREAD_MUTEX_INITIALIZER, 140 .wrapper_cond = PTHREAD_COND_INITIALIZER, 141 }; 142 #endif 143 144 if (thread_attr != NULL) { 145 ret = pthread_attr_init(&attr); 146 if (ret != 0) { 147 EAL_LOG(DEBUG, "pthread_attr_init failed"); 148 goto cleanup; 149 } 150 151 attrp = &attr; 152 153 #ifdef RTE_EAL_PTHREAD_ATTR_SETAFFINITY_NP 154 if (CPU_COUNT(&thread_attr->cpuset) > 0) { 155 ret = pthread_attr_setaffinity_np(attrp, sizeof(thread_attr->cpuset), 156 &thread_attr->cpuset); 157 if (ret != 0) { 158 EAL_LOG(DEBUG, "pthread_attr_setaffinity_np failed"); 159 goto cleanup; 160 } 161 } 162 #endif 163 /* 164 * Set the inherit scheduler parameter to explicit, 165 * otherwise the priority attribute is ignored. 166 */ 167 ret = pthread_attr_setinheritsched(attrp, 168 PTHREAD_EXPLICIT_SCHED); 169 if (ret != 0) { 170 EAL_LOG(DEBUG, "pthread_attr_setinheritsched failed"); 171 goto cleanup; 172 } 173 174 if (thread_attr->priority == 175 RTE_THREAD_PRIORITY_REALTIME_CRITICAL) { 176 ret = ENOTSUP; 177 goto cleanup; 178 } 179 ret = thread_map_priority_to_os_value(thread_attr->priority, 180 ¶m.sched_priority, &policy); 181 if (ret != 0) 182 goto cleanup; 183 184 ret = pthread_attr_setschedpolicy(attrp, policy); 185 if (ret != 0) { 186 EAL_LOG(DEBUG, "pthread_attr_setschedpolicy failed"); 187 goto cleanup; 188 } 189 190 ret = pthread_attr_setschedparam(attrp, ¶m); 191 if (ret != 0) { 192 EAL_LOG(DEBUG, "pthread_attr_setschedparam failed"); 193 goto cleanup; 194 } 195 } 196 197 #ifdef RTE_EAL_PTHREAD_ATTR_SETAFFINITY_NP 198 ret = pthread_create((pthread_t *)&thread_id->opaque_id, attrp, 199 (void *)(void *)thread_func, args); 200 if (ret != 0) { 201 EAL_LOG(DEBUG, "pthread_create failed"); 202 goto cleanup; 203 } 204 #else /* !RTE_EAL_PTHREAD_ATTR_SETAFFINITY_NP */ 205 ret = pthread_create((pthread_t *)&thread_id->opaque_id, attrp, 206 thread_start_wrapper, &ctx); 207 if (ret != 0) { 208 EAL_LOG(DEBUG, "pthread_create failed"); 209 goto cleanup; 210 } 211 212 pthread_mutex_lock(&ctx.wrapper_mutex); 213 while (!ctx.wrapper_done) 214 pthread_cond_wait(&ctx.wrapper_cond, &ctx.wrapper_mutex); 215 ret = ctx.wrapper_ret; 216 pthread_mutex_unlock(&ctx.wrapper_mutex); 217 218 if (ret != 0) 219 rte_thread_join(*thread_id, NULL); 220 #endif /* RTE_EAL_PTHREAD_ATTR_SETAFFINITY_NP */ 221 222 cleanup: 223 if (attrp != NULL) 224 pthread_attr_destroy(&attr); 225 226 return ret; 227 } 228 229 int 230 rte_thread_join(rte_thread_t thread_id, uint32_t *value_ptr) 231 { 232 int ret = 0; 233 void *res = (void *)(uintptr_t)0; 234 void **pres = NULL; 235 236 if (value_ptr != NULL) 237 pres = &res; 238 239 ret = pthread_join((pthread_t)thread_id.opaque_id, pres); 240 if (ret != 0) { 241 EAL_LOG(DEBUG, "pthread_join failed"); 242 return ret; 243 } 244 245 if (value_ptr != NULL) 246 *value_ptr = (uint32_t)(uintptr_t)res; 247 248 return 0; 249 } 250 251 int 252 rte_thread_detach(rte_thread_t thread_id) 253 { 254 return pthread_detach((pthread_t)thread_id.opaque_id); 255 } 256 257 int 258 rte_thread_equal(rte_thread_t t1, rte_thread_t t2) 259 { 260 return pthread_equal((pthread_t)t1.opaque_id, (pthread_t)t2.opaque_id); 261 } 262 263 rte_thread_t 264 rte_thread_self(void) 265 { 266 RTE_BUILD_BUG_ON(sizeof(pthread_t) > sizeof(uintptr_t)); 267 268 rte_thread_t thread_id; 269 270 thread_id.opaque_id = (uintptr_t)pthread_self(); 271 272 return thread_id; 273 } 274 275 int 276 rte_thread_get_priority(rte_thread_t thread_id, 277 enum rte_thread_priority *priority) 278 { 279 struct sched_param param; 280 int policy; 281 int ret; 282 283 ret = pthread_getschedparam((pthread_t)thread_id.opaque_id, &policy, 284 ¶m); 285 if (ret != 0) { 286 EAL_LOG(DEBUG, "pthread_getschedparam failed"); 287 goto cleanup; 288 } 289 290 return thread_map_os_priority_to_eal_priority(policy, 291 param.sched_priority, priority); 292 293 cleanup: 294 return ret; 295 } 296 297 int 298 rte_thread_set_priority(rte_thread_t thread_id, 299 enum rte_thread_priority priority) 300 { 301 struct sched_param param; 302 int policy; 303 int ret; 304 305 /* Realtime priority can cause crashes on non-Windows platforms. */ 306 if (priority == RTE_THREAD_PRIORITY_REALTIME_CRITICAL) 307 return ENOTSUP; 308 309 ret = thread_map_priority_to_os_value(priority, ¶m.sched_priority, 310 &policy); 311 if (ret != 0) 312 return ret; 313 314 return pthread_setschedparam((pthread_t)thread_id.opaque_id, policy, 315 ¶m); 316 } 317 318 int 319 rte_thread_key_create(rte_thread_key *key, void (*destructor)(void *)) 320 { 321 int err; 322 323 *key = malloc(sizeof(**key)); 324 if ((*key) == NULL) { 325 EAL_LOG(DEBUG, "Cannot allocate TLS key."); 326 rte_errno = ENOMEM; 327 return -1; 328 } 329 err = pthread_key_create(&((*key)->thread_index), destructor); 330 if (err) { 331 EAL_LOG(DEBUG, "pthread_key_create failed: %s", 332 strerror(err)); 333 free(*key); 334 rte_errno = ENOEXEC; 335 return -1; 336 } 337 return 0; 338 } 339 340 int 341 rte_thread_key_delete(rte_thread_key key) 342 { 343 int err; 344 345 if (!key) { 346 EAL_LOG(DEBUG, "Invalid TLS key."); 347 rte_errno = EINVAL; 348 return -1; 349 } 350 err = pthread_key_delete(key->thread_index); 351 if (err) { 352 EAL_LOG(DEBUG, "pthread_key_delete failed: %s", 353 strerror(err)); 354 free(key); 355 rte_errno = ENOEXEC; 356 return -1; 357 } 358 free(key); 359 return 0; 360 } 361 362 int 363 rte_thread_value_set(rte_thread_key key, const void *value) 364 { 365 int err; 366 367 if (!key) { 368 EAL_LOG(DEBUG, "Invalid TLS key."); 369 rte_errno = EINVAL; 370 return -1; 371 } 372 err = pthread_setspecific(key->thread_index, value); 373 if (err) { 374 EAL_LOG(DEBUG, "pthread_setspecific failed: %s", 375 strerror(err)); 376 rte_errno = ENOEXEC; 377 return -1; 378 } 379 return 0; 380 } 381 382 void * 383 rte_thread_value_get(rte_thread_key key) 384 { 385 if (!key) { 386 EAL_LOG(DEBUG, "Invalid TLS key."); 387 rte_errno = EINVAL; 388 return NULL; 389 } 390 return pthread_getspecific(key->thread_index); 391 } 392 393 int 394 rte_thread_set_affinity_by_id(rte_thread_t thread_id, 395 const rte_cpuset_t *cpuset) 396 { 397 return pthread_setaffinity_np((pthread_t)thread_id.opaque_id, 398 sizeof(*cpuset), cpuset); 399 } 400 401 int 402 rte_thread_get_affinity_by_id(rte_thread_t thread_id, 403 rte_cpuset_t *cpuset) 404 { 405 return pthread_getaffinity_np((pthread_t)thread_id.opaque_id, 406 sizeof(*cpuset), cpuset); 407 } 408