1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2021 Mellanox Technologies, Ltd 3 * Copyright (C) 2022 Microsoft Corporation 4 */ 5 6 #include <errno.h> 7 #include <pthread.h> 8 #include <stdbool.h> 9 #include <stdlib.h> 10 #include <string.h> 11 12 #include <rte_errno.h> 13 #include <rte_log.h> 14 #include <rte_thread.h> 15 16 #include "eal_private.h" 17 18 struct eal_tls_key { 19 pthread_key_t thread_index; 20 }; 21 22 struct thread_start_context { 23 rte_thread_func thread_func; 24 void *thread_args; 25 const rte_thread_attr_t *thread_attr; 26 pthread_mutex_t wrapper_mutex; 27 pthread_cond_t wrapper_cond; 28 int wrapper_ret; 29 bool wrapper_done; 30 }; 31 32 static int 33 thread_map_priority_to_os_value(enum rte_thread_priority eal_pri, int *os_pri, 34 int *pol) 35 { 36 /* Clear the output parameters. */ 37 *os_pri = sched_get_priority_min(SCHED_OTHER) - 1; 38 *pol = -1; 39 40 switch (eal_pri) { 41 case RTE_THREAD_PRIORITY_NORMAL: 42 *pol = SCHED_OTHER; 43 44 /* 45 * Choose the middle of the range to represent the priority 46 * 'normal'. 47 * On Linux, this should be 0, since both 48 * sched_get_priority_min/_max return 0 for SCHED_OTHER. 49 */ 50 *os_pri = (sched_get_priority_min(SCHED_OTHER) + 51 sched_get_priority_max(SCHED_OTHER)) / 2; 52 break; 53 case RTE_THREAD_PRIORITY_REALTIME_CRITICAL: 54 *pol = SCHED_RR; 55 *os_pri = sched_get_priority_max(SCHED_RR); 56 break; 57 default: 58 EAL_LOG(DEBUG, "The requested priority value is invalid."); 59 return EINVAL; 60 } 61 62 return 0; 63 } 64 65 static int 66 thread_map_os_priority_to_eal_priority(int policy, int os_pri, 67 enum rte_thread_priority *eal_pri) 68 { 69 switch (policy) { 70 case SCHED_OTHER: 71 if (os_pri == (sched_get_priority_min(SCHED_OTHER) + 72 sched_get_priority_max(SCHED_OTHER)) / 2) { 73 *eal_pri = RTE_THREAD_PRIORITY_NORMAL; 74 return 0; 75 } 76 break; 77 case SCHED_RR: 78 if (os_pri == sched_get_priority_max(SCHED_RR)) { 79 *eal_pri = RTE_THREAD_PRIORITY_REALTIME_CRITICAL; 80 return 0; 81 } 82 break; 83 default: 84 EAL_LOG(DEBUG, "The OS priority value does not map to an EAL-defined priority."); 85 return EINVAL; 86 } 87 88 return 0; 89 } 90 91 static void * 92 thread_start_wrapper(void *arg) 93 { 94 struct thread_start_context *ctx = (struct thread_start_context *)arg; 95 rte_thread_func thread_func = ctx->thread_func; 96 void *thread_args = ctx->thread_args; 97 int ret = 0; 98 99 if (ctx->thread_attr != NULL && CPU_COUNT(&ctx->thread_attr->cpuset) > 0) { 100 ret = rte_thread_set_affinity_by_id(rte_thread_self(), &ctx->thread_attr->cpuset); 101 if (ret != 0) 102 EAL_LOG(DEBUG, "rte_thread_set_affinity_by_id failed"); 103 } 104 105 pthread_mutex_lock(&ctx->wrapper_mutex); 106 ctx->wrapper_ret = ret; 107 ctx->wrapper_done = true; 108 pthread_cond_signal(&ctx->wrapper_cond); 109 pthread_mutex_unlock(&ctx->wrapper_mutex); 110 111 if (ret != 0) 112 return NULL; 113 114 return (void *)(uintptr_t)thread_func(thread_args); 115 } 116 117 int 118 rte_thread_create(rte_thread_t *thread_id, 119 const rte_thread_attr_t *thread_attr, 120 rte_thread_func thread_func, void *args) 121 { 122 int ret = 0; 123 pthread_attr_t attr; 124 pthread_attr_t *attrp = NULL; 125 struct sched_param param = { 126 .sched_priority = 0, 127 }; 128 int policy = SCHED_OTHER; 129 struct thread_start_context ctx = { 130 .thread_func = thread_func, 131 .thread_args = args, 132 .thread_attr = thread_attr, 133 .wrapper_done = false, 134 .wrapper_mutex = PTHREAD_MUTEX_INITIALIZER, 135 .wrapper_cond = PTHREAD_COND_INITIALIZER, 136 }; 137 138 if (thread_attr != NULL) { 139 ret = pthread_attr_init(&attr); 140 if (ret != 0) { 141 EAL_LOG(DEBUG, "pthread_attr_init failed"); 142 goto cleanup; 143 } 144 145 attrp = &attr; 146 147 /* 148 * Set the inherit scheduler parameter to explicit, 149 * otherwise the priority attribute is ignored. 150 */ 151 ret = pthread_attr_setinheritsched(attrp, 152 PTHREAD_EXPLICIT_SCHED); 153 if (ret != 0) { 154 EAL_LOG(DEBUG, "pthread_attr_setinheritsched failed"); 155 goto cleanup; 156 } 157 158 if (thread_attr->priority == 159 RTE_THREAD_PRIORITY_REALTIME_CRITICAL) { 160 ret = ENOTSUP; 161 goto cleanup; 162 } 163 ret = thread_map_priority_to_os_value(thread_attr->priority, 164 ¶m.sched_priority, &policy); 165 if (ret != 0) 166 goto cleanup; 167 168 ret = pthread_attr_setschedpolicy(attrp, policy); 169 if (ret != 0) { 170 EAL_LOG(DEBUG, "pthread_attr_setschedpolicy failed"); 171 goto cleanup; 172 } 173 174 ret = pthread_attr_setschedparam(attrp, ¶m); 175 if (ret != 0) { 176 EAL_LOG(DEBUG, "pthread_attr_setschedparam failed"); 177 goto cleanup; 178 } 179 } 180 181 ret = pthread_create((pthread_t *)&thread_id->opaque_id, attrp, 182 thread_start_wrapper, &ctx); 183 if (ret != 0) { 184 EAL_LOG(DEBUG, "pthread_create failed"); 185 goto cleanup; 186 } 187 188 pthread_mutex_lock(&ctx.wrapper_mutex); 189 while (!ctx.wrapper_done) 190 pthread_cond_wait(&ctx.wrapper_cond, &ctx.wrapper_mutex); 191 ret = ctx.wrapper_ret; 192 pthread_mutex_unlock(&ctx.wrapper_mutex); 193 194 if (ret != 0) 195 rte_thread_join(*thread_id, NULL); 196 197 cleanup: 198 if (attrp != NULL) 199 pthread_attr_destroy(&attr); 200 201 return ret; 202 } 203 204 int 205 rte_thread_join(rte_thread_t thread_id, uint32_t *value_ptr) 206 { 207 int ret = 0; 208 void *res = (void *)(uintptr_t)0; 209 void **pres = NULL; 210 211 if (value_ptr != NULL) 212 pres = &res; 213 214 ret = pthread_join((pthread_t)thread_id.opaque_id, pres); 215 if (ret != 0) { 216 EAL_LOG(DEBUG, "pthread_join failed"); 217 return ret; 218 } 219 220 if (value_ptr != NULL) 221 *value_ptr = (uint32_t)(uintptr_t)res; 222 223 return 0; 224 } 225 226 int 227 rte_thread_detach(rte_thread_t thread_id) 228 { 229 return pthread_detach((pthread_t)thread_id.opaque_id); 230 } 231 232 int 233 rte_thread_equal(rte_thread_t t1, rte_thread_t t2) 234 { 235 return pthread_equal((pthread_t)t1.opaque_id, (pthread_t)t2.opaque_id); 236 } 237 238 rte_thread_t 239 rte_thread_self(void) 240 { 241 RTE_BUILD_BUG_ON(sizeof(pthread_t) > sizeof(uintptr_t)); 242 243 rte_thread_t thread_id; 244 245 thread_id.opaque_id = (uintptr_t)pthread_self(); 246 247 return thread_id; 248 } 249 250 int 251 rte_thread_get_priority(rte_thread_t thread_id, 252 enum rte_thread_priority *priority) 253 { 254 struct sched_param param; 255 int policy; 256 int ret; 257 258 ret = pthread_getschedparam((pthread_t)thread_id.opaque_id, &policy, 259 ¶m); 260 if (ret != 0) { 261 EAL_LOG(DEBUG, "pthread_getschedparam failed"); 262 goto cleanup; 263 } 264 265 return thread_map_os_priority_to_eal_priority(policy, 266 param.sched_priority, priority); 267 268 cleanup: 269 return ret; 270 } 271 272 int 273 rte_thread_set_priority(rte_thread_t thread_id, 274 enum rte_thread_priority priority) 275 { 276 struct sched_param param; 277 int policy; 278 int ret; 279 280 /* Realtime priority can cause crashes on non-Windows platforms. */ 281 if (priority == RTE_THREAD_PRIORITY_REALTIME_CRITICAL) 282 return ENOTSUP; 283 284 ret = thread_map_priority_to_os_value(priority, ¶m.sched_priority, 285 &policy); 286 if (ret != 0) 287 return ret; 288 289 return pthread_setschedparam((pthread_t)thread_id.opaque_id, policy, 290 ¶m); 291 } 292 293 int 294 rte_thread_key_create(rte_thread_key *key, void (*destructor)(void *)) 295 { 296 int err; 297 298 *key = malloc(sizeof(**key)); 299 if ((*key) == NULL) { 300 EAL_LOG(DEBUG, "Cannot allocate TLS key."); 301 rte_errno = ENOMEM; 302 return -1; 303 } 304 err = pthread_key_create(&((*key)->thread_index), destructor); 305 if (err) { 306 EAL_LOG(DEBUG, "pthread_key_create failed: %s", 307 strerror(err)); 308 free(*key); 309 rte_errno = ENOEXEC; 310 return -1; 311 } 312 return 0; 313 } 314 315 int 316 rte_thread_key_delete(rte_thread_key key) 317 { 318 int err; 319 320 if (!key) { 321 EAL_LOG(DEBUG, "Invalid TLS key."); 322 rte_errno = EINVAL; 323 return -1; 324 } 325 err = pthread_key_delete(key->thread_index); 326 if (err) { 327 EAL_LOG(DEBUG, "pthread_key_delete failed: %s", 328 strerror(err)); 329 free(key); 330 rte_errno = ENOEXEC; 331 return -1; 332 } 333 free(key); 334 return 0; 335 } 336 337 int 338 rte_thread_value_set(rte_thread_key key, const void *value) 339 { 340 int err; 341 342 if (!key) { 343 EAL_LOG(DEBUG, "Invalid TLS key."); 344 rte_errno = EINVAL; 345 return -1; 346 } 347 err = pthread_setspecific(key->thread_index, value); 348 if (err) { 349 EAL_LOG(DEBUG, "pthread_setspecific failed: %s", 350 strerror(err)); 351 rte_errno = ENOEXEC; 352 return -1; 353 } 354 return 0; 355 } 356 357 void * 358 rte_thread_value_get(rte_thread_key key) 359 { 360 if (!key) { 361 EAL_LOG(DEBUG, "Invalid TLS key."); 362 rte_errno = EINVAL; 363 return NULL; 364 } 365 return pthread_getspecific(key->thread_index); 366 } 367 368 int 369 rte_thread_set_affinity_by_id(rte_thread_t thread_id, 370 const rte_cpuset_t *cpuset) 371 { 372 return pthread_setaffinity_np((pthread_t)thread_id.opaque_id, 373 sizeof(*cpuset), cpuset); 374 } 375 376 int 377 rte_thread_get_affinity_by_id(rte_thread_t thread_id, 378 rte_cpuset_t *cpuset) 379 { 380 return pthread_getaffinity_np((pthread_t)thread_id.opaque_id, 381 sizeof(*cpuset), cpuset); 382 } 383