1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2021 Mellanox Technologies, Ltd 3 * Copyright (C) 2022 Microsoft Corporation 4 */ 5 6 #include <errno.h> 7 #include <pthread.h> 8 #include <stdbool.h> 9 #include <stdlib.h> 10 #include <string.h> 11 12 #include <rte_errno.h> 13 #include <rte_log.h> 14 #include <rte_thread.h> 15 16 struct eal_tls_key { 17 pthread_key_t thread_index; 18 }; 19 20 struct thread_start_context { 21 rte_thread_func thread_func; 22 void *thread_args; 23 const rte_thread_attr_t *thread_attr; 24 pthread_mutex_t wrapper_mutex; 25 pthread_cond_t wrapper_cond; 26 int wrapper_ret; 27 bool wrapper_done; 28 }; 29 30 static int 31 thread_map_priority_to_os_value(enum rte_thread_priority eal_pri, int *os_pri, 32 int *pol) 33 { 34 /* Clear the output parameters. */ 35 *os_pri = sched_get_priority_min(SCHED_OTHER) - 1; 36 *pol = -1; 37 38 switch (eal_pri) { 39 case RTE_THREAD_PRIORITY_NORMAL: 40 *pol = SCHED_OTHER; 41 42 /* 43 * Choose the middle of the range to represent the priority 44 * 'normal'. 45 * On Linux, this should be 0, since both 46 * sched_get_priority_min/_max return 0 for SCHED_OTHER. 47 */ 48 *os_pri = (sched_get_priority_min(SCHED_OTHER) + 49 sched_get_priority_max(SCHED_OTHER)) / 2; 50 break; 51 case RTE_THREAD_PRIORITY_REALTIME_CRITICAL: 52 *pol = SCHED_RR; 53 *os_pri = sched_get_priority_max(SCHED_RR); 54 break; 55 default: 56 RTE_LOG(DEBUG, EAL, "The requested priority value is invalid.\n"); 57 return EINVAL; 58 } 59 60 return 0; 61 } 62 63 static int 64 thread_map_os_priority_to_eal_priority(int policy, int os_pri, 65 enum rte_thread_priority *eal_pri) 66 { 67 switch (policy) { 68 case SCHED_OTHER: 69 if (os_pri == (sched_get_priority_min(SCHED_OTHER) + 70 sched_get_priority_max(SCHED_OTHER)) / 2) { 71 *eal_pri = RTE_THREAD_PRIORITY_NORMAL; 72 return 0; 73 } 74 break; 75 case SCHED_RR: 76 if (os_pri == sched_get_priority_max(SCHED_RR)) { 77 *eal_pri = RTE_THREAD_PRIORITY_REALTIME_CRITICAL; 78 return 0; 79 } 80 break; 81 default: 82 RTE_LOG(DEBUG, EAL, "The OS priority value does not map to an EAL-defined priority.\n"); 83 return EINVAL; 84 } 85 86 return 0; 87 } 88 89 static void * 90 thread_start_wrapper(void *arg) 91 { 92 struct thread_start_context *ctx = (struct thread_start_context *)arg; 93 rte_thread_func thread_func = ctx->thread_func; 94 void *thread_args = ctx->thread_args; 95 int ret = 0; 96 97 if (ctx->thread_attr != NULL && CPU_COUNT(&ctx->thread_attr->cpuset) > 0) { 98 ret = rte_thread_set_affinity_by_id(rte_thread_self(), &ctx->thread_attr->cpuset); 99 if (ret != 0) 100 RTE_LOG(DEBUG, EAL, "rte_thread_set_affinity_by_id failed\n"); 101 } 102 103 pthread_mutex_lock(&ctx->wrapper_mutex); 104 ctx->wrapper_ret = ret; 105 ctx->wrapper_done = true; 106 pthread_cond_signal(&ctx->wrapper_cond); 107 pthread_mutex_unlock(&ctx->wrapper_mutex); 108 109 if (ret != 0) 110 return NULL; 111 112 return (void *)(uintptr_t)thread_func(thread_args); 113 } 114 115 int 116 rte_thread_create(rte_thread_t *thread_id, 117 const rte_thread_attr_t *thread_attr, 118 rte_thread_func thread_func, void *args) 119 { 120 int ret = 0; 121 pthread_attr_t attr; 122 pthread_attr_t *attrp = NULL; 123 struct sched_param param = { 124 .sched_priority = 0, 125 }; 126 int policy = SCHED_OTHER; 127 struct thread_start_context ctx = { 128 .thread_func = thread_func, 129 .thread_args = args, 130 .thread_attr = thread_attr, 131 .wrapper_done = false, 132 .wrapper_mutex = PTHREAD_MUTEX_INITIALIZER, 133 .wrapper_cond = PTHREAD_COND_INITIALIZER, 134 }; 135 136 if (thread_attr != NULL) { 137 ret = pthread_attr_init(&attr); 138 if (ret != 0) { 139 RTE_LOG(DEBUG, EAL, "pthread_attr_init failed\n"); 140 goto cleanup; 141 } 142 143 attrp = &attr; 144 145 /* 146 * Set the inherit scheduler parameter to explicit, 147 * otherwise the priority attribute is ignored. 148 */ 149 ret = pthread_attr_setinheritsched(attrp, 150 PTHREAD_EXPLICIT_SCHED); 151 if (ret != 0) { 152 RTE_LOG(DEBUG, EAL, "pthread_attr_setinheritsched failed\n"); 153 goto cleanup; 154 } 155 156 if (thread_attr->priority == 157 RTE_THREAD_PRIORITY_REALTIME_CRITICAL) { 158 ret = ENOTSUP; 159 goto cleanup; 160 } 161 ret = thread_map_priority_to_os_value(thread_attr->priority, 162 ¶m.sched_priority, &policy); 163 if (ret != 0) 164 goto cleanup; 165 166 ret = pthread_attr_setschedpolicy(attrp, policy); 167 if (ret != 0) { 168 RTE_LOG(DEBUG, EAL, "pthread_attr_setschedpolicy failed\n"); 169 goto cleanup; 170 } 171 172 ret = pthread_attr_setschedparam(attrp, ¶m); 173 if (ret != 0) { 174 RTE_LOG(DEBUG, EAL, "pthread_attr_setschedparam failed\n"); 175 goto cleanup; 176 } 177 } 178 179 ret = pthread_create((pthread_t *)&thread_id->opaque_id, attrp, 180 thread_start_wrapper, &ctx); 181 if (ret != 0) { 182 RTE_LOG(DEBUG, EAL, "pthread_create failed\n"); 183 goto cleanup; 184 } 185 186 pthread_mutex_lock(&ctx.wrapper_mutex); 187 while (!ctx.wrapper_done) 188 pthread_cond_wait(&ctx.wrapper_cond, &ctx.wrapper_mutex); 189 ret = ctx.wrapper_ret; 190 pthread_mutex_unlock(&ctx.wrapper_mutex); 191 192 if (ret != 0) 193 rte_thread_join(*thread_id, NULL); 194 195 cleanup: 196 if (attrp != NULL) 197 pthread_attr_destroy(&attr); 198 199 return ret; 200 } 201 202 int 203 rte_thread_join(rte_thread_t thread_id, uint32_t *value_ptr) 204 { 205 int ret = 0; 206 void *res = (void *)(uintptr_t)0; 207 void **pres = NULL; 208 209 if (value_ptr != NULL) 210 pres = &res; 211 212 ret = pthread_join((pthread_t)thread_id.opaque_id, pres); 213 if (ret != 0) { 214 RTE_LOG(DEBUG, EAL, "pthread_join failed\n"); 215 return ret; 216 } 217 218 if (value_ptr != NULL) 219 *value_ptr = (uint32_t)(uintptr_t)res; 220 221 return 0; 222 } 223 224 int 225 rte_thread_detach(rte_thread_t thread_id) 226 { 227 return pthread_detach((pthread_t)thread_id.opaque_id); 228 } 229 230 int 231 rte_thread_equal(rte_thread_t t1, rte_thread_t t2) 232 { 233 return pthread_equal((pthread_t)t1.opaque_id, (pthread_t)t2.opaque_id); 234 } 235 236 rte_thread_t 237 rte_thread_self(void) 238 { 239 RTE_BUILD_BUG_ON(sizeof(pthread_t) > sizeof(uintptr_t)); 240 241 rte_thread_t thread_id; 242 243 thread_id.opaque_id = (uintptr_t)pthread_self(); 244 245 return thread_id; 246 } 247 248 int 249 rte_thread_get_priority(rte_thread_t thread_id, 250 enum rte_thread_priority *priority) 251 { 252 struct sched_param param; 253 int policy; 254 int ret; 255 256 ret = pthread_getschedparam((pthread_t)thread_id.opaque_id, &policy, 257 ¶m); 258 if (ret != 0) { 259 RTE_LOG(DEBUG, EAL, "pthread_getschedparam failed\n"); 260 goto cleanup; 261 } 262 263 return thread_map_os_priority_to_eal_priority(policy, 264 param.sched_priority, priority); 265 266 cleanup: 267 return ret; 268 } 269 270 int 271 rte_thread_set_priority(rte_thread_t thread_id, 272 enum rte_thread_priority priority) 273 { 274 struct sched_param param; 275 int policy; 276 int ret; 277 278 /* Realtime priority can cause crashes on non-Windows platforms. */ 279 if (priority == RTE_THREAD_PRIORITY_REALTIME_CRITICAL) 280 return ENOTSUP; 281 282 ret = thread_map_priority_to_os_value(priority, ¶m.sched_priority, 283 &policy); 284 if (ret != 0) 285 return ret; 286 287 return pthread_setschedparam((pthread_t)thread_id.opaque_id, policy, 288 ¶m); 289 } 290 291 int 292 rte_thread_key_create(rte_thread_key *key, void (*destructor)(void *)) 293 { 294 int err; 295 296 *key = malloc(sizeof(**key)); 297 if ((*key) == NULL) { 298 RTE_LOG(DEBUG, EAL, "Cannot allocate TLS key.\n"); 299 rte_errno = ENOMEM; 300 return -1; 301 } 302 err = pthread_key_create(&((*key)->thread_index), destructor); 303 if (err) { 304 RTE_LOG(DEBUG, EAL, "pthread_key_create failed: %s\n", 305 strerror(err)); 306 free(*key); 307 rte_errno = ENOEXEC; 308 return -1; 309 } 310 return 0; 311 } 312 313 int 314 rte_thread_key_delete(rte_thread_key key) 315 { 316 int err; 317 318 if (!key) { 319 RTE_LOG(DEBUG, EAL, "Invalid TLS key.\n"); 320 rte_errno = EINVAL; 321 return -1; 322 } 323 err = pthread_key_delete(key->thread_index); 324 if (err) { 325 RTE_LOG(DEBUG, EAL, "pthread_key_delete failed: %s\n", 326 strerror(err)); 327 free(key); 328 rte_errno = ENOEXEC; 329 return -1; 330 } 331 free(key); 332 return 0; 333 } 334 335 int 336 rte_thread_value_set(rte_thread_key key, const void *value) 337 { 338 int err; 339 340 if (!key) { 341 RTE_LOG(DEBUG, EAL, "Invalid TLS key.\n"); 342 rte_errno = EINVAL; 343 return -1; 344 } 345 err = pthread_setspecific(key->thread_index, value); 346 if (err) { 347 RTE_LOG(DEBUG, EAL, "pthread_setspecific failed: %s\n", 348 strerror(err)); 349 rte_errno = ENOEXEC; 350 return -1; 351 } 352 return 0; 353 } 354 355 void * 356 rte_thread_value_get(rte_thread_key key) 357 { 358 if (!key) { 359 RTE_LOG(DEBUG, EAL, "Invalid TLS key.\n"); 360 rte_errno = EINVAL; 361 return NULL; 362 } 363 return pthread_getspecific(key->thread_index); 364 } 365 366 int 367 rte_thread_set_affinity_by_id(rte_thread_t thread_id, 368 const rte_cpuset_t *cpuset) 369 { 370 return pthread_setaffinity_np((pthread_t)thread_id.opaque_id, 371 sizeof(*cpuset), cpuset); 372 } 373 374 int 375 rte_thread_get_affinity_by_id(rte_thread_t thread_id, 376 rte_cpuset_t *cpuset) 377 { 378 return pthread_getaffinity_np((pthread_t)thread_id.opaque_id, 379 sizeof(*cpuset), cpuset); 380 } 381