1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2021 Mellanox Technologies, Ltd 3 * Copyright (C) 2022 Microsoft Corporation 4 */ 5 6 #include <errno.h> 7 8 #include <rte_common.h> 9 #include <rte_errno.h> 10 #include <rte_thread.h> 11 12 #include "eal_windows.h" 13 14 struct eal_tls_key { 15 DWORD thread_index; 16 }; 17 18 struct thread_routine_ctx { 19 rte_thread_func thread_func; 20 void *routine_args; 21 }; 22 23 /* Translates the most common error codes related to threads */ 24 static int 25 thread_translate_win32_error(DWORD error) 26 { 27 switch (error) { 28 case ERROR_SUCCESS: 29 return 0; 30 31 case ERROR_INVALID_PARAMETER: 32 return EINVAL; 33 34 case ERROR_INVALID_HANDLE: 35 return EFAULT; 36 37 case ERROR_NOT_ENOUGH_MEMORY: 38 /* FALLTHROUGH */ 39 case ERROR_NO_SYSTEM_RESOURCES: 40 return ENOMEM; 41 42 case ERROR_PRIVILEGE_NOT_HELD: 43 /* FALLTHROUGH */ 44 case ERROR_ACCESS_DENIED: 45 return EACCES; 46 47 case ERROR_ALREADY_EXISTS: 48 return EEXIST; 49 50 case ERROR_POSSIBLE_DEADLOCK: 51 return EDEADLK; 52 53 case ERROR_INVALID_FUNCTION: 54 /* FALLTHROUGH */ 55 case ERROR_CALL_NOT_IMPLEMENTED: 56 return ENOSYS; 57 } 58 59 return EINVAL; 60 } 61 62 static int 63 thread_log_last_error(const char *message) 64 { 65 DWORD error = GetLastError(); 66 RTE_LOG(DEBUG, EAL, "GetLastError()=%lu: %s\n", error, message); 67 68 return thread_translate_win32_error(error); 69 } 70 71 static int 72 thread_map_priority_to_os_value(enum rte_thread_priority eal_pri, int *os_pri, 73 DWORD *pri_class) 74 { 75 /* Clear the output parameters. */ 76 *os_pri = -1; 77 *pri_class = -1; 78 79 switch (eal_pri) { 80 case RTE_THREAD_PRIORITY_NORMAL: 81 *pri_class = NORMAL_PRIORITY_CLASS; 82 *os_pri = THREAD_PRIORITY_NORMAL; 83 break; 84 case RTE_THREAD_PRIORITY_REALTIME_CRITICAL: 85 *pri_class = REALTIME_PRIORITY_CLASS; 86 *os_pri = THREAD_PRIORITY_TIME_CRITICAL; 87 break; 88 default: 89 RTE_LOG(DEBUG, EAL, "The requested priority value is invalid.\n"); 90 return EINVAL; 91 } 92 93 return 0; 94 } 95 96 static int 97 thread_map_os_priority_to_eal_value(int os_pri, DWORD pri_class, 98 enum rte_thread_priority *eal_pri) 99 { 100 switch (pri_class) { 101 case NORMAL_PRIORITY_CLASS: 102 if (os_pri == THREAD_PRIORITY_NORMAL) { 103 *eal_pri = RTE_THREAD_PRIORITY_NORMAL; 104 return 0; 105 } 106 break; 107 case HIGH_PRIORITY_CLASS: 108 RTE_LOG(WARNING, EAL, "The OS priority class is high not real-time.\n"); 109 /* FALLTHROUGH */ 110 case REALTIME_PRIORITY_CLASS: 111 if (os_pri == THREAD_PRIORITY_TIME_CRITICAL) { 112 *eal_pri = RTE_THREAD_PRIORITY_REALTIME_CRITICAL; 113 return 0; 114 } 115 break; 116 default: 117 RTE_LOG(DEBUG, EAL, "The OS priority value does not map to an EAL-defined priority.\n"); 118 return EINVAL; 119 } 120 121 return 0; 122 } 123 124 static int 125 convert_cpuset_to_affinity(const rte_cpuset_t *cpuset, 126 PGROUP_AFFINITY affinity) 127 { 128 int ret = 0; 129 PGROUP_AFFINITY cpu_affinity = NULL; 130 unsigned int cpu_idx; 131 132 memset(affinity, 0, sizeof(GROUP_AFFINITY)); 133 affinity->Group = (USHORT)-1; 134 135 /* Check that all cpus of the set belong to the same processor group and 136 * accumulate thread affinity to be applied. 137 */ 138 for (cpu_idx = 0; cpu_idx < CPU_SETSIZE; cpu_idx++) { 139 if (!CPU_ISSET(cpu_idx, cpuset)) 140 continue; 141 142 cpu_affinity = eal_get_cpu_affinity(cpu_idx); 143 144 if (affinity->Group == (USHORT)-1) { 145 affinity->Group = cpu_affinity->Group; 146 } else if (affinity->Group != cpu_affinity->Group) { 147 RTE_LOG(DEBUG, EAL, "All processors must belong to the same processor group\n"); 148 ret = ENOTSUP; 149 goto cleanup; 150 } 151 152 affinity->Mask |= cpu_affinity->Mask; 153 } 154 155 if (affinity->Mask == 0) { 156 ret = EINVAL; 157 goto cleanup; 158 } 159 160 cleanup: 161 return ret; 162 } 163 164 static DWORD 165 thread_func_wrapper(void *arg) 166 { 167 struct thread_routine_ctx ctx = *(struct thread_routine_ctx *)arg; 168 169 free(arg); 170 171 return (DWORD)ctx.thread_func(ctx.routine_args); 172 } 173 174 int 175 rte_thread_create(rte_thread_t *thread_id, 176 const rte_thread_attr_t *thread_attr, 177 rte_thread_func thread_func, void *args) 178 { 179 int ret = 0; 180 DWORD tid; 181 HANDLE thread_handle = NULL; 182 GROUP_AFFINITY thread_affinity; 183 struct thread_routine_ctx *ctx; 184 185 ctx = calloc(1, sizeof(*ctx)); 186 if (ctx == NULL) { 187 RTE_LOG(DEBUG, EAL, "Insufficient memory for thread context allocations\n"); 188 ret = ENOMEM; 189 goto cleanup; 190 } 191 ctx->routine_args = args; 192 ctx->thread_func = thread_func; 193 194 thread_handle = CreateThread(NULL, 0, thread_func_wrapper, ctx, 195 CREATE_SUSPENDED, &tid); 196 if (thread_handle == NULL) { 197 ret = thread_log_last_error("CreateThread()"); 198 goto cleanup; 199 } 200 thread_id->opaque_id = tid; 201 202 if (thread_attr != NULL) { 203 if (CPU_COUNT(&thread_attr->cpuset) > 0) { 204 ret = convert_cpuset_to_affinity( 205 &thread_attr->cpuset, 206 &thread_affinity 207 ); 208 if (ret != 0) { 209 RTE_LOG(DEBUG, EAL, "Unable to convert cpuset to thread affinity\n"); 210 goto cleanup; 211 } 212 213 if (!SetThreadGroupAffinity(thread_handle, 214 &thread_affinity, NULL)) { 215 ret = thread_log_last_error("SetThreadGroupAffinity()"); 216 goto cleanup; 217 } 218 } 219 ret = rte_thread_set_priority(*thread_id, 220 thread_attr->priority); 221 if (ret != 0) { 222 RTE_LOG(DEBUG, EAL, "Unable to set thread priority\n"); 223 goto cleanup; 224 } 225 } 226 227 if (ResumeThread(thread_handle) == (DWORD)-1) { 228 ret = thread_log_last_error("ResumeThread()"); 229 goto cleanup; 230 } 231 232 ctx = NULL; 233 cleanup: 234 free(ctx); 235 if (thread_handle != NULL) { 236 CloseHandle(thread_handle); 237 thread_handle = NULL; 238 } 239 240 return ret; 241 } 242 243 int 244 rte_thread_join(rte_thread_t thread_id, uint32_t *value_ptr) 245 { 246 HANDLE thread_handle; 247 DWORD result; 248 DWORD exit_code = 0; 249 BOOL err; 250 int ret = 0; 251 252 thread_handle = OpenThread(SYNCHRONIZE | THREAD_QUERY_INFORMATION, 253 FALSE, thread_id.opaque_id); 254 if (thread_handle == NULL) { 255 ret = thread_log_last_error("OpenThread()"); 256 goto cleanup; 257 } 258 259 result = WaitForSingleObject(thread_handle, INFINITE); 260 if (result != WAIT_OBJECT_0) { 261 ret = thread_log_last_error("WaitForSingleObject()"); 262 goto cleanup; 263 } 264 265 if (value_ptr != NULL) { 266 err = GetExitCodeThread(thread_handle, &exit_code); 267 if (err == 0) { 268 ret = thread_log_last_error("GetExitCodeThread()"); 269 goto cleanup; 270 } 271 *value_ptr = exit_code; 272 } 273 274 cleanup: 275 if (thread_handle != NULL) { 276 CloseHandle(thread_handle); 277 thread_handle = NULL; 278 } 279 280 return ret; 281 } 282 283 int 284 rte_thread_detach(rte_thread_t thread_id) 285 { 286 /* No resources that need to be released. */ 287 RTE_SET_USED(thread_id); 288 289 return 0; 290 } 291 292 int 293 rte_thread_equal(rte_thread_t t1, rte_thread_t t2) 294 { 295 return t1.opaque_id == t2.opaque_id; 296 } 297 298 rte_thread_t 299 rte_thread_self(void) 300 { 301 rte_thread_t thread_id; 302 303 thread_id.opaque_id = GetCurrentThreadId(); 304 305 return thread_id; 306 } 307 308 int 309 rte_thread_get_priority(rte_thread_t thread_id, 310 enum rte_thread_priority *priority) 311 { 312 HANDLE thread_handle = NULL; 313 DWORD pri_class; 314 int os_pri; 315 int ret; 316 317 pri_class = GetPriorityClass(GetCurrentProcess()); 318 if (pri_class == 0) { 319 ret = thread_log_last_error("GetPriorityClass()"); 320 goto cleanup; 321 } 322 323 thread_handle = OpenThread(THREAD_SET_INFORMATION | 324 THREAD_QUERY_INFORMATION, FALSE, thread_id.opaque_id); 325 if (thread_handle == NULL) { 326 ret = thread_log_last_error("OpenThread()"); 327 goto cleanup; 328 } 329 330 os_pri = GetThreadPriority(thread_handle); 331 if (os_pri == THREAD_PRIORITY_ERROR_RETURN) { 332 ret = thread_log_last_error("GetThreadPriority()"); 333 goto cleanup; 334 } 335 336 ret = thread_map_os_priority_to_eal_value(os_pri, pri_class, priority); 337 if (ret != 0) 338 goto cleanup; 339 340 cleanup: 341 if (thread_handle != NULL) 342 CloseHandle(thread_handle); 343 344 return ret; 345 } 346 347 int 348 rte_thread_set_priority(rte_thread_t thread_id, 349 enum rte_thread_priority priority) 350 { 351 HANDLE thread_handle; 352 DWORD priority_class; 353 int os_priority; 354 int ret = 0; 355 356 thread_handle = OpenThread(THREAD_SET_INFORMATION | 357 THREAD_QUERY_INFORMATION, FALSE, thread_id.opaque_id); 358 if (thread_handle == NULL) { 359 ret = thread_log_last_error("OpenThread()"); 360 goto cleanup; 361 } 362 363 ret = thread_map_priority_to_os_value(priority, &os_priority, 364 &priority_class); 365 if (ret != 0) 366 goto cleanup; 367 368 if (!SetPriorityClass(GetCurrentProcess(), priority_class)) { 369 ret = thread_log_last_error("SetPriorityClass()"); 370 goto cleanup; 371 } 372 373 if (!SetThreadPriority(thread_handle, os_priority)) { 374 ret = thread_log_last_error("SetThreadPriority()"); 375 goto cleanup; 376 } 377 378 cleanup: 379 if (thread_handle != NULL) 380 CloseHandle(thread_handle); 381 382 return ret; 383 } 384 385 int 386 rte_thread_key_create(rte_thread_key *key, 387 __rte_unused void (*destructor)(void *)) 388 { 389 *key = malloc(sizeof(**key)); 390 if ((*key) == NULL) { 391 RTE_LOG(DEBUG, EAL, "Cannot allocate TLS key.\n"); 392 rte_errno = ENOMEM; 393 return -1; 394 } 395 (*key)->thread_index = TlsAlloc(); 396 if ((*key)->thread_index == TLS_OUT_OF_INDEXES) { 397 RTE_LOG_WIN32_ERR("TlsAlloc()"); 398 free(*key); 399 rte_errno = ENOEXEC; 400 return -1; 401 } 402 return 0; 403 } 404 405 int 406 rte_thread_key_delete(rte_thread_key key) 407 { 408 if (!key) { 409 RTE_LOG(DEBUG, EAL, "Invalid TLS key.\n"); 410 rte_errno = EINVAL; 411 return -1; 412 } 413 if (!TlsFree(key->thread_index)) { 414 RTE_LOG_WIN32_ERR("TlsFree()"); 415 free(key); 416 rte_errno = ENOEXEC; 417 return -1; 418 } 419 free(key); 420 return 0; 421 } 422 423 int 424 rte_thread_value_set(rte_thread_key key, const void *value) 425 { 426 char *p; 427 428 if (!key) { 429 RTE_LOG(DEBUG, EAL, "Invalid TLS key.\n"); 430 rte_errno = EINVAL; 431 return -1; 432 } 433 /* discard const qualifier */ 434 p = (char *) (uintptr_t) value; 435 if (!TlsSetValue(key->thread_index, p)) { 436 RTE_LOG_WIN32_ERR("TlsSetValue()"); 437 rte_errno = ENOEXEC; 438 return -1; 439 } 440 return 0; 441 } 442 443 void * 444 rte_thread_value_get(rte_thread_key key) 445 { 446 void *output; 447 448 if (!key) { 449 RTE_LOG(DEBUG, EAL, "Invalid TLS key.\n"); 450 rte_errno = EINVAL; 451 return NULL; 452 } 453 output = TlsGetValue(key->thread_index); 454 if (GetLastError() != ERROR_SUCCESS) { 455 RTE_LOG_WIN32_ERR("TlsGetValue()"); 456 rte_errno = ENOEXEC; 457 return NULL; 458 } 459 return output; 460 } 461 462 int 463 rte_thread_set_affinity_by_id(rte_thread_t thread_id, 464 const rte_cpuset_t *cpuset) 465 { 466 int ret = 0; 467 GROUP_AFFINITY thread_affinity; 468 HANDLE thread_handle = NULL; 469 470 if (cpuset == NULL) { 471 ret = EINVAL; 472 goto cleanup; 473 } 474 475 ret = convert_cpuset_to_affinity(cpuset, &thread_affinity); 476 if (ret != 0) { 477 RTE_LOG(DEBUG, EAL, "Unable to convert cpuset to thread affinity\n"); 478 goto cleanup; 479 } 480 481 thread_handle = OpenThread(THREAD_ALL_ACCESS, FALSE, 482 thread_id.opaque_id); 483 if (thread_handle == NULL) { 484 ret = thread_log_last_error("OpenThread()"); 485 goto cleanup; 486 } 487 488 if (!SetThreadGroupAffinity(thread_handle, &thread_affinity, NULL)) { 489 ret = thread_log_last_error("SetThreadGroupAffinity()"); 490 goto cleanup; 491 } 492 493 cleanup: 494 if (thread_handle != NULL) { 495 CloseHandle(thread_handle); 496 thread_handle = NULL; 497 } 498 499 return ret; 500 } 501 502 int 503 rte_thread_get_affinity_by_id(rte_thread_t thread_id, 504 rte_cpuset_t *cpuset) 505 { 506 HANDLE thread_handle = NULL; 507 PGROUP_AFFINITY cpu_affinity; 508 GROUP_AFFINITY thread_affinity; 509 unsigned int cpu_idx; 510 int ret = 0; 511 512 if (cpuset == NULL) { 513 ret = EINVAL; 514 goto cleanup; 515 } 516 517 thread_handle = OpenThread(THREAD_ALL_ACCESS, FALSE, 518 thread_id.opaque_id); 519 if (thread_handle == NULL) { 520 ret = thread_log_last_error("OpenThread()"); 521 goto cleanup; 522 } 523 524 /* obtain previous thread affinity */ 525 if (!GetThreadGroupAffinity(thread_handle, &thread_affinity)) { 526 ret = thread_log_last_error("GetThreadGroupAffinity()"); 527 goto cleanup; 528 } 529 530 CPU_ZERO(cpuset); 531 532 /* Convert affinity to DPDK cpu set */ 533 for (cpu_idx = 0; cpu_idx < CPU_SETSIZE; cpu_idx++) { 534 535 cpu_affinity = eal_get_cpu_affinity(cpu_idx); 536 537 if ((cpu_affinity->Group == thread_affinity.Group) && 538 ((cpu_affinity->Mask & thread_affinity.Mask) != 0)) { 539 CPU_SET(cpu_idx, cpuset); 540 } 541 } 542 543 cleanup: 544 if (thread_handle != NULL) { 545 CloseHandle(thread_handle); 546 thread_handle = NULL; 547 } 548 return ret; 549 } 550