1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2021 Mellanox Technologies, Ltd 3 * Copyright (C) 2022 Microsoft Corporation 4 */ 5 6 #include <errno.h> 7 #include <wchar.h> 8 9 #include <rte_eal.h> 10 #include <rte_common.h> 11 #include <rte_errno.h> 12 #include <rte_thread.h> 13 14 #include "eal_windows.h" 15 16 struct eal_tls_key { 17 DWORD thread_index; 18 }; 19 20 struct thread_routine_ctx { 21 rte_thread_func thread_func; 22 void *routine_args; 23 }; 24 25 /* Translates the most common error codes related to threads */ 26 static int 27 thread_translate_win32_error(DWORD error) 28 { 29 switch (error) { 30 case ERROR_SUCCESS: 31 return 0; 32 33 case ERROR_INVALID_PARAMETER: 34 return EINVAL; 35 36 case ERROR_INVALID_HANDLE: 37 return EFAULT; 38 39 case ERROR_NOT_ENOUGH_MEMORY: 40 /* FALLTHROUGH */ 41 case ERROR_NO_SYSTEM_RESOURCES: 42 return ENOMEM; 43 44 case ERROR_PRIVILEGE_NOT_HELD: 45 /* FALLTHROUGH */ 46 case ERROR_ACCESS_DENIED: 47 return EACCES; 48 49 case ERROR_ALREADY_EXISTS: 50 return EEXIST; 51 52 case ERROR_POSSIBLE_DEADLOCK: 53 return EDEADLK; 54 55 case ERROR_INVALID_FUNCTION: 56 /* FALLTHROUGH */ 57 case ERROR_CALL_NOT_IMPLEMENTED: 58 return ENOSYS; 59 } 60 61 return EINVAL; 62 } 63 64 static int 65 thread_log_last_error(const char *message) 66 { 67 DWORD error = GetLastError(); 68 RTE_LOG(DEBUG, EAL, "GetLastError()=%lu: %s\n", error, message); 69 70 return thread_translate_win32_error(error); 71 } 72 73 static int 74 thread_map_priority_to_os_value(enum rte_thread_priority eal_pri, int *os_pri, 75 DWORD *pri_class) 76 { 77 /* Clear the output parameters. */ 78 *os_pri = -1; 79 *pri_class = -1; 80 81 switch (eal_pri) { 82 case RTE_THREAD_PRIORITY_NORMAL: 83 *pri_class = NORMAL_PRIORITY_CLASS; 84 *os_pri = THREAD_PRIORITY_NORMAL; 85 break; 86 case RTE_THREAD_PRIORITY_REALTIME_CRITICAL: 87 *pri_class = REALTIME_PRIORITY_CLASS; 88 *os_pri = THREAD_PRIORITY_TIME_CRITICAL; 89 break; 90 default: 91 RTE_LOG(DEBUG, EAL, "The requested priority value is invalid.\n"); 92 return EINVAL; 93 } 94 95 return 0; 96 } 97 98 static int 99 thread_map_os_priority_to_eal_value(int os_pri, DWORD pri_class, 100 enum rte_thread_priority *eal_pri) 101 { 102 switch (pri_class) { 103 case NORMAL_PRIORITY_CLASS: 104 if (os_pri == THREAD_PRIORITY_NORMAL) { 105 *eal_pri = RTE_THREAD_PRIORITY_NORMAL; 106 return 0; 107 } 108 break; 109 case HIGH_PRIORITY_CLASS: 110 RTE_LOG(WARNING, EAL, "The OS priority class is high not real-time.\n"); 111 /* FALLTHROUGH */ 112 case REALTIME_PRIORITY_CLASS: 113 if (os_pri == THREAD_PRIORITY_TIME_CRITICAL) { 114 *eal_pri = RTE_THREAD_PRIORITY_REALTIME_CRITICAL; 115 return 0; 116 } 117 break; 118 default: 119 RTE_LOG(DEBUG, EAL, "The OS priority value does not map to an EAL-defined priority.\n"); 120 return EINVAL; 121 } 122 123 return 0; 124 } 125 126 static int 127 convert_cpuset_to_affinity(const rte_cpuset_t *cpuset, 128 PGROUP_AFFINITY affinity) 129 { 130 int ret = 0; 131 PGROUP_AFFINITY cpu_affinity = NULL; 132 unsigned int cpu_idx; 133 134 memset(affinity, 0, sizeof(GROUP_AFFINITY)); 135 affinity->Group = (USHORT)-1; 136 137 /* Check that all cpus of the set belong to the same processor group and 138 * accumulate thread affinity to be applied. 139 */ 140 for (cpu_idx = 0; cpu_idx < CPU_SETSIZE; cpu_idx++) { 141 if (!CPU_ISSET(cpu_idx, cpuset)) 142 continue; 143 144 cpu_affinity = eal_get_cpu_affinity(cpu_idx); 145 146 if (affinity->Group == (USHORT)-1) { 147 affinity->Group = cpu_affinity->Group; 148 } else if (affinity->Group != cpu_affinity->Group) { 149 RTE_LOG(DEBUG, EAL, "All processors must belong to the same processor group\n"); 150 ret = ENOTSUP; 151 goto cleanup; 152 } 153 154 affinity->Mask |= cpu_affinity->Mask; 155 } 156 157 if (affinity->Mask == 0) { 158 ret = EINVAL; 159 goto cleanup; 160 } 161 162 cleanup: 163 return ret; 164 } 165 166 static DWORD 167 thread_func_wrapper(void *arg) 168 { 169 struct thread_routine_ctx ctx = *(struct thread_routine_ctx *)arg; 170 171 free(arg); 172 173 return (DWORD)ctx.thread_func(ctx.routine_args); 174 } 175 176 int 177 rte_thread_create(rte_thread_t *thread_id, 178 const rte_thread_attr_t *thread_attr, 179 rte_thread_func thread_func, void *args) 180 { 181 int ret = 0; 182 DWORD tid; 183 HANDLE thread_handle = NULL; 184 GROUP_AFFINITY thread_affinity; 185 struct thread_routine_ctx *ctx; 186 187 ctx = calloc(1, sizeof(*ctx)); 188 if (ctx == NULL) { 189 RTE_LOG(DEBUG, EAL, "Insufficient memory for thread context allocations\n"); 190 ret = ENOMEM; 191 goto cleanup; 192 } 193 ctx->routine_args = args; 194 ctx->thread_func = thread_func; 195 196 thread_handle = CreateThread(NULL, 0, thread_func_wrapper, ctx, 197 CREATE_SUSPENDED, &tid); 198 if (thread_handle == NULL) { 199 ret = thread_log_last_error("CreateThread()"); 200 goto cleanup; 201 } 202 thread_id->opaque_id = tid; 203 204 if (thread_attr != NULL) { 205 if (CPU_COUNT(&thread_attr->cpuset) > 0) { 206 ret = convert_cpuset_to_affinity( 207 &thread_attr->cpuset, 208 &thread_affinity 209 ); 210 if (ret != 0) { 211 RTE_LOG(DEBUG, EAL, "Unable to convert cpuset to thread affinity\n"); 212 goto cleanup; 213 } 214 215 if (!SetThreadGroupAffinity(thread_handle, 216 &thread_affinity, NULL)) { 217 ret = thread_log_last_error("SetThreadGroupAffinity()"); 218 goto cleanup; 219 } 220 } 221 ret = rte_thread_set_priority(*thread_id, 222 thread_attr->priority); 223 if (ret != 0) { 224 RTE_LOG(DEBUG, EAL, "Unable to set thread priority\n"); 225 goto cleanup; 226 } 227 } 228 229 if (ResumeThread(thread_handle) == (DWORD)-1) { 230 ret = thread_log_last_error("ResumeThread()"); 231 goto cleanup; 232 } 233 234 ctx = NULL; 235 cleanup: 236 free(ctx); 237 if (thread_handle != NULL) { 238 CloseHandle(thread_handle); 239 thread_handle = NULL; 240 } 241 242 return ret; 243 } 244 245 int 246 rte_thread_join(rte_thread_t thread_id, uint32_t *value_ptr) 247 { 248 HANDLE thread_handle; 249 DWORD result; 250 DWORD exit_code = 0; 251 BOOL err; 252 int ret = 0; 253 254 thread_handle = OpenThread(SYNCHRONIZE | THREAD_QUERY_INFORMATION, 255 FALSE, thread_id.opaque_id); 256 if (thread_handle == NULL) { 257 ret = thread_log_last_error("OpenThread()"); 258 goto cleanup; 259 } 260 261 result = WaitForSingleObject(thread_handle, INFINITE); 262 if (result != WAIT_OBJECT_0) { 263 ret = thread_log_last_error("WaitForSingleObject()"); 264 goto cleanup; 265 } 266 267 if (value_ptr != NULL) { 268 err = GetExitCodeThread(thread_handle, &exit_code); 269 if (err == 0) { 270 ret = thread_log_last_error("GetExitCodeThread()"); 271 goto cleanup; 272 } 273 *value_ptr = exit_code; 274 } 275 276 cleanup: 277 if (thread_handle != NULL) { 278 CloseHandle(thread_handle); 279 thread_handle = NULL; 280 } 281 282 return ret; 283 } 284 285 int 286 rte_thread_detach(rte_thread_t thread_id) 287 { 288 /* No resources that need to be released. */ 289 RTE_SET_USED(thread_id); 290 291 return 0; 292 } 293 294 int 295 rte_thread_equal(rte_thread_t t1, rte_thread_t t2) 296 { 297 return t1.opaque_id == t2.opaque_id; 298 } 299 300 rte_thread_t 301 rte_thread_self(void) 302 { 303 rte_thread_t thread_id; 304 305 thread_id.opaque_id = GetCurrentThreadId(); 306 307 return thread_id; 308 } 309 310 void 311 rte_thread_set_name(rte_thread_t thread_id, const char *thread_name) 312 { 313 int ret = 0; 314 wchar_t wname[RTE_MAX_THREAD_NAME_LEN]; 315 mbstate_t state = {0}; 316 size_t rv; 317 HANDLE thread_handle; 318 319 thread_handle = OpenThread(THREAD_ALL_ACCESS, FALSE, 320 thread_id.opaque_id); 321 if (thread_handle == NULL) { 322 ret = thread_log_last_error("OpenThread()"); 323 goto cleanup; 324 } 325 326 memset(wname, 0, sizeof(wname)); 327 rv = mbsrtowcs(wname, &thread_name, RTE_DIM(wname) - 1, &state); 328 if (rv == (size_t)-1) { 329 ret = EILSEQ; 330 goto cleanup; 331 } 332 333 #ifndef RTE_TOOLCHAIN_GCC 334 if (FAILED(SetThreadDescription(thread_handle, wname))) { 335 ret = EINVAL; 336 goto cleanup; 337 } 338 #else 339 ret = ENOTSUP; 340 goto cleanup; 341 #endif 342 343 cleanup: 344 if (thread_handle != NULL) 345 CloseHandle(thread_handle); 346 347 if (ret != 0) 348 RTE_LOG(DEBUG, EAL, "Failed to set thread name\n"); 349 } 350 351 int 352 rte_thread_get_priority(rte_thread_t thread_id, 353 enum rte_thread_priority *priority) 354 { 355 HANDLE thread_handle = NULL; 356 DWORD pri_class; 357 int os_pri; 358 int ret; 359 360 pri_class = GetPriorityClass(GetCurrentProcess()); 361 if (pri_class == 0) { 362 ret = thread_log_last_error("GetPriorityClass()"); 363 goto cleanup; 364 } 365 366 thread_handle = OpenThread(THREAD_SET_INFORMATION | 367 THREAD_QUERY_INFORMATION, FALSE, thread_id.opaque_id); 368 if (thread_handle == NULL) { 369 ret = thread_log_last_error("OpenThread()"); 370 goto cleanup; 371 } 372 373 os_pri = GetThreadPriority(thread_handle); 374 if (os_pri == THREAD_PRIORITY_ERROR_RETURN) { 375 ret = thread_log_last_error("GetThreadPriority()"); 376 goto cleanup; 377 } 378 379 ret = thread_map_os_priority_to_eal_value(os_pri, pri_class, priority); 380 if (ret != 0) 381 goto cleanup; 382 383 cleanup: 384 if (thread_handle != NULL) 385 CloseHandle(thread_handle); 386 387 return ret; 388 } 389 390 int 391 rte_thread_set_priority(rte_thread_t thread_id, 392 enum rte_thread_priority priority) 393 { 394 HANDLE thread_handle; 395 DWORD priority_class; 396 int os_priority; 397 int ret = 0; 398 399 thread_handle = OpenThread(THREAD_SET_INFORMATION | 400 THREAD_QUERY_INFORMATION, FALSE, thread_id.opaque_id); 401 if (thread_handle == NULL) { 402 ret = thread_log_last_error("OpenThread()"); 403 goto cleanup; 404 } 405 406 ret = thread_map_priority_to_os_value(priority, &os_priority, 407 &priority_class); 408 if (ret != 0) 409 goto cleanup; 410 411 if (!SetPriorityClass(GetCurrentProcess(), priority_class)) { 412 ret = thread_log_last_error("SetPriorityClass()"); 413 goto cleanup; 414 } 415 416 if (!SetThreadPriority(thread_handle, os_priority)) { 417 ret = thread_log_last_error("SetThreadPriority()"); 418 goto cleanup; 419 } 420 421 cleanup: 422 if (thread_handle != NULL) 423 CloseHandle(thread_handle); 424 425 return ret; 426 } 427 428 int 429 rte_thread_key_create(rte_thread_key *key, 430 __rte_unused void (*destructor)(void *)) 431 { 432 *key = malloc(sizeof(**key)); 433 if ((*key) == NULL) { 434 RTE_LOG(DEBUG, EAL, "Cannot allocate TLS key.\n"); 435 rte_errno = ENOMEM; 436 return -1; 437 } 438 (*key)->thread_index = TlsAlloc(); 439 if ((*key)->thread_index == TLS_OUT_OF_INDEXES) { 440 RTE_LOG_WIN32_ERR("TlsAlloc()"); 441 free(*key); 442 rte_errno = ENOEXEC; 443 return -1; 444 } 445 return 0; 446 } 447 448 int 449 rte_thread_key_delete(rte_thread_key key) 450 { 451 if (!key) { 452 RTE_LOG(DEBUG, EAL, "Invalid TLS key.\n"); 453 rte_errno = EINVAL; 454 return -1; 455 } 456 if (!TlsFree(key->thread_index)) { 457 RTE_LOG_WIN32_ERR("TlsFree()"); 458 free(key); 459 rte_errno = ENOEXEC; 460 return -1; 461 } 462 free(key); 463 return 0; 464 } 465 466 int 467 rte_thread_value_set(rte_thread_key key, const void *value) 468 { 469 char *p; 470 471 if (!key) { 472 RTE_LOG(DEBUG, EAL, "Invalid TLS key.\n"); 473 rte_errno = EINVAL; 474 return -1; 475 } 476 /* discard const qualifier */ 477 p = (char *) (uintptr_t) value; 478 if (!TlsSetValue(key->thread_index, p)) { 479 RTE_LOG_WIN32_ERR("TlsSetValue()"); 480 rte_errno = ENOEXEC; 481 return -1; 482 } 483 return 0; 484 } 485 486 void * 487 rte_thread_value_get(rte_thread_key key) 488 { 489 void *output; 490 491 if (!key) { 492 RTE_LOG(DEBUG, EAL, "Invalid TLS key.\n"); 493 rte_errno = EINVAL; 494 return NULL; 495 } 496 output = TlsGetValue(key->thread_index); 497 if (GetLastError() != ERROR_SUCCESS) { 498 RTE_LOG_WIN32_ERR("TlsGetValue()"); 499 rte_errno = ENOEXEC; 500 return NULL; 501 } 502 return output; 503 } 504 505 int 506 rte_thread_set_affinity_by_id(rte_thread_t thread_id, 507 const rte_cpuset_t *cpuset) 508 { 509 int ret = 0; 510 GROUP_AFFINITY thread_affinity; 511 HANDLE thread_handle = NULL; 512 513 if (cpuset == NULL) { 514 ret = EINVAL; 515 goto cleanup; 516 } 517 518 ret = convert_cpuset_to_affinity(cpuset, &thread_affinity); 519 if (ret != 0) { 520 RTE_LOG(DEBUG, EAL, "Unable to convert cpuset to thread affinity\n"); 521 goto cleanup; 522 } 523 524 thread_handle = OpenThread(THREAD_ALL_ACCESS, FALSE, 525 thread_id.opaque_id); 526 if (thread_handle == NULL) { 527 ret = thread_log_last_error("OpenThread()"); 528 goto cleanup; 529 } 530 531 if (!SetThreadGroupAffinity(thread_handle, &thread_affinity, NULL)) { 532 ret = thread_log_last_error("SetThreadGroupAffinity()"); 533 goto cleanup; 534 } 535 536 cleanup: 537 if (thread_handle != NULL) { 538 CloseHandle(thread_handle); 539 thread_handle = NULL; 540 } 541 542 return ret; 543 } 544 545 int 546 rte_thread_get_affinity_by_id(rte_thread_t thread_id, 547 rte_cpuset_t *cpuset) 548 { 549 HANDLE thread_handle = NULL; 550 PGROUP_AFFINITY cpu_affinity; 551 GROUP_AFFINITY thread_affinity; 552 unsigned int cpu_idx; 553 int ret = 0; 554 555 if (cpuset == NULL) { 556 ret = EINVAL; 557 goto cleanup; 558 } 559 560 thread_handle = OpenThread(THREAD_ALL_ACCESS, FALSE, 561 thread_id.opaque_id); 562 if (thread_handle == NULL) { 563 ret = thread_log_last_error("OpenThread()"); 564 goto cleanup; 565 } 566 567 /* obtain previous thread affinity */ 568 if (!GetThreadGroupAffinity(thread_handle, &thread_affinity)) { 569 ret = thread_log_last_error("GetThreadGroupAffinity()"); 570 goto cleanup; 571 } 572 573 CPU_ZERO(cpuset); 574 575 /* Convert affinity to DPDK cpu set */ 576 for (cpu_idx = 0; cpu_idx < CPU_SETSIZE; cpu_idx++) { 577 578 cpu_affinity = eal_get_cpu_affinity(cpu_idx); 579 580 if ((cpu_affinity->Group == thread_affinity.Group) && 581 ((cpu_affinity->Mask & thread_affinity.Mask) != 0)) { 582 CPU_SET(cpu_idx, cpuset); 583 } 584 } 585 586 cleanup: 587 if (thread_handle != NULL) { 588 CloseHandle(thread_handle); 589 thread_handle = NULL; 590 } 591 return ret; 592 } 593