1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "spdk_cunit.h" 9 #include "common/lib/test_env.c" 10 #include "event/reactor.c" 11 #include "spdk/thread.h" 12 #include "spdk_internal/thread.h" 13 #include "event/scheduler_static.c" 14 #include "../module/scheduler/dynamic/scheduler_dynamic.c" 15 16 struct spdk_thread * 17 _spdk_get_app_thread(void) 18 { 19 struct spdk_lw_thread *lw_thread; 20 struct spdk_thread *thread; 21 22 /* Assume there has to be at least one thread on main 23 * reactor, that has at least one thread. */ 24 lw_thread = TAILQ_FIRST(&g_scheduling_reactor->threads); 25 SPDK_CU_ASSERT_FATAL(lw_thread != NULL); 26 thread = spdk_thread_get_from_ctx(lw_thread); 27 SPDK_CU_ASSERT_FATAL(thread != NULL); 28 29 return thread; 30 } 31 32 static void 33 test_create_reactor(void) 34 { 35 struct spdk_reactor reactor = {}; 36 37 g_reactors = &reactor; 38 g_reactor_count = 1; 39 40 reactor_construct(&reactor, 0); 41 42 CU_ASSERT(spdk_reactor_get(0) == &reactor); 43 44 spdk_ring_free(reactor.events); 45 reactor_interrupt_fini(&reactor); 46 g_reactors = NULL; 47 } 48 49 static void 50 test_init_reactors(void) 51 { 52 uint32_t core; 53 54 MOCK_SET(spdk_env_get_current_core, 0); 55 56 allocate_cores(3); 57 58 CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0); 59 60 CU_ASSERT(g_reactor_state == SPDK_REACTOR_STATE_INITIALIZED); 61 for (core = 0; core < 3; core++) { 62 CU_ASSERT(spdk_reactor_get(core) != NULL); 63 } 64 65 spdk_reactors_fini(); 66 67 free_cores(); 68 69 MOCK_CLEAR(spdk_env_get_current_core); 70 } 71 72 static void 73 ut_event_fn(void *arg1, void *arg2) 74 { 75 uint8_t *test1 = arg1; 76 uint8_t *test2 = arg2; 77 78 *test1 = 1; 79 *test2 = 0xFF; 80 } 81 82 static void 83 test_event_call(void) 84 { 85 uint8_t test1 = 0, test2 = 0; 86 struct spdk_event *evt; 87 struct spdk_reactor *reactor; 88 89 MOCK_SET(spdk_env_get_current_core, 0); 90 91 allocate_cores(1); 92 93 CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0); 94 95 evt = spdk_event_allocate(0, ut_event_fn, &test1, &test2); 96 CU_ASSERT(evt != NULL); 97 98 MOCK_SET(spdk_env_get_current_core, 0); 99 100 spdk_event_call(evt); 101 102 reactor = spdk_reactor_get(0); 103 CU_ASSERT(reactor != NULL); 104 105 CU_ASSERT(event_queue_run_batch(reactor) == 1); 106 CU_ASSERT(test1 == 1); 107 CU_ASSERT(test2 == 0xFF); 108 109 MOCK_CLEAR(spdk_env_get_current_core); 110 111 spdk_reactors_fini(); 112 113 free_cores(); 114 115 MOCK_CLEAR(spdk_env_get_current_core); 116 } 117 118 static void 119 test_schedule_thread(void) 120 { 121 struct spdk_cpuset cpuset = {}; 122 struct spdk_thread *thread; 123 struct spdk_reactor *reactor; 124 struct spdk_lw_thread *lw_thread; 125 126 MOCK_SET(spdk_env_get_current_core, 0); 127 128 allocate_cores(5); 129 130 CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0); 131 132 spdk_cpuset_set_cpu(&cpuset, 3, true); 133 g_next_core = 4; 134 135 MOCK_SET(spdk_env_get_current_core, 3); 136 137 /* _reactor_schedule_thread() will be called in spdk_thread_create() 138 * at its end because it is passed to SPDK thread library by 139 * spdk_thread_lib_init(). 140 */ 141 thread = spdk_thread_create(NULL, &cpuset); 142 CU_ASSERT(thread != NULL); 143 144 reactor = spdk_reactor_get(3); 145 CU_ASSERT(reactor != NULL); 146 147 CU_ASSERT(event_queue_run_batch(reactor) == 1); 148 149 MOCK_CLEAR(spdk_env_get_current_core); 150 151 lw_thread = TAILQ_FIRST(&reactor->threads); 152 CU_ASSERT(lw_thread != NULL); 153 CU_ASSERT(spdk_thread_get_from_ctx(lw_thread) == thread); 154 155 TAILQ_REMOVE(&reactor->threads, lw_thread, link); 156 reactor->thread_count--; 157 spdk_set_thread(thread); 158 spdk_thread_exit(thread); 159 while (!spdk_thread_is_exited(thread)) { 160 spdk_thread_poll(thread, 0, 0); 161 } 162 spdk_thread_destroy(thread); 163 spdk_set_thread(NULL); 164 165 spdk_reactors_fini(); 166 167 free_cores(); 168 } 169 170 static void 171 test_reschedule_thread(void) 172 { 173 struct spdk_cpuset cpuset = {}; 174 struct spdk_thread *thread; 175 struct spdk_reactor *reactor; 176 struct spdk_lw_thread *lw_thread; 177 178 MOCK_SET(spdk_env_get_current_core, 0); 179 180 allocate_cores(3); 181 182 CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0); 183 184 spdk_cpuset_set_cpu(&g_reactor_core_mask, 0, true); 185 spdk_cpuset_set_cpu(&g_reactor_core_mask, 1, true); 186 spdk_cpuset_set_cpu(&g_reactor_core_mask, 2, true); 187 g_next_core = 0; 188 189 MOCK_SET(spdk_env_get_current_core, 1); 190 /* Create and schedule the thread to core 1. */ 191 spdk_cpuset_set_cpu(&cpuset, 1, true); 192 193 thread = spdk_thread_create(NULL, &cpuset); 194 CU_ASSERT(thread != NULL); 195 lw_thread = spdk_thread_get_ctx(thread); 196 197 reactor = spdk_reactor_get(1); 198 CU_ASSERT(reactor != NULL); 199 200 CU_ASSERT(event_queue_run_batch(reactor) == 1); 201 CU_ASSERT(TAILQ_FIRST(&reactor->threads) == lw_thread); 202 203 spdk_set_thread(thread); 204 205 /* Call spdk_thread_set_cpumask() twice with different cpumask values. 206 * The cpumask of the 2nd call will be used in reschedule operation. 207 */ 208 209 spdk_cpuset_zero(&cpuset); 210 spdk_cpuset_set_cpu(&cpuset, 0, true); 211 CU_ASSERT(spdk_thread_set_cpumask(&cpuset) == 0); 212 213 spdk_cpuset_zero(&cpuset); 214 spdk_cpuset_set_cpu(&cpuset, 2, true); 215 CU_ASSERT(spdk_thread_set_cpumask(&cpuset) == 0); 216 217 CU_ASSERT(lw_thread->resched == true); 218 219 reactor_run(reactor); 220 221 CU_ASSERT(lw_thread->resched == false); 222 CU_ASSERT(TAILQ_EMPTY(&reactor->threads)); 223 224 reactor = spdk_reactor_get(0); 225 CU_ASSERT(reactor != NULL); 226 MOCK_SET(spdk_env_get_current_core, 0); 227 228 CU_ASSERT(event_queue_run_batch(reactor) == 0); 229 230 reactor = spdk_reactor_get(2); 231 CU_ASSERT(reactor != NULL); 232 MOCK_SET(spdk_env_get_current_core, 2); 233 234 CU_ASSERT(event_queue_run_batch(reactor) == 1); 235 236 CU_ASSERT(TAILQ_FIRST(&reactor->threads) == lw_thread); 237 238 MOCK_CLEAR(spdk_env_get_current_core); 239 240 TAILQ_REMOVE(&reactor->threads, lw_thread, link); 241 reactor->thread_count--; 242 spdk_set_thread(thread); 243 spdk_thread_exit(thread); 244 while (!spdk_thread_is_exited(thread)) { 245 spdk_thread_poll(thread, 0, 0); 246 } 247 spdk_thread_destroy(thread); 248 spdk_set_thread(NULL); 249 250 spdk_reactors_fini(); 251 252 free_cores(); 253 } 254 255 static void 256 for_each_reactor_done(void *arg1, void *arg2) 257 { 258 uint32_t *count = arg1; 259 bool *done = arg2; 260 261 (*count)++; 262 *done = true; 263 } 264 265 static void 266 for_each_reactor_cb(void *arg1, void *arg2) 267 { 268 uint32_t *count = arg1; 269 270 (*count)++; 271 } 272 273 static void 274 test_for_each_reactor(void) 275 { 276 uint32_t count = 0, i; 277 bool done = false; 278 struct spdk_reactor *reactor; 279 280 MOCK_SET(spdk_env_get_current_core, 0); 281 282 allocate_cores(5); 283 284 CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0); 285 286 spdk_for_each_reactor(for_each_reactor_cb, &count, &done, for_each_reactor_done); 287 288 MOCK_CLEAR(spdk_env_get_current_core); 289 290 /* We have not processed any event yet, so count and done should be 0 and false, 291 * respectively. 292 */ 293 CU_ASSERT(count == 0); 294 295 /* Poll each reactor to verify the event is passed to each */ 296 for (i = 0; i < 5; i++) { 297 reactor = spdk_reactor_get(i); 298 CU_ASSERT(reactor != NULL); 299 MOCK_SET(spdk_env_get_current_core, i); 300 301 event_queue_run_batch(reactor); 302 CU_ASSERT(count == (i + 1)); 303 CU_ASSERT(done == false); 304 MOCK_CLEAR(spdk_env_get_current_core); 305 } 306 307 MOCK_SET(spdk_env_get_current_core, 0); 308 /* After each reactor is called, the completion calls it one more time. */ 309 reactor = spdk_reactor_get(0); 310 CU_ASSERT(reactor != NULL); 311 312 event_queue_run_batch(reactor); 313 CU_ASSERT(count == 6); 314 CU_ASSERT(done == true); 315 MOCK_CLEAR(spdk_env_get_current_core); 316 317 spdk_reactors_fini(); 318 319 free_cores(); 320 } 321 322 static int 323 poller_run_idle(void *ctx) 324 { 325 uint64_t delay_us = (uint64_t)ctx; 326 327 spdk_delay_us(delay_us); 328 329 return 0; 330 } 331 332 static int 333 poller_run_busy(void *ctx) 334 { 335 uint64_t delay_us = (uint64_t)ctx; 336 337 spdk_delay_us(delay_us); 338 339 return 1; 340 } 341 342 static void 343 test_reactor_stats(void) 344 { 345 struct spdk_cpuset cpuset = {}; 346 struct spdk_thread *thread1, *thread2; 347 struct spdk_reactor *reactor; 348 struct spdk_poller *busy1, *idle1, *busy2, *idle2; 349 struct spdk_thread_stats stats; 350 int rc __attribute__((unused)); 351 352 /* Test case is the following: 353 * Create a reactor on CPU core0. 354 * Create thread1 and thread2 simultaneously on reactor0 at TSC = 100. 355 * Reactor runs 356 * - thread1 for 100 with busy 357 * - thread2 for 200 with idle 358 * - thread1 for 300 with idle 359 * - thread2 for 400 with busy. 360 * Then, 361 * - both elapsed TSC of thread1 and thread2 should be 1100 (= 100 + 1000). 362 * - busy TSC of reactor should be 500 (= 100 + 400). 363 * - idle TSC of reactor should be 500 (= 200 + 300). 364 * 365 * After that reactor0 runs with no threads for 900 TSC. 366 * Create thread1 on reactor0 at TSC = 2000. 367 * Reactor runs 368 * - thread1 for 100 with busy 369 * Then, 370 * - elapsed TSC of thread1 should be 2100 (= 2000+ 100). 371 * - busy TSC of reactor should be 600 (= 500 + 100). 372 * - idle TSC of reactor should be 500 (= 500 + 900). 373 */ 374 375 MOCK_SET(spdk_env_get_current_core, 0); 376 377 allocate_cores(1); 378 379 CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0); 380 381 spdk_cpuset_set_cpu(&cpuset, 0, true); 382 383 reactor = spdk_reactor_get(0); 384 SPDK_CU_ASSERT_FATAL(reactor != NULL); 385 386 /* First reactor_run() sets the tsc_last. */ 387 MOCK_SET(spdk_get_ticks, 100); 388 reactor->tsc_last = spdk_get_ticks(); 389 390 thread1 = spdk_thread_create(NULL, &cpuset); 391 SPDK_CU_ASSERT_FATAL(thread1 != NULL); 392 393 thread2 = spdk_thread_create(NULL, &cpuset); 394 SPDK_CU_ASSERT_FATAL(thread2 != NULL); 395 396 spdk_set_thread(thread1); 397 busy1 = spdk_poller_register(poller_run_busy, (void *)100, 0); 398 CU_ASSERT(busy1 != NULL); 399 400 spdk_set_thread(thread2); 401 idle2 = spdk_poller_register(poller_run_idle, (void *)300, 0); 402 CU_ASSERT(idle2 != NULL); 403 404 _reactor_run(reactor); 405 406 spdk_set_thread(thread1); 407 CU_ASSERT(spdk_thread_get_last_tsc(thread1) == 200); 408 CU_ASSERT(spdk_thread_get_stats(&stats) == 0); 409 CU_ASSERT(stats.busy_tsc == 100); 410 CU_ASSERT(stats.idle_tsc == 0); 411 spdk_set_thread(thread2); 412 CU_ASSERT(spdk_thread_get_last_tsc(thread2) == 500); 413 CU_ASSERT(spdk_thread_get_stats(&stats) == 0); 414 CU_ASSERT(stats.busy_tsc == 0); 415 CU_ASSERT(stats.idle_tsc == 300); 416 417 CU_ASSERT(reactor->busy_tsc == 100); 418 CU_ASSERT(reactor->idle_tsc == 300); 419 420 /* 100 + 100 + 300 = 500 ticks elapsed */ 421 CU_ASSERT(reactor->tsc_last == 500); 422 423 spdk_set_thread(thread1); 424 spdk_poller_unregister(&busy1); 425 idle1 = spdk_poller_register(poller_run_idle, (void *)200, 0); 426 CU_ASSERT(idle1 != NULL); 427 428 spdk_set_thread(thread2); 429 spdk_poller_unregister(&idle2); 430 busy2 = spdk_poller_register(poller_run_busy, (void *)400, 0); 431 CU_ASSERT(busy2 != NULL); 432 433 _reactor_run(reactor); 434 435 spdk_set_thread(thread1); 436 CU_ASSERT(spdk_thread_get_last_tsc(thread1) == 700); 437 CU_ASSERT(spdk_thread_get_stats(&stats) == 0); 438 CU_ASSERT(stats.busy_tsc == 100); 439 CU_ASSERT(stats.idle_tsc == 200); 440 spdk_set_thread(thread2); 441 CU_ASSERT(spdk_thread_get_last_tsc(thread2) == 1100); 442 CU_ASSERT(spdk_thread_get_stats(&stats) == 0); 443 CU_ASSERT(stats.busy_tsc == 400); 444 CU_ASSERT(stats.idle_tsc == 300); 445 446 CU_ASSERT(reactor->busy_tsc == 500); 447 CU_ASSERT(reactor->idle_tsc == 500); 448 449 /* 500 + 200 + 400 = 1100 ticks elapsed */ 450 CU_ASSERT(reactor->tsc_last == 1100); 451 452 spdk_set_thread(thread1); 453 spdk_poller_unregister(&idle1); 454 spdk_thread_exit(thread1); 455 456 spdk_set_thread(thread2); 457 spdk_poller_unregister(&busy2); 458 spdk_thread_exit(thread2); 459 460 _reactor_run(reactor); 461 462 /* After 900 ticks new thread is created. */ 463 /* 1100 + 900 = 2000 ticks elapsed */ 464 MOCK_SET(spdk_get_ticks, 2000); 465 _reactor_run(reactor); 466 CU_ASSERT(reactor->tsc_last == 2000); 467 468 thread1 = spdk_thread_create(NULL, &cpuset); 469 SPDK_CU_ASSERT_FATAL(thread1 != NULL); 470 471 spdk_set_thread(thread1); 472 busy1 = spdk_poller_register(poller_run_busy, (void *)100, 0); 473 CU_ASSERT(busy1 != NULL); 474 475 _reactor_run(reactor); 476 477 spdk_set_thread(thread1); 478 CU_ASSERT(spdk_thread_get_last_tsc(thread1) == 2100); 479 CU_ASSERT(spdk_thread_get_stats(&stats) == 0); 480 CU_ASSERT(stats.busy_tsc == 100); 481 CU_ASSERT(stats.idle_tsc == 0); 482 483 CU_ASSERT(reactor->busy_tsc == 600); 484 CU_ASSERT(reactor->idle_tsc == 1400); 485 486 /* 2000 + 100 = 2100 ticks elapsed */ 487 CU_ASSERT(reactor->tsc_last == 2100); 488 489 spdk_set_thread(thread1); 490 spdk_poller_unregister(&busy1); 491 spdk_thread_exit(thread1); 492 493 _reactor_run(reactor); 494 495 CU_ASSERT(TAILQ_EMPTY(&reactor->threads)); 496 497 /* No further than 2100 ticks elapsed */ 498 CU_ASSERT(reactor->tsc_last == 2100); 499 500 spdk_reactors_fini(); 501 502 free_cores(); 503 504 MOCK_CLEAR(spdk_env_get_current_core); 505 } 506 507 static uint32_t 508 _run_events_till_completion(uint32_t reactor_count) 509 { 510 struct spdk_reactor *reactor; 511 struct spdk_thread *app_thread = _spdk_get_app_thread(); 512 uint32_t i, events; 513 uint32_t total_events = 0; 514 515 do { 516 events = 0; 517 for (i = 0; i < reactor_count; i++) { 518 reactor = spdk_reactor_get(i); 519 CU_ASSERT(reactor != NULL); 520 MOCK_SET(spdk_env_get_current_core, i); 521 events += event_queue_run_batch(reactor); 522 523 /* Some events still require app_thread to run */ 524 MOCK_SET(spdk_env_get_current_core, g_scheduling_reactor->lcore); 525 spdk_thread_poll(app_thread, 0, 0); 526 527 MOCK_CLEAR(spdk_env_get_current_core); 528 } 529 total_events += events; 530 } while (events > 0); 531 532 return total_events; 533 } 534 535 static void 536 test_scheduler(void) 537 { 538 struct spdk_cpuset cpuset = {}; 539 struct spdk_thread *thread[3]; 540 struct spdk_reactor *reactor; 541 struct spdk_poller *busy, *idle; 542 uint64_t reactor_busy_tsc[3], reactor_idle_tsc[3]; 543 uint64_t thread_busy_tsc[3], thread_idle_tsc[3]; 544 uint64_t current_time, end_time, busy_time, idle_time; 545 struct spdk_thread_stats stats; 546 int i; 547 548 MOCK_SET(spdk_env_get_current_core, 0); 549 550 allocate_cores(3); 551 552 CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0); 553 554 spdk_scheduler_set("dynamic"); 555 556 for (i = 0; i < 3; i++) { 557 spdk_cpuset_set_cpu(&g_reactor_core_mask, i, true); 558 } 559 g_next_core = 0; 560 561 /* Create threads. */ 562 for (i = 0; i < 3; i++) { 563 spdk_cpuset_zero(&cpuset); 564 spdk_cpuset_set_cpu(&cpuset, i, true); 565 thread[i] = spdk_thread_create(NULL, &cpuset); 566 CU_ASSERT(thread[i] != NULL); 567 thread_busy_tsc[i] = 0; 568 thread_idle_tsc[i] = 0; 569 } 570 571 for (i = 0; i < 3; i++) { 572 reactor = spdk_reactor_get(i); 573 CU_ASSERT(reactor != NULL); 574 MOCK_SET(spdk_env_get_current_core, i); 575 event_queue_run_batch(reactor); 576 CU_ASSERT(!TAILQ_EMPTY(&reactor->threads)); 577 reactor_busy_tsc[i] = 0; 578 reactor_idle_tsc[i] = 0; 579 } 580 581 g_reactor_state = SPDK_REACTOR_STATE_RUNNING; 582 583 MOCK_SET(spdk_env_get_current_core, 0); 584 585 /* Init threads stats (low load) */ 586 /* Each reactor starts at 100 tsc, 587 * ends at 100 + 100 = 200 tsc. */ 588 current_time = 100; 589 idle_time = 100; 590 busy_time = 0; 591 end_time = current_time + idle_time + busy_time; 592 for (i = 0; i < 3; i++) { 593 spdk_set_thread(thread[i]); 594 idle = spdk_poller_register(poller_run_idle, (void *)idle_time, 0); 595 reactor = spdk_reactor_get(i); 596 CU_ASSERT(reactor != NULL); 597 MOCK_SET(spdk_get_ticks, current_time); 598 reactor->tsc_last = spdk_get_ticks(); 599 _reactor_run(reactor); 600 CU_ASSERT(reactor->tsc_last == end_time); 601 spdk_poller_unregister(&idle); 602 603 CU_ASSERT(spdk_thread_get_last_tsc(thread[i]) == end_time); 604 CU_ASSERT(spdk_thread_get_stats(&stats) == 0); 605 CU_ASSERT(stats.busy_tsc == busy_time); 606 thread_busy_tsc[i] = stats.busy_tsc; 607 CU_ASSERT(stats.idle_tsc == idle_time); 608 thread_idle_tsc[i] = stats.idle_tsc; 609 CU_ASSERT(reactor->busy_tsc == busy_time); 610 reactor_busy_tsc[i] = reactor->busy_tsc; 611 CU_ASSERT(reactor->idle_tsc == idle_time); 612 reactor_idle_tsc[i] = reactor->idle_tsc; 613 } 614 CU_ASSERT(spdk_get_ticks() == end_time); 615 current_time = 200; 616 617 MOCK_SET(spdk_env_get_current_core, 0); 618 _reactors_scheduler_gather_metrics(NULL, NULL); 619 620 _run_events_till_completion(3); 621 MOCK_SET(spdk_env_get_current_core, 0); 622 623 /* Threads were idle, so all of them should be placed on core 0. 624 * All reactors start and end at 200 tsc, since for this iteration 625 * the threads have no pollers (so they consume no idle or busy tsc). 626 */ 627 for (i = 0; i < 3; i++) { 628 reactor = spdk_reactor_get(i); 629 CU_ASSERT(reactor != NULL); 630 MOCK_SET(spdk_get_ticks, current_time); 631 _reactor_run(reactor); 632 CU_ASSERT(reactor->tsc_last == current_time); 633 CU_ASSERT(reactor->busy_tsc == reactor_busy_tsc[i]); 634 CU_ASSERT(reactor->idle_tsc == reactor_idle_tsc[i]); 635 spdk_set_thread(thread[i]); 636 CU_ASSERT(spdk_thread_get_last_tsc(thread[i]) == current_time); 637 CU_ASSERT(spdk_thread_get_stats(&stats) == 0); 638 CU_ASSERT(stats.busy_tsc == thread_busy_tsc[i]); 639 CU_ASSERT(stats.idle_tsc == thread_idle_tsc[i]); 640 } 641 CU_ASSERT(spdk_get_ticks() == current_time); 642 643 /* 2 threads should be scheduled to core 0 */ 644 reactor = spdk_reactor_get(0); 645 CU_ASSERT(reactor != NULL); 646 MOCK_SET(spdk_env_get_current_core, 0); 647 event_queue_run_batch(reactor); 648 649 reactor = spdk_reactor_get(0); 650 CU_ASSERT(reactor != NULL); 651 CU_ASSERT(!TAILQ_EMPTY(&reactor->threads)); 652 reactor = spdk_reactor_get(1); 653 CU_ASSERT(reactor != NULL); 654 CU_ASSERT(TAILQ_EMPTY(&reactor->threads)); 655 reactor = spdk_reactor_get(2); 656 CU_ASSERT(reactor != NULL); 657 CU_ASSERT(TAILQ_EMPTY(&reactor->threads)); 658 659 /* Make threads busy */ 660 reactor = spdk_reactor_get(0); 661 CU_ASSERT(reactor != NULL); 662 663 /* All threads run on single reactor, 664 * reactor 0 starts at 200 tsc, 665 * ending at 200 + (100 * 3) = 500 tsc. */ 666 MOCK_SET(spdk_get_ticks, current_time); 667 busy_time = 100; 668 idle_time = 0; 669 for (i = 0; i < 3; i++) { 670 spdk_set_thread(thread[i]); 671 busy = spdk_poller_register(poller_run_busy, (void *)busy_time, 0); 672 _reactor_run(reactor); 673 spdk_poller_unregister(&busy); 674 current_time += busy_time; 675 676 CU_ASSERT(reactor->tsc_last == current_time); 677 CU_ASSERT(spdk_thread_get_last_tsc(thread[i]) == current_time); 678 CU_ASSERT(spdk_thread_get_stats(&stats) == 0); 679 CU_ASSERT(stats.busy_tsc == thread_busy_tsc[i] + busy_time); 680 CU_ASSERT(stats.idle_tsc == thread_idle_tsc[i] + idle_time);; 681 } 682 CU_ASSERT(reactor->busy_tsc == reactor_busy_tsc[0] + 3 * busy_time); 683 CU_ASSERT(reactor->idle_tsc == reactor_idle_tsc[0] + 3 * idle_time); 684 CU_ASSERT(spdk_get_ticks() == current_time); 685 686 /* Run scheduler again, this time all threads are busy */ 687 MOCK_SET(spdk_env_get_current_core, 0); 688 _reactors_scheduler_gather_metrics(NULL, NULL); 689 690 _run_events_till_completion(3); 691 MOCK_SET(spdk_env_get_current_core, 0); 692 693 /* Threads were busy, 2 will stay on core 0, 1 will move to core 1 */ 694 for (i = 0; i < 3; i++) { 695 MOCK_SET(spdk_env_get_current_core, i); 696 reactor = spdk_reactor_get(i); 697 CU_ASSERT(reactor != NULL); 698 _reactor_run(reactor); 699 } 700 701 for (i = 0; i < 3; i++) { 702 reactor = spdk_reactor_get(i); 703 CU_ASSERT(reactor != NULL); 704 CU_ASSERT(!TAILQ_EMPTY(&reactor->threads)); 705 } 706 707 g_reactor_state = SPDK_REACTOR_STATE_INITIALIZED; 708 709 /* Destroy threads */ 710 for (i = 0; i < 3; i++) { 711 spdk_set_thread(thread[i]); 712 spdk_thread_exit(thread[i]); 713 } 714 for (i = 0; i < 3; i++) { 715 reactor = spdk_reactor_get(i); 716 CU_ASSERT(reactor != NULL); 717 reactor_run(reactor); 718 } 719 720 spdk_set_thread(NULL); 721 722 MOCK_CLEAR(spdk_env_get_current_core); 723 724 spdk_reactors_fini(); 725 726 free_cores(); 727 } 728 729 uint8_t g_curr_freq; 730 731 static int 732 core_freq_up(uint32_t lcore) 733 { 734 if (g_curr_freq != UINT8_MAX) { 735 g_curr_freq++; 736 } 737 738 return 0; 739 } 740 741 static int 742 core_freq_down(uint32_t lcore) 743 { 744 if (g_curr_freq != 0) { 745 g_curr_freq--; 746 } 747 748 return 0; 749 } 750 751 static int 752 core_freq_max(uint32_t lcore) 753 { 754 g_curr_freq = UINT8_MAX; 755 756 return 0; 757 } 758 759 DEFINE_STUB(core_freq_min, int, (uint32_t lcore_id), 0); 760 DEFINE_STUB(core_caps, int, 761 (uint32_t lcore_id, struct spdk_governor_capabilities *capabilities), 0); 762 DEFINE_STUB(governor_init, int, (void), 0); 763 DEFINE_STUB_V(governor_deinit, (void)); 764 765 static struct spdk_governor governor = { 766 .name = "dpdk_governor", 767 .get_core_curr_freq = NULL, 768 .core_freq_up = core_freq_up, 769 .core_freq_down = core_freq_down, 770 .set_core_freq_max = core_freq_max, 771 .set_core_freq_min = core_freq_min, 772 .get_core_capabilities = core_caps, 773 .init = governor_init, 774 .deinit = governor_deinit, 775 }; 776 777 static void 778 test_governor(void) 779 { 780 struct spdk_cpuset cpuset = {}; 781 struct spdk_thread *thread[2]; 782 struct spdk_lw_thread *lw_thread; 783 struct spdk_reactor *reactor; 784 struct spdk_poller *busy, *idle; 785 uint8_t last_freq = 100; 786 int i; 787 788 MOCK_SET(spdk_env_get_current_core, 0); 789 790 g_curr_freq = last_freq; 791 spdk_governor_register(&governor); 792 793 allocate_cores(2); 794 795 CU_ASSERT(spdk_reactors_init(SPDK_DEFAULT_MSG_MEMPOOL_SIZE) == 0); 796 797 spdk_scheduler_set("dynamic"); 798 spdk_governor_set("dpdk_governor"); 799 800 for (i = 0; i < 2; i++) { 801 spdk_cpuset_set_cpu(&g_reactor_core_mask, i, true); 802 } 803 804 /* Create threads. */ 805 for (i = 0; i < 2; i++) { 806 spdk_cpuset_zero(&cpuset); 807 spdk_cpuset_set_cpu(&cpuset, i, true); 808 thread[i] = spdk_thread_create(NULL, &cpuset); 809 CU_ASSERT(thread[i] != NULL); 810 } 811 812 for (i = 0; i < 2; i++) { 813 reactor = spdk_reactor_get(i); 814 CU_ASSERT(reactor != NULL); 815 MOCK_SET(spdk_env_get_current_core, i); 816 CU_ASSERT(event_queue_run_batch(reactor) == 1); 817 CU_ASSERT(!TAILQ_EMPTY(&reactor->threads)); 818 } 819 820 reactor = spdk_reactor_get(0); 821 CU_ASSERT(reactor != NULL); 822 MOCK_SET(spdk_env_get_current_core, 0); 823 824 g_reactor_state = SPDK_REACTOR_STATE_RUNNING; 825 826 /* TEST 1 */ 827 /* Init thread stats (low load) */ 828 MOCK_SET(spdk_get_ticks, 100); 829 reactor->tsc_last = 100; 830 831 for (i = 0; i < 2; i++) { 832 spdk_set_thread(thread[i]); 833 idle = spdk_poller_register(poller_run_idle, (void *)200, 0); 834 reactor = spdk_reactor_get(i); 835 CU_ASSERT(reactor != NULL); 836 MOCK_SET(spdk_env_get_current_core, i); 837 _reactor_run(reactor); 838 spdk_poller_unregister(&idle); 839 840 /* Update last stats so that we don't have to call scheduler twice */ 841 lw_thread = spdk_thread_get_ctx(thread[i]); 842 lw_thread->current_stats.idle_tsc = 1; 843 } 844 845 MOCK_SET(spdk_env_get_current_core, 0); 846 _reactors_scheduler_gather_metrics(NULL, NULL); 847 848 CU_ASSERT(_run_events_till_completion(2) == 2); 849 MOCK_SET(spdk_env_get_current_core, 0); 850 851 /* Threads were idle, so all of them should be placed on core 0 */ 852 for (i = 0; i < 2; i++) { 853 reactor = spdk_reactor_get(i); 854 CU_ASSERT(reactor != NULL); 855 _reactor_run(reactor); 856 } 857 858 /* 1 thread should be scheduled to core 0 */ 859 reactor = spdk_reactor_get(0); 860 CU_ASSERT(reactor != NULL); 861 MOCK_SET(spdk_env_get_current_core, 0); 862 CU_ASSERT(event_queue_run_batch(reactor) == 1); 863 864 /* Main core should be busy less than 50% time now - frequency should be lowered */ 865 CU_ASSERT(g_curr_freq == last_freq - 1); 866 867 last_freq = g_curr_freq; 868 869 /* TEST 2 */ 870 /* Make first threads busy - both threads will be still on core 0, but frequency will have to be raised */ 871 spdk_set_thread(thread[0]); 872 busy = spdk_poller_register(poller_run_busy, (void *)1000, 0); 873 _reactor_run(reactor); 874 spdk_poller_unregister(&busy); 875 876 spdk_set_thread(thread[1]); 877 idle = spdk_poller_register(poller_run_idle, (void *)100, 0); 878 _reactor_run(reactor); 879 spdk_poller_unregister(&idle); 880 881 /* Run scheduler again */ 882 MOCK_SET(spdk_env_get_current_core, 0); 883 _reactors_scheduler_gather_metrics(NULL, NULL); 884 885 i = _run_events_till_completion(2); 886 /* Six runs when interrupt mode is supported, two if not. */ 887 CU_ASSERT(i == 6 || i == 2); 888 MOCK_SET(spdk_env_get_current_core, 0); 889 890 /* Main core should be busy more than 50% time now - frequency should be raised */ 891 CU_ASSERT(g_curr_freq == last_freq + 1); 892 893 /* TEST 3 */ 894 /* Make second thread very busy so that it will be moved to second core */ 895 spdk_set_thread(thread[1]); 896 busy = spdk_poller_register(poller_run_busy, (void *)2000, 0); 897 _reactor_run(reactor); 898 spdk_poller_unregister(&busy); 899 900 /* Update first thread stats */ 901 spdk_set_thread(thread[0]); 902 idle = spdk_poller_register(poller_run_idle, (void *)100, 0); 903 _reactor_run(reactor); 904 spdk_poller_unregister(&idle); 905 906 /* Run scheduler again */ 907 MOCK_SET(spdk_env_get_current_core, 0); 908 _reactors_scheduler_gather_metrics(NULL, NULL); 909 910 i = _run_events_till_completion(2); 911 /* Six runs when interrupt mode is supported, two if not. */ 912 CU_ASSERT(i == 6 || i == 2); 913 MOCK_SET(spdk_env_get_current_core, 0); 914 915 for (i = 0; i < 2; i++) { 916 reactor = spdk_reactor_get(i); 917 CU_ASSERT(reactor != NULL); 918 _reactor_run(reactor); 919 } 920 921 /* Main core frequency should be set to max when we have busy threads on other cores */ 922 CU_ASSERT(g_curr_freq == UINT8_MAX); 923 924 g_reactor_state = SPDK_REACTOR_STATE_INITIALIZED; 925 926 /* Destroy threads */ 927 for (i = 0; i < 2; i++) { 928 spdk_set_thread(thread[i]); 929 spdk_thread_exit(thread[i]); 930 } 931 for (i = 0; i < 2; i++) { 932 reactor = spdk_reactor_get(i); 933 CU_ASSERT(reactor != NULL); 934 reactor_run(reactor); 935 } 936 937 spdk_set_thread(NULL); 938 939 MOCK_CLEAR(spdk_env_get_current_core); 940 941 spdk_reactors_fini(); 942 943 free_cores(); 944 } 945 946 int 947 main(int argc, char **argv) 948 { 949 CU_pSuite suite = NULL; 950 unsigned int num_failures; 951 952 CU_set_error_action(CUEA_ABORT); 953 CU_initialize_registry(); 954 955 suite = CU_add_suite("app_suite", NULL, NULL); 956 957 CU_ADD_TEST(suite, test_create_reactor); 958 CU_ADD_TEST(suite, test_init_reactors); 959 CU_ADD_TEST(suite, test_event_call); 960 CU_ADD_TEST(suite, test_schedule_thread); 961 CU_ADD_TEST(suite, test_reschedule_thread); 962 CU_ADD_TEST(suite, test_for_each_reactor); 963 CU_ADD_TEST(suite, test_reactor_stats); 964 CU_ADD_TEST(suite, test_scheduler); 965 CU_ADD_TEST(suite, test_governor); 966 967 CU_basic_set_mode(CU_BRM_VERBOSE); 968 CU_basic_run_tests(); 969 num_failures = CU_get_number_of_failures(); 970 CU_cleanup_registry(); 971 972 return num_failures; 973 } 974