1 /* 2 * Copyright (c) 2012-2017 The DragonFly Project. All rights reserved. 3 * Copyright (c) 1999 Peter Wemm <peter@FreeBSD.org>. All rights reserved. 4 * 5 * This code is derived from software contributed to The DragonFly Project 6 * by Matthew Dillon <dillon@backplane.com>, 7 * by Mihai Carabas <mihai.carabas@gmail.com> 8 * and many others. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/kernel.h> 40 #include <sys/lock.h> 41 #include <sys/queue.h> 42 #include <sys/proc.h> 43 #include <sys/rtprio.h> 44 #include <sys/uio.h> 45 #include <sys/sysctl.h> 46 #include <sys/resourcevar.h> 47 #include <sys/spinlock.h> 48 #include <sys/cpu_topology.h> 49 #include <sys/thread2.h> 50 #include <sys/spinlock2.h> 51 52 #include <sys/ktr.h> 53 54 #include <machine/cpu.h> 55 #include <machine/smp.h> 56 57 /* 58 * Priorities. Note that with 32 run queues per scheduler each queue 59 * represents four priority levels. 60 */ 61 62 int dfly_rebalanced; 63 64 #define MAXPRI 128 65 #define PRIMASK (MAXPRI - 1) 66 #define PRIBASE_REALTIME 0 67 #define PRIBASE_NORMAL MAXPRI 68 #define PRIBASE_IDLE (MAXPRI * 2) 69 #define PRIBASE_THREAD (MAXPRI * 3) 70 #define PRIBASE_NULL (MAXPRI * 4) 71 72 #define NQS 32 /* 32 run queues. */ 73 #define PPQ (MAXPRI / NQS) /* priorities per queue */ 74 #define PPQMASK (PPQ - 1) 75 76 /* 77 * NICE_QS - maximum queues nice can shift the process 78 * EST_QS - maximum queues estcpu can shift the process 79 * 80 * ESTCPUPPQ - number of estcpu units per priority queue 81 * ESTCPUMAX - number of estcpu units 82 * 83 * Remember that NICE runs over the whole -20 to +20 range. 84 */ 85 #define NICE_QS 24 /* -20 to +20 shift in whole queues */ 86 #define EST_QS 20 /* 0-MAX shift in whole queues */ 87 #define ESTCPUPPQ 512 88 #define ESTCPUMAX (ESTCPUPPQ * EST_QS) 89 #define PRIO_RANGE (PRIO_MAX - PRIO_MIN + 1) 90 91 #define ESTCPULIM(v) min((v), ESTCPUMAX) 92 93 TAILQ_HEAD(rq, lwp); 94 95 #define lwp_priority lwp_usdata.dfly.priority 96 #define lwp_forked lwp_usdata.dfly.forked 97 #define lwp_rqindex lwp_usdata.dfly.rqindex 98 #define lwp_estcpu lwp_usdata.dfly.estcpu 99 #define lwp_estfast lwp_usdata.dfly.estfast 100 #define lwp_uload lwp_usdata.dfly.uload 101 #define lwp_rqtype lwp_usdata.dfly.rqtype 102 #define lwp_qcpu lwp_usdata.dfly.qcpu 103 #define lwp_rrcount lwp_usdata.dfly.rrcount 104 105 static __inline int 106 lptouload(struct lwp *lp) 107 { 108 int uload; 109 110 uload = lp->lwp_estcpu / NQS; 111 uload -= uload * lp->lwp_proc->p_nice / (PRIO_MAX + 1); 112 113 return uload; 114 } 115 116 /* 117 * DFly scheduler pcpu structure. Note that the pcpu uload field must 118 * be 64-bits to avoid overflowing in the situation where more than 32768 119 * processes are on a single cpu's queue. Since high-end systems can 120 * easily run 900,000+ processes, we have to deal with it. 121 */ 122 struct usched_dfly_pcpu { 123 struct spinlock spin; 124 struct thread *helper_thread; 125 struct globaldata *gd; 126 u_short scancpu; 127 short upri; 128 long uload; /* 64-bits to avoid overflow (1) */ 129 int ucount; 130 int flags; 131 struct lwp *uschedcp; 132 struct rq queues[NQS]; 133 struct rq rtqueues[NQS]; 134 struct rq idqueues[NQS]; 135 u_int32_t queuebits; 136 u_int32_t rtqueuebits; 137 u_int32_t idqueuebits; 138 int runqcount; 139 int cpuid; 140 cpumask_t cpumask; 141 cpu_node_t *cpunode; 142 } __cachealign; 143 144 /* 145 * Reflecting bits in the global atomic masks allows us to avoid 146 * a certain degree of global ping-ponging. 147 */ 148 #define DFLY_PCPU_RDYMASK 0x0001 /* reflect rdyprocmask */ 149 #define DFLY_PCPU_CURMASK 0x0002 /* reflect curprocmask */ 150 151 typedef struct usched_dfly_pcpu *dfly_pcpu_t; 152 153 static void dfly_acquire_curproc(struct lwp *lp); 154 static void dfly_release_curproc(struct lwp *lp); 155 static void dfly_select_curproc(globaldata_t gd); 156 static void dfly_setrunqueue(struct lwp *lp); 157 static void dfly_setrunqueue_dd(dfly_pcpu_t rdd, struct lwp *lp); 158 static void dfly_schedulerclock(struct lwp *lp, sysclock_t period, 159 sysclock_t cpstamp); 160 static void dfly_recalculate_estcpu(struct lwp *lp); 161 static void dfly_resetpriority(struct lwp *lp); 162 static void dfly_forking(struct lwp *plp, struct lwp *lp); 163 static void dfly_exiting(struct lwp *lp, struct proc *); 164 static void dfly_uload_update(struct lwp *lp); 165 static void dfly_yield(struct lwp *lp); 166 static void dfly_changeqcpu_locked(struct lwp *lp, 167 dfly_pcpu_t dd, dfly_pcpu_t rdd); 168 static dfly_pcpu_t dfly_choose_best_queue(struct lwp *lp); 169 static dfly_pcpu_t dfly_choose_worst_queue(dfly_pcpu_t dd, int forceit); 170 static dfly_pcpu_t dfly_choose_queue_simple(dfly_pcpu_t dd, struct lwp *lp); 171 static void dfly_need_user_resched_remote(void *dummy); 172 static struct lwp *dfly_chooseproc_locked(dfly_pcpu_t rdd, dfly_pcpu_t dd, 173 struct lwp *chklp, int worst); 174 static void dfly_remrunqueue_locked(dfly_pcpu_t dd, struct lwp *lp); 175 static void dfly_setrunqueue_locked(dfly_pcpu_t dd, struct lwp *lp); 176 static void dfly_changedcpu(struct lwp *lp); 177 178 struct usched usched_dfly = { 179 { NULL }, 180 "dfly", "Original DragonFly Scheduler", 181 NULL, /* default registration */ 182 NULL, /* default deregistration */ 183 dfly_acquire_curproc, 184 dfly_release_curproc, 185 dfly_setrunqueue, 186 dfly_schedulerclock, 187 dfly_recalculate_estcpu, 188 dfly_resetpriority, 189 dfly_forking, 190 dfly_exiting, 191 dfly_uload_update, 192 NULL, /* setcpumask not supported */ 193 dfly_yield, 194 dfly_changedcpu 195 }; 196 197 /* 198 * We have NQS (32) run queues per scheduling class. For the normal 199 * class, there are 128 priorities scaled onto these 32 queues. New 200 * processes are added to the last entry in each queue, and processes 201 * are selected for running by taking them from the head and maintaining 202 * a simple FIFO arrangement. Realtime and Idle priority processes have 203 * and explicit 0-31 priority which maps directly onto their class queue 204 * index. When a queue has something in it, the corresponding bit is 205 * set in the queuebits variable, allowing a single read to determine 206 * the state of all 32 queues and then a ffs() to find the first busy 207 * queue. 208 * 209 * curprocmask is used to publish cpus with assigned curprocs to the rest 210 * of the cpus. In certain situations curprocmask may leave a bit set 211 * (e.g. a yield or a token-based yield) even though dd->uschedcp is 212 * NULL'd out temporarily). 213 */ 214 /* currently running a user process */ 215 static cpumask_t dfly_curprocmask = CPUMASK_INITIALIZER_ALLONES; 216 static cpumask_t dfly_rdyprocmask; /* ready to accept a user process */ 217 static struct usched_dfly_pcpu dfly_pcpu[MAXCPU]; 218 static struct sysctl_ctx_list usched_dfly_sysctl_ctx; 219 static struct sysctl_oid *usched_dfly_sysctl_tree; 220 221 /* Debug info exposed through debug.* sysctl */ 222 223 static int usched_dfly_debug = -1; 224 SYSCTL_INT(_debug, OID_AUTO, dfly_scdebug, CTLFLAG_RW, 225 &usched_dfly_debug, 0, 226 "Print debug information for this pid"); 227 228 static int usched_dfly_pid_debug = -1; 229 SYSCTL_INT(_debug, OID_AUTO, dfly_pid_debug, CTLFLAG_RW, 230 &usched_dfly_pid_debug, 0, 231 "Print KTR debug information for this pid"); 232 233 static int usched_dfly_chooser = 0; 234 SYSCTL_INT(_debug, OID_AUTO, dfly_chooser, CTLFLAG_RW, 235 &usched_dfly_chooser, 0, 236 "Print KTR debug information for this pid"); 237 238 /* 239 * WARNING! 240 * 241 * The fork bias can have a large effect on the system in the face of a 242 * make -j N or other high-forking applications. 243 * 244 * Larger values are much less invasive vs other things that 245 * might be running in the system, but can cause exec chains 246 * such as those typically generated by make to have higher 247 * latencies in the face of modest load. 248 * 249 * Lower values are more invasive but have reduced latencies 250 * for such exec chains. 251 * 252 * make -j 10 buildkernel example, build times: 253 * 254 * +0 3:04 255 * +1 3:14 -5.2% <-- default 256 * +2 3:22 -8.9% 257 * 258 * This issue occurs due to the way the scheduler affinity heuristics work. 259 * There is no way to really 'fix' the affinity heuristics because when it 260 * comes right down to it trying to instantly schedule a process on an 261 * available cpu (even if it will become unavailable a microsecond later) 262 * tends to cause processes to shift around between cpus and sockets too much 263 * and breaks the affinity. 264 * 265 * NOTE: Heavily concurrent builds typically have enough things on the pan 266 * that they remain time-efficient even with a higher bias. 267 */ 268 static int usched_dfly_forkbias = 1; 269 SYSCTL_INT(_debug, OID_AUTO, dfly_forkbias, CTLFLAG_RW, 270 &usched_dfly_forkbias, 0, 271 "Fork bias for estcpu in whole queues"); 272 273 /* 274 * Tunning usched_dfly - configurable through kern.usched_dfly. 275 * 276 * weight1 - Tries to keep threads on their current cpu. If you 277 * make this value too large the scheduler will not be 278 * able to load-balance large loads. 279 * 280 * Generally set to a fairly low value, but high enough 281 * such that estcpu jitter doesn't move threads around. 282 * 283 * weight2 - If non-zero, detects thread pairs undergoing synchronous 284 * communications and tries to move them closer together. 285 * Behavior is adjusted by bit 4 of features (0x10). 286 * 287 * WARNING! Weight2 is a ridiculously sensitive parameter, 288 * change the default at your peril. 289 * 290 * weight3 - Weighting based on the number of recently runnable threads 291 * on the userland scheduling queue (ignoring their loads). 292 * 293 * A nominal value here prevents high-priority (low-load) 294 * threads from accumulating on one cpu core when other 295 * cores are available. 296 * 297 * This value should be left fairly small because low-load 298 * high priority threads can still be mostly idle and too 299 * high a value will kick cpu-bound processes off the cpu 300 * unnecessarily. 301 * 302 * weight4 - Weighting based on other cpu queues being available 303 * or running processes with higher lwp_priority's. 304 * 305 * This allows a thread to migrate to another nearby cpu if it 306 * is unable to run on the current cpu based on the other cpu 307 * being idle or running a lower priority (higher lwp_priority) 308 * thread. This value should be large enough to override weight1 309 * 310 * weight5 - Weighting based on the relative amount of ram connected 311 * to the node a cpu resides on. 312 * 313 * This value should remain fairly low to allow assymetric 314 * NUMA nodes to get threads scheduled to them. Setting a very 315 * high level will prevent scheduling on assymetric NUMA nodes 316 * with low amounts of directly-attached memory. 317 * 318 * Note that when testing e.g. N threads on a machine with N 319 * cpu cores with assymtric NUMA nodes, a non-zero value will 320 * cause some cpu threads on the low-priority NUMA nodes to remain 321 * idle even when a few process threads are doubled-up on other 322 * cpus. But this is typically more ideal because it deschedules 323 * low-priority NUMA nodes at lighter nodes. 324 * 325 * Values between 50 and 200 are recommended. Default is 50. 326 * 327 * weight6 - rdd transfer weight hysteresis. Defaults to 0, can be increased 328 * to improve stabillity at the cost of more mis-schedules. 329 * 330 * features - These flags can be set or cleared to enable or disable various 331 * features. 332 * 333 * 0x01 Enable idle-cpu pulling (default) 334 * 0x02 Enable proactive pushing (default) 335 * 0x04 Enable rebalancing rover (default) 336 * 0x08 Enable more proactive pushing (default) 337 * 0x10 (flip weight2 limit on same cpu) (default) 338 * 0x20 choose best cpu for forked process 339 * 0x40 choose current cpu for forked process 340 * 0x80 choose random cpu for forked process (default) 341 */ 342 static int usched_dfly_smt = 0; 343 static int usched_dfly_cache_coherent = 0; 344 static int usched_dfly_weight1 = 10; /* keep thread on current cpu */ 345 static int usched_dfly_weight2 = 180; /* synchronous peer's current cpu */ 346 static int usched_dfly_weight3 = 10; /* number of threads on queue */ 347 static int usched_dfly_weight4 = 160; /* availability of idle cores */ 348 static int usched_dfly_weight5 = 50; /* node attached memory */ 349 static int usched_dfly_weight6 = 0; /* rdd trasnfer weight */ 350 static int usched_dfly_features = 0x8F; /* allow pulls */ 351 static int usched_dfly_fast_resched = PPQ / 2; /* delta priority / resched */ 352 static int usched_dfly_swmask = ~PPQMASK; /* allow pulls */ 353 static int usched_dfly_rrinterval = (ESTCPUFREQ + 9) / 10; 354 static int usched_dfly_decay = 8; 355 static long usched_dfly_node_mem; 356 357 /* KTR debug printings */ 358 359 KTR_INFO_MASTER(usched); 360 361 #if !defined(KTR_USCHED_DFLY) 362 #define KTR_USCHED_DFLY KTR_ALL 363 #endif 364 365 KTR_INFO(KTR_USCHED_DFLY, usched, chooseproc, 0, 366 "USCHED_DFLY(chooseproc: pid %d, old_cpuid %d, curr_cpuid %d)", 367 pid_t pid, int old_cpuid, int curr); 368 369 /* 370 * This function is called when the kernel intends to return to userland. 371 * It is responsible for making the thread the current designated userland 372 * thread for this cpu, blocking if necessary. 373 * 374 * The kernel will not depress our LWKT priority until after we return, 375 * in case we have to shove over to another cpu. 376 * 377 * We must determine our thread's disposition before we switch away. This 378 * is very sensitive code. 379 * 380 * WARNING! THIS FUNCTION IS ALLOWED TO CAUSE THE CURRENT THREAD TO MIGRATE 381 * TO ANOTHER CPU! Because most of the kernel assumes that no migration will 382 * occur, this function is called only under very controlled circumstances. 383 */ 384 static void 385 dfly_acquire_curproc(struct lwp *lp) 386 { 387 globaldata_t gd; 388 dfly_pcpu_t dd; 389 dfly_pcpu_t rdd; 390 thread_t td; 391 int force_resched; 392 393 /* 394 * Make sure we aren't sitting on a tsleep queue. 395 */ 396 td = lp->lwp_thread; 397 crit_enter_quick(td); 398 if (td->td_flags & TDF_TSLEEPQ) 399 tsleep_remove(td); 400 dfly_recalculate_estcpu(lp); 401 402 gd = mycpu; 403 dd = &dfly_pcpu[gd->gd_cpuid]; 404 405 /* 406 * Process any pending interrupts/ipi's, then handle reschedule 407 * requests. dfly_release_curproc() will try to assign a new 408 * uschedcp that isn't us and otherwise NULL it out. 409 */ 410 force_resched = 0; 411 if ((td->td_mpflags & TDF_MP_BATCH_DEMARC) && 412 lp->lwp_rrcount >= usched_dfly_rrinterval / 2) { 413 force_resched = 1; 414 } 415 416 if (user_resched_wanted()) { 417 if (dd->uschedcp == lp) 418 force_resched = 1; 419 clear_user_resched(); 420 dfly_release_curproc(lp); 421 } 422 423 /* 424 * Loop until we are the current user thread. 425 * 426 * NOTE: dd spinlock not held at top of loop. 427 */ 428 if (dd->uschedcp == lp) 429 lwkt_yield_quick(); 430 431 while (dd->uschedcp != lp) { 432 /* 433 * Do not do a lwkt_yield_quick() here as it will prevent 434 * the lwp from being placed on the dfly_bsd runqueue for 435 * one cycle (possibly an entire round-robin), preventing 436 * it from being scheduled to another cpu. 437 */ 438 /* lwkt_yield_quick(); */ 439 440 spin_lock(&dd->spin); 441 442 /* This lwp is an outcast; force reschedule. */ 443 if (__predict_false( 444 CPUMASK_TESTBIT(lp->lwp_cpumask, gd->gd_cpuid) == 0) && 445 (rdd = dfly_choose_best_queue(lp)) != dd) { 446 dfly_changeqcpu_locked(lp, dd, rdd); 447 spin_unlock(&dd->spin); 448 lwkt_deschedule(lp->lwp_thread); 449 dfly_setrunqueue_dd(rdd, lp); 450 lwkt_switch(); 451 gd = mycpu; 452 dd = &dfly_pcpu[gd->gd_cpuid]; 453 continue; 454 } 455 456 /* 457 * We are not or are no longer the current lwp and a forced 458 * reschedule was requested. Figure out the best cpu to 459 * run on (our current cpu will be given significant weight). 460 * 461 * Doing this on many cpus simultaneously leads to 462 * instability so pace the operation. 463 * 464 * (if a reschedule was not requested we want to move this 465 * step after the uschedcp tests). 466 */ 467 if (force_resched && 468 (usched_dfly_features & 0x08) && 469 (u_int)sched_ticks / 8 % ncpus == gd->gd_cpuid && 470 (rdd = dfly_choose_best_queue(lp)) != dd) { 471 dfly_changeqcpu_locked(lp, dd, rdd); 472 spin_unlock(&dd->spin); 473 lwkt_deschedule(lp->lwp_thread); 474 dfly_setrunqueue_dd(rdd, lp); 475 lwkt_switch(); 476 gd = mycpu; 477 dd = &dfly_pcpu[gd->gd_cpuid]; 478 continue; 479 } 480 481 /* 482 * Either no reschedule was requested or the best queue was 483 * dd, and no current process has been selected. We can 484 * trivially become the current lwp on the current cpu. 485 */ 486 if (dd->uschedcp == NULL) { 487 atomic_clear_int(&lp->lwp_thread->td_mpflags, 488 TDF_MP_DIDYIELD); 489 if ((dd->flags & DFLY_PCPU_CURMASK) == 0) { 490 ATOMIC_CPUMASK_ORBIT(dfly_curprocmask, 491 gd->gd_cpuid); 492 dd->flags |= DFLY_PCPU_CURMASK; 493 } 494 dd->uschedcp = lp; 495 dd->upri = lp->lwp_priority; 496 KKASSERT(lp->lwp_qcpu == dd->cpuid); 497 spin_unlock(&dd->spin); 498 break; 499 } 500 501 /* 502 * Can we steal the current designated user thread? 503 * 504 * If we do the other thread will stall when it tries to 505 * return to userland, possibly rescheduling elsewhere. 506 * Set need_user_resched() to get the thread to cycle soonest. 507 * 508 * It is important to do a masked test to avoid the edge 509 * case where two near-equal-priority threads are constantly 510 * interrupting each other. 511 * 512 * In the exact match case another thread has already gained 513 * uschedcp and lowered its priority, if we steal it the 514 * other thread will stay stuck on the LWKT runq and not 515 * push to another cpu. So don't steal on equal-priority even 516 * though it might appear to be more beneficial due to not 517 * having to switch back to the other thread's context. 518 * 519 * usched_dfly_fast_resched requires that two threads be 520 * significantly far apart in priority in order to interrupt. 521 * 522 * If better but not sufficiently far apart, the current 523 * uschedcp will be interrupted at the next scheduler clock. 524 */ 525 if (dd->uschedcp && 526 (dd->upri & ~PPQMASK) > 527 (lp->lwp_priority & ~PPQMASK) + usched_dfly_fast_resched) { 528 dd->uschedcp = lp; 529 dd->upri = lp->lwp_priority; 530 KKASSERT(lp->lwp_qcpu == dd->cpuid); 531 need_user_resched(); 532 spin_unlock(&dd->spin); 533 break; 534 } 535 536 /* 537 * Requeue us at lwp_priority, which recalculate_estcpu() 538 * set for us. Reset the rrcount to force placement 539 * at the end of the queue. 540 * 541 * We used to move ourselves to the worst queue, but 542 * this creates a fairly serious priority inversion 543 * problem. 544 */ 545 if (lp->lwp_thread->td_mpflags & TDF_MP_DIDYIELD) { 546 spin_unlock(&dd->spin); 547 lp->lwp_rrcount = usched_dfly_rrinterval; 548 lp->lwp_rqindex = (lp->lwp_priority & PRIMASK) / PPQ; 549 550 lwkt_deschedule(lp->lwp_thread); 551 dfly_setrunqueue_dd(dd, lp); 552 atomic_clear_int(&lp->lwp_thread->td_mpflags, 553 TDF_MP_DIDYIELD); 554 lwkt_switch(); 555 gd = mycpu; 556 dd = &dfly_pcpu[gd->gd_cpuid]; 557 continue; 558 } 559 560 /* 561 * We are not the current lwp, figure out the best cpu 562 * to run on (our current cpu will be given significant 563 * weight). Loop on cpu change. 564 */ 565 if ((usched_dfly_features & 0x02) && 566 force_resched == 0 && 567 (rdd = dfly_choose_best_queue(lp)) != dd) { 568 dfly_changeqcpu_locked(lp, dd, rdd); 569 spin_unlock(&dd->spin); 570 lwkt_deschedule(lp->lwp_thread); 571 dfly_setrunqueue_dd(rdd, lp); 572 lwkt_switch(); 573 gd = mycpu; 574 dd = &dfly_pcpu[gd->gd_cpuid]; 575 continue; 576 } 577 578 /* 579 * We cannot become the current lwp, place the lp on the 580 * run-queue of this or another cpu and deschedule ourselves. 581 * 582 * When we are reactivated we will have another chance. 583 * 584 * Reload after a switch or setrunqueue/switch possibly 585 * moved us to another cpu. 586 */ 587 spin_unlock(&dd->spin); 588 lwkt_deschedule(lp->lwp_thread); 589 dfly_setrunqueue_dd(dd, lp); 590 lwkt_switch(); 591 gd = mycpu; 592 dd = &dfly_pcpu[gd->gd_cpuid]; 593 } 594 595 /* 596 * Make sure upri is synchronized, then yield to LWKT threads as 597 * needed before returning. This could result in another reschedule. 598 * XXX 599 */ 600 crit_exit_quick(td); 601 602 KKASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0); 603 } 604 605 /* 606 * DFLY_RELEASE_CURPROC 607 * 608 * This routine detaches the current thread from the userland scheduler, 609 * usually because the thread needs to run or block in the kernel (at 610 * kernel priority) for a while. 611 * 612 * This routine is also responsible for selecting a new thread to 613 * make the current thread. 614 * 615 * NOTE: This implementation differs from the dummy example in that 616 * dfly_select_curproc() is able to select the current process, whereas 617 * dummy_select_curproc() is not able to select the current process. 618 * This means we have to NULL out uschedcp. 619 * 620 * Additionally, note that we may already be on a run queue if releasing 621 * via the lwkt_switch() in dfly_setrunqueue(). 622 */ 623 static void 624 dfly_release_curproc(struct lwp *lp) 625 { 626 globaldata_t gd = mycpu; 627 dfly_pcpu_t dd = &dfly_pcpu[gd->gd_cpuid]; 628 629 /* 630 * Make sure td_wakefromcpu is defaulted. This will be overwritten 631 * by wakeup(). 632 */ 633 if (dd->uschedcp == lp) { 634 KKASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0); 635 spin_lock(&dd->spin); 636 if (dd->uschedcp == lp) { 637 dd->uschedcp = NULL; /* don't let lp be selected */ 638 dd->upri = PRIBASE_NULL; 639 640 /* 641 * We're just going to set it again, avoid the global 642 * cache line ping-pong. 643 */ 644 if ((lp->lwp_thread->td_mpflags & TDF_MP_DIDYIELD) == 0) { 645 if (dd->flags & DFLY_PCPU_CURMASK) { 646 ATOMIC_CPUMASK_NANDBIT(dfly_curprocmask, 647 gd->gd_cpuid); 648 dd->flags &= ~DFLY_PCPU_CURMASK; 649 } 650 } 651 spin_unlock(&dd->spin); 652 dfly_select_curproc(gd); 653 } else { 654 spin_unlock(&dd->spin); 655 } 656 } 657 } 658 659 /* 660 * DFLY_SELECT_CURPROC 661 * 662 * Select a new current process for this cpu and clear any pending user 663 * reschedule request. The cpu currently has no current process. 664 * 665 * This routine is also responsible for equal-priority round-robining, 666 * typically triggered from dfly_schedulerclock(). In our dummy example 667 * all the 'user' threads are LWKT scheduled all at once and we just 668 * call lwkt_switch(). 669 * 670 * The calling process is not on the queue and cannot be selected. 671 */ 672 static 673 void 674 dfly_select_curproc(globaldata_t gd) 675 { 676 dfly_pcpu_t dd = &dfly_pcpu[gd->gd_cpuid]; 677 struct lwp *nlp; 678 int cpuid = gd->gd_cpuid; 679 680 crit_enter_gd(gd); 681 682 spin_lock(&dd->spin); 683 nlp = dfly_chooseproc_locked(dd, dd, dd->uschedcp, 0); 684 685 if (nlp) { 686 if ((dd->flags & DFLY_PCPU_CURMASK) == 0) { 687 ATOMIC_CPUMASK_ORBIT(dfly_curprocmask, cpuid); 688 dd->flags |= DFLY_PCPU_CURMASK; 689 } 690 dd->upri = nlp->lwp_priority; 691 dd->uschedcp = nlp; 692 #if 0 693 dd->rrcount = 0; /* reset round robin */ 694 #endif 695 spin_unlock(&dd->spin); 696 lwkt_acquire(nlp->lwp_thread); 697 lwkt_schedule(nlp->lwp_thread); 698 } else { 699 spin_unlock(&dd->spin); 700 } 701 crit_exit_gd(gd); 702 } 703 704 /* 705 * Place the specified lwp on the user scheduler's run queue. This routine 706 * must be called with the thread descheduled. The lwp must be runnable. 707 * It must not be possible for anyone else to explicitly schedule this thread. 708 * 709 * The thread may be the current thread as a special case. 710 */ 711 static void 712 dfly_setrunqueue(struct lwp *lp) 713 { 714 dfly_pcpu_t dd; 715 dfly_pcpu_t rdd; 716 717 /* 718 * First validate the process LWKT state. 719 */ 720 KASSERT(lp->lwp_stat == LSRUN, ("setrunqueue: lwp not LSRUN")); 721 KASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0, 722 ("lwp %d/%d already on runq! flag %08x/%08x", lp->lwp_proc->p_pid, 723 lp->lwp_tid, lp->lwp_proc->p_flags, lp->lwp_flags)); 724 KKASSERT((lp->lwp_thread->td_flags & TDF_RUNQ) == 0); 725 726 /* 727 * NOTE: dd/rdd do not necessarily represent the current cpu. 728 * Instead they may represent the cpu the thread was last 729 * scheduled on or inherited by its parent. 730 */ 731 dd = &dfly_pcpu[lp->lwp_qcpu]; 732 rdd = dd; 733 734 /* 735 * This process is not supposed to be scheduled anywhere or assigned 736 * as the current process anywhere. Assert the condition. 737 */ 738 KKASSERT(rdd->uschedcp != lp); 739 740 /* 741 * Ok, we have to setrunqueue some target cpu and request a reschedule 742 * if necessary. 743 * 744 * We have to choose the best target cpu. It might not be the current 745 * target even if the current cpu has no running user thread (for 746 * example, because the current cpu might be a hyperthread and its 747 * sibling has a thread assigned). 748 * 749 * If we just forked it is most optimal to run the child on the same 750 * cpu just in case the parent decides to wait for it (thus getting 751 * off that cpu). As long as there is nothing else runnable on the 752 * cpu, that is. If we did this unconditionally a parent forking 753 * multiple children before waiting (e.g. make -j N) leaves other 754 * cpus idle that could be working. 755 */ 756 if (lp->lwp_forked) { 757 lp->lwp_forked = 0; 758 if (usched_dfly_features & 0x20) 759 rdd = dfly_choose_best_queue(lp); 760 else if (usched_dfly_features & 0x40) 761 rdd = &dfly_pcpu[lp->lwp_qcpu]; 762 else if (usched_dfly_features & 0x80) 763 rdd = dfly_choose_queue_simple(rdd, lp); 764 else if (dfly_pcpu[lp->lwp_qcpu].runqcount) 765 rdd = dfly_choose_best_queue(lp); 766 else 767 rdd = &dfly_pcpu[lp->lwp_qcpu]; 768 } else { 769 rdd = dfly_choose_best_queue(lp); 770 /* rdd = &dfly_pcpu[lp->lwp_qcpu]; */ 771 } 772 if (lp->lwp_qcpu != rdd->cpuid) { 773 spin_lock(&dd->spin); 774 dfly_changeqcpu_locked(lp, dd, rdd); 775 spin_unlock(&dd->spin); 776 } 777 dfly_setrunqueue_dd(rdd, lp); 778 } 779 780 /* 781 * Change qcpu to rdd->cpuid. The dd the lp is CURRENTLY on must be 782 * spin-locked on-call. rdd does not have to be. 783 */ 784 static void 785 dfly_changeqcpu_locked(struct lwp *lp, dfly_pcpu_t dd, dfly_pcpu_t rdd) 786 { 787 if (lp->lwp_qcpu != rdd->cpuid) { 788 if (lp->lwp_mpflags & LWP_MP_ULOAD) { 789 atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ULOAD); 790 atomic_add_long(&dd->uload, -lp->lwp_uload); 791 atomic_add_int(&dd->ucount, -1); 792 } 793 lp->lwp_qcpu = rdd->cpuid; 794 } 795 } 796 797 /* 798 * Place lp on rdd's runqueue. Nothing is locked on call. This function 799 * also performs all necessary ancillary notification actions. 800 */ 801 static void 802 dfly_setrunqueue_dd(dfly_pcpu_t rdd, struct lwp *lp) 803 { 804 globaldata_t rgd; 805 806 /* 807 * We might be moving the lp to another cpu's run queue, and once 808 * on the runqueue (even if it is our cpu's), another cpu can rip 809 * it away from us. 810 * 811 * TDF_MIGRATING might already be set if this is part of a 812 * remrunqueue+setrunqueue sequence. 813 */ 814 if ((lp->lwp_thread->td_flags & TDF_MIGRATING) == 0) 815 lwkt_giveaway(lp->lwp_thread); 816 817 rgd = rdd->gd; 818 819 /* 820 * We lose control of the lp the moment we release the spinlock 821 * after having placed it on the queue. i.e. another cpu could pick 822 * it up, or it could exit, or its priority could be further 823 * adjusted, or something like that. 824 * 825 * WARNING! rdd can point to a foreign cpu! 826 */ 827 spin_lock(&rdd->spin); 828 dfly_setrunqueue_locked(rdd, lp); 829 830 /* 831 * Potentially interrupt the currently-running thread 832 */ 833 if ((rdd->upri & ~PPQMASK) <= (lp->lwp_priority & ~PPQMASK)) { 834 /* 835 * Currently running thread is better or same, do not 836 * interrupt. 837 */ 838 spin_unlock(&rdd->spin); 839 } else if ((rdd->upri & ~PPQMASK) <= (lp->lwp_priority & ~PPQMASK) + 840 usched_dfly_fast_resched) { 841 /* 842 * Currently running thread is not better, but not so bad 843 * that we need to interrupt it. Let it run for one more 844 * scheduler tick. 845 */ 846 if (rdd->uschedcp && 847 rdd->uschedcp->lwp_rrcount < usched_dfly_rrinterval) { 848 rdd->uschedcp->lwp_rrcount = usched_dfly_rrinterval - 1; 849 } 850 spin_unlock(&rdd->spin); 851 } else if (rgd == mycpu) { 852 /* 853 * We should interrupt the currently running thread, which 854 * is on the current cpu. However, if DIDYIELD is set we 855 * round-robin unconditionally and do not interrupt it. 856 */ 857 spin_unlock(&rdd->spin); 858 if (rdd->uschedcp == NULL) 859 wakeup_mycpu(rdd->helper_thread); /* XXX */ 860 if ((lp->lwp_thread->td_mpflags & TDF_MP_DIDYIELD) == 0) 861 need_user_resched(); 862 } else { 863 /* 864 * We should interrupt the currently running thread, which 865 * is on a different cpu. 866 */ 867 spin_unlock(&rdd->spin); 868 lwkt_send_ipiq(rgd, dfly_need_user_resched_remote, NULL); 869 } 870 } 871 872 /* 873 * This routine is called from a systimer IPI. It MUST be MP-safe and 874 * the BGL IS NOT HELD ON ENTRY. This routine is called at ESTCPUFREQ on 875 * each cpu. 876 */ 877 static 878 void 879 dfly_schedulerclock(struct lwp *lp, sysclock_t period, sysclock_t cpstamp) 880 { 881 globaldata_t gd = mycpu; 882 dfly_pcpu_t dd = &dfly_pcpu[gd->gd_cpuid]; 883 884 /* 885 * Spinlocks also hold a critical section so there should not be 886 * any active. 887 */ 888 KKASSERT(gd->gd_spinlocks == 0 || dumping); 889 890 /* 891 * If lp is NULL we might be contended and lwkt_switch() may have 892 * cycled into the idle thread. Apply the tick to the current 893 * process on this cpu if it is contended. 894 */ 895 if (gd->gd_curthread == &gd->gd_idlethread) { 896 lp = dd->uschedcp; 897 if (lp && (lp->lwp_thread == NULL || 898 lp->lwp_thread->td_contended == 0)) { 899 lp = NULL; 900 } 901 } 902 903 /* 904 * Dock thread for tick 905 */ 906 if (lp) { 907 /* 908 * Do we need to round-robin? We round-robin 10 times a 909 * second. This should only occur for cpu-bound batch 910 * processes. 911 */ 912 if (++lp->lwp_rrcount >= usched_dfly_rrinterval) { 913 lp->lwp_thread->td_wakefromcpu = -1; 914 need_user_resched(); 915 } 916 917 /* 918 * Adjust estcpu upward using a real time equivalent 919 * calculation, and recalculate lp's priority. Estcpu 920 * is increased such that it will cap-out over a period 921 * of one second. 922 */ 923 lp->lwp_estcpu = ESTCPULIM(lp->lwp_estcpu + 924 ESTCPUMAX / ESTCPUFREQ + 1); 925 dfly_resetpriority(lp); 926 } 927 928 /* 929 * Rebalance two cpus every 8 ticks, pulling the worst thread 930 * from the worst cpu's queue into a rotating cpu number. 931 * Also require that the moving of the highest-load thread 932 * from rdd to dd does not cause the uload to cross over. 933 * 934 * This mechanic is needed because the push algorithms can 935 * steady-state in an non-optimal configuration. We need to mix it 936 * up a little, even if it means breaking up a paired thread, so 937 * the push algorithms can rebalance the degenerate conditions. 938 * This portion of the algorithm exists to ensure stability at the 939 * selected weightings. 940 * 941 * Because we might be breaking up optimal conditions we do not want 942 * to execute this too quickly, hence we only rebalance approximately 943 * ~7-8 times per second. The push's, on the otherhand, are capable 944 * moving threads to other cpus at a much higher rate. 945 * 946 * We choose the most heavily loaded thread from the worst queue 947 * in order to ensure that multiple heavy-weight threads on the same 948 * queue get broken up, and also because these threads are the most 949 * likely to be able to remain in place. Hopefully then any pairings, 950 * if applicable, migrate to where these threads are. 951 */ 952 if ((usched_dfly_features & 0x04) && 953 ((u_int)sched_ticks & 7) == 0 && 954 (u_int)sched_ticks / 8 % ncpus == gd->gd_cpuid) { 955 /* 956 * Our cpu is up. 957 */ 958 struct lwp *nlp; 959 dfly_pcpu_t rdd; 960 961 rdd = dfly_choose_worst_queue(dd, 1); 962 if (rdd && dd->uload + usched_dfly_weight6 / 2 < rdd->uload) { 963 spin_lock(&dd->spin); 964 if (spin_trylock(&rdd->spin)) { 965 nlp = dfly_chooseproc_locked(rdd, dd, NULL, 1); 966 spin_unlock(&rdd->spin); 967 if (nlp == NULL) 968 spin_unlock(&dd->spin); 969 } else { 970 spin_unlock(&dd->spin); 971 nlp = NULL; 972 } 973 } else { 974 nlp = NULL; 975 } 976 /* dd->spin held if nlp != NULL */ 977 978 /* 979 * Either schedule it or add it to our queue. 980 */ 981 if (nlp && 982 (nlp->lwp_priority & ~PPQMASK) < (dd->upri & ~PPQMASK)) { 983 if ((dd->flags & DFLY_PCPU_CURMASK) == 0) { 984 ATOMIC_CPUMASK_ORMASK(dfly_curprocmask, 985 dd->cpumask); 986 dd->flags |= DFLY_PCPU_CURMASK; 987 } 988 dd->upri = nlp->lwp_priority; 989 dd->uschedcp = nlp; 990 #if 0 991 dd->rrcount = 0; /* reset round robin */ 992 #endif 993 spin_unlock(&dd->spin); 994 lwkt_acquire(nlp->lwp_thread); 995 lwkt_schedule(nlp->lwp_thread); 996 } else if (nlp) { 997 dfly_setrunqueue_locked(dd, nlp); 998 spin_unlock(&dd->spin); 999 } 1000 } 1001 } 1002 1003 /* 1004 * Called from acquire and from kern_synch's one-second timer (one of the 1005 * callout helper threads) with a critical section held. 1006 * 1007 * Adjust p_estcpu based on our single-cpu load, p_nice, and compensate for 1008 * overall system load. 1009 * 1010 * Note that no recalculation occurs for a process which sleeps and wakes 1011 * up in the same tick. That is, a system doing thousands of context 1012 * switches per second will still only do serious estcpu calculations 1013 * ESTCPUFREQ times per second. 1014 */ 1015 static 1016 void 1017 dfly_recalculate_estcpu(struct lwp *lp) 1018 { 1019 globaldata_t gd = mycpu; 1020 sysclock_t cpbase; 1021 sysclock_t ttlticks; 1022 int estcpu; 1023 int decay_factor; 1024 int ucount; 1025 1026 /* 1027 * We have to subtract periodic to get the last schedclock 1028 * timeout time, otherwise we would get the upcoming timeout. 1029 * Keep in mind that a process can migrate between cpus and 1030 * while the scheduler clock should be very close, boundary 1031 * conditions could lead to a small negative delta. 1032 */ 1033 cpbase = gd->gd_schedclock.time - gd->gd_schedclock.periodic; 1034 1035 if (lp->lwp_slptime > 1) { 1036 /* 1037 * Too much time has passed, do a coarse correction. 1038 */ 1039 lp->lwp_estcpu = lp->lwp_estcpu >> 1; 1040 dfly_resetpriority(lp); 1041 lp->lwp_cpbase = cpbase; 1042 lp->lwp_cpticks = 0; 1043 lp->lwp_estfast = 0; 1044 } else if (lp->lwp_cpbase != cpbase) { 1045 /* 1046 * Adjust estcpu if we are in a different tick. Don't waste 1047 * time if we are in the same tick. 1048 * 1049 * First calculate the number of ticks in the measurement 1050 * interval. The ttlticks calculation can wind up 0 due to 1051 * a bug in the handling of lwp_slptime (as yet not found), 1052 * so make sure we do not get a divide by 0 panic. 1053 */ 1054 ttlticks = (cpbase - lp->lwp_cpbase) / 1055 gd->gd_schedclock.periodic; 1056 if ((ssysclock_t)ttlticks < 0) { 1057 ttlticks = 0; 1058 lp->lwp_cpbase = cpbase; 1059 } 1060 if (ttlticks < 4) 1061 return; 1062 updatepcpu(lp, lp->lwp_cpticks, ttlticks); 1063 1064 /* 1065 * Calculate instant estcpu based percentage of (one) cpu 1066 * used and exponentially average it into the current 1067 * lwp_estcpu. 1068 */ 1069 ucount = dfly_pcpu[lp->lwp_qcpu].ucount; 1070 estcpu = lp->lwp_cpticks * ESTCPUMAX / ttlticks; 1071 1072 /* 1073 * The higher ttlticks gets, the more meaning the calculation 1074 * has and the smaller our decay_factor in the exponential 1075 * average. 1076 * 1077 * The uload calculation has been removed because it actually 1078 * makes things worse, causing processes which use less cpu 1079 * (such as a browser) to be pumped up and treated the same 1080 * as a cpu-bound process (such as a make). The same effect 1081 * can occur with sufficient load without the uload 1082 * calculation, but occurs less quickly and takes more load. 1083 * In addition, the less cpu a process uses the smaller the 1084 * effect of the overload. 1085 */ 1086 if (ttlticks >= hz) 1087 decay_factor = 1; 1088 else 1089 decay_factor = hz - ttlticks; 1090 1091 lp->lwp_estcpu = ESTCPULIM( 1092 (lp->lwp_estcpu * ttlticks + estcpu) / 1093 (ttlticks + 1)); 1094 if (usched_dfly_debug == lp->lwp_proc->p_pid) 1095 kprintf(" finalestcpu %d %d\n", estcpu, lp->lwp_estcpu); 1096 1097 dfly_resetpriority(lp); 1098 lp->lwp_cpbase += ttlticks * gd->gd_schedclock.periodic; 1099 lp->lwp_cpticks = 0; 1100 } 1101 } 1102 1103 /* 1104 * Compute the priority of a process when running in user mode. 1105 * Arrange to reschedule if the resulting priority is better 1106 * than that of the current process. 1107 * 1108 * This routine may be called with any process. 1109 * 1110 * This routine is called by fork1() for initial setup with the process of 1111 * the run queue, and also may be called normally with the process on or 1112 * off the run queue. 1113 */ 1114 static void 1115 dfly_resetpriority(struct lwp *lp) 1116 { 1117 dfly_pcpu_t rdd; 1118 int newpriority; 1119 u_short newrqtype; 1120 int rcpu; 1121 int checkpri; 1122 int estcpu; 1123 int delta_uload; 1124 1125 crit_enter(); 1126 1127 /* 1128 * Lock the scheduler (lp) belongs to. This can be on a different 1129 * cpu. Handle races. This loop breaks out with the appropriate 1130 * rdd locked. 1131 */ 1132 for (;;) { 1133 rcpu = lp->lwp_qcpu; 1134 cpu_ccfence(); 1135 rdd = &dfly_pcpu[rcpu]; 1136 spin_lock(&rdd->spin); 1137 if (rcpu == lp->lwp_qcpu) 1138 break; 1139 spin_unlock(&rdd->spin); 1140 } 1141 1142 /* 1143 * Calculate the new priority and queue type 1144 */ 1145 newrqtype = lp->lwp_rtprio.type; 1146 1147 switch(newrqtype) { 1148 case RTP_PRIO_REALTIME: 1149 case RTP_PRIO_FIFO: 1150 newpriority = PRIBASE_REALTIME + 1151 (lp->lwp_rtprio.prio & PRIMASK); 1152 break; 1153 case RTP_PRIO_NORMAL: 1154 /* 1155 * Calculate the new priority. 1156 * 1157 * nice contributes up to NICE_QS queues (typ 32 - full range) 1158 * estcpu contributes up to EST_QS queues (typ 24) 1159 * 1160 * A nice +20 process receives 1/10 cpu vs nice+0. Niced 1161 * process more than 20 apart may receive no cpu, so cpu 1162 * bound nice -20 can prevent a nice +5 from getting any 1163 * cpu. A nice+0, being in the middle, always gets some cpu 1164 * no matter what. 1165 */ 1166 estcpu = lp->lwp_estcpu; 1167 newpriority = (lp->lwp_proc->p_nice - PRIO_MIN) * 1168 (NICE_QS * PPQ) / PRIO_RANGE; 1169 newpriority += estcpu * PPQ / ESTCPUPPQ; 1170 if (newpriority < 0) 1171 newpriority = 0; 1172 if (newpriority >= MAXPRI) 1173 newpriority = MAXPRI - 1; 1174 newpriority += PRIBASE_NORMAL; 1175 break; 1176 case RTP_PRIO_IDLE: 1177 newpriority = PRIBASE_IDLE + (lp->lwp_rtprio.prio & PRIMASK); 1178 break; 1179 case RTP_PRIO_THREAD: 1180 newpriority = PRIBASE_THREAD + (lp->lwp_rtprio.prio & PRIMASK); 1181 break; 1182 default: 1183 panic("Bad RTP_PRIO %d", newrqtype); 1184 /* NOT REACHED */ 1185 } 1186 1187 /* 1188 * The LWKT scheduler doesn't dive usched structures, give it a hint 1189 * on the relative priority of user threads running in the kernel. 1190 * The LWKT scheduler will always ensure that a user thread running 1191 * in the kernel will get cpu some time, regardless of its upri, 1192 * but can decide not to instantly switch from one kernel or user 1193 * mode user thread to a kernel-mode user thread when it has a less 1194 * desireable user priority. 1195 * 1196 * td_upri has normal sense (higher values are more desireable), so 1197 * negate it (this is a different field lp->lwp_priority) 1198 */ 1199 lp->lwp_thread->td_upri = -(newpriority & usched_dfly_swmask); 1200 1201 /* 1202 * The newpriority incorporates the queue type so do a simple masked 1203 * check to determine if the process has moved to another queue. If 1204 * it has, and it is currently on a run queue, then move it. 1205 * 1206 * Since uload is ~PPQMASK masked, no modifications are necessary if 1207 * we end up in the same run queue. 1208 * 1209 * Reset rrcount if moving to a higher-priority queue, otherwise 1210 * retain rrcount. 1211 */ 1212 if ((lp->lwp_priority ^ newpriority) & ~PPQMASK) { 1213 if (lp->lwp_priority < newpriority) 1214 lp->lwp_rrcount = 0; 1215 if (lp->lwp_mpflags & LWP_MP_ONRUNQ) { 1216 dfly_remrunqueue_locked(rdd, lp); 1217 lp->lwp_priority = newpriority; 1218 lp->lwp_rqtype = newrqtype; 1219 lp->lwp_rqindex = (newpriority & PRIMASK) / PPQ; 1220 dfly_setrunqueue_locked(rdd, lp); 1221 checkpri = 1; 1222 } else { 1223 lp->lwp_priority = newpriority; 1224 lp->lwp_rqtype = newrqtype; 1225 lp->lwp_rqindex = (newpriority & PRIMASK) / PPQ; 1226 checkpri = 0; 1227 } 1228 } else { 1229 /* 1230 * In the same PPQ, uload cannot change. 1231 */ 1232 lp->lwp_priority = newpriority; 1233 checkpri = 1; 1234 rcpu = -1; 1235 } 1236 1237 /* 1238 * Adjust effective load. 1239 * 1240 * Calculate load then scale up or down geometrically based on p_nice. 1241 * Processes niced up (positive) are less important, and processes 1242 * niced downard (negative) are more important. The higher the uload, 1243 * the more important the thread. 1244 */ 1245 /* 0-511, 0-100% cpu */ 1246 delta_uload = lptouload(lp); 1247 delta_uload -= lp->lwp_uload; 1248 if (lp->lwp_uload + delta_uload < -32767) { 1249 delta_uload = -32768 - lp->lwp_uload; 1250 } else if (lp->lwp_uload + delta_uload > 32767) { 1251 delta_uload = 32767 - lp->lwp_uload; 1252 } 1253 lp->lwp_uload += delta_uload; 1254 if (lp->lwp_mpflags & LWP_MP_ULOAD) 1255 atomic_add_long(&dfly_pcpu[lp->lwp_qcpu].uload, delta_uload); 1256 1257 /* 1258 * Determine if we need to reschedule the target cpu. This only 1259 * occurs if the LWP is already on a scheduler queue, which means 1260 * that idle cpu notification has already occured. At most we 1261 * need only issue a need_user_resched() on the appropriate cpu. 1262 * 1263 * The LWP may be owned by a CPU different from the current one, 1264 * in which case dd->uschedcp may be modified without an MP lock 1265 * or a spinlock held. The worst that happens is that the code 1266 * below causes a spurious need_user_resched() on the target CPU 1267 * and dd->pri to be wrong for a short period of time, both of 1268 * which are harmless. 1269 * 1270 * If checkpri is 0 we are adjusting the priority of the current 1271 * process, possibly higher (less desireable), so ignore the upri 1272 * check which will fail in that case. 1273 */ 1274 if (rcpu >= 0) { 1275 if (CPUMASK_TESTBIT(dfly_rdyprocmask, rcpu) && 1276 (checkpri == 0 || 1277 (rdd->upri & ~PRIMASK) > 1278 (lp->lwp_priority & ~PRIMASK))) { 1279 if (rcpu == mycpu->gd_cpuid) { 1280 spin_unlock(&rdd->spin); 1281 need_user_resched(); 1282 } else { 1283 spin_unlock(&rdd->spin); 1284 lwkt_send_ipiq(globaldata_find(rcpu), 1285 dfly_need_user_resched_remote, 1286 NULL); 1287 } 1288 } else { 1289 spin_unlock(&rdd->spin); 1290 } 1291 } else { 1292 spin_unlock(&rdd->spin); 1293 } 1294 crit_exit(); 1295 } 1296 1297 static 1298 void 1299 dfly_yield(struct lwp *lp) 1300 { 1301 if (lp->lwp_qcpu != mycpu->gd_cpuid) 1302 return; 1303 KKASSERT(lp == curthread->td_lwp); 1304 1305 /* 1306 * Don't set need_user_resched() or mess with rrcount or anything. 1307 * the TDF flag will override everything as long as we release. 1308 */ 1309 atomic_set_int(&lp->lwp_thread->td_mpflags, TDF_MP_DIDYIELD); 1310 dfly_release_curproc(lp); 1311 } 1312 1313 /* 1314 * Thread was forcefully migrated to another cpu. Normally forced migrations 1315 * are used for iterations and the kernel returns to the original cpu before 1316 * returning and this is not needed. However, if the kernel migrates a 1317 * thread to another cpu and wants to leave it there, it has to call this 1318 * scheduler helper. 1319 * 1320 * Note that the lwkt_migratecpu() function also released the thread, so 1321 * we don't have to worry about that. 1322 */ 1323 static 1324 void 1325 dfly_changedcpu(struct lwp *lp) 1326 { 1327 dfly_pcpu_t dd = &dfly_pcpu[lp->lwp_qcpu]; 1328 dfly_pcpu_t rdd = &dfly_pcpu[mycpu->gd_cpuid]; 1329 1330 if (dd != rdd) { 1331 spin_lock(&dd->spin); 1332 dfly_changeqcpu_locked(lp, dd, rdd); 1333 spin_unlock(&dd->spin); 1334 } 1335 } 1336 1337 /* 1338 * Called from fork1() when a new child process is being created. 1339 * 1340 * Give the child process an initial estcpu that is more batch then 1341 * its parent and dock the parent for the fork (but do not 1342 * reschedule the parent). 1343 * 1344 * fast 1345 * 1346 * XXX lwp should be "spawning" instead of "forking" 1347 */ 1348 static void 1349 dfly_forking(struct lwp *plp, struct lwp *lp) 1350 { 1351 int estcpu; 1352 1353 /* 1354 * Put the child 4 queue slots (out of 32) higher than the parent 1355 * (less desireable than the parent). 1356 */ 1357 lp->lwp_estcpu = ESTCPULIM(plp->lwp_estcpu + 1358 ESTCPUPPQ * usched_dfly_forkbias); 1359 lp->lwp_forked = 1; 1360 lp->lwp_estfast = 0; 1361 1362 /* 1363 * Even though the lp will be scheduled specially the first time 1364 * due to lp->lwp_forked, it is important to initialize lwp_qcpu 1365 * to avoid favoring a fixed cpu. 1366 */ 1367 #if 0 1368 static uint16_t save_cpu; 1369 lp->lwp_qcpu = ++save_cpu % ncpus; 1370 #else 1371 lp->lwp_qcpu = plp->lwp_qcpu; 1372 if (CPUMASK_TESTBIT(lp->lwp_cpumask, lp->lwp_qcpu) == 0) 1373 lp->lwp_qcpu = BSFCPUMASK(lp->lwp_cpumask); 1374 #endif 1375 1376 /* 1377 * Dock the parent a cost for the fork, protecting us from fork 1378 * bombs. If the parent is forking quickly this makes both the 1379 * parent and child more batchy. 1380 */ 1381 estcpu = plp->lwp_estcpu + ESTCPUPPQ / 16; 1382 plp->lwp_estcpu = ESTCPULIM(estcpu); 1383 } 1384 1385 /* 1386 * Called when a lwp is being removed from this scheduler, typically 1387 * during lwp_exit(). We have to clean out any ULOAD accounting before 1388 * we can let the lp go. The dd->spin lock is not needed for uload 1389 * updates. 1390 * 1391 * Scheduler dequeueing has already occurred, no further action in that 1392 * regard is needed. 1393 */ 1394 static void 1395 dfly_exiting(struct lwp *lp, struct proc *child_proc) 1396 { 1397 dfly_pcpu_t dd = &dfly_pcpu[lp->lwp_qcpu]; 1398 1399 if (lp->lwp_mpflags & LWP_MP_ULOAD) { 1400 atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ULOAD); 1401 atomic_add_long(&dd->uload, -lp->lwp_uload); 1402 atomic_add_int(&dd->ucount, -1); 1403 } 1404 } 1405 1406 /* 1407 * This function cannot block in any way, but spinlocks are ok. 1408 * 1409 * Update the uload based on the state of the thread (whether it is going 1410 * to sleep or running again). The uload is meant to be a longer-term 1411 * load and not an instantanious load. 1412 */ 1413 static void 1414 dfly_uload_update(struct lwp *lp) 1415 { 1416 dfly_pcpu_t dd = &dfly_pcpu[lp->lwp_qcpu]; 1417 1418 if (lp->lwp_thread->td_flags & TDF_RUNQ) { 1419 if ((lp->lwp_mpflags & LWP_MP_ULOAD) == 0) { 1420 spin_lock(&dd->spin); 1421 if ((lp->lwp_mpflags & LWP_MP_ULOAD) == 0) { 1422 atomic_set_int(&lp->lwp_mpflags, 1423 LWP_MP_ULOAD); 1424 atomic_add_long(&dd->uload, lp->lwp_uload); 1425 atomic_add_int(&dd->ucount, 1); 1426 } 1427 spin_unlock(&dd->spin); 1428 } 1429 } else if (lp->lwp_slptime > 0) { 1430 if (lp->lwp_mpflags & LWP_MP_ULOAD) { 1431 spin_lock(&dd->spin); 1432 if (lp->lwp_mpflags & LWP_MP_ULOAD) { 1433 atomic_clear_int(&lp->lwp_mpflags, 1434 LWP_MP_ULOAD); 1435 atomic_add_long(&dd->uload, -lp->lwp_uload); 1436 atomic_add_int(&dd->ucount, -1); 1437 } 1438 spin_unlock(&dd->spin); 1439 } 1440 } 1441 } 1442 1443 /* 1444 * chooseproc() is called when a cpu needs a user process to LWKT schedule, 1445 * it selects a user process and returns it. If chklp is non-NULL and chklp 1446 * has a better or equal priority then the process that would otherwise be 1447 * chosen, NULL is returned. 1448 * 1449 * Until we fix the RUNQ code the chklp test has to be strict or we may 1450 * bounce between processes trying to acquire the current process designation. 1451 * 1452 * Must be called with rdd->spin locked. The spinlock is left intact through 1453 * the entire routine. dd->spin does not have to be locked. 1454 * 1455 * If worst is non-zero this function finds the worst thread instead of the 1456 * best thread (used by the schedulerclock-based rover). 1457 */ 1458 static 1459 struct lwp * 1460 dfly_chooseproc_locked(dfly_pcpu_t rdd, dfly_pcpu_t dd, 1461 struct lwp *chklp, int worst) 1462 { 1463 struct lwp *lp; 1464 struct rq *q; 1465 u_int32_t *which; 1466 u_int32_t pri; 1467 u_int32_t rtqbits; 1468 u_int32_t tsqbits; 1469 u_int32_t idqbits; 1470 1471 /* 1472 * Select best or worst process. Once selected, clear the bit 1473 * in our local variable (idqbits, tsqbits, or rtqbits) just 1474 * in case we have to loop. 1475 */ 1476 rtqbits = rdd->rtqueuebits; 1477 tsqbits = rdd->queuebits; 1478 idqbits = rdd->idqueuebits; 1479 1480 loopfar: 1481 if (worst) { 1482 if (idqbits) { 1483 pri = bsrl(idqbits); 1484 idqbits &= ~(1U << pri); 1485 q = &rdd->idqueues[pri]; 1486 which = &rdd->idqueuebits; 1487 } else if (tsqbits) { 1488 pri = bsrl(tsqbits); 1489 tsqbits &= ~(1U << pri); 1490 q = &rdd->queues[pri]; 1491 which = &rdd->queuebits; 1492 } else if (rtqbits) { 1493 pri = bsrl(rtqbits); 1494 rtqbits &= ~(1U << pri); 1495 q = &rdd->rtqueues[pri]; 1496 which = &rdd->rtqueuebits; 1497 } else { 1498 return (NULL); 1499 } 1500 lp = TAILQ_LAST(q, rq); 1501 } else { 1502 if (rtqbits) { 1503 pri = bsfl(rtqbits); 1504 rtqbits &= ~(1U << pri); 1505 q = &rdd->rtqueues[pri]; 1506 which = &rdd->rtqueuebits; 1507 } else if (tsqbits) { 1508 pri = bsfl(tsqbits); 1509 tsqbits &= ~(1U << pri); 1510 q = &rdd->queues[pri]; 1511 which = &rdd->queuebits; 1512 } else if (idqbits) { 1513 pri = bsfl(idqbits); 1514 idqbits &= ~(1U << pri); 1515 q = &rdd->idqueues[pri]; 1516 which = &rdd->idqueuebits; 1517 } else { 1518 return (NULL); 1519 } 1520 lp = TAILQ_FIRST(q); 1521 } 1522 KASSERT(lp, ("chooseproc: no lwp on busy queue")); 1523 1524 loopnear: 1525 /* 1526 * If the passed lwp <chklp> is reasonably close to the selected 1527 * lwp <lp>, return NULL (indicating that <chklp> should be kept). 1528 * 1529 * Note that we must error on the side of <chklp> to avoid bouncing 1530 * between threads in the acquire code. 1531 */ 1532 if (chklp) { 1533 if (chklp->lwp_priority < lp->lwp_priority + PPQ) 1534 return(NULL); 1535 } 1536 1537 /* 1538 * When rdd != dd, we have to make sure that the process we 1539 * are pulling is allow to run on our cpu. This alternative 1540 * path is a bit more expensive but its not considered to be 1541 * in the critical path. 1542 */ 1543 if (rdd != dd && CPUMASK_TESTBIT(lp->lwp_cpumask, dd->cpuid) == 0) { 1544 if (worst) 1545 lp = TAILQ_PREV(lp, rq, lwp_procq); 1546 else 1547 lp = TAILQ_NEXT(lp, lwp_procq); 1548 if (lp) 1549 goto loopnear; 1550 goto loopfar; 1551 } 1552 1553 KTR_COND_LOG(usched_chooseproc, 1554 lp->lwp_proc->p_pid == usched_dfly_pid_debug, 1555 lp->lwp_proc->p_pid, 1556 lp->lwp_thread->td_gd->gd_cpuid, 1557 mycpu->gd_cpuid); 1558 1559 KASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) != 0, ("not on runq6!")); 1560 atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ); 1561 TAILQ_REMOVE(q, lp, lwp_procq); 1562 --rdd->runqcount; 1563 if (TAILQ_EMPTY(q)) 1564 *which &= ~(1 << pri); 1565 1566 /* 1567 * If we are choosing a process from rdd with the intent to 1568 * move it to dd, lwp_qcpu must be adjusted while rdd's spinlock 1569 * is still held. 1570 */ 1571 if (rdd != dd) { 1572 if (lp->lwp_mpflags & LWP_MP_ULOAD) { 1573 atomic_add_long(&rdd->uload, -lp->lwp_uload); 1574 atomic_add_int(&rdd->ucount, -1); 1575 } 1576 lp->lwp_qcpu = dd->cpuid; 1577 atomic_add_long(&dd->uload, lp->lwp_uload); 1578 atomic_add_int(&dd->ucount, 1); 1579 atomic_set_int(&lp->lwp_mpflags, LWP_MP_ULOAD); 1580 } 1581 return lp; 1582 } 1583 1584 /* 1585 * USED TO PUSH RUNNABLE LWPS TO THE LEAST LOADED CPU. 1586 * 1587 * Choose a cpu node to schedule lp on, hopefully nearby its current 1588 * node. 1589 * 1590 * We give the current node a modest advantage for obvious reasons. 1591 * 1592 * We also give the node the thread was woken up FROM a slight advantage 1593 * in order to try to schedule paired threads which synchronize/block waiting 1594 * for each other fairly close to each other. Similarly in a network setting 1595 * this feature will also attempt to place a user process near the kernel 1596 * protocol thread that is feeding it data. THIS IS A CRITICAL PART of the 1597 * algorithm as it heuristically groups synchronizing processes for locality 1598 * of reference in multi-socket systems. 1599 * 1600 * We check against running processes and give a big advantage if there 1601 * are none running. 1602 * 1603 * The caller will normally dfly_setrunqueue() lp on the returned queue. 1604 * 1605 * When the topology is known choose a cpu whos group has, in aggregate, 1606 * has the lowest weighted load. 1607 */ 1608 static 1609 dfly_pcpu_t 1610 dfly_choose_best_queue(struct lwp *lp) 1611 { 1612 cpumask_t wakemask; 1613 cpumask_t mask; 1614 cpu_node_t *cpup; 1615 cpu_node_t *cpun; 1616 cpu_node_t *cpub; 1617 dfly_pcpu_t dd = &dfly_pcpu[lp->lwp_qcpu]; 1618 dfly_pcpu_t rdd; 1619 int wakecpu; 1620 int cpuid; 1621 int n; 1622 long load; 1623 long lowest_load; 1624 1625 /* 1626 * When the topology is unknown choose a random cpu that is hopefully 1627 * idle. 1628 */ 1629 if (dd->cpunode == NULL) 1630 return (dfly_choose_queue_simple(dd, lp)); 1631 1632 /* 1633 * Pairing mask 1634 */ 1635 if ((wakecpu = lp->lwp_thread->td_wakefromcpu) >= 0) 1636 wakemask = dfly_pcpu[wakecpu].cpumask; 1637 else 1638 CPUMASK_ASSZERO(wakemask); 1639 1640 /* 1641 * When the topology is known choose a cpu whos group has, in 1642 * aggregate, has the lowest weighted load. 1643 */ 1644 cpup = root_cpu_node; 1645 rdd = dd; 1646 1647 while (cpup) { 1648 /* 1649 * Degenerate case super-root 1650 */ 1651 if (cpup->child_no == 1) { 1652 cpup = cpup->child_node[0]; 1653 continue; 1654 } 1655 1656 /* 1657 * Terminal cpunode 1658 */ 1659 if (cpup->child_no == 0) { 1660 rdd = &dfly_pcpu[BSFCPUMASK(cpup->members)]; 1661 break; 1662 } 1663 1664 cpub = NULL; 1665 lowest_load = 0x7FFFFFFFFFFFFFFFLL; 1666 1667 for (n = 0; n < cpup->child_no; ++n) { 1668 /* 1669 * Accumulate load information for all cpus 1670 * which are members of this node. 1671 */ 1672 int count; 1673 1674 cpun = cpup->child_node[n]; 1675 mask = cpun->members; 1676 CPUMASK_ANDMASK(mask, usched_global_cpumask); 1677 CPUMASK_ANDMASK(mask, smp_active_mask); 1678 CPUMASK_ANDMASK(mask, lp->lwp_cpumask); 1679 if (CPUMASK_TESTZERO(mask)) 1680 continue; 1681 1682 load = 0; 1683 count = 0; 1684 1685 while (CPUMASK_TESTNZERO(mask)) { 1686 cpuid = BSFCPUMASK(mask); 1687 rdd = &dfly_pcpu[cpuid]; 1688 1689 if (rdd->uschedcp == NULL && 1690 rdd->runqcount == 0 && 1691 rdd->gd->gd_tdrunqcount == 0 1692 ) { 1693 load += rdd->uload / 2; 1694 load += rdd->ucount * 1695 usched_dfly_weight3 / 2; 1696 } else { 1697 load += rdd->uload; 1698 load += rdd->ucount * 1699 usched_dfly_weight3; 1700 } 1701 CPUMASK_NANDBIT(mask, cpuid); 1702 ++count; 1703 } 1704 1705 /* 1706 * Compensate if the lp is already accounted for in 1707 * the aggregate uload for this mask set. We want 1708 * to calculate the loads as if lp were not present, 1709 * otherwise the calculation is bogus. 1710 */ 1711 if ((lp->lwp_mpflags & LWP_MP_ULOAD) && 1712 CPUMASK_TESTMASK(dd->cpumask, cpun->members)) { 1713 load -= lp->lwp_uload; 1714 load -= usched_dfly_weight3; /* ucount */ 1715 } 1716 1717 load /= count; 1718 1719 /* 1720 * Advantage the cpu group (lp) is already on. 1721 */ 1722 if (CPUMASK_TESTMASK(cpun->members, dd->cpumask)) 1723 load -= usched_dfly_weight1; 1724 1725 /* 1726 * Advantage nodes with more memory 1727 */ 1728 if (usched_dfly_node_mem) { 1729 load -= cpun->phys_mem * usched_dfly_weight5 / 1730 usched_dfly_node_mem; 1731 } 1732 1733 /* 1734 * Advantage the cpu group we want to pair (lp) to, 1735 * but don't let it go to the exact same cpu as 1736 * the wakecpu target. 1737 * 1738 * We do this by checking whether cpun is a 1739 * terminal node or not. All cpun's at the same 1740 * level will either all be terminal or all not 1741 * terminal. 1742 * 1743 * If it is and we match we disadvantage the load. 1744 * If it is and we don't match we advantage the load. 1745 * 1746 * Also note that we are effectively disadvantaging 1747 * all-but-one by the same amount, so it won't effect 1748 * the weight1 factor for the all-but-one nodes. 1749 */ 1750 if (CPUMASK_TESTMASK(cpun->members, wakemask)) { 1751 if (cpun->child_no != 0) { 1752 /* advantage */ 1753 load -= usched_dfly_weight2; 1754 } else { 1755 if (usched_dfly_features & 0x10) 1756 load += usched_dfly_weight2; 1757 else 1758 load -= usched_dfly_weight2; 1759 } 1760 } 1761 1762 /* 1763 * Calculate the best load 1764 */ 1765 if (cpub == NULL || lowest_load > load || 1766 (lowest_load == load && 1767 CPUMASK_TESTMASK(cpun->members, dd->cpumask)) 1768 ) { 1769 lowest_load = load; 1770 cpub = cpun; 1771 } 1772 } 1773 cpup = cpub; 1774 } 1775 /* Dispatch this outcast to a proper CPU. */ 1776 if (__predict_false(CPUMASK_TESTBIT(lp->lwp_cpumask, rdd->cpuid) == 0)) 1777 rdd = &dfly_pcpu[BSFCPUMASK(lp->lwp_cpumask)]; 1778 if (usched_dfly_chooser > 0) { 1779 --usched_dfly_chooser; /* only N lines */ 1780 kprintf("lp %02d->%02d %s\n", 1781 lp->lwp_qcpu, rdd->cpuid, lp->lwp_proc->p_comm); 1782 } 1783 return (rdd); 1784 } 1785 1786 /* 1787 * USED TO PULL RUNNABLE LWPS FROM THE MOST LOADED CPU. 1788 * 1789 * Choose the worst queue close to dd's cpu node with a non-empty runq 1790 * that is NOT dd. 1791 * 1792 * This is used by the thread chooser when the current cpu's queues are 1793 * empty to steal a thread from another cpu's queue. We want to offload 1794 * the most heavily-loaded queue. 1795 * 1796 * However, we do not want to steal from far-away nodes who themselves 1797 * have idle cpu's that are more suitable to distribute the far-away 1798 * thread to. 1799 */ 1800 static 1801 dfly_pcpu_t 1802 dfly_choose_worst_queue(dfly_pcpu_t dd, int forceit) 1803 { 1804 cpumask_t mask; 1805 cpu_node_t *cpup; 1806 cpu_node_t *cpun; 1807 cpu_node_t *cpub; 1808 dfly_pcpu_t rdd; 1809 int cpuid; 1810 int n; 1811 long load; 1812 long highest_load; 1813 #if 0 1814 int pri; 1815 int hpri; 1816 #endif 1817 1818 /* 1819 * When the topology is unknown choose a random cpu that is hopefully 1820 * idle. 1821 */ 1822 if (dd->cpunode == NULL) { 1823 return (NULL); 1824 } 1825 1826 /* 1827 * When the topology is known choose a cpu whos group has, in 1828 * aggregate, has the highest weighted load. 1829 */ 1830 cpup = root_cpu_node; 1831 rdd = dd; 1832 while (cpup) { 1833 /* 1834 * Degenerate case super-root 1835 */ 1836 if (cpup->child_no == 1) { 1837 cpup = cpup->child_node[0]; 1838 continue; 1839 } 1840 1841 /* 1842 * Terminal cpunode 1843 */ 1844 if (cpup->child_no == 0) { 1845 rdd = &dfly_pcpu[BSFCPUMASK(cpup->members)]; 1846 break; 1847 } 1848 1849 cpub = NULL; 1850 highest_load = -0x7FFFFFFFFFFFFFFFLL; 1851 1852 for (n = 0; n < cpup->child_no; ++n) { 1853 /* 1854 * Accumulate load information for all cpus 1855 * which are members of this node. 1856 */ 1857 int count; 1858 1859 cpun = cpup->child_node[n]; 1860 mask = cpun->members; 1861 CPUMASK_ANDMASK(mask, usched_global_cpumask); 1862 CPUMASK_ANDMASK(mask, smp_active_mask); 1863 if (CPUMASK_TESTZERO(mask)) 1864 continue; 1865 1866 load = 0; 1867 count = 0; 1868 1869 while (CPUMASK_TESTNZERO(mask)) { 1870 cpuid = BSFCPUMASK(mask); 1871 rdd = &dfly_pcpu[cpuid]; 1872 1873 if (rdd->uschedcp == NULL && 1874 rdd->runqcount == 0 && 1875 rdd->gd->gd_tdrunqcount == 0 1876 ) { 1877 load += rdd->uload / 2; 1878 load += rdd->ucount * 1879 usched_dfly_weight3 / 2; 1880 } else { 1881 load += rdd->uload; 1882 load += rdd->ucount * 1883 usched_dfly_weight3; 1884 } 1885 CPUMASK_NANDBIT(mask, cpuid); 1886 ++count; 1887 } 1888 load /= count; 1889 1890 /* 1891 * Advantage the cpu group (dd) is already on. 1892 * 1893 * When choosing the worst queue we reverse the 1894 * sign, but only count half the weight. 1895 * 1896 * weight1 needs to be high enough to be stable, 1897 * but this can also cause it to be too sticky, 1898 * so the iterator which rebalances the load sets 1899 * forceit to ignore it. 1900 */ 1901 if (forceit == 0 && 1902 CPUMASK_TESTMASK(dd->cpumask, cpun->members)) { 1903 load += usched_dfly_weight1 / 2; 1904 } 1905 1906 /* 1907 * Disadvantage nodes with more memory (same sign). 1908 */ 1909 if (usched_dfly_node_mem) { 1910 load -= cpun->phys_mem * usched_dfly_weight5 / 1911 usched_dfly_node_mem; 1912 } 1913 1914 1915 /* 1916 * The best candidate is the one with the worst 1917 * (highest) load. 1918 */ 1919 if (cpub == NULL || highest_load < load || 1920 (highest_load == load && 1921 CPUMASK_TESTMASK(cpun->members, dd->cpumask))) { 1922 highest_load = load; 1923 cpub = cpun; 1924 } 1925 } 1926 cpup = cpub; 1927 } 1928 1929 /* 1930 * We never return our own node (dd), and only return a remote 1931 * node if it's load is significantly worse than ours (i.e. where 1932 * stealing a thread would be considered reasonable). 1933 * 1934 * This also helps us avoid breaking paired threads apart which 1935 * can have disastrous effects on performance. 1936 */ 1937 if (rdd == dd) 1938 return(NULL); 1939 1940 #if 0 1941 hpri = 0; 1942 if (rdd->rtqueuebits && hpri < (pri = bsrl(rdd->rtqueuebits))) 1943 hpri = pri; 1944 if (rdd->queuebits && hpri < (pri = bsrl(rdd->queuebits))) 1945 hpri = pri; 1946 if (rdd->idqueuebits && hpri < (pri = bsrl(rdd->idqueuebits))) 1947 hpri = pri; 1948 hpri *= PPQ; 1949 if (rdd->uload - hpri < dd->uload + hpri) 1950 return(NULL); 1951 #endif 1952 return (rdd); 1953 } 1954 1955 static 1956 dfly_pcpu_t 1957 dfly_choose_queue_simple(dfly_pcpu_t dd, struct lwp *lp) 1958 { 1959 dfly_pcpu_t rdd; 1960 cpumask_t tmpmask; 1961 cpumask_t mask; 1962 int cpubase; 1963 int cpuid; 1964 1965 /* 1966 * Fallback to the original heuristic, select random cpu, 1967 * first checking the cpus not currently running a user thread. 1968 * 1969 * Use cpuid as the base cpu in our scan, first checking 1970 * cpuid...(ncpus-1), then 0...(cpuid-1). This avoid favoring 1971 * lower-numbered cpus. 1972 */ 1973 ++dd->scancpu; /* SMP race ok */ 1974 mask = dfly_rdyprocmask; 1975 CPUMASK_NANDMASK(mask, dfly_curprocmask); 1976 CPUMASK_ANDMASK(mask, lp->lwp_cpumask); 1977 CPUMASK_ANDMASK(mask, smp_active_mask); 1978 CPUMASK_ANDMASK(mask, usched_global_cpumask); 1979 1980 cpubase = (int)(dd->scancpu % ncpus); 1981 CPUMASK_ASSBMASK(tmpmask, cpubase); 1982 CPUMASK_INVMASK(tmpmask); 1983 CPUMASK_ANDMASK(tmpmask, mask); 1984 while (CPUMASK_TESTNZERO(tmpmask)) { 1985 cpuid = BSFCPUMASK(tmpmask); 1986 rdd = &dfly_pcpu[cpuid]; 1987 1988 if ((rdd->upri & ~PPQMASK) >= (lp->lwp_priority & ~PPQMASK)) 1989 goto found; 1990 CPUMASK_NANDBIT(tmpmask, cpuid); 1991 } 1992 1993 CPUMASK_ASSBMASK(tmpmask, cpubase); 1994 CPUMASK_ANDMASK(tmpmask, mask); 1995 while (CPUMASK_TESTNZERO(tmpmask)) { 1996 cpuid = BSFCPUMASK(tmpmask); 1997 rdd = &dfly_pcpu[cpuid]; 1998 1999 if ((rdd->upri & ~PPQMASK) >= (lp->lwp_priority & ~PPQMASK)) 2000 goto found; 2001 CPUMASK_NANDBIT(tmpmask, cpuid); 2002 } 2003 2004 /* 2005 * Then cpus which might have a currently running lp 2006 */ 2007 mask = dfly_rdyprocmask; 2008 CPUMASK_ANDMASK(mask, dfly_curprocmask); 2009 CPUMASK_ANDMASK(mask, lp->lwp_cpumask); 2010 CPUMASK_ANDMASK(mask, smp_active_mask); 2011 CPUMASK_ANDMASK(mask, usched_global_cpumask); 2012 2013 CPUMASK_ASSBMASK(tmpmask, cpubase); 2014 CPUMASK_INVMASK(tmpmask); 2015 CPUMASK_ANDMASK(tmpmask, mask); 2016 while (CPUMASK_TESTNZERO(tmpmask)) { 2017 cpuid = BSFCPUMASK(tmpmask); 2018 rdd = &dfly_pcpu[cpuid]; 2019 2020 if ((rdd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) 2021 goto found; 2022 CPUMASK_NANDBIT(tmpmask, cpuid); 2023 } 2024 2025 CPUMASK_ASSBMASK(tmpmask, cpubase); 2026 CPUMASK_ANDMASK(tmpmask, mask); 2027 while (CPUMASK_TESTNZERO(tmpmask)) { 2028 cpuid = BSFCPUMASK(tmpmask); 2029 rdd = &dfly_pcpu[cpuid]; 2030 2031 if ((rdd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) 2032 goto found; 2033 CPUMASK_NANDBIT(tmpmask, cpuid); 2034 } 2035 2036 /* 2037 * If we cannot find a suitable cpu we round-robin using scancpu. 2038 * Other cpus will pickup as they release their current lwps or 2039 * become ready. 2040 * 2041 * Avoid a degenerate system lockup case if usched_global_cpumask 2042 * is set to 0 or otherwise does not cover lwp_cpumask. 2043 * 2044 * We only kick the target helper thread in this case, we do not 2045 * set the user resched flag because 2046 */ 2047 cpuid = cpubase; 2048 if (CPUMASK_TESTBIT(lp->lwp_cpumask, cpuid) == 0) 2049 cpuid = BSFCPUMASK(lp->lwp_cpumask); 2050 else if (CPUMASK_TESTBIT(usched_global_cpumask, cpuid) == 0) 2051 cpuid = 0; 2052 rdd = &dfly_pcpu[cpuid]; 2053 found: 2054 return (rdd); 2055 } 2056 2057 static 2058 void 2059 dfly_need_user_resched_remote(void *dummy) 2060 { 2061 globaldata_t gd = mycpu; 2062 dfly_pcpu_t dd = &dfly_pcpu[gd->gd_cpuid]; 2063 2064 /* 2065 * Flag reschedule needed 2066 */ 2067 need_user_resched(); 2068 2069 /* 2070 * If no user thread is currently running we need to kick the helper 2071 * on our cpu to recover. Otherwise the cpu will never schedule 2072 * anything again. 2073 * 2074 * We cannot schedule the process ourselves because this is an 2075 * IPI callback and we cannot acquire spinlocks in an IPI callback. 2076 * 2077 * Call wakeup_mycpu to avoid sending IPIs to other CPUs 2078 */ 2079 if (dd->uschedcp == NULL && (dd->flags & DFLY_PCPU_RDYMASK)) { 2080 ATOMIC_CPUMASK_NANDBIT(dfly_rdyprocmask, gd->gd_cpuid); 2081 dd->flags &= ~DFLY_PCPU_RDYMASK; 2082 wakeup_mycpu(dd->helper_thread); 2083 } 2084 } 2085 2086 /* 2087 * dfly_remrunqueue_locked() removes a given process from the run queue 2088 * that it is on, clearing the queue busy bit if it becomes empty. 2089 * 2090 * Note that user process scheduler is different from the LWKT schedule. 2091 * The user process scheduler only manages user processes but it uses LWKT 2092 * underneath, and a user process operating in the kernel will often be 2093 * 'released' from our management. 2094 * 2095 * uload is NOT adjusted here. It is only adjusted if the lwkt_thread goes 2096 * to sleep or the lwp is moved to a different runq. 2097 */ 2098 static void 2099 dfly_remrunqueue_locked(dfly_pcpu_t rdd, struct lwp *lp) 2100 { 2101 struct rq *q; 2102 u_int32_t *which; 2103 u_int8_t pri; 2104 2105 KKASSERT(rdd->runqcount >= 0); 2106 2107 pri = lp->lwp_rqindex; 2108 2109 switch(lp->lwp_rqtype) { 2110 case RTP_PRIO_NORMAL: 2111 q = &rdd->queues[pri]; 2112 which = &rdd->queuebits; 2113 break; 2114 case RTP_PRIO_REALTIME: 2115 case RTP_PRIO_FIFO: 2116 q = &rdd->rtqueues[pri]; 2117 which = &rdd->rtqueuebits; 2118 break; 2119 case RTP_PRIO_IDLE: 2120 q = &rdd->idqueues[pri]; 2121 which = &rdd->idqueuebits; 2122 break; 2123 default: 2124 panic("remrunqueue: invalid rtprio type"); 2125 /* NOT REACHED */ 2126 } 2127 KKASSERT(lp->lwp_mpflags & LWP_MP_ONRUNQ); 2128 atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ); 2129 TAILQ_REMOVE(q, lp, lwp_procq); 2130 --rdd->runqcount; 2131 if (TAILQ_EMPTY(q)) { 2132 KASSERT((*which & (1 << pri)) != 0, 2133 ("remrunqueue: remove from empty queue")); 2134 *which &= ~(1 << pri); 2135 } 2136 } 2137 2138 /* 2139 * dfly_setrunqueue_locked() 2140 * 2141 * Add a process whos rqtype and rqindex had previously been calculated 2142 * onto the appropriate run queue. Determine if the addition requires 2143 * a reschedule on a cpu and return the cpuid or -1. 2144 * 2145 * NOTE: Lower priorities are better priorities. 2146 * 2147 * NOTE ON ULOAD: This variable specifies the aggregate load on a cpu, the 2148 * sum of the rough lwp_priority for all running and runnable 2149 * processes. Lower priority processes (higher lwp_priority 2150 * values) actually DO count as more load, not less, because 2151 * these are the programs which require the most care with 2152 * regards to cpu selection. 2153 */ 2154 static void 2155 dfly_setrunqueue_locked(dfly_pcpu_t rdd, struct lwp *lp) 2156 { 2157 u_int32_t *which; 2158 struct rq *q; 2159 int pri; 2160 2161 KKASSERT(lp->lwp_qcpu == rdd->cpuid); 2162 2163 if ((lp->lwp_mpflags & LWP_MP_ULOAD) == 0) { 2164 atomic_set_int(&lp->lwp_mpflags, LWP_MP_ULOAD); 2165 atomic_add_long(&rdd->uload, lp->lwp_uload); 2166 atomic_add_int(&rdd->ucount, 1); 2167 } 2168 2169 pri = lp->lwp_rqindex; 2170 2171 switch(lp->lwp_rqtype) { 2172 case RTP_PRIO_NORMAL: 2173 q = &rdd->queues[pri]; 2174 which = &rdd->queuebits; 2175 break; 2176 case RTP_PRIO_REALTIME: 2177 case RTP_PRIO_FIFO: 2178 q = &rdd->rtqueues[pri]; 2179 which = &rdd->rtqueuebits; 2180 break; 2181 case RTP_PRIO_IDLE: 2182 q = &rdd->idqueues[pri]; 2183 which = &rdd->idqueuebits; 2184 break; 2185 default: 2186 panic("remrunqueue: invalid rtprio type"); 2187 /* NOT REACHED */ 2188 } 2189 2190 /* 2191 * Place us on the selected queue. Determine if we should be 2192 * placed at the head of the queue or at the end. 2193 * 2194 * We are placed at the tail if our round-robin count has expired, 2195 * or is about to expire and the system thinks its a good place to 2196 * round-robin, or there is already a next thread on the queue 2197 * (it might be trying to pick up where it left off and we don't 2198 * want to interfere). 2199 */ 2200 KKASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0); 2201 atomic_set_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ); 2202 ++rdd->runqcount; 2203 2204 if (lp->lwp_rrcount >= usched_dfly_rrinterval || 2205 (lp->lwp_rrcount >= usched_dfly_rrinterval / 2 && 2206 (lp->lwp_thread->td_mpflags & TDF_MP_BATCH_DEMARC)) 2207 ) { 2208 /* 2209 * Place on tail 2210 */ 2211 atomic_clear_int(&lp->lwp_thread->td_mpflags, 2212 TDF_MP_BATCH_DEMARC); 2213 lp->lwp_rrcount = 0; 2214 TAILQ_INSERT_TAIL(q, lp, lwp_procq); 2215 } else { 2216 /* 2217 * Retain rrcount and place on head. Count is retained 2218 * even if the queue is empty. 2219 */ 2220 TAILQ_INSERT_HEAD(q, lp, lwp_procq); 2221 } 2222 *which |= 1 << pri; 2223 } 2224 2225 /* 2226 * For SMP systems a user scheduler helper thread is created for each 2227 * cpu and is used to allow one cpu to wakeup another for the purposes of 2228 * scheduling userland threads from setrunqueue(). 2229 * 2230 * UP systems do not need the helper since there is only one cpu. 2231 * 2232 * We can't use the idle thread for this because we might block. 2233 * Additionally, doing things this way allows us to HLT idle cpus 2234 * on MP systems. 2235 */ 2236 static void 2237 dfly_helper_thread(void *dummy) 2238 { 2239 globaldata_t gd; 2240 dfly_pcpu_t dd; 2241 dfly_pcpu_t rdd; 2242 struct lwp *nlp; 2243 cpumask_t mask; 2244 int cpuid; 2245 2246 gd = mycpu; 2247 cpuid = gd->gd_cpuid; /* doesn't change */ 2248 mask = gd->gd_cpumask; /* doesn't change */ 2249 dd = &dfly_pcpu[cpuid]; 2250 2251 /* 2252 * Since we only want to be woken up only when no user processes 2253 * are scheduled on a cpu, run at an ultra low priority. 2254 */ 2255 lwkt_setpri_self(TDPRI_USER_SCHEDULER); 2256 2257 tsleep(dd->helper_thread, 0, "schslp", 0); 2258 2259 for (;;) { 2260 /* 2261 * We use the LWKT deschedule-interlock trick to avoid racing 2262 * dfly_rdyprocmask. This means we cannot block through to the 2263 * manual lwkt_switch() call we make below. 2264 */ 2265 crit_enter_gd(gd); 2266 tsleep_interlock(dd->helper_thread, 0); 2267 2268 spin_lock(&dd->spin); 2269 if ((dd->flags & DFLY_PCPU_RDYMASK) == 0) { 2270 ATOMIC_CPUMASK_ORMASK(dfly_rdyprocmask, mask); 2271 dd->flags |= DFLY_PCPU_RDYMASK; 2272 } 2273 clear_user_resched(); /* This satisfied the reschedule request */ 2274 #if 0 2275 dd->rrcount = 0; /* Reset the round-robin counter */ 2276 #endif 2277 2278 if (dd->runqcount || dd->uschedcp != NULL) { 2279 /* 2280 * Threads are available. A thread may or may not be 2281 * currently scheduled. Get the best thread already queued 2282 * to this cpu. 2283 */ 2284 nlp = dfly_chooseproc_locked(dd, dd, dd->uschedcp, 0); 2285 if (nlp) { 2286 if ((dd->flags & DFLY_PCPU_CURMASK) == 0) { 2287 ATOMIC_CPUMASK_ORMASK(dfly_curprocmask, mask); 2288 dd->flags |= DFLY_PCPU_CURMASK; 2289 } 2290 dd->upri = nlp->lwp_priority; 2291 dd->uschedcp = nlp; 2292 #if 0 2293 dd->rrcount = 0; /* reset round robin */ 2294 #endif 2295 spin_unlock(&dd->spin); 2296 lwkt_acquire(nlp->lwp_thread); 2297 lwkt_schedule(nlp->lwp_thread); 2298 } else { 2299 /* 2300 * This situation should not occur because we had 2301 * at least one thread available. 2302 */ 2303 spin_unlock(&dd->spin); 2304 } 2305 } else if (usched_dfly_features & 0x01) { 2306 /* 2307 * This cpu is devoid of runnable threads, steal a thread 2308 * from another cpu. Since we're stealing, might as well 2309 * load balance at the same time. 2310 * 2311 * We choose the highest-loaded thread from the worst queue. 2312 * 2313 * NOTE! This function only returns a non-NULL rdd when 2314 * another cpu's queue is obviously overloaded. We 2315 * do not want to perform the type of rebalancing 2316 * the schedclock does here because it would result 2317 * in insane process pulling when 'steady' state is 2318 * partially unbalanced (e.g. 6 runnables and only 2319 * 4 cores). 2320 */ 2321 rdd = dfly_choose_worst_queue(dd, 0); 2322 if (rdd && dd->uload + usched_dfly_weight6 < rdd->uload && 2323 spin_trylock(&rdd->spin)) { 2324 nlp = dfly_chooseproc_locked(rdd, dd, NULL, 1); 2325 spin_unlock(&rdd->spin); 2326 } else { 2327 nlp = NULL; 2328 } 2329 if (nlp) { 2330 if ((dd->flags & DFLY_PCPU_CURMASK) == 0) { 2331 ATOMIC_CPUMASK_ORMASK(dfly_curprocmask, mask); 2332 dd->flags |= DFLY_PCPU_CURMASK; 2333 } 2334 dd->upri = nlp->lwp_priority; 2335 dd->uschedcp = nlp; 2336 #if 0 2337 dd->rrcount = 0; /* reset round robin */ 2338 #endif 2339 spin_unlock(&dd->spin); 2340 lwkt_acquire(nlp->lwp_thread); 2341 lwkt_schedule(nlp->lwp_thread); 2342 } else { 2343 /* 2344 * Leave the thread on our run queue. Another 2345 * scheduler will try to pull it later. 2346 */ 2347 spin_unlock(&dd->spin); 2348 } 2349 } else { 2350 /* 2351 * devoid of runnable threads and not allowed to steal 2352 * any. 2353 */ 2354 spin_unlock(&dd->spin); 2355 } 2356 2357 /* 2358 * We're descheduled unless someone scheduled us. Switch away. 2359 * Exiting the critical section will cause splz() to be called 2360 * for us if interrupts and such are pending. 2361 */ 2362 crit_exit_gd(gd); 2363 tsleep(dd->helper_thread, PINTERLOCKED, "schslp", 0); 2364 } 2365 } 2366 2367 #if 0 2368 static int 2369 sysctl_usched_dfly_stick_to_level(SYSCTL_HANDLER_ARGS) 2370 { 2371 int error, new_val; 2372 2373 new_val = usched_dfly_stick_to_level; 2374 2375 error = sysctl_handle_int(oidp, &new_val, 0, req); 2376 if (error != 0 || req->newptr == NULL) 2377 return (error); 2378 if (new_val > cpu_topology_levels_number - 1 || new_val < 0) 2379 return (EINVAL); 2380 usched_dfly_stick_to_level = new_val; 2381 return (0); 2382 } 2383 #endif 2384 2385 /* 2386 * Setup the queues and scheduler helpers (scheduler helpers are SMP only). 2387 * Note that curprocmask bit 0 has already been cleared by rqinit() and 2388 * we should not mess with it further. 2389 */ 2390 static void 2391 usched_dfly_cpu_init(void) 2392 { 2393 int i; 2394 int j; 2395 int smt_not_supported = 0; 2396 int cache_coherent_not_supported = 0; 2397 2398 if (bootverbose) 2399 kprintf("Start usched_dfly helpers on cpus:\n"); 2400 2401 sysctl_ctx_init(&usched_dfly_sysctl_ctx); 2402 usched_dfly_sysctl_tree = 2403 SYSCTL_ADD_NODE(&usched_dfly_sysctl_ctx, 2404 SYSCTL_STATIC_CHILDREN(_kern), OID_AUTO, 2405 "usched_dfly", CTLFLAG_RD, 0, ""); 2406 2407 usched_dfly_node_mem = get_highest_node_memory(); 2408 2409 for (i = 0; i < ncpus; ++i) { 2410 dfly_pcpu_t dd = &dfly_pcpu[i]; 2411 cpumask_t mask; 2412 2413 CPUMASK_ASSBIT(mask, i); 2414 if (CPUMASK_TESTMASK(mask, smp_active_mask) == 0) 2415 continue; 2416 2417 spin_init(&dd->spin, "uschedcpuinit"); 2418 dd->cpunode = get_cpu_node_by_cpuid(i); 2419 dd->cpuid = i; 2420 dd->gd = globaldata_find(i); 2421 CPUMASK_ASSBIT(dd->cpumask, i); 2422 for (j = 0; j < NQS; j++) { 2423 TAILQ_INIT(&dd->queues[j]); 2424 TAILQ_INIT(&dd->rtqueues[j]); 2425 TAILQ_INIT(&dd->idqueues[j]); 2426 } 2427 ATOMIC_CPUMASK_NANDBIT(dfly_curprocmask, 0); 2428 if (i == 0) 2429 dd->flags &= ~DFLY_PCPU_CURMASK; 2430 2431 if (dd->cpunode == NULL) { 2432 smt_not_supported = 1; 2433 cache_coherent_not_supported = 1; 2434 if (bootverbose) 2435 kprintf (" cpu%d - WARNING: No CPU NODE " 2436 "found for cpu\n", i); 2437 } else { 2438 switch (dd->cpunode->type) { 2439 case THREAD_LEVEL: 2440 if (bootverbose) 2441 kprintf (" cpu%d - HyperThreading " 2442 "available. Core siblings: ", 2443 i); 2444 break; 2445 case CORE_LEVEL: 2446 smt_not_supported = 1; 2447 2448 if (bootverbose) 2449 kprintf (" cpu%d - No HT available, " 2450 "multi-core/physical " 2451 "cpu. Physical siblings: ", 2452 i); 2453 break; 2454 case CHIP_LEVEL: 2455 smt_not_supported = 1; 2456 2457 if (bootverbose) 2458 kprintf (" cpu%d - No HT available, " 2459 "single-core/physical cpu. " 2460 "Package siblings: ", 2461 i); 2462 break; 2463 default: 2464 /* Let's go for safe defaults here */ 2465 smt_not_supported = 1; 2466 cache_coherent_not_supported = 1; 2467 if (bootverbose) 2468 kprintf (" cpu%d - Unknown cpunode->" 2469 "type=%u. siblings: ", 2470 i, 2471 (u_int)dd->cpunode->type); 2472 break; 2473 } 2474 2475 if (bootverbose) { 2476 if (dd->cpunode->parent_node != NULL) { 2477 kprint_cpuset(&dd->cpunode-> 2478 parent_node->members); 2479 kprintf("\n"); 2480 } else { 2481 kprintf(" no siblings\n"); 2482 } 2483 } 2484 } 2485 2486 lwkt_create(dfly_helper_thread, NULL, &dd->helper_thread, NULL, 2487 0, i, "usched %d", i); 2488 2489 /* 2490 * Allow user scheduling on the target cpu. cpu #0 has already 2491 * been enabled in rqinit(). 2492 */ 2493 if (i) { 2494 ATOMIC_CPUMASK_NANDMASK(dfly_curprocmask, mask); 2495 dd->flags &= ~DFLY_PCPU_CURMASK; 2496 } 2497 if ((dd->flags & DFLY_PCPU_RDYMASK) == 0) { 2498 ATOMIC_CPUMASK_ORMASK(dfly_rdyprocmask, mask); 2499 dd->flags |= DFLY_PCPU_RDYMASK; 2500 } 2501 dd->upri = PRIBASE_NULL; 2502 2503 } 2504 2505 /* usched_dfly sysctl configurable parameters */ 2506 2507 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx, 2508 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2509 OID_AUTO, "rrinterval", CTLFLAG_RW, 2510 &usched_dfly_rrinterval, 0, ""); 2511 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx, 2512 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2513 OID_AUTO, "decay", CTLFLAG_RW, 2514 &usched_dfly_decay, 0, "Extra decay when not running"); 2515 2516 /* Add enable/disable option for SMT scheduling if supported */ 2517 if (smt_not_supported) { 2518 usched_dfly_smt = 0; 2519 SYSCTL_ADD_STRING(&usched_dfly_sysctl_ctx, 2520 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2521 OID_AUTO, "smt", CTLFLAG_RD, 2522 "NOT SUPPORTED", 0, "SMT NOT SUPPORTED"); 2523 } else { 2524 usched_dfly_smt = 1; 2525 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx, 2526 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2527 OID_AUTO, "smt", CTLFLAG_RW, 2528 &usched_dfly_smt, 0, "Enable SMT scheduling"); 2529 } 2530 2531 /* 2532 * Add enable/disable option for cache coherent scheduling 2533 * if supported 2534 */ 2535 if (cache_coherent_not_supported) { 2536 usched_dfly_cache_coherent = 0; 2537 SYSCTL_ADD_STRING(&usched_dfly_sysctl_ctx, 2538 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2539 OID_AUTO, "cache_coherent", CTLFLAG_RD, 2540 "NOT SUPPORTED", 0, 2541 "Cache coherence NOT SUPPORTED"); 2542 } else { 2543 usched_dfly_cache_coherent = 1; 2544 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx, 2545 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2546 OID_AUTO, "cache_coherent", CTLFLAG_RW, 2547 &usched_dfly_cache_coherent, 0, 2548 "Enable/Disable cache coherent scheduling"); 2549 2550 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx, 2551 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2552 OID_AUTO, "weight1", CTLFLAG_RW, 2553 &usched_dfly_weight1, 200, 2554 "Weight selection for current cpu"); 2555 2556 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx, 2557 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2558 OID_AUTO, "weight2", CTLFLAG_RW, 2559 &usched_dfly_weight2, 180, 2560 "Weight selection for wakefrom cpu"); 2561 2562 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx, 2563 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2564 OID_AUTO, "weight3", CTLFLAG_RW, 2565 &usched_dfly_weight3, 40, 2566 "Weight selection for num threads on queue"); 2567 2568 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx, 2569 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2570 OID_AUTO, "weight4", CTLFLAG_RW, 2571 &usched_dfly_weight4, 160, 2572 "Availability of other idle cpus"); 2573 2574 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx, 2575 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2576 OID_AUTO, "weight5", CTLFLAG_RW, 2577 &usched_dfly_weight5, 50, 2578 "Memory attached to node"); 2579 2580 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx, 2581 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2582 OID_AUTO, "weight6", CTLFLAG_RW, 2583 &usched_dfly_weight6, 150, 2584 "Transfer weight"); 2585 2586 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx, 2587 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2588 OID_AUTO, "fast_resched", CTLFLAG_RW, 2589 &usched_dfly_fast_resched, 0, 2590 "Availability of other idle cpus"); 2591 2592 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx, 2593 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2594 OID_AUTO, "features", CTLFLAG_RW, 2595 &usched_dfly_features, 0x8F, 2596 "Allow pulls into empty queues"); 2597 2598 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx, 2599 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2600 OID_AUTO, "swmask", CTLFLAG_RW, 2601 &usched_dfly_swmask, ~PPQMASK, 2602 "Queue mask to force thread switch"); 2603 2604 #if 0 2605 SYSCTL_ADD_PROC(&usched_dfly_sysctl_ctx, 2606 SYSCTL_CHILDREN(usched_dfly_sysctl_tree), 2607 OID_AUTO, "stick_to_level", 2608 CTLTYPE_INT | CTLFLAG_RW, 2609 NULL, sizeof usched_dfly_stick_to_level, 2610 sysctl_usched_dfly_stick_to_level, "I", 2611 "Stick a process to this level. See sysctl" 2612 "paremter hw.cpu_topology.level_description"); 2613 #endif 2614 } 2615 } 2616 SYSINIT(uschedtd, SI_BOOT2_USCHED, SI_ORDER_SECOND, 2617 usched_dfly_cpu_init, NULL); 2618