1 /* $NetBSD: kern_timeout.c,v 1.37 2008/04/22 12:04:22 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Copyright (c) 2001 Thomas Nordin <nordin@openbsd.org> 41 * Copyright (c) 2000-2001 Artur Grabowski <art@openbsd.org> 42 * All rights reserved. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 48 * 1. Redistributions of source code must retain the above copyright 49 * notice, this list of conditions and the following disclaimer. 50 * 2. Redistributions in binary form must reproduce the above copyright 51 * notice, this list of conditions and the following disclaimer in the 52 * documentation and/or other materials provided with the distribution. 53 * 3. The name of the author may not be used to endorse or promote products 54 * derived from this software without specific prior written permission. 55 * 56 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 57 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY 58 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 59 * THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 60 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 61 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 62 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 63 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 64 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 65 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 66 */ 67 68 #include <sys/cdefs.h> 69 __KERNEL_RCSID(0, "$NetBSD: kern_timeout.c,v 1.37 2008/04/22 12:04:22 ad Exp $"); 70 71 /* 72 * Timeouts are kept in a hierarchical timing wheel. The c_time is the 73 * value of c_cpu->cc_ticks when the timeout should be called. There are 74 * four levels with 256 buckets each. See 'Scheme 7' in "Hashed and 75 * Hierarchical Timing Wheels: Efficient Data Structures for Implementing 76 * a Timer Facility" by George Varghese and Tony Lauck. 77 * 78 * Some of the "math" in here is a bit tricky. We have to beware of 79 * wrapping ints. 80 * 81 * We use the fact that any element added to the queue must be added with 82 * a positive time. That means that any element `to' on the queue cannot 83 * be scheduled to timeout further in time than INT_MAX, but c->c_time can 84 * be positive or negative so comparing it with anything is dangerous. 85 * The only way we can use the c->c_time value in any predictable way is 86 * when we calculate how far in the future `to' will timeout - "c->c_time 87 * - c->c_cpu->cc_ticks". The result will always be positive for future 88 * timeouts and 0 or negative for due timeouts. 89 */ 90 91 #define _CALLOUT_PRIVATE 92 93 #include <sys/param.h> 94 #include <sys/systm.h> 95 #include <sys/kernel.h> 96 #include <sys/callout.h> 97 #include <sys/mutex.h> 98 #include <sys/proc.h> 99 #include <sys/sleepq.h> 100 #include <sys/syncobj.h> 101 #include <sys/evcnt.h> 102 #include <sys/intr.h> 103 #include <sys/cpu.h> 104 #include <sys/kmem.h> 105 106 #ifdef DDB 107 #include <machine/db_machdep.h> 108 #include <ddb/db_interface.h> 109 #include <ddb/db_access.h> 110 #include <ddb/db_sym.h> 111 #include <ddb/db_output.h> 112 #endif 113 114 #define BUCKETS 1024 115 #define WHEELSIZE 256 116 #define WHEELMASK 255 117 #define WHEELBITS 8 118 119 #define MASKWHEEL(wheel, time) (((time) >> ((wheel)*WHEELBITS)) & WHEELMASK) 120 121 #define BUCKET(cc, rel, abs) \ 122 (((rel) <= (1 << (2*WHEELBITS))) \ 123 ? ((rel) <= (1 << WHEELBITS)) \ 124 ? &(cc)->cc_wheel[MASKWHEEL(0, (abs))] \ 125 : &(cc)->cc_wheel[MASKWHEEL(1, (abs)) + WHEELSIZE] \ 126 : ((rel) <= (1 << (3*WHEELBITS))) \ 127 ? &(cc)->cc_wheel[MASKWHEEL(2, (abs)) + 2*WHEELSIZE] \ 128 : &(cc)->cc_wheel[MASKWHEEL(3, (abs)) + 3*WHEELSIZE]) 129 130 #define MOVEBUCKET(cc, wheel, time) \ 131 CIRCQ_APPEND(&(cc)->cc_todo, \ 132 &(cc)->cc_wheel[MASKWHEEL((wheel), (time)) + (wheel)*WHEELSIZE]) 133 134 /* 135 * Circular queue definitions. 136 */ 137 138 #define CIRCQ_INIT(list) \ 139 do { \ 140 (list)->cq_next_l = (list); \ 141 (list)->cq_prev_l = (list); \ 142 } while (/*CONSTCOND*/0) 143 144 #define CIRCQ_INSERT(elem, list) \ 145 do { \ 146 (elem)->cq_prev_e = (list)->cq_prev_e; \ 147 (elem)->cq_next_l = (list); \ 148 (list)->cq_prev_l->cq_next_l = (elem); \ 149 (list)->cq_prev_l = (elem); \ 150 } while (/*CONSTCOND*/0) 151 152 #define CIRCQ_APPEND(fst, snd) \ 153 do { \ 154 if (!CIRCQ_EMPTY(snd)) { \ 155 (fst)->cq_prev_l->cq_next_l = (snd)->cq_next_l; \ 156 (snd)->cq_next_l->cq_prev_l = (fst)->cq_prev_l; \ 157 (snd)->cq_prev_l->cq_next_l = (fst); \ 158 (fst)->cq_prev_l = (snd)->cq_prev_l; \ 159 CIRCQ_INIT(snd); \ 160 } \ 161 } while (/*CONSTCOND*/0) 162 163 #define CIRCQ_REMOVE(elem) \ 164 do { \ 165 (elem)->cq_next_l->cq_prev_e = (elem)->cq_prev_e; \ 166 (elem)->cq_prev_l->cq_next_e = (elem)->cq_next_e; \ 167 } while (/*CONSTCOND*/0) 168 169 #define CIRCQ_FIRST(list) ((list)->cq_next_e) 170 #define CIRCQ_NEXT(elem) ((elem)->cq_next_e) 171 #define CIRCQ_LAST(elem,list) ((elem)->cq_next_l == (list)) 172 #define CIRCQ_EMPTY(list) ((list)->cq_next_l == (list)) 173 174 static void callout_softclock(void *); 175 176 struct callout_cpu { 177 kmutex_t cc_lock; 178 sleepq_t cc_sleepq; 179 u_int cc_nwait; 180 u_int cc_ticks; 181 lwp_t *cc_lwp; 182 callout_impl_t *cc_active; 183 callout_impl_t *cc_cancel; 184 struct evcnt cc_ev_late; 185 struct evcnt cc_ev_block; 186 struct callout_circq cc_todo; /* Worklist */ 187 struct callout_circq cc_wheel[BUCKETS]; /* Queues of timeouts */ 188 char cc_name1[12]; 189 char cc_name2[12]; 190 }; 191 192 static struct callout_cpu callout_cpu0; 193 static void *callout_sih; 194 195 static inline kmutex_t * 196 callout_lock(callout_impl_t *c) 197 { 198 kmutex_t *lock; 199 200 for (;;) { 201 lock = &c->c_cpu->cc_lock; 202 mutex_spin_enter(lock); 203 if (__predict_true(lock == &c->c_cpu->cc_lock)) 204 return lock; 205 mutex_spin_exit(lock); 206 } 207 } 208 209 /* 210 * callout_startup: 211 * 212 * Initialize the callout facility, called at system startup time. 213 * Do just enough to allow callouts to be safely registered. 214 */ 215 void 216 callout_startup(void) 217 { 218 struct callout_cpu *cc; 219 int b; 220 221 KASSERT(curcpu()->ci_data.cpu_callout == NULL); 222 223 cc = &callout_cpu0; 224 mutex_init(&cc->cc_lock, MUTEX_DEFAULT, IPL_SCHED); 225 CIRCQ_INIT(&cc->cc_todo); 226 for (b = 0; b < BUCKETS; b++) 227 CIRCQ_INIT(&cc->cc_wheel[b]); 228 curcpu()->ci_data.cpu_callout = cc; 229 } 230 231 /* 232 * callout_init_cpu: 233 * 234 * Per-CPU initialization. 235 */ 236 void 237 callout_init_cpu(struct cpu_info *ci) 238 { 239 struct callout_cpu *cc; 240 int b; 241 242 KASSERT(sizeof(callout_impl_t) <= sizeof(callout_t)); 243 244 if ((cc = ci->ci_data.cpu_callout) == NULL) { 245 cc = kmem_zalloc(sizeof(*cc), KM_SLEEP); 246 if (cc == NULL) 247 panic("callout_init_cpu (1)"); 248 mutex_init(&cc->cc_lock, MUTEX_DEFAULT, IPL_SCHED); 249 CIRCQ_INIT(&cc->cc_todo); 250 for (b = 0; b < BUCKETS; b++) 251 CIRCQ_INIT(&cc->cc_wheel[b]); 252 } else { 253 /* Boot CPU, one time only. */ 254 callout_sih = softint_establish(SOFTINT_CLOCK | SOFTINT_MPSAFE, 255 callout_softclock, NULL); 256 if (callout_sih == NULL) 257 panic("callout_init_cpu (2)"); 258 } 259 260 sleepq_init(&cc->cc_sleepq, &cc->cc_lock); 261 262 snprintf(cc->cc_name1, sizeof(cc->cc_name1), "late/%u", 263 cpu_index(ci)); 264 evcnt_attach_dynamic(&cc->cc_ev_late, EVCNT_TYPE_MISC, 265 NULL, "callout", cc->cc_name1); 266 267 snprintf(cc->cc_name2, sizeof(cc->cc_name2), "wait/%u", 268 cpu_index(ci)); 269 evcnt_attach_dynamic(&cc->cc_ev_block, EVCNT_TYPE_MISC, 270 NULL, "callout", cc->cc_name2); 271 272 ci->ci_data.cpu_callout = cc; 273 } 274 275 /* 276 * callout_init: 277 * 278 * Initialize a callout structure. This must be quick, so we fill 279 * only the minimum number of fields. 280 */ 281 void 282 callout_init(callout_t *cs, u_int flags) 283 { 284 callout_impl_t *c = (callout_impl_t *)cs; 285 struct callout_cpu *cc; 286 287 KASSERT((flags & ~CALLOUT_FLAGMASK) == 0); 288 289 cc = curcpu()->ci_data.cpu_callout; 290 c->c_func = NULL; 291 c->c_magic = CALLOUT_MAGIC; 292 if (__predict_true((flags & CALLOUT_MPSAFE) != 0 && cc != NULL)) { 293 c->c_flags = flags; 294 c->c_cpu = cc; 295 return; 296 } 297 c->c_flags = flags | CALLOUT_BOUND; 298 c->c_cpu = &callout_cpu0; 299 } 300 301 /* 302 * callout_destroy: 303 * 304 * Destroy a callout structure. The callout must be stopped. 305 */ 306 void 307 callout_destroy(callout_t *cs) 308 { 309 callout_impl_t *c = (callout_impl_t *)cs; 310 311 /* 312 * It's not necessary to lock in order to see the correct value 313 * of c->c_flags. If the callout could potentially have been 314 * running, the current thread should have stopped it. 315 */ 316 KASSERT((c->c_flags & CALLOUT_PENDING) == 0); 317 KASSERT(c->c_cpu->cc_lwp == curlwp || c->c_cpu->cc_active != c); 318 KASSERT(c->c_magic == CALLOUT_MAGIC); 319 c->c_magic = 0; 320 } 321 322 /* 323 * callout_schedule_locked: 324 * 325 * Schedule a callout to run. The function and argument must 326 * already be set in the callout structure. Must be called with 327 * callout_lock. 328 */ 329 static void 330 callout_schedule_locked(callout_impl_t *c, kmutex_t *lock, int to_ticks) 331 { 332 struct callout_cpu *cc, *occ; 333 int old_time; 334 335 KASSERT(to_ticks >= 0); 336 KASSERT(c->c_func != NULL); 337 338 /* Initialize the time here, it won't change. */ 339 occ = c->c_cpu; 340 c->c_flags &= ~CALLOUT_FIRED; 341 342 /* 343 * If this timeout is already scheduled and now is moved 344 * earlier, reschedule it now. Otherwise leave it in place 345 * and let it be rescheduled later. 346 */ 347 if ((c->c_flags & CALLOUT_PENDING) != 0) { 348 /* Leave on existing CPU. */ 349 old_time = c->c_time; 350 c->c_time = to_ticks + occ->cc_ticks; 351 if (c->c_time - old_time < 0) { 352 CIRCQ_REMOVE(&c->c_list); 353 CIRCQ_INSERT(&c->c_list, &occ->cc_todo); 354 } 355 mutex_spin_exit(lock); 356 return; 357 } 358 359 cc = curcpu()->ci_data.cpu_callout; 360 if ((c->c_flags & CALLOUT_BOUND) != 0 || cc == occ || 361 !mutex_tryenter(&cc->cc_lock)) { 362 /* Leave on existing CPU. */ 363 c->c_time = to_ticks + occ->cc_ticks; 364 c->c_flags |= CALLOUT_PENDING; 365 CIRCQ_INSERT(&c->c_list, &occ->cc_todo); 366 } else { 367 /* Move to this CPU. */ 368 c->c_cpu = cc; 369 c->c_time = to_ticks + cc->cc_ticks; 370 c->c_flags |= CALLOUT_PENDING; 371 CIRCQ_INSERT(&c->c_list, &cc->cc_todo); 372 mutex_spin_exit(&cc->cc_lock); 373 } 374 mutex_spin_exit(lock); 375 } 376 377 /* 378 * callout_reset: 379 * 380 * Reset a callout structure with a new function and argument, and 381 * schedule it to run. 382 */ 383 void 384 callout_reset(callout_t *cs, int to_ticks, void (*func)(void *), void *arg) 385 { 386 callout_impl_t *c = (callout_impl_t *)cs; 387 kmutex_t *lock; 388 389 KASSERT(c->c_magic == CALLOUT_MAGIC); 390 391 lock = callout_lock(c); 392 c->c_func = func; 393 c->c_arg = arg; 394 callout_schedule_locked(c, lock, to_ticks); 395 } 396 397 /* 398 * callout_schedule: 399 * 400 * Schedule a callout to run. The function and argument must 401 * already be set in the callout structure. 402 */ 403 void 404 callout_schedule(callout_t *cs, int to_ticks) 405 { 406 callout_impl_t *c = (callout_impl_t *)cs; 407 kmutex_t *lock; 408 409 KASSERT(c->c_magic == CALLOUT_MAGIC); 410 411 lock = callout_lock(c); 412 callout_schedule_locked(c, lock, to_ticks); 413 } 414 415 /* 416 * callout_stop: 417 * 418 * Try to cancel a pending callout. It may be too late: the callout 419 * could be running on another CPU. If called from interrupt context, 420 * the callout could already be in progress at a lower priority. 421 */ 422 bool 423 callout_stop(callout_t *cs) 424 { 425 callout_impl_t *c = (callout_impl_t *)cs; 426 struct callout_cpu *cc; 427 kmutex_t *lock; 428 bool expired; 429 430 KASSERT(c->c_magic == CALLOUT_MAGIC); 431 432 lock = callout_lock(c); 433 434 if ((c->c_flags & CALLOUT_PENDING) != 0) 435 CIRCQ_REMOVE(&c->c_list); 436 expired = ((c->c_flags & CALLOUT_FIRED) != 0); 437 c->c_flags &= ~(CALLOUT_PENDING|CALLOUT_FIRED); 438 439 cc = c->c_cpu; 440 if (cc->cc_active == c) { 441 /* 442 * This is for non-MPSAFE callouts only. To synchronize 443 * effectively we must be called with kernel_lock held. 444 * It's also taken in callout_softclock. 445 */ 446 cc->cc_cancel = c; 447 } 448 449 mutex_spin_exit(lock); 450 451 return expired; 452 } 453 454 /* 455 * callout_halt: 456 * 457 * Cancel a pending callout. If in-flight, block until it completes. 458 * May not be called from a hard interrupt handler. If the callout 459 * can take locks, the caller of callout_halt() must not hold any of 460 * those locks, otherwise the two could deadlock. If 'interlock' is 461 * non-NULL and we must wait for the callout to complete, it will be 462 * released and re-acquired before returning. 463 */ 464 bool 465 callout_halt(callout_t *cs, kmutex_t *interlock) 466 { 467 callout_impl_t *c = (callout_impl_t *)cs; 468 struct callout_cpu *cc; 469 struct lwp *l; 470 kmutex_t *lock, *relock; 471 bool expired; 472 473 KASSERT(c->c_magic == CALLOUT_MAGIC); 474 KASSERT(!cpu_intr_p()); 475 476 lock = callout_lock(c); 477 relock = NULL; 478 479 expired = ((c->c_flags & CALLOUT_FIRED) != 0); 480 if ((c->c_flags & CALLOUT_PENDING) != 0) 481 CIRCQ_REMOVE(&c->c_list); 482 c->c_flags &= ~(CALLOUT_PENDING|CALLOUT_FIRED); 483 484 l = curlwp; 485 for (;;) { 486 cc = c->c_cpu; 487 if (__predict_true(cc->cc_active != c || cc->cc_lwp == l)) 488 break; 489 if (interlock != NULL) { 490 /* 491 * Avoid potential scheduler lock order problems by 492 * dropping the interlock without the callout lock 493 * held. 494 */ 495 mutex_spin_exit(lock); 496 mutex_exit(interlock); 497 relock = interlock; 498 interlock = NULL; 499 } else { 500 /* XXX Better to do priority inheritance. */ 501 KASSERT(l->l_wchan == NULL); 502 cc->cc_nwait++; 503 cc->cc_ev_block.ev_count++; 504 l->l_kpriority = true; 505 sleepq_enter(&cc->cc_sleepq, l); 506 sleepq_enqueue(&cc->cc_sleepq, cc, "callout", 507 &sleep_syncobj); 508 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 509 sleepq_block(0, false); 510 } 511 lock = callout_lock(c); 512 } 513 514 mutex_spin_exit(lock); 515 if (__predict_false(relock != NULL)) 516 mutex_enter(relock); 517 518 return expired; 519 } 520 521 #ifdef notyet 522 /* 523 * callout_bind: 524 * 525 * Bind a callout so that it will only execute on one CPU. 526 * The callout must be stopped, and must be MPSAFE. 527 * 528 * XXX Disabled for now until it is decided how to handle 529 * offlined CPUs. We may want weak+strong binding. 530 */ 531 void 532 callout_bind(callout_t *cs, struct cpu_info *ci) 533 { 534 callout_impl_t *c = (callout_impl_t *)cs; 535 struct callout_cpu *cc; 536 kmutex_t *lock; 537 538 KASSERT((c->c_flags & CALLOUT_PENDING) == 0); 539 KASSERT(c->c_cpu->cc_active != c); 540 KASSERT(c->c_magic == CALLOUT_MAGIC); 541 KASSERT((c->c_flags & CALLOUT_MPSAFE) != 0); 542 543 lock = callout_lock(c); 544 cc = ci->ci_data.cpu_callout; 545 c->c_flags |= CALLOUT_BOUND; 546 if (c->c_cpu != cc) { 547 /* 548 * Assigning c_cpu effectively unlocks the callout 549 * structure, as we don't hold the new CPU's lock. 550 * Issue memory barrier to prevent accesses being 551 * reordered. 552 */ 553 membar_exit(); 554 c->c_cpu = cc; 555 } 556 mutex_spin_exit(lock); 557 } 558 #endif 559 560 void 561 callout_setfunc(callout_t *cs, void (*func)(void *), void *arg) 562 { 563 callout_impl_t *c = (callout_impl_t *)cs; 564 kmutex_t *lock; 565 566 KASSERT(c->c_magic == CALLOUT_MAGIC); 567 568 lock = callout_lock(c); 569 c->c_func = func; 570 c->c_arg = arg; 571 mutex_spin_exit(lock); 572 } 573 574 bool 575 callout_expired(callout_t *cs) 576 { 577 callout_impl_t *c = (callout_impl_t *)cs; 578 kmutex_t *lock; 579 bool rv; 580 581 KASSERT(c->c_magic == CALLOUT_MAGIC); 582 583 lock = callout_lock(c); 584 rv = ((c->c_flags & CALLOUT_FIRED) != 0); 585 mutex_spin_exit(lock); 586 587 return rv; 588 } 589 590 bool 591 callout_active(callout_t *cs) 592 { 593 callout_impl_t *c = (callout_impl_t *)cs; 594 kmutex_t *lock; 595 bool rv; 596 597 KASSERT(c->c_magic == CALLOUT_MAGIC); 598 599 lock = callout_lock(c); 600 rv = ((c->c_flags & (CALLOUT_PENDING|CALLOUT_FIRED)) != 0); 601 mutex_spin_exit(lock); 602 603 return rv; 604 } 605 606 bool 607 callout_pending(callout_t *cs) 608 { 609 callout_impl_t *c = (callout_impl_t *)cs; 610 kmutex_t *lock; 611 bool rv; 612 613 KASSERT(c->c_magic == CALLOUT_MAGIC); 614 615 lock = callout_lock(c); 616 rv = ((c->c_flags & CALLOUT_PENDING) != 0); 617 mutex_spin_exit(lock); 618 619 return rv; 620 } 621 622 bool 623 callout_invoking(callout_t *cs) 624 { 625 callout_impl_t *c = (callout_impl_t *)cs; 626 kmutex_t *lock; 627 bool rv; 628 629 KASSERT(c->c_magic == CALLOUT_MAGIC); 630 631 lock = callout_lock(c); 632 rv = ((c->c_flags & CALLOUT_INVOKING) != 0); 633 mutex_spin_exit(lock); 634 635 return rv; 636 } 637 638 void 639 callout_ack(callout_t *cs) 640 { 641 callout_impl_t *c = (callout_impl_t *)cs; 642 kmutex_t *lock; 643 644 KASSERT(c->c_magic == CALLOUT_MAGIC); 645 646 lock = callout_lock(c); 647 c->c_flags &= ~CALLOUT_INVOKING; 648 mutex_spin_exit(lock); 649 } 650 651 /* 652 * callout_hardclock: 653 * 654 * Called from hardclock() once every tick. We schedule a soft 655 * interrupt if there is work to be done. 656 */ 657 void 658 callout_hardclock(void) 659 { 660 struct callout_cpu *cc; 661 int needsoftclock, ticks; 662 663 cc = curcpu()->ci_data.cpu_callout; 664 mutex_spin_enter(&cc->cc_lock); 665 666 ticks = ++cc->cc_ticks; 667 668 MOVEBUCKET(cc, 0, ticks); 669 if (MASKWHEEL(0, ticks) == 0) { 670 MOVEBUCKET(cc, 1, ticks); 671 if (MASKWHEEL(1, ticks) == 0) { 672 MOVEBUCKET(cc, 2, ticks); 673 if (MASKWHEEL(2, ticks) == 0) 674 MOVEBUCKET(cc, 3, ticks); 675 } 676 } 677 678 needsoftclock = !CIRCQ_EMPTY(&cc->cc_todo); 679 mutex_spin_exit(&cc->cc_lock); 680 681 if (needsoftclock) 682 softint_schedule(callout_sih); 683 } 684 685 /* 686 * callout_softclock: 687 * 688 * Soft interrupt handler, scheduled above if there is work to 689 * be done. Callouts are made in soft interrupt context. 690 */ 691 static void 692 callout_softclock(void *v) 693 { 694 callout_impl_t *c; 695 struct callout_cpu *cc; 696 void (*func)(void *); 697 void *arg; 698 int mpsafe, count, ticks, delta; 699 lwp_t *l; 700 701 l = curlwp; 702 KASSERT(l->l_cpu == curcpu()); 703 cc = l->l_cpu->ci_data.cpu_callout; 704 705 mutex_spin_enter(&cc->cc_lock); 706 cc->cc_lwp = l; 707 while (!CIRCQ_EMPTY(&cc->cc_todo)) { 708 c = CIRCQ_FIRST(&cc->cc_todo); 709 KASSERT(c->c_magic == CALLOUT_MAGIC); 710 KASSERT(c->c_func != NULL); 711 KASSERT(c->c_cpu == cc); 712 KASSERT((c->c_flags & CALLOUT_PENDING) != 0); 713 KASSERT((c->c_flags & CALLOUT_FIRED) == 0); 714 CIRCQ_REMOVE(&c->c_list); 715 716 /* If due run it, otherwise insert it into the right bucket. */ 717 ticks = cc->cc_ticks; 718 delta = c->c_time - ticks; 719 if (delta > 0) { 720 CIRCQ_INSERT(&c->c_list, BUCKET(cc, delta, c->c_time)); 721 continue; 722 } 723 if (delta < 0) 724 cc->cc_ev_late.ev_count++; 725 726 c->c_flags ^= (CALLOUT_PENDING | CALLOUT_FIRED); 727 mpsafe = (c->c_flags & CALLOUT_MPSAFE); 728 func = c->c_func; 729 arg = c->c_arg; 730 cc->cc_active = c; 731 732 mutex_spin_exit(&cc->cc_lock); 733 if (!mpsafe) { 734 KERNEL_LOCK(1, NULL); 735 (*func)(arg); 736 KERNEL_UNLOCK_ONE(NULL); 737 } else 738 (*func)(arg); 739 mutex_spin_enter(&cc->cc_lock); 740 741 /* 742 * We can't touch 'c' here because it might be 743 * freed already. If LWPs waiting for callout 744 * to complete, awaken them. 745 */ 746 cc->cc_active = NULL; 747 if ((count = cc->cc_nwait) != 0) { 748 cc->cc_nwait = 0; 749 /* sleepq_wake() drops the lock. */ 750 sleepq_wake(&cc->cc_sleepq, cc, count); 751 mutex_spin_enter(&cc->cc_lock); 752 } 753 } 754 cc->cc_lwp = NULL; 755 mutex_spin_exit(&cc->cc_lock); 756 } 757 758 #ifdef DDB 759 static void 760 db_show_callout_bucket(struct callout_cpu *cc, struct callout_circq *bucket) 761 { 762 callout_impl_t *c; 763 db_expr_t offset; 764 const char *name; 765 static char question[] = "?"; 766 int b; 767 768 if (CIRCQ_EMPTY(bucket)) 769 return; 770 771 for (c = CIRCQ_FIRST(bucket); /*nothing*/; c = CIRCQ_NEXT(&c->c_list)) { 772 db_find_sym_and_offset((db_addr_t)(intptr_t)c->c_func, &name, 773 &offset); 774 name = name ? name : question; 775 b = (bucket - cc->cc_wheel); 776 if (b < 0) 777 b = -WHEELSIZE; 778 db_printf("%9d %2d/%-4d %16lx %s\n", 779 c->c_time - cc->cc_ticks, b / WHEELSIZE, b, 780 (u_long)c->c_arg, name); 781 if (CIRCQ_LAST(&c->c_list, bucket)) 782 break; 783 } 784 } 785 786 void 787 db_show_callout(db_expr_t addr, bool haddr, db_expr_t count, const char *modif) 788 { 789 CPU_INFO_ITERATOR cii; 790 struct callout_cpu *cc; 791 struct cpu_info *ci; 792 int b; 793 794 db_printf("hardclock_ticks now: %d\n", hardclock_ticks); 795 db_printf(" ticks wheel arg func\n"); 796 797 /* 798 * Don't lock the callwheel; all the other CPUs are paused 799 * anyhow, and we might be called in a circumstance where 800 * some other CPU was paused while holding the lock. 801 */ 802 for (CPU_INFO_FOREACH(cii, ci)) { 803 cc = ci->ci_data.cpu_callout; 804 db_show_callout_bucket(cc, &cc->cc_todo); 805 } 806 for (b = 0; b < BUCKETS; b++) { 807 for (CPU_INFO_FOREACH(cii, ci)) { 808 cc = ci->ci_data.cpu_callout; 809 db_show_callout_bucket(cc, &cc->cc_wheel[b]); 810 } 811 } 812 } 813 #endif /* DDB */ 814