1 /*- 2 * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer, 10 * without modification, immediately at the beginning of the file. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 /* 31 * Common routines to manage event timers hardware. 32 */ 33 34 /* XEN has own timer routines now. */ 35 #ifndef XEN 36 37 #include "opt_kdtrace.h" 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/bus.h> 42 #include <sys/lock.h> 43 #include <sys/kdb.h> 44 #include <sys/mutex.h> 45 #include <sys/proc.h> 46 #include <sys/kernel.h> 47 #include <sys/sched.h> 48 #include <sys/smp.h> 49 #include <sys/sysctl.h> 50 #include <sys/timeet.h> 51 52 #include <machine/atomic.h> 53 #include <machine/clock.h> 54 #include <machine/cpu.h> 55 #include <machine/smp.h> 56 57 #ifdef KDTRACE_HOOKS 58 #include <sys/dtrace_bsd.h> 59 cyclic_clock_func_t cyclic_clock_func[MAXCPU]; 60 #endif 61 62 static void cpu_restartclocks(void); 63 static void timercheck(void); 64 inline static int doconfigtimer(int i); 65 static void configtimer(int i); 66 67 static struct eventtimer *timer[2] = { NULL, NULL }; 68 static int timertest = 0; 69 static int timerticks[2] = { 0, 0 }; 70 static int profiling_on = 0; 71 static struct bintime timerperiod[2]; 72 73 static char timername[2][32]; 74 TUNABLE_STR("kern.eventtimer.timer1", timername[0], sizeof(*timername)); 75 TUNABLE_STR("kern.eventtimer.timer2", timername[1], sizeof(*timername)); 76 77 static u_int singlemul = 0; 78 TUNABLE_INT("kern.eventtimer.singlemul", &singlemul); 79 SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RW, &singlemul, 80 0, "Multiplier, used in single timer mode"); 81 82 typedef u_int tc[2]; 83 static DPCPU_DEFINE(tc, configtimer); 84 85 #define FREQ2BT(freq, bt) \ 86 { \ 87 (bt)->sec = 0; \ 88 (bt)->frac = ((uint64_t)0x8000000000000000 / (freq)) << 1; \ 89 } 90 #define BT2FREQ(bt, freq) \ 91 { \ 92 *(freq) = ((uint64_t)0x8000000000000000 + ((bt)->frac >> 2)) / \ 93 ((bt)->frac >> 1); \ 94 } 95 96 /* Per-CPU timer1 handler. */ 97 static int 98 hardclockhandler(struct trapframe *frame) 99 { 100 101 #ifdef KDTRACE_HOOKS 102 /* 103 * If the DTrace hooks are configured and a callback function 104 * has been registered, then call it to process the high speed 105 * timers. 106 */ 107 int cpu = curcpu; 108 if (cyclic_clock_func[cpu] != NULL) 109 (*cyclic_clock_func[cpu])(frame); 110 #endif 111 112 timer1clock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); 113 return (FILTER_HANDLED); 114 } 115 116 /* Per-CPU timer2 handler. */ 117 static int 118 statclockhandler(struct trapframe *frame) 119 { 120 121 timer2clock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); 122 return (FILTER_HANDLED); 123 } 124 125 /* timer1 broadcast IPI handler. */ 126 int 127 hardclockintr(struct trapframe *frame) 128 { 129 130 if (doconfigtimer(0)) 131 return (FILTER_HANDLED); 132 return (hardclockhandler(frame)); 133 } 134 135 /* timer2 broadcast IPI handler. */ 136 int 137 statclockintr(struct trapframe *frame) 138 { 139 140 if (doconfigtimer(1)) 141 return (FILTER_HANDLED); 142 return (statclockhandler(frame)); 143 } 144 145 /* timer1 callback. */ 146 static void 147 timer1cb(struct eventtimer *et, void *arg) 148 { 149 150 #ifdef SMP 151 /* Broadcast interrupt to other CPUs for non-per-CPU timers */ 152 if (smp_started && (et->et_flags & ET_FLAGS_PERCPU) == 0) 153 ipi_all_but_self(IPI_HARDCLOCK); 154 #endif 155 if (timertest) { 156 if ((et->et_flags & ET_FLAGS_PERCPU) == 0 || curcpu == 0) { 157 timerticks[0]++; 158 if (timerticks[0] >= timer1hz) { 159 ET_LOCK(); 160 timercheck(); 161 ET_UNLOCK(); 162 } 163 } 164 } 165 hardclockhandler(curthread->td_intr_frame); 166 } 167 168 /* timer2 callback. */ 169 static void 170 timer2cb(struct eventtimer *et, void *arg) 171 { 172 173 #ifdef SMP 174 /* Broadcast interrupt to other CPUs for non-per-CPU timers */ 175 if (smp_started && (et->et_flags & ET_FLAGS_PERCPU) == 0) 176 ipi_all_but_self(IPI_STATCLOCK); 177 #endif 178 if (timertest) { 179 if ((et->et_flags & ET_FLAGS_PERCPU) == 0 || curcpu == 0) { 180 timerticks[1]++; 181 if (timerticks[1] >= timer2hz * 2) { 182 ET_LOCK(); 183 timercheck(); 184 ET_UNLOCK(); 185 } 186 } 187 } 188 statclockhandler(curthread->td_intr_frame); 189 } 190 191 /* 192 * Check that both timers are running with at least 1/4 of configured rate. 193 * If not - replace the broken one. 194 */ 195 static void 196 timercheck(void) 197 { 198 199 if (!timertest) 200 return; 201 timertest = 0; 202 if (timerticks[0] * 4 < timer1hz) { 203 printf("Event timer \"%s\" is dead.\n", timer[0]->et_name); 204 timer1hz = 0; 205 configtimer(0); 206 et_ban(timer[0]); 207 et_free(timer[0]); 208 timer[0] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 209 if (timer[0] == NULL) { 210 timer2hz = 0; 211 configtimer(1); 212 et_free(timer[1]); 213 timer[1] = NULL; 214 timer[0] = timer[1]; 215 } 216 et_init(timer[0], timer1cb, NULL, NULL); 217 cpu_restartclocks(); 218 return; 219 } 220 if (timerticks[1] * 4 < timer2hz) { 221 printf("Event timer \"%s\" is dead.\n", timer[1]->et_name); 222 timer2hz = 0; 223 configtimer(1); 224 et_ban(timer[1]); 225 et_free(timer[1]); 226 timer[1] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 227 if (timer[1] != NULL) 228 et_init(timer[1], timer2cb, NULL, NULL); 229 cpu_restartclocks(); 230 return; 231 } 232 } 233 234 /* 235 * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler. 236 */ 237 inline static int 238 doconfigtimer(int i) 239 { 240 tc *conf; 241 242 conf = DPCPU_PTR(configtimer); 243 if (atomic_load_acq_int(*conf + i)) { 244 if (i == 0 ? timer1hz : timer2hz) 245 et_start(timer[i], NULL, &timerperiod[i]); 246 else 247 et_stop(timer[i]); 248 atomic_store_rel_int(*conf + i, 0); 249 return (1); 250 } 251 return (0); 252 } 253 254 /* 255 * Reconfigure specified timer. 256 * For per-CPU timers use IPI to make other CPUs to reconfigure. 257 */ 258 static void 259 configtimer(int i) 260 { 261 #ifdef SMP 262 tc *conf; 263 int cpu; 264 265 critical_enter(); 266 #endif 267 /* Start/stop global timer or per-CPU timer of this CPU. */ 268 if (i == 0 ? timer1hz : timer2hz) 269 et_start(timer[i], NULL, &timerperiod[i]); 270 else 271 et_stop(timer[i]); 272 #ifdef SMP 273 if ((timer[i]->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) { 274 critical_exit(); 275 return; 276 } 277 /* Set reconfigure flags for other CPUs. */ 278 CPU_FOREACH(cpu) { 279 conf = DPCPU_ID_PTR(cpu, configtimer); 280 atomic_store_rel_int(*conf + i, (cpu == curcpu) ? 0 : 1); 281 } 282 /* Send reconfigure IPI. */ 283 ipi_all_but_self(i == 0 ? IPI_HARDCLOCK : IPI_STATCLOCK); 284 /* Wait for reconfiguration completed. */ 285 restart: 286 cpu_spinwait(); 287 CPU_FOREACH(cpu) { 288 if (cpu == curcpu) 289 continue; 290 conf = DPCPU_ID_PTR(cpu, configtimer); 291 if (atomic_load_acq_int(*conf + i)) 292 goto restart; 293 } 294 critical_exit(); 295 #endif 296 } 297 298 /* 299 * Configure and start event timers. 300 */ 301 void 302 cpu_initclocks_bsp(void) 303 { 304 int base, div; 305 306 timer[0] = et_find(timername[0], ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 307 if (timer[0] == NULL) 308 timer[0] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 309 if (timer[0] == NULL) 310 panic("No usable event timer found!"); 311 et_init(timer[0], timer1cb, NULL, NULL); 312 timer[1] = et_find(timername[1][0] ? timername[1] : NULL, 313 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 314 if (timer[1]) 315 et_init(timer[1], timer2cb, NULL, NULL); 316 /* 317 * We honor the requested 'hz' value. 318 * We want to run stathz in the neighborhood of 128hz. 319 * We would like profhz to run as often as possible. 320 */ 321 if (singlemul == 0) { 322 if (hz >= 1500 || (hz % 128) == 0) 323 singlemul = 1; 324 else if (hz >= 750) 325 singlemul = 2; 326 else 327 singlemul = 4; 328 } 329 if (timer[1] == NULL) { 330 base = hz * singlemul; 331 if (base < 128) 332 stathz = base; 333 else { 334 div = base / 128; 335 if (div % 2 == 0) 336 div++; 337 stathz = base / div; 338 } 339 profhz = stathz; 340 while ((profhz + stathz) <= 8192) 341 profhz += stathz; 342 } else { 343 stathz = 128; 344 profhz = stathz * 64; 345 } 346 ET_LOCK(); 347 cpu_restartclocks(); 348 ET_UNLOCK(); 349 } 350 351 /* Start per-CPU event timers on APs. */ 352 void 353 cpu_initclocks_ap(void) 354 { 355 356 ET_LOCK(); 357 if (timer[0]->et_flags & ET_FLAGS_PERCPU) 358 et_start(timer[0], NULL, &timerperiod[0]); 359 if (timer[1] && timer[1]->et_flags & ET_FLAGS_PERCPU) 360 et_start(timer[1], NULL, &timerperiod[1]); 361 ET_UNLOCK(); 362 } 363 364 /* Reconfigure and restart event timers after configuration changes. */ 365 static void 366 cpu_restartclocks(void) 367 { 368 369 /* Stop all event timers. */ 370 timertest = 0; 371 if (timer1hz) { 372 timer1hz = 0; 373 configtimer(0); 374 } 375 if (timer[1] && timer2hz) { 376 timer2hz = 0; 377 configtimer(1); 378 } 379 /* Calculate new event timers parameters. */ 380 if (timer[1] == NULL) { 381 timer1hz = hz * singlemul; 382 while (timer1hz < (profiling_on ? profhz : stathz)) 383 timer1hz += hz; 384 timer2hz = 0; 385 } else { 386 timer1hz = hz; 387 timer2hz = profiling_on ? profhz : stathz; 388 } 389 printf("Starting kernel event timers: %s @ %dHz, %s @ %dHz\n", 390 timer[0]->et_name, timer1hz, 391 timer[1] ? timer[1]->et_name : "NONE", timer2hz); 392 /* Restart event timers. */ 393 FREQ2BT(timer1hz, &timerperiod[0]); 394 configtimer(0); 395 if (timer[1]) { 396 timerticks[0] = 0; 397 timerticks[1] = 0; 398 FREQ2BT(timer2hz, &timerperiod[1]); 399 configtimer(1); 400 timertest = 1; 401 } 402 } 403 404 /* Switch to profiling clock rates. */ 405 void 406 cpu_startprofclock(void) 407 { 408 409 ET_LOCK(); 410 profiling_on = 1; 411 cpu_restartclocks(); 412 ET_UNLOCK(); 413 } 414 415 /* Switch to regular clock rates. */ 416 void 417 cpu_stopprofclock(void) 418 { 419 420 ET_LOCK(); 421 profiling_on = 0; 422 cpu_restartclocks(); 423 ET_UNLOCK(); 424 } 425 426 /* Report or change the active event timers hardware. */ 427 static int 428 sysctl_kern_eventtimer_timer1(SYSCTL_HANDLER_ARGS) 429 { 430 char buf[32]; 431 struct eventtimer *et; 432 int error; 433 434 ET_LOCK(); 435 et = timer[0]; 436 snprintf(buf, sizeof(buf), "%s", et->et_name); 437 ET_UNLOCK(); 438 error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 439 ET_LOCK(); 440 et = timer[0]; 441 if (error != 0 || req->newptr == NULL || 442 strcmp(buf, et->et_name) == 0) { 443 ET_UNLOCK(); 444 return (error); 445 } 446 et = et_find(buf, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 447 if (et == NULL) { 448 ET_UNLOCK(); 449 return (ENOENT); 450 } 451 timer1hz = 0; 452 configtimer(0); 453 et_free(timer[0]); 454 timer[0] = et; 455 et_init(timer[0], timer1cb, NULL, NULL); 456 cpu_restartclocks(); 457 ET_UNLOCK(); 458 return (error); 459 } 460 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer1, 461 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, 462 0, 0, sysctl_kern_eventtimer_timer1, "A", "Primary event timer"); 463 464 static int 465 sysctl_kern_eventtimer_timer2(SYSCTL_HANDLER_ARGS) 466 { 467 char buf[32]; 468 struct eventtimer *et; 469 int error; 470 471 ET_LOCK(); 472 et = timer[1]; 473 if (et == NULL) 474 snprintf(buf, sizeof(buf), "NONE"); 475 else 476 snprintf(buf, sizeof(buf), "%s", et->et_name); 477 ET_UNLOCK(); 478 error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 479 ET_LOCK(); 480 et = timer[1]; 481 if (error != 0 || req->newptr == NULL || 482 strcmp(buf, et ? et->et_name : "NONE") == 0) { 483 ET_UNLOCK(); 484 return (error); 485 } 486 et = et_find(buf, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 487 if (et == NULL && strcasecmp(buf, "NONE") != 0) { 488 ET_UNLOCK(); 489 return (ENOENT); 490 } 491 if (timer[1] != NULL) { 492 timer2hz = 0; 493 configtimer(1); 494 et_free(timer[1]); 495 } 496 timer[1] = et; 497 if (timer[1] != NULL) 498 et_init(timer[1], timer2cb, NULL, NULL); 499 cpu_restartclocks(); 500 ET_UNLOCK(); 501 return (error); 502 } 503 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer2, 504 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, 505 0, 0, sysctl_kern_eventtimer_timer2, "A", "Secondary event timer"); 506 507 #endif 508 509