1 /* $OpenBSD: agtimer.c,v 1.15 2020/07/15 22:58:33 kettenis Exp $ */ 2 /* 3 * Copyright (c) 2011 Dale Rahn <drahn@openbsd.org> 4 * Copyright (c) 2013 Patrick Wildt <patrick@blueri.se> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/param.h> 20 #include <sys/systm.h> 21 #include <sys/queue.h> 22 #include <sys/malloc.h> 23 #include <sys/device.h> 24 #include <sys/kernel.h> 25 #include <sys/timetc.h> 26 #include <sys/evcount.h> 27 28 #include <machine/intr.h> 29 #include <machine/bus.h> 30 #include <machine/fdt.h> 31 32 #include <dev/ofw/fdt.h> 33 #include <dev/ofw/openfirm.h> 34 35 /* registers */ 36 #define GTIMER_CNTV_CTL_ENABLE (1 << 0) 37 #define GTIMER_CNTV_CTL_IMASK (1 << 1) 38 #define GTIMER_CNTV_CTL_ISTATUS (1 << 2) 39 40 #define TIMER_FREQUENCY 24 * 1000 * 1000 /* ARM core clock */ 41 int32_t agtimer_frequency = TIMER_FREQUENCY; 42 43 u_int agtimer_get_timecount(struct timecounter *); 44 45 static struct timecounter agtimer_timecounter = { 46 agtimer_get_timecount, NULL, 0xffffffff, 0, "agtimer", 0, NULL, 47 TC_AGTIMER 48 }; 49 50 struct agtimer_pcpu_softc { 51 uint64_t pc_nexttickevent; 52 uint64_t pc_nextstatevent; 53 u_int32_t pc_ticks_err_sum; 54 }; 55 56 struct agtimer_softc { 57 struct device sc_dev; 58 int sc_node; 59 60 struct agtimer_pcpu_softc sc_pstat[MAXCPUS]; 61 62 u_int32_t sc_ticks_err_cnt; 63 u_int32_t sc_ticks_per_second; 64 u_int32_t sc_ticks_per_intr; 65 u_int32_t sc_statvar; 66 u_int32_t sc_statmin; 67 68 #ifdef AMPTIMER_DEBUG 69 struct evcount sc_clk_count; 70 struct evcount sc_stat_count; 71 #endif 72 void *sc_ih; 73 }; 74 75 int agtimer_match(struct device *, void *, void *); 76 void agtimer_attach(struct device *, struct device *, void *); 77 uint64_t agtimer_readcnt64(void); 78 int agtimer_intr(void *); 79 void agtimer_cpu_initclocks(void); 80 void agtimer_delay(u_int); 81 void agtimer_setstatclockrate(int stathz); 82 void agtimer_set_clockrate(int32_t new_frequency); 83 void agtimer_startclock(void); 84 85 struct cfattach agtimer_ca = { 86 sizeof (struct agtimer_softc), agtimer_match, agtimer_attach 87 }; 88 89 struct cfdriver agtimer_cd = { 90 NULL, "agtimer", DV_DULL 91 }; 92 93 uint64_t 94 agtimer_readcnt64(void) 95 { 96 uint64_t val0, val1; 97 98 /* 99 * Work around Cortex-A73 errata 858921, where there is a 100 * one-cycle window where the read might return the old value 101 * for the low 32 bits and the new value for the high 32 bits 102 * upon roll-over of the low 32 bits. 103 */ 104 __asm volatile("isb" ::: "memory"); 105 __asm volatile("mrs %x0, CNTVCT_EL0" : "=r" (val0)); 106 __asm volatile("mrs %x0, CNTVCT_EL0" : "=r" (val1)); 107 return ((val0 ^ val1) & 0x100000000ULL) ? val0 : val1; 108 } 109 110 static inline uint64_t 111 agtimer_get_freq(void) 112 { 113 uint64_t val; 114 115 __asm volatile("mrs %x0, CNTFRQ_EL0" : "=r" (val)); 116 117 return (val); 118 } 119 120 static inline int 121 agtimer_get_ctrl(void) 122 { 123 uint32_t val; 124 125 __asm volatile("mrs %x0, CNTV_CTL_EL0" : "=r" (val)); 126 127 return (val); 128 } 129 130 static inline int 131 agtimer_set_ctrl(uint32_t val) 132 { 133 __asm volatile("msr CNTV_CTL_EL0, %x0" :: "r" (val)); 134 __asm volatile("isb" ::: "memory"); 135 136 return (0); 137 } 138 139 static inline int 140 agtimer_set_tval(uint32_t val) 141 { 142 __asm volatile("msr CNTV_TVAL_EL0, %x0" :: "r" (val)); 143 __asm volatile("isb" ::: "memory"); 144 145 return (0); 146 } 147 148 int 149 agtimer_match(struct device *parent, void *cfdata, void *aux) 150 { 151 struct fdt_attach_args *faa = (struct fdt_attach_args *)aux; 152 153 return (OF_is_compatible(faa->fa_node, "arm,armv7-timer") || 154 OF_is_compatible(faa->fa_node, "arm,armv8-timer")); 155 } 156 157 void 158 agtimer_attach(struct device *parent, struct device *self, void *aux) 159 { 160 struct agtimer_softc *sc = (struct agtimer_softc *)self; 161 struct fdt_attach_args *faa = aux; 162 163 sc->sc_node = faa->fa_node; 164 165 if (agtimer_get_freq() != 0) 166 agtimer_frequency = agtimer_get_freq(); 167 agtimer_frequency = 168 OF_getpropint(sc->sc_node, "clock-frequency", agtimer_frequency); 169 sc->sc_ticks_per_second = agtimer_frequency; 170 171 printf(": tick rate %d KHz\n", sc->sc_ticks_per_second /1000); 172 173 #ifdef AMPTIMER_DEBUG 174 evcount_attach(&sc->sc_clk_count, "clock", NULL); 175 evcount_attach(&sc->sc_stat_count, "stat", NULL); 176 #endif 177 178 /* 179 * private timer and interrupts not enabled until 180 * timer configures 181 */ 182 183 arm_clock_register(agtimer_cpu_initclocks, agtimer_delay, 184 agtimer_setstatclockrate, agtimer_startclock); 185 186 agtimer_timecounter.tc_frequency = sc->sc_ticks_per_second; 187 agtimer_timecounter.tc_priv = sc; 188 189 tc_init(&agtimer_timecounter); 190 } 191 192 u_int 193 agtimer_get_timecount(struct timecounter *tc) 194 { 195 uint64_t val; 196 197 /* 198 * No need to work around Cortex-A73 errata 858921 since we 199 * only look at the low 32 bits here. 200 */ 201 __asm volatile("isb" ::: "memory"); 202 __asm volatile("mrs %x0, CNTVCT_EL0" : "=r" (val)); 203 return (val & 0xffffffff); 204 } 205 206 int 207 agtimer_intr(void *frame) 208 { 209 struct agtimer_softc *sc = agtimer_cd.cd_devs[0]; 210 struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())]; 211 uint64_t now; 212 uint64_t nextevent; 213 uint32_t r; 214 #if defined(USE_GTIMER_CMP) 215 int skip = 1; 216 #else 217 int64_t delay; 218 #endif 219 int rc = 0; 220 221 /* 222 * DSR - I know that the tick timer is 64 bits, but the following 223 * code deals with rollover, so there is no point in dealing 224 * with the 64 bit math, just let the 32 bit rollover 225 * do the right thing 226 */ 227 228 now = agtimer_readcnt64(); 229 230 while (pc->pc_nexttickevent <= now) { 231 pc->pc_nexttickevent += sc->sc_ticks_per_intr; 232 pc->pc_ticks_err_sum += sc->sc_ticks_err_cnt; 233 234 /* looping a few times is faster than divide */ 235 while (pc->pc_ticks_err_sum > hz) { 236 pc->pc_nexttickevent += 1; 237 pc->pc_ticks_err_sum -= hz; 238 } 239 240 #ifdef AMPTIMER_DEBUG 241 sc->sc_clk_count.ec_count++; 242 #endif 243 rc = 1; 244 hardclock(frame); 245 } 246 while (pc->pc_nextstatevent <= now) { 247 do { 248 r = random() & (sc->sc_statvar -1); 249 } while (r == 0); /* random == 0 not allowed */ 250 pc->pc_nextstatevent += sc->sc_statmin + r; 251 252 /* XXX - correct nextstatevent? */ 253 #ifdef AMPTIMER_DEBUG 254 sc->sc_stat_count.ec_count++; 255 #endif 256 rc = 1; 257 statclock(frame); 258 } 259 260 if (pc->pc_nexttickevent < pc->pc_nextstatevent) 261 nextevent = pc->pc_nexttickevent; 262 else 263 nextevent = pc->pc_nextstatevent; 264 265 delay = nextevent - now; 266 if (delay < 0) 267 delay = 1; 268 269 agtimer_set_tval(delay); 270 271 return (rc); 272 } 273 274 void 275 agtimer_set_clockrate(int32_t new_frequency) 276 { 277 struct agtimer_softc *sc = agtimer_cd.cd_devs[0]; 278 279 agtimer_frequency = new_frequency; 280 281 if (sc == NULL) 282 return; 283 284 sc->sc_ticks_per_second = agtimer_frequency; 285 agtimer_timecounter.tc_frequency = sc->sc_ticks_per_second; 286 printf("agtimer0: adjusting clock: new tick rate %d KHz\n", 287 sc->sc_ticks_per_second /1000); 288 } 289 290 void 291 agtimer_cpu_initclocks(void) 292 { 293 struct agtimer_softc *sc = agtimer_cd.cd_devs[0]; 294 struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())]; 295 uint32_t reg; 296 uint64_t next; 297 uint64_t kctl; 298 299 stathz = hz; 300 profhz = hz * 10; 301 302 if (sc->sc_ticks_per_second != agtimer_frequency) { 303 agtimer_set_clockrate(agtimer_frequency); 304 } 305 306 agtimer_setstatclockrate(stathz); 307 308 sc->sc_ticks_per_intr = sc->sc_ticks_per_second / hz; 309 sc->sc_ticks_err_cnt = sc->sc_ticks_per_second % hz; 310 pc->pc_ticks_err_sum = 0; 311 312 /* configure virtual timer interupt */ 313 sc->sc_ih = arm_intr_establish_fdt_idx(sc->sc_node, 2, 314 IPL_CLOCK|IPL_MPSAFE, agtimer_intr, NULL, "tick"); 315 316 next = agtimer_readcnt64() + sc->sc_ticks_per_intr; 317 pc->pc_nexttickevent = pc->pc_nextstatevent = next; 318 319 reg = agtimer_get_ctrl(); 320 reg &= ~GTIMER_CNTV_CTL_IMASK; 321 reg |= GTIMER_CNTV_CTL_ENABLE; 322 agtimer_set_tval(sc->sc_ticks_per_second); 323 agtimer_set_ctrl(reg); 324 325 /* enable userland access to virtual counter */ 326 kctl = READ_SPECIALREG(CNTKCTL_EL1); 327 WRITE_SPECIALREG(CNTKCTL_EL1, kctl | CNTKCTL_EL0VCTEN); 328 } 329 330 void 331 agtimer_delay(u_int usecs) 332 { 333 uint64_t clock, oclock, delta, delaycnt; 334 uint64_t csec, usec; 335 volatile int j; 336 337 if (usecs > (0x80000000 / agtimer_frequency)) { 338 csec = usecs / 10000; 339 usec = usecs % 10000; 340 341 delaycnt = (agtimer_frequency / 100) * csec + 342 (agtimer_frequency / 100) * usec / 10000; 343 } else { 344 delaycnt = agtimer_frequency * usecs / 1000000; 345 } 346 if (delaycnt <= 1) 347 for (j = 100; j > 0; j--) 348 ; 349 350 oclock = agtimer_readcnt64(); 351 while (1) { 352 for (j = 100; j > 0; j--) 353 ; 354 clock = agtimer_readcnt64(); 355 delta = clock - oclock; 356 if (delta > delaycnt) 357 break; 358 } 359 } 360 361 void 362 agtimer_setstatclockrate(int newhz) 363 { 364 struct agtimer_softc *sc = agtimer_cd.cd_devs[0]; 365 int minint, statint; 366 int s; 367 368 s = splclock(); 369 370 statint = sc->sc_ticks_per_second / newhz; 371 /* calculate largest 2^n which is smaller that just over half statint */ 372 sc->sc_statvar = 0x40000000; /* really big power of two */ 373 minint = statint / 2 + 100; 374 while (sc->sc_statvar > minint) 375 sc->sc_statvar >>= 1; 376 377 sc->sc_statmin = statint - (sc->sc_statvar >> 1); 378 379 splx(s); 380 381 /* 382 * XXX this allows the next stat timer to occur then it switches 383 * to the new frequency. Rather than switching instantly. 384 */ 385 } 386 387 void 388 agtimer_startclock(void) 389 { 390 struct agtimer_softc *sc = agtimer_cd.cd_devs[0]; 391 struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())]; 392 uint64_t nextevent; 393 uint64_t kctl; 394 uint32_t reg; 395 396 nextevent = agtimer_readcnt64() + sc->sc_ticks_per_intr; 397 pc->pc_nexttickevent = pc->pc_nextstatevent = nextevent; 398 399 arm_intr_route(sc->sc_ih, 1, curcpu()); 400 401 reg = agtimer_get_ctrl(); 402 reg &= ~GTIMER_CNTV_CTL_IMASK; 403 reg |= GTIMER_CNTV_CTL_ENABLE; 404 agtimer_set_tval(sc->sc_ticks_per_second); 405 agtimer_set_ctrl(reg); 406 407 /* enable userland access to virtual counter */ 408 kctl = READ_SPECIALREG(CNTKCTL_EL1); 409 WRITE_SPECIALREG(CNTKCTL_EL1, kctl | CNTKCTL_EL0VCTEN); 410 } 411 412 void 413 agtimer_init(void) 414 { 415 uint64_t cntfrq = 0; 416 417 /* XXX: Check for Generic Timer support. */ 418 cntfrq = agtimer_get_freq(); 419 420 if (cntfrq != 0) { 421 agtimer_frequency = cntfrq; 422 arm_clock_register(NULL, agtimer_delay, NULL, NULL); 423 } 424 } 425