1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2004 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*0Sstevel@tonic-gate 29*0Sstevel@tonic-gate /* 30*0Sstevel@tonic-gate * i86pc Memory Scrubbing 31*0Sstevel@tonic-gate * 32*0Sstevel@tonic-gate * On detection of a correctable memory ECC error, the i86pc hardware 33*0Sstevel@tonic-gate * returns the corrected data to the requester and may re-write it 34*0Sstevel@tonic-gate * to memory (DRAM or NVRAM). Machines which do not re-write this to 35*0Sstevel@tonic-gate * memory should add an NMI handler to correct and rewrite. 36*0Sstevel@tonic-gate * 37*0Sstevel@tonic-gate * Scrubbing thus reduces the likelyhood that multiple transient errors 38*0Sstevel@tonic-gate * will occur in the same memory word, making uncorrectable errors due 39*0Sstevel@tonic-gate * to transients less likely. 40*0Sstevel@tonic-gate * 41*0Sstevel@tonic-gate * Thus is born the desire that every memory location be periodically 42*0Sstevel@tonic-gate * accessed. 43*0Sstevel@tonic-gate * 44*0Sstevel@tonic-gate * This file implements a memory scrubbing thread. This scrubber 45*0Sstevel@tonic-gate * guarantees that all of physical memory is accessed periodically 46*0Sstevel@tonic-gate * (memscrub_period_sec -- 12 hours). 47*0Sstevel@tonic-gate * 48*0Sstevel@tonic-gate * It attempts to do this as unobtrusively as possible. The thread 49*0Sstevel@tonic-gate * schedules itself to wake up at an interval such that if it reads 50*0Sstevel@tonic-gate * memscrub_span_pages (4MB) on each wakeup, it will read all of physical 51*0Sstevel@tonic-gate * memory in in memscrub_period_sec (12 hours). 52*0Sstevel@tonic-gate * 53*0Sstevel@tonic-gate * The scrubber uses the REP LODS so it reads 4MB in 0.15 secs (on P5-200). 54*0Sstevel@tonic-gate * When it completes a span, if all the CPUs are idle, it reads another span. 55*0Sstevel@tonic-gate * Typically it soaks up idle time this way to reach its deadline early 56*0Sstevel@tonic-gate * -- and sleeps until the next period begins. 57*0Sstevel@tonic-gate * 58*0Sstevel@tonic-gate * Maximal Cost Estimate: 8GB @ xxMB/s = xxx seconds spent in 640 wakeups 59*0Sstevel@tonic-gate * that run for 0.15 seconds at intervals of 67 seconds. 60*0Sstevel@tonic-gate * 61*0Sstevel@tonic-gate * In practice, the scrubber finds enough idle time to finish in a few 62*0Sstevel@tonic-gate * minutes, and sleeps until its 12 hour deadline. 63*0Sstevel@tonic-gate * 64*0Sstevel@tonic-gate * The scrubber maintains a private copy of the phys_install memory list 65*0Sstevel@tonic-gate * to keep track of what memory should be scrubbed. 66*0Sstevel@tonic-gate * 67*0Sstevel@tonic-gate * The following parameters can be set via /etc/system 68*0Sstevel@tonic-gate * 69*0Sstevel@tonic-gate * memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES (4MB) 70*0Sstevel@tonic-gate * memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC (12 hours) 71*0Sstevel@tonic-gate * memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI (0) 72*0Sstevel@tonic-gate * memscrub_delay_start_sec = (10 seconds) 73*0Sstevel@tonic-gate * disable_memscrub = (0) 74*0Sstevel@tonic-gate * 75*0Sstevel@tonic-gate * the scrubber will exit (or never be started) if it finds the variable 76*0Sstevel@tonic-gate * "disable_memscrub" set. 77*0Sstevel@tonic-gate * 78*0Sstevel@tonic-gate * MEMSCRUB_DFL_SPAN_PAGES is based on the guess that 0.15 sec 79*0Sstevel@tonic-gate * is a "good" amount of minimum time for the thread to run at a time. 80*0Sstevel@tonic-gate * 81*0Sstevel@tonic-gate * MEMSCRUB_DFL_PERIOD_SEC (12 hours) is nearly a total guess -- 82*0Sstevel@tonic-gate * twice the frequency the hardware folk estimated would be necessary. 83*0Sstevel@tonic-gate * 84*0Sstevel@tonic-gate * MEMSCRUB_DFL_THREAD_PRI (0) is based on the assumption that nearly 85*0Sstevel@tonic-gate * any other use of the system should be higher priority than scrubbing. 86*0Sstevel@tonic-gate */ 87*0Sstevel@tonic-gate 88*0Sstevel@tonic-gate #include <sys/types.h> 89*0Sstevel@tonic-gate #include <sys/systm.h> /* timeout, types, t_lock */ 90*0Sstevel@tonic-gate #include <sys/cmn_err.h> 91*0Sstevel@tonic-gate #include <sys/sysmacros.h> /* MIN */ 92*0Sstevel@tonic-gate #include <sys/memlist.h> /* memlist */ 93*0Sstevel@tonic-gate #include <sys/kmem.h> /* KMEM_NOSLEEP */ 94*0Sstevel@tonic-gate #include <sys/cpuvar.h> /* ncpus_online */ 95*0Sstevel@tonic-gate #include <sys/debug.h> /* ASSERTs */ 96*0Sstevel@tonic-gate #include <sys/vmem.h> 97*0Sstevel@tonic-gate #include <sys/mman.h> 98*0Sstevel@tonic-gate #include <vm/seg_kmem.h> 99*0Sstevel@tonic-gate #include <vm/seg_kpm.h> 100*0Sstevel@tonic-gate #include <vm/hat_i86.h> 101*0Sstevel@tonic-gate 102*0Sstevel@tonic-gate static caddr_t memscrub_window; 103*0Sstevel@tonic-gate static void *memscrub_pte; 104*0Sstevel@tonic-gate 105*0Sstevel@tonic-gate /* 106*0Sstevel@tonic-gate * Global Data: 107*0Sstevel@tonic-gate */ 108*0Sstevel@tonic-gate /* 109*0Sstevel@tonic-gate * scan all of physical memory at least once every MEMSCRUB_PERIOD_SEC 110*0Sstevel@tonic-gate */ 111*0Sstevel@tonic-gate #define MEMSCRUB_DFL_PERIOD_SEC (12 * 60 * 60) /* 12 hours */ 112*0Sstevel@tonic-gate 113*0Sstevel@tonic-gate /* 114*0Sstevel@tonic-gate * start only if at least MEMSCRUB_MIN_PAGES in system 115*0Sstevel@tonic-gate */ 116*0Sstevel@tonic-gate #define MEMSCRUB_MIN_PAGES ((32 * 1024 * 1024) / PAGESIZE) 117*0Sstevel@tonic-gate 118*0Sstevel@tonic-gate /* 119*0Sstevel@tonic-gate * scan at least MEMSCRUB_DFL_SPAN_PAGES each iteration 120*0Sstevel@tonic-gate */ 121*0Sstevel@tonic-gate #define MEMSCRUB_DFL_SPAN_PAGES ((4 * 1024 * 1024) / PAGESIZE) 122*0Sstevel@tonic-gate 123*0Sstevel@tonic-gate /* 124*0Sstevel@tonic-gate * almost anything is higher priority than scrubbing 125*0Sstevel@tonic-gate */ 126*0Sstevel@tonic-gate #define MEMSCRUB_DFL_THREAD_PRI 0 127*0Sstevel@tonic-gate 128*0Sstevel@tonic-gate /* 129*0Sstevel@tonic-gate * we can patch these defaults in /etc/system if necessary 130*0Sstevel@tonic-gate */ 131*0Sstevel@tonic-gate uint_t disable_memscrub = 0; 132*0Sstevel@tonic-gate pgcnt_t memscrub_min_pages = MEMSCRUB_MIN_PAGES; 133*0Sstevel@tonic-gate pgcnt_t memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES; 134*0Sstevel@tonic-gate time_t memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC; 135*0Sstevel@tonic-gate uint_t memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI; 136*0Sstevel@tonic-gate time_t memscrub_delay_start_sec = 10; 137*0Sstevel@tonic-gate 138*0Sstevel@tonic-gate /* 139*0Sstevel@tonic-gate * Static Routines 140*0Sstevel@tonic-gate */ 141*0Sstevel@tonic-gate static void memscrubber(void); 142*0Sstevel@tonic-gate static int system_is_idle(void); 143*0Sstevel@tonic-gate static int memscrub_add_span(uint64_t, uint64_t); 144*0Sstevel@tonic-gate 145*0Sstevel@tonic-gate /* 146*0Sstevel@tonic-gate * Static Data 147*0Sstevel@tonic-gate */ 148*0Sstevel@tonic-gate static struct memlist *memscrub_memlist; 149*0Sstevel@tonic-gate static uint_t memscrub_phys_pages; 150*0Sstevel@tonic-gate 151*0Sstevel@tonic-gate static kcondvar_t memscrub_cv; 152*0Sstevel@tonic-gate static kmutex_t memscrub_lock; 153*0Sstevel@tonic-gate /* 154*0Sstevel@tonic-gate * memscrub_lock protects memscrub_memlist 155*0Sstevel@tonic-gate */ 156*0Sstevel@tonic-gate uint_t memscrub_scans_done; 157*0Sstevel@tonic-gate 158*0Sstevel@tonic-gate uint_t memscrub_done_early; 159*0Sstevel@tonic-gate uint_t memscrub_early_sec; 160*0Sstevel@tonic-gate 161*0Sstevel@tonic-gate uint_t memscrub_done_late; 162*0Sstevel@tonic-gate time_t memscrub_late_sec; 163*0Sstevel@tonic-gate 164*0Sstevel@tonic-gate /* 165*0Sstevel@tonic-gate * create memscrub_memlist from phys_install list 166*0Sstevel@tonic-gate * initialize locks, set memscrub_phys_pages. 167*0Sstevel@tonic-gate */ 168*0Sstevel@tonic-gate void 169*0Sstevel@tonic-gate memscrub_init() 170*0Sstevel@tonic-gate { 171*0Sstevel@tonic-gate struct memlist *src; 172*0Sstevel@tonic-gate 173*0Sstevel@tonic-gate if (physmem < memscrub_min_pages) 174*0Sstevel@tonic-gate return; 175*0Sstevel@tonic-gate 176*0Sstevel@tonic-gate if (!kpm_enable) { 177*0Sstevel@tonic-gate memscrub_window = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP); 178*0Sstevel@tonic-gate memscrub_pte = hat_mempte_setup(memscrub_window); 179*0Sstevel@tonic-gate } 180*0Sstevel@tonic-gate 181*0Sstevel@tonic-gate /* 182*0Sstevel@tonic-gate * copy phys_install to memscrub_memlist 183*0Sstevel@tonic-gate */ 184*0Sstevel@tonic-gate for (src = phys_install; src; src = src->next) { 185*0Sstevel@tonic-gate if (memscrub_add_span(src->address, src->size)) { 186*0Sstevel@tonic-gate cmn_err(CE_WARN, 187*0Sstevel@tonic-gate "Memory scrubber failed to initialize\n"); 188*0Sstevel@tonic-gate return; 189*0Sstevel@tonic-gate } 190*0Sstevel@tonic-gate } 191*0Sstevel@tonic-gate 192*0Sstevel@tonic-gate mutex_init(&memscrub_lock, NULL, MUTEX_DRIVER, NULL); 193*0Sstevel@tonic-gate cv_init(&memscrub_cv, NULL, CV_DRIVER, NULL); 194*0Sstevel@tonic-gate 195*0Sstevel@tonic-gate /* 196*0Sstevel@tonic-gate * create memscrubber thread 197*0Sstevel@tonic-gate */ 198*0Sstevel@tonic-gate (void) thread_create(NULL, 0, (void (*)())memscrubber, NULL, 0, &p0, 199*0Sstevel@tonic-gate TS_RUN, memscrub_thread_pri); 200*0Sstevel@tonic-gate } 201*0Sstevel@tonic-gate 202*0Sstevel@tonic-gate #ifdef MEMSCRUB_DEBUG 203*0Sstevel@tonic-gate void 204*0Sstevel@tonic-gate memscrub_printmemlist(char *title, struct memlist *listp) 205*0Sstevel@tonic-gate { 206*0Sstevel@tonic-gate struct memlist *list; 207*0Sstevel@tonic-gate 208*0Sstevel@tonic-gate cmn_err(CE_CONT, "%s:\n", title); 209*0Sstevel@tonic-gate 210*0Sstevel@tonic-gate for (list = listp; list; list = list->next) { 211*0Sstevel@tonic-gate cmn_err(CE_CONT, "addr = 0x%llx, size = 0x%llx\n", 212*0Sstevel@tonic-gate list->address, list->size); 213*0Sstevel@tonic-gate } 214*0Sstevel@tonic-gate } 215*0Sstevel@tonic-gate #endif /* MEMSCRUB_DEBUG */ 216*0Sstevel@tonic-gate 217*0Sstevel@tonic-gate /* ARGSUSED */ 218*0Sstevel@tonic-gate void 219*0Sstevel@tonic-gate memscrub_wakeup(void *c) 220*0Sstevel@tonic-gate { 221*0Sstevel@tonic-gate /* 222*0Sstevel@tonic-gate * grab mutex to guarantee that our wakeup call 223*0Sstevel@tonic-gate * arrives after we go to sleep -- so we can't sleep forever. 224*0Sstevel@tonic-gate */ 225*0Sstevel@tonic-gate mutex_enter(&memscrub_lock); 226*0Sstevel@tonic-gate cv_signal(&memscrub_cv); 227*0Sstevel@tonic-gate mutex_exit(&memscrub_lock); 228*0Sstevel@tonic-gate } 229*0Sstevel@tonic-gate 230*0Sstevel@tonic-gate /* 231*0Sstevel@tonic-gate * this calculation doesn't account for the time that the actual scan 232*0Sstevel@tonic-gate * consumes -- so we'd fall slightly behind schedule with this 233*0Sstevel@tonic-gate * interval_sec. but the idle loop optimization below usually makes us 234*0Sstevel@tonic-gate * come in way ahead of schedule. 235*0Sstevel@tonic-gate */ 236*0Sstevel@tonic-gate static int 237*0Sstevel@tonic-gate compute_interval_sec() 238*0Sstevel@tonic-gate { 239*0Sstevel@tonic-gate if (memscrub_phys_pages <= memscrub_span_pages) 240*0Sstevel@tonic-gate return (memscrub_period_sec); 241*0Sstevel@tonic-gate else 242*0Sstevel@tonic-gate return (memscrub_period_sec/ 243*0Sstevel@tonic-gate (memscrub_phys_pages/memscrub_span_pages)); 244*0Sstevel@tonic-gate } 245*0Sstevel@tonic-gate 246*0Sstevel@tonic-gate void 247*0Sstevel@tonic-gate memscrubber() 248*0Sstevel@tonic-gate { 249*0Sstevel@tonic-gate time_t deadline; 250*0Sstevel@tonic-gate uint64_t mlp_last_addr; 251*0Sstevel@tonic-gate uint64_t mlp_next_addr; 252*0Sstevel@tonic-gate int reached_end = 1; 253*0Sstevel@tonic-gate time_t interval_sec = 0; 254*0Sstevel@tonic-gate struct memlist *mlp; 255*0Sstevel@tonic-gate 256*0Sstevel@tonic-gate extern void scan_memory(caddr_t, size_t); 257*0Sstevel@tonic-gate 258*0Sstevel@tonic-gate if (memscrub_memlist == NULL) { 259*0Sstevel@tonic-gate cmn_err(CE_WARN, "memscrub_memlist not initialized."); 260*0Sstevel@tonic-gate goto memscrub_exit; 261*0Sstevel@tonic-gate } 262*0Sstevel@tonic-gate 263*0Sstevel@tonic-gate mlp = memscrub_memlist; 264*0Sstevel@tonic-gate mlp_next_addr = mlp->address; 265*0Sstevel@tonic-gate mlp_last_addr = mlp->address + mlp->size; 266*0Sstevel@tonic-gate 267*0Sstevel@tonic-gate deadline = gethrestime_sec() + memscrub_delay_start_sec; 268*0Sstevel@tonic-gate 269*0Sstevel@tonic-gate for (;;) { 270*0Sstevel@tonic-gate if (disable_memscrub) 271*0Sstevel@tonic-gate break; 272*0Sstevel@tonic-gate 273*0Sstevel@tonic-gate mutex_enter(&memscrub_lock); 274*0Sstevel@tonic-gate 275*0Sstevel@tonic-gate /* 276*0Sstevel@tonic-gate * did we just reach the end of memory? 277*0Sstevel@tonic-gate */ 278*0Sstevel@tonic-gate if (reached_end) { 279*0Sstevel@tonic-gate time_t now = gethrestime_sec(); 280*0Sstevel@tonic-gate 281*0Sstevel@tonic-gate if (now >= deadline) { 282*0Sstevel@tonic-gate memscrub_done_late++; 283*0Sstevel@tonic-gate memscrub_late_sec += (now - deadline); 284*0Sstevel@tonic-gate /* 285*0Sstevel@tonic-gate * past deadline, start right away 286*0Sstevel@tonic-gate */ 287*0Sstevel@tonic-gate interval_sec = 0; 288*0Sstevel@tonic-gate 289*0Sstevel@tonic-gate deadline = now + memscrub_period_sec; 290*0Sstevel@tonic-gate } else { 291*0Sstevel@tonic-gate /* 292*0Sstevel@tonic-gate * we finished ahead of schedule. 293*0Sstevel@tonic-gate * wait till previous dealine before re-start. 294*0Sstevel@tonic-gate */ 295*0Sstevel@tonic-gate interval_sec = deadline - now; 296*0Sstevel@tonic-gate memscrub_done_early++; 297*0Sstevel@tonic-gate memscrub_early_sec += interval_sec; 298*0Sstevel@tonic-gate deadline += memscrub_period_sec; 299*0Sstevel@tonic-gate } 300*0Sstevel@tonic-gate } else { 301*0Sstevel@tonic-gate interval_sec = compute_interval_sec(); 302*0Sstevel@tonic-gate } 303*0Sstevel@tonic-gate 304*0Sstevel@tonic-gate /* 305*0Sstevel@tonic-gate * hit the snooze bar 306*0Sstevel@tonic-gate */ 307*0Sstevel@tonic-gate (void) timeout(memscrub_wakeup, NULL, interval_sec * hz); 308*0Sstevel@tonic-gate 309*0Sstevel@tonic-gate /* 310*0Sstevel@tonic-gate * go to sleep 311*0Sstevel@tonic-gate */ 312*0Sstevel@tonic-gate cv_wait(&memscrub_cv, &memscrub_lock); 313*0Sstevel@tonic-gate 314*0Sstevel@tonic-gate mutex_exit(&memscrub_lock); 315*0Sstevel@tonic-gate 316*0Sstevel@tonic-gate do { 317*0Sstevel@tonic-gate pgcnt_t pages = memscrub_span_pages; 318*0Sstevel@tonic-gate uint64_t address = mlp_next_addr; 319*0Sstevel@tonic-gate 320*0Sstevel@tonic-gate if (disable_memscrub) 321*0Sstevel@tonic-gate break; 322*0Sstevel@tonic-gate 323*0Sstevel@tonic-gate mutex_enter(&memscrub_lock); 324*0Sstevel@tonic-gate 325*0Sstevel@tonic-gate /* 326*0Sstevel@tonic-gate * Make sure we don't try to scan beyond the end of 327*0Sstevel@tonic-gate * the current memlist. If we would, then resize 328*0Sstevel@tonic-gate * our scan target for this iteration, and prepare 329*0Sstevel@tonic-gate * to read the next memlist entry on the next 330*0Sstevel@tonic-gate * iteration. 331*0Sstevel@tonic-gate */ 332*0Sstevel@tonic-gate reached_end = 0; 333*0Sstevel@tonic-gate if (address + mmu_ptob(pages) >= mlp_last_addr) { 334*0Sstevel@tonic-gate pages = mmu_btop(mlp_last_addr - address); 335*0Sstevel@tonic-gate mlp = mlp->next; 336*0Sstevel@tonic-gate if (mlp == NULL) { 337*0Sstevel@tonic-gate reached_end = 1; 338*0Sstevel@tonic-gate mlp = memscrub_memlist; 339*0Sstevel@tonic-gate } 340*0Sstevel@tonic-gate mlp_next_addr = mlp->address; 341*0Sstevel@tonic-gate mlp_last_addr = mlp->address + mlp->size; 342*0Sstevel@tonic-gate } else { 343*0Sstevel@tonic-gate mlp_next_addr += mmu_ptob(pages); 344*0Sstevel@tonic-gate } 345*0Sstevel@tonic-gate 346*0Sstevel@tonic-gate mutex_exit(&memscrub_lock); 347*0Sstevel@tonic-gate 348*0Sstevel@tonic-gate while (pages--) { 349*0Sstevel@tonic-gate pfn_t pfn = btop(address); 350*0Sstevel@tonic-gate 351*0Sstevel@tonic-gate /* 352*0Sstevel@tonic-gate * Without segkpm, the memscrubber cannot 353*0Sstevel@tonic-gate * be allowed to migrate across CPUs, as 354*0Sstevel@tonic-gate * the CPU-specific mapping of 355*0Sstevel@tonic-gate * memscrub_window would be incorrect. 356*0Sstevel@tonic-gate * With segkpm, switching CPUs is legal, but 357*0Sstevel@tonic-gate * inefficient. We don't use 358*0Sstevel@tonic-gate * kpreempt_disable as it might hold a 359*0Sstevel@tonic-gate * higher priority thread (eg, RT) too long 360*0Sstevel@tonic-gate * off CPU. 361*0Sstevel@tonic-gate */ 362*0Sstevel@tonic-gate thread_affinity_set(curthread, CPU_CURRENT); 363*0Sstevel@tonic-gate if (kpm_enable) 364*0Sstevel@tonic-gate memscrub_window = hat_kpm_pfn2va(pfn); 365*0Sstevel@tonic-gate else 366*0Sstevel@tonic-gate hat_mempte_remap(pfn, memscrub_window, 367*0Sstevel@tonic-gate memscrub_pte, 368*0Sstevel@tonic-gate PROT_READ, HAT_LOAD_NOCONSIST); 369*0Sstevel@tonic-gate 370*0Sstevel@tonic-gate scan_memory(memscrub_window, PAGESIZE); 371*0Sstevel@tonic-gate 372*0Sstevel@tonic-gate thread_affinity_clear(curthread); 373*0Sstevel@tonic-gate address += MMU_PAGESIZE; 374*0Sstevel@tonic-gate } 375*0Sstevel@tonic-gate 376*0Sstevel@tonic-gate memscrub_scans_done++; 377*0Sstevel@tonic-gate } while (!reached_end && system_is_idle()); 378*0Sstevel@tonic-gate } 379*0Sstevel@tonic-gate 380*0Sstevel@tonic-gate memscrub_exit: 381*0Sstevel@tonic-gate 382*0Sstevel@tonic-gate cmn_err(CE_NOTE, "memory scrubber exiting."); 383*0Sstevel@tonic-gate 384*0Sstevel@tonic-gate cv_destroy(&memscrub_cv); 385*0Sstevel@tonic-gate 386*0Sstevel@tonic-gate thread_exit(); 387*0Sstevel@tonic-gate } 388*0Sstevel@tonic-gate 389*0Sstevel@tonic-gate 390*0Sstevel@tonic-gate /* 391*0Sstevel@tonic-gate * return 1 if we're MP and all the other CPUs are idle 392*0Sstevel@tonic-gate */ 393*0Sstevel@tonic-gate static int 394*0Sstevel@tonic-gate system_is_idle() 395*0Sstevel@tonic-gate { 396*0Sstevel@tonic-gate int cpu_id; 397*0Sstevel@tonic-gate int found = 0; 398*0Sstevel@tonic-gate 399*0Sstevel@tonic-gate if (1 == ncpus_online) 400*0Sstevel@tonic-gate return (0); 401*0Sstevel@tonic-gate 402*0Sstevel@tonic-gate for (cpu_id = 0; cpu_id < NCPU; ++cpu_id) { 403*0Sstevel@tonic-gate if (!cpu[cpu_id]) 404*0Sstevel@tonic-gate continue; 405*0Sstevel@tonic-gate 406*0Sstevel@tonic-gate found++; 407*0Sstevel@tonic-gate 408*0Sstevel@tonic-gate if (cpu[cpu_id]->cpu_thread != cpu[cpu_id]->cpu_idle_thread) { 409*0Sstevel@tonic-gate if (CPU->cpu_id == cpu_id && 410*0Sstevel@tonic-gate CPU->cpu_disp->disp_nrunnable == 0) 411*0Sstevel@tonic-gate continue; 412*0Sstevel@tonic-gate return (0); 413*0Sstevel@tonic-gate } 414*0Sstevel@tonic-gate 415*0Sstevel@tonic-gate if (found == ncpus) 416*0Sstevel@tonic-gate break; 417*0Sstevel@tonic-gate } 418*0Sstevel@tonic-gate return (1); 419*0Sstevel@tonic-gate } 420*0Sstevel@tonic-gate 421*0Sstevel@tonic-gate /* 422*0Sstevel@tonic-gate * add a span to the memscrub list 423*0Sstevel@tonic-gate */ 424*0Sstevel@tonic-gate static int 425*0Sstevel@tonic-gate memscrub_add_span(uint64_t start, uint64_t bytes) 426*0Sstevel@tonic-gate { 427*0Sstevel@tonic-gate struct memlist *dst; 428*0Sstevel@tonic-gate struct memlist *prev, *next; 429*0Sstevel@tonic-gate uint64_t end = start + bytes - 1; 430*0Sstevel@tonic-gate int retval = 0; 431*0Sstevel@tonic-gate 432*0Sstevel@tonic-gate mutex_enter(&memscrub_lock); 433*0Sstevel@tonic-gate 434*0Sstevel@tonic-gate #ifdef MEMSCRUB_DEBUG 435*0Sstevel@tonic-gate memscrub_printmemlist("memscrub_memlist before", memscrub_memlist); 436*0Sstevel@tonic-gate cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages); 437*0Sstevel@tonic-gate cmn_err(CE_CONT, "memscrub_add_span: address: 0x%llx" 438*0Sstevel@tonic-gate " size: 0x%llx\n", start, bytes); 439*0Sstevel@tonic-gate #endif /* MEMSCRUB_DEBUG */ 440*0Sstevel@tonic-gate 441*0Sstevel@tonic-gate /* 442*0Sstevel@tonic-gate * Scan through the list to find the proper place to install it. 443*0Sstevel@tonic-gate */ 444*0Sstevel@tonic-gate prev = NULL; 445*0Sstevel@tonic-gate next = memscrub_memlist; 446*0Sstevel@tonic-gate while (next) { 447*0Sstevel@tonic-gate uint64_t ns = next->address; 448*0Sstevel@tonic-gate uint64_t ne = next->address + next->size - 1; 449*0Sstevel@tonic-gate 450*0Sstevel@tonic-gate /* 451*0Sstevel@tonic-gate * If this span overlaps with an existing span, then 452*0Sstevel@tonic-gate * something has gone horribly wrong with the phys_install 453*0Sstevel@tonic-gate * list. In fact, I'm surprised we made it this far. 454*0Sstevel@tonic-gate */ 455*0Sstevel@tonic-gate if ((start >= ns && start <= ne) || (end >= ns && end <= ne) || 456*0Sstevel@tonic-gate (start < ns && end > ne)) 457*0Sstevel@tonic-gate panic("memscrub found overlapping memory ranges " 458*0Sstevel@tonic-gate "(0x%p-0x%p) and (0x%p-0x%p)", 459*0Sstevel@tonic-gate (void *)start, (void *)end, (void *)ns, (void *)ne); 460*0Sstevel@tonic-gate 461*0Sstevel@tonic-gate /* 462*0Sstevel@tonic-gate * New span can be appended to an existing one. 463*0Sstevel@tonic-gate */ 464*0Sstevel@tonic-gate if (start == ne + 1) { 465*0Sstevel@tonic-gate next->size += bytes; 466*0Sstevel@tonic-gate goto add_done; 467*0Sstevel@tonic-gate } 468*0Sstevel@tonic-gate 469*0Sstevel@tonic-gate /* 470*0Sstevel@tonic-gate * New span can be prepended to an existing one. 471*0Sstevel@tonic-gate */ 472*0Sstevel@tonic-gate if (end + 1 == ns) { 473*0Sstevel@tonic-gate next->size += bytes; 474*0Sstevel@tonic-gate next->address = start; 475*0Sstevel@tonic-gate goto add_done; 476*0Sstevel@tonic-gate } 477*0Sstevel@tonic-gate 478*0Sstevel@tonic-gate /* 479*0Sstevel@tonic-gate * If the next span has a higher start address than the new 480*0Sstevel@tonic-gate * one, then we have found the right spot for our 481*0Sstevel@tonic-gate * insertion. 482*0Sstevel@tonic-gate */ 483*0Sstevel@tonic-gate if (ns > start) 484*0Sstevel@tonic-gate break; 485*0Sstevel@tonic-gate 486*0Sstevel@tonic-gate prev = next; 487*0Sstevel@tonic-gate next = next->next; 488*0Sstevel@tonic-gate } 489*0Sstevel@tonic-gate 490*0Sstevel@tonic-gate /* 491*0Sstevel@tonic-gate * allocate a new struct memlist 492*0Sstevel@tonic-gate */ 493*0Sstevel@tonic-gate dst = kmem_alloc(sizeof (struct memlist), KM_NOSLEEP); 494*0Sstevel@tonic-gate if (dst == NULL) { 495*0Sstevel@tonic-gate retval = -1; 496*0Sstevel@tonic-gate goto add_done; 497*0Sstevel@tonic-gate } 498*0Sstevel@tonic-gate dst->address = start; 499*0Sstevel@tonic-gate dst->size = bytes; 500*0Sstevel@tonic-gate dst->prev = prev; 501*0Sstevel@tonic-gate dst->next = next; 502*0Sstevel@tonic-gate 503*0Sstevel@tonic-gate if (prev) 504*0Sstevel@tonic-gate prev->next = dst; 505*0Sstevel@tonic-gate else 506*0Sstevel@tonic-gate memscrub_memlist = dst; 507*0Sstevel@tonic-gate 508*0Sstevel@tonic-gate if (next) 509*0Sstevel@tonic-gate next->prev = dst; 510*0Sstevel@tonic-gate 511*0Sstevel@tonic-gate add_done: 512*0Sstevel@tonic-gate 513*0Sstevel@tonic-gate if (retval != -1) 514*0Sstevel@tonic-gate memscrub_phys_pages += mmu_btop(bytes); 515*0Sstevel@tonic-gate 516*0Sstevel@tonic-gate #ifdef MEMSCRUB_DEBUG 517*0Sstevel@tonic-gate memscrub_printmemlist("memscrub_memlist after", memscrub_memlist); 518*0Sstevel@tonic-gate cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages); 519*0Sstevel@tonic-gate #endif /* MEMSCRUB_DEBUG */ 520*0Sstevel@tonic-gate 521*0Sstevel@tonic-gate mutex_exit(&memscrub_lock); 522*0Sstevel@tonic-gate return (retval); 523*0Sstevel@tonic-gate } 524