1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2004 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*0Sstevel@tonic-gate 29*0Sstevel@tonic-gate /* 30*0Sstevel@tonic-gate * The following routines implement the hat layer's 31*0Sstevel@tonic-gate * recording of the referenced and modified bits. 32*0Sstevel@tonic-gate */ 33*0Sstevel@tonic-gate 34*0Sstevel@tonic-gate #include <sys/types.h> 35*0Sstevel@tonic-gate #include <sys/param.h> 36*0Sstevel@tonic-gate #include <sys/systm.h> 37*0Sstevel@tonic-gate #include <sys/debug.h> 38*0Sstevel@tonic-gate #include <sys/kmem.h> 39*0Sstevel@tonic-gate 40*0Sstevel@tonic-gate /* 41*0Sstevel@tonic-gate * Note, usage of cmn_err requires you not hold any hat layer locks. 42*0Sstevel@tonic-gate */ 43*0Sstevel@tonic-gate #include <sys/cmn_err.h> 44*0Sstevel@tonic-gate 45*0Sstevel@tonic-gate #include <vm/as.h> 46*0Sstevel@tonic-gate #include <vm/hat.h> 47*0Sstevel@tonic-gate 48*0Sstevel@tonic-gate kmutex_t hat_statlock; /* protects all hat statistics data */ 49*0Sstevel@tonic-gate struct hrmstat *hrm_memlist; /* tracks memory alloced for hrm_blist blocks */ 50*0Sstevel@tonic-gate struct hrmstat **hrm_hashtab; /* hash table for finding blocks quickly */ 51*0Sstevel@tonic-gate struct hrmstat *hrm_blist; 52*0Sstevel@tonic-gate int hrm_blist_incr = HRM_BLIST_INCR; 53*0Sstevel@tonic-gate int hrm_blist_lowater = HRM_BLIST_INCR/2; 54*0Sstevel@tonic-gate int hrm_blist_num = 0; 55*0Sstevel@tonic-gate int hrm_blist_total = 0; 56*0Sstevel@tonic-gate int hrm_mlockinited = 0; 57*0Sstevel@tonic-gate int hrm_allocfailmsg = 0; /* print a message when allocations fail */ 58*0Sstevel@tonic-gate int hrm_allocfail = 0; 59*0Sstevel@tonic-gate 60*0Sstevel@tonic-gate static struct hrmstat *hrm_balloc(void); 61*0Sstevel@tonic-gate static int hrm_init(void); 62*0Sstevel@tonic-gate static void hrm_link(struct hrmstat *); 63*0Sstevel@tonic-gate static void hrm_setbits(struct hrmstat *, caddr_t, uint_t); 64*0Sstevel@tonic-gate static void hrm_hashout(struct hrmstat *); 65*0Sstevel@tonic-gate static void hrm_getblk(int); 66*0Sstevel@tonic-gate 67*0Sstevel@tonic-gate #define hrm_hash(as, addr) \ 68*0Sstevel@tonic-gate (HRM_HASHMASK & \ 69*0Sstevel@tonic-gate (((uintptr_t)(addr) >> HRM_BASESHIFT) ^ ((uintptr_t)(as) >> 2))) 70*0Sstevel@tonic-gate 71*0Sstevel@tonic-gate #define hrm_match(hrm, as, addr) \ 72*0Sstevel@tonic-gate (((hrm)->hrm_as == (as) && \ 73*0Sstevel@tonic-gate ((hrm)->hrm_base == ((uintptr_t)(addr) & HRM_BASEMASK))) ? 1 : 0) 74*0Sstevel@tonic-gate 75*0Sstevel@tonic-gate /* 76*0Sstevel@tonic-gate * reserve enough statistic blocks for 77*0Sstevel@tonic-gate * chunk of bytes (pages) in a given as. 78*0Sstevel@tonic-gate */ 79*0Sstevel@tonic-gate /* ARGSUSED */ 80*0Sstevel@tonic-gate void 81*0Sstevel@tonic-gate hat_resvstat(size_t chunk, struct as *as, caddr_t addr) 82*0Sstevel@tonic-gate { 83*0Sstevel@tonic-gate int nhrm = btop(chunk)/HRM_PAGES; 84*0Sstevel@tonic-gate 85*0Sstevel@tonic-gate if (nhrm < HRM_BLIST_INCR) 86*0Sstevel@tonic-gate nhrm = 0; /* preallocate at least HRM_BLIST_INCR */ 87*0Sstevel@tonic-gate hrm_getblk(nhrm); 88*0Sstevel@tonic-gate } 89*0Sstevel@tonic-gate 90*0Sstevel@tonic-gate /* 91*0Sstevel@tonic-gate * Start the statistics gathering for an address space. 92*0Sstevel@tonic-gate * Return -1 if we can't do it, otherwise return an opaque 93*0Sstevel@tonic-gate * identifier to be used when querying for the gathered statistics. 94*0Sstevel@tonic-gate * The identifier is an unused bit in a_vbits. 95*0Sstevel@tonic-gate * Bit 0 is reserved for swsmon. 96*0Sstevel@tonic-gate */ 97*0Sstevel@tonic-gate int 98*0Sstevel@tonic-gate hat_startstat(struct as *as) 99*0Sstevel@tonic-gate { 100*0Sstevel@tonic-gate uint_t nbits; /* number of bits */ 101*0Sstevel@tonic-gate uint_t bn; /* bit number */ 102*0Sstevel@tonic-gate uint_t id; /* new vbit, identifier */ 103*0Sstevel@tonic-gate uint_t vbits; /* used vbits of address space */ 104*0Sstevel@tonic-gate size_t chunk; /* mapped size for stats */ 105*0Sstevel@tonic-gate /* 106*0Sstevel@tonic-gate * Initialize global data, if needed. 107*0Sstevel@tonic-gate */ 108*0Sstevel@tonic-gate if (hrm_init() == -1) 109*0Sstevel@tonic-gate return (-1); 110*0Sstevel@tonic-gate 111*0Sstevel@tonic-gate /* 112*0Sstevel@tonic-gate * If the refmod saving memory allocator runs out, print 113*0Sstevel@tonic-gate * a warning message about how to fix it, see comment at 114*0Sstevel@tonic-gate * the beginning of hat_setstat. 115*0Sstevel@tonic-gate */ 116*0Sstevel@tonic-gate if (hrm_allocfailmsg) { 117*0Sstevel@tonic-gate cmn_err(CE_WARN, 118*0Sstevel@tonic-gate "hrm_balloc failures occured, increase hrm_blist_incr"); 119*0Sstevel@tonic-gate hrm_allocfailmsg = 0; 120*0Sstevel@tonic-gate } 121*0Sstevel@tonic-gate 122*0Sstevel@tonic-gate /* 123*0Sstevel@tonic-gate * Verify that a buffer of statistics blocks exists 124*0Sstevel@tonic-gate * and allocate more, if needed. 125*0Sstevel@tonic-gate */ 126*0Sstevel@tonic-gate 127*0Sstevel@tonic-gate chunk = hat_get_mapped_size(as->a_hat); 128*0Sstevel@tonic-gate chunk = (btop(chunk)/HRM_PAGES); 129*0Sstevel@tonic-gate if (chunk < HRM_BLIST_INCR) 130*0Sstevel@tonic-gate chunk = 0; 131*0Sstevel@tonic-gate 132*0Sstevel@tonic-gate hrm_getblk((int)chunk); 133*0Sstevel@tonic-gate 134*0Sstevel@tonic-gate /* 135*0Sstevel@tonic-gate * Find a unused id in the given address space. 136*0Sstevel@tonic-gate */ 137*0Sstevel@tonic-gate hat_enter(as->a_hat); 138*0Sstevel@tonic-gate vbits = as->a_vbits; 139*0Sstevel@tonic-gate nbits = sizeof (as->a_vbits) * NBBY; 140*0Sstevel@tonic-gate for (bn = 1, id = 2; bn < (nbits - 1); bn++, id <<= 1) 141*0Sstevel@tonic-gate if ((id & vbits) == 0) 142*0Sstevel@tonic-gate break; 143*0Sstevel@tonic-gate if (bn >= (nbits - 1)) { 144*0Sstevel@tonic-gate hat_exit(as->a_hat); 145*0Sstevel@tonic-gate return (-1); 146*0Sstevel@tonic-gate } 147*0Sstevel@tonic-gate as->a_vbits |= id; 148*0Sstevel@tonic-gate hat_exit(as->a_hat); 149*0Sstevel@tonic-gate (void) hat_stats_enable(as->a_hat); 150*0Sstevel@tonic-gate return (id); 151*0Sstevel@tonic-gate } 152*0Sstevel@tonic-gate 153*0Sstevel@tonic-gate /* 154*0Sstevel@tonic-gate * Record referenced and modified information for an address space. 155*0Sstevel@tonic-gate * Rmbits is a word containing the referenced bit in bit position 1 156*0Sstevel@tonic-gate * and the modified bit in bit position 0. 157*0Sstevel@tonic-gate * 158*0Sstevel@tonic-gate * For current informational uses, one can rerun any program using 159*0Sstevel@tonic-gate * this facility after modifying the hrm_blist_incr to be a larger 160*0Sstevel@tonic-gate * amount so that a larger buffer of blocks will be maintained. 161*0Sstevel@tonic-gate */ 162*0Sstevel@tonic-gate void 163*0Sstevel@tonic-gate hat_setstat(struct as *as, caddr_t addr, size_t len, uint_t rmbits) 164*0Sstevel@tonic-gate { 165*0Sstevel@tonic-gate struct hrmstat *hrm; 166*0Sstevel@tonic-gate uint_t vbits, newbits, nb; 167*0Sstevel@tonic-gate int h; 168*0Sstevel@tonic-gate 169*0Sstevel@tonic-gate ASSERT(len == PAGESIZE); 170*0Sstevel@tonic-gate ASSERT((rmbits & ~(P_MOD|P_REF)) == 0); 171*0Sstevel@tonic-gate 172*0Sstevel@tonic-gate if (rmbits == 0) 173*0Sstevel@tonic-gate return; 174*0Sstevel@tonic-gate 175*0Sstevel@tonic-gate /* 176*0Sstevel@tonic-gate * Initialize global data, if needed. 177*0Sstevel@tonic-gate */ 178*0Sstevel@tonic-gate if (hrm_init() == -1) 179*0Sstevel@tonic-gate return; 180*0Sstevel@tonic-gate 181*0Sstevel@tonic-gate mutex_enter(&hat_statlock); 182*0Sstevel@tonic-gate 183*0Sstevel@tonic-gate /* 184*0Sstevel@tonic-gate * Search the hash list for the as and addr we are looking for 185*0Sstevel@tonic-gate * and set the ref and mod bits in every block that matches. 186*0Sstevel@tonic-gate */ 187*0Sstevel@tonic-gate vbits = 0; 188*0Sstevel@tonic-gate h = hrm_hash(as, addr); 189*0Sstevel@tonic-gate for (hrm = hrm_hashtab[h]; hrm; hrm = hrm->hrm_hnext) { 190*0Sstevel@tonic-gate if (hrm_match(hrm, as, addr)) { 191*0Sstevel@tonic-gate hrm_setbits(hrm, addr, rmbits); 192*0Sstevel@tonic-gate vbits |= hrm->hrm_id; 193*0Sstevel@tonic-gate } 194*0Sstevel@tonic-gate } 195*0Sstevel@tonic-gate 196*0Sstevel@tonic-gate /* 197*0Sstevel@tonic-gate * If we didn't find a block for all of the enabled 198*0Sstevel@tonic-gate * vpages bits, then allocate and initialize a block 199*0Sstevel@tonic-gate * for each bit that was not found. 200*0Sstevel@tonic-gate */ 201*0Sstevel@tonic-gate if (vbits != as->a_vbits) { 202*0Sstevel@tonic-gate newbits = vbits ^ as->a_vbits; 203*0Sstevel@tonic-gate while (newbits) { 204*0Sstevel@tonic-gate if (ffs(newbits)) 205*0Sstevel@tonic-gate nb = 1 << (ffs(newbits)-1); 206*0Sstevel@tonic-gate hrm = (struct hrmstat *)hrm_balloc(); 207*0Sstevel@tonic-gate if (hrm == NULL) { 208*0Sstevel@tonic-gate hrm_allocfailmsg = 1; 209*0Sstevel@tonic-gate hrm_allocfail++; 210*0Sstevel@tonic-gate mutex_exit(&hat_statlock); 211*0Sstevel@tonic-gate return; 212*0Sstevel@tonic-gate } 213*0Sstevel@tonic-gate hrm->hrm_as = as; 214*0Sstevel@tonic-gate hrm->hrm_base = (uintptr_t)addr & HRM_BASEMASK; 215*0Sstevel@tonic-gate hrm->hrm_id = nb; 216*0Sstevel@tonic-gate hrm_link(hrm); 217*0Sstevel@tonic-gate hrm_setbits(hrm, addr, rmbits); 218*0Sstevel@tonic-gate newbits &= ~nb; 219*0Sstevel@tonic-gate } 220*0Sstevel@tonic-gate } 221*0Sstevel@tonic-gate mutex_exit(&hat_statlock); 222*0Sstevel@tonic-gate } 223*0Sstevel@tonic-gate 224*0Sstevel@tonic-gate /* 225*0Sstevel@tonic-gate * Free the resources used to maintain the referenced and modified 226*0Sstevel@tonic-gate * statistics for the virtual page view of an address space 227*0Sstevel@tonic-gate * identified by id. 228*0Sstevel@tonic-gate */ 229*0Sstevel@tonic-gate void 230*0Sstevel@tonic-gate hat_freestat(struct as *as, int id) 231*0Sstevel@tonic-gate { 232*0Sstevel@tonic-gate struct hrmstat *hrm, *prev_ahrm; 233*0Sstevel@tonic-gate 234*0Sstevel@tonic-gate hat_stats_disable(as->a_hat); /* tell the hat layer to stop */ 235*0Sstevel@tonic-gate hat_enter(as->a_hat); 236*0Sstevel@tonic-gate if (id == 0) 237*0Sstevel@tonic-gate as->a_vbits = 0; 238*0Sstevel@tonic-gate else 239*0Sstevel@tonic-gate as->a_vbits &= ~id; 240*0Sstevel@tonic-gate 241*0Sstevel@tonic-gate if ((hrm = as->a_hrm) == NULL) { 242*0Sstevel@tonic-gate hat_exit(as->a_hat); 243*0Sstevel@tonic-gate return; 244*0Sstevel@tonic-gate } 245*0Sstevel@tonic-gate hat_exit(as->a_hat); 246*0Sstevel@tonic-gate 247*0Sstevel@tonic-gate mutex_enter(&hat_statlock); 248*0Sstevel@tonic-gate if (hrm_hashtab == NULL) { 249*0Sstevel@tonic-gate /* can't happen? */ 250*0Sstevel@tonic-gate mutex_exit(&hat_statlock); 251*0Sstevel@tonic-gate return; 252*0Sstevel@tonic-gate } 253*0Sstevel@tonic-gate for (prev_ahrm = NULL; hrm; hrm = hrm->hrm_anext) { 254*0Sstevel@tonic-gate if ((id == hrm->hrm_id) || (id == NULL)) { 255*0Sstevel@tonic-gate 256*0Sstevel@tonic-gate hrm_hashout(hrm); 257*0Sstevel@tonic-gate hrm->hrm_hnext = hrm_blist; 258*0Sstevel@tonic-gate hrm_blist = hrm; 259*0Sstevel@tonic-gate hrm_blist_num++; 260*0Sstevel@tonic-gate 261*0Sstevel@tonic-gate if (prev_ahrm == NULL) 262*0Sstevel@tonic-gate as->a_hrm = hrm->hrm_anext; 263*0Sstevel@tonic-gate else 264*0Sstevel@tonic-gate prev_ahrm->hrm_anext = hrm->hrm_anext; 265*0Sstevel@tonic-gate 266*0Sstevel@tonic-gate } else 267*0Sstevel@tonic-gate prev_ahrm = hrm; 268*0Sstevel@tonic-gate } 269*0Sstevel@tonic-gate 270*0Sstevel@tonic-gate /* 271*0Sstevel@tonic-gate * If all statistics blocks are free, 272*0Sstevel@tonic-gate * return the memory to the system. 273*0Sstevel@tonic-gate */ 274*0Sstevel@tonic-gate if (hrm_blist_num == hrm_blist_total) { 275*0Sstevel@tonic-gate /* zero the block list since we are giving back its memory */ 276*0Sstevel@tonic-gate hrm_blist = NULL; 277*0Sstevel@tonic-gate hrm_blist_num = 0; 278*0Sstevel@tonic-gate hrm_blist_total = 0; 279*0Sstevel@tonic-gate while (hrm_memlist) { 280*0Sstevel@tonic-gate hrm = hrm_memlist; 281*0Sstevel@tonic-gate hrm_memlist = hrm->hrm_hnext; 282*0Sstevel@tonic-gate kmem_free(hrm, hrm->hrm_base); 283*0Sstevel@tonic-gate } 284*0Sstevel@tonic-gate ASSERT(hrm_memlist == NULL); 285*0Sstevel@tonic-gate kmem_free(hrm_hashtab, HRM_HASHSIZE * sizeof (char *)); 286*0Sstevel@tonic-gate hrm_hashtab = NULL; 287*0Sstevel@tonic-gate } 288*0Sstevel@tonic-gate mutex_exit(&hat_statlock); 289*0Sstevel@tonic-gate } 290*0Sstevel@tonic-gate 291*0Sstevel@tonic-gate /* 292*0Sstevel@tonic-gate * Initialize any global state for the statistics handling. 293*0Sstevel@tonic-gate * Hrm_lock protects the globally allocted memory: 294*0Sstevel@tonic-gate * hrm_memlist and hrm_hashtab. 295*0Sstevel@tonic-gate */ 296*0Sstevel@tonic-gate static int 297*0Sstevel@tonic-gate hrm_init(void) 298*0Sstevel@tonic-gate { 299*0Sstevel@tonic-gate /* 300*0Sstevel@tonic-gate * Alloacte the hashtable if it doesn't exist yet. 301*0Sstevel@tonic-gate */ 302*0Sstevel@tonic-gate mutex_enter(&hat_statlock); 303*0Sstevel@tonic-gate if (hrm_hashtab == NULL) 304*0Sstevel@tonic-gate hrm_hashtab = 305*0Sstevel@tonic-gate kmem_zalloc(HRM_HASHSIZE * sizeof (char *), KM_SLEEP); 306*0Sstevel@tonic-gate mutex_exit(&hat_statlock); 307*0Sstevel@tonic-gate return (0); 308*0Sstevel@tonic-gate } 309*0Sstevel@tonic-gate 310*0Sstevel@tonic-gate /* 311*0Sstevel@tonic-gate * Grab memory for statistics gathering of the hat layer. 312*0Sstevel@tonic-gate */ 313*0Sstevel@tonic-gate static void 314*0Sstevel@tonic-gate hrm_getblk(int chunk) 315*0Sstevel@tonic-gate { 316*0Sstevel@tonic-gate struct hrmstat *hrm, *l; 317*0Sstevel@tonic-gate int i; 318*0Sstevel@tonic-gate int hrm_incr; 319*0Sstevel@tonic-gate 320*0Sstevel@tonic-gate mutex_enter(&hat_statlock); 321*0Sstevel@tonic-gate if ((hrm_blist == NULL) || 322*0Sstevel@tonic-gate (hrm_blist_num <= hrm_blist_lowater) || 323*0Sstevel@tonic-gate chunk) { 324*0Sstevel@tonic-gate 325*0Sstevel@tonic-gate mutex_exit(&hat_statlock); 326*0Sstevel@tonic-gate 327*0Sstevel@tonic-gate hrm_incr = chunk? chunk : hrm_blist_incr; 328*0Sstevel@tonic-gate hrm = kmem_zalloc(sizeof (struct hrmstat) * hrm_incr, KM_SLEEP); 329*0Sstevel@tonic-gate hrm->hrm_base = sizeof (struct hrmstat) * hrm_incr; 330*0Sstevel@tonic-gate 331*0Sstevel@tonic-gate /* 332*0Sstevel@tonic-gate * thread the allocated blocks onto a freelist 333*0Sstevel@tonic-gate * using the first block to hold information for 334*0Sstevel@tonic-gate * freeing them all later 335*0Sstevel@tonic-gate */ 336*0Sstevel@tonic-gate mutex_enter(&hat_statlock); 337*0Sstevel@tonic-gate hrm->hrm_hnext = hrm_memlist; 338*0Sstevel@tonic-gate hrm_memlist = hrm; 339*0Sstevel@tonic-gate 340*0Sstevel@tonic-gate hrm_blist_total += (hrm_incr - 1); 341*0Sstevel@tonic-gate for (i = 1; i < hrm_incr; i++) { 342*0Sstevel@tonic-gate l = &hrm[i]; 343*0Sstevel@tonic-gate l->hrm_hnext = hrm_blist; 344*0Sstevel@tonic-gate hrm_blist = l; 345*0Sstevel@tonic-gate hrm_blist_num++; 346*0Sstevel@tonic-gate } 347*0Sstevel@tonic-gate } 348*0Sstevel@tonic-gate mutex_exit(&hat_statlock); 349*0Sstevel@tonic-gate } 350*0Sstevel@tonic-gate 351*0Sstevel@tonic-gate static void 352*0Sstevel@tonic-gate hrm_hashin(struct hrmstat *hrm) 353*0Sstevel@tonic-gate { 354*0Sstevel@tonic-gate int h; 355*0Sstevel@tonic-gate 356*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&hat_statlock)); 357*0Sstevel@tonic-gate h = hrm_hash(hrm->hrm_as, hrm->hrm_base); 358*0Sstevel@tonic-gate 359*0Sstevel@tonic-gate hrm->hrm_hnext = hrm_hashtab[h]; 360*0Sstevel@tonic-gate hrm_hashtab[h] = hrm; 361*0Sstevel@tonic-gate } 362*0Sstevel@tonic-gate 363*0Sstevel@tonic-gate static void 364*0Sstevel@tonic-gate hrm_hashout(struct hrmstat *hrm) 365*0Sstevel@tonic-gate { 366*0Sstevel@tonic-gate struct hrmstat *list, **prev_hrm; 367*0Sstevel@tonic-gate int h; 368*0Sstevel@tonic-gate 369*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&hat_statlock)); 370*0Sstevel@tonic-gate h = hrm_hash(hrm->hrm_as, hrm->hrm_base); 371*0Sstevel@tonic-gate list = hrm_hashtab[h]; 372*0Sstevel@tonic-gate prev_hrm = &hrm_hashtab[h]; 373*0Sstevel@tonic-gate 374*0Sstevel@tonic-gate while (list) { 375*0Sstevel@tonic-gate if (list == hrm) { 376*0Sstevel@tonic-gate *prev_hrm = list->hrm_hnext; 377*0Sstevel@tonic-gate return; 378*0Sstevel@tonic-gate } 379*0Sstevel@tonic-gate prev_hrm = &list->hrm_hnext; 380*0Sstevel@tonic-gate list = list->hrm_hnext; 381*0Sstevel@tonic-gate } 382*0Sstevel@tonic-gate } 383*0Sstevel@tonic-gate 384*0Sstevel@tonic-gate 385*0Sstevel@tonic-gate /* 386*0Sstevel@tonic-gate * Link a statistic block into an address space and also put it 387*0Sstevel@tonic-gate * on the hash list for future references. 388*0Sstevel@tonic-gate */ 389*0Sstevel@tonic-gate static void 390*0Sstevel@tonic-gate hrm_link(struct hrmstat *hrm) 391*0Sstevel@tonic-gate { 392*0Sstevel@tonic-gate struct as *as = hrm->hrm_as; 393*0Sstevel@tonic-gate 394*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&hat_statlock)); 395*0Sstevel@tonic-gate hrm->hrm_anext = as->a_hrm; 396*0Sstevel@tonic-gate as->a_hrm = hrm; 397*0Sstevel@tonic-gate hrm_hashin(hrm); 398*0Sstevel@tonic-gate } 399*0Sstevel@tonic-gate 400*0Sstevel@tonic-gate /* 401*0Sstevel@tonic-gate * Allocate a block for statistics keeping. 402*0Sstevel@tonic-gate * Returns NULL if blocks are unavailable. 403*0Sstevel@tonic-gate */ 404*0Sstevel@tonic-gate static struct hrmstat * 405*0Sstevel@tonic-gate hrm_balloc(void) 406*0Sstevel@tonic-gate { 407*0Sstevel@tonic-gate struct hrmstat *hrm; 408*0Sstevel@tonic-gate 409*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&hat_statlock)); 410*0Sstevel@tonic-gate 411*0Sstevel@tonic-gate hrm = hrm_blist; 412*0Sstevel@tonic-gate if (hrm != NULL) { 413*0Sstevel@tonic-gate hrm_blist = hrm->hrm_hnext; 414*0Sstevel@tonic-gate hrm_blist_num--; 415*0Sstevel@tonic-gate hrm->hrm_hnext = NULL; 416*0Sstevel@tonic-gate } 417*0Sstevel@tonic-gate return (hrm); 418*0Sstevel@tonic-gate } 419*0Sstevel@tonic-gate 420*0Sstevel@tonic-gate /* 421*0Sstevel@tonic-gate * Set the ref and mod bits for addr within statistics block hrm. 422*0Sstevel@tonic-gate */ 423*0Sstevel@tonic-gate static void 424*0Sstevel@tonic-gate hrm_setbits(struct hrmstat *hrm, caddr_t addr, uint_t bits) 425*0Sstevel@tonic-gate { 426*0Sstevel@tonic-gate uint_t po, bo, spb; 427*0Sstevel@tonic-gate uint_t nbits; 428*0Sstevel@tonic-gate 429*0Sstevel@tonic-gate po = ((uintptr_t)addr & HRM_BASEOFFSET) >> MMU_PAGESHIFT; /* pg off */ 430*0Sstevel@tonic-gate bo = po / (NBBY / 2); /* which byte in bit array */ 431*0Sstevel@tonic-gate spb = (3 - (po & 3)) * 2; /* shift position within byte */ 432*0Sstevel@tonic-gate nbits = bits << spb; /* bit mask */ 433*0Sstevel@tonic-gate hrm->hrm_bits[bo] |= nbits; 434*0Sstevel@tonic-gate } 435*0Sstevel@tonic-gate 436*0Sstevel@tonic-gate /* 437*0Sstevel@tonic-gate * Return collected statistics about an address space. 438*0Sstevel@tonic-gate * If clearflag is set, atomically read and zero the bits. 439*0Sstevel@tonic-gate * 440*0Sstevel@tonic-gate * Fill in the data array supplied with the referenced and 441*0Sstevel@tonic-gate * modified bits collected for address range [addr ... addr + len] 442*0Sstevel@tonic-gate * in address space, as, uniquely identified by id. 443*0Sstevel@tonic-gate * The destination is a byte array. We fill in three bits per byte: 444*0Sstevel@tonic-gate * referenced, modified, and hwmapped bits. 445*0Sstevel@tonic-gate * Kernel only interface, can't fault on destination data array. 446*0Sstevel@tonic-gate * 447*0Sstevel@tonic-gate */ 448*0Sstevel@tonic-gate void 449*0Sstevel@tonic-gate hat_getstat(struct as *as, caddr_t addr, size_t len, uint_t id, 450*0Sstevel@tonic-gate caddr_t datap, int clearflag) 451*0Sstevel@tonic-gate { 452*0Sstevel@tonic-gate size_t np; /* number of pages */ 453*0Sstevel@tonic-gate caddr_t a; 454*0Sstevel@tonic-gate char *dp; 455*0Sstevel@tonic-gate 456*0Sstevel@tonic-gate np = btop(len); 457*0Sstevel@tonic-gate bzero(datap, np); 458*0Sstevel@tonic-gate 459*0Sstevel@tonic-gate hat_sync(as->a_hat, addr, len, clearflag); 460*0Sstevel@tonic-gate 461*0Sstevel@tonic-gate /* allocate more statistics blocks if needed */ 462*0Sstevel@tonic-gate hrm_getblk(0); 463*0Sstevel@tonic-gate 464*0Sstevel@tonic-gate mutex_enter(&hat_statlock); 465*0Sstevel@tonic-gate if (hrm_hashtab == NULL) { 466*0Sstevel@tonic-gate /* can happen when victim process exits */ 467*0Sstevel@tonic-gate mutex_exit(&hat_statlock); 468*0Sstevel@tonic-gate return; 469*0Sstevel@tonic-gate } 470*0Sstevel@tonic-gate dp = datap; 471*0Sstevel@tonic-gate a = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 472*0Sstevel@tonic-gate while (a < addr + len) { 473*0Sstevel@tonic-gate struct hrmstat *hrm; 474*0Sstevel@tonic-gate size_t n; /* number of pages, temp */ 475*0Sstevel@tonic-gate int h; /* hash index */ 476*0Sstevel@tonic-gate uint_t po; 477*0Sstevel@tonic-gate 478*0Sstevel@tonic-gate h = hrm_hash(as, a); 479*0Sstevel@tonic-gate n = (HRM_PAGES - 480*0Sstevel@tonic-gate (((uintptr_t)a & HRM_PAGEMASK) >> MMU_PAGESHIFT)); 481*0Sstevel@tonic-gate if (n > np) 482*0Sstevel@tonic-gate n = np; 483*0Sstevel@tonic-gate po = ((uintptr_t)a & HRM_BASEOFFSET) >> MMU_PAGESHIFT; 484*0Sstevel@tonic-gate 485*0Sstevel@tonic-gate for (hrm = hrm_hashtab[h]; hrm; hrm = hrm->hrm_hnext) { 486*0Sstevel@tonic-gate if (hrm->hrm_as == as && 487*0Sstevel@tonic-gate hrm->hrm_base == ((uintptr_t)a & HRM_BASEMASK) && 488*0Sstevel@tonic-gate id == hrm->hrm_id) { 489*0Sstevel@tonic-gate int i, nr; 490*0Sstevel@tonic-gate uint_t bo, spb; 491*0Sstevel@tonic-gate 492*0Sstevel@tonic-gate /* 493*0Sstevel@tonic-gate * Extract leading unaligned bits. 494*0Sstevel@tonic-gate */ 495*0Sstevel@tonic-gate i = 0; 496*0Sstevel@tonic-gate while (i < n && (po & 3)) { 497*0Sstevel@tonic-gate bo = po / (NBBY / 2); 498*0Sstevel@tonic-gate spb = (3 - (po & 3)) * 2; 499*0Sstevel@tonic-gate *dp++ |= (hrm->hrm_bits[bo] >> spb) & 3; 500*0Sstevel@tonic-gate if (clearflag) 501*0Sstevel@tonic-gate hrm->hrm_bits[bo] &= ~(3<<spb); 502*0Sstevel@tonic-gate po++; 503*0Sstevel@tonic-gate i++; 504*0Sstevel@tonic-gate } 505*0Sstevel@tonic-gate /* 506*0Sstevel@tonic-gate * Extract aligned bits. 507*0Sstevel@tonic-gate */ 508*0Sstevel@tonic-gate nr = n/4*4; 509*0Sstevel@tonic-gate bo = po / (NBBY / 2); 510*0Sstevel@tonic-gate while (i < nr) { 511*0Sstevel@tonic-gate int bits = hrm->hrm_bits[bo]; 512*0Sstevel@tonic-gate *dp++ |= (bits >> 6) & 3; 513*0Sstevel@tonic-gate *dp++ |= (bits >> 4) & 3; 514*0Sstevel@tonic-gate *dp++ |= (bits >> 2) & 3; 515*0Sstevel@tonic-gate *dp++ |= (bits >> 0) & 3; 516*0Sstevel@tonic-gate if (clearflag) 517*0Sstevel@tonic-gate hrm->hrm_bits[bo] = 0; 518*0Sstevel@tonic-gate bo++; 519*0Sstevel@tonic-gate po += 4; 520*0Sstevel@tonic-gate i += 4; 521*0Sstevel@tonic-gate } 522*0Sstevel@tonic-gate /* 523*0Sstevel@tonic-gate * Extract trailing unaligned bits. 524*0Sstevel@tonic-gate */ 525*0Sstevel@tonic-gate while (i < n) { 526*0Sstevel@tonic-gate bo = po / (NBBY / 2); 527*0Sstevel@tonic-gate spb = (3 - (po & 3)) * 2; 528*0Sstevel@tonic-gate *dp++ |= (hrm->hrm_bits[bo] >> spb) & 3; 529*0Sstevel@tonic-gate if (clearflag) 530*0Sstevel@tonic-gate hrm->hrm_bits[bo] &= ~(3<<spb); 531*0Sstevel@tonic-gate po++; 532*0Sstevel@tonic-gate i++; 533*0Sstevel@tonic-gate } 534*0Sstevel@tonic-gate 535*0Sstevel@tonic-gate break; 536*0Sstevel@tonic-gate } 537*0Sstevel@tonic-gate } 538*0Sstevel@tonic-gate if (hrm == NULL) 539*0Sstevel@tonic-gate dp += n; 540*0Sstevel@tonic-gate np -= n; 541*0Sstevel@tonic-gate a += n * MMU_PAGESIZE; 542*0Sstevel@tonic-gate } 543*0Sstevel@tonic-gate mutex_exit(&hat_statlock); 544*0Sstevel@tonic-gate } 545