1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*0Sstevel@tonic-gate 29*0Sstevel@tonic-gate #include <sys/types.h> 30*0Sstevel@tonic-gate #include <sys/sysmacros.h> 31*0Sstevel@tonic-gate #include <sys/kmem.h> 32*0Sstevel@tonic-gate #include <sys/atomic.h> 33*0Sstevel@tonic-gate #include <sys/bitmap.h> 34*0Sstevel@tonic-gate #include <sys/systm.h> 35*0Sstevel@tonic-gate #include <vm/seg_kmem.h> 36*0Sstevel@tonic-gate #include <vm/hat.h> 37*0Sstevel@tonic-gate #include <vm/vm_dep.h> 38*0Sstevel@tonic-gate #include <vm/hat_i86.h> 39*0Sstevel@tonic-gate #include <sys/cmn_err.h> 40*0Sstevel@tonic-gate 41*0Sstevel@tonic-gate 42*0Sstevel@tonic-gate /* 43*0Sstevel@tonic-gate * When pages are shared by more than one mapping, a list of these 44*0Sstevel@tonic-gate * structs hangs off of the page_t connected by the hm_next and hm_prev 45*0Sstevel@tonic-gate * fields. Every hment is also indexed by a system-wide hash table, using 46*0Sstevel@tonic-gate * hm_hashnext to connect it to the chain of hments in a single hash 47*0Sstevel@tonic-gate * bucket. 48*0Sstevel@tonic-gate */ 49*0Sstevel@tonic-gate struct hment { 50*0Sstevel@tonic-gate struct hment *hm_hashnext; /* next mapping on hash chain */ 51*0Sstevel@tonic-gate struct hment *hm_next; /* next mapping of same page */ 52*0Sstevel@tonic-gate struct hment *hm_prev; /* previous mapping of same page */ 53*0Sstevel@tonic-gate htable_t *hm_htable; /* corresponding htable_t */ 54*0Sstevel@tonic-gate uint16_t hm_entry; /* index of pte in htable */ 55*0Sstevel@tonic-gate uint16_t hm_pad; /* explicitly expose compiler padding */ 56*0Sstevel@tonic-gate #ifdef __amd64 57*0Sstevel@tonic-gate uint32_t hm_pad2; /* explicitly expose compiler padding */ 58*0Sstevel@tonic-gate #endif 59*0Sstevel@tonic-gate }; 60*0Sstevel@tonic-gate 61*0Sstevel@tonic-gate /* 62*0Sstevel@tonic-gate * Value returned by hment_walk() when dealing with a single mapping 63*0Sstevel@tonic-gate * embedded in the page_t. 64*0Sstevel@tonic-gate */ 65*0Sstevel@tonic-gate #define HMENT_EMBEDDED ((hment_t *)(uintptr_t)1) 66*0Sstevel@tonic-gate 67*0Sstevel@tonic-gate kmem_cache_t *hment_cache; 68*0Sstevel@tonic-gate 69*0Sstevel@tonic-gate /* 70*0Sstevel@tonic-gate * The hment reserve is similar to the htable reserve, with the following 71*0Sstevel@tonic-gate * exception. Hment's are never needed for HAT kmem allocs. 72*0Sstevel@tonic-gate * 73*0Sstevel@tonic-gate * The hment_reserve_amount variable is used, so that you can change it's 74*0Sstevel@tonic-gate * value to zero via a kernel debugger to force stealing to get tested. 75*0Sstevel@tonic-gate */ 76*0Sstevel@tonic-gate #define HMENT_RESERVE_AMOUNT (200) /* currently a guess at right value. */ 77*0Sstevel@tonic-gate uint_t hment_reserve_amount = HMENT_RESERVE_AMOUNT; 78*0Sstevel@tonic-gate kmutex_t hment_reserve_mutex; 79*0Sstevel@tonic-gate uint_t hment_reserve_count; 80*0Sstevel@tonic-gate hment_t *hment_reserve_pool; 81*0Sstevel@tonic-gate extern kthread_t *hat_reserves_thread; 82*0Sstevel@tonic-gate 83*0Sstevel@tonic-gate /* 84*0Sstevel@tonic-gate * Possible performance RFE: we might need to make this dynamic, perhaps 85*0Sstevel@tonic-gate * based on the number of pages in the system. 86*0Sstevel@tonic-gate */ 87*0Sstevel@tonic-gate #define HMENT_HASH_SIZE (64 * 1024) 88*0Sstevel@tonic-gate static uint_t hment_hash_entries = HMENT_HASH_SIZE; 89*0Sstevel@tonic-gate static hment_t **hment_hash; 90*0Sstevel@tonic-gate 91*0Sstevel@tonic-gate /* 92*0Sstevel@tonic-gate * Lots of highly shared pages will have the same value for "entry" (consider 93*0Sstevel@tonic-gate * the starting address of "xterm" or "sh"). So we'll distinguish them by 94*0Sstevel@tonic-gate * adding the pfn of the page table into both the high bits. 95*0Sstevel@tonic-gate * The shift by 9 corresponds to the range of values for entry (0..511). 96*0Sstevel@tonic-gate */ 97*0Sstevel@tonic-gate #define HMENT_HASH(pfn, entry) (uint32_t) \ 98*0Sstevel@tonic-gate ((((pfn) << 9) + entry + pfn) & (hment_hash_entries - 1)) 99*0Sstevel@tonic-gate 100*0Sstevel@tonic-gate /* 101*0Sstevel@tonic-gate * "mlist_lock" is a hashed mutex lock for protecting per-page mapping 102*0Sstevel@tonic-gate * lists and "hash_lock" is a similar lock protecting the hment hash 103*0Sstevel@tonic-gate * table. The hashed approach is taken to avoid the spatial overhead of 104*0Sstevel@tonic-gate * maintaining a separate lock for each page, while still achieving better 105*0Sstevel@tonic-gate * scalability than a single lock would allow. 106*0Sstevel@tonic-gate */ 107*0Sstevel@tonic-gate #define MLIST_NUM_LOCK 256 /* must be power of two */ 108*0Sstevel@tonic-gate static kmutex_t mlist_lock[MLIST_NUM_LOCK]; 109*0Sstevel@tonic-gate 110*0Sstevel@tonic-gate /* 111*0Sstevel@tonic-gate * the shift by 9 is so that all large pages don't use the same hash bucket 112*0Sstevel@tonic-gate */ 113*0Sstevel@tonic-gate #define MLIST_MUTEX(pp) \ 114*0Sstevel@tonic-gate &mlist_lock[((pp)->p_pagenum + ((pp)->p_pagenum >> 9)) & \ 115*0Sstevel@tonic-gate (MLIST_NUM_LOCK - 1)] 116*0Sstevel@tonic-gate 117*0Sstevel@tonic-gate #define HASH_NUM_LOCK 256 /* must be power of two */ 118*0Sstevel@tonic-gate static kmutex_t hash_lock[HASH_NUM_LOCK]; 119*0Sstevel@tonic-gate 120*0Sstevel@tonic-gate #define HASH_MUTEX(idx) &hash_lock[(idx) & (HASH_NUM_LOCK-1)] 121*0Sstevel@tonic-gate 122*0Sstevel@tonic-gate static hment_t *hment_steal(void); 123*0Sstevel@tonic-gate 124*0Sstevel@tonic-gate /* 125*0Sstevel@tonic-gate * put one hment onto the reserves list 126*0Sstevel@tonic-gate */ 127*0Sstevel@tonic-gate static void 128*0Sstevel@tonic-gate hment_put_reserve(hment_t *hm) 129*0Sstevel@tonic-gate { 130*0Sstevel@tonic-gate HATSTAT_INC(hs_hm_put_reserve); 131*0Sstevel@tonic-gate mutex_enter(&hment_reserve_mutex); 132*0Sstevel@tonic-gate hm->hm_next = hment_reserve_pool; 133*0Sstevel@tonic-gate hment_reserve_pool = hm; 134*0Sstevel@tonic-gate ++hment_reserve_count; 135*0Sstevel@tonic-gate mutex_exit(&hment_reserve_mutex); 136*0Sstevel@tonic-gate } 137*0Sstevel@tonic-gate 138*0Sstevel@tonic-gate /* 139*0Sstevel@tonic-gate * Take one hment from the reserve. 140*0Sstevel@tonic-gate */ 141*0Sstevel@tonic-gate static hment_t * 142*0Sstevel@tonic-gate hment_get_reserve(void) 143*0Sstevel@tonic-gate { 144*0Sstevel@tonic-gate hment_t *hm = NULL; 145*0Sstevel@tonic-gate 146*0Sstevel@tonic-gate /* 147*0Sstevel@tonic-gate * We rely on a "donation system" to refill the hment reserve 148*0Sstevel@tonic-gate * list, which only takes place when we are allocating hments for 149*0Sstevel@tonic-gate * user mappings. It is theoretically possible that an incredibly 150*0Sstevel@tonic-gate * long string of kernel hment_alloc()s with no intervening user 151*0Sstevel@tonic-gate * hment_alloc()s could exhaust that pool. 152*0Sstevel@tonic-gate */ 153*0Sstevel@tonic-gate HATSTAT_INC(hs_hm_get_reserve); 154*0Sstevel@tonic-gate mutex_enter(&hment_reserve_mutex); 155*0Sstevel@tonic-gate if (hment_reserve_count != 0) { 156*0Sstevel@tonic-gate hm = hment_reserve_pool; 157*0Sstevel@tonic-gate hment_reserve_pool = hm->hm_next; 158*0Sstevel@tonic-gate --hment_reserve_count; 159*0Sstevel@tonic-gate } 160*0Sstevel@tonic-gate mutex_exit(&hment_reserve_mutex); 161*0Sstevel@tonic-gate return (hm); 162*0Sstevel@tonic-gate } 163*0Sstevel@tonic-gate 164*0Sstevel@tonic-gate /* 165*0Sstevel@tonic-gate * Allocate an hment 166*0Sstevel@tonic-gate */ 167*0Sstevel@tonic-gate static hment_t * 168*0Sstevel@tonic-gate hment_alloc() 169*0Sstevel@tonic-gate { 170*0Sstevel@tonic-gate int km_flag = can_steal_post_boot ? KM_NOSLEEP : KM_SLEEP; 171*0Sstevel@tonic-gate hment_t *hm = NULL; 172*0Sstevel@tonic-gate int use_reserves = (use_boot_reserve || 173*0Sstevel@tonic-gate curthread == hat_reserves_thread || panicstr != NULL); 174*0Sstevel@tonic-gate 175*0Sstevel@tonic-gate /* 176*0Sstevel@tonic-gate * If we aren't using the reserves, try using kmem to get an hment. 177*0Sstevel@tonic-gate * Donate any successful allocations to reserves if low. 178*0Sstevel@tonic-gate * 179*0Sstevel@tonic-gate * If we're in panic, resort to using the reserves. 180*0Sstevel@tonic-gate */ 181*0Sstevel@tonic-gate HATSTAT_INC(hs_hm_alloc); 182*0Sstevel@tonic-gate if (!use_reserves) { 183*0Sstevel@tonic-gate for (;;) { 184*0Sstevel@tonic-gate hm = kmem_cache_alloc(hment_cache, km_flag); 185*0Sstevel@tonic-gate if (hment_reserve_count >= hment_reserve_amount || 186*0Sstevel@tonic-gate hm == NULL || panicstr != NULL || 187*0Sstevel@tonic-gate curthread == hat_reserves_thread) 188*0Sstevel@tonic-gate break; 189*0Sstevel@tonic-gate hment_put_reserve(hm); 190*0Sstevel@tonic-gate } 191*0Sstevel@tonic-gate } 192*0Sstevel@tonic-gate 193*0Sstevel@tonic-gate /* 194*0Sstevel@tonic-gate * If allocation failed, we need to tap the reserves or steal 195*0Sstevel@tonic-gate */ 196*0Sstevel@tonic-gate if (hm == NULL) { 197*0Sstevel@tonic-gate if (use_reserves) 198*0Sstevel@tonic-gate hm = hment_get_reserve(); 199*0Sstevel@tonic-gate 200*0Sstevel@tonic-gate /* 201*0Sstevel@tonic-gate * If we still haven't gotten an hment, attempt to steal one by 202*0Sstevel@tonic-gate * victimizing a mapping in a user htable. 203*0Sstevel@tonic-gate */ 204*0Sstevel@tonic-gate if (hm == NULL && can_steal_post_boot) 205*0Sstevel@tonic-gate hm = hment_steal(); 206*0Sstevel@tonic-gate 207*0Sstevel@tonic-gate /* 208*0Sstevel@tonic-gate * we're in dire straights, try the reserve 209*0Sstevel@tonic-gate */ 210*0Sstevel@tonic-gate if (hm == NULL) 211*0Sstevel@tonic-gate hm = hment_get_reserve(); 212*0Sstevel@tonic-gate 213*0Sstevel@tonic-gate /* 214*0Sstevel@tonic-gate * still no hment is a serious problem. 215*0Sstevel@tonic-gate */ 216*0Sstevel@tonic-gate if (hm == NULL) 217*0Sstevel@tonic-gate panic("hment_alloc(): no reserve, couldn't steal"); 218*0Sstevel@tonic-gate } 219*0Sstevel@tonic-gate 220*0Sstevel@tonic-gate 221*0Sstevel@tonic-gate hm->hm_entry = 0; 222*0Sstevel@tonic-gate hm->hm_htable = NULL; 223*0Sstevel@tonic-gate hm->hm_hashnext = NULL; 224*0Sstevel@tonic-gate hm->hm_next = NULL; 225*0Sstevel@tonic-gate hm->hm_prev = NULL; 226*0Sstevel@tonic-gate return (hm); 227*0Sstevel@tonic-gate } 228*0Sstevel@tonic-gate 229*0Sstevel@tonic-gate /* 230*0Sstevel@tonic-gate * Free an hment, possibly to the reserves list when called from the 231*0Sstevel@tonic-gate * thread using the reserves. For example, when freeing an hment during an 232*0Sstevel@tonic-gate * htable_steal(), we can't recurse into the kmem allocator, so we just 233*0Sstevel@tonic-gate * push the hment onto the reserve list. 234*0Sstevel@tonic-gate */ 235*0Sstevel@tonic-gate void 236*0Sstevel@tonic-gate hment_free(hment_t *hm) 237*0Sstevel@tonic-gate { 238*0Sstevel@tonic-gate #ifdef DEBUG 239*0Sstevel@tonic-gate /* 240*0Sstevel@tonic-gate * zero out all fields to try and force any race conditions to segfault 241*0Sstevel@tonic-gate */ 242*0Sstevel@tonic-gate bzero(hm, sizeof (*hm)); 243*0Sstevel@tonic-gate #endif 244*0Sstevel@tonic-gate HATSTAT_INC(hs_hm_free); 245*0Sstevel@tonic-gate if (curthread == hat_reserves_thread || 246*0Sstevel@tonic-gate hment_reserve_count < hment_reserve_amount) 247*0Sstevel@tonic-gate hment_put_reserve(hm); 248*0Sstevel@tonic-gate else 249*0Sstevel@tonic-gate kmem_cache_free(hment_cache, hm); 250*0Sstevel@tonic-gate } 251*0Sstevel@tonic-gate 252*0Sstevel@tonic-gate int 253*0Sstevel@tonic-gate x86_hm_held(page_t *pp) 254*0Sstevel@tonic-gate { 255*0Sstevel@tonic-gate ASSERT(pp != NULL); 256*0Sstevel@tonic-gate return (MUTEX_HELD(MLIST_MUTEX(pp))); 257*0Sstevel@tonic-gate } 258*0Sstevel@tonic-gate 259*0Sstevel@tonic-gate void 260*0Sstevel@tonic-gate x86_hm_enter(page_t *pp) 261*0Sstevel@tonic-gate { 262*0Sstevel@tonic-gate ASSERT(pp != NULL); 263*0Sstevel@tonic-gate mutex_enter(MLIST_MUTEX(pp)); 264*0Sstevel@tonic-gate } 265*0Sstevel@tonic-gate 266*0Sstevel@tonic-gate void 267*0Sstevel@tonic-gate x86_hm_exit(page_t *pp) 268*0Sstevel@tonic-gate { 269*0Sstevel@tonic-gate ASSERT(pp != NULL); 270*0Sstevel@tonic-gate mutex_exit(MLIST_MUTEX(pp)); 271*0Sstevel@tonic-gate } 272*0Sstevel@tonic-gate 273*0Sstevel@tonic-gate /* 274*0Sstevel@tonic-gate * Internal routine to add a full hment to a page_t mapping list 275*0Sstevel@tonic-gate */ 276*0Sstevel@tonic-gate static void 277*0Sstevel@tonic-gate hment_insert(hment_t *hm, page_t *pp) 278*0Sstevel@tonic-gate { 279*0Sstevel@tonic-gate uint_t idx; 280*0Sstevel@tonic-gate 281*0Sstevel@tonic-gate ASSERT(x86_hm_held(pp)); 282*0Sstevel@tonic-gate ASSERT(!pp->p_embed); 283*0Sstevel@tonic-gate 284*0Sstevel@tonic-gate /* 285*0Sstevel@tonic-gate * Add the hment to the page's mapping list. 286*0Sstevel@tonic-gate */ 287*0Sstevel@tonic-gate ++pp->p_share; 288*0Sstevel@tonic-gate hm->hm_next = pp->p_mapping; 289*0Sstevel@tonic-gate if (pp->p_mapping != NULL) 290*0Sstevel@tonic-gate ((hment_t *)pp->p_mapping)->hm_prev = hm; 291*0Sstevel@tonic-gate pp->p_mapping = hm; 292*0Sstevel@tonic-gate 293*0Sstevel@tonic-gate /* 294*0Sstevel@tonic-gate * Add the hment to the system-wide hash table. 295*0Sstevel@tonic-gate */ 296*0Sstevel@tonic-gate idx = HMENT_HASH(hm->hm_htable->ht_pfn, hm->hm_entry); 297*0Sstevel@tonic-gate 298*0Sstevel@tonic-gate mutex_enter(HASH_MUTEX(idx)); 299*0Sstevel@tonic-gate hm->hm_hashnext = hment_hash[idx]; 300*0Sstevel@tonic-gate hment_hash[idx] = hm; 301*0Sstevel@tonic-gate mutex_exit(HASH_MUTEX(idx)); 302*0Sstevel@tonic-gate } 303*0Sstevel@tonic-gate 304*0Sstevel@tonic-gate /* 305*0Sstevel@tonic-gate * Prepare a mapping list entry to the given page. 306*0Sstevel@tonic-gate * 307*0Sstevel@tonic-gate * There are 4 different situations to deal with: 308*0Sstevel@tonic-gate * 309*0Sstevel@tonic-gate * - Adding the first mapping to a page_t as an embedded hment 310*0Sstevel@tonic-gate * - Refaulting on an existing embedded mapping 311*0Sstevel@tonic-gate * - Upgrading an embedded mapping when adding a 2nd mapping 312*0Sstevel@tonic-gate * - Adding another mapping to a page_t that already has multiple mappings 313*0Sstevel@tonic-gate * note we don't optimized for the refaulting case here. 314*0Sstevel@tonic-gate * 315*0Sstevel@tonic-gate * Due to competition with other threads that may be mapping/unmapping the 316*0Sstevel@tonic-gate * same page and the need to drop all locks while allocating hments, any or 317*0Sstevel@tonic-gate * all of the 3 situations can occur (and in almost any order) in any given 318*0Sstevel@tonic-gate * call. Isn't this fun! 319*0Sstevel@tonic-gate */ 320*0Sstevel@tonic-gate hment_t * 321*0Sstevel@tonic-gate hment_prepare(htable_t *htable, uint_t entry, page_t *pp) 322*0Sstevel@tonic-gate { 323*0Sstevel@tonic-gate hment_t *hm = NULL; 324*0Sstevel@tonic-gate 325*0Sstevel@tonic-gate ASSERT(x86_hm_held(pp)); 326*0Sstevel@tonic-gate 327*0Sstevel@tonic-gate for (;;) { 328*0Sstevel@tonic-gate 329*0Sstevel@tonic-gate /* 330*0Sstevel@tonic-gate * The most common case is establishing the first mapping to a 331*0Sstevel@tonic-gate * page, so check that first. This doesn't need any allocated 332*0Sstevel@tonic-gate * hment. 333*0Sstevel@tonic-gate */ 334*0Sstevel@tonic-gate if (pp->p_mapping == NULL) { 335*0Sstevel@tonic-gate ASSERT(!pp->p_embed); 336*0Sstevel@tonic-gate ASSERT(pp->p_share == 0); 337*0Sstevel@tonic-gate if (hm == NULL) 338*0Sstevel@tonic-gate break; 339*0Sstevel@tonic-gate 340*0Sstevel@tonic-gate /* 341*0Sstevel@tonic-gate * we had an hment already, so free it and retry 342*0Sstevel@tonic-gate */ 343*0Sstevel@tonic-gate goto free_and_continue; 344*0Sstevel@tonic-gate } 345*0Sstevel@tonic-gate 346*0Sstevel@tonic-gate /* 347*0Sstevel@tonic-gate * If there is an embedded mapping, we may need to 348*0Sstevel@tonic-gate * convert it to an hment. 349*0Sstevel@tonic-gate */ 350*0Sstevel@tonic-gate if (pp->p_embed) { 351*0Sstevel@tonic-gate 352*0Sstevel@tonic-gate /* should point to htable */ 353*0Sstevel@tonic-gate ASSERT(pp->p_mapping != NULL); 354*0Sstevel@tonic-gate 355*0Sstevel@tonic-gate /* 356*0Sstevel@tonic-gate * If we are faulting on a pre-existing mapping 357*0Sstevel@tonic-gate * there is no need to promote/allocate a new hment. 358*0Sstevel@tonic-gate * This happens a lot due to segmap. 359*0Sstevel@tonic-gate */ 360*0Sstevel@tonic-gate if (pp->p_mapping == htable && pp->p_mlentry == entry) { 361*0Sstevel@tonic-gate if (hm == NULL) 362*0Sstevel@tonic-gate break; 363*0Sstevel@tonic-gate goto free_and_continue; 364*0Sstevel@tonic-gate } 365*0Sstevel@tonic-gate 366*0Sstevel@tonic-gate /* 367*0Sstevel@tonic-gate * If we have an hment allocated, use it to promote the 368*0Sstevel@tonic-gate * existing embedded mapping. 369*0Sstevel@tonic-gate */ 370*0Sstevel@tonic-gate if (hm != NULL) { 371*0Sstevel@tonic-gate hm->hm_htable = pp->p_mapping; 372*0Sstevel@tonic-gate hm->hm_entry = pp->p_mlentry; 373*0Sstevel@tonic-gate pp->p_mapping = NULL; 374*0Sstevel@tonic-gate pp->p_share = 0; 375*0Sstevel@tonic-gate pp->p_embed = 0; 376*0Sstevel@tonic-gate hment_insert(hm, pp); 377*0Sstevel@tonic-gate } 378*0Sstevel@tonic-gate 379*0Sstevel@tonic-gate /* 380*0Sstevel@tonic-gate * We either didn't have an hment allocated or we just 381*0Sstevel@tonic-gate * used it for the embedded mapping. In either case, 382*0Sstevel@tonic-gate * allocate another hment and restart. 383*0Sstevel@tonic-gate */ 384*0Sstevel@tonic-gate goto allocate_and_continue; 385*0Sstevel@tonic-gate } 386*0Sstevel@tonic-gate 387*0Sstevel@tonic-gate /* 388*0Sstevel@tonic-gate * Last possibility is that we're adding an hment to a list 389*0Sstevel@tonic-gate * of hments. 390*0Sstevel@tonic-gate */ 391*0Sstevel@tonic-gate if (hm != NULL) 392*0Sstevel@tonic-gate break; 393*0Sstevel@tonic-gate allocate_and_continue: 394*0Sstevel@tonic-gate x86_hm_exit(pp); 395*0Sstevel@tonic-gate hm = hment_alloc(); 396*0Sstevel@tonic-gate x86_hm_enter(pp); 397*0Sstevel@tonic-gate continue; 398*0Sstevel@tonic-gate 399*0Sstevel@tonic-gate free_and_continue: 400*0Sstevel@tonic-gate /* 401*0Sstevel@tonic-gate * we allocated an hment already, free it and retry 402*0Sstevel@tonic-gate */ 403*0Sstevel@tonic-gate x86_hm_exit(pp); 404*0Sstevel@tonic-gate hment_free(hm); 405*0Sstevel@tonic-gate hm = NULL; 406*0Sstevel@tonic-gate x86_hm_enter(pp); 407*0Sstevel@tonic-gate } 408*0Sstevel@tonic-gate ASSERT(x86_hm_held(pp)); 409*0Sstevel@tonic-gate return (hm); 410*0Sstevel@tonic-gate } 411*0Sstevel@tonic-gate 412*0Sstevel@tonic-gate /* 413*0Sstevel@tonic-gate * Record a mapping list entry for the htable/entry to the given page. 414*0Sstevel@tonic-gate * 415*0Sstevel@tonic-gate * hment_prepare() should have properly set up the situation. 416*0Sstevel@tonic-gate */ 417*0Sstevel@tonic-gate void 418*0Sstevel@tonic-gate hment_assign(htable_t *htable, uint_t entry, page_t *pp, hment_t *hm) 419*0Sstevel@tonic-gate { 420*0Sstevel@tonic-gate ASSERT(x86_hm_held(pp)); 421*0Sstevel@tonic-gate 422*0Sstevel@tonic-gate /* 423*0Sstevel@tonic-gate * The most common case is establishing the first mapping to a 424*0Sstevel@tonic-gate * page, so check that first. This doesn't need any allocated 425*0Sstevel@tonic-gate * hment. 426*0Sstevel@tonic-gate */ 427*0Sstevel@tonic-gate if (pp->p_mapping == NULL) { 428*0Sstevel@tonic-gate ASSERT(hm == NULL); 429*0Sstevel@tonic-gate ASSERT(!pp->p_embed); 430*0Sstevel@tonic-gate ASSERT(pp->p_share == 0); 431*0Sstevel@tonic-gate pp->p_embed = 1; 432*0Sstevel@tonic-gate pp->p_mapping = htable; 433*0Sstevel@tonic-gate pp->p_mlentry = entry; 434*0Sstevel@tonic-gate return; 435*0Sstevel@tonic-gate } 436*0Sstevel@tonic-gate 437*0Sstevel@tonic-gate /* 438*0Sstevel@tonic-gate * We should never get here with a pre-existing embedded maping 439*0Sstevel@tonic-gate */ 440*0Sstevel@tonic-gate ASSERT(!pp->p_embed); 441*0Sstevel@tonic-gate 442*0Sstevel@tonic-gate /* 443*0Sstevel@tonic-gate * add the new hment to the mapping list 444*0Sstevel@tonic-gate */ 445*0Sstevel@tonic-gate ASSERT(hm != NULL); 446*0Sstevel@tonic-gate hm->hm_htable = htable; 447*0Sstevel@tonic-gate hm->hm_entry = entry; 448*0Sstevel@tonic-gate hment_insert(hm, pp); 449*0Sstevel@tonic-gate } 450*0Sstevel@tonic-gate 451*0Sstevel@tonic-gate /* 452*0Sstevel@tonic-gate * Walk through the mappings for a page. 453*0Sstevel@tonic-gate * 454*0Sstevel@tonic-gate * must already have done an x86_hm_enter() 455*0Sstevel@tonic-gate */ 456*0Sstevel@tonic-gate hment_t * 457*0Sstevel@tonic-gate hment_walk(page_t *pp, htable_t **ht, uint_t *entry, hment_t *prev) 458*0Sstevel@tonic-gate { 459*0Sstevel@tonic-gate hment_t *hm; 460*0Sstevel@tonic-gate 461*0Sstevel@tonic-gate ASSERT(x86_hm_held(pp)); 462*0Sstevel@tonic-gate 463*0Sstevel@tonic-gate if (pp->p_embed) { 464*0Sstevel@tonic-gate if (prev == NULL) { 465*0Sstevel@tonic-gate *ht = (htable_t *)pp->p_mapping; 466*0Sstevel@tonic-gate *entry = pp->p_mlentry; 467*0Sstevel@tonic-gate hm = HMENT_EMBEDDED; 468*0Sstevel@tonic-gate } else { 469*0Sstevel@tonic-gate ASSERT(prev == HMENT_EMBEDDED); 470*0Sstevel@tonic-gate hm = NULL; 471*0Sstevel@tonic-gate } 472*0Sstevel@tonic-gate } else { 473*0Sstevel@tonic-gate if (prev == NULL) { 474*0Sstevel@tonic-gate ASSERT(prev != HMENT_EMBEDDED); 475*0Sstevel@tonic-gate hm = (hment_t *)pp->p_mapping; 476*0Sstevel@tonic-gate } else { 477*0Sstevel@tonic-gate hm = prev->hm_next; 478*0Sstevel@tonic-gate } 479*0Sstevel@tonic-gate 480*0Sstevel@tonic-gate if (hm != NULL) { 481*0Sstevel@tonic-gate *ht = hm->hm_htable; 482*0Sstevel@tonic-gate *entry = hm->hm_entry; 483*0Sstevel@tonic-gate } 484*0Sstevel@tonic-gate } 485*0Sstevel@tonic-gate return (hm); 486*0Sstevel@tonic-gate } 487*0Sstevel@tonic-gate 488*0Sstevel@tonic-gate /* 489*0Sstevel@tonic-gate * Remove a mapping to a page from its mapping list. Must have 490*0Sstevel@tonic-gate * the corresponding mapping list locked. 491*0Sstevel@tonic-gate * Finds the mapping list entry with the given pte_t and 492*0Sstevel@tonic-gate * unlinks it from the mapping list. 493*0Sstevel@tonic-gate */ 494*0Sstevel@tonic-gate hment_t * 495*0Sstevel@tonic-gate hment_remove(page_t *pp, htable_t *ht, uint_t entry) 496*0Sstevel@tonic-gate { 497*0Sstevel@tonic-gate hment_t *prev = NULL; 498*0Sstevel@tonic-gate hment_t *hm; 499*0Sstevel@tonic-gate uint_t idx; 500*0Sstevel@tonic-gate 501*0Sstevel@tonic-gate ASSERT(x86_hm_held(pp)); 502*0Sstevel@tonic-gate 503*0Sstevel@tonic-gate /* 504*0Sstevel@tonic-gate * Check if we have only one mapping embedded in the page_t. 505*0Sstevel@tonic-gate */ 506*0Sstevel@tonic-gate if (pp->p_embed) { 507*0Sstevel@tonic-gate ASSERT(ht == (htable_t *)pp->p_mapping); 508*0Sstevel@tonic-gate ASSERT(entry == pp->p_mlentry); 509*0Sstevel@tonic-gate ASSERT(pp->p_share == 0); 510*0Sstevel@tonic-gate pp->p_mapping = NULL; 511*0Sstevel@tonic-gate pp->p_mlentry = 0; 512*0Sstevel@tonic-gate pp->p_embed = 0; 513*0Sstevel@tonic-gate return (NULL); 514*0Sstevel@tonic-gate } 515*0Sstevel@tonic-gate 516*0Sstevel@tonic-gate /* 517*0Sstevel@tonic-gate * Otherwise it must be in the list of hments. 518*0Sstevel@tonic-gate * Find the hment in the system-wide hash table and remove it. 519*0Sstevel@tonic-gate */ 520*0Sstevel@tonic-gate ASSERT(pp->p_share != 0); 521*0Sstevel@tonic-gate idx = HMENT_HASH(ht->ht_pfn, entry); 522*0Sstevel@tonic-gate mutex_enter(HASH_MUTEX(idx)); 523*0Sstevel@tonic-gate hm = hment_hash[idx]; 524*0Sstevel@tonic-gate while (hm && (hm->hm_htable != ht || hm->hm_entry != entry)) { 525*0Sstevel@tonic-gate prev = hm; 526*0Sstevel@tonic-gate hm = hm->hm_hashnext; 527*0Sstevel@tonic-gate } 528*0Sstevel@tonic-gate if (hm == NULL) 529*0Sstevel@tonic-gate panic("hment_remove() mapping not found in hash table"); 530*0Sstevel@tonic-gate 531*0Sstevel@tonic-gate if (prev) 532*0Sstevel@tonic-gate prev->hm_hashnext = hm->hm_hashnext; 533*0Sstevel@tonic-gate else 534*0Sstevel@tonic-gate hment_hash[idx] = hm->hm_hashnext; 535*0Sstevel@tonic-gate mutex_exit(HASH_MUTEX(idx)); 536*0Sstevel@tonic-gate 537*0Sstevel@tonic-gate /* 538*0Sstevel@tonic-gate * Remove the hment from the page's mapping list 539*0Sstevel@tonic-gate */ 540*0Sstevel@tonic-gate if (hm->hm_next) 541*0Sstevel@tonic-gate hm->hm_next->hm_prev = hm->hm_prev; 542*0Sstevel@tonic-gate if (hm->hm_prev) 543*0Sstevel@tonic-gate hm->hm_prev->hm_next = hm->hm_next; 544*0Sstevel@tonic-gate else 545*0Sstevel@tonic-gate pp->p_mapping = hm->hm_next; 546*0Sstevel@tonic-gate 547*0Sstevel@tonic-gate --pp->p_share; 548*0Sstevel@tonic-gate hm->hm_hashnext = NULL; 549*0Sstevel@tonic-gate hm->hm_next = NULL; 550*0Sstevel@tonic-gate hm->hm_prev = NULL; 551*0Sstevel@tonic-gate 552*0Sstevel@tonic-gate return (hm); 553*0Sstevel@tonic-gate } 554*0Sstevel@tonic-gate 555*0Sstevel@tonic-gate /* 556*0Sstevel@tonic-gate * Put initial hment's in the reserve pool. 557*0Sstevel@tonic-gate */ 558*0Sstevel@tonic-gate void 559*0Sstevel@tonic-gate hment_reserve(uint_t count) 560*0Sstevel@tonic-gate { 561*0Sstevel@tonic-gate hment_t *hm; 562*0Sstevel@tonic-gate 563*0Sstevel@tonic-gate count += hment_reserve_amount; 564*0Sstevel@tonic-gate 565*0Sstevel@tonic-gate while (hment_reserve_count < count) { 566*0Sstevel@tonic-gate hm = kmem_cache_alloc(hment_cache, KM_NOSLEEP); 567*0Sstevel@tonic-gate if (hm == NULL) 568*0Sstevel@tonic-gate return; 569*0Sstevel@tonic-gate hment_put_reserve(hm); 570*0Sstevel@tonic-gate } 571*0Sstevel@tonic-gate } 572*0Sstevel@tonic-gate 573*0Sstevel@tonic-gate /* 574*0Sstevel@tonic-gate * Readjust the hment reserves after they may have been used. 575*0Sstevel@tonic-gate */ 576*0Sstevel@tonic-gate void 577*0Sstevel@tonic-gate hment_adjust_reserve() 578*0Sstevel@tonic-gate { 579*0Sstevel@tonic-gate hment_t *hm; 580*0Sstevel@tonic-gate 581*0Sstevel@tonic-gate /* 582*0Sstevel@tonic-gate * Free up any excess reserves 583*0Sstevel@tonic-gate */ 584*0Sstevel@tonic-gate while (hment_reserve_count > hment_reserve_amount) { 585*0Sstevel@tonic-gate ASSERT(curthread != hat_reserves_thread); 586*0Sstevel@tonic-gate hm = hment_get_reserve(); 587*0Sstevel@tonic-gate if (hm == NULL) 588*0Sstevel@tonic-gate return; 589*0Sstevel@tonic-gate hment_free(hm); 590*0Sstevel@tonic-gate } 591*0Sstevel@tonic-gate } 592*0Sstevel@tonic-gate 593*0Sstevel@tonic-gate /* 594*0Sstevel@tonic-gate * initialize hment data structures 595*0Sstevel@tonic-gate */ 596*0Sstevel@tonic-gate void 597*0Sstevel@tonic-gate hment_init(void) 598*0Sstevel@tonic-gate { 599*0Sstevel@tonic-gate int i; 600*0Sstevel@tonic-gate int flags = KMC_NOHASH | KMC_NODEBUG; 601*0Sstevel@tonic-gate 602*0Sstevel@tonic-gate /* 603*0Sstevel@tonic-gate * Initialize kmem caches. On 32 bit kernel's we shut off 604*0Sstevel@tonic-gate * debug information to save on precious kernel VA usage. 605*0Sstevel@tonic-gate */ 606*0Sstevel@tonic-gate hment_cache = kmem_cache_create("hment_t", 607*0Sstevel@tonic-gate sizeof (hment_t), 0, NULL, NULL, NULL, 608*0Sstevel@tonic-gate NULL, hat_memload_arena, flags); 609*0Sstevel@tonic-gate 610*0Sstevel@tonic-gate hment_hash = kmem_zalloc(hment_hash_entries * sizeof (hment_t *), 611*0Sstevel@tonic-gate KM_SLEEP); 612*0Sstevel@tonic-gate 613*0Sstevel@tonic-gate for (i = 0; i < MLIST_NUM_LOCK; i++) 614*0Sstevel@tonic-gate mutex_init(&mlist_lock[i], NULL, MUTEX_DEFAULT, NULL); 615*0Sstevel@tonic-gate 616*0Sstevel@tonic-gate for (i = 0; i < HASH_NUM_LOCK; i++) 617*0Sstevel@tonic-gate mutex_init(&hash_lock[i], NULL, MUTEX_DEFAULT, NULL); 618*0Sstevel@tonic-gate 619*0Sstevel@tonic-gate 620*0Sstevel@tonic-gate } 621*0Sstevel@tonic-gate 622*0Sstevel@tonic-gate /* 623*0Sstevel@tonic-gate * return the number of mappings to a page 624*0Sstevel@tonic-gate * 625*0Sstevel@tonic-gate * Note there is no ASSERT() that the MUTEX is held for this. 626*0Sstevel@tonic-gate * Hence the return value might be inaccurate if this is called without 627*0Sstevel@tonic-gate * doing an x86_hm_enter(). 628*0Sstevel@tonic-gate */ 629*0Sstevel@tonic-gate uint_t 630*0Sstevel@tonic-gate hment_mapcnt(page_t *pp) 631*0Sstevel@tonic-gate { 632*0Sstevel@tonic-gate uint_t cnt; 633*0Sstevel@tonic-gate uint_t szc; 634*0Sstevel@tonic-gate page_t *larger; 635*0Sstevel@tonic-gate hment_t *hm; 636*0Sstevel@tonic-gate 637*0Sstevel@tonic-gate x86_hm_enter(pp); 638*0Sstevel@tonic-gate if (pp->p_mapping == NULL) 639*0Sstevel@tonic-gate cnt = 0; 640*0Sstevel@tonic-gate else if (pp->p_embed) 641*0Sstevel@tonic-gate cnt = 1; 642*0Sstevel@tonic-gate else 643*0Sstevel@tonic-gate cnt = pp->p_share; 644*0Sstevel@tonic-gate x86_hm_exit(pp); 645*0Sstevel@tonic-gate 646*0Sstevel@tonic-gate /* 647*0Sstevel@tonic-gate * walk through all larger mapping sizes counting mappings 648*0Sstevel@tonic-gate */ 649*0Sstevel@tonic-gate for (szc = 1; szc <= pp->p_szc; ++szc) { 650*0Sstevel@tonic-gate larger = PP_GROUPLEADER(pp, szc); 651*0Sstevel@tonic-gate if (larger == pp) /* don't double count large mappings */ 652*0Sstevel@tonic-gate continue; 653*0Sstevel@tonic-gate 654*0Sstevel@tonic-gate x86_hm_enter(larger); 655*0Sstevel@tonic-gate if (larger->p_mapping != NULL) { 656*0Sstevel@tonic-gate if (larger->p_embed && 657*0Sstevel@tonic-gate ((htable_t *)larger->p_mapping)->ht_level == szc) { 658*0Sstevel@tonic-gate ++cnt; 659*0Sstevel@tonic-gate } else if (!larger->p_embed) { 660*0Sstevel@tonic-gate for (hm = larger->p_mapping; hm; 661*0Sstevel@tonic-gate hm = hm->hm_next) { 662*0Sstevel@tonic-gate if (hm->hm_htable->ht_level == szc) 663*0Sstevel@tonic-gate ++cnt; 664*0Sstevel@tonic-gate } 665*0Sstevel@tonic-gate } 666*0Sstevel@tonic-gate } 667*0Sstevel@tonic-gate x86_hm_exit(larger); 668*0Sstevel@tonic-gate } 669*0Sstevel@tonic-gate return (cnt); 670*0Sstevel@tonic-gate } 671*0Sstevel@tonic-gate 672*0Sstevel@tonic-gate /* 673*0Sstevel@tonic-gate * We need to steal an hment. Walk through all the page_t's until we 674*0Sstevel@tonic-gate * find one that has multiple mappings. Unload one of the mappings 675*0Sstevel@tonic-gate * and reclaim that hment. Note that we'll save/restart the starting 676*0Sstevel@tonic-gate * page to try and spread the pain. 677*0Sstevel@tonic-gate */ 678*0Sstevel@tonic-gate static page_t *last_page = NULL; 679*0Sstevel@tonic-gate 680*0Sstevel@tonic-gate static hment_t * 681*0Sstevel@tonic-gate hment_steal(void) 682*0Sstevel@tonic-gate { 683*0Sstevel@tonic-gate page_t *last = last_page; 684*0Sstevel@tonic-gate page_t *pp = last; 685*0Sstevel@tonic-gate hment_t *hm = NULL; 686*0Sstevel@tonic-gate hment_t *hm2; 687*0Sstevel@tonic-gate htable_t *ht; 688*0Sstevel@tonic-gate uint_t found_one = 0; 689*0Sstevel@tonic-gate 690*0Sstevel@tonic-gate HATSTAT_INC(hs_hm_steals); 691*0Sstevel@tonic-gate if (pp == NULL) 692*0Sstevel@tonic-gate last = pp = page_first(); 693*0Sstevel@tonic-gate 694*0Sstevel@tonic-gate while (!found_one) { 695*0Sstevel@tonic-gate HATSTAT_INC(hs_hm_steal_exam); 696*0Sstevel@tonic-gate pp = page_next(pp); 697*0Sstevel@tonic-gate if (pp == NULL) 698*0Sstevel@tonic-gate pp = page_first(); 699*0Sstevel@tonic-gate 700*0Sstevel@tonic-gate /* 701*0Sstevel@tonic-gate * The loop and function exit here if nothing found to steal. 702*0Sstevel@tonic-gate */ 703*0Sstevel@tonic-gate if (pp == last) 704*0Sstevel@tonic-gate return (NULL); 705*0Sstevel@tonic-gate 706*0Sstevel@tonic-gate /* 707*0Sstevel@tonic-gate * Only lock the page_t if it has hments. 708*0Sstevel@tonic-gate */ 709*0Sstevel@tonic-gate if (pp->p_mapping == NULL || pp->p_embed) 710*0Sstevel@tonic-gate continue; 711*0Sstevel@tonic-gate 712*0Sstevel@tonic-gate /* 713*0Sstevel@tonic-gate * Search the mapping list for a usable mapping. 714*0Sstevel@tonic-gate */ 715*0Sstevel@tonic-gate x86_hm_enter(pp); 716*0Sstevel@tonic-gate if (!pp->p_embed) { 717*0Sstevel@tonic-gate for (hm = pp->p_mapping; hm; hm = hm->hm_next) { 718*0Sstevel@tonic-gate ht = hm->hm_htable; 719*0Sstevel@tonic-gate if (ht->ht_hat != kas.a_hat && 720*0Sstevel@tonic-gate ht->ht_busy == 0 && 721*0Sstevel@tonic-gate ht->ht_lock_cnt == 0) { 722*0Sstevel@tonic-gate found_one = 1; 723*0Sstevel@tonic-gate break; 724*0Sstevel@tonic-gate } 725*0Sstevel@tonic-gate } 726*0Sstevel@tonic-gate } 727*0Sstevel@tonic-gate if (!found_one) 728*0Sstevel@tonic-gate x86_hm_exit(pp); 729*0Sstevel@tonic-gate } 730*0Sstevel@tonic-gate 731*0Sstevel@tonic-gate /* 732*0Sstevel@tonic-gate * Steal the mapping we found. Note that hati_page_unmap() will 733*0Sstevel@tonic-gate * do the x86_hm_exit(). 734*0Sstevel@tonic-gate */ 735*0Sstevel@tonic-gate hm2 = hati_page_unmap(pp, ht, hm->hm_entry); 736*0Sstevel@tonic-gate ASSERT(hm2 == hm); 737*0Sstevel@tonic-gate last_page = pp; 738*0Sstevel@tonic-gate return (hm); 739*0Sstevel@tonic-gate } 740