10Sstevel@tonic-gate /* 20Sstevel@tonic-gate * CDDL HEADER START 30Sstevel@tonic-gate * 40Sstevel@tonic-gate * The contents of this file are subject to the terms of the 51498Sbs21162 * Common Development and Distribution License (the "License"). 61498Sbs21162 * You may not use this file except in compliance with the License. 70Sstevel@tonic-gate * 80Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 90Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 100Sstevel@tonic-gate * See the License for the specific language governing permissions 110Sstevel@tonic-gate * and limitations under the License. 120Sstevel@tonic-gate * 130Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 140Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 150Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 160Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 170Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 180Sstevel@tonic-gate * 190Sstevel@tonic-gate * CDDL HEADER END 200Sstevel@tonic-gate */ 210Sstevel@tonic-gate /* 22*3379Ssl108498 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 230Sstevel@tonic-gate * Use is subject to license terms. 240Sstevel@tonic-gate */ 250Sstevel@tonic-gate 260Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 270Sstevel@tonic-gate 280Sstevel@tonic-gate #include <sys/param.h> 290Sstevel@tonic-gate #include <sys/user.h> 300Sstevel@tonic-gate #include <sys/mman.h> 310Sstevel@tonic-gate #include <sys/kmem.h> 320Sstevel@tonic-gate #include <sys/sysmacros.h> 330Sstevel@tonic-gate #include <sys/cmn_err.h> 340Sstevel@tonic-gate #include <sys/systm.h> 350Sstevel@tonic-gate #include <sys/tuneable.h> 360Sstevel@tonic-gate #include <vm/hat.h> 370Sstevel@tonic-gate #include <vm/seg.h> 380Sstevel@tonic-gate #include <vm/as.h> 390Sstevel@tonic-gate #include <vm/anon.h> 400Sstevel@tonic-gate #include <vm/page.h> 410Sstevel@tonic-gate #include <sys/buf.h> 420Sstevel@tonic-gate #include <sys/swap.h> 430Sstevel@tonic-gate #include <sys/atomic.h> 440Sstevel@tonic-gate #include <vm/seg_spt.h> 450Sstevel@tonic-gate #include <sys/debug.h> 460Sstevel@tonic-gate #include <sys/vtrace.h> 470Sstevel@tonic-gate #include <sys/shm.h> 482768Ssl108498 #include <sys/shm_impl.h> 490Sstevel@tonic-gate #include <sys/lgrp.h> 500Sstevel@tonic-gate #include <sys/vmsystm.h> 512768Ssl108498 #include <sys/policy.h> 522768Ssl108498 #include <sys/project.h> 530Sstevel@tonic-gate #include <sys/tnf_probe.h> 542768Ssl108498 #include <sys/zone.h> 550Sstevel@tonic-gate 560Sstevel@tonic-gate #define SEGSPTADDR (caddr_t)0x0 570Sstevel@tonic-gate 580Sstevel@tonic-gate /* 590Sstevel@tonic-gate * # pages used for spt 600Sstevel@tonic-gate */ 610Sstevel@tonic-gate static size_t spt_used; 620Sstevel@tonic-gate 630Sstevel@tonic-gate /* 640Sstevel@tonic-gate * segspt_minfree is the memory left for system after ISM 650Sstevel@tonic-gate * locked its pages; it is set up to 5% of availrmem in 660Sstevel@tonic-gate * sptcreate when ISM is created. ISM should not use more 670Sstevel@tonic-gate * than ~90% of availrmem; if it does, then the performance 680Sstevel@tonic-gate * of the system may decrease. Machines with large memories may 690Sstevel@tonic-gate * be able to use up more memory for ISM so we set the default 700Sstevel@tonic-gate * segspt_minfree to 5% (which gives ISM max 95% of availrmem. 710Sstevel@tonic-gate * If somebody wants even more memory for ISM (risking hanging 720Sstevel@tonic-gate * the system) they can patch the segspt_minfree to smaller number. 730Sstevel@tonic-gate */ 740Sstevel@tonic-gate pgcnt_t segspt_minfree = 0; 750Sstevel@tonic-gate 760Sstevel@tonic-gate static int segspt_create(struct seg *seg, caddr_t argsp); 770Sstevel@tonic-gate static int segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize); 780Sstevel@tonic-gate static void segspt_free(struct seg *seg); 790Sstevel@tonic-gate static void segspt_free_pages(struct seg *seg, caddr_t addr, size_t len); 800Sstevel@tonic-gate static lgrp_mem_policy_info_t *segspt_getpolicy(struct seg *seg, caddr_t addr); 810Sstevel@tonic-gate 820Sstevel@tonic-gate static void 830Sstevel@tonic-gate segspt_badop() 840Sstevel@tonic-gate { 850Sstevel@tonic-gate panic("segspt_badop called"); 860Sstevel@tonic-gate /*NOTREACHED*/ 870Sstevel@tonic-gate } 880Sstevel@tonic-gate 890Sstevel@tonic-gate #define SEGSPT_BADOP(t) (t(*)())segspt_badop 900Sstevel@tonic-gate 910Sstevel@tonic-gate struct seg_ops segspt_ops = { 920Sstevel@tonic-gate SEGSPT_BADOP(int), /* dup */ 930Sstevel@tonic-gate segspt_unmap, 940Sstevel@tonic-gate segspt_free, 950Sstevel@tonic-gate SEGSPT_BADOP(int), /* fault */ 960Sstevel@tonic-gate SEGSPT_BADOP(faultcode_t), /* faulta */ 970Sstevel@tonic-gate SEGSPT_BADOP(int), /* setprot */ 980Sstevel@tonic-gate SEGSPT_BADOP(int), /* checkprot */ 990Sstevel@tonic-gate SEGSPT_BADOP(int), /* kluster */ 1000Sstevel@tonic-gate SEGSPT_BADOP(size_t), /* swapout */ 1010Sstevel@tonic-gate SEGSPT_BADOP(int), /* sync */ 1020Sstevel@tonic-gate SEGSPT_BADOP(size_t), /* incore */ 1030Sstevel@tonic-gate SEGSPT_BADOP(int), /* lockop */ 1040Sstevel@tonic-gate SEGSPT_BADOP(int), /* getprot */ 1050Sstevel@tonic-gate SEGSPT_BADOP(u_offset_t), /* getoffset */ 1060Sstevel@tonic-gate SEGSPT_BADOP(int), /* gettype */ 1070Sstevel@tonic-gate SEGSPT_BADOP(int), /* getvp */ 1080Sstevel@tonic-gate SEGSPT_BADOP(int), /* advise */ 1090Sstevel@tonic-gate SEGSPT_BADOP(void), /* dump */ 1100Sstevel@tonic-gate SEGSPT_BADOP(int), /* pagelock */ 1110Sstevel@tonic-gate SEGSPT_BADOP(int), /* setpgsz */ 1120Sstevel@tonic-gate SEGSPT_BADOP(int), /* getmemid */ 1130Sstevel@tonic-gate segspt_getpolicy, /* getpolicy */ 114670Selowe SEGSPT_BADOP(int), /* capable */ 1150Sstevel@tonic-gate }; 1160Sstevel@tonic-gate 1170Sstevel@tonic-gate static int segspt_shmdup(struct seg *seg, struct seg *newseg); 1180Sstevel@tonic-gate static int segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize); 1190Sstevel@tonic-gate static void segspt_shmfree(struct seg *seg); 1200Sstevel@tonic-gate static faultcode_t segspt_shmfault(struct hat *hat, struct seg *seg, 1210Sstevel@tonic-gate caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw); 1220Sstevel@tonic-gate static faultcode_t segspt_shmfaulta(struct seg *seg, caddr_t addr); 1230Sstevel@tonic-gate static int segspt_shmsetprot(register struct seg *seg, register caddr_t addr, 1240Sstevel@tonic-gate register size_t len, register uint_t prot); 1250Sstevel@tonic-gate static int segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, 1260Sstevel@tonic-gate uint_t prot); 1270Sstevel@tonic-gate static int segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta); 1280Sstevel@tonic-gate static size_t segspt_shmswapout(struct seg *seg); 1290Sstevel@tonic-gate static size_t segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, 1300Sstevel@tonic-gate register char *vec); 1310Sstevel@tonic-gate static int segspt_shmsync(struct seg *seg, register caddr_t addr, size_t len, 1320Sstevel@tonic-gate int attr, uint_t flags); 1330Sstevel@tonic-gate static int segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len, 1340Sstevel@tonic-gate int attr, int op, ulong_t *lockmap, size_t pos); 1350Sstevel@tonic-gate static int segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, 1360Sstevel@tonic-gate uint_t *protv); 1370Sstevel@tonic-gate static u_offset_t segspt_shmgetoffset(struct seg *seg, caddr_t addr); 1380Sstevel@tonic-gate static int segspt_shmgettype(struct seg *seg, caddr_t addr); 1390Sstevel@tonic-gate static int segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp); 1400Sstevel@tonic-gate static int segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, 1410Sstevel@tonic-gate uint_t behav); 1420Sstevel@tonic-gate static void segspt_shmdump(struct seg *seg); 1430Sstevel@tonic-gate static int segspt_shmpagelock(struct seg *, caddr_t, size_t, 1440Sstevel@tonic-gate struct page ***, enum lock_type, enum seg_rw); 1450Sstevel@tonic-gate static int segspt_shmsetpgsz(struct seg *, caddr_t, size_t, uint_t); 1460Sstevel@tonic-gate static int segspt_shmgetmemid(struct seg *, caddr_t, memid_t *); 1470Sstevel@tonic-gate static lgrp_mem_policy_info_t *segspt_shmgetpolicy(struct seg *, caddr_t); 148670Selowe static int segspt_shmcapable(struct seg *, segcapability_t); 1490Sstevel@tonic-gate 1500Sstevel@tonic-gate struct seg_ops segspt_shmops = { 1510Sstevel@tonic-gate segspt_shmdup, 1520Sstevel@tonic-gate segspt_shmunmap, 1530Sstevel@tonic-gate segspt_shmfree, 1540Sstevel@tonic-gate segspt_shmfault, 1550Sstevel@tonic-gate segspt_shmfaulta, 1560Sstevel@tonic-gate segspt_shmsetprot, 1570Sstevel@tonic-gate segspt_shmcheckprot, 1580Sstevel@tonic-gate segspt_shmkluster, 1590Sstevel@tonic-gate segspt_shmswapout, 1600Sstevel@tonic-gate segspt_shmsync, 1610Sstevel@tonic-gate segspt_shmincore, 1620Sstevel@tonic-gate segspt_shmlockop, 1630Sstevel@tonic-gate segspt_shmgetprot, 1640Sstevel@tonic-gate segspt_shmgetoffset, 1650Sstevel@tonic-gate segspt_shmgettype, 1660Sstevel@tonic-gate segspt_shmgetvp, 1670Sstevel@tonic-gate segspt_shmadvise, /* advise */ 1680Sstevel@tonic-gate segspt_shmdump, 1690Sstevel@tonic-gate segspt_shmpagelock, 1700Sstevel@tonic-gate segspt_shmsetpgsz, 1710Sstevel@tonic-gate segspt_shmgetmemid, 1720Sstevel@tonic-gate segspt_shmgetpolicy, 173670Selowe segspt_shmcapable, 1740Sstevel@tonic-gate }; 1750Sstevel@tonic-gate 1760Sstevel@tonic-gate static void segspt_purge(struct seg *seg); 1770Sstevel@tonic-gate static int segspt_reclaim(struct seg *, caddr_t, size_t, struct page **, 1780Sstevel@tonic-gate enum seg_rw); 1790Sstevel@tonic-gate static int spt_anon_getpages(struct seg *seg, caddr_t addr, size_t len, 1800Sstevel@tonic-gate page_t **ppa); 1810Sstevel@tonic-gate 1820Sstevel@tonic-gate 1830Sstevel@tonic-gate 1840Sstevel@tonic-gate /*ARGSUSED*/ 1850Sstevel@tonic-gate int 1860Sstevel@tonic-gate sptcreate(size_t size, struct seg **sptseg, struct anon_map *amp, 1872768Ssl108498 uint_t prot, uint_t flags, uint_t share_szc) 1880Sstevel@tonic-gate { 1890Sstevel@tonic-gate int err; 1900Sstevel@tonic-gate struct as *newas; 1910Sstevel@tonic-gate struct segspt_crargs sptcargs; 1920Sstevel@tonic-gate 1930Sstevel@tonic-gate #ifdef DEBUG 1940Sstevel@tonic-gate TNF_PROBE_1(sptcreate, "spt", /* CSTYLED */, 1952768Ssl108498 tnf_ulong, size, size ); 1960Sstevel@tonic-gate #endif 1970Sstevel@tonic-gate if (segspt_minfree == 0) /* leave min 5% of availrmem for */ 1980Sstevel@tonic-gate segspt_minfree = availrmem/20; /* for the system */ 1990Sstevel@tonic-gate 2000Sstevel@tonic-gate if (!hat_supported(HAT_SHARED_PT, (void *)0)) 2010Sstevel@tonic-gate return (EINVAL); 2020Sstevel@tonic-gate 2030Sstevel@tonic-gate /* 2040Sstevel@tonic-gate * get a new as for this shared memory segment 2050Sstevel@tonic-gate */ 2060Sstevel@tonic-gate newas = as_alloc(); 2072768Ssl108498 newas->a_proc = NULL; 2080Sstevel@tonic-gate sptcargs.amp = amp; 2090Sstevel@tonic-gate sptcargs.prot = prot; 2100Sstevel@tonic-gate sptcargs.flags = flags; 2110Sstevel@tonic-gate sptcargs.szc = share_szc; 2120Sstevel@tonic-gate /* 2130Sstevel@tonic-gate * create a shared page table (spt) segment 2140Sstevel@tonic-gate */ 2150Sstevel@tonic-gate 2160Sstevel@tonic-gate if (err = as_map(newas, SEGSPTADDR, size, segspt_create, &sptcargs)) { 2170Sstevel@tonic-gate as_free(newas); 2180Sstevel@tonic-gate return (err); 2190Sstevel@tonic-gate } 2200Sstevel@tonic-gate *sptseg = sptcargs.seg_spt; 2210Sstevel@tonic-gate return (0); 2220Sstevel@tonic-gate } 2230Sstevel@tonic-gate 2240Sstevel@tonic-gate void 2250Sstevel@tonic-gate sptdestroy(struct as *as, struct anon_map *amp) 2260Sstevel@tonic-gate { 2270Sstevel@tonic-gate 2280Sstevel@tonic-gate #ifdef DEBUG 2290Sstevel@tonic-gate TNF_PROBE_0(sptdestroy, "spt", /* CSTYLED */); 2300Sstevel@tonic-gate #endif 2310Sstevel@tonic-gate (void) as_unmap(as, SEGSPTADDR, amp->size); 2320Sstevel@tonic-gate as_free(as); 2330Sstevel@tonic-gate } 2340Sstevel@tonic-gate 2350Sstevel@tonic-gate /* 2360Sstevel@tonic-gate * called from seg_free(). 2370Sstevel@tonic-gate * free (i.e., unlock, unmap, return to free list) 2380Sstevel@tonic-gate * all the pages in the given seg. 2390Sstevel@tonic-gate */ 2400Sstevel@tonic-gate void 2410Sstevel@tonic-gate segspt_free(struct seg *seg) 2420Sstevel@tonic-gate { 2430Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)seg->s_data; 2440Sstevel@tonic-gate 2450Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 2460Sstevel@tonic-gate 2470Sstevel@tonic-gate if (sptd != NULL) { 2480Sstevel@tonic-gate if (sptd->spt_realsize) 2490Sstevel@tonic-gate segspt_free_pages(seg, seg->s_base, sptd->spt_realsize); 2500Sstevel@tonic-gate 2512768Ssl108498 if (sptd->spt_ppa_lckcnt) 2522768Ssl108498 kmem_free(sptd->spt_ppa_lckcnt, 2532768Ssl108498 sizeof (*sptd->spt_ppa_lckcnt) 2542768Ssl108498 * btopr(sptd->spt_amp->size)); 2550Sstevel@tonic-gate kmem_free(sptd->spt_vp, sizeof (*sptd->spt_vp)); 2560Sstevel@tonic-gate mutex_destroy(&sptd->spt_lock); 2570Sstevel@tonic-gate kmem_free(sptd, sizeof (*sptd)); 2580Sstevel@tonic-gate } 2590Sstevel@tonic-gate } 2600Sstevel@tonic-gate 2610Sstevel@tonic-gate /*ARGSUSED*/ 2620Sstevel@tonic-gate static int 2630Sstevel@tonic-gate segspt_shmsync(struct seg *seg, caddr_t addr, size_t len, int attr, 2640Sstevel@tonic-gate uint_t flags) 2650Sstevel@tonic-gate { 2660Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2670Sstevel@tonic-gate 2680Sstevel@tonic-gate return (0); 2690Sstevel@tonic-gate } 2700Sstevel@tonic-gate 2710Sstevel@tonic-gate /*ARGSUSED*/ 2720Sstevel@tonic-gate static size_t 2730Sstevel@tonic-gate segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, char *vec) 2740Sstevel@tonic-gate { 2750Sstevel@tonic-gate caddr_t eo_seg; 2760Sstevel@tonic-gate pgcnt_t npages; 2770Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 2780Sstevel@tonic-gate struct seg *sptseg; 2790Sstevel@tonic-gate struct spt_data *sptd; 2800Sstevel@tonic-gate 2810Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2820Sstevel@tonic-gate #ifdef lint 2830Sstevel@tonic-gate seg = seg; 2840Sstevel@tonic-gate #endif 2850Sstevel@tonic-gate sptseg = shmd->shm_sptseg; 2860Sstevel@tonic-gate sptd = sptseg->s_data; 2870Sstevel@tonic-gate 2880Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 2890Sstevel@tonic-gate eo_seg = addr + len; 2900Sstevel@tonic-gate while (addr < eo_seg) { 2910Sstevel@tonic-gate /* page exists, and it's locked. */ 2920Sstevel@tonic-gate *vec++ = SEG_PAGE_INCORE | SEG_PAGE_LOCKED | 2930Sstevel@tonic-gate SEG_PAGE_ANON; 2940Sstevel@tonic-gate addr += PAGESIZE; 2950Sstevel@tonic-gate } 2960Sstevel@tonic-gate return (len); 2970Sstevel@tonic-gate } else { 2980Sstevel@tonic-gate struct anon_map *amp = shmd->shm_amp; 2990Sstevel@tonic-gate struct anon *ap; 3000Sstevel@tonic-gate page_t *pp; 3010Sstevel@tonic-gate pgcnt_t anon_index; 3020Sstevel@tonic-gate struct vnode *vp; 3030Sstevel@tonic-gate u_offset_t off; 3040Sstevel@tonic-gate ulong_t i; 3050Sstevel@tonic-gate int ret; 3060Sstevel@tonic-gate anon_sync_obj_t cookie; 3070Sstevel@tonic-gate 3080Sstevel@tonic-gate addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 3090Sstevel@tonic-gate anon_index = seg_page(seg, addr); 3100Sstevel@tonic-gate npages = btopr(len); 3110Sstevel@tonic-gate if (anon_index + npages > btopr(shmd->shm_amp->size)) { 3120Sstevel@tonic-gate return (EINVAL); 3130Sstevel@tonic-gate } 3140Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 3150Sstevel@tonic-gate for (i = 0; i < npages; i++, anon_index++) { 3160Sstevel@tonic-gate ret = 0; 3170Sstevel@tonic-gate anon_array_enter(amp, anon_index, &cookie); 3180Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index); 3190Sstevel@tonic-gate if (ap != NULL) { 3200Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 3210Sstevel@tonic-gate anon_array_exit(&cookie); 3220Sstevel@tonic-gate pp = page_lookup_nowait(vp, off, SE_SHARED); 3230Sstevel@tonic-gate if (pp != NULL) { 3240Sstevel@tonic-gate ret |= SEG_PAGE_INCORE | SEG_PAGE_ANON; 3250Sstevel@tonic-gate page_unlock(pp); 3260Sstevel@tonic-gate } 3270Sstevel@tonic-gate } else { 3280Sstevel@tonic-gate anon_array_exit(&cookie); 3290Sstevel@tonic-gate } 3300Sstevel@tonic-gate if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) { 3310Sstevel@tonic-gate ret |= SEG_PAGE_LOCKED; 3320Sstevel@tonic-gate } 3330Sstevel@tonic-gate *vec++ = (char)ret; 3340Sstevel@tonic-gate } 3350Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 3360Sstevel@tonic-gate return (len); 3370Sstevel@tonic-gate } 3380Sstevel@tonic-gate } 3390Sstevel@tonic-gate 3400Sstevel@tonic-gate static int 3410Sstevel@tonic-gate segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize) 3420Sstevel@tonic-gate { 3430Sstevel@tonic-gate size_t share_size; 3440Sstevel@tonic-gate 3450Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 3460Sstevel@tonic-gate 3470Sstevel@tonic-gate /* 3480Sstevel@tonic-gate * seg.s_size may have been rounded up to the largest page size 3490Sstevel@tonic-gate * in shmat(). 3500Sstevel@tonic-gate * XXX This should be cleanedup. sptdestroy should take a length 3510Sstevel@tonic-gate * argument which should be the same as sptcreate. Then 3520Sstevel@tonic-gate * this rounding would not be needed (or is done in shm.c) 3530Sstevel@tonic-gate * Only the check for full segment will be needed. 3540Sstevel@tonic-gate * 3550Sstevel@tonic-gate * XXX -- shouldn't raddr == 0 always? These tests don't seem 3560Sstevel@tonic-gate * to be useful at all. 3570Sstevel@tonic-gate */ 3580Sstevel@tonic-gate share_size = page_get_pagesize(seg->s_szc); 3590Sstevel@tonic-gate ssize = P2ROUNDUP(ssize, share_size); 3600Sstevel@tonic-gate 3610Sstevel@tonic-gate if (raddr == seg->s_base && ssize == seg->s_size) { 3620Sstevel@tonic-gate seg_free(seg); 3630Sstevel@tonic-gate return (0); 3640Sstevel@tonic-gate } else 3650Sstevel@tonic-gate return (EINVAL); 3660Sstevel@tonic-gate } 3670Sstevel@tonic-gate 3680Sstevel@tonic-gate int 3690Sstevel@tonic-gate segspt_create(struct seg *seg, caddr_t argsp) 3700Sstevel@tonic-gate { 3710Sstevel@tonic-gate int err; 3720Sstevel@tonic-gate caddr_t addr = seg->s_base; 3730Sstevel@tonic-gate struct spt_data *sptd; 3740Sstevel@tonic-gate struct segspt_crargs *sptcargs = (struct segspt_crargs *)argsp; 3750Sstevel@tonic-gate struct anon_map *amp = sptcargs->amp; 3762768Ssl108498 struct kshmid *sp = amp->a_sp; 3770Sstevel@tonic-gate struct cred *cred = CRED(); 3780Sstevel@tonic-gate ulong_t i, j, anon_index = 0; 3790Sstevel@tonic-gate pgcnt_t npages = btopr(amp->size); 3800Sstevel@tonic-gate struct vnode *vp; 3810Sstevel@tonic-gate page_t **ppa; 3820Sstevel@tonic-gate uint_t hat_flags; 3832414Saguzovsk size_t pgsz; 3842414Saguzovsk pgcnt_t pgcnt; 3852414Saguzovsk caddr_t a; 3862414Saguzovsk pgcnt_t pidx; 3872414Saguzovsk size_t sz; 3882768Ssl108498 proc_t *procp = curproc; 3892768Ssl108498 rctl_qty_t lockedbytes = 0; 3902768Ssl108498 kproject_t *proj; 3910Sstevel@tonic-gate 3920Sstevel@tonic-gate /* 3930Sstevel@tonic-gate * We are holding the a_lock on the underlying dummy as, 3940Sstevel@tonic-gate * so we can make calls to the HAT layer. 3950Sstevel@tonic-gate */ 3960Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 3972768Ssl108498 ASSERT(sp != NULL); 3980Sstevel@tonic-gate 3990Sstevel@tonic-gate #ifdef DEBUG 4000Sstevel@tonic-gate TNF_PROBE_2(segspt_create, "spt", /* CSTYLED */, 4012768Ssl108498 tnf_opaque, addr, addr, 4020Sstevel@tonic-gate tnf_ulong, len, seg->s_size); 4030Sstevel@tonic-gate #endif 4040Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0) { 4050Sstevel@tonic-gate if (err = anon_swap_adjust(npages)) 4060Sstevel@tonic-gate return (err); 4070Sstevel@tonic-gate } 4080Sstevel@tonic-gate err = ENOMEM; 4090Sstevel@tonic-gate 4100Sstevel@tonic-gate if ((sptd = kmem_zalloc(sizeof (*sptd), KM_NOSLEEP)) == NULL) 4110Sstevel@tonic-gate goto out1; 4120Sstevel@tonic-gate 4130Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0) { 4140Sstevel@tonic-gate if ((ppa = kmem_zalloc(((sizeof (page_t *)) * npages), 4150Sstevel@tonic-gate KM_NOSLEEP)) == NULL) 4160Sstevel@tonic-gate goto out2; 4170Sstevel@tonic-gate } 4180Sstevel@tonic-gate 4190Sstevel@tonic-gate mutex_init(&sptd->spt_lock, NULL, MUTEX_DEFAULT, NULL); 4200Sstevel@tonic-gate 4210Sstevel@tonic-gate if ((vp = kmem_zalloc(sizeof (*vp), KM_NOSLEEP)) == NULL) 4220Sstevel@tonic-gate goto out3; 4230Sstevel@tonic-gate 4240Sstevel@tonic-gate seg->s_ops = &segspt_ops; 4250Sstevel@tonic-gate sptd->spt_vp = vp; 4260Sstevel@tonic-gate sptd->spt_amp = amp; 4270Sstevel@tonic-gate sptd->spt_prot = sptcargs->prot; 4280Sstevel@tonic-gate sptd->spt_flags = sptcargs->flags; 4290Sstevel@tonic-gate seg->s_data = (caddr_t)sptd; 4300Sstevel@tonic-gate sptd->spt_ppa = NULL; 4310Sstevel@tonic-gate sptd->spt_ppa_lckcnt = NULL; 4320Sstevel@tonic-gate seg->s_szc = sptcargs->szc; 4330Sstevel@tonic-gate 4340Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 4352414Saguzovsk if (seg->s_szc > amp->a_szc) { 4362414Saguzovsk amp->a_szc = seg->s_szc; 4372414Saguzovsk } 4380Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 4390Sstevel@tonic-gate 4400Sstevel@tonic-gate /* 4410Sstevel@tonic-gate * Set policy to affect initial allocation of pages in 4420Sstevel@tonic-gate * anon_map_createpages() 4430Sstevel@tonic-gate */ 4440Sstevel@tonic-gate (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, amp, anon_index, 4450Sstevel@tonic-gate NULL, 0, ptob(npages)); 4460Sstevel@tonic-gate 4470Sstevel@tonic-gate if (sptcargs->flags & SHM_PAGEABLE) { 4480Sstevel@tonic-gate size_t share_sz; 4490Sstevel@tonic-gate pgcnt_t new_npgs, more_pgs; 4500Sstevel@tonic-gate struct anon_hdr *nahp; 451*3379Ssl108498 zone_t *zone; 4520Sstevel@tonic-gate 4530Sstevel@tonic-gate share_sz = page_get_pagesize(seg->s_szc); 4540Sstevel@tonic-gate if (!IS_P2ALIGNED(amp->size, share_sz)) { 4550Sstevel@tonic-gate /* 4560Sstevel@tonic-gate * We are rounding up the size of the anon array 4570Sstevel@tonic-gate * on 4 M boundary because we always create 4 M 4580Sstevel@tonic-gate * of page(s) when locking, faulting pages and we 4590Sstevel@tonic-gate * don't have to check for all corner cases e.g. 4600Sstevel@tonic-gate * if there is enough space to allocate 4 M 4610Sstevel@tonic-gate * page. 4620Sstevel@tonic-gate */ 4630Sstevel@tonic-gate new_npgs = btop(P2ROUNDUP(amp->size, share_sz)); 4640Sstevel@tonic-gate more_pgs = new_npgs - npages; 4650Sstevel@tonic-gate 466*3379Ssl108498 /* 467*3379Ssl108498 * This may return NULL if global zone is removing a 468*3379Ssl108498 * shm created by a non-global zone that has been 469*3379Ssl108498 * destroyed. 470*3379Ssl108498 */ 471*3379Ssl108498 zone = 472*3379Ssl108498 zone_find_by_id(sp->shm_perm.ipc_proj->kpj_zoneid); 473*3379Ssl108498 474*3379Ssl108498 if (anon_resv_zone(ptob(more_pgs), zone) == 0) { 475*3379Ssl108498 if (zone != NULL) 476*3379Ssl108498 zone_rele(zone); 4770Sstevel@tonic-gate err = ENOMEM; 4780Sstevel@tonic-gate goto out4; 4790Sstevel@tonic-gate } 480*3379Ssl108498 if (zone != NULL) 481*3379Ssl108498 zone_rele(zone); 482*3379Ssl108498 4830Sstevel@tonic-gate nahp = anon_create(new_npgs, ANON_SLEEP); 4840Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 4850Sstevel@tonic-gate (void) anon_copy_ptr(amp->ahp, 0, nahp, 0, npages, 4860Sstevel@tonic-gate ANON_SLEEP); 4870Sstevel@tonic-gate anon_release(amp->ahp, npages); 4880Sstevel@tonic-gate amp->ahp = nahp; 489*3379Ssl108498 ASSERT(amp->swresv == ptob(npages)); 4900Sstevel@tonic-gate amp->swresv = amp->size = ptob(new_npgs); 4910Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 4920Sstevel@tonic-gate npages = new_npgs; 4930Sstevel@tonic-gate } 4940Sstevel@tonic-gate 4950Sstevel@tonic-gate sptd->spt_ppa_lckcnt = kmem_zalloc(npages * 4960Sstevel@tonic-gate sizeof (*sptd->spt_ppa_lckcnt), KM_SLEEP); 4970Sstevel@tonic-gate sptd->spt_pcachecnt = 0; 4980Sstevel@tonic-gate sptd->spt_realsize = ptob(npages); 4990Sstevel@tonic-gate sptcargs->seg_spt = seg; 5000Sstevel@tonic-gate return (0); 5010Sstevel@tonic-gate } 5020Sstevel@tonic-gate 5030Sstevel@tonic-gate /* 5040Sstevel@tonic-gate * get array of pages for each anon slot in amp 5050Sstevel@tonic-gate */ 5060Sstevel@tonic-gate if ((err = anon_map_createpages(amp, anon_index, ptob(npages), ppa, 5070Sstevel@tonic-gate seg, addr, S_CREATE, cred)) != 0) 5080Sstevel@tonic-gate goto out4; 5090Sstevel@tonic-gate 5102768Ssl108498 mutex_enter(&sp->shm_mlock); 5112768Ssl108498 5122768Ssl108498 /* May be partially locked, so, count bytes to charge for locking */ 5132768Ssl108498 for (i = 0; i < npages; i++) 5142768Ssl108498 if (ppa[i]->p_lckcnt == 0) 5152768Ssl108498 lockedbytes += PAGESIZE; 5162768Ssl108498 5172768Ssl108498 proj = sp->shm_perm.ipc_proj; 5182768Ssl108498 5192768Ssl108498 if (lockedbytes > 0) { 5202768Ssl108498 mutex_enter(&procp->p_lock); 5212768Ssl108498 if (rctl_incr_locked_mem(procp, proj, lockedbytes, 0)) { 5222768Ssl108498 mutex_exit(&procp->p_lock); 5232768Ssl108498 mutex_exit(&sp->shm_mlock); 5242768Ssl108498 for (i = 0; i < npages; i++) 5252768Ssl108498 page_unlock(ppa[i]); 5262768Ssl108498 err = ENOMEM; 5272768Ssl108498 goto out4; 5282768Ssl108498 } 5292768Ssl108498 mutex_exit(&procp->p_lock); 5302768Ssl108498 } 5312768Ssl108498 5320Sstevel@tonic-gate /* 5330Sstevel@tonic-gate * addr is initial address corresponding to the first page on ppa list 5340Sstevel@tonic-gate */ 5350Sstevel@tonic-gate for (i = 0; i < npages; i++) { 5360Sstevel@tonic-gate /* attempt to lock all pages */ 5372768Ssl108498 if (page_pp_lock(ppa[i], 0, 1) == 0) { 5380Sstevel@tonic-gate /* 5390Sstevel@tonic-gate * if unable to lock any page, unlock all 5400Sstevel@tonic-gate * of them and return error 5410Sstevel@tonic-gate */ 5420Sstevel@tonic-gate for (j = 0; j < i; j++) 5430Sstevel@tonic-gate page_pp_unlock(ppa[j], 0, 1); 5442768Ssl108498 for (i = 0; i < npages; i++) 5450Sstevel@tonic-gate page_unlock(ppa[i]); 5462768Ssl108498 rctl_decr_locked_mem(NULL, proj, lockedbytes, 0); 5472768Ssl108498 mutex_exit(&sp->shm_mlock); 5480Sstevel@tonic-gate err = ENOMEM; 5490Sstevel@tonic-gate goto out4; 5500Sstevel@tonic-gate } 5510Sstevel@tonic-gate } 5522768Ssl108498 mutex_exit(&sp->shm_mlock); 5530Sstevel@tonic-gate 5540Sstevel@tonic-gate /* 5550Sstevel@tonic-gate * Some platforms assume that ISM mappings are HAT_LOAD_LOCK 5560Sstevel@tonic-gate * for the entire life of the segment. For example platforms 5570Sstevel@tonic-gate * that do not support Dynamic Reconfiguration. 5580Sstevel@tonic-gate */ 5590Sstevel@tonic-gate hat_flags = HAT_LOAD_SHARE; 5600Sstevel@tonic-gate if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) 5610Sstevel@tonic-gate hat_flags |= HAT_LOAD_LOCK; 5620Sstevel@tonic-gate 5632414Saguzovsk /* 5642414Saguzovsk * Load translations one lare page at a time 5652414Saguzovsk * to make sure we don't create mappings bigger than 5662414Saguzovsk * segment's size code in case underlying pages 5672414Saguzovsk * are shared with segvn's segment that uses bigger 5682414Saguzovsk * size code than we do. 5692414Saguzovsk */ 5702414Saguzovsk pgsz = page_get_pagesize(seg->s_szc); 5712414Saguzovsk pgcnt = page_get_pagecnt(seg->s_szc); 5722414Saguzovsk for (a = addr, pidx = 0; pidx < npages; a += pgsz, pidx += pgcnt) { 5732414Saguzovsk sz = MIN(pgsz, ptob(npages - pidx)); 5742414Saguzovsk hat_memload_array(seg->s_as->a_hat, a, sz, 5752414Saguzovsk &ppa[pidx], sptd->spt_prot, hat_flags); 5762414Saguzovsk } 5770Sstevel@tonic-gate 5780Sstevel@tonic-gate /* 5790Sstevel@tonic-gate * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP, 5800Sstevel@tonic-gate * we will leave the pages locked SE_SHARED for the life 5810Sstevel@tonic-gate * of the ISM segment. This will prevent any calls to 5820Sstevel@tonic-gate * hat_pageunload() on this ISM segment for those platforms. 5830Sstevel@tonic-gate */ 5840Sstevel@tonic-gate if (!(hat_flags & HAT_LOAD_LOCK)) { 5850Sstevel@tonic-gate /* 5860Sstevel@tonic-gate * On platforms that support HAT_DYNAMIC_ISM_UNMAP, 5870Sstevel@tonic-gate * we no longer need to hold the SE_SHARED lock on the pages, 5880Sstevel@tonic-gate * since L_PAGELOCK and F_SOFTLOCK calls will grab the 5890Sstevel@tonic-gate * SE_SHARED lock on the pages as necessary. 5900Sstevel@tonic-gate */ 5910Sstevel@tonic-gate for (i = 0; i < npages; i++) 5920Sstevel@tonic-gate page_unlock(ppa[i]); 5930Sstevel@tonic-gate } 5940Sstevel@tonic-gate sptd->spt_pcachecnt = 0; 5950Sstevel@tonic-gate kmem_free(ppa, ((sizeof (page_t *)) * npages)); 5960Sstevel@tonic-gate sptd->spt_realsize = ptob(npages); 5970Sstevel@tonic-gate atomic_add_long(&spt_used, npages); 5980Sstevel@tonic-gate sptcargs->seg_spt = seg; 5990Sstevel@tonic-gate return (0); 6000Sstevel@tonic-gate 6010Sstevel@tonic-gate out4: 6020Sstevel@tonic-gate seg->s_data = NULL; 6030Sstevel@tonic-gate kmem_free(vp, sizeof (*vp)); 6040Sstevel@tonic-gate out3: 6050Sstevel@tonic-gate mutex_destroy(&sptd->spt_lock); 6060Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0) 6070Sstevel@tonic-gate kmem_free(ppa, (sizeof (*ppa) * npages)); 6080Sstevel@tonic-gate out2: 6090Sstevel@tonic-gate kmem_free(sptd, sizeof (*sptd)); 6100Sstevel@tonic-gate out1: 6110Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0) 6120Sstevel@tonic-gate anon_swap_restore(npages); 6130Sstevel@tonic-gate return (err); 6140Sstevel@tonic-gate } 6150Sstevel@tonic-gate 6160Sstevel@tonic-gate /*ARGSUSED*/ 6170Sstevel@tonic-gate void 6180Sstevel@tonic-gate segspt_free_pages(struct seg *seg, caddr_t addr, size_t len) 6190Sstevel@tonic-gate { 6200Sstevel@tonic-gate struct page *pp; 6210Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)seg->s_data; 6220Sstevel@tonic-gate pgcnt_t npages; 6230Sstevel@tonic-gate ulong_t anon_idx; 6240Sstevel@tonic-gate struct anon_map *amp; 6250Sstevel@tonic-gate struct anon *ap; 6260Sstevel@tonic-gate struct vnode *vp; 6270Sstevel@tonic-gate u_offset_t off; 6280Sstevel@tonic-gate uint_t hat_flags; 6290Sstevel@tonic-gate int root = 0; 6300Sstevel@tonic-gate pgcnt_t pgs, curnpgs = 0; 6310Sstevel@tonic-gate page_t *rootpp; 6322768Ssl108498 rctl_qty_t unlocked_bytes = 0; 6332768Ssl108498 kproject_t *proj; 6342768Ssl108498 kshmid_t *sp; 6350Sstevel@tonic-gate 6360Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 6370Sstevel@tonic-gate 6380Sstevel@tonic-gate len = P2ROUNDUP(len, PAGESIZE); 6390Sstevel@tonic-gate 6400Sstevel@tonic-gate npages = btop(len); 6410Sstevel@tonic-gate 6420Sstevel@tonic-gate hat_flags = HAT_UNLOAD_UNLOCK; 6430Sstevel@tonic-gate if ((hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) || 6440Sstevel@tonic-gate (sptd->spt_flags & SHM_PAGEABLE)) { 6450Sstevel@tonic-gate hat_flags = HAT_UNLOAD; 6460Sstevel@tonic-gate } 6470Sstevel@tonic-gate 6480Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, len, hat_flags); 6490Sstevel@tonic-gate 6500Sstevel@tonic-gate amp = sptd->spt_amp; 6510Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) 6520Sstevel@tonic-gate npages = btop(amp->size); 6530Sstevel@tonic-gate 6542768Ssl108498 ASSERT(amp != NULL); 6552768Ssl108498 6562768Ssl108498 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 6572768Ssl108498 sp = amp->a_sp; 6582768Ssl108498 proj = sp->shm_perm.ipc_proj; 6592768Ssl108498 mutex_enter(&sp->shm_mlock); 6602768Ssl108498 } 6610Sstevel@tonic-gate for (anon_idx = 0; anon_idx < npages; anon_idx++) { 6620Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 6630Sstevel@tonic-gate if ((ap = anon_get_ptr(amp->ahp, anon_idx)) == NULL) { 6640Sstevel@tonic-gate panic("segspt_free_pages: null app"); 6650Sstevel@tonic-gate /*NOTREACHED*/ 6660Sstevel@tonic-gate } 6670Sstevel@tonic-gate } else { 6680Sstevel@tonic-gate if ((ap = anon_get_next_ptr(amp->ahp, &anon_idx)) 6690Sstevel@tonic-gate == NULL) 6700Sstevel@tonic-gate continue; 6710Sstevel@tonic-gate } 6720Sstevel@tonic-gate ASSERT(ANON_ISBUSY(anon_get_slot(amp->ahp, anon_idx)) == 0); 6730Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 6740Sstevel@tonic-gate 6750Sstevel@tonic-gate /* 6760Sstevel@tonic-gate * If this platform supports HAT_DYNAMIC_ISM_UNMAP, 6770Sstevel@tonic-gate * the pages won't be having SE_SHARED lock at this 6780Sstevel@tonic-gate * point. 6790Sstevel@tonic-gate * 6800Sstevel@tonic-gate * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP, 6810Sstevel@tonic-gate * the pages are still held SE_SHARED locked from the 6820Sstevel@tonic-gate * original segspt_create() 6830Sstevel@tonic-gate * 6840Sstevel@tonic-gate * Our goal is to get SE_EXCL lock on each page, remove 6850Sstevel@tonic-gate * permanent lock on it and invalidate the page. 6860Sstevel@tonic-gate */ 6870Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 6880Sstevel@tonic-gate if (hat_flags == HAT_UNLOAD) 6890Sstevel@tonic-gate pp = page_lookup(vp, off, SE_EXCL); 6900Sstevel@tonic-gate else { 6910Sstevel@tonic-gate if ((pp = page_find(vp, off)) == NULL) { 6920Sstevel@tonic-gate panic("segspt_free_pages: " 6930Sstevel@tonic-gate "page not locked"); 6940Sstevel@tonic-gate /*NOTREACHED*/ 6950Sstevel@tonic-gate } 6960Sstevel@tonic-gate if (!page_tryupgrade(pp)) { 6970Sstevel@tonic-gate page_unlock(pp); 6980Sstevel@tonic-gate pp = page_lookup(vp, off, SE_EXCL); 6990Sstevel@tonic-gate } 7000Sstevel@tonic-gate } 7010Sstevel@tonic-gate if (pp == NULL) { 7020Sstevel@tonic-gate panic("segspt_free_pages: " 7030Sstevel@tonic-gate "page not in the system"); 7040Sstevel@tonic-gate /*NOTREACHED*/ 7050Sstevel@tonic-gate } 7062768Ssl108498 ASSERT(pp->p_lckcnt > 0); 7070Sstevel@tonic-gate page_pp_unlock(pp, 0, 1); 7082768Ssl108498 if (pp->p_lckcnt == 0) 7092768Ssl108498 unlocked_bytes += PAGESIZE; 7100Sstevel@tonic-gate } else { 7110Sstevel@tonic-gate if ((pp = page_lookup(vp, off, SE_EXCL)) == NULL) 7120Sstevel@tonic-gate continue; 7130Sstevel@tonic-gate } 7140Sstevel@tonic-gate /* 7150Sstevel@tonic-gate * It's logical to invalidate the pages here as in most cases 7160Sstevel@tonic-gate * these were created by segspt. 7170Sstevel@tonic-gate */ 7180Sstevel@tonic-gate if (pp->p_szc != 0) { 7190Sstevel@tonic-gate /* 7200Sstevel@tonic-gate * For DISM swap is released in shm_rm_amp. 7210Sstevel@tonic-gate */ 7220Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0 && 7230Sstevel@tonic-gate ap->an_pvp != NULL) { 7240Sstevel@tonic-gate panic("segspt_free_pages: pvp non NULL"); 7250Sstevel@tonic-gate /*NOTREACHED*/ 7260Sstevel@tonic-gate } 7270Sstevel@tonic-gate if (root == 0) { 7280Sstevel@tonic-gate ASSERT(curnpgs == 0); 7290Sstevel@tonic-gate root = 1; 7300Sstevel@tonic-gate rootpp = pp; 7310Sstevel@tonic-gate pgs = curnpgs = page_get_pagecnt(pp->p_szc); 7320Sstevel@tonic-gate ASSERT(pgs > 1); 7330Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(pgs, pgs)); 7340Sstevel@tonic-gate ASSERT(!(page_pptonum(pp) & (pgs - 1))); 7350Sstevel@tonic-gate curnpgs--; 7360Sstevel@tonic-gate } else if ((page_pptonum(pp) & (pgs - 1)) == pgs - 1) { 7370Sstevel@tonic-gate ASSERT(curnpgs == 1); 7380Sstevel@tonic-gate ASSERT(page_pptonum(pp) == 7390Sstevel@tonic-gate page_pptonum(rootpp) + (pgs - 1)); 7400Sstevel@tonic-gate page_destroy_pages(rootpp); 7410Sstevel@tonic-gate root = 0; 7420Sstevel@tonic-gate curnpgs = 0; 7430Sstevel@tonic-gate } else { 7440Sstevel@tonic-gate ASSERT(curnpgs > 1); 7450Sstevel@tonic-gate ASSERT(page_pptonum(pp) == 7460Sstevel@tonic-gate page_pptonum(rootpp) + (pgs - curnpgs)); 7470Sstevel@tonic-gate curnpgs--; 7480Sstevel@tonic-gate } 7490Sstevel@tonic-gate } else { 7500Sstevel@tonic-gate if (root != 0 || curnpgs != 0) { 7510Sstevel@tonic-gate panic("segspt_free_pages: bad large page"); 7520Sstevel@tonic-gate /*NOTREACHED*/ 7530Sstevel@tonic-gate } 7540Sstevel@tonic-gate /*LINTED: constant in conditional context */ 7550Sstevel@tonic-gate VN_DISPOSE(pp, B_INVAL, 0, kcred); 7560Sstevel@tonic-gate } 7570Sstevel@tonic-gate } 7582768Ssl108498 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 7592768Ssl108498 if (unlocked_bytes > 0) 7602768Ssl108498 rctl_decr_locked_mem(NULL, proj, unlocked_bytes, 0); 7612768Ssl108498 mutex_exit(&sp->shm_mlock); 7622768Ssl108498 } 7630Sstevel@tonic-gate if (root != 0 || curnpgs != 0) { 7640Sstevel@tonic-gate panic("segspt_free_pages: bad large page"); 7650Sstevel@tonic-gate /*NOTREACHED*/ 7660Sstevel@tonic-gate } 7670Sstevel@tonic-gate 7680Sstevel@tonic-gate /* 7690Sstevel@tonic-gate * mark that pages have been released 7700Sstevel@tonic-gate */ 7710Sstevel@tonic-gate sptd->spt_realsize = 0; 7720Sstevel@tonic-gate 7730Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 7740Sstevel@tonic-gate atomic_add_long(&spt_used, -npages); 7750Sstevel@tonic-gate anon_swap_restore(npages); 7760Sstevel@tonic-gate } 7770Sstevel@tonic-gate } 7780Sstevel@tonic-gate 7790Sstevel@tonic-gate /* 7800Sstevel@tonic-gate * Get memory allocation policy info for specified address in given segment 7810Sstevel@tonic-gate */ 7820Sstevel@tonic-gate static lgrp_mem_policy_info_t * 7830Sstevel@tonic-gate segspt_getpolicy(struct seg *seg, caddr_t addr) 7840Sstevel@tonic-gate { 7850Sstevel@tonic-gate struct anon_map *amp; 7860Sstevel@tonic-gate ulong_t anon_index; 7870Sstevel@tonic-gate lgrp_mem_policy_info_t *policy_info; 7880Sstevel@tonic-gate struct spt_data *spt_data; 7890Sstevel@tonic-gate 7900Sstevel@tonic-gate ASSERT(seg != NULL); 7910Sstevel@tonic-gate 7920Sstevel@tonic-gate /* 7930Sstevel@tonic-gate * Get anon_map from segspt 7940Sstevel@tonic-gate * 7950Sstevel@tonic-gate * Assume that no lock needs to be held on anon_map, since 7960Sstevel@tonic-gate * it should be protected by its reference count which must be 7970Sstevel@tonic-gate * nonzero for an existing segment 7980Sstevel@tonic-gate * Need to grab readers lock on policy tree though 7990Sstevel@tonic-gate */ 8000Sstevel@tonic-gate spt_data = (struct spt_data *)seg->s_data; 8010Sstevel@tonic-gate if (spt_data == NULL) 8020Sstevel@tonic-gate return (NULL); 8030Sstevel@tonic-gate amp = spt_data->spt_amp; 8040Sstevel@tonic-gate ASSERT(amp->refcnt != 0); 8050Sstevel@tonic-gate 8060Sstevel@tonic-gate /* 8070Sstevel@tonic-gate * Get policy info 8080Sstevel@tonic-gate * 8090Sstevel@tonic-gate * Assume starting anon index of 0 8100Sstevel@tonic-gate */ 8110Sstevel@tonic-gate anon_index = seg_page(seg, addr); 8120Sstevel@tonic-gate policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0); 8130Sstevel@tonic-gate 8140Sstevel@tonic-gate return (policy_info); 8150Sstevel@tonic-gate } 8160Sstevel@tonic-gate 8170Sstevel@tonic-gate /* 8180Sstevel@tonic-gate * DISM only. 8190Sstevel@tonic-gate * Return locked pages over a given range. 8200Sstevel@tonic-gate * 8210Sstevel@tonic-gate * We will cache all DISM locked pages and save the pplist for the 8220Sstevel@tonic-gate * entire segment in the ppa field of the underlying DISM segment structure. 8230Sstevel@tonic-gate * Later, during a call to segspt_reclaim() we will use this ppa array 8240Sstevel@tonic-gate * to page_unlock() all of the pages and then we will free this ppa list. 8250Sstevel@tonic-gate */ 8260Sstevel@tonic-gate /*ARGSUSED*/ 8270Sstevel@tonic-gate static int 8280Sstevel@tonic-gate segspt_dismpagelock(struct seg *seg, caddr_t addr, size_t len, 8290Sstevel@tonic-gate struct page ***ppp, enum lock_type type, enum seg_rw rw) 8300Sstevel@tonic-gate { 8310Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 8320Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 8330Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 8340Sstevel@tonic-gate pgcnt_t pg_idx, npages, tot_npages, npgs; 8350Sstevel@tonic-gate struct page **pplist, **pl, **ppa, *pp; 8360Sstevel@tonic-gate struct anon_map *amp; 8370Sstevel@tonic-gate spgcnt_t an_idx; 8380Sstevel@tonic-gate int ret = ENOTSUP; 8390Sstevel@tonic-gate uint_t pl_built = 0; 8400Sstevel@tonic-gate struct anon *ap; 8410Sstevel@tonic-gate struct vnode *vp; 8420Sstevel@tonic-gate u_offset_t off; 8430Sstevel@tonic-gate pgcnt_t claim_availrmem = 0; 8440Sstevel@tonic-gate uint_t szc; 8450Sstevel@tonic-gate 8460Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 8470Sstevel@tonic-gate 8480Sstevel@tonic-gate /* 8490Sstevel@tonic-gate * We want to lock/unlock the entire ISM segment. Therefore, 8500Sstevel@tonic-gate * we will be using the underlying sptseg and it's base address 8510Sstevel@tonic-gate * and length for the caching arguments. 8520Sstevel@tonic-gate */ 8530Sstevel@tonic-gate ASSERT(sptseg); 8540Sstevel@tonic-gate ASSERT(sptd); 8550Sstevel@tonic-gate 8560Sstevel@tonic-gate pg_idx = seg_page(seg, addr); 8570Sstevel@tonic-gate npages = btopr(len); 8580Sstevel@tonic-gate 8590Sstevel@tonic-gate /* 8600Sstevel@tonic-gate * check if the request is larger than number of pages covered 8610Sstevel@tonic-gate * by amp 8620Sstevel@tonic-gate */ 8630Sstevel@tonic-gate if (pg_idx + npages > btopr(sptd->spt_amp->size)) { 8640Sstevel@tonic-gate *ppp = NULL; 8650Sstevel@tonic-gate return (ENOTSUP); 8660Sstevel@tonic-gate } 8670Sstevel@tonic-gate 8680Sstevel@tonic-gate if (type == L_PAGEUNLOCK) { 8690Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 8700Sstevel@tonic-gate 8710Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, sptd->spt_amp->size, 8720Sstevel@tonic-gate sptd->spt_ppa, sptd->spt_prot, segspt_reclaim); 8730Sstevel@tonic-gate 8740Sstevel@tonic-gate /* 8750Sstevel@tonic-gate * If someone is blocked while unmapping, we purge 8760Sstevel@tonic-gate * segment page cache and thus reclaim pplist synchronously 8770Sstevel@tonic-gate * without waiting for seg_pasync_thread. This speeds up 8780Sstevel@tonic-gate * unmapping in cases where munmap(2) is called, while 8790Sstevel@tonic-gate * raw async i/o is still in progress or where a thread 8800Sstevel@tonic-gate * exits on data fault in a multithreaded application. 8810Sstevel@tonic-gate */ 8820Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) { 8830Sstevel@tonic-gate segspt_purge(seg); 8840Sstevel@tonic-gate } 8850Sstevel@tonic-gate return (0); 8860Sstevel@tonic-gate } else if (type == L_PAGERECLAIM) { 8870Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 8880Sstevel@tonic-gate (void) segspt_reclaim(seg, seg->s_base, sptd->spt_amp->size, 8890Sstevel@tonic-gate sptd->spt_ppa, sptd->spt_prot); 8900Sstevel@tonic-gate return (0); 8910Sstevel@tonic-gate } 8920Sstevel@tonic-gate 8930Sstevel@tonic-gate if (sptd->spt_flags & DISM_PPA_CHANGED) { 8940Sstevel@tonic-gate segspt_purge(seg); 8950Sstevel@tonic-gate /* 8960Sstevel@tonic-gate * for DISM ppa needs to be rebuild since 8970Sstevel@tonic-gate * number of locked pages could be changed 8980Sstevel@tonic-gate */ 8990Sstevel@tonic-gate *ppp = NULL; 9000Sstevel@tonic-gate return (ENOTSUP); 9010Sstevel@tonic-gate } 9020Sstevel@tonic-gate 9030Sstevel@tonic-gate /* 9040Sstevel@tonic-gate * First try to find pages in segment page cache, without 9050Sstevel@tonic-gate * holding the segment lock. 9060Sstevel@tonic-gate */ 9070Sstevel@tonic-gate pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size, 9080Sstevel@tonic-gate sptd->spt_prot); 9090Sstevel@tonic-gate if (pplist != NULL) { 9100Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 9110Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 9120Sstevel@tonic-gate ppa = sptd->spt_ppa; 9130Sstevel@tonic-gate for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 9140Sstevel@tonic-gate if (ppa[an_idx] == NULL) { 9150Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, 9160Sstevel@tonic-gate sptd->spt_amp->size, ppa, 9170Sstevel@tonic-gate sptd->spt_prot, segspt_reclaim); 9180Sstevel@tonic-gate *ppp = NULL; 9190Sstevel@tonic-gate return (ENOTSUP); 9200Sstevel@tonic-gate } 9210Sstevel@tonic-gate if ((szc = ppa[an_idx]->p_szc) != 0) { 9220Sstevel@tonic-gate npgs = page_get_pagecnt(szc); 9230Sstevel@tonic-gate an_idx = P2ROUNDUP(an_idx + 1, npgs); 9240Sstevel@tonic-gate } else { 9250Sstevel@tonic-gate an_idx++; 9260Sstevel@tonic-gate } 9270Sstevel@tonic-gate } 9280Sstevel@tonic-gate /* 9290Sstevel@tonic-gate * Since we cache the entire DISM segment, we want to 9300Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 9310Sstevel@tonic-gate * to the requested addr, i.e. pg_idx. 9320Sstevel@tonic-gate */ 9330Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[pg_idx]); 9340Sstevel@tonic-gate return (0); 9350Sstevel@tonic-gate } 9360Sstevel@tonic-gate 9370Sstevel@tonic-gate /* The L_PAGELOCK case... */ 9380Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 9390Sstevel@tonic-gate /* 9400Sstevel@tonic-gate * try to find pages in segment page cache with mutex 9410Sstevel@tonic-gate */ 9420Sstevel@tonic-gate pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size, 9430Sstevel@tonic-gate sptd->spt_prot); 9440Sstevel@tonic-gate if (pplist != NULL) { 9450Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 9460Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 9470Sstevel@tonic-gate ppa = sptd->spt_ppa; 9480Sstevel@tonic-gate for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 9490Sstevel@tonic-gate if (ppa[an_idx] == NULL) { 9500Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 9510Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, 9520Sstevel@tonic-gate sptd->spt_amp->size, ppa, 9530Sstevel@tonic-gate sptd->spt_prot, segspt_reclaim); 9540Sstevel@tonic-gate *ppp = NULL; 9550Sstevel@tonic-gate return (ENOTSUP); 9560Sstevel@tonic-gate } 9570Sstevel@tonic-gate if ((szc = ppa[an_idx]->p_szc) != 0) { 9580Sstevel@tonic-gate npgs = page_get_pagecnt(szc); 9590Sstevel@tonic-gate an_idx = P2ROUNDUP(an_idx + 1, npgs); 9600Sstevel@tonic-gate } else { 9610Sstevel@tonic-gate an_idx++; 9620Sstevel@tonic-gate } 9630Sstevel@tonic-gate } 9640Sstevel@tonic-gate /* 9650Sstevel@tonic-gate * Since we cache the entire DISM segment, we want to 9660Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 9670Sstevel@tonic-gate * to the requested addr, i.e. pg_idx. 9680Sstevel@tonic-gate */ 9690Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 9700Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[pg_idx]); 9710Sstevel@tonic-gate return (0); 9720Sstevel@tonic-gate } 9730Sstevel@tonic-gate if (seg_pinsert_check(seg, sptd->spt_amp->size, SEGP_FORCE_WIRED) == 9740Sstevel@tonic-gate SEGP_FAIL) { 9750Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 9760Sstevel@tonic-gate *ppp = NULL; 9770Sstevel@tonic-gate return (ENOTSUP); 9780Sstevel@tonic-gate } 9790Sstevel@tonic-gate 9800Sstevel@tonic-gate /* 9810Sstevel@tonic-gate * No need to worry about protections because DISM pages are always rw. 9820Sstevel@tonic-gate */ 9830Sstevel@tonic-gate pl = pplist = NULL; 9840Sstevel@tonic-gate amp = sptd->spt_amp; 9850Sstevel@tonic-gate 9860Sstevel@tonic-gate /* 9870Sstevel@tonic-gate * Do we need to build the ppa array? 9880Sstevel@tonic-gate */ 9890Sstevel@tonic-gate if (sptd->spt_ppa == NULL) { 9900Sstevel@tonic-gate pgcnt_t lpg_cnt = 0; 9910Sstevel@tonic-gate 9920Sstevel@tonic-gate pl_built = 1; 9930Sstevel@tonic-gate tot_npages = btopr(sptd->spt_amp->size); 9940Sstevel@tonic-gate 9950Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt == 0); 9960Sstevel@tonic-gate pplist = kmem_zalloc(sizeof (page_t *) * tot_npages, KM_SLEEP); 9970Sstevel@tonic-gate pl = pplist; 9980Sstevel@tonic-gate 9990Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 10000Sstevel@tonic-gate for (an_idx = 0; an_idx < tot_npages; ) { 10010Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, an_idx); 10020Sstevel@tonic-gate /* 10030Sstevel@tonic-gate * Cache only mlocked pages. For large pages 10040Sstevel@tonic-gate * if one (constituent) page is mlocked 10050Sstevel@tonic-gate * all pages for that large page 10060Sstevel@tonic-gate * are cached also. This is for quick 10070Sstevel@tonic-gate * lookups of ppa array; 10080Sstevel@tonic-gate */ 10090Sstevel@tonic-gate if ((ap != NULL) && (lpg_cnt != 0 || 10100Sstevel@tonic-gate (sptd->spt_ppa_lckcnt[an_idx] != 0))) { 10110Sstevel@tonic-gate 10120Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 10130Sstevel@tonic-gate pp = page_lookup(vp, off, SE_SHARED); 10140Sstevel@tonic-gate ASSERT(pp != NULL); 10150Sstevel@tonic-gate if (lpg_cnt == 0) { 10161498Sbs21162 lpg_cnt++; 10171498Sbs21162 /* 10181498Sbs21162 * For a small page, we are done -- 10191498Sbs21162 * lpg_count is reset to 0 below. 10201498Sbs21162 * 10211498Sbs21162 * For a large page, we are guaranteed 10221498Sbs21162 * to find the anon structures of all 10231498Sbs21162 * constituent pages and a non-zero 10241498Sbs21162 * lpg_cnt ensures that we don't test 10251498Sbs21162 * for mlock for these. We are done 10261498Sbs21162 * when lpg_count reaches (npgs + 1). 10271498Sbs21162 * If we are not the first constituent 10281498Sbs21162 * page, restart at the first one. 10291498Sbs21162 */ 10300Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 10310Sstevel@tonic-gate if (!IS_P2ALIGNED(an_idx, npgs)) { 10320Sstevel@tonic-gate an_idx = P2ALIGN(an_idx, npgs); 10330Sstevel@tonic-gate page_unlock(pp); 10340Sstevel@tonic-gate continue; 10350Sstevel@tonic-gate } 10360Sstevel@tonic-gate } 10371498Sbs21162 if (++lpg_cnt > npgs) 10380Sstevel@tonic-gate lpg_cnt = 0; 10390Sstevel@tonic-gate 10400Sstevel@tonic-gate /* 10410Sstevel@tonic-gate * availrmem is decremented only 10420Sstevel@tonic-gate * for unlocked pages 10430Sstevel@tonic-gate */ 10440Sstevel@tonic-gate if (sptd->spt_ppa_lckcnt[an_idx] == 0) 10450Sstevel@tonic-gate claim_availrmem++; 10460Sstevel@tonic-gate pplist[an_idx] = pp; 10470Sstevel@tonic-gate } 10480Sstevel@tonic-gate an_idx++; 10490Sstevel@tonic-gate } 10500Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 10510Sstevel@tonic-gate 10520Sstevel@tonic-gate mutex_enter(&freemem_lock); 10530Sstevel@tonic-gate if (availrmem < tune.t_minarmem + claim_availrmem) { 10540Sstevel@tonic-gate mutex_exit(&freemem_lock); 10550Sstevel@tonic-gate ret = FC_MAKE_ERR(ENOMEM); 10560Sstevel@tonic-gate claim_availrmem = 0; 10570Sstevel@tonic-gate goto insert_fail; 10580Sstevel@tonic-gate } else { 10590Sstevel@tonic-gate availrmem -= claim_availrmem; 10600Sstevel@tonic-gate } 10610Sstevel@tonic-gate mutex_exit(&freemem_lock); 10620Sstevel@tonic-gate 10630Sstevel@tonic-gate sptd->spt_ppa = pl; 10640Sstevel@tonic-gate } else { 10650Sstevel@tonic-gate /* 10660Sstevel@tonic-gate * We already have a valid ppa[]. 10670Sstevel@tonic-gate */ 10680Sstevel@tonic-gate pl = sptd->spt_ppa; 10690Sstevel@tonic-gate } 10700Sstevel@tonic-gate 10710Sstevel@tonic-gate ASSERT(pl != NULL); 10720Sstevel@tonic-gate 10730Sstevel@tonic-gate ret = seg_pinsert(seg, seg->s_base, sptd->spt_amp->size, 10740Sstevel@tonic-gate pl, sptd->spt_prot, SEGP_FORCE_WIRED | SEGP_ASYNC_FLUSH, 10750Sstevel@tonic-gate segspt_reclaim); 10760Sstevel@tonic-gate if (ret == SEGP_FAIL) { 10770Sstevel@tonic-gate /* 10780Sstevel@tonic-gate * seg_pinsert failed. We return 10790Sstevel@tonic-gate * ENOTSUP, so that the as_pagelock() code will 10800Sstevel@tonic-gate * then try the slower F_SOFTLOCK path. 10810Sstevel@tonic-gate */ 1082934Srd117015 if (pl_built) { 1083934Srd117015 /* 1084934Srd117015 * No one else has referenced the ppa[]. 1085934Srd117015 * We created it and we need to destroy it. 1086934Srd117015 */ 1087934Srd117015 sptd->spt_ppa = NULL; 1088934Srd117015 } 10890Sstevel@tonic-gate ret = ENOTSUP; 10900Sstevel@tonic-gate goto insert_fail; 10910Sstevel@tonic-gate } 10920Sstevel@tonic-gate 10930Sstevel@tonic-gate /* 10940Sstevel@tonic-gate * In either case, we increment softlockcnt on the 'real' segment. 10950Sstevel@tonic-gate */ 10960Sstevel@tonic-gate sptd->spt_pcachecnt++; 10970Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), 1); 10980Sstevel@tonic-gate 10990Sstevel@tonic-gate ppa = sptd->spt_ppa; 11000Sstevel@tonic-gate for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 11010Sstevel@tonic-gate if (ppa[an_idx] == NULL) { 11020Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 11030Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, sptd->spt_amp->size, 11040Sstevel@tonic-gate pl, sptd->spt_prot, segspt_reclaim); 11050Sstevel@tonic-gate *ppp = NULL; 11060Sstevel@tonic-gate return (ENOTSUP); 11070Sstevel@tonic-gate } 11080Sstevel@tonic-gate if ((szc = ppa[an_idx]->p_szc) != 0) { 11090Sstevel@tonic-gate npgs = page_get_pagecnt(szc); 11100Sstevel@tonic-gate an_idx = P2ROUNDUP(an_idx + 1, npgs); 11110Sstevel@tonic-gate } else { 11120Sstevel@tonic-gate an_idx++; 11130Sstevel@tonic-gate } 11140Sstevel@tonic-gate } 11150Sstevel@tonic-gate /* 11160Sstevel@tonic-gate * We can now drop the sptd->spt_lock since the ppa[] 11170Sstevel@tonic-gate * exists and he have incremented pacachecnt. 11180Sstevel@tonic-gate */ 11190Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 11200Sstevel@tonic-gate 11210Sstevel@tonic-gate /* 11220Sstevel@tonic-gate * Since we cache the entire segment, we want to 11230Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 11240Sstevel@tonic-gate * to the requested addr, i.e. pg_idx. 11250Sstevel@tonic-gate */ 11260Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[pg_idx]); 11270Sstevel@tonic-gate return (ret); 11280Sstevel@tonic-gate 11290Sstevel@tonic-gate insert_fail: 11300Sstevel@tonic-gate /* 11310Sstevel@tonic-gate * We will only reach this code if we tried and failed. 11320Sstevel@tonic-gate * 11330Sstevel@tonic-gate * And we can drop the lock on the dummy seg, once we've failed 11340Sstevel@tonic-gate * to set up a new ppa[]. 11350Sstevel@tonic-gate */ 11360Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 11370Sstevel@tonic-gate 11380Sstevel@tonic-gate if (pl_built) { 11390Sstevel@tonic-gate mutex_enter(&freemem_lock); 11400Sstevel@tonic-gate availrmem += claim_availrmem; 11410Sstevel@tonic-gate mutex_exit(&freemem_lock); 11420Sstevel@tonic-gate 11430Sstevel@tonic-gate /* 11440Sstevel@tonic-gate * We created pl and we need to destroy it. 11450Sstevel@tonic-gate */ 11460Sstevel@tonic-gate pplist = pl; 11470Sstevel@tonic-gate for (an_idx = 0; an_idx < tot_npages; an_idx++) { 11480Sstevel@tonic-gate if (pplist[an_idx] != NULL) 11490Sstevel@tonic-gate page_unlock(pplist[an_idx]); 11500Sstevel@tonic-gate } 11510Sstevel@tonic-gate kmem_free(pl, sizeof (page_t *) * tot_npages); 11520Sstevel@tonic-gate } 11530Sstevel@tonic-gate 11540Sstevel@tonic-gate if (shmd->shm_softlockcnt <= 0) { 11550Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 11560Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents); 11570Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 11580Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as); 11590Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv); 11600Sstevel@tonic-gate } 11610Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents); 11620Sstevel@tonic-gate } 11630Sstevel@tonic-gate } 11640Sstevel@tonic-gate *ppp = NULL; 11650Sstevel@tonic-gate return (ret); 11660Sstevel@tonic-gate } 11670Sstevel@tonic-gate 11680Sstevel@tonic-gate 11690Sstevel@tonic-gate 11700Sstevel@tonic-gate /* 11710Sstevel@tonic-gate * return locked pages over a given range. 11720Sstevel@tonic-gate * 11730Sstevel@tonic-gate * We will cache the entire ISM segment and save the pplist for the 11740Sstevel@tonic-gate * entire segment in the ppa field of the underlying ISM segment structure. 11750Sstevel@tonic-gate * Later, during a call to segspt_reclaim() we will use this ppa array 11760Sstevel@tonic-gate * to page_unlock() all of the pages and then we will free this ppa list. 11770Sstevel@tonic-gate */ 11780Sstevel@tonic-gate /*ARGSUSED*/ 11790Sstevel@tonic-gate static int 11800Sstevel@tonic-gate segspt_shmpagelock(struct seg *seg, caddr_t addr, size_t len, 11810Sstevel@tonic-gate struct page ***ppp, enum lock_type type, enum seg_rw rw) 11820Sstevel@tonic-gate { 11830Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 11840Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 11850Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 11860Sstevel@tonic-gate pgcnt_t np, page_index, npages; 11870Sstevel@tonic-gate caddr_t a, spt_base; 11880Sstevel@tonic-gate struct page **pplist, **pl, *pp; 11890Sstevel@tonic-gate struct anon_map *amp; 11900Sstevel@tonic-gate ulong_t anon_index; 11910Sstevel@tonic-gate int ret = ENOTSUP; 11920Sstevel@tonic-gate uint_t pl_built = 0; 11930Sstevel@tonic-gate struct anon *ap; 11940Sstevel@tonic-gate struct vnode *vp; 11950Sstevel@tonic-gate u_offset_t off; 11960Sstevel@tonic-gate 11970Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 11980Sstevel@tonic-gate 11990Sstevel@tonic-gate /* 12000Sstevel@tonic-gate * We want to lock/unlock the entire ISM segment. Therefore, 12010Sstevel@tonic-gate * we will be using the underlying sptseg and it's base address 12020Sstevel@tonic-gate * and length for the caching arguments. 12030Sstevel@tonic-gate */ 12040Sstevel@tonic-gate ASSERT(sptseg); 12050Sstevel@tonic-gate ASSERT(sptd); 12060Sstevel@tonic-gate 12070Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 12080Sstevel@tonic-gate return (segspt_dismpagelock(seg, addr, len, ppp, type, rw)); 12090Sstevel@tonic-gate } 12100Sstevel@tonic-gate 12110Sstevel@tonic-gate page_index = seg_page(seg, addr); 12120Sstevel@tonic-gate npages = btopr(len); 12130Sstevel@tonic-gate 12140Sstevel@tonic-gate /* 12150Sstevel@tonic-gate * check if the request is larger than number of pages covered 12160Sstevel@tonic-gate * by amp 12170Sstevel@tonic-gate */ 12180Sstevel@tonic-gate if (page_index + npages > btopr(sptd->spt_amp->size)) { 12190Sstevel@tonic-gate *ppp = NULL; 12200Sstevel@tonic-gate return (ENOTSUP); 12210Sstevel@tonic-gate } 12220Sstevel@tonic-gate 12230Sstevel@tonic-gate if (type == L_PAGEUNLOCK) { 12240Sstevel@tonic-gate 12250Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 12260Sstevel@tonic-gate 12270Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, sptd->spt_amp->size, 12280Sstevel@tonic-gate sptd->spt_ppa, sptd->spt_prot, segspt_reclaim); 12290Sstevel@tonic-gate 12300Sstevel@tonic-gate /* 12310Sstevel@tonic-gate * If someone is blocked while unmapping, we purge 12320Sstevel@tonic-gate * segment page cache and thus reclaim pplist synchronously 12330Sstevel@tonic-gate * without waiting for seg_pasync_thread. This speeds up 12340Sstevel@tonic-gate * unmapping in cases where munmap(2) is called, while 12350Sstevel@tonic-gate * raw async i/o is still in progress or where a thread 12360Sstevel@tonic-gate * exits on data fault in a multithreaded application. 12370Sstevel@tonic-gate */ 12380Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) { 12390Sstevel@tonic-gate segspt_purge(seg); 12400Sstevel@tonic-gate } 12410Sstevel@tonic-gate return (0); 12420Sstevel@tonic-gate } else if (type == L_PAGERECLAIM) { 12430Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 12440Sstevel@tonic-gate 12450Sstevel@tonic-gate (void) segspt_reclaim(seg, seg->s_base, sptd->spt_amp->size, 12460Sstevel@tonic-gate sptd->spt_ppa, sptd->spt_prot); 12470Sstevel@tonic-gate return (0); 12480Sstevel@tonic-gate } 12490Sstevel@tonic-gate 12500Sstevel@tonic-gate /* 12510Sstevel@tonic-gate * First try to find pages in segment page cache, without 12520Sstevel@tonic-gate * holding the segment lock. 12530Sstevel@tonic-gate */ 12540Sstevel@tonic-gate pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size, 12550Sstevel@tonic-gate sptd->spt_prot); 12560Sstevel@tonic-gate if (pplist != NULL) { 12570Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 12580Sstevel@tonic-gate ASSERT(sptd->spt_ppa[page_index]); 12590Sstevel@tonic-gate /* 12600Sstevel@tonic-gate * Since we cache the entire ISM segment, we want to 12610Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 12620Sstevel@tonic-gate * to the requested addr, i.e. page_index. 12630Sstevel@tonic-gate */ 12640Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[page_index]); 12650Sstevel@tonic-gate return (0); 12660Sstevel@tonic-gate } 12670Sstevel@tonic-gate 12680Sstevel@tonic-gate /* The L_PAGELOCK case... */ 12690Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 12700Sstevel@tonic-gate 12710Sstevel@tonic-gate /* 12720Sstevel@tonic-gate * try to find pages in segment page cache 12730Sstevel@tonic-gate */ 12740Sstevel@tonic-gate pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size, 12750Sstevel@tonic-gate sptd->spt_prot); 12760Sstevel@tonic-gate if (pplist != NULL) { 12770Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 12780Sstevel@tonic-gate /* 12790Sstevel@tonic-gate * Since we cache the entire segment, we want to 12800Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 12810Sstevel@tonic-gate * to the requested addr, i.e. page_index. 12820Sstevel@tonic-gate */ 12830Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 12840Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[page_index]); 12850Sstevel@tonic-gate return (0); 12860Sstevel@tonic-gate } 12870Sstevel@tonic-gate 12880Sstevel@tonic-gate if (seg_pinsert_check(seg, sptd->spt_amp->size, SEGP_FORCE_WIRED) == 12890Sstevel@tonic-gate SEGP_FAIL) { 12900Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 12910Sstevel@tonic-gate *ppp = NULL; 12920Sstevel@tonic-gate return (ENOTSUP); 12930Sstevel@tonic-gate } 12940Sstevel@tonic-gate 12950Sstevel@tonic-gate /* 12960Sstevel@tonic-gate * No need to worry about protections because ISM pages 12970Sstevel@tonic-gate * are always rw. 12980Sstevel@tonic-gate */ 12990Sstevel@tonic-gate pl = pplist = NULL; 13000Sstevel@tonic-gate 13010Sstevel@tonic-gate /* 13020Sstevel@tonic-gate * Do we need to build the ppa array? 13030Sstevel@tonic-gate */ 13040Sstevel@tonic-gate if (sptd->spt_ppa == NULL) { 13050Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 13060Sstevel@tonic-gate 13070Sstevel@tonic-gate spt_base = sptseg->s_base; 13080Sstevel@tonic-gate pl_built = 1; 13090Sstevel@tonic-gate 13100Sstevel@tonic-gate /* 13110Sstevel@tonic-gate * availrmem is decremented once during anon_swap_adjust() 13120Sstevel@tonic-gate * and is incremented during the anon_unresv(), which is 13130Sstevel@tonic-gate * called from shm_rm_amp() when the segment is destroyed. 13140Sstevel@tonic-gate */ 13150Sstevel@tonic-gate amp = sptd->spt_amp; 13160Sstevel@tonic-gate ASSERT(amp != NULL); 13170Sstevel@tonic-gate 13180Sstevel@tonic-gate /* pcachecnt is protected by sptd->spt_lock */ 13190Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt == 0); 13200Sstevel@tonic-gate pplist = kmem_zalloc(sizeof (page_t *) 13210Sstevel@tonic-gate * btopr(sptd->spt_amp->size), KM_SLEEP); 13220Sstevel@tonic-gate pl = pplist; 13230Sstevel@tonic-gate 13240Sstevel@tonic-gate anon_index = seg_page(sptseg, spt_base); 13250Sstevel@tonic-gate 13260Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 13270Sstevel@tonic-gate for (a = spt_base; a < (spt_base + sptd->spt_amp->size); 13280Sstevel@tonic-gate a += PAGESIZE, anon_index++, pplist++) { 13290Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index); 13300Sstevel@tonic-gate ASSERT(ap != NULL); 13310Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 13320Sstevel@tonic-gate pp = page_lookup(vp, off, SE_SHARED); 13330Sstevel@tonic-gate ASSERT(pp != NULL); 13340Sstevel@tonic-gate *pplist = pp; 13350Sstevel@tonic-gate } 13360Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 13370Sstevel@tonic-gate 13380Sstevel@tonic-gate if (a < (spt_base + sptd->spt_amp->size)) { 13390Sstevel@tonic-gate ret = ENOTSUP; 13400Sstevel@tonic-gate goto insert_fail; 13410Sstevel@tonic-gate } 13420Sstevel@tonic-gate sptd->spt_ppa = pl; 13430Sstevel@tonic-gate } else { 13440Sstevel@tonic-gate /* 13450Sstevel@tonic-gate * We already have a valid ppa[]. 13460Sstevel@tonic-gate */ 13470Sstevel@tonic-gate pl = sptd->spt_ppa; 13480Sstevel@tonic-gate } 13490Sstevel@tonic-gate 13500Sstevel@tonic-gate ASSERT(pl != NULL); 13510Sstevel@tonic-gate 13520Sstevel@tonic-gate ret = seg_pinsert(seg, seg->s_base, sptd->spt_amp->size, 13530Sstevel@tonic-gate pl, sptd->spt_prot, SEGP_FORCE_WIRED, segspt_reclaim); 13540Sstevel@tonic-gate if (ret == SEGP_FAIL) { 13550Sstevel@tonic-gate /* 13560Sstevel@tonic-gate * seg_pinsert failed. We return 13570Sstevel@tonic-gate * ENOTSUP, so that the as_pagelock() code will 13580Sstevel@tonic-gate * then try the slower F_SOFTLOCK path. 13590Sstevel@tonic-gate */ 13600Sstevel@tonic-gate if (pl_built) { 13610Sstevel@tonic-gate /* 13620Sstevel@tonic-gate * No one else has referenced the ppa[]. 13630Sstevel@tonic-gate * We created it and we need to destroy it. 13640Sstevel@tonic-gate */ 13650Sstevel@tonic-gate sptd->spt_ppa = NULL; 13660Sstevel@tonic-gate } 13670Sstevel@tonic-gate ret = ENOTSUP; 13680Sstevel@tonic-gate goto insert_fail; 13690Sstevel@tonic-gate } 13700Sstevel@tonic-gate 13710Sstevel@tonic-gate /* 13720Sstevel@tonic-gate * In either case, we increment softlockcnt on the 'real' segment. 13730Sstevel@tonic-gate */ 13740Sstevel@tonic-gate sptd->spt_pcachecnt++; 13750Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), 1); 13760Sstevel@tonic-gate 13770Sstevel@tonic-gate /* 13780Sstevel@tonic-gate * We can now drop the sptd->spt_lock since the ppa[] 13790Sstevel@tonic-gate * exists and he have incremented pacachecnt. 13800Sstevel@tonic-gate */ 13810Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 13820Sstevel@tonic-gate 13830Sstevel@tonic-gate /* 13840Sstevel@tonic-gate * Since we cache the entire segment, we want to 13850Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 13860Sstevel@tonic-gate * to the requested addr, i.e. page_index. 13870Sstevel@tonic-gate */ 13880Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[page_index]); 13890Sstevel@tonic-gate return (ret); 13900Sstevel@tonic-gate 13910Sstevel@tonic-gate insert_fail: 13920Sstevel@tonic-gate /* 13930Sstevel@tonic-gate * We will only reach this code if we tried and failed. 13940Sstevel@tonic-gate * 13950Sstevel@tonic-gate * And we can drop the lock on the dummy seg, once we've failed 13960Sstevel@tonic-gate * to set up a new ppa[]. 13970Sstevel@tonic-gate */ 13980Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 13990Sstevel@tonic-gate 14000Sstevel@tonic-gate if (pl_built) { 14010Sstevel@tonic-gate /* 14020Sstevel@tonic-gate * We created pl and we need to destroy it. 14030Sstevel@tonic-gate */ 14040Sstevel@tonic-gate pplist = pl; 14050Sstevel@tonic-gate np = (((uintptr_t)(a - spt_base)) >> PAGESHIFT); 14060Sstevel@tonic-gate while (np) { 14070Sstevel@tonic-gate page_unlock(*pplist); 14080Sstevel@tonic-gate np--; 14090Sstevel@tonic-gate pplist++; 14100Sstevel@tonic-gate } 14110Sstevel@tonic-gate kmem_free(pl, sizeof (page_t *) * 14120Sstevel@tonic-gate btopr(sptd->spt_amp->size)); 14130Sstevel@tonic-gate } 14140Sstevel@tonic-gate if (shmd->shm_softlockcnt <= 0) { 14150Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 14160Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents); 14170Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 14180Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as); 14190Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv); 14200Sstevel@tonic-gate } 14210Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents); 14220Sstevel@tonic-gate } 14230Sstevel@tonic-gate } 14240Sstevel@tonic-gate *ppp = NULL; 14250Sstevel@tonic-gate return (ret); 14260Sstevel@tonic-gate } 14270Sstevel@tonic-gate 14280Sstevel@tonic-gate /* 14290Sstevel@tonic-gate * purge any cached pages in the I/O page cache 14300Sstevel@tonic-gate */ 14310Sstevel@tonic-gate static void 14320Sstevel@tonic-gate segspt_purge(struct seg *seg) 14330Sstevel@tonic-gate { 14340Sstevel@tonic-gate seg_ppurge(seg); 14350Sstevel@tonic-gate } 14360Sstevel@tonic-gate 14370Sstevel@tonic-gate static int 14380Sstevel@tonic-gate segspt_reclaim(struct seg *seg, caddr_t addr, size_t len, struct page **pplist, 14390Sstevel@tonic-gate enum seg_rw rw) 14400Sstevel@tonic-gate { 14410Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 14420Sstevel@tonic-gate struct seg *sptseg; 14430Sstevel@tonic-gate struct spt_data *sptd; 14440Sstevel@tonic-gate pgcnt_t npages, i, free_availrmem = 0; 14450Sstevel@tonic-gate int done = 0; 14460Sstevel@tonic-gate 14470Sstevel@tonic-gate #ifdef lint 14480Sstevel@tonic-gate addr = addr; 14490Sstevel@tonic-gate #endif 14500Sstevel@tonic-gate sptseg = shmd->shm_sptseg; 14510Sstevel@tonic-gate sptd = sptseg->s_data; 14520Sstevel@tonic-gate npages = (len >> PAGESHIFT); 14530Sstevel@tonic-gate ASSERT(npages); 14540Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt != 0); 14550Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 14560Sstevel@tonic-gate ASSERT(npages == btopr(sptd->spt_amp->size)); 14570Sstevel@tonic-gate /* 14580Sstevel@tonic-gate * Acquire the lock on the dummy seg and destroy the 14590Sstevel@tonic-gate * ppa array IF this is the last pcachecnt. 14600Sstevel@tonic-gate */ 14610Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 14620Sstevel@tonic-gate if (--sptd->spt_pcachecnt == 0) { 14630Sstevel@tonic-gate for (i = 0; i < npages; i++) { 14640Sstevel@tonic-gate if (pplist[i] == NULL) { 14650Sstevel@tonic-gate continue; 14660Sstevel@tonic-gate } 14670Sstevel@tonic-gate if (rw == S_WRITE) { 14680Sstevel@tonic-gate hat_setrefmod(pplist[i]); 14690Sstevel@tonic-gate } else { 14700Sstevel@tonic-gate hat_setref(pplist[i]); 14710Sstevel@tonic-gate } 14720Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) && 14732768Ssl108498 (sptd->spt_ppa_lckcnt[i] == 0)) 14740Sstevel@tonic-gate free_availrmem++; 14750Sstevel@tonic-gate page_unlock(pplist[i]); 14760Sstevel@tonic-gate } 14770Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 14780Sstevel@tonic-gate mutex_enter(&freemem_lock); 14790Sstevel@tonic-gate availrmem += free_availrmem; 14800Sstevel@tonic-gate mutex_exit(&freemem_lock); 14810Sstevel@tonic-gate } 14820Sstevel@tonic-gate /* 14830Sstevel@tonic-gate * Since we want to cach/uncache the entire ISM segment, 14840Sstevel@tonic-gate * we will track the pplist in a segspt specific field 14850Sstevel@tonic-gate * ppa, that is initialized at the time we add an entry to 14860Sstevel@tonic-gate * the cache. 14870Sstevel@tonic-gate */ 14880Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt == 0); 14890Sstevel@tonic-gate kmem_free(pplist, sizeof (page_t *) * npages); 14900Sstevel@tonic-gate sptd->spt_ppa = NULL; 14910Sstevel@tonic-gate sptd->spt_flags &= ~DISM_PPA_CHANGED; 14920Sstevel@tonic-gate done = 1; 14930Sstevel@tonic-gate } 14940Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 14950Sstevel@tonic-gate /* 14960Sstevel@tonic-gate * Now decrement softlockcnt. 14970Sstevel@tonic-gate */ 14980Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -1); 14990Sstevel@tonic-gate 15000Sstevel@tonic-gate if (shmd->shm_softlockcnt <= 0) { 15010Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 15020Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents); 15030Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 15040Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as); 15050Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv); 15060Sstevel@tonic-gate } 15070Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents); 15080Sstevel@tonic-gate } 15090Sstevel@tonic-gate } 15100Sstevel@tonic-gate return (done); 15110Sstevel@tonic-gate } 15120Sstevel@tonic-gate 15130Sstevel@tonic-gate /* 15140Sstevel@tonic-gate * Do a F_SOFTUNLOCK call over the range requested. 15150Sstevel@tonic-gate * The range must have already been F_SOFTLOCK'ed. 15160Sstevel@tonic-gate * 15170Sstevel@tonic-gate * The calls to acquire and release the anon map lock mutex were 15180Sstevel@tonic-gate * removed in order to avoid a deadly embrace during a DR 15190Sstevel@tonic-gate * memory delete operation. (Eg. DR blocks while waiting for a 15200Sstevel@tonic-gate * exclusive lock on a page that is being used for kaio; the 15210Sstevel@tonic-gate * thread that will complete the kaio and call segspt_softunlock 15220Sstevel@tonic-gate * blocks on the anon map lock; another thread holding the anon 15230Sstevel@tonic-gate * map lock blocks on another page lock via the segspt_shmfault 15240Sstevel@tonic-gate * -> page_lookup -> page_lookup_create -> page_lock_es code flow.) 15250Sstevel@tonic-gate * 15260Sstevel@tonic-gate * The appropriateness of the removal is based upon the following: 15270Sstevel@tonic-gate * 1. If we are holding a segment's reader lock and the page is held 15280Sstevel@tonic-gate * shared, then the corresponding element in anonmap which points to 15290Sstevel@tonic-gate * anon struct cannot change and there is no need to acquire the 15300Sstevel@tonic-gate * anonymous map lock. 15310Sstevel@tonic-gate * 2. Threads in segspt_softunlock have a reader lock on the segment 15320Sstevel@tonic-gate * and already have the shared page lock, so we are guaranteed that 15330Sstevel@tonic-gate * the anon map slot cannot change and therefore can call anon_get_ptr() 15340Sstevel@tonic-gate * without grabbing the anonymous map lock. 15350Sstevel@tonic-gate * 3. Threads that softlock a shared page break copy-on-write, even if 15360Sstevel@tonic-gate * its a read. Thus cow faults can be ignored with respect to soft 15370Sstevel@tonic-gate * unlocking, since the breaking of cow means that the anon slot(s) will 15380Sstevel@tonic-gate * not be shared. 15390Sstevel@tonic-gate */ 15400Sstevel@tonic-gate static void 15410Sstevel@tonic-gate segspt_softunlock(struct seg *seg, caddr_t sptseg_addr, 15420Sstevel@tonic-gate size_t len, enum seg_rw rw) 15430Sstevel@tonic-gate { 15440Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 15450Sstevel@tonic-gate struct seg *sptseg; 15460Sstevel@tonic-gate struct spt_data *sptd; 15470Sstevel@tonic-gate page_t *pp; 15480Sstevel@tonic-gate caddr_t adr; 15490Sstevel@tonic-gate struct vnode *vp; 15500Sstevel@tonic-gate u_offset_t offset; 15510Sstevel@tonic-gate ulong_t anon_index; 15520Sstevel@tonic-gate struct anon_map *amp; /* XXX - for locknest */ 15530Sstevel@tonic-gate struct anon *ap = NULL; 15540Sstevel@tonic-gate pgcnt_t npages; 15550Sstevel@tonic-gate 15560Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 15570Sstevel@tonic-gate 15580Sstevel@tonic-gate sptseg = shmd->shm_sptseg; 15590Sstevel@tonic-gate sptd = sptseg->s_data; 15600Sstevel@tonic-gate 15610Sstevel@tonic-gate /* 15620Sstevel@tonic-gate * Some platforms assume that ISM mappings are HAT_LOAD_LOCK 15630Sstevel@tonic-gate * and therefore their pages are SE_SHARED locked 15640Sstevel@tonic-gate * for the entire life of the segment. 15650Sstevel@tonic-gate */ 15660Sstevel@tonic-gate if ((!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) && 15670Sstevel@tonic-gate ((sptd->spt_flags & SHM_PAGEABLE) == 0)) { 15680Sstevel@tonic-gate goto softlock_decrement; 15690Sstevel@tonic-gate } 15700Sstevel@tonic-gate 15710Sstevel@tonic-gate /* 15720Sstevel@tonic-gate * Any thread is free to do a page_find and 15730Sstevel@tonic-gate * page_unlock() on the pages within this seg. 15740Sstevel@tonic-gate * 15750Sstevel@tonic-gate * We are already holding the as->a_lock on the user's 15760Sstevel@tonic-gate * real segment, but we need to hold the a_lock on the 15770Sstevel@tonic-gate * underlying dummy as. This is mostly to satisfy the 15780Sstevel@tonic-gate * underlying HAT layer. 15790Sstevel@tonic-gate */ 15800Sstevel@tonic-gate AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 15810Sstevel@tonic-gate hat_unlock(sptseg->s_as->a_hat, sptseg_addr, len); 15820Sstevel@tonic-gate AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 15830Sstevel@tonic-gate 15840Sstevel@tonic-gate amp = sptd->spt_amp; 15850Sstevel@tonic-gate ASSERT(amp != NULL); 15860Sstevel@tonic-gate anon_index = seg_page(sptseg, sptseg_addr); 15870Sstevel@tonic-gate 15880Sstevel@tonic-gate for (adr = sptseg_addr; adr < sptseg_addr + len; adr += PAGESIZE) { 15890Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index++); 15900Sstevel@tonic-gate ASSERT(ap != NULL); 15910Sstevel@tonic-gate swap_xlate(ap, &vp, &offset); 15920Sstevel@tonic-gate 15930Sstevel@tonic-gate /* 15940Sstevel@tonic-gate * Use page_find() instead of page_lookup() to 15950Sstevel@tonic-gate * find the page since we know that it has a 15960Sstevel@tonic-gate * "shared" lock. 15970Sstevel@tonic-gate */ 15980Sstevel@tonic-gate pp = page_find(vp, offset); 15990Sstevel@tonic-gate ASSERT(ap == anon_get_ptr(amp->ahp, anon_index - 1)); 16000Sstevel@tonic-gate if (pp == NULL) { 16010Sstevel@tonic-gate panic("segspt_softunlock: " 16020Sstevel@tonic-gate "addr %p, ap %p, vp %p, off %llx", 16030Sstevel@tonic-gate (void *)adr, (void *)ap, (void *)vp, offset); 16040Sstevel@tonic-gate /*NOTREACHED*/ 16050Sstevel@tonic-gate } 16060Sstevel@tonic-gate 16070Sstevel@tonic-gate if (rw == S_WRITE) { 16080Sstevel@tonic-gate hat_setrefmod(pp); 16090Sstevel@tonic-gate } else if (rw != S_OTHER) { 16100Sstevel@tonic-gate hat_setref(pp); 16110Sstevel@tonic-gate } 16120Sstevel@tonic-gate page_unlock(pp); 16130Sstevel@tonic-gate } 16140Sstevel@tonic-gate 16150Sstevel@tonic-gate softlock_decrement: 16160Sstevel@tonic-gate npages = btopr(len); 16170Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -npages); 16180Sstevel@tonic-gate if (shmd->shm_softlockcnt == 0) { 16190Sstevel@tonic-gate /* 16200Sstevel@tonic-gate * All SOFTLOCKS are gone. Wakeup any waiting 16210Sstevel@tonic-gate * unmappers so they can try again to unmap. 16220Sstevel@tonic-gate * Check for waiters first without the mutex 16230Sstevel@tonic-gate * held so we don't always grab the mutex on 16240Sstevel@tonic-gate * softunlocks. 16250Sstevel@tonic-gate */ 16260Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 16270Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents); 16280Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 16290Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as); 16300Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv); 16310Sstevel@tonic-gate } 16320Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents); 16330Sstevel@tonic-gate } 16340Sstevel@tonic-gate } 16350Sstevel@tonic-gate } 16360Sstevel@tonic-gate 16370Sstevel@tonic-gate int 16380Sstevel@tonic-gate segspt_shmattach(struct seg *seg, caddr_t *argsp) 16390Sstevel@tonic-gate { 16400Sstevel@tonic-gate struct shm_data *shmd_arg = (struct shm_data *)argsp; 16410Sstevel@tonic-gate struct shm_data *shmd; 16420Sstevel@tonic-gate struct anon_map *shm_amp = shmd_arg->shm_amp; 16430Sstevel@tonic-gate struct spt_data *sptd; 16440Sstevel@tonic-gate int error = 0; 16450Sstevel@tonic-gate 16460Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 16470Sstevel@tonic-gate 16480Sstevel@tonic-gate shmd = kmem_zalloc((sizeof (*shmd)), KM_NOSLEEP); 16490Sstevel@tonic-gate if (shmd == NULL) 16500Sstevel@tonic-gate return (ENOMEM); 16510Sstevel@tonic-gate 16520Sstevel@tonic-gate shmd->shm_sptas = shmd_arg->shm_sptas; 16530Sstevel@tonic-gate shmd->shm_amp = shm_amp; 16540Sstevel@tonic-gate shmd->shm_sptseg = shmd_arg->shm_sptseg; 16550Sstevel@tonic-gate 16560Sstevel@tonic-gate (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, shm_amp, 0, 16570Sstevel@tonic-gate NULL, 0, seg->s_size); 16580Sstevel@tonic-gate 16590Sstevel@tonic-gate seg->s_data = (void *)shmd; 16600Sstevel@tonic-gate seg->s_ops = &segspt_shmops; 16610Sstevel@tonic-gate seg->s_szc = shmd->shm_sptseg->s_szc; 16620Sstevel@tonic-gate sptd = shmd->shm_sptseg->s_data; 16630Sstevel@tonic-gate 16640Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 16650Sstevel@tonic-gate if ((shmd->shm_vpage = kmem_zalloc(btopr(shm_amp->size), 16660Sstevel@tonic-gate KM_NOSLEEP)) == NULL) { 16670Sstevel@tonic-gate seg->s_data = (void *)NULL; 16680Sstevel@tonic-gate kmem_free(shmd, (sizeof (*shmd))); 16690Sstevel@tonic-gate return (ENOMEM); 16700Sstevel@tonic-gate } 16710Sstevel@tonic-gate shmd->shm_lckpgs = 0; 16720Sstevel@tonic-gate if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 16730Sstevel@tonic-gate if ((error = hat_share(seg->s_as->a_hat, seg->s_base, 16740Sstevel@tonic-gate shmd_arg->shm_sptas->a_hat, SEGSPTADDR, 16750Sstevel@tonic-gate seg->s_size, seg->s_szc)) != 0) { 16760Sstevel@tonic-gate kmem_free(shmd->shm_vpage, 16770Sstevel@tonic-gate btopr(shm_amp->size)); 16780Sstevel@tonic-gate } 16790Sstevel@tonic-gate } 16800Sstevel@tonic-gate } else { 16810Sstevel@tonic-gate error = hat_share(seg->s_as->a_hat, seg->s_base, 16820Sstevel@tonic-gate shmd_arg->shm_sptas->a_hat, SEGSPTADDR, 16830Sstevel@tonic-gate seg->s_size, seg->s_szc); 16840Sstevel@tonic-gate } 16850Sstevel@tonic-gate if (error) { 16860Sstevel@tonic-gate seg->s_szc = 0; 16870Sstevel@tonic-gate seg->s_data = (void *)NULL; 16880Sstevel@tonic-gate kmem_free(shmd, (sizeof (*shmd))); 16890Sstevel@tonic-gate } else { 16900Sstevel@tonic-gate ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER); 16910Sstevel@tonic-gate shm_amp->refcnt++; 16920Sstevel@tonic-gate ANON_LOCK_EXIT(&shm_amp->a_rwlock); 16930Sstevel@tonic-gate } 16940Sstevel@tonic-gate return (error); 16950Sstevel@tonic-gate } 16960Sstevel@tonic-gate 16970Sstevel@tonic-gate int 16980Sstevel@tonic-gate segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize) 16990Sstevel@tonic-gate { 17000Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 17010Sstevel@tonic-gate int reclaim = 1; 17020Sstevel@tonic-gate 17030Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 17040Sstevel@tonic-gate retry: 17050Sstevel@tonic-gate if (shmd->shm_softlockcnt > 0) { 17060Sstevel@tonic-gate if (reclaim == 1) { 17070Sstevel@tonic-gate segspt_purge(seg); 17080Sstevel@tonic-gate reclaim = 0; 17090Sstevel@tonic-gate goto retry; 17100Sstevel@tonic-gate } 17110Sstevel@tonic-gate return (EAGAIN); 17120Sstevel@tonic-gate } 17130Sstevel@tonic-gate 17140Sstevel@tonic-gate if (ssize != seg->s_size) { 17150Sstevel@tonic-gate #ifdef DEBUG 17160Sstevel@tonic-gate cmn_err(CE_WARN, "Incompatible ssize %lx s_size %lx\n", 17170Sstevel@tonic-gate ssize, seg->s_size); 17180Sstevel@tonic-gate #endif 17190Sstevel@tonic-gate return (EINVAL); 17200Sstevel@tonic-gate } 17210Sstevel@tonic-gate 17220Sstevel@tonic-gate (void) segspt_shmlockop(seg, raddr, shmd->shm_amp->size, 0, MC_UNLOCK, 17230Sstevel@tonic-gate NULL, 0); 17240Sstevel@tonic-gate hat_unshare(seg->s_as->a_hat, raddr, ssize, seg->s_szc); 17250Sstevel@tonic-gate 17260Sstevel@tonic-gate seg_free(seg); 17270Sstevel@tonic-gate 17280Sstevel@tonic-gate return (0); 17290Sstevel@tonic-gate } 17300Sstevel@tonic-gate 17310Sstevel@tonic-gate void 17320Sstevel@tonic-gate segspt_shmfree(struct seg *seg) 17330Sstevel@tonic-gate { 17340Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 17350Sstevel@tonic-gate struct anon_map *shm_amp = shmd->shm_amp; 17360Sstevel@tonic-gate 17370Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 17380Sstevel@tonic-gate 17390Sstevel@tonic-gate (void) segspt_shmlockop(seg, seg->s_base, shm_amp->size, 0, 17400Sstevel@tonic-gate MC_UNLOCK, NULL, 0); 17410Sstevel@tonic-gate 17420Sstevel@tonic-gate /* 17430Sstevel@tonic-gate * Need to increment refcnt when attaching 17440Sstevel@tonic-gate * and decrement when detaching because of dup(). 17450Sstevel@tonic-gate */ 17460Sstevel@tonic-gate ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER); 17470Sstevel@tonic-gate shm_amp->refcnt--; 17480Sstevel@tonic-gate ANON_LOCK_EXIT(&shm_amp->a_rwlock); 17490Sstevel@tonic-gate 17500Sstevel@tonic-gate if (shmd->shm_vpage) { /* only for DISM */ 17510Sstevel@tonic-gate kmem_free(shmd->shm_vpage, btopr(shm_amp->size)); 17520Sstevel@tonic-gate shmd->shm_vpage = NULL; 17530Sstevel@tonic-gate } 17540Sstevel@tonic-gate kmem_free(shmd, sizeof (*shmd)); 17550Sstevel@tonic-gate } 17560Sstevel@tonic-gate 17570Sstevel@tonic-gate /*ARGSUSED*/ 17580Sstevel@tonic-gate int 17590Sstevel@tonic-gate segspt_shmsetprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) 17600Sstevel@tonic-gate { 17610Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 17620Sstevel@tonic-gate 17630Sstevel@tonic-gate /* 17640Sstevel@tonic-gate * Shared page table is more than shared mapping. 17650Sstevel@tonic-gate * Individual process sharing page tables can't change prot 17660Sstevel@tonic-gate * because there is only one set of page tables. 17670Sstevel@tonic-gate * This will be allowed after private page table is 17680Sstevel@tonic-gate * supported. 17690Sstevel@tonic-gate */ 17700Sstevel@tonic-gate /* need to return correct status error? */ 17710Sstevel@tonic-gate return (0); 17720Sstevel@tonic-gate } 17730Sstevel@tonic-gate 17740Sstevel@tonic-gate 17750Sstevel@tonic-gate faultcode_t 17760Sstevel@tonic-gate segspt_dismfault(struct hat *hat, struct seg *seg, caddr_t addr, 17770Sstevel@tonic-gate size_t len, enum fault_type type, enum seg_rw rw) 17780Sstevel@tonic-gate { 17790Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 17800Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 17810Sstevel@tonic-gate struct as *curspt = shmd->shm_sptas; 17820Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 17830Sstevel@tonic-gate pgcnt_t npages; 17842414Saguzovsk size_t size; 17850Sstevel@tonic-gate caddr_t segspt_addr, shm_addr; 17860Sstevel@tonic-gate page_t **ppa; 17870Sstevel@tonic-gate int i; 17880Sstevel@tonic-gate ulong_t an_idx = 0; 17890Sstevel@tonic-gate int err = 0; 1790721Smec int dyn_ism_unmap = hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0); 17912414Saguzovsk size_t pgsz; 17922414Saguzovsk pgcnt_t pgcnt; 17932414Saguzovsk caddr_t a; 17942414Saguzovsk pgcnt_t pidx; 17950Sstevel@tonic-gate 17960Sstevel@tonic-gate #ifdef lint 17970Sstevel@tonic-gate hat = hat; 17980Sstevel@tonic-gate #endif 17990Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 18000Sstevel@tonic-gate 18010Sstevel@tonic-gate /* 18020Sstevel@tonic-gate * Because of the way spt is implemented 18030Sstevel@tonic-gate * the realsize of the segment does not have to be 18040Sstevel@tonic-gate * equal to the segment size itself. The segment size is 18050Sstevel@tonic-gate * often in multiples of a page size larger than PAGESIZE. 18060Sstevel@tonic-gate * The realsize is rounded up to the nearest PAGESIZE 18070Sstevel@tonic-gate * based on what the user requested. This is a bit of 18080Sstevel@tonic-gate * ungliness that is historical but not easily fixed 18090Sstevel@tonic-gate * without re-designing the higher levels of ISM. 18100Sstevel@tonic-gate */ 18110Sstevel@tonic-gate ASSERT(addr >= seg->s_base); 18120Sstevel@tonic-gate if (((addr + len) - seg->s_base) > sptd->spt_realsize) 18130Sstevel@tonic-gate return (FC_NOMAP); 18140Sstevel@tonic-gate /* 18150Sstevel@tonic-gate * For all of the following cases except F_PROT, we need to 18160Sstevel@tonic-gate * make any necessary adjustments to addr and len 18170Sstevel@tonic-gate * and get all of the necessary page_t's into an array called ppa[]. 18180Sstevel@tonic-gate * 18190Sstevel@tonic-gate * The code in shmat() forces base addr and len of ISM segment 18200Sstevel@tonic-gate * to be aligned to largest page size supported. Therefore, 18210Sstevel@tonic-gate * we are able to handle F_SOFTLOCK and F_INVAL calls in "large 18220Sstevel@tonic-gate * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK 18230Sstevel@tonic-gate * in large pagesize chunks, or else we will screw up the HAT 18240Sstevel@tonic-gate * layer by calling hat_memload_array() with differing page sizes 18250Sstevel@tonic-gate * over a given virtual range. 18260Sstevel@tonic-gate */ 18272414Saguzovsk pgsz = page_get_pagesize(sptseg->s_szc); 18282414Saguzovsk pgcnt = page_get_pagecnt(sptseg->s_szc); 18292414Saguzovsk shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz); 18302414Saguzovsk size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz); 18310Sstevel@tonic-gate npages = btopr(size); 18320Sstevel@tonic-gate 18330Sstevel@tonic-gate /* 18340Sstevel@tonic-gate * Now we need to convert from addr in segshm to addr in segspt. 18350Sstevel@tonic-gate */ 18360Sstevel@tonic-gate an_idx = seg_page(seg, shm_addr); 18370Sstevel@tonic-gate segspt_addr = sptseg->s_base + ptob(an_idx); 18380Sstevel@tonic-gate 18390Sstevel@tonic-gate ASSERT((segspt_addr + ptob(npages)) <= 18400Sstevel@tonic-gate (sptseg->s_base + sptd->spt_realsize)); 18410Sstevel@tonic-gate ASSERT(segspt_addr < (sptseg->s_base + sptseg->s_size)); 18420Sstevel@tonic-gate 18430Sstevel@tonic-gate switch (type) { 18440Sstevel@tonic-gate 18450Sstevel@tonic-gate case F_SOFTLOCK: 18460Sstevel@tonic-gate 18470Sstevel@tonic-gate mutex_enter(&freemem_lock); 18480Sstevel@tonic-gate if (availrmem < tune.t_minarmem + npages) { 18490Sstevel@tonic-gate mutex_exit(&freemem_lock); 18500Sstevel@tonic-gate return (FC_MAKE_ERR(ENOMEM)); 18510Sstevel@tonic-gate } else { 18520Sstevel@tonic-gate availrmem -= npages; 18530Sstevel@tonic-gate } 18540Sstevel@tonic-gate mutex_exit(&freemem_lock); 18550Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages); 18560Sstevel@tonic-gate /* 18570Sstevel@tonic-gate * Fall through to the F_INVAL case to load up the hat layer 18580Sstevel@tonic-gate * entries with the HAT_LOAD_LOCK flag. 18590Sstevel@tonic-gate */ 18600Sstevel@tonic-gate /* FALLTHRU */ 18610Sstevel@tonic-gate case F_INVAL: 18620Sstevel@tonic-gate 18630Sstevel@tonic-gate if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC)) 18640Sstevel@tonic-gate return (FC_NOMAP); 18650Sstevel@tonic-gate 18660Sstevel@tonic-gate ppa = kmem_zalloc(npages * sizeof (page_t *), KM_SLEEP); 18670Sstevel@tonic-gate 18680Sstevel@tonic-gate err = spt_anon_getpages(sptseg, segspt_addr, size, ppa); 18690Sstevel@tonic-gate if (err != 0) { 18700Sstevel@tonic-gate if (type == F_SOFTLOCK) { 18710Sstevel@tonic-gate mutex_enter(&freemem_lock); 18720Sstevel@tonic-gate availrmem += npages; 18730Sstevel@tonic-gate mutex_exit(&freemem_lock); 18740Sstevel@tonic-gate atomic_add_long((ulong_t *)( 18750Sstevel@tonic-gate &(shmd->shm_softlockcnt)), -npages); 18760Sstevel@tonic-gate } 18770Sstevel@tonic-gate goto dism_err; 18780Sstevel@tonic-gate } 18790Sstevel@tonic-gate AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 18802414Saguzovsk a = segspt_addr; 18812414Saguzovsk pidx = 0; 18820Sstevel@tonic-gate if (type == F_SOFTLOCK) { 18830Sstevel@tonic-gate 18840Sstevel@tonic-gate /* 18850Sstevel@tonic-gate * Load up the translation keeping it 18860Sstevel@tonic-gate * locked and don't unlock the page. 18870Sstevel@tonic-gate */ 18882414Saguzovsk for (; pidx < npages; a += pgsz, pidx += pgcnt) { 18892414Saguzovsk hat_memload_array(sptseg->s_as->a_hat, 18902414Saguzovsk a, pgsz, &ppa[pidx], sptd->spt_prot, 18912414Saguzovsk HAT_LOAD_LOCK | HAT_LOAD_SHARE); 18922414Saguzovsk } 18930Sstevel@tonic-gate } else { 18940Sstevel@tonic-gate if (hat == seg->s_as->a_hat) { 18950Sstevel@tonic-gate 18960Sstevel@tonic-gate /* 18970Sstevel@tonic-gate * Migrate pages marked for migration 18980Sstevel@tonic-gate */ 18990Sstevel@tonic-gate if (lgrp_optimizations()) 19000Sstevel@tonic-gate page_migrate(seg, shm_addr, ppa, 19010Sstevel@tonic-gate npages); 19020Sstevel@tonic-gate 19030Sstevel@tonic-gate /* CPU HAT */ 19042414Saguzovsk for (; pidx < npages; 19052414Saguzovsk a += pgsz, pidx += pgcnt) { 19062414Saguzovsk hat_memload_array(sptseg->s_as->a_hat, 19072414Saguzovsk a, pgsz, &ppa[pidx], 19082414Saguzovsk sptd->spt_prot, 19092414Saguzovsk HAT_LOAD_SHARE); 19102414Saguzovsk } 19110Sstevel@tonic-gate } else { 19120Sstevel@tonic-gate /* XHAT. Pass real address */ 19130Sstevel@tonic-gate hat_memload_array(hat, shm_addr, 19140Sstevel@tonic-gate size, ppa, sptd->spt_prot, HAT_LOAD_SHARE); 19150Sstevel@tonic-gate } 19160Sstevel@tonic-gate 19170Sstevel@tonic-gate /* 19180Sstevel@tonic-gate * And now drop the SE_SHARED lock(s). 19190Sstevel@tonic-gate */ 1920721Smec if (dyn_ism_unmap) { 1921721Smec for (i = 0; i < npages; i++) { 1922721Smec page_unlock(ppa[i]); 1923721Smec } 1924721Smec } 19250Sstevel@tonic-gate } 19260Sstevel@tonic-gate 1927721Smec if (!dyn_ism_unmap) { 19280Sstevel@tonic-gate if (hat_share(seg->s_as->a_hat, shm_addr, 19290Sstevel@tonic-gate curspt->a_hat, segspt_addr, ptob(npages), 19300Sstevel@tonic-gate seg->s_szc) != 0) { 19310Sstevel@tonic-gate panic("hat_share err in DISM fault"); 19320Sstevel@tonic-gate /* NOTREACHED */ 19330Sstevel@tonic-gate } 1934721Smec if (type == F_INVAL) { 1935721Smec for (i = 0; i < npages; i++) { 1936721Smec page_unlock(ppa[i]); 1937721Smec } 1938721Smec } 19390Sstevel@tonic-gate } 19400Sstevel@tonic-gate AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 19410Sstevel@tonic-gate dism_err: 19420Sstevel@tonic-gate kmem_free(ppa, npages * sizeof (page_t *)); 19430Sstevel@tonic-gate return (err); 19440Sstevel@tonic-gate 19450Sstevel@tonic-gate case F_SOFTUNLOCK: 19460Sstevel@tonic-gate 19470Sstevel@tonic-gate mutex_enter(&freemem_lock); 19480Sstevel@tonic-gate availrmem += npages; 19490Sstevel@tonic-gate mutex_exit(&freemem_lock); 19500Sstevel@tonic-gate 19510Sstevel@tonic-gate /* 19520Sstevel@tonic-gate * This is a bit ugly, we pass in the real seg pointer, 19530Sstevel@tonic-gate * but the segspt_addr is the virtual address within the 19540Sstevel@tonic-gate * dummy seg. 19550Sstevel@tonic-gate */ 19560Sstevel@tonic-gate segspt_softunlock(seg, segspt_addr, size, rw); 19570Sstevel@tonic-gate return (0); 19580Sstevel@tonic-gate 19590Sstevel@tonic-gate case F_PROT: 19600Sstevel@tonic-gate 19610Sstevel@tonic-gate /* 19620Sstevel@tonic-gate * This takes care of the unusual case where a user 19630Sstevel@tonic-gate * allocates a stack in shared memory and a register 19640Sstevel@tonic-gate * window overflow is written to that stack page before 19650Sstevel@tonic-gate * it is otherwise modified. 19660Sstevel@tonic-gate * 19670Sstevel@tonic-gate * We can get away with this because ISM segments are 19680Sstevel@tonic-gate * always rw. Other than this unusual case, there 19690Sstevel@tonic-gate * should be no instances of protection violations. 19700Sstevel@tonic-gate */ 19710Sstevel@tonic-gate return (0); 19720Sstevel@tonic-gate 19730Sstevel@tonic-gate default: 19740Sstevel@tonic-gate #ifdef DEBUG 19750Sstevel@tonic-gate panic("segspt_dismfault default type?"); 19760Sstevel@tonic-gate #else 19770Sstevel@tonic-gate return (FC_NOMAP); 19780Sstevel@tonic-gate #endif 19790Sstevel@tonic-gate } 19800Sstevel@tonic-gate } 19810Sstevel@tonic-gate 19820Sstevel@tonic-gate 19830Sstevel@tonic-gate faultcode_t 19840Sstevel@tonic-gate segspt_shmfault(struct hat *hat, struct seg *seg, caddr_t addr, 19850Sstevel@tonic-gate size_t len, enum fault_type type, enum seg_rw rw) 19860Sstevel@tonic-gate { 19870Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 19880Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 19890Sstevel@tonic-gate struct as *curspt = shmd->shm_sptas; 19900Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 19910Sstevel@tonic-gate pgcnt_t npages; 19922414Saguzovsk size_t size; 19930Sstevel@tonic-gate caddr_t sptseg_addr, shm_addr; 19940Sstevel@tonic-gate page_t *pp, **ppa; 19950Sstevel@tonic-gate int i; 19960Sstevel@tonic-gate u_offset_t offset; 19970Sstevel@tonic-gate ulong_t anon_index = 0; 19980Sstevel@tonic-gate struct vnode *vp; 19990Sstevel@tonic-gate struct anon_map *amp; /* XXX - for locknest */ 20000Sstevel@tonic-gate struct anon *ap = NULL; 20010Sstevel@tonic-gate anon_sync_obj_t cookie; 20022414Saguzovsk size_t pgsz; 20032414Saguzovsk pgcnt_t pgcnt; 20042414Saguzovsk caddr_t a; 20052414Saguzovsk pgcnt_t pidx; 20062414Saguzovsk size_t sz; 20070Sstevel@tonic-gate 20080Sstevel@tonic-gate #ifdef lint 20090Sstevel@tonic-gate hat = hat; 20100Sstevel@tonic-gate #endif 20110Sstevel@tonic-gate 20120Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 20130Sstevel@tonic-gate 20140Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 20150Sstevel@tonic-gate return (segspt_dismfault(hat, seg, addr, len, type, rw)); 20160Sstevel@tonic-gate } 20170Sstevel@tonic-gate 20180Sstevel@tonic-gate /* 20190Sstevel@tonic-gate * Because of the way spt is implemented 20200Sstevel@tonic-gate * the realsize of the segment does not have to be 20210Sstevel@tonic-gate * equal to the segment size itself. The segment size is 20220Sstevel@tonic-gate * often in multiples of a page size larger than PAGESIZE. 20230Sstevel@tonic-gate * The realsize is rounded up to the nearest PAGESIZE 20240Sstevel@tonic-gate * based on what the user requested. This is a bit of 20250Sstevel@tonic-gate * ungliness that is historical but not easily fixed 20260Sstevel@tonic-gate * without re-designing the higher levels of ISM. 20270Sstevel@tonic-gate */ 20280Sstevel@tonic-gate ASSERT(addr >= seg->s_base); 20290Sstevel@tonic-gate if (((addr + len) - seg->s_base) > sptd->spt_realsize) 20300Sstevel@tonic-gate return (FC_NOMAP); 20310Sstevel@tonic-gate /* 20320Sstevel@tonic-gate * For all of the following cases except F_PROT, we need to 20330Sstevel@tonic-gate * make any necessary adjustments to addr and len 20340Sstevel@tonic-gate * and get all of the necessary page_t's into an array called ppa[]. 20350Sstevel@tonic-gate * 20360Sstevel@tonic-gate * The code in shmat() forces base addr and len of ISM segment 20370Sstevel@tonic-gate * to be aligned to largest page size supported. Therefore, 20380Sstevel@tonic-gate * we are able to handle F_SOFTLOCK and F_INVAL calls in "large 20390Sstevel@tonic-gate * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK 20400Sstevel@tonic-gate * in large pagesize chunks, or else we will screw up the HAT 20410Sstevel@tonic-gate * layer by calling hat_memload_array() with differing page sizes 20420Sstevel@tonic-gate * over a given virtual range. 20430Sstevel@tonic-gate */ 20442414Saguzovsk pgsz = page_get_pagesize(sptseg->s_szc); 20452414Saguzovsk pgcnt = page_get_pagecnt(sptseg->s_szc); 20462414Saguzovsk shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz); 20472414Saguzovsk size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz); 20480Sstevel@tonic-gate npages = btopr(size); 20490Sstevel@tonic-gate 20500Sstevel@tonic-gate /* 20510Sstevel@tonic-gate * Now we need to convert from addr in segshm to addr in segspt. 20520Sstevel@tonic-gate */ 20530Sstevel@tonic-gate anon_index = seg_page(seg, shm_addr); 20540Sstevel@tonic-gate sptseg_addr = sptseg->s_base + ptob(anon_index); 20550Sstevel@tonic-gate 20560Sstevel@tonic-gate /* 20570Sstevel@tonic-gate * And now we may have to adjust npages downward if we have 20580Sstevel@tonic-gate * exceeded the realsize of the segment or initial anon 20590Sstevel@tonic-gate * allocations. 20600Sstevel@tonic-gate */ 20610Sstevel@tonic-gate if ((sptseg_addr + ptob(npages)) > 20620Sstevel@tonic-gate (sptseg->s_base + sptd->spt_realsize)) 20630Sstevel@tonic-gate size = (sptseg->s_base + sptd->spt_realsize) - sptseg_addr; 20640Sstevel@tonic-gate 20650Sstevel@tonic-gate npages = btopr(size); 20660Sstevel@tonic-gate 20670Sstevel@tonic-gate ASSERT(sptseg_addr < (sptseg->s_base + sptseg->s_size)); 20680Sstevel@tonic-gate ASSERT((sptd->spt_flags & SHM_PAGEABLE) == 0); 20690Sstevel@tonic-gate 20700Sstevel@tonic-gate switch (type) { 20710Sstevel@tonic-gate 20720Sstevel@tonic-gate case F_SOFTLOCK: 20730Sstevel@tonic-gate 20740Sstevel@tonic-gate /* 20750Sstevel@tonic-gate * availrmem is decremented once during anon_swap_adjust() 20760Sstevel@tonic-gate * and is incremented during the anon_unresv(), which is 20770Sstevel@tonic-gate * called from shm_rm_amp() when the segment is destroyed. 20780Sstevel@tonic-gate */ 20790Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages); 20800Sstevel@tonic-gate /* 20810Sstevel@tonic-gate * Some platforms assume that ISM pages are SE_SHARED 20820Sstevel@tonic-gate * locked for the entire life of the segment. 20830Sstevel@tonic-gate */ 20840Sstevel@tonic-gate if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) 20850Sstevel@tonic-gate return (0); 20860Sstevel@tonic-gate /* 20870Sstevel@tonic-gate * Fall through to the F_INVAL case to load up the hat layer 20880Sstevel@tonic-gate * entries with the HAT_LOAD_LOCK flag. 20890Sstevel@tonic-gate */ 20900Sstevel@tonic-gate 20910Sstevel@tonic-gate /* FALLTHRU */ 20920Sstevel@tonic-gate case F_INVAL: 20930Sstevel@tonic-gate 20940Sstevel@tonic-gate if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC)) 20950Sstevel@tonic-gate return (FC_NOMAP); 20960Sstevel@tonic-gate 20970Sstevel@tonic-gate /* 20980Sstevel@tonic-gate * Some platforms that do NOT support DYNAMIC_ISM_UNMAP 20990Sstevel@tonic-gate * may still rely on this call to hat_share(). That 21000Sstevel@tonic-gate * would imply that those hat's can fault on a 21010Sstevel@tonic-gate * HAT_LOAD_LOCK translation, which would seem 21020Sstevel@tonic-gate * contradictory. 21030Sstevel@tonic-gate */ 21040Sstevel@tonic-gate if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 21050Sstevel@tonic-gate if (hat_share(seg->s_as->a_hat, seg->s_base, 21060Sstevel@tonic-gate curspt->a_hat, sptseg->s_base, 21070Sstevel@tonic-gate sptseg->s_size, sptseg->s_szc) != 0) { 21080Sstevel@tonic-gate panic("hat_share error in ISM fault"); 21090Sstevel@tonic-gate /*NOTREACHED*/ 21100Sstevel@tonic-gate } 21110Sstevel@tonic-gate return (0); 21120Sstevel@tonic-gate } 21130Sstevel@tonic-gate ppa = kmem_zalloc(sizeof (page_t *) * npages, KM_SLEEP); 21140Sstevel@tonic-gate 21150Sstevel@tonic-gate /* 21160Sstevel@tonic-gate * I see no need to lock the real seg, 21170Sstevel@tonic-gate * here, because all of our work will be on the underlying 21180Sstevel@tonic-gate * dummy seg. 21190Sstevel@tonic-gate * 21200Sstevel@tonic-gate * sptseg_addr and npages now account for large pages. 21210Sstevel@tonic-gate */ 21220Sstevel@tonic-gate amp = sptd->spt_amp; 21230Sstevel@tonic-gate ASSERT(amp != NULL); 21240Sstevel@tonic-gate anon_index = seg_page(sptseg, sptseg_addr); 21250Sstevel@tonic-gate 21260Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 21270Sstevel@tonic-gate for (i = 0; i < npages; i++) { 21280Sstevel@tonic-gate anon_array_enter(amp, anon_index, &cookie); 21290Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index++); 21300Sstevel@tonic-gate ASSERT(ap != NULL); 21310Sstevel@tonic-gate swap_xlate(ap, &vp, &offset); 21320Sstevel@tonic-gate anon_array_exit(&cookie); 21330Sstevel@tonic-gate pp = page_lookup(vp, offset, SE_SHARED); 21340Sstevel@tonic-gate ASSERT(pp != NULL); 21350Sstevel@tonic-gate ppa[i] = pp; 21360Sstevel@tonic-gate } 21370Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 21380Sstevel@tonic-gate ASSERT(i == npages); 21390Sstevel@tonic-gate 21400Sstevel@tonic-gate /* 21410Sstevel@tonic-gate * We are already holding the as->a_lock on the user's 21420Sstevel@tonic-gate * real segment, but we need to hold the a_lock on the 21430Sstevel@tonic-gate * underlying dummy as. This is mostly to satisfy the 21440Sstevel@tonic-gate * underlying HAT layer. 21450Sstevel@tonic-gate */ 21460Sstevel@tonic-gate AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 21472414Saguzovsk a = sptseg_addr; 21482414Saguzovsk pidx = 0; 21490Sstevel@tonic-gate if (type == F_SOFTLOCK) { 21500Sstevel@tonic-gate /* 21510Sstevel@tonic-gate * Load up the translation keeping it 21520Sstevel@tonic-gate * locked and don't unlock the page. 21530Sstevel@tonic-gate */ 21542414Saguzovsk for (; pidx < npages; a += pgsz, pidx += pgcnt) { 21552414Saguzovsk sz = MIN(pgsz, ptob(npages - pidx)); 21562414Saguzovsk hat_memload_array(sptseg->s_as->a_hat, a, 21572414Saguzovsk sz, &ppa[pidx], sptd->spt_prot, 21582414Saguzovsk HAT_LOAD_LOCK | HAT_LOAD_SHARE); 21592414Saguzovsk } 21600Sstevel@tonic-gate } else { 21610Sstevel@tonic-gate if (hat == seg->s_as->a_hat) { 21620Sstevel@tonic-gate 21630Sstevel@tonic-gate /* 21640Sstevel@tonic-gate * Migrate pages marked for migration. 21650Sstevel@tonic-gate */ 21660Sstevel@tonic-gate if (lgrp_optimizations()) 21670Sstevel@tonic-gate page_migrate(seg, shm_addr, ppa, 21680Sstevel@tonic-gate npages); 21690Sstevel@tonic-gate 21700Sstevel@tonic-gate /* CPU HAT */ 21712414Saguzovsk for (; pidx < npages; 21722414Saguzovsk a += pgsz, pidx += pgcnt) { 21732414Saguzovsk sz = MIN(pgsz, ptob(npages - pidx)); 21742414Saguzovsk hat_memload_array(sptseg->s_as->a_hat, 21752414Saguzovsk a, sz, &ppa[pidx], 21762414Saguzovsk sptd->spt_prot, HAT_LOAD_SHARE); 21772414Saguzovsk } 21780Sstevel@tonic-gate } else { 21790Sstevel@tonic-gate /* XHAT. Pass real address */ 21800Sstevel@tonic-gate hat_memload_array(hat, shm_addr, 21810Sstevel@tonic-gate ptob(npages), ppa, sptd->spt_prot, 21820Sstevel@tonic-gate HAT_LOAD_SHARE); 21830Sstevel@tonic-gate } 21840Sstevel@tonic-gate 21850Sstevel@tonic-gate /* 21860Sstevel@tonic-gate * And now drop the SE_SHARED lock(s). 21870Sstevel@tonic-gate */ 21880Sstevel@tonic-gate for (i = 0; i < npages; i++) 21890Sstevel@tonic-gate page_unlock(ppa[i]); 21900Sstevel@tonic-gate } 21910Sstevel@tonic-gate AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 21920Sstevel@tonic-gate 21930Sstevel@tonic-gate kmem_free(ppa, sizeof (page_t *) * npages); 21940Sstevel@tonic-gate return (0); 21950Sstevel@tonic-gate case F_SOFTUNLOCK: 21960Sstevel@tonic-gate 21970Sstevel@tonic-gate /* 21980Sstevel@tonic-gate * This is a bit ugly, we pass in the real seg pointer, 21990Sstevel@tonic-gate * but the sptseg_addr is the virtual address within the 22000Sstevel@tonic-gate * dummy seg. 22010Sstevel@tonic-gate */ 22020Sstevel@tonic-gate segspt_softunlock(seg, sptseg_addr, ptob(npages), rw); 22030Sstevel@tonic-gate return (0); 22040Sstevel@tonic-gate 22050Sstevel@tonic-gate case F_PROT: 22060Sstevel@tonic-gate 22070Sstevel@tonic-gate /* 22080Sstevel@tonic-gate * This takes care of the unusual case where a user 22090Sstevel@tonic-gate * allocates a stack in shared memory and a register 22100Sstevel@tonic-gate * window overflow is written to that stack page before 22110Sstevel@tonic-gate * it is otherwise modified. 22120Sstevel@tonic-gate * 22130Sstevel@tonic-gate * We can get away with this because ISM segments are 22140Sstevel@tonic-gate * always rw. Other than this unusual case, there 22150Sstevel@tonic-gate * should be no instances of protection violations. 22160Sstevel@tonic-gate */ 22170Sstevel@tonic-gate return (0); 22180Sstevel@tonic-gate 22190Sstevel@tonic-gate default: 22200Sstevel@tonic-gate #ifdef DEBUG 22210Sstevel@tonic-gate cmn_err(CE_WARN, "segspt_shmfault default type?"); 22220Sstevel@tonic-gate #endif 22230Sstevel@tonic-gate return (FC_NOMAP); 22240Sstevel@tonic-gate } 22250Sstevel@tonic-gate } 22260Sstevel@tonic-gate 22270Sstevel@tonic-gate /*ARGSUSED*/ 22280Sstevel@tonic-gate static faultcode_t 22290Sstevel@tonic-gate segspt_shmfaulta(struct seg *seg, caddr_t addr) 22300Sstevel@tonic-gate { 22310Sstevel@tonic-gate return (0); 22320Sstevel@tonic-gate } 22330Sstevel@tonic-gate 22340Sstevel@tonic-gate /*ARGSUSED*/ 22350Sstevel@tonic-gate static int 22360Sstevel@tonic-gate segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta) 22370Sstevel@tonic-gate { 22380Sstevel@tonic-gate return (0); 22390Sstevel@tonic-gate } 22400Sstevel@tonic-gate 22410Sstevel@tonic-gate /*ARGSUSED*/ 22420Sstevel@tonic-gate static size_t 22430Sstevel@tonic-gate segspt_shmswapout(struct seg *seg) 22440Sstevel@tonic-gate { 22450Sstevel@tonic-gate return (0); 22460Sstevel@tonic-gate } 22470Sstevel@tonic-gate 22480Sstevel@tonic-gate /* 22490Sstevel@tonic-gate * duplicate the shared page tables 22500Sstevel@tonic-gate */ 22510Sstevel@tonic-gate int 22520Sstevel@tonic-gate segspt_shmdup(struct seg *seg, struct seg *newseg) 22530Sstevel@tonic-gate { 22540Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 22550Sstevel@tonic-gate struct anon_map *amp = shmd->shm_amp; 22560Sstevel@tonic-gate struct shm_data *shmd_new; 22570Sstevel@tonic-gate struct seg *spt_seg = shmd->shm_sptseg; 22580Sstevel@tonic-gate struct spt_data *sptd = spt_seg->s_data; 2259721Smec int error = 0; 22600Sstevel@tonic-gate 22610Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 22620Sstevel@tonic-gate 22630Sstevel@tonic-gate shmd_new = kmem_zalloc((sizeof (*shmd_new)), KM_SLEEP); 22640Sstevel@tonic-gate newseg->s_data = (void *)shmd_new; 22650Sstevel@tonic-gate shmd_new->shm_sptas = shmd->shm_sptas; 22660Sstevel@tonic-gate shmd_new->shm_amp = amp; 22670Sstevel@tonic-gate shmd_new->shm_sptseg = shmd->shm_sptseg; 22680Sstevel@tonic-gate newseg->s_ops = &segspt_shmops; 22690Sstevel@tonic-gate newseg->s_szc = seg->s_szc; 22700Sstevel@tonic-gate ASSERT(seg->s_szc == shmd->shm_sptseg->s_szc); 22710Sstevel@tonic-gate 22720Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 22730Sstevel@tonic-gate amp->refcnt++; 22740Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 22750Sstevel@tonic-gate 22760Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 22770Sstevel@tonic-gate shmd_new->shm_vpage = kmem_zalloc(btopr(amp->size), KM_SLEEP); 22780Sstevel@tonic-gate shmd_new->shm_lckpgs = 0; 2279721Smec if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 2280721Smec if ((error = hat_share(newseg->s_as->a_hat, 2281721Smec newseg->s_base, shmd->shm_sptas->a_hat, SEGSPTADDR, 2282721Smec seg->s_size, seg->s_szc)) != 0) { 2283721Smec kmem_free(shmd_new->shm_vpage, 2284721Smec btopr(amp->size)); 2285721Smec } 2286721Smec } 2287721Smec return (error); 2288721Smec } else { 2289721Smec return (hat_share(newseg->s_as->a_hat, newseg->s_base, 2290721Smec shmd->shm_sptas->a_hat, SEGSPTADDR, seg->s_size, 2291721Smec seg->s_szc)); 2292721Smec 22930Sstevel@tonic-gate } 22940Sstevel@tonic-gate } 22950Sstevel@tonic-gate 22960Sstevel@tonic-gate /*ARGSUSED*/ 22970Sstevel@tonic-gate int 22980Sstevel@tonic-gate segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot) 22990Sstevel@tonic-gate { 23000Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 23010Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 23020Sstevel@tonic-gate 23030Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 23040Sstevel@tonic-gate 23050Sstevel@tonic-gate /* 23060Sstevel@tonic-gate * ISM segment is always rw. 23070Sstevel@tonic-gate */ 23080Sstevel@tonic-gate return (((sptd->spt_prot & prot) != prot) ? EACCES : 0); 23090Sstevel@tonic-gate } 23100Sstevel@tonic-gate 23110Sstevel@tonic-gate /* 23120Sstevel@tonic-gate * Return an array of locked large pages, for empty slots allocate 23130Sstevel@tonic-gate * private zero-filled anon pages. 23140Sstevel@tonic-gate */ 23150Sstevel@tonic-gate static int 23160Sstevel@tonic-gate spt_anon_getpages( 23170Sstevel@tonic-gate struct seg *sptseg, 23180Sstevel@tonic-gate caddr_t sptaddr, 23190Sstevel@tonic-gate size_t len, 23200Sstevel@tonic-gate page_t *ppa[]) 23210Sstevel@tonic-gate { 23220Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 23230Sstevel@tonic-gate struct anon_map *amp = sptd->spt_amp; 23240Sstevel@tonic-gate enum seg_rw rw = sptd->spt_prot; 23250Sstevel@tonic-gate uint_t szc = sptseg->s_szc; 23260Sstevel@tonic-gate size_t pg_sz, share_sz = page_get_pagesize(szc); 23270Sstevel@tonic-gate pgcnt_t lp_npgs; 23280Sstevel@tonic-gate caddr_t lp_addr, e_sptaddr; 23290Sstevel@tonic-gate uint_t vpprot, ppa_szc = 0; 23300Sstevel@tonic-gate struct vpage *vpage = NULL; 23310Sstevel@tonic-gate ulong_t j, ppa_idx; 23320Sstevel@tonic-gate int err, ierr = 0; 23330Sstevel@tonic-gate pgcnt_t an_idx; 23340Sstevel@tonic-gate anon_sync_obj_t cookie; 23350Sstevel@tonic-gate 23360Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(sptaddr, share_sz) && IS_P2ALIGNED(len, share_sz)); 23370Sstevel@tonic-gate ASSERT(len != 0); 23380Sstevel@tonic-gate 23390Sstevel@tonic-gate pg_sz = share_sz; 23400Sstevel@tonic-gate lp_npgs = btop(pg_sz); 23410Sstevel@tonic-gate lp_addr = sptaddr; 23420Sstevel@tonic-gate e_sptaddr = sptaddr + len; 23430Sstevel@tonic-gate an_idx = seg_page(sptseg, sptaddr); 23440Sstevel@tonic-gate ppa_idx = 0; 23450Sstevel@tonic-gate 23460Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 23470Sstevel@tonic-gate /*CONSTCOND*/ 23480Sstevel@tonic-gate while (1) { 23490Sstevel@tonic-gate for (; lp_addr < e_sptaddr; 23500Sstevel@tonic-gate an_idx += lp_npgs, lp_addr += pg_sz, 23510Sstevel@tonic-gate ppa_idx += lp_npgs) { 23520Sstevel@tonic-gate 23530Sstevel@tonic-gate anon_array_enter(amp, an_idx, &cookie); 23540Sstevel@tonic-gate ppa_szc = (uint_t)-1; 23550Sstevel@tonic-gate ierr = anon_map_getpages(amp, an_idx, szc, sptseg, 23560Sstevel@tonic-gate lp_addr, sptd->spt_prot, &vpprot, &ppa[ppa_idx], 23570Sstevel@tonic-gate &ppa_szc, vpage, rw, 0, segvn_anypgsz, kcred); 23580Sstevel@tonic-gate anon_array_exit(&cookie); 23590Sstevel@tonic-gate 23600Sstevel@tonic-gate if (ierr != 0) { 23610Sstevel@tonic-gate if (ierr > 0) { 23620Sstevel@tonic-gate err = FC_MAKE_ERR(ierr); 23630Sstevel@tonic-gate goto lpgs_err; 23640Sstevel@tonic-gate } 23650Sstevel@tonic-gate break; 23660Sstevel@tonic-gate } 23670Sstevel@tonic-gate } 23680Sstevel@tonic-gate if (lp_addr == e_sptaddr) { 23690Sstevel@tonic-gate break; 23700Sstevel@tonic-gate } 23710Sstevel@tonic-gate ASSERT(lp_addr < e_sptaddr); 23720Sstevel@tonic-gate 23730Sstevel@tonic-gate /* 23740Sstevel@tonic-gate * ierr == -1 means we failed to allocate a large page. 23750Sstevel@tonic-gate * so do a size down operation. 23760Sstevel@tonic-gate * 23770Sstevel@tonic-gate * ierr == -2 means some other process that privately shares 23780Sstevel@tonic-gate * pages with this process has allocated a larger page and we 23790Sstevel@tonic-gate * need to retry with larger pages. So do a size up 23800Sstevel@tonic-gate * operation. This relies on the fact that large pages are 23810Sstevel@tonic-gate * never partially shared i.e. if we share any constituent 23820Sstevel@tonic-gate * page of a large page with another process we must share the 23830Sstevel@tonic-gate * entire large page. Note this cannot happen for SOFTLOCK 23840Sstevel@tonic-gate * case, unless current address (lpaddr) is at the beginning 23850Sstevel@tonic-gate * of the next page size boundary because the other process 23860Sstevel@tonic-gate * couldn't have relocated locked pages. 23870Sstevel@tonic-gate */ 23880Sstevel@tonic-gate ASSERT(ierr == -1 || ierr == -2); 23890Sstevel@tonic-gate if (segvn_anypgsz) { 23900Sstevel@tonic-gate ASSERT(ierr == -2 || szc != 0); 23910Sstevel@tonic-gate ASSERT(ierr == -1 || szc < sptseg->s_szc); 23920Sstevel@tonic-gate szc = (ierr == -1) ? szc - 1 : szc + 1; 23930Sstevel@tonic-gate } else { 23940Sstevel@tonic-gate /* 23950Sstevel@tonic-gate * For faults and segvn_anypgsz == 0 23960Sstevel@tonic-gate * we need to be careful not to loop forever 23970Sstevel@tonic-gate * if existing page is found with szc other 23980Sstevel@tonic-gate * than 0 or seg->s_szc. This could be due 23990Sstevel@tonic-gate * to page relocations on behalf of DR or 24000Sstevel@tonic-gate * more likely large page creation. For this 24010Sstevel@tonic-gate * case simply re-size to existing page's szc 24020Sstevel@tonic-gate * if returned by anon_map_getpages(). 24030Sstevel@tonic-gate */ 24040Sstevel@tonic-gate if (ppa_szc == (uint_t)-1) { 24050Sstevel@tonic-gate szc = (ierr == -1) ? 0 : sptseg->s_szc; 24060Sstevel@tonic-gate } else { 24070Sstevel@tonic-gate ASSERT(ppa_szc <= sptseg->s_szc); 24080Sstevel@tonic-gate ASSERT(ierr == -2 || ppa_szc < szc); 24090Sstevel@tonic-gate ASSERT(ierr == -1 || ppa_szc > szc); 24100Sstevel@tonic-gate szc = ppa_szc; 24110Sstevel@tonic-gate } 24120Sstevel@tonic-gate } 24130Sstevel@tonic-gate pg_sz = page_get_pagesize(szc); 24140Sstevel@tonic-gate lp_npgs = btop(pg_sz); 24150Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(lp_addr, pg_sz)); 24160Sstevel@tonic-gate } 24170Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 24180Sstevel@tonic-gate return (0); 24190Sstevel@tonic-gate 24200Sstevel@tonic-gate lpgs_err: 24210Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 24220Sstevel@tonic-gate for (j = 0; j < ppa_idx; j++) 24230Sstevel@tonic-gate page_unlock(ppa[j]); 24240Sstevel@tonic-gate return (err); 24250Sstevel@tonic-gate } 24260Sstevel@tonic-gate 24272768Ssl108498 /* 24282768Ssl108498 * count the number of bytes in a set of spt pages that are currently not 24292768Ssl108498 * locked 24302768Ssl108498 */ 24312768Ssl108498 static rctl_qty_t 24322768Ssl108498 spt_unlockedbytes(pgcnt_t npages, page_t **ppa) 24332768Ssl108498 { 24342768Ssl108498 ulong_t i; 24352768Ssl108498 rctl_qty_t unlocked = 0; 24362768Ssl108498 24372768Ssl108498 for (i = 0; i < npages; i++) { 24382768Ssl108498 if (ppa[i]->p_lckcnt == 0) 24392768Ssl108498 unlocked += PAGESIZE; 24402768Ssl108498 } 24412768Ssl108498 return (unlocked); 24422768Ssl108498 } 24432768Ssl108498 24440Sstevel@tonic-gate int 24450Sstevel@tonic-gate spt_lockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages, 24462768Ssl108498 page_t **ppa, ulong_t *lockmap, size_t pos, 24472768Ssl108498 rctl_qty_t *locked) 24480Sstevel@tonic-gate { 24490Sstevel@tonic-gate struct shm_data *shmd = seg->s_data; 24500Sstevel@tonic-gate struct spt_data *sptd = shmd->shm_sptseg->s_data; 24510Sstevel@tonic-gate ulong_t i; 24520Sstevel@tonic-gate int kernel; 24530Sstevel@tonic-gate 24542768Ssl108498 /* return the number of bytes actually locked */ 24552768Ssl108498 *locked = 0; 24560Sstevel@tonic-gate for (i = 0; i < npages; anon_index++, pos++, i++) { 24570Sstevel@tonic-gate if (!(shmd->shm_vpage[anon_index] & DISM_PG_LOCKED)) { 24580Sstevel@tonic-gate if (sptd->spt_ppa_lckcnt[anon_index] < 24590Sstevel@tonic-gate (ushort_t)DISM_LOCK_MAX) { 24600Sstevel@tonic-gate if (++sptd->spt_ppa_lckcnt[anon_index] == 24610Sstevel@tonic-gate (ushort_t)DISM_LOCK_MAX) { 24620Sstevel@tonic-gate cmn_err(CE_WARN, 24630Sstevel@tonic-gate "DISM page lock limit " 24640Sstevel@tonic-gate "reached on DISM offset 0x%lx\n", 24650Sstevel@tonic-gate anon_index << PAGESHIFT); 24660Sstevel@tonic-gate } 24670Sstevel@tonic-gate kernel = (sptd->spt_ppa && 24680Sstevel@tonic-gate sptd->spt_ppa[anon_index]) ? 1 : 0; 24690Sstevel@tonic-gate if (!page_pp_lock(ppa[i], 0, kernel)) { 24700Sstevel@tonic-gate sptd->spt_ppa_lckcnt[anon_index]--; 24710Sstevel@tonic-gate return (EAGAIN); 24720Sstevel@tonic-gate } 24732768Ssl108498 /* if this is a newly locked page, count it */ 24742768Ssl108498 if (ppa[i]->p_lckcnt == 1) { 24752768Ssl108498 *locked += PAGESIZE; 24762768Ssl108498 } 24770Sstevel@tonic-gate shmd->shm_lckpgs++; 24780Sstevel@tonic-gate shmd->shm_vpage[anon_index] |= DISM_PG_LOCKED; 24790Sstevel@tonic-gate if (lockmap != NULL) 24800Sstevel@tonic-gate BT_SET(lockmap, pos); 24810Sstevel@tonic-gate } 24820Sstevel@tonic-gate } 24830Sstevel@tonic-gate } 24840Sstevel@tonic-gate return (0); 24850Sstevel@tonic-gate } 24860Sstevel@tonic-gate 24870Sstevel@tonic-gate /*ARGSUSED*/ 24880Sstevel@tonic-gate static int 24890Sstevel@tonic-gate segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len, 24900Sstevel@tonic-gate int attr, int op, ulong_t *lockmap, size_t pos) 24910Sstevel@tonic-gate { 24920Sstevel@tonic-gate struct shm_data *shmd = seg->s_data; 24930Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 24940Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 24952768Ssl108498 struct kshmid *sp = sptd->spt_amp->a_sp; 24960Sstevel@tonic-gate pgcnt_t npages, a_npages; 24970Sstevel@tonic-gate page_t **ppa; 24980Sstevel@tonic-gate pgcnt_t an_idx, a_an_idx, ppa_idx; 24990Sstevel@tonic-gate caddr_t spt_addr, a_addr; /* spt and aligned address */ 25000Sstevel@tonic-gate size_t a_len; /* aligned len */ 25010Sstevel@tonic-gate size_t share_sz; 25020Sstevel@tonic-gate ulong_t i; 25030Sstevel@tonic-gate int sts = 0; 25042768Ssl108498 rctl_qty_t unlocked = 0; 25052768Ssl108498 rctl_qty_t locked = 0; 25062768Ssl108498 struct proc *p = curproc; 25072768Ssl108498 kproject_t *proj; 25080Sstevel@tonic-gate 25090Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 25102768Ssl108498 ASSERT(sp != NULL); 25110Sstevel@tonic-gate 25120Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 25130Sstevel@tonic-gate return (0); 25140Sstevel@tonic-gate } 25150Sstevel@tonic-gate 25160Sstevel@tonic-gate addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 25170Sstevel@tonic-gate an_idx = seg_page(seg, addr); 25180Sstevel@tonic-gate npages = btopr(len); 25190Sstevel@tonic-gate 25200Sstevel@tonic-gate if (an_idx + npages > btopr(shmd->shm_amp->size)) { 25210Sstevel@tonic-gate return (ENOMEM); 25220Sstevel@tonic-gate } 25230Sstevel@tonic-gate 25242768Ssl108498 /* 25252768Ssl108498 * A shm's project never changes, so no lock needed. 25262768Ssl108498 * The shm has a hold on the project, so it will not go away. 25272768Ssl108498 * Since we have a mapping to shm within this zone, we know 25282768Ssl108498 * that the zone will not go away. 25292768Ssl108498 */ 25302768Ssl108498 proj = sp->shm_perm.ipc_proj; 25312768Ssl108498 25320Sstevel@tonic-gate if (op == MC_LOCK) { 25332768Ssl108498 25340Sstevel@tonic-gate /* 25350Sstevel@tonic-gate * Need to align addr and size request if they are not 25360Sstevel@tonic-gate * aligned so we can always allocate large page(s) however 25370Sstevel@tonic-gate * we only lock what was requested in initial request. 25380Sstevel@tonic-gate */ 25390Sstevel@tonic-gate share_sz = page_get_pagesize(sptseg->s_szc); 25400Sstevel@tonic-gate a_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_sz); 25410Sstevel@tonic-gate a_len = P2ROUNDUP((uintptr_t)(((addr + len) - a_addr)), 25420Sstevel@tonic-gate share_sz); 25430Sstevel@tonic-gate a_npages = btop(a_len); 25440Sstevel@tonic-gate a_an_idx = seg_page(seg, a_addr); 25450Sstevel@tonic-gate spt_addr = sptseg->s_base + ptob(a_an_idx); 25460Sstevel@tonic-gate ppa_idx = an_idx - a_an_idx; 25470Sstevel@tonic-gate 25480Sstevel@tonic-gate if ((ppa = kmem_zalloc(((sizeof (page_t *)) * a_npages), 25490Sstevel@tonic-gate KM_NOSLEEP)) == NULL) { 25500Sstevel@tonic-gate return (ENOMEM); 25510Sstevel@tonic-gate } 25520Sstevel@tonic-gate 25530Sstevel@tonic-gate /* 25540Sstevel@tonic-gate * Don't cache any new pages for IO and 25550Sstevel@tonic-gate * flush any cached pages. 25560Sstevel@tonic-gate */ 25570Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 25580Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 25590Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 25600Sstevel@tonic-gate 25610Sstevel@tonic-gate sts = spt_anon_getpages(sptseg, spt_addr, a_len, ppa); 25620Sstevel@tonic-gate if (sts != 0) { 25630Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 25640Sstevel@tonic-gate kmem_free(ppa, ((sizeof (page_t *)) * a_npages)); 25650Sstevel@tonic-gate return (sts); 25660Sstevel@tonic-gate } 25670Sstevel@tonic-gate 25682768Ssl108498 mutex_enter(&sp->shm_mlock); 25692768Ssl108498 /* enforce locked memory rctl */ 25702768Ssl108498 unlocked = spt_unlockedbytes(npages, &ppa[ppa_idx]); 25712768Ssl108498 25722768Ssl108498 mutex_enter(&p->p_lock); 25732768Ssl108498 if (rctl_incr_locked_mem(p, proj, unlocked, 0)) { 25742768Ssl108498 mutex_exit(&p->p_lock); 25752768Ssl108498 sts = EAGAIN; 25762768Ssl108498 } else { 25772768Ssl108498 mutex_exit(&p->p_lock); 25782768Ssl108498 sts = spt_lockpages(seg, an_idx, npages, 25792768Ssl108498 &ppa[ppa_idx], lockmap, pos, &locked); 25802768Ssl108498 25812768Ssl108498 /* 25822768Ssl108498 * correct locked count if not all pages could be 25832768Ssl108498 * locked 25842768Ssl108498 */ 25852768Ssl108498 if ((unlocked - locked) > 0) { 25862768Ssl108498 rctl_decr_locked_mem(NULL, proj, 25872768Ssl108498 (unlocked - locked), 0); 25882768Ssl108498 } 25892768Ssl108498 } 25900Sstevel@tonic-gate /* 25912768Ssl108498 * unlock pages 25920Sstevel@tonic-gate */ 25932768Ssl108498 for (i = 0; i < a_npages; i++) 25940Sstevel@tonic-gate page_unlock(ppa[i]); 25950Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 25960Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 25972768Ssl108498 mutex_exit(&sp->shm_mlock); 25980Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 25990Sstevel@tonic-gate 26000Sstevel@tonic-gate kmem_free(ppa, ((sizeof (page_t *)) * a_npages)); 26010Sstevel@tonic-gate 26020Sstevel@tonic-gate } else if (op == MC_UNLOCK) { /* unlock */ 26030Sstevel@tonic-gate struct anon_map *amp; 26040Sstevel@tonic-gate struct anon *ap; 26050Sstevel@tonic-gate struct vnode *vp; 26060Sstevel@tonic-gate u_offset_t off; 26070Sstevel@tonic-gate struct page *pp; 26080Sstevel@tonic-gate int kernel; 26090Sstevel@tonic-gate anon_sync_obj_t cookie; 26102768Ssl108498 rctl_qty_t unlocked = 0; 26110Sstevel@tonic-gate 26120Sstevel@tonic-gate amp = sptd->spt_amp; 26130Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 26140Sstevel@tonic-gate if (shmd->shm_lckpgs == 0) { 26150Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 26160Sstevel@tonic-gate return (0); 26170Sstevel@tonic-gate } 26180Sstevel@tonic-gate /* 26190Sstevel@tonic-gate * Don't cache new IO pages. 26200Sstevel@tonic-gate */ 26210Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 26220Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 26230Sstevel@tonic-gate 26242768Ssl108498 mutex_enter(&sp->shm_mlock); 26250Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 26260Sstevel@tonic-gate for (i = 0; i < npages; i++, an_idx++) { 26270Sstevel@tonic-gate if (shmd->shm_vpage[an_idx] & DISM_PG_LOCKED) { 26280Sstevel@tonic-gate anon_array_enter(amp, an_idx, &cookie); 26290Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, an_idx); 26300Sstevel@tonic-gate ASSERT(ap); 26310Sstevel@tonic-gate 26320Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 26330Sstevel@tonic-gate anon_array_exit(&cookie); 26340Sstevel@tonic-gate pp = page_lookup(vp, off, SE_SHARED); 26350Sstevel@tonic-gate ASSERT(pp); 26360Sstevel@tonic-gate /* 26370Sstevel@tonic-gate * the availrmem is decremented only for 26380Sstevel@tonic-gate * pages which are not in seg pcache, 26390Sstevel@tonic-gate * for pages in seg pcache availrmem was 26400Sstevel@tonic-gate * decremented in _dismpagelock() (if 26410Sstevel@tonic-gate * they were not locked here) 26420Sstevel@tonic-gate */ 26430Sstevel@tonic-gate kernel = (sptd->spt_ppa && 26440Sstevel@tonic-gate sptd->spt_ppa[an_idx]) ? 1 : 0; 26452768Ssl108498 ASSERT(pp->p_lckcnt > 0); 26460Sstevel@tonic-gate page_pp_unlock(pp, 0, kernel); 26472768Ssl108498 if (pp->p_lckcnt == 0) 26482768Ssl108498 unlocked += PAGESIZE; 26490Sstevel@tonic-gate page_unlock(pp); 26500Sstevel@tonic-gate shmd->shm_vpage[an_idx] &= ~DISM_PG_LOCKED; 26510Sstevel@tonic-gate sptd->spt_ppa_lckcnt[an_idx]--; 26520Sstevel@tonic-gate shmd->shm_lckpgs--; 26530Sstevel@tonic-gate } 26540Sstevel@tonic-gate } 26550Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 26560Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 26570Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 26580Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 26592768Ssl108498 26602768Ssl108498 rctl_decr_locked_mem(NULL, proj, unlocked, 0); 26612768Ssl108498 mutex_exit(&sp->shm_mlock); 26620Sstevel@tonic-gate } 26630Sstevel@tonic-gate return (sts); 26640Sstevel@tonic-gate } 26650Sstevel@tonic-gate 26660Sstevel@tonic-gate /*ARGSUSED*/ 26670Sstevel@tonic-gate int 26680Sstevel@tonic-gate segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv) 26690Sstevel@tonic-gate { 26700Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 26710Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 26720Sstevel@tonic-gate spgcnt_t pgno = seg_page(seg, addr+len) - seg_page(seg, addr) + 1; 26730Sstevel@tonic-gate 26740Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 26750Sstevel@tonic-gate 26760Sstevel@tonic-gate /* 26770Sstevel@tonic-gate * ISM segment is always rw. 26780Sstevel@tonic-gate */ 26790Sstevel@tonic-gate while (--pgno >= 0) 26800Sstevel@tonic-gate *protv++ = sptd->spt_prot; 26810Sstevel@tonic-gate return (0); 26820Sstevel@tonic-gate } 26830Sstevel@tonic-gate 26840Sstevel@tonic-gate /*ARGSUSED*/ 26850Sstevel@tonic-gate u_offset_t 26860Sstevel@tonic-gate segspt_shmgetoffset(struct seg *seg, caddr_t addr) 26870Sstevel@tonic-gate { 26880Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 26890Sstevel@tonic-gate 26900Sstevel@tonic-gate /* Offset does not matter in ISM memory */ 26910Sstevel@tonic-gate 26920Sstevel@tonic-gate return ((u_offset_t)0); 26930Sstevel@tonic-gate } 26940Sstevel@tonic-gate 26950Sstevel@tonic-gate /* ARGSUSED */ 26960Sstevel@tonic-gate int 26970Sstevel@tonic-gate segspt_shmgettype(struct seg *seg, caddr_t addr) 26980Sstevel@tonic-gate { 26990Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 27000Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 27010Sstevel@tonic-gate 27020Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 27030Sstevel@tonic-gate 27040Sstevel@tonic-gate /* 27050Sstevel@tonic-gate * The shared memory mapping is always MAP_SHARED, SWAP is only 27060Sstevel@tonic-gate * reserved for DISM 27070Sstevel@tonic-gate */ 27080Sstevel@tonic-gate return (MAP_SHARED | 27090Sstevel@tonic-gate ((sptd->spt_flags & SHM_PAGEABLE) ? 0 : MAP_NORESERVE)); 27100Sstevel@tonic-gate } 27110Sstevel@tonic-gate 27120Sstevel@tonic-gate /*ARGSUSED*/ 27130Sstevel@tonic-gate int 27140Sstevel@tonic-gate segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp) 27150Sstevel@tonic-gate { 27160Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 27170Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 27180Sstevel@tonic-gate 27190Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 27200Sstevel@tonic-gate 27210Sstevel@tonic-gate *vpp = sptd->spt_vp; 27220Sstevel@tonic-gate return (0); 27230Sstevel@tonic-gate } 27240Sstevel@tonic-gate 27250Sstevel@tonic-gate /*ARGSUSED*/ 27260Sstevel@tonic-gate static int 27270Sstevel@tonic-gate segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, uint_t behav) 27280Sstevel@tonic-gate { 27290Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 27300Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 27310Sstevel@tonic-gate struct anon_map *amp; 27320Sstevel@tonic-gate pgcnt_t pg_idx; 27330Sstevel@tonic-gate 27340Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 27350Sstevel@tonic-gate 27360Sstevel@tonic-gate if (behav == MADV_FREE) { 27370Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) 27380Sstevel@tonic-gate return (0); 27390Sstevel@tonic-gate 27400Sstevel@tonic-gate amp = sptd->spt_amp; 27410Sstevel@tonic-gate pg_idx = seg_page(seg, addr); 27420Sstevel@tonic-gate 27430Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 27440Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 27450Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 27460Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 27470Sstevel@tonic-gate 27480Sstevel@tonic-gate /* 27490Sstevel@tonic-gate * Purge all DISM cached pages 27500Sstevel@tonic-gate */ 27510Sstevel@tonic-gate seg_ppurge_seg(segspt_reclaim); 27520Sstevel@tonic-gate 27530Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 27540Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 27550Sstevel@tonic-gate anon_disclaim(amp, pg_idx, len, ANON_PGLOOKUP_BLK); 27560Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 27570Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 27580Sstevel@tonic-gate } else if (lgrp_optimizations() && (behav == MADV_ACCESS_LWP || 27590Sstevel@tonic-gate behav == MADV_ACCESS_MANY || behav == MADV_ACCESS_DEFAULT)) { 27600Sstevel@tonic-gate int already_set; 27610Sstevel@tonic-gate ulong_t anon_index; 27620Sstevel@tonic-gate lgrp_mem_policy_t policy; 27630Sstevel@tonic-gate caddr_t shm_addr; 27640Sstevel@tonic-gate size_t share_size; 27650Sstevel@tonic-gate size_t size; 27660Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 27670Sstevel@tonic-gate caddr_t sptseg_addr; 27680Sstevel@tonic-gate 27690Sstevel@tonic-gate /* 27700Sstevel@tonic-gate * Align address and length to page size of underlying segment 27710Sstevel@tonic-gate */ 27720Sstevel@tonic-gate share_size = page_get_pagesize(shmd->shm_sptseg->s_szc); 27730Sstevel@tonic-gate shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_size); 27740Sstevel@tonic-gate size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), 27750Sstevel@tonic-gate share_size); 27760Sstevel@tonic-gate 27770Sstevel@tonic-gate amp = shmd->shm_amp; 27780Sstevel@tonic-gate anon_index = seg_page(seg, shm_addr); 27790Sstevel@tonic-gate 27800Sstevel@tonic-gate /* 27810Sstevel@tonic-gate * And now we may have to adjust size downward if we have 27820Sstevel@tonic-gate * exceeded the realsize of the segment or initial anon 27830Sstevel@tonic-gate * allocations. 27840Sstevel@tonic-gate */ 27850Sstevel@tonic-gate sptseg_addr = sptseg->s_base + ptob(anon_index); 27860Sstevel@tonic-gate if ((sptseg_addr + size) > 27870Sstevel@tonic-gate (sptseg->s_base + sptd->spt_realsize)) 27880Sstevel@tonic-gate size = (sptseg->s_base + sptd->spt_realsize) - 27890Sstevel@tonic-gate sptseg_addr; 27900Sstevel@tonic-gate 27910Sstevel@tonic-gate /* 27920Sstevel@tonic-gate * Set memory allocation policy for this segment 27930Sstevel@tonic-gate */ 27940Sstevel@tonic-gate policy = lgrp_madv_to_policy(behav, len, MAP_SHARED); 27950Sstevel@tonic-gate already_set = lgrp_shm_policy_set(policy, amp, anon_index, 27960Sstevel@tonic-gate NULL, 0, len); 27970Sstevel@tonic-gate 27980Sstevel@tonic-gate /* 27990Sstevel@tonic-gate * If random memory allocation policy set already, 28000Sstevel@tonic-gate * don't bother reapplying it. 28010Sstevel@tonic-gate */ 28020Sstevel@tonic-gate if (already_set && !LGRP_MEM_POLICY_REAPPLICABLE(policy)) 28030Sstevel@tonic-gate return (0); 28040Sstevel@tonic-gate 28050Sstevel@tonic-gate /* 28060Sstevel@tonic-gate * Mark any existing pages in the given range for 28070Sstevel@tonic-gate * migration, flushing the I/O page cache, and using 28080Sstevel@tonic-gate * underlying segment to calculate anon index and get 28090Sstevel@tonic-gate * anonmap and vnode pointer from 28100Sstevel@tonic-gate */ 28110Sstevel@tonic-gate if (shmd->shm_softlockcnt > 0) 28120Sstevel@tonic-gate segspt_purge(seg); 28130Sstevel@tonic-gate 28140Sstevel@tonic-gate page_mark_migrate(seg, shm_addr, size, amp, 0, NULL, 0, 0); 28150Sstevel@tonic-gate } 28160Sstevel@tonic-gate 28170Sstevel@tonic-gate return (0); 28180Sstevel@tonic-gate } 28190Sstevel@tonic-gate 28200Sstevel@tonic-gate /*ARGSUSED*/ 28210Sstevel@tonic-gate void 28220Sstevel@tonic-gate segspt_shmdump(struct seg *seg) 28230Sstevel@tonic-gate { 28240Sstevel@tonic-gate /* no-op for ISM segment */ 28250Sstevel@tonic-gate } 28260Sstevel@tonic-gate 28270Sstevel@tonic-gate /*ARGSUSED*/ 28280Sstevel@tonic-gate static faultcode_t 28290Sstevel@tonic-gate segspt_shmsetpgsz(struct seg *seg, caddr_t addr, size_t len, uint_t szc) 28300Sstevel@tonic-gate { 28310Sstevel@tonic-gate return (ENOTSUP); 28320Sstevel@tonic-gate } 28330Sstevel@tonic-gate 28340Sstevel@tonic-gate /* 28350Sstevel@tonic-gate * get a memory ID for an addr in a given segment 28360Sstevel@tonic-gate */ 28370Sstevel@tonic-gate static int 28380Sstevel@tonic-gate segspt_shmgetmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 28390Sstevel@tonic-gate { 28400Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 28410Sstevel@tonic-gate struct anon *ap; 28420Sstevel@tonic-gate size_t anon_index; 28430Sstevel@tonic-gate struct anon_map *amp = shmd->shm_amp; 28440Sstevel@tonic-gate struct spt_data *sptd = shmd->shm_sptseg->s_data; 28450Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 28460Sstevel@tonic-gate anon_sync_obj_t cookie; 28470Sstevel@tonic-gate 28480Sstevel@tonic-gate anon_index = seg_page(seg, addr); 28490Sstevel@tonic-gate 28500Sstevel@tonic-gate if (addr > (seg->s_base + sptd->spt_realsize)) { 28510Sstevel@tonic-gate return (EFAULT); 28520Sstevel@tonic-gate } 28530Sstevel@tonic-gate 28540Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 28550Sstevel@tonic-gate anon_array_enter(amp, anon_index, &cookie); 28560Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index); 28570Sstevel@tonic-gate if (ap == NULL) { 28580Sstevel@tonic-gate struct page *pp; 28590Sstevel@tonic-gate caddr_t spt_addr = sptseg->s_base + ptob(anon_index); 28600Sstevel@tonic-gate 28610Sstevel@tonic-gate pp = anon_zero(sptseg, spt_addr, &ap, kcred); 28620Sstevel@tonic-gate if (pp == NULL) { 28630Sstevel@tonic-gate anon_array_exit(&cookie); 28640Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 28650Sstevel@tonic-gate return (ENOMEM); 28660Sstevel@tonic-gate } 28670Sstevel@tonic-gate (void) anon_set_ptr(amp->ahp, anon_index, ap, ANON_SLEEP); 28680Sstevel@tonic-gate page_unlock(pp); 28690Sstevel@tonic-gate } 28700Sstevel@tonic-gate anon_array_exit(&cookie); 28710Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 28720Sstevel@tonic-gate memidp->val[0] = (uintptr_t)ap; 28730Sstevel@tonic-gate memidp->val[1] = (uintptr_t)addr & PAGEOFFSET; 28740Sstevel@tonic-gate return (0); 28750Sstevel@tonic-gate } 28760Sstevel@tonic-gate 28770Sstevel@tonic-gate /* 28780Sstevel@tonic-gate * Get memory allocation policy info for specified address in given segment 28790Sstevel@tonic-gate */ 28800Sstevel@tonic-gate static lgrp_mem_policy_info_t * 28810Sstevel@tonic-gate segspt_shmgetpolicy(struct seg *seg, caddr_t addr) 28820Sstevel@tonic-gate { 28830Sstevel@tonic-gate struct anon_map *amp; 28840Sstevel@tonic-gate ulong_t anon_index; 28850Sstevel@tonic-gate lgrp_mem_policy_info_t *policy_info; 28860Sstevel@tonic-gate struct shm_data *shm_data; 28870Sstevel@tonic-gate 28880Sstevel@tonic-gate ASSERT(seg != NULL); 28890Sstevel@tonic-gate 28900Sstevel@tonic-gate /* 28910Sstevel@tonic-gate * Get anon_map from segshm 28920Sstevel@tonic-gate * 28930Sstevel@tonic-gate * Assume that no lock needs to be held on anon_map, since 28940Sstevel@tonic-gate * it should be protected by its reference count which must be 28950Sstevel@tonic-gate * nonzero for an existing segment 28960Sstevel@tonic-gate * Need to grab readers lock on policy tree though 28970Sstevel@tonic-gate */ 28980Sstevel@tonic-gate shm_data = (struct shm_data *)seg->s_data; 28990Sstevel@tonic-gate if (shm_data == NULL) 29000Sstevel@tonic-gate return (NULL); 29010Sstevel@tonic-gate amp = shm_data->shm_amp; 29020Sstevel@tonic-gate ASSERT(amp->refcnt != 0); 29030Sstevel@tonic-gate 29040Sstevel@tonic-gate /* 29050Sstevel@tonic-gate * Get policy info 29060Sstevel@tonic-gate * 29070Sstevel@tonic-gate * Assume starting anon index of 0 29080Sstevel@tonic-gate */ 29090Sstevel@tonic-gate anon_index = seg_page(seg, addr); 29100Sstevel@tonic-gate policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0); 29110Sstevel@tonic-gate 29120Sstevel@tonic-gate return (policy_info); 29130Sstevel@tonic-gate } 2914670Selowe 2915670Selowe /*ARGSUSED*/ 2916670Selowe static int 2917670Selowe segspt_shmcapable(struct seg *seg, segcapability_t capability) 2918670Selowe { 2919670Selowe return (0); 2920670Selowe } 2921