xref: /onnv-gate/usr/src/uts/common/vm/seg_spt.c (revision 3379:ab5e51073e69)
10Sstevel@tonic-gate /*
20Sstevel@tonic-gate  * CDDL HEADER START
30Sstevel@tonic-gate  *
40Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
51498Sbs21162  * Common Development and Distribution License (the "License").
61498Sbs21162  * You may not use this file except in compliance with the License.
70Sstevel@tonic-gate  *
80Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
90Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
100Sstevel@tonic-gate  * See the License for the specific language governing permissions
110Sstevel@tonic-gate  * and limitations under the License.
120Sstevel@tonic-gate  *
130Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
140Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
150Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
160Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
170Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
180Sstevel@tonic-gate  *
190Sstevel@tonic-gate  * CDDL HEADER END
200Sstevel@tonic-gate  */
210Sstevel@tonic-gate /*
22*3379Ssl108498  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
230Sstevel@tonic-gate  * Use is subject to license terms.
240Sstevel@tonic-gate  */
250Sstevel@tonic-gate 
260Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
270Sstevel@tonic-gate 
280Sstevel@tonic-gate #include <sys/param.h>
290Sstevel@tonic-gate #include <sys/user.h>
300Sstevel@tonic-gate #include <sys/mman.h>
310Sstevel@tonic-gate #include <sys/kmem.h>
320Sstevel@tonic-gate #include <sys/sysmacros.h>
330Sstevel@tonic-gate #include <sys/cmn_err.h>
340Sstevel@tonic-gate #include <sys/systm.h>
350Sstevel@tonic-gate #include <sys/tuneable.h>
360Sstevel@tonic-gate #include <vm/hat.h>
370Sstevel@tonic-gate #include <vm/seg.h>
380Sstevel@tonic-gate #include <vm/as.h>
390Sstevel@tonic-gate #include <vm/anon.h>
400Sstevel@tonic-gate #include <vm/page.h>
410Sstevel@tonic-gate #include <sys/buf.h>
420Sstevel@tonic-gate #include <sys/swap.h>
430Sstevel@tonic-gate #include <sys/atomic.h>
440Sstevel@tonic-gate #include <vm/seg_spt.h>
450Sstevel@tonic-gate #include <sys/debug.h>
460Sstevel@tonic-gate #include <sys/vtrace.h>
470Sstevel@tonic-gate #include <sys/shm.h>
482768Ssl108498 #include <sys/shm_impl.h>
490Sstevel@tonic-gate #include <sys/lgrp.h>
500Sstevel@tonic-gate #include <sys/vmsystm.h>
512768Ssl108498 #include <sys/policy.h>
522768Ssl108498 #include <sys/project.h>
530Sstevel@tonic-gate #include <sys/tnf_probe.h>
542768Ssl108498 #include <sys/zone.h>
550Sstevel@tonic-gate 
560Sstevel@tonic-gate #define	SEGSPTADDR	(caddr_t)0x0
570Sstevel@tonic-gate 
580Sstevel@tonic-gate /*
590Sstevel@tonic-gate  * # pages used for spt
600Sstevel@tonic-gate  */
610Sstevel@tonic-gate static size_t	spt_used;
620Sstevel@tonic-gate 
630Sstevel@tonic-gate /*
640Sstevel@tonic-gate  * segspt_minfree is the memory left for system after ISM
650Sstevel@tonic-gate  * locked its pages; it is set up to 5% of availrmem in
660Sstevel@tonic-gate  * sptcreate when ISM is created.  ISM should not use more
670Sstevel@tonic-gate  * than ~90% of availrmem; if it does, then the performance
680Sstevel@tonic-gate  * of the system may decrease. Machines with large memories may
690Sstevel@tonic-gate  * be able to use up more memory for ISM so we set the default
700Sstevel@tonic-gate  * segspt_minfree to 5% (which gives ISM max 95% of availrmem.
710Sstevel@tonic-gate  * If somebody wants even more memory for ISM (risking hanging
720Sstevel@tonic-gate  * the system) they can patch the segspt_minfree to smaller number.
730Sstevel@tonic-gate  */
740Sstevel@tonic-gate pgcnt_t segspt_minfree = 0;
750Sstevel@tonic-gate 
760Sstevel@tonic-gate static int segspt_create(struct seg *seg, caddr_t argsp);
770Sstevel@tonic-gate static int segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize);
780Sstevel@tonic-gate static void segspt_free(struct seg *seg);
790Sstevel@tonic-gate static void segspt_free_pages(struct seg *seg, caddr_t addr, size_t len);
800Sstevel@tonic-gate static lgrp_mem_policy_info_t *segspt_getpolicy(struct seg *seg, caddr_t addr);
810Sstevel@tonic-gate 
820Sstevel@tonic-gate static void
830Sstevel@tonic-gate segspt_badop()
840Sstevel@tonic-gate {
850Sstevel@tonic-gate 	panic("segspt_badop called");
860Sstevel@tonic-gate 	/*NOTREACHED*/
870Sstevel@tonic-gate }
880Sstevel@tonic-gate 
890Sstevel@tonic-gate #define	SEGSPT_BADOP(t)	(t(*)())segspt_badop
900Sstevel@tonic-gate 
910Sstevel@tonic-gate struct seg_ops segspt_ops = {
920Sstevel@tonic-gate 	SEGSPT_BADOP(int),		/* dup */
930Sstevel@tonic-gate 	segspt_unmap,
940Sstevel@tonic-gate 	segspt_free,
950Sstevel@tonic-gate 	SEGSPT_BADOP(int),		/* fault */
960Sstevel@tonic-gate 	SEGSPT_BADOP(faultcode_t),	/* faulta */
970Sstevel@tonic-gate 	SEGSPT_BADOP(int),		/* setprot */
980Sstevel@tonic-gate 	SEGSPT_BADOP(int),		/* checkprot */
990Sstevel@tonic-gate 	SEGSPT_BADOP(int),		/* kluster */
1000Sstevel@tonic-gate 	SEGSPT_BADOP(size_t),		/* swapout */
1010Sstevel@tonic-gate 	SEGSPT_BADOP(int),		/* sync */
1020Sstevel@tonic-gate 	SEGSPT_BADOP(size_t),		/* incore */
1030Sstevel@tonic-gate 	SEGSPT_BADOP(int),		/* lockop */
1040Sstevel@tonic-gate 	SEGSPT_BADOP(int),		/* getprot */
1050Sstevel@tonic-gate 	SEGSPT_BADOP(u_offset_t), 	/* getoffset */
1060Sstevel@tonic-gate 	SEGSPT_BADOP(int),		/* gettype */
1070Sstevel@tonic-gate 	SEGSPT_BADOP(int),		/* getvp */
1080Sstevel@tonic-gate 	SEGSPT_BADOP(int),		/* advise */
1090Sstevel@tonic-gate 	SEGSPT_BADOP(void),		/* dump */
1100Sstevel@tonic-gate 	SEGSPT_BADOP(int),		/* pagelock */
1110Sstevel@tonic-gate 	SEGSPT_BADOP(int),		/* setpgsz */
1120Sstevel@tonic-gate 	SEGSPT_BADOP(int),		/* getmemid */
1130Sstevel@tonic-gate 	segspt_getpolicy,		/* getpolicy */
114670Selowe 	SEGSPT_BADOP(int),		/* capable */
1150Sstevel@tonic-gate };
1160Sstevel@tonic-gate 
1170Sstevel@tonic-gate static int segspt_shmdup(struct seg *seg, struct seg *newseg);
1180Sstevel@tonic-gate static int segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize);
1190Sstevel@tonic-gate static void segspt_shmfree(struct seg *seg);
1200Sstevel@tonic-gate static faultcode_t segspt_shmfault(struct hat *hat, struct seg *seg,
1210Sstevel@tonic-gate 		caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw);
1220Sstevel@tonic-gate static faultcode_t segspt_shmfaulta(struct seg *seg, caddr_t addr);
1230Sstevel@tonic-gate static int segspt_shmsetprot(register struct seg *seg, register caddr_t addr,
1240Sstevel@tonic-gate 			register size_t len, register uint_t prot);
1250Sstevel@tonic-gate static int segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size,
1260Sstevel@tonic-gate 			uint_t prot);
1270Sstevel@tonic-gate static int	segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta);
1280Sstevel@tonic-gate static size_t	segspt_shmswapout(struct seg *seg);
1290Sstevel@tonic-gate static size_t segspt_shmincore(struct seg *seg, caddr_t addr, size_t len,
1300Sstevel@tonic-gate 			register char *vec);
1310Sstevel@tonic-gate static int segspt_shmsync(struct seg *seg, register caddr_t addr, size_t len,
1320Sstevel@tonic-gate 			int attr, uint_t flags);
1330Sstevel@tonic-gate static int segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len,
1340Sstevel@tonic-gate 			int attr, int op, ulong_t *lockmap, size_t pos);
1350Sstevel@tonic-gate static int segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len,
1360Sstevel@tonic-gate 			uint_t *protv);
1370Sstevel@tonic-gate static u_offset_t segspt_shmgetoffset(struct seg *seg, caddr_t addr);
1380Sstevel@tonic-gate static int segspt_shmgettype(struct seg *seg, caddr_t addr);
1390Sstevel@tonic-gate static int segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp);
1400Sstevel@tonic-gate static int segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len,
1410Sstevel@tonic-gate 			uint_t behav);
1420Sstevel@tonic-gate static void segspt_shmdump(struct seg *seg);
1430Sstevel@tonic-gate static int segspt_shmpagelock(struct seg *, caddr_t, size_t,
1440Sstevel@tonic-gate 			struct page ***, enum lock_type, enum seg_rw);
1450Sstevel@tonic-gate static int segspt_shmsetpgsz(struct seg *, caddr_t, size_t, uint_t);
1460Sstevel@tonic-gate static int segspt_shmgetmemid(struct seg *, caddr_t, memid_t *);
1470Sstevel@tonic-gate static lgrp_mem_policy_info_t *segspt_shmgetpolicy(struct seg *, caddr_t);
148670Selowe static int segspt_shmcapable(struct seg *, segcapability_t);
1490Sstevel@tonic-gate 
1500Sstevel@tonic-gate struct seg_ops segspt_shmops = {
1510Sstevel@tonic-gate 	segspt_shmdup,
1520Sstevel@tonic-gate 	segspt_shmunmap,
1530Sstevel@tonic-gate 	segspt_shmfree,
1540Sstevel@tonic-gate 	segspt_shmfault,
1550Sstevel@tonic-gate 	segspt_shmfaulta,
1560Sstevel@tonic-gate 	segspt_shmsetprot,
1570Sstevel@tonic-gate 	segspt_shmcheckprot,
1580Sstevel@tonic-gate 	segspt_shmkluster,
1590Sstevel@tonic-gate 	segspt_shmswapout,
1600Sstevel@tonic-gate 	segspt_shmsync,
1610Sstevel@tonic-gate 	segspt_shmincore,
1620Sstevel@tonic-gate 	segspt_shmlockop,
1630Sstevel@tonic-gate 	segspt_shmgetprot,
1640Sstevel@tonic-gate 	segspt_shmgetoffset,
1650Sstevel@tonic-gate 	segspt_shmgettype,
1660Sstevel@tonic-gate 	segspt_shmgetvp,
1670Sstevel@tonic-gate 	segspt_shmadvise,	/* advise */
1680Sstevel@tonic-gate 	segspt_shmdump,
1690Sstevel@tonic-gate 	segspt_shmpagelock,
1700Sstevel@tonic-gate 	segspt_shmsetpgsz,
1710Sstevel@tonic-gate 	segspt_shmgetmemid,
1720Sstevel@tonic-gate 	segspt_shmgetpolicy,
173670Selowe 	segspt_shmcapable,
1740Sstevel@tonic-gate };
1750Sstevel@tonic-gate 
1760Sstevel@tonic-gate static void segspt_purge(struct seg *seg);
1770Sstevel@tonic-gate static int segspt_reclaim(struct seg *, caddr_t, size_t, struct page **,
1780Sstevel@tonic-gate 		enum seg_rw);
1790Sstevel@tonic-gate static int spt_anon_getpages(struct seg *seg, caddr_t addr, size_t len,
1800Sstevel@tonic-gate 		page_t **ppa);
1810Sstevel@tonic-gate 
1820Sstevel@tonic-gate 
1830Sstevel@tonic-gate 
1840Sstevel@tonic-gate /*ARGSUSED*/
1850Sstevel@tonic-gate int
1860Sstevel@tonic-gate sptcreate(size_t size, struct seg **sptseg, struct anon_map *amp,
1872768Ssl108498 	uint_t prot, uint_t flags, uint_t share_szc)
1880Sstevel@tonic-gate {
1890Sstevel@tonic-gate 	int 	err;
1900Sstevel@tonic-gate 	struct  as	*newas;
1910Sstevel@tonic-gate 	struct	segspt_crargs sptcargs;
1920Sstevel@tonic-gate 
1930Sstevel@tonic-gate #ifdef DEBUG
1940Sstevel@tonic-gate 	TNF_PROBE_1(sptcreate, "spt", /* CSTYLED */,
1952768Ssl108498 			tnf_ulong, size, size );
1960Sstevel@tonic-gate #endif
1970Sstevel@tonic-gate 	if (segspt_minfree == 0)	/* leave min 5% of availrmem for */
1980Sstevel@tonic-gate 		segspt_minfree = availrmem/20;	/* for the system */
1990Sstevel@tonic-gate 
2000Sstevel@tonic-gate 	if (!hat_supported(HAT_SHARED_PT, (void *)0))
2010Sstevel@tonic-gate 		return (EINVAL);
2020Sstevel@tonic-gate 
2030Sstevel@tonic-gate 	/*
2040Sstevel@tonic-gate 	 * get a new as for this shared memory segment
2050Sstevel@tonic-gate 	 */
2060Sstevel@tonic-gate 	newas = as_alloc();
2072768Ssl108498 	newas->a_proc = NULL;
2080Sstevel@tonic-gate 	sptcargs.amp = amp;
2090Sstevel@tonic-gate 	sptcargs.prot = prot;
2100Sstevel@tonic-gate 	sptcargs.flags = flags;
2110Sstevel@tonic-gate 	sptcargs.szc = share_szc;
2120Sstevel@tonic-gate 	/*
2130Sstevel@tonic-gate 	 * create a shared page table (spt) segment
2140Sstevel@tonic-gate 	 */
2150Sstevel@tonic-gate 
2160Sstevel@tonic-gate 	if (err = as_map(newas, SEGSPTADDR, size, segspt_create, &sptcargs)) {
2170Sstevel@tonic-gate 		as_free(newas);
2180Sstevel@tonic-gate 		return (err);
2190Sstevel@tonic-gate 	}
2200Sstevel@tonic-gate 	*sptseg = sptcargs.seg_spt;
2210Sstevel@tonic-gate 	return (0);
2220Sstevel@tonic-gate }
2230Sstevel@tonic-gate 
2240Sstevel@tonic-gate void
2250Sstevel@tonic-gate sptdestroy(struct as *as, struct anon_map *amp)
2260Sstevel@tonic-gate {
2270Sstevel@tonic-gate 
2280Sstevel@tonic-gate #ifdef DEBUG
2290Sstevel@tonic-gate 	TNF_PROBE_0(sptdestroy, "spt", /* CSTYLED */);
2300Sstevel@tonic-gate #endif
2310Sstevel@tonic-gate 	(void) as_unmap(as, SEGSPTADDR, amp->size);
2320Sstevel@tonic-gate 	as_free(as);
2330Sstevel@tonic-gate }
2340Sstevel@tonic-gate 
2350Sstevel@tonic-gate /*
2360Sstevel@tonic-gate  * called from seg_free().
2370Sstevel@tonic-gate  * free (i.e., unlock, unmap, return to free list)
2380Sstevel@tonic-gate  *  all the pages in the given seg.
2390Sstevel@tonic-gate  */
2400Sstevel@tonic-gate void
2410Sstevel@tonic-gate segspt_free(struct seg	*seg)
2420Sstevel@tonic-gate {
2430Sstevel@tonic-gate 	struct spt_data *sptd = (struct spt_data *)seg->s_data;
2440Sstevel@tonic-gate 
2450Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
2460Sstevel@tonic-gate 
2470Sstevel@tonic-gate 	if (sptd != NULL) {
2480Sstevel@tonic-gate 		if (sptd->spt_realsize)
2490Sstevel@tonic-gate 			segspt_free_pages(seg, seg->s_base, sptd->spt_realsize);
2500Sstevel@tonic-gate 
2512768Ssl108498 	if (sptd->spt_ppa_lckcnt)
2522768Ssl108498 		kmem_free(sptd->spt_ppa_lckcnt,
2532768Ssl108498 		    sizeof (*sptd->spt_ppa_lckcnt)
2542768Ssl108498 		    * btopr(sptd->spt_amp->size));
2550Sstevel@tonic-gate 		kmem_free(sptd->spt_vp, sizeof (*sptd->spt_vp));
2560Sstevel@tonic-gate 		mutex_destroy(&sptd->spt_lock);
2570Sstevel@tonic-gate 		kmem_free(sptd, sizeof (*sptd));
2580Sstevel@tonic-gate 	}
2590Sstevel@tonic-gate }
2600Sstevel@tonic-gate 
2610Sstevel@tonic-gate /*ARGSUSED*/
2620Sstevel@tonic-gate static int
2630Sstevel@tonic-gate segspt_shmsync(struct seg *seg, caddr_t addr, size_t len, int attr,
2640Sstevel@tonic-gate 	uint_t flags)
2650Sstevel@tonic-gate {
2660Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2670Sstevel@tonic-gate 
2680Sstevel@tonic-gate 	return (0);
2690Sstevel@tonic-gate }
2700Sstevel@tonic-gate 
2710Sstevel@tonic-gate /*ARGSUSED*/
2720Sstevel@tonic-gate static size_t
2730Sstevel@tonic-gate segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, char *vec)
2740Sstevel@tonic-gate {
2750Sstevel@tonic-gate 	caddr_t	eo_seg;
2760Sstevel@tonic-gate 	pgcnt_t	npages;
2770Sstevel@tonic-gate 	struct shm_data *shmd = (struct shm_data *)seg->s_data;
2780Sstevel@tonic-gate 	struct seg	*sptseg;
2790Sstevel@tonic-gate 	struct spt_data *sptd;
2800Sstevel@tonic-gate 
2810Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2820Sstevel@tonic-gate #ifdef lint
2830Sstevel@tonic-gate 	seg = seg;
2840Sstevel@tonic-gate #endif
2850Sstevel@tonic-gate 	sptseg = shmd->shm_sptseg;
2860Sstevel@tonic-gate 	sptd = sptseg->s_data;
2870Sstevel@tonic-gate 
2880Sstevel@tonic-gate 	if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
2890Sstevel@tonic-gate 		eo_seg = addr + len;
2900Sstevel@tonic-gate 		while (addr < eo_seg) {
2910Sstevel@tonic-gate 			/* page exists, and it's locked. */
2920Sstevel@tonic-gate 			*vec++ = SEG_PAGE_INCORE | SEG_PAGE_LOCKED |
2930Sstevel@tonic-gate 				SEG_PAGE_ANON;
2940Sstevel@tonic-gate 			addr += PAGESIZE;
2950Sstevel@tonic-gate 		}
2960Sstevel@tonic-gate 		return (len);
2970Sstevel@tonic-gate 	} else {
2980Sstevel@tonic-gate 		struct  anon_map *amp = shmd->shm_amp;
2990Sstevel@tonic-gate 		struct  anon	*ap;
3000Sstevel@tonic-gate 		page_t		*pp;
3010Sstevel@tonic-gate 		pgcnt_t 	anon_index;
3020Sstevel@tonic-gate 		struct vnode 	*vp;
3030Sstevel@tonic-gate 		u_offset_t 	off;
3040Sstevel@tonic-gate 		ulong_t		i;
3050Sstevel@tonic-gate 		int		ret;
3060Sstevel@tonic-gate 		anon_sync_obj_t	cookie;
3070Sstevel@tonic-gate 
3080Sstevel@tonic-gate 		addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
3090Sstevel@tonic-gate 		anon_index = seg_page(seg, addr);
3100Sstevel@tonic-gate 		npages = btopr(len);
3110Sstevel@tonic-gate 		if (anon_index + npages > btopr(shmd->shm_amp->size)) {
3120Sstevel@tonic-gate 			return (EINVAL);
3130Sstevel@tonic-gate 		}
3140Sstevel@tonic-gate 		ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
3150Sstevel@tonic-gate 		for (i = 0; i < npages; i++, anon_index++) {
3160Sstevel@tonic-gate 			ret = 0;
3170Sstevel@tonic-gate 			anon_array_enter(amp, anon_index, &cookie);
3180Sstevel@tonic-gate 			ap = anon_get_ptr(amp->ahp, anon_index);
3190Sstevel@tonic-gate 			if (ap != NULL) {
3200Sstevel@tonic-gate 				swap_xlate(ap, &vp, &off);
3210Sstevel@tonic-gate 				anon_array_exit(&cookie);
3220Sstevel@tonic-gate 				pp = page_lookup_nowait(vp, off, SE_SHARED);
3230Sstevel@tonic-gate 				if (pp != NULL) {
3240Sstevel@tonic-gate 					ret |= SEG_PAGE_INCORE | SEG_PAGE_ANON;
3250Sstevel@tonic-gate 					page_unlock(pp);
3260Sstevel@tonic-gate 				}
3270Sstevel@tonic-gate 			} else {
3280Sstevel@tonic-gate 				anon_array_exit(&cookie);
3290Sstevel@tonic-gate 			}
3300Sstevel@tonic-gate 			if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) {
3310Sstevel@tonic-gate 				ret |= SEG_PAGE_LOCKED;
3320Sstevel@tonic-gate 			}
3330Sstevel@tonic-gate 			*vec++ = (char)ret;
3340Sstevel@tonic-gate 		}
3350Sstevel@tonic-gate 		ANON_LOCK_EXIT(&amp->a_rwlock);
3360Sstevel@tonic-gate 		return (len);
3370Sstevel@tonic-gate 	}
3380Sstevel@tonic-gate }
3390Sstevel@tonic-gate 
3400Sstevel@tonic-gate static int
3410Sstevel@tonic-gate segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize)
3420Sstevel@tonic-gate {
3430Sstevel@tonic-gate 	size_t share_size;
3440Sstevel@tonic-gate 
3450Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
3460Sstevel@tonic-gate 
3470Sstevel@tonic-gate 	/*
3480Sstevel@tonic-gate 	 * seg.s_size may have been rounded up to the largest page size
3490Sstevel@tonic-gate 	 * in shmat().
3500Sstevel@tonic-gate 	 * XXX This should be cleanedup. sptdestroy should take a length
3510Sstevel@tonic-gate 	 * argument which should be the same as sptcreate. Then
3520Sstevel@tonic-gate 	 * this rounding would not be needed (or is done in shm.c)
3530Sstevel@tonic-gate 	 * Only the check for full segment will be needed.
3540Sstevel@tonic-gate 	 *
3550Sstevel@tonic-gate 	 * XXX -- shouldn't raddr == 0 always? These tests don't seem
3560Sstevel@tonic-gate 	 * to be useful at all.
3570Sstevel@tonic-gate 	 */
3580Sstevel@tonic-gate 	share_size = page_get_pagesize(seg->s_szc);
3590Sstevel@tonic-gate 	ssize = P2ROUNDUP(ssize, share_size);
3600Sstevel@tonic-gate 
3610Sstevel@tonic-gate 	if (raddr == seg->s_base && ssize == seg->s_size) {
3620Sstevel@tonic-gate 		seg_free(seg);
3630Sstevel@tonic-gate 		return (0);
3640Sstevel@tonic-gate 	} else
3650Sstevel@tonic-gate 		return (EINVAL);
3660Sstevel@tonic-gate }
3670Sstevel@tonic-gate 
3680Sstevel@tonic-gate int
3690Sstevel@tonic-gate segspt_create(struct seg *seg, caddr_t argsp)
3700Sstevel@tonic-gate {
3710Sstevel@tonic-gate 	int		err;
3720Sstevel@tonic-gate 	caddr_t		addr = seg->s_base;
3730Sstevel@tonic-gate 	struct spt_data *sptd;
3740Sstevel@tonic-gate 	struct 	segspt_crargs *sptcargs = (struct segspt_crargs *)argsp;
3750Sstevel@tonic-gate 	struct anon_map *amp = sptcargs->amp;
3762768Ssl108498 	struct kshmid	*sp = amp->a_sp;
3770Sstevel@tonic-gate 	struct	cred	*cred = CRED();
3780Sstevel@tonic-gate 	ulong_t		i, j, anon_index = 0;
3790Sstevel@tonic-gate 	pgcnt_t		npages = btopr(amp->size);
3800Sstevel@tonic-gate 	struct vnode	*vp;
3810Sstevel@tonic-gate 	page_t		**ppa;
3820Sstevel@tonic-gate 	uint_t		hat_flags;
3832414Saguzovsk 	size_t		pgsz;
3842414Saguzovsk 	pgcnt_t		pgcnt;
3852414Saguzovsk 	caddr_t		a;
3862414Saguzovsk 	pgcnt_t		pidx;
3872414Saguzovsk 	size_t		sz;
3882768Ssl108498 	proc_t		*procp = curproc;
3892768Ssl108498 	rctl_qty_t	lockedbytes = 0;
3902768Ssl108498 	kproject_t	*proj;
3910Sstevel@tonic-gate 
3920Sstevel@tonic-gate 	/*
3930Sstevel@tonic-gate 	 * We are holding the a_lock on the underlying dummy as,
3940Sstevel@tonic-gate 	 * so we can make calls to the HAT layer.
3950Sstevel@tonic-gate 	 */
3960Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
3972768Ssl108498 	ASSERT(sp != NULL);
3980Sstevel@tonic-gate 
3990Sstevel@tonic-gate #ifdef DEBUG
4000Sstevel@tonic-gate 	TNF_PROBE_2(segspt_create, "spt", /* CSTYLED */,
4012768Ssl108498 				tnf_opaque, addr, addr,
4020Sstevel@tonic-gate 				tnf_ulong, len, seg->s_size);
4030Sstevel@tonic-gate #endif
4040Sstevel@tonic-gate 	if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
4050Sstevel@tonic-gate 		if (err = anon_swap_adjust(npages))
4060Sstevel@tonic-gate 			return (err);
4070Sstevel@tonic-gate 	}
4080Sstevel@tonic-gate 	err = ENOMEM;
4090Sstevel@tonic-gate 
4100Sstevel@tonic-gate 	if ((sptd = kmem_zalloc(sizeof (*sptd), KM_NOSLEEP)) == NULL)
4110Sstevel@tonic-gate 		goto out1;
4120Sstevel@tonic-gate 
4130Sstevel@tonic-gate 	if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
4140Sstevel@tonic-gate 		if ((ppa = kmem_zalloc(((sizeof (page_t *)) * npages),
4150Sstevel@tonic-gate 		    KM_NOSLEEP)) == NULL)
4160Sstevel@tonic-gate 			goto out2;
4170Sstevel@tonic-gate 	}
4180Sstevel@tonic-gate 
4190Sstevel@tonic-gate 	mutex_init(&sptd->spt_lock, NULL, MUTEX_DEFAULT, NULL);
4200Sstevel@tonic-gate 
4210Sstevel@tonic-gate 	if ((vp = kmem_zalloc(sizeof (*vp), KM_NOSLEEP)) == NULL)
4220Sstevel@tonic-gate 		goto out3;
4230Sstevel@tonic-gate 
4240Sstevel@tonic-gate 	seg->s_ops = &segspt_ops;
4250Sstevel@tonic-gate 	sptd->spt_vp = vp;
4260Sstevel@tonic-gate 	sptd->spt_amp = amp;
4270Sstevel@tonic-gate 	sptd->spt_prot = sptcargs->prot;
4280Sstevel@tonic-gate 	sptd->spt_flags = sptcargs->flags;
4290Sstevel@tonic-gate 	seg->s_data = (caddr_t)sptd;
4300Sstevel@tonic-gate 	sptd->spt_ppa = NULL;
4310Sstevel@tonic-gate 	sptd->spt_ppa_lckcnt = NULL;
4320Sstevel@tonic-gate 	seg->s_szc = sptcargs->szc;
4330Sstevel@tonic-gate 
4340Sstevel@tonic-gate 	ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
4352414Saguzovsk 	if (seg->s_szc > amp->a_szc) {
4362414Saguzovsk 		amp->a_szc = seg->s_szc;
4372414Saguzovsk 	}
4380Sstevel@tonic-gate 	ANON_LOCK_EXIT(&amp->a_rwlock);
4390Sstevel@tonic-gate 
4400Sstevel@tonic-gate 	/*
4410Sstevel@tonic-gate 	 * Set policy to affect initial allocation of pages in
4420Sstevel@tonic-gate 	 * anon_map_createpages()
4430Sstevel@tonic-gate 	 */
4440Sstevel@tonic-gate 	(void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, amp, anon_index,
4450Sstevel@tonic-gate 	    NULL, 0, ptob(npages));
4460Sstevel@tonic-gate 
4470Sstevel@tonic-gate 	if (sptcargs->flags & SHM_PAGEABLE) {
4480Sstevel@tonic-gate 		size_t  share_sz;
4490Sstevel@tonic-gate 		pgcnt_t new_npgs, more_pgs;
4500Sstevel@tonic-gate 		struct anon_hdr *nahp;
451*3379Ssl108498 		zone_t *zone;
4520Sstevel@tonic-gate 
4530Sstevel@tonic-gate 		share_sz = page_get_pagesize(seg->s_szc);
4540Sstevel@tonic-gate 		if (!IS_P2ALIGNED(amp->size, share_sz)) {
4550Sstevel@tonic-gate 			/*
4560Sstevel@tonic-gate 			 * We are rounding up the size of the anon array
4570Sstevel@tonic-gate 			 * on 4 M boundary because we always create 4 M
4580Sstevel@tonic-gate 			 * of page(s) when locking, faulting pages and we
4590Sstevel@tonic-gate 			 * don't have to check for all corner cases e.g.
4600Sstevel@tonic-gate 			 * if there is enough space to allocate 4 M
4610Sstevel@tonic-gate 			 * page.
4620Sstevel@tonic-gate 			 */
4630Sstevel@tonic-gate 			new_npgs = btop(P2ROUNDUP(amp->size, share_sz));
4640Sstevel@tonic-gate 			more_pgs = new_npgs - npages;
4650Sstevel@tonic-gate 
466*3379Ssl108498 			/*
467*3379Ssl108498 			 * This may return NULL if global zone is removing a
468*3379Ssl108498 			 * shm created by a non-global zone that has been
469*3379Ssl108498 			 * destroyed.
470*3379Ssl108498 			 */
471*3379Ssl108498 			zone =
472*3379Ssl108498 			    zone_find_by_id(sp->shm_perm.ipc_proj->kpj_zoneid);
473*3379Ssl108498 
474*3379Ssl108498 			if (anon_resv_zone(ptob(more_pgs), zone) == 0) {
475*3379Ssl108498 				if (zone != NULL)
476*3379Ssl108498 					zone_rele(zone);
4770Sstevel@tonic-gate 				err = ENOMEM;
4780Sstevel@tonic-gate 				goto out4;
4790Sstevel@tonic-gate 			}
480*3379Ssl108498 			if (zone != NULL)
481*3379Ssl108498 				zone_rele(zone);
482*3379Ssl108498 
4830Sstevel@tonic-gate 			nahp = anon_create(new_npgs, ANON_SLEEP);
4840Sstevel@tonic-gate 			ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
4850Sstevel@tonic-gate 			(void) anon_copy_ptr(amp->ahp, 0, nahp, 0, npages,
4860Sstevel@tonic-gate 			    ANON_SLEEP);
4870Sstevel@tonic-gate 			anon_release(amp->ahp, npages);
4880Sstevel@tonic-gate 			amp->ahp = nahp;
489*3379Ssl108498 			ASSERT(amp->swresv == ptob(npages));
4900Sstevel@tonic-gate 			amp->swresv = amp->size = ptob(new_npgs);
4910Sstevel@tonic-gate 			ANON_LOCK_EXIT(&amp->a_rwlock);
4920Sstevel@tonic-gate 			npages = new_npgs;
4930Sstevel@tonic-gate 		}
4940Sstevel@tonic-gate 
4950Sstevel@tonic-gate 		sptd->spt_ppa_lckcnt = kmem_zalloc(npages *
4960Sstevel@tonic-gate 		    sizeof (*sptd->spt_ppa_lckcnt), KM_SLEEP);
4970Sstevel@tonic-gate 		sptd->spt_pcachecnt = 0;
4980Sstevel@tonic-gate 		sptd->spt_realsize = ptob(npages);
4990Sstevel@tonic-gate 		sptcargs->seg_spt = seg;
5000Sstevel@tonic-gate 		return (0);
5010Sstevel@tonic-gate 	}
5020Sstevel@tonic-gate 
5030Sstevel@tonic-gate 	/*
5040Sstevel@tonic-gate 	 * get array of pages for each anon slot in amp
5050Sstevel@tonic-gate 	 */
5060Sstevel@tonic-gate 	if ((err = anon_map_createpages(amp, anon_index, ptob(npages), ppa,
5070Sstevel@tonic-gate 	    seg, addr, S_CREATE, cred)) != 0)
5080Sstevel@tonic-gate 		goto out4;
5090Sstevel@tonic-gate 
5102768Ssl108498 	mutex_enter(&sp->shm_mlock);
5112768Ssl108498 
5122768Ssl108498 	/* May be partially locked, so, count bytes to charge for locking */
5132768Ssl108498 	for (i = 0; i < npages; i++)
5142768Ssl108498 		if (ppa[i]->p_lckcnt == 0)
5152768Ssl108498 			lockedbytes += PAGESIZE;
5162768Ssl108498 
5172768Ssl108498 	proj = sp->shm_perm.ipc_proj;
5182768Ssl108498 
5192768Ssl108498 	if (lockedbytes > 0) {
5202768Ssl108498 		mutex_enter(&procp->p_lock);
5212768Ssl108498 		if (rctl_incr_locked_mem(procp, proj, lockedbytes, 0)) {
5222768Ssl108498 			mutex_exit(&procp->p_lock);
5232768Ssl108498 			mutex_exit(&sp->shm_mlock);
5242768Ssl108498 			for (i = 0; i < npages; i++)
5252768Ssl108498 				page_unlock(ppa[i]);
5262768Ssl108498 			err = ENOMEM;
5272768Ssl108498 			goto out4;
5282768Ssl108498 		}
5292768Ssl108498 		mutex_exit(&procp->p_lock);
5302768Ssl108498 	}
5312768Ssl108498 
5320Sstevel@tonic-gate 	/*
5330Sstevel@tonic-gate 	 * addr is initial address corresponding to the first page on ppa list
5340Sstevel@tonic-gate 	 */
5350Sstevel@tonic-gate 	for (i = 0; i < npages; i++) {
5360Sstevel@tonic-gate 		/* attempt to lock all pages */
5372768Ssl108498 		if (page_pp_lock(ppa[i], 0, 1) == 0) {
5380Sstevel@tonic-gate 			/*
5390Sstevel@tonic-gate 			 * if unable to lock any page, unlock all
5400Sstevel@tonic-gate 			 * of them and return error
5410Sstevel@tonic-gate 			 */
5420Sstevel@tonic-gate 			for (j = 0; j < i; j++)
5430Sstevel@tonic-gate 				page_pp_unlock(ppa[j], 0, 1);
5442768Ssl108498 			for (i = 0; i < npages; i++)
5450Sstevel@tonic-gate 				page_unlock(ppa[i]);
5462768Ssl108498 			rctl_decr_locked_mem(NULL, proj, lockedbytes, 0);
5472768Ssl108498 			mutex_exit(&sp->shm_mlock);
5480Sstevel@tonic-gate 			err = ENOMEM;
5490Sstevel@tonic-gate 			goto out4;
5500Sstevel@tonic-gate 		}
5510Sstevel@tonic-gate 	}
5522768Ssl108498 	mutex_exit(&sp->shm_mlock);
5530Sstevel@tonic-gate 
5540Sstevel@tonic-gate 	/*
5550Sstevel@tonic-gate 	 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK
5560Sstevel@tonic-gate 	 * for the entire life of the segment. For example platforms
5570Sstevel@tonic-gate 	 * that do not support Dynamic Reconfiguration.
5580Sstevel@tonic-gate 	 */
5590Sstevel@tonic-gate 	hat_flags = HAT_LOAD_SHARE;
5600Sstevel@tonic-gate 	if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL))
5610Sstevel@tonic-gate 		hat_flags |= HAT_LOAD_LOCK;
5620Sstevel@tonic-gate 
5632414Saguzovsk 	/*
5642414Saguzovsk 	 * Load translations one lare page at a time
5652414Saguzovsk 	 * to make sure we don't create mappings bigger than
5662414Saguzovsk 	 * segment's size code in case underlying pages
5672414Saguzovsk 	 * are shared with segvn's segment that uses bigger
5682414Saguzovsk 	 * size code than we do.
5692414Saguzovsk 	 */
5702414Saguzovsk 	pgsz = page_get_pagesize(seg->s_szc);
5712414Saguzovsk 	pgcnt = page_get_pagecnt(seg->s_szc);
5722414Saguzovsk 	for (a = addr, pidx = 0; pidx < npages; a += pgsz, pidx += pgcnt) {
5732414Saguzovsk 		sz = MIN(pgsz, ptob(npages - pidx));
5742414Saguzovsk 		hat_memload_array(seg->s_as->a_hat, a, sz,
5752414Saguzovsk 		    &ppa[pidx], sptd->spt_prot, hat_flags);
5762414Saguzovsk 	}
5770Sstevel@tonic-gate 
5780Sstevel@tonic-gate 	/*
5790Sstevel@tonic-gate 	 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP,
5800Sstevel@tonic-gate 	 * we will leave the pages locked SE_SHARED for the life
5810Sstevel@tonic-gate 	 * of the ISM segment. This will prevent any calls to
5820Sstevel@tonic-gate 	 * hat_pageunload() on this ISM segment for those platforms.
5830Sstevel@tonic-gate 	 */
5840Sstevel@tonic-gate 	if (!(hat_flags & HAT_LOAD_LOCK)) {
5850Sstevel@tonic-gate 		/*
5860Sstevel@tonic-gate 		 * On platforms that support HAT_DYNAMIC_ISM_UNMAP,
5870Sstevel@tonic-gate 		 * we no longer need to hold the SE_SHARED lock on the pages,
5880Sstevel@tonic-gate 		 * since L_PAGELOCK and F_SOFTLOCK calls will grab the
5890Sstevel@tonic-gate 		 * SE_SHARED lock on the pages as necessary.
5900Sstevel@tonic-gate 		 */
5910Sstevel@tonic-gate 		for (i = 0; i < npages; i++)
5920Sstevel@tonic-gate 			page_unlock(ppa[i]);
5930Sstevel@tonic-gate 	}
5940Sstevel@tonic-gate 	sptd->spt_pcachecnt = 0;
5950Sstevel@tonic-gate 	kmem_free(ppa, ((sizeof (page_t *)) * npages));
5960Sstevel@tonic-gate 	sptd->spt_realsize = ptob(npages);
5970Sstevel@tonic-gate 	atomic_add_long(&spt_used, npages);
5980Sstevel@tonic-gate 	sptcargs->seg_spt = seg;
5990Sstevel@tonic-gate 	return (0);
6000Sstevel@tonic-gate 
6010Sstevel@tonic-gate out4:
6020Sstevel@tonic-gate 	seg->s_data = NULL;
6030Sstevel@tonic-gate 	kmem_free(vp, sizeof (*vp));
6040Sstevel@tonic-gate out3:
6050Sstevel@tonic-gate 	mutex_destroy(&sptd->spt_lock);
6060Sstevel@tonic-gate 	if ((sptcargs->flags & SHM_PAGEABLE) == 0)
6070Sstevel@tonic-gate 		kmem_free(ppa, (sizeof (*ppa) * npages));
6080Sstevel@tonic-gate out2:
6090Sstevel@tonic-gate 	kmem_free(sptd, sizeof (*sptd));
6100Sstevel@tonic-gate out1:
6110Sstevel@tonic-gate 	if ((sptcargs->flags & SHM_PAGEABLE) == 0)
6120Sstevel@tonic-gate 		anon_swap_restore(npages);
6130Sstevel@tonic-gate 	return (err);
6140Sstevel@tonic-gate }
6150Sstevel@tonic-gate 
6160Sstevel@tonic-gate /*ARGSUSED*/
6170Sstevel@tonic-gate void
6180Sstevel@tonic-gate segspt_free_pages(struct seg *seg, caddr_t addr, size_t len)
6190Sstevel@tonic-gate {
6200Sstevel@tonic-gate 	struct page 	*pp;
6210Sstevel@tonic-gate 	struct spt_data *sptd = (struct spt_data *)seg->s_data;
6220Sstevel@tonic-gate 	pgcnt_t		npages;
6230Sstevel@tonic-gate 	ulong_t		anon_idx;
6240Sstevel@tonic-gate 	struct anon_map *amp;
6250Sstevel@tonic-gate 	struct anon 	*ap;
6260Sstevel@tonic-gate 	struct vnode 	*vp;
6270Sstevel@tonic-gate 	u_offset_t 	off;
6280Sstevel@tonic-gate 	uint_t		hat_flags;
6290Sstevel@tonic-gate 	int		root = 0;
6300Sstevel@tonic-gate 	pgcnt_t		pgs, curnpgs = 0;
6310Sstevel@tonic-gate 	page_t		*rootpp;
6322768Ssl108498 	rctl_qty_t	unlocked_bytes = 0;
6332768Ssl108498 	kproject_t	*proj;
6342768Ssl108498 	kshmid_t	*sp;
6350Sstevel@tonic-gate 
6360Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
6370Sstevel@tonic-gate 
6380Sstevel@tonic-gate 	len = P2ROUNDUP(len, PAGESIZE);
6390Sstevel@tonic-gate 
6400Sstevel@tonic-gate 	npages = btop(len);
6410Sstevel@tonic-gate 
6420Sstevel@tonic-gate 	hat_flags = HAT_UNLOAD_UNLOCK;
6430Sstevel@tonic-gate 	if ((hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) ||
6440Sstevel@tonic-gate 	    (sptd->spt_flags & SHM_PAGEABLE)) {
6450Sstevel@tonic-gate 		hat_flags = HAT_UNLOAD;
6460Sstevel@tonic-gate 	}
6470Sstevel@tonic-gate 
6480Sstevel@tonic-gate 	hat_unload(seg->s_as->a_hat, addr, len, hat_flags);
6490Sstevel@tonic-gate 
6500Sstevel@tonic-gate 	amp = sptd->spt_amp;
6510Sstevel@tonic-gate 	if (sptd->spt_flags & SHM_PAGEABLE)
6520Sstevel@tonic-gate 		npages = btop(amp->size);
6530Sstevel@tonic-gate 
6542768Ssl108498 	ASSERT(amp != NULL);
6552768Ssl108498 
6562768Ssl108498 	if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
6572768Ssl108498 		sp = amp->a_sp;
6582768Ssl108498 		proj = sp->shm_perm.ipc_proj;
6592768Ssl108498 		mutex_enter(&sp->shm_mlock);
6602768Ssl108498 	}
6610Sstevel@tonic-gate 	for (anon_idx = 0; anon_idx < npages; anon_idx++) {
6620Sstevel@tonic-gate 		if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
6630Sstevel@tonic-gate 			if ((ap = anon_get_ptr(amp->ahp, anon_idx)) == NULL) {
6640Sstevel@tonic-gate 				panic("segspt_free_pages: null app");
6650Sstevel@tonic-gate 				/*NOTREACHED*/
6660Sstevel@tonic-gate 			}
6670Sstevel@tonic-gate 		} else {
6680Sstevel@tonic-gate 			if ((ap = anon_get_next_ptr(amp->ahp, &anon_idx))
6690Sstevel@tonic-gate 			    == NULL)
6700Sstevel@tonic-gate 				continue;
6710Sstevel@tonic-gate 		}
6720Sstevel@tonic-gate 		ASSERT(ANON_ISBUSY(anon_get_slot(amp->ahp, anon_idx)) == 0);
6730Sstevel@tonic-gate 		swap_xlate(ap, &vp, &off);
6740Sstevel@tonic-gate 
6750Sstevel@tonic-gate 		/*
6760Sstevel@tonic-gate 		 * If this platform supports HAT_DYNAMIC_ISM_UNMAP,
6770Sstevel@tonic-gate 		 * the pages won't be having SE_SHARED lock at this
6780Sstevel@tonic-gate 		 * point.
6790Sstevel@tonic-gate 		 *
6800Sstevel@tonic-gate 		 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP,
6810Sstevel@tonic-gate 		 * the pages are still held SE_SHARED locked from the
6820Sstevel@tonic-gate 		 * original segspt_create()
6830Sstevel@tonic-gate 		 *
6840Sstevel@tonic-gate 		 * Our goal is to get SE_EXCL lock on each page, remove
6850Sstevel@tonic-gate 		 * permanent lock on it and invalidate the page.
6860Sstevel@tonic-gate 		 */
6870Sstevel@tonic-gate 		if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
6880Sstevel@tonic-gate 			if (hat_flags == HAT_UNLOAD)
6890Sstevel@tonic-gate 				pp = page_lookup(vp, off, SE_EXCL);
6900Sstevel@tonic-gate 			else {
6910Sstevel@tonic-gate 				if ((pp = page_find(vp, off)) == NULL) {
6920Sstevel@tonic-gate 					panic("segspt_free_pages: "
6930Sstevel@tonic-gate 					    "page not locked");
6940Sstevel@tonic-gate 					/*NOTREACHED*/
6950Sstevel@tonic-gate 				}
6960Sstevel@tonic-gate 				if (!page_tryupgrade(pp)) {
6970Sstevel@tonic-gate 					page_unlock(pp);
6980Sstevel@tonic-gate 					pp = page_lookup(vp, off, SE_EXCL);
6990Sstevel@tonic-gate 				}
7000Sstevel@tonic-gate 			}
7010Sstevel@tonic-gate 			if (pp == NULL) {
7020Sstevel@tonic-gate 				panic("segspt_free_pages: "
7030Sstevel@tonic-gate 				    "page not in the system");
7040Sstevel@tonic-gate 				/*NOTREACHED*/
7050Sstevel@tonic-gate 			}
7062768Ssl108498 			ASSERT(pp->p_lckcnt > 0);
7070Sstevel@tonic-gate 			page_pp_unlock(pp, 0, 1);
7082768Ssl108498 			if (pp->p_lckcnt == 0)
7092768Ssl108498 				    unlocked_bytes += PAGESIZE;
7100Sstevel@tonic-gate 		} else {
7110Sstevel@tonic-gate 			if ((pp = page_lookup(vp, off, SE_EXCL)) == NULL)
7120Sstevel@tonic-gate 				continue;
7130Sstevel@tonic-gate 		}
7140Sstevel@tonic-gate 		/*
7150Sstevel@tonic-gate 		 * It's logical to invalidate the pages here as in most cases
7160Sstevel@tonic-gate 		 * these were created by segspt.
7170Sstevel@tonic-gate 		 */
7180Sstevel@tonic-gate 		if (pp->p_szc != 0) {
7190Sstevel@tonic-gate 			/*
7200Sstevel@tonic-gate 			 * For DISM swap is released in shm_rm_amp.
7210Sstevel@tonic-gate 			 */
7220Sstevel@tonic-gate 			if ((sptd->spt_flags & SHM_PAGEABLE) == 0 &&
7230Sstevel@tonic-gate 			    ap->an_pvp != NULL) {
7240Sstevel@tonic-gate 				panic("segspt_free_pages: pvp non NULL");
7250Sstevel@tonic-gate 				/*NOTREACHED*/
7260Sstevel@tonic-gate 			}
7270Sstevel@tonic-gate 			if (root == 0) {
7280Sstevel@tonic-gate 				ASSERT(curnpgs == 0);
7290Sstevel@tonic-gate 				root = 1;
7300Sstevel@tonic-gate 				rootpp = pp;
7310Sstevel@tonic-gate 				pgs = curnpgs = page_get_pagecnt(pp->p_szc);
7320Sstevel@tonic-gate 				ASSERT(pgs > 1);
7330Sstevel@tonic-gate 				ASSERT(IS_P2ALIGNED(pgs, pgs));
7340Sstevel@tonic-gate 				ASSERT(!(page_pptonum(pp) & (pgs - 1)));
7350Sstevel@tonic-gate 				curnpgs--;
7360Sstevel@tonic-gate 			} else if ((page_pptonum(pp) & (pgs - 1)) == pgs - 1) {
7370Sstevel@tonic-gate 				ASSERT(curnpgs == 1);
7380Sstevel@tonic-gate 				ASSERT(page_pptonum(pp) ==
7390Sstevel@tonic-gate 				    page_pptonum(rootpp) + (pgs - 1));
7400Sstevel@tonic-gate 				page_destroy_pages(rootpp);
7410Sstevel@tonic-gate 				root = 0;
7420Sstevel@tonic-gate 				curnpgs = 0;
7430Sstevel@tonic-gate 			} else {
7440Sstevel@tonic-gate 				ASSERT(curnpgs > 1);
7450Sstevel@tonic-gate 				ASSERT(page_pptonum(pp) ==
7460Sstevel@tonic-gate 				    page_pptonum(rootpp) + (pgs - curnpgs));
7470Sstevel@tonic-gate 				curnpgs--;
7480Sstevel@tonic-gate 			}
7490Sstevel@tonic-gate 		} else {
7500Sstevel@tonic-gate 			if (root != 0 || curnpgs != 0) {
7510Sstevel@tonic-gate 				panic("segspt_free_pages: bad large page");
7520Sstevel@tonic-gate 				/*NOTREACHED*/
7530Sstevel@tonic-gate 			}
7540Sstevel@tonic-gate 			/*LINTED: constant in conditional context */
7550Sstevel@tonic-gate 			VN_DISPOSE(pp, B_INVAL, 0, kcred);
7560Sstevel@tonic-gate 		}
7570Sstevel@tonic-gate 	}
7582768Ssl108498 	if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
7592768Ssl108498 		if (unlocked_bytes > 0)
7602768Ssl108498 			rctl_decr_locked_mem(NULL, proj, unlocked_bytes, 0);
7612768Ssl108498 		mutex_exit(&sp->shm_mlock);
7622768Ssl108498 	}
7630Sstevel@tonic-gate 	if (root != 0 || curnpgs != 0) {
7640Sstevel@tonic-gate 		panic("segspt_free_pages: bad large page");
7650Sstevel@tonic-gate 		/*NOTREACHED*/
7660Sstevel@tonic-gate 	}
7670Sstevel@tonic-gate 
7680Sstevel@tonic-gate 	/*
7690Sstevel@tonic-gate 	 * mark that pages have been released
7700Sstevel@tonic-gate 	 */
7710Sstevel@tonic-gate 	sptd->spt_realsize = 0;
7720Sstevel@tonic-gate 
7730Sstevel@tonic-gate 	if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
7740Sstevel@tonic-gate 		atomic_add_long(&spt_used, -npages);
7750Sstevel@tonic-gate 		anon_swap_restore(npages);
7760Sstevel@tonic-gate 	}
7770Sstevel@tonic-gate }
7780Sstevel@tonic-gate 
7790Sstevel@tonic-gate /*
7800Sstevel@tonic-gate  * Get memory allocation policy info for specified address in given segment
7810Sstevel@tonic-gate  */
7820Sstevel@tonic-gate static lgrp_mem_policy_info_t *
7830Sstevel@tonic-gate segspt_getpolicy(struct seg *seg, caddr_t addr)
7840Sstevel@tonic-gate {
7850Sstevel@tonic-gate 	struct anon_map		*amp;
7860Sstevel@tonic-gate 	ulong_t			anon_index;
7870Sstevel@tonic-gate 	lgrp_mem_policy_info_t	*policy_info;
7880Sstevel@tonic-gate 	struct spt_data		*spt_data;
7890Sstevel@tonic-gate 
7900Sstevel@tonic-gate 	ASSERT(seg != NULL);
7910Sstevel@tonic-gate 
7920Sstevel@tonic-gate 	/*
7930Sstevel@tonic-gate 	 * Get anon_map from segspt
7940Sstevel@tonic-gate 	 *
7950Sstevel@tonic-gate 	 * Assume that no lock needs to be held on anon_map, since
7960Sstevel@tonic-gate 	 * it should be protected by its reference count which must be
7970Sstevel@tonic-gate 	 * nonzero for an existing segment
7980Sstevel@tonic-gate 	 * Need to grab readers lock on policy tree though
7990Sstevel@tonic-gate 	 */
8000Sstevel@tonic-gate 	spt_data = (struct spt_data *)seg->s_data;
8010Sstevel@tonic-gate 	if (spt_data == NULL)
8020Sstevel@tonic-gate 		return (NULL);
8030Sstevel@tonic-gate 	amp = spt_data->spt_amp;
8040Sstevel@tonic-gate 	ASSERT(amp->refcnt != 0);
8050Sstevel@tonic-gate 
8060Sstevel@tonic-gate 	/*
8070Sstevel@tonic-gate 	 * Get policy info
8080Sstevel@tonic-gate 	 *
8090Sstevel@tonic-gate 	 * Assume starting anon index of 0
8100Sstevel@tonic-gate 	 */
8110Sstevel@tonic-gate 	anon_index = seg_page(seg, addr);
8120Sstevel@tonic-gate 	policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0);
8130Sstevel@tonic-gate 
8140Sstevel@tonic-gate 	return (policy_info);
8150Sstevel@tonic-gate }
8160Sstevel@tonic-gate 
8170Sstevel@tonic-gate /*
8180Sstevel@tonic-gate  * DISM only.
8190Sstevel@tonic-gate  * Return locked pages over a given range.
8200Sstevel@tonic-gate  *
8210Sstevel@tonic-gate  * We will cache all DISM locked pages and save the pplist for the
8220Sstevel@tonic-gate  * entire segment in the ppa field of the underlying DISM segment structure.
8230Sstevel@tonic-gate  * Later, during a call to segspt_reclaim() we will use this ppa array
8240Sstevel@tonic-gate  * to page_unlock() all of the pages and then we will free this ppa list.
8250Sstevel@tonic-gate  */
8260Sstevel@tonic-gate /*ARGSUSED*/
8270Sstevel@tonic-gate static int
8280Sstevel@tonic-gate segspt_dismpagelock(struct seg *seg, caddr_t addr, size_t len,
8290Sstevel@tonic-gate     struct page ***ppp, enum lock_type type, enum seg_rw rw)
8300Sstevel@tonic-gate {
8310Sstevel@tonic-gate 	struct  shm_data *shmd = (struct shm_data *)seg->s_data;
8320Sstevel@tonic-gate 	struct  seg	*sptseg = shmd->shm_sptseg;
8330Sstevel@tonic-gate 	struct  spt_data *sptd = sptseg->s_data;
8340Sstevel@tonic-gate 	pgcnt_t pg_idx, npages, tot_npages, npgs;
8350Sstevel@tonic-gate 	struct  page **pplist, **pl, **ppa, *pp;
8360Sstevel@tonic-gate 	struct  anon_map *amp;
8370Sstevel@tonic-gate 	spgcnt_t	an_idx;
8380Sstevel@tonic-gate 	int 	ret = ENOTSUP;
8390Sstevel@tonic-gate 	uint_t	pl_built = 0;
8400Sstevel@tonic-gate 	struct  anon *ap;
8410Sstevel@tonic-gate 	struct  vnode *vp;
8420Sstevel@tonic-gate 	u_offset_t off;
8430Sstevel@tonic-gate 	pgcnt_t claim_availrmem = 0;
8440Sstevel@tonic-gate 	uint_t	szc;
8450Sstevel@tonic-gate 
8460Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
8470Sstevel@tonic-gate 
8480Sstevel@tonic-gate 	/*
8490Sstevel@tonic-gate 	 * We want to lock/unlock the entire ISM segment. Therefore,
8500Sstevel@tonic-gate 	 * we will be using the underlying sptseg and it's base address
8510Sstevel@tonic-gate 	 * and length for the caching arguments.
8520Sstevel@tonic-gate 	 */
8530Sstevel@tonic-gate 	ASSERT(sptseg);
8540Sstevel@tonic-gate 	ASSERT(sptd);
8550Sstevel@tonic-gate 
8560Sstevel@tonic-gate 	pg_idx = seg_page(seg, addr);
8570Sstevel@tonic-gate 	npages = btopr(len);
8580Sstevel@tonic-gate 
8590Sstevel@tonic-gate 	/*
8600Sstevel@tonic-gate 	 * check if the request is larger than number of pages covered
8610Sstevel@tonic-gate 	 * by amp
8620Sstevel@tonic-gate 	 */
8630Sstevel@tonic-gate 	if (pg_idx + npages > btopr(sptd->spt_amp->size)) {
8640Sstevel@tonic-gate 		*ppp = NULL;
8650Sstevel@tonic-gate 		return (ENOTSUP);
8660Sstevel@tonic-gate 	}
8670Sstevel@tonic-gate 
8680Sstevel@tonic-gate 	if (type == L_PAGEUNLOCK) {
8690Sstevel@tonic-gate 		ASSERT(sptd->spt_ppa != NULL);
8700Sstevel@tonic-gate 
8710Sstevel@tonic-gate 		seg_pinactive(seg, seg->s_base, sptd->spt_amp->size,
8720Sstevel@tonic-gate 		    sptd->spt_ppa, sptd->spt_prot, segspt_reclaim);
8730Sstevel@tonic-gate 
8740Sstevel@tonic-gate 		/*
8750Sstevel@tonic-gate 		 * If someone is blocked while unmapping, we purge
8760Sstevel@tonic-gate 		 * segment page cache and thus reclaim pplist synchronously
8770Sstevel@tonic-gate 		 * without waiting for seg_pasync_thread. This speeds up
8780Sstevel@tonic-gate 		 * unmapping in cases where munmap(2) is called, while
8790Sstevel@tonic-gate 		 * raw async i/o is still in progress or where a thread
8800Sstevel@tonic-gate 		 * exits on data fault in a multithreaded application.
8810Sstevel@tonic-gate 		 */
8820Sstevel@tonic-gate 		if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) {
8830Sstevel@tonic-gate 			segspt_purge(seg);
8840Sstevel@tonic-gate 		}
8850Sstevel@tonic-gate 		return (0);
8860Sstevel@tonic-gate 	} else if (type == L_PAGERECLAIM) {
8870Sstevel@tonic-gate 		ASSERT(sptd->spt_ppa != NULL);
8880Sstevel@tonic-gate 		(void) segspt_reclaim(seg, seg->s_base, sptd->spt_amp->size,
8890Sstevel@tonic-gate 		    sptd->spt_ppa, sptd->spt_prot);
8900Sstevel@tonic-gate 		return (0);
8910Sstevel@tonic-gate 	}
8920Sstevel@tonic-gate 
8930Sstevel@tonic-gate 	if (sptd->spt_flags & DISM_PPA_CHANGED) {
8940Sstevel@tonic-gate 		segspt_purge(seg);
8950Sstevel@tonic-gate 		/*
8960Sstevel@tonic-gate 		 * for DISM ppa needs to be rebuild since
8970Sstevel@tonic-gate 		 * number of locked pages could be changed
8980Sstevel@tonic-gate 		 */
8990Sstevel@tonic-gate 		*ppp = NULL;
9000Sstevel@tonic-gate 		return (ENOTSUP);
9010Sstevel@tonic-gate 	}
9020Sstevel@tonic-gate 
9030Sstevel@tonic-gate 	/*
9040Sstevel@tonic-gate 	 * First try to find pages in segment page cache, without
9050Sstevel@tonic-gate 	 * holding the segment lock.
9060Sstevel@tonic-gate 	 */
9070Sstevel@tonic-gate 	pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size,
9080Sstevel@tonic-gate 	    sptd->spt_prot);
9090Sstevel@tonic-gate 	if (pplist != NULL) {
9100Sstevel@tonic-gate 		ASSERT(sptd->spt_ppa != NULL);
9110Sstevel@tonic-gate 		ASSERT(sptd->spt_ppa == pplist);
9120Sstevel@tonic-gate 		ppa = sptd->spt_ppa;
9130Sstevel@tonic-gate 		for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
9140Sstevel@tonic-gate 			if (ppa[an_idx] == NULL) {
9150Sstevel@tonic-gate 				seg_pinactive(seg, seg->s_base,
9160Sstevel@tonic-gate 				    sptd->spt_amp->size, ppa,
9170Sstevel@tonic-gate 				    sptd->spt_prot, segspt_reclaim);
9180Sstevel@tonic-gate 				*ppp = NULL;
9190Sstevel@tonic-gate 				return (ENOTSUP);
9200Sstevel@tonic-gate 			}
9210Sstevel@tonic-gate 			if ((szc = ppa[an_idx]->p_szc) != 0) {
9220Sstevel@tonic-gate 				npgs = page_get_pagecnt(szc);
9230Sstevel@tonic-gate 				an_idx = P2ROUNDUP(an_idx + 1, npgs);
9240Sstevel@tonic-gate 			} else {
9250Sstevel@tonic-gate 				an_idx++;
9260Sstevel@tonic-gate 			}
9270Sstevel@tonic-gate 		}
9280Sstevel@tonic-gate 		/*
9290Sstevel@tonic-gate 		 * Since we cache the entire DISM segment, we want to
9300Sstevel@tonic-gate 		 * set ppp to point to the first slot that corresponds
9310Sstevel@tonic-gate 		 * to the requested addr, i.e. pg_idx.
9320Sstevel@tonic-gate 		 */
9330Sstevel@tonic-gate 		*ppp = &(sptd->spt_ppa[pg_idx]);
9340Sstevel@tonic-gate 		return (0);
9350Sstevel@tonic-gate 	}
9360Sstevel@tonic-gate 
9370Sstevel@tonic-gate 	/* The L_PAGELOCK case... */
9380Sstevel@tonic-gate 	mutex_enter(&sptd->spt_lock);
9390Sstevel@tonic-gate 	/*
9400Sstevel@tonic-gate 	 * try to find pages in segment page cache with mutex
9410Sstevel@tonic-gate 	 */
9420Sstevel@tonic-gate 	pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size,
9430Sstevel@tonic-gate 	    sptd->spt_prot);
9440Sstevel@tonic-gate 	if (pplist != NULL) {
9450Sstevel@tonic-gate 		ASSERT(sptd->spt_ppa != NULL);
9460Sstevel@tonic-gate 		ASSERT(sptd->spt_ppa == pplist);
9470Sstevel@tonic-gate 		ppa = sptd->spt_ppa;
9480Sstevel@tonic-gate 		for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
9490Sstevel@tonic-gate 			if (ppa[an_idx] == NULL) {
9500Sstevel@tonic-gate 				mutex_exit(&sptd->spt_lock);
9510Sstevel@tonic-gate 				seg_pinactive(seg, seg->s_base,
9520Sstevel@tonic-gate 				    sptd->spt_amp->size, ppa,
9530Sstevel@tonic-gate 				    sptd->spt_prot, segspt_reclaim);
9540Sstevel@tonic-gate 				*ppp = NULL;
9550Sstevel@tonic-gate 				return (ENOTSUP);
9560Sstevel@tonic-gate 			}
9570Sstevel@tonic-gate 			if ((szc = ppa[an_idx]->p_szc) != 0) {
9580Sstevel@tonic-gate 				npgs = page_get_pagecnt(szc);
9590Sstevel@tonic-gate 				an_idx = P2ROUNDUP(an_idx + 1, npgs);
9600Sstevel@tonic-gate 			} else {
9610Sstevel@tonic-gate 				an_idx++;
9620Sstevel@tonic-gate 			}
9630Sstevel@tonic-gate 		}
9640Sstevel@tonic-gate 		/*
9650Sstevel@tonic-gate 		 * Since we cache the entire DISM segment, we want to
9660Sstevel@tonic-gate 		 * set ppp to point to the first slot that corresponds
9670Sstevel@tonic-gate 		 * to the requested addr, i.e. pg_idx.
9680Sstevel@tonic-gate 		 */
9690Sstevel@tonic-gate 		mutex_exit(&sptd->spt_lock);
9700Sstevel@tonic-gate 		*ppp = &(sptd->spt_ppa[pg_idx]);
9710Sstevel@tonic-gate 		return (0);
9720Sstevel@tonic-gate 	}
9730Sstevel@tonic-gate 	if (seg_pinsert_check(seg, sptd->spt_amp->size, SEGP_FORCE_WIRED) ==
9740Sstevel@tonic-gate 	    SEGP_FAIL) {
9750Sstevel@tonic-gate 		mutex_exit(&sptd->spt_lock);
9760Sstevel@tonic-gate 		*ppp = NULL;
9770Sstevel@tonic-gate 		return (ENOTSUP);
9780Sstevel@tonic-gate 	}
9790Sstevel@tonic-gate 
9800Sstevel@tonic-gate 	/*
9810Sstevel@tonic-gate 	 * No need to worry about protections because DISM pages are always rw.
9820Sstevel@tonic-gate 	 */
9830Sstevel@tonic-gate 	pl = pplist = NULL;
9840Sstevel@tonic-gate 	amp = sptd->spt_amp;
9850Sstevel@tonic-gate 
9860Sstevel@tonic-gate 	/*
9870Sstevel@tonic-gate 	 * Do we need to build the ppa array?
9880Sstevel@tonic-gate 	 */
9890Sstevel@tonic-gate 	if (sptd->spt_ppa == NULL) {
9900Sstevel@tonic-gate 		pgcnt_t lpg_cnt = 0;
9910Sstevel@tonic-gate 
9920Sstevel@tonic-gate 		pl_built = 1;
9930Sstevel@tonic-gate 		tot_npages = btopr(sptd->spt_amp->size);
9940Sstevel@tonic-gate 
9950Sstevel@tonic-gate 		ASSERT(sptd->spt_pcachecnt == 0);
9960Sstevel@tonic-gate 		pplist = kmem_zalloc(sizeof (page_t *) * tot_npages, KM_SLEEP);
9970Sstevel@tonic-gate 		pl = pplist;
9980Sstevel@tonic-gate 
9990Sstevel@tonic-gate 		ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
10000Sstevel@tonic-gate 		for (an_idx = 0; an_idx < tot_npages; ) {
10010Sstevel@tonic-gate 			ap = anon_get_ptr(amp->ahp, an_idx);
10020Sstevel@tonic-gate 			/*
10030Sstevel@tonic-gate 			 * Cache only mlocked pages. For large pages
10040Sstevel@tonic-gate 			 * if one (constituent) page is mlocked
10050Sstevel@tonic-gate 			 * all pages for that large page
10060Sstevel@tonic-gate 			 * are cached also. This is for quick
10070Sstevel@tonic-gate 			 * lookups of ppa array;
10080Sstevel@tonic-gate 			 */
10090Sstevel@tonic-gate 			if ((ap != NULL) && (lpg_cnt != 0 ||
10100Sstevel@tonic-gate 			    (sptd->spt_ppa_lckcnt[an_idx] != 0))) {
10110Sstevel@tonic-gate 
10120Sstevel@tonic-gate 				swap_xlate(ap, &vp, &off);
10130Sstevel@tonic-gate 				pp = page_lookup(vp, off, SE_SHARED);
10140Sstevel@tonic-gate 				ASSERT(pp != NULL);
10150Sstevel@tonic-gate 				if (lpg_cnt == 0) {
10161498Sbs21162 					lpg_cnt++;
10171498Sbs21162 					/*
10181498Sbs21162 					 * For a small page, we are done --
10191498Sbs21162 					 * lpg_count is reset to 0 below.
10201498Sbs21162 					 *
10211498Sbs21162 					 * For a large page, we are guaranteed
10221498Sbs21162 					 * to find the anon structures of all
10231498Sbs21162 					 * constituent pages and a non-zero
10241498Sbs21162 					 * lpg_cnt ensures that we don't test
10251498Sbs21162 					 * for mlock for these. We are done
10261498Sbs21162 					 * when lpg_count reaches (npgs + 1).
10271498Sbs21162 					 * If we are not the first constituent
10281498Sbs21162 					 * page, restart at the first one.
10291498Sbs21162 					 */
10300Sstevel@tonic-gate 					npgs = page_get_pagecnt(pp->p_szc);
10310Sstevel@tonic-gate 					if (!IS_P2ALIGNED(an_idx, npgs)) {
10320Sstevel@tonic-gate 						an_idx = P2ALIGN(an_idx, npgs);
10330Sstevel@tonic-gate 						page_unlock(pp);
10340Sstevel@tonic-gate 						continue;
10350Sstevel@tonic-gate 					}
10360Sstevel@tonic-gate 				}
10371498Sbs21162 				if (++lpg_cnt > npgs)
10380Sstevel@tonic-gate 					lpg_cnt = 0;
10390Sstevel@tonic-gate 
10400Sstevel@tonic-gate 				/*
10410Sstevel@tonic-gate 				 * availrmem is decremented only
10420Sstevel@tonic-gate 				 * for unlocked pages
10430Sstevel@tonic-gate 				 */
10440Sstevel@tonic-gate 				if (sptd->spt_ppa_lckcnt[an_idx] == 0)
10450Sstevel@tonic-gate 					claim_availrmem++;
10460Sstevel@tonic-gate 				pplist[an_idx] = pp;
10470Sstevel@tonic-gate 			}
10480Sstevel@tonic-gate 			an_idx++;
10490Sstevel@tonic-gate 		}
10500Sstevel@tonic-gate 		ANON_LOCK_EXIT(&amp->a_rwlock);
10510Sstevel@tonic-gate 
10520Sstevel@tonic-gate 		mutex_enter(&freemem_lock);
10530Sstevel@tonic-gate 		if (availrmem < tune.t_minarmem + claim_availrmem) {
10540Sstevel@tonic-gate 			mutex_exit(&freemem_lock);
10550Sstevel@tonic-gate 			ret = FC_MAKE_ERR(ENOMEM);
10560Sstevel@tonic-gate 			claim_availrmem = 0;
10570Sstevel@tonic-gate 			goto insert_fail;
10580Sstevel@tonic-gate 		} else {
10590Sstevel@tonic-gate 			availrmem -= claim_availrmem;
10600Sstevel@tonic-gate 		}
10610Sstevel@tonic-gate 		mutex_exit(&freemem_lock);
10620Sstevel@tonic-gate 
10630Sstevel@tonic-gate 		sptd->spt_ppa = pl;
10640Sstevel@tonic-gate 	} else {
10650Sstevel@tonic-gate 		/*
10660Sstevel@tonic-gate 		 * We already have a valid ppa[].
10670Sstevel@tonic-gate 		 */
10680Sstevel@tonic-gate 		pl = sptd->spt_ppa;
10690Sstevel@tonic-gate 	}
10700Sstevel@tonic-gate 
10710Sstevel@tonic-gate 	ASSERT(pl != NULL);
10720Sstevel@tonic-gate 
10730Sstevel@tonic-gate 	ret = seg_pinsert(seg, seg->s_base, sptd->spt_amp->size,
10740Sstevel@tonic-gate 	    pl, sptd->spt_prot, SEGP_FORCE_WIRED | SEGP_ASYNC_FLUSH,
10750Sstevel@tonic-gate 	    segspt_reclaim);
10760Sstevel@tonic-gate 	if (ret == SEGP_FAIL) {
10770Sstevel@tonic-gate 		/*
10780Sstevel@tonic-gate 		 * seg_pinsert failed. We return
10790Sstevel@tonic-gate 		 * ENOTSUP, so that the as_pagelock() code will
10800Sstevel@tonic-gate 		 * then try the slower F_SOFTLOCK path.
10810Sstevel@tonic-gate 		 */
1082934Srd117015 		if (pl_built) {
1083934Srd117015 			/*
1084934Srd117015 			 * No one else has referenced the ppa[].
1085934Srd117015 			 * We created it and we need to destroy it.
1086934Srd117015 			 */
1087934Srd117015 			sptd->spt_ppa = NULL;
1088934Srd117015 		}
10890Sstevel@tonic-gate 		ret = ENOTSUP;
10900Sstevel@tonic-gate 		goto insert_fail;
10910Sstevel@tonic-gate 	}
10920Sstevel@tonic-gate 
10930Sstevel@tonic-gate 	/*
10940Sstevel@tonic-gate 	 * In either case, we increment softlockcnt on the 'real' segment.
10950Sstevel@tonic-gate 	 */
10960Sstevel@tonic-gate 	sptd->spt_pcachecnt++;
10970Sstevel@tonic-gate 	atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), 1);
10980Sstevel@tonic-gate 
10990Sstevel@tonic-gate 	ppa = sptd->spt_ppa;
11000Sstevel@tonic-gate 	for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
11010Sstevel@tonic-gate 		if (ppa[an_idx] == NULL) {
11020Sstevel@tonic-gate 			mutex_exit(&sptd->spt_lock);
11030Sstevel@tonic-gate 			seg_pinactive(seg, seg->s_base, sptd->spt_amp->size,
11040Sstevel@tonic-gate 			    pl, sptd->spt_prot, segspt_reclaim);
11050Sstevel@tonic-gate 			*ppp = NULL;
11060Sstevel@tonic-gate 			return (ENOTSUP);
11070Sstevel@tonic-gate 		}
11080Sstevel@tonic-gate 		if ((szc = ppa[an_idx]->p_szc) != 0) {
11090Sstevel@tonic-gate 			npgs = page_get_pagecnt(szc);
11100Sstevel@tonic-gate 			an_idx = P2ROUNDUP(an_idx + 1, npgs);
11110Sstevel@tonic-gate 		} else {
11120Sstevel@tonic-gate 			an_idx++;
11130Sstevel@tonic-gate 		}
11140Sstevel@tonic-gate 	}
11150Sstevel@tonic-gate 	/*
11160Sstevel@tonic-gate 	 * We can now drop the sptd->spt_lock since the ppa[]
11170Sstevel@tonic-gate 	 * exists and he have incremented pacachecnt.
11180Sstevel@tonic-gate 	 */
11190Sstevel@tonic-gate 	mutex_exit(&sptd->spt_lock);
11200Sstevel@tonic-gate 
11210Sstevel@tonic-gate 	/*
11220Sstevel@tonic-gate 	 * Since we cache the entire segment, we want to
11230Sstevel@tonic-gate 	 * set ppp to point to the first slot that corresponds
11240Sstevel@tonic-gate 	 * to the requested addr, i.e. pg_idx.
11250Sstevel@tonic-gate 	 */
11260Sstevel@tonic-gate 	*ppp = &(sptd->spt_ppa[pg_idx]);
11270Sstevel@tonic-gate 	return (ret);
11280Sstevel@tonic-gate 
11290Sstevel@tonic-gate insert_fail:
11300Sstevel@tonic-gate 	/*
11310Sstevel@tonic-gate 	 * We will only reach this code if we tried and failed.
11320Sstevel@tonic-gate 	 *
11330Sstevel@tonic-gate 	 * And we can drop the lock on the dummy seg, once we've failed
11340Sstevel@tonic-gate 	 * to set up a new ppa[].
11350Sstevel@tonic-gate 	 */
11360Sstevel@tonic-gate 	mutex_exit(&sptd->spt_lock);
11370Sstevel@tonic-gate 
11380Sstevel@tonic-gate 	if (pl_built) {
11390Sstevel@tonic-gate 		mutex_enter(&freemem_lock);
11400Sstevel@tonic-gate 		availrmem += claim_availrmem;
11410Sstevel@tonic-gate 		mutex_exit(&freemem_lock);
11420Sstevel@tonic-gate 
11430Sstevel@tonic-gate 		/*
11440Sstevel@tonic-gate 		 * We created pl and we need to destroy it.
11450Sstevel@tonic-gate 		 */
11460Sstevel@tonic-gate 		pplist = pl;
11470Sstevel@tonic-gate 		for (an_idx = 0; an_idx < tot_npages; an_idx++) {
11480Sstevel@tonic-gate 			if (pplist[an_idx] != NULL)
11490Sstevel@tonic-gate 				page_unlock(pplist[an_idx]);
11500Sstevel@tonic-gate 		}
11510Sstevel@tonic-gate 		kmem_free(pl, sizeof (page_t *) * tot_npages);
11520Sstevel@tonic-gate 	}
11530Sstevel@tonic-gate 
11540Sstevel@tonic-gate 	if (shmd->shm_softlockcnt <= 0) {
11550Sstevel@tonic-gate 		if (AS_ISUNMAPWAIT(seg->s_as)) {
11560Sstevel@tonic-gate 			mutex_enter(&seg->s_as->a_contents);
11570Sstevel@tonic-gate 			if (AS_ISUNMAPWAIT(seg->s_as)) {
11580Sstevel@tonic-gate 				AS_CLRUNMAPWAIT(seg->s_as);
11590Sstevel@tonic-gate 				cv_broadcast(&seg->s_as->a_cv);
11600Sstevel@tonic-gate 			}
11610Sstevel@tonic-gate 			mutex_exit(&seg->s_as->a_contents);
11620Sstevel@tonic-gate 		}
11630Sstevel@tonic-gate 	}
11640Sstevel@tonic-gate 	*ppp = NULL;
11650Sstevel@tonic-gate 	return (ret);
11660Sstevel@tonic-gate }
11670Sstevel@tonic-gate 
11680Sstevel@tonic-gate 
11690Sstevel@tonic-gate 
11700Sstevel@tonic-gate /*
11710Sstevel@tonic-gate  * return locked pages over a given range.
11720Sstevel@tonic-gate  *
11730Sstevel@tonic-gate  * We will cache the entire ISM segment and save the pplist for the
11740Sstevel@tonic-gate  * entire segment in the ppa field of the underlying ISM segment structure.
11750Sstevel@tonic-gate  * Later, during a call to segspt_reclaim() we will use this ppa array
11760Sstevel@tonic-gate  * to page_unlock() all of the pages and then we will free this ppa list.
11770Sstevel@tonic-gate  */
11780Sstevel@tonic-gate /*ARGSUSED*/
11790Sstevel@tonic-gate static int
11800Sstevel@tonic-gate segspt_shmpagelock(struct seg *seg, caddr_t addr, size_t len,
11810Sstevel@tonic-gate     struct page ***ppp, enum lock_type type, enum seg_rw rw)
11820Sstevel@tonic-gate {
11830Sstevel@tonic-gate 	struct shm_data *shmd = (struct shm_data *)seg->s_data;
11840Sstevel@tonic-gate 	struct seg	*sptseg = shmd->shm_sptseg;
11850Sstevel@tonic-gate 	struct spt_data *sptd = sptseg->s_data;
11860Sstevel@tonic-gate 	pgcnt_t np, page_index, npages;
11870Sstevel@tonic-gate 	caddr_t a, spt_base;
11880Sstevel@tonic-gate 	struct page **pplist, **pl, *pp;
11890Sstevel@tonic-gate 	struct anon_map *amp;
11900Sstevel@tonic-gate 	ulong_t anon_index;
11910Sstevel@tonic-gate 	int ret = ENOTSUP;
11920Sstevel@tonic-gate 	uint_t	pl_built = 0;
11930Sstevel@tonic-gate 	struct anon *ap;
11940Sstevel@tonic-gate 	struct vnode *vp;
11950Sstevel@tonic-gate 	u_offset_t off;
11960Sstevel@tonic-gate 
11970Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
11980Sstevel@tonic-gate 
11990Sstevel@tonic-gate 	/*
12000Sstevel@tonic-gate 	 * We want to lock/unlock the entire ISM segment. Therefore,
12010Sstevel@tonic-gate 	 * we will be using the underlying sptseg and it's base address
12020Sstevel@tonic-gate 	 * and length for the caching arguments.
12030Sstevel@tonic-gate 	 */
12040Sstevel@tonic-gate 	ASSERT(sptseg);
12050Sstevel@tonic-gate 	ASSERT(sptd);
12060Sstevel@tonic-gate 
12070Sstevel@tonic-gate 	if (sptd->spt_flags & SHM_PAGEABLE) {
12080Sstevel@tonic-gate 		return (segspt_dismpagelock(seg, addr, len, ppp, type, rw));
12090Sstevel@tonic-gate 	}
12100Sstevel@tonic-gate 
12110Sstevel@tonic-gate 	page_index = seg_page(seg, addr);
12120Sstevel@tonic-gate 	npages = btopr(len);
12130Sstevel@tonic-gate 
12140Sstevel@tonic-gate 	/*
12150Sstevel@tonic-gate 	 * check if the request is larger than number of pages covered
12160Sstevel@tonic-gate 	 * by amp
12170Sstevel@tonic-gate 	 */
12180Sstevel@tonic-gate 	if (page_index + npages > btopr(sptd->spt_amp->size)) {
12190Sstevel@tonic-gate 		*ppp = NULL;
12200Sstevel@tonic-gate 		return (ENOTSUP);
12210Sstevel@tonic-gate 	}
12220Sstevel@tonic-gate 
12230Sstevel@tonic-gate 	if (type == L_PAGEUNLOCK) {
12240Sstevel@tonic-gate 
12250Sstevel@tonic-gate 		ASSERT(sptd->spt_ppa != NULL);
12260Sstevel@tonic-gate 
12270Sstevel@tonic-gate 		seg_pinactive(seg, seg->s_base, sptd->spt_amp->size,
12280Sstevel@tonic-gate 		    sptd->spt_ppa, sptd->spt_prot, segspt_reclaim);
12290Sstevel@tonic-gate 
12300Sstevel@tonic-gate 		/*
12310Sstevel@tonic-gate 		 * If someone is blocked while unmapping, we purge
12320Sstevel@tonic-gate 		 * segment page cache and thus reclaim pplist synchronously
12330Sstevel@tonic-gate 		 * without waiting for seg_pasync_thread. This speeds up
12340Sstevel@tonic-gate 		 * unmapping in cases where munmap(2) is called, while
12350Sstevel@tonic-gate 		 * raw async i/o is still in progress or where a thread
12360Sstevel@tonic-gate 		 * exits on data fault in a multithreaded application.
12370Sstevel@tonic-gate 		 */
12380Sstevel@tonic-gate 		if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) {
12390Sstevel@tonic-gate 			segspt_purge(seg);
12400Sstevel@tonic-gate 		}
12410Sstevel@tonic-gate 		return (0);
12420Sstevel@tonic-gate 	} else if (type == L_PAGERECLAIM) {
12430Sstevel@tonic-gate 		ASSERT(sptd->spt_ppa != NULL);
12440Sstevel@tonic-gate 
12450Sstevel@tonic-gate 		(void) segspt_reclaim(seg, seg->s_base, sptd->spt_amp->size,
12460Sstevel@tonic-gate 		    sptd->spt_ppa, sptd->spt_prot);
12470Sstevel@tonic-gate 		return (0);
12480Sstevel@tonic-gate 	}
12490Sstevel@tonic-gate 
12500Sstevel@tonic-gate 	/*
12510Sstevel@tonic-gate 	 * First try to find pages in segment page cache, without
12520Sstevel@tonic-gate 	 * holding the segment lock.
12530Sstevel@tonic-gate 	 */
12540Sstevel@tonic-gate 	pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size,
12550Sstevel@tonic-gate 	    sptd->spt_prot);
12560Sstevel@tonic-gate 	if (pplist != NULL) {
12570Sstevel@tonic-gate 		ASSERT(sptd->spt_ppa == pplist);
12580Sstevel@tonic-gate 		ASSERT(sptd->spt_ppa[page_index]);
12590Sstevel@tonic-gate 		/*
12600Sstevel@tonic-gate 		 * Since we cache the entire ISM segment, we want to
12610Sstevel@tonic-gate 		 * set ppp to point to the first slot that corresponds
12620Sstevel@tonic-gate 		 * to the requested addr, i.e. page_index.
12630Sstevel@tonic-gate 		 */
12640Sstevel@tonic-gate 		*ppp = &(sptd->spt_ppa[page_index]);
12650Sstevel@tonic-gate 		return (0);
12660Sstevel@tonic-gate 	}
12670Sstevel@tonic-gate 
12680Sstevel@tonic-gate 	/* The L_PAGELOCK case... */
12690Sstevel@tonic-gate 	mutex_enter(&sptd->spt_lock);
12700Sstevel@tonic-gate 
12710Sstevel@tonic-gate 	/*
12720Sstevel@tonic-gate 	 * try to find pages in segment page cache
12730Sstevel@tonic-gate 	 */
12740Sstevel@tonic-gate 	pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size,
12750Sstevel@tonic-gate 	    sptd->spt_prot);
12760Sstevel@tonic-gate 	if (pplist != NULL) {
12770Sstevel@tonic-gate 		ASSERT(sptd->spt_ppa == pplist);
12780Sstevel@tonic-gate 		/*
12790Sstevel@tonic-gate 		 * Since we cache the entire segment, we want to
12800Sstevel@tonic-gate 		 * set ppp to point to the first slot that corresponds
12810Sstevel@tonic-gate 		 * to the requested addr, i.e. page_index.
12820Sstevel@tonic-gate 		 */
12830Sstevel@tonic-gate 		mutex_exit(&sptd->spt_lock);
12840Sstevel@tonic-gate 		*ppp = &(sptd->spt_ppa[page_index]);
12850Sstevel@tonic-gate 		return (0);
12860Sstevel@tonic-gate 	}
12870Sstevel@tonic-gate 
12880Sstevel@tonic-gate 	if (seg_pinsert_check(seg, sptd->spt_amp->size, SEGP_FORCE_WIRED) ==
12890Sstevel@tonic-gate 	    SEGP_FAIL) {
12900Sstevel@tonic-gate 		mutex_exit(&sptd->spt_lock);
12910Sstevel@tonic-gate 		*ppp = NULL;
12920Sstevel@tonic-gate 		return (ENOTSUP);
12930Sstevel@tonic-gate 	}
12940Sstevel@tonic-gate 
12950Sstevel@tonic-gate 	/*
12960Sstevel@tonic-gate 	 * No need to worry about protections because ISM pages
12970Sstevel@tonic-gate 	 * are always rw.
12980Sstevel@tonic-gate 	 */
12990Sstevel@tonic-gate 	pl = pplist = NULL;
13000Sstevel@tonic-gate 
13010Sstevel@tonic-gate 	/*
13020Sstevel@tonic-gate 	 * Do we need to build the ppa array?
13030Sstevel@tonic-gate 	 */
13040Sstevel@tonic-gate 	if (sptd->spt_ppa == NULL) {
13050Sstevel@tonic-gate 		ASSERT(sptd->spt_ppa == pplist);
13060Sstevel@tonic-gate 
13070Sstevel@tonic-gate 		spt_base = sptseg->s_base;
13080Sstevel@tonic-gate 		pl_built = 1;
13090Sstevel@tonic-gate 
13100Sstevel@tonic-gate 		/*
13110Sstevel@tonic-gate 		 * availrmem is decremented once during anon_swap_adjust()
13120Sstevel@tonic-gate 		 * and is incremented during the anon_unresv(), which is
13130Sstevel@tonic-gate 		 * called from shm_rm_amp() when the segment is destroyed.
13140Sstevel@tonic-gate 		 */
13150Sstevel@tonic-gate 		amp = sptd->spt_amp;
13160Sstevel@tonic-gate 		ASSERT(amp != NULL);
13170Sstevel@tonic-gate 
13180Sstevel@tonic-gate 		/* pcachecnt is protected by sptd->spt_lock */
13190Sstevel@tonic-gate 		ASSERT(sptd->spt_pcachecnt == 0);
13200Sstevel@tonic-gate 		pplist = kmem_zalloc(sizeof (page_t *)
13210Sstevel@tonic-gate 		    * btopr(sptd->spt_amp->size), KM_SLEEP);
13220Sstevel@tonic-gate 		pl = pplist;
13230Sstevel@tonic-gate 
13240Sstevel@tonic-gate 		anon_index = seg_page(sptseg, spt_base);
13250Sstevel@tonic-gate 
13260Sstevel@tonic-gate 		ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
13270Sstevel@tonic-gate 		for (a = spt_base; a < (spt_base + sptd->spt_amp->size);
13280Sstevel@tonic-gate 		    a += PAGESIZE, anon_index++, pplist++) {
13290Sstevel@tonic-gate 			ap = anon_get_ptr(amp->ahp, anon_index);
13300Sstevel@tonic-gate 			ASSERT(ap != NULL);
13310Sstevel@tonic-gate 			swap_xlate(ap, &vp, &off);
13320Sstevel@tonic-gate 			pp = page_lookup(vp, off, SE_SHARED);
13330Sstevel@tonic-gate 			ASSERT(pp != NULL);
13340Sstevel@tonic-gate 			*pplist = pp;
13350Sstevel@tonic-gate 		}
13360Sstevel@tonic-gate 		ANON_LOCK_EXIT(&amp->a_rwlock);
13370Sstevel@tonic-gate 
13380Sstevel@tonic-gate 		if (a < (spt_base + sptd->spt_amp->size)) {
13390Sstevel@tonic-gate 			ret = ENOTSUP;
13400Sstevel@tonic-gate 			goto insert_fail;
13410Sstevel@tonic-gate 		}
13420Sstevel@tonic-gate 		sptd->spt_ppa = pl;
13430Sstevel@tonic-gate 	} else {
13440Sstevel@tonic-gate 		/*
13450Sstevel@tonic-gate 		 * We already have a valid ppa[].
13460Sstevel@tonic-gate 		 */
13470Sstevel@tonic-gate 		pl = sptd->spt_ppa;
13480Sstevel@tonic-gate 	}
13490Sstevel@tonic-gate 
13500Sstevel@tonic-gate 	ASSERT(pl != NULL);
13510Sstevel@tonic-gate 
13520Sstevel@tonic-gate 	ret = seg_pinsert(seg, seg->s_base, sptd->spt_amp->size,
13530Sstevel@tonic-gate 	    pl, sptd->spt_prot, SEGP_FORCE_WIRED, segspt_reclaim);
13540Sstevel@tonic-gate 	if (ret == SEGP_FAIL) {
13550Sstevel@tonic-gate 		/*
13560Sstevel@tonic-gate 		 * seg_pinsert failed. We return
13570Sstevel@tonic-gate 		 * ENOTSUP, so that the as_pagelock() code will
13580Sstevel@tonic-gate 		 * then try the slower F_SOFTLOCK path.
13590Sstevel@tonic-gate 		 */
13600Sstevel@tonic-gate 		if (pl_built) {
13610Sstevel@tonic-gate 			/*
13620Sstevel@tonic-gate 			 * No one else has referenced the ppa[].
13630Sstevel@tonic-gate 			 * We created it and we need to destroy it.
13640Sstevel@tonic-gate 			 */
13650Sstevel@tonic-gate 			sptd->spt_ppa = NULL;
13660Sstevel@tonic-gate 		}
13670Sstevel@tonic-gate 		ret = ENOTSUP;
13680Sstevel@tonic-gate 		goto insert_fail;
13690Sstevel@tonic-gate 	}
13700Sstevel@tonic-gate 
13710Sstevel@tonic-gate 	/*
13720Sstevel@tonic-gate 	 * In either case, we increment softlockcnt on the 'real' segment.
13730Sstevel@tonic-gate 	 */
13740Sstevel@tonic-gate 	sptd->spt_pcachecnt++;
13750Sstevel@tonic-gate 	atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), 1);
13760Sstevel@tonic-gate 
13770Sstevel@tonic-gate 	/*
13780Sstevel@tonic-gate 	 * We can now drop the sptd->spt_lock since the ppa[]
13790Sstevel@tonic-gate 	 * exists and he have incremented pacachecnt.
13800Sstevel@tonic-gate 	 */
13810Sstevel@tonic-gate 	mutex_exit(&sptd->spt_lock);
13820Sstevel@tonic-gate 
13830Sstevel@tonic-gate 	/*
13840Sstevel@tonic-gate 	 * Since we cache the entire segment, we want to
13850Sstevel@tonic-gate 	 * set ppp to point to the first slot that corresponds
13860Sstevel@tonic-gate 	 * to the requested addr, i.e. page_index.
13870Sstevel@tonic-gate 	 */
13880Sstevel@tonic-gate 	*ppp = &(sptd->spt_ppa[page_index]);
13890Sstevel@tonic-gate 	return (ret);
13900Sstevel@tonic-gate 
13910Sstevel@tonic-gate insert_fail:
13920Sstevel@tonic-gate 	/*
13930Sstevel@tonic-gate 	 * We will only reach this code if we tried and failed.
13940Sstevel@tonic-gate 	 *
13950Sstevel@tonic-gate 	 * And we can drop the lock on the dummy seg, once we've failed
13960Sstevel@tonic-gate 	 * to set up a new ppa[].
13970Sstevel@tonic-gate 	 */
13980Sstevel@tonic-gate 	mutex_exit(&sptd->spt_lock);
13990Sstevel@tonic-gate 
14000Sstevel@tonic-gate 	if (pl_built) {
14010Sstevel@tonic-gate 		/*
14020Sstevel@tonic-gate 		 * We created pl and we need to destroy it.
14030Sstevel@tonic-gate 		 */
14040Sstevel@tonic-gate 		pplist = pl;
14050Sstevel@tonic-gate 		np = (((uintptr_t)(a - spt_base)) >> PAGESHIFT);
14060Sstevel@tonic-gate 		while (np) {
14070Sstevel@tonic-gate 			page_unlock(*pplist);
14080Sstevel@tonic-gate 			np--;
14090Sstevel@tonic-gate 			pplist++;
14100Sstevel@tonic-gate 		}
14110Sstevel@tonic-gate 		kmem_free(pl, sizeof (page_t *) *
14120Sstevel@tonic-gate 				btopr(sptd->spt_amp->size));
14130Sstevel@tonic-gate 	}
14140Sstevel@tonic-gate 	if (shmd->shm_softlockcnt <= 0) {
14150Sstevel@tonic-gate 		if (AS_ISUNMAPWAIT(seg->s_as)) {
14160Sstevel@tonic-gate 			mutex_enter(&seg->s_as->a_contents);
14170Sstevel@tonic-gate 			if (AS_ISUNMAPWAIT(seg->s_as)) {
14180Sstevel@tonic-gate 				AS_CLRUNMAPWAIT(seg->s_as);
14190Sstevel@tonic-gate 				cv_broadcast(&seg->s_as->a_cv);
14200Sstevel@tonic-gate 			}
14210Sstevel@tonic-gate 			mutex_exit(&seg->s_as->a_contents);
14220Sstevel@tonic-gate 		}
14230Sstevel@tonic-gate 	}
14240Sstevel@tonic-gate 	*ppp = NULL;
14250Sstevel@tonic-gate 	return (ret);
14260Sstevel@tonic-gate }
14270Sstevel@tonic-gate 
14280Sstevel@tonic-gate /*
14290Sstevel@tonic-gate  * purge any cached pages in the I/O page cache
14300Sstevel@tonic-gate  */
14310Sstevel@tonic-gate static void
14320Sstevel@tonic-gate segspt_purge(struct seg *seg)
14330Sstevel@tonic-gate {
14340Sstevel@tonic-gate 	seg_ppurge(seg);
14350Sstevel@tonic-gate }
14360Sstevel@tonic-gate 
14370Sstevel@tonic-gate static int
14380Sstevel@tonic-gate segspt_reclaim(struct seg *seg, caddr_t addr, size_t len, struct page **pplist,
14390Sstevel@tonic-gate 	enum seg_rw rw)
14400Sstevel@tonic-gate {
14410Sstevel@tonic-gate 	struct	shm_data *shmd = (struct shm_data *)seg->s_data;
14420Sstevel@tonic-gate 	struct	seg	*sptseg;
14430Sstevel@tonic-gate 	struct	spt_data *sptd;
14440Sstevel@tonic-gate 	pgcnt_t npages, i, free_availrmem = 0;
14450Sstevel@tonic-gate 	int	done = 0;
14460Sstevel@tonic-gate 
14470Sstevel@tonic-gate #ifdef lint
14480Sstevel@tonic-gate 	addr = addr;
14490Sstevel@tonic-gate #endif
14500Sstevel@tonic-gate 	sptseg = shmd->shm_sptseg;
14510Sstevel@tonic-gate 	sptd = sptseg->s_data;
14520Sstevel@tonic-gate 	npages = (len >> PAGESHIFT);
14530Sstevel@tonic-gate 	ASSERT(npages);
14540Sstevel@tonic-gate 	ASSERT(sptd->spt_pcachecnt != 0);
14550Sstevel@tonic-gate 	ASSERT(sptd->spt_ppa == pplist);
14560Sstevel@tonic-gate 	ASSERT(npages == btopr(sptd->spt_amp->size));
14570Sstevel@tonic-gate 	/*
14580Sstevel@tonic-gate 	 * Acquire the lock on the dummy seg and destroy the
14590Sstevel@tonic-gate 	 * ppa array IF this is the last pcachecnt.
14600Sstevel@tonic-gate 	 */
14610Sstevel@tonic-gate 	mutex_enter(&sptd->spt_lock);
14620Sstevel@tonic-gate 	if (--sptd->spt_pcachecnt == 0) {
14630Sstevel@tonic-gate 		for (i = 0; i < npages; i++) {
14640Sstevel@tonic-gate 			if (pplist[i] == NULL) {
14650Sstevel@tonic-gate 				continue;
14660Sstevel@tonic-gate 			}
14670Sstevel@tonic-gate 			if (rw == S_WRITE) {
14680Sstevel@tonic-gate 				hat_setrefmod(pplist[i]);
14690Sstevel@tonic-gate 			} else {
14700Sstevel@tonic-gate 				hat_setref(pplist[i]);
14710Sstevel@tonic-gate 			}
14720Sstevel@tonic-gate 			if ((sptd->spt_flags & SHM_PAGEABLE) &&
14732768Ssl108498 			    (sptd->spt_ppa_lckcnt[i] == 0))
14740Sstevel@tonic-gate 				free_availrmem++;
14750Sstevel@tonic-gate 			page_unlock(pplist[i]);
14760Sstevel@tonic-gate 		}
14770Sstevel@tonic-gate 		if (sptd->spt_flags & SHM_PAGEABLE) {
14780Sstevel@tonic-gate 			mutex_enter(&freemem_lock);
14790Sstevel@tonic-gate 			availrmem += free_availrmem;
14800Sstevel@tonic-gate 			mutex_exit(&freemem_lock);
14810Sstevel@tonic-gate 		}
14820Sstevel@tonic-gate 		/*
14830Sstevel@tonic-gate 		 * Since we want to cach/uncache the entire ISM segment,
14840Sstevel@tonic-gate 		 * we will track the pplist in a segspt specific field
14850Sstevel@tonic-gate 		 * ppa, that is initialized at the time we add an entry to
14860Sstevel@tonic-gate 		 * the cache.
14870Sstevel@tonic-gate 		 */
14880Sstevel@tonic-gate 		ASSERT(sptd->spt_pcachecnt == 0);
14890Sstevel@tonic-gate 		kmem_free(pplist, sizeof (page_t *) * npages);
14900Sstevel@tonic-gate 		sptd->spt_ppa = NULL;
14910Sstevel@tonic-gate 		sptd->spt_flags &= ~DISM_PPA_CHANGED;
14920Sstevel@tonic-gate 		done = 1;
14930Sstevel@tonic-gate 	}
14940Sstevel@tonic-gate 	mutex_exit(&sptd->spt_lock);
14950Sstevel@tonic-gate 	/*
14960Sstevel@tonic-gate 	 * Now decrement softlockcnt.
14970Sstevel@tonic-gate 	 */
14980Sstevel@tonic-gate 	atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -1);
14990Sstevel@tonic-gate 
15000Sstevel@tonic-gate 	if (shmd->shm_softlockcnt <= 0) {
15010Sstevel@tonic-gate 		if (AS_ISUNMAPWAIT(seg->s_as)) {
15020Sstevel@tonic-gate 			mutex_enter(&seg->s_as->a_contents);
15030Sstevel@tonic-gate 			if (AS_ISUNMAPWAIT(seg->s_as)) {
15040Sstevel@tonic-gate 				AS_CLRUNMAPWAIT(seg->s_as);
15050Sstevel@tonic-gate 				cv_broadcast(&seg->s_as->a_cv);
15060Sstevel@tonic-gate 			}
15070Sstevel@tonic-gate 			mutex_exit(&seg->s_as->a_contents);
15080Sstevel@tonic-gate 		}
15090Sstevel@tonic-gate 	}
15100Sstevel@tonic-gate 	return (done);
15110Sstevel@tonic-gate }
15120Sstevel@tonic-gate 
15130Sstevel@tonic-gate /*
15140Sstevel@tonic-gate  * Do a F_SOFTUNLOCK call over the range requested.
15150Sstevel@tonic-gate  * The range must have already been F_SOFTLOCK'ed.
15160Sstevel@tonic-gate  *
15170Sstevel@tonic-gate  * The calls to acquire and release the anon map lock mutex were
15180Sstevel@tonic-gate  * removed in order to avoid a deadly embrace during a DR
15190Sstevel@tonic-gate  * memory delete operation.  (Eg. DR blocks while waiting for a
15200Sstevel@tonic-gate  * exclusive lock on a page that is being used for kaio; the
15210Sstevel@tonic-gate  * thread that will complete the kaio and call segspt_softunlock
15220Sstevel@tonic-gate  * blocks on the anon map lock; another thread holding the anon
15230Sstevel@tonic-gate  * map lock blocks on another page lock via the segspt_shmfault
15240Sstevel@tonic-gate  * -> page_lookup -> page_lookup_create -> page_lock_es code flow.)
15250Sstevel@tonic-gate  *
15260Sstevel@tonic-gate  * The appropriateness of the removal is based upon the following:
15270Sstevel@tonic-gate  * 1. If we are holding a segment's reader lock and the page is held
15280Sstevel@tonic-gate  * shared, then the corresponding element in anonmap which points to
15290Sstevel@tonic-gate  * anon struct cannot change and there is no need to acquire the
15300Sstevel@tonic-gate  * anonymous map lock.
15310Sstevel@tonic-gate  * 2. Threads in segspt_softunlock have a reader lock on the segment
15320Sstevel@tonic-gate  * and already have the shared page lock, so we are guaranteed that
15330Sstevel@tonic-gate  * the anon map slot cannot change and therefore can call anon_get_ptr()
15340Sstevel@tonic-gate  * without grabbing the anonymous map lock.
15350Sstevel@tonic-gate  * 3. Threads that softlock a shared page break copy-on-write, even if
15360Sstevel@tonic-gate  * its a read.  Thus cow faults can be ignored with respect to soft
15370Sstevel@tonic-gate  * unlocking, since the breaking of cow means that the anon slot(s) will
15380Sstevel@tonic-gate  * not be shared.
15390Sstevel@tonic-gate  */
15400Sstevel@tonic-gate static void
15410Sstevel@tonic-gate segspt_softunlock(struct seg *seg, caddr_t sptseg_addr,
15420Sstevel@tonic-gate 	size_t len, enum seg_rw rw)
15430Sstevel@tonic-gate {
15440Sstevel@tonic-gate 	struct shm_data *shmd = (struct shm_data *)seg->s_data;
15450Sstevel@tonic-gate 	struct seg	*sptseg;
15460Sstevel@tonic-gate 	struct spt_data *sptd;
15470Sstevel@tonic-gate 	page_t *pp;
15480Sstevel@tonic-gate 	caddr_t adr;
15490Sstevel@tonic-gate 	struct vnode *vp;
15500Sstevel@tonic-gate 	u_offset_t offset;
15510Sstevel@tonic-gate 	ulong_t anon_index;
15520Sstevel@tonic-gate 	struct anon_map *amp;		/* XXX - for locknest */
15530Sstevel@tonic-gate 	struct anon *ap = NULL;
15540Sstevel@tonic-gate 	pgcnt_t npages;
15550Sstevel@tonic-gate 
15560Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
15570Sstevel@tonic-gate 
15580Sstevel@tonic-gate 	sptseg = shmd->shm_sptseg;
15590Sstevel@tonic-gate 	sptd = sptseg->s_data;
15600Sstevel@tonic-gate 
15610Sstevel@tonic-gate 	/*
15620Sstevel@tonic-gate 	 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK
15630Sstevel@tonic-gate 	 * and therefore their pages are SE_SHARED locked
15640Sstevel@tonic-gate 	 * for the entire life of the segment.
15650Sstevel@tonic-gate 	 */
15660Sstevel@tonic-gate 	if ((!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) &&
15670Sstevel@tonic-gate 		((sptd->spt_flags & SHM_PAGEABLE) == 0)) {
15680Sstevel@tonic-gate 		goto softlock_decrement;
15690Sstevel@tonic-gate 	}
15700Sstevel@tonic-gate 
15710Sstevel@tonic-gate 	/*
15720Sstevel@tonic-gate 	 * Any thread is free to do a page_find and
15730Sstevel@tonic-gate 	 * page_unlock() on the pages within this seg.
15740Sstevel@tonic-gate 	 *
15750Sstevel@tonic-gate 	 * We are already holding the as->a_lock on the user's
15760Sstevel@tonic-gate 	 * real segment, but we need to hold the a_lock on the
15770Sstevel@tonic-gate 	 * underlying dummy as. This is mostly to satisfy the
15780Sstevel@tonic-gate 	 * underlying HAT layer.
15790Sstevel@tonic-gate 	 */
15800Sstevel@tonic-gate 	AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER);
15810Sstevel@tonic-gate 	hat_unlock(sptseg->s_as->a_hat, sptseg_addr, len);
15820Sstevel@tonic-gate 	AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock);
15830Sstevel@tonic-gate 
15840Sstevel@tonic-gate 	amp = sptd->spt_amp;
15850Sstevel@tonic-gate 	ASSERT(amp != NULL);
15860Sstevel@tonic-gate 	anon_index = seg_page(sptseg, sptseg_addr);
15870Sstevel@tonic-gate 
15880Sstevel@tonic-gate 	for (adr = sptseg_addr; adr < sptseg_addr + len; adr += PAGESIZE) {
15890Sstevel@tonic-gate 		ap = anon_get_ptr(amp->ahp, anon_index++);
15900Sstevel@tonic-gate 		ASSERT(ap != NULL);
15910Sstevel@tonic-gate 		swap_xlate(ap, &vp, &offset);
15920Sstevel@tonic-gate 
15930Sstevel@tonic-gate 		/*
15940Sstevel@tonic-gate 		 * Use page_find() instead of page_lookup() to
15950Sstevel@tonic-gate 		 * find the page since we know that it has a
15960Sstevel@tonic-gate 		 * "shared" lock.
15970Sstevel@tonic-gate 		 */
15980Sstevel@tonic-gate 		pp = page_find(vp, offset);
15990Sstevel@tonic-gate 		ASSERT(ap == anon_get_ptr(amp->ahp, anon_index - 1));
16000Sstevel@tonic-gate 		if (pp == NULL) {
16010Sstevel@tonic-gate 			panic("segspt_softunlock: "
16020Sstevel@tonic-gate 			    "addr %p, ap %p, vp %p, off %llx",
16030Sstevel@tonic-gate 			    (void *)adr, (void *)ap, (void *)vp, offset);
16040Sstevel@tonic-gate 			/*NOTREACHED*/
16050Sstevel@tonic-gate 		}
16060Sstevel@tonic-gate 
16070Sstevel@tonic-gate 		if (rw == S_WRITE) {
16080Sstevel@tonic-gate 			hat_setrefmod(pp);
16090Sstevel@tonic-gate 		} else if (rw != S_OTHER) {
16100Sstevel@tonic-gate 			hat_setref(pp);
16110Sstevel@tonic-gate 		}
16120Sstevel@tonic-gate 		page_unlock(pp);
16130Sstevel@tonic-gate 	}
16140Sstevel@tonic-gate 
16150Sstevel@tonic-gate softlock_decrement:
16160Sstevel@tonic-gate 	npages = btopr(len);
16170Sstevel@tonic-gate 	atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -npages);
16180Sstevel@tonic-gate 	if (shmd->shm_softlockcnt == 0) {
16190Sstevel@tonic-gate 		/*
16200Sstevel@tonic-gate 		 * All SOFTLOCKS are gone. Wakeup any waiting
16210Sstevel@tonic-gate 		 * unmappers so they can try again to unmap.
16220Sstevel@tonic-gate 		 * Check for waiters first without the mutex
16230Sstevel@tonic-gate 		 * held so we don't always grab the mutex on
16240Sstevel@tonic-gate 		 * softunlocks.
16250Sstevel@tonic-gate 		 */
16260Sstevel@tonic-gate 		if (AS_ISUNMAPWAIT(seg->s_as)) {
16270Sstevel@tonic-gate 			mutex_enter(&seg->s_as->a_contents);
16280Sstevel@tonic-gate 			if (AS_ISUNMAPWAIT(seg->s_as)) {
16290Sstevel@tonic-gate 				AS_CLRUNMAPWAIT(seg->s_as);
16300Sstevel@tonic-gate 				cv_broadcast(&seg->s_as->a_cv);
16310Sstevel@tonic-gate 			}
16320Sstevel@tonic-gate 			mutex_exit(&seg->s_as->a_contents);
16330Sstevel@tonic-gate 		}
16340Sstevel@tonic-gate 	}
16350Sstevel@tonic-gate }
16360Sstevel@tonic-gate 
16370Sstevel@tonic-gate int
16380Sstevel@tonic-gate segspt_shmattach(struct seg *seg, caddr_t *argsp)
16390Sstevel@tonic-gate {
16400Sstevel@tonic-gate 	struct shm_data *shmd_arg = (struct shm_data *)argsp;
16410Sstevel@tonic-gate 	struct shm_data *shmd;
16420Sstevel@tonic-gate 	struct anon_map *shm_amp = shmd_arg->shm_amp;
16430Sstevel@tonic-gate 	struct spt_data *sptd;
16440Sstevel@tonic-gate 	int error = 0;
16450Sstevel@tonic-gate 
16460Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
16470Sstevel@tonic-gate 
16480Sstevel@tonic-gate 	shmd = kmem_zalloc((sizeof (*shmd)), KM_NOSLEEP);
16490Sstevel@tonic-gate 	if (shmd == NULL)
16500Sstevel@tonic-gate 		return (ENOMEM);
16510Sstevel@tonic-gate 
16520Sstevel@tonic-gate 	shmd->shm_sptas = shmd_arg->shm_sptas;
16530Sstevel@tonic-gate 	shmd->shm_amp = shm_amp;
16540Sstevel@tonic-gate 	shmd->shm_sptseg = shmd_arg->shm_sptseg;
16550Sstevel@tonic-gate 
16560Sstevel@tonic-gate 	(void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, shm_amp, 0,
16570Sstevel@tonic-gate 	    NULL, 0, seg->s_size);
16580Sstevel@tonic-gate 
16590Sstevel@tonic-gate 	seg->s_data = (void *)shmd;
16600Sstevel@tonic-gate 	seg->s_ops = &segspt_shmops;
16610Sstevel@tonic-gate 	seg->s_szc = shmd->shm_sptseg->s_szc;
16620Sstevel@tonic-gate 	sptd = shmd->shm_sptseg->s_data;
16630Sstevel@tonic-gate 
16640Sstevel@tonic-gate 	if (sptd->spt_flags & SHM_PAGEABLE) {
16650Sstevel@tonic-gate 		if ((shmd->shm_vpage = kmem_zalloc(btopr(shm_amp->size),
16660Sstevel@tonic-gate 		    KM_NOSLEEP)) == NULL) {
16670Sstevel@tonic-gate 			seg->s_data = (void *)NULL;
16680Sstevel@tonic-gate 			kmem_free(shmd, (sizeof (*shmd)));
16690Sstevel@tonic-gate 			return (ENOMEM);
16700Sstevel@tonic-gate 		}
16710Sstevel@tonic-gate 		shmd->shm_lckpgs = 0;
16720Sstevel@tonic-gate 		if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
16730Sstevel@tonic-gate 			if ((error = hat_share(seg->s_as->a_hat, seg->s_base,
16740Sstevel@tonic-gate 			    shmd_arg->shm_sptas->a_hat, SEGSPTADDR,
16750Sstevel@tonic-gate 			    seg->s_size, seg->s_szc)) != 0) {
16760Sstevel@tonic-gate 				kmem_free(shmd->shm_vpage,
16770Sstevel@tonic-gate 					btopr(shm_amp->size));
16780Sstevel@tonic-gate 			}
16790Sstevel@tonic-gate 		}
16800Sstevel@tonic-gate 	} else {
16810Sstevel@tonic-gate 		error = hat_share(seg->s_as->a_hat, seg->s_base,
16820Sstevel@tonic-gate 				shmd_arg->shm_sptas->a_hat, SEGSPTADDR,
16830Sstevel@tonic-gate 				seg->s_size, seg->s_szc);
16840Sstevel@tonic-gate 	}
16850Sstevel@tonic-gate 	if (error) {
16860Sstevel@tonic-gate 		seg->s_szc = 0;
16870Sstevel@tonic-gate 		seg->s_data = (void *)NULL;
16880Sstevel@tonic-gate 		kmem_free(shmd, (sizeof (*shmd)));
16890Sstevel@tonic-gate 	} else {
16900Sstevel@tonic-gate 		ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER);
16910Sstevel@tonic-gate 		shm_amp->refcnt++;
16920Sstevel@tonic-gate 		ANON_LOCK_EXIT(&shm_amp->a_rwlock);
16930Sstevel@tonic-gate 	}
16940Sstevel@tonic-gate 	return (error);
16950Sstevel@tonic-gate }
16960Sstevel@tonic-gate 
16970Sstevel@tonic-gate int
16980Sstevel@tonic-gate segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize)
16990Sstevel@tonic-gate {
17000Sstevel@tonic-gate 	struct shm_data *shmd = (struct shm_data *)seg->s_data;
17010Sstevel@tonic-gate 	int reclaim = 1;
17020Sstevel@tonic-gate 
17030Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
17040Sstevel@tonic-gate retry:
17050Sstevel@tonic-gate 	if (shmd->shm_softlockcnt > 0) {
17060Sstevel@tonic-gate 		if (reclaim == 1) {
17070Sstevel@tonic-gate 			segspt_purge(seg);
17080Sstevel@tonic-gate 			reclaim = 0;
17090Sstevel@tonic-gate 			goto retry;
17100Sstevel@tonic-gate 		}
17110Sstevel@tonic-gate 		return (EAGAIN);
17120Sstevel@tonic-gate 	}
17130Sstevel@tonic-gate 
17140Sstevel@tonic-gate 	if (ssize != seg->s_size) {
17150Sstevel@tonic-gate #ifdef DEBUG
17160Sstevel@tonic-gate 		cmn_err(CE_WARN, "Incompatible ssize %lx s_size %lx\n",
17170Sstevel@tonic-gate 		    ssize, seg->s_size);
17180Sstevel@tonic-gate #endif
17190Sstevel@tonic-gate 		return (EINVAL);
17200Sstevel@tonic-gate 	}
17210Sstevel@tonic-gate 
17220Sstevel@tonic-gate 	(void) segspt_shmlockop(seg, raddr, shmd->shm_amp->size, 0, MC_UNLOCK,
17230Sstevel@tonic-gate 	    NULL, 0);
17240Sstevel@tonic-gate 	hat_unshare(seg->s_as->a_hat, raddr, ssize, seg->s_szc);
17250Sstevel@tonic-gate 
17260Sstevel@tonic-gate 	seg_free(seg);
17270Sstevel@tonic-gate 
17280Sstevel@tonic-gate 	return (0);
17290Sstevel@tonic-gate }
17300Sstevel@tonic-gate 
17310Sstevel@tonic-gate void
17320Sstevel@tonic-gate segspt_shmfree(struct seg *seg)
17330Sstevel@tonic-gate {
17340Sstevel@tonic-gate 	struct shm_data *shmd = (struct shm_data *)seg->s_data;
17350Sstevel@tonic-gate 	struct anon_map *shm_amp = shmd->shm_amp;
17360Sstevel@tonic-gate 
17370Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
17380Sstevel@tonic-gate 
17390Sstevel@tonic-gate 	(void) segspt_shmlockop(seg, seg->s_base, shm_amp->size, 0,
17400Sstevel@tonic-gate 		MC_UNLOCK, NULL, 0);
17410Sstevel@tonic-gate 
17420Sstevel@tonic-gate 	/*
17430Sstevel@tonic-gate 	 * Need to increment refcnt when attaching
17440Sstevel@tonic-gate 	 * and decrement when detaching because of dup().
17450Sstevel@tonic-gate 	 */
17460Sstevel@tonic-gate 	ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER);
17470Sstevel@tonic-gate 	shm_amp->refcnt--;
17480Sstevel@tonic-gate 	ANON_LOCK_EXIT(&shm_amp->a_rwlock);
17490Sstevel@tonic-gate 
17500Sstevel@tonic-gate 	if (shmd->shm_vpage) {	/* only for DISM */
17510Sstevel@tonic-gate 		kmem_free(shmd->shm_vpage, btopr(shm_amp->size));
17520Sstevel@tonic-gate 		shmd->shm_vpage = NULL;
17530Sstevel@tonic-gate 	}
17540Sstevel@tonic-gate 	kmem_free(shmd, sizeof (*shmd));
17550Sstevel@tonic-gate }
17560Sstevel@tonic-gate 
17570Sstevel@tonic-gate /*ARGSUSED*/
17580Sstevel@tonic-gate int
17590Sstevel@tonic-gate segspt_shmsetprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
17600Sstevel@tonic-gate {
17610Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
17620Sstevel@tonic-gate 
17630Sstevel@tonic-gate 	/*
17640Sstevel@tonic-gate 	 * Shared page table is more than shared mapping.
17650Sstevel@tonic-gate 	 *  Individual process sharing page tables can't change prot
17660Sstevel@tonic-gate 	 *  because there is only one set of page tables.
17670Sstevel@tonic-gate 	 *  This will be allowed after private page table is
17680Sstevel@tonic-gate 	 *  supported.
17690Sstevel@tonic-gate 	 */
17700Sstevel@tonic-gate /* need to return correct status error? */
17710Sstevel@tonic-gate 	return (0);
17720Sstevel@tonic-gate }
17730Sstevel@tonic-gate 
17740Sstevel@tonic-gate 
17750Sstevel@tonic-gate faultcode_t
17760Sstevel@tonic-gate segspt_dismfault(struct hat *hat, struct seg *seg, caddr_t addr,
17770Sstevel@tonic-gate     size_t len, enum fault_type type, enum seg_rw rw)
17780Sstevel@tonic-gate {
17790Sstevel@tonic-gate 	struct  shm_data 	*shmd = (struct shm_data *)seg->s_data;
17800Sstevel@tonic-gate 	struct  seg		*sptseg = shmd->shm_sptseg;
17810Sstevel@tonic-gate 	struct  as		*curspt = shmd->shm_sptas;
17820Sstevel@tonic-gate 	struct  spt_data 	*sptd = sptseg->s_data;
17830Sstevel@tonic-gate 	pgcnt_t npages;
17842414Saguzovsk 	size_t  size;
17850Sstevel@tonic-gate 	caddr_t segspt_addr, shm_addr;
17860Sstevel@tonic-gate 	page_t  **ppa;
17870Sstevel@tonic-gate 	int	i;
17880Sstevel@tonic-gate 	ulong_t an_idx = 0;
17890Sstevel@tonic-gate 	int	err = 0;
1790721Smec 	int	dyn_ism_unmap = hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0);
17912414Saguzovsk 	size_t	pgsz;
17922414Saguzovsk 	pgcnt_t	pgcnt;
17932414Saguzovsk 	caddr_t	a;
17942414Saguzovsk 	pgcnt_t	pidx;
17950Sstevel@tonic-gate 
17960Sstevel@tonic-gate #ifdef lint
17970Sstevel@tonic-gate 	hat = hat;
17980Sstevel@tonic-gate #endif
17990Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
18000Sstevel@tonic-gate 
18010Sstevel@tonic-gate 	/*
18020Sstevel@tonic-gate 	 * Because of the way spt is implemented
18030Sstevel@tonic-gate 	 * the realsize of the segment does not have to be
18040Sstevel@tonic-gate 	 * equal to the segment size itself. The segment size is
18050Sstevel@tonic-gate 	 * often in multiples of a page size larger than PAGESIZE.
18060Sstevel@tonic-gate 	 * The realsize is rounded up to the nearest PAGESIZE
18070Sstevel@tonic-gate 	 * based on what the user requested. This is a bit of
18080Sstevel@tonic-gate 	 * ungliness that is historical but not easily fixed
18090Sstevel@tonic-gate 	 * without re-designing the higher levels of ISM.
18100Sstevel@tonic-gate 	 */
18110Sstevel@tonic-gate 	ASSERT(addr >= seg->s_base);
18120Sstevel@tonic-gate 	if (((addr + len) - seg->s_base) > sptd->spt_realsize)
18130Sstevel@tonic-gate 		return (FC_NOMAP);
18140Sstevel@tonic-gate 	/*
18150Sstevel@tonic-gate 	 * For all of the following cases except F_PROT, we need to
18160Sstevel@tonic-gate 	 * make any necessary adjustments to addr and len
18170Sstevel@tonic-gate 	 * and get all of the necessary page_t's into an array called ppa[].
18180Sstevel@tonic-gate 	 *
18190Sstevel@tonic-gate 	 * The code in shmat() forces base addr and len of ISM segment
18200Sstevel@tonic-gate 	 * to be aligned to largest page size supported. Therefore,
18210Sstevel@tonic-gate 	 * we are able to handle F_SOFTLOCK and F_INVAL calls in "large
18220Sstevel@tonic-gate 	 * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK
18230Sstevel@tonic-gate 	 * in large pagesize chunks, or else we will screw up the HAT
18240Sstevel@tonic-gate 	 * layer by calling hat_memload_array() with differing page sizes
18250Sstevel@tonic-gate 	 * over a given virtual range.
18260Sstevel@tonic-gate 	 */
18272414Saguzovsk 	pgsz = page_get_pagesize(sptseg->s_szc);
18282414Saguzovsk 	pgcnt = page_get_pagecnt(sptseg->s_szc);
18292414Saguzovsk 	shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz);
18302414Saguzovsk 	size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz);
18310Sstevel@tonic-gate 	npages = btopr(size);
18320Sstevel@tonic-gate 
18330Sstevel@tonic-gate 	/*
18340Sstevel@tonic-gate 	 * Now we need to convert from addr in segshm to addr in segspt.
18350Sstevel@tonic-gate 	 */
18360Sstevel@tonic-gate 	an_idx = seg_page(seg, shm_addr);
18370Sstevel@tonic-gate 	segspt_addr = sptseg->s_base + ptob(an_idx);
18380Sstevel@tonic-gate 
18390Sstevel@tonic-gate 	ASSERT((segspt_addr + ptob(npages)) <=
18400Sstevel@tonic-gate 		(sptseg->s_base + sptd->spt_realsize));
18410Sstevel@tonic-gate 	ASSERT(segspt_addr < (sptseg->s_base + sptseg->s_size));
18420Sstevel@tonic-gate 
18430Sstevel@tonic-gate 	switch (type) {
18440Sstevel@tonic-gate 
18450Sstevel@tonic-gate 	case F_SOFTLOCK:
18460Sstevel@tonic-gate 
18470Sstevel@tonic-gate 		mutex_enter(&freemem_lock);
18480Sstevel@tonic-gate 		if (availrmem < tune.t_minarmem + npages) {
18490Sstevel@tonic-gate 			mutex_exit(&freemem_lock);
18500Sstevel@tonic-gate 			return (FC_MAKE_ERR(ENOMEM));
18510Sstevel@tonic-gate 		} else {
18520Sstevel@tonic-gate 			availrmem -= npages;
18530Sstevel@tonic-gate 		}
18540Sstevel@tonic-gate 		mutex_exit(&freemem_lock);
18550Sstevel@tonic-gate 		atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages);
18560Sstevel@tonic-gate 		/*
18570Sstevel@tonic-gate 		 * Fall through to the F_INVAL case to load up the hat layer
18580Sstevel@tonic-gate 		 * entries with the HAT_LOAD_LOCK flag.
18590Sstevel@tonic-gate 		 */
18600Sstevel@tonic-gate 		/* FALLTHRU */
18610Sstevel@tonic-gate 	case F_INVAL:
18620Sstevel@tonic-gate 
18630Sstevel@tonic-gate 		if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC))
18640Sstevel@tonic-gate 			return (FC_NOMAP);
18650Sstevel@tonic-gate 
18660Sstevel@tonic-gate 		ppa = kmem_zalloc(npages * sizeof (page_t *), KM_SLEEP);
18670Sstevel@tonic-gate 
18680Sstevel@tonic-gate 		err = spt_anon_getpages(sptseg, segspt_addr, size, ppa);
18690Sstevel@tonic-gate 		if (err != 0) {
18700Sstevel@tonic-gate 			if (type == F_SOFTLOCK) {
18710Sstevel@tonic-gate 				mutex_enter(&freemem_lock);
18720Sstevel@tonic-gate 				availrmem += npages;
18730Sstevel@tonic-gate 				mutex_exit(&freemem_lock);
18740Sstevel@tonic-gate 				atomic_add_long((ulong_t *)(
18750Sstevel@tonic-gate 				    &(shmd->shm_softlockcnt)), -npages);
18760Sstevel@tonic-gate 			}
18770Sstevel@tonic-gate 			goto dism_err;
18780Sstevel@tonic-gate 		}
18790Sstevel@tonic-gate 		AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER);
18802414Saguzovsk 		a = segspt_addr;
18812414Saguzovsk 		pidx = 0;
18820Sstevel@tonic-gate 		if (type == F_SOFTLOCK) {
18830Sstevel@tonic-gate 
18840Sstevel@tonic-gate 			/*
18850Sstevel@tonic-gate 			 * Load up the translation keeping it
18860Sstevel@tonic-gate 			 * locked and don't unlock the page.
18870Sstevel@tonic-gate 			 */
18882414Saguzovsk 			for (; pidx < npages; a += pgsz, pidx += pgcnt) {
18892414Saguzovsk 				hat_memload_array(sptseg->s_as->a_hat,
18902414Saguzovsk 				    a, pgsz, &ppa[pidx], sptd->spt_prot,
18912414Saguzovsk 				    HAT_LOAD_LOCK | HAT_LOAD_SHARE);
18922414Saguzovsk 			}
18930Sstevel@tonic-gate 		} else {
18940Sstevel@tonic-gate 			if (hat == seg->s_as->a_hat) {
18950Sstevel@tonic-gate 
18960Sstevel@tonic-gate 				/*
18970Sstevel@tonic-gate 				 * Migrate pages marked for migration
18980Sstevel@tonic-gate 				 */
18990Sstevel@tonic-gate 				if (lgrp_optimizations())
19000Sstevel@tonic-gate 					page_migrate(seg, shm_addr, ppa,
19010Sstevel@tonic-gate 					    npages);
19020Sstevel@tonic-gate 
19030Sstevel@tonic-gate 				/* CPU HAT */
19042414Saguzovsk 				for (; pidx < npages;
19052414Saguzovsk 				    a += pgsz, pidx += pgcnt) {
19062414Saguzovsk 					hat_memload_array(sptseg->s_as->a_hat,
19072414Saguzovsk 					    a, pgsz, &ppa[pidx],
19082414Saguzovsk 					    sptd->spt_prot,
19092414Saguzovsk 					    HAT_LOAD_SHARE);
19102414Saguzovsk 				}
19110Sstevel@tonic-gate 			} else {
19120Sstevel@tonic-gate 				/* XHAT. Pass real address */
19130Sstevel@tonic-gate 				hat_memload_array(hat, shm_addr,
19140Sstevel@tonic-gate 				    size, ppa, sptd->spt_prot, HAT_LOAD_SHARE);
19150Sstevel@tonic-gate 			}
19160Sstevel@tonic-gate 
19170Sstevel@tonic-gate 			/*
19180Sstevel@tonic-gate 			 * And now drop the SE_SHARED lock(s).
19190Sstevel@tonic-gate 			 */
1920721Smec 			if (dyn_ism_unmap) {
1921721Smec 				for (i = 0; i < npages; i++) {
1922721Smec 					page_unlock(ppa[i]);
1923721Smec 				}
1924721Smec 			}
19250Sstevel@tonic-gate 		}
19260Sstevel@tonic-gate 
1927721Smec 		if (!dyn_ism_unmap) {
19280Sstevel@tonic-gate 			if (hat_share(seg->s_as->a_hat, shm_addr,
19290Sstevel@tonic-gate 			    curspt->a_hat, segspt_addr, ptob(npages),
19300Sstevel@tonic-gate 			    seg->s_szc) != 0) {
19310Sstevel@tonic-gate 				panic("hat_share err in DISM fault");
19320Sstevel@tonic-gate 				/* NOTREACHED */
19330Sstevel@tonic-gate 			}
1934721Smec 			if (type == F_INVAL) {
1935721Smec 				for (i = 0; i < npages; i++) {
1936721Smec 					page_unlock(ppa[i]);
1937721Smec 				}
1938721Smec 			}
19390Sstevel@tonic-gate 		}
19400Sstevel@tonic-gate 		AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock);
19410Sstevel@tonic-gate dism_err:
19420Sstevel@tonic-gate 		kmem_free(ppa, npages * sizeof (page_t *));
19430Sstevel@tonic-gate 		return (err);
19440Sstevel@tonic-gate 
19450Sstevel@tonic-gate 	case F_SOFTUNLOCK:
19460Sstevel@tonic-gate 
19470Sstevel@tonic-gate 		mutex_enter(&freemem_lock);
19480Sstevel@tonic-gate 		availrmem += npages;
19490Sstevel@tonic-gate 		mutex_exit(&freemem_lock);
19500Sstevel@tonic-gate 
19510Sstevel@tonic-gate 		/*
19520Sstevel@tonic-gate 		 * This is a bit ugly, we pass in the real seg pointer,
19530Sstevel@tonic-gate 		 * but the segspt_addr is the virtual address within the
19540Sstevel@tonic-gate 		 * dummy seg.
19550Sstevel@tonic-gate 		 */
19560Sstevel@tonic-gate 		segspt_softunlock(seg, segspt_addr, size, rw);
19570Sstevel@tonic-gate 		return (0);
19580Sstevel@tonic-gate 
19590Sstevel@tonic-gate 	case F_PROT:
19600Sstevel@tonic-gate 
19610Sstevel@tonic-gate 		/*
19620Sstevel@tonic-gate 		 * This takes care of the unusual case where a user
19630Sstevel@tonic-gate 		 * allocates a stack in shared memory and a register
19640Sstevel@tonic-gate 		 * window overflow is written to that stack page before
19650Sstevel@tonic-gate 		 * it is otherwise modified.
19660Sstevel@tonic-gate 		 *
19670Sstevel@tonic-gate 		 * We can get away with this because ISM segments are
19680Sstevel@tonic-gate 		 * always rw. Other than this unusual case, there
19690Sstevel@tonic-gate 		 * should be no instances of protection violations.
19700Sstevel@tonic-gate 		 */
19710Sstevel@tonic-gate 		return (0);
19720Sstevel@tonic-gate 
19730Sstevel@tonic-gate 	default:
19740Sstevel@tonic-gate #ifdef DEBUG
19750Sstevel@tonic-gate 		panic("segspt_dismfault default type?");
19760Sstevel@tonic-gate #else
19770Sstevel@tonic-gate 		return (FC_NOMAP);
19780Sstevel@tonic-gate #endif
19790Sstevel@tonic-gate 	}
19800Sstevel@tonic-gate }
19810Sstevel@tonic-gate 
19820Sstevel@tonic-gate 
19830Sstevel@tonic-gate faultcode_t
19840Sstevel@tonic-gate segspt_shmfault(struct hat *hat, struct seg *seg, caddr_t addr,
19850Sstevel@tonic-gate     size_t len, enum fault_type type, enum seg_rw rw)
19860Sstevel@tonic-gate {
19870Sstevel@tonic-gate 	struct shm_data 	*shmd = (struct shm_data *)seg->s_data;
19880Sstevel@tonic-gate 	struct seg		*sptseg = shmd->shm_sptseg;
19890Sstevel@tonic-gate 	struct as		*curspt = shmd->shm_sptas;
19900Sstevel@tonic-gate 	struct spt_data 	*sptd   = sptseg->s_data;
19910Sstevel@tonic-gate 	pgcnt_t npages;
19922414Saguzovsk 	size_t size;
19930Sstevel@tonic-gate 	caddr_t sptseg_addr, shm_addr;
19940Sstevel@tonic-gate 	page_t *pp, **ppa;
19950Sstevel@tonic-gate 	int	i;
19960Sstevel@tonic-gate 	u_offset_t offset;
19970Sstevel@tonic-gate 	ulong_t anon_index = 0;
19980Sstevel@tonic-gate 	struct vnode *vp;
19990Sstevel@tonic-gate 	struct anon_map *amp;		/* XXX - for locknest */
20000Sstevel@tonic-gate 	struct anon *ap = NULL;
20010Sstevel@tonic-gate 	anon_sync_obj_t cookie;
20022414Saguzovsk 	size_t		pgsz;
20032414Saguzovsk 	pgcnt_t		pgcnt;
20042414Saguzovsk 	caddr_t		a;
20052414Saguzovsk 	pgcnt_t		pidx;
20062414Saguzovsk 	size_t		sz;
20070Sstevel@tonic-gate 
20080Sstevel@tonic-gate #ifdef lint
20090Sstevel@tonic-gate 	hat = hat;
20100Sstevel@tonic-gate #endif
20110Sstevel@tonic-gate 
20120Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
20130Sstevel@tonic-gate 
20140Sstevel@tonic-gate 	if (sptd->spt_flags & SHM_PAGEABLE) {
20150Sstevel@tonic-gate 		return (segspt_dismfault(hat, seg, addr, len, type, rw));
20160Sstevel@tonic-gate 	}
20170Sstevel@tonic-gate 
20180Sstevel@tonic-gate 	/*
20190Sstevel@tonic-gate 	 * Because of the way spt is implemented
20200Sstevel@tonic-gate 	 * the realsize of the segment does not have to be
20210Sstevel@tonic-gate 	 * equal to the segment size itself. The segment size is
20220Sstevel@tonic-gate 	 * often in multiples of a page size larger than PAGESIZE.
20230Sstevel@tonic-gate 	 * The realsize is rounded up to the nearest PAGESIZE
20240Sstevel@tonic-gate 	 * based on what the user requested. This is a bit of
20250Sstevel@tonic-gate 	 * ungliness that is historical but not easily fixed
20260Sstevel@tonic-gate 	 * without re-designing the higher levels of ISM.
20270Sstevel@tonic-gate 	 */
20280Sstevel@tonic-gate 	ASSERT(addr >= seg->s_base);
20290Sstevel@tonic-gate 	if (((addr + len) - seg->s_base) > sptd->spt_realsize)
20300Sstevel@tonic-gate 		return (FC_NOMAP);
20310Sstevel@tonic-gate 	/*
20320Sstevel@tonic-gate 	 * For all of the following cases except F_PROT, we need to
20330Sstevel@tonic-gate 	 * make any necessary adjustments to addr and len
20340Sstevel@tonic-gate 	 * and get all of the necessary page_t's into an array called ppa[].
20350Sstevel@tonic-gate 	 *
20360Sstevel@tonic-gate 	 * The code in shmat() forces base addr and len of ISM segment
20370Sstevel@tonic-gate 	 * to be aligned to largest page size supported. Therefore,
20380Sstevel@tonic-gate 	 * we are able to handle F_SOFTLOCK and F_INVAL calls in "large
20390Sstevel@tonic-gate 	 * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK
20400Sstevel@tonic-gate 	 * in large pagesize chunks, or else we will screw up the HAT
20410Sstevel@tonic-gate 	 * layer by calling hat_memload_array() with differing page sizes
20420Sstevel@tonic-gate 	 * over a given virtual range.
20430Sstevel@tonic-gate 	 */
20442414Saguzovsk 	pgsz = page_get_pagesize(sptseg->s_szc);
20452414Saguzovsk 	pgcnt = page_get_pagecnt(sptseg->s_szc);
20462414Saguzovsk 	shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz);
20472414Saguzovsk 	size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz);
20480Sstevel@tonic-gate 	npages = btopr(size);
20490Sstevel@tonic-gate 
20500Sstevel@tonic-gate 	/*
20510Sstevel@tonic-gate 	 * Now we need to convert from addr in segshm to addr in segspt.
20520Sstevel@tonic-gate 	 */
20530Sstevel@tonic-gate 	anon_index = seg_page(seg, shm_addr);
20540Sstevel@tonic-gate 	sptseg_addr = sptseg->s_base + ptob(anon_index);
20550Sstevel@tonic-gate 
20560Sstevel@tonic-gate 	/*
20570Sstevel@tonic-gate 	 * And now we may have to adjust npages downward if we have
20580Sstevel@tonic-gate 	 * exceeded the realsize of the segment or initial anon
20590Sstevel@tonic-gate 	 * allocations.
20600Sstevel@tonic-gate 	 */
20610Sstevel@tonic-gate 	if ((sptseg_addr + ptob(npages)) >
20620Sstevel@tonic-gate 	    (sptseg->s_base + sptd->spt_realsize))
20630Sstevel@tonic-gate 		size = (sptseg->s_base + sptd->spt_realsize) - sptseg_addr;
20640Sstevel@tonic-gate 
20650Sstevel@tonic-gate 	npages = btopr(size);
20660Sstevel@tonic-gate 
20670Sstevel@tonic-gate 	ASSERT(sptseg_addr < (sptseg->s_base + sptseg->s_size));
20680Sstevel@tonic-gate 	ASSERT((sptd->spt_flags & SHM_PAGEABLE) == 0);
20690Sstevel@tonic-gate 
20700Sstevel@tonic-gate 	switch (type) {
20710Sstevel@tonic-gate 
20720Sstevel@tonic-gate 	case F_SOFTLOCK:
20730Sstevel@tonic-gate 
20740Sstevel@tonic-gate 		/*
20750Sstevel@tonic-gate 		 * availrmem is decremented once during anon_swap_adjust()
20760Sstevel@tonic-gate 		 * and is incremented during the anon_unresv(), which is
20770Sstevel@tonic-gate 		 * called from shm_rm_amp() when the segment is destroyed.
20780Sstevel@tonic-gate 		 */
20790Sstevel@tonic-gate 		atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages);
20800Sstevel@tonic-gate 		/*
20810Sstevel@tonic-gate 		 * Some platforms assume that ISM pages are SE_SHARED
20820Sstevel@tonic-gate 		 * locked for the entire life of the segment.
20830Sstevel@tonic-gate 		 */
20840Sstevel@tonic-gate 		if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0))
20850Sstevel@tonic-gate 			return (0);
20860Sstevel@tonic-gate 		/*
20870Sstevel@tonic-gate 		 * Fall through to the F_INVAL case to load up the hat layer
20880Sstevel@tonic-gate 		 * entries with the HAT_LOAD_LOCK flag.
20890Sstevel@tonic-gate 		 */
20900Sstevel@tonic-gate 
20910Sstevel@tonic-gate 		/* FALLTHRU */
20920Sstevel@tonic-gate 	case F_INVAL:
20930Sstevel@tonic-gate 
20940Sstevel@tonic-gate 		if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC))
20950Sstevel@tonic-gate 			return (FC_NOMAP);
20960Sstevel@tonic-gate 
20970Sstevel@tonic-gate 		/*
20980Sstevel@tonic-gate 		 * Some platforms that do NOT support DYNAMIC_ISM_UNMAP
20990Sstevel@tonic-gate 		 * may still rely on this call to hat_share(). That
21000Sstevel@tonic-gate 		 * would imply that those hat's can fault on a
21010Sstevel@tonic-gate 		 * HAT_LOAD_LOCK translation, which would seem
21020Sstevel@tonic-gate 		 * contradictory.
21030Sstevel@tonic-gate 		 */
21040Sstevel@tonic-gate 		if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
21050Sstevel@tonic-gate 			if (hat_share(seg->s_as->a_hat, seg->s_base,
21060Sstevel@tonic-gate 			    curspt->a_hat, sptseg->s_base,
21070Sstevel@tonic-gate 			    sptseg->s_size, sptseg->s_szc) != 0) {
21080Sstevel@tonic-gate 				panic("hat_share error in ISM fault");
21090Sstevel@tonic-gate 				/*NOTREACHED*/
21100Sstevel@tonic-gate 			}
21110Sstevel@tonic-gate 			return (0);
21120Sstevel@tonic-gate 		}
21130Sstevel@tonic-gate 		ppa = kmem_zalloc(sizeof (page_t *) * npages, KM_SLEEP);
21140Sstevel@tonic-gate 
21150Sstevel@tonic-gate 		/*
21160Sstevel@tonic-gate 		 * I see no need to lock the real seg,
21170Sstevel@tonic-gate 		 * here, because all of our work will be on the underlying
21180Sstevel@tonic-gate 		 * dummy seg.
21190Sstevel@tonic-gate 		 *
21200Sstevel@tonic-gate 		 * sptseg_addr and npages now account for large pages.
21210Sstevel@tonic-gate 		 */
21220Sstevel@tonic-gate 		amp = sptd->spt_amp;
21230Sstevel@tonic-gate 		ASSERT(amp != NULL);
21240Sstevel@tonic-gate 		anon_index = seg_page(sptseg, sptseg_addr);
21250Sstevel@tonic-gate 
21260Sstevel@tonic-gate 		ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
21270Sstevel@tonic-gate 		for (i = 0; i < npages; i++) {
21280Sstevel@tonic-gate 			anon_array_enter(amp, anon_index, &cookie);
21290Sstevel@tonic-gate 			ap = anon_get_ptr(amp->ahp, anon_index++);
21300Sstevel@tonic-gate 			ASSERT(ap != NULL);
21310Sstevel@tonic-gate 			swap_xlate(ap, &vp, &offset);
21320Sstevel@tonic-gate 			anon_array_exit(&cookie);
21330Sstevel@tonic-gate 			pp = page_lookup(vp, offset, SE_SHARED);
21340Sstevel@tonic-gate 			ASSERT(pp != NULL);
21350Sstevel@tonic-gate 			ppa[i] = pp;
21360Sstevel@tonic-gate 		}
21370Sstevel@tonic-gate 		ANON_LOCK_EXIT(&amp->a_rwlock);
21380Sstevel@tonic-gate 		ASSERT(i == npages);
21390Sstevel@tonic-gate 
21400Sstevel@tonic-gate 		/*
21410Sstevel@tonic-gate 		 * We are already holding the as->a_lock on the user's
21420Sstevel@tonic-gate 		 * real segment, but we need to hold the a_lock on the
21430Sstevel@tonic-gate 		 * underlying dummy as. This is mostly to satisfy the
21440Sstevel@tonic-gate 		 * underlying HAT layer.
21450Sstevel@tonic-gate 		 */
21460Sstevel@tonic-gate 		AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER);
21472414Saguzovsk 		a = sptseg_addr;
21482414Saguzovsk 		pidx = 0;
21490Sstevel@tonic-gate 		if (type == F_SOFTLOCK) {
21500Sstevel@tonic-gate 			/*
21510Sstevel@tonic-gate 			 * Load up the translation keeping it
21520Sstevel@tonic-gate 			 * locked and don't unlock the page.
21530Sstevel@tonic-gate 			 */
21542414Saguzovsk 			for (; pidx < npages; a += pgsz, pidx += pgcnt) {
21552414Saguzovsk 				sz = MIN(pgsz, ptob(npages - pidx));
21562414Saguzovsk 				hat_memload_array(sptseg->s_as->a_hat, a,
21572414Saguzovsk 				    sz, &ppa[pidx], sptd->spt_prot,
21582414Saguzovsk 				    HAT_LOAD_LOCK | HAT_LOAD_SHARE);
21592414Saguzovsk 			}
21600Sstevel@tonic-gate 		} else {
21610Sstevel@tonic-gate 			if (hat == seg->s_as->a_hat) {
21620Sstevel@tonic-gate 
21630Sstevel@tonic-gate 				/*
21640Sstevel@tonic-gate 				 * Migrate pages marked for migration.
21650Sstevel@tonic-gate 				 */
21660Sstevel@tonic-gate 				if (lgrp_optimizations())
21670Sstevel@tonic-gate 					page_migrate(seg, shm_addr, ppa,
21680Sstevel@tonic-gate 					    npages);
21690Sstevel@tonic-gate 
21700Sstevel@tonic-gate 				/* CPU HAT */
21712414Saguzovsk 				for (; pidx < npages;
21722414Saguzovsk 				    a += pgsz, pidx += pgcnt) {
21732414Saguzovsk 					sz = MIN(pgsz, ptob(npages - pidx));
21742414Saguzovsk 					hat_memload_array(sptseg->s_as->a_hat,
21752414Saguzovsk 					    a, sz, &ppa[pidx],
21762414Saguzovsk 					    sptd->spt_prot, HAT_LOAD_SHARE);
21772414Saguzovsk 				}
21780Sstevel@tonic-gate 			} else {
21790Sstevel@tonic-gate 				/* XHAT. Pass real address */
21800Sstevel@tonic-gate 				hat_memload_array(hat, shm_addr,
21810Sstevel@tonic-gate 				    ptob(npages), ppa, sptd->spt_prot,
21820Sstevel@tonic-gate 				    HAT_LOAD_SHARE);
21830Sstevel@tonic-gate 			}
21840Sstevel@tonic-gate 
21850Sstevel@tonic-gate 			/*
21860Sstevel@tonic-gate 			 * And now drop the SE_SHARED lock(s).
21870Sstevel@tonic-gate 			 */
21880Sstevel@tonic-gate 			for (i = 0; i < npages; i++)
21890Sstevel@tonic-gate 				page_unlock(ppa[i]);
21900Sstevel@tonic-gate 		}
21910Sstevel@tonic-gate 		AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock);
21920Sstevel@tonic-gate 
21930Sstevel@tonic-gate 		kmem_free(ppa, sizeof (page_t *) * npages);
21940Sstevel@tonic-gate 		return (0);
21950Sstevel@tonic-gate 	case F_SOFTUNLOCK:
21960Sstevel@tonic-gate 
21970Sstevel@tonic-gate 		/*
21980Sstevel@tonic-gate 		 * This is a bit ugly, we pass in the real seg pointer,
21990Sstevel@tonic-gate 		 * but the sptseg_addr is the virtual address within the
22000Sstevel@tonic-gate 		 * dummy seg.
22010Sstevel@tonic-gate 		 */
22020Sstevel@tonic-gate 		segspt_softunlock(seg, sptseg_addr, ptob(npages), rw);
22030Sstevel@tonic-gate 		return (0);
22040Sstevel@tonic-gate 
22050Sstevel@tonic-gate 	case F_PROT:
22060Sstevel@tonic-gate 
22070Sstevel@tonic-gate 		/*
22080Sstevel@tonic-gate 		 * This takes care of the unusual case where a user
22090Sstevel@tonic-gate 		 * allocates a stack in shared memory and a register
22100Sstevel@tonic-gate 		 * window overflow is written to that stack page before
22110Sstevel@tonic-gate 		 * it is otherwise modified.
22120Sstevel@tonic-gate 		 *
22130Sstevel@tonic-gate 		 * We can get away with this because ISM segments are
22140Sstevel@tonic-gate 		 * always rw. Other than this unusual case, there
22150Sstevel@tonic-gate 		 * should be no instances of protection violations.
22160Sstevel@tonic-gate 		 */
22170Sstevel@tonic-gate 		return (0);
22180Sstevel@tonic-gate 
22190Sstevel@tonic-gate 	default:
22200Sstevel@tonic-gate #ifdef DEBUG
22210Sstevel@tonic-gate 		cmn_err(CE_WARN, "segspt_shmfault default type?");
22220Sstevel@tonic-gate #endif
22230Sstevel@tonic-gate 		return (FC_NOMAP);
22240Sstevel@tonic-gate 	}
22250Sstevel@tonic-gate }
22260Sstevel@tonic-gate 
22270Sstevel@tonic-gate /*ARGSUSED*/
22280Sstevel@tonic-gate static faultcode_t
22290Sstevel@tonic-gate segspt_shmfaulta(struct seg *seg, caddr_t addr)
22300Sstevel@tonic-gate {
22310Sstevel@tonic-gate 	return (0);
22320Sstevel@tonic-gate }
22330Sstevel@tonic-gate 
22340Sstevel@tonic-gate /*ARGSUSED*/
22350Sstevel@tonic-gate static int
22360Sstevel@tonic-gate segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta)
22370Sstevel@tonic-gate {
22380Sstevel@tonic-gate 	return (0);
22390Sstevel@tonic-gate }
22400Sstevel@tonic-gate 
22410Sstevel@tonic-gate /*ARGSUSED*/
22420Sstevel@tonic-gate static size_t
22430Sstevel@tonic-gate segspt_shmswapout(struct seg *seg)
22440Sstevel@tonic-gate {
22450Sstevel@tonic-gate 	return (0);
22460Sstevel@tonic-gate }
22470Sstevel@tonic-gate 
22480Sstevel@tonic-gate /*
22490Sstevel@tonic-gate  * duplicate the shared page tables
22500Sstevel@tonic-gate  */
22510Sstevel@tonic-gate int
22520Sstevel@tonic-gate segspt_shmdup(struct seg *seg, struct seg *newseg)
22530Sstevel@tonic-gate {
22540Sstevel@tonic-gate 	struct shm_data		*shmd = (struct shm_data *)seg->s_data;
22550Sstevel@tonic-gate 	struct anon_map 	*amp = shmd->shm_amp;
22560Sstevel@tonic-gate 	struct shm_data 	*shmd_new;
22570Sstevel@tonic-gate 	struct seg		*spt_seg = shmd->shm_sptseg;
22580Sstevel@tonic-gate 	struct spt_data		*sptd = spt_seg->s_data;
2259721Smec 	int			error = 0;
22600Sstevel@tonic-gate 
22610Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
22620Sstevel@tonic-gate 
22630Sstevel@tonic-gate 	shmd_new = kmem_zalloc((sizeof (*shmd_new)), KM_SLEEP);
22640Sstevel@tonic-gate 	newseg->s_data = (void *)shmd_new;
22650Sstevel@tonic-gate 	shmd_new->shm_sptas = shmd->shm_sptas;
22660Sstevel@tonic-gate 	shmd_new->shm_amp = amp;
22670Sstevel@tonic-gate 	shmd_new->shm_sptseg = shmd->shm_sptseg;
22680Sstevel@tonic-gate 	newseg->s_ops = &segspt_shmops;
22690Sstevel@tonic-gate 	newseg->s_szc = seg->s_szc;
22700Sstevel@tonic-gate 	ASSERT(seg->s_szc == shmd->shm_sptseg->s_szc);
22710Sstevel@tonic-gate 
22720Sstevel@tonic-gate 	ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
22730Sstevel@tonic-gate 	amp->refcnt++;
22740Sstevel@tonic-gate 	ANON_LOCK_EXIT(&amp->a_rwlock);
22750Sstevel@tonic-gate 
22760Sstevel@tonic-gate 	if (sptd->spt_flags & SHM_PAGEABLE) {
22770Sstevel@tonic-gate 		shmd_new->shm_vpage = kmem_zalloc(btopr(amp->size), KM_SLEEP);
22780Sstevel@tonic-gate 		shmd_new->shm_lckpgs = 0;
2279721Smec 		if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
2280721Smec 			if ((error = hat_share(newseg->s_as->a_hat,
2281721Smec 			    newseg->s_base, shmd->shm_sptas->a_hat, SEGSPTADDR,
2282721Smec 			    seg->s_size, seg->s_szc)) != 0) {
2283721Smec 				kmem_free(shmd_new->shm_vpage,
2284721Smec 				btopr(amp->size));
2285721Smec 			}
2286721Smec 		}
2287721Smec 		return (error);
2288721Smec 	} else {
2289721Smec 		return (hat_share(newseg->s_as->a_hat, newseg->s_base,
2290721Smec 		    shmd->shm_sptas->a_hat, SEGSPTADDR, seg->s_size,
2291721Smec 		    seg->s_szc));
2292721Smec 
22930Sstevel@tonic-gate 	}
22940Sstevel@tonic-gate }
22950Sstevel@tonic-gate 
22960Sstevel@tonic-gate /*ARGSUSED*/
22970Sstevel@tonic-gate int
22980Sstevel@tonic-gate segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
22990Sstevel@tonic-gate {
23000Sstevel@tonic-gate 	struct shm_data *shmd = (struct shm_data *)seg->s_data;
23010Sstevel@tonic-gate 	struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
23020Sstevel@tonic-gate 
23030Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
23040Sstevel@tonic-gate 
23050Sstevel@tonic-gate 	/*
23060Sstevel@tonic-gate 	 * ISM segment is always rw.
23070Sstevel@tonic-gate 	 */
23080Sstevel@tonic-gate 	return (((sptd->spt_prot & prot) != prot) ? EACCES : 0);
23090Sstevel@tonic-gate }
23100Sstevel@tonic-gate 
23110Sstevel@tonic-gate /*
23120Sstevel@tonic-gate  * Return an array of locked large pages, for empty slots allocate
23130Sstevel@tonic-gate  * private zero-filled anon pages.
23140Sstevel@tonic-gate  */
23150Sstevel@tonic-gate static int
23160Sstevel@tonic-gate spt_anon_getpages(
23170Sstevel@tonic-gate 	struct seg *sptseg,
23180Sstevel@tonic-gate 	caddr_t sptaddr,
23190Sstevel@tonic-gate 	size_t len,
23200Sstevel@tonic-gate 	page_t *ppa[])
23210Sstevel@tonic-gate {
23220Sstevel@tonic-gate 	struct  spt_data *sptd = sptseg->s_data;
23230Sstevel@tonic-gate 	struct  anon_map *amp = sptd->spt_amp;
23240Sstevel@tonic-gate 	enum 	seg_rw rw = sptd->spt_prot;
23250Sstevel@tonic-gate 	uint_t	szc = sptseg->s_szc;
23260Sstevel@tonic-gate 	size_t	pg_sz, share_sz = page_get_pagesize(szc);
23270Sstevel@tonic-gate 	pgcnt_t	lp_npgs;
23280Sstevel@tonic-gate 	caddr_t	lp_addr, e_sptaddr;
23290Sstevel@tonic-gate 	uint_t	vpprot, ppa_szc = 0;
23300Sstevel@tonic-gate 	struct  vpage *vpage = NULL;
23310Sstevel@tonic-gate 	ulong_t	j, ppa_idx;
23320Sstevel@tonic-gate 	int	err, ierr = 0;
23330Sstevel@tonic-gate 	pgcnt_t	an_idx;
23340Sstevel@tonic-gate 	anon_sync_obj_t cookie;
23350Sstevel@tonic-gate 
23360Sstevel@tonic-gate 	ASSERT(IS_P2ALIGNED(sptaddr, share_sz) && IS_P2ALIGNED(len, share_sz));
23370Sstevel@tonic-gate 	ASSERT(len != 0);
23380Sstevel@tonic-gate 
23390Sstevel@tonic-gate 	pg_sz = share_sz;
23400Sstevel@tonic-gate 	lp_npgs = btop(pg_sz);
23410Sstevel@tonic-gate 	lp_addr = sptaddr;
23420Sstevel@tonic-gate 	e_sptaddr = sptaddr + len;
23430Sstevel@tonic-gate 	an_idx = seg_page(sptseg, sptaddr);
23440Sstevel@tonic-gate 	ppa_idx = 0;
23450Sstevel@tonic-gate 
23460Sstevel@tonic-gate 	ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
23470Sstevel@tonic-gate 	/*CONSTCOND*/
23480Sstevel@tonic-gate 	while (1) {
23490Sstevel@tonic-gate 		for (; lp_addr < e_sptaddr;
23500Sstevel@tonic-gate 			an_idx += lp_npgs, lp_addr += pg_sz,
23510Sstevel@tonic-gate 			ppa_idx += lp_npgs) {
23520Sstevel@tonic-gate 
23530Sstevel@tonic-gate 			anon_array_enter(amp, an_idx, &cookie);
23540Sstevel@tonic-gate 			ppa_szc = (uint_t)-1;
23550Sstevel@tonic-gate 			ierr = anon_map_getpages(amp, an_idx, szc, sptseg,
23560Sstevel@tonic-gate 			    lp_addr, sptd->spt_prot, &vpprot, &ppa[ppa_idx],
23570Sstevel@tonic-gate 			    &ppa_szc, vpage, rw, 0, segvn_anypgsz, kcred);
23580Sstevel@tonic-gate 			anon_array_exit(&cookie);
23590Sstevel@tonic-gate 
23600Sstevel@tonic-gate 			if (ierr != 0) {
23610Sstevel@tonic-gate 				if (ierr > 0) {
23620Sstevel@tonic-gate 					err = FC_MAKE_ERR(ierr);
23630Sstevel@tonic-gate 					goto lpgs_err;
23640Sstevel@tonic-gate 				}
23650Sstevel@tonic-gate 				break;
23660Sstevel@tonic-gate 			}
23670Sstevel@tonic-gate 		}
23680Sstevel@tonic-gate 		if (lp_addr == e_sptaddr) {
23690Sstevel@tonic-gate 			break;
23700Sstevel@tonic-gate 		}
23710Sstevel@tonic-gate 		ASSERT(lp_addr < e_sptaddr);
23720Sstevel@tonic-gate 
23730Sstevel@tonic-gate 		/*
23740Sstevel@tonic-gate 		 * ierr == -1 means we failed to allocate a large page.
23750Sstevel@tonic-gate 		 * so do a size down operation.
23760Sstevel@tonic-gate 		 *
23770Sstevel@tonic-gate 		 * ierr == -2 means some other process that privately shares
23780Sstevel@tonic-gate 		 * pages with this process has allocated a larger page and we
23790Sstevel@tonic-gate 		 * need to retry with larger pages. So do a size up
23800Sstevel@tonic-gate 		 * operation. This relies on the fact that large pages are
23810Sstevel@tonic-gate 		 * never partially shared i.e. if we share any constituent
23820Sstevel@tonic-gate 		 * page of a large page with another process we must share the
23830Sstevel@tonic-gate 		 * entire large page. Note this cannot happen for SOFTLOCK
23840Sstevel@tonic-gate 		 * case, unless current address (lpaddr) is at the beginning
23850Sstevel@tonic-gate 		 * of the next page size boundary because the other process
23860Sstevel@tonic-gate 		 * couldn't have relocated locked pages.
23870Sstevel@tonic-gate 		 */
23880Sstevel@tonic-gate 		ASSERT(ierr == -1 || ierr == -2);
23890Sstevel@tonic-gate 		if (segvn_anypgsz) {
23900Sstevel@tonic-gate 			ASSERT(ierr == -2 || szc != 0);
23910Sstevel@tonic-gate 			ASSERT(ierr == -1 || szc < sptseg->s_szc);
23920Sstevel@tonic-gate 			szc = (ierr == -1) ? szc - 1 : szc + 1;
23930Sstevel@tonic-gate 		} else {
23940Sstevel@tonic-gate 			/*
23950Sstevel@tonic-gate 			 * For faults and segvn_anypgsz == 0
23960Sstevel@tonic-gate 			 * we need to be careful not to loop forever
23970Sstevel@tonic-gate 			 * if existing page is found with szc other
23980Sstevel@tonic-gate 			 * than 0 or seg->s_szc. This could be due
23990Sstevel@tonic-gate 			 * to page relocations on behalf of DR or
24000Sstevel@tonic-gate 			 * more likely large page creation. For this
24010Sstevel@tonic-gate 			 * case simply re-size to existing page's szc
24020Sstevel@tonic-gate 			 * if returned by anon_map_getpages().
24030Sstevel@tonic-gate 			 */
24040Sstevel@tonic-gate 			if (ppa_szc == (uint_t)-1) {
24050Sstevel@tonic-gate 				szc = (ierr == -1) ? 0 : sptseg->s_szc;
24060Sstevel@tonic-gate 			} else {
24070Sstevel@tonic-gate 				ASSERT(ppa_szc <= sptseg->s_szc);
24080Sstevel@tonic-gate 				ASSERT(ierr == -2 || ppa_szc < szc);
24090Sstevel@tonic-gate 				ASSERT(ierr == -1 || ppa_szc > szc);
24100Sstevel@tonic-gate 				szc = ppa_szc;
24110Sstevel@tonic-gate 			}
24120Sstevel@tonic-gate 		}
24130Sstevel@tonic-gate 		pg_sz = page_get_pagesize(szc);
24140Sstevel@tonic-gate 		lp_npgs = btop(pg_sz);
24150Sstevel@tonic-gate 		ASSERT(IS_P2ALIGNED(lp_addr, pg_sz));
24160Sstevel@tonic-gate 	}
24170Sstevel@tonic-gate 	ANON_LOCK_EXIT(&amp->a_rwlock);
24180Sstevel@tonic-gate 	return (0);
24190Sstevel@tonic-gate 
24200Sstevel@tonic-gate lpgs_err:
24210Sstevel@tonic-gate 	ANON_LOCK_EXIT(&amp->a_rwlock);
24220Sstevel@tonic-gate 	for (j = 0; j < ppa_idx; j++)
24230Sstevel@tonic-gate 		page_unlock(ppa[j]);
24240Sstevel@tonic-gate 	return (err);
24250Sstevel@tonic-gate }
24260Sstevel@tonic-gate 
24272768Ssl108498 /*
24282768Ssl108498  * count the number of bytes in a set of spt pages that are currently not
24292768Ssl108498  * locked
24302768Ssl108498  */
24312768Ssl108498 static rctl_qty_t
24322768Ssl108498 spt_unlockedbytes(pgcnt_t npages, page_t **ppa)
24332768Ssl108498 {
24342768Ssl108498 	ulong_t	i;
24352768Ssl108498 	rctl_qty_t unlocked = 0;
24362768Ssl108498 
24372768Ssl108498 	for (i = 0; i < npages; i++) {
24382768Ssl108498 		if (ppa[i]->p_lckcnt == 0)
24392768Ssl108498 			unlocked += PAGESIZE;
24402768Ssl108498 	}
24412768Ssl108498 	return (unlocked);
24422768Ssl108498 }
24432768Ssl108498 
24440Sstevel@tonic-gate int
24450Sstevel@tonic-gate spt_lockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages,
24462768Ssl108498     page_t **ppa, ulong_t *lockmap, size_t pos,
24472768Ssl108498     rctl_qty_t *locked)
24480Sstevel@tonic-gate {
24490Sstevel@tonic-gate 	struct shm_data *shmd = seg->s_data;
24500Sstevel@tonic-gate 	struct spt_data *sptd = shmd->shm_sptseg->s_data;
24510Sstevel@tonic-gate 	ulong_t	i;
24520Sstevel@tonic-gate 	int	kernel;
24530Sstevel@tonic-gate 
24542768Ssl108498 	/* return the number of bytes actually locked */
24552768Ssl108498 	*locked = 0;
24560Sstevel@tonic-gate 	for (i = 0; i < npages; anon_index++, pos++, i++) {
24570Sstevel@tonic-gate 		if (!(shmd->shm_vpage[anon_index] & DISM_PG_LOCKED)) {
24580Sstevel@tonic-gate 			if (sptd->spt_ppa_lckcnt[anon_index] <
24590Sstevel@tonic-gate 			    (ushort_t)DISM_LOCK_MAX) {
24600Sstevel@tonic-gate 				if (++sptd->spt_ppa_lckcnt[anon_index] ==
24610Sstevel@tonic-gate 				    (ushort_t)DISM_LOCK_MAX) {
24620Sstevel@tonic-gate 					cmn_err(CE_WARN,
24630Sstevel@tonic-gate 					    "DISM page lock limit "
24640Sstevel@tonic-gate 					    "reached on DISM offset 0x%lx\n",
24650Sstevel@tonic-gate 					    anon_index << PAGESHIFT);
24660Sstevel@tonic-gate 				}
24670Sstevel@tonic-gate 				kernel = (sptd->spt_ppa &&
24680Sstevel@tonic-gate 				    sptd->spt_ppa[anon_index]) ? 1 : 0;
24690Sstevel@tonic-gate 				if (!page_pp_lock(ppa[i], 0, kernel)) {
24700Sstevel@tonic-gate 					sptd->spt_ppa_lckcnt[anon_index]--;
24710Sstevel@tonic-gate 					return (EAGAIN);
24720Sstevel@tonic-gate 				}
24732768Ssl108498 				/* if this is a newly locked page, count it */
24742768Ssl108498 				if (ppa[i]->p_lckcnt == 1) {
24752768Ssl108498 					*locked += PAGESIZE;
24762768Ssl108498 				}
24770Sstevel@tonic-gate 				shmd->shm_lckpgs++;
24780Sstevel@tonic-gate 				shmd->shm_vpage[anon_index] |= DISM_PG_LOCKED;
24790Sstevel@tonic-gate 				if (lockmap != NULL)
24800Sstevel@tonic-gate 					BT_SET(lockmap, pos);
24810Sstevel@tonic-gate 			}
24820Sstevel@tonic-gate 		}
24830Sstevel@tonic-gate 	}
24840Sstevel@tonic-gate 	return (0);
24850Sstevel@tonic-gate }
24860Sstevel@tonic-gate 
24870Sstevel@tonic-gate /*ARGSUSED*/
24880Sstevel@tonic-gate static int
24890Sstevel@tonic-gate segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len,
24900Sstevel@tonic-gate     int attr, int op, ulong_t *lockmap, size_t pos)
24910Sstevel@tonic-gate {
24920Sstevel@tonic-gate 	struct shm_data *shmd = seg->s_data;
24930Sstevel@tonic-gate 	struct seg	*sptseg = shmd->shm_sptseg;
24940Sstevel@tonic-gate 	struct spt_data *sptd = sptseg->s_data;
24952768Ssl108498 	struct kshmid	*sp = sptd->spt_amp->a_sp;
24960Sstevel@tonic-gate 	pgcnt_t		npages, a_npages;
24970Sstevel@tonic-gate 	page_t		**ppa;
24980Sstevel@tonic-gate 	pgcnt_t 	an_idx, a_an_idx, ppa_idx;
24990Sstevel@tonic-gate 	caddr_t		spt_addr, a_addr;	/* spt and aligned address */
25000Sstevel@tonic-gate 	size_t		a_len;			/* aligned len */
25010Sstevel@tonic-gate 	size_t		share_sz;
25020Sstevel@tonic-gate 	ulong_t		i;
25030Sstevel@tonic-gate 	int		sts = 0;
25042768Ssl108498 	rctl_qty_t	unlocked = 0;
25052768Ssl108498 	rctl_qty_t	locked = 0;
25062768Ssl108498 	struct proc	*p = curproc;
25072768Ssl108498 	kproject_t	*proj;
25080Sstevel@tonic-gate 
25090Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
25102768Ssl108498 	ASSERT(sp != NULL);
25110Sstevel@tonic-gate 
25120Sstevel@tonic-gate 	if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
25130Sstevel@tonic-gate 		return (0);
25140Sstevel@tonic-gate 	}
25150Sstevel@tonic-gate 
25160Sstevel@tonic-gate 	addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
25170Sstevel@tonic-gate 	an_idx = seg_page(seg, addr);
25180Sstevel@tonic-gate 	npages = btopr(len);
25190Sstevel@tonic-gate 
25200Sstevel@tonic-gate 	if (an_idx + npages > btopr(shmd->shm_amp->size)) {
25210Sstevel@tonic-gate 		return (ENOMEM);
25220Sstevel@tonic-gate 	}
25230Sstevel@tonic-gate 
25242768Ssl108498 	/*
25252768Ssl108498 	 * A shm's project never changes, so no lock needed.
25262768Ssl108498 	 * The shm has a hold on the project, so it will not go away.
25272768Ssl108498 	 * Since we have a mapping to shm within this zone, we know
25282768Ssl108498 	 * that the zone will not go away.
25292768Ssl108498 	 */
25302768Ssl108498 	proj = sp->shm_perm.ipc_proj;
25312768Ssl108498 
25320Sstevel@tonic-gate 	if (op == MC_LOCK) {
25332768Ssl108498 
25340Sstevel@tonic-gate 		/*
25350Sstevel@tonic-gate 		 * Need to align addr and size request if they are not
25360Sstevel@tonic-gate 		 * aligned so we can always allocate large page(s) however
25370Sstevel@tonic-gate 		 * we only lock what was requested in initial request.
25380Sstevel@tonic-gate 		 */
25390Sstevel@tonic-gate 		share_sz = page_get_pagesize(sptseg->s_szc);
25400Sstevel@tonic-gate 		a_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_sz);
25410Sstevel@tonic-gate 		a_len = P2ROUNDUP((uintptr_t)(((addr + len) - a_addr)),
25420Sstevel@tonic-gate 				share_sz);
25430Sstevel@tonic-gate 		a_npages = btop(a_len);
25440Sstevel@tonic-gate 		a_an_idx = seg_page(seg, a_addr);
25450Sstevel@tonic-gate 		spt_addr = sptseg->s_base + ptob(a_an_idx);
25460Sstevel@tonic-gate 		ppa_idx = an_idx - a_an_idx;
25470Sstevel@tonic-gate 
25480Sstevel@tonic-gate 		if ((ppa = kmem_zalloc(((sizeof (page_t *)) * a_npages),
25490Sstevel@tonic-gate 			KM_NOSLEEP)) == NULL) {
25500Sstevel@tonic-gate 			return (ENOMEM);
25510Sstevel@tonic-gate 		}
25520Sstevel@tonic-gate 
25530Sstevel@tonic-gate 		/*
25540Sstevel@tonic-gate 		 * Don't cache any new pages for IO and
25550Sstevel@tonic-gate 		 * flush any cached pages.
25560Sstevel@tonic-gate 		 */
25570Sstevel@tonic-gate 		mutex_enter(&sptd->spt_lock);
25580Sstevel@tonic-gate 		if (sptd->spt_ppa != NULL)
25590Sstevel@tonic-gate 			sptd->spt_flags |= DISM_PPA_CHANGED;
25600Sstevel@tonic-gate 
25610Sstevel@tonic-gate 		sts = spt_anon_getpages(sptseg, spt_addr, a_len, ppa);
25620Sstevel@tonic-gate 		if (sts != 0) {
25630Sstevel@tonic-gate 			mutex_exit(&sptd->spt_lock);
25640Sstevel@tonic-gate 			kmem_free(ppa, ((sizeof (page_t *)) * a_npages));
25650Sstevel@tonic-gate 			return (sts);
25660Sstevel@tonic-gate 		}
25670Sstevel@tonic-gate 
25682768Ssl108498 		mutex_enter(&sp->shm_mlock);
25692768Ssl108498 		/* enforce locked memory rctl */
25702768Ssl108498 		unlocked = spt_unlockedbytes(npages, &ppa[ppa_idx]);
25712768Ssl108498 
25722768Ssl108498 		mutex_enter(&p->p_lock);
25732768Ssl108498 		if (rctl_incr_locked_mem(p, proj, unlocked, 0)) {
25742768Ssl108498 			mutex_exit(&p->p_lock);
25752768Ssl108498 			sts = EAGAIN;
25762768Ssl108498 		} else {
25772768Ssl108498 			mutex_exit(&p->p_lock);
25782768Ssl108498 			sts = spt_lockpages(seg, an_idx, npages,
25792768Ssl108498 			    &ppa[ppa_idx], lockmap, pos, &locked);
25802768Ssl108498 
25812768Ssl108498 			/*
25822768Ssl108498 			 * correct locked count if not all pages could be
25832768Ssl108498 			 * locked
25842768Ssl108498 			 */
25852768Ssl108498 			if ((unlocked - locked) > 0) {
25862768Ssl108498 				rctl_decr_locked_mem(NULL, proj,
25872768Ssl108498 				    (unlocked - locked), 0);
25882768Ssl108498 			}
25892768Ssl108498 		}
25900Sstevel@tonic-gate 		/*
25912768Ssl108498 		 * unlock pages
25920Sstevel@tonic-gate 		 */
25932768Ssl108498 		for (i = 0; i < a_npages; i++)
25940Sstevel@tonic-gate 			page_unlock(ppa[i]);
25950Sstevel@tonic-gate 		if (sptd->spt_ppa != NULL)
25960Sstevel@tonic-gate 			sptd->spt_flags |= DISM_PPA_CHANGED;
25972768Ssl108498 		mutex_exit(&sp->shm_mlock);
25980Sstevel@tonic-gate 		mutex_exit(&sptd->spt_lock);
25990Sstevel@tonic-gate 
26000Sstevel@tonic-gate 		kmem_free(ppa, ((sizeof (page_t *)) * a_npages));
26010Sstevel@tonic-gate 
26020Sstevel@tonic-gate 	} else if (op == MC_UNLOCK) { /* unlock */
26030Sstevel@tonic-gate 		struct anon_map *amp;
26040Sstevel@tonic-gate 		struct anon 	*ap;
26050Sstevel@tonic-gate 		struct vnode 	*vp;
26060Sstevel@tonic-gate 		u_offset_t 	off;
26070Sstevel@tonic-gate 		struct page	*pp;
26080Sstevel@tonic-gate 		int		kernel;
26090Sstevel@tonic-gate 		anon_sync_obj_t cookie;
26102768Ssl108498 		rctl_qty_t	unlocked = 0;
26110Sstevel@tonic-gate 
26120Sstevel@tonic-gate 		amp = sptd->spt_amp;
26130Sstevel@tonic-gate 		mutex_enter(&sptd->spt_lock);
26140Sstevel@tonic-gate 		if (shmd->shm_lckpgs == 0) {
26150Sstevel@tonic-gate 			mutex_exit(&sptd->spt_lock);
26160Sstevel@tonic-gate 			return (0);
26170Sstevel@tonic-gate 		}
26180Sstevel@tonic-gate 		/*
26190Sstevel@tonic-gate 		 * Don't cache new IO pages.
26200Sstevel@tonic-gate 		 */
26210Sstevel@tonic-gate 		if (sptd->spt_ppa != NULL)
26220Sstevel@tonic-gate 			sptd->spt_flags |= DISM_PPA_CHANGED;
26230Sstevel@tonic-gate 
26242768Ssl108498 		mutex_enter(&sp->shm_mlock);
26250Sstevel@tonic-gate 		ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
26260Sstevel@tonic-gate 		for (i = 0; i < npages; i++, an_idx++) {
26270Sstevel@tonic-gate 			if (shmd->shm_vpage[an_idx] & DISM_PG_LOCKED) {
26280Sstevel@tonic-gate 				anon_array_enter(amp, an_idx, &cookie);
26290Sstevel@tonic-gate 				ap = anon_get_ptr(amp->ahp, an_idx);
26300Sstevel@tonic-gate 				ASSERT(ap);
26310Sstevel@tonic-gate 
26320Sstevel@tonic-gate 				swap_xlate(ap, &vp, &off);
26330Sstevel@tonic-gate 				anon_array_exit(&cookie);
26340Sstevel@tonic-gate 				pp = page_lookup(vp, off, SE_SHARED);
26350Sstevel@tonic-gate 				ASSERT(pp);
26360Sstevel@tonic-gate 				/*
26370Sstevel@tonic-gate 				 * the availrmem is decremented only for
26380Sstevel@tonic-gate 				 * pages which are not in seg pcache,
26390Sstevel@tonic-gate 				 * for pages in seg pcache availrmem was
26400Sstevel@tonic-gate 				 * decremented in _dismpagelock() (if
26410Sstevel@tonic-gate 				 * they were not locked here)
26420Sstevel@tonic-gate 				 */
26430Sstevel@tonic-gate 				kernel = (sptd->spt_ppa &&
26440Sstevel@tonic-gate 				    sptd->spt_ppa[an_idx]) ? 1 : 0;
26452768Ssl108498 				ASSERT(pp->p_lckcnt > 0);
26460Sstevel@tonic-gate 				page_pp_unlock(pp, 0, kernel);
26472768Ssl108498 				if (pp->p_lckcnt == 0)
26482768Ssl108498 					unlocked += PAGESIZE;
26490Sstevel@tonic-gate 				page_unlock(pp);
26500Sstevel@tonic-gate 				shmd->shm_vpage[an_idx] &= ~DISM_PG_LOCKED;
26510Sstevel@tonic-gate 				sptd->spt_ppa_lckcnt[an_idx]--;
26520Sstevel@tonic-gate 				shmd->shm_lckpgs--;
26530Sstevel@tonic-gate 			}
26540Sstevel@tonic-gate 		}
26550Sstevel@tonic-gate 		ANON_LOCK_EXIT(&amp->a_rwlock);
26560Sstevel@tonic-gate 		if (sptd->spt_ppa != NULL)
26570Sstevel@tonic-gate 			sptd->spt_flags |= DISM_PPA_CHANGED;
26580Sstevel@tonic-gate 		mutex_exit(&sptd->spt_lock);
26592768Ssl108498 
26602768Ssl108498 		rctl_decr_locked_mem(NULL, proj, unlocked, 0);
26612768Ssl108498 		mutex_exit(&sp->shm_mlock);
26620Sstevel@tonic-gate 	}
26630Sstevel@tonic-gate 	return (sts);
26640Sstevel@tonic-gate }
26650Sstevel@tonic-gate 
26660Sstevel@tonic-gate /*ARGSUSED*/
26670Sstevel@tonic-gate int
26680Sstevel@tonic-gate segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
26690Sstevel@tonic-gate {
26700Sstevel@tonic-gate 	struct shm_data *shmd = (struct shm_data *)seg->s_data;
26710Sstevel@tonic-gate 	struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
26720Sstevel@tonic-gate 	spgcnt_t pgno = seg_page(seg, addr+len) - seg_page(seg, addr) + 1;
26730Sstevel@tonic-gate 
26740Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
26750Sstevel@tonic-gate 
26760Sstevel@tonic-gate 	/*
26770Sstevel@tonic-gate 	 * ISM segment is always rw.
26780Sstevel@tonic-gate 	 */
26790Sstevel@tonic-gate 	while (--pgno >= 0)
26800Sstevel@tonic-gate 		*protv++ = sptd->spt_prot;
26810Sstevel@tonic-gate 	return (0);
26820Sstevel@tonic-gate }
26830Sstevel@tonic-gate 
26840Sstevel@tonic-gate /*ARGSUSED*/
26850Sstevel@tonic-gate u_offset_t
26860Sstevel@tonic-gate segspt_shmgetoffset(struct seg *seg, caddr_t addr)
26870Sstevel@tonic-gate {
26880Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
26890Sstevel@tonic-gate 
26900Sstevel@tonic-gate 	/* Offset does not matter in ISM memory */
26910Sstevel@tonic-gate 
26920Sstevel@tonic-gate 	return ((u_offset_t)0);
26930Sstevel@tonic-gate }
26940Sstevel@tonic-gate 
26950Sstevel@tonic-gate /* ARGSUSED */
26960Sstevel@tonic-gate int
26970Sstevel@tonic-gate segspt_shmgettype(struct seg *seg, caddr_t addr)
26980Sstevel@tonic-gate {
26990Sstevel@tonic-gate 	struct shm_data *shmd = (struct shm_data *)seg->s_data;
27000Sstevel@tonic-gate 	struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
27010Sstevel@tonic-gate 
27020Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
27030Sstevel@tonic-gate 
27040Sstevel@tonic-gate 	/*
27050Sstevel@tonic-gate 	 * The shared memory mapping is always MAP_SHARED, SWAP is only
27060Sstevel@tonic-gate 	 * reserved for DISM
27070Sstevel@tonic-gate 	 */
27080Sstevel@tonic-gate 	return (MAP_SHARED |
27090Sstevel@tonic-gate 		((sptd->spt_flags & SHM_PAGEABLE) ? 0 : MAP_NORESERVE));
27100Sstevel@tonic-gate }
27110Sstevel@tonic-gate 
27120Sstevel@tonic-gate /*ARGSUSED*/
27130Sstevel@tonic-gate int
27140Sstevel@tonic-gate segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
27150Sstevel@tonic-gate {
27160Sstevel@tonic-gate 	struct shm_data *shmd = (struct shm_data *)seg->s_data;
27170Sstevel@tonic-gate 	struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
27180Sstevel@tonic-gate 
27190Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
27200Sstevel@tonic-gate 
27210Sstevel@tonic-gate 	*vpp = sptd->spt_vp;
27220Sstevel@tonic-gate 	return (0);
27230Sstevel@tonic-gate }
27240Sstevel@tonic-gate 
27250Sstevel@tonic-gate /*ARGSUSED*/
27260Sstevel@tonic-gate static int
27270Sstevel@tonic-gate segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
27280Sstevel@tonic-gate {
27290Sstevel@tonic-gate 	struct shm_data 	*shmd = (struct shm_data *)seg->s_data;
27300Sstevel@tonic-gate 	struct spt_data	*sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
27310Sstevel@tonic-gate 	struct anon_map	*amp;
27320Sstevel@tonic-gate 	pgcnt_t		pg_idx;
27330Sstevel@tonic-gate 
27340Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
27350Sstevel@tonic-gate 
27360Sstevel@tonic-gate 	if (behav == MADV_FREE) {
27370Sstevel@tonic-gate 		if ((sptd->spt_flags & SHM_PAGEABLE) == 0)
27380Sstevel@tonic-gate 			return (0);
27390Sstevel@tonic-gate 
27400Sstevel@tonic-gate 		amp = sptd->spt_amp;
27410Sstevel@tonic-gate 		pg_idx = seg_page(seg, addr);
27420Sstevel@tonic-gate 
27430Sstevel@tonic-gate 		mutex_enter(&sptd->spt_lock);
27440Sstevel@tonic-gate 		if (sptd->spt_ppa != NULL)
27450Sstevel@tonic-gate 			sptd->spt_flags |= DISM_PPA_CHANGED;
27460Sstevel@tonic-gate 		mutex_exit(&sptd->spt_lock);
27470Sstevel@tonic-gate 
27480Sstevel@tonic-gate 		/*
27490Sstevel@tonic-gate 		 * Purge all DISM cached pages
27500Sstevel@tonic-gate 		 */
27510Sstevel@tonic-gate 		seg_ppurge_seg(segspt_reclaim);
27520Sstevel@tonic-gate 
27530Sstevel@tonic-gate 		mutex_enter(&sptd->spt_lock);
27540Sstevel@tonic-gate 		ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
27550Sstevel@tonic-gate 		anon_disclaim(amp, pg_idx, len, ANON_PGLOOKUP_BLK);
27560Sstevel@tonic-gate 		ANON_LOCK_EXIT(&amp->a_rwlock);
27570Sstevel@tonic-gate 		mutex_exit(&sptd->spt_lock);
27580Sstevel@tonic-gate 	} else if (lgrp_optimizations() && (behav == MADV_ACCESS_LWP ||
27590Sstevel@tonic-gate 	    behav == MADV_ACCESS_MANY || behav == MADV_ACCESS_DEFAULT)) {
27600Sstevel@tonic-gate 		int			already_set;
27610Sstevel@tonic-gate 		ulong_t			anon_index;
27620Sstevel@tonic-gate 		lgrp_mem_policy_t	policy;
27630Sstevel@tonic-gate 		caddr_t			shm_addr;
27640Sstevel@tonic-gate 		size_t			share_size;
27650Sstevel@tonic-gate 		size_t			size;
27660Sstevel@tonic-gate 		struct seg		*sptseg = shmd->shm_sptseg;
27670Sstevel@tonic-gate 		caddr_t			sptseg_addr;
27680Sstevel@tonic-gate 
27690Sstevel@tonic-gate 		/*
27700Sstevel@tonic-gate 		 * Align address and length to page size of underlying segment
27710Sstevel@tonic-gate 		 */
27720Sstevel@tonic-gate 		share_size = page_get_pagesize(shmd->shm_sptseg->s_szc);
27730Sstevel@tonic-gate 		shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_size);
27740Sstevel@tonic-gate 		size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)),
27750Sstevel@tonic-gate 		    share_size);
27760Sstevel@tonic-gate 
27770Sstevel@tonic-gate 		amp = shmd->shm_amp;
27780Sstevel@tonic-gate 		anon_index = seg_page(seg, shm_addr);
27790Sstevel@tonic-gate 
27800Sstevel@tonic-gate 		/*
27810Sstevel@tonic-gate 		 * And now we may have to adjust size downward if we have
27820Sstevel@tonic-gate 		 * exceeded the realsize of the segment or initial anon
27830Sstevel@tonic-gate 		 * allocations.
27840Sstevel@tonic-gate 		 */
27850Sstevel@tonic-gate 		sptseg_addr = sptseg->s_base + ptob(anon_index);
27860Sstevel@tonic-gate 		if ((sptseg_addr + size) >
27870Sstevel@tonic-gate 		    (sptseg->s_base + sptd->spt_realsize))
27880Sstevel@tonic-gate 			size = (sptseg->s_base + sptd->spt_realsize) -
27890Sstevel@tonic-gate 			    sptseg_addr;
27900Sstevel@tonic-gate 
27910Sstevel@tonic-gate 		/*
27920Sstevel@tonic-gate 		 * Set memory allocation policy for this segment
27930Sstevel@tonic-gate 		 */
27940Sstevel@tonic-gate 		policy = lgrp_madv_to_policy(behav, len, MAP_SHARED);
27950Sstevel@tonic-gate 		already_set = lgrp_shm_policy_set(policy, amp, anon_index,
27960Sstevel@tonic-gate 		    NULL, 0, len);
27970Sstevel@tonic-gate 
27980Sstevel@tonic-gate 		/*
27990Sstevel@tonic-gate 		 * If random memory allocation policy set already,
28000Sstevel@tonic-gate 		 * don't bother reapplying it.
28010Sstevel@tonic-gate 		 */
28020Sstevel@tonic-gate 		if (already_set && !LGRP_MEM_POLICY_REAPPLICABLE(policy))
28030Sstevel@tonic-gate 			return (0);
28040Sstevel@tonic-gate 
28050Sstevel@tonic-gate 		/*
28060Sstevel@tonic-gate 		 * Mark any existing pages in the given range for
28070Sstevel@tonic-gate 		 * migration, flushing the I/O page cache, and using
28080Sstevel@tonic-gate 		 * underlying segment to calculate anon index and get
28090Sstevel@tonic-gate 		 * anonmap and vnode pointer from
28100Sstevel@tonic-gate 		 */
28110Sstevel@tonic-gate 		if (shmd->shm_softlockcnt > 0)
28120Sstevel@tonic-gate 			segspt_purge(seg);
28130Sstevel@tonic-gate 
28140Sstevel@tonic-gate 		page_mark_migrate(seg, shm_addr, size, amp, 0, NULL, 0, 0);
28150Sstevel@tonic-gate 	}
28160Sstevel@tonic-gate 
28170Sstevel@tonic-gate 	return (0);
28180Sstevel@tonic-gate }
28190Sstevel@tonic-gate 
28200Sstevel@tonic-gate /*ARGSUSED*/
28210Sstevel@tonic-gate void
28220Sstevel@tonic-gate segspt_shmdump(struct seg *seg)
28230Sstevel@tonic-gate {
28240Sstevel@tonic-gate 	/* no-op for ISM segment */
28250Sstevel@tonic-gate }
28260Sstevel@tonic-gate 
28270Sstevel@tonic-gate /*ARGSUSED*/
28280Sstevel@tonic-gate static faultcode_t
28290Sstevel@tonic-gate segspt_shmsetpgsz(struct seg *seg, caddr_t addr, size_t len, uint_t szc)
28300Sstevel@tonic-gate {
28310Sstevel@tonic-gate 	return (ENOTSUP);
28320Sstevel@tonic-gate }
28330Sstevel@tonic-gate 
28340Sstevel@tonic-gate /*
28350Sstevel@tonic-gate  * get a memory ID for an addr in a given segment
28360Sstevel@tonic-gate  */
28370Sstevel@tonic-gate static int
28380Sstevel@tonic-gate segspt_shmgetmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
28390Sstevel@tonic-gate {
28400Sstevel@tonic-gate 	struct shm_data *shmd = (struct shm_data *)seg->s_data;
28410Sstevel@tonic-gate 	struct anon 	*ap;
28420Sstevel@tonic-gate 	size_t		anon_index;
28430Sstevel@tonic-gate 	struct anon_map	*amp = shmd->shm_amp;
28440Sstevel@tonic-gate 	struct spt_data	*sptd = shmd->shm_sptseg->s_data;
28450Sstevel@tonic-gate 	struct seg	*sptseg = shmd->shm_sptseg;
28460Sstevel@tonic-gate 	anon_sync_obj_t	cookie;
28470Sstevel@tonic-gate 
28480Sstevel@tonic-gate 	anon_index = seg_page(seg, addr);
28490Sstevel@tonic-gate 
28500Sstevel@tonic-gate 	if (addr > (seg->s_base + sptd->spt_realsize)) {
28510Sstevel@tonic-gate 		return (EFAULT);
28520Sstevel@tonic-gate 	}
28530Sstevel@tonic-gate 
28540Sstevel@tonic-gate 	ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
28550Sstevel@tonic-gate 	anon_array_enter(amp, anon_index, &cookie);
28560Sstevel@tonic-gate 	ap = anon_get_ptr(amp->ahp, anon_index);
28570Sstevel@tonic-gate 	if (ap == NULL) {
28580Sstevel@tonic-gate 		struct page *pp;
28590Sstevel@tonic-gate 		caddr_t spt_addr = sptseg->s_base + ptob(anon_index);
28600Sstevel@tonic-gate 
28610Sstevel@tonic-gate 		pp = anon_zero(sptseg, spt_addr, &ap, kcred);
28620Sstevel@tonic-gate 		if (pp == NULL) {
28630Sstevel@tonic-gate 			anon_array_exit(&cookie);
28640Sstevel@tonic-gate 			ANON_LOCK_EXIT(&amp->a_rwlock);
28650Sstevel@tonic-gate 			return (ENOMEM);
28660Sstevel@tonic-gate 		}
28670Sstevel@tonic-gate 		(void) anon_set_ptr(amp->ahp, anon_index, ap, ANON_SLEEP);
28680Sstevel@tonic-gate 		page_unlock(pp);
28690Sstevel@tonic-gate 	}
28700Sstevel@tonic-gate 	anon_array_exit(&cookie);
28710Sstevel@tonic-gate 	ANON_LOCK_EXIT(&amp->a_rwlock);
28720Sstevel@tonic-gate 	memidp->val[0] = (uintptr_t)ap;
28730Sstevel@tonic-gate 	memidp->val[1] = (uintptr_t)addr & PAGEOFFSET;
28740Sstevel@tonic-gate 	return (0);
28750Sstevel@tonic-gate }
28760Sstevel@tonic-gate 
28770Sstevel@tonic-gate /*
28780Sstevel@tonic-gate  * Get memory allocation policy info for specified address in given segment
28790Sstevel@tonic-gate  */
28800Sstevel@tonic-gate static lgrp_mem_policy_info_t *
28810Sstevel@tonic-gate segspt_shmgetpolicy(struct seg *seg, caddr_t addr)
28820Sstevel@tonic-gate {
28830Sstevel@tonic-gate 	struct anon_map		*amp;
28840Sstevel@tonic-gate 	ulong_t			anon_index;
28850Sstevel@tonic-gate 	lgrp_mem_policy_info_t	*policy_info;
28860Sstevel@tonic-gate 	struct shm_data		*shm_data;
28870Sstevel@tonic-gate 
28880Sstevel@tonic-gate 	ASSERT(seg != NULL);
28890Sstevel@tonic-gate 
28900Sstevel@tonic-gate 	/*
28910Sstevel@tonic-gate 	 * Get anon_map from segshm
28920Sstevel@tonic-gate 	 *
28930Sstevel@tonic-gate 	 * Assume that no lock needs to be held on anon_map, since
28940Sstevel@tonic-gate 	 * it should be protected by its reference count which must be
28950Sstevel@tonic-gate 	 * nonzero for an existing segment
28960Sstevel@tonic-gate 	 * Need to grab readers lock on policy tree though
28970Sstevel@tonic-gate 	 */
28980Sstevel@tonic-gate 	shm_data = (struct shm_data *)seg->s_data;
28990Sstevel@tonic-gate 	if (shm_data == NULL)
29000Sstevel@tonic-gate 		return (NULL);
29010Sstevel@tonic-gate 	amp = shm_data->shm_amp;
29020Sstevel@tonic-gate 	ASSERT(amp->refcnt != 0);
29030Sstevel@tonic-gate 
29040Sstevel@tonic-gate 	/*
29050Sstevel@tonic-gate 	 * Get policy info
29060Sstevel@tonic-gate 	 *
29070Sstevel@tonic-gate 	 * Assume starting anon index of 0
29080Sstevel@tonic-gate 	 */
29090Sstevel@tonic-gate 	anon_index = seg_page(seg, addr);
29100Sstevel@tonic-gate 	policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0);
29110Sstevel@tonic-gate 
29120Sstevel@tonic-gate 	return (policy_info);
29130Sstevel@tonic-gate }
2914670Selowe 
2915670Selowe /*ARGSUSED*/
2916670Selowe static int
2917670Selowe segspt_shmcapable(struct seg *seg, segcapability_t capability)
2918670Selowe {
2919670Selowe 	return (0);
2920670Selowe }
2921