10Sstevel@tonic-gate /* 20Sstevel@tonic-gate * CDDL HEADER START 30Sstevel@tonic-gate * 40Sstevel@tonic-gate * The contents of this file are subject to the terms of the 51899Svsakar * Common Development and Distribution License (the "License"). 61899Svsakar * You may not use this file except in compliance with the License. 70Sstevel@tonic-gate * 80Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 90Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 100Sstevel@tonic-gate * See the License for the specific language governing permissions 110Sstevel@tonic-gate * and limitations under the License. 120Sstevel@tonic-gate * 130Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 140Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 150Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 160Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 170Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 180Sstevel@tonic-gate * 190Sstevel@tonic-gate * CDDL HEADER END 200Sstevel@tonic-gate */ 210Sstevel@tonic-gate /* 226695Saguzovsk * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 230Sstevel@tonic-gate * Use is subject to license terms. 240Sstevel@tonic-gate */ 250Sstevel@tonic-gate 260Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 270Sstevel@tonic-gate /* All Rights Reserved */ 280Sstevel@tonic-gate 290Sstevel@tonic-gate /* 300Sstevel@tonic-gate * University Copyright- Copyright (c) 1982, 1986, 1988 310Sstevel@tonic-gate * The Regents of the University of California 320Sstevel@tonic-gate * All Rights Reserved 330Sstevel@tonic-gate * 340Sstevel@tonic-gate * University Acknowledgment- Portions of this document are derived from 350Sstevel@tonic-gate * software developed by the University of California, Berkeley, and its 360Sstevel@tonic-gate * contributors. 370Sstevel@tonic-gate */ 380Sstevel@tonic-gate 390Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 400Sstevel@tonic-gate 410Sstevel@tonic-gate /* 420Sstevel@tonic-gate * VM - address spaces. 430Sstevel@tonic-gate */ 440Sstevel@tonic-gate 450Sstevel@tonic-gate #include <sys/types.h> 460Sstevel@tonic-gate #include <sys/t_lock.h> 470Sstevel@tonic-gate #include <sys/param.h> 480Sstevel@tonic-gate #include <sys/errno.h> 490Sstevel@tonic-gate #include <sys/systm.h> 500Sstevel@tonic-gate #include <sys/mman.h> 510Sstevel@tonic-gate #include <sys/sysmacros.h> 520Sstevel@tonic-gate #include <sys/cpuvar.h> 530Sstevel@tonic-gate #include <sys/sysinfo.h> 540Sstevel@tonic-gate #include <sys/kmem.h> 550Sstevel@tonic-gate #include <sys/vnode.h> 560Sstevel@tonic-gate #include <sys/vmsystm.h> 570Sstevel@tonic-gate #include <sys/cmn_err.h> 580Sstevel@tonic-gate #include <sys/debug.h> 590Sstevel@tonic-gate #include <sys/tnf_probe.h> 600Sstevel@tonic-gate #include <sys/vtrace.h> 610Sstevel@tonic-gate 620Sstevel@tonic-gate #include <vm/hat.h> 630Sstevel@tonic-gate #include <vm/xhat.h> 640Sstevel@tonic-gate #include <vm/as.h> 650Sstevel@tonic-gate #include <vm/seg.h> 660Sstevel@tonic-gate #include <vm/seg_vn.h> 670Sstevel@tonic-gate #include <vm/seg_dev.h> 680Sstevel@tonic-gate #include <vm/seg_kmem.h> 690Sstevel@tonic-gate #include <vm/seg_map.h> 700Sstevel@tonic-gate #include <vm/seg_spt.h> 710Sstevel@tonic-gate #include <vm/page.h> 720Sstevel@tonic-gate 730Sstevel@tonic-gate clock_t deadlk_wait = 1; /* number of ticks to wait before retrying */ 740Sstevel@tonic-gate 750Sstevel@tonic-gate static struct kmem_cache *as_cache; 760Sstevel@tonic-gate 770Sstevel@tonic-gate static void as_setwatchprot(struct as *, caddr_t, size_t, uint_t); 780Sstevel@tonic-gate static void as_clearwatchprot(struct as *, caddr_t, size_t); 791899Svsakar int as_map_locked(struct as *, caddr_t, size_t, int ((*)()), void *); 800Sstevel@tonic-gate 810Sstevel@tonic-gate 820Sstevel@tonic-gate /* 830Sstevel@tonic-gate * Verifying the segment lists is very time-consuming; it may not be 840Sstevel@tonic-gate * desirable always to define VERIFY_SEGLIST when DEBUG is set. 850Sstevel@tonic-gate */ 860Sstevel@tonic-gate #ifdef DEBUG 870Sstevel@tonic-gate #define VERIFY_SEGLIST 880Sstevel@tonic-gate int do_as_verify = 0; 890Sstevel@tonic-gate #endif 900Sstevel@tonic-gate 910Sstevel@tonic-gate /* 920Sstevel@tonic-gate * Allocate a new callback data structure entry and fill in the events of 930Sstevel@tonic-gate * interest, the address range of interest, and the callback argument. 940Sstevel@tonic-gate * Link the entry on the as->a_callbacks list. A callback entry for the 950Sstevel@tonic-gate * entire address space may be specified with vaddr = 0 and size = -1. 960Sstevel@tonic-gate * 970Sstevel@tonic-gate * CALLERS RESPONSIBILITY: If not calling from within the process context for 980Sstevel@tonic-gate * the specified as, the caller must guarantee persistence of the specified as 990Sstevel@tonic-gate * for the duration of this function (eg. pages being locked within the as 1000Sstevel@tonic-gate * will guarantee persistence). 1010Sstevel@tonic-gate */ 1020Sstevel@tonic-gate int 1030Sstevel@tonic-gate as_add_callback(struct as *as, void (*cb_func)(), void *arg, uint_t events, 1040Sstevel@tonic-gate caddr_t vaddr, size_t size, int sleepflag) 1050Sstevel@tonic-gate { 1060Sstevel@tonic-gate struct as_callback *current_head, *cb; 1070Sstevel@tonic-gate caddr_t saddr; 1080Sstevel@tonic-gate size_t rsize; 1090Sstevel@tonic-gate 1100Sstevel@tonic-gate /* callback function and an event are mandatory */ 1110Sstevel@tonic-gate if ((cb_func == NULL) || ((events & AS_ALL_EVENT) == 0)) 1120Sstevel@tonic-gate return (EINVAL); 1130Sstevel@tonic-gate 1140Sstevel@tonic-gate /* Adding a callback after as_free has been called is not allowed */ 1150Sstevel@tonic-gate if (as == &kas) 1160Sstevel@tonic-gate return (ENOMEM); 1170Sstevel@tonic-gate 1180Sstevel@tonic-gate /* 1190Sstevel@tonic-gate * vaddr = 0 and size = -1 is used to indicate that the callback range 1200Sstevel@tonic-gate * is the entire address space so no rounding is done in that case. 1210Sstevel@tonic-gate */ 1220Sstevel@tonic-gate if (size != -1) { 1230Sstevel@tonic-gate saddr = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK); 1240Sstevel@tonic-gate rsize = (((size_t)(vaddr + size) + PAGEOFFSET) & PAGEMASK) - 1255084Sjohnlev (size_t)saddr; 1260Sstevel@tonic-gate /* check for wraparound */ 1270Sstevel@tonic-gate if (saddr + rsize < saddr) 1280Sstevel@tonic-gate return (ENOMEM); 1290Sstevel@tonic-gate } else { 1300Sstevel@tonic-gate if (vaddr != 0) 1310Sstevel@tonic-gate return (EINVAL); 1320Sstevel@tonic-gate saddr = vaddr; 1330Sstevel@tonic-gate rsize = size; 1340Sstevel@tonic-gate } 1350Sstevel@tonic-gate 1360Sstevel@tonic-gate /* Allocate and initialize a callback entry */ 1370Sstevel@tonic-gate cb = kmem_zalloc(sizeof (struct as_callback), sleepflag); 1380Sstevel@tonic-gate if (cb == NULL) 1390Sstevel@tonic-gate return (EAGAIN); 1400Sstevel@tonic-gate 1410Sstevel@tonic-gate cb->ascb_func = cb_func; 1420Sstevel@tonic-gate cb->ascb_arg = arg; 1430Sstevel@tonic-gate cb->ascb_events = events; 1440Sstevel@tonic-gate cb->ascb_saddr = saddr; 1450Sstevel@tonic-gate cb->ascb_len = rsize; 1460Sstevel@tonic-gate 1470Sstevel@tonic-gate /* Add the entry to the list */ 1480Sstevel@tonic-gate mutex_enter(&as->a_contents); 1490Sstevel@tonic-gate current_head = as->a_callbacks; 1500Sstevel@tonic-gate as->a_callbacks = cb; 1510Sstevel@tonic-gate cb->ascb_next = current_head; 1520Sstevel@tonic-gate 1530Sstevel@tonic-gate /* 1540Sstevel@tonic-gate * The call to this function may lose in a race with 1550Sstevel@tonic-gate * a pertinent event - eg. a thread does long term memory locking 1560Sstevel@tonic-gate * but before the callback is added another thread executes as_unmap. 1570Sstevel@tonic-gate * A broadcast here resolves that. 1580Sstevel@tonic-gate */ 1590Sstevel@tonic-gate if ((cb->ascb_events & AS_UNMAPWAIT_EVENT) && AS_ISUNMAPWAIT(as)) { 1600Sstevel@tonic-gate AS_CLRUNMAPWAIT(as); 1610Sstevel@tonic-gate cv_broadcast(&as->a_cv); 1620Sstevel@tonic-gate } 1630Sstevel@tonic-gate 1640Sstevel@tonic-gate mutex_exit(&as->a_contents); 1650Sstevel@tonic-gate return (0); 1660Sstevel@tonic-gate } 1670Sstevel@tonic-gate 1680Sstevel@tonic-gate /* 1690Sstevel@tonic-gate * Search the callback list for an entry which pertains to arg. 1700Sstevel@tonic-gate * 1710Sstevel@tonic-gate * This is called from within the client upon completion of the callback. 1720Sstevel@tonic-gate * RETURN VALUES: 1730Sstevel@tonic-gate * AS_CALLBACK_DELETED (callback entry found and deleted) 1740Sstevel@tonic-gate * AS_CALLBACK_NOTFOUND (no callback entry found - this is ok) 1750Sstevel@tonic-gate * AS_CALLBACK_DELETE_DEFERRED (callback is in process, delete of this 1760Sstevel@tonic-gate * entry will be made in as_do_callbacks) 1770Sstevel@tonic-gate * 1780Sstevel@tonic-gate * If as_delete_callback encounters a matching entry with AS_CALLBACK_CALLED 1790Sstevel@tonic-gate * set, it indicates that as_do_callbacks is processing this entry. The 1800Sstevel@tonic-gate * AS_ALL_EVENT events are cleared in the entry, and a broadcast is made 1810Sstevel@tonic-gate * to unblock as_do_callbacks, in case it is blocked. 1820Sstevel@tonic-gate * 1830Sstevel@tonic-gate * CALLERS RESPONSIBILITY: If not calling from within the process context for 1840Sstevel@tonic-gate * the specified as, the caller must guarantee persistence of the specified as 1850Sstevel@tonic-gate * for the duration of this function (eg. pages being locked within the as 1860Sstevel@tonic-gate * will guarantee persistence). 1870Sstevel@tonic-gate */ 1880Sstevel@tonic-gate uint_t 1890Sstevel@tonic-gate as_delete_callback(struct as *as, void *arg) 1900Sstevel@tonic-gate { 1910Sstevel@tonic-gate struct as_callback **prevcb = &as->a_callbacks; 1920Sstevel@tonic-gate struct as_callback *cb; 1930Sstevel@tonic-gate uint_t rc = AS_CALLBACK_NOTFOUND; 1940Sstevel@tonic-gate 1950Sstevel@tonic-gate mutex_enter(&as->a_contents); 1960Sstevel@tonic-gate for (cb = as->a_callbacks; cb; prevcb = &cb->ascb_next, cb = *prevcb) { 1970Sstevel@tonic-gate if (cb->ascb_arg != arg) 1980Sstevel@tonic-gate continue; 1990Sstevel@tonic-gate 2000Sstevel@tonic-gate /* 2010Sstevel@tonic-gate * If the events indicate AS_CALLBACK_CALLED, just clear 2020Sstevel@tonic-gate * AS_ALL_EVENT in the events field and wakeup the thread 2030Sstevel@tonic-gate * that may be waiting in as_do_callbacks. as_do_callbacks 2040Sstevel@tonic-gate * will take care of removing this entry from the list. In 2050Sstevel@tonic-gate * that case, return AS_CALLBACK_DELETE_DEFERRED. Otherwise 2060Sstevel@tonic-gate * (AS_CALLBACK_CALLED not set), just remove it from the 2070Sstevel@tonic-gate * list, return the memory and return AS_CALLBACK_DELETED. 2080Sstevel@tonic-gate */ 2090Sstevel@tonic-gate if ((cb->ascb_events & AS_CALLBACK_CALLED) != 0) { 2100Sstevel@tonic-gate /* leave AS_CALLBACK_CALLED */ 2110Sstevel@tonic-gate cb->ascb_events &= ~AS_ALL_EVENT; 2120Sstevel@tonic-gate rc = AS_CALLBACK_DELETE_DEFERRED; 2130Sstevel@tonic-gate cv_broadcast(&as->a_cv); 2140Sstevel@tonic-gate } else { 2150Sstevel@tonic-gate *prevcb = cb->ascb_next; 2160Sstevel@tonic-gate kmem_free(cb, sizeof (struct as_callback)); 2170Sstevel@tonic-gate rc = AS_CALLBACK_DELETED; 2180Sstevel@tonic-gate } 2190Sstevel@tonic-gate break; 2200Sstevel@tonic-gate } 2210Sstevel@tonic-gate mutex_exit(&as->a_contents); 2220Sstevel@tonic-gate return (rc); 2230Sstevel@tonic-gate } 2240Sstevel@tonic-gate 2250Sstevel@tonic-gate /* 2260Sstevel@tonic-gate * Searches the as callback list for a matching entry. 2270Sstevel@tonic-gate * Returns a pointer to the first matching callback, or NULL if 2280Sstevel@tonic-gate * nothing is found. 2290Sstevel@tonic-gate * This function never sleeps so it is ok to call it with more 2300Sstevel@tonic-gate * locks held but the (required) a_contents mutex. 2310Sstevel@tonic-gate * 2320Sstevel@tonic-gate * See also comment on as_do_callbacks below. 2330Sstevel@tonic-gate */ 2340Sstevel@tonic-gate static struct as_callback * 2350Sstevel@tonic-gate as_find_callback(struct as *as, uint_t events, caddr_t event_addr, 2360Sstevel@tonic-gate size_t event_len) 2370Sstevel@tonic-gate { 2380Sstevel@tonic-gate struct as_callback *cb; 2390Sstevel@tonic-gate 2400Sstevel@tonic-gate ASSERT(MUTEX_HELD(&as->a_contents)); 2410Sstevel@tonic-gate for (cb = as->a_callbacks; cb != NULL; cb = cb->ascb_next) { 2420Sstevel@tonic-gate /* 2430Sstevel@tonic-gate * If the callback has not already been called, then 2440Sstevel@tonic-gate * check if events or address range pertains. An event_len 2450Sstevel@tonic-gate * of zero means do an unconditional callback. 2460Sstevel@tonic-gate */ 2470Sstevel@tonic-gate if (((cb->ascb_events & AS_CALLBACK_CALLED) != 0) || 2480Sstevel@tonic-gate ((event_len != 0) && (((cb->ascb_events & events) == 0) || 2490Sstevel@tonic-gate (event_addr + event_len < cb->ascb_saddr) || 2500Sstevel@tonic-gate (event_addr > (cb->ascb_saddr + cb->ascb_len))))) { 2510Sstevel@tonic-gate continue; 2520Sstevel@tonic-gate } 2530Sstevel@tonic-gate break; 2540Sstevel@tonic-gate } 2550Sstevel@tonic-gate return (cb); 2560Sstevel@tonic-gate } 2570Sstevel@tonic-gate 2580Sstevel@tonic-gate /* 2590Sstevel@tonic-gate * Executes a given callback and removes it from the callback list for 2600Sstevel@tonic-gate * this address space. 2610Sstevel@tonic-gate * This function may sleep so the caller must drop all locks except 2620Sstevel@tonic-gate * a_contents before calling this func. 2630Sstevel@tonic-gate * 2640Sstevel@tonic-gate * See also comments on as_do_callbacks below. 2650Sstevel@tonic-gate */ 2660Sstevel@tonic-gate static void 2670Sstevel@tonic-gate as_execute_callback(struct as *as, struct as_callback *cb, 2680Sstevel@tonic-gate uint_t events) 2690Sstevel@tonic-gate { 2700Sstevel@tonic-gate struct as_callback **prevcb; 2710Sstevel@tonic-gate void *cb_arg; 2720Sstevel@tonic-gate 2730Sstevel@tonic-gate ASSERT(MUTEX_HELD(&as->a_contents) && (cb->ascb_events & events)); 2740Sstevel@tonic-gate cb->ascb_events |= AS_CALLBACK_CALLED; 2750Sstevel@tonic-gate mutex_exit(&as->a_contents); 2760Sstevel@tonic-gate (*cb->ascb_func)(as, cb->ascb_arg, events); 2770Sstevel@tonic-gate mutex_enter(&as->a_contents); 2780Sstevel@tonic-gate /* 2790Sstevel@tonic-gate * the callback function is required to delete the callback 2800Sstevel@tonic-gate * when the callback function determines it is OK for 2810Sstevel@tonic-gate * this thread to continue. as_delete_callback will clear 2820Sstevel@tonic-gate * the AS_ALL_EVENT in the events field when it is deleted. 2830Sstevel@tonic-gate * If the callback function called as_delete_callback, 2840Sstevel@tonic-gate * events will already be cleared and there will be no blocking. 2850Sstevel@tonic-gate */ 2860Sstevel@tonic-gate while ((cb->ascb_events & events) != 0) { 2870Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 2880Sstevel@tonic-gate } 2890Sstevel@tonic-gate /* 2900Sstevel@tonic-gate * This entry needs to be taken off the list. Normally, the 2910Sstevel@tonic-gate * callback func itself does that, but unfortunately the list 2920Sstevel@tonic-gate * may have changed while the callback was running because the 2930Sstevel@tonic-gate * a_contents mutex was dropped and someone else other than the 2940Sstevel@tonic-gate * callback func itself could have called as_delete_callback, 2950Sstevel@tonic-gate * so we have to search to find this entry again. The entry 2960Sstevel@tonic-gate * must have AS_CALLBACK_CALLED, and have the same 'arg'. 2970Sstevel@tonic-gate */ 2980Sstevel@tonic-gate cb_arg = cb->ascb_arg; 2990Sstevel@tonic-gate prevcb = &as->a_callbacks; 3000Sstevel@tonic-gate for (cb = as->a_callbacks; cb != NULL; 3010Sstevel@tonic-gate prevcb = &cb->ascb_next, cb = *prevcb) { 3020Sstevel@tonic-gate if (((cb->ascb_events & AS_CALLBACK_CALLED) == 0) || 3030Sstevel@tonic-gate (cb_arg != cb->ascb_arg)) { 3040Sstevel@tonic-gate continue; 3050Sstevel@tonic-gate } 3060Sstevel@tonic-gate *prevcb = cb->ascb_next; 3070Sstevel@tonic-gate kmem_free(cb, sizeof (struct as_callback)); 3080Sstevel@tonic-gate break; 3090Sstevel@tonic-gate } 3100Sstevel@tonic-gate } 3110Sstevel@tonic-gate 3120Sstevel@tonic-gate /* 3130Sstevel@tonic-gate * Check the callback list for a matching event and intersection of 3140Sstevel@tonic-gate * address range. If there is a match invoke the callback. Skip an entry if: 3150Sstevel@tonic-gate * - a callback is already in progress for this entry (AS_CALLBACK_CALLED) 3160Sstevel@tonic-gate * - not event of interest 3170Sstevel@tonic-gate * - not address range of interest 3180Sstevel@tonic-gate * 3190Sstevel@tonic-gate * An event_len of zero indicates a request for an unconditional callback 3200Sstevel@tonic-gate * (regardless of event), only the AS_CALLBACK_CALLED is checked. The 3210Sstevel@tonic-gate * a_contents lock must be dropped before a callback, so only one callback 3220Sstevel@tonic-gate * can be done before returning. Return -1 (true) if a callback was 3230Sstevel@tonic-gate * executed and removed from the list, else return 0 (false). 3240Sstevel@tonic-gate * 3250Sstevel@tonic-gate * The logically separate parts, i.e. finding a matching callback and 3260Sstevel@tonic-gate * executing a given callback have been separated into two functions 3270Sstevel@tonic-gate * so that they can be called with different sets of locks held beyond 3280Sstevel@tonic-gate * the always-required a_contents. as_find_callback does not sleep so 3290Sstevel@tonic-gate * it is ok to call it if more locks than a_contents (i.e. the a_lock 3300Sstevel@tonic-gate * rwlock) are held. as_execute_callback on the other hand may sleep 3310Sstevel@tonic-gate * so all locks beyond a_contents must be dropped by the caller if one 3320Sstevel@tonic-gate * does not want to end comatose. 3330Sstevel@tonic-gate */ 3340Sstevel@tonic-gate static int 3350Sstevel@tonic-gate as_do_callbacks(struct as *as, uint_t events, caddr_t event_addr, 3360Sstevel@tonic-gate size_t event_len) 3370Sstevel@tonic-gate { 3380Sstevel@tonic-gate struct as_callback *cb; 3390Sstevel@tonic-gate 3400Sstevel@tonic-gate if ((cb = as_find_callback(as, events, event_addr, event_len))) { 3410Sstevel@tonic-gate as_execute_callback(as, cb, events); 3420Sstevel@tonic-gate return (-1); 3430Sstevel@tonic-gate } 3440Sstevel@tonic-gate return (0); 3450Sstevel@tonic-gate } 3460Sstevel@tonic-gate 3470Sstevel@tonic-gate /* 3480Sstevel@tonic-gate * Search for the segment containing addr. If a segment containing addr 3490Sstevel@tonic-gate * exists, that segment is returned. If no such segment exists, and 3500Sstevel@tonic-gate * the list spans addresses greater than addr, then the first segment 3510Sstevel@tonic-gate * whose base is greater than addr is returned; otherwise, NULL is 3520Sstevel@tonic-gate * returned unless tail is true, in which case the last element of the 3530Sstevel@tonic-gate * list is returned. 3540Sstevel@tonic-gate * 3550Sstevel@tonic-gate * a_seglast is used to cache the last found segment for repeated 3560Sstevel@tonic-gate * searches to the same addr (which happens frequently). 3570Sstevel@tonic-gate */ 3580Sstevel@tonic-gate struct seg * 3590Sstevel@tonic-gate as_findseg(struct as *as, caddr_t addr, int tail) 3600Sstevel@tonic-gate { 3610Sstevel@tonic-gate struct seg *seg = as->a_seglast; 3620Sstevel@tonic-gate avl_index_t where; 3630Sstevel@tonic-gate 3640Sstevel@tonic-gate ASSERT(AS_LOCK_HELD(as, &as->a_lock)); 3650Sstevel@tonic-gate 3660Sstevel@tonic-gate if (seg != NULL && 3670Sstevel@tonic-gate seg->s_base <= addr && 3680Sstevel@tonic-gate addr < seg->s_base + seg->s_size) 3690Sstevel@tonic-gate return (seg); 3700Sstevel@tonic-gate 3710Sstevel@tonic-gate seg = avl_find(&as->a_segtree, &addr, &where); 3720Sstevel@tonic-gate if (seg != NULL) 3730Sstevel@tonic-gate return (as->a_seglast = seg); 3740Sstevel@tonic-gate 3750Sstevel@tonic-gate seg = avl_nearest(&as->a_segtree, where, AVL_AFTER); 3760Sstevel@tonic-gate if (seg == NULL && tail) 3770Sstevel@tonic-gate seg = avl_last(&as->a_segtree); 3780Sstevel@tonic-gate return (as->a_seglast = seg); 3790Sstevel@tonic-gate } 3800Sstevel@tonic-gate 3810Sstevel@tonic-gate #ifdef VERIFY_SEGLIST 3820Sstevel@tonic-gate /* 3830Sstevel@tonic-gate * verify that the linked list is coherent 3840Sstevel@tonic-gate */ 3850Sstevel@tonic-gate static void 3860Sstevel@tonic-gate as_verify(struct as *as) 3870Sstevel@tonic-gate { 3880Sstevel@tonic-gate struct seg *seg, *seglast, *p, *n; 3890Sstevel@tonic-gate uint_t nsegs = 0; 3900Sstevel@tonic-gate 3910Sstevel@tonic-gate if (do_as_verify == 0) 3920Sstevel@tonic-gate return; 3930Sstevel@tonic-gate 3940Sstevel@tonic-gate seglast = as->a_seglast; 3950Sstevel@tonic-gate 3960Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) { 3970Sstevel@tonic-gate ASSERT(seg->s_as == as); 3980Sstevel@tonic-gate p = AS_SEGPREV(as, seg); 3990Sstevel@tonic-gate n = AS_SEGNEXT(as, seg); 4000Sstevel@tonic-gate ASSERT(p == NULL || p->s_as == as); 4010Sstevel@tonic-gate ASSERT(p == NULL || p->s_base < seg->s_base); 4020Sstevel@tonic-gate ASSERT(n == NULL || n->s_base > seg->s_base); 4030Sstevel@tonic-gate ASSERT(n != NULL || seg == avl_last(&as->a_segtree)); 4040Sstevel@tonic-gate if (seg == seglast) 4050Sstevel@tonic-gate seglast = NULL; 4060Sstevel@tonic-gate nsegs++; 4070Sstevel@tonic-gate } 4080Sstevel@tonic-gate ASSERT(seglast == NULL); 4090Sstevel@tonic-gate ASSERT(avl_numnodes(&as->a_segtree) == nsegs); 4100Sstevel@tonic-gate } 4110Sstevel@tonic-gate #endif /* VERIFY_SEGLIST */ 4120Sstevel@tonic-gate 4130Sstevel@tonic-gate /* 4140Sstevel@tonic-gate * Add a new segment to the address space. The avl_find() 4150Sstevel@tonic-gate * may be expensive so we attempt to use last segment accessed 4160Sstevel@tonic-gate * in as_gap() as an insertion point. 4170Sstevel@tonic-gate */ 4180Sstevel@tonic-gate int 4190Sstevel@tonic-gate as_addseg(struct as *as, struct seg *newseg) 4200Sstevel@tonic-gate { 4210Sstevel@tonic-gate struct seg *seg; 4220Sstevel@tonic-gate caddr_t addr; 4230Sstevel@tonic-gate caddr_t eaddr; 4240Sstevel@tonic-gate avl_index_t where; 4250Sstevel@tonic-gate 4260Sstevel@tonic-gate ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 4270Sstevel@tonic-gate 4280Sstevel@tonic-gate as->a_updatedir = 1; /* inform /proc */ 4290Sstevel@tonic-gate gethrestime(&as->a_updatetime); 4300Sstevel@tonic-gate 4310Sstevel@tonic-gate if (as->a_lastgaphl != NULL) { 4320Sstevel@tonic-gate struct seg *hseg = NULL; 4330Sstevel@tonic-gate struct seg *lseg = NULL; 4340Sstevel@tonic-gate 4350Sstevel@tonic-gate if (as->a_lastgaphl->s_base > newseg->s_base) { 4360Sstevel@tonic-gate hseg = as->a_lastgaphl; 4370Sstevel@tonic-gate lseg = AVL_PREV(&as->a_segtree, hseg); 4380Sstevel@tonic-gate } else { 4390Sstevel@tonic-gate lseg = as->a_lastgaphl; 4400Sstevel@tonic-gate hseg = AVL_NEXT(&as->a_segtree, lseg); 4410Sstevel@tonic-gate } 4420Sstevel@tonic-gate 4430Sstevel@tonic-gate if (hseg && lseg && lseg->s_base < newseg->s_base && 4440Sstevel@tonic-gate hseg->s_base > newseg->s_base) { 4450Sstevel@tonic-gate avl_insert_here(&as->a_segtree, newseg, lseg, 4460Sstevel@tonic-gate AVL_AFTER); 4470Sstevel@tonic-gate as->a_lastgaphl = NULL; 4480Sstevel@tonic-gate as->a_seglast = newseg; 4490Sstevel@tonic-gate return (0); 4500Sstevel@tonic-gate } 4510Sstevel@tonic-gate as->a_lastgaphl = NULL; 4520Sstevel@tonic-gate } 4530Sstevel@tonic-gate 4540Sstevel@tonic-gate addr = newseg->s_base; 4550Sstevel@tonic-gate eaddr = addr + newseg->s_size; 4560Sstevel@tonic-gate again: 4570Sstevel@tonic-gate 4580Sstevel@tonic-gate seg = avl_find(&as->a_segtree, &addr, &where); 4590Sstevel@tonic-gate 4600Sstevel@tonic-gate if (seg == NULL) 4610Sstevel@tonic-gate seg = avl_nearest(&as->a_segtree, where, AVL_AFTER); 4620Sstevel@tonic-gate 4630Sstevel@tonic-gate if (seg == NULL) 4640Sstevel@tonic-gate seg = avl_last(&as->a_segtree); 4650Sstevel@tonic-gate 4660Sstevel@tonic-gate if (seg != NULL) { 4670Sstevel@tonic-gate caddr_t base = seg->s_base; 4680Sstevel@tonic-gate 4690Sstevel@tonic-gate /* 4700Sstevel@tonic-gate * If top of seg is below the requested address, then 4710Sstevel@tonic-gate * the insertion point is at the end of the linked list, 4720Sstevel@tonic-gate * and seg points to the tail of the list. Otherwise, 4730Sstevel@tonic-gate * the insertion point is immediately before seg. 4740Sstevel@tonic-gate */ 4750Sstevel@tonic-gate if (base + seg->s_size > addr) { 4760Sstevel@tonic-gate if (addr >= base || eaddr > base) { 4770Sstevel@tonic-gate #ifdef __sparc 4780Sstevel@tonic-gate extern struct seg_ops segnf_ops; 4790Sstevel@tonic-gate 4800Sstevel@tonic-gate /* 4810Sstevel@tonic-gate * no-fault segs must disappear if overlaid. 4820Sstevel@tonic-gate * XXX need new segment type so 4830Sstevel@tonic-gate * we don't have to check s_ops 4840Sstevel@tonic-gate */ 4850Sstevel@tonic-gate if (seg->s_ops == &segnf_ops) { 4860Sstevel@tonic-gate seg_unmap(seg); 4870Sstevel@tonic-gate goto again; 4880Sstevel@tonic-gate } 4890Sstevel@tonic-gate #endif 4900Sstevel@tonic-gate return (-1); /* overlapping segment */ 4910Sstevel@tonic-gate } 4920Sstevel@tonic-gate } 4930Sstevel@tonic-gate } 4940Sstevel@tonic-gate as->a_seglast = newseg; 4950Sstevel@tonic-gate avl_insert(&as->a_segtree, newseg, where); 4960Sstevel@tonic-gate 4970Sstevel@tonic-gate #ifdef VERIFY_SEGLIST 4980Sstevel@tonic-gate as_verify(as); 4990Sstevel@tonic-gate #endif 5000Sstevel@tonic-gate return (0); 5010Sstevel@tonic-gate } 5020Sstevel@tonic-gate 5030Sstevel@tonic-gate struct seg * 5040Sstevel@tonic-gate as_removeseg(struct as *as, struct seg *seg) 5050Sstevel@tonic-gate { 5060Sstevel@tonic-gate avl_tree_t *t; 5070Sstevel@tonic-gate 5080Sstevel@tonic-gate ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 5090Sstevel@tonic-gate 5100Sstevel@tonic-gate as->a_updatedir = 1; /* inform /proc */ 5110Sstevel@tonic-gate gethrestime(&as->a_updatetime); 5120Sstevel@tonic-gate 5130Sstevel@tonic-gate if (seg == NULL) 5140Sstevel@tonic-gate return (NULL); 5150Sstevel@tonic-gate 5160Sstevel@tonic-gate t = &as->a_segtree; 5170Sstevel@tonic-gate if (as->a_seglast == seg) 5180Sstevel@tonic-gate as->a_seglast = NULL; 5190Sstevel@tonic-gate as->a_lastgaphl = NULL; 5200Sstevel@tonic-gate 5210Sstevel@tonic-gate /* 5220Sstevel@tonic-gate * if this segment is at an address higher than 5230Sstevel@tonic-gate * a_lastgap, set a_lastgap to the next segment (NULL if last segment) 5240Sstevel@tonic-gate */ 5250Sstevel@tonic-gate if (as->a_lastgap && 5260Sstevel@tonic-gate (seg == as->a_lastgap || seg->s_base > as->a_lastgap->s_base)) 5270Sstevel@tonic-gate as->a_lastgap = AVL_NEXT(t, seg); 5280Sstevel@tonic-gate 5290Sstevel@tonic-gate /* 5300Sstevel@tonic-gate * remove the segment from the seg tree 5310Sstevel@tonic-gate */ 5320Sstevel@tonic-gate avl_remove(t, seg); 5330Sstevel@tonic-gate 5340Sstevel@tonic-gate #ifdef VERIFY_SEGLIST 5350Sstevel@tonic-gate as_verify(as); 5360Sstevel@tonic-gate #endif 5370Sstevel@tonic-gate return (seg); 5380Sstevel@tonic-gate } 5390Sstevel@tonic-gate 5400Sstevel@tonic-gate /* 5410Sstevel@tonic-gate * Find a segment containing addr. 5420Sstevel@tonic-gate */ 5430Sstevel@tonic-gate struct seg * 5440Sstevel@tonic-gate as_segat(struct as *as, caddr_t addr) 5450Sstevel@tonic-gate { 5460Sstevel@tonic-gate struct seg *seg = as->a_seglast; 5470Sstevel@tonic-gate 5480Sstevel@tonic-gate ASSERT(AS_LOCK_HELD(as, &as->a_lock)); 5490Sstevel@tonic-gate 5500Sstevel@tonic-gate if (seg != NULL && seg->s_base <= addr && 5510Sstevel@tonic-gate addr < seg->s_base + seg->s_size) 5520Sstevel@tonic-gate return (seg); 5530Sstevel@tonic-gate 5540Sstevel@tonic-gate seg = avl_find(&as->a_segtree, &addr, NULL); 5550Sstevel@tonic-gate return (seg); 5560Sstevel@tonic-gate } 5570Sstevel@tonic-gate 5580Sstevel@tonic-gate /* 5590Sstevel@tonic-gate * Serialize all searches for holes in an address space to 5600Sstevel@tonic-gate * prevent two or more threads from allocating the same virtual 5610Sstevel@tonic-gate * address range. The address space must not be "read/write" 5620Sstevel@tonic-gate * locked by the caller since we may block. 5630Sstevel@tonic-gate */ 5640Sstevel@tonic-gate void 5650Sstevel@tonic-gate as_rangelock(struct as *as) 5660Sstevel@tonic-gate { 5670Sstevel@tonic-gate mutex_enter(&as->a_contents); 5680Sstevel@tonic-gate while (AS_ISCLAIMGAP(as)) 5690Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 5700Sstevel@tonic-gate AS_SETCLAIMGAP(as); 5710Sstevel@tonic-gate mutex_exit(&as->a_contents); 5720Sstevel@tonic-gate } 5730Sstevel@tonic-gate 5740Sstevel@tonic-gate /* 5750Sstevel@tonic-gate * Release hold on a_state & AS_CLAIMGAP and signal any other blocked threads. 5760Sstevel@tonic-gate */ 5770Sstevel@tonic-gate void 5780Sstevel@tonic-gate as_rangeunlock(struct as *as) 5790Sstevel@tonic-gate { 5800Sstevel@tonic-gate mutex_enter(&as->a_contents); 5810Sstevel@tonic-gate AS_CLRCLAIMGAP(as); 5820Sstevel@tonic-gate cv_signal(&as->a_cv); 5830Sstevel@tonic-gate mutex_exit(&as->a_contents); 5840Sstevel@tonic-gate } 5850Sstevel@tonic-gate 5860Sstevel@tonic-gate /* 5870Sstevel@tonic-gate * compar segments (or just an address) by segment address range 5880Sstevel@tonic-gate */ 5890Sstevel@tonic-gate static int 5900Sstevel@tonic-gate as_segcompar(const void *x, const void *y) 5910Sstevel@tonic-gate { 5920Sstevel@tonic-gate struct seg *a = (struct seg *)x; 5930Sstevel@tonic-gate struct seg *b = (struct seg *)y; 5940Sstevel@tonic-gate 5950Sstevel@tonic-gate if (a->s_base < b->s_base) 5960Sstevel@tonic-gate return (-1); 5970Sstevel@tonic-gate if (a->s_base >= b->s_base + b->s_size) 5980Sstevel@tonic-gate return (1); 5990Sstevel@tonic-gate return (0); 6000Sstevel@tonic-gate } 6010Sstevel@tonic-gate 6020Sstevel@tonic-gate 6030Sstevel@tonic-gate void 6040Sstevel@tonic-gate as_avlinit(struct as *as) 6050Sstevel@tonic-gate { 6060Sstevel@tonic-gate avl_create(&as->a_segtree, as_segcompar, sizeof (struct seg), 6070Sstevel@tonic-gate offsetof(struct seg, s_tree)); 6080Sstevel@tonic-gate avl_create(&as->a_wpage, wp_compare, sizeof (struct watched_page), 6090Sstevel@tonic-gate offsetof(struct watched_page, wp_link)); 6100Sstevel@tonic-gate } 6110Sstevel@tonic-gate 6120Sstevel@tonic-gate /*ARGSUSED*/ 6130Sstevel@tonic-gate static int 6140Sstevel@tonic-gate as_constructor(void *buf, void *cdrarg, int kmflags) 6150Sstevel@tonic-gate { 6160Sstevel@tonic-gate struct as *as = buf; 6170Sstevel@tonic-gate 6180Sstevel@tonic-gate mutex_init(&as->a_contents, NULL, MUTEX_DEFAULT, NULL); 6190Sstevel@tonic-gate cv_init(&as->a_cv, NULL, CV_DEFAULT, NULL); 6200Sstevel@tonic-gate rw_init(&as->a_lock, NULL, RW_DEFAULT, NULL); 6210Sstevel@tonic-gate as_avlinit(as); 6220Sstevel@tonic-gate return (0); 6230Sstevel@tonic-gate } 6240Sstevel@tonic-gate 6250Sstevel@tonic-gate /*ARGSUSED1*/ 6260Sstevel@tonic-gate static void 6270Sstevel@tonic-gate as_destructor(void *buf, void *cdrarg) 6280Sstevel@tonic-gate { 6290Sstevel@tonic-gate struct as *as = buf; 6300Sstevel@tonic-gate 6310Sstevel@tonic-gate avl_destroy(&as->a_segtree); 6320Sstevel@tonic-gate mutex_destroy(&as->a_contents); 6330Sstevel@tonic-gate cv_destroy(&as->a_cv); 6340Sstevel@tonic-gate rw_destroy(&as->a_lock); 6350Sstevel@tonic-gate } 6360Sstevel@tonic-gate 6370Sstevel@tonic-gate void 6380Sstevel@tonic-gate as_init(void) 6390Sstevel@tonic-gate { 6400Sstevel@tonic-gate as_cache = kmem_cache_create("as_cache", sizeof (struct as), 0, 6415084Sjohnlev as_constructor, as_destructor, NULL, NULL, NULL, 0); 6420Sstevel@tonic-gate } 6430Sstevel@tonic-gate 6440Sstevel@tonic-gate /* 6450Sstevel@tonic-gate * Allocate and initialize an address space data structure. 6460Sstevel@tonic-gate * We call hat_alloc to allow any machine dependent 6470Sstevel@tonic-gate * information in the hat structure to be initialized. 6480Sstevel@tonic-gate */ 6490Sstevel@tonic-gate struct as * 6500Sstevel@tonic-gate as_alloc(void) 6510Sstevel@tonic-gate { 6520Sstevel@tonic-gate struct as *as; 6530Sstevel@tonic-gate 6540Sstevel@tonic-gate as = kmem_cache_alloc(as_cache, KM_SLEEP); 6550Sstevel@tonic-gate 6560Sstevel@tonic-gate as->a_flags = 0; 6570Sstevel@tonic-gate as->a_vbits = 0; 6580Sstevel@tonic-gate as->a_hrm = NULL; 6590Sstevel@tonic-gate as->a_seglast = NULL; 6600Sstevel@tonic-gate as->a_size = 0; 6610Sstevel@tonic-gate as->a_updatedir = 0; 6620Sstevel@tonic-gate gethrestime(&as->a_updatetime); 6630Sstevel@tonic-gate as->a_objectdir = NULL; 6640Sstevel@tonic-gate as->a_sizedir = 0; 6650Sstevel@tonic-gate as->a_userlimit = (caddr_t)USERLIMIT; 6660Sstevel@tonic-gate as->a_lastgap = NULL; 6670Sstevel@tonic-gate as->a_lastgaphl = NULL; 6680Sstevel@tonic-gate as->a_callbacks = NULL; 6690Sstevel@tonic-gate 6700Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 6710Sstevel@tonic-gate as->a_hat = hat_alloc(as); /* create hat for default system mmu */ 6720Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 6730Sstevel@tonic-gate 6740Sstevel@tonic-gate as->a_xhat = NULL; 6750Sstevel@tonic-gate 6760Sstevel@tonic-gate return (as); 6770Sstevel@tonic-gate } 6780Sstevel@tonic-gate 6790Sstevel@tonic-gate /* 6800Sstevel@tonic-gate * Free an address space data structure. 6810Sstevel@tonic-gate * Need to free the hat first and then 6820Sstevel@tonic-gate * all the segments on this as and finally 6830Sstevel@tonic-gate * the space for the as struct itself. 6840Sstevel@tonic-gate */ 6850Sstevel@tonic-gate void 6860Sstevel@tonic-gate as_free(struct as *as) 6870Sstevel@tonic-gate { 6880Sstevel@tonic-gate struct hat *hat = as->a_hat; 6890Sstevel@tonic-gate struct seg *seg, *next; 6900Sstevel@tonic-gate int called = 0; 6910Sstevel@tonic-gate 6920Sstevel@tonic-gate top: 6930Sstevel@tonic-gate /* 6940Sstevel@tonic-gate * Invoke ALL callbacks. as_do_callbacks will do one callback 6950Sstevel@tonic-gate * per call, and not return (-1) until the callback has completed. 6960Sstevel@tonic-gate * When as_do_callbacks returns zero, all callbacks have completed. 6970Sstevel@tonic-gate */ 6980Sstevel@tonic-gate mutex_enter(&as->a_contents); 6995084Sjohnlev while (as->a_callbacks && as_do_callbacks(as, AS_ALL_EVENT, 0, 0)) 7005084Sjohnlev ; 7010Sstevel@tonic-gate 7020Sstevel@tonic-gate /* This will prevent new XHATs from attaching to as */ 7030Sstevel@tonic-gate if (!called) 7040Sstevel@tonic-gate AS_SETBUSY(as); 7050Sstevel@tonic-gate mutex_exit(&as->a_contents); 7060Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 7070Sstevel@tonic-gate 7080Sstevel@tonic-gate if (!called) { 7090Sstevel@tonic-gate called = 1; 7100Sstevel@tonic-gate hat_free_start(hat); 7110Sstevel@tonic-gate if (as->a_xhat != NULL) 7120Sstevel@tonic-gate xhat_free_start_all(as); 7130Sstevel@tonic-gate } 7140Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; seg = next) { 7150Sstevel@tonic-gate int err; 7160Sstevel@tonic-gate 7170Sstevel@tonic-gate next = AS_SEGNEXT(as, seg); 7186695Saguzovsk retry: 7190Sstevel@tonic-gate err = SEGOP_UNMAP(seg, seg->s_base, seg->s_size); 7200Sstevel@tonic-gate if (err == EAGAIN) { 7210Sstevel@tonic-gate mutex_enter(&as->a_contents); 7220Sstevel@tonic-gate if (as->a_callbacks) { 7230Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 7246695Saguzovsk } else if (!AS_ISNOUNMAPWAIT(as)) { 7250Sstevel@tonic-gate /* 7260Sstevel@tonic-gate * Memory is currently locked. Wait for a 7270Sstevel@tonic-gate * cv_signal that it has been unlocked, then 7280Sstevel@tonic-gate * try the operation again. 7290Sstevel@tonic-gate */ 7300Sstevel@tonic-gate if (AS_ISUNMAPWAIT(as) == 0) 7310Sstevel@tonic-gate cv_broadcast(&as->a_cv); 7320Sstevel@tonic-gate AS_SETUNMAPWAIT(as); 7330Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 7340Sstevel@tonic-gate while (AS_ISUNMAPWAIT(as)) 7350Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 7366695Saguzovsk } else { 7376695Saguzovsk /* 7386695Saguzovsk * We may have raced with 7396695Saguzovsk * segvn_reclaim()/segspt_reclaim(). In this 7406695Saguzovsk * case clean nounmapwait flag and retry since 7416695Saguzovsk * softlockcnt in this segment may be already 7426695Saguzovsk * 0. We don't drop as writer lock so our 7436695Saguzovsk * number of retries without sleeping should 7446695Saguzovsk * be very small. See segvn_reclaim() for 7456695Saguzovsk * more comments. 7466695Saguzovsk */ 7476695Saguzovsk AS_CLRNOUNMAPWAIT(as); 7486695Saguzovsk mutex_exit(&as->a_contents); 7496695Saguzovsk goto retry; 7500Sstevel@tonic-gate } 7510Sstevel@tonic-gate mutex_exit(&as->a_contents); 7520Sstevel@tonic-gate goto top; 7530Sstevel@tonic-gate } else { 7540Sstevel@tonic-gate /* 7550Sstevel@tonic-gate * We do not expect any other error return at this 7560Sstevel@tonic-gate * time. This is similar to an ASSERT in seg_unmap() 7570Sstevel@tonic-gate */ 7580Sstevel@tonic-gate ASSERT(err == 0); 7590Sstevel@tonic-gate } 7600Sstevel@tonic-gate } 7610Sstevel@tonic-gate hat_free_end(hat); 7620Sstevel@tonic-gate if (as->a_xhat != NULL) 7630Sstevel@tonic-gate xhat_free_end_all(as); 7640Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 7650Sstevel@tonic-gate 7660Sstevel@tonic-gate /* /proc stuff */ 7670Sstevel@tonic-gate ASSERT(avl_numnodes(&as->a_wpage) == 0); 7680Sstevel@tonic-gate if (as->a_objectdir) { 7690Sstevel@tonic-gate kmem_free(as->a_objectdir, as->a_sizedir * sizeof (vnode_t *)); 7700Sstevel@tonic-gate as->a_objectdir = NULL; 7710Sstevel@tonic-gate as->a_sizedir = 0; 7720Sstevel@tonic-gate } 7730Sstevel@tonic-gate 7740Sstevel@tonic-gate /* 7750Sstevel@tonic-gate * Free the struct as back to kmem. Assert it has no segments. 7760Sstevel@tonic-gate */ 7770Sstevel@tonic-gate ASSERT(avl_numnodes(&as->a_segtree) == 0); 7780Sstevel@tonic-gate kmem_cache_free(as_cache, as); 7790Sstevel@tonic-gate } 7800Sstevel@tonic-gate 7810Sstevel@tonic-gate int 7820Sstevel@tonic-gate as_dup(struct as *as, struct as **outas) 7830Sstevel@tonic-gate { 7840Sstevel@tonic-gate struct as *newas; 7850Sstevel@tonic-gate struct seg *seg, *newseg; 7860Sstevel@tonic-gate int error; 7870Sstevel@tonic-gate 7880Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 7890Sstevel@tonic-gate as_clearwatch(as); 7900Sstevel@tonic-gate newas = as_alloc(); 7910Sstevel@tonic-gate newas->a_userlimit = as->a_userlimit; 792*7209Svk210190 newas->a_proc = as->a_proc->p_child; 793*7209Svk210190 7940Sstevel@tonic-gate AS_LOCK_ENTER(newas, &newas->a_lock, RW_WRITER); 7950Sstevel@tonic-gate 7960Sstevel@tonic-gate /* This will prevent new XHATs from attaching */ 7970Sstevel@tonic-gate mutex_enter(&as->a_contents); 7980Sstevel@tonic-gate AS_SETBUSY(as); 7990Sstevel@tonic-gate mutex_exit(&as->a_contents); 8000Sstevel@tonic-gate mutex_enter(&newas->a_contents); 8010Sstevel@tonic-gate AS_SETBUSY(newas); 8020Sstevel@tonic-gate mutex_exit(&newas->a_contents); 8030Sstevel@tonic-gate 8044528Spaulsan (void) hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_SRD); 8050Sstevel@tonic-gate 8060Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) { 8070Sstevel@tonic-gate 8080Sstevel@tonic-gate if (seg->s_flags & S_PURGE) 8090Sstevel@tonic-gate continue; 8100Sstevel@tonic-gate 8110Sstevel@tonic-gate newseg = seg_alloc(newas, seg->s_base, seg->s_size); 8120Sstevel@tonic-gate if (newseg == NULL) { 8130Sstevel@tonic-gate AS_LOCK_EXIT(newas, &newas->a_lock); 8140Sstevel@tonic-gate as_setwatch(as); 8150Sstevel@tonic-gate mutex_enter(&as->a_contents); 8160Sstevel@tonic-gate AS_CLRBUSY(as); 8170Sstevel@tonic-gate mutex_exit(&as->a_contents); 8180Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 8190Sstevel@tonic-gate as_free(newas); 8200Sstevel@tonic-gate return (-1); 8210Sstevel@tonic-gate } 8220Sstevel@tonic-gate if ((error = SEGOP_DUP(seg, newseg)) != 0) { 8230Sstevel@tonic-gate /* 8240Sstevel@tonic-gate * We call seg_free() on the new seg 8250Sstevel@tonic-gate * because the segment is not set up 8260Sstevel@tonic-gate * completely; i.e. it has no ops. 8270Sstevel@tonic-gate */ 8280Sstevel@tonic-gate as_setwatch(as); 8290Sstevel@tonic-gate mutex_enter(&as->a_contents); 8300Sstevel@tonic-gate AS_CLRBUSY(as); 8310Sstevel@tonic-gate mutex_exit(&as->a_contents); 8320Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 8330Sstevel@tonic-gate seg_free(newseg); 8340Sstevel@tonic-gate AS_LOCK_EXIT(newas, &newas->a_lock); 8350Sstevel@tonic-gate as_free(newas); 8360Sstevel@tonic-gate return (error); 8370Sstevel@tonic-gate } 8380Sstevel@tonic-gate newas->a_size += seg->s_size; 8390Sstevel@tonic-gate } 8400Sstevel@tonic-gate 8410Sstevel@tonic-gate error = hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_ALL); 8420Sstevel@tonic-gate if (as->a_xhat != NULL) 8430Sstevel@tonic-gate error |= xhat_dup_all(as, newas, NULL, 0, HAT_DUP_ALL); 8440Sstevel@tonic-gate 8450Sstevel@tonic-gate mutex_enter(&newas->a_contents); 8460Sstevel@tonic-gate AS_CLRBUSY(newas); 8470Sstevel@tonic-gate mutex_exit(&newas->a_contents); 8480Sstevel@tonic-gate AS_LOCK_EXIT(newas, &newas->a_lock); 8490Sstevel@tonic-gate 8500Sstevel@tonic-gate as_setwatch(as); 8510Sstevel@tonic-gate mutex_enter(&as->a_contents); 8520Sstevel@tonic-gate AS_CLRBUSY(as); 8530Sstevel@tonic-gate mutex_exit(&as->a_contents); 8540Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 8550Sstevel@tonic-gate if (error != 0) { 8560Sstevel@tonic-gate as_free(newas); 8570Sstevel@tonic-gate return (error); 8580Sstevel@tonic-gate } 8590Sstevel@tonic-gate *outas = newas; 8600Sstevel@tonic-gate return (0); 8610Sstevel@tonic-gate } 8620Sstevel@tonic-gate 8630Sstevel@tonic-gate /* 8640Sstevel@tonic-gate * Handle a ``fault'' at addr for size bytes. 8650Sstevel@tonic-gate */ 8660Sstevel@tonic-gate faultcode_t 8670Sstevel@tonic-gate as_fault(struct hat *hat, struct as *as, caddr_t addr, size_t size, 8680Sstevel@tonic-gate enum fault_type type, enum seg_rw rw) 8690Sstevel@tonic-gate { 8700Sstevel@tonic-gate struct seg *seg; 8710Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 8720Sstevel@tonic-gate size_t rsize; /* rounded up size */ 8730Sstevel@tonic-gate size_t ssize; 8740Sstevel@tonic-gate faultcode_t res = 0; 8750Sstevel@tonic-gate caddr_t addrsav; 8760Sstevel@tonic-gate struct seg *segsav; 8770Sstevel@tonic-gate int as_lock_held; 8780Sstevel@tonic-gate klwp_t *lwp = ttolwp(curthread); 8790Sstevel@tonic-gate int is_xhat = 0; 8800Sstevel@tonic-gate int holding_wpage = 0; 8810Sstevel@tonic-gate extern struct seg_ops segdev_ops; 8820Sstevel@tonic-gate 8830Sstevel@tonic-gate 8840Sstevel@tonic-gate 8850Sstevel@tonic-gate if (as->a_hat != hat) { 8860Sstevel@tonic-gate /* This must be an XHAT then */ 8870Sstevel@tonic-gate is_xhat = 1; 8880Sstevel@tonic-gate 8890Sstevel@tonic-gate if ((type != F_INVAL) || (as == &kas)) 8900Sstevel@tonic-gate return (FC_NOSUPPORT); 8910Sstevel@tonic-gate } 8920Sstevel@tonic-gate 8930Sstevel@tonic-gate retry: 8940Sstevel@tonic-gate if (!is_xhat) { 8950Sstevel@tonic-gate /* 8960Sstevel@tonic-gate * Indicate that the lwp is not to be stopped while waiting 8970Sstevel@tonic-gate * for a pagefault. This is to avoid deadlock while debugging 8980Sstevel@tonic-gate * a process via /proc over NFS (in particular). 8990Sstevel@tonic-gate */ 9003930Snr123932 if (lwp != NULL) 9010Sstevel@tonic-gate lwp->lwp_nostop++; 9020Sstevel@tonic-gate 9030Sstevel@tonic-gate /* 9040Sstevel@tonic-gate * same length must be used when we softlock and softunlock. 9050Sstevel@tonic-gate * We don't support softunlocking lengths less than 9060Sstevel@tonic-gate * the original length when there is largepage support. 9070Sstevel@tonic-gate * See seg_dev.c for more comments. 9080Sstevel@tonic-gate */ 9090Sstevel@tonic-gate switch (type) { 9100Sstevel@tonic-gate 9110Sstevel@tonic-gate case F_SOFTLOCK: 9120Sstevel@tonic-gate CPU_STATS_ADD_K(vm, softlock, 1); 9130Sstevel@tonic-gate break; 9140Sstevel@tonic-gate 9150Sstevel@tonic-gate case F_SOFTUNLOCK: 9160Sstevel@tonic-gate break; 9170Sstevel@tonic-gate 9180Sstevel@tonic-gate case F_PROT: 9190Sstevel@tonic-gate CPU_STATS_ADD_K(vm, prot_fault, 1); 9200Sstevel@tonic-gate break; 9210Sstevel@tonic-gate 9220Sstevel@tonic-gate case F_INVAL: 9230Sstevel@tonic-gate CPU_STATS_ENTER_K(); 9240Sstevel@tonic-gate CPU_STATS_ADDQ(CPU, vm, as_fault, 1); 9250Sstevel@tonic-gate if (as == &kas) 9260Sstevel@tonic-gate CPU_STATS_ADDQ(CPU, vm, kernel_asflt, 1); 9270Sstevel@tonic-gate CPU_STATS_EXIT_K(); 9280Sstevel@tonic-gate break; 9290Sstevel@tonic-gate } 9300Sstevel@tonic-gate } 9310Sstevel@tonic-gate 9320Sstevel@tonic-gate /* Kernel probe */ 9330Sstevel@tonic-gate TNF_PROBE_3(address_fault, "vm pagefault", /* CSTYLED */, 9345084Sjohnlev tnf_opaque, address, addr, 9355084Sjohnlev tnf_fault_type, fault_type, type, 9365084Sjohnlev tnf_seg_access, access, rw); 9370Sstevel@tonic-gate 9380Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 9390Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 9405084Sjohnlev (size_t)raddr; 9410Sstevel@tonic-gate 9420Sstevel@tonic-gate /* 9430Sstevel@tonic-gate * XXX -- Don't grab the as lock for segkmap. We should grab it for 9440Sstevel@tonic-gate * correctness, but then we could be stuck holding this lock for 9450Sstevel@tonic-gate * a LONG time if the fault needs to be resolved on a slow 9460Sstevel@tonic-gate * filesystem, and then no-one will be able to exec new commands, 9470Sstevel@tonic-gate * as exec'ing requires the write lock on the as. 9480Sstevel@tonic-gate */ 9490Sstevel@tonic-gate if (as == &kas && segkmap && segkmap->s_base <= raddr && 9500Sstevel@tonic-gate raddr + size < segkmap->s_base + segkmap->s_size) { 9510Sstevel@tonic-gate /* 9520Sstevel@tonic-gate * if (as==&kas), this can't be XHAT: we've already returned 9530Sstevel@tonic-gate * FC_NOSUPPORT. 9540Sstevel@tonic-gate */ 9550Sstevel@tonic-gate seg = segkmap; 9560Sstevel@tonic-gate as_lock_held = 0; 9570Sstevel@tonic-gate } else { 9580Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 9590Sstevel@tonic-gate if (is_xhat && avl_numnodes(&as->a_wpage) != 0) { 9600Sstevel@tonic-gate /* 9610Sstevel@tonic-gate * Grab and hold the writers' lock on the as 9620Sstevel@tonic-gate * if the fault is to a watched page. 9630Sstevel@tonic-gate * This will keep CPUs from "peeking" at the 9640Sstevel@tonic-gate * address range while we're temporarily boosting 9650Sstevel@tonic-gate * the permissions for the XHAT device to 9660Sstevel@tonic-gate * resolve the fault in the segment layer. 9670Sstevel@tonic-gate * 9680Sstevel@tonic-gate * We could check whether faulted address 9690Sstevel@tonic-gate * is within a watched page and only then grab 9700Sstevel@tonic-gate * the writer lock, but this is simpler. 9710Sstevel@tonic-gate */ 9720Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 9730Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 9740Sstevel@tonic-gate } 9750Sstevel@tonic-gate 9760Sstevel@tonic-gate seg = as_segat(as, raddr); 9770Sstevel@tonic-gate if (seg == NULL) { 9780Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 9793930Snr123932 if ((lwp != NULL) && (!is_xhat)) 9800Sstevel@tonic-gate lwp->lwp_nostop--; 9810Sstevel@tonic-gate return (FC_NOMAP); 9820Sstevel@tonic-gate } 9830Sstevel@tonic-gate 9840Sstevel@tonic-gate as_lock_held = 1; 9850Sstevel@tonic-gate } 9860Sstevel@tonic-gate 9870Sstevel@tonic-gate addrsav = raddr; 9880Sstevel@tonic-gate segsav = seg; 9890Sstevel@tonic-gate 9900Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 9910Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 9920Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 9930Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 9940Sstevel@tonic-gate res = FC_NOMAP; 9950Sstevel@tonic-gate break; 9960Sstevel@tonic-gate } 9970Sstevel@tonic-gate } 9980Sstevel@tonic-gate if (raddr + rsize > seg->s_base + seg->s_size) 9990Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 10000Sstevel@tonic-gate else 10010Sstevel@tonic-gate ssize = rsize; 10020Sstevel@tonic-gate 10030Sstevel@tonic-gate if (!is_xhat || (seg->s_ops != &segdev_ops)) { 10040Sstevel@tonic-gate 10050Sstevel@tonic-gate if (is_xhat && avl_numnodes(&as->a_wpage) != 0 && 10060Sstevel@tonic-gate pr_is_watchpage_as(raddr, rw, as)) { 10070Sstevel@tonic-gate /* 10080Sstevel@tonic-gate * Handle watch pages. If we're faulting on a 10090Sstevel@tonic-gate * watched page from an X-hat, we have to 10100Sstevel@tonic-gate * restore the original permissions while we 10110Sstevel@tonic-gate * handle the fault. 10120Sstevel@tonic-gate */ 10130Sstevel@tonic-gate as_clearwatch(as); 10140Sstevel@tonic-gate holding_wpage = 1; 10150Sstevel@tonic-gate } 10160Sstevel@tonic-gate 10170Sstevel@tonic-gate res = SEGOP_FAULT(hat, seg, raddr, ssize, type, rw); 10180Sstevel@tonic-gate 10190Sstevel@tonic-gate /* Restore watchpoints */ 10200Sstevel@tonic-gate if (holding_wpage) { 10210Sstevel@tonic-gate as_setwatch(as); 10220Sstevel@tonic-gate holding_wpage = 0; 10230Sstevel@tonic-gate } 10240Sstevel@tonic-gate 10250Sstevel@tonic-gate if (res != 0) 10260Sstevel@tonic-gate break; 10270Sstevel@tonic-gate } else { 10280Sstevel@tonic-gate /* XHAT does not support seg_dev */ 10290Sstevel@tonic-gate res = FC_NOSUPPORT; 10300Sstevel@tonic-gate break; 10310Sstevel@tonic-gate } 10320Sstevel@tonic-gate } 10330Sstevel@tonic-gate 10340Sstevel@tonic-gate /* 10350Sstevel@tonic-gate * If we were SOFTLOCKing and encountered a failure, 10360Sstevel@tonic-gate * we must SOFTUNLOCK the range we already did. (Maybe we 10370Sstevel@tonic-gate * should just panic if we are SOFTLOCKing or even SOFTUNLOCKing 10380Sstevel@tonic-gate * right here...) 10390Sstevel@tonic-gate */ 10400Sstevel@tonic-gate if (res != 0 && type == F_SOFTLOCK) { 10410Sstevel@tonic-gate for (seg = segsav; addrsav < raddr; addrsav += ssize) { 10420Sstevel@tonic-gate if (addrsav >= seg->s_base + seg->s_size) 10430Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 10440Sstevel@tonic-gate ASSERT(seg != NULL); 10450Sstevel@tonic-gate /* 10460Sstevel@tonic-gate * Now call the fault routine again to perform the 10470Sstevel@tonic-gate * unlock using S_OTHER instead of the rw variable 10480Sstevel@tonic-gate * since we never got a chance to touch the pages. 10490Sstevel@tonic-gate */ 10500Sstevel@tonic-gate if (raddr > seg->s_base + seg->s_size) 10510Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - addrsav; 10520Sstevel@tonic-gate else 10530Sstevel@tonic-gate ssize = raddr - addrsav; 10540Sstevel@tonic-gate (void) SEGOP_FAULT(hat, seg, addrsav, ssize, 10550Sstevel@tonic-gate F_SOFTUNLOCK, S_OTHER); 10560Sstevel@tonic-gate } 10570Sstevel@tonic-gate } 10580Sstevel@tonic-gate if (as_lock_held) 10590Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 10603930Snr123932 if ((lwp != NULL) && (!is_xhat)) 10610Sstevel@tonic-gate lwp->lwp_nostop--; 10623930Snr123932 10630Sstevel@tonic-gate /* 10640Sstevel@tonic-gate * If the lower levels returned EDEADLK for a fault, 10650Sstevel@tonic-gate * It means that we should retry the fault. Let's wait 10660Sstevel@tonic-gate * a bit also to let the deadlock causing condition clear. 10670Sstevel@tonic-gate * This is part of a gross hack to work around a design flaw 10680Sstevel@tonic-gate * in the ufs/sds logging code and should go away when the 10690Sstevel@tonic-gate * logging code is re-designed to fix the problem. See bug 10700Sstevel@tonic-gate * 4125102 for details of the problem. 10710Sstevel@tonic-gate */ 10720Sstevel@tonic-gate if (FC_ERRNO(res) == EDEADLK) { 10730Sstevel@tonic-gate delay(deadlk_wait); 10740Sstevel@tonic-gate res = 0; 10750Sstevel@tonic-gate goto retry; 10760Sstevel@tonic-gate } 10770Sstevel@tonic-gate return (res); 10780Sstevel@tonic-gate } 10790Sstevel@tonic-gate 10800Sstevel@tonic-gate 10810Sstevel@tonic-gate 10820Sstevel@tonic-gate /* 10830Sstevel@tonic-gate * Asynchronous ``fault'' at addr for size bytes. 10840Sstevel@tonic-gate */ 10850Sstevel@tonic-gate faultcode_t 10860Sstevel@tonic-gate as_faulta(struct as *as, caddr_t addr, size_t size) 10870Sstevel@tonic-gate { 10880Sstevel@tonic-gate struct seg *seg; 10890Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 10900Sstevel@tonic-gate size_t rsize; /* rounded up size */ 10910Sstevel@tonic-gate faultcode_t res = 0; 10920Sstevel@tonic-gate klwp_t *lwp = ttolwp(curthread); 10930Sstevel@tonic-gate 10940Sstevel@tonic-gate retry: 10950Sstevel@tonic-gate /* 10960Sstevel@tonic-gate * Indicate that the lwp is not to be stopped while waiting 10970Sstevel@tonic-gate * for a pagefault. This is to avoid deadlock while debugging 10980Sstevel@tonic-gate * a process via /proc over NFS (in particular). 10990Sstevel@tonic-gate */ 11003930Snr123932 if (lwp != NULL) 11010Sstevel@tonic-gate lwp->lwp_nostop++; 11020Sstevel@tonic-gate 11030Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 11040Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 11055084Sjohnlev (size_t)raddr; 11060Sstevel@tonic-gate 11070Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 11080Sstevel@tonic-gate seg = as_segat(as, raddr); 11090Sstevel@tonic-gate if (seg == NULL) { 11100Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 11113930Snr123932 if (lwp != NULL) 11120Sstevel@tonic-gate lwp->lwp_nostop--; 11130Sstevel@tonic-gate return (FC_NOMAP); 11140Sstevel@tonic-gate } 11150Sstevel@tonic-gate 11160Sstevel@tonic-gate for (; rsize != 0; rsize -= PAGESIZE, raddr += PAGESIZE) { 11170Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 11180Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 11190Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 11200Sstevel@tonic-gate res = FC_NOMAP; 11210Sstevel@tonic-gate break; 11220Sstevel@tonic-gate } 11230Sstevel@tonic-gate } 11240Sstevel@tonic-gate res = SEGOP_FAULTA(seg, raddr); 11250Sstevel@tonic-gate if (res != 0) 11260Sstevel@tonic-gate break; 11270Sstevel@tonic-gate } 11280Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 11293930Snr123932 if (lwp != NULL) 11300Sstevel@tonic-gate lwp->lwp_nostop--; 11310Sstevel@tonic-gate /* 11320Sstevel@tonic-gate * If the lower levels returned EDEADLK for a fault, 11330Sstevel@tonic-gate * It means that we should retry the fault. Let's wait 11340Sstevel@tonic-gate * a bit also to let the deadlock causing condition clear. 11350Sstevel@tonic-gate * This is part of a gross hack to work around a design flaw 11360Sstevel@tonic-gate * in the ufs/sds logging code and should go away when the 11370Sstevel@tonic-gate * logging code is re-designed to fix the problem. See bug 11380Sstevel@tonic-gate * 4125102 for details of the problem. 11390Sstevel@tonic-gate */ 11400Sstevel@tonic-gate if (FC_ERRNO(res) == EDEADLK) { 11410Sstevel@tonic-gate delay(deadlk_wait); 11420Sstevel@tonic-gate res = 0; 11430Sstevel@tonic-gate goto retry; 11440Sstevel@tonic-gate } 11450Sstevel@tonic-gate return (res); 11460Sstevel@tonic-gate } 11470Sstevel@tonic-gate 11480Sstevel@tonic-gate /* 11490Sstevel@tonic-gate * Set the virtual mapping for the interval from [addr : addr + size) 11500Sstevel@tonic-gate * in address space `as' to have the specified protection. 11510Sstevel@tonic-gate * It is ok for the range to cross over several segments, 11520Sstevel@tonic-gate * as long as they are contiguous. 11530Sstevel@tonic-gate */ 11540Sstevel@tonic-gate int 11550Sstevel@tonic-gate as_setprot(struct as *as, caddr_t addr, size_t size, uint_t prot) 11560Sstevel@tonic-gate { 11570Sstevel@tonic-gate struct seg *seg; 11580Sstevel@tonic-gate struct as_callback *cb; 11590Sstevel@tonic-gate size_t ssize; 11600Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 11610Sstevel@tonic-gate size_t rsize; /* rounded up size */ 11620Sstevel@tonic-gate int error = 0, writer = 0; 11630Sstevel@tonic-gate caddr_t saveraddr; 11640Sstevel@tonic-gate size_t saversize; 11650Sstevel@tonic-gate 11660Sstevel@tonic-gate setprot_top: 11670Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 11680Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 11695084Sjohnlev (size_t)raddr; 11700Sstevel@tonic-gate 11710Sstevel@tonic-gate if (raddr + rsize < raddr) /* check for wraparound */ 11720Sstevel@tonic-gate return (ENOMEM); 11730Sstevel@tonic-gate 11740Sstevel@tonic-gate saveraddr = raddr; 11750Sstevel@tonic-gate saversize = rsize; 11760Sstevel@tonic-gate 11770Sstevel@tonic-gate /* 11780Sstevel@tonic-gate * Normally we only lock the as as a reader. But 11790Sstevel@tonic-gate * if due to setprot the segment driver needs to split 11805331Samw * a segment it will return IE_RETRY. Therefore we re-acquire 11810Sstevel@tonic-gate * the as lock as a writer so the segment driver can change 11820Sstevel@tonic-gate * the seg list. Also the segment driver will return IE_RETRY 11830Sstevel@tonic-gate * after it has changed the segment list so we therefore keep 11840Sstevel@tonic-gate * locking as a writer. Since these opeartions should be rare 11850Sstevel@tonic-gate * want to only lock as a writer when necessary. 11860Sstevel@tonic-gate */ 11870Sstevel@tonic-gate if (writer || avl_numnodes(&as->a_wpage) != 0) { 11880Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 11890Sstevel@tonic-gate } else { 11900Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 11910Sstevel@tonic-gate } 11920Sstevel@tonic-gate 11930Sstevel@tonic-gate as_clearwatchprot(as, raddr, rsize); 11940Sstevel@tonic-gate seg = as_segat(as, raddr); 11950Sstevel@tonic-gate if (seg == NULL) { 11960Sstevel@tonic-gate as_setwatch(as); 11970Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 11980Sstevel@tonic-gate return (ENOMEM); 11990Sstevel@tonic-gate } 12000Sstevel@tonic-gate 12010Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 12020Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 12030Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 12040Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 12050Sstevel@tonic-gate error = ENOMEM; 12060Sstevel@tonic-gate break; 12070Sstevel@tonic-gate } 12080Sstevel@tonic-gate } 12090Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 12100Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 12110Sstevel@tonic-gate else 12120Sstevel@tonic-gate ssize = rsize; 12136695Saguzovsk retry: 12140Sstevel@tonic-gate error = SEGOP_SETPROT(seg, raddr, ssize, prot); 12150Sstevel@tonic-gate 12160Sstevel@tonic-gate if (error == IE_NOMEM) { 12170Sstevel@tonic-gate error = EAGAIN; 12180Sstevel@tonic-gate break; 12190Sstevel@tonic-gate } 12200Sstevel@tonic-gate 12210Sstevel@tonic-gate if (error == IE_RETRY) { 12220Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 12230Sstevel@tonic-gate writer = 1; 12240Sstevel@tonic-gate goto setprot_top; 12250Sstevel@tonic-gate } 12260Sstevel@tonic-gate 12270Sstevel@tonic-gate if (error == EAGAIN) { 12280Sstevel@tonic-gate /* 12290Sstevel@tonic-gate * Make sure we have a_lock as writer. 12300Sstevel@tonic-gate */ 12310Sstevel@tonic-gate if (writer == 0) { 12320Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 12330Sstevel@tonic-gate writer = 1; 12340Sstevel@tonic-gate goto setprot_top; 12350Sstevel@tonic-gate } 12360Sstevel@tonic-gate 12370Sstevel@tonic-gate /* 12380Sstevel@tonic-gate * Memory is currently locked. It must be unlocked 12390Sstevel@tonic-gate * before this operation can succeed through a retry. 12400Sstevel@tonic-gate * The possible reasons for locked memory and 12410Sstevel@tonic-gate * corresponding strategies for unlocking are: 12420Sstevel@tonic-gate * (1) Normal I/O 12430Sstevel@tonic-gate * wait for a signal that the I/O operation 12440Sstevel@tonic-gate * has completed and the memory is unlocked. 12450Sstevel@tonic-gate * (2) Asynchronous I/O 12460Sstevel@tonic-gate * The aio subsystem does not unlock pages when 12470Sstevel@tonic-gate * the I/O is completed. Those pages are unlocked 12480Sstevel@tonic-gate * when the application calls aiowait/aioerror. 12490Sstevel@tonic-gate * So, to prevent blocking forever, cv_broadcast() 12500Sstevel@tonic-gate * is done to wake up aio_cleanup_thread. 12510Sstevel@tonic-gate * Subsequently, segvn_reclaim will be called, and 12520Sstevel@tonic-gate * that will do AS_CLRUNMAPWAIT() and wake us up. 12530Sstevel@tonic-gate * (3) Long term page locking: 12540Sstevel@tonic-gate * Drivers intending to have pages locked for a 12550Sstevel@tonic-gate * period considerably longer than for normal I/O 12560Sstevel@tonic-gate * (essentially forever) may have registered for a 12570Sstevel@tonic-gate * callback so they may unlock these pages on 12580Sstevel@tonic-gate * request. This is needed to allow this operation 12590Sstevel@tonic-gate * to succeed. Each entry on the callback list is 12600Sstevel@tonic-gate * examined. If the event or address range pertains 12610Sstevel@tonic-gate * the callback is invoked (unless it already is in 12620Sstevel@tonic-gate * progress). The a_contents lock must be dropped 12630Sstevel@tonic-gate * before the callback, so only one callback can 12640Sstevel@tonic-gate * be done at a time. Go to the top and do more 12650Sstevel@tonic-gate * until zero is returned. If zero is returned, 12660Sstevel@tonic-gate * either there were no callbacks for this event 12670Sstevel@tonic-gate * or they were already in progress. 12680Sstevel@tonic-gate */ 12690Sstevel@tonic-gate mutex_enter(&as->a_contents); 12700Sstevel@tonic-gate if (as->a_callbacks && 12715084Sjohnlev (cb = as_find_callback(as, AS_SETPROT_EVENT, 12725084Sjohnlev seg->s_base, seg->s_size))) { 12730Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 12740Sstevel@tonic-gate as_execute_callback(as, cb, AS_SETPROT_EVENT); 12756695Saguzovsk } else if (!AS_ISNOUNMAPWAIT(as)) { 12760Sstevel@tonic-gate if (AS_ISUNMAPWAIT(as) == 0) 12770Sstevel@tonic-gate cv_broadcast(&as->a_cv); 12780Sstevel@tonic-gate AS_SETUNMAPWAIT(as); 12790Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 12800Sstevel@tonic-gate while (AS_ISUNMAPWAIT(as)) 12810Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 12826695Saguzovsk } else { 12836695Saguzovsk /* 12846695Saguzovsk * We may have raced with 12856695Saguzovsk * segvn_reclaim()/segspt_reclaim(). In this 12866695Saguzovsk * case clean nounmapwait flag and retry since 12876695Saguzovsk * softlockcnt in this segment may be already 12886695Saguzovsk * 0. We don't drop as writer lock so our 12896695Saguzovsk * number of retries without sleeping should 12906695Saguzovsk * be very small. See segvn_reclaim() for 12916695Saguzovsk * more comments. 12926695Saguzovsk */ 12936695Saguzovsk AS_CLRNOUNMAPWAIT(as); 12946695Saguzovsk mutex_exit(&as->a_contents); 12956695Saguzovsk goto retry; 12960Sstevel@tonic-gate } 12970Sstevel@tonic-gate mutex_exit(&as->a_contents); 12980Sstevel@tonic-gate goto setprot_top; 12990Sstevel@tonic-gate } else if (error != 0) 13000Sstevel@tonic-gate break; 13010Sstevel@tonic-gate } 13020Sstevel@tonic-gate if (error != 0) { 13030Sstevel@tonic-gate as_setwatch(as); 13040Sstevel@tonic-gate } else { 13050Sstevel@tonic-gate as_setwatchprot(as, saveraddr, saversize, prot); 13060Sstevel@tonic-gate } 13070Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 13080Sstevel@tonic-gate return (error); 13090Sstevel@tonic-gate } 13100Sstevel@tonic-gate 13110Sstevel@tonic-gate /* 13120Sstevel@tonic-gate * Check to make sure that the interval [addr, addr + size) 13130Sstevel@tonic-gate * in address space `as' has at least the specified protection. 13140Sstevel@tonic-gate * It is ok for the range to cross over several segments, as long 13150Sstevel@tonic-gate * as they are contiguous. 13160Sstevel@tonic-gate */ 13170Sstevel@tonic-gate int 13180Sstevel@tonic-gate as_checkprot(struct as *as, caddr_t addr, size_t size, uint_t prot) 13190Sstevel@tonic-gate { 13200Sstevel@tonic-gate struct seg *seg; 13210Sstevel@tonic-gate size_t ssize; 13220Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 13230Sstevel@tonic-gate size_t rsize; /* rounded up size */ 13240Sstevel@tonic-gate int error = 0; 13250Sstevel@tonic-gate 13260Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 13270Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 13285084Sjohnlev (size_t)raddr; 13290Sstevel@tonic-gate 13300Sstevel@tonic-gate if (raddr + rsize < raddr) /* check for wraparound */ 13310Sstevel@tonic-gate return (ENOMEM); 13320Sstevel@tonic-gate 13330Sstevel@tonic-gate /* 13340Sstevel@tonic-gate * This is ugly as sin... 13350Sstevel@tonic-gate * Normally, we only acquire the address space readers lock. 13360Sstevel@tonic-gate * However, if the address space has watchpoints present, 13370Sstevel@tonic-gate * we must acquire the writer lock on the address space for 13380Sstevel@tonic-gate * the benefit of as_clearwatchprot() and as_setwatchprot(). 13390Sstevel@tonic-gate */ 13400Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) != 0) 13410Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 13420Sstevel@tonic-gate else 13430Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 13440Sstevel@tonic-gate as_clearwatchprot(as, raddr, rsize); 13450Sstevel@tonic-gate seg = as_segat(as, raddr); 13460Sstevel@tonic-gate if (seg == NULL) { 13470Sstevel@tonic-gate as_setwatch(as); 13480Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 13490Sstevel@tonic-gate return (ENOMEM); 13500Sstevel@tonic-gate } 13510Sstevel@tonic-gate 13520Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 13530Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 13540Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 13550Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 13560Sstevel@tonic-gate error = ENOMEM; 13570Sstevel@tonic-gate break; 13580Sstevel@tonic-gate } 13590Sstevel@tonic-gate } 13600Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 13610Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 13620Sstevel@tonic-gate else 13630Sstevel@tonic-gate ssize = rsize; 13640Sstevel@tonic-gate 13650Sstevel@tonic-gate error = SEGOP_CHECKPROT(seg, raddr, ssize, prot); 13660Sstevel@tonic-gate if (error != 0) 13670Sstevel@tonic-gate break; 13680Sstevel@tonic-gate } 13690Sstevel@tonic-gate as_setwatch(as); 13700Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 13710Sstevel@tonic-gate return (error); 13720Sstevel@tonic-gate } 13730Sstevel@tonic-gate 13740Sstevel@tonic-gate int 13750Sstevel@tonic-gate as_unmap(struct as *as, caddr_t addr, size_t size) 13760Sstevel@tonic-gate { 13770Sstevel@tonic-gate struct seg *seg, *seg_next; 13780Sstevel@tonic-gate struct as_callback *cb; 13790Sstevel@tonic-gate caddr_t raddr, eaddr; 13800Sstevel@tonic-gate size_t ssize; 13810Sstevel@tonic-gate int err; 13820Sstevel@tonic-gate 13830Sstevel@tonic-gate top: 13840Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 13850Sstevel@tonic-gate eaddr = (caddr_t)(((uintptr_t)(addr + size) + PAGEOFFSET) & 13860Sstevel@tonic-gate (uintptr_t)PAGEMASK); 13870Sstevel@tonic-gate 13880Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 13890Sstevel@tonic-gate 13900Sstevel@tonic-gate as->a_updatedir = 1; /* inform /proc */ 13910Sstevel@tonic-gate gethrestime(&as->a_updatetime); 13920Sstevel@tonic-gate 13930Sstevel@tonic-gate /* 13940Sstevel@tonic-gate * Use as_findseg to find the first segment in the range, then 13950Sstevel@tonic-gate * step through the segments in order, following s_next. 13960Sstevel@tonic-gate */ 13970Sstevel@tonic-gate as_clearwatchprot(as, raddr, eaddr - raddr); 13980Sstevel@tonic-gate 13990Sstevel@tonic-gate for (seg = as_findseg(as, raddr, 0); seg != NULL; seg = seg_next) { 14000Sstevel@tonic-gate if (eaddr <= seg->s_base) 14010Sstevel@tonic-gate break; /* eaddr was in a gap; all done */ 14020Sstevel@tonic-gate 14030Sstevel@tonic-gate /* this is implied by the test above */ 14040Sstevel@tonic-gate ASSERT(raddr < eaddr); 14050Sstevel@tonic-gate 14060Sstevel@tonic-gate if (raddr < seg->s_base) 14070Sstevel@tonic-gate raddr = seg->s_base; /* raddr was in a gap */ 14080Sstevel@tonic-gate 14090Sstevel@tonic-gate if (eaddr > (seg->s_base + seg->s_size)) 14100Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 14110Sstevel@tonic-gate else 14120Sstevel@tonic-gate ssize = eaddr - raddr; 14130Sstevel@tonic-gate 14140Sstevel@tonic-gate /* 14150Sstevel@tonic-gate * Save next segment pointer since seg can be 14160Sstevel@tonic-gate * destroyed during the segment unmap operation. 14170Sstevel@tonic-gate */ 14180Sstevel@tonic-gate seg_next = AS_SEGNEXT(as, seg); 14190Sstevel@tonic-gate 14206695Saguzovsk retry: 14210Sstevel@tonic-gate err = SEGOP_UNMAP(seg, raddr, ssize); 14220Sstevel@tonic-gate if (err == EAGAIN) { 14230Sstevel@tonic-gate /* 14240Sstevel@tonic-gate * Memory is currently locked. It must be unlocked 14250Sstevel@tonic-gate * before this operation can succeed through a retry. 14260Sstevel@tonic-gate * The possible reasons for locked memory and 14270Sstevel@tonic-gate * corresponding strategies for unlocking are: 14280Sstevel@tonic-gate * (1) Normal I/O 14290Sstevel@tonic-gate * wait for a signal that the I/O operation 14300Sstevel@tonic-gate * has completed and the memory is unlocked. 14310Sstevel@tonic-gate * (2) Asynchronous I/O 14320Sstevel@tonic-gate * The aio subsystem does not unlock pages when 14330Sstevel@tonic-gate * the I/O is completed. Those pages are unlocked 14340Sstevel@tonic-gate * when the application calls aiowait/aioerror. 14350Sstevel@tonic-gate * So, to prevent blocking forever, cv_broadcast() 14360Sstevel@tonic-gate * is done to wake up aio_cleanup_thread. 14370Sstevel@tonic-gate * Subsequently, segvn_reclaim will be called, and 14380Sstevel@tonic-gate * that will do AS_CLRUNMAPWAIT() and wake us up. 14390Sstevel@tonic-gate * (3) Long term page locking: 14400Sstevel@tonic-gate * Drivers intending to have pages locked for a 14410Sstevel@tonic-gate * period considerably longer than for normal I/O 14420Sstevel@tonic-gate * (essentially forever) may have registered for a 14430Sstevel@tonic-gate * callback so they may unlock these pages on 14440Sstevel@tonic-gate * request. This is needed to allow this operation 14450Sstevel@tonic-gate * to succeed. Each entry on the callback list is 14460Sstevel@tonic-gate * examined. If the event or address range pertains 14470Sstevel@tonic-gate * the callback is invoked (unless it already is in 14480Sstevel@tonic-gate * progress). The a_contents lock must be dropped 14490Sstevel@tonic-gate * before the callback, so only one callback can 14500Sstevel@tonic-gate * be done at a time. Go to the top and do more 14510Sstevel@tonic-gate * until zero is returned. If zero is returned, 14520Sstevel@tonic-gate * either there were no callbacks for this event 14530Sstevel@tonic-gate * or they were already in progress. 14540Sstevel@tonic-gate */ 14550Sstevel@tonic-gate mutex_enter(&as->a_contents); 14560Sstevel@tonic-gate if (as->a_callbacks && 14575084Sjohnlev (cb = as_find_callback(as, AS_UNMAP_EVENT, 14585084Sjohnlev seg->s_base, seg->s_size))) { 14590Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 14600Sstevel@tonic-gate as_execute_callback(as, cb, AS_UNMAP_EVENT); 14616695Saguzovsk } else if (!AS_ISNOUNMAPWAIT(as)) { 14620Sstevel@tonic-gate if (AS_ISUNMAPWAIT(as) == 0) 14630Sstevel@tonic-gate cv_broadcast(&as->a_cv); 14640Sstevel@tonic-gate AS_SETUNMAPWAIT(as); 14650Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 14660Sstevel@tonic-gate while (AS_ISUNMAPWAIT(as)) 14670Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 14686695Saguzovsk } else { 14696695Saguzovsk /* 14706695Saguzovsk * We may have raced with 14716695Saguzovsk * segvn_reclaim()/segspt_reclaim(). In this 14726695Saguzovsk * case clean nounmapwait flag and retry since 14736695Saguzovsk * softlockcnt in this segment may be already 14746695Saguzovsk * 0. We don't drop as writer lock so our 14756695Saguzovsk * number of retries without sleeping should 14766695Saguzovsk * be very small. See segvn_reclaim() for 14776695Saguzovsk * more comments. 14786695Saguzovsk */ 14796695Saguzovsk AS_CLRNOUNMAPWAIT(as); 14806695Saguzovsk mutex_exit(&as->a_contents); 14816695Saguzovsk goto retry; 14820Sstevel@tonic-gate } 14830Sstevel@tonic-gate mutex_exit(&as->a_contents); 14840Sstevel@tonic-gate goto top; 14850Sstevel@tonic-gate } else if (err == IE_RETRY) { 14860Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 14870Sstevel@tonic-gate goto top; 14880Sstevel@tonic-gate } else if (err) { 14890Sstevel@tonic-gate as_setwatch(as); 14900Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 14910Sstevel@tonic-gate return (-1); 14920Sstevel@tonic-gate } 14930Sstevel@tonic-gate 14940Sstevel@tonic-gate as->a_size -= ssize; 14950Sstevel@tonic-gate raddr += ssize; 14960Sstevel@tonic-gate } 14970Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 14980Sstevel@tonic-gate return (0); 14990Sstevel@tonic-gate } 15000Sstevel@tonic-gate 15010Sstevel@tonic-gate static int 15022414Saguzovsk as_map_segvn_segs(struct as *as, caddr_t addr, size_t size, uint_t szcvec, 15030Sstevel@tonic-gate int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated) 15040Sstevel@tonic-gate { 15050Sstevel@tonic-gate uint_t szc; 15060Sstevel@tonic-gate uint_t nszc; 15070Sstevel@tonic-gate int error; 15080Sstevel@tonic-gate caddr_t a; 15090Sstevel@tonic-gate caddr_t eaddr; 15100Sstevel@tonic-gate size_t segsize; 15110Sstevel@tonic-gate struct seg *seg; 15122414Saguzovsk size_t pgsz; 15132414Saguzovsk int do_off = (vn_a->vp != NULL || vn_a->amp != NULL); 15140Sstevel@tonic-gate uint_t save_szcvec; 15152414Saguzovsk 15162414Saguzovsk ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 15172414Saguzovsk ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 15182414Saguzovsk ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 15192414Saguzovsk ASSERT(vn_a->vp == NULL || vn_a->amp == NULL); 15202414Saguzovsk if (!do_off) { 15212414Saguzovsk vn_a->offset = 0; 15222414Saguzovsk } 15232414Saguzovsk 15242414Saguzovsk if (szcvec <= 1) { 15252414Saguzovsk seg = seg_alloc(as, addr, size); 15262414Saguzovsk if (seg == NULL) { 15272414Saguzovsk return (ENOMEM); 15282414Saguzovsk } 15292414Saguzovsk vn_a->szc = 0; 15302414Saguzovsk error = (*crfp)(seg, vn_a); 15312414Saguzovsk if (error != 0) { 15322414Saguzovsk seg_free(seg); 15334225Ssusans } else { 15344225Ssusans as->a_size += size; 15352414Saguzovsk } 15362414Saguzovsk return (error); 15372414Saguzovsk } 15382414Saguzovsk 15392414Saguzovsk eaddr = addr + size; 15402414Saguzovsk save_szcvec = szcvec; 15412414Saguzovsk szcvec >>= 1; 15422414Saguzovsk szc = 0; 15432414Saguzovsk nszc = 0; 15442414Saguzovsk while (szcvec) { 15452414Saguzovsk if ((szcvec & 0x1) == 0) { 15462414Saguzovsk nszc++; 15472414Saguzovsk szcvec >>= 1; 15482414Saguzovsk continue; 15492414Saguzovsk } 15502414Saguzovsk nszc++; 15512414Saguzovsk pgsz = page_get_pagesize(nszc); 15522414Saguzovsk a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz); 15532414Saguzovsk if (a != addr) { 15542414Saguzovsk ASSERT(a < eaddr); 15552414Saguzovsk segsize = a - addr; 15562414Saguzovsk seg = seg_alloc(as, addr, segsize); 15572414Saguzovsk if (seg == NULL) { 15582414Saguzovsk return (ENOMEM); 15592414Saguzovsk } 15602414Saguzovsk vn_a->szc = szc; 15612414Saguzovsk error = (*crfp)(seg, vn_a); 15622414Saguzovsk if (error != 0) { 15632414Saguzovsk seg_free(seg); 15642414Saguzovsk return (error); 15652414Saguzovsk } 15664225Ssusans as->a_size += segsize; 15672414Saguzovsk *segcreated = 1; 15682414Saguzovsk if (do_off) { 15692414Saguzovsk vn_a->offset += segsize; 15702414Saguzovsk } 15712414Saguzovsk addr = a; 15722414Saguzovsk } 15732414Saguzovsk szc = nszc; 15742414Saguzovsk szcvec >>= 1; 15752414Saguzovsk } 15762414Saguzovsk 15772414Saguzovsk ASSERT(addr < eaddr); 15782414Saguzovsk szcvec = save_szcvec | 1; /* add 8K pages */ 15792414Saguzovsk while (szcvec) { 15802414Saguzovsk a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz); 15812414Saguzovsk ASSERT(a >= addr); 15822414Saguzovsk if (a != addr) { 15832414Saguzovsk segsize = a - addr; 15842414Saguzovsk seg = seg_alloc(as, addr, segsize); 15852414Saguzovsk if (seg == NULL) { 15862414Saguzovsk return (ENOMEM); 15872414Saguzovsk } 15882414Saguzovsk vn_a->szc = szc; 15892414Saguzovsk error = (*crfp)(seg, vn_a); 15902414Saguzovsk if (error != 0) { 15912414Saguzovsk seg_free(seg); 15922414Saguzovsk return (error); 15932414Saguzovsk } 15944225Ssusans as->a_size += segsize; 15952414Saguzovsk *segcreated = 1; 15962414Saguzovsk if (do_off) { 15972414Saguzovsk vn_a->offset += segsize; 15982414Saguzovsk } 15992414Saguzovsk addr = a; 16002414Saguzovsk } 16012414Saguzovsk szcvec &= ~(1 << szc); 16022414Saguzovsk if (szcvec) { 16032414Saguzovsk szc = highbit(szcvec) - 1; 16042414Saguzovsk pgsz = page_get_pagesize(szc); 16052414Saguzovsk } 16062414Saguzovsk } 16072414Saguzovsk ASSERT(addr == eaddr); 16082414Saguzovsk 16092414Saguzovsk return (0); 16102414Saguzovsk } 16112414Saguzovsk 16122414Saguzovsk static int 16132414Saguzovsk as_map_vnsegs(struct as *as, caddr_t addr, size_t size, 16142414Saguzovsk int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated) 16152414Saguzovsk { 16162991Ssusans uint_t mapflags = vn_a->flags & (MAP_TEXT | MAP_INITDATA); 16172991Ssusans int type = (vn_a->type == MAP_SHARED) ? MAPPGSZC_SHM : MAPPGSZC_PRIVM; 16182991Ssusans uint_t szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, mapflags, 16192991Ssusans type, 0); 16202414Saguzovsk int error; 16212414Saguzovsk struct seg *seg; 16220Sstevel@tonic-gate struct vattr va; 16230Sstevel@tonic-gate u_offset_t eoff; 16240Sstevel@tonic-gate size_t save_size = 0; 16254426Saguzovsk extern size_t textrepl_size_thresh; 16260Sstevel@tonic-gate 16270Sstevel@tonic-gate ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 16280Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 16290Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 16300Sstevel@tonic-gate ASSERT(vn_a->vp != NULL); 16310Sstevel@tonic-gate ASSERT(vn_a->amp == NULL); 16320Sstevel@tonic-gate 16330Sstevel@tonic-gate again: 16340Sstevel@tonic-gate if (szcvec <= 1) { 16350Sstevel@tonic-gate seg = seg_alloc(as, addr, size); 16360Sstevel@tonic-gate if (seg == NULL) { 16370Sstevel@tonic-gate return (ENOMEM); 16380Sstevel@tonic-gate } 16390Sstevel@tonic-gate vn_a->szc = 0; 16400Sstevel@tonic-gate error = (*crfp)(seg, vn_a); 16410Sstevel@tonic-gate if (error != 0) { 16420Sstevel@tonic-gate seg_free(seg); 16434225Ssusans } else { 16444225Ssusans as->a_size += size; 16450Sstevel@tonic-gate } 16460Sstevel@tonic-gate return (error); 16470Sstevel@tonic-gate } 16480Sstevel@tonic-gate 16490Sstevel@tonic-gate va.va_mask = AT_SIZE; 16505331Samw if (VOP_GETATTR(vn_a->vp, &va, ATTR_HINT, vn_a->cred, NULL) != 0) { 16510Sstevel@tonic-gate szcvec = 0; 16520Sstevel@tonic-gate goto again; 16530Sstevel@tonic-gate } 16540Sstevel@tonic-gate eoff = vn_a->offset & PAGEMASK; 16550Sstevel@tonic-gate if (eoff >= va.va_size) { 16560Sstevel@tonic-gate szcvec = 0; 16570Sstevel@tonic-gate goto again; 16580Sstevel@tonic-gate } 16590Sstevel@tonic-gate eoff += size; 16600Sstevel@tonic-gate if (btopr(va.va_size) < btopr(eoff)) { 16610Sstevel@tonic-gate save_size = size; 16620Sstevel@tonic-gate size = va.va_size - (vn_a->offset & PAGEMASK); 16630Sstevel@tonic-gate size = P2ROUNDUP_TYPED(size, PAGESIZE, size_t); 16642991Ssusans szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, mapflags, 16652991Ssusans type, 0); 16660Sstevel@tonic-gate if (szcvec <= 1) { 16670Sstevel@tonic-gate size = save_size; 16680Sstevel@tonic-gate goto again; 16690Sstevel@tonic-gate } 16700Sstevel@tonic-gate } 16710Sstevel@tonic-gate 16724426Saguzovsk if (size > textrepl_size_thresh) { 16734426Saguzovsk vn_a->flags |= _MAP_TEXTREPL; 16744426Saguzovsk } 16752414Saguzovsk error = as_map_segvn_segs(as, addr, size, szcvec, crfp, vn_a, 16762414Saguzovsk segcreated); 16772414Saguzovsk if (error != 0) { 16782414Saguzovsk return (error); 16790Sstevel@tonic-gate } 16800Sstevel@tonic-gate if (save_size) { 16812414Saguzovsk addr += size; 16820Sstevel@tonic-gate size = save_size - size; 16832414Saguzovsk szcvec = 0; 16840Sstevel@tonic-gate goto again; 16850Sstevel@tonic-gate } 16862414Saguzovsk return (0); 16872414Saguzovsk } 16880Sstevel@tonic-gate 16892991Ssusans /* 16902991Ssusans * as_map_ansegs: shared or private anonymous memory. Note that the flags 16912991Ssusans * passed to map_pgszvec cannot be MAP_INITDATA, for anon. 16922991Ssusans */ 16932414Saguzovsk static int 16942991Ssusans as_map_ansegs(struct as *as, caddr_t addr, size_t size, 16952414Saguzovsk int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated) 16962414Saguzovsk { 16972991Ssusans uint_t szcvec; 16982991Ssusans uchar_t type; 16992991Ssusans 17002991Ssusans ASSERT(vn_a->type == MAP_SHARED || vn_a->type == MAP_PRIVATE); 17012991Ssusans if (vn_a->type == MAP_SHARED) { 17022991Ssusans type = MAPPGSZC_SHM; 17032991Ssusans } else if (vn_a->type == MAP_PRIVATE) { 17042991Ssusans if (vn_a->szc == AS_MAP_HEAP) { 17052991Ssusans type = MAPPGSZC_HEAP; 17062991Ssusans } else if (vn_a->szc == AS_MAP_STACK) { 17072991Ssusans type = MAPPGSZC_STACK; 17082991Ssusans } else { 17092991Ssusans type = MAPPGSZC_PRIVM; 17102991Ssusans } 17112991Ssusans } 17122991Ssusans szcvec = map_pgszcvec(addr, size, vn_a->amp == NULL ? 17132991Ssusans (uintptr_t)addr : (uintptr_t)P2ROUNDUP(vn_a->offset, PAGESIZE), 17142991Ssusans (vn_a->flags & MAP_TEXT), type, 0); 17152414Saguzovsk ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 17162414Saguzovsk ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 17172414Saguzovsk ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 17182414Saguzovsk ASSERT(vn_a->vp == NULL); 17192414Saguzovsk 17202414Saguzovsk return (as_map_segvn_segs(as, addr, size, szcvec, 17212414Saguzovsk crfp, vn_a, segcreated)); 17220Sstevel@tonic-gate } 17230Sstevel@tonic-gate 17240Sstevel@tonic-gate int 17250Sstevel@tonic-gate as_map(struct as *as, caddr_t addr, size_t size, int (*crfp)(), void *argsp) 17260Sstevel@tonic-gate { 17271899Svsakar AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 17281899Svsakar return (as_map_locked(as, addr, size, crfp, argsp)); 17291899Svsakar } 17301899Svsakar 17311899Svsakar int 17321899Svsakar as_map_locked(struct as *as, caddr_t addr, size_t size, int (*crfp)(), 17331899Svsakar void *argsp) 17341899Svsakar { 17350Sstevel@tonic-gate struct seg *seg = NULL; 17360Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 17370Sstevel@tonic-gate size_t rsize; /* rounded up size */ 17380Sstevel@tonic-gate int error; 17392991Ssusans int unmap = 0; 17400Sstevel@tonic-gate struct proc *p = curproc; 17413183Ssusans struct segvn_crargs crargs; 17420Sstevel@tonic-gate 17430Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 17440Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 17455084Sjohnlev (size_t)raddr; 17460Sstevel@tonic-gate 17470Sstevel@tonic-gate /* 17480Sstevel@tonic-gate * check for wrap around 17490Sstevel@tonic-gate */ 17500Sstevel@tonic-gate if ((raddr + rsize < raddr) || (as->a_size > (ULONG_MAX - size))) { 17510Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 17520Sstevel@tonic-gate return (ENOMEM); 17530Sstevel@tonic-gate } 17540Sstevel@tonic-gate 17550Sstevel@tonic-gate as->a_updatedir = 1; /* inform /proc */ 17560Sstevel@tonic-gate gethrestime(&as->a_updatetime); 17570Sstevel@tonic-gate 17580Sstevel@tonic-gate if (as != &kas && as->a_size + rsize > (size_t)p->p_vmem_ctl) { 17590Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 17600Sstevel@tonic-gate 17610Sstevel@tonic-gate (void) rctl_action(rctlproc_legacy[RLIMIT_VMEM], p->p_rctls, p, 17620Sstevel@tonic-gate RCA_UNSAFE_ALL); 17630Sstevel@tonic-gate 17640Sstevel@tonic-gate return (ENOMEM); 17650Sstevel@tonic-gate } 17660Sstevel@tonic-gate 17672991Ssusans if (AS_MAP_CHECK_VNODE_LPOOB(crfp, argsp)) { 17683183Ssusans crargs = *(struct segvn_crargs *)argsp; 17693183Ssusans error = as_map_vnsegs(as, raddr, rsize, crfp, &crargs, &unmap); 17702991Ssusans if (error != 0) { 17712991Ssusans AS_LOCK_EXIT(as, &as->a_lock); 17722991Ssusans if (unmap) { 17732991Ssusans (void) as_unmap(as, addr, size); 17742991Ssusans } 17752991Ssusans return (error); 17762414Saguzovsk } 17772991Ssusans } else if (AS_MAP_CHECK_ANON_LPOOB(crfp, argsp)) { 17783183Ssusans crargs = *(struct segvn_crargs *)argsp; 17793183Ssusans error = as_map_ansegs(as, raddr, rsize, crfp, &crargs, &unmap); 17800Sstevel@tonic-gate if (error != 0) { 17810Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 17820Sstevel@tonic-gate if (unmap) { 17830Sstevel@tonic-gate (void) as_unmap(as, addr, size); 17840Sstevel@tonic-gate } 17850Sstevel@tonic-gate return (error); 17860Sstevel@tonic-gate } 17870Sstevel@tonic-gate } else { 17880Sstevel@tonic-gate seg = seg_alloc(as, addr, size); 17890Sstevel@tonic-gate if (seg == NULL) { 17900Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 17910Sstevel@tonic-gate return (ENOMEM); 17920Sstevel@tonic-gate } 17930Sstevel@tonic-gate 17940Sstevel@tonic-gate error = (*crfp)(seg, argsp); 17950Sstevel@tonic-gate if (error != 0) { 17960Sstevel@tonic-gate seg_free(seg); 17970Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 17980Sstevel@tonic-gate return (error); 17990Sstevel@tonic-gate } 18004225Ssusans /* 18014225Ssusans * Add size now so as_unmap will work if as_ctl fails. 18024225Ssusans */ 18034225Ssusans as->a_size += rsize; 18040Sstevel@tonic-gate } 18050Sstevel@tonic-gate 18060Sstevel@tonic-gate as_setwatch(as); 18070Sstevel@tonic-gate 18080Sstevel@tonic-gate /* 18090Sstevel@tonic-gate * If the address space is locked, 18100Sstevel@tonic-gate * establish memory locks for the new segment. 18110Sstevel@tonic-gate */ 18120Sstevel@tonic-gate mutex_enter(&as->a_contents); 18130Sstevel@tonic-gate if (AS_ISPGLCK(as)) { 18140Sstevel@tonic-gate mutex_exit(&as->a_contents); 18150Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 18160Sstevel@tonic-gate error = as_ctl(as, addr, size, MC_LOCK, 0, 0, NULL, 0); 18170Sstevel@tonic-gate if (error != 0) 18180Sstevel@tonic-gate (void) as_unmap(as, addr, size); 18190Sstevel@tonic-gate } else { 18200Sstevel@tonic-gate mutex_exit(&as->a_contents); 18210Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 18220Sstevel@tonic-gate } 18230Sstevel@tonic-gate return (error); 18240Sstevel@tonic-gate } 18250Sstevel@tonic-gate 18260Sstevel@tonic-gate 18270Sstevel@tonic-gate /* 18280Sstevel@tonic-gate * Delete all segments in the address space marked with S_PURGE. 18290Sstevel@tonic-gate * This is currently used for Sparc V9 nofault ASI segments (seg_nf.c). 18300Sstevel@tonic-gate * These segments are deleted as a first step before calls to as_gap(), so 18310Sstevel@tonic-gate * that they don't affect mmap() or shmat(). 18320Sstevel@tonic-gate */ 18330Sstevel@tonic-gate void 18340Sstevel@tonic-gate as_purge(struct as *as) 18350Sstevel@tonic-gate { 18360Sstevel@tonic-gate struct seg *seg; 18370Sstevel@tonic-gate struct seg *next_seg; 18380Sstevel@tonic-gate 18390Sstevel@tonic-gate /* 18400Sstevel@tonic-gate * the setting of NEEDSPURGE is protect by as_rangelock(), so 18410Sstevel@tonic-gate * no need to grab a_contents mutex for this check 18420Sstevel@tonic-gate */ 18430Sstevel@tonic-gate if ((as->a_flags & AS_NEEDSPURGE) == 0) 18440Sstevel@tonic-gate return; 18450Sstevel@tonic-gate 18460Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 18470Sstevel@tonic-gate next_seg = NULL; 18480Sstevel@tonic-gate seg = AS_SEGFIRST(as); 18490Sstevel@tonic-gate while (seg != NULL) { 18500Sstevel@tonic-gate next_seg = AS_SEGNEXT(as, seg); 18510Sstevel@tonic-gate if (seg->s_flags & S_PURGE) 18520Sstevel@tonic-gate SEGOP_UNMAP(seg, seg->s_base, seg->s_size); 18530Sstevel@tonic-gate seg = next_seg; 18540Sstevel@tonic-gate } 18550Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 18560Sstevel@tonic-gate 18570Sstevel@tonic-gate mutex_enter(&as->a_contents); 18580Sstevel@tonic-gate as->a_flags &= ~AS_NEEDSPURGE; 18590Sstevel@tonic-gate mutex_exit(&as->a_contents); 18600Sstevel@tonic-gate } 18610Sstevel@tonic-gate 18620Sstevel@tonic-gate /* 18635668Smec * Find a hole within [*basep, *basep + *lenp), which contains a mappable 18645668Smec * range of addresses at least "minlen" long, where the base of the range is 18655668Smec * at "off" phase from an "align" boundary and there is space for a 18665668Smec * "redzone"-sized redzone on eithe rside of the range. Thus, 18675668Smec * if align was 4M and off was 16k, the user wants a hole which will start 18685668Smec * 16k into a 4M page. 18690Sstevel@tonic-gate * 18700Sstevel@tonic-gate * If flags specifies AH_HI, the hole will have the highest possible address 18710Sstevel@tonic-gate * in the range. We use the as->a_lastgap field to figure out where to 18720Sstevel@tonic-gate * start looking for a gap. 18730Sstevel@tonic-gate * 18740Sstevel@tonic-gate * Otherwise, the gap will have the lowest possible address. 18750Sstevel@tonic-gate * 18760Sstevel@tonic-gate * If flags specifies AH_CONTAIN, the hole will contain the address addr. 18770Sstevel@tonic-gate * 18785668Smec * If an adequate hole is found, *basep and *lenp are set to reflect the part of 18795668Smec * the hole that is within range, and 0 is returned. On failure, -1 is returned. 18800Sstevel@tonic-gate * 18810Sstevel@tonic-gate * NOTE: This routine is not correct when base+len overflows caddr_t. 18820Sstevel@tonic-gate */ 18830Sstevel@tonic-gate int 18845668Smec as_gap_aligned(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp, 18855668Smec uint_t flags, caddr_t addr, size_t align, size_t redzone, size_t off) 18860Sstevel@tonic-gate { 18870Sstevel@tonic-gate caddr_t lobound = *basep; 18880Sstevel@tonic-gate caddr_t hibound = lobound + *lenp; 18890Sstevel@tonic-gate struct seg *lseg, *hseg; 18900Sstevel@tonic-gate caddr_t lo, hi; 18910Sstevel@tonic-gate int forward; 18920Sstevel@tonic-gate caddr_t save_base; 18930Sstevel@tonic-gate size_t save_len; 18946978Smec size_t save_minlen; 18956978Smec size_t save_redzone; 18966978Smec int fast_path = 1; 18970Sstevel@tonic-gate 18980Sstevel@tonic-gate save_base = *basep; 18990Sstevel@tonic-gate save_len = *lenp; 19006978Smec save_minlen = minlen; 19016978Smec save_redzone = redzone; 19026978Smec 19036978Smec /* 19046978Smec * For the first pass/fast_path, just add align and redzone into 19056978Smec * minlen since if we get an allocation, we can guarantee that it 19066978Smec * will fit the alignment and redzone requested. 19076978Smec * This increases the chance that hibound will be adjusted to 19086978Smec * a_lastgap->s_base which will likely allow us to find an 19096978Smec * acceptable hole in the address space quicker. 19106978Smec * If we can't find a hole with this fast_path, then we look for 19116978Smec * smaller holes in which the alignment and offset may allow 19126978Smec * the allocation to fit. 19136978Smec */ 19146978Smec minlen += align; 19156978Smec minlen += 2 * redzone; 19166978Smec redzone = 0; 19176978Smec 19180Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 19190Sstevel@tonic-gate if (AS_SEGFIRST(as) == NULL) { 19205668Smec if (valid_va_range_aligned(basep, lenp, minlen, flags & AH_DIR, 19215668Smec align, redzone, off)) { 19220Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 19230Sstevel@tonic-gate return (0); 19240Sstevel@tonic-gate } else { 19250Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 19260Sstevel@tonic-gate *basep = save_base; 19270Sstevel@tonic-gate *lenp = save_len; 19280Sstevel@tonic-gate return (-1); 19290Sstevel@tonic-gate } 19300Sstevel@tonic-gate } 19310Sstevel@tonic-gate 19326978Smec retry: 19330Sstevel@tonic-gate /* 19340Sstevel@tonic-gate * Set up to iterate over all the inter-segment holes in the given 19350Sstevel@tonic-gate * direction. lseg is NULL for the lowest-addressed hole and hseg is 19360Sstevel@tonic-gate * NULL for the highest-addressed hole. If moving backwards, we reset 19370Sstevel@tonic-gate * sseg to denote the highest-addressed segment. 19380Sstevel@tonic-gate */ 19390Sstevel@tonic-gate forward = (flags & AH_DIR) == AH_LO; 19400Sstevel@tonic-gate if (forward) { 19410Sstevel@tonic-gate hseg = as_findseg(as, lobound, 1); 19420Sstevel@tonic-gate lseg = AS_SEGPREV(as, hseg); 19430Sstevel@tonic-gate } else { 19440Sstevel@tonic-gate 19450Sstevel@tonic-gate /* 19460Sstevel@tonic-gate * If allocating at least as much as the last allocation, 19470Sstevel@tonic-gate * use a_lastgap's base as a better estimate of hibound. 19480Sstevel@tonic-gate */ 19490Sstevel@tonic-gate if (as->a_lastgap && 19500Sstevel@tonic-gate minlen >= as->a_lastgap->s_size && 19510Sstevel@tonic-gate hibound >= as->a_lastgap->s_base) 19520Sstevel@tonic-gate hibound = as->a_lastgap->s_base; 19530Sstevel@tonic-gate 19540Sstevel@tonic-gate hseg = as_findseg(as, hibound, 1); 19550Sstevel@tonic-gate if (hseg->s_base + hseg->s_size < hibound) { 19560Sstevel@tonic-gate lseg = hseg; 19570Sstevel@tonic-gate hseg = NULL; 19580Sstevel@tonic-gate } else { 19590Sstevel@tonic-gate lseg = AS_SEGPREV(as, hseg); 19600Sstevel@tonic-gate } 19610Sstevel@tonic-gate } 19620Sstevel@tonic-gate 19630Sstevel@tonic-gate for (;;) { 19640Sstevel@tonic-gate /* 19650Sstevel@tonic-gate * Set lo and hi to the hole's boundaries. (We should really 19660Sstevel@tonic-gate * use MAXADDR in place of hibound in the expression below, 19670Sstevel@tonic-gate * but can't express it easily; using hibound in its place is 19680Sstevel@tonic-gate * harmless.) 19690Sstevel@tonic-gate */ 19700Sstevel@tonic-gate lo = (lseg == NULL) ? 0 : lseg->s_base + lseg->s_size; 19710Sstevel@tonic-gate hi = (hseg == NULL) ? hibound : hseg->s_base; 19720Sstevel@tonic-gate /* 19730Sstevel@tonic-gate * If the iteration has moved past the interval from lobound 19740Sstevel@tonic-gate * to hibound it's pointless to continue. 19750Sstevel@tonic-gate */ 19760Sstevel@tonic-gate if ((forward && lo > hibound) || (!forward && hi < lobound)) 19770Sstevel@tonic-gate break; 19780Sstevel@tonic-gate else if (lo > hibound || hi < lobound) 19790Sstevel@tonic-gate goto cont; 19800Sstevel@tonic-gate /* 19810Sstevel@tonic-gate * Candidate hole lies at least partially within the allowable 19820Sstevel@tonic-gate * range. Restrict it to fall completely within that range, 19830Sstevel@tonic-gate * i.e., to [max(lo, lobound), min(hi, hibound)]. 19840Sstevel@tonic-gate */ 19850Sstevel@tonic-gate if (lo < lobound) 19860Sstevel@tonic-gate lo = lobound; 19870Sstevel@tonic-gate if (hi > hibound) 19880Sstevel@tonic-gate hi = hibound; 19890Sstevel@tonic-gate /* 19900Sstevel@tonic-gate * Verify that the candidate hole is big enough and meets 19916978Smec * hardware constraints. If the hole is too small, no need 19926978Smec * to do the further checks since they will fail. 19930Sstevel@tonic-gate */ 19940Sstevel@tonic-gate *basep = lo; 19950Sstevel@tonic-gate *lenp = hi - lo; 19966978Smec if (*lenp >= minlen && valid_va_range_aligned(basep, lenp, 19976978Smec minlen, forward ? AH_LO : AH_HI, align, redzone, off) && 19980Sstevel@tonic-gate ((flags & AH_CONTAIN) == 0 || 19990Sstevel@tonic-gate (*basep <= addr && *basep + *lenp > addr))) { 20000Sstevel@tonic-gate if (!forward) 20010Sstevel@tonic-gate as->a_lastgap = hseg; 20020Sstevel@tonic-gate if (hseg != NULL) 20030Sstevel@tonic-gate as->a_lastgaphl = hseg; 20040Sstevel@tonic-gate else 20050Sstevel@tonic-gate as->a_lastgaphl = lseg; 20060Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 20070Sstevel@tonic-gate return (0); 20080Sstevel@tonic-gate } 20090Sstevel@tonic-gate cont: 20100Sstevel@tonic-gate /* 20110Sstevel@tonic-gate * Move to the next hole. 20120Sstevel@tonic-gate */ 20130Sstevel@tonic-gate if (forward) { 20140Sstevel@tonic-gate lseg = hseg; 20150Sstevel@tonic-gate if (lseg == NULL) 20160Sstevel@tonic-gate break; 20170Sstevel@tonic-gate hseg = AS_SEGNEXT(as, hseg); 20180Sstevel@tonic-gate } else { 20190Sstevel@tonic-gate hseg = lseg; 20200Sstevel@tonic-gate if (hseg == NULL) 20210Sstevel@tonic-gate break; 20220Sstevel@tonic-gate lseg = AS_SEGPREV(as, lseg); 20230Sstevel@tonic-gate } 20240Sstevel@tonic-gate } 20256978Smec if (fast_path && (align != 0 || save_redzone != 0)) { 20266978Smec fast_path = 0; 20276978Smec minlen = save_minlen; 20286978Smec redzone = save_redzone; 20296978Smec goto retry; 20306978Smec } 20310Sstevel@tonic-gate *basep = save_base; 20320Sstevel@tonic-gate *lenp = save_len; 20330Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 20340Sstevel@tonic-gate return (-1); 20350Sstevel@tonic-gate } 20360Sstevel@tonic-gate 20370Sstevel@tonic-gate /* 20385668Smec * Find a hole of at least size minlen within [*basep, *basep + *lenp). 20395668Smec * 20405668Smec * If flags specifies AH_HI, the hole will have the highest possible address 20415668Smec * in the range. We use the as->a_lastgap field to figure out where to 20425668Smec * start looking for a gap. 20435668Smec * 20445668Smec * Otherwise, the gap will have the lowest possible address. 20455668Smec * 20465668Smec * If flags specifies AH_CONTAIN, the hole will contain the address addr. 20475668Smec * 20485668Smec * If an adequate hole is found, base and len are set to reflect the part of 20495668Smec * the hole that is within range, and 0 is returned, otherwise, 20505668Smec * -1 is returned. 20515668Smec * 20525668Smec * NOTE: This routine is not correct when base+len overflows caddr_t. 20535668Smec */ 20545668Smec int 20555668Smec as_gap(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp, uint_t flags, 20565668Smec caddr_t addr) 20575668Smec { 20585668Smec 20595668Smec return (as_gap_aligned(as, minlen, basep, lenp, flags, addr, 0, 0, 0)); 20605668Smec } 20615668Smec 20625668Smec /* 20630Sstevel@tonic-gate * Return the next range within [base, base + len) that is backed 20640Sstevel@tonic-gate * with "real memory". Skip holes and non-seg_vn segments. 20650Sstevel@tonic-gate * We're lazy and only return one segment at a time. 20660Sstevel@tonic-gate */ 20670Sstevel@tonic-gate int 20680Sstevel@tonic-gate as_memory(struct as *as, caddr_t *basep, size_t *lenp) 20690Sstevel@tonic-gate { 20700Sstevel@tonic-gate extern struct seg_ops segspt_shmops; /* needs a header file */ 20710Sstevel@tonic-gate struct seg *seg; 20720Sstevel@tonic-gate caddr_t addr, eaddr; 20730Sstevel@tonic-gate caddr_t segend; 20740Sstevel@tonic-gate 20750Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 20760Sstevel@tonic-gate 20770Sstevel@tonic-gate addr = *basep; 20780Sstevel@tonic-gate eaddr = addr + *lenp; 20790Sstevel@tonic-gate 20800Sstevel@tonic-gate seg = as_findseg(as, addr, 0); 20810Sstevel@tonic-gate if (seg != NULL) 20820Sstevel@tonic-gate addr = MAX(seg->s_base, addr); 20830Sstevel@tonic-gate 20840Sstevel@tonic-gate for (;;) { 20850Sstevel@tonic-gate if (seg == NULL || addr >= eaddr || eaddr <= seg->s_base) { 20860Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 20870Sstevel@tonic-gate return (EINVAL); 20880Sstevel@tonic-gate } 20890Sstevel@tonic-gate 20900Sstevel@tonic-gate if (seg->s_ops == &segvn_ops) { 20910Sstevel@tonic-gate segend = seg->s_base + seg->s_size; 20920Sstevel@tonic-gate break; 20930Sstevel@tonic-gate } 20940Sstevel@tonic-gate 20950Sstevel@tonic-gate /* 20960Sstevel@tonic-gate * We do ISM by looking into the private data 20970Sstevel@tonic-gate * to determine the real size of the segment. 20980Sstevel@tonic-gate */ 20990Sstevel@tonic-gate if (seg->s_ops == &segspt_shmops) { 21000Sstevel@tonic-gate segend = seg->s_base + spt_realsize(seg); 21010Sstevel@tonic-gate if (addr < segend) 21020Sstevel@tonic-gate break; 21030Sstevel@tonic-gate } 21040Sstevel@tonic-gate 21050Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 21060Sstevel@tonic-gate 21070Sstevel@tonic-gate if (seg != NULL) 21080Sstevel@tonic-gate addr = seg->s_base; 21090Sstevel@tonic-gate } 21100Sstevel@tonic-gate 21110Sstevel@tonic-gate *basep = addr; 21120Sstevel@tonic-gate 21130Sstevel@tonic-gate if (segend > eaddr) 21140Sstevel@tonic-gate *lenp = eaddr - addr; 21150Sstevel@tonic-gate else 21160Sstevel@tonic-gate *lenp = segend - addr; 21170Sstevel@tonic-gate 21180Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 21190Sstevel@tonic-gate return (0); 21200Sstevel@tonic-gate } 21210Sstevel@tonic-gate 21220Sstevel@tonic-gate /* 21230Sstevel@tonic-gate * Swap the pages associated with the address space as out to 21240Sstevel@tonic-gate * secondary storage, returning the number of bytes actually 21250Sstevel@tonic-gate * swapped. 21260Sstevel@tonic-gate * 21270Sstevel@tonic-gate * The value returned is intended to correlate well with the process's 21280Sstevel@tonic-gate * memory requirements. Its usefulness for this purpose depends on 21290Sstevel@tonic-gate * how well the segment-level routines do at returning accurate 21300Sstevel@tonic-gate * information. 21310Sstevel@tonic-gate */ 21320Sstevel@tonic-gate size_t 21330Sstevel@tonic-gate as_swapout(struct as *as) 21340Sstevel@tonic-gate { 21350Sstevel@tonic-gate struct seg *seg; 21360Sstevel@tonic-gate size_t swpcnt = 0; 21370Sstevel@tonic-gate 21380Sstevel@tonic-gate /* 21390Sstevel@tonic-gate * Kernel-only processes have given up their address 21400Sstevel@tonic-gate * spaces. Of course, we shouldn't be attempting to 21410Sstevel@tonic-gate * swap out such processes in the first place... 21420Sstevel@tonic-gate */ 21430Sstevel@tonic-gate if (as == NULL) 21440Sstevel@tonic-gate return (0); 21450Sstevel@tonic-gate 21460Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 21470Sstevel@tonic-gate 21480Sstevel@tonic-gate /* Prevent XHATs from attaching */ 21490Sstevel@tonic-gate mutex_enter(&as->a_contents); 21500Sstevel@tonic-gate AS_SETBUSY(as); 21510Sstevel@tonic-gate mutex_exit(&as->a_contents); 21520Sstevel@tonic-gate 21530Sstevel@tonic-gate 21540Sstevel@tonic-gate /* 21550Sstevel@tonic-gate * Free all mapping resources associated with the address 21560Sstevel@tonic-gate * space. The segment-level swapout routines capitalize 21570Sstevel@tonic-gate * on this unmapping by scavanging pages that have become 21580Sstevel@tonic-gate * unmapped here. 21590Sstevel@tonic-gate */ 21600Sstevel@tonic-gate hat_swapout(as->a_hat); 21610Sstevel@tonic-gate if (as->a_xhat != NULL) 21620Sstevel@tonic-gate xhat_swapout_all(as); 21630Sstevel@tonic-gate 21640Sstevel@tonic-gate mutex_enter(&as->a_contents); 21650Sstevel@tonic-gate AS_CLRBUSY(as); 21660Sstevel@tonic-gate mutex_exit(&as->a_contents); 21670Sstevel@tonic-gate 21680Sstevel@tonic-gate /* 21690Sstevel@tonic-gate * Call the swapout routines of all segments in the address 21700Sstevel@tonic-gate * space to do the actual work, accumulating the amount of 21710Sstevel@tonic-gate * space reclaimed. 21720Sstevel@tonic-gate */ 21730Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) { 21740Sstevel@tonic-gate struct seg_ops *ov = seg->s_ops; 21750Sstevel@tonic-gate 21760Sstevel@tonic-gate /* 21770Sstevel@tonic-gate * We have to check to see if the seg has 21780Sstevel@tonic-gate * an ops vector because the seg may have 21790Sstevel@tonic-gate * been in the middle of being set up when 21800Sstevel@tonic-gate * the process was picked for swapout. 21810Sstevel@tonic-gate */ 21820Sstevel@tonic-gate if ((ov != NULL) && (ov->swapout != NULL)) 21830Sstevel@tonic-gate swpcnt += SEGOP_SWAPOUT(seg); 21840Sstevel@tonic-gate } 21850Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 21860Sstevel@tonic-gate return (swpcnt); 21870Sstevel@tonic-gate } 21880Sstevel@tonic-gate 21890Sstevel@tonic-gate /* 21900Sstevel@tonic-gate * Determine whether data from the mappings in interval [addr, addr + size) 21910Sstevel@tonic-gate * are in the primary memory (core) cache. 21920Sstevel@tonic-gate */ 21930Sstevel@tonic-gate int 21940Sstevel@tonic-gate as_incore(struct as *as, caddr_t addr, 21950Sstevel@tonic-gate size_t size, char *vec, size_t *sizep) 21960Sstevel@tonic-gate { 21970Sstevel@tonic-gate struct seg *seg; 21980Sstevel@tonic-gate size_t ssize; 21990Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 22000Sstevel@tonic-gate size_t rsize; /* rounded up size */ 22010Sstevel@tonic-gate size_t isize; /* iteration size */ 22020Sstevel@tonic-gate int error = 0; /* result, assume success */ 22030Sstevel@tonic-gate 22040Sstevel@tonic-gate *sizep = 0; 22050Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 22060Sstevel@tonic-gate rsize = ((((size_t)addr + size) + PAGEOFFSET) & PAGEMASK) - 22075084Sjohnlev (size_t)raddr; 22080Sstevel@tonic-gate 22090Sstevel@tonic-gate if (raddr + rsize < raddr) /* check for wraparound */ 22100Sstevel@tonic-gate return (ENOMEM); 22110Sstevel@tonic-gate 22120Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 22130Sstevel@tonic-gate seg = as_segat(as, raddr); 22140Sstevel@tonic-gate if (seg == NULL) { 22150Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 22160Sstevel@tonic-gate return (-1); 22170Sstevel@tonic-gate } 22180Sstevel@tonic-gate 22190Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 22200Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 22210Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 22220Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 22230Sstevel@tonic-gate error = -1; 22240Sstevel@tonic-gate break; 22250Sstevel@tonic-gate } 22260Sstevel@tonic-gate } 22270Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 22280Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 22290Sstevel@tonic-gate else 22300Sstevel@tonic-gate ssize = rsize; 22310Sstevel@tonic-gate *sizep += isize = SEGOP_INCORE(seg, raddr, ssize, vec); 22320Sstevel@tonic-gate if (isize != ssize) { 22330Sstevel@tonic-gate error = -1; 22340Sstevel@tonic-gate break; 22350Sstevel@tonic-gate } 22360Sstevel@tonic-gate vec += btopr(ssize); 22370Sstevel@tonic-gate } 22380Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 22390Sstevel@tonic-gate return (error); 22400Sstevel@tonic-gate } 22410Sstevel@tonic-gate 22420Sstevel@tonic-gate static void 22430Sstevel@tonic-gate as_segunlock(struct seg *seg, caddr_t addr, int attr, 22440Sstevel@tonic-gate ulong_t *bitmap, size_t position, size_t npages) 22450Sstevel@tonic-gate { 22460Sstevel@tonic-gate caddr_t range_start; 22470Sstevel@tonic-gate size_t pos1 = position; 22480Sstevel@tonic-gate size_t pos2; 22490Sstevel@tonic-gate size_t size; 22500Sstevel@tonic-gate size_t end_pos = npages + position; 22510Sstevel@tonic-gate 22520Sstevel@tonic-gate while (bt_range(bitmap, &pos1, &pos2, end_pos)) { 22530Sstevel@tonic-gate size = ptob((pos2 - pos1)); 22540Sstevel@tonic-gate range_start = (caddr_t)((uintptr_t)addr + 22555084Sjohnlev ptob(pos1 - position)); 22560Sstevel@tonic-gate 22570Sstevel@tonic-gate (void) SEGOP_LOCKOP(seg, range_start, size, attr, MC_UNLOCK, 22585084Sjohnlev (ulong_t *)NULL, (size_t)NULL); 22590Sstevel@tonic-gate pos1 = pos2; 22600Sstevel@tonic-gate } 22610Sstevel@tonic-gate } 22620Sstevel@tonic-gate 22630Sstevel@tonic-gate static void 22640Sstevel@tonic-gate as_unlockerr(struct as *as, int attr, ulong_t *mlock_map, 22650Sstevel@tonic-gate caddr_t raddr, size_t rsize) 22660Sstevel@tonic-gate { 22670Sstevel@tonic-gate struct seg *seg = as_segat(as, raddr); 22680Sstevel@tonic-gate size_t ssize; 22690Sstevel@tonic-gate 22700Sstevel@tonic-gate while (rsize != 0) { 22710Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) 22720Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 22730Sstevel@tonic-gate 22740Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 22750Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 22760Sstevel@tonic-gate else 22770Sstevel@tonic-gate ssize = rsize; 22780Sstevel@tonic-gate 22790Sstevel@tonic-gate as_segunlock(seg, raddr, attr, mlock_map, 0, btopr(ssize)); 22800Sstevel@tonic-gate 22810Sstevel@tonic-gate rsize -= ssize; 22820Sstevel@tonic-gate raddr += ssize; 22830Sstevel@tonic-gate } 22840Sstevel@tonic-gate } 22850Sstevel@tonic-gate 22860Sstevel@tonic-gate /* 22870Sstevel@tonic-gate * Cache control operations over the interval [addr, addr + size) in 22880Sstevel@tonic-gate * address space "as". 22890Sstevel@tonic-gate */ 22900Sstevel@tonic-gate /*ARGSUSED*/ 22910Sstevel@tonic-gate int 22920Sstevel@tonic-gate as_ctl(struct as *as, caddr_t addr, size_t size, int func, int attr, 22930Sstevel@tonic-gate uintptr_t arg, ulong_t *lock_map, size_t pos) 22940Sstevel@tonic-gate { 22950Sstevel@tonic-gate struct seg *seg; /* working segment */ 22960Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 22970Sstevel@tonic-gate caddr_t initraddr; /* saved initial rounded down addr */ 22980Sstevel@tonic-gate size_t rsize; /* rounded up size */ 22990Sstevel@tonic-gate size_t initrsize; /* saved initial rounded up size */ 23000Sstevel@tonic-gate size_t ssize; /* size of seg */ 23010Sstevel@tonic-gate int error = 0; /* result */ 23020Sstevel@tonic-gate size_t mlock_size; /* size of bitmap */ 23030Sstevel@tonic-gate ulong_t *mlock_map; /* pointer to bitmap used */ 23040Sstevel@tonic-gate /* to represent the locked */ 23050Sstevel@tonic-gate /* pages. */ 23060Sstevel@tonic-gate retry: 23070Sstevel@tonic-gate if (error == IE_RETRY) 23080Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 23090Sstevel@tonic-gate else 23100Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 23110Sstevel@tonic-gate 23120Sstevel@tonic-gate /* 23130Sstevel@tonic-gate * If these are address space lock/unlock operations, loop over 23140Sstevel@tonic-gate * all segments in the address space, as appropriate. 23150Sstevel@tonic-gate */ 23160Sstevel@tonic-gate if (func == MC_LOCKAS) { 23170Sstevel@tonic-gate size_t npages, idx; 23180Sstevel@tonic-gate size_t rlen = 0; /* rounded as length */ 23190Sstevel@tonic-gate 23200Sstevel@tonic-gate idx = pos; 23210Sstevel@tonic-gate 23220Sstevel@tonic-gate if (arg & MCL_FUTURE) { 23230Sstevel@tonic-gate mutex_enter(&as->a_contents); 23240Sstevel@tonic-gate AS_SETPGLCK(as); 23250Sstevel@tonic-gate mutex_exit(&as->a_contents); 23260Sstevel@tonic-gate } 23270Sstevel@tonic-gate if ((arg & MCL_CURRENT) == 0) { 23280Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 23290Sstevel@tonic-gate return (0); 23300Sstevel@tonic-gate } 23310Sstevel@tonic-gate 23320Sstevel@tonic-gate seg = AS_SEGFIRST(as); 23330Sstevel@tonic-gate if (seg == NULL) { 23340Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 23350Sstevel@tonic-gate return (0); 23360Sstevel@tonic-gate } 23370Sstevel@tonic-gate 23380Sstevel@tonic-gate do { 23390Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)seg->s_base & 23400Sstevel@tonic-gate (uintptr_t)PAGEMASK); 23410Sstevel@tonic-gate rlen += (((uintptr_t)(seg->s_base + seg->s_size) + 23425084Sjohnlev PAGEOFFSET) & PAGEMASK) - (uintptr_t)raddr; 23430Sstevel@tonic-gate } while ((seg = AS_SEGNEXT(as, seg)) != NULL); 23440Sstevel@tonic-gate 23450Sstevel@tonic-gate mlock_size = BT_BITOUL(btopr(rlen)); 23460Sstevel@tonic-gate if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size * 23475084Sjohnlev sizeof (ulong_t), KM_NOSLEEP)) == NULL) { 23480Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 23490Sstevel@tonic-gate return (EAGAIN); 23500Sstevel@tonic-gate } 23510Sstevel@tonic-gate 23520Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) { 23530Sstevel@tonic-gate error = SEGOP_LOCKOP(seg, seg->s_base, 23540Sstevel@tonic-gate seg->s_size, attr, MC_LOCK, mlock_map, pos); 23550Sstevel@tonic-gate if (error != 0) 23560Sstevel@tonic-gate break; 23570Sstevel@tonic-gate pos += seg_pages(seg); 23580Sstevel@tonic-gate } 23590Sstevel@tonic-gate 23600Sstevel@tonic-gate if (error) { 23610Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; 23625084Sjohnlev seg = AS_SEGNEXT(as, seg)) { 23630Sstevel@tonic-gate 23640Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)seg->s_base & 23655084Sjohnlev (uintptr_t)PAGEMASK); 23660Sstevel@tonic-gate npages = seg_pages(seg); 23670Sstevel@tonic-gate as_segunlock(seg, raddr, attr, mlock_map, 23685084Sjohnlev idx, npages); 23690Sstevel@tonic-gate idx += npages; 23700Sstevel@tonic-gate } 23710Sstevel@tonic-gate } 23720Sstevel@tonic-gate 23730Sstevel@tonic-gate kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); 23740Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 23750Sstevel@tonic-gate goto lockerr; 23760Sstevel@tonic-gate } else if (func == MC_UNLOCKAS) { 23770Sstevel@tonic-gate mutex_enter(&as->a_contents); 23780Sstevel@tonic-gate AS_CLRPGLCK(as); 23790Sstevel@tonic-gate mutex_exit(&as->a_contents); 23800Sstevel@tonic-gate 23810Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) { 23820Sstevel@tonic-gate error = SEGOP_LOCKOP(seg, seg->s_base, 23830Sstevel@tonic-gate seg->s_size, attr, MC_UNLOCK, NULL, 0); 23840Sstevel@tonic-gate if (error != 0) 23850Sstevel@tonic-gate break; 23860Sstevel@tonic-gate } 23870Sstevel@tonic-gate 23880Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 23890Sstevel@tonic-gate goto lockerr; 23900Sstevel@tonic-gate } 23910Sstevel@tonic-gate 23920Sstevel@tonic-gate /* 23930Sstevel@tonic-gate * Normalize addresses and sizes. 23940Sstevel@tonic-gate */ 23950Sstevel@tonic-gate initraddr = raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 23960Sstevel@tonic-gate initrsize = rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 23975084Sjohnlev (size_t)raddr; 23980Sstevel@tonic-gate 23990Sstevel@tonic-gate if (raddr + rsize < raddr) { /* check for wraparound */ 24000Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 24010Sstevel@tonic-gate return (ENOMEM); 24020Sstevel@tonic-gate } 24030Sstevel@tonic-gate 24040Sstevel@tonic-gate /* 24050Sstevel@tonic-gate * Get initial segment. 24060Sstevel@tonic-gate */ 24070Sstevel@tonic-gate if ((seg = as_segat(as, raddr)) == NULL) { 24080Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 24090Sstevel@tonic-gate return (ENOMEM); 24100Sstevel@tonic-gate } 24110Sstevel@tonic-gate 24120Sstevel@tonic-gate if (func == MC_LOCK) { 24130Sstevel@tonic-gate mlock_size = BT_BITOUL(btopr(rsize)); 24140Sstevel@tonic-gate if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size * 24155084Sjohnlev sizeof (ulong_t), KM_NOSLEEP)) == NULL) { 24160Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 24170Sstevel@tonic-gate return (EAGAIN); 24180Sstevel@tonic-gate } 24190Sstevel@tonic-gate } 24200Sstevel@tonic-gate 24210Sstevel@tonic-gate /* 24220Sstevel@tonic-gate * Loop over all segments. If a hole in the address range is 24230Sstevel@tonic-gate * discovered, then fail. For each segment, perform the appropriate 24240Sstevel@tonic-gate * control operation. 24250Sstevel@tonic-gate */ 24260Sstevel@tonic-gate while (rsize != 0) { 24270Sstevel@tonic-gate 24280Sstevel@tonic-gate /* 24290Sstevel@tonic-gate * Make sure there's no hole, calculate the portion 24300Sstevel@tonic-gate * of the next segment to be operated over. 24310Sstevel@tonic-gate */ 24320Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 24330Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 24340Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 24350Sstevel@tonic-gate if (func == MC_LOCK) { 24360Sstevel@tonic-gate as_unlockerr(as, attr, mlock_map, 24375084Sjohnlev initraddr, initrsize - rsize); 24380Sstevel@tonic-gate kmem_free(mlock_map, 24395084Sjohnlev mlock_size * sizeof (ulong_t)); 24400Sstevel@tonic-gate } 24410Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 24420Sstevel@tonic-gate return (ENOMEM); 24430Sstevel@tonic-gate } 24440Sstevel@tonic-gate } 24450Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 24460Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 24470Sstevel@tonic-gate else 24480Sstevel@tonic-gate ssize = rsize; 24490Sstevel@tonic-gate 24500Sstevel@tonic-gate /* 24510Sstevel@tonic-gate * Dispatch on specific function. 24520Sstevel@tonic-gate */ 24530Sstevel@tonic-gate switch (func) { 24540Sstevel@tonic-gate 24550Sstevel@tonic-gate /* 24560Sstevel@tonic-gate * Synchronize cached data from mappings with backing 24570Sstevel@tonic-gate * objects. 24580Sstevel@tonic-gate */ 24590Sstevel@tonic-gate case MC_SYNC: 24600Sstevel@tonic-gate if (error = SEGOP_SYNC(seg, raddr, ssize, 24610Sstevel@tonic-gate attr, (uint_t)arg)) { 24620Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 24630Sstevel@tonic-gate return (error); 24640Sstevel@tonic-gate } 24650Sstevel@tonic-gate break; 24660Sstevel@tonic-gate 24670Sstevel@tonic-gate /* 24680Sstevel@tonic-gate * Lock pages in memory. 24690Sstevel@tonic-gate */ 24700Sstevel@tonic-gate case MC_LOCK: 24710Sstevel@tonic-gate if (error = SEGOP_LOCKOP(seg, raddr, ssize, 24725084Sjohnlev attr, func, mlock_map, pos)) { 24730Sstevel@tonic-gate as_unlockerr(as, attr, mlock_map, initraddr, 24745084Sjohnlev initrsize - rsize + ssize); 24750Sstevel@tonic-gate kmem_free(mlock_map, mlock_size * 24765084Sjohnlev sizeof (ulong_t)); 24770Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 24780Sstevel@tonic-gate goto lockerr; 24790Sstevel@tonic-gate } 24800Sstevel@tonic-gate break; 24810Sstevel@tonic-gate 24820Sstevel@tonic-gate /* 24830Sstevel@tonic-gate * Unlock mapped pages. 24840Sstevel@tonic-gate */ 24850Sstevel@tonic-gate case MC_UNLOCK: 24860Sstevel@tonic-gate (void) SEGOP_LOCKOP(seg, raddr, ssize, attr, func, 24875084Sjohnlev (ulong_t *)NULL, (size_t)NULL); 24880Sstevel@tonic-gate break; 24890Sstevel@tonic-gate 24900Sstevel@tonic-gate /* 24910Sstevel@tonic-gate * Store VM advise for mapped pages in segment layer. 24920Sstevel@tonic-gate */ 24930Sstevel@tonic-gate case MC_ADVISE: 24940Sstevel@tonic-gate error = SEGOP_ADVISE(seg, raddr, ssize, (uint_t)arg); 24950Sstevel@tonic-gate 24960Sstevel@tonic-gate /* 24970Sstevel@tonic-gate * Check for regular errors and special retry error 24980Sstevel@tonic-gate */ 24990Sstevel@tonic-gate if (error) { 25000Sstevel@tonic-gate if (error == IE_RETRY) { 25010Sstevel@tonic-gate /* 25020Sstevel@tonic-gate * Need to acquire writers lock, so 25030Sstevel@tonic-gate * have to drop readers lock and start 25040Sstevel@tonic-gate * all over again 25050Sstevel@tonic-gate */ 25060Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 25070Sstevel@tonic-gate goto retry; 25080Sstevel@tonic-gate } else if (error == IE_REATTACH) { 25090Sstevel@tonic-gate /* 25100Sstevel@tonic-gate * Find segment for current address 25110Sstevel@tonic-gate * because current segment just got 25120Sstevel@tonic-gate * split or concatenated 25130Sstevel@tonic-gate */ 25140Sstevel@tonic-gate seg = as_segat(as, raddr); 25150Sstevel@tonic-gate if (seg == NULL) { 25160Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 25170Sstevel@tonic-gate return (ENOMEM); 25180Sstevel@tonic-gate } 25190Sstevel@tonic-gate } else { 25200Sstevel@tonic-gate /* 25210Sstevel@tonic-gate * Regular error 25220Sstevel@tonic-gate */ 25230Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 25240Sstevel@tonic-gate return (error); 25250Sstevel@tonic-gate } 25260Sstevel@tonic-gate } 25270Sstevel@tonic-gate break; 25280Sstevel@tonic-gate 25290Sstevel@tonic-gate /* 25300Sstevel@tonic-gate * Can't happen. 25310Sstevel@tonic-gate */ 25320Sstevel@tonic-gate default: 25330Sstevel@tonic-gate panic("as_ctl: bad operation %d", func); 25340Sstevel@tonic-gate /*NOTREACHED*/ 25350Sstevel@tonic-gate } 25360Sstevel@tonic-gate 25370Sstevel@tonic-gate rsize -= ssize; 25380Sstevel@tonic-gate raddr += ssize; 25390Sstevel@tonic-gate } 25400Sstevel@tonic-gate 25410Sstevel@tonic-gate if (func == MC_LOCK) 25420Sstevel@tonic-gate kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); 25430Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 25440Sstevel@tonic-gate return (0); 25450Sstevel@tonic-gate lockerr: 25460Sstevel@tonic-gate 25470Sstevel@tonic-gate /* 25480Sstevel@tonic-gate * If the lower levels returned EDEADLK for a segment lockop, 25490Sstevel@tonic-gate * it means that we should retry the operation. Let's wait 25500Sstevel@tonic-gate * a bit also to let the deadlock causing condition clear. 25510Sstevel@tonic-gate * This is part of a gross hack to work around a design flaw 25520Sstevel@tonic-gate * in the ufs/sds logging code and should go away when the 25530Sstevel@tonic-gate * logging code is re-designed to fix the problem. See bug 25540Sstevel@tonic-gate * 4125102 for details of the problem. 25550Sstevel@tonic-gate */ 25560Sstevel@tonic-gate if (error == EDEADLK) { 25570Sstevel@tonic-gate delay(deadlk_wait); 25580Sstevel@tonic-gate error = 0; 25590Sstevel@tonic-gate goto retry; 25600Sstevel@tonic-gate } 25610Sstevel@tonic-gate return (error); 25620Sstevel@tonic-gate } 25630Sstevel@tonic-gate 25640Sstevel@tonic-gate /* 25650Sstevel@tonic-gate * Special code for exec to move the stack segment from its interim 25660Sstevel@tonic-gate * place in the old address to the right place in the new address space. 25670Sstevel@tonic-gate */ 25680Sstevel@tonic-gate /*ARGSUSED*/ 25690Sstevel@tonic-gate int 25700Sstevel@tonic-gate as_exec(struct as *oas, caddr_t ostka, size_t stksz, 25710Sstevel@tonic-gate struct as *nas, caddr_t nstka, uint_t hatflag) 25720Sstevel@tonic-gate { 25730Sstevel@tonic-gate struct seg *stkseg; 25740Sstevel@tonic-gate 25750Sstevel@tonic-gate AS_LOCK_ENTER(oas, &oas->a_lock, RW_WRITER); 25760Sstevel@tonic-gate stkseg = as_segat(oas, ostka); 25770Sstevel@tonic-gate stkseg = as_removeseg(oas, stkseg); 25780Sstevel@tonic-gate ASSERT(stkseg != NULL); 25790Sstevel@tonic-gate ASSERT(stkseg->s_base == ostka && stkseg->s_size == stksz); 25800Sstevel@tonic-gate stkseg->s_as = nas; 25810Sstevel@tonic-gate stkseg->s_base = nstka; 25820Sstevel@tonic-gate 25830Sstevel@tonic-gate /* 25840Sstevel@tonic-gate * It's ok to lock the address space we are about to exec to. 25850Sstevel@tonic-gate */ 25860Sstevel@tonic-gate AS_LOCK_ENTER(nas, &nas->a_lock, RW_WRITER); 25870Sstevel@tonic-gate ASSERT(avl_numnodes(&nas->a_wpage) == 0); 25880Sstevel@tonic-gate nas->a_size += stkseg->s_size; 25890Sstevel@tonic-gate oas->a_size -= stkseg->s_size; 25900Sstevel@tonic-gate (void) as_addseg(nas, stkseg); 25910Sstevel@tonic-gate AS_LOCK_EXIT(nas, &nas->a_lock); 25920Sstevel@tonic-gate AS_LOCK_EXIT(oas, &oas->a_lock); 25930Sstevel@tonic-gate return (0); 25940Sstevel@tonic-gate } 25950Sstevel@tonic-gate 25965084Sjohnlev int 25975084Sjohnlev fc_decode(faultcode_t fault_err) 25980Sstevel@tonic-gate { 25990Sstevel@tonic-gate int error = 0; 26000Sstevel@tonic-gate 26010Sstevel@tonic-gate switch (FC_CODE(fault_err)) { 26020Sstevel@tonic-gate case FC_OBJERR: 26030Sstevel@tonic-gate error = FC_ERRNO(fault_err); 26040Sstevel@tonic-gate break; 26050Sstevel@tonic-gate case FC_PROT: 26060Sstevel@tonic-gate error = EACCES; 26070Sstevel@tonic-gate break; 26080Sstevel@tonic-gate default: 26090Sstevel@tonic-gate error = EFAULT; 26100Sstevel@tonic-gate break; 26110Sstevel@tonic-gate } 26120Sstevel@tonic-gate return (error); 26130Sstevel@tonic-gate } 26140Sstevel@tonic-gate 26150Sstevel@tonic-gate /* 26166695Saguzovsk * Pagelock pages from a range that spans more than 1 segment. Obtain shadow 26176695Saguzovsk * lists from each segment and copy them to one contiguous shadow list (plist) 26186695Saguzovsk * as expected by the caller. Save pointers to per segment shadow lists at 26196695Saguzovsk * the tail of plist so that they can be used during as_pageunlock(). 26206695Saguzovsk */ 26216695Saguzovsk static int 26226695Saguzovsk as_pagelock_segs(struct as *as, struct seg *seg, struct page ***ppp, 26236695Saguzovsk caddr_t addr, size_t size, enum seg_rw rw) 26246695Saguzovsk { 26256695Saguzovsk caddr_t sv_addr = addr; 26266695Saguzovsk size_t sv_size = size; 26276695Saguzovsk struct seg *sv_seg = seg; 26286695Saguzovsk ulong_t segcnt = 1; 26296695Saguzovsk ulong_t cnt; 26306695Saguzovsk size_t ssize; 26316695Saguzovsk pgcnt_t npages = btop(size); 26326695Saguzovsk page_t **plist; 26336695Saguzovsk page_t **pl; 26346695Saguzovsk int error; 26356695Saguzovsk caddr_t eaddr; 26366695Saguzovsk faultcode_t fault_err = 0; 26376695Saguzovsk pgcnt_t pl_off; 26386695Saguzovsk extern struct seg_ops segspt_shmops; 26396695Saguzovsk 26406695Saguzovsk ASSERT(AS_LOCK_HELD(as, &as->a_lock)); 26416695Saguzovsk ASSERT(seg != NULL); 26426695Saguzovsk ASSERT(addr >= seg->s_base && addr < seg->s_base + seg->s_size); 26436695Saguzovsk ASSERT(addr + size > seg->s_base + seg->s_size); 26446695Saguzovsk ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 26456695Saguzovsk ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 26466695Saguzovsk 26476695Saguzovsk /* 26486695Saguzovsk * Count the number of segments covered by the range we are about to 26496695Saguzovsk * lock. The segment count is used to size the shadow list we return 26506695Saguzovsk * back to the caller. 26516695Saguzovsk */ 26526695Saguzovsk for (; size != 0; size -= ssize, addr += ssize) { 26536695Saguzovsk if (addr >= seg->s_base + seg->s_size) { 26546695Saguzovsk 26556695Saguzovsk seg = AS_SEGNEXT(as, seg); 26566695Saguzovsk if (seg == NULL || addr != seg->s_base) { 26576695Saguzovsk AS_LOCK_EXIT(as, &as->a_lock); 26586695Saguzovsk return (EFAULT); 26596695Saguzovsk } 26606695Saguzovsk /* 26616695Saguzovsk * Do a quick check if subsequent segments 26626695Saguzovsk * will most likely support pagelock. 26636695Saguzovsk */ 26646695Saguzovsk if (seg->s_ops == &segvn_ops) { 26656695Saguzovsk vnode_t *vp; 26666695Saguzovsk 26676695Saguzovsk if (SEGOP_GETVP(seg, addr, &vp) != 0 || 26686695Saguzovsk vp != NULL) { 26696695Saguzovsk AS_LOCK_EXIT(as, &as->a_lock); 26706695Saguzovsk goto slow; 26716695Saguzovsk } 26726695Saguzovsk } else if (seg->s_ops != &segspt_shmops) { 26736695Saguzovsk AS_LOCK_EXIT(as, &as->a_lock); 26746695Saguzovsk goto slow; 26756695Saguzovsk } 26766695Saguzovsk segcnt++; 26776695Saguzovsk } 26786695Saguzovsk if (addr + size > seg->s_base + seg->s_size) { 26796695Saguzovsk ssize = seg->s_base + seg->s_size - addr; 26806695Saguzovsk } else { 26816695Saguzovsk ssize = size; 26826695Saguzovsk } 26836695Saguzovsk } 26846695Saguzovsk ASSERT(segcnt > 1); 26856695Saguzovsk 26866695Saguzovsk plist = kmem_zalloc((npages + segcnt) * sizeof (page_t *), KM_SLEEP); 26876695Saguzovsk 26886695Saguzovsk addr = sv_addr; 26896695Saguzovsk size = sv_size; 26906695Saguzovsk seg = sv_seg; 26916695Saguzovsk 26926695Saguzovsk for (cnt = 0, pl_off = 0; size != 0; size -= ssize, addr += ssize) { 26936695Saguzovsk if (addr >= seg->s_base + seg->s_size) { 26946695Saguzovsk seg = AS_SEGNEXT(as, seg); 26956695Saguzovsk ASSERT(seg != NULL && addr == seg->s_base); 26966695Saguzovsk cnt++; 26976695Saguzovsk ASSERT(cnt < segcnt); 26986695Saguzovsk } 26996695Saguzovsk if (addr + size > seg->s_base + seg->s_size) { 27006695Saguzovsk ssize = seg->s_base + seg->s_size - addr; 27016695Saguzovsk } else { 27026695Saguzovsk ssize = size; 27036695Saguzovsk } 27046695Saguzovsk pl = &plist[npages + cnt]; 27056695Saguzovsk error = SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl, 27066695Saguzovsk L_PAGELOCK, rw); 27076695Saguzovsk if (error) { 27086695Saguzovsk break; 27096695Saguzovsk } 27106695Saguzovsk ASSERT(plist[npages + cnt] != NULL); 27116695Saguzovsk ASSERT(pl_off + btop(ssize) <= npages); 27126695Saguzovsk bcopy(plist[npages + cnt], &plist[pl_off], 27136695Saguzovsk btop(ssize) * sizeof (page_t *)); 27146695Saguzovsk pl_off += btop(ssize); 27156695Saguzovsk } 27166695Saguzovsk 27176695Saguzovsk if (size == 0) { 27186695Saguzovsk AS_LOCK_EXIT(as, &as->a_lock); 27196695Saguzovsk ASSERT(cnt == segcnt - 1); 27206695Saguzovsk *ppp = plist; 27216695Saguzovsk return (0); 27226695Saguzovsk } 27236695Saguzovsk 27246695Saguzovsk /* 27256695Saguzovsk * one of pagelock calls failed. The error type is in error variable. 27266695Saguzovsk * Unlock what we've locked so far and retry with F_SOFTLOCK if error 27276695Saguzovsk * type is either EFAULT or ENOTSUP. Otherwise just return the error 27286695Saguzovsk * back to the caller. 27296695Saguzovsk */ 27306695Saguzovsk 27316695Saguzovsk eaddr = addr; 27326695Saguzovsk seg = sv_seg; 27336695Saguzovsk 27346695Saguzovsk for (cnt = 0, addr = sv_addr; addr < eaddr; addr += ssize) { 27356695Saguzovsk if (addr >= seg->s_base + seg->s_size) { 27366695Saguzovsk seg = AS_SEGNEXT(as, seg); 27376695Saguzovsk ASSERT(seg != NULL && addr == seg->s_base); 27386695Saguzovsk cnt++; 27396695Saguzovsk ASSERT(cnt < segcnt); 27406695Saguzovsk } 27416695Saguzovsk if (eaddr > seg->s_base + seg->s_size) { 27426695Saguzovsk ssize = seg->s_base + seg->s_size - addr; 27436695Saguzovsk } else { 27446695Saguzovsk ssize = eaddr - addr; 27456695Saguzovsk } 27466695Saguzovsk pl = &plist[npages + cnt]; 27476695Saguzovsk ASSERT(*pl != NULL); 27486695Saguzovsk (void) SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl, 27496695Saguzovsk L_PAGEUNLOCK, rw); 27506695Saguzovsk } 27516695Saguzovsk 27526695Saguzovsk AS_LOCK_EXIT(as, &as->a_lock); 27536695Saguzovsk 27546695Saguzovsk kmem_free(plist, (npages + segcnt) * sizeof (page_t *)); 27556695Saguzovsk 27566695Saguzovsk if (error != ENOTSUP && error != EFAULT) { 27576695Saguzovsk return (error); 27586695Saguzovsk } 27596695Saguzovsk 27606695Saguzovsk slow: 27616695Saguzovsk /* 27626695Saguzovsk * If we are here because pagelock failed due to the need to cow fault 27636695Saguzovsk * in the pages we want to lock F_SOFTLOCK will do this job and in 27646695Saguzovsk * next as_pagelock() call for this address range pagelock will 27656695Saguzovsk * hopefully succeed. 27666695Saguzovsk */ 27676695Saguzovsk fault_err = as_fault(as->a_hat, as, sv_addr, sv_size, F_SOFTLOCK, rw); 27686695Saguzovsk if (fault_err != 0) { 27696695Saguzovsk return (fc_decode(fault_err)); 27706695Saguzovsk } 27716695Saguzovsk *ppp = NULL; 27726695Saguzovsk 27736695Saguzovsk return (0); 27746695Saguzovsk } 27756695Saguzovsk 27766695Saguzovsk /* 27770Sstevel@tonic-gate * lock pages in a given address space. Return shadow list. If 27780Sstevel@tonic-gate * the list is NULL, the MMU mapping is also locked. 27790Sstevel@tonic-gate */ 27800Sstevel@tonic-gate int 27810Sstevel@tonic-gate as_pagelock(struct as *as, struct page ***ppp, caddr_t addr, 27820Sstevel@tonic-gate size_t size, enum seg_rw rw) 27830Sstevel@tonic-gate { 27840Sstevel@tonic-gate size_t rsize; 27850Sstevel@tonic-gate caddr_t raddr; 27860Sstevel@tonic-gate faultcode_t fault_err; 27870Sstevel@tonic-gate struct seg *seg; 27886695Saguzovsk int err; 27890Sstevel@tonic-gate 27900Sstevel@tonic-gate TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_START, 27910Sstevel@tonic-gate "as_pagelock_start: addr %p size %ld", addr, size); 27920Sstevel@tonic-gate 27930Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 27940Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 27955084Sjohnlev (size_t)raddr; 27966695Saguzovsk 27970Sstevel@tonic-gate /* 27980Sstevel@tonic-gate * if the request crosses two segments let 27990Sstevel@tonic-gate * as_fault handle it. 28000Sstevel@tonic-gate */ 28010Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 28026695Saguzovsk 28036695Saguzovsk seg = as_segat(as, raddr); 28046695Saguzovsk if (seg == NULL) { 28050Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 28066695Saguzovsk return (EFAULT); 28076695Saguzovsk } 28086695Saguzovsk ASSERT(raddr >= seg->s_base && raddr < seg->s_base + seg->s_size); 28096695Saguzovsk if (raddr + rsize > seg->s_base + seg->s_size) { 28106695Saguzovsk return (as_pagelock_segs(as, seg, ppp, raddr, rsize, rw)); 28116695Saguzovsk } 28126695Saguzovsk if (raddr + rsize <= raddr) { 28136695Saguzovsk AS_LOCK_EXIT(as, &as->a_lock); 28146695Saguzovsk return (EFAULT); 28150Sstevel@tonic-gate } 28160Sstevel@tonic-gate 28170Sstevel@tonic-gate TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_START, 28180Sstevel@tonic-gate "seg_lock_1_start: raddr %p rsize %ld", raddr, rsize); 28190Sstevel@tonic-gate 28200Sstevel@tonic-gate /* 28210Sstevel@tonic-gate * try to lock pages and pass back shadow list 28220Sstevel@tonic-gate */ 28236695Saguzovsk err = SEGOP_PAGELOCK(seg, raddr, rsize, ppp, L_PAGELOCK, rw); 28240Sstevel@tonic-gate 28250Sstevel@tonic-gate TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_END, "seg_lock_1_end"); 28266695Saguzovsk 28270Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 28286695Saguzovsk 28296695Saguzovsk if (err == 0 || (err != ENOTSUP && err != EFAULT)) { 28306695Saguzovsk return (err); 28310Sstevel@tonic-gate } 28320Sstevel@tonic-gate 28330Sstevel@tonic-gate /* 28346695Saguzovsk * Use F_SOFTLOCK to lock the pages because pagelock failed either due 28356695Saguzovsk * to no pagelock support for this segment or pages need to be cow 28366695Saguzovsk * faulted in. If fault is needed F_SOFTLOCK will do this job for 28376695Saguzovsk * this as_pagelock() call and in the next as_pagelock() call for the 28386695Saguzovsk * same address range pagelock call will hopefull succeed. 28390Sstevel@tonic-gate */ 28400Sstevel@tonic-gate fault_err = as_fault(as->a_hat, as, addr, size, F_SOFTLOCK, rw); 28410Sstevel@tonic-gate if (fault_err != 0) { 28425084Sjohnlev return (fc_decode(fault_err)); 28430Sstevel@tonic-gate } 28440Sstevel@tonic-gate *ppp = NULL; 28450Sstevel@tonic-gate 28460Sstevel@tonic-gate TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_END, "as_pagelock_end"); 28470Sstevel@tonic-gate return (0); 28480Sstevel@tonic-gate } 28490Sstevel@tonic-gate 28500Sstevel@tonic-gate /* 28516695Saguzovsk * unlock pages locked by as_pagelock_segs(). Retrieve per segment shadow 28526695Saguzovsk * lists from the end of plist and call pageunlock interface for each segment. 28536695Saguzovsk * Drop as lock and free plist. 28546695Saguzovsk */ 28556695Saguzovsk static void 28566695Saguzovsk as_pageunlock_segs(struct as *as, struct seg *seg, caddr_t addr, size_t size, 28576695Saguzovsk struct page **plist, enum seg_rw rw) 28586695Saguzovsk { 28596695Saguzovsk ulong_t cnt; 28606695Saguzovsk caddr_t eaddr = addr + size; 28616695Saguzovsk pgcnt_t npages = btop(size); 28626695Saguzovsk size_t ssize; 28636695Saguzovsk page_t **pl; 28646695Saguzovsk 28656695Saguzovsk ASSERT(AS_LOCK_HELD(as, &as->a_lock)); 28666695Saguzovsk ASSERT(seg != NULL); 28676695Saguzovsk ASSERT(addr >= seg->s_base && addr < seg->s_base + seg->s_size); 28686695Saguzovsk ASSERT(addr + size > seg->s_base + seg->s_size); 28696695Saguzovsk ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 28706695Saguzovsk ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 28716695Saguzovsk ASSERT(plist != NULL); 28726695Saguzovsk 28736695Saguzovsk for (cnt = 0; addr < eaddr; addr += ssize) { 28746695Saguzovsk if (addr >= seg->s_base + seg->s_size) { 28756695Saguzovsk seg = AS_SEGNEXT(as, seg); 28766695Saguzovsk ASSERT(seg != NULL && addr == seg->s_base); 28776695Saguzovsk cnt++; 28786695Saguzovsk } 28796695Saguzovsk if (eaddr > seg->s_base + seg->s_size) { 28806695Saguzovsk ssize = seg->s_base + seg->s_size - addr; 28816695Saguzovsk } else { 28826695Saguzovsk ssize = eaddr - addr; 28836695Saguzovsk } 28846695Saguzovsk pl = &plist[npages + cnt]; 28856695Saguzovsk ASSERT(*pl != NULL); 28866695Saguzovsk (void) SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl, 28876695Saguzovsk L_PAGEUNLOCK, rw); 28886695Saguzovsk } 28896695Saguzovsk ASSERT(cnt > 0); 28906695Saguzovsk AS_LOCK_EXIT(as, &as->a_lock); 28916695Saguzovsk 28926695Saguzovsk cnt++; 28936695Saguzovsk kmem_free(plist, (npages + cnt) * sizeof (page_t *)); 28946695Saguzovsk } 28956695Saguzovsk 28966695Saguzovsk /* 28970Sstevel@tonic-gate * unlock pages in a given address range 28980Sstevel@tonic-gate */ 28990Sstevel@tonic-gate void 29000Sstevel@tonic-gate as_pageunlock(struct as *as, struct page **pp, caddr_t addr, size_t size, 29010Sstevel@tonic-gate enum seg_rw rw) 29020Sstevel@tonic-gate { 29030Sstevel@tonic-gate struct seg *seg; 29040Sstevel@tonic-gate size_t rsize; 29050Sstevel@tonic-gate caddr_t raddr; 29060Sstevel@tonic-gate 29070Sstevel@tonic-gate TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_START, 29080Sstevel@tonic-gate "as_pageunlock_start: addr %p size %ld", addr, size); 29090Sstevel@tonic-gate 29100Sstevel@tonic-gate /* 29110Sstevel@tonic-gate * if the shadow list is NULL, as_pagelock was 29120Sstevel@tonic-gate * falling back to as_fault 29130Sstevel@tonic-gate */ 29140Sstevel@tonic-gate if (pp == NULL) { 29150Sstevel@tonic-gate (void) as_fault(as->a_hat, as, addr, size, F_SOFTUNLOCK, rw); 29160Sstevel@tonic-gate return; 29170Sstevel@tonic-gate } 29180Sstevel@tonic-gate 29190Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 29200Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 29215084Sjohnlev (size_t)raddr; 29226695Saguzovsk 29236695Saguzovsk AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 29246695Saguzovsk seg = as_segat(as, raddr); 29256695Saguzovsk ASSERT(seg != NULL); 29266695Saguzovsk 29276695Saguzovsk TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_UNLOCK_START, 29286695Saguzovsk "seg_unlock_start: raddr %p rsize %ld", raddr, rsize); 29296695Saguzovsk 29306695Saguzovsk ASSERT(raddr >= seg->s_base && raddr < seg->s_base + seg->s_size); 29316695Saguzovsk if (raddr + rsize <= seg->s_base + seg->s_size) { 29326695Saguzovsk SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGEUNLOCK, rw); 29336695Saguzovsk } else { 29346695Saguzovsk as_pageunlock_segs(as, seg, raddr, rsize, pp, rw); 29356695Saguzovsk return; 29366695Saguzovsk } 29376695Saguzovsk AS_LOCK_EXIT(as, &as->a_lock); 29386695Saguzovsk TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_END, "as_pageunlock_end"); 29390Sstevel@tonic-gate } 29400Sstevel@tonic-gate 29410Sstevel@tonic-gate int 29420Sstevel@tonic-gate as_setpagesize(struct as *as, caddr_t addr, size_t size, uint_t szc, 29430Sstevel@tonic-gate boolean_t wait) 29440Sstevel@tonic-gate { 29450Sstevel@tonic-gate struct seg *seg; 29460Sstevel@tonic-gate size_t ssize; 29470Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 29480Sstevel@tonic-gate size_t rsize; /* rounded up size */ 29490Sstevel@tonic-gate int error = 0; 29500Sstevel@tonic-gate size_t pgsz = page_get_pagesize(szc); 29510Sstevel@tonic-gate 29520Sstevel@tonic-gate setpgsz_top: 29530Sstevel@tonic-gate if (!IS_P2ALIGNED(addr, pgsz) || !IS_P2ALIGNED(size, pgsz)) { 29540Sstevel@tonic-gate return (EINVAL); 29550Sstevel@tonic-gate } 29560Sstevel@tonic-gate 29570Sstevel@tonic-gate raddr = addr; 29580Sstevel@tonic-gate rsize = size; 29590Sstevel@tonic-gate 29600Sstevel@tonic-gate if (raddr + rsize < raddr) /* check for wraparound */ 29610Sstevel@tonic-gate return (ENOMEM); 29620Sstevel@tonic-gate 29630Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 29640Sstevel@tonic-gate as_clearwatchprot(as, raddr, rsize); 29650Sstevel@tonic-gate seg = as_segat(as, raddr); 29660Sstevel@tonic-gate if (seg == NULL) { 29670Sstevel@tonic-gate as_setwatch(as); 29680Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 29690Sstevel@tonic-gate return (ENOMEM); 29700Sstevel@tonic-gate } 29710Sstevel@tonic-gate 29720Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 29730Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 29740Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 29750Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 29760Sstevel@tonic-gate error = ENOMEM; 29770Sstevel@tonic-gate break; 29780Sstevel@tonic-gate } 29790Sstevel@tonic-gate } 29800Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) { 29810Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 29820Sstevel@tonic-gate } else { 29830Sstevel@tonic-gate ssize = rsize; 29840Sstevel@tonic-gate } 29850Sstevel@tonic-gate 29866695Saguzovsk retry: 29870Sstevel@tonic-gate error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc); 29880Sstevel@tonic-gate 29890Sstevel@tonic-gate if (error == IE_NOMEM) { 29900Sstevel@tonic-gate error = EAGAIN; 29910Sstevel@tonic-gate break; 29920Sstevel@tonic-gate } 29930Sstevel@tonic-gate 29940Sstevel@tonic-gate if (error == IE_RETRY) { 29950Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 29960Sstevel@tonic-gate goto setpgsz_top; 29970Sstevel@tonic-gate } 29980Sstevel@tonic-gate 29990Sstevel@tonic-gate if (error == ENOTSUP) { 30000Sstevel@tonic-gate error = EINVAL; 30010Sstevel@tonic-gate break; 30020Sstevel@tonic-gate } 30030Sstevel@tonic-gate 30040Sstevel@tonic-gate if (wait && (error == EAGAIN)) { 30050Sstevel@tonic-gate /* 30060Sstevel@tonic-gate * Memory is currently locked. It must be unlocked 30070Sstevel@tonic-gate * before this operation can succeed through a retry. 30080Sstevel@tonic-gate * The possible reasons for locked memory and 30090Sstevel@tonic-gate * corresponding strategies for unlocking are: 30100Sstevel@tonic-gate * (1) Normal I/O 30110Sstevel@tonic-gate * wait for a signal that the I/O operation 30120Sstevel@tonic-gate * has completed and the memory is unlocked. 30130Sstevel@tonic-gate * (2) Asynchronous I/O 30140Sstevel@tonic-gate * The aio subsystem does not unlock pages when 30150Sstevel@tonic-gate * the I/O is completed. Those pages are unlocked 30160Sstevel@tonic-gate * when the application calls aiowait/aioerror. 30170Sstevel@tonic-gate * So, to prevent blocking forever, cv_broadcast() 30180Sstevel@tonic-gate * is done to wake up aio_cleanup_thread. 30190Sstevel@tonic-gate * Subsequently, segvn_reclaim will be called, and 30200Sstevel@tonic-gate * that will do AS_CLRUNMAPWAIT() and wake us up. 30210Sstevel@tonic-gate * (3) Long term page locking: 30220Sstevel@tonic-gate * This is not relevant for as_setpagesize() 30230Sstevel@tonic-gate * because we cannot change the page size for 30240Sstevel@tonic-gate * driver memory. The attempt to do so will 30250Sstevel@tonic-gate * fail with a different error than EAGAIN so 30260Sstevel@tonic-gate * there's no need to trigger as callbacks like 30270Sstevel@tonic-gate * as_unmap, as_setprot or as_free would do. 30280Sstevel@tonic-gate */ 30290Sstevel@tonic-gate mutex_enter(&as->a_contents); 30306695Saguzovsk if (!AS_ISNOUNMAPWAIT(as)) { 30316695Saguzovsk if (AS_ISUNMAPWAIT(as) == 0) { 30326695Saguzovsk cv_broadcast(&as->a_cv); 30336695Saguzovsk } 30346695Saguzovsk AS_SETUNMAPWAIT(as); 30356695Saguzovsk AS_LOCK_EXIT(as, &as->a_lock); 30366695Saguzovsk while (AS_ISUNMAPWAIT(as)) { 30376695Saguzovsk cv_wait(&as->a_cv, &as->a_contents); 30386695Saguzovsk } 30396695Saguzovsk } else { 30406695Saguzovsk /* 30416695Saguzovsk * We may have raced with 30426695Saguzovsk * segvn_reclaim()/segspt_reclaim(). In this 30436695Saguzovsk * case clean nounmapwait flag and retry since 30446695Saguzovsk * softlockcnt in this segment may be already 30456695Saguzovsk * 0. We don't drop as writer lock so our 30466695Saguzovsk * number of retries without sleeping should 30476695Saguzovsk * be very small. See segvn_reclaim() for 30486695Saguzovsk * more comments. 30496695Saguzovsk */ 30506695Saguzovsk AS_CLRNOUNMAPWAIT(as); 30516695Saguzovsk mutex_exit(&as->a_contents); 30526695Saguzovsk goto retry; 30530Sstevel@tonic-gate } 30540Sstevel@tonic-gate mutex_exit(&as->a_contents); 30550Sstevel@tonic-gate goto setpgsz_top; 30560Sstevel@tonic-gate } else if (error != 0) { 30570Sstevel@tonic-gate break; 30580Sstevel@tonic-gate } 30590Sstevel@tonic-gate } 30600Sstevel@tonic-gate as_setwatch(as); 30610Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 30620Sstevel@tonic-gate return (error); 30630Sstevel@tonic-gate } 30640Sstevel@tonic-gate 30650Sstevel@tonic-gate /* 30662991Ssusans * as_iset3_default_lpsize() just calls SEGOP_SETPAGESIZE() on all segments 30672991Ssusans * in its chunk where s_szc is less than the szc we want to set. 30682991Ssusans */ 30692991Ssusans static int 30702991Ssusans as_iset3_default_lpsize(struct as *as, caddr_t raddr, size_t rsize, uint_t szc, 30712991Ssusans int *retry) 30722991Ssusans { 30732991Ssusans struct seg *seg; 30742991Ssusans size_t ssize; 30752991Ssusans int error; 30762991Ssusans 30776695Saguzovsk ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 30786695Saguzovsk 30792991Ssusans seg = as_segat(as, raddr); 30802991Ssusans if (seg == NULL) { 30812991Ssusans panic("as_iset3_default_lpsize: no seg"); 30822991Ssusans } 30832991Ssusans 30842991Ssusans for (; rsize != 0; rsize -= ssize, raddr += ssize) { 30852991Ssusans if (raddr >= seg->s_base + seg->s_size) { 30862991Ssusans seg = AS_SEGNEXT(as, seg); 30872991Ssusans if (seg == NULL || raddr != seg->s_base) { 30882991Ssusans panic("as_iset3_default_lpsize: as changed"); 30892991Ssusans } 30902991Ssusans } 30912991Ssusans if ((raddr + rsize) > (seg->s_base + seg->s_size)) { 30922991Ssusans ssize = seg->s_base + seg->s_size - raddr; 30932991Ssusans } else { 30942991Ssusans ssize = rsize; 30952991Ssusans } 30962991Ssusans 30972991Ssusans if (szc > seg->s_szc) { 30982991Ssusans error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc); 30992991Ssusans /* Only retry on EINVAL segments that have no vnode. */ 31002991Ssusans if (error == EINVAL) { 31012991Ssusans vnode_t *vp = NULL; 31022991Ssusans if ((SEGOP_GETTYPE(seg, raddr) & MAP_SHARED) && 31032991Ssusans (SEGOP_GETVP(seg, raddr, &vp) != 0 || 31042991Ssusans vp == NULL)) { 31052991Ssusans *retry = 1; 31062991Ssusans } else { 31072991Ssusans *retry = 0; 31082991Ssusans } 31092991Ssusans } 31102991Ssusans if (error) { 31112991Ssusans return (error); 31122991Ssusans } 31132991Ssusans } 31142991Ssusans } 31152991Ssusans return (0); 31162991Ssusans } 31172991Ssusans 31182991Ssusans /* 31192991Ssusans * as_iset2_default_lpsize() calls as_iset3_default_lpsize() to set the 31202991Ssusans * pagesize on each segment in its range, but if any fails with EINVAL, 31212991Ssusans * then it reduces the pagesizes to the next size in the bitmap and 31222991Ssusans * retries as_iset3_default_lpsize(). The reason why the code retries 31232991Ssusans * smaller allowed sizes on EINVAL is because (a) the anon offset may not 31242991Ssusans * match the bigger sizes, and (b) it's hard to get this offset (to begin 31252991Ssusans * with) to pass to map_pgszcvec(). 31262991Ssusans */ 31272991Ssusans static int 31282991Ssusans as_iset2_default_lpsize(struct as *as, caddr_t addr, size_t size, uint_t szc, 31292991Ssusans uint_t szcvec) 31302991Ssusans { 31312991Ssusans int error; 31322991Ssusans int retry; 31332991Ssusans 31346695Saguzovsk ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 31356695Saguzovsk 31362991Ssusans for (;;) { 31372991Ssusans error = as_iset3_default_lpsize(as, addr, size, szc, &retry); 31382991Ssusans if (error == EINVAL && retry) { 31392991Ssusans szcvec &= ~(1 << szc); 31402991Ssusans if (szcvec <= 1) { 31412991Ssusans return (EINVAL); 31422991Ssusans } 31432991Ssusans szc = highbit(szcvec) - 1; 31442991Ssusans } else { 31452991Ssusans return (error); 31462991Ssusans } 31472991Ssusans } 31482991Ssusans } 31492991Ssusans 31502991Ssusans /* 31512991Ssusans * as_iset1_default_lpsize() breaks its chunk into areas where existing 31522991Ssusans * segments have a smaller szc than we want to set. For each such area, 31532991Ssusans * it calls as_iset2_default_lpsize() 31542991Ssusans */ 31552991Ssusans static int 31562991Ssusans as_iset1_default_lpsize(struct as *as, caddr_t raddr, size_t rsize, uint_t szc, 31572991Ssusans uint_t szcvec) 31582991Ssusans { 31592991Ssusans struct seg *seg; 31602991Ssusans size_t ssize; 31612991Ssusans caddr_t setaddr = raddr; 31622991Ssusans size_t setsize = 0; 31632991Ssusans int set; 31642991Ssusans int error; 31652991Ssusans 31662991Ssusans ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 31672991Ssusans 31682991Ssusans seg = as_segat(as, raddr); 31692991Ssusans if (seg == NULL) { 31702991Ssusans panic("as_iset1_default_lpsize: no seg"); 31712991Ssusans } 31722991Ssusans if (seg->s_szc < szc) { 31732991Ssusans set = 1; 31742991Ssusans } else { 31752991Ssusans set = 0; 31762991Ssusans } 31772991Ssusans 31782991Ssusans for (; rsize != 0; rsize -= ssize, raddr += ssize, setsize += ssize) { 31792991Ssusans if (raddr >= seg->s_base + seg->s_size) { 31802991Ssusans seg = AS_SEGNEXT(as, seg); 31812991Ssusans if (seg == NULL || raddr != seg->s_base) { 31822991Ssusans panic("as_iset1_default_lpsize: as changed"); 31832991Ssusans } 31842991Ssusans if (seg->s_szc >= szc && set) { 31852991Ssusans ASSERT(setsize != 0); 31862991Ssusans error = as_iset2_default_lpsize(as, 31872991Ssusans setaddr, setsize, szc, szcvec); 31882991Ssusans if (error) { 31892991Ssusans return (error); 31902991Ssusans } 31912991Ssusans set = 0; 31922991Ssusans } else if (seg->s_szc < szc && !set) { 31932991Ssusans setaddr = raddr; 31942991Ssusans setsize = 0; 31952991Ssusans set = 1; 31962991Ssusans } 31972991Ssusans } 31982991Ssusans if ((raddr + rsize) > (seg->s_base + seg->s_size)) { 31992991Ssusans ssize = seg->s_base + seg->s_size - raddr; 32002991Ssusans } else { 32012991Ssusans ssize = rsize; 32022991Ssusans } 32032991Ssusans } 32042991Ssusans error = 0; 32052991Ssusans if (set) { 32062991Ssusans ASSERT(setsize != 0); 32072991Ssusans error = as_iset2_default_lpsize(as, setaddr, setsize, 32082991Ssusans szc, szcvec); 32092991Ssusans } 32102991Ssusans return (error); 32112991Ssusans } 32122991Ssusans 32132991Ssusans /* 32142991Ssusans * as_iset_default_lpsize() breaks its chunk according to the size code bitmap 32152991Ssusans * returned by map_pgszcvec() (similar to as_map_segvn_segs()), and passes each 32162991Ssusans * chunk to as_iset1_default_lpsize(). 32172991Ssusans */ 32182991Ssusans static int 32192991Ssusans as_iset_default_lpsize(struct as *as, caddr_t addr, size_t size, int flags, 32202991Ssusans int type) 32212991Ssusans { 32222991Ssusans int rtype = (type & MAP_SHARED) ? MAPPGSZC_SHM : MAPPGSZC_PRIVM; 32232991Ssusans uint_t szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, 32245084Sjohnlev flags, rtype, 1); 32252991Ssusans uint_t szc; 32262991Ssusans uint_t nszc; 32272991Ssusans int error; 32282991Ssusans caddr_t a; 32292991Ssusans caddr_t eaddr; 32302991Ssusans size_t segsize; 32312991Ssusans size_t pgsz; 32322991Ssusans uint_t save_szcvec; 32332991Ssusans 32342991Ssusans ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 32352991Ssusans ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 32362991Ssusans ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 32372991Ssusans 32382991Ssusans szcvec &= ~1; 32392991Ssusans if (szcvec <= 1) { /* skip if base page size */ 32402991Ssusans return (0); 32412991Ssusans } 32422991Ssusans 32432991Ssusans /* Get the pagesize of the first larger page size. */ 32442991Ssusans szc = lowbit(szcvec) - 1; 32452991Ssusans pgsz = page_get_pagesize(szc); 32462991Ssusans eaddr = addr + size; 32472991Ssusans addr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz); 32482991Ssusans eaddr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz); 32492991Ssusans 32502991Ssusans save_szcvec = szcvec; 32512991Ssusans szcvec >>= (szc + 1); 32522991Ssusans nszc = szc; 32532991Ssusans while (szcvec) { 32542991Ssusans if ((szcvec & 0x1) == 0) { 32552991Ssusans nszc++; 32562991Ssusans szcvec >>= 1; 32572991Ssusans continue; 32582991Ssusans } 32592991Ssusans nszc++; 32602991Ssusans pgsz = page_get_pagesize(nszc); 32612991Ssusans a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz); 32622991Ssusans if (a != addr) { 32632991Ssusans ASSERT(szc > 0); 32642991Ssusans ASSERT(a < eaddr); 32652991Ssusans segsize = a - addr; 32662991Ssusans error = as_iset1_default_lpsize(as, addr, segsize, szc, 32672991Ssusans save_szcvec); 32682991Ssusans if (error) { 32692991Ssusans return (error); 32702991Ssusans } 32712991Ssusans addr = a; 32722991Ssusans } 32732991Ssusans szc = nszc; 32742991Ssusans szcvec >>= 1; 32752991Ssusans } 32762991Ssusans 32772991Ssusans ASSERT(addr < eaddr); 32782991Ssusans szcvec = save_szcvec; 32792991Ssusans while (szcvec) { 32802991Ssusans a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz); 32812991Ssusans ASSERT(a >= addr); 32822991Ssusans if (a != addr) { 32832991Ssusans ASSERT(szc > 0); 32842991Ssusans segsize = a - addr; 32852991Ssusans error = as_iset1_default_lpsize(as, addr, segsize, szc, 32862991Ssusans save_szcvec); 32872991Ssusans if (error) { 32882991Ssusans return (error); 32892991Ssusans } 32902991Ssusans addr = a; 32912991Ssusans } 32922991Ssusans szcvec &= ~(1 << szc); 32932991Ssusans if (szcvec) { 32942991Ssusans szc = highbit(szcvec) - 1; 32952991Ssusans pgsz = page_get_pagesize(szc); 32962991Ssusans } 32972991Ssusans } 32982991Ssusans ASSERT(addr == eaddr); 32992991Ssusans 33002991Ssusans return (0); 33012991Ssusans } 33022991Ssusans 33032991Ssusans /* 33042991Ssusans * Set the default large page size for the range. Called via memcntl with 33052991Ssusans * page size set to 0. as_set_default_lpsize breaks the range down into 33062991Ssusans * chunks with the same type/flags, ignores-non segvn segments, and passes 33072991Ssusans * each chunk to as_iset_default_lpsize(). 33082991Ssusans */ 33092991Ssusans int 33102991Ssusans as_set_default_lpsize(struct as *as, caddr_t addr, size_t size) 33112991Ssusans { 33122991Ssusans struct seg *seg; 33132991Ssusans caddr_t raddr; 33142991Ssusans size_t rsize; 33152991Ssusans size_t ssize; 33162991Ssusans int rtype, rflags; 33172991Ssusans int stype, sflags; 33182991Ssusans int error; 33192991Ssusans caddr_t setaddr; 33202991Ssusans size_t setsize; 33212991Ssusans int segvn; 33222991Ssusans 33232991Ssusans if (size == 0) 33242991Ssusans return (0); 33252991Ssusans 33262991Ssusans AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 33272991Ssusans again: 33282991Ssusans error = 0; 33292991Ssusans 33302991Ssusans raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 33312991Ssusans rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 33322991Ssusans (size_t)raddr; 33332991Ssusans 33342991Ssusans if (raddr + rsize < raddr) { /* check for wraparound */ 33352991Ssusans AS_LOCK_EXIT(as, &as->a_lock); 33362991Ssusans return (ENOMEM); 33372991Ssusans } 33382991Ssusans as_clearwatchprot(as, raddr, rsize); 33392991Ssusans seg = as_segat(as, raddr); 33402991Ssusans if (seg == NULL) { 33412991Ssusans as_setwatch(as); 33422991Ssusans AS_LOCK_EXIT(as, &as->a_lock); 33432991Ssusans return (ENOMEM); 33442991Ssusans } 33452991Ssusans if (seg->s_ops == &segvn_ops) { 33462991Ssusans rtype = SEGOP_GETTYPE(seg, addr); 33472991Ssusans rflags = rtype & (MAP_TEXT | MAP_INITDATA); 33482991Ssusans rtype = rtype & (MAP_SHARED | MAP_PRIVATE); 33492991Ssusans segvn = 1; 33502991Ssusans } else { 33512991Ssusans segvn = 0; 33522991Ssusans } 33532991Ssusans setaddr = raddr; 33542991Ssusans setsize = 0; 33552991Ssusans 33562991Ssusans for (; rsize != 0; rsize -= ssize, raddr += ssize, setsize += ssize) { 33572991Ssusans if (raddr >= (seg->s_base + seg->s_size)) { 33582991Ssusans seg = AS_SEGNEXT(as, seg); 33592991Ssusans if (seg == NULL || raddr != seg->s_base) { 33602991Ssusans error = ENOMEM; 33612991Ssusans break; 33622991Ssusans } 33632991Ssusans if (seg->s_ops == &segvn_ops) { 33642991Ssusans stype = SEGOP_GETTYPE(seg, raddr); 33652991Ssusans sflags = stype & (MAP_TEXT | MAP_INITDATA); 33662991Ssusans stype &= (MAP_SHARED | MAP_PRIVATE); 33672991Ssusans if (segvn && (rflags != sflags || 33682991Ssusans rtype != stype)) { 33692991Ssusans /* 33702991Ssusans * The next segment is also segvn but 33712991Ssusans * has different flags and/or type. 33722991Ssusans */ 33732991Ssusans ASSERT(setsize != 0); 33742991Ssusans error = as_iset_default_lpsize(as, 33752991Ssusans setaddr, setsize, rflags, rtype); 33762991Ssusans if (error) { 33772991Ssusans break; 33782991Ssusans } 33792991Ssusans rflags = sflags; 33802991Ssusans rtype = stype; 33812991Ssusans setaddr = raddr; 33822991Ssusans setsize = 0; 33832991Ssusans } else if (!segvn) { 33842991Ssusans rflags = sflags; 33852991Ssusans rtype = stype; 33862991Ssusans setaddr = raddr; 33872991Ssusans setsize = 0; 33882991Ssusans segvn = 1; 33892991Ssusans } 33902991Ssusans } else if (segvn) { 33912991Ssusans /* The next segment is not segvn. */ 33922991Ssusans ASSERT(setsize != 0); 33932991Ssusans error = as_iset_default_lpsize(as, 33942991Ssusans setaddr, setsize, rflags, rtype); 33952991Ssusans if (error) { 33962991Ssusans break; 33972991Ssusans } 33982991Ssusans segvn = 0; 33992991Ssusans } 34002991Ssusans } 34012991Ssusans if ((raddr + rsize) > (seg->s_base + seg->s_size)) { 34022991Ssusans ssize = seg->s_base + seg->s_size - raddr; 34032991Ssusans } else { 34042991Ssusans ssize = rsize; 34052991Ssusans } 34062991Ssusans } 34072991Ssusans if (error == 0 && segvn) { 34082991Ssusans /* The last chunk when rsize == 0. */ 34092991Ssusans ASSERT(setsize != 0); 34102991Ssusans error = as_iset_default_lpsize(as, setaddr, setsize, 34112991Ssusans rflags, rtype); 34122991Ssusans } 34132991Ssusans 34142991Ssusans if (error == IE_RETRY) { 34152991Ssusans goto again; 34162991Ssusans } else if (error == IE_NOMEM) { 34172991Ssusans error = EAGAIN; 34182991Ssusans } else if (error == ENOTSUP) { 34192991Ssusans error = EINVAL; 34202991Ssusans } else if (error == EAGAIN) { 34212991Ssusans mutex_enter(&as->a_contents); 34226695Saguzovsk if (!AS_ISNOUNMAPWAIT(as)) { 34236695Saguzovsk if (AS_ISUNMAPWAIT(as) == 0) { 34246695Saguzovsk cv_broadcast(&as->a_cv); 34256695Saguzovsk } 34266695Saguzovsk AS_SETUNMAPWAIT(as); 34276695Saguzovsk AS_LOCK_EXIT(as, &as->a_lock); 34286695Saguzovsk while (AS_ISUNMAPWAIT(as)) { 34296695Saguzovsk cv_wait(&as->a_cv, &as->a_contents); 34306695Saguzovsk } 34316695Saguzovsk mutex_exit(&as->a_contents); 34326695Saguzovsk AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 34336695Saguzovsk } else { 34346695Saguzovsk /* 34356695Saguzovsk * We may have raced with 34366695Saguzovsk * segvn_reclaim()/segspt_reclaim(). In this case 34376695Saguzovsk * clean nounmapwait flag and retry since softlockcnt 34386695Saguzovsk * in this segment may be already 0. We don't drop as 34396695Saguzovsk * writer lock so our number of retries without 34406695Saguzovsk * sleeping should be very small. See segvn_reclaim() 34416695Saguzovsk * for more comments. 34426695Saguzovsk */ 34436695Saguzovsk AS_CLRNOUNMAPWAIT(as); 34446695Saguzovsk mutex_exit(&as->a_contents); 34452991Ssusans } 34462991Ssusans goto again; 34472991Ssusans } 34482991Ssusans 34492991Ssusans as_setwatch(as); 34502991Ssusans AS_LOCK_EXIT(as, &as->a_lock); 34512991Ssusans return (error); 34522991Ssusans } 34532991Ssusans 34542991Ssusans /* 34550Sstevel@tonic-gate * Setup all of the uninitialized watched pages that we can. 34560Sstevel@tonic-gate */ 34570Sstevel@tonic-gate void 34580Sstevel@tonic-gate as_setwatch(struct as *as) 34590Sstevel@tonic-gate { 34600Sstevel@tonic-gate struct watched_page *pwp; 34610Sstevel@tonic-gate struct seg *seg; 34620Sstevel@tonic-gate caddr_t vaddr; 34630Sstevel@tonic-gate uint_t prot; 34640Sstevel@tonic-gate int err, retrycnt; 34650Sstevel@tonic-gate 34660Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) == 0) 34670Sstevel@tonic-gate return; 34680Sstevel@tonic-gate 34690Sstevel@tonic-gate ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 34700Sstevel@tonic-gate 34710Sstevel@tonic-gate for (pwp = avl_first(&as->a_wpage); pwp != NULL; 34720Sstevel@tonic-gate pwp = AVL_NEXT(&as->a_wpage, pwp)) { 34730Sstevel@tonic-gate retrycnt = 0; 34740Sstevel@tonic-gate retry: 34750Sstevel@tonic-gate vaddr = pwp->wp_vaddr; 34760Sstevel@tonic-gate if (pwp->wp_oprot != 0 || /* already set up */ 34770Sstevel@tonic-gate (seg = as_segat(as, vaddr)) == NULL || 34780Sstevel@tonic-gate SEGOP_GETPROT(seg, vaddr, 0, &prot) != 0) 34790Sstevel@tonic-gate continue; 34800Sstevel@tonic-gate 34810Sstevel@tonic-gate pwp->wp_oprot = prot; 34820Sstevel@tonic-gate if (pwp->wp_read) 34830Sstevel@tonic-gate prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 34840Sstevel@tonic-gate if (pwp->wp_write) 34850Sstevel@tonic-gate prot &= ~PROT_WRITE; 34860Sstevel@tonic-gate if (pwp->wp_exec) 34870Sstevel@tonic-gate prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 34880Sstevel@tonic-gate if (!(pwp->wp_flags & WP_NOWATCH) && prot != pwp->wp_oprot) { 34890Sstevel@tonic-gate err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot); 34900Sstevel@tonic-gate if (err == IE_RETRY) { 34910Sstevel@tonic-gate pwp->wp_oprot = 0; 34920Sstevel@tonic-gate ASSERT(retrycnt == 0); 34930Sstevel@tonic-gate retrycnt++; 34940Sstevel@tonic-gate goto retry; 34950Sstevel@tonic-gate } 34960Sstevel@tonic-gate } 34970Sstevel@tonic-gate pwp->wp_prot = prot; 34980Sstevel@tonic-gate } 34990Sstevel@tonic-gate } 35000Sstevel@tonic-gate 35010Sstevel@tonic-gate /* 35020Sstevel@tonic-gate * Clear all of the watched pages in the address space. 35030Sstevel@tonic-gate */ 35040Sstevel@tonic-gate void 35050Sstevel@tonic-gate as_clearwatch(struct as *as) 35060Sstevel@tonic-gate { 35070Sstevel@tonic-gate struct watched_page *pwp; 35080Sstevel@tonic-gate struct seg *seg; 35090Sstevel@tonic-gate caddr_t vaddr; 35100Sstevel@tonic-gate uint_t prot; 35110Sstevel@tonic-gate int err, retrycnt; 35120Sstevel@tonic-gate 35130Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) == 0) 35140Sstevel@tonic-gate return; 35150Sstevel@tonic-gate 35160Sstevel@tonic-gate ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 35170Sstevel@tonic-gate 35180Sstevel@tonic-gate for (pwp = avl_first(&as->a_wpage); pwp != NULL; 35190Sstevel@tonic-gate pwp = AVL_NEXT(&as->a_wpage, pwp)) { 35200Sstevel@tonic-gate retrycnt = 0; 35210Sstevel@tonic-gate retry: 35220Sstevel@tonic-gate vaddr = pwp->wp_vaddr; 35230Sstevel@tonic-gate if (pwp->wp_oprot == 0 || /* not set up */ 35240Sstevel@tonic-gate (seg = as_segat(as, vaddr)) == NULL) 35250Sstevel@tonic-gate continue; 35260Sstevel@tonic-gate 35270Sstevel@tonic-gate if ((prot = pwp->wp_oprot) != pwp->wp_prot) { 35280Sstevel@tonic-gate err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot); 35290Sstevel@tonic-gate if (err == IE_RETRY) { 35300Sstevel@tonic-gate ASSERT(retrycnt == 0); 35310Sstevel@tonic-gate retrycnt++; 35320Sstevel@tonic-gate goto retry; 35330Sstevel@tonic-gate } 35340Sstevel@tonic-gate } 35350Sstevel@tonic-gate pwp->wp_oprot = 0; 35360Sstevel@tonic-gate pwp->wp_prot = 0; 35370Sstevel@tonic-gate } 35380Sstevel@tonic-gate } 35390Sstevel@tonic-gate 35400Sstevel@tonic-gate /* 35410Sstevel@tonic-gate * Force a new setup for all the watched pages in the range. 35420Sstevel@tonic-gate */ 35430Sstevel@tonic-gate static void 35440Sstevel@tonic-gate as_setwatchprot(struct as *as, caddr_t addr, size_t size, uint_t prot) 35450Sstevel@tonic-gate { 35460Sstevel@tonic-gate struct watched_page *pwp; 35470Sstevel@tonic-gate struct watched_page tpw; 35480Sstevel@tonic-gate caddr_t eaddr = addr + size; 35490Sstevel@tonic-gate caddr_t vaddr; 35500Sstevel@tonic-gate struct seg *seg; 35510Sstevel@tonic-gate int err, retrycnt; 35520Sstevel@tonic-gate uint_t wprot; 35530Sstevel@tonic-gate avl_index_t where; 35540Sstevel@tonic-gate 35550Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) == 0) 35560Sstevel@tonic-gate return; 35570Sstevel@tonic-gate 35580Sstevel@tonic-gate ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 35590Sstevel@tonic-gate 35600Sstevel@tonic-gate tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 35610Sstevel@tonic-gate if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL) 35620Sstevel@tonic-gate pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER); 35630Sstevel@tonic-gate 35640Sstevel@tonic-gate while (pwp != NULL && pwp->wp_vaddr < eaddr) { 35650Sstevel@tonic-gate retrycnt = 0; 35660Sstevel@tonic-gate vaddr = pwp->wp_vaddr; 35670Sstevel@tonic-gate 35680Sstevel@tonic-gate wprot = prot; 35690Sstevel@tonic-gate if (pwp->wp_read) 35700Sstevel@tonic-gate wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 35710Sstevel@tonic-gate if (pwp->wp_write) 35720Sstevel@tonic-gate wprot &= ~PROT_WRITE; 35730Sstevel@tonic-gate if (pwp->wp_exec) 35740Sstevel@tonic-gate wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 35750Sstevel@tonic-gate if (!(pwp->wp_flags & WP_NOWATCH) && wprot != pwp->wp_oprot) { 35760Sstevel@tonic-gate retry: 35770Sstevel@tonic-gate seg = as_segat(as, vaddr); 35780Sstevel@tonic-gate if (seg == NULL) { 35790Sstevel@tonic-gate panic("as_setwatchprot: no seg"); 35800Sstevel@tonic-gate /*NOTREACHED*/ 35810Sstevel@tonic-gate } 35820Sstevel@tonic-gate err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, wprot); 35830Sstevel@tonic-gate if (err == IE_RETRY) { 35840Sstevel@tonic-gate ASSERT(retrycnt == 0); 35850Sstevel@tonic-gate retrycnt++; 35860Sstevel@tonic-gate goto retry; 35870Sstevel@tonic-gate } 35880Sstevel@tonic-gate } 35890Sstevel@tonic-gate pwp->wp_oprot = prot; 35900Sstevel@tonic-gate pwp->wp_prot = wprot; 35910Sstevel@tonic-gate 35920Sstevel@tonic-gate pwp = AVL_NEXT(&as->a_wpage, pwp); 35930Sstevel@tonic-gate } 35940Sstevel@tonic-gate } 35950Sstevel@tonic-gate 35960Sstevel@tonic-gate /* 35970Sstevel@tonic-gate * Clear all of the watched pages in the range. 35980Sstevel@tonic-gate */ 35990Sstevel@tonic-gate static void 36000Sstevel@tonic-gate as_clearwatchprot(struct as *as, caddr_t addr, size_t size) 36010Sstevel@tonic-gate { 36020Sstevel@tonic-gate caddr_t eaddr = addr + size; 36030Sstevel@tonic-gate struct watched_page *pwp; 36040Sstevel@tonic-gate struct watched_page tpw; 36050Sstevel@tonic-gate uint_t prot; 36060Sstevel@tonic-gate struct seg *seg; 36070Sstevel@tonic-gate int err, retrycnt; 36080Sstevel@tonic-gate avl_index_t where; 36090Sstevel@tonic-gate 36100Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) == 0) 36110Sstevel@tonic-gate return; 36120Sstevel@tonic-gate 36130Sstevel@tonic-gate tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 36140Sstevel@tonic-gate if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL) 36150Sstevel@tonic-gate pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER); 36160Sstevel@tonic-gate 36170Sstevel@tonic-gate ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 36180Sstevel@tonic-gate 36190Sstevel@tonic-gate while (pwp != NULL && pwp->wp_vaddr < eaddr) { 36200Sstevel@tonic-gate 36210Sstevel@tonic-gate if ((prot = pwp->wp_oprot) != 0) { 36220Sstevel@tonic-gate retrycnt = 0; 36230Sstevel@tonic-gate 36240Sstevel@tonic-gate if (prot != pwp->wp_prot) { 36250Sstevel@tonic-gate retry: 36260Sstevel@tonic-gate seg = as_segat(as, pwp->wp_vaddr); 36270Sstevel@tonic-gate if (seg == NULL) 36280Sstevel@tonic-gate continue; 36290Sstevel@tonic-gate err = SEGOP_SETPROT(seg, pwp->wp_vaddr, 36300Sstevel@tonic-gate PAGESIZE, prot); 36310Sstevel@tonic-gate if (err == IE_RETRY) { 36320Sstevel@tonic-gate ASSERT(retrycnt == 0); 36330Sstevel@tonic-gate retrycnt++; 36340Sstevel@tonic-gate goto retry; 36350Sstevel@tonic-gate 36360Sstevel@tonic-gate } 36370Sstevel@tonic-gate } 36380Sstevel@tonic-gate pwp->wp_oprot = 0; 36390Sstevel@tonic-gate pwp->wp_prot = 0; 36400Sstevel@tonic-gate } 36410Sstevel@tonic-gate 36420Sstevel@tonic-gate pwp = AVL_NEXT(&as->a_wpage, pwp); 36430Sstevel@tonic-gate } 36440Sstevel@tonic-gate } 36450Sstevel@tonic-gate 36460Sstevel@tonic-gate void 36470Sstevel@tonic-gate as_signal_proc(struct as *as, k_siginfo_t *siginfo) 36480Sstevel@tonic-gate { 36490Sstevel@tonic-gate struct proc *p; 36500Sstevel@tonic-gate 36510Sstevel@tonic-gate mutex_enter(&pidlock); 36520Sstevel@tonic-gate for (p = practive; p; p = p->p_next) { 36530Sstevel@tonic-gate if (p->p_as == as) { 36540Sstevel@tonic-gate mutex_enter(&p->p_lock); 36550Sstevel@tonic-gate if (p->p_as == as) 36560Sstevel@tonic-gate sigaddq(p, NULL, siginfo, KM_NOSLEEP); 36570Sstevel@tonic-gate mutex_exit(&p->p_lock); 36580Sstevel@tonic-gate } 36590Sstevel@tonic-gate } 36600Sstevel@tonic-gate mutex_exit(&pidlock); 36610Sstevel@tonic-gate } 36620Sstevel@tonic-gate 36630Sstevel@tonic-gate /* 36640Sstevel@tonic-gate * return memory object ID 36650Sstevel@tonic-gate */ 36660Sstevel@tonic-gate int 36670Sstevel@tonic-gate as_getmemid(struct as *as, caddr_t addr, memid_t *memidp) 36680Sstevel@tonic-gate { 36690Sstevel@tonic-gate struct seg *seg; 36700Sstevel@tonic-gate int sts; 36710Sstevel@tonic-gate 36720Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 36730Sstevel@tonic-gate seg = as_segat(as, addr); 36740Sstevel@tonic-gate if (seg == NULL) { 36750Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 36760Sstevel@tonic-gate return (EFAULT); 36770Sstevel@tonic-gate } 36780Sstevel@tonic-gate /* 36790Sstevel@tonic-gate * catch old drivers which may not support getmemid 36800Sstevel@tonic-gate */ 36810Sstevel@tonic-gate if (seg->s_ops->getmemid == NULL) { 36820Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 36830Sstevel@tonic-gate return (ENODEV); 36840Sstevel@tonic-gate } 36850Sstevel@tonic-gate 36860Sstevel@tonic-gate sts = SEGOP_GETMEMID(seg, addr, memidp); 36870Sstevel@tonic-gate 36880Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 36890Sstevel@tonic-gate return (sts); 36900Sstevel@tonic-gate } 3691