xref: /netbsd-src/sys/uvm/uvm_device.c (revision daf6c4152fcddc27c445489775ed1f66ab4ea9a9)
1 /*	$NetBSD: uvm_device.c,v 1.60 2011/02/12 14:45:31 jmcneill Exp $	*/
2 
3 /*
4  * Copyright (c) 1997 Charles D. Cranor and Washington University.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  *
27  * from: Id: uvm_device.c,v 1.1.2.9 1998/02/06 05:11:47 chs Exp
28  */
29 
30 /*
31  * uvm_device.c: the device pager.
32  */
33 
34 #include <sys/cdefs.h>
35 __KERNEL_RCSID(0, "$NetBSD: uvm_device.c,v 1.60 2011/02/12 14:45:31 jmcneill Exp $");
36 
37 #include "opt_uvmhist.h"
38 
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/conf.h>
42 #include <sys/proc.h>
43 #include <sys/malloc.h>
44 
45 #include <uvm/uvm.h>
46 #include <uvm/uvm_device.h>
47 #include <uvm/uvm_pmap.h>
48 
49 /*
50  * private global data structure
51  *
52  * we keep a list of active device objects in the system.
53  */
54 
55 LIST_HEAD(udv_list_struct, uvm_device);
56 static struct udv_list_struct udv_list;
57 static kmutex_t udv_lock;
58 
59 /*
60  * functions
61  */
62 
63 static void	udv_init(void);
64 static void	udv_reference(struct uvm_object *);
65 static void	udv_detach(struct uvm_object *);
66 static int	udv_fault(struct uvm_faultinfo *, vaddr_t,
67 			  struct vm_page **, int, int, vm_prot_t,
68 			  int);
69 
70 /*
71  * master pager structure
72  */
73 
74 const struct uvm_pagerops uvm_deviceops = {
75 	.pgo_init = udv_init,
76 	.pgo_reference = udv_reference,
77 	.pgo_detach = udv_detach,
78 	.pgo_fault = udv_fault,
79 };
80 
81 /*
82  * the ops!
83  */
84 
85 /*
86  * udv_init
87  *
88  * init pager private data structures.
89  */
90 
91 static void
92 udv_init(void)
93 {
94 	LIST_INIT(&udv_list);
95 	mutex_init(&udv_lock, MUTEX_DEFAULT, IPL_NONE);
96 }
97 
98 /*
99  * udv_attach
100  *
101  * get a VM object that is associated with a device.   allocate a new
102  * one if needed.
103  *
104  * => caller must _not_ already be holding the lock on the uvm_object.
105  * => in fact, nothing should be locked so that we can sleep here.
106  */
107 
108 struct uvm_object *
109 udv_attach(void *arg, vm_prot_t accessprot,
110     voff_t off,		/* used only for access check */
111     vsize_t size	/* used only for access check */)
112 {
113 	dev_t device = *((dev_t *)arg);
114 	struct uvm_device *udv, *lcv;
115 	const struct cdevsw *cdev;
116 	dev_type_mmap((*mapfn));
117 
118 	UVMHIST_FUNC("udv_attach"); UVMHIST_CALLED(maphist);
119 
120 	UVMHIST_LOG(maphist, "(device=0x%x)", device,0,0,0);
121 
122 	/*
123 	 * before we do anything, ensure this device supports mmap
124 	 */
125 
126 	cdev = cdevsw_lookup(device);
127 	if (cdev == NULL) {
128 		return (NULL);
129 	}
130 	mapfn = cdev->d_mmap;
131 	if (mapfn == NULL || mapfn == nommap || mapfn == nullmmap) {
132 		return(NULL);
133 	}
134 
135 	/*
136 	 * Negative offsets on the object are not allowed.
137 	 */
138 
139 	if ((cdev->d_flag & D_NEGOFFSAFE) == 0 &&
140 	    off != UVM_UNKNOWN_OFFSET && off < 0)
141 		return(NULL);
142 
143 	/*
144 	 * Check that the specified range of the device allows the
145 	 * desired protection.
146 	 *
147 	 * XXX assumes VM_PROT_* == PROT_*
148 	 * XXX clobbers off and size, but nothing else here needs them.
149 	 */
150 
151 	while (size != 0) {
152 		if (cdev_mmap(device, off, accessprot) == -1) {
153 			return (NULL);
154 		}
155 		off += PAGE_SIZE; size -= PAGE_SIZE;
156 	}
157 
158 	/*
159 	 * keep looping until we get it
160 	 */
161 
162 	for (;;) {
163 
164 		/*
165 		 * first, attempt to find it on the main list
166 		 */
167 
168 		mutex_enter(&udv_lock);
169 		LIST_FOREACH(lcv, &udv_list, u_list) {
170 			if (device == lcv->u_device)
171 				break;
172 		}
173 
174 		/*
175 		 * got it on main list.  put a hold on it and unlock udv_lock.
176 		 */
177 
178 		if (lcv) {
179 
180 			/*
181 			 * if someone else has a hold on it, sleep and start
182 			 * over again.
183 			 */
184 
185 			if (lcv->u_flags & UVM_DEVICE_HOLD) {
186 				lcv->u_flags |= UVM_DEVICE_WANTED;
187 				UVM_UNLOCK_AND_WAIT(lcv, &udv_lock, false,
188 				    "udv_attach",0);
189 				continue;
190 			}
191 
192 			/* we are now holding it */
193 			lcv->u_flags |= UVM_DEVICE_HOLD;
194 			mutex_exit(&udv_lock);
195 
196 			/*
197 			 * bump reference count, unhold, return.
198 			 */
199 
200 			mutex_enter(&lcv->u_obj.vmobjlock);
201 			lcv->u_obj.uo_refs++;
202 			mutex_exit(&lcv->u_obj.vmobjlock);
203 
204 			mutex_enter(&udv_lock);
205 			if (lcv->u_flags & UVM_DEVICE_WANTED)
206 				wakeup(lcv);
207 			lcv->u_flags &= ~(UVM_DEVICE_WANTED|UVM_DEVICE_HOLD);
208 			mutex_exit(&udv_lock);
209 			return(&lcv->u_obj);
210 		}
211 
212 		/*
213 		 * did not find it on main list.   need to malloc a new one.
214 		 */
215 
216 		mutex_exit(&udv_lock);
217 		/* NOTE: we could sleep in the following malloc() */
218 		udv = malloc(sizeof(*udv), M_TEMP, M_WAITOK);
219 		mutex_enter(&udv_lock);
220 
221 		/*
222 		 * now we have to double check to make sure no one added it
223 		 * to the list while we were sleeping...
224 		 */
225 
226 		LIST_FOREACH(lcv, &udv_list, u_list) {
227 			if (device == lcv->u_device)
228 				break;
229 		}
230 
231 		/*
232 		 * did we lose a race to someone else?
233 		 * free our memory and retry.
234 		 */
235 
236 		if (lcv) {
237 			mutex_exit(&udv_lock);
238 			free(udv, M_TEMP);
239 			continue;
240 		}
241 
242 		/*
243 		 * we have it!   init the data structures, add to list
244 		 * and return.
245 		 */
246 
247 		UVM_OBJ_INIT(&udv->u_obj, &uvm_deviceops, 1);
248 		udv->u_flags = 0;
249 		udv->u_device = device;
250 		LIST_INSERT_HEAD(&udv_list, udv, u_list);
251 		mutex_exit(&udv_lock);
252 		return(&udv->u_obj);
253 	}
254 	/*NOTREACHED*/
255 }
256 
257 /*
258  * udv_reference
259  *
260  * add a reference to a VM object.   Note that the reference count must
261  * already be one (the passed in reference) so there is no chance of the
262  * udv being released or locked out here.
263  *
264  * => caller must call with object unlocked.
265  */
266 
267 static void
268 udv_reference(struct uvm_object *uobj)
269 {
270 	UVMHIST_FUNC("udv_reference"); UVMHIST_CALLED(maphist);
271 
272 	mutex_enter(&uobj->vmobjlock);
273 	uobj->uo_refs++;
274 	UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)",
275 		    uobj, uobj->uo_refs,0,0);
276 	mutex_exit(&uobj->vmobjlock);
277 }
278 
279 /*
280  * udv_detach
281  *
282  * remove a reference to a VM object.
283  *
284  * => caller must call with object unlocked and map locked.
285  */
286 
287 static void
288 udv_detach(struct uvm_object *uobj)
289 {
290 	struct uvm_device *udv = (struct uvm_device *)uobj;
291 	UVMHIST_FUNC("udv_detach"); UVMHIST_CALLED(maphist);
292 
293 	/*
294 	 * loop until done
295 	 */
296 again:
297 	mutex_enter(&uobj->vmobjlock);
298 	if (uobj->uo_refs > 1) {
299 		uobj->uo_refs--;
300 		mutex_exit(&uobj->vmobjlock);
301 		UVMHIST_LOG(maphist," <- done, uobj=0x%x, ref=%d",
302 			  uobj,uobj->uo_refs,0,0);
303 		return;
304 	}
305 
306 	/*
307 	 * is it being held?   if so, wait until others are done.
308 	 */
309 
310 	mutex_enter(&udv_lock);
311 	if (udv->u_flags & UVM_DEVICE_HOLD) {
312 		udv->u_flags |= UVM_DEVICE_WANTED;
313 		mutex_exit(&uobj->vmobjlock);
314 		UVM_UNLOCK_AND_WAIT(udv, &udv_lock, false, "udv_detach",0);
315 		goto again;
316 	}
317 
318 	/*
319 	 * got it!   nuke it now.
320 	 */
321 
322 	LIST_REMOVE(udv, u_list);
323 	if (udv->u_flags & UVM_DEVICE_WANTED)
324 		wakeup(udv);
325 	mutex_exit(&udv_lock);
326 	mutex_exit(&uobj->vmobjlock);
327 	UVM_OBJ_DESTROY(uobj);
328 	free(udv, M_TEMP);
329 	UVMHIST_LOG(maphist," <- done, freed uobj=0x%x", uobj,0,0,0);
330 }
331 
332 /*
333  * udv_fault: non-standard fault routine for device "pages"
334  *
335  * => rather than having a "get" function, we have a fault routine
336  *	since we don't return vm_pages we need full control over the
337  *	pmap_enter map in
338  * => all the usual fault data structured are locked by the caller
339  *	(i.e. maps(read), amap (if any), uobj)
340  * => on return, we unlock all fault data structures
341  * => flags: PGO_ALLPAGES: get all of the pages
342  *	     PGO_LOCKED: fault data structures are locked
343  *    XXX: currently PGO_LOCKED is always required ... consider removing
344  *	it as a flag
345  * => NOTE: vaddr is the VA of pps[0] in ufi->entry, _NOT_ pps[centeridx]
346  */
347 
348 static int
349 udv_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, struct vm_page **pps,
350     int npages, int centeridx, vm_prot_t access_type,
351     int flags)
352 {
353 	struct vm_map_entry *entry = ufi->entry;
354 	struct uvm_object *uobj = entry->object.uvm_obj;
355 	struct uvm_device *udv = (struct uvm_device *)uobj;
356 	vaddr_t curr_va;
357 	off_t curr_offset;
358 	paddr_t paddr, mdpgno;
359 	u_int mmapflags;
360 	int lcv, retval;
361 	dev_t device;
362 	vm_prot_t mapprot;
363 	UVMHIST_FUNC("udv_fault"); UVMHIST_CALLED(maphist);
364 	UVMHIST_LOG(maphist,"  flags=%d", flags,0,0,0);
365 
366 	/*
367 	 * we do not allow device mappings to be mapped copy-on-write
368 	 * so we kill any attempt to do so here.
369 	 */
370 
371 	if (UVM_ET_ISCOPYONWRITE(entry)) {
372 		UVMHIST_LOG(maphist, "<- failed -- COW entry (etype=0x%x)",
373 		entry->etype, 0,0,0);
374 		uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj, NULL);
375 		return(EIO);
376 	}
377 
378 	/*
379 	 * get device map function.
380 	 */
381 
382 	device = udv->u_device;
383 	if (cdevsw_lookup(device) == NULL) {
384 		/* XXX This should not happen */
385 		uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj, NULL);
386 		return (EIO);
387 	}
388 
389 	/*
390 	 * now we must determine the offset in udv to use and the VA to
391 	 * use for pmap_enter.  note that we always use orig_map's pmap
392 	 * for pmap_enter (even if we have a submap).   since virtual
393 	 * addresses in a submap must match the main map, this is ok.
394 	 */
395 
396 	/* udv offset = (offset from start of entry) + entry's offset */
397 	curr_offset = entry->offset + (vaddr - entry->start);
398 	/* pmap va = vaddr (virtual address of pps[0]) */
399 	curr_va = vaddr;
400 
401 	/*
402 	 * loop over the page range entering in as needed
403 	 */
404 
405 	retval = 0;
406 	for (lcv = 0 ; lcv < npages ; lcv++, curr_offset += PAGE_SIZE,
407 	    curr_va += PAGE_SIZE) {
408 		if ((flags & PGO_ALLPAGES) == 0 && lcv != centeridx)
409 			continue;
410 
411 		if (pps[lcv] == PGO_DONTCARE)
412 			continue;
413 
414 		mdpgno = cdev_mmap(device, curr_offset, access_type);
415 		if (mdpgno == -1) {
416 			retval = EIO;
417 			break;
418 		}
419 		paddr = pmap_phys_address(mdpgno);
420 		mmapflags = pmap_mmap_flags(mdpgno);
421 		mapprot = ufi->entry->protection;
422 		UVMHIST_LOG(maphist,
423 		    "  MAPPING: device: pm=0x%x, va=0x%x, pa=0x%lx, at=%d",
424 		    ufi->orig_map->pmap, curr_va, paddr, mapprot);
425 		if (pmap_enter(ufi->orig_map->pmap, curr_va, paddr, mapprot,
426 		    PMAP_CANFAIL | mapprot | mmapflags) != 0) {
427 			/*
428 			 * pmap_enter() didn't have the resource to
429 			 * enter this mapping.  Unlock everything,
430 			 * wait for the pagedaemon to free up some
431 			 * pages, and then tell uvm_fault() to start
432 			 * the fault again.
433 			 *
434 			 * XXX Needs some rethinking for the PGO_ALLPAGES
435 			 * XXX case.
436 			 */
437 			pmap_update(ufi->orig_map->pmap);	/* sync what we have so far */
438 			uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap,
439 			    uobj, NULL);
440 			uvm_wait("udv_fault");
441 			return (ERESTART);
442 		}
443 	}
444 
445 	pmap_update(ufi->orig_map->pmap);
446 	uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj, NULL);
447 	return (retval);
448 }
449