xref: /netbsd-src/sys/uvm/uvm_page.c (revision 4d342c046e3288fb5a1edcd33cfec48c41c80664)
1 /*	$NetBSD: uvm_page.c,v 1.247 2020/09/20 10:30:05 skrll Exp $	*/
2 
3 /*-
4  * Copyright (c) 2019, 2020 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Andrew Doran.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 /*
33  * Copyright (c) 1997 Charles D. Cranor and Washington University.
34  * Copyright (c) 1991, 1993, The Regents of the University of California.
35  *
36  * All rights reserved.
37  *
38  * This code is derived from software contributed to Berkeley by
39  * The Mach Operating System project at Carnegie-Mellon University.
40  *
41  * Redistribution and use in source and binary forms, with or without
42  * modification, are permitted provided that the following conditions
43  * are met:
44  * 1. Redistributions of source code must retain the above copyright
45  *    notice, this list of conditions and the following disclaimer.
46  * 2. Redistributions in binary form must reproduce the above copyright
47  *    notice, this list of conditions and the following disclaimer in the
48  *    documentation and/or other materials provided with the distribution.
49  * 3. Neither the name of the University nor the names of its contributors
50  *    may be used to endorse or promote products derived from this software
51  *    without specific prior written permission.
52  *
53  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
54  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
57  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63  * SUCH DAMAGE.
64  *
65  *	@(#)vm_page.c   8.3 (Berkeley) 3/21/94
66  * from: Id: uvm_page.c,v 1.1.2.18 1998/02/06 05:24:42 chs Exp
67  *
68  *
69  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
70  * All rights reserved.
71  *
72  * Permission to use, copy, modify and distribute this software and
73  * its documentation is hereby granted, provided that both the copyright
74  * notice and this permission notice appear in all copies of the
75  * software, derivative works or modified versions, and any portions
76  * thereof, and that both notices appear in supporting documentation.
77  *
78  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
79  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
80  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
81  *
82  * Carnegie Mellon requests users of this software to return to
83  *
84  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
85  *  School of Computer Science
86  *  Carnegie Mellon University
87  *  Pittsburgh PA 15213-3890
88  *
89  * any improvements or extensions that they make and grant Carnegie the
90  * rights to redistribute these changes.
91  */
92 
93 /*
94  * uvm_page.c: page ops.
95  */
96 
97 #include <sys/cdefs.h>
98 __KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.247 2020/09/20 10:30:05 skrll Exp $");
99 
100 #include "opt_ddb.h"
101 #include "opt_uvm.h"
102 #include "opt_uvmhist.h"
103 #include "opt_readahead.h"
104 
105 #include <sys/param.h>
106 #include <sys/systm.h>
107 #include <sys/sched.h>
108 #include <sys/kernel.h>
109 #include <sys/vnode.h>
110 #include <sys/proc.h>
111 #include <sys/radixtree.h>
112 #include <sys/atomic.h>
113 #include <sys/cpu.h>
114 
115 #include <uvm/uvm.h>
116 #include <uvm/uvm_ddb.h>
117 #include <uvm/uvm_pdpolicy.h>
118 #include <uvm/uvm_pgflcache.h>
119 
120 /*
121  * number of pages per-CPU to reserve for the kernel.
122  */
123 #ifndef	UVM_RESERVED_PAGES_PER_CPU
124 #define	UVM_RESERVED_PAGES_PER_CPU	5
125 #endif
126 int vm_page_reserve_kernel = UVM_RESERVED_PAGES_PER_CPU;
127 
128 /*
129  * physical memory size;
130  */
131 psize_t physmem;
132 
133 /*
134  * local variables
135  */
136 
137 /*
138  * these variables record the values returned by vm_page_bootstrap,
139  * for debugging purposes.  The implementation of uvm_pageboot_alloc
140  * and pmap_startup here also uses them internally.
141  */
142 
143 static vaddr_t      virtual_space_start;
144 static vaddr_t      virtual_space_end;
145 
146 /*
147  * we allocate an initial number of page colors in uvm_page_init(),
148  * and remember them.  We may re-color pages as cache sizes are
149  * discovered during the autoconfiguration phase.  But we can never
150  * free the initial set of buckets, since they are allocated using
151  * uvm_pageboot_alloc().
152  */
153 
154 static size_t recolored_pages_memsize /* = 0 */;
155 static char *recolored_pages_mem;
156 
157 /*
158  * freelist locks - one per bucket.
159  */
160 
161 union uvm_freelist_lock	uvm_freelist_locks[PGFL_MAX_BUCKETS]
162     __cacheline_aligned;
163 
164 /*
165  * basic NUMA information.
166  */
167 
168 static struct uvm_page_numa_region {
169 	struct uvm_page_numa_region	*next;
170 	paddr_t				start;
171 	paddr_t				size;
172 	u_int				numa_id;
173 } *uvm_page_numa_region;
174 
175 #ifdef DEBUG
176 kmutex_t uvm_zerochecklock __cacheline_aligned;
177 vaddr_t uvm_zerocheckkva;
178 #endif /* DEBUG */
179 
180 /*
181  * These functions are reserved for uvm(9) internal use and are not
182  * exported in the header file uvm_physseg.h
183  *
184  * Thus they are redefined here.
185  */
186 void uvm_physseg_init_seg(uvm_physseg_t, struct vm_page *);
187 void uvm_physseg_seg_chomp_slab(uvm_physseg_t, struct vm_page *, size_t);
188 
189 /* returns a pgs array */
190 struct vm_page *uvm_physseg_seg_alloc_from_slab(uvm_physseg_t, size_t);
191 
192 /*
193  * inline functions
194  */
195 
196 /*
197  * uvm_pageinsert: insert a page in the object.
198  *
199  * => caller must lock object
200  * => call should have already set pg's object and offset pointers
201  *    and bumped the version counter
202  */
203 
204 static inline void
205 uvm_pageinsert_object(struct uvm_object *uobj, struct vm_page *pg)
206 {
207 
208 	KASSERT(uobj == pg->uobject);
209 	KASSERT(rw_write_held(uobj->vmobjlock));
210 	KASSERT((pg->flags & PG_TABLED) == 0);
211 
212 	if ((pg->flags & PG_STAT) != 0) {
213 		/* Cannot use uvm_pagegetdirty(): not yet in radix tree. */
214 		const unsigned int status = pg->flags & (PG_CLEAN | PG_DIRTY);
215 
216 		if ((pg->flags & PG_FILE) != 0) {
217 			if (uobj->uo_npages == 0) {
218 				struct vnode *vp = (struct vnode *)uobj;
219 				mutex_enter(vp->v_interlock);
220 				KASSERT((vp->v_iflag & VI_PAGES) == 0);
221 				vp->v_iflag |= VI_PAGES;
222 				vholdl(vp);
223 				mutex_exit(vp->v_interlock);
224 			}
225 			if (UVM_OBJ_IS_VTEXT(uobj)) {
226 				cpu_count(CPU_COUNT_EXECPAGES, 1);
227 			}
228 			cpu_count(CPU_COUNT_FILEUNKNOWN + status, 1);
229 		} else {
230 			cpu_count(CPU_COUNT_ANONUNKNOWN + status, 1);
231 		}
232 	}
233 	pg->flags |= PG_TABLED;
234 	uobj->uo_npages++;
235 }
236 
237 static inline int
238 uvm_pageinsert_tree(struct uvm_object *uobj, struct vm_page *pg)
239 {
240 	const uint64_t idx = pg->offset >> PAGE_SHIFT;
241 	int error;
242 
243 	KASSERT(rw_write_held(uobj->vmobjlock));
244 
245 	error = radix_tree_insert_node(&uobj->uo_pages, idx, pg);
246 	if (error != 0) {
247 		return error;
248 	}
249 	if ((pg->flags & PG_CLEAN) == 0) {
250 		uvm_obj_page_set_dirty(pg);
251 	}
252 	KASSERT(((pg->flags & PG_CLEAN) == 0) ==
253 		uvm_obj_page_dirty_p(pg));
254 	return 0;
255 }
256 
257 /*
258  * uvm_page_remove: remove page from object.
259  *
260  * => caller must lock object
261  */
262 
263 static inline void
264 uvm_pageremove_object(struct uvm_object *uobj, struct vm_page *pg)
265 {
266 
267 	KASSERT(uobj == pg->uobject);
268 	KASSERT(rw_write_held(uobj->vmobjlock));
269 	KASSERT(pg->flags & PG_TABLED);
270 
271 	if ((pg->flags & PG_STAT) != 0) {
272 		/* Cannot use uvm_pagegetdirty(): no longer in radix tree. */
273 		const unsigned int status = pg->flags & (PG_CLEAN | PG_DIRTY);
274 
275 		if ((pg->flags & PG_FILE) != 0) {
276 			if (uobj->uo_npages == 1) {
277 				struct vnode *vp = (struct vnode *)uobj;
278 				mutex_enter(vp->v_interlock);
279 				KASSERT((vp->v_iflag & VI_PAGES) != 0);
280 				vp->v_iflag &= ~VI_PAGES;
281 				holdrelel(vp);
282 				mutex_exit(vp->v_interlock);
283 			}
284 			if (UVM_OBJ_IS_VTEXT(uobj)) {
285 				cpu_count(CPU_COUNT_EXECPAGES, -1);
286 			}
287 			cpu_count(CPU_COUNT_FILEUNKNOWN + status, -1);
288 		} else {
289 			cpu_count(CPU_COUNT_ANONUNKNOWN + status, -1);
290 		}
291 	}
292 	uobj->uo_npages--;
293 	pg->flags &= ~PG_TABLED;
294 	pg->uobject = NULL;
295 }
296 
297 static inline void
298 uvm_pageremove_tree(struct uvm_object *uobj, struct vm_page *pg)
299 {
300 	struct vm_page *opg __unused;
301 
302 	KASSERT(rw_write_held(uobj->vmobjlock));
303 
304 	opg = radix_tree_remove_node(&uobj->uo_pages, pg->offset >> PAGE_SHIFT);
305 	KASSERT(pg == opg);
306 }
307 
308 static void
309 uvm_page_init_bucket(struct pgfreelist *pgfl, struct pgflbucket *pgb, int num)
310 {
311 	int i;
312 
313 	pgb->pgb_nfree = 0;
314 	for (i = 0; i < uvmexp.ncolors; i++) {
315 		LIST_INIT(&pgb->pgb_colors[i]);
316 	}
317 	pgfl->pgfl_buckets[num] = pgb;
318 }
319 
320 /*
321  * uvm_page_init: init the page system.   called from uvm_init().
322  *
323  * => we return the range of kernel virtual memory in kvm_startp/kvm_endp
324  */
325 
326 void
327 uvm_page_init(vaddr_t *kvm_startp, vaddr_t *kvm_endp)
328 {
329 	static struct uvm_cpu boot_cpu __cacheline_aligned;
330 	psize_t freepages, pagecount, bucketsize, n;
331 	struct pgflbucket *pgb;
332 	struct vm_page *pagearray;
333 	char *bucketarray;
334 	uvm_physseg_t bank;
335 	int fl, b;
336 
337 	KASSERT(ncpu <= 1);
338 
339 	/*
340 	 * init the page queues and free page queue locks, except the
341 	 * free list; we allocate that later (with the initial vm_page
342 	 * structures).
343 	 */
344 
345 	curcpu()->ci_data.cpu_uvm = &boot_cpu;
346 	uvmpdpol_init();
347 	for (b = 0; b < __arraycount(uvm_freelist_locks); b++) {
348 		mutex_init(&uvm_freelist_locks[b].lock, MUTEX_DEFAULT, IPL_VM);
349 	}
350 
351 	/*
352 	 * allocate vm_page structures.
353 	 */
354 
355 	/*
356 	 * sanity check:
357 	 * before calling this function the MD code is expected to register
358 	 * some free RAM with the uvm_page_physload() function.   our job
359 	 * now is to allocate vm_page structures for this memory.
360 	 */
361 
362 	if (uvm_physseg_get_last() == UVM_PHYSSEG_TYPE_INVALID)
363 		panic("uvm_page_bootstrap: no memory pre-allocated");
364 
365 	/*
366 	 * first calculate the number of free pages...
367 	 *
368 	 * note that we use start/end rather than avail_start/avail_end.
369 	 * this allows us to allocate extra vm_page structures in case we
370 	 * want to return some memory to the pool after booting.
371 	 */
372 
373 	freepages = 0;
374 
375 	for (bank = uvm_physseg_get_first();
376 	     uvm_physseg_valid_p(bank) ;
377 	     bank = uvm_physseg_get_next(bank)) {
378 		freepages += (uvm_physseg_get_end(bank) - uvm_physseg_get_start(bank));
379 	}
380 
381 	/*
382 	 * Let MD code initialize the number of colors, or default
383 	 * to 1 color if MD code doesn't care.
384 	 */
385 	if (uvmexp.ncolors == 0)
386 		uvmexp.ncolors = 1;
387 	uvmexp.colormask = uvmexp.ncolors - 1;
388 	KASSERT((uvmexp.colormask & uvmexp.ncolors) == 0);
389 
390 	/* We always start with only 1 bucket. */
391 	uvm.bucketcount = 1;
392 
393 	/*
394 	 * we now know we have (PAGE_SIZE * freepages) bytes of memory we can
395 	 * use.   for each page of memory we use we need a vm_page structure.
396 	 * thus, the total number of pages we can use is the total size of
397 	 * the memory divided by the PAGE_SIZE plus the size of the vm_page
398 	 * structure.   we add one to freepages as a fudge factor to avoid
399 	 * truncation errors (since we can only allocate in terms of whole
400 	 * pages).
401 	 */
402 	pagecount = ((freepages + 1) << PAGE_SHIFT) /
403 	    (PAGE_SIZE + sizeof(struct vm_page));
404 	bucketsize = offsetof(struct pgflbucket, pgb_colors[uvmexp.ncolors]);
405 	bucketsize = roundup2(bucketsize, coherency_unit);
406 	bucketarray = (void *)uvm_pageboot_alloc(
407 	    bucketsize * VM_NFREELIST +
408 	    pagecount * sizeof(struct vm_page));
409 	pagearray = (struct vm_page *)
410 	    (bucketarray + bucketsize * VM_NFREELIST);
411 
412 	for (fl = 0; fl < VM_NFREELIST; fl++) {
413 		pgb = (struct pgflbucket *)(bucketarray + bucketsize * fl);
414 		uvm_page_init_bucket(&uvm.page_free[fl], pgb, 0);
415 	}
416 	memset(pagearray, 0, pagecount * sizeof(struct vm_page));
417 
418 	/*
419 	 * init the freelist cache in the disabled state.
420 	 */
421 	uvm_pgflcache_init();
422 
423 	/*
424 	 * init the vm_page structures and put them in the correct place.
425 	 */
426 	/* First init the extent */
427 
428 	for (bank = uvm_physseg_get_first(),
429 		 uvm_physseg_seg_chomp_slab(bank, pagearray, pagecount);
430 	     uvm_physseg_valid_p(bank);
431 	     bank = uvm_physseg_get_next(bank)) {
432 
433 		n = uvm_physseg_get_end(bank) - uvm_physseg_get_start(bank);
434 		uvm_physseg_seg_alloc_from_slab(bank, n);
435 		uvm_physseg_init_seg(bank, pagearray);
436 
437 		/* set up page array pointers */
438 		pagearray += n;
439 		pagecount -= n;
440 	}
441 
442 	/*
443 	 * pass up the values of virtual_space_start and
444 	 * virtual_space_end (obtained by uvm_pageboot_alloc) to the upper
445 	 * layers of the VM.
446 	 */
447 
448 	*kvm_startp = round_page(virtual_space_start);
449 	*kvm_endp = trunc_page(virtual_space_end);
450 
451 	/*
452 	 * init various thresholds.
453 	 */
454 
455 	uvmexp.reserve_pagedaemon = 1;
456 	uvmexp.reserve_kernel = vm_page_reserve_kernel;
457 
458 	/*
459 	 * done!
460 	 */
461 
462 	uvm.page_init_done = true;
463 }
464 
465 /*
466  * uvm_pgfl_lock: lock all freelist buckets
467  */
468 
469 void
470 uvm_pgfl_lock(void)
471 {
472 	int i;
473 
474 	for (i = 0; i < __arraycount(uvm_freelist_locks); i++) {
475 		mutex_spin_enter(&uvm_freelist_locks[i].lock);
476 	}
477 }
478 
479 /*
480  * uvm_pgfl_unlock: unlock all freelist buckets
481  */
482 
483 void
484 uvm_pgfl_unlock(void)
485 {
486 	int i;
487 
488 	for (i = 0; i < __arraycount(uvm_freelist_locks); i++) {
489 		mutex_spin_exit(&uvm_freelist_locks[i].lock);
490 	}
491 }
492 
493 /*
494  * uvm_setpagesize: set the page size
495  *
496  * => sets page_shift and page_mask from uvmexp.pagesize.
497  */
498 
499 void
500 uvm_setpagesize(void)
501 {
502 
503 	/*
504 	 * If uvmexp.pagesize is 0 at this point, we expect PAGE_SIZE
505 	 * to be a constant (indicated by being a non-zero value).
506 	 */
507 	if (uvmexp.pagesize == 0) {
508 		if (PAGE_SIZE == 0)
509 			panic("uvm_setpagesize: uvmexp.pagesize not set");
510 		uvmexp.pagesize = PAGE_SIZE;
511 	}
512 	uvmexp.pagemask = uvmexp.pagesize - 1;
513 	if ((uvmexp.pagemask & uvmexp.pagesize) != 0)
514 		panic("uvm_setpagesize: page size %u (%#x) not a power of two",
515 		    uvmexp.pagesize, uvmexp.pagesize);
516 	for (uvmexp.pageshift = 0; ; uvmexp.pageshift++)
517 		if ((1 << uvmexp.pageshift) == uvmexp.pagesize)
518 			break;
519 }
520 
521 /*
522  * uvm_pageboot_alloc: steal memory from physmem for bootstrapping
523  */
524 
525 vaddr_t
526 uvm_pageboot_alloc(vsize_t size)
527 {
528 	static bool initialized = false;
529 	vaddr_t addr;
530 #if !defined(PMAP_STEAL_MEMORY)
531 	vaddr_t vaddr;
532 	paddr_t paddr;
533 #endif
534 
535 	/*
536 	 * on first call to this function, initialize ourselves.
537 	 */
538 	if (initialized == false) {
539 		pmap_virtual_space(&virtual_space_start, &virtual_space_end);
540 
541 		/* round it the way we like it */
542 		virtual_space_start = round_page(virtual_space_start);
543 		virtual_space_end = trunc_page(virtual_space_end);
544 
545 		initialized = true;
546 	}
547 
548 	/* round to page size */
549 	size = round_page(size);
550 	uvmexp.bootpages += atop(size);
551 
552 #if defined(PMAP_STEAL_MEMORY)
553 
554 	/*
555 	 * defer bootstrap allocation to MD code (it may want to allocate
556 	 * from a direct-mapped segment).  pmap_steal_memory should adjust
557 	 * virtual_space_start/virtual_space_end if necessary.
558 	 */
559 
560 	addr = pmap_steal_memory(size, &virtual_space_start,
561 	    &virtual_space_end);
562 
563 	return(addr);
564 
565 #else /* !PMAP_STEAL_MEMORY */
566 
567 	/*
568 	 * allocate virtual memory for this request
569 	 */
570 	if (virtual_space_start == virtual_space_end ||
571 	    (virtual_space_end - virtual_space_start) < size)
572 		panic("uvm_pageboot_alloc: out of virtual space");
573 
574 	addr = virtual_space_start;
575 
576 #ifdef PMAP_GROWKERNEL
577 	/*
578 	 * If the kernel pmap can't map the requested space,
579 	 * then allocate more resources for it.
580 	 */
581 	if (uvm_maxkaddr < (addr + size)) {
582 		uvm_maxkaddr = pmap_growkernel(addr + size);
583 		if (uvm_maxkaddr < (addr + size))
584 			panic("uvm_pageboot_alloc: pmap_growkernel() failed");
585 	}
586 #endif
587 
588 	virtual_space_start += size;
589 
590 	/*
591 	 * allocate and mapin physical pages to back new virtual pages
592 	 */
593 
594 	for (vaddr = round_page(addr) ; vaddr < addr + size ;
595 	    vaddr += PAGE_SIZE) {
596 
597 		if (!uvm_page_physget(&paddr))
598 			panic("uvm_pageboot_alloc: out of memory");
599 
600 		/*
601 		 * Note this memory is no longer managed, so using
602 		 * pmap_kenter is safe.
603 		 */
604 		pmap_kenter_pa(vaddr, paddr, VM_PROT_READ|VM_PROT_WRITE, 0);
605 	}
606 	pmap_update(pmap_kernel());
607 	return(addr);
608 #endif	/* PMAP_STEAL_MEMORY */
609 }
610 
611 #if !defined(PMAP_STEAL_MEMORY)
612 /*
613  * uvm_page_physget: "steal" one page from the vm_physmem structure.
614  *
615  * => attempt to allocate it off the end of a segment in which the "avail"
616  *    values match the start/end values.   if we can't do that, then we
617  *    will advance both values (making them equal, and removing some
618  *    vm_page structures from the non-avail area).
619  * => return false if out of memory.
620  */
621 
622 /* subroutine: try to allocate from memory chunks on the specified freelist */
623 static bool uvm_page_physget_freelist(paddr_t *, int);
624 
625 static bool
626 uvm_page_physget_freelist(paddr_t *paddrp, int freelist)
627 {
628 	uvm_physseg_t lcv;
629 
630 	/* pass 1: try allocating from a matching end */
631 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
632 	for (lcv = uvm_physseg_get_last(); uvm_physseg_valid_p(lcv); lcv = uvm_physseg_get_prev(lcv))
633 #else
634 	for (lcv = uvm_physseg_get_first(); uvm_physseg_valid_p(lcv); lcv = uvm_physseg_get_next(lcv))
635 #endif
636 	{
637 		if (uvm.page_init_done == true)
638 			panic("uvm_page_physget: called _after_ bootstrap");
639 
640 		/* Try to match at front or back on unused segment */
641 		if (uvm_page_physunload(lcv, freelist, paddrp))
642 			return true;
643 	}
644 
645 	/* pass2: forget about matching ends, just allocate something */
646 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
647 	for (lcv = uvm_physseg_get_last(); uvm_physseg_valid_p(lcv); lcv = uvm_physseg_get_prev(lcv))
648 #else
649 	for (lcv = uvm_physseg_get_first(); uvm_physseg_valid_p(lcv); lcv = uvm_physseg_get_next(lcv))
650 #endif
651 	{
652 		/* Try the front regardless. */
653 		if (uvm_page_physunload_force(lcv, freelist, paddrp))
654 			return true;
655 	}
656 	return false;
657 }
658 
659 bool
660 uvm_page_physget(paddr_t *paddrp)
661 {
662 	int i;
663 
664 	/* try in the order of freelist preference */
665 	for (i = 0; i < VM_NFREELIST; i++)
666 		if (uvm_page_physget_freelist(paddrp, i) == true)
667 			return (true);
668 	return (false);
669 }
670 #endif /* PMAP_STEAL_MEMORY */
671 
672 /*
673  * PHYS_TO_VM_PAGE: find vm_page for a PA.   used by MI code to get vm_pages
674  * back from an I/O mapping (ugh!).   used in some MD code as well.
675  */
676 struct vm_page *
677 uvm_phys_to_vm_page(paddr_t pa)
678 {
679 	paddr_t pf = atop(pa);
680 	paddr_t	off;
681 	uvm_physseg_t	upm;
682 
683 	upm = uvm_physseg_find(pf, &off);
684 	if (upm != UVM_PHYSSEG_TYPE_INVALID)
685 		return uvm_physseg_get_pg(upm, off);
686 	return(NULL);
687 }
688 
689 paddr_t
690 uvm_vm_page_to_phys(const struct vm_page *pg)
691 {
692 
693 	return pg->phys_addr & ~(PAGE_SIZE - 1);
694 }
695 
696 /*
697  * uvm_page_numa_load: load NUMA range description.
698  */
699 void
700 uvm_page_numa_load(paddr_t start, paddr_t size, u_int numa_id)
701 {
702 	struct uvm_page_numa_region *d;
703 
704 	KASSERT(numa_id < PGFL_MAX_BUCKETS);
705 
706 	d = kmem_alloc(sizeof(*d), KM_SLEEP);
707 	d->start = start;
708 	d->size = size;
709 	d->numa_id = numa_id;
710 	d->next = uvm_page_numa_region;
711 	uvm_page_numa_region = d;
712 }
713 
714 /*
715  * uvm_page_numa_lookup: lookup NUMA node for the given page.
716  */
717 static u_int
718 uvm_page_numa_lookup(struct vm_page *pg)
719 {
720 	struct uvm_page_numa_region *d;
721 	static bool warned;
722 	paddr_t pa;
723 
724 	KASSERT(uvm_page_numa_region != NULL);
725 
726 	pa = VM_PAGE_TO_PHYS(pg);
727 	for (d = uvm_page_numa_region; d != NULL; d = d->next) {
728 		if (pa >= d->start && pa < d->start + d->size) {
729 			return d->numa_id;
730 		}
731 	}
732 
733 	if (!warned) {
734 		printf("uvm_page_numa_lookup: failed, first pg=%p pa=%#"
735 		    PRIxPADDR "\n", pg, VM_PAGE_TO_PHYS(pg));
736 		warned = true;
737 	}
738 
739 	return 0;
740 }
741 
742 /*
743  * uvm_page_redim: adjust freelist dimensions if they have changed.
744  */
745 
746 static void
747 uvm_page_redim(int newncolors, int newnbuckets)
748 {
749 	struct pgfreelist npgfl;
750 	struct pgflbucket *opgb, *npgb;
751 	struct pgflist *ohead, *nhead;
752 	struct vm_page *pg;
753 	size_t bucketsize, bucketmemsize, oldbucketmemsize;
754 	int fl, ob, oc, nb, nc, obuckets, ocolors;
755 	char *bucketarray, *oldbucketmem, *bucketmem;
756 
757 	KASSERT(((newncolors - 1) & newncolors) == 0);
758 
759 	/* Anything to do? */
760 	if (newncolors <= uvmexp.ncolors &&
761 	    newnbuckets == uvm.bucketcount) {
762 		return;
763 	}
764 	if (uvm.page_init_done == false) {
765 		uvmexp.ncolors = newncolors;
766 		return;
767 	}
768 
769 	bucketsize = offsetof(struct pgflbucket, pgb_colors[newncolors]);
770 	bucketsize = roundup2(bucketsize, coherency_unit);
771 	bucketmemsize = bucketsize * newnbuckets * VM_NFREELIST +
772 	    coherency_unit - 1;
773 	bucketmem = kmem_zalloc(bucketmemsize, KM_SLEEP);
774 	bucketarray = (char *)roundup2((uintptr_t)bucketmem, coherency_unit);
775 
776 	ocolors = uvmexp.ncolors;
777 	obuckets = uvm.bucketcount;
778 
779 	/* Freelist cache musn't be enabled. */
780 	uvm_pgflcache_pause();
781 
782 	/* Make sure we should still do this. */
783 	uvm_pgfl_lock();
784 	if (newncolors <= uvmexp.ncolors &&
785 	    newnbuckets == uvm.bucketcount) {
786 		uvm_pgfl_unlock();
787 		uvm_pgflcache_resume();
788 		kmem_free(bucketmem, bucketmemsize);
789 		return;
790 	}
791 
792 	uvmexp.ncolors = newncolors;
793 	uvmexp.colormask = uvmexp.ncolors - 1;
794 	uvm.bucketcount = newnbuckets;
795 
796 	for (fl = 0; fl < VM_NFREELIST; fl++) {
797 		/* Init new buckets in new freelist. */
798 		memset(&npgfl, 0, sizeof(npgfl));
799 		for (nb = 0; nb < newnbuckets; nb++) {
800 			npgb = (struct pgflbucket *)bucketarray;
801 			uvm_page_init_bucket(&npgfl, npgb, nb);
802 			bucketarray += bucketsize;
803 		}
804 		/* Now transfer pages from the old freelist. */
805 		for (nb = ob = 0; ob < obuckets; ob++) {
806 			opgb = uvm.page_free[fl].pgfl_buckets[ob];
807 			for (oc = 0; oc < ocolors; oc++) {
808 				ohead = &opgb->pgb_colors[oc];
809 				while ((pg = LIST_FIRST(ohead)) != NULL) {
810 					LIST_REMOVE(pg, pageq.list);
811 					/*
812 					 * Here we decide on the NEW color &
813 					 * bucket for the page.  For NUMA
814 					 * we'll use the info that the
815 					 * hardware gave us.  For non-NUMA
816 					 * assign take physical page frame
817 					 * number and cache color into
818 					 * account.  We do this to try and
819 					 * avoid defeating any memory
820 					 * interleaving in the hardware.
821 					 */
822 					KASSERT(
823 					    uvm_page_get_bucket(pg) == ob);
824 					KASSERT(fl ==
825 					    uvm_page_get_freelist(pg));
826 					if (uvm_page_numa_region != NULL) {
827 						nb = uvm_page_numa_lookup(pg);
828 					} else {
829 						nb = atop(VM_PAGE_TO_PHYS(pg))
830 						    / uvmexp.ncolors / 8
831 						    % newnbuckets;
832 					}
833 					uvm_page_set_bucket(pg, nb);
834 					npgb = npgfl.pgfl_buckets[nb];
835 					npgb->pgb_nfree++;
836 					nc = VM_PGCOLOR(pg);
837 					nhead = &npgb->pgb_colors[nc];
838 					LIST_INSERT_HEAD(nhead, pg, pageq.list);
839 				}
840 			}
841 		}
842 		/* Install the new freelist. */
843 		memcpy(&uvm.page_free[fl], &npgfl, sizeof(npgfl));
844 	}
845 
846 	/* Unlock and free the old memory. */
847 	oldbucketmemsize = recolored_pages_memsize;
848 	oldbucketmem = recolored_pages_mem;
849 	recolored_pages_memsize = bucketmemsize;
850 	recolored_pages_mem = bucketmem;
851 
852 	uvm_pgfl_unlock();
853 	uvm_pgflcache_resume();
854 
855 	if (oldbucketmemsize) {
856 		kmem_free(oldbucketmem, oldbucketmemsize);
857 	}
858 
859 	/*
860 	 * this calls uvm_km_alloc() which may want to hold
861 	 * uvm_freelist_lock.
862 	 */
863 	uvm_pager_realloc_emerg();
864 }
865 
866 /*
867  * uvm_page_recolor: Recolor the pages if the new color count is
868  * larger than the old one.
869  */
870 
871 void
872 uvm_page_recolor(int newncolors)
873 {
874 
875 	uvm_page_redim(newncolors, uvm.bucketcount);
876 }
877 
878 /*
879  * uvm_page_rebucket: Determine a bucket structure and redim the free
880  * lists to match.
881  */
882 
883 void
884 uvm_page_rebucket(void)
885 {
886 	u_int min_numa, max_numa, npackage, shift;
887 	struct cpu_info *ci, *ci2, *ci3;
888 	CPU_INFO_ITERATOR cii;
889 
890 	/*
891 	 * If we have more than one NUMA node, and the maximum NUMA node ID
892 	 * is less than PGFL_MAX_BUCKETS, then we'll use NUMA distribution
893 	 * for free pages.
894 	 */
895 	min_numa = (u_int)-1;
896 	max_numa = 0;
897 	for (CPU_INFO_FOREACH(cii, ci)) {
898 		if (ci->ci_numa_id < min_numa) {
899 			min_numa = ci->ci_numa_id;
900 		}
901 		if (ci->ci_numa_id > max_numa) {
902 			max_numa = ci->ci_numa_id;
903 		}
904 	}
905 	if (min_numa != max_numa && max_numa < PGFL_MAX_BUCKETS) {
906 		aprint_debug("UVM: using NUMA allocation scheme\n");
907 		for (CPU_INFO_FOREACH(cii, ci)) {
908 			ci->ci_data.cpu_uvm->pgflbucket = ci->ci_numa_id;
909 		}
910 	 	uvm_page_redim(uvmexp.ncolors, max_numa + 1);
911 	 	return;
912 	}
913 
914 	/*
915 	 * Otherwise we'll go with a scheme to maximise L2/L3 cache locality
916 	 * and minimise lock contention.  Count the total number of CPU
917 	 * packages, and then try to distribute the buckets among CPU
918 	 * packages evenly.
919 	 */
920 	npackage = curcpu()->ci_nsibling[CPUREL_PACKAGE1ST];
921 
922 	/*
923 	 * Figure out how to arrange the packages & buckets, and the total
924 	 * number of buckets we need.  XXX 2 may not be the best factor.
925 	 */
926 	for (shift = 0; npackage > PGFL_MAX_BUCKETS; shift++) {
927 		npackage >>= 1;
928 	}
929  	uvm_page_redim(uvmexp.ncolors, npackage);
930 
931  	/*
932  	 * Now tell each CPU which bucket to use.  In the outer loop, scroll
933  	 * through all CPU packages.
934  	 */
935  	npackage = 0;
936 	ci = curcpu();
937 	ci2 = ci->ci_sibling[CPUREL_PACKAGE1ST];
938 	do {
939 		/*
940 		 * In the inner loop, scroll through all CPUs in the package
941 		 * and assign the same bucket ID.
942 		 */
943 		ci3 = ci2;
944 		do {
945 			ci3->ci_data.cpu_uvm->pgflbucket = npackage >> shift;
946 			ci3 = ci3->ci_sibling[CPUREL_PACKAGE];
947 		} while (ci3 != ci2);
948 		npackage++;
949 		ci2 = ci2->ci_sibling[CPUREL_PACKAGE1ST];
950 	} while (ci2 != ci->ci_sibling[CPUREL_PACKAGE1ST]);
951 
952 	aprint_debug("UVM: using package allocation scheme, "
953 	    "%d package(s) per bucket\n", 1 << shift);
954 }
955 
956 /*
957  * uvm_cpu_attach: initialize per-CPU data structures.
958  */
959 
960 void
961 uvm_cpu_attach(struct cpu_info *ci)
962 {
963 	struct uvm_cpu *ucpu;
964 
965 	/* Already done in uvm_page_init(). */
966 	if (!CPU_IS_PRIMARY(ci)) {
967 		/* Add more reserve pages for this CPU. */
968 		uvmexp.reserve_kernel += vm_page_reserve_kernel;
969 
970 		/* Allocate per-CPU data structures. */
971 		ucpu = kmem_zalloc(sizeof(struct uvm_cpu) + coherency_unit - 1,
972 		    KM_SLEEP);
973 		ucpu = (struct uvm_cpu *)roundup2((uintptr_t)ucpu,
974 		    coherency_unit);
975 		ci->ci_data.cpu_uvm = ucpu;
976 	} else {
977 		ucpu = ci->ci_data.cpu_uvm;
978 	}
979 
980 	uvmpdpol_init_cpu(ucpu);
981 
982 	/*
983 	 * Attach RNG source for this CPU's VM events
984 	 */
985         rnd_attach_source(&ucpu->rs, ci->ci_data.cpu_name, RND_TYPE_VM,
986 	    RND_FLAG_COLLECT_TIME|RND_FLAG_COLLECT_VALUE|
987 	    RND_FLAG_ESTIMATE_VALUE);
988 }
989 
990 /*
991  * uvm_availmem: fetch the total amount of free memory in pages.  this can
992  * have a detrimental effect on performance due to false sharing; don't call
993  * unless needed.
994  *
995  * some users can request the amount of free memory so often that it begins
996  * to impact upon performance.  if calling frequently and an inexact value
997  * is okay, call with cached = true.
998  */
999 
1000 int
1001 uvm_availmem(bool cached)
1002 {
1003 	int64_t fp;
1004 
1005 	cpu_count_sync(cached);
1006 	if ((fp = cpu_count_get(CPU_COUNT_FREEPAGES)) < 0) {
1007 		/*
1008 		 * XXXAD could briefly go negative because it's impossible
1009 		 * to get a clean snapshot.  address this for other counters
1010 		 * used as running totals before NetBSD 10 although less
1011 		 * important for those.
1012 		 */
1013 		fp = 0;
1014 	}
1015 	return (int)fp;
1016 }
1017 
1018 /*
1019  * uvm_pagealloc_pgb: helper routine that tries to allocate any color from a
1020  * specific freelist and specific bucket only.
1021  *
1022  * => must be at IPL_VM or higher to protect per-CPU data structures.
1023  */
1024 
1025 static struct vm_page *
1026 uvm_pagealloc_pgb(struct uvm_cpu *ucpu, int f, int b, int *trycolorp, int flags)
1027 {
1028 	int c, trycolor, colormask;
1029 	struct pgflbucket *pgb;
1030 	struct vm_page *pg;
1031 	kmutex_t *lock;
1032 	bool fill;
1033 
1034 	/*
1035 	 * Skip the bucket if empty, no lock needed.  There could be many
1036 	 * empty freelists/buckets.
1037 	 */
1038 	pgb = uvm.page_free[f].pgfl_buckets[b];
1039 	if (pgb->pgb_nfree == 0) {
1040 		return NULL;
1041 	}
1042 
1043 	/* Skip bucket if low on memory. */
1044 	lock = &uvm_freelist_locks[b].lock;
1045 	mutex_spin_enter(lock);
1046 	if (__predict_false(pgb->pgb_nfree <= uvmexp.reserve_kernel)) {
1047 		if ((flags & UVM_PGA_USERESERVE) == 0 ||
1048 		    (pgb->pgb_nfree <= uvmexp.reserve_pagedaemon &&
1049 		     curlwp != uvm.pagedaemon_lwp)) {
1050 			mutex_spin_exit(lock);
1051 		     	return NULL;
1052 		}
1053 		fill = false;
1054 	} else {
1055 		fill = true;
1056 	}
1057 
1058 	/* Try all page colors as needed. */
1059 	c = trycolor = *trycolorp;
1060 	colormask = uvmexp.colormask;
1061 	do {
1062 		pg = LIST_FIRST(&pgb->pgb_colors[c]);
1063 		if (__predict_true(pg != NULL)) {
1064 			/*
1065 			 * Got a free page!  PG_FREE must be cleared under
1066 			 * lock because of uvm_pglistalloc().
1067 			 */
1068 			LIST_REMOVE(pg, pageq.list);
1069 			KASSERT(pg->flags == PG_FREE);
1070 			pg->flags = PG_BUSY | PG_CLEAN | PG_FAKE;
1071 			pgb->pgb_nfree--;
1072 
1073 			/*
1074 			 * While we have the bucket locked and our data
1075 			 * structures fresh in L1 cache, we have an ideal
1076 			 * opportunity to grab some pages for the freelist
1077 			 * cache without causing extra contention.  Only do
1078 			 * so if we found pages in this CPU's preferred
1079 			 * bucket.
1080 			 */
1081 			if (__predict_true(b == ucpu->pgflbucket && fill)) {
1082 				uvm_pgflcache_fill(ucpu, f, b, c);
1083 			}
1084 			mutex_spin_exit(lock);
1085 			KASSERT(uvm_page_get_bucket(pg) == b);
1086 			CPU_COUNT(c == trycolor ?
1087 			    CPU_COUNT_COLORHIT : CPU_COUNT_COLORMISS, 1);
1088 			CPU_COUNT(CPU_COUNT_CPUMISS, 1);
1089 			*trycolorp = c;
1090 			return pg;
1091 		}
1092 		c = (c + 1) & colormask;
1093 	} while (c != trycolor);
1094 	mutex_spin_exit(lock);
1095 
1096 	return NULL;
1097 }
1098 
1099 /*
1100  * uvm_pagealloc_pgfl: helper routine for uvm_pagealloc_strat that allocates
1101  * any color from any bucket, in a specific freelist.
1102  *
1103  * => must be at IPL_VM or higher to protect per-CPU data structures.
1104  */
1105 
1106 static struct vm_page *
1107 uvm_pagealloc_pgfl(struct uvm_cpu *ucpu, int f, int *trycolorp, int flags)
1108 {
1109 	int b, trybucket, bucketcount;
1110 	struct vm_page *pg;
1111 
1112 	/* Try for the exact thing in the per-CPU cache. */
1113 	if ((pg = uvm_pgflcache_alloc(ucpu, f, *trycolorp)) != NULL) {
1114 		CPU_COUNT(CPU_COUNT_CPUHIT, 1);
1115 		CPU_COUNT(CPU_COUNT_COLORHIT, 1);
1116 		return pg;
1117 	}
1118 
1119 	/* Walk through all buckets, trying our preferred bucket first. */
1120 	trybucket = ucpu->pgflbucket;
1121 	b = trybucket;
1122 	bucketcount = uvm.bucketcount;
1123 	do {
1124 		pg = uvm_pagealloc_pgb(ucpu, f, b, trycolorp, flags);
1125 		if (pg != NULL) {
1126 			return pg;
1127 		}
1128 		b = (b + 1 == bucketcount ? 0 : b + 1);
1129 	} while (b != trybucket);
1130 
1131 	return NULL;
1132 }
1133 
1134 /*
1135  * uvm_pagealloc_strat: allocate vm_page from a particular free list.
1136  *
1137  * => return null if no pages free
1138  * => wake up pagedaemon if number of free pages drops below low water mark
1139  * => if obj != NULL, obj must be locked (to put in obj's tree)
1140  * => if anon != NULL, anon must be locked (to put in anon)
1141  * => only one of obj or anon can be non-null
1142  * => caller must activate/deactivate page if it is not wired.
1143  * => free_list is ignored if strat == UVM_PGA_STRAT_NORMAL.
1144  * => policy decision: it is more important to pull a page off of the
1145  *	appropriate priority free list than it is to get a page from the
1146  *	correct bucket or color bin.  This is because we live with the
1147  *	consequences of a bad free list decision for the entire
1148  *	lifetime of the page, e.g. if the page comes from memory that
1149  *	is slower to access.
1150  */
1151 
1152 struct vm_page *
1153 uvm_pagealloc_strat(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
1154     int flags, int strat, int free_list)
1155 {
1156 	int color, lcv, error, s;
1157 	struct uvm_cpu *ucpu;
1158 	struct vm_page *pg;
1159 	lwp_t *l;
1160 
1161 	KASSERT(obj == NULL || anon == NULL);
1162 	KASSERT(anon == NULL || (flags & UVM_FLAG_COLORMATCH) || off == 0);
1163 	KASSERT(off == trunc_page(off));
1164 	KASSERT(obj == NULL || rw_write_held(obj->vmobjlock));
1165 	KASSERT(anon == NULL || anon->an_lock == NULL ||
1166 	    rw_write_held(anon->an_lock));
1167 
1168 	/*
1169 	 * This implements a global round-robin page coloring
1170 	 * algorithm.
1171 	 */
1172 
1173 	s = splvm();
1174 	ucpu = curcpu()->ci_data.cpu_uvm;
1175 	if (flags & UVM_FLAG_COLORMATCH) {
1176 		color = atop(off) & uvmexp.colormask;
1177 	} else {
1178 		color = ucpu->pgflcolor;
1179 	}
1180 
1181 	/*
1182 	 * fail if any of these conditions is true:
1183 	 * [1]  there really are no free pages, or
1184 	 * [2]  only kernel "reserved" pages remain and
1185 	 *        reserved pages have not been requested.
1186 	 * [3]  only pagedaemon "reserved" pages remain and
1187 	 *        the requestor isn't the pagedaemon.
1188 	 * we make kernel reserve pages available if called by a
1189 	 * kernel thread.
1190 	 */
1191 	l = curlwp;
1192 	if (__predict_true(l != NULL) && (l->l_flag & LW_SYSTEM) != 0) {
1193 		flags |= UVM_PGA_USERESERVE;
1194 	}
1195 
1196  again:
1197 	switch (strat) {
1198 	case UVM_PGA_STRAT_NORMAL:
1199 		/* Check freelists: descending priority (ascending id) order. */
1200 		for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
1201 			pg = uvm_pagealloc_pgfl(ucpu, lcv, &color, flags);
1202 			if (pg != NULL) {
1203 				goto gotit;
1204 			}
1205 		}
1206 
1207 		/* No pages free!  Have pagedaemon free some memory. */
1208 		splx(s);
1209 		uvm_kick_pdaemon();
1210 		return NULL;
1211 
1212 	case UVM_PGA_STRAT_ONLY:
1213 	case UVM_PGA_STRAT_FALLBACK:
1214 		/* Attempt to allocate from the specified free list. */
1215 		KASSERT(free_list >= 0 && free_list < VM_NFREELIST);
1216 		pg = uvm_pagealloc_pgfl(ucpu, free_list, &color, flags);
1217 		if (pg != NULL) {
1218 			goto gotit;
1219 		}
1220 
1221 		/* Fall back, if possible. */
1222 		if (strat == UVM_PGA_STRAT_FALLBACK) {
1223 			strat = UVM_PGA_STRAT_NORMAL;
1224 			goto again;
1225 		}
1226 
1227 		/* No pages free!  Have pagedaemon free some memory. */
1228 		splx(s);
1229 		uvm_kick_pdaemon();
1230 		return NULL;
1231 
1232 	case UVM_PGA_STRAT_NUMA:
1233 		/*
1234 		 * NUMA strategy (experimental): allocating from the correct
1235 		 * bucket is more important than observing freelist
1236 		 * priority.  Look only to the current NUMA node; if that
1237 		 * fails, we need to look to other NUMA nodes, so retry with
1238 		 * the normal strategy.
1239 		 */
1240 		for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
1241 			pg = uvm_pgflcache_alloc(ucpu, lcv, color);
1242 			if (pg != NULL) {
1243 				CPU_COUNT(CPU_COUNT_CPUHIT, 1);
1244 				CPU_COUNT(CPU_COUNT_COLORHIT, 1);
1245 				goto gotit;
1246 			}
1247 			pg = uvm_pagealloc_pgb(ucpu, lcv,
1248 			    ucpu->pgflbucket, &color, flags);
1249 			if (pg != NULL) {
1250 				goto gotit;
1251 			}
1252 		}
1253 		strat = UVM_PGA_STRAT_NORMAL;
1254 		goto again;
1255 
1256 	default:
1257 		panic("uvm_pagealloc_strat: bad strat %d", strat);
1258 		/* NOTREACHED */
1259 	}
1260 
1261  gotit:
1262 	/*
1263 	 * We now know which color we actually allocated from; set
1264 	 * the next color accordingly.
1265 	 */
1266 
1267 	ucpu->pgflcolor = (color + 1) & uvmexp.colormask;
1268 
1269 	/*
1270 	 * while still at IPL_VM, update allocation statistics.
1271 	 */
1272 
1273     	CPU_COUNT(CPU_COUNT_FREEPAGES, -1);
1274 	if (anon) {
1275 		CPU_COUNT(CPU_COUNT_ANONCLEAN, 1);
1276 	}
1277 	splx(s);
1278 	KASSERT(pg->flags == (PG_BUSY|PG_CLEAN|PG_FAKE));
1279 
1280 	/*
1281 	 * assign the page to the object.  as the page was free, we know
1282 	 * that pg->uobject and pg->uanon are NULL.  we only need to take
1283 	 * the page's interlock if we are changing the values.
1284 	 */
1285 	if (anon != NULL || obj != NULL) {
1286 		mutex_enter(&pg->interlock);
1287 	}
1288 	pg->offset = off;
1289 	pg->uobject = obj;
1290 	pg->uanon = anon;
1291 	KASSERT(uvm_page_owner_locked_p(pg, true));
1292 	if (anon) {
1293 		anon->an_page = pg;
1294 		pg->flags |= PG_ANON;
1295 		mutex_exit(&pg->interlock);
1296 	} else if (obj) {
1297 		/*
1298 		 * set PG_FILE|PG_AOBJ before the first uvm_pageinsert.
1299 		 */
1300 		if (UVM_OBJ_IS_VNODE(obj)) {
1301 			pg->flags |= PG_FILE;
1302 		} else if (UVM_OBJ_IS_AOBJ(obj)) {
1303 			pg->flags |= PG_AOBJ;
1304 		}
1305 		uvm_pageinsert_object(obj, pg);
1306 		mutex_exit(&pg->interlock);
1307 		error = uvm_pageinsert_tree(obj, pg);
1308 		if (error != 0) {
1309 			mutex_enter(&pg->interlock);
1310 			uvm_pageremove_object(obj, pg);
1311 			mutex_exit(&pg->interlock);
1312 			uvm_pagefree(pg);
1313 			return NULL;
1314 		}
1315 	}
1316 
1317 #if defined(UVM_PAGE_TRKOWN)
1318 	pg->owner_tag = NULL;
1319 #endif
1320 	UVM_PAGE_OWN(pg, "new alloc");
1321 
1322 	if (flags & UVM_PGA_ZERO) {
1323 		/* A zero'd page is not clean. */
1324 		if (obj != NULL || anon != NULL) {
1325 			uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
1326 		}
1327 		pmap_zero_page(VM_PAGE_TO_PHYS(pg));
1328 	}
1329 
1330 	return(pg);
1331 }
1332 
1333 /*
1334  * uvm_pagereplace: replace a page with another
1335  *
1336  * => object must be locked
1337  * => page interlocks must be held
1338  */
1339 
1340 void
1341 uvm_pagereplace(struct vm_page *oldpg, struct vm_page *newpg)
1342 {
1343 	struct uvm_object *uobj = oldpg->uobject;
1344 	struct vm_page *pg __diagused;
1345 	uint64_t idx;
1346 
1347 	KASSERT((oldpg->flags & PG_TABLED) != 0);
1348 	KASSERT(uobj != NULL);
1349 	KASSERT((newpg->flags & PG_TABLED) == 0);
1350 	KASSERT(newpg->uobject == NULL);
1351 	KASSERT(rw_write_held(uobj->vmobjlock));
1352 	KASSERT(mutex_owned(&oldpg->interlock));
1353 	KASSERT(mutex_owned(&newpg->interlock));
1354 
1355 	newpg->uobject = uobj;
1356 	newpg->offset = oldpg->offset;
1357 	idx = newpg->offset >> PAGE_SHIFT;
1358 	pg = radix_tree_replace_node(&uobj->uo_pages, idx, newpg);
1359 	KASSERT(pg == oldpg);
1360 	if (((oldpg->flags ^ newpg->flags) & PG_CLEAN) != 0) {
1361 		if ((newpg->flags & PG_CLEAN) != 0) {
1362 			uvm_obj_page_clear_dirty(newpg);
1363 		} else {
1364 			uvm_obj_page_set_dirty(newpg);
1365 		}
1366 	}
1367 	/*
1368 	 * oldpg's PG_STAT is stable.  newpg is not reachable by others yet.
1369 	 */
1370 	newpg->flags |=
1371 	    (newpg->flags & ~PG_STAT) | (oldpg->flags & PG_STAT);
1372 	uvm_pageinsert_object(uobj, newpg);
1373 	uvm_pageremove_object(uobj, oldpg);
1374 }
1375 
1376 /*
1377  * uvm_pagerealloc: reallocate a page from one object to another
1378  *
1379  * => both objects must be locked
1380  */
1381 
1382 int
1383 uvm_pagerealloc(struct vm_page *pg, struct uvm_object *newobj, voff_t newoff)
1384 {
1385 	int error = 0;
1386 
1387 	/*
1388 	 * remove it from the old object
1389 	 */
1390 
1391 	if (pg->uobject) {
1392 		uvm_pageremove_tree(pg->uobject, pg);
1393 		uvm_pageremove_object(pg->uobject, pg);
1394 	}
1395 
1396 	/*
1397 	 * put it in the new object
1398 	 */
1399 
1400 	if (newobj) {
1401 		mutex_enter(&pg->interlock);
1402 		pg->uobject = newobj;
1403 		pg->offset = newoff;
1404 		if (UVM_OBJ_IS_VNODE(newobj)) {
1405 			pg->flags |= PG_FILE;
1406 		} else if (UVM_OBJ_IS_AOBJ(newobj)) {
1407 			pg->flags |= PG_AOBJ;
1408 		}
1409 		uvm_pageinsert_object(newobj, pg);
1410 		mutex_exit(&pg->interlock);
1411 		error = uvm_pageinsert_tree(newobj, pg);
1412 		if (error != 0) {
1413 			mutex_enter(&pg->interlock);
1414 			uvm_pageremove_object(newobj, pg);
1415 			mutex_exit(&pg->interlock);
1416 		}
1417 	}
1418 
1419 	return error;
1420 }
1421 
1422 /*
1423  * uvm_pagefree: free page
1424  *
1425  * => erase page's identity (i.e. remove from object)
1426  * => put page on free list
1427  * => caller must lock owning object (either anon or uvm_object)
1428  * => assumes all valid mappings of pg are gone
1429  */
1430 
1431 void
1432 uvm_pagefree(struct vm_page *pg)
1433 {
1434 	struct pgfreelist *pgfl;
1435 	struct pgflbucket *pgb;
1436 	struct uvm_cpu *ucpu;
1437 	kmutex_t *lock;
1438 	int bucket, s;
1439 	bool locked;
1440 
1441 #ifdef DEBUG
1442 	if (pg->uobject == (void *)0xdeadbeef &&
1443 	    pg->uanon == (void *)0xdeadbeef) {
1444 		panic("uvm_pagefree: freeing free page %p", pg);
1445 	}
1446 #endif /* DEBUG */
1447 
1448 	KASSERT((pg->flags & PG_PAGEOUT) == 0);
1449 	KASSERT(!(pg->flags & PG_FREE));
1450 	KASSERT(pg->uobject == NULL || rw_write_held(pg->uobject->vmobjlock));
1451 	KASSERT(pg->uobject != NULL || pg->uanon == NULL ||
1452 		rw_write_held(pg->uanon->an_lock));
1453 
1454 	/*
1455 	 * remove the page from the object's tree before acquiring any page
1456 	 * interlocks: this can acquire locks to free radixtree nodes.
1457 	 */
1458 	if (pg->uobject != NULL) {
1459 		uvm_pageremove_tree(pg->uobject, pg);
1460 	}
1461 
1462 	/*
1463 	 * if the page is loaned, resolve the loan instead of freeing.
1464 	 */
1465 
1466 	if (pg->loan_count) {
1467 		KASSERT(pg->wire_count == 0);
1468 
1469 		/*
1470 		 * if the page is owned by an anon then we just want to
1471 		 * drop anon ownership.  the kernel will free the page when
1472 		 * it is done with it.  if the page is owned by an object,
1473 		 * remove it from the object and mark it dirty for the benefit
1474 		 * of possible anon owners.
1475 		 *
1476 		 * regardless of previous ownership, wakeup any waiters,
1477 		 * unbusy the page, and we're done.
1478 		 */
1479 
1480 		uvm_pagelock(pg);
1481 		locked = true;
1482 		if (pg->uobject != NULL) {
1483 			uvm_pageremove_object(pg->uobject, pg);
1484 			pg->flags &= ~(PG_FILE|PG_AOBJ);
1485 		} else if (pg->uanon != NULL) {
1486 			if ((pg->flags & PG_ANON) == 0) {
1487 				pg->loan_count--;
1488 			} else {
1489 				const unsigned status = uvm_pagegetdirty(pg);
1490 				pg->flags &= ~PG_ANON;
1491 				cpu_count(CPU_COUNT_ANONUNKNOWN + status, -1);
1492 			}
1493 			pg->uanon->an_page = NULL;
1494 			pg->uanon = NULL;
1495 		}
1496 		if (pg->pqflags & PQ_WANTED) {
1497 			wakeup(pg);
1498 		}
1499 		pg->pqflags &= ~PQ_WANTED;
1500 		pg->flags &= ~(PG_BUSY|PG_RELEASED|PG_PAGER1);
1501 #ifdef UVM_PAGE_TRKOWN
1502 		pg->owner_tag = NULL;
1503 #endif
1504 		KASSERT((pg->flags & PG_STAT) == 0);
1505 		if (pg->loan_count) {
1506 			KASSERT(pg->uobject == NULL);
1507 			if (pg->uanon == NULL) {
1508 				uvm_pagedequeue(pg);
1509 			}
1510 			uvm_pageunlock(pg);
1511 			return;
1512 		}
1513 	} else if (pg->uobject != NULL || pg->uanon != NULL ||
1514 	           pg->wire_count != 0) {
1515 		uvm_pagelock(pg);
1516 		locked = true;
1517 	} else {
1518 		locked = false;
1519 	}
1520 
1521 	/*
1522 	 * remove page from its object or anon.
1523 	 */
1524 	if (pg->uobject != NULL) {
1525 		uvm_pageremove_object(pg->uobject, pg);
1526 	} else if (pg->uanon != NULL) {
1527 		const unsigned int status = uvm_pagegetdirty(pg);
1528 		pg->uanon->an_page = NULL;
1529 		pg->uanon = NULL;
1530 		cpu_count(CPU_COUNT_ANONUNKNOWN + status, -1);
1531 	}
1532 
1533 	/*
1534 	 * if the page was wired, unwire it now.
1535 	 */
1536 
1537 	if (pg->wire_count) {
1538 		pg->wire_count = 0;
1539 		atomic_dec_uint(&uvmexp.wired);
1540 	}
1541 	if (locked) {
1542 		/*
1543 		 * wake anyone waiting on the page.
1544 		 */
1545 		if ((pg->pqflags & PQ_WANTED) != 0) {
1546 			pg->pqflags &= ~PQ_WANTED;
1547 			wakeup(pg);
1548 		}
1549 
1550 		/*
1551 		 * now remove the page from the queues.
1552 		 */
1553 		uvm_pagedequeue(pg);
1554 		uvm_pageunlock(pg);
1555 	} else {
1556 		KASSERT(!uvmpdpol_pageisqueued_p(pg));
1557 	}
1558 
1559 	/*
1560 	 * and put on free queue
1561 	 */
1562 
1563 #ifdef DEBUG
1564 	pg->uobject = (void *)0xdeadbeef;
1565 	pg->uanon = (void *)0xdeadbeef;
1566 #endif /* DEBUG */
1567 
1568 	/* Try to send the page to the per-CPU cache. */
1569 	s = splvm();
1570     	CPU_COUNT(CPU_COUNT_FREEPAGES, 1);
1571 	ucpu = curcpu()->ci_data.cpu_uvm;
1572 	bucket = uvm_page_get_bucket(pg);
1573 	if (bucket == ucpu->pgflbucket && uvm_pgflcache_free(ucpu, pg)) {
1574 		splx(s);
1575 		return;
1576 	}
1577 
1578 	/* Didn't work.  Never mind, send it to a global bucket. */
1579 	pgfl = &uvm.page_free[uvm_page_get_freelist(pg)];
1580 	pgb = pgfl->pgfl_buckets[bucket];
1581 	lock = &uvm_freelist_locks[bucket].lock;
1582 
1583 	mutex_spin_enter(lock);
1584 	/* PG_FREE must be set under lock because of uvm_pglistalloc(). */
1585 	pg->flags = PG_FREE;
1586 	LIST_INSERT_HEAD(&pgb->pgb_colors[VM_PGCOLOR(pg)], pg, pageq.list);
1587 	pgb->pgb_nfree++;
1588 	mutex_spin_exit(lock);
1589 	splx(s);
1590 }
1591 
1592 /*
1593  * uvm_page_unbusy: unbusy an array of pages.
1594  *
1595  * => pages must either all belong to the same object, or all belong to anons.
1596  * => if pages are object-owned, object must be locked.
1597  * => if pages are anon-owned, anons must be locked.
1598  * => caller must make sure that anon-owned pages are not PG_RELEASED.
1599  */
1600 
1601 void
1602 uvm_page_unbusy(struct vm_page **pgs, int npgs)
1603 {
1604 	struct vm_page *pg;
1605 	int i;
1606 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
1607 
1608 	for (i = 0; i < npgs; i++) {
1609 		pg = pgs[i];
1610 		if (pg == NULL || pg == PGO_DONTCARE) {
1611 			continue;
1612 		}
1613 
1614 		KASSERT(uvm_page_owner_locked_p(pg, true));
1615 		KASSERT(pg->flags & PG_BUSY);
1616 		KASSERT((pg->flags & PG_PAGEOUT) == 0);
1617 		if (pg->flags & PG_RELEASED) {
1618 			UVMHIST_LOG(ubchist, "releasing pg %#jx",
1619 			    (uintptr_t)pg, 0, 0, 0);
1620 			KASSERT(pg->uobject != NULL ||
1621 			    (pg->uanon != NULL && pg->uanon->an_ref > 0));
1622 			pg->flags &= ~PG_RELEASED;
1623 			uvm_pagefree(pg);
1624 		} else {
1625 			UVMHIST_LOG(ubchist, "unbusying pg %#jx",
1626 			    (uintptr_t)pg, 0, 0, 0);
1627 			KASSERT((pg->flags & PG_FAKE) == 0);
1628 			pg->flags &= ~PG_BUSY;
1629 			uvm_pagelock(pg);
1630 			uvm_pagewakeup(pg);
1631 			uvm_pageunlock(pg);
1632 			UVM_PAGE_OWN(pg, NULL);
1633 		}
1634 	}
1635 }
1636 
1637 /*
1638  * uvm_pagewait: wait for a busy page
1639  *
1640  * => page must be known PG_BUSY
1641  * => object must be read or write locked
1642  * => object will be unlocked on return
1643  */
1644 
1645 void
1646 uvm_pagewait(struct vm_page *pg, krwlock_t *lock, const char *wmesg)
1647 {
1648 
1649 	KASSERT(rw_lock_held(lock));
1650 	KASSERT((pg->flags & PG_BUSY) != 0);
1651 	KASSERT(uvm_page_owner_locked_p(pg, false));
1652 
1653 	mutex_enter(&pg->interlock);
1654 	pg->pqflags |= PQ_WANTED;
1655 	rw_exit(lock);
1656 	UVM_UNLOCK_AND_WAIT(pg, &pg->interlock, false, wmesg, 0);
1657 }
1658 
1659 /*
1660  * uvm_pagewakeup: wake anyone waiting on a page
1661  *
1662  * => page interlock must be held
1663  */
1664 
1665 void
1666 uvm_pagewakeup(struct vm_page *pg)
1667 {
1668 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
1669 
1670 	KASSERT(mutex_owned(&pg->interlock));
1671 
1672 	UVMHIST_LOG(ubchist, "waking pg %#jx", (uintptr_t)pg, 0, 0, 0);
1673 
1674 	if ((pg->pqflags & PQ_WANTED) != 0) {
1675 		wakeup(pg);
1676 		pg->pqflags &= ~PQ_WANTED;
1677 	}
1678 }
1679 
1680 /*
1681  * uvm_pagewanted_p: return true if someone is waiting on the page
1682  *
1683  * => object must be write locked (lock out all concurrent access)
1684  */
1685 
1686 bool
1687 uvm_pagewanted_p(struct vm_page *pg)
1688 {
1689 
1690 	KASSERT(uvm_page_owner_locked_p(pg, true));
1691 
1692 	return (atomic_load_relaxed(&pg->pqflags) & PQ_WANTED) != 0;
1693 }
1694 
1695 #if defined(UVM_PAGE_TRKOWN)
1696 /*
1697  * uvm_page_own: set or release page ownership
1698  *
1699  * => this is a debugging function that keeps track of who sets PG_BUSY
1700  *	and where they do it.   it can be used to track down problems
1701  *	such a process setting "PG_BUSY" and never releasing it.
1702  * => page's object [if any] must be locked
1703  * => if "tag" is NULL then we are releasing page ownership
1704  */
1705 void
1706 uvm_page_own(struct vm_page *pg, const char *tag)
1707 {
1708 
1709 	KASSERT((pg->flags & (PG_PAGEOUT|PG_RELEASED)) == 0);
1710 	KASSERT(uvm_page_owner_locked_p(pg, true));
1711 
1712 	/* gain ownership? */
1713 	if (tag) {
1714 		KASSERT((pg->flags & PG_BUSY) != 0);
1715 		if (pg->owner_tag) {
1716 			printf("uvm_page_own: page %p already owned "
1717 			    "by proc %d.%d [%s]\n", pg,
1718 			    pg->owner, pg->lowner, pg->owner_tag);
1719 			panic("uvm_page_own");
1720 		}
1721 		pg->owner = curproc->p_pid;
1722 		pg->lowner = curlwp->l_lid;
1723 		pg->owner_tag = tag;
1724 		return;
1725 	}
1726 
1727 	/* drop ownership */
1728 	KASSERT((pg->flags & PG_BUSY) == 0);
1729 	if (pg->owner_tag == NULL) {
1730 		printf("uvm_page_own: dropping ownership of an non-owned "
1731 		    "page (%p)\n", pg);
1732 		panic("uvm_page_own");
1733 	}
1734 	pg->owner_tag = NULL;
1735 }
1736 #endif
1737 
1738 /*
1739  * uvm_pagelookup: look up a page
1740  *
1741  * => caller should lock object to keep someone from pulling the page
1742  *	out from under it
1743  */
1744 
1745 struct vm_page *
1746 uvm_pagelookup(struct uvm_object *obj, voff_t off)
1747 {
1748 	struct vm_page *pg;
1749 	bool ddb __diagused = false;
1750 #ifdef DDB
1751 	extern int db_active;
1752 	ddb = db_active != 0;
1753 #endif
1754 
1755 	KASSERT(ddb || rw_lock_held(obj->vmobjlock));
1756 
1757 	pg = radix_tree_lookup_node(&obj->uo_pages, off >> PAGE_SHIFT);
1758 
1759 	KASSERT(pg == NULL || obj->uo_npages != 0);
1760 	KASSERT(pg == NULL || (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 ||
1761 		(pg->flags & PG_BUSY) != 0);
1762 	return pg;
1763 }
1764 
1765 /*
1766  * uvm_pagewire: wire the page, thus removing it from the daemon's grasp
1767  *
1768  * => caller must lock objects
1769  * => caller must hold pg->interlock
1770  */
1771 
1772 void
1773 uvm_pagewire(struct vm_page *pg)
1774 {
1775 
1776 	KASSERT(uvm_page_owner_locked_p(pg, true));
1777 	KASSERT(mutex_owned(&pg->interlock));
1778 #if defined(READAHEAD_STATS)
1779 	if ((pg->flags & PG_READAHEAD) != 0) {
1780 		uvm_ra_hit.ev_count++;
1781 		pg->flags &= ~PG_READAHEAD;
1782 	}
1783 #endif /* defined(READAHEAD_STATS) */
1784 	if (pg->wire_count == 0) {
1785 		uvm_pagedequeue(pg);
1786 		atomic_inc_uint(&uvmexp.wired);
1787 	}
1788 	pg->wire_count++;
1789 	KASSERT(pg->wire_count > 0);	/* detect wraparound */
1790 }
1791 
1792 /*
1793  * uvm_pageunwire: unwire the page.
1794  *
1795  * => activate if wire count goes to zero.
1796  * => caller must lock objects
1797  * => caller must hold pg->interlock
1798  */
1799 
1800 void
1801 uvm_pageunwire(struct vm_page *pg)
1802 {
1803 
1804 	KASSERT(uvm_page_owner_locked_p(pg, true));
1805 	KASSERT(pg->wire_count != 0);
1806 	KASSERT(!uvmpdpol_pageisqueued_p(pg));
1807 	KASSERT(mutex_owned(&pg->interlock));
1808 	pg->wire_count--;
1809 	if (pg->wire_count == 0) {
1810 		uvm_pageactivate(pg);
1811 		KASSERT(uvmexp.wired != 0);
1812 		atomic_dec_uint(&uvmexp.wired);
1813 	}
1814 }
1815 
1816 /*
1817  * uvm_pagedeactivate: deactivate page
1818  *
1819  * => caller must lock objects
1820  * => caller must check to make sure page is not wired
1821  * => object that page belongs to must be locked (so we can adjust pg->flags)
1822  * => caller must clear the reference on the page before calling
1823  * => caller must hold pg->interlock
1824  */
1825 
1826 void
1827 uvm_pagedeactivate(struct vm_page *pg)
1828 {
1829 
1830 	KASSERT(uvm_page_owner_locked_p(pg, false));
1831 	KASSERT(mutex_owned(&pg->interlock));
1832 	if (pg->wire_count == 0) {
1833 		KASSERT(uvmpdpol_pageisqueued_p(pg));
1834 		uvmpdpol_pagedeactivate(pg);
1835 	}
1836 }
1837 
1838 /*
1839  * uvm_pageactivate: activate page
1840  *
1841  * => caller must lock objects
1842  * => caller must hold pg->interlock
1843  */
1844 
1845 void
1846 uvm_pageactivate(struct vm_page *pg)
1847 {
1848 
1849 	KASSERT(uvm_page_owner_locked_p(pg, false));
1850 	KASSERT(mutex_owned(&pg->interlock));
1851 #if defined(READAHEAD_STATS)
1852 	if ((pg->flags & PG_READAHEAD) != 0) {
1853 		uvm_ra_hit.ev_count++;
1854 		pg->flags &= ~PG_READAHEAD;
1855 	}
1856 #endif /* defined(READAHEAD_STATS) */
1857 	if (pg->wire_count == 0) {
1858 		uvmpdpol_pageactivate(pg);
1859 	}
1860 }
1861 
1862 /*
1863  * uvm_pagedequeue: remove a page from any paging queue
1864  *
1865  * => caller must lock objects
1866  * => caller must hold pg->interlock
1867  */
1868 void
1869 uvm_pagedequeue(struct vm_page *pg)
1870 {
1871 
1872 	KASSERT(uvm_page_owner_locked_p(pg, true));
1873 	KASSERT(mutex_owned(&pg->interlock));
1874 	if (uvmpdpol_pageisqueued_p(pg)) {
1875 		uvmpdpol_pagedequeue(pg);
1876 	}
1877 }
1878 
1879 /*
1880  * uvm_pageenqueue: add a page to a paging queue without activating.
1881  * used where a page is not really demanded (yet).  eg. read-ahead
1882  *
1883  * => caller must lock objects
1884  * => caller must hold pg->interlock
1885  */
1886 void
1887 uvm_pageenqueue(struct vm_page *pg)
1888 {
1889 
1890 	KASSERT(uvm_page_owner_locked_p(pg, false));
1891 	KASSERT(mutex_owned(&pg->interlock));
1892 	if (pg->wire_count == 0 && !uvmpdpol_pageisqueued_p(pg)) {
1893 		uvmpdpol_pageenqueue(pg);
1894 	}
1895 }
1896 
1897 /*
1898  * uvm_pagelock: acquire page interlock
1899  */
1900 void
1901 uvm_pagelock(struct vm_page *pg)
1902 {
1903 
1904 	mutex_enter(&pg->interlock);
1905 }
1906 
1907 /*
1908  * uvm_pagelock2: acquire two page interlocks
1909  */
1910 void
1911 uvm_pagelock2(struct vm_page *pg1, struct vm_page *pg2)
1912 {
1913 
1914 	if (pg1 < pg2) {
1915 		mutex_enter(&pg1->interlock);
1916 		mutex_enter(&pg2->interlock);
1917 	} else {
1918 		mutex_enter(&pg2->interlock);
1919 		mutex_enter(&pg1->interlock);
1920 	}
1921 }
1922 
1923 /*
1924  * uvm_pageunlock: release page interlock, and if a page replacement intent
1925  * is set on the page, pass it to uvmpdpol to make real.
1926  *
1927  * => caller must hold pg->interlock
1928  */
1929 void
1930 uvm_pageunlock(struct vm_page *pg)
1931 {
1932 
1933 	if ((pg->pqflags & PQ_INTENT_SET) == 0 ||
1934 	    (pg->pqflags & PQ_INTENT_QUEUED) != 0) {
1935 	    	mutex_exit(&pg->interlock);
1936 	    	return;
1937 	}
1938 	pg->pqflags |= PQ_INTENT_QUEUED;
1939 	mutex_exit(&pg->interlock);
1940 	uvmpdpol_pagerealize(pg);
1941 }
1942 
1943 /*
1944  * uvm_pageunlock2: release two page interlocks, and for both pages if a
1945  * page replacement intent is set on the page, pass it to uvmpdpol to make
1946  * real.
1947  *
1948  * => caller must hold pg->interlock
1949  */
1950 void
1951 uvm_pageunlock2(struct vm_page *pg1, struct vm_page *pg2)
1952 {
1953 
1954 	if ((pg1->pqflags & PQ_INTENT_SET) == 0 ||
1955 	    (pg1->pqflags & PQ_INTENT_QUEUED) != 0) {
1956 	    	mutex_exit(&pg1->interlock);
1957 	    	pg1 = NULL;
1958 	} else {
1959 		pg1->pqflags |= PQ_INTENT_QUEUED;
1960 		mutex_exit(&pg1->interlock);
1961 	}
1962 
1963 	if ((pg2->pqflags & PQ_INTENT_SET) == 0 ||
1964 	    (pg2->pqflags & PQ_INTENT_QUEUED) != 0) {
1965 	    	mutex_exit(&pg2->interlock);
1966 	    	pg2 = NULL;
1967 	} else {
1968 		pg2->pqflags |= PQ_INTENT_QUEUED;
1969 		mutex_exit(&pg2->interlock);
1970 	}
1971 
1972 	if (pg1 != NULL) {
1973 		uvmpdpol_pagerealize(pg1);
1974 	}
1975 	if (pg2 != NULL) {
1976 		uvmpdpol_pagerealize(pg2);
1977 	}
1978 }
1979 
1980 /*
1981  * uvm_pagezero: zero fill a page
1982  *
1983  * => if page is part of an object then the object should be locked
1984  *	to protect pg->flags.
1985  */
1986 
1987 void
1988 uvm_pagezero(struct vm_page *pg)
1989 {
1990 
1991 	uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
1992 	pmap_zero_page(VM_PAGE_TO_PHYS(pg));
1993 }
1994 
1995 /*
1996  * uvm_pagecopy: copy a page
1997  *
1998  * => if page is part of an object then the object should be locked
1999  *	to protect pg->flags.
2000  */
2001 
2002 void
2003 uvm_pagecopy(struct vm_page *src, struct vm_page *dst)
2004 {
2005 
2006 	uvm_pagemarkdirty(dst, UVM_PAGE_STATUS_DIRTY);
2007 	pmap_copy_page(VM_PAGE_TO_PHYS(src), VM_PAGE_TO_PHYS(dst));
2008 }
2009 
2010 /*
2011  * uvm_pageismanaged: test it see that a page (specified by PA) is managed.
2012  */
2013 
2014 bool
2015 uvm_pageismanaged(paddr_t pa)
2016 {
2017 
2018 	return (uvm_physseg_find(atop(pa), NULL) != UVM_PHYSSEG_TYPE_INVALID);
2019 }
2020 
2021 /*
2022  * uvm_page_lookup_freelist: look up the free list for the specified page
2023  */
2024 
2025 int
2026 uvm_page_lookup_freelist(struct vm_page *pg)
2027 {
2028 	uvm_physseg_t upm;
2029 
2030 	upm = uvm_physseg_find(atop(VM_PAGE_TO_PHYS(pg)), NULL);
2031 	KASSERT(upm != UVM_PHYSSEG_TYPE_INVALID);
2032 	return uvm_physseg_get_free_list(upm);
2033 }
2034 
2035 /*
2036  * uvm_page_owner_locked_p: return true if object associated with page is
2037  * locked.  this is a weak check for runtime assertions only.
2038  */
2039 
2040 bool
2041 uvm_page_owner_locked_p(struct vm_page *pg, bool exclusive)
2042 {
2043 
2044 	if (pg->uobject != NULL) {
2045 		return exclusive
2046 		    ? rw_write_held(pg->uobject->vmobjlock)
2047 		    : rw_lock_held(pg->uobject->vmobjlock);
2048 	}
2049 	if (pg->uanon != NULL) {
2050 		return exclusive
2051 		    ? rw_write_held(pg->uanon->an_lock)
2052 		    : rw_lock_held(pg->uanon->an_lock);
2053 	}
2054 	return true;
2055 }
2056 
2057 /*
2058  * uvm_pagereadonly_p: return if the page should be mapped read-only
2059  */
2060 
2061 bool
2062 uvm_pagereadonly_p(struct vm_page *pg)
2063 {
2064 	struct uvm_object * const uobj = pg->uobject;
2065 
2066 	KASSERT(uobj == NULL || rw_lock_held(uobj->vmobjlock));
2067 	KASSERT(uobj != NULL || rw_lock_held(pg->uanon->an_lock));
2068 	if ((pg->flags & PG_RDONLY) != 0) {
2069 		return true;
2070 	}
2071 	if (uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN) {
2072 		return true;
2073 	}
2074 	if (uobj == NULL) {
2075 		return false;
2076 	}
2077 	return UVM_OBJ_NEEDS_WRITEFAULT(uobj);
2078 }
2079 
2080 #ifdef PMAP_DIRECT
2081 /*
2082  * Call pmap to translate physical address into a virtual and to run a callback
2083  * for it. Used to avoid actually mapping the pages, pmap most likely uses direct map
2084  * or equivalent.
2085  */
2086 int
2087 uvm_direct_process(struct vm_page **pgs, u_int npages, voff_t off, vsize_t len,
2088             int (*process)(void *, size_t, void *), void *arg)
2089 {
2090 	int error = 0;
2091 	paddr_t pa;
2092 	size_t todo;
2093 	voff_t pgoff = (off & PAGE_MASK);
2094 	struct vm_page *pg;
2095 
2096 	KASSERT(npages > 0 && len > 0);
2097 
2098 	for (int i = 0; i < npages; i++) {
2099 		pg = pgs[i];
2100 
2101 		KASSERT(len > 0);
2102 
2103 		/*
2104 		 * Caller is responsible for ensuring all the pages are
2105 		 * available.
2106 		 */
2107 		KASSERT(pg != NULL && pg != PGO_DONTCARE);
2108 
2109 		pa = VM_PAGE_TO_PHYS(pg);
2110 		todo = MIN(len, PAGE_SIZE - pgoff);
2111 
2112 		error = pmap_direct_process(pa, pgoff, todo, process, arg);
2113 		if (error)
2114 			break;
2115 
2116 		pgoff = 0;
2117 		len -= todo;
2118 	}
2119 
2120 	KASSERTMSG(error != 0 || len == 0, "len %lu != 0 for non-error", len);
2121 	return error;
2122 }
2123 #endif /* PMAP_DIRECT */
2124 
2125 #if defined(DDB) || defined(DEBUGPRINT)
2126 
2127 /*
2128  * uvm_page_printit: actually print the page
2129  */
2130 
2131 static const char page_flagbits[] = UVM_PGFLAGBITS;
2132 static const char page_pqflagbits[] = UVM_PQFLAGBITS;
2133 
2134 void
2135 uvm_page_printit(struct vm_page *pg, bool full,
2136     void (*pr)(const char *, ...))
2137 {
2138 	struct vm_page *tpg;
2139 	struct uvm_object *uobj;
2140 	struct pgflbucket *pgb;
2141 	struct pgflist *pgl;
2142 	char pgbuf[128];
2143 
2144 	(*pr)("PAGE %p:\n", pg);
2145 	snprintb(pgbuf, sizeof(pgbuf), page_flagbits, pg->flags);
2146 	(*pr)("  flags=%s\n", pgbuf);
2147 	snprintb(pgbuf, sizeof(pgbuf), page_pqflagbits, pg->pqflags);
2148 	(*pr)("  pqflags=%s\n", pgbuf);
2149 	(*pr)("  uobject=%p, uanon=%p, offset=0x%llx\n",
2150 	    pg->uobject, pg->uanon, (long long)pg->offset);
2151 	(*pr)("  loan_count=%d wire_count=%d bucket=%d freelist=%d\n",
2152 	    pg->loan_count, pg->wire_count, uvm_page_get_bucket(pg),
2153 	    uvm_page_get_freelist(pg));
2154 	(*pr)("  pa=0x%lx\n", (long)VM_PAGE_TO_PHYS(pg));
2155 #if defined(UVM_PAGE_TRKOWN)
2156 	if (pg->flags & PG_BUSY)
2157 		(*pr)("  owning process = %d.%d, tag=%s\n",
2158 		    pg->owner, pg->lowner, pg->owner_tag);
2159 	else
2160 		(*pr)("  page not busy, no owner\n");
2161 #else
2162 	(*pr)("  [page ownership tracking disabled]\n");
2163 #endif
2164 
2165 	if (!full)
2166 		return;
2167 
2168 	/* cross-verify object/anon */
2169 	if ((pg->flags & PG_FREE) == 0) {
2170 		if (pg->flags & PG_ANON) {
2171 			if (pg->uanon == NULL || pg->uanon->an_page != pg)
2172 			    (*pr)("  >>> ANON DOES NOT POINT HERE <<< (%p)\n",
2173 				(pg->uanon) ? pg->uanon->an_page : NULL);
2174 			else
2175 				(*pr)("  anon backpointer is OK\n");
2176 		} else {
2177 			uobj = pg->uobject;
2178 			if (uobj) {
2179 				(*pr)("  checking object list\n");
2180 				tpg = uvm_pagelookup(uobj, pg->offset);
2181 				if (tpg)
2182 					(*pr)("  page found on object list\n");
2183 				else
2184 			(*pr)("  >>> PAGE NOT FOUND ON OBJECT LIST! <<<\n");
2185 			}
2186 		}
2187 	}
2188 
2189 	/* cross-verify page queue */
2190 	if (pg->flags & PG_FREE) {
2191 		int fl = uvm_page_get_freelist(pg);
2192 		int b = uvm_page_get_bucket(pg);
2193 		pgb = uvm.page_free[fl].pgfl_buckets[b];
2194 		pgl = &pgb->pgb_colors[VM_PGCOLOR(pg)];
2195 		(*pr)("  checking pageq list\n");
2196 		LIST_FOREACH(tpg, pgl, pageq.list) {
2197 			if (tpg == pg) {
2198 				break;
2199 			}
2200 		}
2201 		if (tpg)
2202 			(*pr)("  page found on pageq list\n");
2203 		else
2204 			(*pr)("  >>> PAGE NOT FOUND ON PAGEQ LIST! <<<\n");
2205 	}
2206 }
2207 
2208 /*
2209  * uvm_page_printall - print a summary of all managed pages
2210  */
2211 
2212 void
2213 uvm_page_printall(void (*pr)(const char *, ...))
2214 {
2215 	uvm_physseg_t i;
2216 	paddr_t pfn;
2217 	struct vm_page *pg;
2218 
2219 	(*pr)("%18s %4s %4s %18s %18s"
2220 #ifdef UVM_PAGE_TRKOWN
2221 	    " OWNER"
2222 #endif
2223 	    "\n", "PAGE", "FLAG", "PQ", "UOBJECT", "UANON");
2224 	for (i = uvm_physseg_get_first();
2225 	     uvm_physseg_valid_p(i);
2226 	     i = uvm_physseg_get_next(i)) {
2227 		for (pfn = uvm_physseg_get_start(i);
2228 		     pfn < uvm_physseg_get_end(i);
2229 		     pfn++) {
2230 			pg = PHYS_TO_VM_PAGE(ptoa(pfn));
2231 
2232 			(*pr)("%18p %04x %08x %18p %18p",
2233 			    pg, pg->flags, pg->pqflags, pg->uobject,
2234 			    pg->uanon);
2235 #ifdef UVM_PAGE_TRKOWN
2236 			if (pg->flags & PG_BUSY)
2237 				(*pr)(" %d [%s]", pg->owner, pg->owner_tag);
2238 #endif
2239 			(*pr)("\n");
2240 		}
2241 	}
2242 }
2243 
2244 /*
2245  * uvm_page_print_freelists - print a summary freelists
2246  */
2247 
2248 void
2249 uvm_page_print_freelists(void (*pr)(const char *, ...))
2250 {
2251 	struct pgfreelist *pgfl;
2252 	struct pgflbucket *pgb;
2253 	int fl, b, c;
2254 
2255 	(*pr)("There are %d freelists with %d buckets of %d colors.\n\n",
2256 	    VM_NFREELIST, uvm.bucketcount, uvmexp.ncolors);
2257 
2258 	for (fl = 0; fl < VM_NFREELIST; fl++) {
2259 		pgfl = &uvm.page_free[fl];
2260 		(*pr)("freelist(%d) @ %p\n", fl, pgfl);
2261 		for (b = 0; b < uvm.bucketcount; b++) {
2262 			pgb = uvm.page_free[fl].pgfl_buckets[b];
2263 			(*pr)("    bucket(%d) @ %p, nfree = %d, lock @ %p:\n",
2264 			    b, pgb, pgb->pgb_nfree,
2265 			    &uvm_freelist_locks[b].lock);
2266 			for (c = 0; c < uvmexp.ncolors; c++) {
2267 				(*pr)("        color(%d) @ %p, ", c,
2268 				    &pgb->pgb_colors[c]);
2269 				(*pr)("first page = %p\n",
2270 				    LIST_FIRST(&pgb->pgb_colors[c]));
2271 			}
2272 		}
2273 	}
2274 }
2275 
2276 #endif /* DDB || DEBUGPRINT */
2277