xref: /netbsd-src/sys/uvm/uvm_pgflcache.c (revision 31e69e6c91e24dcbcfae5ad9aba96f2dd8254078)
1*31e69e6cSchs /*	$NetBSD: uvm_pgflcache.c,v 1.6 2020/10/18 18:31:31 chs Exp $	*/
29b1e2fa2Sad 
39b1e2fa2Sad /*-
49b1e2fa2Sad  * Copyright (c) 2019 The NetBSD Foundation, Inc.
59b1e2fa2Sad  * All rights reserved.
69b1e2fa2Sad  *
79b1e2fa2Sad  * This code is derived from software contributed to The NetBSD Foundation
89b1e2fa2Sad  * by Andrew Doran.
99b1e2fa2Sad  *
109b1e2fa2Sad  * Redistribution and use in source and binary forms, with or without
119b1e2fa2Sad  * modification, are permitted provided that the following conditions
129b1e2fa2Sad  * are met:
139b1e2fa2Sad  * 1. Redistributions of source code must retain the above copyright
149b1e2fa2Sad  *    notice, this list of conditions and the following disclaimer.
159b1e2fa2Sad  * 2. Redistributions in binary form must reproduce the above copyright
169b1e2fa2Sad  *    notice, this list of conditions and the following disclaimer in the
179b1e2fa2Sad  *    documentation and/or other materials provided with the distribution.
189b1e2fa2Sad  *
199b1e2fa2Sad  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
209b1e2fa2Sad  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
219b1e2fa2Sad  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
229b1e2fa2Sad  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
239b1e2fa2Sad  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
249b1e2fa2Sad  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
259b1e2fa2Sad  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
269b1e2fa2Sad  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
279b1e2fa2Sad  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
289b1e2fa2Sad  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
299b1e2fa2Sad  * POSSIBILITY OF SUCH DAMAGE.
309b1e2fa2Sad  */
319b1e2fa2Sad 
329b1e2fa2Sad /*
339b1e2fa2Sad  * uvm_pgflcache.c: page freelist cache.
349b1e2fa2Sad  *
359b1e2fa2Sad  * This implements a tiny per-CPU cache of pages that sits between the main
369b1e2fa2Sad  * page allocator and the freelists.  By allocating and freeing pages in
379b1e2fa2Sad  * batch, it reduces freelist contention by an order of magnitude.
389b1e2fa2Sad  *
399b1e2fa2Sad  * The cache can be paused & resumed at runtime so that UVM_HOTPLUG,
409b1e2fa2Sad  * uvm_pglistalloc() and uvm_page_redim() can have a consistent view of the
419b1e2fa2Sad  * world.  On system with one CPU per physical package (e.g. a uniprocessor)
429b1e2fa2Sad  * the cache is not enabled.
439b1e2fa2Sad  */
449b1e2fa2Sad 
459b1e2fa2Sad #include <sys/cdefs.h>
46*31e69e6cSchs __KERNEL_RCSID(0, "$NetBSD: uvm_pgflcache.c,v 1.6 2020/10/18 18:31:31 chs Exp $");
479b1e2fa2Sad 
489b1e2fa2Sad #include "opt_uvm.h"
499b1e2fa2Sad #include "opt_multiprocessor.h"
509b1e2fa2Sad 
519b1e2fa2Sad #include <sys/param.h>
529b1e2fa2Sad #include <sys/systm.h>
539b1e2fa2Sad #include <sys/sched.h>
549b1e2fa2Sad #include <sys/kernel.h>
559b1e2fa2Sad #include <sys/vnode.h>
569b1e2fa2Sad #include <sys/proc.h>
579b1e2fa2Sad #include <sys/atomic.h>
589b1e2fa2Sad #include <sys/cpu.h>
599b1e2fa2Sad #include <sys/xcall.h>
609b1e2fa2Sad 
619b1e2fa2Sad #include <uvm/uvm.h>
629b1e2fa2Sad #include <uvm/uvm_pglist.h>
639b1e2fa2Sad #include <uvm/uvm_pgflcache.h>
649b1e2fa2Sad 
659b1e2fa2Sad /* There is no point doing any of this on a uniprocessor. */
669b1e2fa2Sad #ifdef MULTIPROCESSOR
679b1e2fa2Sad 
689b1e2fa2Sad /*
699b1e2fa2Sad  * MAXPGS - maximum pages per color, per bucket.
709b1e2fa2Sad  * FILLPGS - number of pages to allocate at once, per color, per bucket.
719b1e2fa2Sad  *
729b1e2fa2Sad  * Why the chosen values:
739b1e2fa2Sad  *
749b1e2fa2Sad  * (1) In 2019, an average Intel system has 4kB pages and 8x L2 cache
759b1e2fa2Sad  * colors.  We make the assumption that most of the time allocation activity
769b1e2fa2Sad  * will be centered around one UVM freelist, so most of the time there will
779b1e2fa2Sad  * be no more than 224kB worth of cached pages per-CPU.  That's tiny, but
789b1e2fa2Sad  * enough to hugely reduce contention on the freelist locks, and give us a
799b1e2fa2Sad  * small pool of pages which if we're very lucky may have some L1/L2 cache
809b1e2fa2Sad  * locality, and do so without subtracting too much from the L2/L3 cache
819b1e2fa2Sad  * benefits of having per-package free lists in the page allocator.
829b1e2fa2Sad  *
839b1e2fa2Sad  * (2) With the chosen values on _LP64, the data structure for each color
849b1e2fa2Sad  * takes up a single cache line (64 bytes) giving this very low overhead
859b1e2fa2Sad  * even in the "miss" case.
869b1e2fa2Sad  *
879b1e2fa2Sad  * (3) We don't want to cause too much pressure by hiding away memory that
889b1e2fa2Sad  * could otherwise be put to good use.
899b1e2fa2Sad  */
909b1e2fa2Sad #define	MAXPGS		7
919b1e2fa2Sad #define	FILLPGS		6
929b1e2fa2Sad 
939b1e2fa2Sad /* Variable size, according to # colors. */
949b1e2fa2Sad struct pgflcache {
959b1e2fa2Sad 	struct pccolor {
969b1e2fa2Sad 		intptr_t	count;
979b1e2fa2Sad 		struct vm_page	*pages[MAXPGS];
989b1e2fa2Sad 	} color[1];
999b1e2fa2Sad };
1009b1e2fa2Sad 
1019b1e2fa2Sad static kmutex_t		uvm_pgflcache_lock;
1029b1e2fa2Sad static int		uvm_pgflcache_sem;
1039b1e2fa2Sad 
1049b1e2fa2Sad /*
1059b1e2fa2Sad  * uvm_pgflcache_fill: fill specified freelist/color from global list
1069b1e2fa2Sad  *
1079b1e2fa2Sad  * => must be called at IPL_VM
1089b1e2fa2Sad  * => must be called with given bucket lock held
1099b1e2fa2Sad  * => must only fill from the correct bucket for this CPU
1109b1e2fa2Sad  */
1119b1e2fa2Sad 
1129b1e2fa2Sad void
uvm_pgflcache_fill(struct uvm_cpu * ucpu,int fl,int b,int c)1139b1e2fa2Sad uvm_pgflcache_fill(struct uvm_cpu *ucpu, int fl, int b, int c)
1149b1e2fa2Sad {
1159b1e2fa2Sad 	struct pgflbucket *pgb;
1169b1e2fa2Sad 	struct pgflcache *pc;
1179b1e2fa2Sad 	struct pccolor *pcc;
1189b1e2fa2Sad 	struct pgflist *head;
1199b1e2fa2Sad 	struct vm_page *pg;
1209b1e2fa2Sad 	int count;
1219b1e2fa2Sad 
1229b1e2fa2Sad 	KASSERT(mutex_owned(&uvm_freelist_locks[b].lock));
1239b1e2fa2Sad 	KASSERT(ucpu->pgflbucket == b);
1249b1e2fa2Sad 
1259b1e2fa2Sad 	/* If caching is off, then bail out. */
1269b1e2fa2Sad 	if (__predict_false((pc = ucpu->pgflcache[fl]) == NULL)) {
1279b1e2fa2Sad 		return;
1289b1e2fa2Sad 	}
1299b1e2fa2Sad 
1309b1e2fa2Sad 	/* Fill only to the limit. */
1319b1e2fa2Sad 	pcc = &pc->color[c];
1329b1e2fa2Sad 	pgb = uvm.page_free[fl].pgfl_buckets[b];
1339b1e2fa2Sad 	head = &pgb->pgb_colors[c];
1349b1e2fa2Sad 	if (pcc->count >= FILLPGS) {
1359b1e2fa2Sad 		return;
1369b1e2fa2Sad 	}
1379b1e2fa2Sad 
1389b1e2fa2Sad 	/* Pull pages from the bucket until it's empty, or we are full. */
1399b1e2fa2Sad 	count = pcc->count;
1409b1e2fa2Sad 	pg = LIST_FIRST(head);
1419b1e2fa2Sad 	while (__predict_true(pg != NULL && count < FILLPGS)) {
1429b1e2fa2Sad 		KASSERT(pg->flags & PG_FREE);
1439b1e2fa2Sad 		KASSERT(uvm_page_get_bucket(pg) == b);
1449b1e2fa2Sad 		pcc->pages[count++] = pg;
1459b1e2fa2Sad 		pg = LIST_NEXT(pg, pageq.list);
1469b1e2fa2Sad 	}
1479b1e2fa2Sad 
1489b1e2fa2Sad 	/* Violate LIST abstraction to remove all pages at once. */
1499b1e2fa2Sad 	head->lh_first = pg;
1509b1e2fa2Sad 	if (__predict_true(pg != NULL)) {
1519b1e2fa2Sad 		pg->pageq.list.le_prev = &head->lh_first;
1529b1e2fa2Sad 	}
1539b1e2fa2Sad 	pgb->pgb_nfree -= (count - pcc->count);
154*31e69e6cSchs 	CPU_COUNT(CPU_COUNT_FREEPAGES, -(count - pcc->count));
1559b1e2fa2Sad 	pcc->count = count;
1569b1e2fa2Sad }
1579b1e2fa2Sad 
1589b1e2fa2Sad /*
1599b1e2fa2Sad  * uvm_pgflcache_spill: spill specified freelist/color to global list
1609b1e2fa2Sad  *
1619b1e2fa2Sad  * => must be called at IPL_VM
1629b1e2fa2Sad  * => mark __noinline so we don't pull it into uvm_pgflcache_free()
1639b1e2fa2Sad  */
1649b1e2fa2Sad 
1659b1e2fa2Sad static void __noinline
uvm_pgflcache_spill(struct uvm_cpu * ucpu,int fl,int c)1669b1e2fa2Sad uvm_pgflcache_spill(struct uvm_cpu *ucpu, int fl, int c)
1679b1e2fa2Sad {
1689b1e2fa2Sad 	struct pgflbucket *pgb;
1699b1e2fa2Sad 	struct pgfreelist *pgfl;
1709b1e2fa2Sad 	struct pgflcache *pc;
1719b1e2fa2Sad 	struct pccolor *pcc;
1729b1e2fa2Sad 	struct pgflist *head;
1739b1e2fa2Sad 	kmutex_t *lock;
1749b1e2fa2Sad 	int b, adj;
1759b1e2fa2Sad 
1769b1e2fa2Sad 	pc = ucpu->pgflcache[fl];
1779b1e2fa2Sad 	pcc = &pc->color[c];
1789b1e2fa2Sad 	pgfl = &uvm.page_free[fl];
1799b1e2fa2Sad 	b = ucpu->pgflbucket;
1809b1e2fa2Sad 	pgb = pgfl->pgfl_buckets[b];
1819b1e2fa2Sad 	head = &pgb->pgb_colors[c];
1829b1e2fa2Sad 	lock = &uvm_freelist_locks[b].lock;
1839b1e2fa2Sad 
1849b1e2fa2Sad 	mutex_spin_enter(lock);
1859b1e2fa2Sad 	for (adj = pcc->count; pcc->count != 0;) {
1869b1e2fa2Sad 		pcc->count--;
1879b1e2fa2Sad 		KASSERT(pcc->pages[pcc->count] != NULL);
1889b1e2fa2Sad 		KASSERT(pcc->pages[pcc->count]->flags & PG_FREE);
1899b1e2fa2Sad 		LIST_INSERT_HEAD(head, pcc->pages[pcc->count], pageq.list);
1909b1e2fa2Sad 	}
1919b1e2fa2Sad 	pgb->pgb_nfree += adj;
192*31e69e6cSchs 	CPU_COUNT(CPU_COUNT_FREEPAGES, adj);
1939b1e2fa2Sad 	mutex_spin_exit(lock);
1949b1e2fa2Sad }
1959b1e2fa2Sad 
1969b1e2fa2Sad /*
1979b1e2fa2Sad  * uvm_pgflcache_alloc: try to allocate a cached page.
1989b1e2fa2Sad  *
1999b1e2fa2Sad  * => must be called at IPL_VM
2009b1e2fa2Sad  * => allocate only from the given freelist and given page color
2019b1e2fa2Sad  */
2029b1e2fa2Sad 
2039b1e2fa2Sad struct vm_page *
uvm_pgflcache_alloc(struct uvm_cpu * ucpu,int fl,int c)2049b1e2fa2Sad uvm_pgflcache_alloc(struct uvm_cpu *ucpu, int fl, int c)
2059b1e2fa2Sad {
2069b1e2fa2Sad 	struct pgflcache *pc;
2079b1e2fa2Sad 	struct pccolor *pcc;
2089b1e2fa2Sad 	struct vm_page *pg;
2099b1e2fa2Sad 
2109b1e2fa2Sad 	/* If caching is off, then bail out. */
2119b1e2fa2Sad 	if (__predict_false((pc = ucpu->pgflcache[fl]) == NULL)) {
2129b1e2fa2Sad 		return NULL;
2139b1e2fa2Sad 	}
2149b1e2fa2Sad 
2159b1e2fa2Sad 	/* Very simple: if we have a page then return it. */
2169b1e2fa2Sad 	pcc = &pc->color[c];
2179b1e2fa2Sad 	if (__predict_false(pcc->count == 0)) {
2189b1e2fa2Sad 		return NULL;
2199b1e2fa2Sad 	}
2209b1e2fa2Sad 	pg = pcc->pages[--(pcc->count)];
2219b1e2fa2Sad 	KASSERT(pg != NULL);
222748d5a7eSad 	KASSERT(pg->flags == PG_FREE);
2239b1e2fa2Sad 	KASSERT(uvm_page_get_freelist(pg) == fl);
2249b1e2fa2Sad 	KASSERT(uvm_page_get_bucket(pg) == ucpu->pgflbucket);
225748d5a7eSad 	pg->flags = PG_BUSY | PG_CLEAN | PG_FAKE;
2269b1e2fa2Sad 	return pg;
2279b1e2fa2Sad }
2289b1e2fa2Sad 
2299b1e2fa2Sad /*
2309b1e2fa2Sad  * uvm_pgflcache_free: cache a page, if possible.
2319b1e2fa2Sad  *
2329b1e2fa2Sad  * => must be called at IPL_VM
2339b1e2fa2Sad  * => must only send pages for the correct bucket for this CPU
2349b1e2fa2Sad  */
2359b1e2fa2Sad 
2369b1e2fa2Sad bool
uvm_pgflcache_free(struct uvm_cpu * ucpu,struct vm_page * pg)2379b1e2fa2Sad uvm_pgflcache_free(struct uvm_cpu *ucpu, struct vm_page *pg)
2389b1e2fa2Sad {
2399b1e2fa2Sad 	struct pgflcache *pc;
2409b1e2fa2Sad 	struct pccolor *pcc;
2419b1e2fa2Sad 	int fl, c;
2429b1e2fa2Sad 
2439b1e2fa2Sad 	KASSERT(uvm_page_get_bucket(pg) == ucpu->pgflbucket);
2449b1e2fa2Sad 
2459b1e2fa2Sad 	/* If caching is off, then bail out. */
2469b1e2fa2Sad  	fl = uvm_page_get_freelist(pg);
2479b1e2fa2Sad 	if (__predict_false((pc = ucpu->pgflcache[fl]) == NULL)) {
2489b1e2fa2Sad 		return false;
2499b1e2fa2Sad 	}
2509b1e2fa2Sad 
2519b1e2fa2Sad 	/* If the array is full spill it first, then add page to array. */
2529b1e2fa2Sad 	c = VM_PGCOLOR(pg);
2539b1e2fa2Sad 	pcc = &pc->color[c];
2549b1e2fa2Sad 	KASSERT((pg->flags & PG_FREE) == 0);
2559b1e2fa2Sad 	if (__predict_false(pcc->count == MAXPGS)) {
2569b1e2fa2Sad 		uvm_pgflcache_spill(ucpu, fl, c);
2579b1e2fa2Sad 	}
258748d5a7eSad 	pg->flags = PG_FREE;
2599b1e2fa2Sad 	pcc->pages[pcc->count] = pg;
2609b1e2fa2Sad 	pcc->count++;
2619b1e2fa2Sad 	return true;
2629b1e2fa2Sad }
2639b1e2fa2Sad 
2649b1e2fa2Sad /*
2659b1e2fa2Sad  * uvm_pgflcache_init: allocate and initialize per-CPU data structures for
2669b1e2fa2Sad  * the free page cache.  Don't set anything in motion - that's taken care
2679b1e2fa2Sad  * of by uvm_pgflcache_resume().
2689b1e2fa2Sad  */
2699b1e2fa2Sad 
2709b1e2fa2Sad static void
uvm_pgflcache_init_cpu(struct cpu_info * ci)2719b1e2fa2Sad uvm_pgflcache_init_cpu(struct cpu_info *ci)
2729b1e2fa2Sad {
2739b1e2fa2Sad 	struct uvm_cpu *ucpu;
2749b1e2fa2Sad 	size_t sz;
2759b1e2fa2Sad 
2769b1e2fa2Sad 	ucpu = ci->ci_data.cpu_uvm;
2779b1e2fa2Sad 	KASSERT(ucpu->pgflcachemem == NULL);
2789b1e2fa2Sad 	KASSERT(ucpu->pgflcache[0] == NULL);
2799b1e2fa2Sad 
2809b1e2fa2Sad 	sz = offsetof(struct pgflcache, color[uvmexp.ncolors]);
2819b1e2fa2Sad 	ucpu->pgflcachememsz =
2829b1e2fa2Sad 	    (roundup2(sz * VM_NFREELIST, coherency_unit) + coherency_unit - 1);
2839b1e2fa2Sad 	ucpu->pgflcachemem = kmem_zalloc(ucpu->pgflcachememsz, KM_SLEEP);
2849b1e2fa2Sad }
2859b1e2fa2Sad 
2869b1e2fa2Sad /*
2879b1e2fa2Sad  * uvm_pgflcache_fini_cpu: dump all cached pages back to global free list
2889b1e2fa2Sad  * and shut down caching on the CPU.  Called on each CPU in the system via
2899b1e2fa2Sad  * xcall.
2909b1e2fa2Sad  */
2919b1e2fa2Sad 
2929b1e2fa2Sad static void
uvm_pgflcache_fini_cpu(void * arg1 __unused,void * arg2 __unused)2939b1e2fa2Sad uvm_pgflcache_fini_cpu(void *arg1 __unused, void *arg2 __unused)
2949b1e2fa2Sad {
2959b1e2fa2Sad 	struct uvm_cpu *ucpu;
2969b1e2fa2Sad 	int fl, color, s;
2979b1e2fa2Sad 
2989b1e2fa2Sad 	ucpu = curcpu()->ci_data.cpu_uvm;
2999b1e2fa2Sad 	for (fl = 0; fl < VM_NFREELIST; fl++) {
3009b1e2fa2Sad 		s = splvm();
3019b1e2fa2Sad 		for (color = 0; color < uvmexp.ncolors; color++) {
3029b1e2fa2Sad 			uvm_pgflcache_spill(ucpu, fl, color);
3039b1e2fa2Sad 		}
3049b1e2fa2Sad 		ucpu->pgflcache[fl] = NULL;
3059b1e2fa2Sad 		splx(s);
3069b1e2fa2Sad 	}
3079b1e2fa2Sad }
3089b1e2fa2Sad 
3099b1e2fa2Sad /*
3109b1e2fa2Sad  * uvm_pgflcache_pause: pause operation of the caches
3119b1e2fa2Sad  */
3129b1e2fa2Sad 
3139b1e2fa2Sad void
uvm_pgflcache_pause(void)3149b1e2fa2Sad uvm_pgflcache_pause(void)
3159b1e2fa2Sad {
3169b1e2fa2Sad 	uint64_t where;
3179b1e2fa2Sad 
3189b1e2fa2Sad 	/* First one in starts draining.  Everyone else waits. */
3199b1e2fa2Sad 	mutex_enter(&uvm_pgflcache_lock);
3209b1e2fa2Sad 	if (uvm_pgflcache_sem++ == 0) {
3217875bd6dSad 		where = xc_broadcast(XC_HIGHPRI, uvm_pgflcache_fini_cpu,
3227875bd6dSad 		    (void *)1, NULL);
3239b1e2fa2Sad 		xc_wait(where);
3249b1e2fa2Sad 	}
3259b1e2fa2Sad 	mutex_exit(&uvm_pgflcache_lock);
3269b1e2fa2Sad }
3279b1e2fa2Sad 
3289b1e2fa2Sad /*
3299b1e2fa2Sad  * uvm_pgflcache_resume: resume operation of the caches
3309b1e2fa2Sad  */
3319b1e2fa2Sad 
3329b1e2fa2Sad void
uvm_pgflcache_resume(void)3339b1e2fa2Sad uvm_pgflcache_resume(void)
3349b1e2fa2Sad {
3359b1e2fa2Sad 	CPU_INFO_ITERATOR cii;
3369b1e2fa2Sad 	struct cpu_info *ci;
3379b1e2fa2Sad 	struct uvm_cpu *ucpu;
3389b1e2fa2Sad 	uintptr_t addr;
3399b1e2fa2Sad 	size_t sz;
3409b1e2fa2Sad 	int fl;
3419b1e2fa2Sad 
3429b1e2fa2Sad 	/* Last guy out takes care of business. */
3439b1e2fa2Sad 	mutex_enter(&uvm_pgflcache_lock);
3449b1e2fa2Sad 	KASSERT(uvm_pgflcache_sem > 0);
3459b1e2fa2Sad 	if (uvm_pgflcache_sem-- > 1) {
3469b1e2fa2Sad 		mutex_exit(&uvm_pgflcache_lock);
3479b1e2fa2Sad 		return;
3489b1e2fa2Sad 	}
3499b1e2fa2Sad 
3509b1e2fa2Sad 	/*
3519b1e2fa2Sad 	 * Make sure dependant data structure updates are remotely visible.
3529b1e2fa2Sad 	 * Essentially this functions as a global memory barrier.
3539b1e2fa2Sad 	 */
3549b1e2fa2Sad 	xc_barrier(XC_HIGHPRI);
3559b1e2fa2Sad 
3569b1e2fa2Sad 	/*
3579b1e2fa2Sad 	 * Then set all of the pointers in place on each CPU.  As soon as
3589b1e2fa2Sad 	 * each pointer is set, caching is operational in that dimension.
3599b1e2fa2Sad 	 */
3609b1e2fa2Sad 	sz = offsetof(struct pgflcache, color[uvmexp.ncolors]);
3619b1e2fa2Sad 	for (CPU_INFO_FOREACH(cii, ci)) {
3629b1e2fa2Sad 		ucpu = ci->ci_data.cpu_uvm;
3639b1e2fa2Sad 		addr = roundup2((uintptr_t)ucpu->pgflcachemem, coherency_unit);
3649b1e2fa2Sad 		for (fl = 0; fl < VM_NFREELIST; fl++) {
3659b1e2fa2Sad 			ucpu->pgflcache[fl] = (struct pgflcache *)addr;
3669b1e2fa2Sad 			addr += sz;
3679b1e2fa2Sad 		}
3689b1e2fa2Sad 	}
3699b1e2fa2Sad 	mutex_exit(&uvm_pgflcache_lock);
3709b1e2fa2Sad }
3719b1e2fa2Sad 
3729b1e2fa2Sad /*
3739b1e2fa2Sad  * uvm_pgflcache_start: start operation of the cache.
3749b1e2fa2Sad  *
3759b1e2fa2Sad  * => called once only, when init(8) is about to be started
3769b1e2fa2Sad  */
3779b1e2fa2Sad 
3789b1e2fa2Sad void
uvm_pgflcache_start(void)3799b1e2fa2Sad uvm_pgflcache_start(void)
3809b1e2fa2Sad {
3819b1e2fa2Sad 	CPU_INFO_ITERATOR cii;
3829b1e2fa2Sad 	struct cpu_info *ci;
3839b1e2fa2Sad 
3849b1e2fa2Sad 	KASSERT(uvm_pgflcache_sem > 0);
3859b1e2fa2Sad 
3869b1e2fa2Sad 	/*
3879b1e2fa2Sad 	 * There's not much point doing this if every CPU has its own
3889b1e2fa2Sad 	 * bucket (and that includes the uniprocessor case).
3899b1e2fa2Sad 	 */
3909b1e2fa2Sad 	if (ncpu == uvm.bucketcount) {
3919b1e2fa2Sad 		return;
3929b1e2fa2Sad 	}
3939b1e2fa2Sad 
3946e083f05Sad 	/* Create data structures for each CPU. */
3959b1e2fa2Sad 	for (CPU_INFO_FOREACH(cii, ci)) {
3969b1e2fa2Sad 		uvm_pgflcache_init_cpu(ci);
3979b1e2fa2Sad 	}
3989b1e2fa2Sad 
3999b1e2fa2Sad 	/* Kick it into action. */
4007875bd6dSad 	uvm_pgflcache_resume();
4019b1e2fa2Sad }
4029b1e2fa2Sad 
4039b1e2fa2Sad /*
4049b1e2fa2Sad  * uvm_pgflcache_init: set up data structures for the free page cache.
4059b1e2fa2Sad  */
4069b1e2fa2Sad 
4079b1e2fa2Sad void
uvm_pgflcache_init(void)4089b1e2fa2Sad uvm_pgflcache_init(void)
4099b1e2fa2Sad {
4109b1e2fa2Sad 
4119b1e2fa2Sad 	uvm_pgflcache_sem = 1;
4129b1e2fa2Sad 	mutex_init(&uvm_pgflcache_lock, MUTEX_DEFAULT, IPL_NONE);
4139b1e2fa2Sad }
4149b1e2fa2Sad 
4159b1e2fa2Sad #else	/* MULTIPROCESSOR */
4169b1e2fa2Sad 
4179b1e2fa2Sad struct vm_page *
uvm_pgflcache_alloc(struct uvm_cpu * ucpu,int fl,int c)4189b1e2fa2Sad uvm_pgflcache_alloc(struct uvm_cpu *ucpu, int fl, int c)
4199b1e2fa2Sad {
4209b1e2fa2Sad 
4219b1e2fa2Sad 	return NULL;
4229b1e2fa2Sad }
4239b1e2fa2Sad 
4249b1e2fa2Sad bool
uvm_pgflcache_free(struct uvm_cpu * ucpu,struct vm_page * pg)4259b1e2fa2Sad uvm_pgflcache_free(struct uvm_cpu *ucpu, struct vm_page *pg)
4269b1e2fa2Sad {
4279b1e2fa2Sad 
4289b1e2fa2Sad 	return false;
4299b1e2fa2Sad }
4309b1e2fa2Sad 
4319b1e2fa2Sad void
uvm_pgflcache_fill(struct uvm_cpu * ucpu,int fl,int b,int c)4329b1e2fa2Sad uvm_pgflcache_fill(struct uvm_cpu *ucpu, int fl, int b, int c)
4339b1e2fa2Sad {
4349b1e2fa2Sad 
4359b1e2fa2Sad }
4369b1e2fa2Sad 
4379b1e2fa2Sad void
uvm_pgflcache_pause(void)4389b1e2fa2Sad uvm_pgflcache_pause(void)
4399b1e2fa2Sad {
4409b1e2fa2Sad 
4419b1e2fa2Sad }
4429b1e2fa2Sad 
4439b1e2fa2Sad void
uvm_pgflcache_resume(void)4449b1e2fa2Sad uvm_pgflcache_resume(void)
4459b1e2fa2Sad {
4469b1e2fa2Sad 
4479b1e2fa2Sad }
4489b1e2fa2Sad 
4499b1e2fa2Sad void
uvm_pgflcache_start(void)4509b1e2fa2Sad uvm_pgflcache_start(void)
4519b1e2fa2Sad {
4529b1e2fa2Sad 
4539b1e2fa2Sad }
4549b1e2fa2Sad 
4559b1e2fa2Sad void
uvm_pgflcache_init(void)4569b1e2fa2Sad uvm_pgflcache_init(void)
4579b1e2fa2Sad {
4589b1e2fa2Sad 
4599b1e2fa2Sad }
4609b1e2fa2Sad 
4619b1e2fa2Sad #endif	/* MULTIPROCESSOR */
462