1*31e69e6cSchs /* $NetBSD: uvm_pgflcache.c,v 1.6 2020/10/18 18:31:31 chs Exp $ */
29b1e2fa2Sad
39b1e2fa2Sad /*-
49b1e2fa2Sad * Copyright (c) 2019 The NetBSD Foundation, Inc.
59b1e2fa2Sad * All rights reserved.
69b1e2fa2Sad *
79b1e2fa2Sad * This code is derived from software contributed to The NetBSD Foundation
89b1e2fa2Sad * by Andrew Doran.
99b1e2fa2Sad *
109b1e2fa2Sad * Redistribution and use in source and binary forms, with or without
119b1e2fa2Sad * modification, are permitted provided that the following conditions
129b1e2fa2Sad * are met:
139b1e2fa2Sad * 1. Redistributions of source code must retain the above copyright
149b1e2fa2Sad * notice, this list of conditions and the following disclaimer.
159b1e2fa2Sad * 2. Redistributions in binary form must reproduce the above copyright
169b1e2fa2Sad * notice, this list of conditions and the following disclaimer in the
179b1e2fa2Sad * documentation and/or other materials provided with the distribution.
189b1e2fa2Sad *
199b1e2fa2Sad * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
209b1e2fa2Sad * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
219b1e2fa2Sad * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
229b1e2fa2Sad * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
239b1e2fa2Sad * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
249b1e2fa2Sad * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
259b1e2fa2Sad * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
269b1e2fa2Sad * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
279b1e2fa2Sad * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
289b1e2fa2Sad * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
299b1e2fa2Sad * POSSIBILITY OF SUCH DAMAGE.
309b1e2fa2Sad */
319b1e2fa2Sad
329b1e2fa2Sad /*
339b1e2fa2Sad * uvm_pgflcache.c: page freelist cache.
349b1e2fa2Sad *
359b1e2fa2Sad * This implements a tiny per-CPU cache of pages that sits between the main
369b1e2fa2Sad * page allocator and the freelists. By allocating and freeing pages in
379b1e2fa2Sad * batch, it reduces freelist contention by an order of magnitude.
389b1e2fa2Sad *
399b1e2fa2Sad * The cache can be paused & resumed at runtime so that UVM_HOTPLUG,
409b1e2fa2Sad * uvm_pglistalloc() and uvm_page_redim() can have a consistent view of the
419b1e2fa2Sad * world. On system with one CPU per physical package (e.g. a uniprocessor)
429b1e2fa2Sad * the cache is not enabled.
439b1e2fa2Sad */
449b1e2fa2Sad
459b1e2fa2Sad #include <sys/cdefs.h>
46*31e69e6cSchs __KERNEL_RCSID(0, "$NetBSD: uvm_pgflcache.c,v 1.6 2020/10/18 18:31:31 chs Exp $");
479b1e2fa2Sad
489b1e2fa2Sad #include "opt_uvm.h"
499b1e2fa2Sad #include "opt_multiprocessor.h"
509b1e2fa2Sad
519b1e2fa2Sad #include <sys/param.h>
529b1e2fa2Sad #include <sys/systm.h>
539b1e2fa2Sad #include <sys/sched.h>
549b1e2fa2Sad #include <sys/kernel.h>
559b1e2fa2Sad #include <sys/vnode.h>
569b1e2fa2Sad #include <sys/proc.h>
579b1e2fa2Sad #include <sys/atomic.h>
589b1e2fa2Sad #include <sys/cpu.h>
599b1e2fa2Sad #include <sys/xcall.h>
609b1e2fa2Sad
619b1e2fa2Sad #include <uvm/uvm.h>
629b1e2fa2Sad #include <uvm/uvm_pglist.h>
639b1e2fa2Sad #include <uvm/uvm_pgflcache.h>
649b1e2fa2Sad
659b1e2fa2Sad /* There is no point doing any of this on a uniprocessor. */
669b1e2fa2Sad #ifdef MULTIPROCESSOR
679b1e2fa2Sad
689b1e2fa2Sad /*
699b1e2fa2Sad * MAXPGS - maximum pages per color, per bucket.
709b1e2fa2Sad * FILLPGS - number of pages to allocate at once, per color, per bucket.
719b1e2fa2Sad *
729b1e2fa2Sad * Why the chosen values:
739b1e2fa2Sad *
749b1e2fa2Sad * (1) In 2019, an average Intel system has 4kB pages and 8x L2 cache
759b1e2fa2Sad * colors. We make the assumption that most of the time allocation activity
769b1e2fa2Sad * will be centered around one UVM freelist, so most of the time there will
779b1e2fa2Sad * be no more than 224kB worth of cached pages per-CPU. That's tiny, but
789b1e2fa2Sad * enough to hugely reduce contention on the freelist locks, and give us a
799b1e2fa2Sad * small pool of pages which if we're very lucky may have some L1/L2 cache
809b1e2fa2Sad * locality, and do so without subtracting too much from the L2/L3 cache
819b1e2fa2Sad * benefits of having per-package free lists in the page allocator.
829b1e2fa2Sad *
839b1e2fa2Sad * (2) With the chosen values on _LP64, the data structure for each color
849b1e2fa2Sad * takes up a single cache line (64 bytes) giving this very low overhead
859b1e2fa2Sad * even in the "miss" case.
869b1e2fa2Sad *
879b1e2fa2Sad * (3) We don't want to cause too much pressure by hiding away memory that
889b1e2fa2Sad * could otherwise be put to good use.
899b1e2fa2Sad */
909b1e2fa2Sad #define MAXPGS 7
919b1e2fa2Sad #define FILLPGS 6
929b1e2fa2Sad
939b1e2fa2Sad /* Variable size, according to # colors. */
949b1e2fa2Sad struct pgflcache {
959b1e2fa2Sad struct pccolor {
969b1e2fa2Sad intptr_t count;
979b1e2fa2Sad struct vm_page *pages[MAXPGS];
989b1e2fa2Sad } color[1];
999b1e2fa2Sad };
1009b1e2fa2Sad
1019b1e2fa2Sad static kmutex_t uvm_pgflcache_lock;
1029b1e2fa2Sad static int uvm_pgflcache_sem;
1039b1e2fa2Sad
1049b1e2fa2Sad /*
1059b1e2fa2Sad * uvm_pgflcache_fill: fill specified freelist/color from global list
1069b1e2fa2Sad *
1079b1e2fa2Sad * => must be called at IPL_VM
1089b1e2fa2Sad * => must be called with given bucket lock held
1099b1e2fa2Sad * => must only fill from the correct bucket for this CPU
1109b1e2fa2Sad */
1119b1e2fa2Sad
1129b1e2fa2Sad void
uvm_pgflcache_fill(struct uvm_cpu * ucpu,int fl,int b,int c)1139b1e2fa2Sad uvm_pgflcache_fill(struct uvm_cpu *ucpu, int fl, int b, int c)
1149b1e2fa2Sad {
1159b1e2fa2Sad struct pgflbucket *pgb;
1169b1e2fa2Sad struct pgflcache *pc;
1179b1e2fa2Sad struct pccolor *pcc;
1189b1e2fa2Sad struct pgflist *head;
1199b1e2fa2Sad struct vm_page *pg;
1209b1e2fa2Sad int count;
1219b1e2fa2Sad
1229b1e2fa2Sad KASSERT(mutex_owned(&uvm_freelist_locks[b].lock));
1239b1e2fa2Sad KASSERT(ucpu->pgflbucket == b);
1249b1e2fa2Sad
1259b1e2fa2Sad /* If caching is off, then bail out. */
1269b1e2fa2Sad if (__predict_false((pc = ucpu->pgflcache[fl]) == NULL)) {
1279b1e2fa2Sad return;
1289b1e2fa2Sad }
1299b1e2fa2Sad
1309b1e2fa2Sad /* Fill only to the limit. */
1319b1e2fa2Sad pcc = &pc->color[c];
1329b1e2fa2Sad pgb = uvm.page_free[fl].pgfl_buckets[b];
1339b1e2fa2Sad head = &pgb->pgb_colors[c];
1349b1e2fa2Sad if (pcc->count >= FILLPGS) {
1359b1e2fa2Sad return;
1369b1e2fa2Sad }
1379b1e2fa2Sad
1389b1e2fa2Sad /* Pull pages from the bucket until it's empty, or we are full. */
1399b1e2fa2Sad count = pcc->count;
1409b1e2fa2Sad pg = LIST_FIRST(head);
1419b1e2fa2Sad while (__predict_true(pg != NULL && count < FILLPGS)) {
1429b1e2fa2Sad KASSERT(pg->flags & PG_FREE);
1439b1e2fa2Sad KASSERT(uvm_page_get_bucket(pg) == b);
1449b1e2fa2Sad pcc->pages[count++] = pg;
1459b1e2fa2Sad pg = LIST_NEXT(pg, pageq.list);
1469b1e2fa2Sad }
1479b1e2fa2Sad
1489b1e2fa2Sad /* Violate LIST abstraction to remove all pages at once. */
1499b1e2fa2Sad head->lh_first = pg;
1509b1e2fa2Sad if (__predict_true(pg != NULL)) {
1519b1e2fa2Sad pg->pageq.list.le_prev = &head->lh_first;
1529b1e2fa2Sad }
1539b1e2fa2Sad pgb->pgb_nfree -= (count - pcc->count);
154*31e69e6cSchs CPU_COUNT(CPU_COUNT_FREEPAGES, -(count - pcc->count));
1559b1e2fa2Sad pcc->count = count;
1569b1e2fa2Sad }
1579b1e2fa2Sad
1589b1e2fa2Sad /*
1599b1e2fa2Sad * uvm_pgflcache_spill: spill specified freelist/color to global list
1609b1e2fa2Sad *
1619b1e2fa2Sad * => must be called at IPL_VM
1629b1e2fa2Sad * => mark __noinline so we don't pull it into uvm_pgflcache_free()
1639b1e2fa2Sad */
1649b1e2fa2Sad
1659b1e2fa2Sad static void __noinline
uvm_pgflcache_spill(struct uvm_cpu * ucpu,int fl,int c)1669b1e2fa2Sad uvm_pgflcache_spill(struct uvm_cpu *ucpu, int fl, int c)
1679b1e2fa2Sad {
1689b1e2fa2Sad struct pgflbucket *pgb;
1699b1e2fa2Sad struct pgfreelist *pgfl;
1709b1e2fa2Sad struct pgflcache *pc;
1719b1e2fa2Sad struct pccolor *pcc;
1729b1e2fa2Sad struct pgflist *head;
1739b1e2fa2Sad kmutex_t *lock;
1749b1e2fa2Sad int b, adj;
1759b1e2fa2Sad
1769b1e2fa2Sad pc = ucpu->pgflcache[fl];
1779b1e2fa2Sad pcc = &pc->color[c];
1789b1e2fa2Sad pgfl = &uvm.page_free[fl];
1799b1e2fa2Sad b = ucpu->pgflbucket;
1809b1e2fa2Sad pgb = pgfl->pgfl_buckets[b];
1819b1e2fa2Sad head = &pgb->pgb_colors[c];
1829b1e2fa2Sad lock = &uvm_freelist_locks[b].lock;
1839b1e2fa2Sad
1849b1e2fa2Sad mutex_spin_enter(lock);
1859b1e2fa2Sad for (adj = pcc->count; pcc->count != 0;) {
1869b1e2fa2Sad pcc->count--;
1879b1e2fa2Sad KASSERT(pcc->pages[pcc->count] != NULL);
1889b1e2fa2Sad KASSERT(pcc->pages[pcc->count]->flags & PG_FREE);
1899b1e2fa2Sad LIST_INSERT_HEAD(head, pcc->pages[pcc->count], pageq.list);
1909b1e2fa2Sad }
1919b1e2fa2Sad pgb->pgb_nfree += adj;
192*31e69e6cSchs CPU_COUNT(CPU_COUNT_FREEPAGES, adj);
1939b1e2fa2Sad mutex_spin_exit(lock);
1949b1e2fa2Sad }
1959b1e2fa2Sad
1969b1e2fa2Sad /*
1979b1e2fa2Sad * uvm_pgflcache_alloc: try to allocate a cached page.
1989b1e2fa2Sad *
1999b1e2fa2Sad * => must be called at IPL_VM
2009b1e2fa2Sad * => allocate only from the given freelist and given page color
2019b1e2fa2Sad */
2029b1e2fa2Sad
2039b1e2fa2Sad struct vm_page *
uvm_pgflcache_alloc(struct uvm_cpu * ucpu,int fl,int c)2049b1e2fa2Sad uvm_pgflcache_alloc(struct uvm_cpu *ucpu, int fl, int c)
2059b1e2fa2Sad {
2069b1e2fa2Sad struct pgflcache *pc;
2079b1e2fa2Sad struct pccolor *pcc;
2089b1e2fa2Sad struct vm_page *pg;
2099b1e2fa2Sad
2109b1e2fa2Sad /* If caching is off, then bail out. */
2119b1e2fa2Sad if (__predict_false((pc = ucpu->pgflcache[fl]) == NULL)) {
2129b1e2fa2Sad return NULL;
2139b1e2fa2Sad }
2149b1e2fa2Sad
2159b1e2fa2Sad /* Very simple: if we have a page then return it. */
2169b1e2fa2Sad pcc = &pc->color[c];
2179b1e2fa2Sad if (__predict_false(pcc->count == 0)) {
2189b1e2fa2Sad return NULL;
2199b1e2fa2Sad }
2209b1e2fa2Sad pg = pcc->pages[--(pcc->count)];
2219b1e2fa2Sad KASSERT(pg != NULL);
222748d5a7eSad KASSERT(pg->flags == PG_FREE);
2239b1e2fa2Sad KASSERT(uvm_page_get_freelist(pg) == fl);
2249b1e2fa2Sad KASSERT(uvm_page_get_bucket(pg) == ucpu->pgflbucket);
225748d5a7eSad pg->flags = PG_BUSY | PG_CLEAN | PG_FAKE;
2269b1e2fa2Sad return pg;
2279b1e2fa2Sad }
2289b1e2fa2Sad
2299b1e2fa2Sad /*
2309b1e2fa2Sad * uvm_pgflcache_free: cache a page, if possible.
2319b1e2fa2Sad *
2329b1e2fa2Sad * => must be called at IPL_VM
2339b1e2fa2Sad * => must only send pages for the correct bucket for this CPU
2349b1e2fa2Sad */
2359b1e2fa2Sad
2369b1e2fa2Sad bool
uvm_pgflcache_free(struct uvm_cpu * ucpu,struct vm_page * pg)2379b1e2fa2Sad uvm_pgflcache_free(struct uvm_cpu *ucpu, struct vm_page *pg)
2389b1e2fa2Sad {
2399b1e2fa2Sad struct pgflcache *pc;
2409b1e2fa2Sad struct pccolor *pcc;
2419b1e2fa2Sad int fl, c;
2429b1e2fa2Sad
2439b1e2fa2Sad KASSERT(uvm_page_get_bucket(pg) == ucpu->pgflbucket);
2449b1e2fa2Sad
2459b1e2fa2Sad /* If caching is off, then bail out. */
2469b1e2fa2Sad fl = uvm_page_get_freelist(pg);
2479b1e2fa2Sad if (__predict_false((pc = ucpu->pgflcache[fl]) == NULL)) {
2489b1e2fa2Sad return false;
2499b1e2fa2Sad }
2509b1e2fa2Sad
2519b1e2fa2Sad /* If the array is full spill it first, then add page to array. */
2529b1e2fa2Sad c = VM_PGCOLOR(pg);
2539b1e2fa2Sad pcc = &pc->color[c];
2549b1e2fa2Sad KASSERT((pg->flags & PG_FREE) == 0);
2559b1e2fa2Sad if (__predict_false(pcc->count == MAXPGS)) {
2569b1e2fa2Sad uvm_pgflcache_spill(ucpu, fl, c);
2579b1e2fa2Sad }
258748d5a7eSad pg->flags = PG_FREE;
2599b1e2fa2Sad pcc->pages[pcc->count] = pg;
2609b1e2fa2Sad pcc->count++;
2619b1e2fa2Sad return true;
2629b1e2fa2Sad }
2639b1e2fa2Sad
2649b1e2fa2Sad /*
2659b1e2fa2Sad * uvm_pgflcache_init: allocate and initialize per-CPU data structures for
2669b1e2fa2Sad * the free page cache. Don't set anything in motion - that's taken care
2679b1e2fa2Sad * of by uvm_pgflcache_resume().
2689b1e2fa2Sad */
2699b1e2fa2Sad
2709b1e2fa2Sad static void
uvm_pgflcache_init_cpu(struct cpu_info * ci)2719b1e2fa2Sad uvm_pgflcache_init_cpu(struct cpu_info *ci)
2729b1e2fa2Sad {
2739b1e2fa2Sad struct uvm_cpu *ucpu;
2749b1e2fa2Sad size_t sz;
2759b1e2fa2Sad
2769b1e2fa2Sad ucpu = ci->ci_data.cpu_uvm;
2779b1e2fa2Sad KASSERT(ucpu->pgflcachemem == NULL);
2789b1e2fa2Sad KASSERT(ucpu->pgflcache[0] == NULL);
2799b1e2fa2Sad
2809b1e2fa2Sad sz = offsetof(struct pgflcache, color[uvmexp.ncolors]);
2819b1e2fa2Sad ucpu->pgflcachememsz =
2829b1e2fa2Sad (roundup2(sz * VM_NFREELIST, coherency_unit) + coherency_unit - 1);
2839b1e2fa2Sad ucpu->pgflcachemem = kmem_zalloc(ucpu->pgflcachememsz, KM_SLEEP);
2849b1e2fa2Sad }
2859b1e2fa2Sad
2869b1e2fa2Sad /*
2879b1e2fa2Sad * uvm_pgflcache_fini_cpu: dump all cached pages back to global free list
2889b1e2fa2Sad * and shut down caching on the CPU. Called on each CPU in the system via
2899b1e2fa2Sad * xcall.
2909b1e2fa2Sad */
2919b1e2fa2Sad
2929b1e2fa2Sad static void
uvm_pgflcache_fini_cpu(void * arg1 __unused,void * arg2 __unused)2939b1e2fa2Sad uvm_pgflcache_fini_cpu(void *arg1 __unused, void *arg2 __unused)
2949b1e2fa2Sad {
2959b1e2fa2Sad struct uvm_cpu *ucpu;
2969b1e2fa2Sad int fl, color, s;
2979b1e2fa2Sad
2989b1e2fa2Sad ucpu = curcpu()->ci_data.cpu_uvm;
2999b1e2fa2Sad for (fl = 0; fl < VM_NFREELIST; fl++) {
3009b1e2fa2Sad s = splvm();
3019b1e2fa2Sad for (color = 0; color < uvmexp.ncolors; color++) {
3029b1e2fa2Sad uvm_pgflcache_spill(ucpu, fl, color);
3039b1e2fa2Sad }
3049b1e2fa2Sad ucpu->pgflcache[fl] = NULL;
3059b1e2fa2Sad splx(s);
3069b1e2fa2Sad }
3079b1e2fa2Sad }
3089b1e2fa2Sad
3099b1e2fa2Sad /*
3109b1e2fa2Sad * uvm_pgflcache_pause: pause operation of the caches
3119b1e2fa2Sad */
3129b1e2fa2Sad
3139b1e2fa2Sad void
uvm_pgflcache_pause(void)3149b1e2fa2Sad uvm_pgflcache_pause(void)
3159b1e2fa2Sad {
3169b1e2fa2Sad uint64_t where;
3179b1e2fa2Sad
3189b1e2fa2Sad /* First one in starts draining. Everyone else waits. */
3199b1e2fa2Sad mutex_enter(&uvm_pgflcache_lock);
3209b1e2fa2Sad if (uvm_pgflcache_sem++ == 0) {
3217875bd6dSad where = xc_broadcast(XC_HIGHPRI, uvm_pgflcache_fini_cpu,
3227875bd6dSad (void *)1, NULL);
3239b1e2fa2Sad xc_wait(where);
3249b1e2fa2Sad }
3259b1e2fa2Sad mutex_exit(&uvm_pgflcache_lock);
3269b1e2fa2Sad }
3279b1e2fa2Sad
3289b1e2fa2Sad /*
3299b1e2fa2Sad * uvm_pgflcache_resume: resume operation of the caches
3309b1e2fa2Sad */
3319b1e2fa2Sad
3329b1e2fa2Sad void
uvm_pgflcache_resume(void)3339b1e2fa2Sad uvm_pgflcache_resume(void)
3349b1e2fa2Sad {
3359b1e2fa2Sad CPU_INFO_ITERATOR cii;
3369b1e2fa2Sad struct cpu_info *ci;
3379b1e2fa2Sad struct uvm_cpu *ucpu;
3389b1e2fa2Sad uintptr_t addr;
3399b1e2fa2Sad size_t sz;
3409b1e2fa2Sad int fl;
3419b1e2fa2Sad
3429b1e2fa2Sad /* Last guy out takes care of business. */
3439b1e2fa2Sad mutex_enter(&uvm_pgflcache_lock);
3449b1e2fa2Sad KASSERT(uvm_pgflcache_sem > 0);
3459b1e2fa2Sad if (uvm_pgflcache_sem-- > 1) {
3469b1e2fa2Sad mutex_exit(&uvm_pgflcache_lock);
3479b1e2fa2Sad return;
3489b1e2fa2Sad }
3499b1e2fa2Sad
3509b1e2fa2Sad /*
3519b1e2fa2Sad * Make sure dependant data structure updates are remotely visible.
3529b1e2fa2Sad * Essentially this functions as a global memory barrier.
3539b1e2fa2Sad */
3549b1e2fa2Sad xc_barrier(XC_HIGHPRI);
3559b1e2fa2Sad
3569b1e2fa2Sad /*
3579b1e2fa2Sad * Then set all of the pointers in place on each CPU. As soon as
3589b1e2fa2Sad * each pointer is set, caching is operational in that dimension.
3599b1e2fa2Sad */
3609b1e2fa2Sad sz = offsetof(struct pgflcache, color[uvmexp.ncolors]);
3619b1e2fa2Sad for (CPU_INFO_FOREACH(cii, ci)) {
3629b1e2fa2Sad ucpu = ci->ci_data.cpu_uvm;
3639b1e2fa2Sad addr = roundup2((uintptr_t)ucpu->pgflcachemem, coherency_unit);
3649b1e2fa2Sad for (fl = 0; fl < VM_NFREELIST; fl++) {
3659b1e2fa2Sad ucpu->pgflcache[fl] = (struct pgflcache *)addr;
3669b1e2fa2Sad addr += sz;
3679b1e2fa2Sad }
3689b1e2fa2Sad }
3699b1e2fa2Sad mutex_exit(&uvm_pgflcache_lock);
3709b1e2fa2Sad }
3719b1e2fa2Sad
3729b1e2fa2Sad /*
3739b1e2fa2Sad * uvm_pgflcache_start: start operation of the cache.
3749b1e2fa2Sad *
3759b1e2fa2Sad * => called once only, when init(8) is about to be started
3769b1e2fa2Sad */
3779b1e2fa2Sad
3789b1e2fa2Sad void
uvm_pgflcache_start(void)3799b1e2fa2Sad uvm_pgflcache_start(void)
3809b1e2fa2Sad {
3819b1e2fa2Sad CPU_INFO_ITERATOR cii;
3829b1e2fa2Sad struct cpu_info *ci;
3839b1e2fa2Sad
3849b1e2fa2Sad KASSERT(uvm_pgflcache_sem > 0);
3859b1e2fa2Sad
3869b1e2fa2Sad /*
3879b1e2fa2Sad * There's not much point doing this if every CPU has its own
3889b1e2fa2Sad * bucket (and that includes the uniprocessor case).
3899b1e2fa2Sad */
3909b1e2fa2Sad if (ncpu == uvm.bucketcount) {
3919b1e2fa2Sad return;
3929b1e2fa2Sad }
3939b1e2fa2Sad
3946e083f05Sad /* Create data structures for each CPU. */
3959b1e2fa2Sad for (CPU_INFO_FOREACH(cii, ci)) {
3969b1e2fa2Sad uvm_pgflcache_init_cpu(ci);
3979b1e2fa2Sad }
3989b1e2fa2Sad
3999b1e2fa2Sad /* Kick it into action. */
4007875bd6dSad uvm_pgflcache_resume();
4019b1e2fa2Sad }
4029b1e2fa2Sad
4039b1e2fa2Sad /*
4049b1e2fa2Sad * uvm_pgflcache_init: set up data structures for the free page cache.
4059b1e2fa2Sad */
4069b1e2fa2Sad
4079b1e2fa2Sad void
uvm_pgflcache_init(void)4089b1e2fa2Sad uvm_pgflcache_init(void)
4099b1e2fa2Sad {
4109b1e2fa2Sad
4119b1e2fa2Sad uvm_pgflcache_sem = 1;
4129b1e2fa2Sad mutex_init(&uvm_pgflcache_lock, MUTEX_DEFAULT, IPL_NONE);
4139b1e2fa2Sad }
4149b1e2fa2Sad
4159b1e2fa2Sad #else /* MULTIPROCESSOR */
4169b1e2fa2Sad
4179b1e2fa2Sad struct vm_page *
uvm_pgflcache_alloc(struct uvm_cpu * ucpu,int fl,int c)4189b1e2fa2Sad uvm_pgflcache_alloc(struct uvm_cpu *ucpu, int fl, int c)
4199b1e2fa2Sad {
4209b1e2fa2Sad
4219b1e2fa2Sad return NULL;
4229b1e2fa2Sad }
4239b1e2fa2Sad
4249b1e2fa2Sad bool
uvm_pgflcache_free(struct uvm_cpu * ucpu,struct vm_page * pg)4259b1e2fa2Sad uvm_pgflcache_free(struct uvm_cpu *ucpu, struct vm_page *pg)
4269b1e2fa2Sad {
4279b1e2fa2Sad
4289b1e2fa2Sad return false;
4299b1e2fa2Sad }
4309b1e2fa2Sad
4319b1e2fa2Sad void
uvm_pgflcache_fill(struct uvm_cpu * ucpu,int fl,int b,int c)4329b1e2fa2Sad uvm_pgflcache_fill(struct uvm_cpu *ucpu, int fl, int b, int c)
4339b1e2fa2Sad {
4349b1e2fa2Sad
4359b1e2fa2Sad }
4369b1e2fa2Sad
4379b1e2fa2Sad void
uvm_pgflcache_pause(void)4389b1e2fa2Sad uvm_pgflcache_pause(void)
4399b1e2fa2Sad {
4409b1e2fa2Sad
4419b1e2fa2Sad }
4429b1e2fa2Sad
4439b1e2fa2Sad void
uvm_pgflcache_resume(void)4449b1e2fa2Sad uvm_pgflcache_resume(void)
4459b1e2fa2Sad {
4469b1e2fa2Sad
4479b1e2fa2Sad }
4489b1e2fa2Sad
4499b1e2fa2Sad void
uvm_pgflcache_start(void)4509b1e2fa2Sad uvm_pgflcache_start(void)
4519b1e2fa2Sad {
4529b1e2fa2Sad
4539b1e2fa2Sad }
4549b1e2fa2Sad
4559b1e2fa2Sad void
uvm_pgflcache_init(void)4569b1e2fa2Sad uvm_pgflcache_init(void)
4579b1e2fa2Sad {
4589b1e2fa2Sad
4599b1e2fa2Sad }
4609b1e2fa2Sad
4619b1e2fa2Sad #endif /* MULTIPROCESSOR */
462