1 /* $NetBSD: subr_pool.c,v 1.262 2019/11/14 16:23:53 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010, 2014, 2015, 2018 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace 10 * Simulation Facility, NASA Ames Research Center; by Andrew Doran, and by 11 * Maxime Villard. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 #include <sys/cdefs.h> 36 __KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.262 2019/11/14 16:23:53 maxv Exp $"); 37 38 #ifdef _KERNEL_OPT 39 #include "opt_ddb.h" 40 #include "opt_lockdebug.h" 41 #include "opt_pool.h" 42 #include "opt_kleak.h" 43 #endif 44 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/sysctl.h> 48 #include <sys/bitops.h> 49 #include <sys/proc.h> 50 #include <sys/errno.h> 51 #include <sys/kernel.h> 52 #include <sys/vmem.h> 53 #include <sys/pool.h> 54 #include <sys/syslog.h> 55 #include <sys/debug.h> 56 #include <sys/lockdebug.h> 57 #include <sys/xcall.h> 58 #include <sys/cpu.h> 59 #include <sys/atomic.h> 60 #include <sys/asan.h> 61 #include <sys/msan.h> 62 63 #include <uvm/uvm_extern.h> 64 65 /* 66 * Pool resource management utility. 67 * 68 * Memory is allocated in pages which are split into pieces according to 69 * the pool item size. Each page is kept on one of three lists in the 70 * pool structure: `pr_emptypages', `pr_fullpages' and `pr_partpages', 71 * for empty, full and partially-full pages respectively. The individual 72 * pool items are on a linked list headed by `ph_itemlist' in each page 73 * header. The memory for building the page list is either taken from 74 * the allocated pages themselves (for small pool items) or taken from 75 * an internal pool of page headers (`phpool'). 76 */ 77 78 /* List of all pools. Non static as needed by 'vmstat -m' */ 79 TAILQ_HEAD(, pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); 80 81 /* Private pool for page header structures */ 82 #define PHPOOL_MAX 8 83 static struct pool phpool[PHPOOL_MAX]; 84 #define PHPOOL_FREELIST_NELEM(idx) \ 85 (((idx) == 0) ? BITMAP_MIN_SIZE : BITMAP_SIZE * (1 << (idx))) 86 87 #if !defined(KMSAN) && (defined(DIAGNOSTIC) || defined(KASAN)) 88 #define POOL_REDZONE 89 #endif 90 91 #ifdef POOL_REDZONE 92 # ifdef KASAN 93 # define POOL_REDZONE_SIZE 8 94 # else 95 # define POOL_REDZONE_SIZE 2 96 # endif 97 static void pool_redzone_init(struct pool *, size_t); 98 static void pool_redzone_fill(struct pool *, void *); 99 static void pool_redzone_check(struct pool *, void *); 100 static void pool_cache_redzone_check(pool_cache_t, void *); 101 #else 102 # define pool_redzone_init(pp, sz) __nothing 103 # define pool_redzone_fill(pp, ptr) __nothing 104 # define pool_redzone_check(pp, ptr) __nothing 105 # define pool_cache_redzone_check(pc, ptr) __nothing 106 #endif 107 108 #ifdef KMSAN 109 static inline void pool_get_kmsan(struct pool *, void *); 110 static inline void pool_put_kmsan(struct pool *, void *); 111 static inline void pool_cache_get_kmsan(pool_cache_t, void *); 112 static inline void pool_cache_put_kmsan(pool_cache_t, void *); 113 #else 114 #define pool_get_kmsan(pp, ptr) __nothing 115 #define pool_put_kmsan(pp, ptr) __nothing 116 #define pool_cache_get_kmsan(pc, ptr) __nothing 117 #define pool_cache_put_kmsan(pc, ptr) __nothing 118 #endif 119 120 #ifdef KLEAK 121 static void pool_kleak_fill(struct pool *, void *); 122 static void pool_cache_kleak_fill(pool_cache_t, void *); 123 #else 124 #define pool_kleak_fill(pp, ptr) __nothing 125 #define pool_cache_kleak_fill(pc, ptr) __nothing 126 #endif 127 128 #ifdef POOL_QUARANTINE 129 static void pool_quarantine_init(struct pool *); 130 static void pool_quarantine_flush(struct pool *); 131 static bool pool_put_quarantine(struct pool *, void *, 132 struct pool_pagelist *); 133 static bool pool_cache_put_quarantine(pool_cache_t, void *, paddr_t); 134 #else 135 #define pool_quarantine_init(a) __nothing 136 #define pool_quarantine_flush(a) __nothing 137 #define pool_put_quarantine(a, b, c) false 138 #define pool_cache_put_quarantine(a, b, c) false 139 #endif 140 141 #define NO_CTOR __FPTRCAST(int (*)(void *, void *, int), nullop) 142 #define NO_DTOR __FPTRCAST(void (*)(void *, void *), nullop) 143 144 #define pc_has_ctor(pc) ((pc)->pc_ctor != NO_CTOR) 145 #define pc_has_dtor(pc) ((pc)->pc_dtor != NO_DTOR) 146 147 /* 148 * Pool backend allocators. 149 * 150 * Each pool has a backend allocator that handles allocation, deallocation, 151 * and any additional draining that might be needed. 152 * 153 * We provide two standard allocators: 154 * 155 * pool_allocator_kmem - the default when no allocator is specified 156 * 157 * pool_allocator_nointr - used for pools that will not be accessed 158 * in interrupt context. 159 */ 160 void *pool_page_alloc(struct pool *, int); 161 void pool_page_free(struct pool *, void *); 162 163 static void *pool_page_alloc_meta(struct pool *, int); 164 static void pool_page_free_meta(struct pool *, void *); 165 166 struct pool_allocator pool_allocator_kmem = { 167 .pa_alloc = pool_page_alloc, 168 .pa_free = pool_page_free, 169 .pa_pagesz = 0 170 }; 171 172 struct pool_allocator pool_allocator_nointr = { 173 .pa_alloc = pool_page_alloc, 174 .pa_free = pool_page_free, 175 .pa_pagesz = 0 176 }; 177 178 struct pool_allocator pool_allocator_meta = { 179 .pa_alloc = pool_page_alloc_meta, 180 .pa_free = pool_page_free_meta, 181 .pa_pagesz = 0 182 }; 183 184 #define POOL_ALLOCATOR_BIG_BASE 13 185 static struct pool_allocator pool_allocator_big[] = { 186 { 187 .pa_alloc = pool_page_alloc, 188 .pa_free = pool_page_free, 189 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 0), 190 }, 191 { 192 .pa_alloc = pool_page_alloc, 193 .pa_free = pool_page_free, 194 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 1), 195 }, 196 { 197 .pa_alloc = pool_page_alloc, 198 .pa_free = pool_page_free, 199 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 2), 200 }, 201 { 202 .pa_alloc = pool_page_alloc, 203 .pa_free = pool_page_free, 204 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 3), 205 }, 206 { 207 .pa_alloc = pool_page_alloc, 208 .pa_free = pool_page_free, 209 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 4), 210 }, 211 { 212 .pa_alloc = pool_page_alloc, 213 .pa_free = pool_page_free, 214 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 5), 215 }, 216 { 217 .pa_alloc = pool_page_alloc, 218 .pa_free = pool_page_free, 219 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 6), 220 }, 221 { 222 .pa_alloc = pool_page_alloc, 223 .pa_free = pool_page_free, 224 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 7), 225 } 226 }; 227 228 static int pool_bigidx(size_t); 229 230 /* # of seconds to retain page after last use */ 231 int pool_inactive_time = 10; 232 233 /* Next candidate for drainage (see pool_drain()) */ 234 static struct pool *drainpp; 235 236 /* This lock protects both pool_head and drainpp. */ 237 static kmutex_t pool_head_lock; 238 static kcondvar_t pool_busy; 239 240 /* This lock protects initialization of a potentially shared pool allocator */ 241 static kmutex_t pool_allocator_lock; 242 243 static unsigned int poolid_counter = 0; 244 245 typedef uint32_t pool_item_bitmap_t; 246 #define BITMAP_SIZE (CHAR_BIT * sizeof(pool_item_bitmap_t)) 247 #define BITMAP_MASK (BITMAP_SIZE - 1) 248 #define BITMAP_MIN_SIZE (CHAR_BIT * sizeof(((struct pool_item_header *)NULL)->ph_u2)) 249 250 struct pool_item_header { 251 /* Page headers */ 252 LIST_ENTRY(pool_item_header) 253 ph_pagelist; /* pool page list */ 254 union { 255 /* !PR_PHINPAGE */ 256 struct { 257 SPLAY_ENTRY(pool_item_header) 258 phu_node; /* off-page page headers */ 259 } phu_offpage; 260 /* PR_PHINPAGE */ 261 struct { 262 unsigned int phu_poolid; 263 } phu_onpage; 264 } ph_u1; 265 void * ph_page; /* this page's address */ 266 uint32_t ph_time; /* last referenced */ 267 uint16_t ph_nmissing; /* # of chunks in use */ 268 uint16_t ph_off; /* start offset in page */ 269 union { 270 /* !PR_USEBMAP */ 271 struct { 272 LIST_HEAD(, pool_item) 273 phu_itemlist; /* chunk list for this page */ 274 } phu_normal; 275 /* PR_USEBMAP */ 276 struct { 277 pool_item_bitmap_t phu_bitmap[1]; 278 } phu_notouch; 279 } ph_u2; 280 }; 281 #define ph_node ph_u1.phu_offpage.phu_node 282 #define ph_poolid ph_u1.phu_onpage.phu_poolid 283 #define ph_itemlist ph_u2.phu_normal.phu_itemlist 284 #define ph_bitmap ph_u2.phu_notouch.phu_bitmap 285 286 #define PHSIZE ALIGN(sizeof(struct pool_item_header)) 287 288 CTASSERT(offsetof(struct pool_item_header, ph_u2) + 289 BITMAP_MIN_SIZE / CHAR_BIT == sizeof(struct pool_item_header)); 290 291 #if defined(DIAGNOSTIC) && !defined(KASAN) 292 #define POOL_CHECK_MAGIC 293 #endif 294 295 struct pool_item { 296 #ifdef POOL_CHECK_MAGIC 297 u_int pi_magic; 298 #endif 299 #define PI_MAGIC 0xdeaddeadU 300 /* Other entries use only this list entry */ 301 LIST_ENTRY(pool_item) pi_list; 302 }; 303 304 #define POOL_NEEDS_CATCHUP(pp) \ 305 ((pp)->pr_nitems < (pp)->pr_minitems) 306 #define POOL_OBJ_TO_PAGE(pp, v) \ 307 (void *)((uintptr_t)v & pp->pr_alloc->pa_pagemask) 308 309 /* 310 * Pool cache management. 311 * 312 * Pool caches provide a way for constructed objects to be cached by the 313 * pool subsystem. This can lead to performance improvements by avoiding 314 * needless object construction/destruction; it is deferred until absolutely 315 * necessary. 316 * 317 * Caches are grouped into cache groups. Each cache group references up 318 * to PCG_NUMOBJECTS constructed objects. When a cache allocates an 319 * object from the pool, it calls the object's constructor and places it 320 * into a cache group. When a cache group frees an object back to the 321 * pool, it first calls the object's destructor. This allows the object 322 * to persist in constructed form while freed to the cache. 323 * 324 * The pool references each cache, so that when a pool is drained by the 325 * pagedaemon, it can drain each individual cache as well. Each time a 326 * cache is drained, the most idle cache group is freed to the pool in 327 * its entirety. 328 * 329 * Pool caches are layed on top of pools. By layering them, we can avoid 330 * the complexity of cache management for pools which would not benefit 331 * from it. 332 */ 333 334 static struct pool pcg_normal_pool; 335 static struct pool pcg_large_pool; 336 static struct pool cache_pool; 337 static struct pool cache_cpu_pool; 338 339 /* List of all caches. */ 340 TAILQ_HEAD(,pool_cache) pool_cache_head = 341 TAILQ_HEAD_INITIALIZER(pool_cache_head); 342 343 int pool_cache_disable; /* global disable for caching */ 344 static const pcg_t pcg_dummy; /* zero sized: always empty, yet always full */ 345 346 static bool pool_cache_put_slow(pool_cache_cpu_t *, int, 347 void *); 348 static bool pool_cache_get_slow(pool_cache_cpu_t *, int, 349 void **, paddr_t *, int); 350 static void pool_cache_cpu_init1(struct cpu_info *, pool_cache_t); 351 static void pool_cache_invalidate_groups(pool_cache_t, pcg_t *); 352 static void pool_cache_invalidate_cpu(pool_cache_t, u_int); 353 static void pool_cache_transfer(pool_cache_t); 354 355 static int pool_catchup(struct pool *); 356 static void pool_prime_page(struct pool *, void *, 357 struct pool_item_header *); 358 static void pool_update_curpage(struct pool *); 359 360 static int pool_grow(struct pool *, int); 361 static void *pool_allocator_alloc(struct pool *, int); 362 static void pool_allocator_free(struct pool *, void *); 363 364 static void pool_print_pagelist(struct pool *, struct pool_pagelist *, 365 void (*)(const char *, ...) __printflike(1, 2)); 366 static void pool_print1(struct pool *, const char *, 367 void (*)(const char *, ...) __printflike(1, 2)); 368 369 static int pool_chk_page(struct pool *, const char *, 370 struct pool_item_header *); 371 372 /* -------------------------------------------------------------------------- */ 373 374 static inline unsigned int 375 pr_item_bitmap_index(const struct pool *pp, const struct pool_item_header *ph, 376 const void *v) 377 { 378 const char *cp = v; 379 unsigned int idx; 380 381 KASSERT(pp->pr_roflags & PR_USEBMAP); 382 idx = (cp - (char *)ph->ph_page - ph->ph_off) / pp->pr_size; 383 384 if (__predict_false(idx >= pp->pr_itemsperpage)) { 385 panic("%s: [%s] %u >= %u", __func__, pp->pr_wchan, idx, 386 pp->pr_itemsperpage); 387 } 388 389 return idx; 390 } 391 392 static inline void 393 pr_item_bitmap_put(const struct pool *pp, struct pool_item_header *ph, 394 void *obj) 395 { 396 unsigned int idx = pr_item_bitmap_index(pp, ph, obj); 397 pool_item_bitmap_t *bitmap = ph->ph_bitmap + (idx / BITMAP_SIZE); 398 pool_item_bitmap_t mask = 1U << (idx & BITMAP_MASK); 399 400 if (__predict_false((*bitmap & mask) != 0)) { 401 panic("%s: [%s] %p already freed", __func__, pp->pr_wchan, obj); 402 } 403 404 *bitmap |= mask; 405 } 406 407 static inline void * 408 pr_item_bitmap_get(const struct pool *pp, struct pool_item_header *ph) 409 { 410 pool_item_bitmap_t *bitmap = ph->ph_bitmap; 411 unsigned int idx; 412 int i; 413 414 for (i = 0; ; i++) { 415 int bit; 416 417 KASSERT((i * BITMAP_SIZE) < pp->pr_itemsperpage); 418 bit = ffs32(bitmap[i]); 419 if (bit) { 420 pool_item_bitmap_t mask; 421 422 bit--; 423 idx = (i * BITMAP_SIZE) + bit; 424 mask = 1U << bit; 425 KASSERT((bitmap[i] & mask) != 0); 426 bitmap[i] &= ~mask; 427 break; 428 } 429 } 430 KASSERT(idx < pp->pr_itemsperpage); 431 return (char *)ph->ph_page + ph->ph_off + idx * pp->pr_size; 432 } 433 434 static inline void 435 pr_item_bitmap_init(const struct pool *pp, struct pool_item_header *ph) 436 { 437 pool_item_bitmap_t *bitmap = ph->ph_bitmap; 438 const int n = howmany(pp->pr_itemsperpage, BITMAP_SIZE); 439 int i; 440 441 for (i = 0; i < n; i++) { 442 bitmap[i] = (pool_item_bitmap_t)-1; 443 } 444 } 445 446 /* -------------------------------------------------------------------------- */ 447 448 static inline void 449 pr_item_linkedlist_put(const struct pool *pp, struct pool_item_header *ph, 450 void *obj) 451 { 452 struct pool_item *pi = obj; 453 454 #ifdef POOL_CHECK_MAGIC 455 pi->pi_magic = PI_MAGIC; 456 #endif 457 458 if (pp->pr_redzone) { 459 /* 460 * Mark the pool_item as valid. The rest is already 461 * invalid. 462 */ 463 kasan_mark(pi, sizeof(*pi), sizeof(*pi), 0); 464 } 465 466 LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); 467 } 468 469 static inline void * 470 pr_item_linkedlist_get(struct pool *pp, struct pool_item_header *ph) 471 { 472 struct pool_item *pi; 473 void *v; 474 475 v = pi = LIST_FIRST(&ph->ph_itemlist); 476 if (__predict_false(v == NULL)) { 477 mutex_exit(&pp->pr_lock); 478 panic("%s: [%s] page empty", __func__, pp->pr_wchan); 479 } 480 KASSERTMSG((pp->pr_nitems > 0), 481 "%s: [%s] nitems %u inconsistent on itemlist", 482 __func__, pp->pr_wchan, pp->pr_nitems); 483 #ifdef POOL_CHECK_MAGIC 484 KASSERTMSG((pi->pi_magic == PI_MAGIC), 485 "%s: [%s] free list modified: " 486 "magic=%x; page %p; item addr %p", __func__, 487 pp->pr_wchan, pi->pi_magic, ph->ph_page, pi); 488 #endif 489 490 /* 491 * Remove from item list. 492 */ 493 LIST_REMOVE(pi, pi_list); 494 495 return v; 496 } 497 498 /* -------------------------------------------------------------------------- */ 499 500 static inline void 501 pr_phinpage_check(struct pool *pp, struct pool_item_header *ph, void *page, 502 void *object) 503 { 504 if (__predict_false((void *)ph->ph_page != page)) { 505 panic("%s: [%s] item %p not part of pool", __func__, 506 pp->pr_wchan, object); 507 } 508 if (__predict_false((char *)object < (char *)page + ph->ph_off)) { 509 panic("%s: [%s] item %p below item space", __func__, 510 pp->pr_wchan, object); 511 } 512 if (__predict_false(ph->ph_poolid != pp->pr_poolid)) { 513 panic("%s: [%s] item %p poolid %u != %u", __func__, 514 pp->pr_wchan, object, ph->ph_poolid, pp->pr_poolid); 515 } 516 } 517 518 static inline void 519 pc_phinpage_check(pool_cache_t pc, void *object) 520 { 521 struct pool_item_header *ph; 522 struct pool *pp; 523 void *page; 524 525 pp = &pc->pc_pool; 526 page = POOL_OBJ_TO_PAGE(pp, object); 527 ph = (struct pool_item_header *)page; 528 529 pr_phinpage_check(pp, ph, page, object); 530 } 531 532 /* -------------------------------------------------------------------------- */ 533 534 static inline int 535 phtree_compare(struct pool_item_header *a, struct pool_item_header *b) 536 { 537 538 /* 539 * We consider pool_item_header with smaller ph_page bigger. This 540 * unnatural ordering is for the benefit of pr_find_pagehead. 541 */ 542 if (a->ph_page < b->ph_page) 543 return 1; 544 else if (a->ph_page > b->ph_page) 545 return -1; 546 else 547 return 0; 548 } 549 550 SPLAY_PROTOTYPE(phtree, pool_item_header, ph_node, phtree_compare); 551 SPLAY_GENERATE(phtree, pool_item_header, ph_node, phtree_compare); 552 553 static inline struct pool_item_header * 554 pr_find_pagehead_noalign(struct pool *pp, void *v) 555 { 556 struct pool_item_header *ph, tmp; 557 558 tmp.ph_page = (void *)(uintptr_t)v; 559 ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); 560 if (ph == NULL) { 561 ph = SPLAY_ROOT(&pp->pr_phtree); 562 if (ph != NULL && phtree_compare(&tmp, ph) >= 0) { 563 ph = SPLAY_NEXT(phtree, &pp->pr_phtree, ph); 564 } 565 KASSERT(ph == NULL || phtree_compare(&tmp, ph) < 0); 566 } 567 568 return ph; 569 } 570 571 /* 572 * Return the pool page header based on item address. 573 */ 574 static inline struct pool_item_header * 575 pr_find_pagehead(struct pool *pp, void *v) 576 { 577 struct pool_item_header *ph, tmp; 578 579 if ((pp->pr_roflags & PR_NOALIGN) != 0) { 580 ph = pr_find_pagehead_noalign(pp, v); 581 } else { 582 void *page = POOL_OBJ_TO_PAGE(pp, v); 583 if ((pp->pr_roflags & PR_PHINPAGE) != 0) { 584 ph = (struct pool_item_header *)page; 585 pr_phinpage_check(pp, ph, page, v); 586 } else { 587 tmp.ph_page = page; 588 ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); 589 } 590 } 591 592 KASSERT(ph == NULL || ((pp->pr_roflags & PR_PHINPAGE) != 0) || 593 ((char *)ph->ph_page <= (char *)v && 594 (char *)v < (char *)ph->ph_page + pp->pr_alloc->pa_pagesz)); 595 return ph; 596 } 597 598 static void 599 pr_pagelist_free(struct pool *pp, struct pool_pagelist *pq) 600 { 601 struct pool_item_header *ph; 602 603 while ((ph = LIST_FIRST(pq)) != NULL) { 604 LIST_REMOVE(ph, ph_pagelist); 605 pool_allocator_free(pp, ph->ph_page); 606 if ((pp->pr_roflags & PR_PHINPAGE) == 0) 607 pool_put(pp->pr_phpool, ph); 608 } 609 } 610 611 /* 612 * Remove a page from the pool. 613 */ 614 static inline void 615 pr_rmpage(struct pool *pp, struct pool_item_header *ph, 616 struct pool_pagelist *pq) 617 { 618 619 KASSERT(mutex_owned(&pp->pr_lock)); 620 621 /* 622 * If the page was idle, decrement the idle page count. 623 */ 624 if (ph->ph_nmissing == 0) { 625 KASSERT(pp->pr_nidle != 0); 626 KASSERTMSG((pp->pr_nitems >= pp->pr_itemsperpage), 627 "%s: [%s] nitems=%u < itemsperpage=%u", __func__, 628 pp->pr_wchan, pp->pr_nitems, pp->pr_itemsperpage); 629 pp->pr_nidle--; 630 } 631 632 pp->pr_nitems -= pp->pr_itemsperpage; 633 634 /* 635 * Unlink the page from the pool and queue it for release. 636 */ 637 LIST_REMOVE(ph, ph_pagelist); 638 if (pp->pr_roflags & PR_PHINPAGE) { 639 if (__predict_false(ph->ph_poolid != pp->pr_poolid)) { 640 panic("%s: [%s] ph %p poolid %u != %u", 641 __func__, pp->pr_wchan, ph, ph->ph_poolid, 642 pp->pr_poolid); 643 } 644 } else { 645 SPLAY_REMOVE(phtree, &pp->pr_phtree, ph); 646 } 647 LIST_INSERT_HEAD(pq, ph, ph_pagelist); 648 649 pp->pr_npages--; 650 pp->pr_npagefree++; 651 652 pool_update_curpage(pp); 653 } 654 655 /* 656 * Initialize all the pools listed in the "pools" link set. 657 */ 658 void 659 pool_subsystem_init(void) 660 { 661 size_t size; 662 int idx; 663 664 mutex_init(&pool_head_lock, MUTEX_DEFAULT, IPL_NONE); 665 mutex_init(&pool_allocator_lock, MUTEX_DEFAULT, IPL_NONE); 666 cv_init(&pool_busy, "poolbusy"); 667 668 /* 669 * Initialize private page header pool and cache magazine pool if we 670 * haven't done so yet. 671 */ 672 for (idx = 0; idx < PHPOOL_MAX; idx++) { 673 static char phpool_names[PHPOOL_MAX][6+1+6+1]; 674 int nelem; 675 size_t sz; 676 677 nelem = PHPOOL_FREELIST_NELEM(idx); 678 KASSERT(nelem != 0); 679 snprintf(phpool_names[idx], sizeof(phpool_names[idx]), 680 "phpool-%d", nelem); 681 sz = offsetof(struct pool_item_header, 682 ph_bitmap[howmany(nelem, BITMAP_SIZE)]); 683 pool_init(&phpool[idx], sz, 0, 0, 0, 684 phpool_names[idx], &pool_allocator_meta, IPL_VM); 685 } 686 687 size = sizeof(pcg_t) + 688 (PCG_NOBJECTS_NORMAL - 1) * sizeof(pcgpair_t); 689 pool_init(&pcg_normal_pool, size, coherency_unit, 0, 0, 690 "pcgnormal", &pool_allocator_meta, IPL_VM); 691 692 size = sizeof(pcg_t) + 693 (PCG_NOBJECTS_LARGE - 1) * sizeof(pcgpair_t); 694 pool_init(&pcg_large_pool, size, coherency_unit, 0, 0, 695 "pcglarge", &pool_allocator_meta, IPL_VM); 696 697 pool_init(&cache_pool, sizeof(struct pool_cache), coherency_unit, 698 0, 0, "pcache", &pool_allocator_meta, IPL_NONE); 699 700 pool_init(&cache_cpu_pool, sizeof(pool_cache_cpu_t), coherency_unit, 701 0, 0, "pcachecpu", &pool_allocator_meta, IPL_NONE); 702 } 703 704 static inline bool 705 pool_init_is_phinpage(const struct pool *pp) 706 { 707 size_t pagesize; 708 709 if (pp->pr_roflags & PR_PHINPAGE) { 710 return true; 711 } 712 if (pp->pr_roflags & (PR_NOTOUCH | PR_NOALIGN)) { 713 return false; 714 } 715 716 pagesize = pp->pr_alloc->pa_pagesz; 717 718 /* 719 * Threshold: the item size is below 1/16 of a page size, and below 720 * 8 times the page header size. The latter ensures we go off-page 721 * if the page header would make us waste a rather big item. 722 */ 723 if (pp->pr_size < MIN(pagesize / 16, PHSIZE * 8)) { 724 return true; 725 } 726 727 /* Put the header into the page if it doesn't waste any items. */ 728 if (pagesize / pp->pr_size == (pagesize - PHSIZE) / pp->pr_size) { 729 return true; 730 } 731 732 return false; 733 } 734 735 static inline bool 736 pool_init_is_usebmap(const struct pool *pp) 737 { 738 size_t bmapsize; 739 740 if (pp->pr_roflags & PR_NOTOUCH) { 741 return true; 742 } 743 744 /* 745 * If we're off-page, go with a bitmap. 746 */ 747 if (!(pp->pr_roflags & PR_PHINPAGE)) { 748 return true; 749 } 750 751 /* 752 * If we're on-page, and the page header can already contain a bitmap 753 * big enough to cover all the items of the page, go with a bitmap. 754 */ 755 bmapsize = roundup(PHSIZE, pp->pr_align) - 756 offsetof(struct pool_item_header, ph_bitmap[0]); 757 KASSERT(bmapsize % sizeof(pool_item_bitmap_t) == 0); 758 if (pp->pr_itemsperpage <= bmapsize * CHAR_BIT) { 759 return true; 760 } 761 762 return false; 763 } 764 765 /* 766 * Initialize the given pool resource structure. 767 * 768 * We export this routine to allow other kernel parts to declare 769 * static pools that must be initialized before kmem(9) is available. 770 */ 771 void 772 pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, 773 const char *wchan, struct pool_allocator *palloc, int ipl) 774 { 775 struct pool *pp1; 776 size_t prsize; 777 int itemspace, slack; 778 779 /* XXX ioff will be removed. */ 780 KASSERT(ioff == 0); 781 782 #ifdef DEBUG 783 if (__predict_true(!cold)) 784 mutex_enter(&pool_head_lock); 785 /* 786 * Check that the pool hasn't already been initialised and 787 * added to the list of all pools. 788 */ 789 TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { 790 if (pp == pp1) 791 panic("%s: [%s] already initialised", __func__, 792 wchan); 793 } 794 if (__predict_true(!cold)) 795 mutex_exit(&pool_head_lock); 796 #endif 797 798 if (palloc == NULL) 799 palloc = &pool_allocator_kmem; 800 801 if (!cold) 802 mutex_enter(&pool_allocator_lock); 803 if (palloc->pa_refcnt++ == 0) { 804 if (palloc->pa_pagesz == 0) 805 palloc->pa_pagesz = PAGE_SIZE; 806 807 TAILQ_INIT(&palloc->pa_list); 808 809 mutex_init(&palloc->pa_lock, MUTEX_DEFAULT, IPL_VM); 810 palloc->pa_pagemask = ~(palloc->pa_pagesz - 1); 811 palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1; 812 } 813 if (!cold) 814 mutex_exit(&pool_allocator_lock); 815 816 if (align == 0) 817 align = ALIGN(1); 818 819 prsize = size; 820 if ((flags & PR_NOTOUCH) == 0 && prsize < sizeof(struct pool_item)) 821 prsize = sizeof(struct pool_item); 822 823 prsize = roundup(prsize, align); 824 KASSERTMSG((prsize <= palloc->pa_pagesz), 825 "%s: [%s] pool item size (%zu) larger than page size (%u)", 826 __func__, wchan, prsize, palloc->pa_pagesz); 827 828 /* 829 * Initialize the pool structure. 830 */ 831 LIST_INIT(&pp->pr_emptypages); 832 LIST_INIT(&pp->pr_fullpages); 833 LIST_INIT(&pp->pr_partpages); 834 pp->pr_cache = NULL; 835 pp->pr_curpage = NULL; 836 pp->pr_npages = 0; 837 pp->pr_minitems = 0; 838 pp->pr_minpages = 0; 839 pp->pr_maxpages = UINT_MAX; 840 pp->pr_roflags = flags; 841 pp->pr_flags = 0; 842 pp->pr_size = prsize; 843 pp->pr_reqsize = size; 844 pp->pr_align = align; 845 pp->pr_wchan = wchan; 846 pp->pr_alloc = palloc; 847 pp->pr_poolid = atomic_inc_uint_nv(&poolid_counter); 848 pp->pr_nitems = 0; 849 pp->pr_nout = 0; 850 pp->pr_hardlimit = UINT_MAX; 851 pp->pr_hardlimit_warning = NULL; 852 pp->pr_hardlimit_ratecap.tv_sec = 0; 853 pp->pr_hardlimit_ratecap.tv_usec = 0; 854 pp->pr_hardlimit_warning_last.tv_sec = 0; 855 pp->pr_hardlimit_warning_last.tv_usec = 0; 856 pp->pr_drain_hook = NULL; 857 pp->pr_drain_hook_arg = NULL; 858 pp->pr_freecheck = NULL; 859 pp->pr_redzone = false; 860 pool_redzone_init(pp, size); 861 pool_quarantine_init(pp); 862 863 /* 864 * Decide whether to put the page header off-page to avoid wasting too 865 * large a part of the page or too big an item. Off-page page headers 866 * go on a hash table, so we can match a returned item with its header 867 * based on the page address. 868 */ 869 if (pool_init_is_phinpage(pp)) { 870 /* Use the beginning of the page for the page header */ 871 itemspace = palloc->pa_pagesz - roundup(PHSIZE, align); 872 pp->pr_itemoffset = roundup(PHSIZE, align); 873 pp->pr_roflags |= PR_PHINPAGE; 874 } else { 875 /* The page header will be taken from our page header pool */ 876 itemspace = palloc->pa_pagesz; 877 pp->pr_itemoffset = 0; 878 SPLAY_INIT(&pp->pr_phtree); 879 } 880 881 pp->pr_itemsperpage = itemspace / pp->pr_size; 882 KASSERT(pp->pr_itemsperpage != 0); 883 884 /* 885 * Decide whether to use a bitmap or a linked list to manage freed 886 * items. 887 */ 888 if (pool_init_is_usebmap(pp)) { 889 pp->pr_roflags |= PR_USEBMAP; 890 } 891 892 /* 893 * If we're off-page, then we're using a bitmap; choose the appropriate 894 * pool to allocate page headers, whose size varies depending on the 895 * bitmap. If we're on-page, nothing to do. 896 */ 897 if (!(pp->pr_roflags & PR_PHINPAGE)) { 898 int idx; 899 900 KASSERT(pp->pr_roflags & PR_USEBMAP); 901 902 for (idx = 0; pp->pr_itemsperpage > PHPOOL_FREELIST_NELEM(idx); 903 idx++) { 904 /* nothing */ 905 } 906 if (idx >= PHPOOL_MAX) { 907 /* 908 * if you see this panic, consider to tweak 909 * PHPOOL_MAX and PHPOOL_FREELIST_NELEM. 910 */ 911 panic("%s: [%s] too large itemsperpage(%d) for " 912 "PR_USEBMAP", __func__, 913 pp->pr_wchan, pp->pr_itemsperpage); 914 } 915 pp->pr_phpool = &phpool[idx]; 916 } else { 917 pp->pr_phpool = NULL; 918 } 919 920 /* 921 * Use the slack between the chunks and the page header 922 * for "cache coloring". 923 */ 924 slack = itemspace - pp->pr_itemsperpage * pp->pr_size; 925 pp->pr_maxcolor = rounddown(slack, align); 926 pp->pr_curcolor = 0; 927 928 pp->pr_nget = 0; 929 pp->pr_nfail = 0; 930 pp->pr_nput = 0; 931 pp->pr_npagealloc = 0; 932 pp->pr_npagefree = 0; 933 pp->pr_hiwat = 0; 934 pp->pr_nidle = 0; 935 pp->pr_refcnt = 0; 936 937 mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl); 938 cv_init(&pp->pr_cv, wchan); 939 pp->pr_ipl = ipl; 940 941 /* Insert into the list of all pools. */ 942 if (!cold) 943 mutex_enter(&pool_head_lock); 944 TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { 945 if (strcmp(pp1->pr_wchan, pp->pr_wchan) > 0) 946 break; 947 } 948 if (pp1 == NULL) 949 TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist); 950 else 951 TAILQ_INSERT_BEFORE(pp1, pp, pr_poollist); 952 if (!cold) 953 mutex_exit(&pool_head_lock); 954 955 /* Insert this into the list of pools using this allocator. */ 956 if (!cold) 957 mutex_enter(&palloc->pa_lock); 958 TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); 959 if (!cold) 960 mutex_exit(&palloc->pa_lock); 961 } 962 963 /* 964 * De-commision a pool resource. 965 */ 966 void 967 pool_destroy(struct pool *pp) 968 { 969 struct pool_pagelist pq; 970 struct pool_item_header *ph; 971 972 pool_quarantine_flush(pp); 973 974 /* Remove from global pool list */ 975 mutex_enter(&pool_head_lock); 976 while (pp->pr_refcnt != 0) 977 cv_wait(&pool_busy, &pool_head_lock); 978 TAILQ_REMOVE(&pool_head, pp, pr_poollist); 979 if (drainpp == pp) 980 drainpp = NULL; 981 mutex_exit(&pool_head_lock); 982 983 /* Remove this pool from its allocator's list of pools. */ 984 mutex_enter(&pp->pr_alloc->pa_lock); 985 TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list); 986 mutex_exit(&pp->pr_alloc->pa_lock); 987 988 mutex_enter(&pool_allocator_lock); 989 if (--pp->pr_alloc->pa_refcnt == 0) 990 mutex_destroy(&pp->pr_alloc->pa_lock); 991 mutex_exit(&pool_allocator_lock); 992 993 mutex_enter(&pp->pr_lock); 994 995 KASSERT(pp->pr_cache == NULL); 996 KASSERTMSG((pp->pr_nout == 0), 997 "%s: [%s] pool busy: still out: %u", __func__, pp->pr_wchan, 998 pp->pr_nout); 999 KASSERT(LIST_EMPTY(&pp->pr_fullpages)); 1000 KASSERT(LIST_EMPTY(&pp->pr_partpages)); 1001 1002 /* Remove all pages */ 1003 LIST_INIT(&pq); 1004 while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) 1005 pr_rmpage(pp, ph, &pq); 1006 1007 mutex_exit(&pp->pr_lock); 1008 1009 pr_pagelist_free(pp, &pq); 1010 cv_destroy(&pp->pr_cv); 1011 mutex_destroy(&pp->pr_lock); 1012 } 1013 1014 void 1015 pool_set_drain_hook(struct pool *pp, void (*fn)(void *, int), void *arg) 1016 { 1017 1018 /* XXX no locking -- must be used just after pool_init() */ 1019 KASSERTMSG((pp->pr_drain_hook == NULL), 1020 "%s: [%s] already set", __func__, pp->pr_wchan); 1021 pp->pr_drain_hook = fn; 1022 pp->pr_drain_hook_arg = arg; 1023 } 1024 1025 static struct pool_item_header * 1026 pool_alloc_item_header(struct pool *pp, void *storage, int flags) 1027 { 1028 struct pool_item_header *ph; 1029 1030 if ((pp->pr_roflags & PR_PHINPAGE) != 0) 1031 ph = storage; 1032 else 1033 ph = pool_get(pp->pr_phpool, flags); 1034 1035 return ph; 1036 } 1037 1038 /* 1039 * Grab an item from the pool. 1040 */ 1041 void * 1042 pool_get(struct pool *pp, int flags) 1043 { 1044 struct pool_item_header *ph; 1045 void *v; 1046 1047 KASSERT(!(flags & PR_NOWAIT) != !(flags & PR_WAITOK)); 1048 KASSERTMSG((pp->pr_itemsperpage != 0), 1049 "%s: [%s] pr_itemsperpage is zero, " 1050 "pool not initialized?", __func__, pp->pr_wchan); 1051 KASSERTMSG((!(cpu_intr_p() || cpu_softintr_p()) 1052 || pp->pr_ipl != IPL_NONE || cold || panicstr != NULL), 1053 "%s: [%s] is IPL_NONE, but called from interrupt context", 1054 __func__, pp->pr_wchan); 1055 if (flags & PR_WAITOK) { 1056 ASSERT_SLEEPABLE(); 1057 } 1058 1059 mutex_enter(&pp->pr_lock); 1060 startover: 1061 /* 1062 * Check to see if we've reached the hard limit. If we have, 1063 * and we can wait, then wait until an item has been returned to 1064 * the pool. 1065 */ 1066 KASSERTMSG((pp->pr_nout <= pp->pr_hardlimit), 1067 "%s: %s: crossed hard limit", __func__, pp->pr_wchan); 1068 if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) { 1069 if (pp->pr_drain_hook != NULL) { 1070 /* 1071 * Since the drain hook is going to free things 1072 * back to the pool, unlock, call the hook, re-lock, 1073 * and check the hardlimit condition again. 1074 */ 1075 mutex_exit(&pp->pr_lock); 1076 (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags); 1077 mutex_enter(&pp->pr_lock); 1078 if (pp->pr_nout < pp->pr_hardlimit) 1079 goto startover; 1080 } 1081 1082 if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) { 1083 /* 1084 * XXX: A warning isn't logged in this case. Should 1085 * it be? 1086 */ 1087 pp->pr_flags |= PR_WANTED; 1088 do { 1089 cv_wait(&pp->pr_cv, &pp->pr_lock); 1090 } while (pp->pr_flags & PR_WANTED); 1091 goto startover; 1092 } 1093 1094 /* 1095 * Log a message that the hard limit has been hit. 1096 */ 1097 if (pp->pr_hardlimit_warning != NULL && 1098 ratecheck(&pp->pr_hardlimit_warning_last, 1099 &pp->pr_hardlimit_ratecap)) 1100 log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning); 1101 1102 pp->pr_nfail++; 1103 1104 mutex_exit(&pp->pr_lock); 1105 KASSERT((flags & (PR_NOWAIT|PR_LIMITFAIL)) != 0); 1106 return NULL; 1107 } 1108 1109 /* 1110 * The convention we use is that if `curpage' is not NULL, then 1111 * it points at a non-empty bucket. In particular, `curpage' 1112 * never points at a page header which has PR_PHINPAGE set and 1113 * has no items in its bucket. 1114 */ 1115 if ((ph = pp->pr_curpage) == NULL) { 1116 int error; 1117 1118 KASSERTMSG((pp->pr_nitems == 0), 1119 "%s: [%s] curpage NULL, inconsistent nitems %u", 1120 __func__, pp->pr_wchan, pp->pr_nitems); 1121 1122 /* 1123 * Call the back-end page allocator for more memory. 1124 * Release the pool lock, as the back-end page allocator 1125 * may block. 1126 */ 1127 error = pool_grow(pp, flags); 1128 if (error != 0) { 1129 /* 1130 * pool_grow aborts when another thread 1131 * is allocating a new page. Retry if it 1132 * waited for it. 1133 */ 1134 if (error == ERESTART) 1135 goto startover; 1136 1137 /* 1138 * We were unable to allocate a page or item 1139 * header, but we released the lock during 1140 * allocation, so perhaps items were freed 1141 * back to the pool. Check for this case. 1142 */ 1143 if (pp->pr_curpage != NULL) 1144 goto startover; 1145 1146 pp->pr_nfail++; 1147 mutex_exit(&pp->pr_lock); 1148 KASSERT((flags & (PR_WAITOK|PR_NOWAIT)) == PR_NOWAIT); 1149 return NULL; 1150 } 1151 1152 /* Start the allocation process over. */ 1153 goto startover; 1154 } 1155 if (pp->pr_roflags & PR_USEBMAP) { 1156 KASSERTMSG((ph->ph_nmissing < pp->pr_itemsperpage), 1157 "%s: [%s] pool page empty", __func__, pp->pr_wchan); 1158 v = pr_item_bitmap_get(pp, ph); 1159 } else { 1160 v = pr_item_linkedlist_get(pp, ph); 1161 } 1162 pp->pr_nitems--; 1163 pp->pr_nout++; 1164 if (ph->ph_nmissing == 0) { 1165 KASSERT(pp->pr_nidle > 0); 1166 pp->pr_nidle--; 1167 1168 /* 1169 * This page was previously empty. Move it to the list of 1170 * partially-full pages. This page is already curpage. 1171 */ 1172 LIST_REMOVE(ph, ph_pagelist); 1173 LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); 1174 } 1175 ph->ph_nmissing++; 1176 if (ph->ph_nmissing == pp->pr_itemsperpage) { 1177 KASSERTMSG(((pp->pr_roflags & PR_USEBMAP) || 1178 LIST_EMPTY(&ph->ph_itemlist)), 1179 "%s: [%s] nmissing (%u) inconsistent", __func__, 1180 pp->pr_wchan, ph->ph_nmissing); 1181 /* 1182 * This page is now full. Move it to the full list 1183 * and select a new current page. 1184 */ 1185 LIST_REMOVE(ph, ph_pagelist); 1186 LIST_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist); 1187 pool_update_curpage(pp); 1188 } 1189 1190 pp->pr_nget++; 1191 1192 /* 1193 * If we have a low water mark and we are now below that low 1194 * water mark, add more items to the pool. 1195 */ 1196 if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) { 1197 /* 1198 * XXX: Should we log a warning? Should we set up a timeout 1199 * to try again in a second or so? The latter could break 1200 * a caller's assumptions about interrupt protection, etc. 1201 */ 1202 } 1203 1204 mutex_exit(&pp->pr_lock); 1205 KASSERT((((vaddr_t)v) & (pp->pr_align - 1)) == 0); 1206 FREECHECK_OUT(&pp->pr_freecheck, v); 1207 pool_redzone_fill(pp, v); 1208 pool_get_kmsan(pp, v); 1209 if (flags & PR_ZERO) 1210 memset(v, 0, pp->pr_reqsize); 1211 else 1212 pool_kleak_fill(pp, v); 1213 return v; 1214 } 1215 1216 /* 1217 * Internal version of pool_put(). Pool is already locked/entered. 1218 */ 1219 static void 1220 pool_do_put(struct pool *pp, void *v, struct pool_pagelist *pq) 1221 { 1222 struct pool_item_header *ph; 1223 1224 KASSERT(mutex_owned(&pp->pr_lock)); 1225 pool_redzone_check(pp, v); 1226 pool_put_kmsan(pp, v); 1227 FREECHECK_IN(&pp->pr_freecheck, v); 1228 LOCKDEBUG_MEM_CHECK(v, pp->pr_size); 1229 1230 KASSERTMSG((pp->pr_nout > 0), 1231 "%s: [%s] putting with none out", __func__, pp->pr_wchan); 1232 1233 if (__predict_false((ph = pr_find_pagehead(pp, v)) == NULL)) { 1234 panic("%s: [%s] page header missing", __func__, pp->pr_wchan); 1235 } 1236 1237 /* 1238 * Return to item list. 1239 */ 1240 if (pp->pr_roflags & PR_USEBMAP) { 1241 pr_item_bitmap_put(pp, ph, v); 1242 } else { 1243 pr_item_linkedlist_put(pp, ph, v); 1244 } 1245 KDASSERT(ph->ph_nmissing != 0); 1246 ph->ph_nmissing--; 1247 pp->pr_nput++; 1248 pp->pr_nitems++; 1249 pp->pr_nout--; 1250 1251 /* Cancel "pool empty" condition if it exists */ 1252 if (pp->pr_curpage == NULL) 1253 pp->pr_curpage = ph; 1254 1255 if (pp->pr_flags & PR_WANTED) { 1256 pp->pr_flags &= ~PR_WANTED; 1257 cv_broadcast(&pp->pr_cv); 1258 } 1259 1260 /* 1261 * If this page is now empty, do one of two things: 1262 * 1263 * (1) If we have more pages than the page high water mark, 1264 * free the page back to the system. ONLY CONSIDER 1265 * FREEING BACK A PAGE IF WE HAVE MORE THAN OUR MINIMUM PAGE 1266 * CLAIM. 1267 * 1268 * (2) Otherwise, move the page to the empty page list. 1269 * 1270 * Either way, select a new current page (so we use a partially-full 1271 * page if one is available). 1272 */ 1273 if (ph->ph_nmissing == 0) { 1274 pp->pr_nidle++; 1275 if (pp->pr_npages > pp->pr_minpages && 1276 pp->pr_npages > pp->pr_maxpages) { 1277 pr_rmpage(pp, ph, pq); 1278 } else { 1279 LIST_REMOVE(ph, ph_pagelist); 1280 LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); 1281 1282 /* 1283 * Update the timestamp on the page. A page must 1284 * be idle for some period of time before it can 1285 * be reclaimed by the pagedaemon. This minimizes 1286 * ping-pong'ing for memory. 1287 * 1288 * note for 64-bit time_t: truncating to 32-bit is not 1289 * a problem for our usage. 1290 */ 1291 ph->ph_time = time_uptime; 1292 } 1293 pool_update_curpage(pp); 1294 } 1295 1296 /* 1297 * If the page was previously completely full, move it to the 1298 * partially-full list and make it the current page. The next 1299 * allocation will get the item from this page, instead of 1300 * further fragmenting the pool. 1301 */ 1302 else if (ph->ph_nmissing == (pp->pr_itemsperpage - 1)) { 1303 LIST_REMOVE(ph, ph_pagelist); 1304 LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); 1305 pp->pr_curpage = ph; 1306 } 1307 } 1308 1309 void 1310 pool_put(struct pool *pp, void *v) 1311 { 1312 struct pool_pagelist pq; 1313 1314 LIST_INIT(&pq); 1315 1316 mutex_enter(&pp->pr_lock); 1317 if (!pool_put_quarantine(pp, v, &pq)) { 1318 pool_do_put(pp, v, &pq); 1319 } 1320 mutex_exit(&pp->pr_lock); 1321 1322 pr_pagelist_free(pp, &pq); 1323 } 1324 1325 /* 1326 * pool_grow: grow a pool by a page. 1327 * 1328 * => called with pool locked. 1329 * => unlock and relock the pool. 1330 * => return with pool locked. 1331 */ 1332 1333 static int 1334 pool_grow(struct pool *pp, int flags) 1335 { 1336 struct pool_item_header *ph; 1337 char *storage; 1338 1339 /* 1340 * If there's a pool_grow in progress, wait for it to complete 1341 * and try again from the top. 1342 */ 1343 if (pp->pr_flags & PR_GROWING) { 1344 if (flags & PR_WAITOK) { 1345 do { 1346 cv_wait(&pp->pr_cv, &pp->pr_lock); 1347 } while (pp->pr_flags & PR_GROWING); 1348 return ERESTART; 1349 } else { 1350 if (pp->pr_flags & PR_GROWINGNOWAIT) { 1351 /* 1352 * This needs an unlock/relock dance so 1353 * that the other caller has a chance to 1354 * run and actually do the thing. Note 1355 * that this is effectively a busy-wait. 1356 */ 1357 mutex_exit(&pp->pr_lock); 1358 mutex_enter(&pp->pr_lock); 1359 return ERESTART; 1360 } 1361 return EWOULDBLOCK; 1362 } 1363 } 1364 pp->pr_flags |= PR_GROWING; 1365 if (flags & PR_WAITOK) 1366 mutex_exit(&pp->pr_lock); 1367 else 1368 pp->pr_flags |= PR_GROWINGNOWAIT; 1369 1370 storage = pool_allocator_alloc(pp, flags); 1371 if (__predict_false(storage == NULL)) 1372 goto out; 1373 1374 ph = pool_alloc_item_header(pp, storage, flags); 1375 if (__predict_false(ph == NULL)) { 1376 pool_allocator_free(pp, storage); 1377 goto out; 1378 } 1379 1380 if (flags & PR_WAITOK) 1381 mutex_enter(&pp->pr_lock); 1382 pool_prime_page(pp, storage, ph); 1383 pp->pr_npagealloc++; 1384 KASSERT(pp->pr_flags & PR_GROWING); 1385 pp->pr_flags &= ~(PR_GROWING|PR_GROWINGNOWAIT); 1386 /* 1387 * If anyone was waiting for pool_grow, notify them that we 1388 * may have just done it. 1389 */ 1390 cv_broadcast(&pp->pr_cv); 1391 return 0; 1392 out: 1393 if (flags & PR_WAITOK) 1394 mutex_enter(&pp->pr_lock); 1395 KASSERT(pp->pr_flags & PR_GROWING); 1396 pp->pr_flags &= ~(PR_GROWING|PR_GROWINGNOWAIT); 1397 return ENOMEM; 1398 } 1399 1400 /* 1401 * Add N items to the pool. 1402 */ 1403 int 1404 pool_prime(struct pool *pp, int n) 1405 { 1406 int newpages; 1407 int error = 0; 1408 1409 mutex_enter(&pp->pr_lock); 1410 1411 newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 1412 1413 while (newpages > 0) { 1414 error = pool_grow(pp, PR_NOWAIT); 1415 if (error) { 1416 if (error == ERESTART) 1417 continue; 1418 break; 1419 } 1420 pp->pr_minpages++; 1421 newpages--; 1422 } 1423 1424 if (pp->pr_minpages >= pp->pr_maxpages) 1425 pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ 1426 1427 mutex_exit(&pp->pr_lock); 1428 return error; 1429 } 1430 1431 /* 1432 * Add a page worth of items to the pool. 1433 * 1434 * Note, we must be called with the pool descriptor LOCKED. 1435 */ 1436 static void 1437 pool_prime_page(struct pool *pp, void *storage, struct pool_item_header *ph) 1438 { 1439 const unsigned int align = pp->pr_align; 1440 struct pool_item *pi; 1441 void *cp = storage; 1442 int n; 1443 1444 KASSERT(mutex_owned(&pp->pr_lock)); 1445 KASSERTMSG(((pp->pr_roflags & PR_NOALIGN) || 1446 (((uintptr_t)cp & (pp->pr_alloc->pa_pagesz - 1)) == 0)), 1447 "%s: [%s] unaligned page: %p", __func__, pp->pr_wchan, cp); 1448 1449 /* 1450 * Insert page header. 1451 */ 1452 LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); 1453 LIST_INIT(&ph->ph_itemlist); 1454 ph->ph_page = storage; 1455 ph->ph_nmissing = 0; 1456 ph->ph_time = time_uptime; 1457 if (pp->pr_roflags & PR_PHINPAGE) 1458 ph->ph_poolid = pp->pr_poolid; 1459 else 1460 SPLAY_INSERT(phtree, &pp->pr_phtree, ph); 1461 1462 pp->pr_nidle++; 1463 1464 /* 1465 * The item space starts after the on-page header, if any. 1466 */ 1467 ph->ph_off = pp->pr_itemoffset; 1468 1469 /* 1470 * Color this page. 1471 */ 1472 ph->ph_off += pp->pr_curcolor; 1473 cp = (char *)cp + ph->ph_off; 1474 if ((pp->pr_curcolor += align) > pp->pr_maxcolor) 1475 pp->pr_curcolor = 0; 1476 1477 KASSERT((((vaddr_t)cp) & (align - 1)) == 0); 1478 1479 /* 1480 * Insert remaining chunks on the bucket list. 1481 */ 1482 n = pp->pr_itemsperpage; 1483 pp->pr_nitems += n; 1484 1485 if (pp->pr_roflags & PR_USEBMAP) { 1486 pr_item_bitmap_init(pp, ph); 1487 } else { 1488 while (n--) { 1489 pi = (struct pool_item *)cp; 1490 1491 KASSERT((((vaddr_t)pi) & (align - 1)) == 0); 1492 1493 /* Insert on page list */ 1494 LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); 1495 #ifdef POOL_CHECK_MAGIC 1496 pi->pi_magic = PI_MAGIC; 1497 #endif 1498 cp = (char *)cp + pp->pr_size; 1499 1500 KASSERT((((vaddr_t)cp) & (align - 1)) == 0); 1501 } 1502 } 1503 1504 /* 1505 * If the pool was depleted, point at the new page. 1506 */ 1507 if (pp->pr_curpage == NULL) 1508 pp->pr_curpage = ph; 1509 1510 if (++pp->pr_npages > pp->pr_hiwat) 1511 pp->pr_hiwat = pp->pr_npages; 1512 } 1513 1514 /* 1515 * Used by pool_get() when nitems drops below the low water mark. This 1516 * is used to catch up pr_nitems with the low water mark. 1517 * 1518 * Note 1, we never wait for memory here, we let the caller decide what to do. 1519 * 1520 * Note 2, we must be called with the pool already locked, and we return 1521 * with it locked. 1522 */ 1523 static int 1524 pool_catchup(struct pool *pp) 1525 { 1526 int error = 0; 1527 1528 while (POOL_NEEDS_CATCHUP(pp)) { 1529 error = pool_grow(pp, PR_NOWAIT); 1530 if (error) { 1531 if (error == ERESTART) 1532 continue; 1533 break; 1534 } 1535 } 1536 return error; 1537 } 1538 1539 static void 1540 pool_update_curpage(struct pool *pp) 1541 { 1542 1543 pp->pr_curpage = LIST_FIRST(&pp->pr_partpages); 1544 if (pp->pr_curpage == NULL) { 1545 pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages); 1546 } 1547 KASSERT((pp->pr_curpage == NULL && pp->pr_nitems == 0) || 1548 (pp->pr_curpage != NULL && pp->pr_nitems > 0)); 1549 } 1550 1551 void 1552 pool_setlowat(struct pool *pp, int n) 1553 { 1554 1555 mutex_enter(&pp->pr_lock); 1556 1557 pp->pr_minitems = n; 1558 pp->pr_minpages = (n == 0) 1559 ? 0 1560 : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 1561 1562 /* Make sure we're caught up with the newly-set low water mark. */ 1563 if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) { 1564 /* 1565 * XXX: Should we log a warning? Should we set up a timeout 1566 * to try again in a second or so? The latter could break 1567 * a caller's assumptions about interrupt protection, etc. 1568 */ 1569 } 1570 1571 mutex_exit(&pp->pr_lock); 1572 } 1573 1574 void 1575 pool_sethiwat(struct pool *pp, int n) 1576 { 1577 1578 mutex_enter(&pp->pr_lock); 1579 1580 pp->pr_maxpages = (n == 0) 1581 ? 0 1582 : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 1583 1584 mutex_exit(&pp->pr_lock); 1585 } 1586 1587 void 1588 pool_sethardlimit(struct pool *pp, int n, const char *warnmess, int ratecap) 1589 { 1590 1591 mutex_enter(&pp->pr_lock); 1592 1593 pp->pr_hardlimit = n; 1594 pp->pr_hardlimit_warning = warnmess; 1595 pp->pr_hardlimit_ratecap.tv_sec = ratecap; 1596 pp->pr_hardlimit_warning_last.tv_sec = 0; 1597 pp->pr_hardlimit_warning_last.tv_usec = 0; 1598 1599 /* 1600 * In-line version of pool_sethiwat(), because we don't want to 1601 * release the lock. 1602 */ 1603 pp->pr_maxpages = (n == 0) 1604 ? 0 1605 : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 1606 1607 mutex_exit(&pp->pr_lock); 1608 } 1609 1610 /* 1611 * Release all complete pages that have not been used recently. 1612 * 1613 * Must not be called from interrupt context. 1614 */ 1615 int 1616 pool_reclaim(struct pool *pp) 1617 { 1618 struct pool_item_header *ph, *phnext; 1619 struct pool_pagelist pq; 1620 uint32_t curtime; 1621 bool klock; 1622 int rv; 1623 1624 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 1625 1626 if (pp->pr_drain_hook != NULL) { 1627 /* 1628 * The drain hook must be called with the pool unlocked. 1629 */ 1630 (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, PR_NOWAIT); 1631 } 1632 1633 /* 1634 * XXXSMP Because we do not want to cause non-MPSAFE code 1635 * to block. 1636 */ 1637 if (pp->pr_ipl == IPL_SOFTNET || pp->pr_ipl == IPL_SOFTCLOCK || 1638 pp->pr_ipl == IPL_SOFTSERIAL) { 1639 KERNEL_LOCK(1, NULL); 1640 klock = true; 1641 } else 1642 klock = false; 1643 1644 /* Reclaim items from the pool's cache (if any). */ 1645 if (pp->pr_cache != NULL) 1646 pool_cache_invalidate(pp->pr_cache); 1647 1648 if (mutex_tryenter(&pp->pr_lock) == 0) { 1649 if (klock) { 1650 KERNEL_UNLOCK_ONE(NULL); 1651 } 1652 return 0; 1653 } 1654 1655 LIST_INIT(&pq); 1656 1657 curtime = time_uptime; 1658 1659 for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) { 1660 phnext = LIST_NEXT(ph, ph_pagelist); 1661 1662 /* Check our minimum page claim */ 1663 if (pp->pr_npages <= pp->pr_minpages) 1664 break; 1665 1666 KASSERT(ph->ph_nmissing == 0); 1667 if (curtime - ph->ph_time < pool_inactive_time) 1668 continue; 1669 1670 /* 1671 * If freeing this page would put us below 1672 * the low water mark, stop now. 1673 */ 1674 if ((pp->pr_nitems - pp->pr_itemsperpage) < 1675 pp->pr_minitems) 1676 break; 1677 1678 pr_rmpage(pp, ph, &pq); 1679 } 1680 1681 mutex_exit(&pp->pr_lock); 1682 1683 if (LIST_EMPTY(&pq)) 1684 rv = 0; 1685 else { 1686 pr_pagelist_free(pp, &pq); 1687 rv = 1; 1688 } 1689 1690 if (klock) { 1691 KERNEL_UNLOCK_ONE(NULL); 1692 } 1693 1694 return rv; 1695 } 1696 1697 /* 1698 * Drain pools, one at a time. The drained pool is returned within ppp. 1699 * 1700 * Note, must never be called from interrupt context. 1701 */ 1702 bool 1703 pool_drain(struct pool **ppp) 1704 { 1705 bool reclaimed; 1706 struct pool *pp; 1707 1708 KASSERT(!TAILQ_EMPTY(&pool_head)); 1709 1710 pp = NULL; 1711 1712 /* Find next pool to drain, and add a reference. */ 1713 mutex_enter(&pool_head_lock); 1714 do { 1715 if (drainpp == NULL) { 1716 drainpp = TAILQ_FIRST(&pool_head); 1717 } 1718 if (drainpp != NULL) { 1719 pp = drainpp; 1720 drainpp = TAILQ_NEXT(pp, pr_poollist); 1721 } 1722 /* 1723 * Skip completely idle pools. We depend on at least 1724 * one pool in the system being active. 1725 */ 1726 } while (pp == NULL || pp->pr_npages == 0); 1727 pp->pr_refcnt++; 1728 mutex_exit(&pool_head_lock); 1729 1730 /* Drain the cache (if any) and pool.. */ 1731 reclaimed = pool_reclaim(pp); 1732 1733 /* Finally, unlock the pool. */ 1734 mutex_enter(&pool_head_lock); 1735 pp->pr_refcnt--; 1736 cv_broadcast(&pool_busy); 1737 mutex_exit(&pool_head_lock); 1738 1739 if (ppp != NULL) 1740 *ppp = pp; 1741 1742 return reclaimed; 1743 } 1744 1745 /* 1746 * Calculate the total number of pages consumed by pools. 1747 */ 1748 int 1749 pool_totalpages(void) 1750 { 1751 1752 mutex_enter(&pool_head_lock); 1753 int pages = pool_totalpages_locked(); 1754 mutex_exit(&pool_head_lock); 1755 1756 return pages; 1757 } 1758 1759 int 1760 pool_totalpages_locked(void) 1761 { 1762 struct pool *pp; 1763 uint64_t total = 0; 1764 1765 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 1766 uint64_t bytes = pp->pr_npages * pp->pr_alloc->pa_pagesz; 1767 1768 if ((pp->pr_roflags & PR_RECURSIVE) != 0) 1769 bytes -= (pp->pr_nout * pp->pr_size); 1770 total += bytes; 1771 } 1772 1773 return atop(total); 1774 } 1775 1776 /* 1777 * Diagnostic helpers. 1778 */ 1779 1780 void 1781 pool_printall(const char *modif, void (*pr)(const char *, ...)) 1782 { 1783 struct pool *pp; 1784 1785 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 1786 pool_printit(pp, modif, pr); 1787 } 1788 } 1789 1790 void 1791 pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) 1792 { 1793 1794 if (pp == NULL) { 1795 (*pr)("Must specify a pool to print.\n"); 1796 return; 1797 } 1798 1799 pool_print1(pp, modif, pr); 1800 } 1801 1802 static void 1803 pool_print_pagelist(struct pool *pp, struct pool_pagelist *pl, 1804 void (*pr)(const char *, ...)) 1805 { 1806 struct pool_item_header *ph; 1807 1808 LIST_FOREACH(ph, pl, ph_pagelist) { 1809 (*pr)("\t\tpage %p, nmissing %d, time %" PRIu32 "\n", 1810 ph->ph_page, ph->ph_nmissing, ph->ph_time); 1811 #ifdef POOL_CHECK_MAGIC 1812 struct pool_item *pi; 1813 if (!(pp->pr_roflags & PR_USEBMAP)) { 1814 LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) { 1815 if (pi->pi_magic != PI_MAGIC) { 1816 (*pr)("\t\t\titem %p, magic 0x%x\n", 1817 pi, pi->pi_magic); 1818 } 1819 } 1820 } 1821 #endif 1822 } 1823 } 1824 1825 static void 1826 pool_print1(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) 1827 { 1828 struct pool_item_header *ph; 1829 pool_cache_t pc; 1830 pcg_t *pcg; 1831 pool_cache_cpu_t *cc; 1832 uint64_t cpuhit, cpumiss; 1833 int i, print_log = 0, print_pagelist = 0, print_cache = 0; 1834 char c; 1835 1836 while ((c = *modif++) != '\0') { 1837 if (c == 'l') 1838 print_log = 1; 1839 if (c == 'p') 1840 print_pagelist = 1; 1841 if (c == 'c') 1842 print_cache = 1; 1843 } 1844 1845 if ((pc = pp->pr_cache) != NULL) { 1846 (*pr)("POOL CACHE"); 1847 } else { 1848 (*pr)("POOL"); 1849 } 1850 1851 (*pr)(" %s: size %u, align %u, ioff %u, roflags 0x%08x\n", 1852 pp->pr_wchan, pp->pr_size, pp->pr_align, pp->pr_itemoffset, 1853 pp->pr_roflags); 1854 (*pr)("\talloc %p\n", pp->pr_alloc); 1855 (*pr)("\tminitems %u, minpages %u, maxpages %u, npages %u\n", 1856 pp->pr_minitems, pp->pr_minpages, pp->pr_maxpages, pp->pr_npages); 1857 (*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n", 1858 pp->pr_itemsperpage, pp->pr_nitems, pp->pr_nout, pp->pr_hardlimit); 1859 1860 (*pr)("\tnget %lu, nfail %lu, nput %lu\n", 1861 pp->pr_nget, pp->pr_nfail, pp->pr_nput); 1862 (*pr)("\tnpagealloc %lu, npagefree %lu, hiwat %u, nidle %lu\n", 1863 pp->pr_npagealloc, pp->pr_npagefree, pp->pr_hiwat, pp->pr_nidle); 1864 1865 if (print_pagelist == 0) 1866 goto skip_pagelist; 1867 1868 if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) 1869 (*pr)("\n\tempty page list:\n"); 1870 pool_print_pagelist(pp, &pp->pr_emptypages, pr); 1871 if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL) 1872 (*pr)("\n\tfull page list:\n"); 1873 pool_print_pagelist(pp, &pp->pr_fullpages, pr); 1874 if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL) 1875 (*pr)("\n\tpartial-page list:\n"); 1876 pool_print_pagelist(pp, &pp->pr_partpages, pr); 1877 1878 if (pp->pr_curpage == NULL) 1879 (*pr)("\tno current page\n"); 1880 else 1881 (*pr)("\tcurpage %p\n", pp->pr_curpage->ph_page); 1882 1883 skip_pagelist: 1884 if (print_log == 0) 1885 goto skip_log; 1886 1887 (*pr)("\n"); 1888 1889 skip_log: 1890 1891 #define PR_GROUPLIST(pcg) \ 1892 (*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \ 1893 for (i = 0; i < pcg->pcg_size; i++) { \ 1894 if (pcg->pcg_objects[i].pcgo_pa != \ 1895 POOL_PADDR_INVALID) { \ 1896 (*pr)("\t\t\t%p, 0x%llx\n", \ 1897 pcg->pcg_objects[i].pcgo_va, \ 1898 (unsigned long long) \ 1899 pcg->pcg_objects[i].pcgo_pa); \ 1900 } else { \ 1901 (*pr)("\t\t\t%p\n", \ 1902 pcg->pcg_objects[i].pcgo_va); \ 1903 } \ 1904 } 1905 1906 if (pc != NULL) { 1907 cpuhit = 0; 1908 cpumiss = 0; 1909 for (i = 0; i < __arraycount(pc->pc_cpus); i++) { 1910 if ((cc = pc->pc_cpus[i]) == NULL) 1911 continue; 1912 cpuhit += cc->cc_hits; 1913 cpumiss += cc->cc_misses; 1914 } 1915 (*pr)("\tcpu layer hits %llu misses %llu\n", cpuhit, cpumiss); 1916 (*pr)("\tcache layer hits %llu misses %llu\n", 1917 pc->pc_hits, pc->pc_misses); 1918 (*pr)("\tcache layer entry uncontended %llu contended %llu\n", 1919 pc->pc_hits + pc->pc_misses - pc->pc_contended, 1920 pc->pc_contended); 1921 (*pr)("\tcache layer empty groups %u full groups %u\n", 1922 pc->pc_nempty, pc->pc_nfull); 1923 if (print_cache) { 1924 (*pr)("\tfull cache groups:\n"); 1925 for (pcg = pc->pc_fullgroups; pcg != NULL; 1926 pcg = pcg->pcg_next) { 1927 PR_GROUPLIST(pcg); 1928 } 1929 (*pr)("\tempty cache groups:\n"); 1930 for (pcg = pc->pc_emptygroups; pcg != NULL; 1931 pcg = pcg->pcg_next) { 1932 PR_GROUPLIST(pcg); 1933 } 1934 } 1935 } 1936 #undef PR_GROUPLIST 1937 } 1938 1939 static int 1940 pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph) 1941 { 1942 struct pool_item *pi; 1943 void *page; 1944 int n; 1945 1946 if ((pp->pr_roflags & PR_NOALIGN) == 0) { 1947 page = POOL_OBJ_TO_PAGE(pp, ph); 1948 if (page != ph->ph_page && 1949 (pp->pr_roflags & PR_PHINPAGE) != 0) { 1950 if (label != NULL) 1951 printf("%s: ", label); 1952 printf("pool(%p:%s): page inconsistency: page %p;" 1953 " at page head addr %p (p %p)\n", pp, 1954 pp->pr_wchan, ph->ph_page, 1955 ph, page); 1956 return 1; 1957 } 1958 } 1959 1960 if ((pp->pr_roflags & PR_USEBMAP) != 0) 1961 return 0; 1962 1963 for (pi = LIST_FIRST(&ph->ph_itemlist), n = 0; 1964 pi != NULL; 1965 pi = LIST_NEXT(pi,pi_list), n++) { 1966 1967 #ifdef POOL_CHECK_MAGIC 1968 if (pi->pi_magic != PI_MAGIC) { 1969 if (label != NULL) 1970 printf("%s: ", label); 1971 printf("pool(%s): free list modified: magic=%x;" 1972 " page %p; item ordinal %d; addr %p\n", 1973 pp->pr_wchan, pi->pi_magic, ph->ph_page, 1974 n, pi); 1975 panic("pool"); 1976 } 1977 #endif 1978 if ((pp->pr_roflags & PR_NOALIGN) != 0) { 1979 continue; 1980 } 1981 page = POOL_OBJ_TO_PAGE(pp, pi); 1982 if (page == ph->ph_page) 1983 continue; 1984 1985 if (label != NULL) 1986 printf("%s: ", label); 1987 printf("pool(%p:%s): page inconsistency: page %p;" 1988 " item ordinal %d; addr %p (p %p)\n", pp, 1989 pp->pr_wchan, ph->ph_page, 1990 n, pi, page); 1991 return 1; 1992 } 1993 return 0; 1994 } 1995 1996 1997 int 1998 pool_chk(struct pool *pp, const char *label) 1999 { 2000 struct pool_item_header *ph; 2001 int r = 0; 2002 2003 mutex_enter(&pp->pr_lock); 2004 LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) { 2005 r = pool_chk_page(pp, label, ph); 2006 if (r) { 2007 goto out; 2008 } 2009 } 2010 LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { 2011 r = pool_chk_page(pp, label, ph); 2012 if (r) { 2013 goto out; 2014 } 2015 } 2016 LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { 2017 r = pool_chk_page(pp, label, ph); 2018 if (r) { 2019 goto out; 2020 } 2021 } 2022 2023 out: 2024 mutex_exit(&pp->pr_lock); 2025 return r; 2026 } 2027 2028 /* 2029 * pool_cache_init: 2030 * 2031 * Initialize a pool cache. 2032 */ 2033 pool_cache_t 2034 pool_cache_init(size_t size, u_int align, u_int align_offset, u_int flags, 2035 const char *wchan, struct pool_allocator *palloc, int ipl, 2036 int (*ctor)(void *, void *, int), void (*dtor)(void *, void *), void *arg) 2037 { 2038 pool_cache_t pc; 2039 2040 pc = pool_get(&cache_pool, PR_WAITOK); 2041 if (pc == NULL) 2042 return NULL; 2043 2044 pool_cache_bootstrap(pc, size, align, align_offset, flags, wchan, 2045 palloc, ipl, ctor, dtor, arg); 2046 2047 return pc; 2048 } 2049 2050 /* 2051 * pool_cache_bootstrap: 2052 * 2053 * Kernel-private version of pool_cache_init(). The caller 2054 * provides initial storage. 2055 */ 2056 void 2057 pool_cache_bootstrap(pool_cache_t pc, size_t size, u_int align, 2058 u_int align_offset, u_int flags, const char *wchan, 2059 struct pool_allocator *palloc, int ipl, 2060 int (*ctor)(void *, void *, int), void (*dtor)(void *, void *), 2061 void *arg) 2062 { 2063 CPU_INFO_ITERATOR cii; 2064 pool_cache_t pc1; 2065 struct cpu_info *ci; 2066 struct pool *pp; 2067 2068 pp = &pc->pc_pool; 2069 if (palloc == NULL && ipl == IPL_NONE) { 2070 if (size > PAGE_SIZE) { 2071 int bigidx = pool_bigidx(size); 2072 2073 palloc = &pool_allocator_big[bigidx]; 2074 flags |= PR_NOALIGN; 2075 } else 2076 palloc = &pool_allocator_nointr; 2077 } 2078 pool_init(pp, size, align, align_offset, flags, wchan, palloc, ipl); 2079 mutex_init(&pc->pc_lock, MUTEX_DEFAULT, ipl); 2080 2081 if (ctor == NULL) { 2082 ctor = NO_CTOR; 2083 } 2084 if (dtor == NULL) { 2085 dtor = NO_DTOR; 2086 } 2087 2088 pc->pc_emptygroups = NULL; 2089 pc->pc_fullgroups = NULL; 2090 pc->pc_partgroups = NULL; 2091 pc->pc_ctor = ctor; 2092 pc->pc_dtor = dtor; 2093 pc->pc_arg = arg; 2094 pc->pc_hits = 0; 2095 pc->pc_misses = 0; 2096 pc->pc_nempty = 0; 2097 pc->pc_npart = 0; 2098 pc->pc_nfull = 0; 2099 pc->pc_contended = 0; 2100 pc->pc_refcnt = 0; 2101 pc->pc_freecheck = NULL; 2102 2103 if ((flags & PR_LARGECACHE) != 0) { 2104 pc->pc_pcgsize = PCG_NOBJECTS_LARGE; 2105 pc->pc_pcgpool = &pcg_large_pool; 2106 } else { 2107 pc->pc_pcgsize = PCG_NOBJECTS_NORMAL; 2108 pc->pc_pcgpool = &pcg_normal_pool; 2109 } 2110 2111 /* Allocate per-CPU caches. */ 2112 memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); 2113 pc->pc_ncpu = 0; 2114 if (ncpu < 2) { 2115 /* XXX For sparc: boot CPU is not attached yet. */ 2116 pool_cache_cpu_init1(curcpu(), pc); 2117 } else { 2118 for (CPU_INFO_FOREACH(cii, ci)) { 2119 pool_cache_cpu_init1(ci, pc); 2120 } 2121 } 2122 2123 /* Add to list of all pools. */ 2124 if (__predict_true(!cold)) 2125 mutex_enter(&pool_head_lock); 2126 TAILQ_FOREACH(pc1, &pool_cache_head, pc_cachelist) { 2127 if (strcmp(pc1->pc_pool.pr_wchan, pc->pc_pool.pr_wchan) > 0) 2128 break; 2129 } 2130 if (pc1 == NULL) 2131 TAILQ_INSERT_TAIL(&pool_cache_head, pc, pc_cachelist); 2132 else 2133 TAILQ_INSERT_BEFORE(pc1, pc, pc_cachelist); 2134 if (__predict_true(!cold)) 2135 mutex_exit(&pool_head_lock); 2136 2137 membar_sync(); 2138 pp->pr_cache = pc; 2139 } 2140 2141 /* 2142 * pool_cache_destroy: 2143 * 2144 * Destroy a pool cache. 2145 */ 2146 void 2147 pool_cache_destroy(pool_cache_t pc) 2148 { 2149 2150 pool_cache_bootstrap_destroy(pc); 2151 pool_put(&cache_pool, pc); 2152 } 2153 2154 /* 2155 * pool_cache_bootstrap_destroy: 2156 * 2157 * Destroy a pool cache. 2158 */ 2159 void 2160 pool_cache_bootstrap_destroy(pool_cache_t pc) 2161 { 2162 struct pool *pp = &pc->pc_pool; 2163 u_int i; 2164 2165 /* Remove it from the global list. */ 2166 mutex_enter(&pool_head_lock); 2167 while (pc->pc_refcnt != 0) 2168 cv_wait(&pool_busy, &pool_head_lock); 2169 TAILQ_REMOVE(&pool_cache_head, pc, pc_cachelist); 2170 mutex_exit(&pool_head_lock); 2171 2172 /* First, invalidate the entire cache. */ 2173 pool_cache_invalidate(pc); 2174 2175 /* Disassociate it from the pool. */ 2176 mutex_enter(&pp->pr_lock); 2177 pp->pr_cache = NULL; 2178 mutex_exit(&pp->pr_lock); 2179 2180 /* Destroy per-CPU data */ 2181 for (i = 0; i < __arraycount(pc->pc_cpus); i++) 2182 pool_cache_invalidate_cpu(pc, i); 2183 2184 /* Finally, destroy it. */ 2185 mutex_destroy(&pc->pc_lock); 2186 pool_destroy(pp); 2187 } 2188 2189 /* 2190 * pool_cache_cpu_init1: 2191 * 2192 * Called for each pool_cache whenever a new CPU is attached. 2193 */ 2194 static void 2195 pool_cache_cpu_init1(struct cpu_info *ci, pool_cache_t pc) 2196 { 2197 pool_cache_cpu_t *cc; 2198 int index; 2199 2200 index = ci->ci_index; 2201 2202 KASSERT(index < __arraycount(pc->pc_cpus)); 2203 2204 if ((cc = pc->pc_cpus[index]) != NULL) { 2205 KASSERT(cc->cc_cpuindex == index); 2206 return; 2207 } 2208 2209 /* 2210 * The first CPU is 'free'. This needs to be the case for 2211 * bootstrap - we may not be able to allocate yet. 2212 */ 2213 if (pc->pc_ncpu == 0) { 2214 cc = &pc->pc_cpu0; 2215 pc->pc_ncpu = 1; 2216 } else { 2217 mutex_enter(&pc->pc_lock); 2218 pc->pc_ncpu++; 2219 mutex_exit(&pc->pc_lock); 2220 cc = pool_get(&cache_cpu_pool, PR_WAITOK); 2221 } 2222 2223 cc->cc_ipl = pc->pc_pool.pr_ipl; 2224 cc->cc_iplcookie = makeiplcookie(cc->cc_ipl); 2225 cc->cc_cache = pc; 2226 cc->cc_cpuindex = index; 2227 cc->cc_hits = 0; 2228 cc->cc_misses = 0; 2229 cc->cc_current = __UNCONST(&pcg_dummy); 2230 cc->cc_previous = __UNCONST(&pcg_dummy); 2231 2232 pc->pc_cpus[index] = cc; 2233 } 2234 2235 /* 2236 * pool_cache_cpu_init: 2237 * 2238 * Called whenever a new CPU is attached. 2239 */ 2240 void 2241 pool_cache_cpu_init(struct cpu_info *ci) 2242 { 2243 pool_cache_t pc; 2244 2245 mutex_enter(&pool_head_lock); 2246 TAILQ_FOREACH(pc, &pool_cache_head, pc_cachelist) { 2247 pc->pc_refcnt++; 2248 mutex_exit(&pool_head_lock); 2249 2250 pool_cache_cpu_init1(ci, pc); 2251 2252 mutex_enter(&pool_head_lock); 2253 pc->pc_refcnt--; 2254 cv_broadcast(&pool_busy); 2255 } 2256 mutex_exit(&pool_head_lock); 2257 } 2258 2259 /* 2260 * pool_cache_reclaim: 2261 * 2262 * Reclaim memory from a pool cache. 2263 */ 2264 bool 2265 pool_cache_reclaim(pool_cache_t pc) 2266 { 2267 2268 return pool_reclaim(&pc->pc_pool); 2269 } 2270 2271 static void 2272 pool_cache_destruct_object1(pool_cache_t pc, void *object) 2273 { 2274 (*pc->pc_dtor)(pc->pc_arg, object); 2275 pool_put(&pc->pc_pool, object); 2276 } 2277 2278 /* 2279 * pool_cache_destruct_object: 2280 * 2281 * Force destruction of an object and its release back into 2282 * the pool. 2283 */ 2284 void 2285 pool_cache_destruct_object(pool_cache_t pc, void *object) 2286 { 2287 2288 FREECHECK_IN(&pc->pc_freecheck, object); 2289 2290 pool_cache_destruct_object1(pc, object); 2291 } 2292 2293 /* 2294 * pool_cache_invalidate_groups: 2295 * 2296 * Invalidate a chain of groups and destruct all objects. 2297 */ 2298 static void 2299 pool_cache_invalidate_groups(pool_cache_t pc, pcg_t *pcg) 2300 { 2301 void *object; 2302 pcg_t *next; 2303 int i; 2304 2305 for (; pcg != NULL; pcg = next) { 2306 next = pcg->pcg_next; 2307 2308 for (i = 0; i < pcg->pcg_avail; i++) { 2309 object = pcg->pcg_objects[i].pcgo_va; 2310 pool_cache_destruct_object1(pc, object); 2311 } 2312 2313 if (pcg->pcg_size == PCG_NOBJECTS_LARGE) { 2314 pool_put(&pcg_large_pool, pcg); 2315 } else { 2316 KASSERT(pcg->pcg_size == PCG_NOBJECTS_NORMAL); 2317 pool_put(&pcg_normal_pool, pcg); 2318 } 2319 } 2320 } 2321 2322 /* 2323 * pool_cache_invalidate: 2324 * 2325 * Invalidate a pool cache (destruct and release all of the 2326 * cached objects). Does not reclaim objects from the pool. 2327 * 2328 * Note: For pool caches that provide constructed objects, there 2329 * is an assumption that another level of synchronization is occurring 2330 * between the input to the constructor and the cache invalidation. 2331 * 2332 * Invalidation is a costly process and should not be called from 2333 * interrupt context. 2334 */ 2335 void 2336 pool_cache_invalidate(pool_cache_t pc) 2337 { 2338 uint64_t where; 2339 pcg_t *full, *empty, *part; 2340 2341 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 2342 2343 if (ncpu < 2 || !mp_online) { 2344 /* 2345 * We might be called early enough in the boot process 2346 * for the CPU data structures to not be fully initialized. 2347 * In this case, transfer the content of the local CPU's 2348 * cache back into global cache as only this CPU is currently 2349 * running. 2350 */ 2351 pool_cache_transfer(pc); 2352 } else { 2353 /* 2354 * Signal all CPUs that they must transfer their local 2355 * cache back to the global pool then wait for the xcall to 2356 * complete. 2357 */ 2358 where = xc_broadcast(0, 2359 __FPTRCAST(xcfunc_t, pool_cache_transfer), pc, NULL); 2360 xc_wait(where); 2361 } 2362 2363 /* Empty pool caches, then invalidate objects */ 2364 mutex_enter(&pc->pc_lock); 2365 full = pc->pc_fullgroups; 2366 empty = pc->pc_emptygroups; 2367 part = pc->pc_partgroups; 2368 pc->pc_fullgroups = NULL; 2369 pc->pc_emptygroups = NULL; 2370 pc->pc_partgroups = NULL; 2371 pc->pc_nfull = 0; 2372 pc->pc_nempty = 0; 2373 pc->pc_npart = 0; 2374 mutex_exit(&pc->pc_lock); 2375 2376 pool_cache_invalidate_groups(pc, full); 2377 pool_cache_invalidate_groups(pc, empty); 2378 pool_cache_invalidate_groups(pc, part); 2379 } 2380 2381 /* 2382 * pool_cache_invalidate_cpu: 2383 * 2384 * Invalidate all CPU-bound cached objects in pool cache, the CPU being 2385 * identified by its associated index. 2386 * It is caller's responsibility to ensure that no operation is 2387 * taking place on this pool cache while doing this invalidation. 2388 * WARNING: as no inter-CPU locking is enforced, trying to invalidate 2389 * pool cached objects from a CPU different from the one currently running 2390 * may result in an undefined behaviour. 2391 */ 2392 static void 2393 pool_cache_invalidate_cpu(pool_cache_t pc, u_int index) 2394 { 2395 pool_cache_cpu_t *cc; 2396 pcg_t *pcg; 2397 2398 if ((cc = pc->pc_cpus[index]) == NULL) 2399 return; 2400 2401 if ((pcg = cc->cc_current) != &pcg_dummy) { 2402 pcg->pcg_next = NULL; 2403 pool_cache_invalidate_groups(pc, pcg); 2404 } 2405 if ((pcg = cc->cc_previous) != &pcg_dummy) { 2406 pcg->pcg_next = NULL; 2407 pool_cache_invalidate_groups(pc, pcg); 2408 } 2409 if (cc != &pc->pc_cpu0) 2410 pool_put(&cache_cpu_pool, cc); 2411 2412 } 2413 2414 void 2415 pool_cache_set_drain_hook(pool_cache_t pc, void (*fn)(void *, int), void *arg) 2416 { 2417 2418 pool_set_drain_hook(&pc->pc_pool, fn, arg); 2419 } 2420 2421 void 2422 pool_cache_setlowat(pool_cache_t pc, int n) 2423 { 2424 2425 pool_setlowat(&pc->pc_pool, n); 2426 } 2427 2428 void 2429 pool_cache_sethiwat(pool_cache_t pc, int n) 2430 { 2431 2432 pool_sethiwat(&pc->pc_pool, n); 2433 } 2434 2435 void 2436 pool_cache_sethardlimit(pool_cache_t pc, int n, const char *warnmess, int ratecap) 2437 { 2438 2439 pool_sethardlimit(&pc->pc_pool, n, warnmess, ratecap); 2440 } 2441 2442 static bool __noinline 2443 pool_cache_get_slow(pool_cache_cpu_t *cc, int s, void **objectp, 2444 paddr_t *pap, int flags) 2445 { 2446 pcg_t *pcg, *cur; 2447 uint64_t ncsw; 2448 pool_cache_t pc; 2449 void *object; 2450 2451 KASSERT(cc->cc_current->pcg_avail == 0); 2452 KASSERT(cc->cc_previous->pcg_avail == 0); 2453 2454 pc = cc->cc_cache; 2455 cc->cc_misses++; 2456 2457 /* 2458 * Nothing was available locally. Try and grab a group 2459 * from the cache. 2460 */ 2461 if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { 2462 ncsw = curlwp->l_ncsw; 2463 mutex_enter(&pc->pc_lock); 2464 pc->pc_contended++; 2465 2466 /* 2467 * If we context switched while locking, then 2468 * our view of the per-CPU data is invalid: 2469 * retry. 2470 */ 2471 if (curlwp->l_ncsw != ncsw) { 2472 mutex_exit(&pc->pc_lock); 2473 return true; 2474 } 2475 } 2476 2477 if (__predict_true((pcg = pc->pc_fullgroups) != NULL)) { 2478 /* 2479 * If there's a full group, release our empty 2480 * group back to the cache. Install the full 2481 * group as cc_current and return. 2482 */ 2483 if (__predict_true((cur = cc->cc_current) != &pcg_dummy)) { 2484 KASSERT(cur->pcg_avail == 0); 2485 cur->pcg_next = pc->pc_emptygroups; 2486 pc->pc_emptygroups = cur; 2487 pc->pc_nempty++; 2488 } 2489 KASSERT(pcg->pcg_avail == pcg->pcg_size); 2490 cc->cc_current = pcg; 2491 pc->pc_fullgroups = pcg->pcg_next; 2492 pc->pc_hits++; 2493 pc->pc_nfull--; 2494 mutex_exit(&pc->pc_lock); 2495 return true; 2496 } 2497 2498 /* 2499 * Nothing available locally or in cache. Take the slow 2500 * path: fetch a new object from the pool and construct 2501 * it. 2502 */ 2503 pc->pc_misses++; 2504 mutex_exit(&pc->pc_lock); 2505 splx(s); 2506 2507 object = pool_get(&pc->pc_pool, flags); 2508 *objectp = object; 2509 if (__predict_false(object == NULL)) { 2510 KASSERT((flags & (PR_WAITOK|PR_NOWAIT)) == PR_NOWAIT); 2511 return false; 2512 } 2513 2514 if (__predict_false((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0)) { 2515 pool_put(&pc->pc_pool, object); 2516 *objectp = NULL; 2517 return false; 2518 } 2519 2520 KASSERT((((vaddr_t)object) & (pc->pc_pool.pr_align - 1)) == 0); 2521 2522 if (pap != NULL) { 2523 #ifdef POOL_VTOPHYS 2524 *pap = POOL_VTOPHYS(object); 2525 #else 2526 *pap = POOL_PADDR_INVALID; 2527 #endif 2528 } 2529 2530 FREECHECK_OUT(&pc->pc_freecheck, object); 2531 pool_cache_kleak_fill(pc, object); 2532 return false; 2533 } 2534 2535 /* 2536 * pool_cache_get{,_paddr}: 2537 * 2538 * Get an object from a pool cache (optionally returning 2539 * the physical address of the object). 2540 */ 2541 void * 2542 pool_cache_get_paddr(pool_cache_t pc, int flags, paddr_t *pap) 2543 { 2544 pool_cache_cpu_t *cc; 2545 pcg_t *pcg; 2546 void *object; 2547 int s; 2548 2549 KASSERT(!(flags & PR_NOWAIT) != !(flags & PR_WAITOK)); 2550 KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()) || 2551 (pc->pc_pool.pr_ipl != IPL_NONE || cold || panicstr != NULL), 2552 "%s: [%s] is IPL_NONE, but called from interrupt context", 2553 __func__, pc->pc_pool.pr_wchan); 2554 2555 if (flags & PR_WAITOK) { 2556 ASSERT_SLEEPABLE(); 2557 } 2558 2559 /* Lock out interrupts and disable preemption. */ 2560 s = splvm(); 2561 while (/* CONSTCOND */ true) { 2562 /* Try and allocate an object from the current group. */ 2563 cc = pc->pc_cpus[curcpu()->ci_index]; 2564 KASSERT(cc->cc_cache == pc); 2565 pcg = cc->cc_current; 2566 if (__predict_true(pcg->pcg_avail > 0)) { 2567 object = pcg->pcg_objects[--pcg->pcg_avail].pcgo_va; 2568 if (__predict_false(pap != NULL)) 2569 *pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa; 2570 #if defined(DIAGNOSTIC) 2571 pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL; 2572 KASSERT(pcg->pcg_avail < pcg->pcg_size); 2573 KASSERT(object != NULL); 2574 #endif 2575 cc->cc_hits++; 2576 splx(s); 2577 FREECHECK_OUT(&pc->pc_freecheck, object); 2578 pool_redzone_fill(&pc->pc_pool, object); 2579 pool_cache_get_kmsan(pc, object); 2580 pool_cache_kleak_fill(pc, object); 2581 return object; 2582 } 2583 2584 /* 2585 * That failed. If the previous group isn't empty, swap 2586 * it with the current group and allocate from there. 2587 */ 2588 pcg = cc->cc_previous; 2589 if (__predict_true(pcg->pcg_avail > 0)) { 2590 cc->cc_previous = cc->cc_current; 2591 cc->cc_current = pcg; 2592 continue; 2593 } 2594 2595 /* 2596 * Can't allocate from either group: try the slow path. 2597 * If get_slow() allocated an object for us, or if 2598 * no more objects are available, it will return false. 2599 * Otherwise, we need to retry. 2600 */ 2601 if (!pool_cache_get_slow(cc, s, &object, pap, flags)) 2602 break; 2603 } 2604 2605 /* 2606 * We would like to KASSERT(object || (flags & PR_NOWAIT)), but 2607 * pool_cache_get can fail even in the PR_WAITOK case, if the 2608 * constructor fails. 2609 */ 2610 return object; 2611 } 2612 2613 static bool __noinline 2614 pool_cache_put_slow(pool_cache_cpu_t *cc, int s, void *object) 2615 { 2616 struct lwp *l = curlwp; 2617 pcg_t *pcg, *cur; 2618 uint64_t ncsw; 2619 pool_cache_t pc; 2620 2621 KASSERT(cc->cc_current->pcg_avail == cc->cc_current->pcg_size); 2622 KASSERT(cc->cc_previous->pcg_avail == cc->cc_previous->pcg_size); 2623 2624 pc = cc->cc_cache; 2625 pcg = NULL; 2626 cc->cc_misses++; 2627 ncsw = l->l_ncsw; 2628 2629 /* 2630 * If there are no empty groups in the cache then allocate one 2631 * while still unlocked. 2632 */ 2633 if (__predict_false(pc->pc_emptygroups == NULL)) { 2634 if (__predict_true(!pool_cache_disable)) { 2635 pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT); 2636 } 2637 /* 2638 * If pool_get() blocked, then our view of 2639 * the per-CPU data is invalid: retry. 2640 */ 2641 if (__predict_false(l->l_ncsw != ncsw)) { 2642 if (pcg != NULL) { 2643 pool_put(pc->pc_pcgpool, pcg); 2644 } 2645 return true; 2646 } 2647 if (__predict_true(pcg != NULL)) { 2648 pcg->pcg_avail = 0; 2649 pcg->pcg_size = pc->pc_pcgsize; 2650 } 2651 } 2652 2653 /* Lock the cache. */ 2654 if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { 2655 mutex_enter(&pc->pc_lock); 2656 pc->pc_contended++; 2657 2658 /* 2659 * If we context switched while locking, then our view of 2660 * the per-CPU data is invalid: retry. 2661 */ 2662 if (__predict_false(l->l_ncsw != ncsw)) { 2663 mutex_exit(&pc->pc_lock); 2664 if (pcg != NULL) { 2665 pool_put(pc->pc_pcgpool, pcg); 2666 } 2667 return true; 2668 } 2669 } 2670 2671 /* If there are no empty groups in the cache then allocate one. */ 2672 if (pcg == NULL && pc->pc_emptygroups != NULL) { 2673 pcg = pc->pc_emptygroups; 2674 pc->pc_emptygroups = pcg->pcg_next; 2675 pc->pc_nempty--; 2676 } 2677 2678 /* 2679 * If there's a empty group, release our full group back 2680 * to the cache. Install the empty group to the local CPU 2681 * and return. 2682 */ 2683 if (pcg != NULL) { 2684 KASSERT(pcg->pcg_avail == 0); 2685 if (__predict_false(cc->cc_previous == &pcg_dummy)) { 2686 cc->cc_previous = pcg; 2687 } else { 2688 cur = cc->cc_current; 2689 if (__predict_true(cur != &pcg_dummy)) { 2690 KASSERT(cur->pcg_avail == cur->pcg_size); 2691 cur->pcg_next = pc->pc_fullgroups; 2692 pc->pc_fullgroups = cur; 2693 pc->pc_nfull++; 2694 } 2695 cc->cc_current = pcg; 2696 } 2697 pc->pc_hits++; 2698 mutex_exit(&pc->pc_lock); 2699 return true; 2700 } 2701 2702 /* 2703 * Nothing available locally or in cache, and we didn't 2704 * allocate an empty group. Take the slow path and destroy 2705 * the object here and now. 2706 */ 2707 pc->pc_misses++; 2708 mutex_exit(&pc->pc_lock); 2709 splx(s); 2710 pool_cache_destruct_object(pc, object); 2711 2712 return false; 2713 } 2714 2715 /* 2716 * pool_cache_put{,_paddr}: 2717 * 2718 * Put an object back to the pool cache (optionally caching the 2719 * physical address of the object). 2720 */ 2721 void 2722 pool_cache_put_paddr(pool_cache_t pc, void *object, paddr_t pa) 2723 { 2724 pool_cache_cpu_t *cc; 2725 pcg_t *pcg; 2726 int s; 2727 2728 KASSERT(object != NULL); 2729 pool_cache_put_kmsan(pc, object); 2730 pool_cache_redzone_check(pc, object); 2731 FREECHECK_IN(&pc->pc_freecheck, object); 2732 2733 if (pc->pc_pool.pr_roflags & PR_PHINPAGE) { 2734 pc_phinpage_check(pc, object); 2735 } 2736 2737 if (pool_cache_put_quarantine(pc, object, pa)) { 2738 return; 2739 } 2740 2741 /* Lock out interrupts and disable preemption. */ 2742 s = splvm(); 2743 while (/* CONSTCOND */ true) { 2744 /* If the current group isn't full, release it there. */ 2745 cc = pc->pc_cpus[curcpu()->ci_index]; 2746 KASSERT(cc->cc_cache == pc); 2747 pcg = cc->cc_current; 2748 if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) { 2749 pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object; 2750 pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa; 2751 pcg->pcg_avail++; 2752 cc->cc_hits++; 2753 splx(s); 2754 return; 2755 } 2756 2757 /* 2758 * That failed. If the previous group isn't full, swap 2759 * it with the current group and try again. 2760 */ 2761 pcg = cc->cc_previous; 2762 if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) { 2763 cc->cc_previous = cc->cc_current; 2764 cc->cc_current = pcg; 2765 continue; 2766 } 2767 2768 /* 2769 * Can't free to either group: try the slow path. 2770 * If put_slow() releases the object for us, it 2771 * will return false. Otherwise we need to retry. 2772 */ 2773 if (!pool_cache_put_slow(cc, s, object)) 2774 break; 2775 } 2776 } 2777 2778 /* 2779 * pool_cache_transfer: 2780 * 2781 * Transfer objects from the per-CPU cache to the global cache. 2782 * Run within a cross-call thread. 2783 */ 2784 static void 2785 pool_cache_transfer(pool_cache_t pc) 2786 { 2787 pool_cache_cpu_t *cc; 2788 pcg_t *prev, *cur, **list; 2789 int s; 2790 2791 s = splvm(); 2792 mutex_enter(&pc->pc_lock); 2793 cc = pc->pc_cpus[curcpu()->ci_index]; 2794 cur = cc->cc_current; 2795 cc->cc_current = __UNCONST(&pcg_dummy); 2796 prev = cc->cc_previous; 2797 cc->cc_previous = __UNCONST(&pcg_dummy); 2798 if (cur != &pcg_dummy) { 2799 if (cur->pcg_avail == cur->pcg_size) { 2800 list = &pc->pc_fullgroups; 2801 pc->pc_nfull++; 2802 } else if (cur->pcg_avail == 0) { 2803 list = &pc->pc_emptygroups; 2804 pc->pc_nempty++; 2805 } else { 2806 list = &pc->pc_partgroups; 2807 pc->pc_npart++; 2808 } 2809 cur->pcg_next = *list; 2810 *list = cur; 2811 } 2812 if (prev != &pcg_dummy) { 2813 if (prev->pcg_avail == prev->pcg_size) { 2814 list = &pc->pc_fullgroups; 2815 pc->pc_nfull++; 2816 } else if (prev->pcg_avail == 0) { 2817 list = &pc->pc_emptygroups; 2818 pc->pc_nempty++; 2819 } else { 2820 list = &pc->pc_partgroups; 2821 pc->pc_npart++; 2822 } 2823 prev->pcg_next = *list; 2824 *list = prev; 2825 } 2826 mutex_exit(&pc->pc_lock); 2827 splx(s); 2828 } 2829 2830 static int 2831 pool_bigidx(size_t size) 2832 { 2833 int i; 2834 2835 for (i = 0; i < __arraycount(pool_allocator_big); i++) { 2836 if (1 << (i + POOL_ALLOCATOR_BIG_BASE) >= size) 2837 return i; 2838 } 2839 panic("pool item size %zu too large, use a custom allocator", size); 2840 } 2841 2842 static void * 2843 pool_allocator_alloc(struct pool *pp, int flags) 2844 { 2845 struct pool_allocator *pa = pp->pr_alloc; 2846 void *res; 2847 2848 res = (*pa->pa_alloc)(pp, flags); 2849 if (res == NULL && (flags & PR_WAITOK) == 0) { 2850 /* 2851 * We only run the drain hook here if PR_NOWAIT. 2852 * In other cases, the hook will be run in 2853 * pool_reclaim(). 2854 */ 2855 if (pp->pr_drain_hook != NULL) { 2856 (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags); 2857 res = (*pa->pa_alloc)(pp, flags); 2858 } 2859 } 2860 return res; 2861 } 2862 2863 static void 2864 pool_allocator_free(struct pool *pp, void *v) 2865 { 2866 struct pool_allocator *pa = pp->pr_alloc; 2867 2868 if (pp->pr_redzone) { 2869 kasan_mark(v, pa->pa_pagesz, pa->pa_pagesz, 0); 2870 } 2871 (*pa->pa_free)(pp, v); 2872 } 2873 2874 void * 2875 pool_page_alloc(struct pool *pp, int flags) 2876 { 2877 const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP; 2878 vmem_addr_t va; 2879 int ret; 2880 2881 ret = uvm_km_kmem_alloc(kmem_va_arena, pp->pr_alloc->pa_pagesz, 2882 vflags | VM_INSTANTFIT, &va); 2883 2884 return ret ? NULL : (void *)va; 2885 } 2886 2887 void 2888 pool_page_free(struct pool *pp, void *v) 2889 { 2890 2891 uvm_km_kmem_free(kmem_va_arena, (vaddr_t)v, pp->pr_alloc->pa_pagesz); 2892 } 2893 2894 static void * 2895 pool_page_alloc_meta(struct pool *pp, int flags) 2896 { 2897 const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP; 2898 vmem_addr_t va; 2899 int ret; 2900 2901 ret = vmem_alloc(kmem_meta_arena, pp->pr_alloc->pa_pagesz, 2902 vflags | VM_INSTANTFIT, &va); 2903 2904 return ret ? NULL : (void *)va; 2905 } 2906 2907 static void 2908 pool_page_free_meta(struct pool *pp, void *v) 2909 { 2910 2911 vmem_free(kmem_meta_arena, (vmem_addr_t)v, pp->pr_alloc->pa_pagesz); 2912 } 2913 2914 #ifdef KMSAN 2915 static inline void 2916 pool_get_kmsan(struct pool *pp, void *p) 2917 { 2918 kmsan_orig(p, pp->pr_size, KMSAN_TYPE_POOL, __RET_ADDR); 2919 kmsan_mark(p, pp->pr_size, KMSAN_STATE_UNINIT); 2920 } 2921 2922 static inline void 2923 pool_put_kmsan(struct pool *pp, void *p) 2924 { 2925 kmsan_mark(p, pp->pr_size, KMSAN_STATE_INITED); 2926 } 2927 2928 static inline void 2929 pool_cache_get_kmsan(pool_cache_t pc, void *p) 2930 { 2931 if (__predict_false(pc_has_ctor(pc))) { 2932 return; 2933 } 2934 pool_get_kmsan(&pc->pc_pool, p); 2935 } 2936 2937 static inline void 2938 pool_cache_put_kmsan(pool_cache_t pc, void *p) 2939 { 2940 pool_put_kmsan(&pc->pc_pool, p); 2941 } 2942 #endif 2943 2944 #ifdef KLEAK 2945 static void 2946 pool_kleak_fill(struct pool *pp, void *p) 2947 { 2948 if (__predict_false(pp->pr_roflags & PR_NOTOUCH)) { 2949 return; 2950 } 2951 kleak_fill_area(p, pp->pr_size); 2952 } 2953 2954 static void 2955 pool_cache_kleak_fill(pool_cache_t pc, void *p) 2956 { 2957 if (__predict_false(pc_has_ctor(pc) || pc_has_dtor(pc))) { 2958 return; 2959 } 2960 pool_kleak_fill(&pc->pc_pool, p); 2961 } 2962 #endif 2963 2964 #ifdef POOL_QUARANTINE 2965 static void 2966 pool_quarantine_init(struct pool *pp) 2967 { 2968 pp->pr_quar.rotor = 0; 2969 memset(&pp->pr_quar, 0, sizeof(pp->pr_quar)); 2970 } 2971 2972 static void 2973 pool_quarantine_flush(struct pool *pp) 2974 { 2975 pool_quar_t *quar = &pp->pr_quar; 2976 struct pool_pagelist pq; 2977 size_t i; 2978 2979 LIST_INIT(&pq); 2980 2981 mutex_enter(&pp->pr_lock); 2982 for (i = 0; i < POOL_QUARANTINE_DEPTH; i++) { 2983 if (quar->list[i] == 0) 2984 continue; 2985 pool_do_put(pp, (void *)quar->list[i], &pq); 2986 } 2987 mutex_exit(&pp->pr_lock); 2988 2989 pr_pagelist_free(pp, &pq); 2990 } 2991 2992 static bool 2993 pool_put_quarantine(struct pool *pp, void *v, struct pool_pagelist *pq) 2994 { 2995 pool_quar_t *quar = &pp->pr_quar; 2996 uintptr_t old; 2997 2998 if (pp->pr_roflags & PR_NOTOUCH) { 2999 return false; 3000 } 3001 3002 pool_redzone_check(pp, v); 3003 3004 old = quar->list[quar->rotor]; 3005 quar->list[quar->rotor] = (uintptr_t)v; 3006 quar->rotor = (quar->rotor + 1) % POOL_QUARANTINE_DEPTH; 3007 if (old != 0) { 3008 pool_do_put(pp, (void *)old, pq); 3009 } 3010 3011 return true; 3012 } 3013 3014 static bool 3015 pool_cache_put_quarantine(pool_cache_t pc, void *p, paddr_t pa) 3016 { 3017 pool_cache_destruct_object(pc, p); 3018 return true; 3019 } 3020 #endif 3021 3022 #ifdef POOL_REDZONE 3023 #if defined(_LP64) 3024 # define PRIME 0x9e37fffffffc0000UL 3025 #else /* defined(_LP64) */ 3026 # define PRIME 0x9e3779b1 3027 #endif /* defined(_LP64) */ 3028 #define STATIC_BYTE 0xFE 3029 CTASSERT(POOL_REDZONE_SIZE > 1); 3030 3031 #ifndef KASAN 3032 static inline uint8_t 3033 pool_pattern_generate(const void *p) 3034 { 3035 return (uint8_t)(((uintptr_t)p) * PRIME 3036 >> ((sizeof(uintptr_t) - sizeof(uint8_t))) * CHAR_BIT); 3037 } 3038 #endif 3039 3040 static void 3041 pool_redzone_init(struct pool *pp, size_t requested_size) 3042 { 3043 size_t redzsz; 3044 size_t nsz; 3045 3046 #ifdef KASAN 3047 redzsz = requested_size; 3048 kasan_add_redzone(&redzsz); 3049 redzsz -= requested_size; 3050 #else 3051 redzsz = POOL_REDZONE_SIZE; 3052 #endif 3053 3054 if (pp->pr_roflags & PR_NOTOUCH) { 3055 pp->pr_redzone = false; 3056 return; 3057 } 3058 3059 /* 3060 * We may have extended the requested size earlier; check if 3061 * there's naturally space in the padding for a red zone. 3062 */ 3063 if (pp->pr_size - requested_size >= redzsz) { 3064 pp->pr_reqsize_with_redzone = requested_size + redzsz; 3065 pp->pr_redzone = true; 3066 return; 3067 } 3068 3069 /* 3070 * No space in the natural padding; check if we can extend a 3071 * bit the size of the pool. 3072 */ 3073 nsz = roundup(pp->pr_size + redzsz, pp->pr_align); 3074 if (nsz <= pp->pr_alloc->pa_pagesz) { 3075 /* Ok, we can */ 3076 pp->pr_size = nsz; 3077 pp->pr_reqsize_with_redzone = requested_size + redzsz; 3078 pp->pr_redzone = true; 3079 } else { 3080 /* No space for a red zone... snif :'( */ 3081 pp->pr_redzone = false; 3082 printf("pool redzone disabled for '%s'\n", pp->pr_wchan); 3083 } 3084 } 3085 3086 static void 3087 pool_redzone_fill(struct pool *pp, void *p) 3088 { 3089 if (!pp->pr_redzone) 3090 return; 3091 #ifdef KASAN 3092 kasan_mark(p, pp->pr_reqsize, pp->pr_reqsize_with_redzone, 3093 KASAN_POOL_REDZONE); 3094 #else 3095 uint8_t *cp, pat; 3096 const uint8_t *ep; 3097 3098 cp = (uint8_t *)p + pp->pr_reqsize; 3099 ep = cp + POOL_REDZONE_SIZE; 3100 3101 /* 3102 * We really don't want the first byte of the red zone to be '\0'; 3103 * an off-by-one in a string may not be properly detected. 3104 */ 3105 pat = pool_pattern_generate(cp); 3106 *cp = (pat == '\0') ? STATIC_BYTE: pat; 3107 cp++; 3108 3109 while (cp < ep) { 3110 *cp = pool_pattern_generate(cp); 3111 cp++; 3112 } 3113 #endif 3114 } 3115 3116 static void 3117 pool_redzone_check(struct pool *pp, void *p) 3118 { 3119 if (!pp->pr_redzone) 3120 return; 3121 #ifdef KASAN 3122 kasan_mark(p, 0, pp->pr_reqsize_with_redzone, KASAN_POOL_FREED); 3123 #else 3124 uint8_t *cp, pat, expected; 3125 const uint8_t *ep; 3126 3127 cp = (uint8_t *)p + pp->pr_reqsize; 3128 ep = cp + POOL_REDZONE_SIZE; 3129 3130 pat = pool_pattern_generate(cp); 3131 expected = (pat == '\0') ? STATIC_BYTE: pat; 3132 if (__predict_false(expected != *cp)) { 3133 printf("%s: %p: 0x%02x != 0x%02x\n", 3134 __func__, cp, *cp, expected); 3135 } 3136 cp++; 3137 3138 while (cp < ep) { 3139 expected = pool_pattern_generate(cp); 3140 if (__predict_false(*cp != expected)) { 3141 printf("%s: %p: 0x%02x != 0x%02x\n", 3142 __func__, cp, *cp, expected); 3143 } 3144 cp++; 3145 } 3146 #endif 3147 } 3148 3149 static void 3150 pool_cache_redzone_check(pool_cache_t pc, void *p) 3151 { 3152 #ifdef KASAN 3153 /* If there is a ctor/dtor, leave the data as valid. */ 3154 if (__predict_false(pc_has_ctor(pc) || pc_has_dtor(pc))) { 3155 return; 3156 } 3157 #endif 3158 pool_redzone_check(&pc->pc_pool, p); 3159 } 3160 3161 #endif /* POOL_REDZONE */ 3162 3163 #if defined(DDB) 3164 static bool 3165 pool_in_page(struct pool *pp, struct pool_item_header *ph, uintptr_t addr) 3166 { 3167 3168 return (uintptr_t)ph->ph_page <= addr && 3169 addr < (uintptr_t)ph->ph_page + pp->pr_alloc->pa_pagesz; 3170 } 3171 3172 static bool 3173 pool_in_item(struct pool *pp, void *item, uintptr_t addr) 3174 { 3175 3176 return (uintptr_t)item <= addr && addr < (uintptr_t)item + pp->pr_size; 3177 } 3178 3179 static bool 3180 pool_in_cg(struct pool *pp, struct pool_cache_group *pcg, uintptr_t addr) 3181 { 3182 int i; 3183 3184 if (pcg == NULL) { 3185 return false; 3186 } 3187 for (i = 0; i < pcg->pcg_avail; i++) { 3188 if (pool_in_item(pp, pcg->pcg_objects[i].pcgo_va, addr)) { 3189 return true; 3190 } 3191 } 3192 return false; 3193 } 3194 3195 static bool 3196 pool_allocated(struct pool *pp, struct pool_item_header *ph, uintptr_t addr) 3197 { 3198 3199 if ((pp->pr_roflags & PR_USEBMAP) != 0) { 3200 unsigned int idx = pr_item_bitmap_index(pp, ph, (void *)addr); 3201 pool_item_bitmap_t *bitmap = 3202 ph->ph_bitmap + (idx / BITMAP_SIZE); 3203 pool_item_bitmap_t mask = 1 << (idx & BITMAP_MASK); 3204 3205 return (*bitmap & mask) == 0; 3206 } else { 3207 struct pool_item *pi; 3208 3209 LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) { 3210 if (pool_in_item(pp, pi, addr)) { 3211 return false; 3212 } 3213 } 3214 return true; 3215 } 3216 } 3217 3218 void 3219 pool_whatis(uintptr_t addr, void (*pr)(const char *, ...)) 3220 { 3221 struct pool *pp; 3222 3223 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 3224 struct pool_item_header *ph; 3225 uintptr_t item; 3226 bool allocated = true; 3227 bool incache = false; 3228 bool incpucache = false; 3229 char cpucachestr[32]; 3230 3231 if ((pp->pr_roflags & PR_PHINPAGE) != 0) { 3232 LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { 3233 if (pool_in_page(pp, ph, addr)) { 3234 goto found; 3235 } 3236 } 3237 LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { 3238 if (pool_in_page(pp, ph, addr)) { 3239 allocated = 3240 pool_allocated(pp, ph, addr); 3241 goto found; 3242 } 3243 } 3244 LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) { 3245 if (pool_in_page(pp, ph, addr)) { 3246 allocated = false; 3247 goto found; 3248 } 3249 } 3250 continue; 3251 } else { 3252 ph = pr_find_pagehead_noalign(pp, (void *)addr); 3253 if (ph == NULL || !pool_in_page(pp, ph, addr)) { 3254 continue; 3255 } 3256 allocated = pool_allocated(pp, ph, addr); 3257 } 3258 found: 3259 if (allocated && pp->pr_cache) { 3260 pool_cache_t pc = pp->pr_cache; 3261 struct pool_cache_group *pcg; 3262 int i; 3263 3264 for (pcg = pc->pc_fullgroups; pcg != NULL; 3265 pcg = pcg->pcg_next) { 3266 if (pool_in_cg(pp, pcg, addr)) { 3267 incache = true; 3268 goto print; 3269 } 3270 } 3271 for (i = 0; i < __arraycount(pc->pc_cpus); i++) { 3272 pool_cache_cpu_t *cc; 3273 3274 if ((cc = pc->pc_cpus[i]) == NULL) { 3275 continue; 3276 } 3277 if (pool_in_cg(pp, cc->cc_current, addr) || 3278 pool_in_cg(pp, cc->cc_previous, addr)) { 3279 struct cpu_info *ci = 3280 cpu_lookup(i); 3281 3282 incpucache = true; 3283 snprintf(cpucachestr, 3284 sizeof(cpucachestr), 3285 "cached by CPU %u", 3286 ci->ci_index); 3287 goto print; 3288 } 3289 } 3290 } 3291 print: 3292 item = (uintptr_t)ph->ph_page + ph->ph_off; 3293 item = item + rounddown(addr - item, pp->pr_size); 3294 (*pr)("%p is %p+%zu in POOL '%s' (%s)\n", 3295 (void *)addr, item, (size_t)(addr - item), 3296 pp->pr_wchan, 3297 incpucache ? cpucachestr : 3298 incache ? "cached" : allocated ? "allocated" : "free"); 3299 } 3300 } 3301 #endif /* defined(DDB) */ 3302 3303 static int 3304 pool_sysctl(SYSCTLFN_ARGS) 3305 { 3306 struct pool_sysctl data; 3307 struct pool *pp; 3308 struct pool_cache *pc; 3309 pool_cache_cpu_t *cc; 3310 int error; 3311 size_t i, written; 3312 3313 if (oldp == NULL) { 3314 *oldlenp = 0; 3315 TAILQ_FOREACH(pp, &pool_head, pr_poollist) 3316 *oldlenp += sizeof(data); 3317 return 0; 3318 } 3319 3320 memset(&data, 0, sizeof(data)); 3321 error = 0; 3322 written = 0; 3323 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 3324 if (written + sizeof(data) > *oldlenp) 3325 break; 3326 strlcpy(data.pr_wchan, pp->pr_wchan, sizeof(data.pr_wchan)); 3327 data.pr_pagesize = pp->pr_alloc->pa_pagesz; 3328 data.pr_flags = pp->pr_roflags | pp->pr_flags; 3329 #define COPY(field) data.field = pp->field 3330 COPY(pr_size); 3331 3332 COPY(pr_itemsperpage); 3333 COPY(pr_nitems); 3334 COPY(pr_nout); 3335 COPY(pr_hardlimit); 3336 COPY(pr_npages); 3337 COPY(pr_minpages); 3338 COPY(pr_maxpages); 3339 3340 COPY(pr_nget); 3341 COPY(pr_nfail); 3342 COPY(pr_nput); 3343 COPY(pr_npagealloc); 3344 COPY(pr_npagefree); 3345 COPY(pr_hiwat); 3346 COPY(pr_nidle); 3347 #undef COPY 3348 3349 data.pr_cache_nmiss_pcpu = 0; 3350 data.pr_cache_nhit_pcpu = 0; 3351 if (pp->pr_cache) { 3352 pc = pp->pr_cache; 3353 data.pr_cache_meta_size = pc->pc_pcgsize; 3354 data.pr_cache_nfull = pc->pc_nfull; 3355 data.pr_cache_npartial = pc->pc_npart; 3356 data.pr_cache_nempty = pc->pc_nempty; 3357 data.pr_cache_ncontended = pc->pc_contended; 3358 data.pr_cache_nmiss_global = pc->pc_misses; 3359 data.pr_cache_nhit_global = pc->pc_hits; 3360 for (i = 0; i < pc->pc_ncpu; ++i) { 3361 cc = pc->pc_cpus[i]; 3362 if (cc == NULL) 3363 continue; 3364 data.pr_cache_nmiss_pcpu += cc->cc_misses; 3365 data.pr_cache_nhit_pcpu += cc->cc_hits; 3366 } 3367 } else { 3368 data.pr_cache_meta_size = 0; 3369 data.pr_cache_nfull = 0; 3370 data.pr_cache_npartial = 0; 3371 data.pr_cache_nempty = 0; 3372 data.pr_cache_ncontended = 0; 3373 data.pr_cache_nmiss_global = 0; 3374 data.pr_cache_nhit_global = 0; 3375 } 3376 3377 error = sysctl_copyout(l, &data, oldp, sizeof(data)); 3378 if (error) 3379 break; 3380 written += sizeof(data); 3381 oldp = (char *)oldp + sizeof(data); 3382 } 3383 3384 *oldlenp = written; 3385 return error; 3386 } 3387 3388 SYSCTL_SETUP(sysctl_pool_setup, "sysctl kern.pool setup") 3389 { 3390 const struct sysctlnode *rnode = NULL; 3391 3392 sysctl_createv(clog, 0, NULL, &rnode, 3393 CTLFLAG_PERMANENT, 3394 CTLTYPE_STRUCT, "pool", 3395 SYSCTL_DESCR("Get pool statistics"), 3396 pool_sysctl, 0, NULL, 0, 3397 CTL_KERN, CTL_CREATE, CTL_EOL); 3398 } 3399