1 /* $OpenBSD: subr_pool.c,v 1.13 2001/09/19 20:50:58 mickey Exp $ */ 2 /* $NetBSD: subr_pool.c,v 1.59 2001/06/05 18:51:04 thorpej Exp $ */ 3 4 /*- 5 * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace 10 * Simulation Facility, NASA Ames Research Center. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the NetBSD 23 * Foundation, Inc. and its contributors. 24 * 4. Neither the name of The NetBSD Foundation nor the names of its 25 * contributors may be used to endorse or promote products derived 26 * from this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 29 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 30 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 31 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 32 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 33 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 34 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 35 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 36 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 37 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 38 * POSSIBILITY OF SUCH DAMAGE. 39 */ 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/proc.h> 44 #include <sys/errno.h> 45 #include <sys/kernel.h> 46 #include <sys/malloc.h> 47 #include <sys/lock.h> 48 #include <sys/pool.h> 49 #include <sys/syslog.h> 50 #include <sys/sysctl.h> 51 52 #include <vm/vm.h> 53 #include <uvm/uvm.h> 54 55 /* 56 * XXX - for now. 57 */ 58 #define SIMPLELOCK_INITIALIZER { SLOCK_UNLOCKED } 59 #ifdef LOCKDEBUG 60 #define simple_lock_freecheck(a, s) do { /* nothing */ } while (0) 61 #define simple_lock_only_held(lkp, str) do { /* nothing */ } while (0) 62 #endif 63 #define LOCK_ASSERT(x) /* nothing */ 64 65 /* 66 * Pool resource management utility. 67 * 68 * Memory is allocated in pages which are split into pieces according 69 * to the pool item size. Each page is kept on a list headed by `pr_pagelist' 70 * in the pool structure and the individual pool items are on a linked list 71 * headed by `ph_itemlist' in each page header. The memory for building 72 * the page list is either taken from the allocated pages themselves (for 73 * small pool items) or taken from an internal pool of page headers (`phpool'). 74 */ 75 76 /* List of all pools */ 77 TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); 78 79 /* Private pool for page header structures */ 80 static struct pool phpool; 81 82 /* # of seconds to retain page after last use */ 83 int pool_inactive_time = 10; 84 85 /* Next candidate for drainage (see pool_drain()) */ 86 static struct pool *drainpp; 87 88 /* This spin lock protects both pool_head and drainpp. */ 89 struct simplelock pool_head_slock = SIMPLELOCK_INITIALIZER; 90 91 struct pool_item_header { 92 /* Page headers */ 93 TAILQ_ENTRY(pool_item_header) 94 ph_pagelist; /* pool page list */ 95 TAILQ_HEAD(,pool_item) ph_itemlist; /* chunk list for this page */ 96 LIST_ENTRY(pool_item_header) 97 ph_hashlist; /* Off-page page headers */ 98 int ph_nmissing; /* # of chunks in use */ 99 caddr_t ph_page; /* this page's address */ 100 struct timeval ph_time; /* last referenced */ 101 }; 102 103 struct pool_item { 104 #ifdef DIAGNOSTIC 105 int pi_magic; 106 #endif 107 #define PI_MAGIC 0xdeadbeef 108 /* Other entries use only this list entry */ 109 TAILQ_ENTRY(pool_item) pi_list; 110 }; 111 112 113 #define PR_HASH_INDEX(pp,addr) \ 114 (((u_long)(addr) >> (pp)->pr_pageshift) & (PR_HASHTABSIZE - 1)) 115 116 #define POOL_NEEDS_CATCHUP(pp) \ 117 ((pp)->pr_nitems < (pp)->pr_minitems) 118 119 /* 120 * Every pool get a unique serial number assigned to it. If this counter 121 * wraps, we're screwed, but we shouldn't create so many pools anyway. 122 */ 123 unsigned int pool_serial; 124 125 /* 126 * Pool cache management. 127 * 128 * Pool caches provide a way for constructed objects to be cached by the 129 * pool subsystem. This can lead to performance improvements by avoiding 130 * needless object construction/destruction; it is deferred until absolutely 131 * necessary. 132 * 133 * Caches are grouped into cache groups. Each cache group references 134 * up to 16 constructed objects. When a cache allocates an object 135 * from the pool, it calls the object's constructor and places it into 136 * a cache group. When a cache group frees an object back to the pool, 137 * it first calls the object's destructor. This allows the object to 138 * persist in constructed form while freed to the cache. 139 * 140 * Multiple caches may exist for each pool. This allows a single 141 * object type to have multiple constructed forms. The pool references 142 * each cache, so that when a pool is drained by the pagedaemon, it can 143 * drain each individual cache as well. Each time a cache is drained, 144 * the most idle cache group is freed to the pool in its entirety. 145 * 146 * Pool caches are layed on top of pools. By layering them, we can avoid 147 * the complexity of cache management for pools which would not benefit 148 * from it. 149 */ 150 151 /* The cache group pool. */ 152 static struct pool pcgpool; 153 154 /* The pool cache group. */ 155 #define PCG_NOBJECTS 16 156 struct pool_cache_group { 157 TAILQ_ENTRY(pool_cache_group) 158 pcg_list; /* link in the pool cache's group list */ 159 u_int pcg_avail; /* # available objects */ 160 /* pointers to the objects */ 161 void *pcg_objects[PCG_NOBJECTS]; 162 }; 163 164 static void pool_cache_reclaim(struct pool_cache *); 165 166 static int pool_catchup(struct pool *); 167 static void pool_prime_page(struct pool *, caddr_t, 168 struct pool_item_header *); 169 static void *pool_page_alloc(unsigned long, int, int); 170 static void pool_page_free(void *, unsigned long, int); 171 172 static void pool_print1(struct pool *, const char *, 173 int (*)(const char *, ...)); 174 175 /* 176 * Pool log entry. An array of these is allocated in pool_init(). 177 */ 178 struct pool_log { 179 const char *pl_file; 180 long pl_line; 181 int pl_action; 182 #define PRLOG_GET 1 183 #define PRLOG_PUT 2 184 void *pl_addr; 185 }; 186 187 /* Number of entries in pool log buffers */ 188 #ifndef POOL_LOGSIZE 189 #define POOL_LOGSIZE 10 190 #endif 191 192 int pool_logsize = POOL_LOGSIZE; 193 194 #ifdef POOL_DIAGNOSTIC 195 static __inline void 196 pr_log(struct pool *pp, void *a, int action, const char *file, long line) 197 { 198 int n = pp->pr_curlogentry; 199 struct pool_log *pl; 200 201 if ((pp->pr_roflags & PR_LOGGING) == 0) 202 return; 203 204 /* 205 * Fill in the current entry. Wrap around and overwrite 206 * the oldest entry if necessary. 207 */ 208 pl = &pp->pr_log[n]; 209 pl->pl_file = file; 210 pl->pl_line = line; 211 pl->pl_action = action; 212 pl->pl_addr = v; 213 if (++n >= pp->pr_logsize) 214 n = 0; 215 pp->pr_curlogentry = n; 216 } 217 218 static void 219 pr_printlog(struct pool *pp, struct pool_item *pi, 220 int (*pr)(const char *, ...)) 221 { 222 int i = pp->pr_logsize; 223 int n = pp->pr_curlogentry; 224 225 if ((pp->pr_roflags & PR_LOGGING) == 0) 226 return; 227 228 /* 229 * Print all entries in this pool's log. 230 */ 231 while (i-- > 0) { 232 struct pool_log *pl = &pp->pr_log[n]; 233 if (pl->pl_action != 0) { 234 if (pi == NULL || pi == pl->pl_addr) { 235 (*pr)("\tlog entry %d:\n", i); 236 (*pr)("\t\taction = %s, addr = %p\n", 237 pl->pl_action == PRLOG_GET ? "get" : "put", 238 pl->pl_addr); 239 (*pr)("\t\tfile: %s at line %lu\n", 240 pl->pl_file, pl->pl_line); 241 } 242 } 243 if (++n >= pp->pr_logsize) 244 n = 0; 245 } 246 } 247 248 static __inline void 249 pr_enter(struct pool *pp, const char *file, long line) 250 { 251 252 if (__predict_false(pp->pr_entered_file != NULL)) { 253 printf("pool %s: reentrancy at file %s line %ld\n", 254 pp->pr_wchan, file, line); 255 printf(" previous entry at file %s line %ld\n", 256 pp->pr_entered_file, pp->pr_entered_line); 257 panic("pr_enter"); 258 } 259 260 pp->pr_entered_file = file; 261 pp->pr_entered_line = line; 262 } 263 264 static __inline void 265 pr_leave(struct pool *pp) 266 { 267 268 if (__predict_false(pp->pr_entered_file == NULL)) { 269 printf("pool %s not entered?\n", pp->pr_wchan); 270 panic("pr_leave"); 271 } 272 273 pp->pr_entered_file = NULL; 274 pp->pr_entered_line = 0; 275 } 276 277 static __inline__ void 278 pr_enter_check(struct pool *pp, int (*pr)(const char *, ...)) 279 { 280 281 if (pp->pr_entered_file != NULL) 282 (*pr)("\n\tcurrently entered from file %s line %ld\n", 283 pp->pr_entered_file, pp->pr_entered_line); 284 } 285 #else 286 #define pr_log(pp, v, action, file, line) 287 #define pr_printlog(pp, pi, pr) 288 #define pr_enter(pp, file, line) 289 #define pr_leave(pp) 290 #define pr_enter_check(pp, pr) 291 #endif /* POOL_DIAGNOSTIC */ 292 293 /* 294 * Return the pool page header based on page address. 295 */ 296 static __inline struct pool_item_header * 297 pr_find_pagehead(struct pool *pp, caddr_t page) 298 { 299 struct pool_item_header *ph; 300 301 if ((pp->pr_roflags & PR_PHINPAGE) != 0) 302 return ((struct pool_item_header *)(page + pp->pr_phoffset)); 303 304 for (ph = LIST_FIRST(&pp->pr_hashtab[PR_HASH_INDEX(pp, page)]); 305 ph != NULL; 306 ph = LIST_NEXT(ph, ph_hashlist)) { 307 if (ph->ph_page == page) 308 return (ph); 309 } 310 return (NULL); 311 } 312 313 /* 314 * Remove a page from the pool. 315 */ 316 static __inline void 317 pr_rmpage(struct pool *pp, struct pool_item_header *ph) 318 { 319 320 /* 321 * If the page was idle, decrement the idle page count. 322 */ 323 if (ph->ph_nmissing == 0) { 324 #ifdef DIAGNOSTIC 325 if (pp->pr_nidle == 0) 326 panic("pr_rmpage: nidle inconsistent"); 327 if (pp->pr_nitems < pp->pr_itemsperpage) 328 panic("pr_rmpage: nitems inconsistent"); 329 #endif 330 pp->pr_nidle--; 331 } 332 333 pp->pr_nitems -= pp->pr_itemsperpage; 334 335 /* 336 * Unlink a page from the pool and release it. 337 */ 338 TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist); 339 (*pp->pr_free)(ph->ph_page, pp->pr_pagesz, pp->pr_mtype); 340 pp->pr_npages--; 341 pp->pr_npagefree++; 342 343 if ((pp->pr_roflags & PR_PHINPAGE) == 0) { 344 int s; 345 LIST_REMOVE(ph, ph_hashlist); 346 s = splhigh(); 347 pool_put(&phpool, ph); 348 splx(s); 349 } 350 351 if (pp->pr_curpage == ph) { 352 /* 353 * Find a new non-empty page header, if any. 354 * Start search from the page head, to increase the 355 * chance for "high water" pages to be freed. 356 */ 357 for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL; 358 ph = TAILQ_NEXT(ph, ph_pagelist)) 359 if (TAILQ_FIRST(&ph->ph_itemlist) != NULL) 360 break; 361 362 pp->pr_curpage = ph; 363 } 364 } 365 366 /* 367 * Initialize the given pool resource structure. 368 * 369 * We export this routine to allow other kernel parts to declare 370 * static pools that must be initialized before malloc() is available. 371 */ 372 void 373 pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, 374 const char *wchan, size_t pagesz, 375 void *(*alloc)(unsigned long, int, int), 376 void (*release)(void *, unsigned long, int), 377 int mtype) 378 { 379 int off, slack, i; 380 381 #ifdef POOL_DIAGNOSTIC 382 /* 383 * Always log if POOL_DIAGNOSTIC is defined. 384 */ 385 if (pool_logsize != 0) 386 flags |= PR_LOGGING; 387 #endif 388 389 /* 390 * Check arguments and construct default values. 391 */ 392 if (!powerof2(pagesz)) 393 panic("pool_init: page size invalid (%lx)\n", (u_long)pagesz); 394 395 if (alloc == NULL && release == NULL) { 396 alloc = pool_page_alloc; 397 release = pool_page_free; 398 pagesz = PAGE_SIZE; /* Rounds to PAGE_SIZE anyhow. */ 399 } else if ((alloc != NULL && release != NULL) == 0) { 400 /* If you specifiy one, must specify both. */ 401 panic("pool_init: must specify alloc and release together"); 402 } 403 404 if (pagesz == 0) 405 pagesz = PAGE_SIZE; 406 407 if (align == 0) 408 align = ALIGN(1); 409 410 if (size < sizeof(struct pool_item)) 411 size = sizeof(struct pool_item); 412 413 size = ALIGN(size); 414 if (size > pagesz) 415 panic("pool_init: pool item size (%lu) too large", 416 (u_long)size); 417 418 /* 419 * Initialize the pool structure. 420 */ 421 TAILQ_INIT(&pp->pr_pagelist); 422 TAILQ_INIT(&pp->pr_cachelist); 423 pp->pr_curpage = NULL; 424 pp->pr_npages = 0; 425 pp->pr_minitems = 0; 426 pp->pr_minpages = 0; 427 pp->pr_maxpages = UINT_MAX; 428 pp->pr_roflags = flags; 429 pp->pr_flags = 0; 430 pp->pr_size = size; 431 pp->pr_align = align; 432 pp->pr_wchan = wchan; 433 pp->pr_mtype = mtype; 434 pp->pr_alloc = alloc; 435 pp->pr_free = release; 436 pp->pr_pagesz = pagesz; 437 pp->pr_pagemask = ~(pagesz - 1); 438 pp->pr_pageshift = ffs(pagesz) - 1; 439 pp->pr_nitems = 0; 440 pp->pr_nout = 0; 441 pp->pr_hardlimit = UINT_MAX; 442 pp->pr_hardlimit_warning = NULL; 443 pp->pr_hardlimit_ratecap.tv_sec = 0; 444 pp->pr_hardlimit_ratecap.tv_usec = 0; 445 pp->pr_hardlimit_warning_last.tv_sec = 0; 446 pp->pr_hardlimit_warning_last.tv_usec = 0; 447 pp->pr_serial = ++pool_serial; 448 if (pool_serial == 0) 449 panic("pool_init: too much uptime"); 450 451 /* 452 * Decide whether to put the page header off page to avoid 453 * wasting too large a part of the page. Off-page page headers 454 * go on a hash table, so we can match a returned item 455 * with its header based on the page address. 456 * We use 1/16 of the page size as the threshold (XXX: tune) 457 */ 458 if (pp->pr_size < pagesz/16) { 459 /* Use the end of the page for the page header */ 460 pp->pr_roflags |= PR_PHINPAGE; 461 pp->pr_phoffset = off = 462 pagesz - ALIGN(sizeof(struct pool_item_header)); 463 } else { 464 /* The page header will be taken from our page header pool */ 465 pp->pr_phoffset = 0; 466 off = pagesz; 467 for (i = 0; i < PR_HASHTABSIZE; i++) { 468 LIST_INIT(&pp->pr_hashtab[i]); 469 } 470 } 471 472 /* 473 * Alignment is to take place at `ioff' within the item. This means 474 * we must reserve up to `align - 1' bytes on the page to allow 475 * appropriate positioning of each item. 476 * 477 * Silently enforce `0 <= ioff < align'. 478 */ 479 pp->pr_itemoffset = ioff = ioff % align; 480 pp->pr_itemsperpage = (off - ((align - ioff) % align)) / pp->pr_size; 481 KASSERT(pp->pr_itemsperpage != 0); 482 483 /* 484 * Use the slack between the chunks and the page header 485 * for "cache coloring". 486 */ 487 slack = off - pp->pr_itemsperpage * pp->pr_size; 488 pp->pr_maxcolor = (slack / align) * align; 489 pp->pr_curcolor = 0; 490 491 pp->pr_nget = 0; 492 pp->pr_nfail = 0; 493 pp->pr_nput = 0; 494 pp->pr_npagealloc = 0; 495 pp->pr_npagefree = 0; 496 pp->pr_hiwat = 0; 497 pp->pr_nidle = 0; 498 499 #ifdef POOL_DIAGNOSTIC 500 if (flags & PR_LOGGING) { 501 if (kmem_map == NULL || 502 (pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log), 503 M_TEMP, M_NOWAIT)) == NULL) 504 pp->pr_roflags &= ~PR_LOGGING; 505 pp->pr_curlogentry = 0; 506 pp->pr_logsize = pool_logsize; 507 } 508 #endif 509 510 pp->pr_entered_file = NULL; 511 pp->pr_entered_line = 0; 512 513 simple_lock_init(&pp->pr_slock); 514 515 /* 516 * Initialize private page header pool and cache magazine pool if we 517 * haven't done so yet. 518 * XXX LOCKING. 519 */ 520 if (phpool.pr_size == 0) { 521 pool_init(&phpool, sizeof(struct pool_item_header), 0, 0, 522 0, "phpool", 0, 0, 0, 0); 523 pool_init(&pcgpool, sizeof(struct pool_cache_group), 0, 0, 524 0, "pcgpool", 0, 0, 0, 0); 525 } 526 527 /* Insert into the list of all pools. */ 528 simple_lock(&pool_head_slock); 529 TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist); 530 simple_unlock(&pool_head_slock); 531 } 532 533 /* 534 * De-commision a pool resource. 535 */ 536 void 537 pool_destroy(struct pool *pp) 538 { 539 struct pool_item_header *ph; 540 struct pool_cache *pc; 541 542 /* Destroy all caches for this pool. */ 543 while ((pc = TAILQ_FIRST(&pp->pr_cachelist)) != NULL) 544 pool_cache_destroy(pc); 545 546 #ifdef DIAGNOSTIC 547 if (pp->pr_nout != 0) { 548 pr_printlog(pp, NULL, printf); 549 panic("pool_destroy: pool busy: still out: %u\n", 550 pp->pr_nout); 551 } 552 #endif 553 554 /* Remove all pages */ 555 if ((pp->pr_roflags & PR_STATIC) == 0) 556 while ((ph = pp->pr_pagelist.tqh_first) != NULL) 557 pr_rmpage(pp, ph); 558 559 /* Remove from global pool list */ 560 simple_lock(&pool_head_slock); 561 TAILQ_REMOVE(&pool_head, pp, pr_poollist); 562 /* XXX Only clear this if we were drainpp? */ 563 drainpp = NULL; 564 simple_unlock(&pool_head_slock); 565 566 #ifdef POOL_DIAGNOSTIC 567 if ((pp->pr_roflags & PR_LOGGING) != 0) 568 free(pp->pr_log, M_TEMP); 569 #endif 570 571 if (pp->pr_roflags & PR_FREEHEADER) 572 free(pp, M_POOL); 573 } 574 575 static __inline struct pool_item_header * 576 pool_alloc_item_header(struct pool *pp, caddr_t storage, int flags) 577 { 578 struct pool_item_header *ph; 579 int s; 580 581 LOCK_ASSERT(simple_lock_held(&pp->pr_slock) == 0); 582 583 if ((pp->pr_roflags & PR_PHINPAGE) != 0) 584 ph = (struct pool_item_header *) (storage + pp->pr_phoffset); 585 else { 586 s = splhigh(); 587 ph = pool_get(&phpool, flags); 588 splx(s); 589 } 590 591 return (ph); 592 } 593 594 /* 595 * Grab an item from the pool; must be called at appropriate spl level 596 */ 597 void * 598 #ifdef POOL_DIAGNOSTIC 599 _pool_get(struct pool *pp, int flags, const char *file, long line) 600 #else 601 pool_get(struct pool *pp, int flags) 602 #endif 603 { 604 struct pool_item *pi; 605 struct pool_item_header *ph; 606 void *v; 607 608 #ifdef DIAGNOSTIC 609 if (__predict_false((pp->pr_roflags & PR_STATIC) && 610 (flags & PR_MALLOCOK))) { 611 pr_printlog(pp, NULL, printf); 612 panic("pool_get: static"); 613 } 614 615 if (__predict_false(curproc == NULL && /* doing_shutdown == 0 && XXX*/ 616 (flags & PR_WAITOK) != 0)) 617 panic("pool_get: must have NOWAIT"); 618 619 #endif 620 simple_lock(&pp->pr_slock); 621 pr_enter(pp, file, line); 622 623 startover: 624 /* 625 * Check to see if we've reached the hard limit. If we have, 626 * and we can wait, then wait until an item has been returned to 627 * the pool. 628 */ 629 #ifdef DIAGNOSTIC 630 if (__predict_false(pp->pr_nout > pp->pr_hardlimit)) { 631 pr_leave(pp); 632 simple_unlock(&pp->pr_slock); 633 panic("pool_get: %s: crossed hard limit", pp->pr_wchan); 634 } 635 #endif 636 if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) { 637 if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) { 638 /* 639 * XXX: A warning isn't logged in this case. Should 640 * it be? 641 */ 642 pp->pr_flags |= PR_WANTED; 643 pr_leave(pp); 644 simple_unlock(&pp->pr_slock); 645 tsleep((caddr_t)pp, PSWP, (char *)pp->pr_wchan, 0); 646 simple_lock(&pp->pr_slock); 647 pr_enter(pp, file, line); 648 goto startover; 649 } 650 651 /* 652 * Log a message that the hard limit has been hit. 653 */ 654 if (pp->pr_hardlimit_warning != NULL && 655 ratecheck(&pp->pr_hardlimit_warning_last, 656 &pp->pr_hardlimit_ratecap)) 657 log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning); 658 659 if (flags & PR_URGENT) 660 panic("pool_get: urgent"); 661 662 pp->pr_nfail++; 663 664 pr_leave(pp); 665 simple_unlock(&pp->pr_slock); 666 return (NULL); 667 } 668 669 /* 670 * The convention we use is that if `curpage' is not NULL, then 671 * it points at a non-empty bucket. In particular, `curpage' 672 * never points at a page header which has PR_PHINPAGE set and 673 * has no items in its bucket. 674 */ 675 if ((ph = pp->pr_curpage) == NULL) { 676 #ifdef DIAGNOSTIC 677 if (pp->pr_nitems != 0) { 678 simple_unlock(&pp->pr_slock); 679 printf("pool_get: %s: curpage NULL, nitems %u\n", 680 pp->pr_wchan, pp->pr_nitems); 681 panic("pool_get: nitems inconsistent\n"); 682 } 683 #endif 684 685 /* 686 * Call the back-end page allocator for more memory. 687 * Release the pool lock, as the back-end page allocator 688 * may block. 689 */ 690 pr_leave(pp); 691 simple_unlock(&pp->pr_slock); 692 v = (*pp->pr_alloc)(pp->pr_pagesz, flags, pp->pr_mtype); 693 if (__predict_true(v != NULL)) 694 ph = pool_alloc_item_header(pp, v, flags); 695 simple_lock(&pp->pr_slock); 696 pr_enter(pp, file, line); 697 698 if (__predict_false(v == NULL || ph == NULL)) { 699 if (v != NULL) 700 (*pp->pr_free)(v, pp->pr_pagesz, pp->pr_mtype); 701 702 /* 703 * We were unable to allocate a page or item 704 * header, but we released the lock during 705 * allocation, so perhaps items were freed 706 * back to the pool. Check for this case. 707 */ 708 if (pp->pr_curpage != NULL) 709 goto startover; 710 711 if (flags & PR_URGENT) 712 panic("pool_get: urgent"); 713 714 if ((flags & PR_WAITOK) == 0) { 715 pp->pr_nfail++; 716 pr_leave(pp); 717 simple_unlock(&pp->pr_slock); 718 return (NULL); 719 } 720 721 /* 722 * Wait for items to be returned to this pool. 723 * 724 * XXX: we actually want to wait just until 725 * the page allocator has memory again. Depending 726 * on this pool's usage, we might get stuck here 727 * for a long time. 728 * 729 * XXX: maybe we should wake up once a second and 730 * try again? 731 */ 732 pp->pr_flags |= PR_WANTED; 733 pr_leave(pp); 734 simple_unlock(&pp->pr_slock); 735 tsleep((caddr_t)pp, PSWP, (char *)pp->pr_wchan, 0); 736 simple_lock(&pp->pr_slock); 737 pr_enter(pp, file, line); 738 goto startover; 739 } 740 741 /* We have more memory; add it to the pool */ 742 pp->pr_npagealloc++; 743 pool_prime_page(pp, v, ph); 744 745 /* Start the allocation process over. */ 746 goto startover; 747 } 748 749 if (__predict_false((v = pi = TAILQ_FIRST(&ph->ph_itemlist)) == NULL)) { 750 pr_leave(pp); 751 simple_unlock(&pp->pr_slock); 752 panic("pool_get: %s: page empty", pp->pr_wchan); 753 } 754 #ifdef DIAGNOSTIC 755 if (__predict_false(pp->pr_nitems == 0)) { 756 pr_leave(pp); 757 simple_unlock(&pp->pr_slock); 758 printf("pool_get: %s: items on itemlist, nitems %u\n", 759 pp->pr_wchan, pp->pr_nitems); 760 panic("pool_get: nitems inconsistent\n"); 761 } 762 763 pr_log(pp, v, PRLOG_GET, file, line); 764 765 if (__predict_false(pi->pi_magic != PI_MAGIC)) { 766 pr_printlog(pp, pi, printf); 767 panic("pool_get(%s): free list modified: magic=%x; page %p;" 768 " item addr %p\n", 769 pp->pr_wchan, pi->pi_magic, ph->ph_page, pi); 770 } 771 #endif 772 773 /* 774 * Remove from item list. 775 */ 776 TAILQ_REMOVE(&ph->ph_itemlist, pi, pi_list); 777 pp->pr_nitems--; 778 pp->pr_nout++; 779 if (ph->ph_nmissing == 0) { 780 #ifdef DIAGNOSTIC 781 if (__predict_false(pp->pr_nidle == 0)) 782 panic("pool_get: nidle inconsistent"); 783 #endif 784 pp->pr_nidle--; 785 } 786 ph->ph_nmissing++; 787 if (TAILQ_FIRST(&ph->ph_itemlist) == NULL) { 788 #ifdef DIAGNOSTIC 789 if (__predict_false(ph->ph_nmissing != pp->pr_itemsperpage)) { 790 pr_leave(pp); 791 simple_unlock(&pp->pr_slock); 792 panic("pool_get: %s: nmissing inconsistent", 793 pp->pr_wchan); 794 } 795 #endif 796 /* 797 * Find a new non-empty page header, if any. 798 * Start search from the page head, to increase 799 * the chance for "high water" pages to be freed. 800 * 801 * Migrate empty pages to the end of the list. This 802 * will speed the update of curpage as pages become 803 * idle. Empty pages intermingled with idle pages 804 * is no big deal. As soon as a page becomes un-empty, 805 * it will move back to the head of the list. 806 */ 807 TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist); 808 TAILQ_INSERT_TAIL(&pp->pr_pagelist, ph, ph_pagelist); 809 for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL; 810 ph = TAILQ_NEXT(ph, ph_pagelist)) 811 if (TAILQ_FIRST(&ph->ph_itemlist) != NULL) 812 break; 813 814 pp->pr_curpage = ph; 815 } 816 817 pp->pr_nget++; 818 819 /* 820 * If we have a low water mark and we are now below that low 821 * water mark, add more items to the pool. 822 */ 823 if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) { 824 /* 825 * XXX: Should we log a warning? Should we set up a timeout 826 * to try again in a second or so? The latter could break 827 * a caller's assumptions about interrupt protection, etc. 828 */ 829 } 830 831 pr_leave(pp); 832 simple_unlock(&pp->pr_slock); 833 return (v); 834 } 835 836 /* 837 * Internal version of pool_put(). Pool is already locked/entered. 838 */ 839 static void 840 pool_do_put(struct pool *pp, void *v) 841 { 842 struct pool_item *pi = v; 843 struct pool_item_header *ph; 844 caddr_t page; 845 int s; 846 847 page = (caddr_t)((u_long)v & pp->pr_pagemask); 848 849 #ifdef DIAGNOSTIC 850 if (__predict_false(pp->pr_nout == 0)) { 851 printf("pool %s: putting with none out\n", 852 pp->pr_wchan); 853 panic("pool_put"); 854 } 855 #endif 856 857 if (__predict_false((ph = pr_find_pagehead(pp, page)) == NULL)) { 858 pr_printlog(pp, NULL, printf); 859 panic("pool_put: %s: page header missing", pp->pr_wchan); 860 } 861 862 #ifdef LOCKDEBUG 863 /* 864 * Check if we're freeing a locked simple lock. 865 */ 866 simple_lock_freecheck((caddr_t)pi, ((caddr_t)pi) + pp->pr_size); 867 #endif 868 869 /* 870 * Return to item list. 871 */ 872 #ifdef DIAGNOSTIC 873 pi->pi_magic = PI_MAGIC; 874 #endif 875 #ifdef DEBUG 876 { 877 int i, *ip = v; 878 879 for (i = 0; i < pp->pr_size / sizeof(int); i++) { 880 *ip++ = PI_MAGIC; 881 } 882 } 883 #endif 884 885 TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); 886 ph->ph_nmissing--; 887 pp->pr_nput++; 888 pp->pr_nitems++; 889 pp->pr_nout--; 890 891 /* Cancel "pool empty" condition if it exists */ 892 if (pp->pr_curpage == NULL) 893 pp->pr_curpage = ph; 894 895 if (pp->pr_flags & PR_WANTED) { 896 pp->pr_flags &= ~PR_WANTED; 897 if (ph->ph_nmissing == 0) 898 pp->pr_nidle++; 899 wakeup((caddr_t)pp); 900 return; 901 } 902 903 /* 904 * If this page is now complete, do one of two things: 905 * 906 * (1) If we have more pages than the page high water 907 * mark, free the page back to the system. 908 * 909 * (2) Move it to the end of the page list, so that 910 * we minimize our chances of fragmenting the 911 * pool. Idle pages migrate to the end (along with 912 * completely empty pages, so that we find un-empty 913 * pages more quickly when we update curpage) of the 914 * list so they can be more easily swept up by 915 * the pagedaemon when pages are scarce. 916 */ 917 if (ph->ph_nmissing == 0) { 918 pp->pr_nidle++; 919 if (pp->pr_npages > pp->pr_maxpages) { 920 pr_rmpage(pp, ph); 921 } else { 922 TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist); 923 TAILQ_INSERT_TAIL(&pp->pr_pagelist, ph, ph_pagelist); 924 925 /* 926 * Update the timestamp on the page. A page must 927 * be idle for some period of time before it can 928 * be reclaimed by the pagedaemon. This minimizes 929 * ping-pong'ing for memory. 930 */ 931 s = splclock(); 932 ph->ph_time = mono_time; 933 splx(s); 934 935 /* 936 * Update the current page pointer. Just look for 937 * the first page with any free items. 938 * 939 * XXX: Maybe we want an option to look for the 940 * page with the fewest available items, to minimize 941 * fragmentation? 942 */ 943 for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL; 944 ph = TAILQ_NEXT(ph, ph_pagelist)) 945 if (TAILQ_FIRST(&ph->ph_itemlist) != NULL) 946 break; 947 948 pp->pr_curpage = ph; 949 } 950 } 951 /* 952 * If the page has just become un-empty, move it to the head of 953 * the list, and make it the current page. The next allocation 954 * will get the item from this page, instead of further fragmenting 955 * the pool. 956 */ 957 else if (ph->ph_nmissing == (pp->pr_itemsperpage - 1)) { 958 TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist); 959 TAILQ_INSERT_HEAD(&pp->pr_pagelist, ph, ph_pagelist); 960 pp->pr_curpage = ph; 961 } 962 } 963 964 /* 965 * Return resource to the pool; must be called at appropriate spl level 966 */ 967 #ifdef POOL_DIAGNOSTIC 968 void 969 _pool_put(struct pool *pp, void *v, const char *file, long line) 970 { 971 972 simple_lock(&pp->pr_slock); 973 pr_enter(pp, file, line); 974 975 pr_log(pp, v, PRLOG_PUT, file, line); 976 977 pool_do_put(pp, v); 978 979 pr_leave(pp); 980 simple_unlock(&pp->pr_slock); 981 } 982 #undef pool_put 983 #endif /* POOL_DIAGNOSTIC */ 984 985 void 986 pool_put(struct pool *pp, void *v) 987 { 988 989 simple_lock(&pp->pr_slock); 990 991 pool_do_put(pp, v); 992 993 simple_unlock(&pp->pr_slock); 994 } 995 996 #ifdef POOL_DIAGNOSTIC 997 #define pool_put(h, v) _pool_put((h), (v), __FILE__, __LINE__) 998 #endif 999 1000 /* 1001 * Add N items to the pool. 1002 */ 1003 int 1004 pool_prime(struct pool *pp, int n) 1005 { 1006 struct pool_item_header *ph; 1007 caddr_t cp; 1008 int newpages, error = 0; 1009 1010 simple_lock(&pp->pr_slock); 1011 1012 newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 1013 1014 while (newpages-- > 0) { 1015 simple_unlock(&pp->pr_slock); 1016 cp = (*pp->pr_alloc)(pp->pr_pagesz, PR_NOWAIT, pp->pr_mtype); 1017 if (__predict_true(cp != NULL)) 1018 ph = pool_alloc_item_header(pp, cp, PR_NOWAIT); 1019 simple_lock(&pp->pr_slock); 1020 1021 if (__predict_false(cp == NULL || ph == NULL)) { 1022 error = ENOMEM; 1023 if (cp != NULL) 1024 (*pp->pr_free)(cp, pp->pr_pagesz, pp->pr_mtype); 1025 break; 1026 } 1027 1028 pool_prime_page(pp, cp, ph); 1029 pp->pr_npagealloc++; 1030 pp->pr_minpages++; 1031 } 1032 1033 if (pp->pr_minpages >= pp->pr_maxpages) 1034 pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ 1035 1036 simple_unlock(&pp->pr_slock); 1037 return (0); 1038 } 1039 1040 /* 1041 * Add a page worth of items to the pool. 1042 * 1043 * Note, we must be called with the pool descriptor LOCKED. 1044 */ 1045 static void 1046 pool_prime_page(struct pool *pp, caddr_t storage, struct pool_item_header *ph) 1047 { 1048 struct pool_item *pi; 1049 caddr_t cp = storage; 1050 unsigned int align = pp->pr_align; 1051 unsigned int ioff = pp->pr_itemoffset; 1052 int n; 1053 1054 if (((u_long)cp & (pp->pr_pagesz - 1)) != 0) 1055 panic("pool_prime_page: %s: unaligned page", pp->pr_wchan); 1056 1057 if ((pp->pr_roflags & PR_PHINPAGE) == 0) 1058 LIST_INSERT_HEAD(&pp->pr_hashtab[PR_HASH_INDEX(pp, cp)], 1059 ph, ph_hashlist); 1060 1061 /* 1062 * Insert page header. 1063 */ 1064 TAILQ_INSERT_HEAD(&pp->pr_pagelist, ph, ph_pagelist); 1065 TAILQ_INIT(&ph->ph_itemlist); 1066 ph->ph_page = storage; 1067 ph->ph_nmissing = 0; 1068 memset(&ph->ph_time, 0, sizeof(ph->ph_time)); 1069 1070 pp->pr_nidle++; 1071 1072 /* 1073 * Color this page. 1074 */ 1075 cp = (caddr_t)(cp + pp->pr_curcolor); 1076 if ((pp->pr_curcolor += align) > pp->pr_maxcolor) 1077 pp->pr_curcolor = 0; 1078 1079 /* 1080 * Adjust storage to apply aligment to `pr_itemoffset' in each item. 1081 */ 1082 if (ioff != 0) 1083 cp = (caddr_t)(cp + (align - ioff)); 1084 1085 /* 1086 * Insert remaining chunks on the bucket list. 1087 */ 1088 n = pp->pr_itemsperpage; 1089 pp->pr_nitems += n; 1090 1091 while (n--) { 1092 pi = (struct pool_item *)cp; 1093 1094 /* Insert on page list */ 1095 TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list); 1096 #ifdef DIAGNOSTIC 1097 pi->pi_magic = PI_MAGIC; 1098 #endif 1099 cp = (caddr_t)(cp + pp->pr_size); 1100 } 1101 1102 /* 1103 * If the pool was depleted, point at the new page. 1104 */ 1105 if (pp->pr_curpage == NULL) 1106 pp->pr_curpage = ph; 1107 1108 if (++pp->pr_npages > pp->pr_hiwat) 1109 pp->pr_hiwat = pp->pr_npages; 1110 } 1111 1112 /* 1113 * Used by pool_get() when nitems drops below the low water mark. This 1114 * is used to catch up nitmes with the low water mark. 1115 * 1116 * Note 1, we never wait for memory here, we let the caller decide what to do. 1117 * 1118 * Note 2, this doesn't work with static pools. 1119 * 1120 * Note 3, we must be called with the pool already locked, and we return 1121 * with it locked. 1122 */ 1123 static int 1124 pool_catchup(struct pool *pp) 1125 { 1126 struct pool_item_header *ph; 1127 caddr_t cp; 1128 int error = 0; 1129 1130 if (pp->pr_roflags & PR_STATIC) { 1131 /* 1132 * We dropped below the low water mark, and this is not a 1133 * good thing. Log a warning. 1134 * 1135 * XXX: rate-limit this? 1136 */ 1137 printf("WARNING: static pool `%s' dropped below low water " 1138 "mark\n", pp->pr_wchan); 1139 return (0); 1140 } 1141 1142 while (POOL_NEEDS_CATCHUP(pp)) { 1143 /* 1144 * Call the page back-end allocator for more memory. 1145 * 1146 * XXX: We never wait, so should we bother unlocking 1147 * the pool descriptor? 1148 */ 1149 simple_unlock(&pp->pr_slock); 1150 cp = (*pp->pr_alloc)(pp->pr_pagesz, PR_NOWAIT, pp->pr_mtype); 1151 if (__predict_true(cp != NULL)) 1152 ph = pool_alloc_item_header(pp, cp, PR_NOWAIT); 1153 simple_lock(&pp->pr_slock); 1154 if (__predict_false(cp == NULL || ph == NULL)) { 1155 if (cp != NULL) 1156 (*pp->pr_free)(cp, pp->pr_pagesz, pp->pr_mtype); 1157 error = ENOMEM; 1158 break; 1159 } 1160 pool_prime_page(pp, cp, ph); 1161 pp->pr_npagealloc++; 1162 } 1163 1164 return (error); 1165 } 1166 1167 void 1168 pool_setlowat(struct pool *pp, int n) 1169 { 1170 int error; 1171 1172 simple_lock(&pp->pr_slock); 1173 1174 pp->pr_minitems = n; 1175 pp->pr_minpages = (n == 0) 1176 ? 0 1177 : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 1178 1179 /* Make sure we're caught up with the newly-set low water mark. */ 1180 if (POOL_NEEDS_CATCHUP(pp) && (error = pool_catchup(pp) != 0)) { 1181 /* 1182 * XXX: Should we log a warning? Should we set up a timeout 1183 * to try again in a second or so? The latter could break 1184 * a caller's assumptions about interrupt protection, etc. 1185 */ 1186 } 1187 1188 simple_unlock(&pp->pr_slock); 1189 } 1190 1191 void 1192 pool_sethiwat(struct pool *pp, int n) 1193 { 1194 1195 simple_lock(&pp->pr_slock); 1196 1197 pp->pr_maxpages = (n == 0) 1198 ? 0 1199 : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 1200 1201 simple_unlock(&pp->pr_slock); 1202 } 1203 1204 void 1205 pool_sethardlimit(struct pool *pp, int n, const char *warnmess, int ratecap) 1206 { 1207 1208 simple_lock(&pp->pr_slock); 1209 1210 pp->pr_hardlimit = n; 1211 pp->pr_hardlimit_warning = warnmess; 1212 pp->pr_hardlimit_ratecap.tv_sec = ratecap; 1213 pp->pr_hardlimit_warning_last.tv_sec = 0; 1214 pp->pr_hardlimit_warning_last.tv_usec = 0; 1215 1216 /* 1217 * In-line version of pool_sethiwat(), because we don't want to 1218 * release the lock. 1219 */ 1220 pp->pr_maxpages = (n == 0) 1221 ? 0 1222 : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 1223 1224 simple_unlock(&pp->pr_slock); 1225 } 1226 1227 /* 1228 * Default page allocator. 1229 */ 1230 static void * 1231 pool_page_alloc(unsigned long sz, int flags, int mtype) 1232 { 1233 boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; 1234 1235 return ((void *)uvm_km_alloc_poolpage(waitok)); 1236 } 1237 1238 static void 1239 pool_page_free(void *v, unsigned long sz, int mtype) 1240 { 1241 uvm_km_free_poolpage((vaddr_t)v); 1242 } 1243 1244 /* 1245 * Alternate pool page allocator for pools that know they will 1246 * never be accessed in interrupt context. 1247 */ 1248 void * 1249 pool_page_alloc_nointr(unsigned long sz, int flags, int mtype) 1250 { 1251 boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; 1252 1253 return ((void *)uvm_km_alloc_poolpage1(kernel_map, uvm.kernel_object, 1254 waitok)); 1255 } 1256 1257 void 1258 pool_page_free_nointr(void *v, unsigned long sz, int mtype) 1259 { 1260 1261 uvm_km_free_poolpage1(kernel_map, (vaddr_t)v); 1262 } 1263 1264 1265 /* 1266 * Release all complete pages that have not been used recently. 1267 */ 1268 void 1269 #ifdef POOL_DIAGNOSTIC 1270 _pool_reclaim(struct pool *pp, const char *file, long line) 1271 #else 1272 pool_reclaim(struct pool *pp) 1273 #endif 1274 { 1275 struct pool_item_header *ph, *phnext; 1276 struct pool_cache *pc; 1277 struct timeval curtime; 1278 int s; 1279 1280 if (pp->pr_roflags & PR_STATIC) 1281 return; 1282 1283 if (simple_lock_try(&pp->pr_slock) == 0) 1284 return; 1285 pr_enter(pp, file, line); 1286 1287 /* 1288 * Reclaim items from the pool's caches. 1289 */ 1290 for (pc = TAILQ_FIRST(&pp->pr_cachelist); pc != NULL; 1291 pc = TAILQ_NEXT(pc, pc_poollist)) 1292 pool_cache_reclaim(pc); 1293 1294 s = splclock(); 1295 curtime = mono_time; 1296 splx(s); 1297 1298 for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL; ph = phnext) { 1299 phnext = TAILQ_NEXT(ph, ph_pagelist); 1300 1301 /* Check our minimum page claim */ 1302 if (pp->pr_npages <= pp->pr_minpages) 1303 break; 1304 1305 if (ph->ph_nmissing == 0) { 1306 struct timeval diff; 1307 timersub(&curtime, &ph->ph_time, &diff); 1308 if (diff.tv_sec < pool_inactive_time) 1309 continue; 1310 1311 /* 1312 * If freeing this page would put us below 1313 * the low water mark, stop now. 1314 */ 1315 if ((pp->pr_nitems - pp->pr_itemsperpage) < 1316 pp->pr_minitems) 1317 break; 1318 1319 pr_rmpage(pp, ph); 1320 } 1321 } 1322 1323 pr_leave(pp); 1324 simple_unlock(&pp->pr_slock); 1325 } 1326 1327 1328 /* 1329 * Drain pools, one at a time. 1330 * 1331 * Note, we must never be called from an interrupt context. 1332 */ 1333 void 1334 pool_drain(void *arg) 1335 { 1336 struct pool *pp; 1337 int s; 1338 1339 s = splvm(); 1340 simple_lock(&pool_head_slock); 1341 1342 if (drainpp == NULL && (drainpp = TAILQ_FIRST(&pool_head)) == NULL) 1343 goto out; 1344 1345 pp = drainpp; 1346 drainpp = TAILQ_NEXT(pp, pr_poollist); 1347 1348 pool_reclaim(pp); 1349 1350 out: 1351 simple_unlock(&pool_head_slock); 1352 splx(s); 1353 } 1354 1355 1356 /* 1357 * Diagnostic helpers. 1358 */ 1359 void 1360 pool_printit(struct pool *pp, const char *modif, int (*pr)(const char *, ...)) 1361 { 1362 int s; 1363 1364 s = splvm(); 1365 if (simple_lock_try(&pp->pr_slock) == 0) { 1366 printf("pool %s is locked; try again later\n", 1367 pp->pr_wchan); 1368 splx(s); 1369 return; 1370 } 1371 pool_print1(pp, modif, printf); 1372 simple_unlock(&pp->pr_slock); 1373 splx(s); 1374 } 1375 1376 static void 1377 pool_print1(struct pool *pp, const char *modif, int (*pr)(const char *, ...)) 1378 { 1379 struct pool_item_header *ph; 1380 struct pool_cache *pc; 1381 struct pool_cache_group *pcg; 1382 #ifdef DIAGNOSTIC 1383 struct pool_item *pi; 1384 #endif 1385 int i, print_log = 0, print_pagelist = 0, print_cache = 0; 1386 char c; 1387 1388 while ((c = *modif++) != '\0') { 1389 if (c == 'l') 1390 print_log = 1; 1391 if (c == 'p') 1392 print_pagelist = 1; 1393 if (c == 'c') 1394 print_cache = 1; 1395 modif++; 1396 } 1397 1398 (*pr)("POOL %s: size %u, align %u, ioff %u, roflags 0x%08x\n", 1399 pp->pr_wchan, pp->pr_size, pp->pr_align, pp->pr_itemoffset, 1400 pp->pr_roflags); 1401 (*pr)("\tpagesz %u, mtype %d\n", pp->pr_pagesz, pp->pr_mtype); 1402 (*pr)("\talloc %p, release %p\n", pp->pr_alloc, pp->pr_free); 1403 (*pr)("\tminitems %u, minpages %u, maxpages %u, npages %u\n", 1404 pp->pr_minitems, pp->pr_minpages, pp->pr_maxpages, pp->pr_npages); 1405 (*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n", 1406 pp->pr_itemsperpage, pp->pr_nitems, pp->pr_nout, pp->pr_hardlimit); 1407 1408 (*pr)("\n\tnget %lu, nfail %lu, nput %lu\n", 1409 pp->pr_nget, pp->pr_nfail, pp->pr_nput); 1410 (*pr)("\tnpagealloc %lu, npagefree %lu, hiwat %u, nidle %lu\n", 1411 pp->pr_npagealloc, pp->pr_npagefree, pp->pr_hiwat, pp->pr_nidle); 1412 1413 if (print_pagelist == 0) 1414 goto skip_pagelist; 1415 1416 if ((ph = TAILQ_FIRST(&pp->pr_pagelist)) != NULL) 1417 (*pr)("\n\tpage list:\n"); 1418 for (; ph != NULL; ph = TAILQ_NEXT(ph, ph_pagelist)) { 1419 (*pr)("\t\tpage %p, nmissing %d, time %lu,%lu\n", 1420 ph->ph_page, ph->ph_nmissing, 1421 (u_long)ph->ph_time.tv_sec, 1422 (u_long)ph->ph_time.tv_usec); 1423 #ifdef DIAGNOSTIC 1424 for (pi = TAILQ_FIRST(&ph->ph_itemlist); pi != NULL; 1425 pi = TAILQ_NEXT(pi, pi_list)) { 1426 if (pi->pi_magic != PI_MAGIC) { 1427 (*pr)("\t\t\titem %p, magic 0x%x\n", 1428 pi, pi->pi_magic); 1429 } 1430 } 1431 #endif 1432 } 1433 if (pp->pr_curpage == NULL) 1434 (*pr)("\tno current page\n"); 1435 else 1436 (*pr)("\tcurpage %p\n", pp->pr_curpage->ph_page); 1437 1438 skip_pagelist: 1439 1440 if (print_log == 0) 1441 goto skip_log; 1442 1443 (*pr)("\n"); 1444 if ((pp->pr_roflags & PR_LOGGING) == 0) 1445 (*pr)("\tno log\n"); 1446 else 1447 pr_printlog(pp, NULL, pr); 1448 1449 skip_log: 1450 1451 if (print_cache == 0) 1452 goto skip_cache; 1453 1454 for (pc = TAILQ_FIRST(&pp->pr_cachelist); pc != NULL; 1455 pc = TAILQ_NEXT(pc, pc_poollist)) { 1456 (*pr)("\tcache %p: allocfrom %p freeto %p\n", pc, 1457 pc->pc_allocfrom, pc->pc_freeto); 1458 (*pr)("\t hits %lu misses %lu ngroups %lu nitems %lu\n", 1459 pc->pc_hits, pc->pc_misses, pc->pc_ngroups, pc->pc_nitems); 1460 for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL; 1461 pcg = TAILQ_NEXT(pcg, pcg_list)) { 1462 (*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); 1463 for (i = 0; i < PCG_NOBJECTS; i++) 1464 (*pr)("\t\t\t%p\n", pcg->pcg_objects[i]); 1465 } 1466 } 1467 1468 skip_cache: 1469 1470 pr_enter_check(pp, pr); 1471 } 1472 1473 int 1474 pool_chk(struct pool *pp, const char *label) 1475 { 1476 struct pool_item_header *ph; 1477 int r = 0; 1478 1479 simple_lock(&pp->pr_slock); 1480 1481 for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL; 1482 ph = TAILQ_NEXT(ph, ph_pagelist)) { 1483 1484 struct pool_item *pi; 1485 int n; 1486 caddr_t page; 1487 1488 page = (caddr_t)((u_long)ph & pp->pr_pagemask); 1489 if (page != ph->ph_page && 1490 (pp->pr_roflags & PR_PHINPAGE) != 0) { 1491 if (label != NULL) 1492 printf("%s: ", label); 1493 printf("pool(%p:%s): page inconsistency: page %p;" 1494 " at page head addr %p (p %p)\n", pp, 1495 pp->pr_wchan, ph->ph_page, 1496 ph, page); 1497 r++; 1498 goto out; 1499 } 1500 1501 for (pi = TAILQ_FIRST(&ph->ph_itemlist), n = 0; 1502 pi != NULL; 1503 pi = TAILQ_NEXT(pi,pi_list), n++) { 1504 1505 #ifdef DIAGNOSTIC 1506 if (pi->pi_magic != PI_MAGIC) { 1507 if (label != NULL) 1508 printf("%s: ", label); 1509 printf("pool(%s): free list modified: magic=%x;" 1510 " page %p; item ordinal %d;" 1511 " addr %p (p %p)\n", 1512 pp->pr_wchan, pi->pi_magic, ph->ph_page, 1513 n, pi, page); 1514 panic("pool"); 1515 } 1516 #endif 1517 page = (caddr_t)((u_long)pi & pp->pr_pagemask); 1518 if (page == ph->ph_page) 1519 continue; 1520 1521 if (label != NULL) 1522 printf("%s: ", label); 1523 printf("pool(%p:%s): page inconsistency: page %p;" 1524 " item ordinal %d; addr %p (p %p)\n", pp, 1525 pp->pr_wchan, ph->ph_page, 1526 n, pi, page); 1527 r++; 1528 goto out; 1529 } 1530 } 1531 out: 1532 simple_unlock(&pp->pr_slock); 1533 return (r); 1534 } 1535 1536 /* 1537 * pool_cache_init: 1538 * 1539 * Initialize a pool cache. 1540 * 1541 * NOTE: If the pool must be protected from interrupts, we expect 1542 * to be called at the appropriate interrupt priority level. 1543 */ 1544 void 1545 pool_cache_init(struct pool_cache *pc, struct pool *pp, 1546 int (*ctor)(void *, void *, int), 1547 void (*dtor)(void *, void *), 1548 void *arg) 1549 { 1550 1551 TAILQ_INIT(&pc->pc_grouplist); 1552 simple_lock_init(&pc->pc_slock); 1553 1554 pc->pc_allocfrom = NULL; 1555 pc->pc_freeto = NULL; 1556 pc->pc_pool = pp; 1557 1558 pc->pc_ctor = ctor; 1559 pc->pc_dtor = dtor; 1560 pc->pc_arg = arg; 1561 1562 pc->pc_hits = 0; 1563 pc->pc_misses = 0; 1564 1565 pc->pc_ngroups = 0; 1566 1567 pc->pc_nitems = 0; 1568 1569 simple_lock(&pp->pr_slock); 1570 TAILQ_INSERT_TAIL(&pp->pr_cachelist, pc, pc_poollist); 1571 simple_unlock(&pp->pr_slock); 1572 } 1573 1574 /* 1575 * pool_cache_destroy: 1576 * 1577 * Destroy a pool cache. 1578 */ 1579 void 1580 pool_cache_destroy(struct pool_cache *pc) 1581 { 1582 struct pool *pp = pc->pc_pool; 1583 1584 /* First, invalidate the entire cache. */ 1585 pool_cache_invalidate(pc); 1586 1587 /* ...and remove it from the pool's cache list. */ 1588 simple_lock(&pp->pr_slock); 1589 TAILQ_REMOVE(&pp->pr_cachelist, pc, pc_poollist); 1590 simple_unlock(&pp->pr_slock); 1591 } 1592 1593 static __inline void * 1594 pcg_get(struct pool_cache_group *pcg) 1595 { 1596 void *object; 1597 u_int idx; 1598 1599 KASSERT(pcg->pcg_avail <= PCG_NOBJECTS); 1600 KASSERT(pcg->pcg_avail != 0); 1601 idx = --pcg->pcg_avail; 1602 1603 KASSERT(pcg->pcg_objects[idx] != NULL); 1604 object = pcg->pcg_objects[idx]; 1605 pcg->pcg_objects[idx] = NULL; 1606 1607 return (object); 1608 } 1609 1610 static __inline void 1611 pcg_put(struct pool_cache_group *pcg, void *object) 1612 { 1613 u_int idx; 1614 1615 KASSERT(pcg->pcg_avail < PCG_NOBJECTS); 1616 idx = pcg->pcg_avail++; 1617 1618 KASSERT(pcg->pcg_objects[idx] == NULL); 1619 pcg->pcg_objects[idx] = object; 1620 } 1621 1622 /* 1623 * pool_cache_get: 1624 * 1625 * Get an object from a pool cache. 1626 */ 1627 void * 1628 pool_cache_get(struct pool_cache *pc, int flags) 1629 { 1630 struct pool_cache_group *pcg; 1631 void *object; 1632 1633 #ifdef LOCKDEBUG 1634 if (flags & PR_WAITOK) 1635 simple_lock_only_held(NULL, "pool_cache_get(PR_WAITOK)"); 1636 #endif 1637 1638 simple_lock(&pc->pc_slock); 1639 1640 if ((pcg = pc->pc_allocfrom) == NULL) { 1641 for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL; 1642 pcg = TAILQ_NEXT(pcg, pcg_list)) { 1643 if (pcg->pcg_avail != 0) { 1644 pc->pc_allocfrom = pcg; 1645 goto have_group; 1646 } 1647 } 1648 1649 /* 1650 * No groups with any available objects. Allocate 1651 * a new object, construct it, and return it to 1652 * the caller. We will allocate a group, if necessary, 1653 * when the object is freed back to the cache. 1654 */ 1655 pc->pc_misses++; 1656 simple_unlock(&pc->pc_slock); 1657 object = pool_get(pc->pc_pool, flags); 1658 if (object != NULL && pc->pc_ctor != NULL) { 1659 if ((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0) { 1660 pool_put(pc->pc_pool, object); 1661 return (NULL); 1662 } 1663 } 1664 return (object); 1665 } 1666 1667 have_group: 1668 pc->pc_hits++; 1669 pc->pc_nitems--; 1670 object = pcg_get(pcg); 1671 1672 if (pcg->pcg_avail == 0) 1673 pc->pc_allocfrom = NULL; 1674 1675 simple_unlock(&pc->pc_slock); 1676 1677 return (object); 1678 } 1679 1680 /* 1681 * pool_cache_put: 1682 * 1683 * Put an object back to the pool cache. 1684 */ 1685 void 1686 pool_cache_put(struct pool_cache *pc, void *object) 1687 { 1688 struct pool_cache_group *pcg; 1689 1690 simple_lock(&pc->pc_slock); 1691 1692 if ((pcg = pc->pc_freeto) == NULL) { 1693 for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL; 1694 pcg = TAILQ_NEXT(pcg, pcg_list)) { 1695 if (pcg->pcg_avail != PCG_NOBJECTS) { 1696 pc->pc_freeto = pcg; 1697 goto have_group; 1698 } 1699 } 1700 1701 /* 1702 * No empty groups to free the object to. Attempt to 1703 * allocate one. 1704 */ 1705 simple_unlock(&pc->pc_slock); 1706 pcg = pool_get(&pcgpool, PR_NOWAIT); 1707 if (pcg != NULL) { 1708 memset(pcg, 0, sizeof(*pcg)); 1709 simple_lock(&pc->pc_slock); 1710 pc->pc_ngroups++; 1711 TAILQ_INSERT_TAIL(&pc->pc_grouplist, pcg, pcg_list); 1712 if (pc->pc_freeto == NULL) 1713 pc->pc_freeto = pcg; 1714 goto have_group; 1715 } 1716 1717 /* 1718 * Unable to allocate a cache group; destruct the object 1719 * and free it back to the pool. 1720 */ 1721 pool_cache_destruct_object(pc, object); 1722 return; 1723 } 1724 1725 have_group: 1726 pc->pc_nitems++; 1727 pcg_put(pcg, object); 1728 1729 if (pcg->pcg_avail == PCG_NOBJECTS) 1730 pc->pc_freeto = NULL; 1731 1732 simple_unlock(&pc->pc_slock); 1733 } 1734 1735 /* 1736 * pool_cache_destruct_object: 1737 * 1738 * Force destruction of an object and its release back into 1739 * the pool. 1740 */ 1741 void 1742 pool_cache_destruct_object(struct pool_cache *pc, void *object) 1743 { 1744 1745 if (pc->pc_dtor != NULL) 1746 (*pc->pc_dtor)(pc->pc_arg, object); 1747 pool_put(pc->pc_pool, object); 1748 } 1749 1750 /* 1751 * pool_cache_do_invalidate: 1752 * 1753 * This internal function implements pool_cache_invalidate() and 1754 * pool_cache_reclaim(). 1755 */ 1756 static void 1757 pool_cache_do_invalidate(struct pool_cache *pc, int free_groups, 1758 void (*putit)(struct pool *, void *)) 1759 { 1760 struct pool_cache_group *pcg, *npcg; 1761 void *object; 1762 1763 for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL; 1764 pcg = npcg) { 1765 npcg = TAILQ_NEXT(pcg, pcg_list); 1766 while (pcg->pcg_avail != 0) { 1767 pc->pc_nitems--; 1768 object = pcg_get(pcg); 1769 if (pcg->pcg_avail == 0 && pc->pc_allocfrom == pcg) 1770 pc->pc_allocfrom = NULL; 1771 if (pc->pc_dtor != NULL) 1772 (*pc->pc_dtor)(pc->pc_arg, object); 1773 (*putit)(pc->pc_pool, object); 1774 } 1775 if (free_groups) { 1776 pc->pc_ngroups--; 1777 TAILQ_REMOVE(&pc->pc_grouplist, pcg, pcg_list); 1778 if (pc->pc_freeto == pcg) 1779 pc->pc_freeto = NULL; 1780 pool_put(&pcgpool, pcg); 1781 } 1782 } 1783 } 1784 1785 /* 1786 * pool_cache_invalidate: 1787 * 1788 * Invalidate a pool cache (destruct and release all of the 1789 * cached objects). 1790 */ 1791 void 1792 pool_cache_invalidate(struct pool_cache *pc) 1793 { 1794 1795 simple_lock(&pc->pc_slock); 1796 pool_cache_do_invalidate(pc, 0, pool_put); 1797 simple_unlock(&pc->pc_slock); 1798 } 1799 1800 /* 1801 * pool_cache_reclaim: 1802 * 1803 * Reclaim a pool cache for pool_reclaim(). 1804 */ 1805 static void 1806 pool_cache_reclaim(struct pool_cache *pc) 1807 { 1808 1809 simple_lock(&pc->pc_slock); 1810 pool_cache_do_invalidate(pc, 1, pool_do_put); 1811 simple_unlock(&pc->pc_slock); 1812 } 1813 1814 /* 1815 * We have three different sysctls. 1816 * kern.pool.npools - the number of pools. 1817 * kern.pool.pool.<pool#> - the pool struct for the pool#. 1818 * kern.pool.name.<pool#> - the name for pool#.[6~ 1819 */ 1820 int 1821 sysctl_dopool(int *name, u_int namelen, char *where, size_t *sizep) 1822 { 1823 struct pool *pp, *foundpool = NULL; 1824 size_t buflen = where != NULL ? *sizep : 0; 1825 int npools = 0, s; 1826 unsigned int lookfor; 1827 size_t len; 1828 1829 switch (*name) { 1830 case KERN_POOL_NPOOLS: 1831 if (namelen != 1 || buflen != sizeof(int)) 1832 return (EINVAL); 1833 lookfor = 0; 1834 break; 1835 case KERN_POOL_NAME: 1836 if (namelen != 2 || buflen < 1) 1837 return (EINVAL); 1838 lookfor = name[1]; 1839 break; 1840 case KERN_POOL_POOL: 1841 if (namelen != 2 || buflen != sizeof(struct pool)) 1842 return (EINVAL); 1843 lookfor = name[1]; 1844 break; 1845 default: 1846 return (EINVAL); 1847 } 1848 1849 s = splvm(); 1850 simple_lock(&pool_head_slock); 1851 1852 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 1853 npools++; 1854 if (lookfor == pp->pr_serial) { 1855 foundpool = pp; 1856 break; 1857 } 1858 } 1859 1860 simple_unlock(&pool_head_slock); 1861 splx(s); 1862 1863 if (lookfor != 0 && foundpool == NULL) 1864 return (ENOENT); 1865 1866 switch (*name) { 1867 case KERN_POOL_NPOOLS: 1868 return copyout(&npools, where, buflen); 1869 case KERN_POOL_NAME: 1870 len = strlen(foundpool->pr_wchan) + 1; 1871 if (*sizep < len) 1872 return (ENOMEM); 1873 *sizep = len; 1874 return copyout(foundpool->pr_wchan, where, len); 1875 case KERN_POOL_POOL: 1876 return copyout(foundpool, where, buflen); 1877 } 1878 /* NOTREACHED */ 1879 return (0); /* XXX - Stupid gcc */ 1880 } 1881