1 /* $OpenBSD: subr_pool.c,v 1.102 2011/04/05 01:28:05 art Exp $ */ 2 /* $NetBSD: subr_pool.c,v 1.61 2001/09/26 07:14:56 chs Exp $ */ 3 4 /*- 5 * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace 10 * Simulation Facility, NASA Ames Research Center. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/proc.h> 37 #include <sys/errno.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 #include <sys/pool.h> 41 #include <sys/syslog.h> 42 #include <sys/sysctl.h> 43 44 #include <uvm/uvm.h> 45 #include <dev/rndvar.h> 46 47 /* 48 * Pool resource management utility. 49 * 50 * Memory is allocated in pages which are split into pieces according to 51 * the pool item size. Each page is kept on one of three lists in the 52 * pool structure: `pr_emptypages', `pr_fullpages' and `pr_partpages', 53 * for empty, full and partially-full pages respectively. The individual 54 * pool items are on a linked list headed by `ph_itemlist' in each page 55 * header. The memory for building the page list is either taken from 56 * the allocated pages themselves (for small pool items) or taken from 57 * an internal pool of page headers (`phpool'). 58 */ 59 60 /* List of all pools */ 61 TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); 62 63 /* Private pool for page header structures */ 64 struct pool phpool; 65 66 struct pool_item_header { 67 /* Page headers */ 68 LIST_ENTRY(pool_item_header) 69 ph_pagelist; /* pool page list */ 70 TAILQ_HEAD(,pool_item) ph_itemlist; /* chunk list for this page */ 71 RB_ENTRY(pool_item_header) 72 ph_node; /* Off-page page headers */ 73 int ph_nmissing; /* # of chunks in use */ 74 caddr_t ph_page; /* this page's address */ 75 caddr_t ph_colored; /* page's colored address */ 76 int ph_pagesize; 77 int ph_magic; 78 }; 79 80 struct pool_item { 81 #ifdef DIAGNOSTIC 82 u_int32_t pi_magic; 83 #endif 84 /* Other entries use only this list entry */ 85 TAILQ_ENTRY(pool_item) pi_list; 86 }; 87 88 #ifdef DEADBEEF1 89 #define PI_MAGIC DEADBEEF1 90 #else 91 #define PI_MAGIC 0xdeafbeef 92 #endif 93 94 #ifdef POOL_DEBUG 95 int pool_debug = 1; 96 #else 97 int pool_debug = 0; 98 #endif 99 100 #define POOL_NEEDS_CATCHUP(pp) \ 101 ((pp)->pr_nitems < (pp)->pr_minitems) 102 103 /* 104 * Every pool gets a unique serial number assigned to it. If this counter 105 * wraps, we're screwed, but we shouldn't create so many pools anyway. 106 */ 107 unsigned int pool_serial; 108 109 int pool_catchup(struct pool *); 110 void pool_prime_page(struct pool *, caddr_t, struct pool_item_header *); 111 void pool_update_curpage(struct pool *); 112 void *pool_do_get(struct pool *, int); 113 void pool_do_put(struct pool *, void *); 114 void pr_rmpage(struct pool *, struct pool_item_header *, 115 struct pool_pagelist *); 116 int pool_chk_page(struct pool *, const char *, struct pool_item_header *); 117 struct pool_item_header *pool_alloc_item_header(struct pool *, caddr_t , int); 118 119 void *pool_allocator_alloc(struct pool *, int, int *); 120 void pool_allocator_free(struct pool *, void *); 121 122 /* 123 * XXX - quick hack. For pools with large items we want to use a special 124 * allocator. For now, instead of having the allocator figure out 125 * the allocation size from the pool (which can be done trivially 126 * with round_page(pr_itemsperpage * pr_size)) which would require 127 * lots of changes everywhere, we just create allocators for each 128 * size. We limit those to 128 pages. 129 */ 130 #define POOL_LARGE_MAXPAGES 128 131 struct pool_allocator pool_allocator_large[POOL_LARGE_MAXPAGES]; 132 struct pool_allocator pool_allocator_large_ni[POOL_LARGE_MAXPAGES]; 133 void *pool_large_alloc(struct pool *, int, int *); 134 void pool_large_free(struct pool *, void *); 135 void *pool_large_alloc_ni(struct pool *, int, int *); 136 void pool_large_free_ni(struct pool *, void *); 137 138 139 #ifdef DDB 140 void pool_print_pagelist(struct pool_pagelist *, 141 int (*)(const char *, ...)); 142 void pool_print1(struct pool *, const char *, int (*)(const char *, ...)); 143 #endif 144 145 #define pool_sleep(pl) msleep(pl, &pl->pr_mtx, PSWP, pl->pr_wchan, 0) 146 147 static __inline int 148 phtree_compare(struct pool_item_header *a, struct pool_item_header *b) 149 { 150 long diff = (vaddr_t)a->ph_page - (vaddr_t)b->ph_page; 151 if (diff < 0) 152 return -(-diff >= a->ph_pagesize); 153 else if (diff > 0) 154 return (diff >= b->ph_pagesize); 155 else 156 return (0); 157 } 158 159 RB_PROTOTYPE(phtree, pool_item_header, ph_node, phtree_compare); 160 RB_GENERATE(phtree, pool_item_header, ph_node, phtree_compare); 161 162 /* 163 * Return the pool page header based on page address. 164 */ 165 static __inline struct pool_item_header * 166 pr_find_pagehead(struct pool *pp, void *v) 167 { 168 struct pool_item_header *ph, tmp; 169 170 if ((pp->pr_roflags & PR_PHINPAGE) != 0) { 171 caddr_t page; 172 173 page = (caddr_t)((vaddr_t)v & pp->pr_alloc->pa_pagemask); 174 175 return ((struct pool_item_header *)(page + pp->pr_phoffset)); 176 } 177 178 /* 179 * The trick we're using in the tree compare function is to compare 180 * two elements equal when they overlap. We want to return the 181 * page header that belongs to the element just before this address. 182 * We don't want this element to compare equal to the next element, 183 * so the compare function takes the pagesize from the lower element. 184 * If this header is the lower, its pagesize is zero, so it can't 185 * overlap with the next header. But if the header we're looking for 186 * is lower, we'll use its pagesize and it will overlap and return 187 * equal. 188 */ 189 tmp.ph_page = v; 190 tmp.ph_pagesize = 0; 191 ph = RB_FIND(phtree, &pp->pr_phtree, &tmp); 192 193 if (ph) { 194 KASSERT(ph->ph_page <= (caddr_t)v); 195 KASSERT(ph->ph_page + ph->ph_pagesize > (caddr_t)v); 196 } 197 return ph; 198 } 199 200 /* 201 * Remove a page from the pool. 202 */ 203 void 204 pr_rmpage(struct pool *pp, struct pool_item_header *ph, 205 struct pool_pagelist *pq) 206 { 207 208 /* 209 * If the page was idle, decrement the idle page count. 210 */ 211 if (ph->ph_nmissing == 0) { 212 #ifdef DIAGNOSTIC 213 if (pp->pr_nidle == 0) 214 panic("pr_rmpage: nidle inconsistent"); 215 if (pp->pr_nitems < pp->pr_itemsperpage) 216 panic("pr_rmpage: nitems inconsistent"); 217 #endif 218 pp->pr_nidle--; 219 } 220 221 pp->pr_nitems -= pp->pr_itemsperpage; 222 223 /* 224 * Unlink a page from the pool and release it (or queue it for release). 225 */ 226 LIST_REMOVE(ph, ph_pagelist); 227 if ((pp->pr_roflags & PR_PHINPAGE) == 0) 228 RB_REMOVE(phtree, &pp->pr_phtree, ph); 229 if (pq) { 230 LIST_INSERT_HEAD(pq, ph, ph_pagelist); 231 } else { 232 pool_allocator_free(pp, ph->ph_page); 233 if ((pp->pr_roflags & PR_PHINPAGE) == 0) 234 pool_put(&phpool, ph); 235 } 236 pp->pr_npages--; 237 pp->pr_npagefree++; 238 239 pool_update_curpage(pp); 240 } 241 242 /* 243 * Initialize the given pool resource structure. 244 * 245 * We export this routine to allow other kernel parts to declare 246 * static pools that must be initialized before malloc() is available. 247 */ 248 void 249 pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, 250 const char *wchan, struct pool_allocator *palloc) 251 { 252 int off, slack; 253 254 #ifdef MALLOC_DEBUG 255 if ((flags & PR_DEBUG) && (ioff != 0 || align != 0)) 256 flags &= ~PR_DEBUG; 257 #endif 258 /* 259 * Check arguments and construct default values. 260 */ 261 if (palloc == NULL) { 262 if (size > PAGE_SIZE) { 263 int psize; 264 265 /* 266 * XXX - should take align into account as well. 267 */ 268 if (size == round_page(size)) 269 psize = size / PAGE_SIZE; 270 else 271 psize = PAGE_SIZE / roundup(size % PAGE_SIZE, 272 1024); 273 if (psize > POOL_LARGE_MAXPAGES) 274 psize = POOL_LARGE_MAXPAGES; 275 if (flags & PR_WAITOK) 276 palloc = &pool_allocator_large_ni[psize-1]; 277 else 278 palloc = &pool_allocator_large[psize-1]; 279 if (palloc->pa_pagesz == 0) { 280 palloc->pa_pagesz = psize * PAGE_SIZE; 281 if (flags & PR_WAITOK) { 282 palloc->pa_alloc = pool_large_alloc_ni; 283 palloc->pa_free = pool_large_free_ni; 284 } else { 285 palloc->pa_alloc = pool_large_alloc; 286 palloc->pa_free = pool_large_free; 287 } 288 } 289 } else { 290 palloc = &pool_allocator_nointr; 291 } 292 } 293 if (palloc->pa_pagesz == 0) { 294 palloc->pa_pagesz = PAGE_SIZE; 295 } 296 if (palloc->pa_pagemask == 0) { 297 palloc->pa_pagemask = ~(palloc->pa_pagesz - 1); 298 palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1; 299 } 300 301 if (align == 0) 302 align = ALIGN(1); 303 304 if (size < sizeof(struct pool_item)) 305 size = sizeof(struct pool_item); 306 307 size = roundup(size, align); 308 #ifdef DIAGNOSTIC 309 if (size > palloc->pa_pagesz) 310 panic("pool_init: pool item size (%lu) too large", 311 (u_long)size); 312 #endif 313 314 /* 315 * Initialize the pool structure. 316 */ 317 LIST_INIT(&pp->pr_emptypages); 318 LIST_INIT(&pp->pr_fullpages); 319 LIST_INIT(&pp->pr_partpages); 320 pp->pr_curpage = NULL; 321 pp->pr_npages = 0; 322 pp->pr_minitems = 0; 323 pp->pr_minpages = 0; 324 pp->pr_maxpages = 8; 325 pp->pr_roflags = flags; 326 pp->pr_flags = 0; 327 pp->pr_size = size; 328 pp->pr_align = align; 329 pp->pr_wchan = wchan; 330 pp->pr_alloc = palloc; 331 pp->pr_nitems = 0; 332 pp->pr_nout = 0; 333 pp->pr_hardlimit = UINT_MAX; 334 pp->pr_hardlimit_warning = NULL; 335 pp->pr_hardlimit_ratecap.tv_sec = 0; 336 pp->pr_hardlimit_ratecap.tv_usec = 0; 337 pp->pr_hardlimit_warning_last.tv_sec = 0; 338 pp->pr_hardlimit_warning_last.tv_usec = 0; 339 pp->pr_serial = ++pool_serial; 340 if (pool_serial == 0) 341 panic("pool_init: too much uptime"); 342 343 /* constructor, destructor, and arg */ 344 pp->pr_ctor = NULL; 345 pp->pr_dtor = NULL; 346 pp->pr_arg = NULL; 347 348 /* 349 * Decide whether to put the page header off page to avoid 350 * wasting too large a part of the page. Off-page page headers 351 * go into an RB tree, so we can match a returned item with 352 * its header based on the page address. 353 * We use 1/16 of the page size as the threshold (XXX: tune) 354 */ 355 if (pp->pr_size < palloc->pa_pagesz/16 && pp->pr_size < PAGE_SIZE) { 356 /* Use the end of the page for the page header */ 357 pp->pr_roflags |= PR_PHINPAGE; 358 pp->pr_phoffset = off = palloc->pa_pagesz - 359 ALIGN(sizeof(struct pool_item_header)); 360 } else { 361 /* The page header will be taken from our page header pool */ 362 pp->pr_phoffset = 0; 363 off = palloc->pa_pagesz; 364 RB_INIT(&pp->pr_phtree); 365 } 366 367 /* 368 * Alignment is to take place at `ioff' within the item. This means 369 * we must reserve up to `align - 1' bytes on the page to allow 370 * appropriate positioning of each item. 371 * 372 * Silently enforce `0 <= ioff < align'. 373 */ 374 pp->pr_itemoffset = ioff = ioff % align; 375 pp->pr_itemsperpage = (off - ((align - ioff) % align)) / pp->pr_size; 376 KASSERT(pp->pr_itemsperpage != 0); 377 378 /* 379 * Use the slack between the chunks and the page header 380 * for "cache coloring". 381 */ 382 slack = off - pp->pr_itemsperpage * pp->pr_size; 383 pp->pr_maxcolor = (slack / align) * align; 384 pp->pr_curcolor = 0; 385 386 pp->pr_nget = 0; 387 pp->pr_nfail = 0; 388 pp->pr_nput = 0; 389 pp->pr_npagealloc = 0; 390 pp->pr_npagefree = 0; 391 pp->pr_hiwat = 0; 392 pp->pr_nidle = 0; 393 394 pp->pr_ipl = -1; 395 mtx_init(&pp->pr_mtx, IPL_NONE); 396 397 if (phpool.pr_size == 0) { 398 pool_init(&phpool, sizeof(struct pool_item_header), 0, 0, 399 0, "phpool", NULL); 400 pool_setipl(&phpool, IPL_HIGH); 401 } 402 403 /* pglistalloc/constraint parameters */ 404 pp->pr_crange = &kp_dirty; 405 406 /* Insert this into the list of all pools. */ 407 TAILQ_INSERT_HEAD(&pool_head, pp, pr_poollist); 408 } 409 410 void 411 pool_setipl(struct pool *pp, int ipl) 412 { 413 pp->pr_ipl = ipl; 414 mtx_init(&pp->pr_mtx, ipl); 415 } 416 417 /* 418 * Decommission a pool resource. 419 */ 420 void 421 pool_destroy(struct pool *pp) 422 { 423 struct pool_item_header *ph; 424 425 #ifdef DIAGNOSTIC 426 if (pp->pr_nout != 0) 427 panic("pool_destroy: pool busy: still out: %u", pp->pr_nout); 428 #endif 429 430 /* Remove all pages */ 431 while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) 432 pr_rmpage(pp, ph, NULL); 433 KASSERT(LIST_EMPTY(&pp->pr_fullpages)); 434 KASSERT(LIST_EMPTY(&pp->pr_partpages)); 435 436 /* Remove from global pool list */ 437 TAILQ_REMOVE(&pool_head, pp, pr_poollist); 438 } 439 440 struct pool_item_header * 441 pool_alloc_item_header(struct pool *pp, caddr_t storage, int flags) 442 { 443 struct pool_item_header *ph; 444 445 if ((pp->pr_roflags & PR_PHINPAGE) != 0) 446 ph = (struct pool_item_header *)(storage + pp->pr_phoffset); 447 else 448 ph = pool_get(&phpool, (flags & ~(PR_WAITOK | PR_ZERO)) | 449 PR_NOWAIT); 450 if (pool_debug) 451 ph->ph_magic = PI_MAGIC; 452 return (ph); 453 } 454 455 /* 456 * Grab an item from the pool; must be called at appropriate spl level 457 */ 458 void * 459 pool_get(struct pool *pp, int flags) 460 { 461 void *v; 462 463 KASSERT(flags & (PR_WAITOK | PR_NOWAIT)); 464 465 #ifdef DIAGNOSTIC 466 if ((flags & PR_WAITOK) != 0) 467 assertwaitok(); 468 #endif /* DIAGNOSTIC */ 469 470 mtx_enter(&pp->pr_mtx); 471 v = pool_do_get(pp, flags); 472 mtx_leave(&pp->pr_mtx); 473 if (v == NULL) 474 return (v); 475 476 if (pp->pr_ctor) { 477 if (flags & PR_ZERO) 478 panic("pool_get: PR_ZERO when ctor set"); 479 if (pp->pr_ctor(pp->pr_arg, v, flags)) { 480 mtx_enter(&pp->pr_mtx); 481 pool_do_put(pp, v); 482 mtx_leave(&pp->pr_mtx); 483 v = NULL; 484 } 485 } else { 486 if (flags & PR_ZERO) 487 memset(v, 0, pp->pr_size); 488 } 489 if (v != NULL) 490 pp->pr_nget++; 491 return (v); 492 } 493 494 void * 495 pool_do_get(struct pool *pp, int flags) 496 { 497 struct pool_item *pi; 498 struct pool_item_header *ph; 499 void *v; 500 int slowdown = 0; 501 #if defined(DIAGNOSTIC) && defined(POOL_DEBUG) 502 int i, *ip; 503 #endif 504 505 #ifdef MALLOC_DEBUG 506 if (pp->pr_roflags & PR_DEBUG) { 507 void *addr; 508 509 addr = NULL; 510 debug_malloc(pp->pr_size, M_DEBUG, 511 (flags & PR_WAITOK) ? M_WAITOK : M_NOWAIT, &addr); 512 return (addr); 513 } 514 #endif 515 516 startover: 517 /* 518 * Check to see if we've reached the hard limit. If we have, 519 * and we can wait, then wait until an item has been returned to 520 * the pool. 521 */ 522 #ifdef DIAGNOSTIC 523 if (__predict_false(pp->pr_nout > pp->pr_hardlimit)) 524 panic("pool_do_get: %s: crossed hard limit", pp->pr_wchan); 525 #endif 526 if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) { 527 if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) { 528 /* 529 * XXX: A warning isn't logged in this case. Should 530 * it be? 531 */ 532 pp->pr_flags |= PR_WANTED; 533 pool_sleep(pp); 534 goto startover; 535 } 536 537 /* 538 * Log a message that the hard limit has been hit. 539 */ 540 if (pp->pr_hardlimit_warning != NULL && 541 ratecheck(&pp->pr_hardlimit_warning_last, 542 &pp->pr_hardlimit_ratecap)) 543 log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning); 544 545 pp->pr_nfail++; 546 return (NULL); 547 } 548 549 /* 550 * The convention we use is that if `curpage' is not NULL, then 551 * it points at a non-empty bucket. In particular, `curpage' 552 * never points at a page header which has PR_PHINPAGE set and 553 * has no items in its bucket. 554 */ 555 if ((ph = pp->pr_curpage) == NULL) { 556 #ifdef DIAGNOSTIC 557 if (pp->pr_nitems != 0) { 558 printf("pool_do_get: %s: curpage NULL, nitems %u\n", 559 pp->pr_wchan, pp->pr_nitems); 560 panic("pool_do_get: nitems inconsistent"); 561 } 562 #endif 563 564 /* 565 * Call the back-end page allocator for more memory. 566 */ 567 v = pool_allocator_alloc(pp, flags, &slowdown); 568 if (__predict_true(v != NULL)) 569 ph = pool_alloc_item_header(pp, v, flags); 570 571 if (__predict_false(v == NULL || ph == NULL)) { 572 if (v != NULL) 573 pool_allocator_free(pp, v); 574 575 if ((flags & PR_WAITOK) == 0) { 576 pp->pr_nfail++; 577 return (NULL); 578 } 579 580 /* 581 * Wait for items to be returned to this pool. 582 * 583 * XXX: maybe we should wake up once a second and 584 * try again? 585 */ 586 pp->pr_flags |= PR_WANTED; 587 pool_sleep(pp); 588 goto startover; 589 } 590 591 /* We have more memory; add it to the pool */ 592 pool_prime_page(pp, v, ph); 593 pp->pr_npagealloc++; 594 595 if (slowdown && (flags & PR_WAITOK)) { 596 mtx_leave(&pp->pr_mtx); 597 yield(); 598 mtx_enter(&pp->pr_mtx); 599 } 600 601 /* Start the allocation process over. */ 602 goto startover; 603 } 604 if (__predict_false((v = pi = TAILQ_FIRST(&ph->ph_itemlist)) == NULL)) { 605 panic("pool_do_get: %s: page empty", pp->pr_wchan); 606 } 607 #ifdef DIAGNOSTIC 608 if (__predict_false(pp->pr_nitems == 0)) { 609 printf("pool_do_get: %s: items on itemlist, nitems %u\n", 610 pp->pr_wchan, pp->pr_nitems); 611 panic("pool_do_get: nitems inconsistent"); 612 } 613 #endif 614 615 #ifdef DIAGNOSTIC 616 if (__predict_false(pi->pi_magic != PI_MAGIC)) 617 panic("pool_do_get(%s): free list modified: " 618 "page %p; item addr %p; offset 0x%x=0x%x", 619 pp->pr_wchan, ph->ph_page, pi, 0, pi->pi_magic); 620 #ifdef POOL_DEBUG 621 if (pool_debug && ph->ph_magic) { 622 for (ip = (int *)pi, i = sizeof(*pi) / sizeof(int); 623 i < pp->pr_size / sizeof(int); i++) { 624 if (ip[i] != ph->ph_magic) { 625 panic("pool_do_get(%s): free list modified: " 626 "page %p; item addr %p; offset 0x%x=0x%x", 627 pp->pr_wchan, ph->ph_page, pi, 628 i * sizeof(int), ip[i]); 629 } 630 } 631 } 632 #endif /* POOL_DEBUG */ 633 #endif /* DIAGNOSTIC */ 634 635 /* 636 * Remove from item list. 637 */ 638 TAILQ_REMOVE(&ph->ph_itemlist, pi, pi_list); 639 pp->pr_nitems--; 640 pp->pr_nout++; 641 if (ph->ph_nmissing == 0) { 642 #ifdef DIAGNOSTIC 643 if (__predict_false(pp->pr_nidle == 0)) 644 panic("pool_do_get: nidle inconsistent"); 645 #endif 646 pp->pr_nidle--; 647 648 /* 649 * This page was previously empty. Move it to the list of 650 * partially-full pages. This page is already curpage. 651 */ 652 LIST_REMOVE(ph, ph_pagelist); 653 LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); 654 } 655 ph->ph_nmissing++; 656 if (TAILQ_EMPTY(&ph->ph_itemlist)) { 657 #ifdef DIAGNOSTIC 658 if (__predict_false(ph->ph_nmissing != pp->pr_itemsperpage)) { 659 panic("pool_do_get: %s: nmissing inconsistent", 660 pp->pr_wchan); 661 } 662 #endif 663 /* 664 * This page is now full. Move it to the full list 665 * and select a new current page. 666 */ 667 LIST_REMOVE(ph, ph_pagelist); 668 LIST_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist); 669 pool_update_curpage(pp); 670 } 671 672 /* 673 * If we have a low water mark and we are now below that low 674 * water mark, add more items to the pool. 675 */ 676 if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) { 677 /* 678 * XXX: Should we log a warning? Should we set up a timeout 679 * to try again in a second or so? The latter could break 680 * a caller's assumptions about interrupt protection, etc. 681 */ 682 } 683 return (v); 684 } 685 686 /* 687 * Return resource to the pool; must be called at appropriate spl level 688 */ 689 void 690 pool_put(struct pool *pp, void *v) 691 { 692 if (pp->pr_dtor) 693 pp->pr_dtor(pp->pr_arg, v); 694 mtx_enter(&pp->pr_mtx); 695 pool_do_put(pp, v); 696 mtx_leave(&pp->pr_mtx); 697 pp->pr_nput++; 698 } 699 700 /* 701 * Internal version of pool_put(). 702 */ 703 void 704 pool_do_put(struct pool *pp, void *v) 705 { 706 struct pool_item *pi = v; 707 struct pool_item_header *ph; 708 #if defined(DIAGNOSTIC) && defined(POOL_DEBUG) 709 int i, *ip; 710 #endif 711 712 if (v == NULL) 713 panic("pool_put of NULL"); 714 715 #ifdef MALLOC_DEBUG 716 if (pp->pr_roflags & PR_DEBUG) { 717 debug_free(v, M_DEBUG); 718 return; 719 } 720 #endif 721 722 #ifdef DIAGNOSTIC 723 if (pp->pr_ipl != -1) 724 splassert(pp->pr_ipl); 725 726 if (__predict_false(pp->pr_nout == 0)) { 727 printf("pool %s: putting with none out\n", 728 pp->pr_wchan); 729 panic("pool_do_put"); 730 } 731 #endif 732 733 if (__predict_false((ph = pr_find_pagehead(pp, v)) == NULL)) { 734 panic("pool_do_put: %s: page header missing", pp->pr_wchan); 735 } 736 737 /* 738 * Return to item list. 739 */ 740 #ifdef DIAGNOSTIC 741 pi->pi_magic = PI_MAGIC; 742 #ifdef POOL_DEBUG 743 if (ph->ph_magic) { 744 for (ip = (int *)pi, i = sizeof(*pi)/sizeof(int); 745 i < pp->pr_size / sizeof(int); i++) 746 ip[i] = ph->ph_magic; 747 } 748 #endif /* POOL_DEBUG */ 749 #endif /* DIAGNOSTIC */ 750 751 TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); 752 ph->ph_nmissing--; 753 pp->pr_nitems++; 754 pp->pr_nout--; 755 756 /* Cancel "pool empty" condition if it exists */ 757 if (pp->pr_curpage == NULL) 758 pp->pr_curpage = ph; 759 760 if (pp->pr_flags & PR_WANTED) { 761 pp->pr_flags &= ~PR_WANTED; 762 if (ph->ph_nmissing == 0) 763 pp->pr_nidle++; 764 wakeup(pp); 765 return; 766 } 767 768 /* 769 * If this page is now empty, do one of two things: 770 * 771 * (1) If we have more pages than the page high water mark, 772 * free the page back to the system. 773 * 774 * (2) Otherwise, move the page to the empty page list. 775 * 776 * Either way, select a new current page (so we use a partially-full 777 * page if one is available). 778 */ 779 if (ph->ph_nmissing == 0) { 780 pp->pr_nidle++; 781 if (pp->pr_nidle > pp->pr_maxpages) { 782 pr_rmpage(pp, ph, NULL); 783 } else { 784 LIST_REMOVE(ph, ph_pagelist); 785 LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); 786 } 787 pool_update_curpage(pp); 788 } 789 790 /* 791 * If the page was previously completely full, move it to the 792 * partially-full list and make it the current page. The next 793 * allocation will get the item from this page, instead of 794 * further fragmenting the pool. 795 */ 796 else if (ph->ph_nmissing == (pp->pr_itemsperpage - 1)) { 797 LIST_REMOVE(ph, ph_pagelist); 798 LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); 799 pp->pr_curpage = ph; 800 } 801 } 802 803 /* 804 * Add N items to the pool. 805 */ 806 int 807 pool_prime(struct pool *pp, int n) 808 { 809 struct pool_item_header *ph; 810 caddr_t cp; 811 int newpages; 812 int slowdown; 813 814 mtx_enter(&pp->pr_mtx); 815 newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 816 817 while (newpages-- > 0) { 818 cp = pool_allocator_alloc(pp, PR_NOWAIT, &slowdown); 819 if (__predict_true(cp != NULL)) 820 ph = pool_alloc_item_header(pp, cp, PR_NOWAIT); 821 if (__predict_false(cp == NULL || ph == NULL)) { 822 if (cp != NULL) 823 pool_allocator_free(pp, cp); 824 break; 825 } 826 827 pool_prime_page(pp, cp, ph); 828 pp->pr_npagealloc++; 829 pp->pr_minpages++; 830 } 831 832 if (pp->pr_minpages >= pp->pr_maxpages) 833 pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ 834 835 mtx_leave(&pp->pr_mtx); 836 return (0); 837 } 838 839 /* 840 * Add a page worth of items to the pool. 841 * 842 * Note, we must be called with the pool descriptor LOCKED. 843 */ 844 void 845 pool_prime_page(struct pool *pp, caddr_t storage, struct pool_item_header *ph) 846 { 847 struct pool_item *pi; 848 caddr_t cp = storage; 849 unsigned int align = pp->pr_align; 850 unsigned int ioff = pp->pr_itemoffset; 851 int n; 852 #if defined(DIAGNOSTIC) && defined(POOL_DEBUG) 853 int i, *ip; 854 #endif 855 856 /* 857 * Insert page header. 858 */ 859 LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); 860 TAILQ_INIT(&ph->ph_itemlist); 861 ph->ph_page = storage; 862 ph->ph_pagesize = pp->pr_alloc->pa_pagesz; 863 ph->ph_nmissing = 0; 864 if ((pp->pr_roflags & PR_PHINPAGE) == 0) 865 RB_INSERT(phtree, &pp->pr_phtree, ph); 866 867 pp->pr_nidle++; 868 869 /* 870 * Color this page. 871 */ 872 cp = (caddr_t)(cp + pp->pr_curcolor); 873 if ((pp->pr_curcolor += align) > pp->pr_maxcolor) 874 pp->pr_curcolor = 0; 875 876 /* 877 * Adjust storage to apply alignment to `pr_itemoffset' in each item. 878 */ 879 if (ioff != 0) 880 cp = (caddr_t)(cp + (align - ioff)); 881 ph->ph_colored = cp; 882 883 /* 884 * Insert remaining chunks on the bucket list. 885 */ 886 n = pp->pr_itemsperpage; 887 pp->pr_nitems += n; 888 889 while (n--) { 890 pi = (struct pool_item *)cp; 891 892 KASSERT(((((vaddr_t)pi) + ioff) & (align - 1)) == 0); 893 894 /* Insert on page list */ 895 TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list); 896 897 #ifdef DIAGNOSTIC 898 pi->pi_magic = PI_MAGIC; 899 #ifdef POOL_DEBUG 900 if (ph->ph_magic) { 901 for (ip = (int *)pi, i = sizeof(*pi)/sizeof(int); 902 i < pp->pr_size / sizeof(int); i++) 903 ip[i] = ph->ph_magic; 904 } 905 #endif /* POOL_DEBUG */ 906 #endif /* DIAGNOSTIC */ 907 cp = (caddr_t)(cp + pp->pr_size); 908 } 909 910 /* 911 * If the pool was depleted, point at the new page. 912 */ 913 if (pp->pr_curpage == NULL) 914 pp->pr_curpage = ph; 915 916 if (++pp->pr_npages > pp->pr_hiwat) 917 pp->pr_hiwat = pp->pr_npages; 918 } 919 920 /* 921 * Used by pool_get() when nitems drops below the low water mark. This 922 * is used to catch up pr_nitems with the low water mark. 923 * 924 * Note we never wait for memory here, we let the caller decide what to do. 925 */ 926 int 927 pool_catchup(struct pool *pp) 928 { 929 struct pool_item_header *ph; 930 caddr_t cp; 931 int error = 0; 932 int slowdown; 933 934 while (POOL_NEEDS_CATCHUP(pp)) { 935 /* 936 * Call the page back-end allocator for more memory. 937 */ 938 cp = pool_allocator_alloc(pp, PR_NOWAIT, &slowdown); 939 if (__predict_true(cp != NULL)) 940 ph = pool_alloc_item_header(pp, cp, PR_NOWAIT); 941 if (__predict_false(cp == NULL || ph == NULL)) { 942 if (cp != NULL) 943 pool_allocator_free(pp, cp); 944 error = ENOMEM; 945 break; 946 } 947 pool_prime_page(pp, cp, ph); 948 pp->pr_npagealloc++; 949 } 950 951 return (error); 952 } 953 954 void 955 pool_update_curpage(struct pool *pp) 956 { 957 958 pp->pr_curpage = LIST_FIRST(&pp->pr_partpages); 959 if (pp->pr_curpage == NULL) { 960 pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages); 961 } 962 } 963 964 void 965 pool_setlowat(struct pool *pp, int n) 966 { 967 968 pp->pr_minitems = n; 969 pp->pr_minpages = (n == 0) 970 ? 0 971 : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 972 973 mtx_enter(&pp->pr_mtx); 974 /* Make sure we're caught up with the newly-set low water mark. */ 975 if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) { 976 /* 977 * XXX: Should we log a warning? Should we set up a timeout 978 * to try again in a second or so? The latter could break 979 * a caller's assumptions about interrupt protection, etc. 980 */ 981 } 982 mtx_leave(&pp->pr_mtx); 983 } 984 985 void 986 pool_sethiwat(struct pool *pp, int n) 987 { 988 989 pp->pr_maxpages = (n == 0) 990 ? 0 991 : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 992 } 993 994 int 995 pool_sethardlimit(struct pool *pp, u_int n, const char *warnmsg, int ratecap) 996 { 997 int error = 0; 998 999 if (n < pp->pr_nout) { 1000 error = EINVAL; 1001 goto done; 1002 } 1003 1004 pp->pr_hardlimit = n; 1005 pp->pr_hardlimit_warning = warnmsg; 1006 pp->pr_hardlimit_ratecap.tv_sec = ratecap; 1007 pp->pr_hardlimit_warning_last.tv_sec = 0; 1008 pp->pr_hardlimit_warning_last.tv_usec = 0; 1009 1010 done: 1011 return (error); 1012 } 1013 1014 void 1015 pool_set_constraints(struct pool *pp, struct kmem_pa_mode *mode) 1016 { 1017 pp->pr_crange = mode; 1018 } 1019 1020 void 1021 pool_set_ctordtor(struct pool *pp, int (*ctor)(void *, void *, int), 1022 void (*dtor)(void *, void *), void *arg) 1023 { 1024 pp->pr_ctor = ctor; 1025 pp->pr_dtor = dtor; 1026 pp->pr_arg = arg; 1027 } 1028 /* 1029 * Release all complete pages that have not been used recently. 1030 * 1031 * Returns non-zero if any pages have been reclaimed. 1032 */ 1033 int 1034 pool_reclaim(struct pool *pp) 1035 { 1036 struct pool_item_header *ph, *phnext; 1037 struct pool_pagelist pq; 1038 1039 LIST_INIT(&pq); 1040 1041 mtx_enter(&pp->pr_mtx); 1042 for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) { 1043 phnext = LIST_NEXT(ph, ph_pagelist); 1044 1045 /* Check our minimum page claim */ 1046 if (pp->pr_npages <= pp->pr_minpages) 1047 break; 1048 1049 KASSERT(ph->ph_nmissing == 0); 1050 1051 /* 1052 * If freeing this page would put us below 1053 * the low water mark, stop now. 1054 */ 1055 if ((pp->pr_nitems - pp->pr_itemsperpage) < 1056 pp->pr_minitems) 1057 break; 1058 1059 pr_rmpage(pp, ph, &pq); 1060 } 1061 mtx_leave(&pp->pr_mtx); 1062 1063 if (LIST_EMPTY(&pq)) 1064 return (0); 1065 while ((ph = LIST_FIRST(&pq)) != NULL) { 1066 LIST_REMOVE(ph, ph_pagelist); 1067 pool_allocator_free(pp, ph->ph_page); 1068 if (pp->pr_roflags & PR_PHINPAGE) 1069 continue; 1070 pool_put(&phpool, ph); 1071 } 1072 1073 return (1); 1074 } 1075 1076 /* 1077 * Release all complete pages that have not been used recently 1078 * from all pools. 1079 */ 1080 void 1081 pool_reclaim_all(void) 1082 { 1083 struct pool *pp; 1084 TAILQ_FOREACH(pp, &pool_head, pr_poollist) 1085 pool_reclaim(pp); 1086 } 1087 1088 #ifdef DDB 1089 #include <machine/db_machdep.h> 1090 #include <ddb/db_interface.h> 1091 #include <ddb/db_output.h> 1092 1093 /* 1094 * Diagnostic helpers. 1095 */ 1096 void 1097 pool_printit(struct pool *pp, const char *modif, int (*pr)(const char *, ...)) 1098 { 1099 pool_print1(pp, modif, pr); 1100 } 1101 1102 void 1103 pool_print_pagelist(struct pool_pagelist *pl, int (*pr)(const char *, ...)) 1104 { 1105 struct pool_item_header *ph; 1106 #ifdef DIAGNOSTIC 1107 struct pool_item *pi; 1108 #endif 1109 1110 LIST_FOREACH(ph, pl, ph_pagelist) { 1111 (*pr)("\t\tpage %p, nmissing %d\n", 1112 ph->ph_page, ph->ph_nmissing); 1113 #ifdef DIAGNOSTIC 1114 TAILQ_FOREACH(pi, &ph->ph_itemlist, pi_list) { 1115 if (pi->pi_magic != PI_MAGIC) { 1116 (*pr)("\t\t\titem %p, magic 0x%x\n", 1117 pi, pi->pi_magic); 1118 } 1119 } 1120 #endif 1121 } 1122 } 1123 1124 void 1125 pool_print1(struct pool *pp, const char *modif, int (*pr)(const char *, ...)) 1126 { 1127 struct pool_item_header *ph; 1128 int print_pagelist = 0; 1129 char c; 1130 1131 while ((c = *modif++) != '\0') { 1132 if (c == 'p') 1133 print_pagelist = 1; 1134 modif++; 1135 } 1136 1137 (*pr)("POOL %s: size %u, align %u, ioff %u, roflags 0x%08x\n", 1138 pp->pr_wchan, pp->pr_size, pp->pr_align, pp->pr_itemoffset, 1139 pp->pr_roflags); 1140 (*pr)("\talloc %p\n", pp->pr_alloc); 1141 (*pr)("\tminitems %u, minpages %u, maxpages %u, npages %u\n", 1142 pp->pr_minitems, pp->pr_minpages, pp->pr_maxpages, pp->pr_npages); 1143 (*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n", 1144 pp->pr_itemsperpage, pp->pr_nitems, pp->pr_nout, pp->pr_hardlimit); 1145 1146 (*pr)("\n\tnget %lu, nfail %lu, nput %lu\n", 1147 pp->pr_nget, pp->pr_nfail, pp->pr_nput); 1148 (*pr)("\tnpagealloc %lu, npagefree %lu, hiwat %u, nidle %lu\n", 1149 pp->pr_npagealloc, pp->pr_npagefree, pp->pr_hiwat, pp->pr_nidle); 1150 1151 if (print_pagelist == 0) 1152 return; 1153 1154 if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) 1155 (*pr)("\n\tempty page list:\n"); 1156 pool_print_pagelist(&pp->pr_emptypages, pr); 1157 if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL) 1158 (*pr)("\n\tfull page list:\n"); 1159 pool_print_pagelist(&pp->pr_fullpages, pr); 1160 if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL) 1161 (*pr)("\n\tpartial-page list:\n"); 1162 pool_print_pagelist(&pp->pr_partpages, pr); 1163 1164 if (pp->pr_curpage == NULL) 1165 (*pr)("\tno current page\n"); 1166 else 1167 (*pr)("\tcurpage %p\n", pp->pr_curpage->ph_page); 1168 } 1169 1170 void 1171 db_show_all_pools(db_expr_t expr, int haddr, db_expr_t count, char *modif) 1172 { 1173 struct pool *pp; 1174 char maxp[16]; 1175 int ovflw; 1176 char mode; 1177 1178 mode = modif[0]; 1179 if (mode != '\0' && mode != 'a') { 1180 db_printf("usage: show all pools [/a]\n"); 1181 return; 1182 } 1183 1184 if (mode == '\0') 1185 db_printf("%-10s%4s%9s%5s%9s%6s%6s%6s%6s%6s%6s%5s\n", 1186 "Name", 1187 "Size", 1188 "Requests", 1189 "Fail", 1190 "Releases", 1191 "Pgreq", 1192 "Pgrel", 1193 "Npage", 1194 "Hiwat", 1195 "Minpg", 1196 "Maxpg", 1197 "Idle"); 1198 else 1199 db_printf("%-10s %18s %18s\n", 1200 "Name", "Address", "Allocator"); 1201 1202 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 1203 if (mode == 'a') { 1204 db_printf("%-10s %18p %18p\n", pp->pr_wchan, pp, 1205 pp->pr_alloc); 1206 continue; 1207 } 1208 1209 if (!pp->pr_nget) 1210 continue; 1211 1212 if (pp->pr_maxpages == UINT_MAX) 1213 snprintf(maxp, sizeof maxp, "inf"); 1214 else 1215 snprintf(maxp, sizeof maxp, "%u", pp->pr_maxpages); 1216 1217 #define PRWORD(ovflw, fmt, width, fixed, val) do { \ 1218 (ovflw) += db_printf((fmt), \ 1219 (width) - (fixed) - (ovflw) > 0 ? \ 1220 (width) - (fixed) - (ovflw) : 0, \ 1221 (val)) - (width); \ 1222 if ((ovflw) < 0) \ 1223 (ovflw) = 0; \ 1224 } while (/* CONSTCOND */0) 1225 1226 ovflw = 0; 1227 PRWORD(ovflw, "%-*s", 10, 0, pp->pr_wchan); 1228 PRWORD(ovflw, " %*u", 4, 1, pp->pr_size); 1229 PRWORD(ovflw, " %*lu", 9, 1, pp->pr_nget); 1230 PRWORD(ovflw, " %*lu", 5, 1, pp->pr_nfail); 1231 PRWORD(ovflw, " %*lu", 9, 1, pp->pr_nput); 1232 PRWORD(ovflw, " %*lu", 6, 1, pp->pr_npagealloc); 1233 PRWORD(ovflw, " %*lu", 6, 1, pp->pr_npagefree); 1234 PRWORD(ovflw, " %*d", 6, 1, pp->pr_npages); 1235 PRWORD(ovflw, " %*d", 6, 1, pp->pr_hiwat); 1236 PRWORD(ovflw, " %*d", 6, 1, pp->pr_minpages); 1237 PRWORD(ovflw, " %*s", 6, 1, maxp); 1238 PRWORD(ovflw, " %*lu\n", 5, 1, pp->pr_nidle); 1239 1240 pool_chk(pp, pp->pr_wchan); 1241 } 1242 } 1243 1244 int 1245 pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph) 1246 { 1247 struct pool_item *pi; 1248 caddr_t page; 1249 int n; 1250 #if defined(DIAGNOSTIC) && defined(POOL_DEBUG) 1251 int i, *ip; 1252 #endif 1253 1254 page = (caddr_t)((u_long)ph & pp->pr_alloc->pa_pagemask); 1255 if (page != ph->ph_page && 1256 (pp->pr_roflags & PR_PHINPAGE) != 0) { 1257 if (label != NULL) 1258 printf("%s: ", label); 1259 printf("pool(%p:%s): page inconsistency: page %p; " 1260 "at page head addr %p (p %p)\n", 1261 pp, pp->pr_wchan, ph->ph_page, ph, page); 1262 return 1; 1263 } 1264 1265 for (pi = TAILQ_FIRST(&ph->ph_itemlist), n = 0; 1266 pi != NULL; 1267 pi = TAILQ_NEXT(pi,pi_list), n++) { 1268 1269 #ifdef DIAGNOSTIC 1270 if (pi->pi_magic != PI_MAGIC) { 1271 if (label != NULL) 1272 printf("%s: ", label); 1273 printf("pool(%s): free list modified: " 1274 "page %p; item ordinal %d; addr %p " 1275 "(p %p); offset 0x%x=0x%x\n", 1276 pp->pr_wchan, ph->ph_page, n, pi, page, 1277 0, pi->pi_magic); 1278 } 1279 #ifdef POOL_DEBUG 1280 if (pool_debug && ph->ph_magic) { 1281 for (ip = (int *)pi, i = sizeof(*pi) / sizeof(int); 1282 i < pp->pr_size / sizeof(int); i++) { 1283 if (ip[i] != ph->ph_magic) { 1284 printf("pool(%s): free list modified: " 1285 "page %p; item ordinal %d; addr %p " 1286 "(p %p); offset 0x%x=0x%x\n", 1287 pp->pr_wchan, ph->ph_page, n, pi, 1288 page, i * sizeof(int), ip[i]); 1289 } 1290 } 1291 } 1292 1293 #endif /* POOL_DEBUG */ 1294 #endif /* DIAGNOSTIC */ 1295 page = 1296 (caddr_t)((u_long)pi & pp->pr_alloc->pa_pagemask); 1297 if (page == ph->ph_page) 1298 continue; 1299 1300 if (label != NULL) 1301 printf("%s: ", label); 1302 printf("pool(%p:%s): page inconsistency: page %p;" 1303 " item ordinal %d; addr %p (p %p)\n", pp, 1304 pp->pr_wchan, ph->ph_page, n, pi, page); 1305 return 1; 1306 } 1307 return 0; 1308 } 1309 1310 int 1311 pool_chk(struct pool *pp, const char *label) 1312 { 1313 struct pool_item_header *ph; 1314 int r = 0; 1315 1316 LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) 1317 r += pool_chk_page(pp, label, ph); 1318 LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) 1319 r += pool_chk_page(pp, label, ph); 1320 LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) 1321 r += pool_chk_page(pp, label, ph); 1322 1323 return (r); 1324 } 1325 1326 void 1327 pool_walk(struct pool *pp, int full, int (*pr)(const char *, ...), 1328 void (*func)(void *, int, int (*)(const char *, ...))) 1329 { 1330 struct pool_item_header *ph; 1331 struct pool_item *pi; 1332 caddr_t cp; 1333 int n; 1334 1335 LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { 1336 cp = ph->ph_colored; 1337 n = ph->ph_nmissing; 1338 1339 while (n--) { 1340 func(cp, full, pr); 1341 cp += pp->pr_size; 1342 } 1343 } 1344 1345 LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { 1346 cp = ph->ph_colored; 1347 n = ph->ph_nmissing; 1348 1349 do { 1350 TAILQ_FOREACH(pi, &ph->ph_itemlist, pi_list) { 1351 if (cp == (caddr_t)pi) 1352 break; 1353 } 1354 if (cp != (caddr_t)pi) { 1355 func(cp, full, pr); 1356 n--; 1357 } 1358 1359 cp += pp->pr_size; 1360 } while (n > 0); 1361 } 1362 } 1363 #endif 1364 1365 /* 1366 * We have three different sysctls. 1367 * kern.pool.npools - the number of pools. 1368 * kern.pool.pool.<pool#> - the pool struct for the pool#. 1369 * kern.pool.name.<pool#> - the name for pool#. 1370 */ 1371 int 1372 sysctl_dopool(int *name, u_int namelen, char *where, size_t *sizep) 1373 { 1374 struct pool *pp, *foundpool = NULL; 1375 size_t buflen = where != NULL ? *sizep : 0; 1376 int npools = 0, s; 1377 unsigned int lookfor; 1378 size_t len; 1379 1380 switch (*name) { 1381 case KERN_POOL_NPOOLS: 1382 if (namelen != 1 || buflen != sizeof(int)) 1383 return (EINVAL); 1384 lookfor = 0; 1385 break; 1386 case KERN_POOL_NAME: 1387 if (namelen != 2 || buflen < 1) 1388 return (EINVAL); 1389 lookfor = name[1]; 1390 break; 1391 case KERN_POOL_POOL: 1392 if (namelen != 2 || buflen != sizeof(struct pool)) 1393 return (EINVAL); 1394 lookfor = name[1]; 1395 break; 1396 default: 1397 return (EINVAL); 1398 } 1399 1400 s = splvm(); 1401 1402 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 1403 npools++; 1404 if (lookfor == pp->pr_serial) { 1405 foundpool = pp; 1406 break; 1407 } 1408 } 1409 1410 splx(s); 1411 1412 if (*name != KERN_POOL_NPOOLS && foundpool == NULL) 1413 return (ENOENT); 1414 1415 switch (*name) { 1416 case KERN_POOL_NPOOLS: 1417 return copyout(&npools, where, buflen); 1418 case KERN_POOL_NAME: 1419 len = strlen(foundpool->pr_wchan) + 1; 1420 if (*sizep < len) 1421 return (ENOMEM); 1422 *sizep = len; 1423 return copyout(foundpool->pr_wchan, where, len); 1424 case KERN_POOL_POOL: 1425 return copyout(foundpool, where, buflen); 1426 } 1427 /* NOTREACHED */ 1428 return (0); /* XXX - Stupid gcc */ 1429 } 1430 1431 /* 1432 * Pool backend allocators. 1433 * 1434 * Each pool has a backend allocator that handles allocation, deallocation 1435 */ 1436 void *pool_page_alloc(struct pool *, int, int *); 1437 void pool_page_free(struct pool *, void *); 1438 1439 /* 1440 * safe for interrupts, name preserved for compat this is the default 1441 * allocator 1442 */ 1443 struct pool_allocator pool_allocator_nointr = { 1444 pool_page_alloc, pool_page_free, 0, 1445 }; 1446 1447 /* 1448 * XXX - we have at least three different resources for the same allocation 1449 * and each resource can be depleted. First we have the ready elements in 1450 * the pool. Then we have the resource (typically a vm_map) for this 1451 * allocator, then we have physical memory. Waiting for any of these can 1452 * be unnecessary when any other is freed, but the kernel doesn't support 1453 * sleeping on multiple addresses, so we have to fake. The caller sleeps on 1454 * the pool (so that we can be awakened when an item is returned to the pool), 1455 * but we set PA_WANT on the allocator. When a page is returned to 1456 * the allocator and PA_WANT is set pool_allocator_free will wakeup all 1457 * sleeping pools belonging to this allocator. (XXX - thundering herd). 1458 * We also wake up the allocator in case someone without a pool (malloc) 1459 * is sleeping waiting for this allocator. 1460 */ 1461 1462 void * 1463 pool_allocator_alloc(struct pool *pp, int flags, int *slowdown) 1464 { 1465 boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; 1466 void *v; 1467 1468 if (waitok) 1469 mtx_leave(&pp->pr_mtx); 1470 v = pp->pr_alloc->pa_alloc(pp, flags, slowdown); 1471 if (waitok) 1472 mtx_enter(&pp->pr_mtx); 1473 1474 return (v); 1475 } 1476 1477 void 1478 pool_allocator_free(struct pool *pp, void *v) 1479 { 1480 struct pool_allocator *pa = pp->pr_alloc; 1481 1482 (*pa->pa_free)(pp, v); 1483 } 1484 1485 void * 1486 pool_page_alloc(struct pool *pp, int flags, int *slowdown) 1487 { 1488 struct kmem_dyn_mode kd = KMEM_DYN_INITIALIZER; 1489 1490 kd.kd_waitok = (flags & PR_WAITOK); 1491 kd.kd_slowdown = slowdown; 1492 1493 return (km_alloc(PAGE_SIZE, &kv_page, pp->pr_crange, &kd)); 1494 } 1495 1496 void 1497 pool_page_free(struct pool *pp, void *v) 1498 { 1499 km_free(v, PAGE_SIZE, &kv_page, pp->pr_crange); 1500 } 1501 1502 void * 1503 pool_large_alloc(struct pool *pp, int flags, int *slowdown) 1504 { 1505 struct kmem_dyn_mode kd = KMEM_DYN_INITIALIZER; 1506 void *v; 1507 int s; 1508 1509 kd.kd_waitok = (flags & PR_WAITOK); 1510 kd.kd_slowdown = slowdown; 1511 1512 s = splvm(); 1513 v = km_alloc(pp->pr_alloc->pa_pagesz, &kv_intrsafe, pp->pr_crange, 1514 &kd); 1515 splx(s); 1516 1517 return (v); 1518 } 1519 1520 void 1521 pool_large_free(struct pool *pp, void *v) 1522 { 1523 int s; 1524 1525 s = splvm(); 1526 km_free(v, pp->pr_alloc->pa_pagesz, &kv_intrsafe, pp->pr_crange); 1527 splx(s); 1528 } 1529 1530 void * 1531 pool_large_alloc_ni(struct pool *pp, int flags, int *slowdown) 1532 { 1533 struct kmem_dyn_mode kd = KMEM_DYN_INITIALIZER; 1534 1535 kd.kd_waitok = (flags & PR_WAITOK); 1536 kd.kd_slowdown = slowdown; 1537 1538 return (km_alloc(pp->pr_alloc->pa_pagesz, &kv_any, pp->pr_crange, &kd)); 1539 } 1540 1541 void 1542 pool_large_free_ni(struct pool *pp, void *v) 1543 { 1544 km_free(v, pp->pr_alloc->pa_pagesz, &kv_any, pp->pr_crange); 1545 } 1546