1 /* $NetBSD: subr_vmem.c,v 1.35 2007/11/07 00:23:23 ad Exp $ */ 2 3 /*- 4 * Copyright (c)2006 YAMAMOTO Takashi, 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * reference: 31 * - Magazines and Vmem: Extending the Slab Allocator 32 * to Many CPUs and Arbitrary Resources 33 * http://www.usenix.org/event/usenix01/bonwick.html 34 * 35 * todo: 36 * - decide how to import segments for vmem_xalloc. 37 * - don't rely on malloc(9). 38 */ 39 40 #include <sys/cdefs.h> 41 __KERNEL_RCSID(0, "$NetBSD: subr_vmem.c,v 1.35 2007/11/07 00:23:23 ad Exp $"); 42 43 #define VMEM_DEBUG 44 #if defined(_KERNEL) 45 #define QCACHE 46 #endif /* defined(_KERNEL) */ 47 48 #include <sys/param.h> 49 #include <sys/hash.h> 50 #include <sys/queue.h> 51 52 #if defined(_KERNEL) 53 #include <sys/systm.h> 54 #include <sys/kernel.h> /* hz */ 55 #include <sys/callout.h> 56 #include <sys/lock.h> 57 #include <sys/malloc.h> 58 #include <sys/once.h> 59 #include <sys/pool.h> 60 #include <sys/proc.h> 61 #include <sys/vmem.h> 62 #include <sys/workqueue.h> 63 #else /* defined(_KERNEL) */ 64 #include "../sys/vmem.h" 65 #endif /* defined(_KERNEL) */ 66 67 #if defined(_KERNEL) 68 #define LOCK_DECL(name) kmutex_t name 69 #else /* defined(_KERNEL) */ 70 #include <errno.h> 71 #include <assert.h> 72 #include <stdlib.h> 73 74 #define KASSERT(a) assert(a) 75 #define LOCK_DECL(name) /* nothing */ 76 #define mutex_init(a, b, c) /* nothing */ 77 #define mutex_destroy(a) /* nothing */ 78 #define mutex_enter(a) /* nothing */ 79 #define mutex_exit(a) /* nothing */ 80 #define mutex_owned(a) /* nothing */ 81 #define ASSERT_SLEEPABLE(lk, msg) /* nothing */ 82 #define IPL_VM 0 83 #endif /* defined(_KERNEL) */ 84 85 struct vmem; 86 struct vmem_btag; 87 88 #if defined(VMEM_DEBUG) 89 void vmem_dump(const vmem_t *); 90 #endif /* defined(VMEM_DEBUG) */ 91 92 #define VMEM_MAXORDER (sizeof(vmem_size_t) * CHAR_BIT) 93 94 #define VMEM_HASHSIZE_MIN 1 /* XXX */ 95 #define VMEM_HASHSIZE_MAX 8192 /* XXX */ 96 #define VMEM_HASHSIZE_INIT VMEM_HASHSIZE_MIN 97 98 #define VM_FITMASK (VM_BESTFIT | VM_INSTANTFIT) 99 100 CIRCLEQ_HEAD(vmem_seglist, vmem_btag); 101 LIST_HEAD(vmem_freelist, vmem_btag); 102 LIST_HEAD(vmem_hashlist, vmem_btag); 103 104 #if defined(QCACHE) 105 #define VMEM_QCACHE_IDX_MAX 32 106 107 #define QC_NAME_MAX 16 108 109 struct qcache { 110 pool_cache_t qc_cache; 111 vmem_t *qc_vmem; 112 char qc_name[QC_NAME_MAX]; 113 }; 114 typedef struct qcache qcache_t; 115 #define QC_POOL_TO_QCACHE(pool) ((qcache_t *)(pool->pr_qcache)) 116 #endif /* defined(QCACHE) */ 117 118 /* vmem arena */ 119 struct vmem { 120 LOCK_DECL(vm_lock); 121 vmem_addr_t (*vm_allocfn)(vmem_t *, vmem_size_t, vmem_size_t *, 122 vm_flag_t); 123 void (*vm_freefn)(vmem_t *, vmem_addr_t, vmem_size_t); 124 vmem_t *vm_source; 125 struct vmem_seglist vm_seglist; 126 struct vmem_freelist vm_freelist[VMEM_MAXORDER]; 127 size_t vm_hashsize; 128 size_t vm_nbusytag; 129 struct vmem_hashlist *vm_hashlist; 130 size_t vm_quantum_mask; 131 int vm_quantum_shift; 132 const char *vm_name; 133 LIST_ENTRY(vmem) vm_alllist; 134 135 #if defined(QCACHE) 136 /* quantum cache */ 137 size_t vm_qcache_max; 138 struct pool_allocator vm_qcache_allocator; 139 qcache_t vm_qcache_store[VMEM_QCACHE_IDX_MAX]; 140 qcache_t *vm_qcache[VMEM_QCACHE_IDX_MAX]; 141 #endif /* defined(QCACHE) */ 142 }; 143 144 #define VMEM_LOCK(vm) mutex_enter(&vm->vm_lock) 145 #define VMEM_TRYLOCK(vm) mutex_tryenter(&vm->vm_lock) 146 #define VMEM_UNLOCK(vm) mutex_exit(&vm->vm_lock) 147 #define VMEM_LOCK_INIT(vm, ipl) mutex_init(&vm->vm_lock, MUTEX_DRIVER, ipl) 148 #define VMEM_LOCK_DESTROY(vm) mutex_destroy(&vm->vm_lock) 149 #define VMEM_ASSERT_LOCKED(vm) KASSERT(mutex_owned(&vm->vm_lock)) 150 151 /* boundary tag */ 152 struct vmem_btag { 153 CIRCLEQ_ENTRY(vmem_btag) bt_seglist; 154 union { 155 LIST_ENTRY(vmem_btag) u_freelist; /* BT_TYPE_FREE */ 156 LIST_ENTRY(vmem_btag) u_hashlist; /* BT_TYPE_BUSY */ 157 } bt_u; 158 #define bt_hashlist bt_u.u_hashlist 159 #define bt_freelist bt_u.u_freelist 160 vmem_addr_t bt_start; 161 vmem_size_t bt_size; 162 int bt_type; 163 }; 164 165 #define BT_TYPE_SPAN 1 166 #define BT_TYPE_SPAN_STATIC 2 167 #define BT_TYPE_FREE 3 168 #define BT_TYPE_BUSY 4 169 #define BT_ISSPAN_P(bt) ((bt)->bt_type <= BT_TYPE_SPAN_STATIC) 170 171 #define BT_END(bt) ((bt)->bt_start + (bt)->bt_size) 172 173 typedef struct vmem_btag bt_t; 174 175 /* ---- misc */ 176 177 #define VMEM_ALIGNUP(addr, align) \ 178 (-(-(addr) & -(align))) 179 #define VMEM_CROSS_P(addr1, addr2, boundary) \ 180 ((((addr1) ^ (addr2)) & -(boundary)) != 0) 181 182 #define ORDER2SIZE(order) ((vmem_size_t)1 << (order)) 183 184 static int 185 calc_order(vmem_size_t size) 186 { 187 vmem_size_t target; 188 int i; 189 190 KASSERT(size != 0); 191 192 i = 0; 193 target = size >> 1; 194 while (ORDER2SIZE(i) <= target) { 195 i++; 196 } 197 198 KASSERT(ORDER2SIZE(i) <= size); 199 KASSERT(size < ORDER2SIZE(i + 1) || ORDER2SIZE(i + 1) < ORDER2SIZE(i)); 200 201 return i; 202 } 203 204 #if defined(_KERNEL) 205 static MALLOC_DEFINE(M_VMEM, "vmem", "vmem"); 206 #endif /* defined(_KERNEL) */ 207 208 static void * 209 xmalloc(size_t sz, vm_flag_t flags) 210 { 211 212 #if defined(_KERNEL) 213 return malloc(sz, M_VMEM, 214 M_CANFAIL | ((flags & VM_SLEEP) ? M_WAITOK : M_NOWAIT)); 215 #else /* defined(_KERNEL) */ 216 return malloc(sz); 217 #endif /* defined(_KERNEL) */ 218 } 219 220 static void 221 xfree(void *p) 222 { 223 224 #if defined(_KERNEL) 225 return free(p, M_VMEM); 226 #else /* defined(_KERNEL) */ 227 return free(p); 228 #endif /* defined(_KERNEL) */ 229 } 230 231 /* ---- boundary tag */ 232 233 #if defined(_KERNEL) 234 static struct pool_cache bt_cache; 235 #endif /* defined(_KERNEL) */ 236 237 static bt_t * 238 bt_alloc(vmem_t *vm, vm_flag_t flags) 239 { 240 bt_t *bt; 241 242 #if defined(_KERNEL) 243 bt = pool_cache_get(&bt_cache, 244 (flags & VM_SLEEP) != 0 ? PR_WAITOK : PR_NOWAIT); 245 #else /* defined(_KERNEL) */ 246 bt = malloc(sizeof *bt); 247 #endif /* defined(_KERNEL) */ 248 249 return bt; 250 } 251 252 static void 253 bt_free(vmem_t *vm, bt_t *bt) 254 { 255 256 #if defined(_KERNEL) 257 pool_cache_put(&bt_cache, bt); 258 #else /* defined(_KERNEL) */ 259 free(bt); 260 #endif /* defined(_KERNEL) */ 261 } 262 263 /* 264 * freelist[0] ... [1, 1] 265 * freelist[1] ... [2, 3] 266 * freelist[2] ... [4, 7] 267 * freelist[3] ... [8, 15] 268 * : 269 * freelist[n] ... [(1 << n), (1 << (n + 1)) - 1] 270 * : 271 */ 272 273 static struct vmem_freelist * 274 bt_freehead_tofree(vmem_t *vm, vmem_size_t size) 275 { 276 const vmem_size_t qsize = size >> vm->vm_quantum_shift; 277 int idx; 278 279 KASSERT((size & vm->vm_quantum_mask) == 0); 280 KASSERT(size != 0); 281 282 idx = calc_order(qsize); 283 KASSERT(idx >= 0); 284 KASSERT(idx < VMEM_MAXORDER); 285 286 return &vm->vm_freelist[idx]; 287 } 288 289 static struct vmem_freelist * 290 bt_freehead_toalloc(vmem_t *vm, vmem_size_t size, vm_flag_t strat) 291 { 292 const vmem_size_t qsize = size >> vm->vm_quantum_shift; 293 int idx; 294 295 KASSERT((size & vm->vm_quantum_mask) == 0); 296 KASSERT(size != 0); 297 298 idx = calc_order(qsize); 299 if (strat == VM_INSTANTFIT && ORDER2SIZE(idx) != qsize) { 300 idx++; 301 /* check too large request? */ 302 } 303 KASSERT(idx >= 0); 304 KASSERT(idx < VMEM_MAXORDER); 305 306 return &vm->vm_freelist[idx]; 307 } 308 309 /* ---- boundary tag hash */ 310 311 static struct vmem_hashlist * 312 bt_hashhead(vmem_t *vm, vmem_addr_t addr) 313 { 314 struct vmem_hashlist *list; 315 unsigned int hash; 316 317 hash = hash32_buf(&addr, sizeof(addr), HASH32_BUF_INIT); 318 list = &vm->vm_hashlist[hash % vm->vm_hashsize]; 319 320 return list; 321 } 322 323 static bt_t * 324 bt_lookupbusy(vmem_t *vm, vmem_addr_t addr) 325 { 326 struct vmem_hashlist *list; 327 bt_t *bt; 328 329 list = bt_hashhead(vm, addr); 330 LIST_FOREACH(bt, list, bt_hashlist) { 331 if (bt->bt_start == addr) { 332 break; 333 } 334 } 335 336 return bt; 337 } 338 339 static void 340 bt_rembusy(vmem_t *vm, bt_t *bt) 341 { 342 343 KASSERT(vm->vm_nbusytag > 0); 344 vm->vm_nbusytag--; 345 LIST_REMOVE(bt, bt_hashlist); 346 } 347 348 static void 349 bt_insbusy(vmem_t *vm, bt_t *bt) 350 { 351 struct vmem_hashlist *list; 352 353 KASSERT(bt->bt_type == BT_TYPE_BUSY); 354 355 list = bt_hashhead(vm, bt->bt_start); 356 LIST_INSERT_HEAD(list, bt, bt_hashlist); 357 vm->vm_nbusytag++; 358 } 359 360 /* ---- boundary tag list */ 361 362 static void 363 bt_remseg(vmem_t *vm, bt_t *bt) 364 { 365 366 CIRCLEQ_REMOVE(&vm->vm_seglist, bt, bt_seglist); 367 } 368 369 static void 370 bt_insseg(vmem_t *vm, bt_t *bt, bt_t *prev) 371 { 372 373 CIRCLEQ_INSERT_AFTER(&vm->vm_seglist, prev, bt, bt_seglist); 374 } 375 376 static void 377 bt_insseg_tail(vmem_t *vm, bt_t *bt) 378 { 379 380 CIRCLEQ_INSERT_TAIL(&vm->vm_seglist, bt, bt_seglist); 381 } 382 383 static void 384 bt_remfree(vmem_t *vm, bt_t *bt) 385 { 386 387 KASSERT(bt->bt_type == BT_TYPE_FREE); 388 389 LIST_REMOVE(bt, bt_freelist); 390 } 391 392 static void 393 bt_insfree(vmem_t *vm, bt_t *bt) 394 { 395 struct vmem_freelist *list; 396 397 list = bt_freehead_tofree(vm, bt->bt_size); 398 LIST_INSERT_HEAD(list, bt, bt_freelist); 399 } 400 401 /* ---- vmem internal functions */ 402 403 #if defined(_KERNEL) 404 static kmutex_t vmem_list_lock; 405 static LIST_HEAD(, vmem) vmem_list = LIST_HEAD_INITIALIZER(vmem_list); 406 #endif /* defined(_KERNEL) */ 407 408 #if defined(QCACHE) 409 static inline vm_flag_t 410 prf_to_vmf(int prflags) 411 { 412 vm_flag_t vmflags; 413 414 KASSERT((prflags & ~(PR_LIMITFAIL | PR_WAITOK | PR_NOWAIT)) == 0); 415 if ((prflags & PR_WAITOK) != 0) { 416 vmflags = VM_SLEEP; 417 } else { 418 vmflags = VM_NOSLEEP; 419 } 420 return vmflags; 421 } 422 423 static inline int 424 vmf_to_prf(vm_flag_t vmflags) 425 { 426 int prflags; 427 428 if ((vmflags & VM_SLEEP) != 0) { 429 prflags = PR_WAITOK; 430 } else { 431 prflags = PR_NOWAIT; 432 } 433 return prflags; 434 } 435 436 static size_t 437 qc_poolpage_size(size_t qcache_max) 438 { 439 int i; 440 441 for (i = 0; ORDER2SIZE(i) <= qcache_max * 3; i++) { 442 /* nothing */ 443 } 444 return ORDER2SIZE(i); 445 } 446 447 static void * 448 qc_poolpage_alloc(struct pool *pool, int prflags) 449 { 450 qcache_t *qc = QC_POOL_TO_QCACHE(pool); 451 vmem_t *vm = qc->qc_vmem; 452 453 return (void *)vmem_alloc(vm, pool->pr_alloc->pa_pagesz, 454 prf_to_vmf(prflags) | VM_INSTANTFIT); 455 } 456 457 static void 458 qc_poolpage_free(struct pool *pool, void *addr) 459 { 460 qcache_t *qc = QC_POOL_TO_QCACHE(pool); 461 vmem_t *vm = qc->qc_vmem; 462 463 vmem_free(vm, (vmem_addr_t)addr, pool->pr_alloc->pa_pagesz); 464 } 465 466 static void 467 qc_init(vmem_t *vm, size_t qcache_max, int ipl) 468 { 469 qcache_t *prevqc; 470 struct pool_allocator *pa; 471 int qcache_idx_max; 472 int i; 473 474 KASSERT((qcache_max & vm->vm_quantum_mask) == 0); 475 if (qcache_max > (VMEM_QCACHE_IDX_MAX << vm->vm_quantum_shift)) { 476 qcache_max = VMEM_QCACHE_IDX_MAX << vm->vm_quantum_shift; 477 } 478 vm->vm_qcache_max = qcache_max; 479 pa = &vm->vm_qcache_allocator; 480 memset(pa, 0, sizeof(*pa)); 481 pa->pa_alloc = qc_poolpage_alloc; 482 pa->pa_free = qc_poolpage_free; 483 pa->pa_pagesz = qc_poolpage_size(qcache_max); 484 485 qcache_idx_max = qcache_max >> vm->vm_quantum_shift; 486 prevqc = NULL; 487 for (i = qcache_idx_max; i > 0; i--) { 488 qcache_t *qc = &vm->vm_qcache_store[i - 1]; 489 size_t size = i << vm->vm_quantum_shift; 490 491 qc->qc_vmem = vm; 492 snprintf(qc->qc_name, sizeof(qc->qc_name), "%s-%zu", 493 vm->vm_name, size); 494 qc->qc_cache = pool_cache_init(size, 495 ORDER2SIZE(vm->vm_quantum_shift), 0, 496 PR_NOALIGN | PR_NOTOUCH /* XXX */, 497 qc->qc_name, pa, ipl, NULL, NULL, NULL); 498 KASSERT(qc->qc_cache != NULL); /* XXX */ 499 if (prevqc != NULL && 500 qc->qc_cache->pc_pool.pr_itemsperpage == 501 prevqc->qc_cache->pc_pool.pr_itemsperpage) { 502 pool_cache_destroy(qc->qc_cache); 503 vm->vm_qcache[i - 1] = prevqc; 504 continue; 505 } 506 qc->qc_cache->pc_pool.pr_qcache = qc; 507 vm->vm_qcache[i - 1] = qc; 508 prevqc = qc; 509 } 510 } 511 512 static void 513 qc_destroy(vmem_t *vm) 514 { 515 const qcache_t *prevqc; 516 int i; 517 int qcache_idx_max; 518 519 qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift; 520 prevqc = NULL; 521 for (i = 0; i < qcache_idx_max; i++) { 522 qcache_t *qc = vm->vm_qcache[i]; 523 524 if (prevqc == qc) { 525 continue; 526 } 527 pool_cache_destroy(qc->qc_cache); 528 prevqc = qc; 529 } 530 } 531 532 static bool 533 qc_reap(vmem_t *vm) 534 { 535 const qcache_t *prevqc; 536 int i; 537 int qcache_idx_max; 538 bool didsomething = false; 539 540 qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift; 541 prevqc = NULL; 542 for (i = 0; i < qcache_idx_max; i++) { 543 qcache_t *qc = vm->vm_qcache[i]; 544 545 if (prevqc == qc) { 546 continue; 547 } 548 if (pool_cache_reclaim(qc->qc_cache) != 0) { 549 didsomething = true; 550 } 551 prevqc = qc; 552 } 553 554 return didsomething; 555 } 556 #endif /* defined(QCACHE) */ 557 558 #if defined(_KERNEL) 559 static int 560 vmem_init(void) 561 { 562 563 mutex_init(&vmem_list_lock, MUTEX_DEFAULT, IPL_NONE); 564 pool_cache_bootstrap(&bt_cache, sizeof(bt_t), 0, 0, 0, "vmembt", 565 NULL, IPL_VM, NULL, NULL, NULL); 566 return 0; 567 } 568 #endif /* defined(_KERNEL) */ 569 570 static vmem_addr_t 571 vmem_add1(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, vm_flag_t flags, 572 int spanbttype) 573 { 574 bt_t *btspan; 575 bt_t *btfree; 576 577 KASSERT((flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 578 KASSERT((~flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 579 580 btspan = bt_alloc(vm, flags); 581 if (btspan == NULL) { 582 return VMEM_ADDR_NULL; 583 } 584 btfree = bt_alloc(vm, flags); 585 if (btfree == NULL) { 586 bt_free(vm, btspan); 587 return VMEM_ADDR_NULL; 588 } 589 590 btspan->bt_type = spanbttype; 591 btspan->bt_start = addr; 592 btspan->bt_size = size; 593 594 btfree->bt_type = BT_TYPE_FREE; 595 btfree->bt_start = addr; 596 btfree->bt_size = size; 597 598 VMEM_LOCK(vm); 599 bt_insseg_tail(vm, btspan); 600 bt_insseg(vm, btfree, btspan); 601 bt_insfree(vm, btfree); 602 VMEM_UNLOCK(vm); 603 604 return addr; 605 } 606 607 static void 608 vmem_destroy1(vmem_t *vm) 609 { 610 611 #if defined(QCACHE) 612 qc_destroy(vm); 613 #endif /* defined(QCACHE) */ 614 if (vm->vm_hashlist != NULL) { 615 int i; 616 617 for (i = 0; i < vm->vm_hashsize; i++) { 618 bt_t *bt; 619 620 while ((bt = LIST_FIRST(&vm->vm_hashlist[i])) != NULL) { 621 KASSERT(bt->bt_type == BT_TYPE_SPAN_STATIC); 622 bt_free(vm, bt); 623 } 624 } 625 xfree(vm->vm_hashlist); 626 } 627 VMEM_LOCK_DESTROY(vm); 628 xfree(vm); 629 } 630 631 static int 632 vmem_import(vmem_t *vm, vmem_size_t size, vm_flag_t flags) 633 { 634 vmem_addr_t addr; 635 636 if (vm->vm_allocfn == NULL) { 637 return EINVAL; 638 } 639 640 addr = (*vm->vm_allocfn)(vm->vm_source, size, &size, flags); 641 if (addr == VMEM_ADDR_NULL) { 642 return ENOMEM; 643 } 644 645 if (vmem_add1(vm, addr, size, flags, BT_TYPE_SPAN) == VMEM_ADDR_NULL) { 646 (*vm->vm_freefn)(vm->vm_source, addr, size); 647 return ENOMEM; 648 } 649 650 return 0; 651 } 652 653 static int 654 vmem_rehash(vmem_t *vm, size_t newhashsize, vm_flag_t flags) 655 { 656 bt_t *bt; 657 int i; 658 struct vmem_hashlist *newhashlist; 659 struct vmem_hashlist *oldhashlist; 660 size_t oldhashsize; 661 662 KASSERT(newhashsize > 0); 663 664 newhashlist = 665 xmalloc(sizeof(struct vmem_hashlist *) * newhashsize, flags); 666 if (newhashlist == NULL) { 667 return ENOMEM; 668 } 669 for (i = 0; i < newhashsize; i++) { 670 LIST_INIT(&newhashlist[i]); 671 } 672 673 if (!VMEM_TRYLOCK(vm)) { 674 xfree(newhashlist); 675 return EBUSY; 676 } 677 oldhashlist = vm->vm_hashlist; 678 oldhashsize = vm->vm_hashsize; 679 vm->vm_hashlist = newhashlist; 680 vm->vm_hashsize = newhashsize; 681 if (oldhashlist == NULL) { 682 VMEM_UNLOCK(vm); 683 return 0; 684 } 685 for (i = 0; i < oldhashsize; i++) { 686 while ((bt = LIST_FIRST(&oldhashlist[i])) != NULL) { 687 bt_rembusy(vm, bt); /* XXX */ 688 bt_insbusy(vm, bt); 689 } 690 } 691 VMEM_UNLOCK(vm); 692 693 xfree(oldhashlist); 694 695 return 0; 696 } 697 698 /* 699 * vmem_fit: check if a bt can satisfy the given restrictions. 700 */ 701 702 static vmem_addr_t 703 vmem_fit(const bt_t *bt, vmem_size_t size, vmem_size_t align, vmem_size_t phase, 704 vmem_size_t nocross, vmem_addr_t minaddr, vmem_addr_t maxaddr) 705 { 706 vmem_addr_t start; 707 vmem_addr_t end; 708 709 KASSERT(bt->bt_size >= size); 710 711 /* 712 * XXX assumption: vmem_addr_t and vmem_size_t are 713 * unsigned integer of the same size. 714 */ 715 716 start = bt->bt_start; 717 if (start < minaddr) { 718 start = minaddr; 719 } 720 end = BT_END(bt); 721 if (end > maxaddr - 1) { 722 end = maxaddr - 1; 723 } 724 if (start >= end) { 725 return VMEM_ADDR_NULL; 726 } 727 728 start = VMEM_ALIGNUP(start - phase, align) + phase; 729 if (start < bt->bt_start) { 730 start += align; 731 } 732 if (VMEM_CROSS_P(start, start + size - 1, nocross)) { 733 KASSERT(align < nocross); 734 start = VMEM_ALIGNUP(start - phase, nocross) + phase; 735 } 736 if (start < end && end - start >= size) { 737 KASSERT((start & (align - 1)) == phase); 738 KASSERT(!VMEM_CROSS_P(start, start + size - 1, nocross)); 739 KASSERT(minaddr <= start); 740 KASSERT(maxaddr == 0 || start + size <= maxaddr); 741 KASSERT(bt->bt_start <= start); 742 KASSERT(start + size <= BT_END(bt)); 743 return start; 744 } 745 return VMEM_ADDR_NULL; 746 } 747 748 /* ---- vmem API */ 749 750 /* 751 * vmem_create: create an arena. 752 * 753 * => must not be called from interrupt context. 754 */ 755 756 vmem_t * 757 vmem_create(const char *name, vmem_addr_t base, vmem_size_t size, 758 vmem_size_t quantum, 759 vmem_addr_t (*allocfn)(vmem_t *, vmem_size_t, vmem_size_t *, vm_flag_t), 760 void (*freefn)(vmem_t *, vmem_addr_t, vmem_size_t), 761 vmem_t *source, vmem_size_t qcache_max, vm_flag_t flags, 762 int ipl) 763 { 764 vmem_t *vm; 765 int i; 766 #if defined(_KERNEL) 767 static ONCE_DECL(control); 768 #endif /* defined(_KERNEL) */ 769 770 KASSERT((flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 771 KASSERT((~flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 772 773 #if defined(_KERNEL) 774 if (RUN_ONCE(&control, vmem_init)) { 775 return NULL; 776 } 777 #endif /* defined(_KERNEL) */ 778 vm = xmalloc(sizeof(*vm), flags); 779 if (vm == NULL) { 780 return NULL; 781 } 782 783 VMEM_LOCK_INIT(vm, ipl); 784 vm->vm_name = name; 785 vm->vm_quantum_mask = quantum - 1; 786 vm->vm_quantum_shift = calc_order(quantum); 787 KASSERT(ORDER2SIZE(vm->vm_quantum_shift) == quantum); 788 vm->vm_allocfn = allocfn; 789 vm->vm_freefn = freefn; 790 vm->vm_source = source; 791 vm->vm_nbusytag = 0; 792 #if defined(QCACHE) 793 qc_init(vm, qcache_max, ipl); 794 #endif /* defined(QCACHE) */ 795 796 CIRCLEQ_INIT(&vm->vm_seglist); 797 for (i = 0; i < VMEM_MAXORDER; i++) { 798 LIST_INIT(&vm->vm_freelist[i]); 799 } 800 vm->vm_hashlist = NULL; 801 if (vmem_rehash(vm, VMEM_HASHSIZE_INIT, flags)) { 802 vmem_destroy1(vm); 803 return NULL; 804 } 805 806 if (size != 0) { 807 if (vmem_add(vm, base, size, flags) == 0) { 808 vmem_destroy1(vm); 809 return NULL; 810 } 811 } 812 813 #if defined(_KERNEL) 814 mutex_enter(&vmem_list_lock); 815 LIST_INSERT_HEAD(&vmem_list, vm, vm_alllist); 816 mutex_exit(&vmem_list_lock); 817 #endif /* defined(_KERNEL) */ 818 819 return vm; 820 } 821 822 void 823 vmem_destroy(vmem_t *vm) 824 { 825 826 #if defined(_KERNEL) 827 mutex_enter(&vmem_list_lock); 828 LIST_REMOVE(vm, vm_alllist); 829 mutex_exit(&vmem_list_lock); 830 #endif /* defined(_KERNEL) */ 831 832 vmem_destroy1(vm); 833 } 834 835 vmem_size_t 836 vmem_roundup_size(vmem_t *vm, vmem_size_t size) 837 { 838 839 return (size + vm->vm_quantum_mask) & ~vm->vm_quantum_mask; 840 } 841 842 /* 843 * vmem_alloc: 844 * 845 * => caller must ensure appropriate spl, 846 * if the arena can be accessed from interrupt context. 847 */ 848 849 vmem_addr_t 850 vmem_alloc(vmem_t *vm, vmem_size_t size0, vm_flag_t flags) 851 { 852 const vmem_size_t size __unused = vmem_roundup_size(vm, size0); 853 const vm_flag_t strat __unused = flags & VM_FITMASK; 854 855 KASSERT((flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 856 KASSERT((~flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 857 858 KASSERT(size0 > 0); 859 KASSERT(size > 0); 860 KASSERT(strat == VM_BESTFIT || strat == VM_INSTANTFIT); 861 if ((flags & VM_SLEEP) != 0) { 862 ASSERT_SLEEPABLE(NULL, __func__); 863 } 864 865 #if defined(QCACHE) 866 if (size <= vm->vm_qcache_max) { 867 int qidx = size >> vm->vm_quantum_shift; 868 qcache_t *qc = vm->vm_qcache[qidx - 1]; 869 870 return (vmem_addr_t)pool_cache_get(qc->qc_cache, 871 vmf_to_prf(flags)); 872 } 873 #endif /* defined(QCACHE) */ 874 875 return vmem_xalloc(vm, size0, 0, 0, 0, 0, 0, flags); 876 } 877 878 vmem_addr_t 879 vmem_xalloc(vmem_t *vm, vmem_size_t size0, vmem_size_t align, vmem_size_t phase, 880 vmem_size_t nocross, vmem_addr_t minaddr, vmem_addr_t maxaddr, 881 vm_flag_t flags) 882 { 883 struct vmem_freelist *list; 884 struct vmem_freelist *first; 885 struct vmem_freelist *end; 886 bt_t *bt; 887 bt_t *btnew; 888 bt_t *btnew2; 889 const vmem_size_t size = vmem_roundup_size(vm, size0); 890 vm_flag_t strat = flags & VM_FITMASK; 891 vmem_addr_t start; 892 893 KASSERT(size0 > 0); 894 KASSERT(size > 0); 895 KASSERT(strat == VM_BESTFIT || strat == VM_INSTANTFIT); 896 if ((flags & VM_SLEEP) != 0) { 897 ASSERT_SLEEPABLE(NULL, __func__); 898 } 899 KASSERT((align & vm->vm_quantum_mask) == 0); 900 KASSERT((align & (align - 1)) == 0); 901 KASSERT((phase & vm->vm_quantum_mask) == 0); 902 KASSERT((nocross & vm->vm_quantum_mask) == 0); 903 KASSERT((nocross & (nocross - 1)) == 0); 904 KASSERT((align == 0 && phase == 0) || phase < align); 905 KASSERT(nocross == 0 || nocross >= size); 906 KASSERT(maxaddr == 0 || minaddr < maxaddr); 907 KASSERT(!VMEM_CROSS_P(phase, phase + size - 1, nocross)); 908 909 if (align == 0) { 910 align = vm->vm_quantum_mask + 1; 911 } 912 btnew = bt_alloc(vm, flags); 913 if (btnew == NULL) { 914 return VMEM_ADDR_NULL; 915 } 916 btnew2 = bt_alloc(vm, flags); /* XXX not necessary if no restrictions */ 917 if (btnew2 == NULL) { 918 bt_free(vm, btnew); 919 return VMEM_ADDR_NULL; 920 } 921 922 retry_strat: 923 first = bt_freehead_toalloc(vm, size, strat); 924 end = &vm->vm_freelist[VMEM_MAXORDER]; 925 retry: 926 bt = NULL; 927 VMEM_LOCK(vm); 928 if (strat == VM_INSTANTFIT) { 929 for (list = first; list < end; list++) { 930 bt = LIST_FIRST(list); 931 if (bt != NULL) { 932 start = vmem_fit(bt, size, align, phase, 933 nocross, minaddr, maxaddr); 934 if (start != VMEM_ADDR_NULL) { 935 goto gotit; 936 } 937 } 938 } 939 } else { /* VM_BESTFIT */ 940 for (list = first; list < end; list++) { 941 LIST_FOREACH(bt, list, bt_freelist) { 942 if (bt->bt_size >= size) { 943 start = vmem_fit(bt, size, align, phase, 944 nocross, minaddr, maxaddr); 945 if (start != VMEM_ADDR_NULL) { 946 goto gotit; 947 } 948 } 949 } 950 } 951 } 952 VMEM_UNLOCK(vm); 953 #if 1 954 if (strat == VM_INSTANTFIT) { 955 strat = VM_BESTFIT; 956 goto retry_strat; 957 } 958 #endif 959 if (align != vm->vm_quantum_mask + 1 || phase != 0 || 960 nocross != 0 || minaddr != 0 || maxaddr != 0) { 961 962 /* 963 * XXX should try to import a region large enough to 964 * satisfy restrictions? 965 */ 966 967 goto fail; 968 } 969 if (vmem_import(vm, size, flags) == 0) { 970 goto retry; 971 } 972 /* XXX */ 973 fail: 974 bt_free(vm, btnew); 975 bt_free(vm, btnew2); 976 return VMEM_ADDR_NULL; 977 978 gotit: 979 KASSERT(bt->bt_type == BT_TYPE_FREE); 980 KASSERT(bt->bt_size >= size); 981 bt_remfree(vm, bt); 982 if (bt->bt_start != start) { 983 btnew2->bt_type = BT_TYPE_FREE; 984 btnew2->bt_start = bt->bt_start; 985 btnew2->bt_size = start - bt->bt_start; 986 bt->bt_start = start; 987 bt->bt_size -= btnew2->bt_size; 988 bt_insfree(vm, btnew2); 989 bt_insseg(vm, btnew2, CIRCLEQ_PREV(bt, bt_seglist)); 990 btnew2 = NULL; 991 } 992 KASSERT(bt->bt_start == start); 993 if (bt->bt_size != size && bt->bt_size - size > vm->vm_quantum_mask) { 994 /* split */ 995 btnew->bt_type = BT_TYPE_BUSY; 996 btnew->bt_start = bt->bt_start; 997 btnew->bt_size = size; 998 bt->bt_start = bt->bt_start + size; 999 bt->bt_size -= size; 1000 bt_insfree(vm, bt); 1001 bt_insseg(vm, btnew, CIRCLEQ_PREV(bt, bt_seglist)); 1002 bt_insbusy(vm, btnew); 1003 VMEM_UNLOCK(vm); 1004 } else { 1005 bt->bt_type = BT_TYPE_BUSY; 1006 bt_insbusy(vm, bt); 1007 VMEM_UNLOCK(vm); 1008 bt_free(vm, btnew); 1009 btnew = bt; 1010 } 1011 if (btnew2 != NULL) { 1012 bt_free(vm, btnew2); 1013 } 1014 KASSERT(btnew->bt_size >= size); 1015 btnew->bt_type = BT_TYPE_BUSY; 1016 1017 return btnew->bt_start; 1018 } 1019 1020 /* 1021 * vmem_free: 1022 * 1023 * => caller must ensure appropriate spl, 1024 * if the arena can be accessed from interrupt context. 1025 */ 1026 1027 void 1028 vmem_free(vmem_t *vm, vmem_addr_t addr, vmem_size_t size) 1029 { 1030 1031 KASSERT(addr != VMEM_ADDR_NULL); 1032 KASSERT(size > 0); 1033 1034 #if defined(QCACHE) 1035 if (size <= vm->vm_qcache_max) { 1036 int qidx = (size + vm->vm_quantum_mask) >> vm->vm_quantum_shift; 1037 qcache_t *qc = vm->vm_qcache[qidx - 1]; 1038 1039 return pool_cache_put(qc->qc_cache, (void *)addr); 1040 } 1041 #endif /* defined(QCACHE) */ 1042 1043 vmem_xfree(vm, addr, size); 1044 } 1045 1046 void 1047 vmem_xfree(vmem_t *vm, vmem_addr_t addr, vmem_size_t size) 1048 { 1049 bt_t *bt; 1050 bt_t *t; 1051 1052 KASSERT(addr != VMEM_ADDR_NULL); 1053 KASSERT(size > 0); 1054 1055 VMEM_LOCK(vm); 1056 1057 bt = bt_lookupbusy(vm, addr); 1058 KASSERT(bt != NULL); 1059 KASSERT(bt->bt_start == addr); 1060 KASSERT(bt->bt_size == vmem_roundup_size(vm, size) || 1061 bt->bt_size - vmem_roundup_size(vm, size) <= vm->vm_quantum_mask); 1062 KASSERT(bt->bt_type == BT_TYPE_BUSY); 1063 bt_rembusy(vm, bt); 1064 bt->bt_type = BT_TYPE_FREE; 1065 1066 /* coalesce */ 1067 t = CIRCLEQ_NEXT(bt, bt_seglist); 1068 if (t != NULL && t->bt_type == BT_TYPE_FREE) { 1069 KASSERT(BT_END(bt) == t->bt_start); 1070 bt_remfree(vm, t); 1071 bt_remseg(vm, t); 1072 bt->bt_size += t->bt_size; 1073 bt_free(vm, t); 1074 } 1075 t = CIRCLEQ_PREV(bt, bt_seglist); 1076 if (t != NULL && t->bt_type == BT_TYPE_FREE) { 1077 KASSERT(BT_END(t) == bt->bt_start); 1078 bt_remfree(vm, t); 1079 bt_remseg(vm, t); 1080 bt->bt_size += t->bt_size; 1081 bt->bt_start = t->bt_start; 1082 bt_free(vm, t); 1083 } 1084 1085 t = CIRCLEQ_PREV(bt, bt_seglist); 1086 KASSERT(t != NULL); 1087 KASSERT(BT_ISSPAN_P(t) || t->bt_type == BT_TYPE_BUSY); 1088 if (vm->vm_freefn != NULL && t->bt_type == BT_TYPE_SPAN && 1089 t->bt_size == bt->bt_size) { 1090 vmem_addr_t spanaddr; 1091 vmem_size_t spansize; 1092 1093 KASSERT(t->bt_start == bt->bt_start); 1094 spanaddr = bt->bt_start; 1095 spansize = bt->bt_size; 1096 bt_remseg(vm, bt); 1097 bt_free(vm, bt); 1098 bt_remseg(vm, t); 1099 bt_free(vm, t); 1100 VMEM_UNLOCK(vm); 1101 (*vm->vm_freefn)(vm->vm_source, spanaddr, spansize); 1102 } else { 1103 bt_insfree(vm, bt); 1104 VMEM_UNLOCK(vm); 1105 } 1106 } 1107 1108 /* 1109 * vmem_add: 1110 * 1111 * => caller must ensure appropriate spl, 1112 * if the arena can be accessed from interrupt context. 1113 */ 1114 1115 vmem_addr_t 1116 vmem_add(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, vm_flag_t flags) 1117 { 1118 1119 return vmem_add1(vm, addr, size, flags, BT_TYPE_SPAN_STATIC); 1120 } 1121 1122 /* 1123 * vmem_reap: reap unused resources. 1124 * 1125 * => return true if we successfully reaped something. 1126 */ 1127 1128 bool 1129 vmem_reap(vmem_t *vm) 1130 { 1131 bool didsomething = false; 1132 1133 #if defined(QCACHE) 1134 didsomething = qc_reap(vm); 1135 #endif /* defined(QCACHE) */ 1136 return didsomething; 1137 } 1138 1139 /* ---- rehash */ 1140 1141 #if defined(_KERNEL) 1142 static struct callout vmem_rehash_ch; 1143 static int vmem_rehash_interval; 1144 static struct workqueue *vmem_rehash_wq; 1145 static struct work vmem_rehash_wk; 1146 1147 static void 1148 vmem_rehash_all(struct work *wk, void *dummy) 1149 { 1150 vmem_t *vm; 1151 1152 KASSERT(wk == &vmem_rehash_wk); 1153 mutex_enter(&vmem_list_lock); 1154 LIST_FOREACH(vm, &vmem_list, vm_alllist) { 1155 size_t desired; 1156 size_t current; 1157 1158 if (!VMEM_TRYLOCK(vm)) { 1159 continue; 1160 } 1161 desired = vm->vm_nbusytag; 1162 current = vm->vm_hashsize; 1163 VMEM_UNLOCK(vm); 1164 1165 if (desired > VMEM_HASHSIZE_MAX) { 1166 desired = VMEM_HASHSIZE_MAX; 1167 } else if (desired < VMEM_HASHSIZE_MIN) { 1168 desired = VMEM_HASHSIZE_MIN; 1169 } 1170 if (desired > current * 2 || desired * 2 < current) { 1171 vmem_rehash(vm, desired, VM_NOSLEEP); 1172 } 1173 } 1174 mutex_exit(&vmem_list_lock); 1175 1176 callout_schedule(&vmem_rehash_ch, vmem_rehash_interval); 1177 } 1178 1179 static void 1180 vmem_rehash_all_kick(void *dummy) 1181 { 1182 1183 workqueue_enqueue(vmem_rehash_wq, &vmem_rehash_wk, NULL); 1184 } 1185 1186 void 1187 vmem_rehash_start(void) 1188 { 1189 int error; 1190 1191 error = workqueue_create(&vmem_rehash_wq, "vmem_rehash", 1192 vmem_rehash_all, NULL, PRI_VM, IPL_SOFTCLOCK, 0); 1193 if (error) { 1194 panic("%s: workqueue_create %d\n", __func__, error); 1195 } 1196 callout_init(&vmem_rehash_ch, 0); 1197 callout_setfunc(&vmem_rehash_ch, vmem_rehash_all_kick, NULL); 1198 1199 vmem_rehash_interval = hz * 10; 1200 callout_schedule(&vmem_rehash_ch, vmem_rehash_interval); 1201 } 1202 #endif /* defined(_KERNEL) */ 1203 1204 /* ---- debug */ 1205 1206 #if defined(VMEM_DEBUG) 1207 1208 #if !defined(_KERNEL) 1209 #include <stdio.h> 1210 #endif /* !defined(_KERNEL) */ 1211 1212 void bt_dump(const bt_t *); 1213 1214 void 1215 bt_dump(const bt_t *bt) 1216 { 1217 1218 printf("\t%p: %" PRIu64 ", %" PRIu64 ", %d\n", 1219 bt, (uint64_t)bt->bt_start, (uint64_t)bt->bt_size, 1220 bt->bt_type); 1221 } 1222 1223 void 1224 vmem_dump(const vmem_t *vm) 1225 { 1226 const bt_t *bt; 1227 int i; 1228 1229 printf("vmem %p '%s'\n", vm, vm->vm_name); 1230 CIRCLEQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) { 1231 bt_dump(bt); 1232 } 1233 1234 for (i = 0; i < VMEM_MAXORDER; i++) { 1235 const struct vmem_freelist *fl = &vm->vm_freelist[i]; 1236 1237 if (LIST_EMPTY(fl)) { 1238 continue; 1239 } 1240 1241 printf("freelist[%d]\n", i); 1242 LIST_FOREACH(bt, fl, bt_freelist) { 1243 bt_dump(bt); 1244 if (bt->bt_size) { 1245 } 1246 } 1247 } 1248 } 1249 1250 #if !defined(_KERNEL) 1251 1252 int 1253 main() 1254 { 1255 vmem_t *vm; 1256 vmem_addr_t p; 1257 struct reg { 1258 vmem_addr_t p; 1259 vmem_size_t sz; 1260 bool x; 1261 } *reg = NULL; 1262 int nreg = 0; 1263 int nalloc = 0; 1264 int nfree = 0; 1265 vmem_size_t total = 0; 1266 #if 1 1267 vm_flag_t strat = VM_INSTANTFIT; 1268 #else 1269 vm_flag_t strat = VM_BESTFIT; 1270 #endif 1271 1272 vm = vmem_create("test", VMEM_ADDR_NULL, 0, 1, 1273 NULL, NULL, NULL, 0, VM_SLEEP); 1274 if (vm == NULL) { 1275 printf("vmem_create\n"); 1276 exit(EXIT_FAILURE); 1277 } 1278 vmem_dump(vm); 1279 1280 p = vmem_add(vm, 100, 200, VM_SLEEP); 1281 p = vmem_add(vm, 2000, 1, VM_SLEEP); 1282 p = vmem_add(vm, 40000, 0x10000000>>12, VM_SLEEP); 1283 p = vmem_add(vm, 10000, 10000, VM_SLEEP); 1284 p = vmem_add(vm, 500, 1000, VM_SLEEP); 1285 vmem_dump(vm); 1286 for (;;) { 1287 struct reg *r; 1288 int t = rand() % 100; 1289 1290 if (t > 45) { 1291 /* alloc */ 1292 vmem_size_t sz = rand() % 500 + 1; 1293 bool x; 1294 vmem_size_t align, phase, nocross; 1295 vmem_addr_t minaddr, maxaddr; 1296 1297 if (t > 70) { 1298 x = true; 1299 /* XXX */ 1300 align = 1 << (rand() % 15); 1301 phase = rand() % 65536; 1302 nocross = 1 << (rand() % 15); 1303 if (align <= phase) { 1304 phase = 0; 1305 } 1306 if (VMEM_CROSS_P(phase, phase + sz - 1, 1307 nocross)) { 1308 nocross = 0; 1309 } 1310 minaddr = rand() % 50000; 1311 maxaddr = rand() % 70000; 1312 if (minaddr > maxaddr) { 1313 minaddr = 0; 1314 maxaddr = 0; 1315 } 1316 printf("=== xalloc %" PRIu64 1317 " align=%" PRIu64 ", phase=%" PRIu64 1318 ", nocross=%" PRIu64 ", min=%" PRIu64 1319 ", max=%" PRIu64 "\n", 1320 (uint64_t)sz, 1321 (uint64_t)align, 1322 (uint64_t)phase, 1323 (uint64_t)nocross, 1324 (uint64_t)minaddr, 1325 (uint64_t)maxaddr); 1326 p = vmem_xalloc(vm, sz, align, phase, nocross, 1327 minaddr, maxaddr, strat|VM_SLEEP); 1328 } else { 1329 x = false; 1330 printf("=== alloc %" PRIu64 "\n", (uint64_t)sz); 1331 p = vmem_alloc(vm, sz, strat|VM_SLEEP); 1332 } 1333 printf("-> %" PRIu64 "\n", (uint64_t)p); 1334 vmem_dump(vm); 1335 if (p == VMEM_ADDR_NULL) { 1336 if (x) { 1337 continue; 1338 } 1339 break; 1340 } 1341 nreg++; 1342 reg = realloc(reg, sizeof(*reg) * nreg); 1343 r = ®[nreg - 1]; 1344 r->p = p; 1345 r->sz = sz; 1346 r->x = x; 1347 total += sz; 1348 nalloc++; 1349 } else if (nreg != 0) { 1350 /* free */ 1351 r = ®[rand() % nreg]; 1352 printf("=== free %" PRIu64 ", %" PRIu64 "\n", 1353 (uint64_t)r->p, (uint64_t)r->sz); 1354 if (r->x) { 1355 vmem_xfree(vm, r->p, r->sz); 1356 } else { 1357 vmem_free(vm, r->p, r->sz); 1358 } 1359 total -= r->sz; 1360 vmem_dump(vm); 1361 *r = reg[nreg - 1]; 1362 nreg--; 1363 nfree++; 1364 } 1365 printf("total=%" PRIu64 "\n", (uint64_t)total); 1366 } 1367 fprintf(stderr, "total=%" PRIu64 ", nalloc=%d, nfree=%d\n", 1368 (uint64_t)total, nalloc, nfree); 1369 exit(EXIT_SUCCESS); 1370 } 1371 #endif /* !defined(_KERNEL) */ 1372 #endif /* defined(VMEM_DEBUG) */ 1373