1 /* $NetBSD: uvm_physseg.c,v 1.18 2023/04/09 09:00:56 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * Copyright (c) 1991, 1993, The Regents of the University of California. 6 * 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * The Mach Operating System project at Carnegie-Mellon University. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)vm_page.h 7.3 (Berkeley) 4/21/91 37 * from: Id: uvm_page.h,v 1.1.2.6 1998/02/04 02:31:42 chuck Exp 38 * 39 * 40 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 41 * All rights reserved. 42 * 43 * Permission to use, copy, modify and distribute this software and 44 * its documentation is hereby granted, provided that both the copyright 45 * notice and this permission notice appear in all copies of the 46 * software, derivative works or modified versions, and any portions 47 * thereof, and that both notices appear in supporting documentation. 48 * 49 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 50 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 51 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 52 * 53 * Carnegie Mellon requests users of this software to return to 54 * 55 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 56 * School of Computer Science 57 * Carnegie Mellon University 58 * Pittsburgh PA 15213-3890 59 * 60 * any improvements or extensions that they make and grant Carnegie the 61 * rights to redistribute these changes. 62 */ 63 64 /* 65 * Consolidated API from uvm_page.c and others. 66 * Consolidated and designed by Cherry G. Mathew <cherry@zyx.in> 67 * rbtree(3) backing implementation by: 68 * Santhosh N. Raju <santhosh.raju@gmail.com> 69 */ 70 71 #ifdef _KERNEL_OPT 72 #include "opt_uvm.h" 73 #endif 74 75 #include <sys/param.h> 76 #include <sys/types.h> 77 #include <sys/extent.h> 78 #include <sys/kmem.h> 79 80 #include <uvm/uvm.h> 81 #include <uvm/uvm_page.h> 82 #include <uvm/uvm_param.h> 83 #include <uvm/uvm_pdpolicy.h> 84 #include <uvm/uvm_physseg.h> 85 86 /* 87 * uvm_physseg: describes one segment of physical memory 88 */ 89 struct uvm_physseg { 90 /* used during RB tree lookup for PHYS_TO_VM_PAGE(). */ 91 struct rb_node rb_node; /* tree information */ 92 paddr_t start; /* PF# of first page in segment */ 93 paddr_t end; /* (PF# of last page in segment) + 1 */ 94 struct vm_page *pgs; /* vm_page structures (from start) */ 95 96 /* less performance sensitive fields. */ 97 paddr_t avail_start; /* PF# of first free page in segment */ 98 paddr_t avail_end; /* (PF# of last free page in segment) +1 */ 99 struct extent *ext; /* extent(9) structure to manage pgs[] */ 100 int free_list; /* which free list they belong on */ 101 u_int start_hint; /* start looking for free pages here */ 102 #ifdef __HAVE_PMAP_PHYSSEG 103 struct pmap_physseg pmseg; /* pmap specific (MD) data */ 104 #endif 105 }; 106 107 /* 108 * These functions are reserved for uvm(9) internal use and are not 109 * exported in the header file uvm_physseg.h 110 * 111 * Thus they are redefined here. 112 */ 113 void uvm_physseg_init_seg(uvm_physseg_t, struct vm_page *); 114 void uvm_physseg_seg_chomp_slab(uvm_physseg_t, struct vm_page *, size_t); 115 116 /* returns a pgs array */ 117 struct vm_page *uvm_physseg_seg_alloc_from_slab(uvm_physseg_t, size_t); 118 119 #if defined(UVM_HOTPLUG) /* rbtree impementation */ 120 121 #define HANDLE_TO_PHYSSEG_NODE(h) ((struct uvm_physseg *)(h)) 122 #define PHYSSEG_NODE_TO_HANDLE(u) ((uvm_physseg_t)(u)) 123 124 struct uvm_physseg_graph { 125 struct rb_tree rb_tree; /* Tree for entries */ 126 int nentries; /* Number of entries */ 127 } __aligned(COHERENCY_UNIT); 128 129 static struct uvm_physseg_graph uvm_physseg_graph __read_mostly; 130 131 /* 132 * Note on kmem(9) allocator usage: 133 * We take the conservative approach that plug/unplug are allowed to 134 * fail in high memory stress situations. 135 * 136 * We want to avoid re-entrant situations in which one plug/unplug 137 * operation is waiting on a previous one to complete, since this 138 * makes the design more complicated than necessary. 139 * 140 * We may review this and change its behaviour, once the use cases 141 * become more obvious. 142 */ 143 144 /* 145 * Special alloc()/free() functions for boot time support: 146 * We assume that alloc() at boot time is only for new 'vm_physseg's 147 * This allows us to use a static array for memory allocation at boot 148 * time. Thus we avoid using kmem(9) which is not ready at this point 149 * in boot. 150 * 151 * After kmem(9) is ready, we use it. We currently discard any free()s 152 * to this static array, since the size is small enough to be a 153 * trivial waste on all architectures we run on. 154 */ 155 156 static size_t nseg = 0; 157 static struct uvm_physseg uvm_physseg[VM_PHYSSEG_MAX]; 158 159 static void * 160 uvm_physseg_alloc(size_t sz) 161 { 162 /* 163 * During boot time, we only support allocating vm_physseg 164 * entries from the static array. 165 * We need to assert for this. 166 */ 167 168 if (__predict_false(uvm.page_init_done == false)) { 169 if (sz % sizeof(struct uvm_physseg)) 170 panic("%s: tried to alloc size other than multiple" 171 " of struct uvm_physseg at boot\n", __func__); 172 173 size_t n = sz / sizeof(struct uvm_physseg); 174 nseg += n; 175 176 KASSERT(nseg > 0); 177 KASSERT(nseg <= VM_PHYSSEG_MAX); 178 179 return &uvm_physseg[nseg - n]; 180 } 181 182 return kmem_zalloc(sz, KM_NOSLEEP); 183 } 184 185 static void 186 uvm_physseg_free(void *p, size_t sz) 187 { 188 /* 189 * This is a bit tricky. We do allow simulation of free() 190 * during boot (for eg: when MD code is "steal"ing memory, 191 * and the segment has been exhausted (and thus needs to be 192 * free() - ed. 193 * free() also complicates things because we leak the 194 * free(). Therefore calling code can't assume that free()-ed 195 * memory is available for alloc() again, at boot time. 196 * 197 * Thus we can't explicitly disallow free()s during 198 * boot time. However, the same restriction for alloc() 199 * applies to free(). We only allow uvm_physseg related free()s 200 * via this function during boot time. 201 */ 202 203 if (__predict_false(uvm.page_init_done == false)) { 204 if (sz % sizeof(struct uvm_physseg)) 205 panic("%s: tried to free size other than struct uvm_physseg" 206 " at boot\n", __func__); 207 208 } 209 210 /* 211 * Could have been in a single if(){} block - split for 212 * clarity 213 */ 214 215 if ((struct uvm_physseg *)p >= uvm_physseg && 216 (struct uvm_physseg *)p < (uvm_physseg + VM_PHYSSEG_MAX)) { 217 if (sz % sizeof(struct uvm_physseg)) 218 panic("%s: tried to free() other than struct uvm_physseg" 219 " from static array\n", __func__); 220 221 if ((sz / sizeof(struct uvm_physseg)) >= VM_PHYSSEG_MAX) 222 panic("%s: tried to free() the entire static array!", __func__); 223 return; /* Nothing to free */ 224 } 225 226 kmem_free(p, sz); 227 } 228 229 /* XXX: Multi page size */ 230 bool 231 uvm_physseg_plug(paddr_t pfn, size_t pages, uvm_physseg_t *psp) 232 { 233 int preload; 234 size_t slabpages; 235 struct uvm_physseg *ps, *current_ps = NULL; 236 struct vm_page *slab = NULL, *pgs = NULL; 237 238 #ifdef DEBUG 239 paddr_t off; 240 uvm_physseg_t upm; 241 upm = uvm_physseg_find(pfn, &off); 242 243 ps = HANDLE_TO_PHYSSEG_NODE(upm); 244 245 if (ps != NULL) /* XXX; do we allow "update" plugs ? */ 246 return false; 247 #endif 248 249 /* 250 * do we have room? 251 */ 252 253 ps = uvm_physseg_alloc(sizeof (struct uvm_physseg)); 254 if (ps == NULL) { 255 printf("uvm_page_physload: unable to load physical memory " 256 "segment\n"); 257 printf("\t%d segments allocated, ignoring 0x%"PRIxPADDR" -> 0x%"PRIxPADDR"\n", 258 VM_PHYSSEG_MAX, pfn, pfn + pages + 1); 259 printf("\tincrease VM_PHYSSEG_MAX\n"); 260 return false; 261 } 262 263 /* span init */ 264 ps->start = pfn; 265 ps->end = pfn + pages; 266 267 /* 268 * XXX: Ugly hack because uvmexp.npages accounts for only 269 * those pages in the segment included below as well - this 270 * should be legacy and removed. 271 */ 272 273 ps->avail_start = ps->start; 274 ps->avail_end = ps->end; 275 276 /* 277 * check to see if this is a "preload" (i.e. uvm_page_init hasn't been 278 * called yet, so kmem is not available). 279 */ 280 281 preload = 1; /* We are going to assume it is a preload */ 282 283 RB_TREE_FOREACH(current_ps, &(uvm_physseg_graph.rb_tree)) { 284 /* If there are non NULL pages then we are not in a preload */ 285 if (current_ps->pgs != NULL) { 286 preload = 0; 287 /* Try to scavenge from earlier unplug()s. */ 288 pgs = uvm_physseg_seg_alloc_from_slab(current_ps, pages); 289 290 if (pgs != NULL) { 291 break; 292 } 293 } 294 } 295 296 297 /* 298 * if VM is already running, attempt to kmem_alloc vm_page structures 299 */ 300 301 if (!preload) { 302 if (pgs == NULL) { /* Brand new */ 303 /* Iteratively try alloc down from uvmexp.npages */ 304 for (slabpages = (size_t) uvmexp.npages; slabpages >= pages; slabpages--) { 305 slab = kmem_zalloc(sizeof *pgs * (long unsigned int)slabpages, KM_NOSLEEP); 306 if (slab != NULL) 307 break; 308 } 309 310 if (slab == NULL) { 311 uvm_physseg_free(ps, sizeof(struct uvm_physseg)); 312 return false; 313 } 314 315 uvm_physseg_seg_chomp_slab(ps, slab, (size_t) slabpages); 316 /* We allocate enough for this plug */ 317 pgs = uvm_physseg_seg_alloc_from_slab(ps, pages); 318 319 if (pgs == NULL) { 320 printf("unable to uvm_physseg_seg_alloc_from_slab() from backend\n"); 321 return false; 322 } 323 } else { 324 /* Reuse scavenged extent */ 325 ps->ext = current_ps->ext; 326 } 327 328 physmem += pages; 329 uvmpdpol_reinit(); 330 } else { /* Boot time - see uvm_page.c:uvm_page_init() */ 331 pgs = NULL; 332 ps->pgs = pgs; 333 } 334 335 /* 336 * now insert us in the proper place in uvm_physseg_graph.rb_tree 337 */ 338 339 current_ps = rb_tree_insert_node(&(uvm_physseg_graph.rb_tree), ps); 340 if (current_ps != ps) { 341 panic("uvm_page_physload: Duplicate address range detected!"); 342 } 343 uvm_physseg_graph.nentries++; 344 345 /* 346 * uvm_pagefree() requires the PHYS_TO_VM_PAGE(pgs[i]) on the 347 * newly allocated pgs[] to return the correct value. This is 348 * a bit of a chicken and egg problem, since it needs 349 * uvm_physseg_find() to succeed. For this, the node needs to 350 * be inserted *before* uvm_physseg_init_seg() happens. 351 * 352 * During boot, this happens anyway, since 353 * uvm_physseg_init_seg() is called later on and separately 354 * from uvm_page.c:uvm_page_init(). 355 * In the case of hotplug we need to ensure this. 356 */ 357 358 if (__predict_true(!preload)) 359 uvm_physseg_init_seg(ps, pgs); 360 361 if (psp != NULL) 362 *psp = ps; 363 364 return true; 365 } 366 367 static int 368 uvm_physseg_compare_nodes(void *ctx, const void *nnode1, const void *nnode2) 369 { 370 const struct uvm_physseg *enode1 = nnode1; 371 const struct uvm_physseg *enode2 = nnode2; 372 373 KASSERT(enode1->start < enode2->start || enode1->start >= enode2->end); 374 KASSERT(enode2->start < enode1->start || enode2->start >= enode1->end); 375 376 if (enode1->start < enode2->start) 377 return -1; 378 if (enode1->start >= enode2->end) 379 return 1; 380 return 0; 381 } 382 383 static int 384 uvm_physseg_compare_key(void *ctx, const void *nnode, const void *pkey) 385 { 386 const struct uvm_physseg *enode = nnode; 387 const paddr_t pa = *(const paddr_t *) pkey; 388 389 if(enode->start <= pa && pa < enode->end) 390 return 0; 391 if (enode->start < pa) 392 return -1; 393 if (enode->end > pa) 394 return 1; 395 396 return 0; 397 } 398 399 static const rb_tree_ops_t uvm_physseg_tree_ops = { 400 .rbto_compare_nodes = uvm_physseg_compare_nodes, 401 .rbto_compare_key = uvm_physseg_compare_key, 402 .rbto_node_offset = offsetof(struct uvm_physseg, rb_node), 403 .rbto_context = NULL 404 }; 405 406 /* 407 * uvm_physseg_init: init the physmem 408 * 409 * => physmem unit should not be in use at this point 410 */ 411 412 void 413 uvm_physseg_init(void) 414 { 415 rb_tree_init(&(uvm_physseg_graph.rb_tree), &uvm_physseg_tree_ops); 416 uvm_physseg_graph.nentries = 0; 417 } 418 419 uvm_physseg_t 420 uvm_physseg_get_next(uvm_physseg_t upm) 421 { 422 /* next of invalid is invalid, not fatal */ 423 if (uvm_physseg_valid_p(upm) == false) 424 return UVM_PHYSSEG_TYPE_INVALID; 425 426 return (uvm_physseg_t) rb_tree_iterate(&(uvm_physseg_graph.rb_tree), upm, 427 RB_DIR_RIGHT); 428 } 429 430 uvm_physseg_t 431 uvm_physseg_get_prev(uvm_physseg_t upm) 432 { 433 /* prev of invalid is invalid, not fatal */ 434 if (uvm_physseg_valid_p(upm) == false) 435 return UVM_PHYSSEG_TYPE_INVALID; 436 437 return (uvm_physseg_t) rb_tree_iterate(&(uvm_physseg_graph.rb_tree), upm, 438 RB_DIR_LEFT); 439 } 440 441 uvm_physseg_t 442 uvm_physseg_get_last(void) 443 { 444 return (uvm_physseg_t) RB_TREE_MAX(&(uvm_physseg_graph.rb_tree)); 445 } 446 447 uvm_physseg_t 448 uvm_physseg_get_first(void) 449 { 450 return (uvm_physseg_t) RB_TREE_MIN(&(uvm_physseg_graph.rb_tree)); 451 } 452 453 paddr_t 454 uvm_physseg_get_highest_frame(void) 455 { 456 struct uvm_physseg *ps = 457 (uvm_physseg_t) RB_TREE_MAX(&(uvm_physseg_graph.rb_tree)); 458 459 return ps->end - 1; 460 } 461 462 /* 463 * uvm_page_physunload: unload physical memory and return it to 464 * caller. 465 */ 466 bool 467 uvm_page_physunload(uvm_physseg_t upm, int freelist, paddr_t *paddrp) 468 { 469 struct uvm_physseg *seg; 470 471 if (__predict_true(uvm.page_init_done == true)) 472 panic("%s: unload attempted after uvm_page_init()\n", __func__); 473 474 seg = HANDLE_TO_PHYSSEG_NODE(upm); 475 476 if (seg->free_list != freelist) { 477 return false; 478 } 479 480 /* 481 * During cold boot, what we're about to unplug hasn't been 482 * put on the uvm freelist, nor has uvmexp.npages been 483 * updated. (This happens in uvm_page.c:uvm_page_init()) 484 * 485 * For hotplug, we assume here that the pages being unloaded 486 * here are completely out of sight of uvm (ie; not on any uvm 487 * lists), and that uvmexp.npages has been suitably 488 * decremented before we're called. 489 * 490 * XXX: will avail_end == start if avail_start < avail_end? 491 */ 492 493 /* try from front */ 494 if (seg->avail_start == seg->start && 495 seg->avail_start < seg->avail_end) { 496 *paddrp = ctob(seg->avail_start); 497 return uvm_physseg_unplug(seg->avail_start, 1); 498 } 499 500 /* try from rear */ 501 if (seg->avail_end == seg->end && 502 seg->avail_start < seg->avail_end) { 503 *paddrp = ctob(seg->avail_end - 1); 504 return uvm_physseg_unplug(seg->avail_end - 1, 1); 505 } 506 507 return false; 508 } 509 510 bool 511 uvm_page_physunload_force(uvm_physseg_t upm, int freelist, paddr_t *paddrp) 512 { 513 struct uvm_physseg *seg; 514 515 seg = HANDLE_TO_PHYSSEG_NODE(upm); 516 517 if (__predict_true(uvm.page_init_done == true)) 518 panic("%s: unload attempted after uvm_page_init()\n", __func__); 519 /* any room in this bank? */ 520 if (seg->avail_start >= seg->avail_end) { 521 return false; /* nope */ 522 } 523 524 *paddrp = ctob(seg->avail_start); 525 526 /* Always unplug from front */ 527 return uvm_physseg_unplug(seg->avail_start, 1); 528 } 529 530 531 /* 532 * vm_physseg_find: find vm_physseg structure that belongs to a PA 533 */ 534 uvm_physseg_t 535 uvm_physseg_find(paddr_t pframe, psize_t *offp) 536 { 537 struct uvm_physseg * ps = NULL; 538 539 ps = rb_tree_find_node(&(uvm_physseg_graph.rb_tree), &pframe); 540 541 if(ps != NULL && offp != NULL) 542 *offp = pframe - ps->start; 543 544 return ps; 545 } 546 547 #else /* UVM_HOTPLUG */ 548 549 /* 550 * physical memory config is stored in vm_physmem. 551 */ 552 553 #define VM_PHYSMEM_PTR(i) (&vm_physmem[i]) 554 #if VM_PHYSSEG_MAX == 1 555 #define VM_PHYSMEM_PTR_SWAP(i, j) /* impossible */ 556 #else 557 #define VM_PHYSMEM_PTR_SWAP(i, j) \ 558 do { vm_physmem[(i)] = vm_physmem[(j)]; } while (0) 559 #endif 560 561 #define HANDLE_TO_PHYSSEG_NODE(h) (VM_PHYSMEM_PTR((int)h)) 562 #define PHYSSEG_NODE_TO_HANDLE(u) ((int)((vsize_t) (u - vm_physmem) / sizeof(struct uvm_physseg))) 563 564 static struct uvm_physseg vm_physmem[VM_PHYSSEG_MAX]; /* XXXCDC: uvm.physmem */ 565 static int vm_nphysseg = 0; /* XXXCDC: uvm.nphysseg */ 566 #define vm_nphysmem vm_nphysseg 567 568 void 569 uvm_physseg_init(void) 570 { 571 /* XXX: Provisioning for rb_tree related init(s) */ 572 return; 573 } 574 575 int 576 uvm_physseg_get_next(uvm_physseg_t lcv) 577 { 578 /* next of invalid is invalid, not fatal */ 579 if (uvm_physseg_valid_p(lcv) == false) 580 return UVM_PHYSSEG_TYPE_INVALID; 581 582 return (lcv + 1); 583 } 584 585 int 586 uvm_physseg_get_prev(uvm_physseg_t lcv) 587 { 588 /* prev of invalid is invalid, not fatal */ 589 if (uvm_physseg_valid_p(lcv) == false) 590 return UVM_PHYSSEG_TYPE_INVALID; 591 592 return (lcv - 1); 593 } 594 595 int 596 uvm_physseg_get_last(void) 597 { 598 return (vm_nphysseg - 1); 599 } 600 601 int 602 uvm_physseg_get_first(void) 603 { 604 return 0; 605 } 606 607 paddr_t 608 uvm_physseg_get_highest_frame(void) 609 { 610 int lcv; 611 paddr_t last = 0; 612 struct uvm_physseg *ps; 613 614 for (lcv = 0; lcv < vm_nphysseg; lcv++) { 615 ps = VM_PHYSMEM_PTR(lcv); 616 if (last < ps->end) 617 last = ps->end; 618 } 619 620 return last; 621 } 622 623 624 static struct vm_page * 625 uvm_post_preload_check(void) 626 { 627 int preload, lcv; 628 629 /* 630 * check to see if this is a "preload" (i.e. uvm_page_init hasn't been 631 * called yet, so kmem is not available). 632 */ 633 634 for (lcv = 0 ; lcv < vm_nphysmem ; lcv++) { 635 if (VM_PHYSMEM_PTR(lcv)->pgs) 636 break; 637 } 638 preload = (lcv == vm_nphysmem); 639 640 /* 641 * if VM is already running, attempt to kmem_alloc vm_page structures 642 */ 643 644 if (!preload) { 645 panic("Tried to add RAM after uvm_page_init"); 646 } 647 648 return NULL; 649 } 650 651 /* 652 * uvm_page_physunload: unload physical memory and return it to 653 * caller. 654 */ 655 bool 656 uvm_page_physunload(uvm_physseg_t psi, int freelist, paddr_t *paddrp) 657 { 658 int x; 659 struct uvm_physseg *seg; 660 661 uvm_post_preload_check(); 662 663 seg = VM_PHYSMEM_PTR(psi); 664 665 if (seg->free_list != freelist) { 666 return false; 667 } 668 669 /* try from front */ 670 if (seg->avail_start == seg->start && 671 seg->avail_start < seg->avail_end) { 672 *paddrp = ctob(seg->avail_start); 673 seg->avail_start++; 674 seg->start++; 675 /* nothing left? nuke it */ 676 if (seg->avail_start == seg->end) { 677 if (vm_nphysmem == 1) 678 panic("uvm_page_physget: out of memory!"); 679 vm_nphysmem--; 680 for (x = psi ; x < vm_nphysmem ; x++) 681 /* structure copy */ 682 VM_PHYSMEM_PTR_SWAP(x, x + 1); 683 } 684 return (true); 685 } 686 687 /* try from rear */ 688 if (seg->avail_end == seg->end && 689 seg->avail_start < seg->avail_end) { 690 *paddrp = ctob(seg->avail_end - 1); 691 seg->avail_end--; 692 seg->end--; 693 /* nothing left? nuke it */ 694 if (seg->avail_end == seg->start) { 695 if (vm_nphysmem == 1) 696 panic("uvm_page_physget: out of memory!"); 697 vm_nphysmem--; 698 for (x = psi ; x < vm_nphysmem ; x++) 699 /* structure copy */ 700 VM_PHYSMEM_PTR_SWAP(x, x + 1); 701 } 702 return (true); 703 } 704 705 return false; 706 } 707 708 bool 709 uvm_page_physunload_force(uvm_physseg_t psi, int freelist, paddr_t *paddrp) 710 { 711 int x; 712 struct uvm_physseg *seg; 713 714 uvm_post_preload_check(); 715 716 seg = VM_PHYSMEM_PTR(psi); 717 718 /* any room in this bank? */ 719 if (seg->avail_start >= seg->avail_end) { 720 return false; /* nope */ 721 } 722 723 *paddrp = ctob(seg->avail_start); 724 seg->avail_start++; 725 /* truncate! */ 726 seg->start = seg->avail_start; 727 728 /* nothing left? nuke it */ 729 if (seg->avail_start == seg->end) { 730 if (vm_nphysmem == 1) 731 panic("uvm_page_physget: out of memory!"); 732 vm_nphysmem--; 733 for (x = psi ; x < vm_nphysmem ; x++) 734 /* structure copy */ 735 VM_PHYSMEM_PTR_SWAP(x, x + 1); 736 } 737 return (true); 738 } 739 740 bool 741 uvm_physseg_plug(paddr_t pfn, size_t pages, uvm_physseg_t *psp) 742 { 743 int lcv; 744 struct vm_page *pgs; 745 struct uvm_physseg *ps; 746 747 #ifdef DEBUG 748 paddr_t off; 749 uvm_physseg_t upm; 750 upm = uvm_physseg_find(pfn, &off); 751 752 if (uvm_physseg_valid_p(upm)) /* XXX; do we allow "update" plugs ? */ 753 return false; 754 #endif 755 756 paddr_t start = pfn; 757 paddr_t end = pfn + pages; 758 paddr_t avail_start = start; 759 paddr_t avail_end = end; 760 761 if (uvmexp.pagesize == 0) 762 panic("uvm_page_physload: page size not set!"); 763 764 /* 765 * do we have room? 766 */ 767 768 if (vm_nphysmem == VM_PHYSSEG_MAX) { 769 printf("uvm_page_physload: unable to load physical memory " 770 "segment\n"); 771 printf("\t%d segments allocated, ignoring 0x%llx -> 0x%llx\n", 772 VM_PHYSSEG_MAX, (long long)start, (long long)end); 773 printf("\tincrease VM_PHYSSEG_MAX\n"); 774 if (psp != NULL) 775 *psp = UVM_PHYSSEG_TYPE_INVALID_OVERFLOW; 776 return false; 777 } 778 779 /* 780 * check to see if this is a "preload" (i.e. uvm_page_init hasn't been 781 * called yet, so kmem is not available). 782 */ 783 pgs = uvm_post_preload_check(); 784 785 /* 786 * now insert us in the proper place in vm_physmem[] 787 */ 788 789 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_RANDOM) 790 /* random: put it at the end (easy!) */ 791 ps = VM_PHYSMEM_PTR(vm_nphysmem); 792 lcv = vm_nphysmem; 793 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH) 794 { 795 int x; 796 /* sort by address for binary search */ 797 for (lcv = 0 ; lcv < vm_nphysmem ; lcv++) 798 if (start < VM_PHYSMEM_PTR(lcv)->start) 799 break; 800 ps = VM_PHYSMEM_PTR(lcv); 801 /* move back other entries, if necessary ... */ 802 for (x = vm_nphysmem ; x > lcv ; x--) 803 /* structure copy */ 804 VM_PHYSMEM_PTR_SWAP(x, x - 1); 805 } 806 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST) 807 { 808 int x; 809 /* sort by largest segment first */ 810 for (lcv = 0 ; lcv < vm_nphysmem ; lcv++) 811 if ((end - start) > 812 (VM_PHYSMEM_PTR(lcv)->end - VM_PHYSMEM_PTR(lcv)->start)) 813 break; 814 ps = VM_PHYSMEM_PTR(lcv); 815 /* move back other entries, if necessary ... */ 816 for (x = vm_nphysmem ; x > lcv ; x--) 817 /* structure copy */ 818 VM_PHYSMEM_PTR_SWAP(x, x - 1); 819 } 820 #else 821 panic("uvm_page_physload: unknown physseg strategy selected!"); 822 #endif 823 824 ps->start = start; 825 ps->end = end; 826 ps->avail_start = avail_start; 827 ps->avail_end = avail_end; 828 829 ps->pgs = pgs; 830 831 vm_nphysmem++; 832 833 if (psp != NULL) 834 *psp = lcv; 835 836 return true; 837 } 838 839 /* 840 * when VM_PHYSSEG_MAX is 1, we can simplify these functions 841 */ 842 843 #if VM_PHYSSEG_MAX == 1 844 static inline int vm_physseg_find_contig(struct uvm_physseg *, int, paddr_t, psize_t *); 845 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH) 846 static inline int vm_physseg_find_bsearch(struct uvm_physseg *, int, paddr_t, psize_t *); 847 #else 848 static inline int vm_physseg_find_linear(struct uvm_physseg *, int, paddr_t, psize_t *); 849 #endif 850 851 /* 852 * vm_physseg_find: find vm_physseg structure that belongs to a PA 853 */ 854 int 855 uvm_physseg_find(paddr_t pframe, psize_t *offp) 856 { 857 858 #if VM_PHYSSEG_MAX == 1 859 return vm_physseg_find_contig(vm_physmem, vm_nphysseg, pframe, offp); 860 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH) 861 return vm_physseg_find_bsearch(vm_physmem, vm_nphysseg, pframe, offp); 862 #else 863 return vm_physseg_find_linear(vm_physmem, vm_nphysseg, pframe, offp); 864 #endif 865 } 866 867 #if VM_PHYSSEG_MAX == 1 868 static inline int 869 vm_physseg_find_contig(struct uvm_physseg *segs, int nsegs, paddr_t pframe, psize_t *offp) 870 { 871 872 /* 'contig' case */ 873 if (pframe >= segs[0].start && pframe < segs[0].end) { 874 if (offp) 875 *offp = pframe - segs[0].start; 876 return(0); 877 } 878 return(-1); 879 } 880 881 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH) 882 883 static inline int 884 vm_physseg_find_bsearch(struct uvm_physseg *segs, int nsegs, paddr_t pframe, psize_t *offp) 885 { 886 /* binary search for it */ 887 int start, len, guess; 888 889 /* 890 * if try is too large (thus target is less than try) we reduce 891 * the length to trunc(len/2) [i.e. everything smaller than "try"] 892 * 893 * if the try is too small (thus target is greater than try) then 894 * we set the new start to be (try + 1). this means we need to 895 * reduce the length to (round(len/2) - 1). 896 * 897 * note "adjust" below which takes advantage of the fact that 898 * (round(len/2) - 1) == trunc((len - 1) / 2) 899 * for any value of len we may have 900 */ 901 902 for (start = 0, len = nsegs ; len != 0 ; len = len / 2) { 903 guess = start + (len / 2); /* try in the middle */ 904 905 /* start past our try? */ 906 if (pframe >= segs[guess].start) { 907 /* was try correct? */ 908 if (pframe < segs[guess].end) { 909 if (offp) 910 *offp = pframe - segs[guess].start; 911 return guess; /* got it */ 912 } 913 start = guess + 1; /* next time, start here */ 914 len--; /* "adjust" */ 915 } else { 916 /* 917 * pframe before try, just reduce length of 918 * region, done in "for" loop 919 */ 920 } 921 } 922 return(-1); 923 } 924 925 #else 926 927 static inline int 928 vm_physseg_find_linear(struct uvm_physseg *segs, int nsegs, paddr_t pframe, psize_t *offp) 929 { 930 /* linear search for it */ 931 int lcv; 932 933 for (lcv = 0; lcv < nsegs; lcv++) { 934 if (pframe >= segs[lcv].start && 935 pframe < segs[lcv].end) { 936 if (offp) 937 *offp = pframe - segs[lcv].start; 938 return(lcv); /* got it */ 939 } 940 } 941 return(-1); 942 } 943 #endif 944 #endif /* UVM_HOTPLUG */ 945 946 bool 947 uvm_physseg_valid_p(uvm_physseg_t upm) 948 { 949 struct uvm_physseg *ps; 950 951 if (upm == UVM_PHYSSEG_TYPE_INVALID || 952 upm == UVM_PHYSSEG_TYPE_INVALID_EMPTY || 953 upm == UVM_PHYSSEG_TYPE_INVALID_OVERFLOW) 954 return false; 955 956 /* 957 * This is the delicate init dance - 958 * needs to go with the dance. 959 */ 960 if (uvm.page_init_done != true) 961 return true; 962 963 ps = HANDLE_TO_PHYSSEG_NODE(upm); 964 965 /* Extra checks needed only post uvm_page_init() */ 966 if (ps->pgs == NULL) 967 return false; 968 969 /* XXX: etc. */ 970 971 return true; 972 973 } 974 975 /* 976 * Boot protocol dictates that these must be able to return partially 977 * initialised segments. 978 */ 979 paddr_t 980 uvm_physseg_get_start(uvm_physseg_t upm) 981 { 982 if (uvm_physseg_valid_p(upm) == false) 983 return (paddr_t) -1; 984 985 return HANDLE_TO_PHYSSEG_NODE(upm)->start; 986 } 987 988 paddr_t 989 uvm_physseg_get_end(uvm_physseg_t upm) 990 { 991 if (uvm_physseg_valid_p(upm) == false) 992 return (paddr_t) -1; 993 994 return HANDLE_TO_PHYSSEG_NODE(upm)->end; 995 } 996 997 paddr_t 998 uvm_physseg_get_avail_start(uvm_physseg_t upm) 999 { 1000 if (uvm_physseg_valid_p(upm) == false) 1001 return (paddr_t) -1; 1002 1003 return HANDLE_TO_PHYSSEG_NODE(upm)->avail_start; 1004 } 1005 1006 #if defined(UVM_PHYSSEG_LEGACY) 1007 void 1008 uvm_physseg_set_avail_start(uvm_physseg_t upm, paddr_t avail_start) 1009 { 1010 struct uvm_physseg *ps = HANDLE_TO_PHYSSEG_NODE(upm); 1011 1012 #if defined(DIAGNOSTIC) 1013 paddr_t avail_end; 1014 avail_end = uvm_physseg_get_avail_end(upm); 1015 KASSERT(uvm_physseg_valid_p(upm)); 1016 KASSERT(avail_start < avail_end); 1017 KASSERT(avail_start >= ps->start); 1018 #endif 1019 1020 ps->avail_start = avail_start; 1021 } 1022 1023 void 1024 uvm_physseg_set_avail_end(uvm_physseg_t upm, paddr_t avail_end) 1025 { 1026 struct uvm_physseg *ps = HANDLE_TO_PHYSSEG_NODE(upm); 1027 1028 #if defined(DIAGNOSTIC) 1029 paddr_t avail_start; 1030 avail_start = uvm_physseg_get_avail_start(upm); 1031 KASSERT(uvm_physseg_valid_p(upm)); 1032 KASSERT(avail_end > avail_start); 1033 KASSERT(avail_end <= ps->end); 1034 #endif 1035 1036 ps->avail_end = avail_end; 1037 } 1038 1039 #endif /* UVM_PHYSSEG_LEGACY */ 1040 1041 paddr_t 1042 uvm_physseg_get_avail_end(uvm_physseg_t upm) 1043 { 1044 if (uvm_physseg_valid_p(upm) == false) 1045 return (paddr_t) -1; 1046 1047 return HANDLE_TO_PHYSSEG_NODE(upm)->avail_end; 1048 } 1049 1050 struct vm_page * 1051 uvm_physseg_get_pg(uvm_physseg_t upm, paddr_t idx) 1052 { 1053 KASSERT(uvm_physseg_valid_p(upm)); 1054 return &HANDLE_TO_PHYSSEG_NODE(upm)->pgs[idx]; 1055 } 1056 1057 #ifdef __HAVE_PMAP_PHYSSEG 1058 struct pmap_physseg * 1059 uvm_physseg_get_pmseg(uvm_physseg_t upm) 1060 { 1061 KASSERT(uvm_physseg_valid_p(upm)); 1062 return &(HANDLE_TO_PHYSSEG_NODE(upm)->pmseg); 1063 } 1064 #endif 1065 1066 int 1067 uvm_physseg_get_free_list(uvm_physseg_t upm) 1068 { 1069 KASSERT(uvm_physseg_valid_p(upm)); 1070 return HANDLE_TO_PHYSSEG_NODE(upm)->free_list; 1071 } 1072 1073 u_int 1074 uvm_physseg_get_start_hint(uvm_physseg_t upm) 1075 { 1076 KASSERT(uvm_physseg_valid_p(upm)); 1077 return HANDLE_TO_PHYSSEG_NODE(upm)->start_hint; 1078 } 1079 1080 bool 1081 uvm_physseg_set_start_hint(uvm_physseg_t upm, u_int start_hint) 1082 { 1083 if (uvm_physseg_valid_p(upm) == false) 1084 return false; 1085 1086 HANDLE_TO_PHYSSEG_NODE(upm)->start_hint = start_hint; 1087 return true; 1088 } 1089 1090 void 1091 uvm_physseg_init_seg(uvm_physseg_t upm, struct vm_page *pgs) 1092 { 1093 psize_t i; 1094 psize_t n; 1095 paddr_t paddr; 1096 struct uvm_physseg *seg; 1097 struct vm_page *pg; 1098 1099 KASSERT(upm != UVM_PHYSSEG_TYPE_INVALID); 1100 KASSERT(pgs != NULL); 1101 1102 seg = HANDLE_TO_PHYSSEG_NODE(upm); 1103 KASSERT(seg != NULL); 1104 KASSERT(seg->pgs == NULL); 1105 1106 n = seg->end - seg->start; 1107 seg->pgs = pgs; 1108 1109 /* init and free vm_pages (we've already zeroed them) */ 1110 paddr = ctob(seg->start); 1111 for (i = 0 ; i < n ; i++, paddr += PAGE_SIZE) { 1112 pg = &seg->pgs[i]; 1113 pg->phys_addr = paddr; 1114 #ifdef __HAVE_VM_PAGE_MD 1115 VM_MDPAGE_INIT(pg); 1116 #endif 1117 if (atop(paddr) >= seg->avail_start && 1118 atop(paddr) < seg->avail_end) { 1119 uvmexp.npages++; 1120 /* add page to free pool */ 1121 uvm_page_set_freelist(pg, 1122 uvm_page_lookup_freelist(pg)); 1123 /* Disable LOCKDEBUG: too many and too early. */ 1124 mutex_init(&pg->interlock, MUTEX_NODEBUG, IPL_NONE); 1125 uvm_pagefree(pg); 1126 } 1127 } 1128 } 1129 1130 void 1131 uvm_physseg_seg_chomp_slab(uvm_physseg_t upm, struct vm_page *pgs, size_t n) 1132 { 1133 struct uvm_physseg *seg = HANDLE_TO_PHYSSEG_NODE(upm); 1134 1135 /* max number of pre-boot unplug()s allowed */ 1136 #define UVM_PHYSSEG_BOOT_UNPLUG_MAX VM_PHYSSEG_MAX 1137 1138 static char btslab_ex_storage[EXTENT_FIXED_STORAGE_SIZE(UVM_PHYSSEG_BOOT_UNPLUG_MAX)]; 1139 1140 if (__predict_false(uvm.page_init_done == false)) { 1141 seg->ext = extent_create("Boot time slab", (u_long) pgs, (u_long) (pgs + n), 1142 (void *)btslab_ex_storage, sizeof(btslab_ex_storage), 0); 1143 } else { 1144 seg->ext = extent_create("Hotplug slab", (u_long) pgs, (u_long) (pgs + n), NULL, 0, 0); 1145 } 1146 1147 KASSERT(seg->ext != NULL); 1148 1149 } 1150 1151 struct vm_page * 1152 uvm_physseg_seg_alloc_from_slab(uvm_physseg_t upm, size_t pages) 1153 { 1154 int err; 1155 struct uvm_physseg *seg; 1156 struct vm_page *pgs = NULL; 1157 1158 KASSERT(pages > 0); 1159 1160 seg = HANDLE_TO_PHYSSEG_NODE(upm); 1161 1162 if (__predict_false(seg->ext == NULL)) { 1163 /* 1164 * This is a situation unique to boot time. 1165 * It shouldn't happen at any point other than from 1166 * the first uvm_page.c:uvm_page_init() call 1167 * Since we're in a loop, we can get away with the 1168 * below. 1169 */ 1170 KASSERT(uvm.page_init_done != true); 1171 1172 uvm_physseg_t upmp = uvm_physseg_get_prev(upm); 1173 KASSERT(upmp != UVM_PHYSSEG_TYPE_INVALID); 1174 1175 seg->ext = HANDLE_TO_PHYSSEG_NODE(upmp)->ext; 1176 1177 KASSERT(seg->ext != NULL); 1178 } 1179 1180 /* We allocate enough for this segment */ 1181 err = extent_alloc(seg->ext, sizeof(*pgs) * pages, 1, 0, EX_BOUNDZERO, (u_long *)&pgs); 1182 1183 if (err != 0) { 1184 #ifdef DEBUG 1185 printf("%s: extent_alloc failed with error: %d \n", 1186 __func__, err); 1187 #endif 1188 } 1189 1190 return pgs; 1191 } 1192 1193 /* 1194 * uvm_page_physload: load physical memory into VM system 1195 * 1196 * => all args are PFs 1197 * => all pages in start/end get vm_page structures 1198 * => areas marked by avail_start/avail_end get added to the free page pool 1199 * => we are limited to VM_PHYSSEG_MAX physical memory segments 1200 */ 1201 1202 uvm_physseg_t 1203 uvm_page_physload(paddr_t start, paddr_t end, paddr_t avail_start, 1204 paddr_t avail_end, int free_list) 1205 { 1206 struct uvm_physseg *ps; 1207 uvm_physseg_t upm; 1208 1209 if (__predict_true(uvm.page_init_done == true)) 1210 panic("%s: unload attempted after uvm_page_init()\n", __func__); 1211 if (uvmexp.pagesize == 0) 1212 panic("uvm_page_physload: page size not set!"); 1213 if (free_list >= VM_NFREELIST || free_list < VM_FREELIST_DEFAULT) 1214 panic("uvm_page_physload: bad free list %d", free_list); 1215 if (start >= end) 1216 panic("uvm_page_physload: start[%" PRIxPADDR "] >= end[%" 1217 PRIxPADDR "]", start, end); 1218 1219 if (uvm_physseg_plug(start, end - start, &upm) == false) { 1220 panic("uvm_physseg_plug() failed at boot."); 1221 /* NOTREACHED */ 1222 return UVM_PHYSSEG_TYPE_INVALID; /* XXX: correct type */ 1223 } 1224 1225 ps = HANDLE_TO_PHYSSEG_NODE(upm); 1226 1227 /* Legacy */ 1228 ps->avail_start = avail_start; 1229 ps->avail_end = avail_end; 1230 1231 ps->free_list = free_list; /* XXX: */ 1232 1233 1234 return upm; 1235 } 1236 1237 bool 1238 uvm_physseg_unplug(paddr_t pfn, size_t pages) 1239 { 1240 uvm_physseg_t upm; 1241 paddr_t off = 0, start __diagused, end; 1242 struct uvm_physseg *seg; 1243 1244 upm = uvm_physseg_find(pfn, &off); 1245 1246 if (!uvm_physseg_valid_p(upm)) { 1247 printf("%s: Tried to unplug from unknown offset\n", __func__); 1248 return false; 1249 } 1250 1251 seg = HANDLE_TO_PHYSSEG_NODE(upm); 1252 1253 start = uvm_physseg_get_start(upm); 1254 end = uvm_physseg_get_end(upm); 1255 1256 if (end < (pfn + pages)) { 1257 printf("%s: Tried to unplug oversized span \n", __func__); 1258 return false; 1259 } 1260 1261 KASSERT(pfn == start + off); /* sanity */ 1262 1263 if (__predict_true(uvm.page_init_done == true)) { 1264 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */ 1265 if (extent_free(seg->ext, (u_long)(seg->pgs + off), sizeof(struct vm_page) * pages, EX_MALLOCOK | EX_NOWAIT) != 0) 1266 return false; 1267 } 1268 1269 if (off == 0 && (pfn + pages) == end) { 1270 #if defined(UVM_HOTPLUG) /* rbtree implementation */ 1271 int segcount = 0; 1272 struct uvm_physseg *current_ps; 1273 /* Complete segment */ 1274 if (uvm_physseg_graph.nentries == 1) 1275 panic("%s: out of memory!", __func__); 1276 1277 if (__predict_true(uvm.page_init_done == true)) { 1278 RB_TREE_FOREACH(current_ps, &(uvm_physseg_graph.rb_tree)) { 1279 if (seg->ext == current_ps->ext) 1280 segcount++; 1281 } 1282 KASSERT(segcount > 0); 1283 1284 if (segcount == 1) { 1285 extent_destroy(seg->ext); 1286 } 1287 1288 /* 1289 * We assume that the unplug will succeed from 1290 * this point onwards 1291 */ 1292 uvmexp.npages -= (int) pages; 1293 } 1294 1295 rb_tree_remove_node(&(uvm_physseg_graph.rb_tree), upm); 1296 memset(seg, 0, sizeof(struct uvm_physseg)); 1297 uvm_physseg_free(seg, sizeof(struct uvm_physseg)); 1298 uvm_physseg_graph.nentries--; 1299 #else /* UVM_HOTPLUG */ 1300 int x; 1301 if (vm_nphysmem == 1) 1302 panic("uvm_page_physget: out of memory!"); 1303 vm_nphysmem--; 1304 for (x = upm ; x < vm_nphysmem ; x++) 1305 /* structure copy */ 1306 VM_PHYSMEM_PTR_SWAP(x, x + 1); 1307 #endif /* UVM_HOTPLUG */ 1308 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */ 1309 return true; 1310 } 1311 1312 if (off > 0 && 1313 (pfn + pages) < end) { 1314 #if defined(UVM_HOTPLUG) /* rbtree implementation */ 1315 /* middle chunk - need a new segment */ 1316 struct uvm_physseg *ps, *current_ps; 1317 ps = uvm_physseg_alloc(sizeof (struct uvm_physseg)); 1318 if (ps == NULL) { 1319 printf("%s: Unable to allocated new fragment vm_physseg \n", 1320 __func__); 1321 return false; 1322 } 1323 1324 /* Remove middle chunk */ 1325 if (__predict_true(uvm.page_init_done == true)) { 1326 KASSERT(seg->ext != NULL); 1327 ps->ext = seg->ext; 1328 1329 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */ 1330 /* 1331 * We assume that the unplug will succeed from 1332 * this point onwards 1333 */ 1334 uvmexp.npages -= (int) pages; 1335 } 1336 1337 ps->start = pfn + pages; 1338 ps->avail_start = ps->start; /* XXX: Legacy */ 1339 1340 ps->end = seg->end; 1341 ps->avail_end = ps->end; /* XXX: Legacy */ 1342 1343 seg->end = pfn; 1344 seg->avail_end = seg->end; /* XXX: Legacy */ 1345 1346 1347 /* 1348 * The new pgs array points to the beginning of the 1349 * tail fragment. 1350 */ 1351 if (__predict_true(uvm.page_init_done == true)) 1352 ps->pgs = seg->pgs + off + pages; 1353 1354 current_ps = rb_tree_insert_node(&(uvm_physseg_graph.rb_tree), ps); 1355 if (current_ps != ps) { 1356 panic("uvm_page_physload: Duplicate address range detected!"); 1357 } 1358 uvm_physseg_graph.nentries++; 1359 #else /* UVM_HOTPLUG */ 1360 panic("%s: can't unplug() from the middle of a segment without" 1361 " UVM_HOTPLUG\n", __func__); 1362 /* NOTREACHED */ 1363 #endif /* UVM_HOTPLUG */ 1364 return true; 1365 } 1366 1367 if (off == 0 && (pfn + pages) < end) { 1368 /* Remove front chunk */ 1369 if (__predict_true(uvm.page_init_done == true)) { 1370 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */ 1371 /* 1372 * We assume that the unplug will succeed from 1373 * this point onwards 1374 */ 1375 uvmexp.npages -= (int) pages; 1376 } 1377 1378 /* Truncate */ 1379 seg->start = pfn + pages; 1380 seg->avail_start = seg->start; /* XXX: Legacy */ 1381 1382 /* 1383 * Move the pgs array start to the beginning of the 1384 * tail end. 1385 */ 1386 if (__predict_true(uvm.page_init_done == true)) 1387 seg->pgs += pages; 1388 1389 return true; 1390 } 1391 1392 if (off > 0 && (pfn + pages) == end) { 1393 /* back chunk */ 1394 1395 1396 /* Truncate! */ 1397 seg->end = pfn; 1398 seg->avail_end = seg->end; /* XXX: Legacy */ 1399 1400 uvmexp.npages -= (int) pages; 1401 1402 return true; 1403 } 1404 1405 printf("%s: Tried to unplug unknown range \n", __func__); 1406 1407 return false; 1408 } 1409