1 /* $NetBSD: uvm_pglist.c,v 1.86 2020/10/07 17:51:50 chs Exp $ */ 2 3 /*- 4 * Copyright (c) 1997, 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * uvm_pglist.c: pglist functions 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: uvm_pglist.c,v 1.86 2020/10/07 17:51:50 chs Exp $"); 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/cpu.h> 43 44 #include <uvm/uvm.h> 45 #include <uvm/uvm_pdpolicy.h> 46 #include <uvm/uvm_pgflcache.h> 47 48 #ifdef VM_PAGE_ALLOC_MEMORY_STATS 49 #define STAT_INCR(v) (v)++ 50 #define STAT_DECR(v) do { \ 51 if ((v) == 0) \ 52 printf("%s:%d -- Already 0!\n", __FILE__, __LINE__); \ 53 else \ 54 (v)--; \ 55 } while (/*CONSTCOND*/ 0) 56 u_long uvm_pglistalloc_npages; 57 #else 58 #define STAT_INCR(v) 59 #define STAT_DECR(v) 60 #endif 61 62 kmutex_t uvm_pglistalloc_contig_lock; 63 64 /* 65 * uvm_pglistalloc: allocate a list of pages 66 * 67 * => allocated pages are placed onto an rlist. rlist is 68 * initialized by uvm_pglistalloc. 69 * => returns 0 on success or errno on failure 70 * => implementation allocates a single segment if any constraints are 71 * imposed by call arguments. 72 * => doesn't take into account clean non-busy pages on inactive list 73 * that could be used(?) 74 * => params: 75 * size the size of the allocation, rounded to page size. 76 * low the low address of the allowed allocation range. 77 * high the high address of the allowed allocation range. 78 * alignment memory must be aligned to this power-of-two boundary. 79 * boundary no segment in the allocation may cross this 80 * power-of-two boundary (relative to zero). 81 */ 82 83 static void 84 uvm_pglist_add(struct vm_page *pg, struct pglist *rlist) 85 { 86 struct pgfreelist *pgfl; 87 struct pgflbucket *pgb; 88 89 pgfl = &uvm.page_free[uvm_page_get_freelist(pg)]; 90 pgb = pgfl->pgfl_buckets[uvm_page_get_bucket(pg)]; 91 92 #ifdef UVMDEBUG 93 struct vm_page *tp; 94 LIST_FOREACH(tp, &pgb->pgb_colors[VM_PGCOLOR(pg)], pageq.list) { 95 if (tp == pg) 96 break; 97 } 98 if (tp == NULL) 99 panic("uvm_pglistalloc: page not on freelist"); 100 #endif 101 LIST_REMOVE(pg, pageq.list); 102 pgb->pgb_nfree--; 103 CPU_COUNT(CPU_COUNT_FREEPAGES, -1); 104 pg->flags = PG_CLEAN; 105 pg->uobject = NULL; 106 pg->uanon = NULL; 107 TAILQ_INSERT_TAIL(rlist, pg, pageq.queue); 108 STAT_INCR(uvm_pglistalloc_npages); 109 } 110 111 static int 112 uvm_pglistalloc_c_ps(uvm_physseg_t psi, int num, paddr_t low, paddr_t high, 113 paddr_t alignment, paddr_t boundary, struct pglist *rlist) 114 { 115 signed int candidate, limit, candidateidx, end, idx, skip; 116 int pagemask; 117 bool second_pass; 118 #ifdef DEBUG 119 paddr_t idxpa, lastidxpa; 120 paddr_t cidx = 0; /* XXX: GCC */ 121 #endif 122 #ifdef PGALLOC_VERBOSE 123 printf("pgalloc: contig %d pgs from psi %d\n", num, psi); 124 #endif 125 126 low = atop(low); 127 high = atop(high); 128 alignment = atop(alignment); 129 130 /* 131 * Make sure that physseg falls within with range to be allocated from. 132 */ 133 if (high <= uvm_physseg_get_avail_start(psi) || low >= uvm_physseg_get_avail_end(psi)) 134 return 0; 135 136 /* 137 * We start our search at the just after where the last allocation 138 * succeeded. 139 */ 140 candidate = roundup2(uimax(low, uvm_physseg_get_avail_start(psi) + 141 uvm_physseg_get_start_hint(psi)), alignment); 142 limit = uimin(high, uvm_physseg_get_avail_end(psi)); 143 pagemask = ~((boundary >> PAGE_SHIFT) - 1); 144 skip = 0; 145 second_pass = false; 146 147 for (;;) { 148 bool ok = true; 149 signed int cnt; 150 151 if (candidate + num > limit) { 152 if (uvm_physseg_get_start_hint(psi) == 0 || second_pass) { 153 /* 154 * We've run past the allowable range. 155 */ 156 return 0; /* FAIL = 0 pages*/ 157 } 158 /* 159 * We've wrapped around the end of this segment 160 * so restart at the beginning but now our limit 161 * is were we started. 162 */ 163 second_pass = true; 164 candidate = roundup2(uimax(low, uvm_physseg_get_avail_start(psi)), alignment); 165 limit = uimin(limit, uvm_physseg_get_avail_start(psi) + 166 uvm_physseg_get_start_hint(psi)); 167 skip = 0; 168 continue; 169 } 170 if (boundary != 0 && 171 ((candidate ^ (candidate + num - 1)) & pagemask) != 0) { 172 /* 173 * Region crosses boundary. Jump to the boundary 174 * just crossed and ensure alignment. 175 */ 176 candidate = (candidate + num - 1) & pagemask; 177 candidate = roundup2(candidate, alignment); 178 skip = 0; 179 continue; 180 } 181 #ifdef DEBUG 182 /* 183 * Make sure this is a managed physical page. 184 */ 185 186 if (uvm_physseg_find(candidate, &cidx) != psi) 187 panic("pgalloc contig: botch1"); 188 if (cidx != candidate - uvm_physseg_get_start(psi)) 189 panic("pgalloc contig: botch2"); 190 if (uvm_physseg_find(candidate + num - 1, &cidx) != psi) 191 panic("pgalloc contig: botch3"); 192 if (cidx != candidate - uvm_physseg_get_start(psi) + num - 1) 193 panic("pgalloc contig: botch4"); 194 #endif 195 candidateidx = candidate - uvm_physseg_get_start(psi); 196 end = candidateidx + num; 197 198 /* 199 * Found a suitable starting page. See if the range is free. 200 */ 201 #ifdef PGALLOC_VERBOSE 202 printf("%s: psi=%d candidate=%#x end=%#x skip=%#x, align=%#"PRIxPADDR, 203 __func__, psi, candidateidx, end, skip, alignment); 204 #endif 205 /* 206 * We start at the end and work backwards since if we find a 207 * non-free page, it makes no sense to continue. 208 * 209 * But on the plus size we have "vetted" some number of free 210 * pages. If this iteration fails, we may be able to skip 211 * testing most of those pages again in the next pass. 212 */ 213 for (idx = end - 1; idx >= candidateidx + skip; idx--) { 214 if (VM_PAGE_IS_FREE(uvm_physseg_get_pg(psi, idx)) == 0) { 215 ok = false; 216 break; 217 } 218 219 #ifdef DEBUG 220 if (idx > candidateidx) { 221 idxpa = VM_PAGE_TO_PHYS(uvm_physseg_get_pg(psi, idx)); 222 lastidxpa = VM_PAGE_TO_PHYS(uvm_physseg_get_pg(psi, idx - 1)); 223 if ((lastidxpa + PAGE_SIZE) != idxpa) { 224 /* 225 * Region not contiguous. 226 */ 227 panic("pgalloc contig: botch5"); 228 } 229 if (boundary != 0 && 230 ((lastidxpa ^ idxpa) & ~(boundary - 1)) 231 != 0) { 232 /* 233 * Region crosses boundary. 234 */ 235 panic("pgalloc contig: botch6"); 236 } 237 } 238 #endif 239 } 240 241 if (ok) { 242 while (skip-- > 0) { 243 KDASSERT(VM_PAGE_IS_FREE(uvm_physseg_get_pg(psi, candidateidx + skip))); 244 } 245 #ifdef PGALLOC_VERBOSE 246 printf(": ok\n"); 247 #endif 248 break; 249 } 250 251 #ifdef PGALLOC_VERBOSE 252 printf(": non-free at %#x\n", idx - candidateidx); 253 #endif 254 /* 255 * count the number of pages we can advance 256 * since we know they aren't all free. 257 */ 258 cnt = idx + 1 - candidateidx; 259 /* 260 * now round up that to the needed alignment. 261 */ 262 cnt = roundup2(cnt, alignment); 263 /* 264 * The number of pages we can skip checking 265 * (might be 0 if cnt > num). 266 */ 267 skip = uimax(num - cnt, 0); 268 candidate += cnt; 269 } 270 271 /* 272 * we have a chunk of memory that conforms to the requested constraints. 273 */ 274 for (idx = candidateidx; idx < end; idx++) 275 uvm_pglist_add(uvm_physseg_get_pg(psi, idx), rlist); 276 277 /* 278 * the next time we need to search this segment, start after this 279 * chunk of pages we just allocated. 280 */ 281 uvm_physseg_set_start_hint(psi, candidate + num - 282 uvm_physseg_get_avail_start(psi)); 283 KASSERTMSG(uvm_physseg_get_start_hint(psi) <= 284 uvm_physseg_get_avail_end(psi) - uvm_physseg_get_avail_start(psi), 285 "%x %u (%#x) <= %#"PRIxPADDR" - %#"PRIxPADDR" (%#"PRIxPADDR")", 286 candidate + num, 287 uvm_physseg_get_start_hint(psi), uvm_physseg_get_start_hint(psi), 288 uvm_physseg_get_avail_end(psi), uvm_physseg_get_avail_start(psi), 289 uvm_physseg_get_avail_end(psi) - uvm_physseg_get_avail_start(psi)); 290 291 #ifdef PGALLOC_VERBOSE 292 printf("got %d pgs\n", num); 293 #endif 294 return num; /* number of pages allocated */ 295 } 296 297 static int 298 uvm_pglistalloc_contig_aggressive(int num, paddr_t low, paddr_t high, 299 paddr_t alignment, paddr_t boundary, struct pglist *rlist) 300 { 301 struct vm_page *pg; 302 struct pglist tmp; 303 paddr_t pa, off, spa, amask, bmask, rlo, rhi; 304 uvm_physseg_t upm; 305 int error, i, run, acnt; 306 307 /* 308 * Allocate pages the normal way and for each new page, check if 309 * the page completes a range satisfying the request. 310 * The pagedaemon will evict pages as we go and we are very likely 311 * to get compatible pages eventually. 312 */ 313 314 error = ENOMEM; 315 TAILQ_INIT(&tmp); 316 acnt = atop(alignment); 317 amask = ~(alignment - 1); 318 bmask = ~(boundary - 1); 319 KASSERT(bmask <= amask); 320 mutex_enter(&uvm_pglistalloc_contig_lock); 321 while (uvm_reclaimable()) { 322 pg = uvm_pagealloc(NULL, 0, NULL, 0); 323 if (pg == NULL) { 324 uvm_wait("pglac2"); 325 continue; 326 } 327 pg->flags |= PG_PGLCA; 328 TAILQ_INSERT_HEAD(&tmp, pg, pageq.queue); 329 330 pa = VM_PAGE_TO_PHYS(pg); 331 if (pa < low || pa >= high) { 332 continue; 333 } 334 335 upm = uvm_physseg_find(atop(pa), &off); 336 KASSERT(uvm_physseg_valid_p(upm)); 337 338 spa = pa & amask; 339 340 /* 341 * Look backward for at most num - 1 pages, back to 342 * the highest of: 343 * - the first page in the physseg 344 * - the specified low address 345 * - num-1 pages before the one we just allocated 346 * - the start of the boundary range containing pa 347 * all rounded up to alignment. 348 */ 349 350 rlo = roundup2(ptoa(uvm_physseg_get_avail_start(upm)), alignment); 351 rlo = MAX(rlo, roundup2(low, alignment)); 352 rlo = MAX(rlo, roundup2(pa - ptoa(num - 1), alignment)); 353 if (boundary) { 354 rlo = MAX(rlo, spa & bmask); 355 } 356 357 /* 358 * Look forward as far as the lowest of: 359 * - the last page of the physseg 360 * - the specified high address 361 * - the boundary after pa 362 */ 363 364 rhi = ptoa(uvm_physseg_get_avail_end(upm)); 365 rhi = MIN(rhi, high); 366 if (boundary) { 367 rhi = MIN(rhi, rounddown2(pa, boundary) + boundary); 368 } 369 370 /* 371 * Make sure our range to consider is big enough. 372 */ 373 374 if (rhi - rlo < ptoa(num)) { 375 continue; 376 } 377 378 run = 0; 379 while (spa > rlo) { 380 381 /* 382 * Examine pages before spa in groups of acnt. 383 * If all the pages in a group are marked then add 384 * these pages to the run. 385 */ 386 387 for (i = 0; i < acnt; i++) { 388 pg = PHYS_TO_VM_PAGE(spa - alignment + ptoa(i)); 389 if ((pg->flags & PG_PGLCA) == 0) { 390 break; 391 } 392 } 393 if (i < acnt) { 394 break; 395 } 396 spa -= alignment; 397 run += acnt; 398 } 399 400 /* 401 * Look forward for any remaining pages. 402 */ 403 404 for (; run < num; run++) { 405 pg = PHYS_TO_VM_PAGE(spa + ptoa(run)); 406 if ((pg->flags & PG_PGLCA) == 0) { 407 break; 408 } 409 } 410 if (run < num) { 411 continue; 412 } 413 414 /* 415 * We found a match. Move these pages from the tmp list to 416 * the caller's list. 417 */ 418 419 for (i = 0; i < num; i++) { 420 pg = PHYS_TO_VM_PAGE(spa + ptoa(i)); 421 TAILQ_REMOVE(&tmp, pg, pageq.queue); 422 pg->flags &= ~PG_PGLCA; 423 TAILQ_INSERT_TAIL(rlist, pg, pageq.queue); 424 STAT_INCR(uvm_pglistalloc_npages); 425 } 426 427 error = 0; 428 break; 429 } 430 431 /* 432 * Free all the pages that we didn't need. 433 */ 434 435 while (!TAILQ_EMPTY(&tmp)) { 436 pg = TAILQ_FIRST(&tmp); 437 TAILQ_REMOVE(&tmp, pg, pageq.queue); 438 pg->flags &= ~PG_PGLCA; 439 uvm_pagefree(pg); 440 } 441 mutex_exit(&uvm_pglistalloc_contig_lock); 442 return error; 443 } 444 445 static int 446 uvm_pglistalloc_contig(int num, paddr_t low, paddr_t high, paddr_t alignment, 447 paddr_t boundary, struct pglist *rlist, int waitok) 448 { 449 int fl; 450 int error; 451 uvm_physseg_t psi; 452 453 /* Default to "lose". */ 454 error = ENOMEM; 455 456 /* 457 * Block all memory allocation and lock the free list. 458 */ 459 uvm_pgfl_lock(); 460 461 /* Are there even any free pages? */ 462 if (uvm_availmem(false) <= 463 (uvmexp.reserve_pagedaemon + uvmexp.reserve_kernel)) 464 goto out; 465 466 for (fl = 0; fl < VM_NFREELIST; fl++) { 467 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST) 468 for (psi = uvm_physseg_get_last(); uvm_physseg_valid_p(psi); psi = uvm_physseg_get_prev(psi)) 469 #else 470 for (psi = uvm_physseg_get_first(); uvm_physseg_valid_p(psi); psi = uvm_physseg_get_next(psi)) 471 #endif 472 { 473 if (uvm_physseg_get_free_list(psi) != fl) 474 continue; 475 476 num -= uvm_pglistalloc_c_ps(psi, num, low, high, 477 alignment, boundary, rlist); 478 if (num == 0) { 479 #ifdef PGALLOC_VERBOSE 480 printf("pgalloc: %"PRIxMAX"-%"PRIxMAX"\n", 481 (uintmax_t) VM_PAGE_TO_PHYS(TAILQ_FIRST(rlist)), 482 (uintmax_t) VM_PAGE_TO_PHYS(TAILQ_LAST(rlist, pglist))); 483 #endif 484 error = 0; 485 goto out; 486 } 487 } 488 } 489 490 out: 491 uvm_pgfl_unlock(); 492 493 /* 494 * If that didn't work, try the more aggressive approach. 495 */ 496 497 if (error) { 498 if (waitok) { 499 error = uvm_pglistalloc_contig_aggressive(num, low, high, 500 alignment, boundary, rlist); 501 } else { 502 uvm_pglistfree(rlist); 503 uvm_kick_pdaemon(); 504 } 505 } 506 return error; 507 } 508 509 static int 510 uvm_pglistalloc_s_ps(uvm_physseg_t psi, int num, paddr_t low, paddr_t high, 511 struct pglist *rlist) 512 { 513 int todo, limit, candidate; 514 struct vm_page *pg; 515 bool second_pass; 516 #ifdef PGALLOC_VERBOSE 517 printf("pgalloc: simple %d pgs from psi %d\n", num, psi); 518 #endif 519 520 KASSERT(uvm_physseg_get_start(psi) <= uvm_physseg_get_avail_start(psi)); 521 KASSERT(uvm_physseg_get_start(psi) <= uvm_physseg_get_avail_end(psi)); 522 KASSERT(uvm_physseg_get_avail_start(psi) <= uvm_physseg_get_end(psi)); 523 KASSERT(uvm_physseg_get_avail_end(psi) <= uvm_physseg_get_end(psi)); 524 525 low = atop(low); 526 high = atop(high); 527 todo = num; 528 candidate = uimax(low, uvm_physseg_get_avail_start(psi) + 529 uvm_physseg_get_start_hint(psi)); 530 limit = uimin(high, uvm_physseg_get_avail_end(psi)); 531 pg = uvm_physseg_get_pg(psi, candidate - uvm_physseg_get_start(psi)); 532 second_pass = false; 533 534 /* 535 * Make sure that physseg falls within with range to be allocated from. 536 */ 537 if (high <= uvm_physseg_get_avail_start(psi) || 538 low >= uvm_physseg_get_avail_end(psi)) 539 return 0; 540 541 again: 542 for (;; candidate++, pg++) { 543 if (candidate >= limit) { 544 if (uvm_physseg_get_start_hint(psi) == 0 || second_pass) { 545 candidate = limit - 1; 546 break; 547 } 548 second_pass = true; 549 candidate = uimax(low, uvm_physseg_get_avail_start(psi)); 550 limit = uimin(limit, uvm_physseg_get_avail_start(psi) + 551 uvm_physseg_get_start_hint(psi)); 552 pg = uvm_physseg_get_pg(psi, candidate - uvm_physseg_get_start(psi)); 553 goto again; 554 } 555 #if defined(DEBUG) 556 { 557 paddr_t cidx = 0; 558 const uvm_physseg_t bank = uvm_physseg_find(candidate, &cidx); 559 KDASSERTMSG(bank == psi, 560 "uvm_physseg_find(%#x) (%"PRIxPHYSSEG ") != psi %"PRIxPHYSSEG, 561 candidate, bank, psi); 562 KDASSERTMSG(cidx == candidate - uvm_physseg_get_start(psi), 563 "uvm_physseg_find(%#x): %#"PRIxPADDR" != off %"PRIxPADDR, 564 candidate, cidx, candidate - uvm_physseg_get_start(psi)); 565 } 566 #endif 567 if (VM_PAGE_IS_FREE(pg) == 0) 568 continue; 569 570 uvm_pglist_add(pg, rlist); 571 if (--todo == 0) { 572 break; 573 } 574 } 575 576 /* 577 * The next time we need to search this segment, 578 * start just after the pages we just allocated. 579 */ 580 uvm_physseg_set_start_hint(psi, candidate + 1 - uvm_physseg_get_avail_start(psi)); 581 KASSERTMSG(uvm_physseg_get_start_hint(psi) <= uvm_physseg_get_avail_end(psi) - 582 uvm_physseg_get_avail_start(psi), 583 "%#x %u (%#x) <= %#"PRIxPADDR" - %#"PRIxPADDR" (%#"PRIxPADDR")", 584 candidate + 1, 585 uvm_physseg_get_start_hint(psi), 586 uvm_physseg_get_start_hint(psi), 587 uvm_physseg_get_avail_end(psi), 588 uvm_physseg_get_avail_start(psi), 589 uvm_physseg_get_avail_end(psi) - uvm_physseg_get_avail_start(psi)); 590 591 #ifdef PGALLOC_VERBOSE 592 printf("got %d pgs\n", num - todo); 593 #endif 594 return (num - todo); /* number of pages allocated */ 595 } 596 597 static int 598 uvm_pglistalloc_simple(int num, paddr_t low, paddr_t high, 599 struct pglist *rlist, int waitok) 600 { 601 int fl, error; 602 uvm_physseg_t psi; 603 int count = 0; 604 605 /* Default to "lose". */ 606 error = ENOMEM; 607 608 again: 609 /* 610 * Block all memory allocation and lock the free list. 611 */ 612 uvm_pgfl_lock(); 613 count++; 614 615 /* Are there even any free pages? */ 616 if (uvm_availmem(false) <= 617 (uvmexp.reserve_pagedaemon + uvmexp.reserve_kernel)) 618 goto out; 619 620 for (fl = 0; fl < VM_NFREELIST; fl++) { 621 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST) 622 for (psi = uvm_physseg_get_last(); uvm_physseg_valid_p(psi); psi = uvm_physseg_get_prev(psi)) 623 #else 624 for (psi = uvm_physseg_get_first(); uvm_physseg_valid_p(psi); psi = uvm_physseg_get_next(psi)) 625 #endif 626 { 627 if (uvm_physseg_get_free_list(psi) != fl) 628 continue; 629 630 num -= uvm_pglistalloc_s_ps(psi, num, low, high, rlist); 631 if (num == 0) { 632 error = 0; 633 goto out; 634 } 635 } 636 637 } 638 639 out: 640 /* 641 * check to see if we need to generate some free pages waking 642 * the pagedaemon. 643 */ 644 645 uvm_pgfl_unlock(); 646 uvm_kick_pdaemon(); 647 648 if (error) { 649 if (waitok) { 650 uvm_wait("pglalloc"); 651 goto again; 652 } else 653 uvm_pglistfree(rlist); 654 } 655 #ifdef PGALLOC_VERBOSE 656 if (!error) 657 printf("pgalloc: %"PRIxMAX"..%"PRIxMAX"\n", 658 (uintmax_t) VM_PAGE_TO_PHYS(TAILQ_FIRST(rlist)), 659 (uintmax_t) VM_PAGE_TO_PHYS(TAILQ_LAST(rlist, pglist))); 660 #endif 661 return (error); 662 } 663 664 int 665 uvm_pglistalloc(psize_t size, paddr_t low, paddr_t high, paddr_t alignment, 666 paddr_t boundary, struct pglist *rlist, int nsegs, int waitok) 667 { 668 int num, res; 669 670 KASSERT(!cpu_intr_p()); 671 KASSERT(!cpu_softintr_p()); 672 KASSERT((alignment & (alignment - 1)) == 0); 673 KASSERT((boundary & (boundary - 1)) == 0); 674 675 /* 676 * Our allocations are always page granularity, so our alignment 677 * must be, too. 678 */ 679 if (alignment < PAGE_SIZE) 680 alignment = PAGE_SIZE; 681 if (boundary != 0 && boundary < size) 682 return (EINVAL); 683 num = atop(round_page(size)); 684 low = roundup2(low, alignment); 685 686 TAILQ_INIT(rlist); 687 688 /* 689 * Turn off the caching of free pages - we need everything to be on 690 * the global freelists. 691 */ 692 uvm_pgflcache_pause(); 693 694 if (nsegs < num || alignment != PAGE_SIZE || boundary != 0) 695 res = uvm_pglistalloc_contig(num, low, high, alignment, 696 boundary, rlist, waitok); 697 else 698 res = uvm_pglistalloc_simple(num, low, high, rlist, waitok); 699 700 uvm_pgflcache_resume(); 701 702 return (res); 703 } 704 705 /* 706 * uvm_pglistfree: free a list of pages 707 * 708 * => pages should already be unmapped 709 */ 710 711 void 712 uvm_pglistfree(struct pglist *list) 713 { 714 struct vm_page *pg; 715 716 KASSERT(!cpu_intr_p()); 717 KASSERT(!cpu_softintr_p()); 718 719 while ((pg = TAILQ_FIRST(list)) != NULL) { 720 TAILQ_REMOVE(list, pg, pageq.queue); 721 uvm_pagefree(pg); 722 STAT_DECR(uvm_pglistalloc_npages); 723 } 724 } 725 726 void 727 uvm_pglistalloc_init(void) 728 { 729 730 mutex_init(&uvm_pglistalloc_contig_lock, MUTEX_DEFAULT, IPL_NONE); 731 } 732