1 /* $NetBSD: uvm_bio.c,v 1.70 2010/06/22 18:34:50 rmind Exp $ */ 2 3 /* 4 * Copyright (c) 1998 Chuck Silvers. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32 /* 33 * uvm_bio.c: buffered i/o object mapping cache 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.70 2010/06/22 18:34:50 rmind Exp $"); 38 39 #include "opt_uvmhist.h" 40 #include "opt_ubc.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kmem.h> 45 #include <sys/kernel.h> 46 #include <sys/proc.h> 47 #include <sys/vnode.h> 48 49 #include <uvm/uvm.h> 50 51 /* 52 * global data structures 53 */ 54 55 /* 56 * local functions 57 */ 58 59 static int ubc_fault(struct uvm_faultinfo *, vaddr_t, struct vm_page **, 60 int, int, vm_prot_t, int); 61 static struct ubc_map *ubc_find_mapping(struct uvm_object *, voff_t); 62 63 /* 64 * local data structues 65 */ 66 67 #define UBC_HASH(uobj, offset) \ 68 (((((u_long)(uobj)) >> 8) + (((u_long)(offset)) >> PAGE_SHIFT)) & \ 69 ubc_object.hashmask) 70 71 #define UBC_QUEUE(offset) \ 72 (&ubc_object.inactive[(((u_long)(offset)) >> ubc_winshift) & \ 73 (UBC_NQUEUES - 1)]) 74 75 #define UBC_UMAP_ADDR(u) \ 76 (vaddr_t)(ubc_object.kva + (((u) - ubc_object.umap) << ubc_winshift)) 77 78 79 #define UMAP_PAGES_LOCKED 0x0001 80 #define UMAP_MAPPING_CACHED 0x0002 81 82 struct ubc_map 83 { 84 struct uvm_object * uobj; /* mapped object */ 85 voff_t offset; /* offset into uobj */ 86 voff_t writeoff; /* write offset */ 87 vsize_t writelen; /* write len */ 88 int refcount; /* refcount on mapping */ 89 int flags; /* extra state */ 90 int advice; 91 92 LIST_ENTRY(ubc_map) hash; /* hash table */ 93 TAILQ_ENTRY(ubc_map) inactive; /* inactive queue */ 94 }; 95 96 static struct ubc_object 97 { 98 struct uvm_object uobj; /* glue for uvm_map() */ 99 char *kva; /* where ubc_object is mapped */ 100 struct ubc_map *umap; /* array of ubc_map's */ 101 102 LIST_HEAD(, ubc_map) *hash; /* hashtable for cached ubc_map's */ 103 u_long hashmask; /* mask for hashtable */ 104 105 TAILQ_HEAD(ubc_inactive_head, ubc_map) *inactive; 106 /* inactive queues for ubc_map's */ 107 108 } ubc_object; 109 110 const struct uvm_pagerops ubc_pager = { 111 .pgo_fault = ubc_fault, 112 /* ... rest are NULL */ 113 }; 114 115 int ubc_nwins = UBC_NWINS; 116 int ubc_winshift = UBC_WINSHIFT; 117 int ubc_winsize; 118 #if defined(PMAP_PREFER) 119 int ubc_nqueues; 120 #define UBC_NQUEUES ubc_nqueues 121 #else 122 #define UBC_NQUEUES 1 123 #endif 124 125 #if defined(UBC_STATS) 126 127 #define UBC_EVCNT_DEFINE(name) \ 128 struct evcnt ubc_evcnt_##name = \ 129 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "ubc", #name); \ 130 EVCNT_ATTACH_STATIC(ubc_evcnt_##name); 131 #define UBC_EVCNT_INCR(name) ubc_evcnt_##name.ev_count++ 132 133 #else /* defined(UBC_STATS) */ 134 135 #define UBC_EVCNT_DEFINE(name) /* nothing */ 136 #define UBC_EVCNT_INCR(name) /* nothing */ 137 138 #endif /* defined(UBC_STATS) */ 139 140 UBC_EVCNT_DEFINE(wincachehit) 141 UBC_EVCNT_DEFINE(wincachemiss) 142 UBC_EVCNT_DEFINE(faultbusy) 143 144 /* 145 * ubc_init 146 * 147 * init pager private data structures. 148 */ 149 150 void 151 ubc_init(void) 152 { 153 struct ubc_map *umap; 154 vaddr_t va; 155 int i; 156 157 /* 158 * Make sure ubc_winshift is sane. 159 */ 160 if (ubc_winshift < PAGE_SHIFT) 161 ubc_winshift = PAGE_SHIFT; 162 163 /* 164 * init ubc_object. 165 * alloc and init ubc_map's. 166 * init inactive queues. 167 * alloc and init hashtable. 168 * map in ubc_object. 169 */ 170 171 UVM_OBJ_INIT(&ubc_object.uobj, &ubc_pager, UVM_OBJ_KERN); 172 173 ubc_object.umap = kmem_zalloc(ubc_nwins * sizeof(struct ubc_map), 174 KM_SLEEP); 175 if (ubc_object.umap == NULL) 176 panic("ubc_init: failed to allocate ubc_map"); 177 178 if (ubc_winshift < PAGE_SHIFT) { 179 ubc_winshift = PAGE_SHIFT; 180 } 181 va = (vaddr_t)1L; 182 #ifdef PMAP_PREFER 183 PMAP_PREFER(0, &va, 0, 0); /* kernel is never topdown */ 184 ubc_nqueues = va >> ubc_winshift; 185 if (ubc_nqueues == 0) { 186 ubc_nqueues = 1; 187 } 188 #endif 189 ubc_winsize = 1 << ubc_winshift; 190 ubc_object.inactive = kmem_alloc(UBC_NQUEUES * 191 sizeof(struct ubc_inactive_head), KM_SLEEP); 192 if (ubc_object.inactive == NULL) 193 panic("ubc_init: failed to allocate inactive queue heads"); 194 for (i = 0; i < UBC_NQUEUES; i++) { 195 TAILQ_INIT(&ubc_object.inactive[i]); 196 } 197 for (i = 0; i < ubc_nwins; i++) { 198 umap = &ubc_object.umap[i]; 199 TAILQ_INSERT_TAIL(&ubc_object.inactive[i & (UBC_NQUEUES - 1)], 200 umap, inactive); 201 } 202 203 ubc_object.hash = hashinit(ubc_nwins, HASH_LIST, true, 204 &ubc_object.hashmask); 205 for (i = 0; i <= ubc_object.hashmask; i++) { 206 LIST_INIT(&ubc_object.hash[i]); 207 } 208 209 if (uvm_map(kernel_map, (vaddr_t *)&ubc_object.kva, 210 ubc_nwins << ubc_winshift, &ubc_object.uobj, 0, (vsize_t)va, 211 UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE, 212 UVM_ADV_RANDOM, UVM_FLAG_NOMERGE)) != 0) { 213 panic("ubc_init: failed to map ubc_object"); 214 } 215 UVMHIST_INIT(ubchist, 300); 216 } 217 218 /* 219 * ubc_fault_page: helper of ubc_fault to handle a single page. 220 * 221 * => Caller has UVM object locked. 222 */ 223 224 static inline int 225 ubc_fault_page(const struct uvm_faultinfo *ufi, const struct ubc_map *umap, 226 struct vm_page *pg, vm_prot_t prot, vm_prot_t access_type, vaddr_t va) 227 { 228 struct uvm_object *uobj; 229 vm_prot_t mask; 230 int error; 231 bool rdonly; 232 233 uobj = pg->uobject; 234 KASSERT(mutex_owned(&uobj->vmobjlock)); 235 236 if (pg->flags & PG_WANTED) { 237 wakeup(pg); 238 } 239 KASSERT((pg->flags & PG_FAKE) == 0); 240 if (pg->flags & PG_RELEASED) { 241 mutex_enter(&uvm_pageqlock); 242 uvm_pagefree(pg); 243 mutex_exit(&uvm_pageqlock); 244 return 0; 245 } 246 if (pg->loan_count != 0) { 247 248 /* 249 * Avoid unneeded loan break, if possible. 250 */ 251 252 if ((access_type & VM_PROT_WRITE) == 0) { 253 prot &= ~VM_PROT_WRITE; 254 } 255 if (prot & VM_PROT_WRITE) { 256 struct vm_page *newpg; 257 258 newpg = uvm_loanbreak(pg); 259 if (newpg == NULL) { 260 uvm_page_unbusy(&pg, 1); 261 return ENOMEM; 262 } 263 pg = newpg; 264 } 265 } 266 267 /* 268 * Note that a page whose backing store is partially allocated 269 * is marked as PG_RDONLY. 270 */ 271 272 KASSERT((pg->flags & PG_RDONLY) == 0 || 273 (access_type & VM_PROT_WRITE) == 0 || 274 pg->offset < umap->writeoff || 275 pg->offset + PAGE_SIZE > umap->writeoff + umap->writelen); 276 277 rdonly = ((access_type & VM_PROT_WRITE) == 0 && 278 (pg->flags & PG_RDONLY) != 0) || 279 UVM_OBJ_NEEDS_WRITEFAULT(uobj); 280 mask = rdonly ? ~VM_PROT_WRITE : VM_PROT_ALL; 281 282 error = pmap_enter(ufi->orig_map->pmap, va, VM_PAGE_TO_PHYS(pg), 283 prot & mask, PMAP_CANFAIL | (access_type & mask)); 284 285 mutex_enter(&uvm_pageqlock); 286 uvm_pageactivate(pg); 287 mutex_exit(&uvm_pageqlock); 288 pg->flags &= ~(PG_BUSY|PG_WANTED); 289 UVM_PAGE_OWN(pg, NULL); 290 291 return error; 292 } 293 294 /* 295 * ubc_fault: fault routine for ubc mapping 296 */ 297 298 static int 299 ubc_fault(struct uvm_faultinfo *ufi, vaddr_t ign1, struct vm_page **ign2, 300 int ign3, int ign4, vm_prot_t access_type, int flags) 301 { 302 struct uvm_object *uobj; 303 struct ubc_map *umap; 304 vaddr_t va, eva, ubc_offset, slot_offset; 305 struct vm_page *pgs[ubc_winsize >> PAGE_SHIFT]; 306 int i, error, npages; 307 vm_prot_t prot; 308 309 UVMHIST_FUNC("ubc_fault"); UVMHIST_CALLED(ubchist); 310 311 /* 312 * no need to try with PGO_LOCKED... 313 * we don't need to have the map locked since we know that 314 * no one will mess with it until our reference is released. 315 */ 316 317 if (flags & PGO_LOCKED) { 318 uvmfault_unlockall(ufi, NULL, &ubc_object.uobj, NULL); 319 flags &= ~PGO_LOCKED; 320 } 321 322 va = ufi->orig_rvaddr; 323 ubc_offset = va - (vaddr_t)ubc_object.kva; 324 umap = &ubc_object.umap[ubc_offset >> ubc_winshift]; 325 KASSERT(umap->refcount != 0); 326 KASSERT((umap->flags & UMAP_PAGES_LOCKED) == 0); 327 slot_offset = ubc_offset & (ubc_winsize - 1); 328 329 /* 330 * some platforms cannot write to individual bytes atomically, so 331 * software has to do read/modify/write of larger quantities instead. 332 * this means that the access_type for "write" operations 333 * can be VM_PROT_READ, which confuses us mightily. 334 * 335 * deal with this by resetting access_type based on the info 336 * that ubc_alloc() stores for us. 337 */ 338 339 access_type = umap->writelen ? VM_PROT_WRITE : VM_PROT_READ; 340 UVMHIST_LOG(ubchist, "va 0x%lx ubc_offset 0x%lx access_type %d", 341 va, ubc_offset, access_type, 0); 342 343 #ifdef DIAGNOSTIC 344 if ((access_type & VM_PROT_WRITE) != 0) { 345 if (slot_offset < trunc_page(umap->writeoff) || 346 umap->writeoff + umap->writelen <= slot_offset) { 347 panic("ubc_fault: out of range write"); 348 } 349 } 350 #endif 351 352 /* no umap locking needed since we have a ref on the umap */ 353 uobj = umap->uobj; 354 355 if ((access_type & VM_PROT_WRITE) == 0) { 356 npages = (ubc_winsize - slot_offset) >> PAGE_SHIFT; 357 } else { 358 npages = (round_page(umap->offset + umap->writeoff + 359 umap->writelen) - (umap->offset + slot_offset)) 360 >> PAGE_SHIFT; 361 flags |= PGO_PASTEOF; 362 } 363 364 again: 365 memset(pgs, 0, sizeof (pgs)); 366 mutex_enter(&uobj->vmobjlock); 367 368 UVMHIST_LOG(ubchist, "slot_offset 0x%x writeoff 0x%x writelen 0x%x ", 369 slot_offset, umap->writeoff, umap->writelen, 0); 370 UVMHIST_LOG(ubchist, "getpages uobj %p offset 0x%x npages %d", 371 uobj, umap->offset + slot_offset, npages, 0); 372 373 error = (*uobj->pgops->pgo_get)(uobj, umap->offset + slot_offset, pgs, 374 &npages, 0, access_type, umap->advice, flags | PGO_NOBLOCKALLOC | 375 PGO_NOTIMESTAMP); 376 UVMHIST_LOG(ubchist, "getpages error %d npages %d", error, npages, 0, 377 0); 378 379 if (error == EAGAIN) { 380 kpause("ubc_fault", false, hz >> 2, NULL); 381 goto again; 382 } 383 if (error) { 384 return error; 385 } 386 387 /* 388 * For virtually-indexed, virtually-tagged caches we should avoid 389 * creating writable mappings when we do not absolutely need them, 390 * since the "compatible alias" trick does not work on such caches. 391 * Otherwise, we can always map the pages writable. 392 */ 393 394 #ifdef PMAP_CACHE_VIVT 395 prot = VM_PROT_READ | access_type; 396 #else 397 prot = VM_PROT_READ | VM_PROT_WRITE; 398 #endif 399 400 /* 401 * Note: in the common case, all returned pages would have the same 402 * UVM object. However, due to layered file-systems and e.g. tmpfs, 403 * returned pages may have different objects. We "remember" the 404 * last object in the loop to reduce locking overhead and to perform 405 * pmap_update() before object unlock. 406 */ 407 uobj = NULL; 408 409 va = ufi->orig_rvaddr; 410 eva = ufi->orig_rvaddr + (npages << PAGE_SHIFT); 411 412 UVMHIST_LOG(ubchist, "va 0x%lx eva 0x%lx", va, eva, 0, 0); 413 for (i = 0; va < eva; i++, va += PAGE_SIZE) { 414 struct vm_page *pg; 415 416 UVMHIST_LOG(ubchist, "pgs[%d] = %p", i, pgs[i], 0, 0); 417 pg = pgs[i]; 418 419 if (pg == NULL || pg == PGO_DONTCARE) { 420 continue; 421 } 422 if (__predict_false(pg->uobject != uobj)) { 423 /* Check for the first iteration and error cases. */ 424 if (uobj != NULL) { 425 /* Must make VA visible before the unlock. */ 426 pmap_update(ufi->orig_map->pmap); 427 mutex_exit(&uobj->vmobjlock); 428 } 429 uobj = pg->uobject; 430 mutex_enter(&uobj->vmobjlock); 431 } 432 error = ubc_fault_page(ufi, umap, pg, prot, access_type, va); 433 if (error) { 434 /* 435 * Flush (there might be pages entered), drop the lock, 436 * "forget" the object and perform uvm_wait(). 437 * Note: page will re-fault. 438 */ 439 pmap_update(ufi->orig_map->pmap); 440 mutex_exit(&uobj->vmobjlock); 441 uobj = NULL; 442 uvm_wait("ubc_fault"); 443 } 444 } 445 if (__predict_true(uobj != NULL)) { 446 pmap_update(ufi->orig_map->pmap); 447 mutex_exit(&uobj->vmobjlock); 448 } 449 return 0; 450 } 451 452 /* 453 * local functions 454 */ 455 456 static struct ubc_map * 457 ubc_find_mapping(struct uvm_object *uobj, voff_t offset) 458 { 459 struct ubc_map *umap; 460 461 LIST_FOREACH(umap, &ubc_object.hash[UBC_HASH(uobj, offset)], hash) { 462 if (umap->uobj == uobj && umap->offset == offset) { 463 return umap; 464 } 465 } 466 return NULL; 467 } 468 469 470 /* 471 * ubc interface functions 472 */ 473 474 /* 475 * ubc_alloc: allocate a file mapping window 476 */ 477 478 void * 479 ubc_alloc(struct uvm_object *uobj, voff_t offset, vsize_t *lenp, int advice, 480 int flags) 481 { 482 vaddr_t slot_offset, va; 483 struct ubc_map *umap; 484 voff_t umap_offset; 485 int error; 486 UVMHIST_FUNC("ubc_alloc"); UVMHIST_CALLED(ubchist); 487 488 UVMHIST_LOG(ubchist, "uobj %p offset 0x%lx len 0x%lx", 489 uobj, offset, *lenp, 0); 490 491 KASSERT(*lenp > 0); 492 umap_offset = (offset & ~((voff_t)ubc_winsize - 1)); 493 slot_offset = (vaddr_t)(offset & ((voff_t)ubc_winsize - 1)); 494 *lenp = MIN(*lenp, ubc_winsize - slot_offset); 495 496 /* 497 * the object is always locked here, so we don't need to add a ref. 498 */ 499 500 again: 501 mutex_enter(&ubc_object.uobj.vmobjlock); 502 umap = ubc_find_mapping(uobj, umap_offset); 503 if (umap == NULL) { 504 UBC_EVCNT_INCR(wincachemiss); 505 umap = TAILQ_FIRST(UBC_QUEUE(offset)); 506 if (umap == NULL) { 507 mutex_exit(&ubc_object.uobj.vmobjlock); 508 kpause("ubc_alloc", false, hz, NULL); 509 goto again; 510 } 511 512 /* 513 * remove from old hash (if any), add to new hash. 514 */ 515 516 if (umap->uobj != NULL) { 517 LIST_REMOVE(umap, hash); 518 } 519 umap->uobj = uobj; 520 umap->offset = umap_offset; 521 LIST_INSERT_HEAD(&ubc_object.hash[UBC_HASH(uobj, umap_offset)], 522 umap, hash); 523 va = UBC_UMAP_ADDR(umap); 524 if (umap->flags & UMAP_MAPPING_CACHED) { 525 umap->flags &= ~UMAP_MAPPING_CACHED; 526 pmap_remove(pmap_kernel(), va, va + ubc_winsize); 527 pmap_update(pmap_kernel()); 528 } 529 } else { 530 UBC_EVCNT_INCR(wincachehit); 531 va = UBC_UMAP_ADDR(umap); 532 } 533 534 if (umap->refcount == 0) { 535 TAILQ_REMOVE(UBC_QUEUE(offset), umap, inactive); 536 } 537 538 #ifdef DIAGNOSTIC 539 if ((flags & UBC_WRITE) && (umap->writeoff || umap->writelen)) { 540 panic("ubc_alloc: concurrent writes uobj %p", uobj); 541 } 542 #endif 543 if (flags & UBC_WRITE) { 544 umap->writeoff = slot_offset; 545 umap->writelen = *lenp; 546 } 547 548 umap->refcount++; 549 umap->advice = advice; 550 mutex_exit(&ubc_object.uobj.vmobjlock); 551 UVMHIST_LOG(ubchist, "umap %p refs %d va %p flags 0x%x", 552 umap, umap->refcount, va, flags); 553 554 if (flags & UBC_FAULTBUSY) { 555 int npages = (*lenp + PAGE_SIZE - 1) >> PAGE_SHIFT; 556 struct vm_page *pgs[npages]; 557 int gpflags = 558 PGO_SYNCIO|PGO_OVERWRITE|PGO_PASTEOF|PGO_NOBLOCKALLOC| 559 PGO_NOTIMESTAMP; 560 int i; 561 KDASSERT(flags & UBC_WRITE); 562 KASSERT(umap->refcount == 1); 563 564 UBC_EVCNT_INCR(faultbusy); 565 if (umap->flags & UMAP_MAPPING_CACHED) { 566 umap->flags &= ~UMAP_MAPPING_CACHED; 567 pmap_remove(pmap_kernel(), va, va + ubc_winsize); 568 } 569 again_faultbusy: 570 memset(pgs, 0, sizeof(pgs)); 571 mutex_enter(&uobj->vmobjlock); 572 error = (*uobj->pgops->pgo_get)(uobj, trunc_page(offset), pgs, 573 &npages, 0, VM_PROT_READ | VM_PROT_WRITE, advice, gpflags); 574 UVMHIST_LOG(ubchist, "faultbusy getpages %d", error, 0, 0, 0); 575 if (error) { 576 goto out; 577 } 578 for (i = 0; i < npages; i++) { 579 struct vm_page *pg = pgs[i]; 580 581 KASSERT(pg->uobject == uobj); 582 if (pg->loan_count != 0) { 583 mutex_enter(&uobj->vmobjlock); 584 if (pg->loan_count != 0) { 585 pg = uvm_loanbreak(pg); 586 } 587 mutex_exit(&uobj->vmobjlock); 588 if (pg == NULL) { 589 pmap_kremove(va, ubc_winsize); 590 pmap_update(pmap_kernel()); 591 mutex_enter(&uobj->vmobjlock); 592 uvm_page_unbusy(pgs, npages); 593 mutex_exit(&uobj->vmobjlock); 594 uvm_wait("ubc_alloc"); 595 goto again_faultbusy; 596 } 597 pgs[i] = pg; 598 } 599 pmap_kenter_pa(va + slot_offset + (i << PAGE_SHIFT), 600 VM_PAGE_TO_PHYS(pg), 601 VM_PROT_READ | VM_PROT_WRITE, 0); 602 } 603 pmap_update(pmap_kernel()); 604 umap->flags |= UMAP_PAGES_LOCKED; 605 } else { 606 KASSERT((umap->flags & UMAP_PAGES_LOCKED) == 0); 607 } 608 609 out: 610 return (void *)(va + slot_offset); 611 } 612 613 /* 614 * ubc_release: free a file mapping window. 615 */ 616 617 void 618 ubc_release(void *va, int flags) 619 { 620 struct ubc_map *umap; 621 struct uvm_object *uobj; 622 vaddr_t umapva; 623 bool unmapped; 624 UVMHIST_FUNC("ubc_release"); UVMHIST_CALLED(ubchist); 625 626 UVMHIST_LOG(ubchist, "va %p", va, 0, 0, 0); 627 umap = &ubc_object.umap[((char *)va - ubc_object.kva) >> ubc_winshift]; 628 umapva = UBC_UMAP_ADDR(umap); 629 uobj = umap->uobj; 630 KASSERT(uobj != NULL); 631 632 if (umap->flags & UMAP_PAGES_LOCKED) { 633 int slot_offset = umap->writeoff; 634 int endoff = umap->writeoff + umap->writelen; 635 int zerolen = round_page(endoff) - endoff; 636 int npages = (int)(round_page(umap->writeoff + umap->writelen) 637 - trunc_page(umap->writeoff)) >> PAGE_SHIFT; 638 struct vm_page *pgs[npages]; 639 paddr_t pa; 640 int i; 641 bool rv; 642 643 KASSERT((umap->flags & UMAP_MAPPING_CACHED) == 0); 644 if (zerolen) { 645 memset((char *)umapva + endoff, 0, zerolen); 646 } 647 umap->flags &= ~UMAP_PAGES_LOCKED; 648 mutex_enter(&uvm_pageqlock); 649 for (i = 0; i < npages; i++) { 650 rv = pmap_extract(pmap_kernel(), 651 umapva + slot_offset + (i << PAGE_SHIFT), &pa); 652 KASSERT(rv); 653 pgs[i] = PHYS_TO_VM_PAGE(pa); 654 pgs[i]->flags &= ~(PG_FAKE|PG_CLEAN); 655 KASSERT(pgs[i]->loan_count == 0); 656 uvm_pageactivate(pgs[i]); 657 } 658 mutex_exit(&uvm_pageqlock); 659 pmap_kremove(umapva, ubc_winsize); 660 pmap_update(pmap_kernel()); 661 mutex_enter(&uobj->vmobjlock); 662 uvm_page_unbusy(pgs, npages); 663 mutex_exit(&uobj->vmobjlock); 664 unmapped = true; 665 } else { 666 unmapped = false; 667 } 668 669 mutex_enter(&ubc_object.uobj.vmobjlock); 670 umap->writeoff = 0; 671 umap->writelen = 0; 672 umap->refcount--; 673 if (umap->refcount == 0) { 674 if (flags & UBC_UNMAP) { 675 676 /* 677 * Invalidate any cached mappings if requested. 678 * This is typically used to avoid leaving 679 * incompatible cache aliases around indefinitely. 680 */ 681 682 pmap_remove(pmap_kernel(), umapva, 683 umapva + ubc_winsize); 684 umap->flags &= ~UMAP_MAPPING_CACHED; 685 pmap_update(pmap_kernel()); 686 LIST_REMOVE(umap, hash); 687 umap->uobj = NULL; 688 TAILQ_INSERT_HEAD(UBC_QUEUE(umap->offset), umap, 689 inactive); 690 } else { 691 if (!unmapped) { 692 umap->flags |= UMAP_MAPPING_CACHED; 693 } 694 TAILQ_INSERT_TAIL(UBC_QUEUE(umap->offset), umap, 695 inactive); 696 } 697 } 698 UVMHIST_LOG(ubchist, "umap %p refs %d", umap, umap->refcount, 0, 0); 699 mutex_exit(&ubc_object.uobj.vmobjlock); 700 } 701 702 /* 703 * ubc_uiomove: move data to/from an object. 704 */ 705 706 int 707 ubc_uiomove(struct uvm_object *uobj, struct uio *uio, vsize_t todo, int advice, 708 int flags) 709 { 710 voff_t off; 711 const bool overwrite = (flags & UBC_FAULTBUSY) != 0; 712 int error; 713 714 KASSERT(todo <= uio->uio_resid); 715 KASSERT(((flags & UBC_WRITE) != 0 && uio->uio_rw == UIO_WRITE) || 716 ((flags & UBC_READ) != 0 && uio->uio_rw == UIO_READ)); 717 718 off = uio->uio_offset; 719 error = 0; 720 while (todo > 0) { 721 vsize_t bytelen = todo; 722 void *win; 723 724 win = ubc_alloc(uobj, off, &bytelen, advice, flags); 725 if (error == 0) { 726 error = uiomove(win, bytelen, uio); 727 } 728 if (error != 0 && overwrite) { 729 /* 730 * if we haven't initialized the pages yet, 731 * do it now. it's safe to use memset here 732 * because we just mapped the pages above. 733 */ 734 printf("%s: error=%d\n", __func__, error); 735 memset(win, 0, bytelen); 736 } 737 ubc_release(win, flags); 738 off += bytelen; 739 todo -= bytelen; 740 if (error != 0 && (flags & UBC_PARTIALOK) != 0) { 741 break; 742 } 743 } 744 745 return error; 746 } 747 748 749 /* 750 * uvm_vnp_zerorange: set a range of bytes in a file to zero. 751 */ 752 753 void 754 uvm_vnp_zerorange(struct vnode *vp, off_t off, size_t len) 755 { 756 void *win; 757 int flags; 758 759 /* 760 * XXXUBC invent kzero() and use it 761 */ 762 763 while (len) { 764 vsize_t bytelen = len; 765 766 win = ubc_alloc(&vp->v_uobj, off, &bytelen, UVM_ADV_NORMAL, 767 UBC_WRITE); 768 memset(win, 0, bytelen); 769 flags = UBC_WANT_UNMAP(vp) ? UBC_UNMAP : 0; 770 ubc_release(win, flags); 771 772 off += bytelen; 773 len -= bytelen; 774 } 775 } 776