1 /* $NetBSD: genfs_io.c,v 1.95 2020/03/22 18:32:41 ad Exp $ */ 2 3 /* 4 * Copyright (c) 1982, 1986, 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.95 2020/03/22 18:32:41 ad Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/proc.h> 39 #include <sys/kernel.h> 40 #include <sys/mount.h> 41 #include <sys/vnode.h> 42 #include <sys/kmem.h> 43 #include <sys/kauth.h> 44 #include <sys/fstrans.h> 45 #include <sys/buf.h> 46 #include <sys/atomic.h> 47 48 #include <miscfs/genfs/genfs.h> 49 #include <miscfs/genfs/genfs_node.h> 50 #include <miscfs/specfs/specdev.h> 51 52 #include <uvm/uvm.h> 53 #include <uvm/uvm_pager.h> 54 #include <uvm/uvm_page_array.h> 55 56 static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *, 57 off_t, enum uio_rw); 58 static void genfs_dio_iodone(struct buf *); 59 60 static int genfs_getpages_read(struct vnode *, struct vm_page **, int, off_t, 61 off_t, bool, bool, bool, bool); 62 static int genfs_do_io(struct vnode *, off_t, vaddr_t, size_t, int, enum uio_rw, 63 void (*)(struct buf *)); 64 static void genfs_rel_pages(struct vm_page **, unsigned int); 65 66 int genfs_maxdio = MAXPHYS; 67 68 static void 69 genfs_rel_pages(struct vm_page **pgs, unsigned int npages) 70 { 71 unsigned int i; 72 73 for (i = 0; i < npages; i++) { 74 struct vm_page *pg = pgs[i]; 75 76 if (pg == NULL || pg == PGO_DONTCARE) 77 continue; 78 KASSERT(uvm_page_owner_locked_p(pg, true)); 79 if (pg->flags & PG_FAKE) { 80 pg->flags |= PG_RELEASED; 81 } 82 } 83 uvm_page_unbusy(pgs, npages); 84 } 85 86 /* 87 * generic VM getpages routine. 88 * Return PG_BUSY pages for the given range, 89 * reading from backing store if necessary. 90 */ 91 92 int 93 genfs_getpages(void *v) 94 { 95 struct vop_getpages_args /* { 96 struct vnode *a_vp; 97 voff_t a_offset; 98 struct vm_page **a_m; 99 int *a_count; 100 int a_centeridx; 101 vm_prot_t a_access_type; 102 int a_advice; 103 int a_flags; 104 } */ * const ap = v; 105 106 off_t diskeof, memeof; 107 int i, error, npages, iflag; 108 const int flags = ap->a_flags; 109 struct vnode * const vp = ap->a_vp; 110 struct uvm_object * const uobj = &vp->v_uobj; 111 const bool async = (flags & PGO_SYNCIO) == 0; 112 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 113 const bool overwrite = (flags & PGO_OVERWRITE) != 0; 114 const bool blockalloc = memwrite && (flags & PGO_NOBLOCKALLOC) == 0; 115 const bool need_wapbl = (vp->v_mount->mnt_wapbl && 116 (flags & PGO_JOURNALLOCKED) == 0); 117 const bool glocked = (flags & PGO_GLOCKHELD) != 0; 118 bool holds_wapbl = false; 119 struct mount *trans_mount = NULL; 120 UVMHIST_FUNC("genfs_getpages"); UVMHIST_CALLED(ubchist); 121 122 UVMHIST_LOG(ubchist, "vp %#jx off 0x%jx/%jx count %jd", 123 (uintptr_t)vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count); 124 125 KASSERT(memwrite >= overwrite); 126 KASSERT(vp->v_type == VREG || vp->v_type == VDIR || 127 vp->v_type == VLNK || vp->v_type == VBLK); 128 129 /* 130 * the object must be locked. it can only be a read lock when 131 * processing a read fault with PGO_LOCKED | PGO_NOBUSY. 132 */ 133 134 KASSERT(rw_lock_held(uobj->vmobjlock)); 135 KASSERT(rw_write_held(uobj->vmobjlock) || 136 ((~flags & (PGO_LOCKED | PGO_NOBUSY)) == 0 && !memwrite)); 137 138 #ifdef DIAGNOSTIC 139 if ((flags & PGO_JOURNALLOCKED) && vp->v_mount->mnt_wapbl) 140 WAPBL_JLOCK_ASSERT(vp->v_mount); 141 #endif 142 143 /* 144 * check for reclaimed vnode. v_interlock is not held here, but 145 * VI_DEADCHECK is set with vmobjlock held. 146 */ 147 148 iflag = atomic_load_relaxed(&vp->v_iflag); 149 if (__predict_false((iflag & VI_DEADCHECK) != 0)) { 150 mutex_enter(vp->v_interlock); 151 error = vdead_check(vp, VDEAD_NOWAIT); 152 mutex_exit(vp->v_interlock); 153 if (error) { 154 if ((flags & PGO_LOCKED) == 0) 155 rw_exit(uobj->vmobjlock); 156 return error; 157 } 158 } 159 160 startover: 161 error = 0; 162 const voff_t origvsize = vp->v_size; 163 const off_t origoffset = ap->a_offset; 164 const int orignpages = *ap->a_count; 165 166 GOP_SIZE(vp, origvsize, &diskeof, 0); 167 if (flags & PGO_PASTEOF) { 168 off_t newsize; 169 #if defined(DIAGNOSTIC) 170 off_t writeeof; 171 #endif /* defined(DIAGNOSTIC) */ 172 173 newsize = MAX(origvsize, 174 origoffset + (orignpages << PAGE_SHIFT)); 175 GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM); 176 #if defined(DIAGNOSTIC) 177 GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM); 178 if (newsize > round_page(writeeof)) { 179 panic("%s: past eof: %" PRId64 " vs. %" PRId64, 180 __func__, newsize, round_page(writeeof)); 181 } 182 #endif /* defined(DIAGNOSTIC) */ 183 } else { 184 GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM); 185 } 186 KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages); 187 KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0); 188 KASSERT(orignpages > 0); 189 190 /* 191 * Bounds-check the request. 192 */ 193 194 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) { 195 if ((flags & PGO_LOCKED) == 0) { 196 rw_exit(uobj->vmobjlock); 197 } 198 UVMHIST_LOG(ubchist, "off 0x%jx count %jd goes past EOF 0x%jx", 199 origoffset, *ap->a_count, memeof,0); 200 error = EINVAL; 201 goto out_err; 202 } 203 204 /* uobj is locked */ 205 206 if ((flags & PGO_NOTIMESTAMP) == 0 && 207 (vp->v_type != VBLK || 208 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 209 int updflags = 0; 210 211 if ((vp->v_mount->mnt_flag & MNT_NOATIME) == 0) { 212 updflags = GOP_UPDATE_ACCESSED; 213 } 214 if (memwrite) { 215 updflags |= GOP_UPDATE_MODIFIED; 216 } 217 if (updflags != 0) { 218 GOP_MARKUPDATE(vp, updflags); 219 } 220 } 221 222 /* 223 * For PGO_LOCKED requests, just return whatever's in memory. 224 */ 225 226 if (flags & PGO_LOCKED) { 227 int nfound; 228 struct vm_page *pg; 229 230 KASSERT(!glocked); 231 npages = *ap->a_count; 232 #if defined(DEBUG) 233 for (i = 0; i < npages; i++) { 234 pg = ap->a_m[i]; 235 KASSERT(pg == NULL || pg == PGO_DONTCARE); 236 } 237 #endif /* defined(DEBUG) */ 238 nfound = uvn_findpages(uobj, origoffset, &npages, 239 ap->a_m, NULL, 240 UFP_NOWAIT | UFP_NOALLOC | 241 (memwrite ? UFP_NORDONLY : 0) | 242 ((flags & PGO_NOBUSY) != 0 ? UFP_NOBUSY : 0)); 243 KASSERT(npages == *ap->a_count); 244 if (nfound == 0) { 245 error = EBUSY; 246 goto out_err; 247 } 248 /* 249 * lock and unlock g_glock to ensure that no one is truncating 250 * the file behind us. 251 */ 252 if (!genfs_node_rdtrylock(vp)) { 253 if ((flags & PGO_NOBUSY) == 0) { 254 genfs_rel_pages(ap->a_m, npages); 255 } 256 257 /* 258 * restore the array. 259 */ 260 261 for (i = 0; i < npages; i++) { 262 pg = ap->a_m[i]; 263 264 if (pg != NULL && pg != PGO_DONTCARE) { 265 ap->a_m[i] = NULL; 266 } 267 KASSERT(ap->a_m[i] == NULL || 268 ap->a_m[i] == PGO_DONTCARE); 269 } 270 } else { 271 genfs_node_unlock(vp); 272 } 273 error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); 274 if (error == 0 && memwrite) { 275 for (i = 0; i < npages; i++) { 276 pg = ap->a_m[i]; 277 if (pg == NULL || pg == PGO_DONTCARE) { 278 continue; 279 } 280 if (uvm_pagegetdirty(pg) == 281 UVM_PAGE_STATUS_CLEAN) { 282 uvm_pagemarkdirty(pg, 283 UVM_PAGE_STATUS_UNKNOWN); 284 } 285 } 286 } 287 goto out_err; 288 } 289 rw_exit(uobj->vmobjlock); 290 291 /* 292 * find the requested pages and make some simple checks. 293 * leave space in the page array for a whole block. 294 */ 295 296 const int fs_bshift = (vp->v_type != VBLK) ? 297 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT; 298 const int fs_bsize = 1 << fs_bshift; 299 #define blk_mask (fs_bsize - 1) 300 #define trunc_blk(x) ((x) & ~blk_mask) 301 #define round_blk(x) (((x) + blk_mask) & ~blk_mask) 302 303 const int orignmempages = MIN(orignpages, 304 round_page(memeof - origoffset) >> PAGE_SHIFT); 305 npages = orignmempages; 306 const off_t startoffset = trunc_blk(origoffset); 307 const off_t endoffset = MIN( 308 round_page(round_blk(origoffset + (npages << PAGE_SHIFT))), 309 round_page(memeof)); 310 const int ridx = (origoffset - startoffset) >> PAGE_SHIFT; 311 312 const int pgs_size = sizeof(struct vm_page *) * 313 ((endoffset - startoffset) >> PAGE_SHIFT); 314 struct vm_page **pgs, *pgs_onstack[UBC_MAX_PAGES]; 315 316 if (pgs_size > sizeof(pgs_onstack)) { 317 pgs = kmem_zalloc(pgs_size, async ? KM_NOSLEEP : KM_SLEEP); 318 if (pgs == NULL) { 319 pgs = pgs_onstack; 320 error = ENOMEM; 321 goto out_err; 322 } 323 } else { 324 pgs = pgs_onstack; 325 (void)memset(pgs, 0, pgs_size); 326 } 327 328 UVMHIST_LOG(ubchist, "ridx %jd npages %jd startoff %jd endoff %jd", 329 ridx, npages, startoffset, endoffset); 330 331 if (trans_mount == NULL) { 332 trans_mount = vp->v_mount; 333 fstrans_start(trans_mount); 334 /* 335 * check if this vnode is still valid. 336 */ 337 mutex_enter(vp->v_interlock); 338 error = vdead_check(vp, 0); 339 mutex_exit(vp->v_interlock); 340 if (error) 341 goto out_err_free; 342 /* 343 * XXX: This assumes that we come here only via 344 * the mmio path 345 */ 346 if (blockalloc && need_wapbl) { 347 error = WAPBL_BEGIN(trans_mount); 348 if (error) 349 goto out_err_free; 350 holds_wapbl = true; 351 } 352 } 353 354 /* 355 * hold g_glock to prevent a race with truncate. 356 * 357 * check if our idea of v_size is still valid. 358 */ 359 360 KASSERT(!glocked || genfs_node_wrlocked(vp)); 361 if (!glocked) { 362 if (blockalloc) { 363 genfs_node_wrlock(vp); 364 } else { 365 genfs_node_rdlock(vp); 366 } 367 } 368 rw_enter(uobj->vmobjlock, RW_WRITER); 369 if (vp->v_size < origvsize) { 370 if (!glocked) { 371 genfs_node_unlock(vp); 372 } 373 if (pgs != pgs_onstack) 374 kmem_free(pgs, pgs_size); 375 goto startover; 376 } 377 378 if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], NULL, 379 async ? UFP_NOWAIT : UFP_ALL) != orignmempages) { 380 if (!glocked) { 381 genfs_node_unlock(vp); 382 } 383 KASSERT(async != 0); 384 genfs_rel_pages(&pgs[ridx], orignmempages); 385 rw_exit(uobj->vmobjlock); 386 error = EBUSY; 387 goto out_err_free; 388 } 389 390 /* 391 * if PGO_OVERWRITE is set, don't bother reading the pages. 392 */ 393 394 if (overwrite) { 395 if (!glocked) { 396 genfs_node_unlock(vp); 397 } 398 UVMHIST_LOG(ubchist, "PGO_OVERWRITE",0,0,0,0); 399 400 for (i = 0; i < npages; i++) { 401 struct vm_page *pg = pgs[ridx + i]; 402 403 /* 404 * it's caller's responsibility to allocate blocks 405 * beforehand for the overwrite case. 406 */ 407 408 KASSERT((pg->flags & PG_RDONLY) == 0 || !blockalloc); 409 pg->flags &= ~PG_RDONLY; 410 411 /* 412 * mark the page DIRTY. 413 * otherwise another thread can do putpages and pull 414 * our vnode from syncer's queue before our caller does 415 * ubc_release. note that putpages won't see CLEAN 416 * pages even if they are BUSY. 417 */ 418 419 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); 420 } 421 npages += ridx; 422 goto out; 423 } 424 425 /* 426 * if the pages are already resident, just return them. 427 */ 428 429 for (i = 0; i < npages; i++) { 430 struct vm_page *pg = pgs[ridx + i]; 431 432 if ((pg->flags & PG_FAKE) || 433 (blockalloc && (pg->flags & PG_RDONLY) != 0)) { 434 break; 435 } 436 } 437 if (i == npages) { 438 if (!glocked) { 439 genfs_node_unlock(vp); 440 } 441 UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0); 442 npages += ridx; 443 goto out; 444 } 445 446 /* 447 * the page wasn't resident and we're not overwriting, 448 * so we're going to have to do some i/o. 449 * find any additional pages needed to cover the expanded range. 450 */ 451 452 npages = (endoffset - startoffset) >> PAGE_SHIFT; 453 if (startoffset != origoffset || npages != orignmempages) { 454 int npgs; 455 456 /* 457 * we need to avoid deadlocks caused by locking 458 * additional pages at lower offsets than pages we 459 * already have locked. unlock them all and start over. 460 */ 461 462 genfs_rel_pages(&pgs[ridx], orignmempages); 463 memset(pgs, 0, pgs_size); 464 465 UVMHIST_LOG(ubchist, "reset npages start 0x%jx end 0x%jx", 466 startoffset, endoffset, 0,0); 467 npgs = npages; 468 if (uvn_findpages(uobj, startoffset, &npgs, pgs, NULL, 469 async ? UFP_NOWAIT : UFP_ALL) != npages) { 470 if (!glocked) { 471 genfs_node_unlock(vp); 472 } 473 KASSERT(async != 0); 474 genfs_rel_pages(pgs, npages); 475 rw_exit(uobj->vmobjlock); 476 error = EBUSY; 477 goto out_err_free; 478 } 479 } 480 481 rw_exit(uobj->vmobjlock); 482 error = genfs_getpages_read(vp, pgs, npages, startoffset, diskeof, 483 async, memwrite, blockalloc, glocked); 484 if (!glocked) { 485 genfs_node_unlock(vp); 486 } 487 if (error == 0 && async) 488 goto out_err_free; 489 rw_enter(uobj->vmobjlock, RW_WRITER); 490 491 /* 492 * we're almost done! release the pages... 493 * for errors, we free the pages. 494 * otherwise we activate them and mark them as valid and clean. 495 * also, unbusy pages that were not actually requested. 496 */ 497 498 if (error) { 499 genfs_rel_pages(pgs, npages); 500 rw_exit(uobj->vmobjlock); 501 UVMHIST_LOG(ubchist, "returning error %jd", error,0,0,0); 502 goto out_err_free; 503 } 504 505 out: 506 UVMHIST_LOG(ubchist, "succeeding, npages %jd", npages,0,0,0); 507 error = 0; 508 for (i = 0; i < npages; i++) { 509 struct vm_page *pg = pgs[i]; 510 if (pg == NULL) { 511 continue; 512 } 513 UVMHIST_LOG(ubchist, "examining pg %#jx flags 0x%jx", 514 (uintptr_t)pg, pg->flags, 0,0); 515 if (pg->flags & PG_FAKE && !overwrite) { 516 /* 517 * we've read page's contents from the backing storage. 518 * 519 * for a read fault, we keep them CLEAN; if we 520 * encountered a hole while reading, the pages can 521 * already been dirtied with zeros. 522 */ 523 KASSERTMSG(blockalloc || uvm_pagegetdirty(pg) == 524 UVM_PAGE_STATUS_CLEAN, "page %p not clean", pg); 525 pg->flags &= ~PG_FAKE; 526 } 527 KASSERT(!memwrite || !blockalloc || (pg->flags & PG_RDONLY) == 0); 528 if (i < ridx || i >= ridx + orignmempages || async) { 529 UVMHIST_LOG(ubchist, "unbusy pg %#jx offset 0x%jx", 530 (uintptr_t)pg, pg->offset,0,0); 531 if (pg->flags & PG_FAKE) { 532 KASSERT(overwrite); 533 uvm_pagezero(pg); 534 } 535 if (pg->flags & PG_RELEASED) { 536 uvm_pagefree(pg); 537 continue; 538 } 539 uvm_pagelock(pg); 540 uvm_pageenqueue(pg); 541 uvm_pagewakeup(pg); 542 uvm_pageunlock(pg); 543 pg->flags &= ~(PG_BUSY|PG_FAKE); 544 UVM_PAGE_OWN(pg, NULL); 545 } else if (memwrite && !overwrite && 546 uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN) { 547 /* 548 * for a write fault, start dirtiness tracking of 549 * requested pages. 550 */ 551 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_UNKNOWN); 552 } 553 } 554 rw_exit(uobj->vmobjlock); 555 if (ap->a_m != NULL) { 556 memcpy(ap->a_m, &pgs[ridx], 557 orignmempages * sizeof(struct vm_page *)); 558 } 559 560 out_err_free: 561 if (pgs != NULL && pgs != pgs_onstack) 562 kmem_free(pgs, pgs_size); 563 out_err: 564 if (trans_mount != NULL) { 565 if (holds_wapbl) 566 WAPBL_END(trans_mount); 567 fstrans_done(trans_mount); 568 } 569 return error; 570 } 571 572 /* 573 * genfs_getpages_read: Read the pages in with VOP_BMAP/VOP_STRATEGY. 574 * 575 * "glocked" (which is currently not actually used) tells us not whether 576 * the genfs_node is locked on entry (it always is) but whether it was 577 * locked on entry to genfs_getpages. 578 */ 579 static int 580 genfs_getpages_read(struct vnode *vp, struct vm_page **pgs, int npages, 581 off_t startoffset, off_t diskeof, 582 bool async, bool memwrite, bool blockalloc, bool glocked) 583 { 584 struct uvm_object * const uobj = &vp->v_uobj; 585 const int fs_bshift = (vp->v_type != VBLK) ? 586 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT; 587 const int dev_bshift = (vp->v_type != VBLK) ? 588 vp->v_mount->mnt_dev_bshift : DEV_BSHIFT; 589 kauth_cred_t const cred = curlwp->l_cred; /* XXXUBC curlwp */ 590 size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes; 591 vaddr_t kva; 592 struct buf *bp, *mbp; 593 bool sawhole = false; 594 int i; 595 int error = 0; 596 597 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 598 599 /* 600 * read the desired page(s). 601 */ 602 603 totalbytes = npages << PAGE_SHIFT; 604 bytes = MIN(totalbytes, MAX(diskeof - startoffset, 0)); 605 tailbytes = totalbytes - bytes; 606 skipbytes = 0; 607 608 kva = uvm_pagermapin(pgs, npages, 609 UVMPAGER_MAPIN_READ | (async ? 0 : UVMPAGER_MAPIN_WAITOK)); 610 if (kva == 0) 611 return EBUSY; 612 613 mbp = getiobuf(vp, true); 614 mbp->b_bufsize = totalbytes; 615 mbp->b_data = (void *)kva; 616 mbp->b_resid = mbp->b_bcount = bytes; 617 mbp->b_cflags |= BC_BUSY; 618 if (async) { 619 mbp->b_flags = B_READ | B_ASYNC; 620 mbp->b_iodone = uvm_aio_aiodone; 621 } else { 622 mbp->b_flags = B_READ; 623 mbp->b_iodone = NULL; 624 } 625 if (async) 626 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 627 else 628 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 629 630 /* 631 * if EOF is in the middle of the range, zero the part past EOF. 632 * skip over pages which are not PG_FAKE since in that case they have 633 * valid data that we need to preserve. 634 */ 635 636 tailstart = bytes; 637 while (tailbytes > 0) { 638 const int len = PAGE_SIZE - (tailstart & PAGE_MASK); 639 640 KASSERT(len <= tailbytes); 641 if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) { 642 memset((void *)(kva + tailstart), 0, len); 643 UVMHIST_LOG(ubchist, "tailbytes %#jx 0x%jx 0x%jx", 644 (uintptr_t)kva, tailstart, len, 0); 645 } 646 tailstart += len; 647 tailbytes -= len; 648 } 649 650 /* 651 * now loop over the pages, reading as needed. 652 */ 653 654 bp = NULL; 655 off_t offset; 656 for (offset = startoffset; 657 bytes > 0; 658 offset += iobytes, bytes -= iobytes) { 659 int run; 660 daddr_t lbn, blkno; 661 int pidx; 662 struct vnode *devvp; 663 664 /* 665 * skip pages which don't need to be read. 666 */ 667 668 pidx = (offset - startoffset) >> PAGE_SHIFT; 669 while ((pgs[pidx]->flags & PG_FAKE) == 0) { 670 size_t b; 671 672 KASSERT((offset & (PAGE_SIZE - 1)) == 0); 673 if ((pgs[pidx]->flags & PG_RDONLY)) { 674 sawhole = true; 675 } 676 b = MIN(PAGE_SIZE, bytes); 677 offset += b; 678 bytes -= b; 679 skipbytes += b; 680 pidx++; 681 UVMHIST_LOG(ubchist, "skipping, new offset 0x%jx", 682 offset, 0,0,0); 683 if (bytes == 0) { 684 goto loopdone; 685 } 686 } 687 688 /* 689 * bmap the file to find out the blkno to read from and 690 * how much we can read in one i/o. if bmap returns an error, 691 * skip the rest of the top-level i/o. 692 */ 693 694 lbn = offset >> fs_bshift; 695 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 696 if (error) { 697 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%jx -> %jd\n", 698 lbn,error,0,0); 699 skipbytes += bytes; 700 bytes = 0; 701 goto loopdone; 702 } 703 704 /* 705 * see how many pages can be read with this i/o. 706 * reduce the i/o size if necessary to avoid 707 * overwriting pages with valid data. 708 */ 709 710 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 711 bytes); 712 if (offset + iobytes > round_page(offset)) { 713 int pcount; 714 715 pcount = 1; 716 while (pidx + pcount < npages && 717 pgs[pidx + pcount]->flags & PG_FAKE) { 718 pcount++; 719 } 720 iobytes = MIN(iobytes, (pcount << PAGE_SHIFT) - 721 (offset - trunc_page(offset))); 722 } 723 724 /* 725 * if this block isn't allocated, zero it instead of 726 * reading it. unless we are going to allocate blocks, 727 * mark the pages we zeroed PG_RDONLY. 728 */ 729 730 if (blkno == (daddr_t)-1) { 731 int holepages = (round_page(offset + iobytes) - 732 trunc_page(offset)) >> PAGE_SHIFT; 733 UVMHIST_LOG(ubchist, "lbn 0x%jx -> HOLE", lbn,0,0,0); 734 735 sawhole = true; 736 memset((char *)kva + (offset - startoffset), 0, 737 iobytes); 738 skipbytes += iobytes; 739 740 if (!blockalloc) { 741 rw_enter(uobj->vmobjlock, RW_WRITER); 742 for (i = 0; i < holepages; i++) { 743 pgs[pidx + i]->flags |= PG_RDONLY; 744 } 745 rw_exit(uobj->vmobjlock); 746 } 747 continue; 748 } 749 750 /* 751 * allocate a sub-buf for this piece of the i/o 752 * (or just use mbp if there's only 1 piece), 753 * and start it going. 754 */ 755 756 if (offset == startoffset && iobytes == bytes) { 757 bp = mbp; 758 } else { 759 UVMHIST_LOG(ubchist, "vp %#jx bp %#jx num now %jd", 760 (uintptr_t)vp, (uintptr_t)bp, vp->v_numoutput, 0); 761 bp = getiobuf(vp, true); 762 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 763 } 764 bp->b_lblkno = 0; 765 766 /* adjust physical blkno for partial blocks */ 767 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 768 dev_bshift); 769 770 UVMHIST_LOG(ubchist, 771 "bp %#jx offset 0x%x bcount 0x%x blkno 0x%x", 772 (uintptr_t)bp, offset, bp->b_bcount, bp->b_blkno); 773 774 VOP_STRATEGY(devvp, bp); 775 } 776 777 loopdone: 778 nestiobuf_done(mbp, skipbytes, error); 779 if (async) { 780 UVMHIST_LOG(ubchist, "returning 0 (async)",0,0,0,0); 781 return 0; 782 } 783 if (bp != NULL) { 784 error = biowait(mbp); 785 } 786 787 /* Remove the mapping (make KVA available as soon as possible) */ 788 uvm_pagermapout(kva, npages); 789 790 /* 791 * if this we encountered a hole then we have to do a little more work. 792 * for read faults, we marked the page PG_RDONLY so that future 793 * write accesses to the page will fault again. 794 * for write faults, we must make sure that the backing store for 795 * the page is completely allocated while the pages are locked. 796 */ 797 798 if (!error && sawhole && blockalloc) { 799 error = GOP_ALLOC(vp, startoffset, 800 npages << PAGE_SHIFT, 0, cred); 801 UVMHIST_LOG(ubchist, "gop_alloc off 0x%jx/0x%jx -> %jd", 802 startoffset, npages << PAGE_SHIFT, error,0); 803 if (!error) { 804 rw_enter(uobj->vmobjlock, RW_WRITER); 805 for (i = 0; i < npages; i++) { 806 struct vm_page *pg = pgs[i]; 807 808 if (pg == NULL) { 809 continue; 810 } 811 pg->flags &= ~PG_RDONLY; 812 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); 813 UVMHIST_LOG(ubchist, "mark dirty pg %#jx", 814 (uintptr_t)pg, 0, 0, 0); 815 } 816 rw_exit(uobj->vmobjlock); 817 } 818 } 819 820 putiobuf(mbp); 821 return error; 822 } 823 824 /* 825 * generic VM putpages routine. 826 * Write the given range of pages to backing store. 827 * 828 * => "offhi == 0" means flush all pages at or after "offlo". 829 * => object should be locked by caller. we return with the 830 * object unlocked. 831 * => if PGO_CLEANIT or PGO_SYNCIO is set, we may block (due to I/O). 832 * thus, a caller might want to unlock higher level resources 833 * (e.g. vm_map) before calling flush. 834 * => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block 835 * => if PGO_ALLPAGES is set, then all pages in the object will be processed. 836 * 837 * note on "cleaning" object and PG_BUSY pages: 838 * this routine is holding the lock on the object. the only time 839 * that it can run into a PG_BUSY page that it does not own is if 840 * some other process has started I/O on the page (e.g. either 841 * a pagein, or a pageout). if the PG_BUSY page is being paged 842 * in, then it can not be dirty (!UVM_PAGE_STATUS_CLEAN) because no 843 * one has had a chance to modify it yet. if the PG_BUSY page is 844 * being paged out then it means that someone else has already started 845 * cleaning the page for us (how nice!). in this case, if we 846 * have syncio specified, then after we make our pass through the 847 * object we need to wait for the other PG_BUSY pages to clear 848 * off (i.e. we need to do an iosync). also note that once a 849 * page is PG_BUSY it must stay in its object until it is un-busyed. 850 */ 851 852 int 853 genfs_putpages(void *v) 854 { 855 struct vop_putpages_args /* { 856 struct vnode *a_vp; 857 voff_t a_offlo; 858 voff_t a_offhi; 859 int a_flags; 860 } */ * const ap = v; 861 862 return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, 863 ap->a_flags, NULL); 864 } 865 866 int 867 genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, 868 int origflags, struct vm_page **busypg) 869 { 870 struct uvm_object * const uobj = &vp->v_uobj; 871 krwlock_t * const slock = uobj->vmobjlock; 872 off_t nextoff; 873 int i, error, npages, nback; 874 int freeflag; 875 /* 876 * This array is larger than it should so that it's size is constant. 877 * The right size is MAXPAGES. 878 */ 879 struct vm_page *pgs[MAXPHYS / MIN_PAGE_SIZE]; 880 #define MAXPAGES (MAXPHYS / PAGE_SIZE) 881 struct vm_page *pg, *tpg; 882 struct uvm_page_array a; 883 bool wasclean, needs_clean; 884 bool async = (origflags & PGO_SYNCIO) == 0; 885 bool pagedaemon = curlwp == uvm.pagedaemon_lwp; 886 struct mount *trans_mp; 887 int flags; 888 bool modified; /* if we write out any pages */ 889 bool holds_wapbl; 890 bool cleanall; /* try to pull off from the syncer's list */ 891 bool onworklst; 892 bool nodirty; 893 const bool dirtyonly = (origflags & (PGO_DEACTIVATE|PGO_FREE)) == 0; 894 895 UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist); 896 897 KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)); 898 KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0); 899 KASSERT(startoff < endoff || endoff == 0); 900 KASSERT(rw_write_held(slock)); 901 902 UVMHIST_LOG(ubchist, "vp %#jx pages %jd off 0x%jx len 0x%jx", 903 (uintptr_t)vp, uobj->uo_npages, startoff, endoff - startoff); 904 905 #ifdef DIAGNOSTIC 906 if ((origflags & PGO_JOURNALLOCKED) && vp->v_mount->mnt_wapbl) 907 WAPBL_JLOCK_ASSERT(vp->v_mount); 908 #endif 909 910 trans_mp = NULL; 911 holds_wapbl = false; 912 913 retry: 914 modified = false; 915 flags = origflags; 916 917 /* 918 * shortcut if we have no pages to process. 919 */ 920 921 nodirty = radix_tree_empty_tagged_tree_p(&uobj->uo_pages, 922 UVM_PAGE_DIRTY_TAG); 923 #ifdef DIAGNOSTIC 924 mutex_enter(vp->v_interlock); 925 KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 || nodirty); 926 mutex_exit(vp->v_interlock); 927 #endif 928 if (uobj->uo_npages == 0 || (dirtyonly && nodirty)) { 929 mutex_enter(vp->v_interlock); 930 if (vp->v_iflag & VI_ONWORKLST) { 931 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 932 vn_syncer_remove_from_worklist(vp); 933 } 934 mutex_exit(vp->v_interlock); 935 if (trans_mp) { 936 if (holds_wapbl) 937 WAPBL_END(trans_mp); 938 fstrans_done(trans_mp); 939 } 940 rw_exit(slock); 941 return (0); 942 } 943 944 /* 945 * the vnode has pages, set up to process the request. 946 */ 947 948 if (trans_mp == NULL && (flags & PGO_CLEANIT) != 0) { 949 if (pagedaemon) { 950 /* Pagedaemon must not sleep here. */ 951 trans_mp = vp->v_mount; 952 error = fstrans_start_nowait(trans_mp); 953 if (error) { 954 rw_exit(slock); 955 return error; 956 } 957 } else { 958 /* 959 * Cannot use vdeadcheck() here as this operation 960 * usually gets used from VOP_RECLAIM(). Test for 961 * change of v_mount instead and retry on change. 962 */ 963 rw_exit(slock); 964 trans_mp = vp->v_mount; 965 fstrans_start(trans_mp); 966 if (vp->v_mount != trans_mp) { 967 fstrans_done(trans_mp); 968 trans_mp = NULL; 969 } else { 970 holds_wapbl = (trans_mp->mnt_wapbl && 971 (origflags & PGO_JOURNALLOCKED) == 0); 972 if (holds_wapbl) { 973 error = WAPBL_BEGIN(trans_mp); 974 if (error) { 975 fstrans_done(trans_mp); 976 return error; 977 } 978 } 979 } 980 rw_enter(slock, RW_WRITER); 981 goto retry; 982 } 983 } 984 985 error = 0; 986 wasclean = radix_tree_empty_tagged_tree_p(&uobj->uo_pages, 987 UVM_PAGE_WRITEBACK_TAG); 988 nextoff = startoff; 989 if (endoff == 0 || flags & PGO_ALLPAGES) { 990 endoff = trunc_page(LLONG_MAX); 991 } 992 993 /* 994 * if this vnode is known not to have dirty pages, 995 * don't bother to clean it out. 996 */ 997 998 if (nodirty) { 999 #if !defined(DEBUG) 1000 if (dirtyonly) { 1001 goto skip_scan; 1002 } 1003 #endif /* !defined(DEBUG) */ 1004 flags &= ~PGO_CLEANIT; 1005 } 1006 1007 /* 1008 * start the loop to scan pages. 1009 */ 1010 1011 cleanall = true; 1012 freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED; 1013 uvm_page_array_init(&a); 1014 for (;;) { 1015 bool pgprotected; 1016 1017 /* 1018 * if !dirtyonly, iterate over all resident pages in the range. 1019 * 1020 * if dirtyonly, only possibly dirty pages are interesting. 1021 * however, if we are asked to sync for integrity, we should 1022 * wait on pages being written back by other threads as well. 1023 */ 1024 1025 pg = uvm_page_array_fill_and_peek(&a, uobj, nextoff, 0, 1026 dirtyonly ? (UVM_PAGE_ARRAY_FILL_DIRTY | 1027 (!async ? UVM_PAGE_ARRAY_FILL_WRITEBACK : 0)) : 0); 1028 if (pg == NULL) { 1029 break; 1030 } 1031 1032 KASSERT(pg->uobject == uobj); 1033 KASSERT((pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 || 1034 (pg->flags & (PG_BUSY)) != 0); 1035 KASSERT(pg->offset >= startoff); 1036 KASSERT(pg->offset >= nextoff); 1037 KASSERT(!dirtyonly || 1038 uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN || 1039 radix_tree_get_tag(&uobj->uo_pages, 1040 pg->offset >> PAGE_SHIFT, UVM_PAGE_WRITEBACK_TAG)); 1041 1042 if (pg->offset >= endoff) { 1043 break; 1044 } 1045 1046 /* 1047 * a preempt point. 1048 */ 1049 1050 if (preempt_needed()) { 1051 nextoff = pg->offset; /* visit this page again */ 1052 rw_exit(slock); 1053 preempt(); 1054 /* 1055 * as we dropped the object lock, our cached pages can 1056 * be stale. 1057 */ 1058 uvm_page_array_clear(&a); 1059 rw_enter(slock, RW_WRITER); 1060 continue; 1061 } 1062 1063 /* 1064 * if the current page is busy, wait for it to become unbusy. 1065 */ 1066 1067 if ((pg->flags & PG_BUSY) != 0) { 1068 UVMHIST_LOG(ubchist, "busy %#jx", (uintptr_t)pg, 1069 0, 0, 0); 1070 if ((pg->flags & (PG_RELEASED|PG_PAGEOUT)) != 0 1071 && (flags & PGO_BUSYFAIL) != 0) { 1072 UVMHIST_LOG(ubchist, "busyfail %#jx", 1073 (uintptr_t)pg, 0, 0, 0); 1074 error = EDEADLK; 1075 if (busypg != NULL) 1076 *busypg = pg; 1077 break; 1078 } 1079 if (pagedaemon) { 1080 /* 1081 * someone has taken the page while we 1082 * dropped the lock for fstrans_start. 1083 */ 1084 break; 1085 } 1086 /* 1087 * don't bother to wait on other's activities 1088 * unless we are asked to sync for integrity. 1089 */ 1090 if (!async && (flags & PGO_RECLAIM) == 0) { 1091 wasclean = false; 1092 nextoff = pg->offset + PAGE_SIZE; 1093 uvm_page_array_advance(&a); 1094 continue; 1095 } 1096 nextoff = pg->offset; /* visit this page again */ 1097 uvm_pagewait(pg, slock, "genput"); 1098 /* 1099 * as we dropped the object lock, our cached pages can 1100 * be stale. 1101 */ 1102 uvm_page_array_clear(&a); 1103 rw_enter(slock, RW_WRITER); 1104 continue; 1105 } 1106 1107 nextoff = pg->offset + PAGE_SIZE; 1108 uvm_page_array_advance(&a); 1109 1110 /* 1111 * if we're freeing, remove all mappings of the page now. 1112 * if we're cleaning, check if the page is needs to be cleaned. 1113 */ 1114 1115 pgprotected = false; 1116 if (flags & PGO_FREE) { 1117 pmap_page_protect(pg, VM_PROT_NONE); 1118 pgprotected = true; 1119 } else if (flags & PGO_CLEANIT) { 1120 1121 /* 1122 * if we still have some hope to pull this vnode off 1123 * from the syncer queue, write-protect the page. 1124 */ 1125 1126 if (cleanall && wasclean) { 1127 1128 /* 1129 * uobj pages get wired only by uvm_fault 1130 * where uobj is locked. 1131 */ 1132 1133 if (pg->wire_count == 0) { 1134 pmap_page_protect(pg, 1135 VM_PROT_READ|VM_PROT_EXECUTE); 1136 pgprotected = true; 1137 } else { 1138 cleanall = false; 1139 } 1140 } 1141 } 1142 1143 if (flags & PGO_CLEANIT) { 1144 needs_clean = uvm_pagecheckdirty(pg, pgprotected); 1145 } else { 1146 needs_clean = false; 1147 } 1148 1149 /* 1150 * if we're cleaning, build a cluster. 1151 * the cluster will consist of pages which are currently dirty. 1152 * if not cleaning, just operate on the one page. 1153 */ 1154 1155 if (needs_clean) { 1156 wasclean = false; 1157 memset(pgs, 0, sizeof(pgs)); 1158 pg->flags |= PG_BUSY; 1159 UVM_PAGE_OWN(pg, "genfs_putpages"); 1160 1161 /* 1162 * let the fs constrain the offset range of the cluster. 1163 * we additionally constrain the range here such that 1164 * it fits in the "pgs" pages array. 1165 */ 1166 1167 off_t fslo, fshi, genlo, lo, off = pg->offset; 1168 GOP_PUTRANGE(vp, off, &fslo, &fshi); 1169 KASSERT(fslo == trunc_page(fslo)); 1170 KASSERT(fslo <= off); 1171 KASSERT(fshi == trunc_page(fshi)); 1172 KASSERT(fshi == 0 || off < fshi); 1173 1174 if (off > MAXPHYS / 2) 1175 genlo = trunc_page(off - (MAXPHYS / 2)); 1176 else 1177 genlo = 0; 1178 lo = MAX(fslo, genlo); 1179 1180 /* 1181 * first look backward. 1182 */ 1183 1184 npages = (off - lo) >> PAGE_SHIFT; 1185 nback = npages; 1186 uvn_findpages(uobj, off - PAGE_SIZE, &nback, 1187 &pgs[0], NULL, 1188 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD); 1189 if (nback) { 1190 memmove(&pgs[0], &pgs[npages - nback], 1191 nback * sizeof(pgs[0])); 1192 if (npages - nback < nback) 1193 memset(&pgs[nback], 0, 1194 (npages - nback) * sizeof(pgs[0])); 1195 else 1196 memset(&pgs[npages - nback], 0, 1197 nback * sizeof(pgs[0])); 1198 } 1199 1200 /* 1201 * then plug in our page of interest. 1202 */ 1203 1204 pgs[nback] = pg; 1205 1206 /* 1207 * then look forward to fill in the remaining space in 1208 * the array of pages. 1209 * 1210 * pass our cached array of pages so that hopefully 1211 * uvn_findpages can find some good pages in it. 1212 * the array a was filled above with the one of 1213 * following sets of flags: 1214 * 0 1215 * UVM_PAGE_ARRAY_FILL_DIRTY 1216 * UVM_PAGE_ARRAY_FILL_DIRTY|WRITEBACK 1217 */ 1218 1219 npages = MAXPAGES - nback - 1; 1220 if (fshi) 1221 npages = MIN(npages, 1222 (fshi - off - 1) >> PAGE_SHIFT); 1223 uvn_findpages(uobj, off + PAGE_SIZE, &npages, 1224 &pgs[nback + 1], NULL, 1225 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY); 1226 npages += nback + 1; 1227 } else { 1228 pgs[0] = pg; 1229 npages = 1; 1230 nback = 0; 1231 } 1232 1233 /* 1234 * apply FREE or DEACTIVATE options if requested. 1235 */ 1236 1237 for (i = 0; i < npages; i++) { 1238 tpg = pgs[i]; 1239 KASSERT(tpg->uobject == uobj); 1240 KASSERT(i == 0 || 1241 pgs[i-1]->offset + PAGE_SIZE == tpg->offset); 1242 KASSERT(!needs_clean || uvm_pagegetdirty(pgs[i]) != 1243 UVM_PAGE_STATUS_DIRTY); 1244 if (needs_clean) { 1245 /* 1246 * mark pages as WRITEBACK so that concurrent 1247 * fsync can find and wait for our activities. 1248 */ 1249 radix_tree_set_tag(&uobj->uo_pages, 1250 pgs[i]->offset >> PAGE_SHIFT, 1251 UVM_PAGE_WRITEBACK_TAG); 1252 } 1253 if (tpg->offset < startoff || tpg->offset >= endoff) 1254 continue; 1255 if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) { 1256 uvm_pagelock(tpg); 1257 uvm_pagedeactivate(tpg); 1258 uvm_pageunlock(tpg); 1259 } else if (flags & PGO_FREE) { 1260 pmap_page_protect(tpg, VM_PROT_NONE); 1261 if (tpg->flags & PG_BUSY) { 1262 tpg->flags |= freeflag; 1263 if (pagedaemon) { 1264 uvm_pageout_start(1); 1265 uvm_pagelock(tpg); 1266 uvm_pagedequeue(tpg); 1267 uvm_pageunlock(tpg); 1268 } 1269 } else { 1270 1271 /* 1272 * ``page is not busy'' 1273 * implies that npages is 1 1274 * and needs_clean is false. 1275 */ 1276 1277 KASSERT(npages == 1); 1278 KASSERT(!needs_clean); 1279 KASSERT(pg == tpg); 1280 KASSERT(nextoff == 1281 tpg->offset + PAGE_SIZE); 1282 uvm_pagefree(tpg); 1283 if (pagedaemon) 1284 uvmexp.pdfreed++; 1285 } 1286 } 1287 } 1288 if (needs_clean) { 1289 modified = true; 1290 KASSERT(nextoff == pg->offset + PAGE_SIZE); 1291 KASSERT(nback < npages); 1292 nextoff = pg->offset + ((npages - nback) << PAGE_SHIFT); 1293 KASSERT(pgs[nback] == pg); 1294 KASSERT(nextoff == pgs[npages - 1]->offset + PAGE_SIZE); 1295 1296 /* 1297 * start the i/o. 1298 */ 1299 rw_exit(slock); 1300 error = GOP_WRITE(vp, pgs, npages, flags); 1301 /* 1302 * as we dropped the object lock, our cached pages can 1303 * be stale. 1304 */ 1305 uvm_page_array_clear(&a); 1306 rw_enter(slock, RW_WRITER); 1307 if (error) { 1308 break; 1309 } 1310 } 1311 } 1312 uvm_page_array_fini(&a); 1313 1314 /* 1315 * update ctime/mtime if the modification we started writing out might 1316 * be from mmap'ed write. 1317 * 1318 * this is necessary when an application keeps a file mmaped and 1319 * repeatedly modifies it via the window. note that, because we 1320 * don't always write-protect pages when cleaning, such modifications 1321 * might not involve any page faults. 1322 */ 1323 1324 mutex_enter(vp->v_interlock); 1325 if (modified && (vp->v_iflag & VI_WRMAP) != 0 && 1326 (vp->v_type != VBLK || 1327 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 1328 GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED); 1329 } 1330 1331 /* 1332 * if we no longer have any possibly dirty pages, take us off the 1333 * syncer list. 1334 */ 1335 1336 if ((vp->v_iflag & VI_ONWORKLST) != 0 && 1337 radix_tree_empty_tagged_tree_p(&uobj->uo_pages, 1338 UVM_PAGE_DIRTY_TAG)) { 1339 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 1340 vn_syncer_remove_from_worklist(vp); 1341 } 1342 1343 #if !defined(DEBUG) 1344 skip_scan: 1345 #endif /* !defined(DEBUG) */ 1346 1347 /* Wait for output to complete. */ 1348 rw_exit(slock); 1349 if (!wasclean && !async && vp->v_numoutput != 0) { 1350 while (vp->v_numoutput != 0) 1351 cv_wait(&vp->v_cv, vp->v_interlock); 1352 } 1353 onworklst = (vp->v_iflag & VI_ONWORKLST) != 0; 1354 mutex_exit(vp->v_interlock); 1355 1356 if ((flags & PGO_RECLAIM) != 0 && onworklst) { 1357 /* 1358 * in the case of PGO_RECLAIM, ensure to make the vnode clean. 1359 * retrying is not a big deal because, in many cases, 1360 * uobj->uo_npages is already 0 here. 1361 */ 1362 rw_enter(slock, RW_WRITER); 1363 goto retry; 1364 } 1365 1366 if (trans_mp) { 1367 if (holds_wapbl) 1368 WAPBL_END(trans_mp); 1369 fstrans_done(trans_mp); 1370 } 1371 1372 return (error); 1373 } 1374 1375 /* 1376 * Default putrange method for file systems that do not care 1377 * how many pages are given to one GOP_WRITE() call. 1378 */ 1379 void 1380 genfs_gop_putrange(struct vnode *vp, off_t off, off_t *lop, off_t *hip) 1381 { 1382 1383 *lop = 0; 1384 *hip = 0; 1385 } 1386 1387 int 1388 genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1389 { 1390 off_t off; 1391 vaddr_t kva; 1392 size_t len; 1393 int error; 1394 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1395 1396 UVMHIST_LOG(ubchist, "vp %#jx pgs %#jx npages %jd flags 0x%jx", 1397 (uintptr_t)vp, (uintptr_t)pgs, npages, flags); 1398 1399 off = pgs[0]->offset; 1400 kva = uvm_pagermapin(pgs, npages, 1401 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1402 len = npages << PAGE_SHIFT; 1403 1404 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1405 uvm_aio_aiodone); 1406 1407 return error; 1408 } 1409 1410 /* 1411 * genfs_gop_write_rwmap: 1412 * 1413 * a variant of genfs_gop_write. it's used by UDF for its directory buffers. 1414 * this maps pages with PROT_WRITE so that VOP_STRATEGY can modifies 1415 * the contents before writing it out to the underlying storage. 1416 */ 1417 1418 int 1419 genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, 1420 int flags) 1421 { 1422 off_t off; 1423 vaddr_t kva; 1424 size_t len; 1425 int error; 1426 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1427 1428 UVMHIST_LOG(ubchist, "vp %#jx pgs %#jx npages %jd flags 0x%jx", 1429 (uintptr_t)vp, (uintptr_t)pgs, npages, flags); 1430 1431 off = pgs[0]->offset; 1432 kva = uvm_pagermapin(pgs, npages, 1433 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1434 len = npages << PAGE_SHIFT; 1435 1436 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1437 uvm_aio_aiodone); 1438 1439 return error; 1440 } 1441 1442 /* 1443 * Backend routine for doing I/O to vnode pages. Pages are already locked 1444 * and mapped into kernel memory. Here we just look up the underlying 1445 * device block addresses and call the strategy routine. 1446 */ 1447 1448 static int 1449 genfs_do_io(struct vnode *vp, off_t off, vaddr_t kva, size_t len, int flags, 1450 enum uio_rw rw, void (*iodone)(struct buf *)) 1451 { 1452 int s, error; 1453 int fs_bshift, dev_bshift; 1454 off_t eof, offset, startoffset; 1455 size_t bytes, iobytes, skipbytes; 1456 struct buf *mbp, *bp; 1457 const bool async = (flags & PGO_SYNCIO) == 0; 1458 const bool lazy = (flags & PGO_LAZY) == 0; 1459 const bool iowrite = rw == UIO_WRITE; 1460 const int brw = iowrite ? B_WRITE : B_READ; 1461 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1462 1463 UVMHIST_LOG(ubchist, "vp %#jx kva %#jx len 0x%jx flags 0x%jx", 1464 (uintptr_t)vp, (uintptr_t)kva, len, flags); 1465 1466 KASSERT(vp->v_size <= vp->v_writesize); 1467 GOP_SIZE(vp, vp->v_writesize, &eof, 0); 1468 if (vp->v_type != VBLK) { 1469 fs_bshift = vp->v_mount->mnt_fs_bshift; 1470 dev_bshift = vp->v_mount->mnt_dev_bshift; 1471 } else { 1472 fs_bshift = DEV_BSHIFT; 1473 dev_bshift = DEV_BSHIFT; 1474 } 1475 error = 0; 1476 startoffset = off; 1477 bytes = MIN(len, eof - startoffset); 1478 skipbytes = 0; 1479 KASSERT(bytes != 0); 1480 1481 if (iowrite) { 1482 /* 1483 * why += 2? 1484 * 1 for biodone, 1 for uvm_aio_aiodone. 1485 */ 1486 mutex_enter(vp->v_interlock); 1487 vp->v_numoutput += 2; 1488 mutex_exit(vp->v_interlock); 1489 } 1490 mbp = getiobuf(vp, true); 1491 UVMHIST_LOG(ubchist, "vp %#jx mbp %#jx num now %jd bytes 0x%jx", 1492 (uintptr_t)vp, (uintptr_t)mbp, vp->v_numoutput, bytes); 1493 mbp->b_bufsize = len; 1494 mbp->b_data = (void *)kva; 1495 mbp->b_resid = mbp->b_bcount = bytes; 1496 mbp->b_cflags |= BC_BUSY | BC_AGE; 1497 if (async) { 1498 mbp->b_flags = brw | B_ASYNC; 1499 mbp->b_iodone = iodone; 1500 } else { 1501 mbp->b_flags = brw; 1502 mbp->b_iodone = NULL; 1503 } 1504 if (curlwp == uvm.pagedaemon_lwp) 1505 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 1506 else if (async || lazy) 1507 BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL); 1508 else 1509 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 1510 1511 bp = NULL; 1512 for (offset = startoffset; 1513 bytes > 0; 1514 offset += iobytes, bytes -= iobytes) { 1515 int run; 1516 daddr_t lbn, blkno; 1517 struct vnode *devvp; 1518 1519 /* 1520 * bmap the file to find out the blkno to read from and 1521 * how much we can read in one i/o. if bmap returns an error, 1522 * skip the rest of the top-level i/o. 1523 */ 1524 1525 lbn = offset >> fs_bshift; 1526 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 1527 if (error) { 1528 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%jx -> %jd\n", 1529 lbn, error, 0, 0); 1530 skipbytes += bytes; 1531 bytes = 0; 1532 goto loopdone; 1533 } 1534 1535 /* 1536 * see how many pages can be read with this i/o. 1537 * reduce the i/o size if necessary to avoid 1538 * overwriting pages with valid data. 1539 */ 1540 1541 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 1542 bytes); 1543 1544 /* 1545 * if this block isn't allocated, zero it instead of 1546 * reading it. unless we are going to allocate blocks, 1547 * mark the pages we zeroed PG_RDONLY. 1548 */ 1549 1550 if (blkno == (daddr_t)-1) { 1551 if (!iowrite) { 1552 memset((char *)kva + (offset - startoffset), 0, 1553 iobytes); 1554 } 1555 skipbytes += iobytes; 1556 continue; 1557 } 1558 1559 /* 1560 * allocate a sub-buf for this piece of the i/o 1561 * (or just use mbp if there's only 1 piece), 1562 * and start it going. 1563 */ 1564 1565 if (offset == startoffset && iobytes == bytes) { 1566 bp = mbp; 1567 } else { 1568 UVMHIST_LOG(ubchist, "vp %#jx bp %#jx num now %jd", 1569 (uintptr_t)vp, (uintptr_t)bp, vp->v_numoutput, 0); 1570 bp = getiobuf(vp, true); 1571 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 1572 } 1573 bp->b_lblkno = 0; 1574 1575 /* adjust physical blkno for partial blocks */ 1576 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 1577 dev_bshift); 1578 1579 UVMHIST_LOG(ubchist, 1580 "bp %#jx offset 0x%jx bcount 0x%jx blkno 0x%jx", 1581 (uintptr_t)bp, offset, bp->b_bcount, bp->b_blkno); 1582 1583 VOP_STRATEGY(devvp, bp); 1584 } 1585 1586 loopdone: 1587 if (skipbytes) { 1588 UVMHIST_LOG(ubchist, "skipbytes %jd", skipbytes, 0,0,0); 1589 } 1590 nestiobuf_done(mbp, skipbytes, error); 1591 if (async) { 1592 UVMHIST_LOG(ubchist, "returning 0 (async)", 0,0,0,0); 1593 return (0); 1594 } 1595 UVMHIST_LOG(ubchist, "waiting for mbp %#jx", (uintptr_t)mbp, 0, 0, 0); 1596 error = biowait(mbp); 1597 s = splbio(); 1598 (*iodone)(mbp); 1599 splx(s); 1600 UVMHIST_LOG(ubchist, "returning, error %jd", error, 0, 0, 0); 1601 return (error); 1602 } 1603 1604 int 1605 genfs_compat_getpages(void *v) 1606 { 1607 struct vop_getpages_args /* { 1608 struct vnode *a_vp; 1609 voff_t a_offset; 1610 struct vm_page **a_m; 1611 int *a_count; 1612 int a_centeridx; 1613 vm_prot_t a_access_type; 1614 int a_advice; 1615 int a_flags; 1616 } */ *ap = v; 1617 1618 off_t origoffset; 1619 struct vnode *vp = ap->a_vp; 1620 struct uvm_object *uobj = &vp->v_uobj; 1621 struct vm_page *pg, **pgs; 1622 vaddr_t kva; 1623 int i, error, orignpages, npages; 1624 struct iovec iov; 1625 struct uio uio; 1626 kauth_cred_t cred = curlwp->l_cred; 1627 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 1628 1629 error = 0; 1630 origoffset = ap->a_offset; 1631 orignpages = *ap->a_count; 1632 pgs = ap->a_m; 1633 1634 if (ap->a_flags & PGO_LOCKED) { 1635 uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, NULL, 1636 UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0)); 1637 1638 error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0; 1639 return error; 1640 } 1641 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { 1642 rw_exit(uobj->vmobjlock); 1643 return EINVAL; 1644 } 1645 if ((ap->a_flags & PGO_SYNCIO) == 0) { 1646 rw_exit(uobj->vmobjlock); 1647 return 0; 1648 } 1649 npages = orignpages; 1650 uvn_findpages(uobj, origoffset, &npages, pgs, NULL, UFP_ALL); 1651 rw_exit(uobj->vmobjlock); 1652 kva = uvm_pagermapin(pgs, npages, 1653 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1654 for (i = 0; i < npages; i++) { 1655 pg = pgs[i]; 1656 if ((pg->flags & PG_FAKE) == 0) { 1657 continue; 1658 } 1659 iov.iov_base = (char *)kva + (i << PAGE_SHIFT); 1660 iov.iov_len = PAGE_SIZE; 1661 uio.uio_iov = &iov; 1662 uio.uio_iovcnt = 1; 1663 uio.uio_offset = origoffset + (i << PAGE_SHIFT); 1664 uio.uio_rw = UIO_READ; 1665 uio.uio_resid = PAGE_SIZE; 1666 UIO_SETUP_SYSSPACE(&uio); 1667 /* XXX vn_lock */ 1668 error = VOP_READ(vp, &uio, 0, cred); 1669 if (error) { 1670 break; 1671 } 1672 if (uio.uio_resid) { 1673 memset(iov.iov_base, 0, uio.uio_resid); 1674 } 1675 } 1676 uvm_pagermapout(kva, npages); 1677 rw_enter(uobj->vmobjlock, RW_WRITER); 1678 for (i = 0; i < npages; i++) { 1679 pg = pgs[i]; 1680 if (error && (pg->flags & PG_FAKE) != 0) { 1681 pg->flags |= PG_RELEASED; 1682 } else { 1683 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_UNKNOWN); 1684 uvm_pagelock(pg); 1685 uvm_pageactivate(pg); 1686 uvm_pageunlock(pg); 1687 } 1688 } 1689 if (error) { 1690 uvm_page_unbusy(pgs, npages); 1691 } 1692 rw_exit(uobj->vmobjlock); 1693 return error; 1694 } 1695 1696 int 1697 genfs_compat_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, 1698 int flags) 1699 { 1700 off_t offset; 1701 struct iovec iov; 1702 struct uio uio; 1703 kauth_cred_t cred = curlwp->l_cred; 1704 struct buf *bp; 1705 vaddr_t kva; 1706 int error; 1707 1708 offset = pgs[0]->offset; 1709 kva = uvm_pagermapin(pgs, npages, 1710 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1711 1712 iov.iov_base = (void *)kva; 1713 iov.iov_len = npages << PAGE_SHIFT; 1714 uio.uio_iov = &iov; 1715 uio.uio_iovcnt = 1; 1716 uio.uio_offset = offset; 1717 uio.uio_rw = UIO_WRITE; 1718 uio.uio_resid = npages << PAGE_SHIFT; 1719 UIO_SETUP_SYSSPACE(&uio); 1720 /* XXX vn_lock */ 1721 error = VOP_WRITE(vp, &uio, 0, cred); 1722 1723 mutex_enter(vp->v_interlock); 1724 vp->v_numoutput++; 1725 mutex_exit(vp->v_interlock); 1726 1727 bp = getiobuf(vp, true); 1728 bp->b_cflags |= BC_BUSY | BC_AGE; 1729 bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift; 1730 bp->b_data = (char *)kva; 1731 bp->b_bcount = npages << PAGE_SHIFT; 1732 bp->b_bufsize = npages << PAGE_SHIFT; 1733 bp->b_resid = 0; 1734 bp->b_error = error; 1735 uvm_aio_aiodone(bp); 1736 return (error); 1737 } 1738 1739 /* 1740 * Process a uio using direct I/O. If we reach a part of the request 1741 * which cannot be processed in this fashion for some reason, just return. 1742 * The caller must handle some additional part of the request using 1743 * buffered I/O before trying direct I/O again. 1744 */ 1745 1746 void 1747 genfs_directio(struct vnode *vp, struct uio *uio, int ioflag) 1748 { 1749 struct vmspace *vs; 1750 struct iovec *iov; 1751 vaddr_t va; 1752 size_t len; 1753 const int mask = DEV_BSIZE - 1; 1754 int error; 1755 bool need_wapbl = (vp->v_mount && vp->v_mount->mnt_wapbl && 1756 (ioflag & IO_JOURNALLOCKED) == 0); 1757 1758 #ifdef DIAGNOSTIC 1759 if ((ioflag & IO_JOURNALLOCKED) && vp->v_mount->mnt_wapbl) 1760 WAPBL_JLOCK_ASSERT(vp->v_mount); 1761 #endif 1762 1763 /* 1764 * We only support direct I/O to user space for now. 1765 */ 1766 1767 if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) { 1768 return; 1769 } 1770 1771 /* 1772 * If the vnode is mapped, we would need to get the getpages lock 1773 * to stabilize the bmap, but then we would get into trouble while 1774 * locking the pages if the pages belong to this same vnode (or a 1775 * multi-vnode cascade to the same effect). Just fall back to 1776 * buffered I/O if the vnode is mapped to avoid this mess. 1777 */ 1778 1779 if (vp->v_vflag & VV_MAPPED) { 1780 return; 1781 } 1782 1783 if (need_wapbl) { 1784 error = WAPBL_BEGIN(vp->v_mount); 1785 if (error) 1786 return; 1787 } 1788 1789 /* 1790 * Do as much of the uio as possible with direct I/O. 1791 */ 1792 1793 vs = uio->uio_vmspace; 1794 while (uio->uio_resid) { 1795 iov = uio->uio_iov; 1796 if (iov->iov_len == 0) { 1797 uio->uio_iov++; 1798 uio->uio_iovcnt--; 1799 continue; 1800 } 1801 va = (vaddr_t)iov->iov_base; 1802 len = MIN(iov->iov_len, genfs_maxdio); 1803 len &= ~mask; 1804 1805 /* 1806 * If the next chunk is smaller than DEV_BSIZE or extends past 1807 * the current EOF, then fall back to buffered I/O. 1808 */ 1809 1810 if (len == 0 || uio->uio_offset + len > vp->v_size) { 1811 break; 1812 } 1813 1814 /* 1815 * Check alignment. The file offset must be at least 1816 * sector-aligned. The exact constraint on memory alignment 1817 * is very hardware-dependent, but requiring sector-aligned 1818 * addresses there too is safe. 1819 */ 1820 1821 if (uio->uio_offset & mask || va & mask) { 1822 break; 1823 } 1824 error = genfs_do_directio(vs, va, len, vp, uio->uio_offset, 1825 uio->uio_rw); 1826 if (error) { 1827 break; 1828 } 1829 iov->iov_base = (char *)iov->iov_base + len; 1830 iov->iov_len -= len; 1831 uio->uio_offset += len; 1832 uio->uio_resid -= len; 1833 } 1834 1835 if (need_wapbl) 1836 WAPBL_END(vp->v_mount); 1837 } 1838 1839 /* 1840 * Iodone routine for direct I/O. We don't do much here since the request is 1841 * always synchronous, so the caller will do most of the work after biowait(). 1842 */ 1843 1844 static void 1845 genfs_dio_iodone(struct buf *bp) 1846 { 1847 1848 KASSERT((bp->b_flags & B_ASYNC) == 0); 1849 if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) { 1850 mutex_enter(bp->b_objlock); 1851 vwakeup(bp); 1852 mutex_exit(bp->b_objlock); 1853 } 1854 putiobuf(bp); 1855 } 1856 1857 /* 1858 * Process one chunk of a direct I/O request. 1859 */ 1860 1861 static int 1862 genfs_do_directio(struct vmspace *vs, vaddr_t uva, size_t len, struct vnode *vp, 1863 off_t off, enum uio_rw rw) 1864 { 1865 struct vm_map *map; 1866 struct pmap *upm, *kpm __unused; 1867 size_t klen = round_page(uva + len) - trunc_page(uva); 1868 off_t spoff, epoff; 1869 vaddr_t kva, puva; 1870 paddr_t pa; 1871 vm_prot_t prot; 1872 int error, rv __diagused, poff, koff; 1873 const int pgoflags = PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED | 1874 (rw == UIO_WRITE ? PGO_FREE : 0); 1875 1876 /* 1877 * For writes, verify that this range of the file already has fully 1878 * allocated backing store. If there are any holes, just punt and 1879 * make the caller take the buffered write path. 1880 */ 1881 1882 if (rw == UIO_WRITE) { 1883 daddr_t lbn, elbn, blkno; 1884 int bsize, bshift, run; 1885 1886 bshift = vp->v_mount->mnt_fs_bshift; 1887 bsize = 1 << bshift; 1888 lbn = off >> bshift; 1889 elbn = (off + len + bsize - 1) >> bshift; 1890 while (lbn < elbn) { 1891 error = VOP_BMAP(vp, lbn, NULL, &blkno, &run); 1892 if (error) { 1893 return error; 1894 } 1895 if (blkno == (daddr_t)-1) { 1896 return ENOSPC; 1897 } 1898 lbn += 1 + run; 1899 } 1900 } 1901 1902 /* 1903 * Flush any cached pages for parts of the file that we're about to 1904 * access. If we're writing, invalidate pages as well. 1905 */ 1906 1907 spoff = trunc_page(off); 1908 epoff = round_page(off + len); 1909 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 1910 error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags); 1911 if (error) { 1912 return error; 1913 } 1914 1915 /* 1916 * Wire the user pages and remap them into kernel memory. 1917 */ 1918 1919 prot = rw == UIO_READ ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; 1920 error = uvm_vslock(vs, (void *)uva, len, prot); 1921 if (error) { 1922 return error; 1923 } 1924 1925 map = &vs->vm_map; 1926 upm = vm_map_pmap(map); 1927 kpm = vm_map_pmap(kernel_map); 1928 puva = trunc_page(uva); 1929 kva = uvm_km_alloc(kernel_map, klen, atop(puva) & uvmexp.colormask, 1930 UVM_KMF_VAONLY | UVM_KMF_WAITVA | UVM_KMF_COLORMATCH); 1931 for (poff = 0; poff < klen; poff += PAGE_SIZE) { 1932 rv = pmap_extract(upm, puva + poff, &pa); 1933 KASSERT(rv); 1934 pmap_kenter_pa(kva + poff, pa, prot, PMAP_WIRED); 1935 } 1936 pmap_update(kpm); 1937 1938 /* 1939 * Do the I/O. 1940 */ 1941 1942 koff = uva - trunc_page(uva); 1943 error = genfs_do_io(vp, off, kva + koff, len, PGO_SYNCIO, rw, 1944 genfs_dio_iodone); 1945 1946 /* 1947 * Tear down the kernel mapping. 1948 */ 1949 1950 pmap_kremove(kva, klen); 1951 pmap_update(kpm); 1952 uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY); 1953 1954 /* 1955 * Unwire the user pages. 1956 */ 1957 1958 uvm_vsunlock(vs, (void *)uva, len); 1959 return error; 1960 } 1961