1 /* $NetBSD: nfs_bio.c,v 1.185 2010/06/12 21:10:55 jakllsch Exp $ */ 2 3 /* 4 * Copyright (c) 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from software contributed to Berkeley by 8 * Rick Macklem at The University of Guelph. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * @(#)nfs_bio.c 8.9 (Berkeley) 3/30/95 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: nfs_bio.c,v 1.185 2010/06/12 21:10:55 jakllsch Exp $"); 39 40 #ifdef _KERNEL_OPT 41 #include "opt_nfs.h" 42 #include "opt_ddb.h" 43 #endif 44 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/resourcevar.h> 48 #include <sys/signalvar.h> 49 #include <sys/proc.h> 50 #include <sys/buf.h> 51 #include <sys/vnode.h> 52 #include <sys/mount.h> 53 #include <sys/kernel.h> 54 #include <sys/namei.h> 55 #include <sys/dirent.h> 56 #include <sys/kauth.h> 57 58 #include <uvm/uvm_extern.h> 59 #include <uvm/uvm.h> 60 61 #include <nfs/rpcv2.h> 62 #include <nfs/nfsproto.h> 63 #include <nfs/nfs.h> 64 #include <nfs/nfsmount.h> 65 #include <nfs/nfsnode.h> 66 #include <nfs/nfs_var.h> 67 68 extern int nfs_numasync; 69 extern int nfs_commitsize; 70 extern struct nfsstats nfsstats; 71 72 static int nfs_doio_read(struct buf *, struct uio *); 73 static int nfs_doio_write(struct buf *, struct uio *); 74 static int nfs_doio_phys(struct buf *, struct uio *); 75 76 /* 77 * Vnode op for read using bio 78 * Any similarity to readip() is purely coincidental 79 */ 80 int 81 nfs_bioread(struct vnode *vp, struct uio *uio, int ioflag, 82 kauth_cred_t cred, int cflag) 83 { 84 struct nfsnode *np = VTONFS(vp); 85 struct buf *bp = NULL, *rabp; 86 struct nfsmount *nmp = VFSTONFS(vp->v_mount); 87 struct nfsdircache *ndp = NULL, *nndp = NULL; 88 void *baddr; 89 int got_buf = 0, error = 0, n = 0, on = 0, en, enn; 90 int enough = 0; 91 struct dirent *dp, *pdp, *edp, *ep; 92 off_t curoff = 0; 93 int advice; 94 struct lwp *l = curlwp; 95 96 #ifdef DIAGNOSTIC 97 if (uio->uio_rw != UIO_READ) 98 panic("nfs_read mode"); 99 #endif 100 if (uio->uio_resid == 0) 101 return (0); 102 if (vp->v_type != VDIR && uio->uio_offset < 0) 103 return (EINVAL); 104 #ifndef NFS_V2_ONLY 105 if ((nmp->nm_flag & NFSMNT_NFSV3) && 106 !(nmp->nm_iflag & NFSMNT_GOTFSINFO)) 107 (void)nfs_fsinfo(nmp, vp, cred, l); 108 #endif 109 if (vp->v_type != VDIR && 110 (uio->uio_offset + uio->uio_resid) > nmp->nm_maxfilesize) 111 return (EFBIG); 112 113 /* 114 * For nfs, cache consistency can only be maintained approximately. 115 * Although RFC1094 does not specify the criteria, the following is 116 * believed to be compatible with the reference port. 117 * 118 * If the file's modify time on the server has changed since the 119 * last read rpc or you have written to the file, 120 * you may have lost data cache consistency with the 121 * server, so flush all of the file's data out of the cache. 122 * Then force a getattr rpc to ensure that you have up to date 123 * attributes. 124 * NB: This implies that cache data can be read when up to 125 * nfs_attrtimeo seconds out of date. If you find that you need current 126 * attributes this could be forced by setting n_attrstamp to 0 before 127 * the VOP_GETATTR() call. 128 */ 129 130 if (vp->v_type != VLNK) { 131 error = nfs_flushstalebuf(vp, cred, l, 132 NFS_FLUSHSTALEBUF_MYWRITE); 133 if (error) 134 return error; 135 } 136 137 do { 138 /* 139 * Don't cache symlinks. 140 */ 141 if ((vp->v_vflag & VV_ROOT) && vp->v_type == VLNK) { 142 return (nfs_readlinkrpc(vp, uio, cred)); 143 } 144 baddr = (void *)0; 145 switch (vp->v_type) { 146 case VREG: 147 nfsstats.biocache_reads++; 148 149 advice = IO_ADV_DECODE(ioflag); 150 error = 0; 151 while (uio->uio_resid > 0) { 152 vsize_t bytelen; 153 154 nfs_delayedtruncate(vp); 155 if (np->n_size <= uio->uio_offset) { 156 break; 157 } 158 bytelen = 159 MIN(np->n_size - uio->uio_offset, uio->uio_resid); 160 error = ubc_uiomove(&vp->v_uobj, uio, bytelen, 161 advice, UBC_READ | UBC_PARTIALOK | 162 (UBC_WANT_UNMAP(vp) ? UBC_UNMAP : 0)); 163 if (error) { 164 /* 165 * XXXkludge 166 * the file has been truncated on the server. 167 * there isn't much we can do. 168 */ 169 if (uio->uio_offset >= np->n_size) { 170 /* end of file */ 171 error = 0; 172 } else { 173 break; 174 } 175 } 176 } 177 break; 178 179 case VLNK: 180 nfsstats.biocache_readlinks++; 181 bp = nfs_getcacheblk(vp, (daddr_t)0, NFS_MAXPATHLEN, l); 182 if (!bp) 183 return (EINTR); 184 if ((bp->b_oflags & BO_DONE) == 0) { 185 bp->b_flags |= B_READ; 186 error = nfs_doio(bp); 187 if (error) { 188 brelse(bp, 0); 189 return (error); 190 } 191 } 192 n = MIN(uio->uio_resid, NFS_MAXPATHLEN - bp->b_resid); 193 got_buf = 1; 194 on = 0; 195 break; 196 case VDIR: 197 diragain: 198 nfsstats.biocache_readdirs++; 199 ndp = nfs_searchdircache(vp, uio->uio_offset, 200 (nmp->nm_flag & NFSMNT_XLATECOOKIE), 0); 201 if (!ndp) { 202 /* 203 * We've been handed a cookie that is not 204 * in the cache. If we're not translating 205 * 32 <-> 64, it may be a value that was 206 * flushed out of the cache because it grew 207 * too big. Let the server judge if it's 208 * valid or not. In the translation case, 209 * we have no way of validating this value, 210 * so punt. 211 */ 212 if (nmp->nm_flag & NFSMNT_XLATECOOKIE) 213 return (EINVAL); 214 ndp = nfs_enterdircache(vp, uio->uio_offset, 215 uio->uio_offset, 0, 0); 216 } 217 218 if (NFS_EOFVALID(np) && 219 ndp->dc_cookie == np->n_direofoffset) { 220 nfs_putdircache(np, ndp); 221 nfsstats.direofcache_hits++; 222 return (0); 223 } 224 225 bp = nfs_getcacheblk(vp, NFSDC_BLKNO(ndp), NFS_DIRBLKSIZ, l); 226 if (!bp) 227 return (EINTR); 228 if ((bp->b_oflags & BO_DONE) == 0) { 229 bp->b_flags |= B_READ; 230 bp->b_dcookie = ndp->dc_blkcookie; 231 error = nfs_doio(bp); 232 if (error) { 233 /* 234 * Yuck! The directory has been modified on the 235 * server. Punt and let the userland code 236 * deal with it. 237 */ 238 nfs_putdircache(np, ndp); 239 brelse(bp, 0); 240 /* 241 * nfs_request maps NFSERR_BAD_COOKIE to EINVAL. 242 */ 243 if (error == EINVAL) { /* NFSERR_BAD_COOKIE */ 244 nfs_invaldircache(vp, 0); 245 nfs_vinvalbuf(vp, 0, cred, l, 1); 246 } 247 return (error); 248 } 249 } 250 251 /* 252 * Just return if we hit EOF right away with this 253 * block. Always check here, because direofoffset 254 * may have been set by an nfsiod since the last 255 * check. 256 * 257 * also, empty block implies EOF. 258 */ 259 260 if (bp->b_bcount == bp->b_resid || 261 (NFS_EOFVALID(np) && 262 ndp->dc_blkcookie == np->n_direofoffset)) { 263 KASSERT(bp->b_bcount != bp->b_resid || 264 ndp->dc_blkcookie == bp->b_dcookie); 265 nfs_putdircache(np, ndp); 266 brelse(bp, BC_NOCACHE); 267 return 0; 268 } 269 270 /* 271 * Find the entry we were looking for in the block. 272 */ 273 274 en = ndp->dc_entry; 275 276 pdp = dp = (struct dirent *)bp->b_data; 277 edp = (struct dirent *)(void *)((char *)bp->b_data + bp->b_bcount - 278 bp->b_resid); 279 enn = 0; 280 while (enn < en && dp < edp) { 281 pdp = dp; 282 dp = _DIRENT_NEXT(dp); 283 enn++; 284 } 285 286 /* 287 * If the entry number was bigger than the number of 288 * entries in the block, or the cookie of the previous 289 * entry doesn't match, the directory cache is 290 * stale. Flush it and try again (i.e. go to 291 * the server). 292 */ 293 if (dp >= edp || (struct dirent *)_DIRENT_NEXT(dp) > edp || 294 (en > 0 && NFS_GETCOOKIE(pdp) != ndp->dc_cookie)) { 295 #ifdef DEBUG 296 printf("invalid cache: %p %p %p off %jx %jx\n", 297 pdp, dp, edp, 298 (uintmax_t)uio->uio_offset, 299 (uintmax_t)NFS_GETCOOKIE(pdp)); 300 #endif 301 nfs_putdircache(np, ndp); 302 brelse(bp, 0); 303 nfs_invaldircache(vp, 0); 304 nfs_vinvalbuf(vp, 0, cred, l, 0); 305 goto diragain; 306 } 307 308 on = (char *)dp - (char *)bp->b_data; 309 310 /* 311 * Cache all entries that may be exported to the 312 * user, as they may be thrown back at us. The 313 * NFSBIO_CACHECOOKIES flag indicates that all 314 * entries are being 'exported', so cache them all. 315 */ 316 317 if (en == 0 && pdp == dp) { 318 dp = _DIRENT_NEXT(dp); 319 enn++; 320 } 321 322 if (uio->uio_resid < (bp->b_bcount - bp->b_resid - on)) { 323 n = uio->uio_resid; 324 enough = 1; 325 } else 326 n = bp->b_bcount - bp->b_resid - on; 327 328 ep = (struct dirent *)(void *)((char *)bp->b_data + on + n); 329 330 /* 331 * Find last complete entry to copy, caching entries 332 * (if requested) as we go. 333 */ 334 335 while (dp < ep && (struct dirent *)_DIRENT_NEXT(dp) <= ep) { 336 if (cflag & NFSBIO_CACHECOOKIES) { 337 nndp = nfs_enterdircache(vp, NFS_GETCOOKIE(pdp), 338 ndp->dc_blkcookie, enn, bp->b_lblkno); 339 if (nmp->nm_flag & NFSMNT_XLATECOOKIE) { 340 NFS_STASHCOOKIE32(pdp, 341 nndp->dc_cookie32); 342 } 343 nfs_putdircache(np, nndp); 344 } 345 pdp = dp; 346 dp = _DIRENT_NEXT(dp); 347 enn++; 348 } 349 nfs_putdircache(np, ndp); 350 351 /* 352 * If the last requested entry was not the last in the 353 * buffer (happens if NFS_DIRFRAGSIZ < NFS_DIRBLKSIZ), 354 * cache the cookie of the last requested one, and 355 * set of the offset to it. 356 */ 357 358 if ((on + n) < bp->b_bcount - bp->b_resid) { 359 curoff = NFS_GETCOOKIE(pdp); 360 nndp = nfs_enterdircache(vp, curoff, ndp->dc_blkcookie, 361 enn, bp->b_lblkno); 362 if (nmp->nm_flag & NFSMNT_XLATECOOKIE) { 363 NFS_STASHCOOKIE32(pdp, nndp->dc_cookie32); 364 curoff = nndp->dc_cookie32; 365 } 366 nfs_putdircache(np, nndp); 367 } else 368 curoff = bp->b_dcookie; 369 370 /* 371 * Always cache the entry for the next block, 372 * so that readaheads can use it. 373 */ 374 nndp = nfs_enterdircache(vp, bp->b_dcookie, bp->b_dcookie, 0,0); 375 if (nmp->nm_flag & NFSMNT_XLATECOOKIE) { 376 if (curoff == bp->b_dcookie) { 377 NFS_STASHCOOKIE32(pdp, nndp->dc_cookie32); 378 curoff = nndp->dc_cookie32; 379 } 380 } 381 382 n = (char *)_DIRENT_NEXT(pdp) - ((char *)bp->b_data + on); 383 384 /* 385 * If not eof and read aheads are enabled, start one. 386 * (You need the current block first, so that you have the 387 * directory offset cookie of the next block.) 388 */ 389 if (nfs_numasync > 0 && nmp->nm_readahead > 0 && 390 !NFS_EOFVALID(np)) { 391 rabp = nfs_getcacheblk(vp, NFSDC_BLKNO(nndp), 392 NFS_DIRBLKSIZ, l); 393 if (rabp) { 394 if ((rabp->b_oflags & (BO_DONE | BO_DELWRI)) == 0) { 395 rabp->b_dcookie = nndp->dc_cookie; 396 rabp->b_flags |= (B_READ | B_ASYNC); 397 if (nfs_asyncio(rabp)) { 398 brelse(rabp, BC_INVAL); 399 } 400 } else 401 brelse(rabp, 0); 402 } 403 } 404 nfs_putdircache(np, nndp); 405 got_buf = 1; 406 break; 407 default: 408 printf(" nfsbioread: type %x unexpected\n",vp->v_type); 409 break; 410 } 411 412 if (n > 0) { 413 if (!baddr) 414 baddr = bp->b_data; 415 error = uiomove((char *)baddr + on, (int)n, uio); 416 } 417 switch (vp->v_type) { 418 case VREG: 419 break; 420 case VLNK: 421 n = 0; 422 break; 423 case VDIR: 424 uio->uio_offset = curoff; 425 if (enough) 426 n = 0; 427 break; 428 default: 429 printf(" nfsbioread: type %x unexpected\n",vp->v_type); 430 } 431 if (got_buf) 432 brelse(bp, 0); 433 } while (error == 0 && uio->uio_resid > 0 && n > 0); 434 return (error); 435 } 436 437 /* 438 * Vnode op for write using bio 439 */ 440 int 441 nfs_write(void *v) 442 { 443 struct vop_write_args /* { 444 struct vnode *a_vp; 445 struct uio *a_uio; 446 int a_ioflag; 447 kauth_cred_t a_cred; 448 } */ *ap = v; 449 struct uio *uio = ap->a_uio; 450 struct lwp *l = curlwp; 451 struct vnode *vp = ap->a_vp; 452 struct nfsnode *np = VTONFS(vp); 453 kauth_cred_t cred = ap->a_cred; 454 struct nfsmount *nmp = VFSTONFS(vp->v_mount); 455 voff_t oldoff, origoff; 456 vsize_t bytelen; 457 int error = 0; 458 int ioflag = ap->a_ioflag; 459 int extended = 0, wrotedata = 0; 460 461 #ifdef DIAGNOSTIC 462 if (uio->uio_rw != UIO_WRITE) 463 panic("nfs_write mode"); 464 #endif 465 if (vp->v_type != VREG) 466 return (EIO); 467 if (np->n_flag & NWRITEERR) { 468 np->n_flag &= ~NWRITEERR; 469 return (np->n_error); 470 } 471 #ifndef NFS_V2_ONLY 472 if ((nmp->nm_flag & NFSMNT_NFSV3) && 473 !(nmp->nm_iflag & NFSMNT_GOTFSINFO)) 474 (void)nfs_fsinfo(nmp, vp, cred, l); 475 #endif 476 if (ioflag & IO_APPEND) { 477 NFS_INVALIDATE_ATTRCACHE(np); 478 error = nfs_flushstalebuf(vp, cred, l, 479 NFS_FLUSHSTALEBUF_MYWRITE); 480 if (error) 481 return (error); 482 uio->uio_offset = np->n_size; 483 484 /* 485 * This is already checked above VOP_WRITE, but recheck 486 * the append case here to make sure our idea of the 487 * file size is as fresh as possible. 488 */ 489 if (uio->uio_offset + uio->uio_resid > 490 l->l_proc->p_rlimit[RLIMIT_FSIZE].rlim_cur) { 491 mutex_enter(proc_lock); 492 psignal(l->l_proc, SIGXFSZ); 493 mutex_exit(proc_lock); 494 return (EFBIG); 495 } 496 } 497 if (uio->uio_offset < 0) 498 return (EINVAL); 499 if ((uio->uio_offset + uio->uio_resid) > nmp->nm_maxfilesize) 500 return (EFBIG); 501 if (uio->uio_resid == 0) 502 return (0); 503 504 origoff = uio->uio_offset; 505 do { 506 bool overwrite; /* if we are overwriting whole pages */ 507 u_quad_t oldsize; 508 oldoff = uio->uio_offset; 509 bytelen = uio->uio_resid; 510 511 nfsstats.biocache_writes++; 512 513 oldsize = np->n_size; 514 np->n_flag |= NMODIFIED; 515 if (np->n_size < uio->uio_offset + bytelen) { 516 np->n_size = uio->uio_offset + bytelen; 517 } 518 overwrite = false; 519 if ((uio->uio_offset & PAGE_MASK) == 0) { 520 if ((vp->v_vflag & VV_MAPPED) == 0 && 521 bytelen > PAGE_SIZE) { 522 bytelen = trunc_page(bytelen); 523 overwrite = true; 524 } else if ((bytelen & PAGE_MASK) == 0 && 525 uio->uio_offset >= vp->v_size) { 526 overwrite = true; 527 } 528 } 529 if (vp->v_size < uio->uio_offset + bytelen) { 530 uvm_vnp_setwritesize(vp, uio->uio_offset + bytelen); 531 } 532 error = ubc_uiomove(&vp->v_uobj, uio, bytelen, 533 UVM_ADV_RANDOM, UBC_WRITE | UBC_PARTIALOK | 534 (overwrite ? UBC_FAULTBUSY : 0) | 535 (UBC_WANT_UNMAP(vp) ? UBC_UNMAP : 0)); 536 if (error) { 537 uvm_vnp_setwritesize(vp, vp->v_size); 538 if (overwrite && np->n_size != oldsize) { 539 /* 540 * backout size and free pages past eof. 541 */ 542 np->n_size = oldsize; 543 mutex_enter(&vp->v_interlock); 544 (void)VOP_PUTPAGES(vp, round_page(vp->v_size), 545 0, PGO_SYNCIO | PGO_FREE); 546 } 547 break; 548 } 549 wrotedata = 1; 550 551 /* 552 * update UVM's notion of the size now that we've 553 * copied the data into the vnode's pages. 554 */ 555 556 if (vp->v_size < uio->uio_offset) { 557 uvm_vnp_setsize(vp, uio->uio_offset); 558 extended = 1; 559 } 560 561 if ((oldoff & ~(nmp->nm_wsize - 1)) != 562 (uio->uio_offset & ~(nmp->nm_wsize - 1))) { 563 mutex_enter(&vp->v_interlock); 564 error = VOP_PUTPAGES(vp, 565 trunc_page(oldoff & ~(nmp->nm_wsize - 1)), 566 round_page((uio->uio_offset + nmp->nm_wsize - 1) & 567 ~(nmp->nm_wsize - 1)), PGO_CLEANIT); 568 } 569 } while (uio->uio_resid > 0); 570 if (wrotedata) 571 VN_KNOTE(vp, NOTE_WRITE | (extended ? NOTE_EXTEND : 0)); 572 if (error == 0 && (ioflag & IO_SYNC) != 0) { 573 mutex_enter(&vp->v_interlock); 574 error = VOP_PUTPAGES(vp, 575 trunc_page(origoff & ~(nmp->nm_wsize - 1)), 576 round_page((uio->uio_offset + nmp->nm_wsize - 1) & 577 ~(nmp->nm_wsize - 1)), 578 PGO_CLEANIT | PGO_SYNCIO); 579 } 580 return error; 581 } 582 583 /* 584 * Get an nfs cache block. 585 * Allocate a new one if the block isn't currently in the cache 586 * and return the block marked busy. If the calling process is 587 * interrupted by a signal for an interruptible mount point, return 588 * NULL. 589 */ 590 struct buf * 591 nfs_getcacheblk(struct vnode *vp, daddr_t bn, int size, struct lwp *l) 592 { 593 struct buf *bp; 594 struct nfsmount *nmp = VFSTONFS(vp->v_mount); 595 596 if (nmp->nm_flag & NFSMNT_INT) { 597 bp = getblk(vp, bn, size, PCATCH, 0); 598 while (bp == NULL) { 599 if (nfs_sigintr(nmp, NULL, l)) 600 return (NULL); 601 bp = getblk(vp, bn, size, 0, 2 * hz); 602 } 603 } else 604 bp = getblk(vp, bn, size, 0, 0); 605 return (bp); 606 } 607 608 /* 609 * Flush and invalidate all dirty buffers. If another process is already 610 * doing the flush, just wait for completion. 611 */ 612 int 613 nfs_vinvalbuf(struct vnode *vp, int flags, kauth_cred_t cred, 614 struct lwp *l, int intrflg) 615 { 616 struct nfsnode *np = VTONFS(vp); 617 struct nfsmount *nmp = VFSTONFS(vp->v_mount); 618 int error = 0, slptimeo; 619 bool catch; 620 621 if ((nmp->nm_flag & NFSMNT_INT) == 0) 622 intrflg = 0; 623 if (intrflg) { 624 catch = true; 625 slptimeo = 2 * hz; 626 } else { 627 catch = false; 628 slptimeo = 0; 629 } 630 /* 631 * First wait for any other process doing a flush to complete. 632 */ 633 mutex_enter(&vp->v_interlock); 634 while (np->n_flag & NFLUSHINPROG) { 635 np->n_flag |= NFLUSHWANT; 636 error = mtsleep(&np->n_flag, PRIBIO + 2, "nfsvinval", 637 slptimeo, &vp->v_interlock); 638 if (error && intrflg && nfs_sigintr(nmp, NULL, l)) { 639 mutex_exit(&vp->v_interlock); 640 return EINTR; 641 } 642 } 643 644 /* 645 * Now, flush as required. 646 */ 647 np->n_flag |= NFLUSHINPROG; 648 mutex_exit(&vp->v_interlock); 649 error = vinvalbuf(vp, flags, cred, l, catch, 0); 650 while (error) { 651 if (intrflg && nfs_sigintr(nmp, NULL, l)) { 652 error = EINTR; 653 break; 654 } 655 error = vinvalbuf(vp, flags, cred, l, 0, slptimeo); 656 } 657 mutex_enter(&vp->v_interlock); 658 if (error == 0) 659 np->n_flag &= ~NMODIFIED; 660 np->n_flag &= ~NFLUSHINPROG; 661 if (np->n_flag & NFLUSHWANT) { 662 np->n_flag &= ~NFLUSHWANT; 663 wakeup(&np->n_flag); 664 } 665 mutex_exit(&vp->v_interlock); 666 return error; 667 } 668 669 /* 670 * nfs_flushstalebuf: flush cache if it's stale. 671 * 672 * => caller shouldn't own any pages or buffers which belong to the vnode. 673 */ 674 675 int 676 nfs_flushstalebuf(struct vnode *vp, kauth_cred_t cred, struct lwp *l, 677 int flags) 678 { 679 struct nfsnode *np = VTONFS(vp); 680 struct vattr vattr; 681 int error; 682 683 if (np->n_flag & NMODIFIED) { 684 if ((flags & NFS_FLUSHSTALEBUF_MYWRITE) == 0 685 || vp->v_type != VREG) { 686 error = nfs_vinvalbuf(vp, V_SAVE, cred, l, 1); 687 if (error) 688 return error; 689 if (vp->v_type == VDIR) { 690 nfs_invaldircache(vp, 0); 691 } 692 } else { 693 /* 694 * XXX assuming writes are ours. 695 */ 696 } 697 NFS_INVALIDATE_ATTRCACHE(np); 698 error = VOP_GETATTR(vp, &vattr, cred); 699 if (error) 700 return error; 701 np->n_mtime = vattr.va_mtime; 702 } else { 703 error = VOP_GETATTR(vp, &vattr, cred); 704 if (error) 705 return error; 706 if (timespeccmp(&np->n_mtime, &vattr.va_mtime, !=)) { 707 if (vp->v_type == VDIR) { 708 nfs_invaldircache(vp, 0); 709 } 710 error = nfs_vinvalbuf(vp, V_SAVE, cred, l, 1); 711 if (error) 712 return error; 713 np->n_mtime = vattr.va_mtime; 714 } 715 } 716 717 return error; 718 } 719 720 /* 721 * Initiate asynchronous I/O. Return an error if no nfsiods are available. 722 * This is mainly to avoid queueing async I/O requests when the nfsiods 723 * are all hung on a dead server. 724 */ 725 726 int 727 nfs_asyncio(struct buf *bp) 728 { 729 struct nfs_iod *iod; 730 struct nfsmount *nmp; 731 int slptimeo = 0, error; 732 bool catch = false; 733 734 if (nfs_numasync == 0) 735 return (EIO); 736 737 nmp = VFSTONFS(bp->b_vp->v_mount); 738 again: 739 if (nmp->nm_flag & NFSMNT_INT) 740 catch = true; 741 742 /* 743 * Find a free iod to process this request. 744 */ 745 746 mutex_enter(&nfs_iodlist_lock); 747 iod = LIST_FIRST(&nfs_iodlist_idle); 748 if (iod) { 749 /* 750 * Found one, so wake it up and tell it which 751 * mount to process. 752 */ 753 LIST_REMOVE(iod, nid_idle); 754 mutex_enter(&iod->nid_lock); 755 mutex_exit(&nfs_iodlist_lock); 756 KASSERT(iod->nid_mount == NULL); 757 iod->nid_mount = nmp; 758 cv_signal(&iod->nid_cv); 759 mutex_enter(&nmp->nm_lock); 760 mutex_exit(&iod->nid_lock); 761 nmp->nm_bufqiods++; 762 if (nmp->nm_bufqlen < 2 * nmp->nm_bufqiods) { 763 cv_broadcast(&nmp->nm_aiocv); 764 } 765 } else { 766 mutex_exit(&nfs_iodlist_lock); 767 mutex_enter(&nmp->nm_lock); 768 } 769 770 KASSERT(mutex_owned(&nmp->nm_lock)); 771 772 /* 773 * If we have an iod which can process the request, then queue 774 * the buffer. However, even if we have an iod, do not initiate 775 * queue cleaning if curproc is the pageout daemon. if the NFS mount 776 * is via local loopback, we may put curproc (pagedaemon) to sleep 777 * waiting for the writes to complete. But the server (ourself) 778 * may block the write, waiting for its (ie., our) pagedaemon 779 * to produce clean pages to handle the write: deadlock. 780 * XXX: start non-loopback mounts straight away? If "lots free", 781 * let pagedaemon start loopback writes anyway? 782 */ 783 if (nmp->nm_bufqiods > 0) { 784 785 /* 786 * Ensure that the queue never grows too large. 787 */ 788 if (curlwp == uvm.pagedaemon_lwp) { 789 /* Enque for later, to avoid free-page deadlock */ 790 } else while (nmp->nm_bufqlen >= 2 * nmp->nm_bufqiods) { 791 if (catch) { 792 error = cv_timedwait_sig(&nmp->nm_aiocv, 793 &nmp->nm_lock, slptimeo); 794 } else { 795 error = cv_timedwait(&nmp->nm_aiocv, 796 &nmp->nm_lock, slptimeo); 797 } 798 if (error) { 799 if (nfs_sigintr(nmp, NULL, curlwp)) { 800 mutex_exit(&nmp->nm_lock); 801 return (EINTR); 802 } 803 if (catch) { 804 catch = false; 805 slptimeo = 2 * hz; 806 } 807 } 808 809 /* 810 * We might have lost our iod while sleeping, 811 * so check and loop if necessary. 812 */ 813 814 if (nmp->nm_bufqiods == 0) { 815 mutex_exit(&nmp->nm_lock); 816 goto again; 817 } 818 } 819 TAILQ_INSERT_TAIL(&nmp->nm_bufq, bp, b_freelist); 820 nmp->nm_bufqlen++; 821 mutex_exit(&nmp->nm_lock); 822 return (0); 823 } 824 mutex_exit(&nmp->nm_lock); 825 826 /* 827 * All the iods are busy on other mounts, so return EIO to 828 * force the caller to process the i/o synchronously. 829 */ 830 831 return (EIO); 832 } 833 834 /* 835 * nfs_doio for read. 836 */ 837 static int 838 nfs_doio_read(struct buf *bp, struct uio *uiop) 839 { 840 struct vnode *vp = bp->b_vp; 841 struct nfsnode *np = VTONFS(vp); 842 struct nfsmount *nmp = VFSTONFS(vp->v_mount); 843 int error = 0; 844 845 uiop->uio_rw = UIO_READ; 846 switch (vp->v_type) { 847 case VREG: 848 nfsstats.read_bios++; 849 error = nfs_readrpc(vp, uiop); 850 if (!error && uiop->uio_resid) { 851 int diff, len; 852 853 /* 854 * If uio_resid > 0, there is a hole in the file and 855 * no writes after the hole have been pushed to 856 * the server yet or the file has been truncated 857 * on the server. 858 * Just zero fill the rest of the valid area. 859 */ 860 861 KASSERT(vp->v_size >= 862 uiop->uio_offset + uiop->uio_resid); 863 diff = bp->b_bcount - uiop->uio_resid; 864 len = uiop->uio_resid; 865 memset((char *)bp->b_data + diff, 0, len); 866 uiop->uio_resid = 0; 867 } 868 #if 0 869 if (uiop->uio_lwp && (vp->v_iflag & VI_TEXT) && 870 timespeccmp(&np->n_mtime, &np->n_vattr->va_mtime, !=)) { 871 mutex_enter(proc_lock); 872 killproc(uiop->uio_lwp->l_proc, "process text file was modified"); 873 mutex_exit(proc_lock); 874 #if 0 /* XXX NJWLWP */ 875 uiop->uio_lwp->l_proc->p_holdcnt++; 876 #endif 877 } 878 #endif 879 break; 880 case VLNK: 881 KASSERT(uiop->uio_offset == (off_t)0); 882 nfsstats.readlink_bios++; 883 error = nfs_readlinkrpc(vp, uiop, np->n_rcred); 884 break; 885 case VDIR: 886 nfsstats.readdir_bios++; 887 uiop->uio_offset = bp->b_dcookie; 888 #ifndef NFS_V2_ONLY 889 if (nmp->nm_flag & NFSMNT_RDIRPLUS) { 890 error = nfs_readdirplusrpc(vp, uiop, 891 curlwp->l_cred); 892 /* 893 * nfs_request maps NFSERR_NOTSUPP to ENOTSUP. 894 */ 895 if (error == ENOTSUP) 896 nmp->nm_flag &= ~NFSMNT_RDIRPLUS; 897 } 898 #else 899 nmp->nm_flag &= ~NFSMNT_RDIRPLUS; 900 #endif 901 if ((nmp->nm_flag & NFSMNT_RDIRPLUS) == 0) 902 error = nfs_readdirrpc(vp, uiop, 903 curlwp->l_cred); 904 if (!error) { 905 bp->b_dcookie = uiop->uio_offset; 906 } 907 break; 908 default: 909 printf("nfs_doio: type %x unexpected\n", vp->v_type); 910 break; 911 } 912 bp->b_error = error; 913 return error; 914 } 915 916 /* 917 * nfs_doio for write. 918 */ 919 static int 920 nfs_doio_write(struct buf *bp, struct uio *uiop) 921 { 922 struct vnode *vp = bp->b_vp; 923 struct nfsnode *np = VTONFS(vp); 924 struct nfsmount *nmp = VFSTONFS(vp->v_mount); 925 int iomode; 926 bool stalewriteverf = false; 927 int i, npages = (bp->b_bcount + PAGE_SIZE - 1) >> PAGE_SHIFT; 928 struct vm_page **pgs, *spgs[UBC_MAX_PAGES]; 929 #ifndef NFS_V2_ONLY 930 bool needcommit = true; /* need only COMMIT RPC */ 931 #else 932 bool needcommit = false; /* need only COMMIT RPC */ 933 #endif 934 bool pageprotected; 935 struct uvm_object *uobj = &vp->v_uobj; 936 int error; 937 off_t off, cnt; 938 939 if (npages < __arraycount(spgs)) 940 pgs = spgs; 941 else { 942 if ((pgs = kmem_alloc(sizeof(*pgs) * npages, KM_NOSLEEP)) == 943 NULL) 944 return ENOMEM; 945 } 946 947 if ((bp->b_flags & B_ASYNC) != 0 && NFS_ISV3(vp)) { 948 iomode = NFSV3WRITE_UNSTABLE; 949 } else { 950 iomode = NFSV3WRITE_FILESYNC; 951 } 952 953 #ifndef NFS_V2_ONLY 954 again: 955 #endif 956 rw_enter(&nmp->nm_writeverflock, RW_READER); 957 958 for (i = 0; i < npages; i++) { 959 pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT)); 960 if (pgs[i]->uobject == uobj && 961 pgs[i]->offset == uiop->uio_offset + (i << PAGE_SHIFT)) { 962 KASSERT(pgs[i]->flags & PG_BUSY); 963 /* 964 * this page belongs to our object. 965 */ 966 mutex_enter(&uobj->vmobjlock); 967 /* 968 * write out the page stably if it's about to 969 * be released because we can't resend it 970 * on the server crash. 971 * 972 * XXX assuming PG_RELEASE|PG_PAGEOUT won't be 973 * changed until unbusy the page. 974 */ 975 if (pgs[i]->flags & (PG_RELEASED|PG_PAGEOUT)) 976 iomode = NFSV3WRITE_FILESYNC; 977 /* 978 * if we met a page which hasn't been sent yet, 979 * we need do WRITE RPC. 980 */ 981 if ((pgs[i]->flags & PG_NEEDCOMMIT) == 0) 982 needcommit = false; 983 mutex_exit(&uobj->vmobjlock); 984 } else { 985 iomode = NFSV3WRITE_FILESYNC; 986 needcommit = false; 987 } 988 } 989 if (!needcommit && iomode == NFSV3WRITE_UNSTABLE) { 990 mutex_enter(&uobj->vmobjlock); 991 for (i = 0; i < npages; i++) { 992 pgs[i]->flags |= PG_NEEDCOMMIT | PG_RDONLY; 993 pmap_page_protect(pgs[i], VM_PROT_READ); 994 } 995 mutex_exit(&uobj->vmobjlock); 996 pageprotected = true; /* pages can't be modified during i/o. */ 997 } else 998 pageprotected = false; 999 1000 /* 1001 * Send the data to the server if necessary, 1002 * otherwise just send a commit rpc. 1003 */ 1004 #ifndef NFS_V2_ONLY 1005 if (needcommit) { 1006 1007 /* 1008 * If the buffer is in the range that we already committed, 1009 * there's nothing to do. 1010 * 1011 * If it's in the range that we need to commit, push the 1012 * whole range at once, otherwise only push the buffer. 1013 * In both these cases, acquire the commit lock to avoid 1014 * other processes modifying the range. 1015 */ 1016 1017 off = uiop->uio_offset; 1018 cnt = bp->b_bcount; 1019 mutex_enter(&np->n_commitlock); 1020 if (!nfs_in_committed_range(vp, off, bp->b_bcount)) { 1021 bool pushedrange; 1022 if (nfs_in_tobecommitted_range(vp, off, bp->b_bcount)) { 1023 pushedrange = true; 1024 off = np->n_pushlo; 1025 cnt = np->n_pushhi - np->n_pushlo; 1026 } else { 1027 pushedrange = false; 1028 } 1029 error = nfs_commit(vp, off, cnt, curlwp); 1030 if (error == 0) { 1031 if (pushedrange) { 1032 nfs_merge_commit_ranges(vp); 1033 } else { 1034 nfs_add_committed_range(vp, off, cnt); 1035 } 1036 } 1037 } else { 1038 error = 0; 1039 } 1040 mutex_exit(&np->n_commitlock); 1041 rw_exit(&nmp->nm_writeverflock); 1042 if (!error) { 1043 /* 1044 * pages are now on stable storage. 1045 */ 1046 uiop->uio_resid = 0; 1047 mutex_enter(&uobj->vmobjlock); 1048 for (i = 0; i < npages; i++) { 1049 pgs[i]->flags &= ~(PG_NEEDCOMMIT | PG_RDONLY); 1050 } 1051 mutex_exit(&uobj->vmobjlock); 1052 goto out; 1053 } else if (error == NFSERR_STALEWRITEVERF) { 1054 nfs_clearcommit(vp->v_mount); 1055 goto again; 1056 } 1057 if (error) { 1058 bp->b_error = np->n_error = error; 1059 np->n_flag |= NWRITEERR; 1060 } 1061 goto out; 1062 } 1063 #endif 1064 off = uiop->uio_offset; 1065 cnt = bp->b_bcount; 1066 uiop->uio_rw = UIO_WRITE; 1067 nfsstats.write_bios++; 1068 error = nfs_writerpc(vp, uiop, &iomode, pageprotected, &stalewriteverf); 1069 #ifndef NFS_V2_ONLY 1070 if (!error && iomode == NFSV3WRITE_UNSTABLE) { 1071 /* 1072 * we need to commit pages later. 1073 */ 1074 mutex_enter(&np->n_commitlock); 1075 nfs_add_tobecommitted_range(vp, off, cnt); 1076 /* 1077 * if there can be too many uncommitted pages, commit them now. 1078 */ 1079 if (np->n_pushhi - np->n_pushlo > nfs_commitsize) { 1080 off = np->n_pushlo; 1081 cnt = nfs_commitsize >> 1; 1082 error = nfs_commit(vp, off, cnt, curlwp); 1083 if (!error) { 1084 nfs_add_committed_range(vp, off, cnt); 1085 nfs_del_tobecommitted_range(vp, off, cnt); 1086 } 1087 if (error == NFSERR_STALEWRITEVERF) { 1088 stalewriteverf = true; 1089 error = 0; /* it isn't a real error */ 1090 } 1091 } else { 1092 /* 1093 * re-dirty pages so that they will be passed 1094 * to us later again. 1095 */ 1096 mutex_enter(&uobj->vmobjlock); 1097 for (i = 0; i < npages; i++) { 1098 pgs[i]->flags &= ~PG_CLEAN; 1099 } 1100 mutex_exit(&uobj->vmobjlock); 1101 } 1102 mutex_exit(&np->n_commitlock); 1103 } else 1104 #endif 1105 if (!error) { 1106 /* 1107 * pages are now on stable storage. 1108 */ 1109 mutex_enter(&np->n_commitlock); 1110 nfs_del_committed_range(vp, off, cnt); 1111 mutex_exit(&np->n_commitlock); 1112 mutex_enter(&uobj->vmobjlock); 1113 for (i = 0; i < npages; i++) { 1114 pgs[i]->flags &= ~(PG_NEEDCOMMIT | PG_RDONLY); 1115 } 1116 mutex_exit(&uobj->vmobjlock); 1117 } else { 1118 /* 1119 * we got an error. 1120 */ 1121 bp->b_error = np->n_error = error; 1122 np->n_flag |= NWRITEERR; 1123 } 1124 1125 rw_exit(&nmp->nm_writeverflock); 1126 1127 1128 if (stalewriteverf) { 1129 nfs_clearcommit(vp->v_mount); 1130 } 1131 #ifndef NFS_V2_ONLY 1132 out: 1133 #endif 1134 if (pgs != spgs) 1135 kmem_free(pgs, sizeof(*pgs) * npages); 1136 return error; 1137 } 1138 1139 /* 1140 * nfs_doio for B_PHYS. 1141 */ 1142 static int 1143 nfs_doio_phys(struct buf *bp, struct uio *uiop) 1144 { 1145 struct vnode *vp = bp->b_vp; 1146 int error; 1147 1148 uiop->uio_offset = ((off_t)bp->b_blkno) << DEV_BSHIFT; 1149 if (bp->b_flags & B_READ) { 1150 uiop->uio_rw = UIO_READ; 1151 nfsstats.read_physios++; 1152 error = nfs_readrpc(vp, uiop); 1153 } else { 1154 int iomode = NFSV3WRITE_DATASYNC; 1155 bool stalewriteverf; 1156 struct nfsmount *nmp = VFSTONFS(vp->v_mount); 1157 1158 uiop->uio_rw = UIO_WRITE; 1159 nfsstats.write_physios++; 1160 rw_enter(&nmp->nm_writeverflock, RW_READER); 1161 error = nfs_writerpc(vp, uiop, &iomode, false, &stalewriteverf); 1162 rw_exit(&nmp->nm_writeverflock); 1163 if (stalewriteverf) { 1164 nfs_clearcommit(bp->b_vp->v_mount); 1165 } 1166 } 1167 bp->b_error = error; 1168 return error; 1169 } 1170 1171 /* 1172 * Do an I/O operation to/from a cache block. This may be called 1173 * synchronously or from an nfsiod. 1174 */ 1175 int 1176 nfs_doio(struct buf *bp) 1177 { 1178 int error; 1179 struct uio uio; 1180 struct uio *uiop = &uio; 1181 struct iovec io; 1182 UVMHIST_FUNC("nfs_doio"); UVMHIST_CALLED(ubchist); 1183 1184 uiop->uio_iov = &io; 1185 uiop->uio_iovcnt = 1; 1186 uiop->uio_offset = (((off_t)bp->b_blkno) << DEV_BSHIFT); 1187 UIO_SETUP_SYSSPACE(uiop); 1188 io.iov_base = bp->b_data; 1189 io.iov_len = uiop->uio_resid = bp->b_bcount; 1190 1191 /* 1192 * Historically, paging was done with physio, but no more... 1193 */ 1194 if (bp->b_flags & B_PHYS) { 1195 /* 1196 * ...though reading /dev/drum still gets us here. 1197 */ 1198 error = nfs_doio_phys(bp, uiop); 1199 } else if (bp->b_flags & B_READ) { 1200 error = nfs_doio_read(bp, uiop); 1201 } else { 1202 error = nfs_doio_write(bp, uiop); 1203 } 1204 bp->b_resid = uiop->uio_resid; 1205 biodone(bp); 1206 return (error); 1207 } 1208 1209 /* 1210 * Vnode op for VM getpages. 1211 */ 1212 1213 int 1214 nfs_getpages(void *v) 1215 { 1216 struct vop_getpages_args /* { 1217 struct vnode *a_vp; 1218 voff_t a_offset; 1219 struct vm_page **a_m; 1220 int *a_count; 1221 int a_centeridx; 1222 vm_prot_t a_access_type; 1223 int a_advice; 1224 int a_flags; 1225 } */ *ap = v; 1226 1227 struct vnode *vp = ap->a_vp; 1228 struct uvm_object *uobj = &vp->v_uobj; 1229 struct nfsnode *np = VTONFS(vp); 1230 const int npages = *ap->a_count; 1231 struct vm_page *pg, **pgs, **opgs, *spgs[UBC_MAX_PAGES]; 1232 off_t origoffset, len; 1233 int i, error; 1234 bool v3 = NFS_ISV3(vp); 1235 bool write = (ap->a_access_type & VM_PROT_WRITE) != 0; 1236 bool locked = (ap->a_flags & PGO_LOCKED) != 0; 1237 1238 /* 1239 * If we are not locked we are not really using opgs, 1240 * so just initialize it 1241 */ 1242 if (!locked || npages < __arraycount(spgs)) 1243 opgs = spgs; 1244 else { 1245 if ((opgs = kmem_alloc(npages * sizeof(*opgs), KM_NOSLEEP)) == 1246 NULL) 1247 return ENOMEM; 1248 } 1249 1250 /* 1251 * call the genfs code to get the pages. `pgs' may be NULL 1252 * when doing read-ahead. 1253 */ 1254 pgs = ap->a_m; 1255 if (write && locked && v3) { 1256 KASSERT(pgs != NULL); 1257 #ifdef DEBUG 1258 1259 /* 1260 * If PGO_LOCKED is set, real pages shouldn't exists 1261 * in the array. 1262 */ 1263 1264 for (i = 0; i < npages; i++) 1265 KDASSERT(pgs[i] == NULL || pgs[i] == PGO_DONTCARE); 1266 #endif 1267 memcpy(opgs, pgs, npages * sizeof(struct vm_pages *)); 1268 } 1269 error = genfs_getpages(v); 1270 if (error) 1271 goto out; 1272 1273 /* 1274 * for read faults where the nfs node is not yet marked NMODIFIED, 1275 * set PG_RDONLY on the pages so that we come back here if someone 1276 * tries to modify later via the mapping that will be entered for 1277 * this fault. 1278 */ 1279 1280 if (!write && (np->n_flag & NMODIFIED) == 0 && pgs != NULL) { 1281 if (!locked) { 1282 mutex_enter(&uobj->vmobjlock); 1283 } 1284 for (i = 0; i < npages; i++) { 1285 pg = pgs[i]; 1286 if (pg == NULL || pg == PGO_DONTCARE) { 1287 continue; 1288 } 1289 pg->flags |= PG_RDONLY; 1290 } 1291 if (!locked) { 1292 mutex_exit(&uobj->vmobjlock); 1293 } 1294 } 1295 if (!write) 1296 goto out; 1297 1298 /* 1299 * this is a write fault, update the commit info. 1300 */ 1301 1302 origoffset = ap->a_offset; 1303 len = npages << PAGE_SHIFT; 1304 1305 if (v3) { 1306 if (!locked) { 1307 mutex_enter(&np->n_commitlock); 1308 } else { 1309 if (!mutex_tryenter(&np->n_commitlock)) { 1310 1311 /* 1312 * Since PGO_LOCKED is set, we need to unbusy 1313 * all pages fetched by genfs_getpages() above, 1314 * tell the caller that there are no pages 1315 * available and put back original pgs array. 1316 */ 1317 1318 mutex_enter(&uvm_pageqlock); 1319 uvm_page_unbusy(pgs, npages); 1320 mutex_exit(&uvm_pageqlock); 1321 *ap->a_count = 0; 1322 memcpy(pgs, opgs, 1323 npages * sizeof(struct vm_pages *)); 1324 error = EBUSY; 1325 goto out; 1326 } 1327 } 1328 nfs_del_committed_range(vp, origoffset, len); 1329 nfs_del_tobecommitted_range(vp, origoffset, len); 1330 } 1331 np->n_flag |= NMODIFIED; 1332 if (!locked) { 1333 mutex_enter(&uobj->vmobjlock); 1334 } 1335 for (i = 0; i < npages; i++) { 1336 pg = pgs[i]; 1337 if (pg == NULL || pg == PGO_DONTCARE) { 1338 continue; 1339 } 1340 pg->flags &= ~(PG_NEEDCOMMIT | PG_RDONLY); 1341 } 1342 if (!locked) { 1343 mutex_exit(&uobj->vmobjlock); 1344 } 1345 if (v3) { 1346 mutex_exit(&np->n_commitlock); 1347 } 1348 out: 1349 if (opgs != spgs) 1350 kmem_free(opgs, sizeof(*opgs) * npages); 1351 return error; 1352 } 1353