1 /* $NetBSD: lfs_pages.c,v 1.7 2015/08/12 18:26:27 dholland Exp $ */
2
3 /*-
4 * Copyright (c) 1999, 2000, 2001, 2002, 2003 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Konrad E. Schroder <perseant@hhhh.org>.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31 /*
32 * Copyright (c) 1986, 1989, 1991, 1993, 1995
33 * The Regents of the University of California. All rights reserved.
34 *
35 * Redistribution and use in source and binary forms, with or without
36 * modification, are permitted provided that the following conditions
37 * are met:
38 * 1. Redistributions of source code must retain the above copyright
39 * notice, this list of conditions and the following disclaimer.
40 * 2. Redistributions in binary form must reproduce the above copyright
41 * notice, this list of conditions and the following disclaimer in the
42 * documentation and/or other materials provided with the distribution.
43 * 3. Neither the name of the University nor the names of its contributors
44 * may be used to endorse or promote products derived from this software
45 * without specific prior written permission.
46 *
47 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
48 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
49 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
50 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
51 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
52 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
53 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
54 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
55 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
56 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
57 * SUCH DAMAGE.
58 *
59 * @(#)lfs_vnops.c 8.13 (Berkeley) 6/10/95
60 */
61
62 #include <sys/cdefs.h>
63 __KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.7 2015/08/12 18:26:27 dholland Exp $");
64
65 #ifdef _KERNEL_OPT
66 #include "opt_compat_netbsd.h"
67 #include "opt_uvm_page_trkown.h"
68 #endif
69
70 #include <sys/param.h>
71 #include <sys/systm.h>
72 #include <sys/namei.h>
73 #include <sys/resourcevar.h>
74 #include <sys/kernel.h>
75 #include <sys/file.h>
76 #include <sys/stat.h>
77 #include <sys/buf.h>
78 #include <sys/proc.h>
79 #include <sys/mount.h>
80 #include <sys/vnode.h>
81 #include <sys/pool.h>
82 #include <sys/signalvar.h>
83 #include <sys/kauth.h>
84 #include <sys/syslog.h>
85 #include <sys/fstrans.h>
86
87 #include <miscfs/fifofs/fifo.h>
88 #include <miscfs/genfs/genfs.h>
89 #include <miscfs/specfs/specdev.h>
90
91 #include <ufs/lfs/ulfs_inode.h>
92 #include <ufs/lfs/ulfsmount.h>
93 #include <ufs/lfs/ulfs_bswap.h>
94 #include <ufs/lfs/ulfs_extern.h>
95
96 #include <uvm/uvm.h>
97 #include <uvm/uvm_pmap.h>
98 #include <uvm/uvm_stat.h>
99 #include <uvm/uvm_pager.h>
100
101 #include <ufs/lfs/lfs.h>
102 #include <ufs/lfs/lfs_accessors.h>
103 #include <ufs/lfs/lfs_kernel.h>
104 #include <ufs/lfs/lfs_extern.h>
105
106 extern pid_t lfs_writer_daemon;
107
108 static int check_dirty(struct lfs *, struct vnode *, off_t, off_t, off_t, int, int, struct vm_page **);
109
110 int
lfs_getpages(void * v)111 lfs_getpages(void *v)
112 {
113 struct vop_getpages_args /* {
114 struct vnode *a_vp;
115 voff_t a_offset;
116 struct vm_page **a_m;
117 int *a_count;
118 int a_centeridx;
119 vm_prot_t a_access_type;
120 int a_advice;
121 int a_flags;
122 } */ *ap = v;
123
124 if (VTOI(ap->a_vp)->i_number == LFS_IFILE_INUM &&
125 (ap->a_access_type & VM_PROT_WRITE) != 0) {
126 return EPERM;
127 }
128 if ((ap->a_access_type & VM_PROT_WRITE) != 0) {
129 mutex_enter(&lfs_lock);
130 LFS_SET_UINO(VTOI(ap->a_vp), IN_MODIFIED);
131 mutex_exit(&lfs_lock);
132 }
133
134 /*
135 * we're relying on the fact that genfs_getpages() always read in
136 * entire filesystem blocks.
137 */
138 return genfs_getpages(v);
139 }
140
141 /*
142 * Wait for a page to become unbusy, possibly printing diagnostic messages
143 * as well.
144 *
145 * Called with vp->v_interlock held; return with it held.
146 */
147 static void
wait_for_page(struct vnode * vp,struct vm_page * pg,const char * label)148 wait_for_page(struct vnode *vp, struct vm_page *pg, const char *label)
149 {
150 KASSERT(mutex_owned(vp->v_interlock));
151 if ((pg->flags & PG_BUSY) == 0)
152 return; /* Nothing to wait for! */
153
154 #if defined(DEBUG) && defined(UVM_PAGE_TRKOWN)
155 static struct vm_page *lastpg;
156
157 if (label != NULL && pg != lastpg) {
158 if (pg->owner_tag) {
159 printf("lfs_putpages[%d.%d]: %s: page %p owner %d.%d [%s]\n",
160 curproc->p_pid, curlwp->l_lid, label,
161 pg, pg->owner, pg->lowner, pg->owner_tag);
162 } else {
163 printf("lfs_putpages[%d.%d]: %s: page %p unowned?!\n",
164 curproc->p_pid, curlwp->l_lid, label, pg);
165 }
166 }
167 lastpg = pg;
168 #endif
169
170 pg->flags |= PG_WANTED;
171 UVM_UNLOCK_AND_WAIT(pg, vp->v_interlock, 0, "lfsput", 0);
172 mutex_enter(vp->v_interlock);
173 }
174
175 /*
176 * This routine is called by lfs_putpages() when it can't complete the
177 * write because a page is busy. This means that either (1) someone,
178 * possibly the pagedaemon, is looking at this page, and will give it up
179 * presently; or (2) we ourselves are holding the page busy in the
180 * process of being written (either gathered or actually on its way to
181 * disk). We don't need to give up the segment lock, but we might need
182 * to call lfs_writeseg() to expedite the page's journey to disk.
183 *
184 * Called with vp->v_interlock held; return with it held.
185 */
186 /* #define BUSYWAIT */
187 static void
write_and_wait(struct lfs * fs,struct vnode * vp,struct vm_page * pg,int seglocked,const char * label)188 write_and_wait(struct lfs *fs, struct vnode *vp, struct vm_page *pg,
189 int seglocked, const char *label)
190 {
191 KASSERT(mutex_owned(vp->v_interlock));
192 #ifndef BUSYWAIT
193 struct inode *ip = VTOI(vp);
194 struct segment *sp = fs->lfs_sp;
195 int count = 0;
196
197 if (pg == NULL)
198 return;
199
200 while (pg->flags & PG_BUSY &&
201 pg->uobject == &vp->v_uobj) {
202 mutex_exit(vp->v_interlock);
203 if (sp->cbpp - sp->bpp > 1) {
204 /* Write gathered pages */
205 lfs_updatemeta(sp);
206 lfs_release_finfo(fs);
207 (void) lfs_writeseg(fs, sp);
208
209 /*
210 * Reinitialize FIP
211 */
212 KASSERT(sp->vp == vp);
213 lfs_acquire_finfo(fs, ip->i_number,
214 ip->i_gen);
215 }
216 ++count;
217 mutex_enter(vp->v_interlock);
218 wait_for_page(vp, pg, label);
219 }
220 if (label != NULL && count > 1) {
221 DLOG((DLOG_PAGE, "lfs_putpages[%d]: %s: %sn = %d\n",
222 curproc->p_pid, label, (count > 0 ? "looping, " : ""),
223 count));
224 }
225 #else
226 preempt(1);
227 #endif
228 KASSERT(mutex_owned(vp->v_interlock));
229 }
230
231 /*
232 * Make sure that for all pages in every block in the given range,
233 * either all are dirty or all are clean. If any of the pages
234 * we've seen so far are dirty, put the vnode on the paging chain,
235 * and mark it IN_PAGING.
236 *
237 * If checkfirst != 0, don't check all the pages but return at the
238 * first dirty page.
239 */
240 static int
check_dirty(struct lfs * fs,struct vnode * vp,off_t startoffset,off_t endoffset,off_t blkeof,int flags,int checkfirst,struct vm_page ** pgp)241 check_dirty(struct lfs *fs, struct vnode *vp,
242 off_t startoffset, off_t endoffset, off_t blkeof,
243 int flags, int checkfirst, struct vm_page **pgp)
244 {
245 int by_list;
246 struct vm_page *curpg = NULL; /* XXX: gcc */
247 struct vm_page *pgs[MAXBSIZE / PAGE_SIZE], *pg;
248 off_t soff = 0; /* XXX: gcc */
249 voff_t off;
250 int i;
251 int nonexistent;
252 int any_dirty; /* number of dirty pages */
253 int dirty; /* number of dirty pages in a block */
254 int tdirty;
255 int pages_per_block = lfs_sb_getbsize(fs) >> PAGE_SHIFT;
256 int pagedaemon = (curlwp == uvm.pagedaemon_lwp);
257
258 KASSERT(mutex_owned(vp->v_interlock));
259 ASSERT_MAYBE_SEGLOCK(fs);
260 top:
261 by_list = (vp->v_uobj.uo_npages <=
262 ((endoffset - startoffset) >> PAGE_SHIFT) *
263 UVM_PAGE_TREE_PENALTY);
264 any_dirty = 0;
265
266 if (by_list) {
267 curpg = TAILQ_FIRST(&vp->v_uobj.memq);
268 } else {
269 soff = startoffset;
270 }
271 while (by_list || soff < MIN(blkeof, endoffset)) {
272 if (by_list) {
273 /*
274 * Find the first page in a block. Skip
275 * blocks outside our area of interest or beyond
276 * the end of file.
277 */
278 KASSERT(curpg == NULL
279 || (curpg->flags & PG_MARKER) == 0);
280 if (pages_per_block > 1) {
281 while (curpg &&
282 ((curpg->offset & lfs_sb_getbmask(fs)) ||
283 curpg->offset >= vp->v_size ||
284 curpg->offset >= endoffset)) {
285 curpg = TAILQ_NEXT(curpg, listq.queue);
286 KASSERT(curpg == NULL ||
287 (curpg->flags & PG_MARKER) == 0);
288 }
289 }
290 if (curpg == NULL)
291 break;
292 soff = curpg->offset;
293 }
294
295 /*
296 * Mark all pages in extended range busy; find out if any
297 * of them are dirty.
298 */
299 nonexistent = dirty = 0;
300 for (i = 0; i == 0 || i < pages_per_block; i++) {
301 KASSERT(mutex_owned(vp->v_interlock));
302 if (by_list && pages_per_block <= 1) {
303 pgs[i] = pg = curpg;
304 } else {
305 off = soff + (i << PAGE_SHIFT);
306 pgs[i] = pg = uvm_pagelookup(&vp->v_uobj, off);
307 if (pg == NULL) {
308 ++nonexistent;
309 continue;
310 }
311 }
312 KASSERT(pg != NULL);
313
314 /*
315 * If we're holding the segment lock, we can deadlock
316 * against a process that has our page and is waiting
317 * for the cleaner, while the cleaner waits for the
318 * segment lock. Just bail in that case.
319 */
320 if ((pg->flags & PG_BUSY) &&
321 (pagedaemon || LFS_SEGLOCK_HELD(fs))) {
322 if (i > 0)
323 uvm_page_unbusy(pgs, i);
324 DLOG((DLOG_PAGE, "lfs_putpages: avoiding 3-way or pagedaemon deadlock\n"));
325 if (pgp)
326 *pgp = pg;
327 KASSERT(mutex_owned(vp->v_interlock));
328 return -1;
329 }
330
331 while (pg->flags & PG_BUSY) {
332 wait_for_page(vp, pg, NULL);
333 KASSERT(mutex_owned(vp->v_interlock));
334 if (i > 0)
335 uvm_page_unbusy(pgs, i);
336 KASSERT(mutex_owned(vp->v_interlock));
337 goto top;
338 }
339 pg->flags |= PG_BUSY;
340 UVM_PAGE_OWN(pg, "lfs_putpages");
341
342 pmap_page_protect(pg, VM_PROT_NONE);
343 tdirty = (pmap_clear_modify(pg) ||
344 (pg->flags & PG_CLEAN) == 0);
345 dirty += tdirty;
346 }
347 if (pages_per_block > 0 && nonexistent >= pages_per_block) {
348 if (by_list) {
349 curpg = TAILQ_NEXT(curpg, listq.queue);
350 } else {
351 soff += lfs_sb_getbsize(fs);
352 }
353 continue;
354 }
355
356 any_dirty += dirty;
357 KASSERT(nonexistent == 0);
358 KASSERT(mutex_owned(vp->v_interlock));
359
360 /*
361 * If any are dirty make all dirty; unbusy them,
362 * but if we were asked to clean, wire them so that
363 * the pagedaemon doesn't bother us about them while
364 * they're on their way to disk.
365 */
366 for (i = 0; i == 0 || i < pages_per_block; i++) {
367 KASSERT(mutex_owned(vp->v_interlock));
368 pg = pgs[i];
369 KASSERT(!((pg->flags & PG_CLEAN) && (pg->flags & PG_DELWRI)));
370 KASSERT(pg->flags & PG_BUSY);
371 if (dirty) {
372 pg->flags &= ~PG_CLEAN;
373 if (flags & PGO_FREE) {
374 /*
375 * Wire the page so that
376 * pdaemon doesn't see it again.
377 */
378 mutex_enter(&uvm_pageqlock);
379 uvm_pagewire(pg);
380 mutex_exit(&uvm_pageqlock);
381
382 /* Suspended write flag */
383 pg->flags |= PG_DELWRI;
384 }
385 }
386 if (pg->flags & PG_WANTED)
387 wakeup(pg);
388 pg->flags &= ~(PG_WANTED|PG_BUSY);
389 UVM_PAGE_OWN(pg, NULL);
390 }
391
392 if (checkfirst && any_dirty)
393 break;
394
395 if (by_list) {
396 curpg = TAILQ_NEXT(curpg, listq.queue);
397 } else {
398 soff += MAX(PAGE_SIZE, lfs_sb_getbsize(fs));
399 }
400 }
401
402 KASSERT(mutex_owned(vp->v_interlock));
403 return any_dirty;
404 }
405
406 /*
407 * lfs_putpages functions like genfs_putpages except that
408 *
409 * (1) It needs to bounds-check the incoming requests to ensure that
410 * they are block-aligned; if they are not, expand the range and
411 * do the right thing in case, e.g., the requested range is clean
412 * but the expanded range is dirty.
413 *
414 * (2) It needs to explicitly send blocks to be written when it is done.
415 * If VOP_PUTPAGES is called without the seglock held, we simply take
416 * the seglock and let lfs_segunlock wait for us.
417 * XXX There might be a bad situation if we have to flush a vnode while
418 * XXX lfs_markv is in operation. As of this writing we panic in this
419 * XXX case.
420 *
421 * Assumptions:
422 *
423 * (1) The caller does not hold any pages in this vnode busy. If it does,
424 * there is a danger that when we expand the page range and busy the
425 * pages we will deadlock.
426 *
427 * (2) We are called with vp->v_interlock held; we must return with it
428 * released.
429 *
430 * (3) We don't absolutely have to free pages right away, provided that
431 * the request does not have PGO_SYNCIO. When the pagedaemon gives
432 * us a request with PGO_FREE, we take the pages out of the paging
433 * queue and wake up the writer, which will handle freeing them for us.
434 *
435 * We ensure that for any filesystem block, all pages for that
436 * block are either resident or not, even if those pages are higher
437 * than EOF; that means that we will be getting requests to free
438 * "unused" pages above EOF all the time, and should ignore them.
439 *
440 * (4) If we are called with PGO_LOCKED, the finfo array we are to write
441 * into has been set up for us by lfs_writefile. If not, we will
442 * have to handle allocating and/or freeing an finfo entry.
443 *
444 * XXX note that we're (ab)using PGO_LOCKED as "seglock held".
445 */
446
447 /* How many times to loop before we should start to worry */
448 #define TOOMANY 4
449
450 int
lfs_putpages(void * v)451 lfs_putpages(void *v)
452 {
453 int error;
454 struct vop_putpages_args /* {
455 struct vnode *a_vp;
456 voff_t a_offlo;
457 voff_t a_offhi;
458 int a_flags;
459 } */ *ap = v;
460 struct vnode *vp;
461 struct inode *ip;
462 struct lfs *fs;
463 struct segment *sp;
464 off_t origoffset, startoffset, endoffset, origendoffset, blkeof;
465 off_t off, max_endoffset;
466 bool seglocked, sync, pagedaemon, reclaim;
467 struct vm_page *pg, *busypg;
468 UVMHIST_FUNC("lfs_putpages"); UVMHIST_CALLED(ubchist);
469 int oreclaim = 0;
470 int donewriting = 0;
471 #ifdef DEBUG
472 int debug_n_again, debug_n_dirtyclean;
473 #endif
474
475 vp = ap->a_vp;
476 ip = VTOI(vp);
477 fs = ip->i_lfs;
478 sync = (ap->a_flags & PGO_SYNCIO) != 0;
479 reclaim = (ap->a_flags & PGO_RECLAIM) != 0;
480 pagedaemon = (curlwp == uvm.pagedaemon_lwp);
481
482 KASSERT(mutex_owned(vp->v_interlock));
483
484 /* Putpages does nothing for metadata. */
485 if (vp == fs->lfs_ivnode || vp->v_type != VREG) {
486 mutex_exit(vp->v_interlock);
487 return 0;
488 }
489
490 /*
491 * If there are no pages, don't do anything.
492 */
493 if (vp->v_uobj.uo_npages == 0) {
494 if (TAILQ_EMPTY(&vp->v_uobj.memq) &&
495 (vp->v_iflag & VI_ONWORKLST) &&
496 LIST_FIRST(&vp->v_dirtyblkhd) == NULL) {
497 vp->v_iflag &= ~VI_WRMAPDIRTY;
498 vn_syncer_remove_from_worklist(vp);
499 }
500 mutex_exit(vp->v_interlock);
501
502 /* Remove us from paging queue, if we were on it */
503 mutex_enter(&lfs_lock);
504 if (ip->i_flags & IN_PAGING) {
505 ip->i_flags &= ~IN_PAGING;
506 TAILQ_REMOVE(&fs->lfs_pchainhd, ip, i_lfs_pchain);
507 }
508 mutex_exit(&lfs_lock);
509
510 KASSERT(!mutex_owned(vp->v_interlock));
511 return 0;
512 }
513
514 blkeof = lfs_blkroundup(fs, ip->i_size);
515
516 /*
517 * Ignore requests to free pages past EOF but in the same block
518 * as EOF, unless the vnode is being reclaimed or the request
519 * is synchronous. (If the request is sync, it comes from
520 * lfs_truncate.)
521 *
522 * To avoid being flooded with this request, make these pages
523 * look "active".
524 */
525 if (!sync && !reclaim &&
526 ap->a_offlo >= ip->i_size && ap->a_offlo < blkeof) {
527 origoffset = ap->a_offlo;
528 for (off = origoffset; off < blkeof; off += lfs_sb_getbsize(fs)) {
529 pg = uvm_pagelookup(&vp->v_uobj, off);
530 KASSERT(pg != NULL);
531 while (pg->flags & PG_BUSY) {
532 pg->flags |= PG_WANTED;
533 UVM_UNLOCK_AND_WAIT(pg, vp->v_interlock, 0,
534 "lfsput2", 0);
535 mutex_enter(vp->v_interlock);
536 }
537 mutex_enter(&uvm_pageqlock);
538 uvm_pageactivate(pg);
539 mutex_exit(&uvm_pageqlock);
540 }
541 ap->a_offlo = blkeof;
542 if (ap->a_offhi > 0 && ap->a_offhi <= ap->a_offlo) {
543 mutex_exit(vp->v_interlock);
544 return 0;
545 }
546 }
547
548 /*
549 * Extend page range to start and end at block boundaries.
550 * (For the purposes of VOP_PUTPAGES, fragments don't exist.)
551 */
552 origoffset = ap->a_offlo;
553 origendoffset = ap->a_offhi;
554 startoffset = origoffset & ~(lfs_sb_getbmask(fs));
555 max_endoffset = (trunc_page(LLONG_MAX) >> lfs_sb_getbshift(fs))
556 << lfs_sb_getbshift(fs);
557
558 if (origendoffset == 0 || ap->a_flags & PGO_ALLPAGES) {
559 endoffset = max_endoffset;
560 origendoffset = endoffset;
561 } else {
562 origendoffset = round_page(ap->a_offhi);
563 endoffset = round_page(lfs_blkroundup(fs, origendoffset));
564 }
565
566 KASSERT(startoffset > 0 || endoffset >= startoffset);
567 if (startoffset == endoffset) {
568 /* Nothing to do, why were we called? */
569 mutex_exit(vp->v_interlock);
570 DLOG((DLOG_PAGE, "lfs_putpages: startoffset = endoffset = %"
571 PRId64 "\n", startoffset));
572 return 0;
573 }
574
575 ap->a_offlo = startoffset;
576 ap->a_offhi = endoffset;
577
578 /*
579 * If not cleaning, just send the pages through genfs_putpages
580 * to be returned to the pool.
581 */
582 if (!(ap->a_flags & PGO_CLEANIT)) {
583 DLOG((DLOG_PAGE, "lfs_putpages: no cleanit vn %p ino %d (flags %x)\n",
584 vp, (int)ip->i_number, ap->a_flags));
585 int r = genfs_putpages(v);
586 KASSERT(!mutex_owned(vp->v_interlock));
587 return r;
588 }
589
590 /* Set PGO_BUSYFAIL to avoid deadlocks */
591 ap->a_flags |= PGO_BUSYFAIL;
592
593 /*
594 * Likewise, if we are asked to clean but the pages are not
595 * dirty, we can just free them using genfs_putpages.
596 */
597 #ifdef DEBUG
598 debug_n_dirtyclean = 0;
599 #endif
600 do {
601 int r;
602 KASSERT(mutex_owned(vp->v_interlock));
603
604 /* Count the number of dirty pages */
605 r = check_dirty(fs, vp, startoffset, endoffset, blkeof,
606 ap->a_flags, 1, NULL);
607 if (r < 0) {
608 /* Pages are busy with another process */
609 mutex_exit(vp->v_interlock);
610 return EDEADLK;
611 }
612 if (r > 0) /* Some pages are dirty */
613 break;
614
615 /*
616 * Sometimes pages are dirtied between the time that
617 * we check and the time we try to clean them.
618 * Instruct lfs_gop_write to return EDEADLK in this case
619 * so we can write them properly.
620 */
621 ip->i_lfs_iflags |= LFSI_NO_GOP_WRITE;
622 r = genfs_do_putpages(vp, startoffset, endoffset,
623 ap->a_flags & ~PGO_SYNCIO, &busypg);
624 ip->i_lfs_iflags &= ~LFSI_NO_GOP_WRITE;
625 if (r != EDEADLK) {
626 KASSERT(!mutex_owned(vp->v_interlock));
627 return r;
628 }
629
630 /* One of the pages was busy. Start over. */
631 mutex_enter(vp->v_interlock);
632 wait_for_page(vp, busypg, "dirtyclean");
633 #ifdef DEBUG
634 ++debug_n_dirtyclean;
635 #endif
636 } while(1);
637
638 #ifdef DEBUG
639 if (debug_n_dirtyclean > TOOMANY)
640 DLOG((DLOG_PAGE, "lfs_putpages: dirtyclean: looping, n = %d\n",
641 debug_n_dirtyclean));
642 #endif
643
644 /*
645 * Dirty and asked to clean.
646 *
647 * Pagedaemon can't actually write LFS pages; wake up
648 * the writer to take care of that. The writer will
649 * notice the pager inode queue and act on that.
650 *
651 * XXX We must drop the vp->interlock before taking the lfs_lock or we
652 * get a nasty deadlock with lfs_flush_pchain().
653 */
654 if (pagedaemon) {
655 mutex_exit(vp->v_interlock);
656 mutex_enter(&lfs_lock);
657 if (!(ip->i_flags & IN_PAGING)) {
658 ip->i_flags |= IN_PAGING;
659 TAILQ_INSERT_TAIL(&fs->lfs_pchainhd, ip, i_lfs_pchain);
660 }
661 wakeup(&lfs_writer_daemon);
662 mutex_exit(&lfs_lock);
663 preempt();
664 KASSERT(!mutex_owned(vp->v_interlock));
665 return EWOULDBLOCK;
666 }
667
668 /*
669 * If this is a file created in a recent dirop, we can't flush its
670 * inode until the dirop is complete. Drain dirops, then flush the
671 * filesystem (taking care of any other pending dirops while we're
672 * at it).
673 */
674 if ((ap->a_flags & (PGO_CLEANIT|PGO_LOCKED)) == PGO_CLEANIT &&
675 (vp->v_uflag & VU_DIROP)) {
676 DLOG((DLOG_PAGE, "lfs_putpages: flushing VU_DIROP\n"));
677
678 lfs_writer_enter(fs, "ppdirop");
679
680 /* Note if we hold the vnode locked */
681 if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE)
682 {
683 DLOG((DLOG_PAGE, "lfs_putpages: dirop inode already locked\n"));
684 } else {
685 DLOG((DLOG_PAGE, "lfs_putpages: dirop inode not locked\n"));
686 }
687 mutex_exit(vp->v_interlock);
688
689 mutex_enter(&lfs_lock);
690 lfs_flush_fs(fs, sync ? SEGM_SYNC : 0);
691 mutex_exit(&lfs_lock);
692
693 mutex_enter(vp->v_interlock);
694 lfs_writer_leave(fs);
695
696 /* The flush will have cleaned out this vnode as well,
697 no need to do more to it. */
698 }
699
700 /*
701 * This is it. We are going to write some pages. From here on
702 * down it's all just mechanics.
703 *
704 * Don't let genfs_putpages wait; lfs_segunlock will wait for us.
705 */
706 ap->a_flags &= ~PGO_SYNCIO;
707
708 /*
709 * If we've already got the seglock, flush the node and return.
710 * The FIP has already been set up for us by lfs_writefile,
711 * and FIP cleanup and lfs_updatemeta will also be done there,
712 * unless genfs_putpages returns EDEADLK; then we must flush
713 * what we have, and correct FIP and segment header accounting.
714 */
715 get_seglock:
716 /*
717 * If we are not called with the segment locked, lock it.
718 * Account for a new FIP in the segment header, and set sp->vp.
719 * (This should duplicate the setup at the top of lfs_writefile().)
720 */
721 seglocked = (ap->a_flags & PGO_LOCKED) != 0;
722 if (!seglocked) {
723 mutex_exit(vp->v_interlock);
724 error = lfs_seglock(fs, SEGM_PROT | (sync ? SEGM_SYNC : 0));
725 if (error != 0) {
726 KASSERT(!mutex_owned(vp->v_interlock));
727 return error;
728 }
729 mutex_enter(vp->v_interlock);
730 lfs_acquire_finfo(fs, ip->i_number, ip->i_gen);
731 }
732 sp = fs->lfs_sp;
733 KASSERT(sp->vp == NULL);
734 sp->vp = vp;
735
736 /* Note segments written by reclaim; only for debugging */
737 if (vdead_check(vp, VDEAD_NOWAIT) != 0) {
738 sp->seg_flags |= SEGM_RECLAIM;
739 fs->lfs_reclino = ip->i_number;
740 }
741
742 /*
743 * Ensure that the partial segment is marked SS_DIROP if this
744 * vnode is a DIROP.
745 */
746 if (!seglocked && vp->v_uflag & VU_DIROP) {
747 SEGSUM *ssp = sp->segsum;
748
749 lfs_ss_setflags(fs, ssp,
750 lfs_ss_getflags(fs, ssp) | (SS_DIROP|SS_CONT));
751 }
752
753 /*
754 * Loop over genfs_putpages until all pages are gathered.
755 * genfs_putpages() drops the interlock, so reacquire it if necessary.
756 * Whenever we lose the interlock we have to rerun check_dirty, as
757 * well, since more pages might have been dirtied in our absence.
758 */
759 #ifdef DEBUG
760 debug_n_again = 0;
761 #endif
762 do {
763 busypg = NULL;
764 KASSERT(mutex_owned(vp->v_interlock));
765 if (check_dirty(fs, vp, startoffset, endoffset, blkeof,
766 ap->a_flags, 0, &busypg) < 0) {
767 mutex_exit(vp->v_interlock);
768 /* XXX why? --ks */
769 mutex_enter(vp->v_interlock);
770 write_and_wait(fs, vp, busypg, seglocked, NULL);
771 if (!seglocked) {
772 mutex_exit(vp->v_interlock);
773 lfs_release_finfo(fs);
774 lfs_segunlock(fs);
775 mutex_enter(vp->v_interlock);
776 }
777 sp->vp = NULL;
778 goto get_seglock;
779 }
780
781 busypg = NULL;
782 KASSERT(!mutex_owned(&uvm_pageqlock));
783 oreclaim = (ap->a_flags & PGO_RECLAIM);
784 ap->a_flags &= ~PGO_RECLAIM;
785 error = genfs_do_putpages(vp, startoffset, endoffset,
786 ap->a_flags, &busypg);
787 ap->a_flags |= oreclaim;
788
789 if (error == EDEADLK || error == EAGAIN) {
790 DLOG((DLOG_PAGE, "lfs_putpages: genfs_putpages returned"
791 " %d ino %d off %jx (seg %d)\n", error,
792 ip->i_number, (uintmax_t)lfs_sb_getoffset(fs),
793 lfs_dtosn(fs, lfs_sb_getoffset(fs))));
794
795 if (oreclaim) {
796 mutex_enter(vp->v_interlock);
797 write_and_wait(fs, vp, busypg, seglocked, "again");
798 mutex_exit(vp->v_interlock);
799 } else {
800 if ((sp->seg_flags & SEGM_SINGLE) &&
801 lfs_sb_getcurseg(fs) != fs->lfs_startseg)
802 donewriting = 1;
803 }
804 } else if (error) {
805 DLOG((DLOG_PAGE, "lfs_putpages: genfs_putpages returned"
806 " %d ino %d off %jx (seg %d)\n", error,
807 (int)ip->i_number, (uintmax_t)lfs_sb_getoffset(fs),
808 lfs_dtosn(fs, lfs_sb_getoffset(fs))));
809 }
810 /* genfs_do_putpages loses the interlock */
811 #ifdef DEBUG
812 ++debug_n_again;
813 #endif
814 if (oreclaim && error == EAGAIN) {
815 DLOG((DLOG_PAGE, "vp %p ino %d vi_flags %x a_flags %x avoiding vclean panic\n",
816 vp, (int)ip->i_number, vp->v_iflag, ap->a_flags));
817 mutex_enter(vp->v_interlock);
818 }
819 if (error == EDEADLK)
820 mutex_enter(vp->v_interlock);
821 } while (error == EDEADLK || (oreclaim && error == EAGAIN));
822 #ifdef DEBUG
823 if (debug_n_again > TOOMANY)
824 DLOG((DLOG_PAGE, "lfs_putpages: again: looping, n = %d\n", debug_n_again));
825 #endif
826
827 KASSERT(sp != NULL && sp->vp == vp);
828 if (!seglocked && !donewriting) {
829 sp->vp = NULL;
830
831 /* Write indirect blocks as well */
832 lfs_gather(fs, fs->lfs_sp, vp, lfs_match_indir);
833 lfs_gather(fs, fs->lfs_sp, vp, lfs_match_dindir);
834 lfs_gather(fs, fs->lfs_sp, vp, lfs_match_tindir);
835
836 KASSERT(sp->vp == NULL);
837 sp->vp = vp;
838 }
839
840 /*
841 * Blocks are now gathered into a segment waiting to be written.
842 * All that's left to do is update metadata, and write them.
843 */
844 lfs_updatemeta(sp);
845 KASSERT(sp->vp == vp);
846 sp->vp = NULL;
847
848 /*
849 * If we were called from lfs_writefile, we don't need to clean up
850 * the FIP or unlock the segment lock. We're done.
851 */
852 if (seglocked) {
853 KASSERT(!mutex_owned(vp->v_interlock));
854 return error;
855 }
856
857 /* Clean up FIP and send it to disk. */
858 lfs_release_finfo(fs);
859 lfs_writeseg(fs, fs->lfs_sp);
860
861 /*
862 * Remove us from paging queue if we wrote all our pages.
863 */
864 if (origendoffset == 0 || ap->a_flags & PGO_ALLPAGES) {
865 mutex_enter(&lfs_lock);
866 if (ip->i_flags & IN_PAGING) {
867 ip->i_flags &= ~IN_PAGING;
868 TAILQ_REMOVE(&fs->lfs_pchainhd, ip, i_lfs_pchain);
869 }
870 mutex_exit(&lfs_lock);
871 }
872
873 /*
874 * XXX - with the malloc/copy writeseg, the pages are freed by now
875 * even if we don't wait (e.g. if we hold a nested lock). This
876 * will not be true if we stop using malloc/copy.
877 */
878 KASSERT(fs->lfs_sp->seg_flags & SEGM_PROT);
879 lfs_segunlock(fs);
880
881 /*
882 * Wait for v_numoutput to drop to zero. The seglock should
883 * take care of this, but there is a slight possibility that
884 * aiodoned might not have got around to our buffers yet.
885 */
886 if (sync) {
887 mutex_enter(vp->v_interlock);
888 while (vp->v_numoutput > 0) {
889 DLOG((DLOG_PAGE, "lfs_putpages: ino %d sleeping on"
890 " num %d\n", ip->i_number, vp->v_numoutput));
891 cv_wait(&vp->v_cv, vp->v_interlock);
892 }
893 mutex_exit(vp->v_interlock);
894 }
895 KASSERT(!mutex_owned(vp->v_interlock));
896 return error;
897 }
898
899