xref: /netbsd-src/sys/kern/vfs_trans.c (revision 37afb7eb6895c833050f8bfb1d1bb2f99f332539)
1 /*	$NetBSD: vfs_trans.c,v 1.33 2015/05/06 15:57:08 hannken Exp $	*/
2 
3 /*-
4  * Copyright (c) 2007 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Juergen Hannken-Illjes.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 #include <sys/cdefs.h>
33 __KERNEL_RCSID(0, "$NetBSD: vfs_trans.c,v 1.33 2015/05/06 15:57:08 hannken Exp $");
34 
35 /*
36  * File system transaction operations.
37  */
38 
39 #include "opt_ddb.h"
40 
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/atomic.h>
44 #include <sys/buf.h>
45 #include <sys/kmem.h>
46 #include <sys/mount.h>
47 #include <sys/pserialize.h>
48 #include <sys/vnode.h>
49 #define _FSTRANS_API_PRIVATE
50 #include <sys/fstrans.h>
51 #include <sys/proc.h>
52 
53 #include <miscfs/specfs/specdev.h>
54 
55 struct fscow_handler {
56 	LIST_ENTRY(fscow_handler) ch_list;
57 	int (*ch_func)(void *, struct buf *, bool);
58 	void *ch_arg;
59 };
60 struct fstrans_lwp_info {
61 	struct fstrans_lwp_info *fli_succ;
62 	struct lwp *fli_self;
63 	struct mount *fli_mount;
64 	int fli_trans_cnt;
65 	int fli_cow_cnt;
66 	enum fstrans_lock_type fli_lock_type;
67 	LIST_ENTRY(fstrans_lwp_info) fli_list;
68 };
69 struct fstrans_mount_info {
70 	enum fstrans_state fmi_state;
71 	unsigned int fmi_ref_cnt;
72 	bool fmi_cow_change;
73 	LIST_HEAD(, fscow_handler) fmi_cow_handler;
74 };
75 
76 static specificdata_key_t lwp_data_key;	/* Our specific data key. */
77 static kmutex_t vfs_suspend_lock;	/* Serialize suspensions. */
78 static kmutex_t fstrans_lock;		/* Fstrans big lock. */
79 static kcondvar_t fstrans_state_cv;	/* Fstrans or cow state changed. */
80 static kcondvar_t fstrans_count_cv;	/* Fstrans or cow count changed. */
81 static pserialize_t fstrans_psz;	/* Pserialize state. */
82 static LIST_HEAD(fstrans_lwp_head, fstrans_lwp_info) fstrans_fli_head;
83 					/* List of all fstrans_lwp_info. */
84 
85 static void fstrans_lwp_dtor(void *);
86 static void fstrans_mount_dtor(struct mount *);
87 static struct fstrans_lwp_info *fstrans_get_lwp_info(struct mount *, bool);
88 static bool grant_lock(const enum fstrans_state, const enum fstrans_lock_type);
89 static bool state_change_done(const struct mount *);
90 static bool cow_state_change_done(const struct mount *);
91 static void cow_change_enter(const struct mount *);
92 static void cow_change_done(const struct mount *);
93 
94 /*
95  * Initialize.
96  */
97 void
98 fstrans_init(void)
99 {
100 	int error __diagused;
101 
102 	error = lwp_specific_key_create(&lwp_data_key, fstrans_lwp_dtor);
103 	KASSERT(error == 0);
104 
105 	mutex_init(&vfs_suspend_lock, MUTEX_DEFAULT, IPL_NONE);
106 	mutex_init(&fstrans_lock, MUTEX_DEFAULT, IPL_NONE);
107 	cv_init(&fstrans_state_cv, "fstchg");
108 	cv_init(&fstrans_count_cv, "fstcnt");
109 	fstrans_psz = pserialize_create();
110 	LIST_INIT(&fstrans_fli_head);
111 }
112 
113 /*
114  * Deallocate lwp state.
115  */
116 static void
117 fstrans_lwp_dtor(void *arg)
118 {
119 	struct fstrans_lwp_info *fli, *fli_next;
120 
121 	for (fli = arg; fli; fli = fli_next) {
122 		KASSERT(fli->fli_trans_cnt == 0);
123 		KASSERT(fli->fli_cow_cnt == 0);
124 		if (fli->fli_mount != NULL)
125 			fstrans_mount_dtor(fli->fli_mount);
126 		fli_next = fli->fli_succ;
127 		fli->fli_mount = NULL;
128 		membar_sync();
129 		fli->fli_self = NULL;
130 	}
131 }
132 
133 /*
134  * Dereference mount state.
135  */
136 static void
137 fstrans_mount_dtor(struct mount *mp)
138 {
139 	struct fstrans_mount_info *fmi;
140 
141 	fmi = mp->mnt_transinfo;
142 	if (atomic_dec_uint_nv(&fmi->fmi_ref_cnt) > 0)
143 		return;
144 
145 	KASSERT(fmi->fmi_state == FSTRANS_NORMAL);
146 	KASSERT(LIST_FIRST(&fmi->fmi_cow_handler) == NULL);
147 
148 	kmem_free(fmi, sizeof(*fmi));
149 	mp->mnt_iflag &= ~IMNT_HAS_TRANS;
150 	mp->mnt_transinfo = NULL;
151 
152 	vfs_destroy(mp);
153 }
154 
155 /*
156  * Allocate mount state.
157  */
158 int
159 fstrans_mount(struct mount *mp)
160 {
161 	int error;
162 	struct fstrans_mount_info *newfmi;
163 
164 	error = vfs_busy(mp, NULL);
165 	if (error)
166 		return error;
167 	newfmi = kmem_alloc(sizeof(*newfmi), KM_SLEEP);
168 	newfmi->fmi_state = FSTRANS_NORMAL;
169 	newfmi->fmi_ref_cnt = 1;
170 	LIST_INIT(&newfmi->fmi_cow_handler);
171 	newfmi->fmi_cow_change = false;
172 
173 	mp->mnt_transinfo = newfmi;
174 	mp->mnt_iflag |= IMNT_HAS_TRANS;
175 
176 	vfs_unbusy(mp, true, NULL);
177 
178 	return 0;
179 }
180 
181 /*
182  * Deallocate mount state.
183  */
184 void
185 fstrans_unmount(struct mount *mp)
186 {
187 
188 	KASSERT(mp->mnt_transinfo != NULL);
189 
190 	fstrans_mount_dtor(mp);
191 }
192 
193 /*
194  * Retrieve the per lwp info for this mount allocating if necessary.
195  */
196 static struct fstrans_lwp_info *
197 fstrans_get_lwp_info(struct mount *mp, bool do_alloc)
198 {
199 	struct fstrans_lwp_info *fli, *res;
200 	struct fstrans_mount_info *fmi;
201 
202 	/*
203 	 * Scan our list for a match clearing entries whose mount is gone.
204 	 */
205 	res = NULL;
206 	for (fli = lwp_getspecific(lwp_data_key); fli; fli = fli->fli_succ) {
207 		if (fli->fli_mount == mp) {
208 			KASSERT(res == NULL);
209 			res = fli;
210 		} else if (fli->fli_mount != NULL &&
211 		    (fli->fli_mount->mnt_iflag & IMNT_GONE) != 0 &&
212 		    fli->fli_trans_cnt == 0 && fli->fli_cow_cnt == 0) {
213 			fstrans_mount_dtor(fli->fli_mount);
214 			fli->fli_mount = NULL;
215 		}
216 	}
217 	if (__predict_true(res != NULL))
218 		return res;
219 
220 	if (! do_alloc)
221 		return NULL;
222 
223 	/*
224 	 * Try to reuse a cleared entry or allocate a new one.
225 	 */
226 	for (fli = lwp_getspecific(lwp_data_key); fli; fli = fli->fli_succ) {
227 		if (fli->fli_mount == NULL) {
228 			KASSERT(fli->fli_trans_cnt == 0);
229 			KASSERT(fli->fli_cow_cnt == 0);
230 			break;
231 		}
232 	}
233 	if (fli == NULL) {
234 		mutex_enter(&fstrans_lock);
235 		LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
236 			if (fli->fli_self == NULL) {
237 				KASSERT(fli->fli_trans_cnt == 0);
238 				KASSERT(fli->fli_cow_cnt == 0);
239 				fli->fli_self = curlwp;
240 				fli->fli_succ = lwp_getspecific(lwp_data_key);
241 				lwp_setspecific(lwp_data_key, fli);
242 				break;
243 			}
244 		}
245 		mutex_exit(&fstrans_lock);
246 	}
247 	if (fli == NULL) {
248 		fli = kmem_alloc(sizeof(*fli), KM_SLEEP);
249 		mutex_enter(&fstrans_lock);
250 		memset(fli, 0, sizeof(*fli));
251 		fli->fli_self = curlwp;
252 		LIST_INSERT_HEAD(&fstrans_fli_head, fli, fli_list);
253 		mutex_exit(&fstrans_lock);
254 		fli->fli_succ = lwp_getspecific(lwp_data_key);
255 		lwp_setspecific(lwp_data_key, fli);
256 	}
257 
258 	/*
259 	 * Attach the entry to the mount.
260 	 */
261 	fmi = mp->mnt_transinfo;
262 	fli->fli_mount = mp;
263 	atomic_inc_uint(&fmi->fmi_ref_cnt);
264 
265 	return fli;
266 }
267 
268 /*
269  * Check if this lock type is granted at this state.
270  */
271 static bool
272 grant_lock(const enum fstrans_state state, const enum fstrans_lock_type type)
273 {
274 
275 	if (__predict_true(state == FSTRANS_NORMAL))
276 		return true;
277 	if (type == FSTRANS_EXCL)
278 		return true;
279 	if  (state == FSTRANS_SUSPENDING && type == FSTRANS_LAZY)
280 		return true;
281 
282 	return false;
283 }
284 
285 /*
286  * Start a transaction.  If this thread already has a transaction on this
287  * file system increment the reference counter.
288  */
289 int
290 _fstrans_start(struct mount *mp, enum fstrans_lock_type lock_type, int wait)
291 {
292 	int s;
293 	struct fstrans_lwp_info *fli;
294 	struct fstrans_mount_info *fmi;
295 
296 	ASSERT_SLEEPABLE();
297 
298 	if (mp == NULL || (mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
299 		return 0;
300 
301 	fli = fstrans_get_lwp_info(mp, true);
302 
303 	if (fli->fli_trans_cnt > 0) {
304 		KASSERT(lock_type != FSTRANS_EXCL);
305 		fli->fli_trans_cnt += 1;
306 
307 		return 0;
308 	}
309 
310 	s = pserialize_read_enter();
311 	fmi = mp->mnt_transinfo;
312 	if (__predict_true(grant_lock(fmi->fmi_state, lock_type))) {
313 		fli->fli_trans_cnt = 1;
314 		fli->fli_lock_type = lock_type;
315 		pserialize_read_exit(s);
316 
317 		return 0;
318 	}
319 	pserialize_read_exit(s);
320 
321 	if (! wait)
322 		return EBUSY;
323 
324 	mutex_enter(&fstrans_lock);
325 	while (! grant_lock(fmi->fmi_state, lock_type))
326 		cv_wait(&fstrans_state_cv, &fstrans_lock);
327 	fli->fli_trans_cnt = 1;
328 	fli->fli_lock_type = lock_type;
329 	mutex_exit(&fstrans_lock);
330 
331 	return 0;
332 }
333 
334 /*
335  * Finish a transaction.
336  */
337 void
338 fstrans_done(struct mount *mp)
339 {
340 	int s;
341 	struct fstrans_lwp_info *fli;
342 	struct fstrans_mount_info *fmi;
343 
344 	if (mp == NULL || (mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
345 		return;
346 
347 	fli = fstrans_get_lwp_info(mp, false);
348 	KASSERT(fli != NULL);
349 	KASSERT(fli->fli_trans_cnt > 0);
350 
351 	if (fli->fli_trans_cnt > 1) {
352 		fli->fli_trans_cnt -= 1;
353 
354 		return;
355 	}
356 
357 	s = pserialize_read_enter();
358 	fmi = mp->mnt_transinfo;
359 	if (__predict_true(fmi->fmi_state == FSTRANS_NORMAL)) {
360 		fli->fli_trans_cnt = 0;
361 		pserialize_read_exit(s);
362 
363 		return;
364 	}
365 	pserialize_read_exit(s);
366 
367 	mutex_enter(&fstrans_lock);
368 	fli->fli_trans_cnt = 0;
369 	cv_signal(&fstrans_count_cv);
370 	mutex_exit(&fstrans_lock);
371 }
372 
373 /*
374  * Check if this thread has an exclusive lock.
375  */
376 int
377 fstrans_is_owner(struct mount *mp)
378 {
379 	struct fstrans_lwp_info *fli;
380 
381 	if (mp == NULL || (mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
382 		return 0;
383 
384 	fli = fstrans_get_lwp_info(mp, false);
385 	if (fli == NULL || fli->fli_trans_cnt == 0)
386 		return 0;
387 
388 	KASSERT(fli->fli_mount == mp);
389 	KASSERT(fli->fli_trans_cnt > 0);
390 
391 	return (fli->fli_lock_type == FSTRANS_EXCL);
392 }
393 
394 /*
395  * True, if no thread is in a transaction not granted at the current state.
396  */
397 static bool
398 state_change_done(const struct mount *mp)
399 {
400 	struct fstrans_lwp_info *fli;
401 	struct fstrans_mount_info *fmi;
402 
403 	KASSERT(mutex_owned(&fstrans_lock));
404 
405 	fmi = mp->mnt_transinfo;
406 	LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
407 		if (fli->fli_mount != mp)
408 			continue;
409 		if (fli->fli_trans_cnt == 0)
410 			continue;
411 		if (grant_lock(fmi->fmi_state, fli->fli_lock_type))
412 			continue;
413 
414 		return false;
415 	}
416 
417 	return true;
418 }
419 
420 /*
421  * Set new file system state.
422  */
423 int
424 fstrans_setstate(struct mount *mp, enum fstrans_state new_state)
425 {
426 	int error;
427 	enum fstrans_state old_state;
428 	struct fstrans_mount_info *fmi;
429 
430 	fmi = mp->mnt_transinfo;
431 	old_state = fmi->fmi_state;
432 	if (old_state == new_state)
433 		return 0;
434 
435 	mutex_enter(&fstrans_lock);
436 	fmi->fmi_state = new_state;
437 	pserialize_perform(fstrans_psz);
438 
439 	/*
440 	 * All threads see the new state now.
441 	 * Wait for transactions invalid at this state to leave.
442 	 */
443 	error = 0;
444 	while (! state_change_done(mp)) {
445 		error = cv_wait_sig(&fstrans_count_cv, &fstrans_lock);
446 		if (error) {
447 			new_state = fmi->fmi_state = FSTRANS_NORMAL;
448 			break;
449 		}
450 	}
451 	cv_broadcast(&fstrans_state_cv);
452 	mutex_exit(&fstrans_lock);
453 
454 	if (old_state != new_state) {
455 		if (old_state == FSTRANS_NORMAL)
456 			fstrans_start(mp, FSTRANS_EXCL);
457 		if (new_state == FSTRANS_NORMAL)
458 			fstrans_done(mp);
459 	}
460 
461 	return error;
462 }
463 
464 /*
465  * Get current file system state.
466  */
467 enum fstrans_state
468 fstrans_getstate(struct mount *mp)
469 {
470 	struct fstrans_mount_info *fmi;
471 
472 	fmi = mp->mnt_transinfo;
473 	KASSERT(fmi != NULL);
474 
475 	return fmi->fmi_state;
476 }
477 
478 /*
479  * Request a filesystem to suspend all operations.
480  */
481 int
482 vfs_suspend(struct mount *mp, int nowait)
483 {
484 	int error;
485 
486 	if (nowait) {
487 		if (!mutex_tryenter(&vfs_suspend_lock))
488 			return EWOULDBLOCK;
489 	} else
490 		mutex_enter(&vfs_suspend_lock);
491 
492 	mutex_enter(&syncer_mutex);
493 	if ((error = VFS_SUSPENDCTL(mp, SUSPEND_SUSPEND)) != 0) {
494 		mutex_exit(&syncer_mutex);
495 		mutex_exit(&vfs_suspend_lock);
496 	}
497 
498 	return error;
499 }
500 
501 /*
502  * Request a filesystem to resume all operations.
503  */
504 void
505 vfs_resume(struct mount *mp)
506 {
507 
508 	VFS_SUSPENDCTL(mp, SUSPEND_RESUME);
509 	mutex_exit(&syncer_mutex);
510 	mutex_exit(&vfs_suspend_lock);
511 }
512 
513 
514 /*
515  * True, if no thread is running a cow handler.
516  */
517 static bool
518 cow_state_change_done(const struct mount *mp)
519 {
520 	struct fstrans_lwp_info *fli;
521 	struct fstrans_mount_info *fmi __diagused;
522 
523 	fmi = mp->mnt_transinfo;
524 
525 	KASSERT(mutex_owned(&fstrans_lock));
526 	KASSERT(fmi->fmi_cow_change);
527 
528 	LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
529 		if (fli->fli_mount != mp)
530 			continue;
531 		if (fli->fli_cow_cnt == 0)
532 			continue;
533 
534 		return false;
535 	}
536 
537 	return true;
538 }
539 
540 /*
541  * Prepare for changing this mounts cow list.
542  * Returns with fstrans_lock locked.
543  */
544 static void
545 cow_change_enter(const struct mount *mp)
546 {
547 	struct fstrans_mount_info *fmi;
548 
549 	fmi = mp->mnt_transinfo;
550 
551 	mutex_enter(&fstrans_lock);
552 
553 	/*
554 	 * Wait for other threads changing the list.
555 	 */
556 	while (fmi->fmi_cow_change)
557 		cv_wait(&fstrans_state_cv, &fstrans_lock);
558 
559 	/*
560 	 * Wait until all threads are aware of a state change.
561 	 */
562 	fmi->fmi_cow_change = true;
563 	pserialize_perform(fstrans_psz);
564 
565 	while (! cow_state_change_done(mp))
566 		cv_wait(&fstrans_count_cv, &fstrans_lock);
567 }
568 
569 /*
570  * Done changing this mounts cow list.
571  */
572 static void
573 cow_change_done(const struct mount *mp)
574 {
575 	struct fstrans_mount_info *fmi;
576 
577 	KASSERT(mutex_owned(&fstrans_lock));
578 
579 	fmi = mp->mnt_transinfo;
580 
581 	fmi->fmi_cow_change = false;
582 	pserialize_perform(fstrans_psz);
583 
584 	cv_broadcast(&fstrans_state_cv);
585 
586 	mutex_exit(&fstrans_lock);
587 }
588 
589 /*
590  * Add a handler to this mount.
591  */
592 int
593 fscow_establish(struct mount *mp, int (*func)(void *, struct buf *, bool),
594     void *arg)
595 {
596 	struct fstrans_mount_info *fmi;
597 	struct fscow_handler *newch;
598 
599 	if ((mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
600 		return EINVAL;
601 
602 	fmi = mp->mnt_transinfo;
603 	KASSERT(fmi != NULL);
604 
605 	newch = kmem_alloc(sizeof(*newch), KM_SLEEP);
606 	newch->ch_func = func;
607 	newch->ch_arg = arg;
608 
609 	cow_change_enter(mp);
610 	LIST_INSERT_HEAD(&fmi->fmi_cow_handler, newch, ch_list);
611 	cow_change_done(mp);
612 
613 	return 0;
614 }
615 
616 /*
617  * Remove a handler from this mount.
618  */
619 int
620 fscow_disestablish(struct mount *mp, int (*func)(void *, struct buf *, bool),
621     void *arg)
622 {
623 	struct fstrans_mount_info *fmi;
624 	struct fscow_handler *hp = NULL;
625 
626 	if ((mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
627 		return EINVAL;
628 
629 	fmi = mp->mnt_transinfo;
630 	KASSERT(fmi != NULL);
631 
632 	cow_change_enter(mp);
633 	LIST_FOREACH(hp, &fmi->fmi_cow_handler, ch_list)
634 		if (hp->ch_func == func && hp->ch_arg == arg)
635 			break;
636 	if (hp != NULL) {
637 		LIST_REMOVE(hp, ch_list);
638 		kmem_free(hp, sizeof(*hp));
639 	}
640 	cow_change_done(mp);
641 
642 	return hp ? 0 : EINVAL;
643 }
644 
645 /*
646  * Check for need to copy block that is about to be written.
647  */
648 int
649 fscow_run(struct buf *bp, bool data_valid)
650 {
651 	int error, s;
652 	struct mount *mp;
653 	struct fstrans_lwp_info *fli;
654 	struct fstrans_mount_info *fmi;
655 	struct fscow_handler *hp;
656 
657 	/*
658 	 * First check if we need run the copy-on-write handler.
659 	 */
660 	if ((bp->b_flags & B_COWDONE))
661 		return 0;
662 	if (bp->b_vp == NULL) {
663 		bp->b_flags |= B_COWDONE;
664 		return 0;
665 	}
666 	if (bp->b_vp->v_type == VBLK)
667 		mp = spec_node_getmountedfs(bp->b_vp);
668 	else
669 		mp = bp->b_vp->v_mount;
670 	if (mp == NULL || (mp->mnt_iflag & IMNT_HAS_TRANS) == 0) {
671 		bp->b_flags |= B_COWDONE;
672 		return 0;
673 	}
674 
675 	fli = fstrans_get_lwp_info(mp, true);
676 	fmi = mp->mnt_transinfo;
677 
678 	/*
679 	 * On non-recursed run check if other threads
680 	 * want to change the list.
681 	 */
682 	if (fli->fli_cow_cnt == 0) {
683 		s = pserialize_read_enter();
684 		if (__predict_false(fmi->fmi_cow_change)) {
685 			pserialize_read_exit(s);
686 			mutex_enter(&fstrans_lock);
687 			while (fmi->fmi_cow_change)
688 				cv_wait(&fstrans_state_cv, &fstrans_lock);
689 			fli->fli_cow_cnt = 1;
690 			mutex_exit(&fstrans_lock);
691 		} else {
692 			fli->fli_cow_cnt = 1;
693 			pserialize_read_exit(s);
694 		}
695 	} else
696 		fli->fli_cow_cnt += 1;
697 
698 	/*
699 	 * Run all copy-on-write handlers, stop on error.
700 	 */
701 	error = 0;
702 	LIST_FOREACH(hp, &fmi->fmi_cow_handler, ch_list)
703 		if ((error = (*hp->ch_func)(hp->ch_arg, bp, data_valid)) != 0)
704 			break;
705  	if (error == 0)
706  		bp->b_flags |= B_COWDONE;
707 
708 	/*
709 	 * Check if other threads want to change the list.
710 	 */
711 	if (fli->fli_cow_cnt > 1) {
712 		fli->fli_cow_cnt -= 1;
713 	} else {
714 		s = pserialize_read_enter();
715 		if (__predict_false(fmi->fmi_cow_change)) {
716 			pserialize_read_exit(s);
717 			mutex_enter(&fstrans_lock);
718 			fli->fli_cow_cnt = 0;
719 			cv_signal(&fstrans_count_cv);
720 			mutex_exit(&fstrans_lock);
721 		} else {
722 			fli->fli_cow_cnt = 0;
723 			pserialize_read_exit(s);
724 		}
725 	}
726 
727 	return error;
728 }
729 
730 #if defined(DDB)
731 void fstrans_dump(int);
732 
733 static void
734 fstrans_print_lwp(struct proc *p, struct lwp *l, int verbose)
735 {
736 	char prefix[9];
737 	struct fstrans_lwp_info *fli;
738 
739 	snprintf(prefix, sizeof(prefix), "%d.%d", p->p_pid, l->l_lid);
740 	LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
741 		if (fli->fli_self != l)
742 			continue;
743 		if (fli->fli_trans_cnt == 0 && fli->fli_cow_cnt == 0) {
744 			if (! verbose)
745 				continue;
746 		}
747 		printf("%-8s", prefix);
748 		if (verbose)
749 			printf(" @%p", fli);
750 		if (fli->fli_mount != NULL)
751 			printf(" (%s)", fli->fli_mount->mnt_stat.f_mntonname);
752 		else
753 			printf(" NULL");
754 		if (fli->fli_trans_cnt == 0) {
755 			printf(" -");
756 		} else {
757 			switch (fli->fli_lock_type) {
758 			case FSTRANS_LAZY:
759 				printf(" lazy");
760 				break;
761 			case FSTRANS_SHARED:
762 				printf(" shared");
763 				break;
764 			case FSTRANS_EXCL:
765 				printf(" excl");
766 				break;
767 			default:
768 				printf(" %#x", fli->fli_lock_type);
769 				break;
770 			}
771 		}
772 		printf(" %d cow %d\n", fli->fli_trans_cnt, fli->fli_cow_cnt);
773 		prefix[0] = '\0';
774 	}
775 }
776 
777 static void
778 fstrans_print_mount(struct mount *mp, int verbose)
779 {
780 	struct fstrans_mount_info *fmi;
781 
782 	fmi = mp->mnt_transinfo;
783 	if (!verbose && (fmi == NULL || fmi->fmi_state == FSTRANS_NORMAL))
784 		return;
785 
786 	printf("%-16s ", mp->mnt_stat.f_mntonname);
787 	if (fmi == NULL) {
788 		printf("(null)\n");
789 		return;
790 	}
791 	switch (fmi->fmi_state) {
792 	case FSTRANS_NORMAL:
793 		printf("state normal\n");
794 		break;
795 	case FSTRANS_SUSPENDING:
796 		printf("state suspending\n");
797 		break;
798 	case FSTRANS_SUSPENDED:
799 		printf("state suspended\n");
800 		break;
801 	default:
802 		printf("state %#x\n", fmi->fmi_state);
803 		break;
804 	}
805 }
806 
807 void
808 fstrans_dump(int full)
809 {
810 	const struct proclist_desc *pd;
811 	struct proc *p;
812 	struct lwp *l;
813 	struct mount *mp;
814 
815 	printf("Fstrans locks by lwp:\n");
816 	for (pd = proclists; pd->pd_list != NULL; pd++)
817 		PROCLIST_FOREACH(p, pd->pd_list)
818 			LIST_FOREACH(l, &p->p_lwps, l_sibling)
819 				fstrans_print_lwp(p, l, full == 1);
820 
821 	printf("Fstrans state by mount:\n");
822 	TAILQ_FOREACH(mp, &mountlist, mnt_list)
823 		fstrans_print_mount(mp, full == 1);
824 }
825 #endif /* defined(DDB) */
826