Lines Matching defs:wl
119 struct mount *wl_mount; /* r: mountpoint wl is associated with */
242 #define WAPBL_USE_FUA(wl) \
243 (wapbl_allow_dpofua && ISSET((wl)->wl_dkcache, DKCACHE_FUA))
244 #define WAPBL_JFLAGS(wl) \
245 (WAPBL_USE_FUA(wl) ? (wl)->wl_jwrite_flags : 0)
246 #define WAPBL_JDATA_FLAGS(wl) \
247 (WAPBL_JFLAGS(wl) & B_MEDIA_DPO) /* only DPO */
286 static void wapbl_inodetrk_init(struct wapbl *wl, u_int size);
287 static void wapbl_inodetrk_free(struct wapbl *wl);
288 static struct wapbl_ino *wapbl_inodetrk_get(struct wapbl *wl, ino_t ino);
290 static size_t wapbl_transaction_len(struct wapbl *wl);
291 static inline size_t wapbl_transaction_inodes_len(struct wapbl *wl);
400 wapbl_evcnt_init(struct wapbl *wl)
403 snprintf(wl->wl_ev_group, sizeof(wl->wl_ev_group),
405 wl->wl_mount->mnt_stat.f_fsidx.__fsid_val[0],
406 wl->wl_mount->mnt_stat.f_fsidx.__fsid_val[1]);
408 evcnt_attach_dynamic(&wl->wl_ev_commit, EVCNT_TYPE_MISC,
409 NULL, wl->wl_ev_group, "commit");
410 evcnt_attach_dynamic(&wl->wl_ev_journalwrite, EVCNT_TYPE_MISC,
411 NULL, wl->wl_ev_group, "journal write total");
412 evcnt_attach_dynamic(&wl->wl_ev_jbufs_bio_nowait, EVCNT_TYPE_MISC,
413 NULL, wl->wl_ev_group, "journal write finished async");
414 evcnt_attach_dynamic(&wl->wl_ev_metawrite, EVCNT_TYPE_MISC,
415 NULL, wl->wl_ev_group, "metadata async write");
416 evcnt_attach_dynamic(&wl->wl_ev_cacheflush, EVCNT_TYPE_MISC,
417 NULL, wl->wl_ev_group, "cache flush");
421 wapbl_evcnt_free(struct wapbl *wl)
424 evcnt_detach(&wl->wl_ev_commit);
425 evcnt_detach(&wl->wl_ev_journalwrite);
426 evcnt_detach(&wl->wl_ev_jbufs_bio_nowait);
427 evcnt_detach(&wl->wl_ev_metawrite);
428 evcnt_detach(&wl->wl_ev_cacheflush);
432 wapbl_dkcache_init(struct wapbl *wl)
437 error = VOP_IOCTL(wl->wl_devvp, DIOCGCACHE, &wl->wl_dkcache,
441 wl->wl_dkcache = DKCACHE_WRITE;
445 if (ISSET(wl->wl_dkcache, DKCACHE_FUA))
446 wl->wl_jwrite_flags |= B_MEDIA_FUA;
449 if (ISSET(wl->wl_dkcache, DKCACHE_DPO))
450 wl->wl_jwrite_flags |= B_MEDIA_DPO;
454 wapbl_start_flush_inodes(struct wapbl *wl, struct wapbl_replay *wr)
466 KASSERT(wl->wl_devvp->v_type == VBLK);
468 KASSERT(wl->wl_devvp->v_rdev == wr->wr_devvp->v_rdev);
469 KASSERT(wl->wl_logpbn == wr->wr_logpbn);
470 KASSERT(wl->wl_circ_size == wr->wr_circ_size);
471 KASSERT(wl->wl_circ_off == wr->wr_circ_off);
472 KASSERT(wl->wl_log_dev_bshift == wr->wr_log_dev_bshift);
473 KASSERT(wl->wl_fs_dev_bshift == wr->wr_fs_dev_bshift);
475 wl->wl_wc_header->wc_generation = wr->wr_generation + 1;
478 wapbl_register_inode(wl, wr->wr_inodes[i].wr_inumber,
482 KDASSERT(wapbl_transaction_len(wl) <=
483 wapbl_space_free(wl->wl_circ_size, wr->wr_inodeshead,
486 wl->wl_head = wl->wl_tail = wr->wr_inodeshead;
487 wl->wl_reclaimable_bytes = wl->wl_reserved_bytes =
488 wapbl_transaction_len(wl);
490 error = wapbl_write_inodes(wl, &wl->wl_head);
494 KASSERT(wl->wl_head != wl->wl_tail);
495 KASSERT(wl->wl_head != 0);
505 struct wapbl *wl;
552 wl = wapbl_calloc(1, sizeof(*wl));
553 rw_init(&wl->wl_rwlock);
554 mutex_init(&wl->wl_mtx, MUTEX_DEFAULT, IPL_NONE);
555 cv_init(&wl->wl_reclaimable_cv, "wapblrec");
556 TAILQ_INIT(&wl->wl_bufs);
557 SIMPLEQ_INIT(&wl->wl_entries);
559 wl->wl_logvp = vp;
560 wl->wl_devvp = devvp;
561 wl->wl_mount = mp;
562 wl->wl_logpbn = logpbn;
563 wl->wl_log_dev_bshift = log_dev_bshift;
564 wl->wl_fs_dev_bshift = fs_dev_bshift;
566 wl->wl_flush = flushfn;
567 wl->wl_flush_abort = flushabortfn;
570 wl->wl_circ_off = 2<<wl->wl_log_dev_bshift;
571 wl->wl_circ_size = ((count * blksize) - wl->wl_circ_off);
573 wl->wl_circ_size >>= wl->wl_log_dev_bshift;
574 wl->wl_circ_size <<= wl->wl_log_dev_bshift;
596 wl->wl_bufbytes_max = MIN(wl->wl_circ_size, buf_memcalc() / 2);
599 wl->wl_bufbytes_max >>= PAGE_SHIFT;
600 wl->wl_bufbytes_max <<= PAGE_SHIFT;
601 wl->wl_bufbytes_max >>= wl->wl_log_dev_bshift;
602 wl->wl_bufbytes_max <<= wl->wl_log_dev_bshift;
603 wl->wl_bufbytes_max >>= wl->wl_fs_dev_bshift;
604 wl->wl_bufbytes_max <<= wl->wl_fs_dev_bshift;
608 wl->wl_bufcount_max = (buf_nbuf() / 2) * 1024;
610 wl->wl_brperjblock = ((1<<wl->wl_log_dev_bshift)
613 KASSERT(wl->wl_brperjblock > 0);
616 wl->wl_dealloclim = wl->wl_bufbytes_max / mp->mnt_stat.f_bsize / 2;
617 TAILQ_INIT(&wl->wl_dealloclist);
619 wapbl_inodetrk_init(wl, WAPBL_INODETRK_SIZE);
621 wapbl_evcnt_init(wl);
623 wapbl_dkcache_init(wl);
628 size_t len = 1 << wl->wl_log_dev_bshift;
632 wc->wc_circ_off = wl->wl_circ_off;
633 wc->wc_circ_size = wl->wl_circ_size;
635 wc->wc_log_dev_bshift = wl->wl_log_dev_bshift;
636 wc->wc_fs_dev_bshift = wl->wl_fs_dev_bshift;
637 wl->wl_wc_header = wc;
638 wl->wl_wc_scratch = wapbl_alloc(len);
641 TAILQ_INIT(&wl->wl_iobufs);
642 TAILQ_INIT(&wl->wl_iobufs_busy);
657 TAILQ_INSERT_TAIL(&wl->wl_iobufs, bp, b_wapbllist);
666 error = wapbl_start_flush_inodes(wl, wr);
671 error = wapbl_write_commit(wl, wl->wl_head, wl->wl_tail);
676 *wlp = wl;
678 wapbl_debug_wl = wl;
683 wapbl_discard(wl);
684 wapbl_free(wl->wl_wc_scratch, wl->wl_wc_header->wc_len);
685 wapbl_free(wl->wl_wc_header, wl->wl_wc_header->wc_len);
686 while (!TAILQ_EMPTY(&wl->wl_iobufs)) {
689 bp = TAILQ_FIRST(&wl->wl_iobufs);
690 TAILQ_REMOVE(&wl->wl_iobufs, bp, b_wapbllist);
693 wapbl_inodetrk_free(wl);
694 wapbl_free(wl, sizeof(*wl));
705 wapbl_discard(struct wapbl *wl)
716 rw_enter(&wl->wl_rwlock, RW_WRITER);
717 wl->wl_flush(wl->wl_mount, TAILQ_FIRST(&wl->wl_dealloclist));
736 wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount,
737 wl->wl_dealloccnt, wl->wl_inohashcnt,
738 wl->wl_error_count, wl->wl_reclaimable_bytes,
739 wl->wl_reserved_bytes,
740 wl->wl_unsynced_bufbytes));
741 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) {
755 wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount,
756 wl->wl_dealloccnt, wl->wl_inohashcnt,
757 wl->wl_error_count, wl->wl_reclaimable_bytes,
758 wl->wl_reserved_bytes));
759 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) {
770 for (i = 0; i <= wl->wl_inohashmask; i++) {
774 wih = &wl->wl_inohash[i];
778 KASSERT(wl->wl_inohashcnt > 0);
779 wl->wl_inohashcnt--;
787 mutex_enter(&wl->wl_mtx);
788 while ((bp = TAILQ_FIRST(&wl->wl_bufs)) != NULL) {
789 if (bbusy(bp, 0, 0, &wl->wl_mtx) == 0) {
797 mutex_exit(&wl->wl_mtx);
800 mutex_enter(&wl->wl_mtx);
805 * Remove references to this wl from wl_entries, free any which
809 while ((we = SIMPLEQ_FIRST(&wl->wl_entries)) != NULL) {
810 SIMPLEQ_REMOVE_HEAD(&wl->wl_entries, we_entries);
822 mutex_exit(&wl->wl_mtx);
826 while ((wd = TAILQ_FIRST(&wl->wl_dealloclist)) != NULL)
827 wapbl_deallocation_free(wl, wd, true);
831 KASSERT(wl->wl_bufbytes == 0);
832 KASSERT(wl->wl_bcount == 0);
833 KASSERT(wl->wl_bufcount == 0);
834 KASSERT(TAILQ_EMPTY(&wl->wl_bufs));
835 KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries));
836 KASSERT(wl->wl_inohashcnt == 0);
837 KASSERT(TAILQ_EMPTY(&wl->wl_dealloclist));
838 KASSERT(wl->wl_dealloccnt == 0);
840 rw_exit(&wl->wl_rwlock);
844 wapbl_stop(struct wapbl *wl, int force)
849 error = wapbl_flush(wl, 1);
852 wapbl_discard(wl);
858 if (wl->wl_inohashcnt) {
860 wapbl_discard(wl);
866 KASSERT(wl->wl_bufbytes == 0);
867 KASSERT(wl->wl_bcount == 0);
868 KASSERT(wl->wl_bufcount == 0);
869 KASSERT(TAILQ_EMPTY(&wl->wl_bufs));
870 KASSERT(wl->wl_dealloccnt == 0);
871 KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries));
872 KASSERT(wl->wl_inohashcnt == 0);
873 KASSERT(TAILQ_EMPTY(&wl->wl_dealloclist));
874 KASSERT(wl->wl_dealloccnt == 0);
875 KASSERT(TAILQ_EMPTY(&wl->wl_iobufs_busy));
877 wapbl_free(wl->wl_wc_scratch, wl->wl_wc_header->wc_len);
878 wapbl_free(wl->wl_wc_header, wl->wl_wc_header->wc_len);
879 while (!TAILQ_EMPTY(&wl->wl_iobufs)) {
882 bp = TAILQ_FIRST(&wl->wl_iobufs);
883 TAILQ_REMOVE(&wl->wl_iobufs, bp, b_wapbllist);
886 wapbl_inodetrk_free(wl);
888 wapbl_evcnt_free(wl);
890 cv_destroy(&wl->wl_reclaimable_cv);
891 mutex_destroy(&wl->wl_mtx);
892 rw_destroy(&wl->wl_rwlock);
893 wapbl_free(wl, sizeof(*wl));
993 * wapbl_buffered_write_async(wl, bp)
998 wapbl_buffered_write_async(struct wapbl *wl, struct buf *bp)
1001 wapbl_doio_accounting(wl->wl_devvp, bp->b_flags);
1003 KASSERT(TAILQ_FIRST(&wl->wl_iobufs) == bp);
1004 TAILQ_REMOVE(&wl->wl_iobufs, bp, b_wapbllist);
1012 VOP_STRATEGY(wl->wl_devvp, bp);
1014 wl->wl_ev_journalwrite.ev_count++;
1016 TAILQ_INSERT_TAIL(&wl->wl_iobufs_busy, bp, b_wapbllist);
1020 * wapbl_buffered_flush(wl)
1025 wapbl_buffered_flush(struct wapbl *wl, bool full)
1032 if ((bp = TAILQ_FIRST(&wl->wl_iobufs)) && bp->b_resid > 0)
1033 wapbl_buffered_write_async(wl, bp);
1037 TAILQ_FOREACH_SAFE(bp, &wl->wl_iobufs_busy, b_wapbllist, bnext) {
1045 wl->wl_ev_jbufs_bio_nowait.ev_count++;
1047 TAILQ_REMOVE(&wl->wl_iobufs_busy, bp, b_wapbllist);
1052 TAILQ_INSERT_TAIL(&wl->wl_iobufs, bp, b_wapbllist);
1059 if (!found && only_done && !TAILQ_EMPTY(&wl->wl_iobufs_busy)) {
1068 * wapbl_buffered_write(data, len, wl, pbn)
1071 * wl->wl_devvp. The write may not complete until
1075 wapbl_buffered_write(void *data, size_t len, struct wapbl *wl, daddr_t pbn,
1082 bp = TAILQ_FIRST(&wl->wl_iobufs);
1086 wapbl_buffered_flush(wl, false);
1088 bp = TAILQ_FIRST(&wl->wl_iobufs);
1097 wapbl_buffered_write_async(wl, bp);
1129 wapbl_buffered_write_async(wl, bp);
1143 * wapbl_circ_write(wl, data, len, offp)
1145 * Write len bytes from data to the circular queue of wl, starting
1149 * If the starting linear byte offset precedes wl->wl_circ_off,
1150 * the write instead begins at wl->wl_circ_off. XXX WTF? This
1153 * The write is buffered in wl and must be flushed with
1157 wapbl_circ_write(struct wapbl *wl, void *data, size_t len, off_t *offp)
1164 KDASSERT(((len >> wl->wl_log_dev_bshift) << wl->wl_log_dev_bshift) ==
1167 if (off < wl->wl_circ_off)
1168 off = wl->wl_circ_off;
1169 slen = wl->wl_circ_off + wl->wl_circ_size - off;
1171 pbn = wl->wl_logpbn + (off >> wl->wl_log_dev_bshift);
1173 pbn = btodb(pbn << wl->wl_log_dev_bshift);
1175 error = wapbl_buffered_write(data, slen, wl, pbn,
1176 WAPBL_JDATA_FLAGS(wl));
1181 off = wl->wl_circ_off;
1183 pbn = wl->wl_logpbn + (off >> wl->wl_log_dev_bshift);
1185 pbn = btodb(pbn << wl->wl_log_dev_bshift);
1187 error = wapbl_buffered_write(data, len, wl, pbn,
1188 WAPBL_JDATA_FLAGS(wl));
1192 if (off >= wl->wl_circ_off + wl->wl_circ_size)
1193 off = wl->wl_circ_off;
1204 wapbl_begin(struct wapbl *wl, const char *file, int line)
1209 KDASSERT(wl);
1216 mutex_enter(&wl->wl_mtx);
1217 lockcount = wl->wl_lock_count;
1218 doflush = ((wl->wl_bufbytes + (lockcount * MAXPHYS)) >
1219 wl->wl_bufbytes_max / 2) ||
1220 ((wl->wl_bufcount + (lockcount * 10)) >
1221 wl->wl_bufcount_max / 2) ||
1222 (wapbl_transaction_len(wl) > wl->wl_circ_size / 2) ||
1223 (wl->wl_dealloccnt >= (wl->wl_dealloclim / 2));
1224 mutex_exit(&wl->wl_mtx);
1231 lockcount, wl->wl_bufbytes,
1232 wl->wl_bufbytes_max, wl->wl_bufcount,
1233 wl->wl_bufcount_max,
1234 wl->wl_dealloccnt, wl->wl_dealloclim));
1238 int error = wapbl_flush(wl, 0);
1243 rw_enter(&wl->wl_rwlock, RW_READER);
1244 mutex_enter(&wl->wl_mtx);
1245 wl->wl_lock_count++;
1246 mutex_exit(&wl->wl_mtx);
1252 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount,
1253 wl->wl_bufbytes, wl->wl_bcount, file, line));
1260 wapbl_end(struct wapbl *wl)
1267 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount,
1268 wl->wl_bufbytes, wl->wl_bcount));
1276 KASSERTMSG((wapbl_transaction_len(wl) <=
1277 (wl->wl_circ_size - wl->wl_reserved_bytes)),
1280 mutex_enter(&wl->wl_mtx);
1281 KASSERT(wl->wl_lock_count > 0);
1282 wl->wl_lock_count--;
1283 mutex_exit(&wl->wl_mtx);
1285 rw_exit(&wl->wl_rwlock);
1289 wapbl_add_buf(struct wapbl *wl, struct buf * bp)
1295 wapbl_jlock_assert(wl);
1307 mutex_enter(&wl->wl_mtx);
1309 TAILQ_REMOVE(&wl->wl_bufs, bp, b_wapbllist);
1318 wl->wl_bufbytes += bp->b_bufsize;
1319 wl->wl_bcount += bp->b_bcount;
1320 wl->wl_bufcount++;
1327 TAILQ_INSERT_TAIL(&wl->wl_bufs, bp, b_wapbllist);
1328 mutex_exit(&wl->wl_mtx);
1334 wapbl_remove_buf_locked(struct wapbl * wl, struct buf *bp)
1337 KASSERT(mutex_owned(&wl->wl_mtx));
1339 wapbl_jlock_assert(wl);
1358 KASSERT(wl->wl_bufbytes >= bp->b_bufsize);
1359 wl->wl_bufbytes -= bp->b_bufsize;
1360 KASSERT(wl->wl_bcount >= bp->b_bcount);
1361 wl->wl_bcount -= bp->b_bcount;
1362 KASSERT(wl->wl_bufcount > 0);
1363 wl->wl_bufcount--;
1364 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0));
1365 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0));
1366 TAILQ_REMOVE(&wl->wl_bufs, bp, b_wapbllist);
1373 wapbl_remove_buf(struct wapbl * wl, struct buf *bp)
1376 mutex_enter(&wl->wl_mtx);
1377 wapbl_remove_buf_locked(wl, bp);
1378 mutex_exit(&wl->wl_mtx);
1382 wapbl_resize_buf(struct wapbl *wl, struct buf *bp, long oldsz, long oldcnt)
1393 mutex_enter(&wl->wl_mtx);
1394 wl->wl_bufbytes += bp->b_bufsize - oldsz;
1395 wl->wl_bcount += bp->b_bcount - oldcnt;
1396 mutex_exit(&wl->wl_mtx);
1522 * wapbl_truncate(wl, minfree)
1530 * => Caller must hold wl->wl_rwlock writer lock.
1533 wapbl_truncate(struct wapbl *wl, size_t minfree)
1541 KASSERT(minfree <= (wl->wl_circ_size - wl->wl_reserved_bytes));
1542 KASSERT(rw_write_held(&wl->wl_rwlock));
1544 mutex_enter(&wl->wl_mtx);
1550 avail = wapbl_space_free(wl->wl_circ_size, wl->wl_head, wl->wl_tail);
1552 mutex_exit(&wl->wl_mtx);
1556 while (wl->wl_error_count == 0 &&
1557 wl->wl_reclaimable_bytes < minfree) {
1560 " wl=%p bytes=%zd minfree=%zd\n",
1561 &wl->wl_reclaimable_bytes,
1562 wl, wl->wl_reclaimable_bytes, minfree));
1563 cv_wait(&wl->wl_reclaimable_cv, &wl->wl_mtx);
1565 if (wl->wl_reclaimable_bytes < minfree) {
1566 KASSERT(wl->wl_error_count);
1570 head = wl->wl_head;
1571 tail = wl->wl_tail;
1572 delta = wl->wl_reclaimable_bytes;
1578 if (SIMPLEQ_EMPTY(&wl->wl_entries) && delta >= wl->wl_reserved_bytes) {
1579 delta -= wl->wl_reserved_bytes;
1581 wapbl_advance_tail(wl->wl_circ_size, wl->wl_circ_off, delta, &head,
1583 KDASSERT(wl->wl_reserved_bytes <=
1584 wapbl_space_used(wl->wl_circ_size, head, tail));
1585 mutex_exit(&wl->wl_mtx);
1598 error = wapbl_write_commit(wl, head, tail);
1602 wl->wl_head = head;
1603 wl->wl_tail = tail;
1605 mutex_enter(&wl->wl_mtx);
1606 KASSERT(wl->wl_reclaimable_bytes >= delta);
1607 wl->wl_reclaimable_bytes -= delta;
1608 mutex_exit(&wl->wl_mtx);
1622 struct wapbl *wl;
1628 wl = we->we_wapbl;
1635 if (!wl) {
1672 mutex_enter(&wl->wl_mtx);
1673 if (wl->wl_error_count == 0) {
1674 wl->wl_error_count++;
1675 cv_broadcast(&wl->wl_reclaimable_cv);
1677 mutex_exit(&wl->wl_mtx);
1694 mutex_enter(&wl->wl_mtx);
1701 KASSERT(wl->wl_unsynced_bufbytes >= bufsize);
1702 wl->wl_unsynced_bufbytes -= bufsize;
1704 wl->wl_ev_metawrite.ev_count++;
1722 while ((we = SIMPLEQ_FIRST(&wl->wl_entries)) &&
1727 SIMPLEQ_REMOVE_HEAD(&wl->wl_entries, we_entries);
1732 wl->wl_reclaimable_bytes += delta;
1733 KASSERT(wl->wl_error_count >= errcnt);
1734 wl->wl_error_count -= errcnt;
1735 cv_broadcast(&wl->wl_reclaimable_cv);
1739 mutex_exit(&wl->wl_mtx);
1743 * wapbl_flush(wl, wait)
1761 wapbl_flush(struct wapbl *wl, int waitfor)
1780 mutex_enter(&wl->wl_mtx); /* XXX need mutex here to
1782 nbufs = wl->wl_bufcount;
1783 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0));
1784 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0));
1785 mutex_exit(&wl->wl_mtx);
1794 rw_enter(&wl->wl_rwlock, RW_WRITER);
1795 wl->wl_flush(wl->wl_mount, TAILQ_FIRST(&wl->wl_dealloclist));
1808 if (wl->wl_bufcount == 0) {
1816 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount,
1817 wl->wl_bufbytes));
1821 flushsize = wapbl_transaction_len(wl);
1830 if (flushsize > (wl->wl_circ_size - wl->wl_reserved_bytes)) {
1839 error = wapbl_truncate(wl, flushsize);
1843 off = wl->wl_head;
1844 KASSERT(off == 0 || off >= wl->wl_circ_off);
1845 KASSERT(off == 0 || off < wl->wl_circ_off + wl->wl_circ_size);
1846 error = wapbl_write_blocks(wl, &off);
1849 error = wapbl_write_revocations(wl, &off);
1852 error = wapbl_write_inodes(wl, &off);
1857 if (wl->wl_inohashcnt)
1858 reserved = wapbl_transaction_inodes_len(wl);
1860 head = wl->wl_head;
1861 tail = wl->wl_tail;
1863 wapbl_advance_head(wl->wl_circ_size, wl->wl_circ_off, flushsize,
1873 mutex_enter(&wl->wl_mtx);
1874 delta = wl->wl_reclaimable_bytes;
1875 mutex_exit(&wl->wl_mtx);
1876 wapbl_advance_tail(wl->wl_circ_size, wl->wl_circ_off, delta,
1879 error = wapbl_write_commit(wl, head, tail);
1892 wapbl_space_used(wl->wl_circ_size, head, tail),
1893 wl->wl_unsynced_bufbytes, wl->wl_bufcount,
1894 wl->wl_bufbytes, wl->wl_bcount, wl->wl_dealloccnt,
1895 wl->wl_inohashcnt));
1902 wapbl_space_used(wl->wl_circ_size, head, tail),
1903 wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount,
1904 wl->wl_dealloccnt, wl->wl_inohashcnt));
1909 mutex_enter(&wl->wl_mtx);
1911 wl->wl_reserved_bytes = reserved;
1912 wl->wl_head = head;
1913 wl->wl_tail = tail;
1914 KASSERT(wl->wl_reclaimable_bytes >= delta);
1915 wl->wl_reclaimable_bytes -= delta;
1916 KDASSERT(wl->wl_dealloccnt == 0);
1918 wl->wl_unsynced_bufbytes += wl->wl_bufbytes;
1921 we->we_wapbl = wl;
1922 we->we_bufcount = wl->wl_bufcount;
1924 we->we_unsynced_bufbytes = wl->wl_bufbytes;
1928 SIMPLEQ_INSERT_TAIL(&wl->wl_entries, we, we_entries);
1934 while ((bp = TAILQ_FIRST(&wl->wl_bufs)) != NULL) {
1935 if (bbusy(bp, 0, 0, &wl->wl_mtx)) {
1942 wapbl_remove_buf_locked(wl, bp);
1943 mutex_exit(&wl->wl_mtx);
1947 mutex_enter(&wl->wl_mtx);
1949 mutex_exit(&wl->wl_mtx);
1965 error = wapbl_truncate(wl, wl->wl_circ_size -
1966 wl->wl_reserved_bytes);
1971 wl->wl_flush_abort(wl->wl_mount,
1972 TAILQ_FIRST(&wl->wl_dealloclist));
1983 mutex_enter(&wl->wl_mtx);
1992 pid, lid, error, wl->wl_bufcount,
1993 wl->wl_bufbytes, wl->wl_bcount,
1994 wl->wl_dealloccnt, wl->wl_inohashcnt,
1995 wl->wl_error_count, wl->wl_reclaimable_bytes,
1996 wl->wl_reserved_bytes, wl->wl_unsynced_bufbytes));
1997 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) {
2011 pid, lid, error, wl->wl_bufcount,
2012 wl->wl_bufbytes, wl->wl_bcount,
2013 wl->wl_dealloccnt, wl->wl_inohashcnt,
2014 wl->wl_error_count, wl->wl_reclaimable_bytes,
2015 wl->wl_reserved_bytes));
2016 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) {
2023 mutex_exit(&wl->wl_mtx);
2027 rw_exit(&wl->wl_rwlock);
2034 wapbl_jlock_assert(struct wapbl *wl)
2037 KASSERT(rw_lock_held(&wl->wl_rwlock));
2041 wapbl_junlock_assert(struct wapbl *wl)
2044 KASSERT(!rw_write_held(&wl->wl_rwlock));
2051 wapbl_print(struct wapbl *wl, int full, void (*pr)(const char *, ...))
2055 (*pr)("wapbl %p", wl);
2057 wl->wl_logvp, wl->wl_devvp, wl->wl_logpbn);
2060 wl->wl_circ_size, wl->wl_circ_off,
2061 (intmax_t)wl->wl_head, (intmax_t)wl->wl_tail);
2063 wl->wl_log_dev_bshift, wl->wl_fs_dev_bshift);
2067 wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount,
2068 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes,
2069 wl->wl_error_count, wl->wl_unsynced_bufbytes);
2072 "reserved = %zu errcnt = %d\n", wl->wl_bufcount, wl->wl_bufbytes,
2073 wl->wl_bcount, wl->wl_reclaimable_bytes, wl->wl_reserved_bytes,
2074 wl->wl_error_count);
2077 wl->wl_dealloccnt, wl->wl_dealloclim);
2079 wl->wl_inohashcnt, wl->wl_inohashmask);
2081 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) {
2095 TAILQ_FOREACH(bp, &wl->wl_bufs, b_wapbllist) {
2110 TAILQ_FOREACH(wd, &wl->wl_dealloclist, wd_entries) {
2125 for (i = 0; i <= wl->wl_inohashmask; i++) {
2129 wih = &wl->wl_inohash[i];
2144 TAILQ_FOREACH(bp, &wl->wl_iobufs, b_wapbllist) {
2156 TAILQ_FOREACH(bp, &wl->wl_iobufs_busy, b_wapbllist) {
2171 wapbl_dump(struct wapbl *wl)
2174 if (!wl)
2175 wl = wapbl_debug_wl;
2177 if (!wl)
2179 wapbl_print(wl, 1, printf);
2186 wapbl_register_deallocation(struct wapbl *wl, daddr_t blk, int len, bool force,
2192 wapbl_jlock_assert(wl);
2194 mutex_enter(&wl->wl_mtx);
2196 if (__predict_false(wl->wl_dealloccnt >= wl->wl_dealloclim)) {
2213 wl->wl_mount->mnt_stat.f_mntonname,
2214 wl->wl_dealloccnt, wl->wl_dealloclim);
2217 wl->wl_dealloccnt++;
2218 mutex_exit(&wl->wl_mtx);
2224 mutex_enter(&wl->wl_mtx);
2225 TAILQ_INSERT_TAIL(&wl->wl_dealloclist, wd, wd_entries);
2231 mutex_exit(&wl->wl_mtx);
2241 wapbl_deallocation_free(struct wapbl *wl, struct wapbl_dealloc *wd,
2246 || rw_lock_held(&wl->wl_rwlock) || mutex_owned(&wl->wl_mtx));
2249 mutex_enter(&wl->wl_mtx);
2251 TAILQ_REMOVE(&wl->wl_dealloclist, wd, wd_entries);
2252 wl->wl_dealloccnt--;
2255 mutex_exit(&wl->wl_mtx);
2261 wapbl_unregister_deallocation(struct wapbl *wl, void *cookie)
2265 wapbl_deallocation_free(wl, cookie, false);
2271 wapbl_inodetrk_init(struct wapbl *wl, u_int size)
2274 wl->wl_inohash = hashinit(size, HASH_LIST, true, &wl->wl_inohashmask);
2282 wapbl_inodetrk_free(struct wapbl *wl)
2286 KASSERT(wl->wl_inohashcnt == 0);
2287 hashdone(wl->wl_inohash, HASH_LIST, wl->wl_inohashmask);
2296 wapbl_inodetrk_get(struct wapbl *wl, ino_t ino)
2301 KASSERT(mutex_owned(&wl->wl_mtx));
2303 wih = &wl->wl_inohash[ino & wl->wl_inohashmask];
2312 wapbl_register_inode(struct wapbl *wl, ino_t ino, mode_t mode)
2319 mutex_enter(&wl->wl_mtx);
2320 if (wapbl_inodetrk_get(wl, ino) == NULL) {
2323 wih = &wl->wl_inohash[ino & wl->wl_inohashmask];
2325 wl->wl_inohashcnt++;
2328 mutex_exit(&wl->wl_mtx);
2330 mutex_exit(&wl->wl_mtx);
2336 wapbl_unregister_inode(struct wapbl *wl, ino_t ino, mode_t mode)
2340 mutex_enter(&wl->wl_mtx);
2341 wi = wapbl_inodetrk_get(wl, ino);
2345 KASSERT(wl->wl_inohashcnt > 0);
2346 wl->wl_inohashcnt--;
2348 mutex_exit(&wl->wl_mtx);
2352 mutex_exit(&wl->wl_mtx);
2359 * wapbl_transaction_inodes_len(wl)
2362 * log records in wl.
2365 wapbl_transaction_inodes_len(struct wapbl *wl)
2367 int blocklen = 1<<wl->wl_log_dev_bshift;
2376 return MAX(1, howmany(wl->wl_inohashcnt, iph)) * blocklen;
2381 * wapbl_transaction_len(wl)
2383 * Calculate number of bytes required for all log records in wl.
2386 wapbl_transaction_len(struct wapbl *wl)
2388 int blocklen = 1<<wl->wl_log_dev_bshift;
2392 len = wl->wl_bcount;
2393 len += howmany(wl->wl_bufcount, wl->wl_brperjblock) * blocklen;
2394 len += howmany(wl->wl_dealloccnt, wl->wl_brperjblock) * blocklen;
2395 len += wapbl_transaction_inodes_len(wl);
2401 * wapbl_cache_sync(wl, msg)
2403 * Issue DIOCCACHESYNC to wl->wl_devvp.
2409 wapbl_cache_sync(struct wapbl *wl, const char *msg)
2423 error = VOP_IOCTL(wl->wl_devvp, DIOCCACHESYNC, &force,
2428 "returned %d\n", (uintmax_t)wl->wl_devvp->v_rdev,
2439 msg, (uintmax_t)wl->wl_devvp->v_rdev,
2443 wl->wl_ev_cacheflush.ev_count++;
2449 * wapbl_write_commit(wl, head, tail)
2461 * => Caller must have exclusive access to wl, either by holding
2462 * wl->wl_rwlock for writer or by being wapbl_start before anyone
2463 * else has seen wl.
2466 wapbl_write_commit(struct wapbl *wl, off_t head, off_t tail)
2468 struct wapbl_wc_header *wc = wl->wl_wc_header;
2473 error = wapbl_buffered_flush(wl, true);
2484 wapbl_cache_sync(wl, "1");
2505 pbn = wl->wl_logpbn + (wc->wc_generation % 2);
2509 error = wapbl_buffered_write(wc, wc->wc_len, wl, pbn,
2510 WAPBL_JFLAGS(wl));
2513 error = wapbl_buffered_flush(wl, true);
2522 if (!WAPBL_USE_FUA(wl))
2523 wapbl_cache_sync(wl, "2");
2530 error = wapbl_write_commit(wl, head, tail);
2542 wl->wl_ev_commit.ev_count++;
2548 * wapbl_write_blocks(wl, offp)
2556 wapbl_write_blocks(struct wapbl *wl, off_t *offp)
2559 (struct wapbl_wc_blocklist *)wl->wl_wc_scratch;
2560 int blocklen = 1<<wl->wl_log_dev_bshift;
2566 KASSERT(rw_write_held(&wl->wl_rwlock));
2568 bp = TAILQ_FIRST(&wl->wl_bufs);
2580 while (bp && wc->wc_blkcount < wl->wl_brperjblock) {
2615 error = wapbl_circ_write(wl, wc, blocklen, &off);
2620 while (bp && cnt++ < wl->wl_brperjblock) {
2621 error = wapbl_circ_write(wl, bp->b_data,
2632 error = wapbl_circ_write(wl, zero, padding, &off);
2643 * wapbl_write_revocations(wl, offp)
2651 wapbl_write_revocations(struct wapbl *wl, off_t *offp)
2654 (struct wapbl_wc_blocklist *)wl->wl_wc_scratch;
2656 int blocklen = 1<<wl->wl_log_dev_bshift;
2660 KASSERT(rw_write_held(&wl->wl_rwlock));
2662 if (wl->wl_dealloccnt == 0)
2665 while ((wd = TAILQ_FIRST(&wl->wl_dealloclist)) != NULL) {
2670 while (wd && wc->wc_blkcount < wl->wl_brperjblock) {
2682 error = wapbl_circ_write(wl, wc, blocklen, &off);
2688 while ((wd = TAILQ_FIRST(&wl->wl_dealloclist)) != NULL) {
2691 wapbl_deallocation_free(wl, wd, true);
2699 * wapbl_write_inodes(wl, offp)
2707 wapbl_write_inodes(struct wapbl *wl, off_t *offp)
2710 (struct wapbl_wc_inodelist *)wl->wl_wc_scratch;
2712 int blocklen = 1 << wl->wl_log_dev_bshift;
2724 wih = &wl->wl_inohash[0];
2731 while (i < wl->wl_inohashcnt && wc->wc_inocnt < iph) {
2733 KASSERT((wih - &wl->wl_inohash[0])
2734 <= wl->wl_inohashmask);
2746 error = wapbl_circ_write(wl, wc, blocklen, &off);
2749 } while (i < wl->wl_inohashcnt);