1 /* $NetBSD: vfs_wapbl.c,v 1.113 2024/05/13 00:01:53 msaitoh Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Wasabi Systems, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * This implements file system independent write ahead filesystem logging. 34 */ 35 36 #define WAPBL_INTERNAL 37 38 #include <sys/cdefs.h> 39 __KERNEL_RCSID(0, "$NetBSD: vfs_wapbl.c,v 1.113 2024/05/13 00:01:53 msaitoh Exp $"); 40 41 #include <sys/param.h> 42 #include <sys/bitops.h> 43 #include <sys/time.h> 44 #include <sys/wapbl.h> 45 #include <sys/wapbl_replay.h> 46 47 #ifdef _KERNEL 48 49 #include <sys/atomic.h> 50 #include <sys/conf.h> 51 #include <sys/evcnt.h> 52 #include <sys/file.h> 53 #include <sys/kauth.h> 54 #include <sys/kernel.h> 55 #include <sys/module.h> 56 #include <sys/mount.h> 57 #include <sys/mutex.h> 58 #include <sys/namei.h> 59 #include <sys/proc.h> 60 #include <sys/resourcevar.h> 61 #include <sys/sysctl.h> 62 #include <sys/uio.h> 63 #include <sys/vnode.h> 64 65 #include <miscfs/specfs/specdev.h> 66 67 #define wapbl_alloc(s) kmem_alloc((s), KM_SLEEP) 68 #define wapbl_free(a, s) kmem_free((a), (s)) 69 #define wapbl_calloc(n, s) kmem_zalloc((n)*(s), KM_SLEEP) 70 71 static int wapbl_flush_disk_cache = 1; 72 static int wapbl_verbose_commit = 0; 73 static int wapbl_allow_dpofua = 0; /* switched off by default for now */ 74 static int wapbl_journal_iobufs = 4; 75 76 static inline size_t wapbl_space_free(size_t, off_t, off_t); 77 78 #else /* !_KERNEL */ 79 80 #include <assert.h> 81 #include <errno.h> 82 #include <stdbool.h> 83 #include <stdio.h> 84 #include <stdlib.h> 85 #include <string.h> 86 87 #define KDASSERT(x) assert(x) 88 #define KASSERT(x) assert(x) 89 #define wapbl_alloc(s) malloc(s) 90 #define wapbl_free(a, s) free(a) 91 #define wapbl_calloc(n, s) calloc((n), (s)) 92 93 #endif /* !_KERNEL */ 94 95 /* 96 * INTERNAL DATA STRUCTURES 97 */ 98 99 /* 100 * This structure holds per-mount log information. 101 * 102 * Legend: a = atomic access only 103 * r = read-only after init 104 * l = rwlock held 105 * m = mutex held 106 * lm = rwlock held writing or mutex held 107 * u = unlocked access ok 108 * b = bufcache_lock held 109 */ 110 LIST_HEAD(wapbl_ino_head, wapbl_ino); 111 struct wapbl { 112 struct vnode *wl_logvp; /* r: log here */ 113 struct vnode *wl_devvp; /* r: log on this device */ 114 struct mount *wl_mount; /* r: mountpoint wl is associated with */ 115 daddr_t wl_logpbn; /* r: Physical block number of start of log */ 116 int wl_log_dev_bshift; /* r: logarithm of device block size of log 117 device */ 118 int wl_fs_dev_bshift; /* r: logarithm of device block size of 119 filesystem device */ 120 121 unsigned wl_lock_count; /* m: Count of transactions in progress */ 122 123 size_t wl_circ_size; /* r: Number of bytes in buffer of log */ 124 size_t wl_circ_off; /* r: Number of bytes reserved at start */ 125 126 size_t wl_bufcount_max; /* r: Number of buffers reserved for log */ 127 size_t wl_bufbytes_max; /* r: Number of buf bytes reserved for log */ 128 129 off_t wl_head; /* l: Byte offset of log head */ 130 off_t wl_tail; /* l: Byte offset of log tail */ 131 /* 132 * WAPBL log layout, stored on wl_devvp at wl_logpbn: 133 * 134 * ___________________ wl_circ_size __________________ 135 * / \ 136 * +---------+---------+-------+--------------+--------+ 137 * [ commit0 | commit1 | CCWCW | EEEEEEEEEEEE | CCCWCW ] 138 * +---------+---------+-------+--------------+--------+ 139 * wl_circ_off --^ ^-- wl_head ^-- wl_tail 140 * 141 * commit0 and commit1 are commit headers. A commit header has 142 * a generation number, indicating which of the two headers is 143 * more recent, and an assignment of head and tail pointers. 144 * The rest is a circular queue of log records, starting at 145 * the byte offset wl_circ_off. 146 * 147 * E marks empty space for records. 148 * W marks records for block writes issued but waiting. 149 * C marks completed records. 150 * 151 * wapbl_flush writes new records to empty `E' spaces after 152 * wl_head from the current transaction in memory. 153 * 154 * wapbl_truncate advances wl_tail past any completed `C' 155 * records, freeing them up for use. 156 * 157 * head == tail == 0 means log is empty. 158 * head == tail != 0 means log is full. 159 * 160 * See assertions in wapbl_advance() for other boundary 161 * conditions. 162 * 163 * Only wapbl_flush moves the head, except when wapbl_truncate 164 * sets it to 0 to indicate that the log is empty. 165 * 166 * Only wapbl_truncate moves the tail, except when wapbl_flush 167 * sets it to wl_circ_off to indicate that the log is full. 168 */ 169 170 struct wapbl_wc_header *wl_wc_header; /* l */ 171 void *wl_wc_scratch; /* l: scratch space (XXX: por que?!?) */ 172 173 kmutex_t wl_mtx; /* u: short-term lock */ 174 krwlock_t wl_rwlock; /* u: File system transaction lock */ 175 176 /* 177 * Must be held while accessing 178 * wl_count or wl_bufs or head or tail 179 */ 180 181 #if _KERNEL 182 /* 183 * Callback called from within the flush routine to flush any extra 184 * bits. Note that flush may be skipped without calling this if 185 * there are no outstanding buffers in the transaction. 186 */ 187 wapbl_flush_fn_t wl_flush; /* r */ 188 wapbl_flush_fn_t wl_flush_abort;/* r */ 189 190 /* Event counters */ 191 char wl_ev_group[EVCNT_STRING_MAX]; /* r */ 192 struct evcnt wl_ev_commit; /* l */ 193 struct evcnt wl_ev_journalwrite; /* l */ 194 struct evcnt wl_ev_jbufs_bio_nowait; /* l */ 195 struct evcnt wl_ev_metawrite; /* lm */ 196 struct evcnt wl_ev_cacheflush; /* l */ 197 #endif 198 199 size_t wl_bufbytes; /* m: Byte count of pages in wl_bufs */ 200 size_t wl_bufcount; /* m: Count of buffers in wl_bufs */ 201 size_t wl_bcount; /* m: Total bcount of wl_bufs */ 202 203 TAILQ_HEAD(, buf) wl_bufs; /* m: Buffers in current transaction */ 204 205 kcondvar_t wl_reclaimable_cv; /* m (obviously) */ 206 size_t wl_reclaimable_bytes; /* m: Amount of space available for 207 reclamation by truncate */ 208 int wl_error_count; /* m: # of wl_entries with errors */ 209 size_t wl_reserved_bytes; /* never truncate log smaller than this */ 210 211 #ifdef WAPBL_DEBUG_BUFBYTES 212 size_t wl_unsynced_bufbytes; /* Byte count of unsynced buffers */ 213 #endif 214 215 #if _KERNEL 216 int wl_brperjblock; /* r Block records per journal block */ 217 #endif 218 219 TAILQ_HEAD(, wapbl_dealloc) wl_dealloclist; /* lm: list head */ 220 int wl_dealloccnt; /* lm: total count */ 221 int wl_dealloclim; /* r: max count */ 222 223 /* hashtable of inode numbers for allocated but unlinked inodes */ 224 /* synch ??? */ 225 struct wapbl_ino_head *wl_inohash; 226 u_long wl_inohashmask; 227 int wl_inohashcnt; 228 229 SIMPLEQ_HEAD(, wapbl_entry) wl_entries; /* m: On disk transaction 230 accounting */ 231 232 /* buffers for wapbl_buffered_write() */ 233 TAILQ_HEAD(, buf) wl_iobufs; /* l: Free or filling bufs */ 234 TAILQ_HEAD(, buf) wl_iobufs_busy; /* l: In-transit bufs */ 235 236 int wl_dkcache; /* r: disk cache flags */ 237 #define WAPBL_USE_FUA(wl) \ 238 (wapbl_allow_dpofua && ISSET((wl)->wl_dkcache, DKCACHE_FUA)) 239 #define WAPBL_JFLAGS(wl) \ 240 (WAPBL_USE_FUA(wl) ? (wl)->wl_jwrite_flags : 0) 241 #define WAPBL_JDATA_FLAGS(wl) \ 242 (WAPBL_JFLAGS(wl) & B_MEDIA_DPO) /* only DPO */ 243 int wl_jwrite_flags; /* r: journal write flags */ 244 }; 245 246 #ifdef WAPBL_DEBUG_PRINT 247 int wapbl_debug_print = WAPBL_DEBUG_PRINT; 248 #endif 249 250 /****************************************************************/ 251 #ifdef _KERNEL 252 253 #ifdef WAPBL_DEBUG 254 struct wapbl *wapbl_debug_wl; 255 #endif 256 257 static int wapbl_write_commit(struct wapbl *wl, off_t head, off_t tail); 258 static int wapbl_write_blocks(struct wapbl *wl, off_t *offp); 259 static int wapbl_write_revocations(struct wapbl *wl, off_t *offp); 260 static int wapbl_write_inodes(struct wapbl *wl, off_t *offp); 261 #endif /* _KERNEL */ 262 263 static int wapbl_replay_process(struct wapbl_replay *wr, off_t, off_t); 264 265 static inline size_t wapbl_space_used(size_t avail, off_t head, 266 off_t tail); 267 268 #ifdef _KERNEL 269 270 static struct pool wapbl_entry_pool; 271 static struct pool wapbl_dealloc_pool; 272 273 #define WAPBL_INODETRK_SIZE 83 274 static int wapbl_ino_pool_refcount; 275 static struct pool wapbl_ino_pool; 276 struct wapbl_ino { 277 LIST_ENTRY(wapbl_ino) wi_hash; 278 ino_t wi_ino; 279 mode_t wi_mode; 280 }; 281 282 static void wapbl_inodetrk_init(struct wapbl *wl, u_int size); 283 static void wapbl_inodetrk_free(struct wapbl *wl); 284 static struct wapbl_ino *wapbl_inodetrk_get(struct wapbl *wl, ino_t ino); 285 286 static size_t wapbl_transaction_len(struct wapbl *wl); 287 static inline size_t wapbl_transaction_inodes_len(struct wapbl *wl); 288 289 static void wapbl_deallocation_free(struct wapbl *, struct wapbl_dealloc *, 290 bool); 291 292 static void wapbl_evcnt_init(struct wapbl *); 293 static void wapbl_evcnt_free(struct wapbl *); 294 295 static void wapbl_dkcache_init(struct wapbl *); 296 297 #if 0 298 int wapbl_replay_verify(struct wapbl_replay *, struct vnode *); 299 #endif 300 301 static int wapbl_replay_isopen1(struct wapbl_replay *); 302 303 const struct wapbl_ops wapbl_ops = { 304 .wo_wapbl_discard = wapbl_discard, 305 .wo_wapbl_replay_isopen = wapbl_replay_isopen1, 306 .wo_wapbl_replay_can_read = wapbl_replay_can_read, 307 .wo_wapbl_replay_read = wapbl_replay_read, 308 .wo_wapbl_add_buf = wapbl_add_buf, 309 .wo_wapbl_remove_buf = wapbl_remove_buf, 310 .wo_wapbl_resize_buf = wapbl_resize_buf, 311 .wo_wapbl_begin = wapbl_begin, 312 .wo_wapbl_end = wapbl_end, 313 .wo_wapbl_junlock_assert= wapbl_junlock_assert, 314 .wo_wapbl_jlock_assert = wapbl_jlock_assert, 315 316 /* XXX: the following is only used to say "this is a wapbl buf" */ 317 .wo_wapbl_biodone = wapbl_biodone, 318 }; 319 320 SYSCTL_SETUP(wapbl_sysctl_init, "wapbl sysctl") 321 { 322 int rv; 323 const struct sysctlnode *rnode, *cnode; 324 325 rv = sysctl_createv(clog, 0, NULL, &rnode, 326 CTLFLAG_PERMANENT, 327 CTLTYPE_NODE, "wapbl", 328 SYSCTL_DESCR("WAPBL journaling options"), 329 NULL, 0, NULL, 0, 330 CTL_VFS, CTL_CREATE, CTL_EOL); 331 if (rv) 332 return; 333 334 rv = sysctl_createv(clog, 0, &rnode, &cnode, 335 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 336 CTLTYPE_INT, "flush_disk_cache", 337 SYSCTL_DESCR("flush disk cache"), 338 NULL, 0, &wapbl_flush_disk_cache, 0, 339 CTL_CREATE, CTL_EOL); 340 if (rv) 341 return; 342 343 rv = sysctl_createv(clog, 0, &rnode, &cnode, 344 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 345 CTLTYPE_INT, "verbose_commit", 346 SYSCTL_DESCR("show time and size of wapbl log commits"), 347 NULL, 0, &wapbl_verbose_commit, 0, 348 CTL_CREATE, CTL_EOL); 349 if (rv) 350 return; 351 352 rv = sysctl_createv(clog, 0, &rnode, &cnode, 353 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 354 CTLTYPE_INT, "allow_dpofua", 355 SYSCTL_DESCR("allow use of FUA/DPO instead of cache flush if available"), 356 NULL, 0, &wapbl_allow_dpofua, 0, 357 CTL_CREATE, CTL_EOL); 358 if (rv) 359 return; 360 361 rv = sysctl_createv(clog, 0, &rnode, &cnode, 362 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 363 CTLTYPE_INT, "journal_iobufs", 364 SYSCTL_DESCR("count of bufs used for journal I/O (max async count)"), 365 NULL, 0, &wapbl_journal_iobufs, 0, 366 CTL_CREATE, CTL_EOL); 367 if (rv) 368 return; 369 370 return; 371 } 372 373 static void 374 wapbl_init(void) 375 { 376 377 pool_init(&wapbl_entry_pool, sizeof(struct wapbl_entry), 0, 0, 0, 378 "wapblentrypl", &pool_allocator_kmem, IPL_VM); 379 pool_init(&wapbl_dealloc_pool, sizeof(struct wapbl_dealloc), 0, 0, 0, 380 "wapbldealloc", &pool_allocator_nointr, IPL_NONE); 381 } 382 383 static int 384 wapbl_fini(void) 385 { 386 387 pool_destroy(&wapbl_dealloc_pool); 388 pool_destroy(&wapbl_entry_pool); 389 390 return 0; 391 } 392 393 static void 394 wapbl_evcnt_init(struct wapbl *wl) 395 { 396 snprintf(wl->wl_ev_group, sizeof(wl->wl_ev_group), 397 "wapbl fsid 0x%x/0x%x", 398 wl->wl_mount->mnt_stat.f_fsidx.__fsid_val[0], 399 wl->wl_mount->mnt_stat.f_fsidx.__fsid_val[1] 400 ); 401 402 evcnt_attach_dynamic(&wl->wl_ev_commit, EVCNT_TYPE_MISC, 403 NULL, wl->wl_ev_group, "commit"); 404 evcnt_attach_dynamic(&wl->wl_ev_journalwrite, EVCNT_TYPE_MISC, 405 NULL, wl->wl_ev_group, "journal write total"); 406 evcnt_attach_dynamic(&wl->wl_ev_jbufs_bio_nowait, EVCNT_TYPE_MISC, 407 NULL, wl->wl_ev_group, "journal write finished async"); 408 evcnt_attach_dynamic(&wl->wl_ev_metawrite, EVCNT_TYPE_MISC, 409 NULL, wl->wl_ev_group, "metadata async write"); 410 evcnt_attach_dynamic(&wl->wl_ev_cacheflush, EVCNT_TYPE_MISC, 411 NULL, wl->wl_ev_group, "cache flush"); 412 } 413 414 static void 415 wapbl_evcnt_free(struct wapbl *wl) 416 { 417 evcnt_detach(&wl->wl_ev_commit); 418 evcnt_detach(&wl->wl_ev_journalwrite); 419 evcnt_detach(&wl->wl_ev_jbufs_bio_nowait); 420 evcnt_detach(&wl->wl_ev_metawrite); 421 evcnt_detach(&wl->wl_ev_cacheflush); 422 } 423 424 static void 425 wapbl_dkcache_init(struct wapbl *wl) 426 { 427 int error; 428 429 /* Get disk cache flags */ 430 error = VOP_IOCTL(wl->wl_devvp, DIOCGCACHE, &wl->wl_dkcache, 431 FWRITE, FSCRED); 432 if (error) { 433 /* behave as if there was a write cache */ 434 wl->wl_dkcache = DKCACHE_WRITE; 435 } 436 437 /* Use FUA instead of cache flush if available */ 438 if (ISSET(wl->wl_dkcache, DKCACHE_FUA)) 439 wl->wl_jwrite_flags |= B_MEDIA_FUA; 440 441 /* Use DPO for journal writes if available */ 442 if (ISSET(wl->wl_dkcache, DKCACHE_DPO)) 443 wl->wl_jwrite_flags |= B_MEDIA_DPO; 444 } 445 446 static int 447 wapbl_start_flush_inodes(struct wapbl *wl, struct wapbl_replay *wr) 448 { 449 int error, i; 450 451 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 452 ("wapbl_start: reusing log with %d inodes\n", wr->wr_inodescnt)); 453 454 /* 455 * Its only valid to reuse the replay log if its 456 * the same as the new log we just opened. 457 */ 458 KDASSERT(!wapbl_replay_isopen(wr)); 459 KASSERT(wl->wl_devvp->v_type == VBLK); 460 KASSERT(wr->wr_devvp->v_type == VBLK); 461 KASSERT(wl->wl_devvp->v_rdev == wr->wr_devvp->v_rdev); 462 KASSERT(wl->wl_logpbn == wr->wr_logpbn); 463 KASSERT(wl->wl_circ_size == wr->wr_circ_size); 464 KASSERT(wl->wl_circ_off == wr->wr_circ_off); 465 KASSERT(wl->wl_log_dev_bshift == wr->wr_log_dev_bshift); 466 KASSERT(wl->wl_fs_dev_bshift == wr->wr_fs_dev_bshift); 467 468 wl->wl_wc_header->wc_generation = wr->wr_generation + 1; 469 470 for (i = 0; i < wr->wr_inodescnt; i++) 471 wapbl_register_inode(wl, wr->wr_inodes[i].wr_inumber, 472 wr->wr_inodes[i].wr_imode); 473 474 /* Make sure new transaction won't overwrite old inodes list */ 475 KDASSERT(wapbl_transaction_len(wl) <= 476 wapbl_space_free(wl->wl_circ_size, wr->wr_inodeshead, 477 wr->wr_inodestail)); 478 479 wl->wl_head = wl->wl_tail = wr->wr_inodeshead; 480 wl->wl_reclaimable_bytes = wl->wl_reserved_bytes = 481 wapbl_transaction_len(wl); 482 483 error = wapbl_write_inodes(wl, &wl->wl_head); 484 if (error) 485 return error; 486 487 KASSERT(wl->wl_head != wl->wl_tail); 488 KASSERT(wl->wl_head != 0); 489 490 return 0; 491 } 492 493 int 494 wapbl_start(struct wapbl ** wlp, struct mount *mp, struct vnode *vp, 495 daddr_t off, size_t count, size_t blksize, struct wapbl_replay *wr, 496 wapbl_flush_fn_t flushfn, wapbl_flush_fn_t flushabortfn) 497 { 498 struct wapbl *wl; 499 struct vnode *devvp; 500 daddr_t logpbn; 501 int error; 502 int log_dev_bshift = ilog2(blksize); 503 int fs_dev_bshift = log_dev_bshift; 504 int run; 505 506 WAPBL_PRINTF(WAPBL_PRINT_OPEN, ("wapbl_start: vp=%p off=%" PRId64 507 " count=%zu blksize=%zu\n", vp, off, count, blksize)); 508 509 if (log_dev_bshift > fs_dev_bshift) { 510 WAPBL_PRINTF(WAPBL_PRINT_OPEN, 511 ("wapbl: log device's block size cannot be larger " 512 "than filesystem's\n")); 513 /* 514 * Not currently implemented, although it could be if 515 * needed someday. 516 */ 517 return ENOSYS; 518 } 519 520 if (off < 0) 521 return EINVAL; 522 523 if (blksize < DEV_BSIZE) 524 return EINVAL; 525 if (blksize % DEV_BSIZE) 526 return EINVAL; 527 528 /* XXXTODO: verify that the full load is writable */ 529 530 /* 531 * XXX check for minimum log size 532 * minimum is governed by minimum amount of space 533 * to complete a transaction. (probably truncate) 534 */ 535 /* XXX for now pick something minimal */ 536 if ((count * blksize) < MAXPHYS) { 537 return ENOSPC; 538 } 539 540 if ((error = VOP_BMAP(vp, off, &devvp, &logpbn, &run)) != 0) { 541 return error; 542 } 543 544 wl = wapbl_calloc(1, sizeof(*wl)); 545 rw_init(&wl->wl_rwlock); 546 mutex_init(&wl->wl_mtx, MUTEX_DEFAULT, IPL_NONE); 547 cv_init(&wl->wl_reclaimable_cv, "wapblrec"); 548 TAILQ_INIT(&wl->wl_bufs); 549 SIMPLEQ_INIT(&wl->wl_entries); 550 551 wl->wl_logvp = vp; 552 wl->wl_devvp = devvp; 553 wl->wl_mount = mp; 554 wl->wl_logpbn = logpbn; 555 wl->wl_log_dev_bshift = log_dev_bshift; 556 wl->wl_fs_dev_bshift = fs_dev_bshift; 557 558 wl->wl_flush = flushfn; 559 wl->wl_flush_abort = flushabortfn; 560 561 /* Reserve two log device blocks for the commit headers */ 562 wl->wl_circ_off = 2<<wl->wl_log_dev_bshift; 563 wl->wl_circ_size = ((count * blksize) - wl->wl_circ_off); 564 /* truncate the log usage to a multiple of log_dev_bshift */ 565 wl->wl_circ_size >>= wl->wl_log_dev_bshift; 566 wl->wl_circ_size <<= wl->wl_log_dev_bshift; 567 568 /* 569 * wl_bufbytes_max limits the size of the in memory transaction space. 570 * - Since buffers are allocated and accounted for in units of 571 * PAGE_SIZE it is required to be a multiple of PAGE_SIZE 572 * (i.e. 1<<PAGE_SHIFT) 573 * - Since the log device has to be written in units of 574 * 1<<wl_log_dev_bshift it is required to be a multiple of 575 * 1<<wl_log_dev_bshift. 576 * - Since filesystem will provide data in units of 1<<wl_fs_dev_bshift, 577 * it is convenient to be a multiple of 1<<wl_fs_dev_bshift. 578 * Therefore it must be multiple of the least common multiple of those 579 * three quantities. Fortunately, all of those quantities are 580 * guaranteed to be a power of two, and the least common multiple of 581 * a set of numbers which are all powers of two is simply the maximum 582 * of those numbers. Finally, the maximum logarithm of a power of two 583 * is the same as the log of the maximum power of two. So we can do 584 * the following operations to size wl_bufbytes_max: 585 */ 586 587 /* XXX fix actual number of pages reserved per filesystem. */ 588 wl->wl_bufbytes_max = MIN(wl->wl_circ_size, buf_memcalc() / 2); 589 590 /* Round wl_bufbytes_max to the largest power of two constraint */ 591 wl->wl_bufbytes_max >>= PAGE_SHIFT; 592 wl->wl_bufbytes_max <<= PAGE_SHIFT; 593 wl->wl_bufbytes_max >>= wl->wl_log_dev_bshift; 594 wl->wl_bufbytes_max <<= wl->wl_log_dev_bshift; 595 wl->wl_bufbytes_max >>= wl->wl_fs_dev_bshift; 596 wl->wl_bufbytes_max <<= wl->wl_fs_dev_bshift; 597 598 /* XXX maybe use filesystem fragment size instead of 1024 */ 599 /* XXX fix actual number of buffers reserved per filesystem. */ 600 wl->wl_bufcount_max = (buf_nbuf() / 2) * 1024; 601 602 wl->wl_brperjblock = ((1<<wl->wl_log_dev_bshift) 603 - offsetof(struct wapbl_wc_blocklist, wc_blocks)) / 604 sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); 605 KASSERT(wl->wl_brperjblock > 0); 606 607 /* XXX tie this into resource estimation */ 608 wl->wl_dealloclim = wl->wl_bufbytes_max / mp->mnt_stat.f_bsize / 2; 609 TAILQ_INIT(&wl->wl_dealloclist); 610 611 wapbl_inodetrk_init(wl, WAPBL_INODETRK_SIZE); 612 613 wapbl_evcnt_init(wl); 614 615 wapbl_dkcache_init(wl); 616 617 /* Initialize the commit header */ 618 { 619 struct wapbl_wc_header *wc; 620 size_t len = 1 << wl->wl_log_dev_bshift; 621 wc = wapbl_calloc(1, len); 622 wc->wc_type = WAPBL_WC_HEADER; 623 wc->wc_len = len; 624 wc->wc_circ_off = wl->wl_circ_off; 625 wc->wc_circ_size = wl->wl_circ_size; 626 /* XXX wc->wc_fsid */ 627 wc->wc_log_dev_bshift = wl->wl_log_dev_bshift; 628 wc->wc_fs_dev_bshift = wl->wl_fs_dev_bshift; 629 wl->wl_wc_header = wc; 630 wl->wl_wc_scratch = wapbl_alloc(len); 631 } 632 633 TAILQ_INIT(&wl->wl_iobufs); 634 TAILQ_INIT(&wl->wl_iobufs_busy); 635 for (int i = 0; i < wapbl_journal_iobufs; i++) { 636 struct buf *bp; 637 638 if ((bp = geteblk(MAXPHYS)) == NULL) 639 goto errout; 640 641 mutex_enter(&bufcache_lock); 642 mutex_enter(devvp->v_interlock); 643 bgetvp(devvp, bp); 644 mutex_exit(devvp->v_interlock); 645 mutex_exit(&bufcache_lock); 646 647 bp->b_dev = devvp->v_rdev; 648 649 TAILQ_INSERT_TAIL(&wl->wl_iobufs, bp, b_wapbllist); 650 } 651 652 /* 653 * if there was an existing set of unlinked but 654 * allocated inodes, preserve it in the new 655 * log. 656 */ 657 if (wr && wr->wr_inodescnt) { 658 error = wapbl_start_flush_inodes(wl, wr); 659 if (error) 660 goto errout; 661 } 662 663 error = wapbl_write_commit(wl, wl->wl_head, wl->wl_tail); 664 if (error) { 665 goto errout; 666 } 667 668 *wlp = wl; 669 #if defined(WAPBL_DEBUG) 670 wapbl_debug_wl = wl; 671 #endif 672 673 return 0; 674 errout: 675 wapbl_discard(wl); 676 wapbl_free(wl->wl_wc_scratch, wl->wl_wc_header->wc_len); 677 wapbl_free(wl->wl_wc_header, wl->wl_wc_header->wc_len); 678 while (!TAILQ_EMPTY(&wl->wl_iobufs)) { 679 struct buf *bp; 680 681 bp = TAILQ_FIRST(&wl->wl_iobufs); 682 TAILQ_REMOVE(&wl->wl_iobufs, bp, b_wapbllist); 683 brelse(bp, BC_INVAL); 684 } 685 wapbl_inodetrk_free(wl); 686 wapbl_free(wl, sizeof(*wl)); 687 688 return error; 689 } 690 691 /* 692 * Like wapbl_flush, only discards the transaction 693 * completely 694 */ 695 696 void 697 wapbl_discard(struct wapbl *wl) 698 { 699 struct wapbl_entry *we; 700 struct wapbl_dealloc *wd; 701 struct buf *bp; 702 int i; 703 704 /* 705 * XXX we may consider using upgrade here 706 * if we want to call flush from inside a transaction 707 */ 708 rw_enter(&wl->wl_rwlock, RW_WRITER); 709 wl->wl_flush(wl->wl_mount, TAILQ_FIRST(&wl->wl_dealloclist)); 710 711 #ifdef WAPBL_DEBUG_PRINT 712 { 713 pid_t pid = -1; 714 lwpid_t lid = -1; 715 if (curproc) 716 pid = curproc->p_pid; 717 if (curlwp) 718 lid = curlwp->l_lid; 719 #ifdef WAPBL_DEBUG_BUFBYTES 720 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 721 ("wapbl_discard: thread %d.%d discarding " 722 "transaction\n" 723 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 724 "deallocs=%d inodes=%d\n" 725 "\terrcnt = %u, reclaimable=%zu reserved=%zu " 726 "unsynced=%zu\n", 727 pid, lid, wl->wl_bufcount, wl->wl_bufbytes, 728 wl->wl_bcount, wl->wl_dealloccnt, 729 wl->wl_inohashcnt, wl->wl_error_count, 730 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 731 wl->wl_unsynced_bufbytes)); 732 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 733 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 734 ("\tentry: bufcount = %zu, reclaimable = %zu, " 735 "error = %d, unsynced = %zu\n", 736 we->we_bufcount, we->we_reclaimable_bytes, 737 we->we_error, we->we_unsynced_bufbytes)); 738 } 739 #else /* !WAPBL_DEBUG_BUFBYTES */ 740 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 741 ("wapbl_discard: thread %d.%d discarding transaction\n" 742 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 743 "deallocs=%d inodes=%d\n" 744 "\terrcnt = %u, reclaimable=%zu reserved=%zu\n", 745 pid, lid, wl->wl_bufcount, wl->wl_bufbytes, 746 wl->wl_bcount, wl->wl_dealloccnt, 747 wl->wl_inohashcnt, wl->wl_error_count, 748 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes)); 749 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 750 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 751 ("\tentry: bufcount = %zu, reclaimable = %zu, " 752 "error = %d\n", 753 we->we_bufcount, we->we_reclaimable_bytes, 754 we->we_error)); 755 } 756 #endif /* !WAPBL_DEBUG_BUFBYTES */ 757 } 758 #endif /* WAPBL_DEBUG_PRINT */ 759 760 for (i = 0; i <= wl->wl_inohashmask; i++) { 761 struct wapbl_ino_head *wih; 762 struct wapbl_ino *wi; 763 764 wih = &wl->wl_inohash[i]; 765 while ((wi = LIST_FIRST(wih)) != NULL) { 766 LIST_REMOVE(wi, wi_hash); 767 pool_put(&wapbl_ino_pool, wi); 768 KASSERT(wl->wl_inohashcnt > 0); 769 wl->wl_inohashcnt--; 770 } 771 } 772 773 /* 774 * clean buffer list 775 */ 776 mutex_enter(&bufcache_lock); 777 mutex_enter(&wl->wl_mtx); 778 while ((bp = TAILQ_FIRST(&wl->wl_bufs)) != NULL) { 779 if (bbusy(bp, 0, 0, &wl->wl_mtx) == 0) { 780 KASSERT(bp->b_flags & B_LOCKED); 781 KASSERT(bp->b_oflags & BO_DELWRI); 782 /* 783 * Buffer is already on BQ_LOCKED queue. 784 * The buffer will be unlocked and 785 * removed from the transaction in brelsel() 786 */ 787 mutex_exit(&wl->wl_mtx); 788 bremfree(bp); 789 brelsel(bp, BC_INVAL); 790 mutex_enter(&wl->wl_mtx); 791 } 792 } 793 794 /* 795 * Remove references to this wl from wl_entries, free any which 796 * no longer have buffers, others will be freed in wapbl_biodone() 797 * when they no longer have any buffers. 798 */ 799 while ((we = SIMPLEQ_FIRST(&wl->wl_entries)) != NULL) { 800 SIMPLEQ_REMOVE_HEAD(&wl->wl_entries, we_entries); 801 /* XXX should we be accumulating wl_error_count 802 * and increasing reclaimable bytes ? */ 803 we->we_wapbl = NULL; 804 if (we->we_bufcount == 0) { 805 #ifdef WAPBL_DEBUG_BUFBYTES 806 KASSERT(we->we_unsynced_bufbytes == 0); 807 #endif 808 pool_put(&wapbl_entry_pool, we); 809 } 810 } 811 812 mutex_exit(&wl->wl_mtx); 813 mutex_exit(&bufcache_lock); 814 815 /* Discard list of deallocs */ 816 while ((wd = TAILQ_FIRST(&wl->wl_dealloclist)) != NULL) 817 wapbl_deallocation_free(wl, wd, true); 818 819 /* XXX should we clear wl_reserved_bytes? */ 820 821 KASSERT(wl->wl_bufbytes == 0); 822 KASSERT(wl->wl_bcount == 0); 823 KASSERT(wl->wl_bufcount == 0); 824 KASSERT(TAILQ_EMPTY(&wl->wl_bufs)); 825 KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); 826 KASSERT(wl->wl_inohashcnt == 0); 827 KASSERT(TAILQ_EMPTY(&wl->wl_dealloclist)); 828 KASSERT(wl->wl_dealloccnt == 0); 829 830 rw_exit(&wl->wl_rwlock); 831 } 832 833 int 834 wapbl_stop(struct wapbl *wl, int force) 835 { 836 int error; 837 838 WAPBL_PRINTF(WAPBL_PRINT_OPEN, ("wapbl_stop called\n")); 839 error = wapbl_flush(wl, 1); 840 if (error) { 841 if (force) 842 wapbl_discard(wl); 843 else 844 return error; 845 } 846 847 /* Unlinked inodes persist after a flush */ 848 if (wl->wl_inohashcnt) { 849 if (force) { 850 wapbl_discard(wl); 851 } else { 852 return EBUSY; 853 } 854 } 855 856 KASSERT(wl->wl_bufbytes == 0); 857 KASSERT(wl->wl_bcount == 0); 858 KASSERT(wl->wl_bufcount == 0); 859 KASSERT(TAILQ_EMPTY(&wl->wl_bufs)); 860 KASSERT(wl->wl_dealloccnt == 0); 861 KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); 862 KASSERT(wl->wl_inohashcnt == 0); 863 KASSERT(TAILQ_EMPTY(&wl->wl_dealloclist)); 864 KASSERT(wl->wl_dealloccnt == 0); 865 KASSERT(TAILQ_EMPTY(&wl->wl_iobufs_busy)); 866 867 wapbl_free(wl->wl_wc_scratch, wl->wl_wc_header->wc_len); 868 wapbl_free(wl->wl_wc_header, wl->wl_wc_header->wc_len); 869 while (!TAILQ_EMPTY(&wl->wl_iobufs)) { 870 struct buf *bp; 871 872 bp = TAILQ_FIRST(&wl->wl_iobufs); 873 TAILQ_REMOVE(&wl->wl_iobufs, bp, b_wapbllist); 874 brelse(bp, BC_INVAL); 875 } 876 wapbl_inodetrk_free(wl); 877 878 wapbl_evcnt_free(wl); 879 880 cv_destroy(&wl->wl_reclaimable_cv); 881 mutex_destroy(&wl->wl_mtx); 882 rw_destroy(&wl->wl_rwlock); 883 wapbl_free(wl, sizeof(*wl)); 884 885 return 0; 886 } 887 888 /****************************************************************/ 889 /* 890 * Unbuffered disk I/O 891 */ 892 893 static void 894 wapbl_doio_accounting(struct vnode *devvp, int flags) 895 { 896 struct pstats *pstats = curlwp->l_proc->p_stats; 897 898 if ((flags & (B_WRITE | B_READ)) == B_WRITE) { 899 mutex_enter(devvp->v_interlock); 900 devvp->v_numoutput++; 901 mutex_exit(devvp->v_interlock); 902 pstats->p_ru.ru_oublock++; 903 } else { 904 pstats->p_ru.ru_inblock++; 905 } 906 907 } 908 909 static int 910 wapbl_doio(void *data, size_t len, struct vnode *devvp, daddr_t pbn, int flags) 911 { 912 struct buf *bp; 913 int error; 914 915 KASSERT(devvp->v_type == VBLK); 916 917 wapbl_doio_accounting(devvp, flags); 918 919 bp = getiobuf(devvp, true); 920 bp->b_flags = flags; 921 bp->b_cflags |= BC_BUSY; /* mandatory, asserted by biowait() */ 922 bp->b_dev = devvp->v_rdev; 923 bp->b_data = data; 924 bp->b_bufsize = bp->b_resid = bp->b_bcount = len; 925 bp->b_blkno = pbn; 926 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 927 928 WAPBL_PRINTF(WAPBL_PRINT_IO, 929 ("wapbl_doio: %s %d bytes at block %"PRId64" on dev 0x%"PRIx64"\n", 930 BUF_ISWRITE(bp) ? "write" : "read", bp->b_bcount, 931 bp->b_blkno, bp->b_dev)); 932 933 VOP_STRATEGY(devvp, bp); 934 935 error = biowait(bp); 936 putiobuf(bp); 937 938 if (error) { 939 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 940 ("wapbl_doio: %s %zu bytes at block %" PRId64 941 " on dev 0x%"PRIx64" failed with error %d\n", 942 (((flags & (B_WRITE | B_READ)) == B_WRITE) ? 943 "write" : "read"), 944 len, pbn, devvp->v_rdev, error)); 945 } 946 947 return error; 948 } 949 950 /* 951 * wapbl_write(data, len, devvp, pbn) 952 * 953 * Synchronously write len bytes from data to physical block pbn 954 * on devvp. 955 */ 956 int 957 wapbl_write(void *data, size_t len, struct vnode *devvp, daddr_t pbn) 958 { 959 960 return wapbl_doio(data, len, devvp, pbn, B_WRITE); 961 } 962 963 /* 964 * wapbl_read(data, len, devvp, pbn) 965 * 966 * Synchronously read len bytes into data from physical block pbn 967 * on devvp. 968 */ 969 int 970 wapbl_read(void *data, size_t len, struct vnode *devvp, daddr_t pbn) 971 { 972 973 return wapbl_doio(data, len, devvp, pbn, B_READ); 974 } 975 976 /****************************************************************/ 977 /* 978 * Buffered disk writes -- try to coalesce writes and emit 979 * MAXPHYS-aligned blocks. 980 */ 981 982 /* 983 * wapbl_buffered_write_async(wl, bp) 984 * 985 * Send buffer for asynchronous write. 986 */ 987 static void 988 wapbl_buffered_write_async(struct wapbl *wl, struct buf *bp) 989 { 990 wapbl_doio_accounting(wl->wl_devvp, bp->b_flags); 991 992 KASSERT(TAILQ_FIRST(&wl->wl_iobufs) == bp); 993 TAILQ_REMOVE(&wl->wl_iobufs, bp, b_wapbllist); 994 995 bp->b_flags |= B_WRITE; 996 bp->b_cflags |= BC_BUSY; /* mandatory, asserted by biowait() */ 997 bp->b_oflags = 0; 998 bp->b_bcount = bp->b_resid; 999 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1000 1001 VOP_STRATEGY(wl->wl_devvp, bp); 1002 1003 wl->wl_ev_journalwrite.ev_count++; 1004 1005 TAILQ_INSERT_TAIL(&wl->wl_iobufs_busy, bp, b_wapbllist); 1006 } 1007 1008 /* 1009 * wapbl_buffered_flush(wl) 1010 * 1011 * Flush any buffered writes from wapbl_buffered_write. 1012 */ 1013 static int 1014 wapbl_buffered_flush(struct wapbl *wl, bool full) 1015 { 1016 int error = 0; 1017 struct buf *bp, *bnext; 1018 bool only_done = true, found = false; 1019 1020 /* if there is outstanding buffered write, send it now */ 1021 if ((bp = TAILQ_FIRST(&wl->wl_iobufs)) && bp->b_resid > 0) 1022 wapbl_buffered_write_async(wl, bp); 1023 1024 /* wait for I/O to complete */ 1025 again: 1026 TAILQ_FOREACH_SAFE(bp, &wl->wl_iobufs_busy, b_wapbllist, bnext) { 1027 if (!full && only_done) { 1028 /* skip unfinished */ 1029 if (!ISSET(bp->b_oflags, BO_DONE)) 1030 continue; 1031 } 1032 1033 if (ISSET(bp->b_oflags, BO_DONE)) 1034 wl->wl_ev_jbufs_bio_nowait.ev_count++; 1035 1036 TAILQ_REMOVE(&wl->wl_iobufs_busy, bp, b_wapbllist); 1037 error = biowait(bp); 1038 1039 /* reset for reuse */ 1040 bp->b_blkno = bp->b_resid = bp->b_flags = 0; 1041 TAILQ_INSERT_TAIL(&wl->wl_iobufs, bp, b_wapbllist); 1042 found = true; 1043 1044 if (!full) 1045 break; 1046 } 1047 1048 if (!found && only_done && !TAILQ_EMPTY(&wl->wl_iobufs_busy)) { 1049 only_done = false; 1050 goto again; 1051 } 1052 1053 return error; 1054 } 1055 1056 /* 1057 * wapbl_buffered_write(data, len, wl, pbn) 1058 * 1059 * Write len bytes from data to physical block pbn on 1060 * wl->wl_devvp. The write may not complete until 1061 * wapbl_buffered_flush. 1062 */ 1063 static int 1064 wapbl_buffered_write(void *data, size_t len, struct wapbl *wl, daddr_t pbn, 1065 int bflags) 1066 { 1067 size_t resid; 1068 struct buf *bp; 1069 1070 again: 1071 bp = TAILQ_FIRST(&wl->wl_iobufs); 1072 1073 if (bp == NULL) { 1074 /* No more buffers, wait for any previous I/O to finish. */ 1075 wapbl_buffered_flush(wl, false); 1076 1077 bp = TAILQ_FIRST(&wl->wl_iobufs); 1078 KASSERT(bp != NULL); 1079 } 1080 1081 /* 1082 * If not adjacent to buffered data flush first. Disk block 1083 * address is always valid for non-empty buffer. 1084 */ 1085 if ((bp->b_resid > 0 && pbn != bp->b_blkno + btodb(bp->b_resid))) { 1086 wapbl_buffered_write_async(wl, bp); 1087 goto again; 1088 } 1089 1090 /* 1091 * If this write goes to an empty buffer we have to 1092 * save the disk block address first. 1093 */ 1094 if (bp->b_blkno == 0) { 1095 bp->b_blkno = pbn; 1096 bp->b_flags |= bflags; 1097 } 1098 1099 /* 1100 * Remaining space so this buffer ends on a buffer size boundary. 1101 * 1102 * Cannot become less or equal zero as the buffer would have been 1103 * flushed on the last call then. 1104 */ 1105 resid = bp->b_bufsize - dbtob(bp->b_blkno % btodb(bp->b_bufsize)) - 1106 bp->b_resid; 1107 KASSERT(resid > 0); 1108 KASSERT(dbtob(btodb(resid)) == resid); 1109 1110 if (len < resid) 1111 resid = len; 1112 1113 memcpy((uint8_t *)bp->b_data + bp->b_resid, data, resid); 1114 bp->b_resid += resid; 1115 1116 if (len >= resid) { 1117 /* Just filled the buf, or data did not fit */ 1118 wapbl_buffered_write_async(wl, bp); 1119 1120 data = (uint8_t *)data + resid; 1121 len -= resid; 1122 pbn += btodb(resid); 1123 1124 if (len > 0) 1125 goto again; 1126 } 1127 1128 return 0; 1129 } 1130 1131 /* 1132 * wapbl_circ_write(wl, data, len, offp) 1133 * 1134 * Write len bytes from data to the circular queue of wl, starting 1135 * at linear byte offset *offp, and returning the new linear byte 1136 * offset in *offp. 1137 * 1138 * If the starting linear byte offset precedes wl->wl_circ_off, 1139 * the write instead begins at wl->wl_circ_off. XXX WTF? This 1140 * should be a KASSERT, not a conditional. 1141 * 1142 * The write is buffered in wl and must be flushed with 1143 * wapbl_buffered_flush before it will be submitted to the disk. 1144 */ 1145 static int 1146 wapbl_circ_write(struct wapbl *wl, void *data, size_t len, off_t *offp) 1147 { 1148 size_t slen; 1149 off_t off = *offp; 1150 int error; 1151 daddr_t pbn; 1152 1153 KDASSERT(((len >> wl->wl_log_dev_bshift) << 1154 wl->wl_log_dev_bshift) == len); 1155 1156 if (off < wl->wl_circ_off) 1157 off = wl->wl_circ_off; 1158 slen = wl->wl_circ_off + wl->wl_circ_size - off; 1159 if (slen < len) { 1160 pbn = wl->wl_logpbn + (off >> wl->wl_log_dev_bshift); 1161 #ifdef _KERNEL 1162 pbn = btodb(pbn << wl->wl_log_dev_bshift); 1163 #endif 1164 error = wapbl_buffered_write(data, slen, wl, pbn, 1165 WAPBL_JDATA_FLAGS(wl)); 1166 if (error) 1167 return error; 1168 data = (uint8_t *)data + slen; 1169 len -= slen; 1170 off = wl->wl_circ_off; 1171 } 1172 pbn = wl->wl_logpbn + (off >> wl->wl_log_dev_bshift); 1173 #ifdef _KERNEL 1174 pbn = btodb(pbn << wl->wl_log_dev_bshift); 1175 #endif 1176 error = wapbl_buffered_write(data, len, wl, pbn, 1177 WAPBL_JDATA_FLAGS(wl)); 1178 if (error) 1179 return error; 1180 off += len; 1181 if (off >= wl->wl_circ_off + wl->wl_circ_size) 1182 off = wl->wl_circ_off; 1183 *offp = off; 1184 return 0; 1185 } 1186 1187 /****************************************************************/ 1188 /* 1189 * WAPBL transactions: entering, adding/removing bufs, and exiting 1190 */ 1191 1192 int 1193 wapbl_begin(struct wapbl *wl, const char *file, int line) 1194 { 1195 int doflush; 1196 unsigned lockcount; 1197 1198 KDASSERT(wl); 1199 1200 /* 1201 * XXX this needs to be made much more sophisticated. 1202 * perhaps each wapbl_begin could reserve a specified 1203 * number of buffers and bytes. 1204 */ 1205 mutex_enter(&wl->wl_mtx); 1206 lockcount = wl->wl_lock_count; 1207 doflush = ((wl->wl_bufbytes + (lockcount * MAXPHYS)) > 1208 wl->wl_bufbytes_max / 2) || 1209 ((wl->wl_bufcount + (lockcount * 10)) > 1210 wl->wl_bufcount_max / 2) || 1211 (wapbl_transaction_len(wl) > wl->wl_circ_size / 2) || 1212 (wl->wl_dealloccnt >= (wl->wl_dealloclim / 2)); 1213 mutex_exit(&wl->wl_mtx); 1214 1215 if (doflush) { 1216 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1217 ("force flush lockcnt=%d bufbytes=%zu " 1218 "(max=%zu) bufcount=%zu (max=%zu) " 1219 "dealloccnt %d (lim=%d)\n", 1220 lockcount, wl->wl_bufbytes, 1221 wl->wl_bufbytes_max, wl->wl_bufcount, 1222 wl->wl_bufcount_max, 1223 wl->wl_dealloccnt, wl->wl_dealloclim)); 1224 } 1225 1226 if (doflush) { 1227 int error = wapbl_flush(wl, 0); 1228 if (error) 1229 return error; 1230 } 1231 1232 rw_enter(&wl->wl_rwlock, RW_READER); 1233 mutex_enter(&wl->wl_mtx); 1234 wl->wl_lock_count++; 1235 mutex_exit(&wl->wl_mtx); 1236 1237 #if defined(WAPBL_DEBUG_PRINT) 1238 WAPBL_PRINTF(WAPBL_PRINT_TRANSACTION, 1239 ("wapbl_begin thread %d.%d with bufcount=%zu " 1240 "bufbytes=%zu bcount=%zu at %s:%d\n", 1241 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 1242 wl->wl_bufbytes, wl->wl_bcount, file, line)); 1243 #endif 1244 1245 return 0; 1246 } 1247 1248 void 1249 wapbl_end(struct wapbl *wl) 1250 { 1251 1252 #if defined(WAPBL_DEBUG_PRINT) 1253 WAPBL_PRINTF(WAPBL_PRINT_TRANSACTION, 1254 ("wapbl_end thread %d.%d with bufcount=%zu " 1255 "bufbytes=%zu bcount=%zu\n", 1256 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 1257 wl->wl_bufbytes, wl->wl_bcount)); 1258 #endif 1259 1260 /* 1261 * XXX this could be handled more gracefully, perhaps place 1262 * only a partial transaction in the log and allow the 1263 * remaining to flush without the protection of the journal. 1264 */ 1265 KASSERTMSG((wapbl_transaction_len(wl) <= 1266 (wl->wl_circ_size - wl->wl_reserved_bytes)), 1267 "wapbl_end: current transaction too big to flush"); 1268 1269 mutex_enter(&wl->wl_mtx); 1270 KASSERT(wl->wl_lock_count > 0); 1271 wl->wl_lock_count--; 1272 mutex_exit(&wl->wl_mtx); 1273 1274 rw_exit(&wl->wl_rwlock); 1275 } 1276 1277 void 1278 wapbl_add_buf(struct wapbl *wl, struct buf * bp) 1279 { 1280 1281 KASSERT(bp->b_cflags & BC_BUSY); 1282 KASSERT(bp->b_vp); 1283 1284 wapbl_jlock_assert(wl); 1285 1286 #if 0 1287 /* 1288 * XXX this might be an issue for swapfiles. 1289 * see uvm_swap.c:1702 1290 * 1291 * XXX2 why require it then? leap of semantics? 1292 */ 1293 KASSERT((bp->b_cflags & BC_NOCACHE) == 0); 1294 #endif 1295 1296 mutex_enter(&wl->wl_mtx); 1297 if (bp->b_flags & B_LOCKED) { 1298 TAILQ_REMOVE(&wl->wl_bufs, bp, b_wapbllist); 1299 WAPBL_PRINTF(WAPBL_PRINT_BUFFER2, 1300 ("wapbl_add_buf thread %d.%d re-adding buf %p " 1301 "with %d bytes %d bcount\n", 1302 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, 1303 bp->b_bcount)); 1304 } else { 1305 /* unlocked by dirty buffers shouldn't exist */ 1306 KASSERT(!(bp->b_oflags & BO_DELWRI)); 1307 wl->wl_bufbytes += bp->b_bufsize; 1308 wl->wl_bcount += bp->b_bcount; 1309 wl->wl_bufcount++; 1310 WAPBL_PRINTF(WAPBL_PRINT_BUFFER, 1311 ("wapbl_add_buf thread %d.%d adding buf %p " 1312 "with %d bytes %d bcount\n", 1313 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, 1314 bp->b_bcount)); 1315 } 1316 TAILQ_INSERT_TAIL(&wl->wl_bufs, bp, b_wapbllist); 1317 mutex_exit(&wl->wl_mtx); 1318 1319 bp->b_flags |= B_LOCKED; 1320 } 1321 1322 static void 1323 wapbl_remove_buf_locked(struct wapbl * wl, struct buf *bp) 1324 { 1325 1326 KASSERT(mutex_owned(&wl->wl_mtx)); 1327 KASSERT(bp->b_cflags & BC_BUSY); 1328 wapbl_jlock_assert(wl); 1329 1330 #if 0 1331 /* 1332 * XXX this might be an issue for swapfiles. 1333 * see uvm_swap.c:1725 1334 * 1335 * XXXdeux: see above 1336 */ 1337 KASSERT((bp->b_flags & BC_NOCACHE) == 0); 1338 #endif 1339 KASSERT(bp->b_flags & B_LOCKED); 1340 1341 WAPBL_PRINTF(WAPBL_PRINT_BUFFER, 1342 ("wapbl_remove_buf thread %d.%d removing buf %p with " 1343 "%d bytes %d bcount\n", 1344 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, bp->b_bcount)); 1345 1346 KASSERT(wl->wl_bufbytes >= bp->b_bufsize); 1347 wl->wl_bufbytes -= bp->b_bufsize; 1348 KASSERT(wl->wl_bcount >= bp->b_bcount); 1349 wl->wl_bcount -= bp->b_bcount; 1350 KASSERT(wl->wl_bufcount > 0); 1351 wl->wl_bufcount--; 1352 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0)); 1353 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0)); 1354 TAILQ_REMOVE(&wl->wl_bufs, bp, b_wapbllist); 1355 1356 bp->b_flags &= ~B_LOCKED; 1357 } 1358 1359 /* called from brelsel() in vfs_bio among other places */ 1360 void 1361 wapbl_remove_buf(struct wapbl * wl, struct buf *bp) 1362 { 1363 1364 mutex_enter(&wl->wl_mtx); 1365 wapbl_remove_buf_locked(wl, bp); 1366 mutex_exit(&wl->wl_mtx); 1367 } 1368 1369 void 1370 wapbl_resize_buf(struct wapbl *wl, struct buf *bp, long oldsz, long oldcnt) 1371 { 1372 1373 KASSERT(bp->b_cflags & BC_BUSY); 1374 1375 /* 1376 * XXX: why does this depend on B_LOCKED? otherwise the buf 1377 * is not for a transaction? if so, why is this called in the 1378 * first place? 1379 */ 1380 if (bp->b_flags & B_LOCKED) { 1381 mutex_enter(&wl->wl_mtx); 1382 wl->wl_bufbytes += bp->b_bufsize - oldsz; 1383 wl->wl_bcount += bp->b_bcount - oldcnt; 1384 mutex_exit(&wl->wl_mtx); 1385 } 1386 } 1387 1388 #endif /* _KERNEL */ 1389 1390 /****************************************************************/ 1391 /* Some utility inlines */ 1392 1393 /* 1394 * wapbl_space_used(avail, head, tail) 1395 * 1396 * Number of bytes used in a circular queue of avail total bytes, 1397 * from tail to head. 1398 */ 1399 static inline size_t 1400 wapbl_space_used(size_t avail, off_t head, off_t tail) 1401 { 1402 1403 if (tail == 0) { 1404 KASSERT(head == 0); 1405 return 0; 1406 } 1407 return ((head + (avail - 1) - tail) % avail) + 1; 1408 } 1409 1410 #ifdef _KERNEL 1411 /* 1412 * wapbl_advance(size, off, oldoff, delta) 1413 * 1414 * Given a byte offset oldoff into a circular queue of size bytes 1415 * starting at off, return a new byte offset oldoff + delta into 1416 * the circular queue. 1417 */ 1418 static inline off_t 1419 wapbl_advance(size_t size, size_t off, off_t oldoff, size_t delta) 1420 { 1421 off_t newoff; 1422 1423 /* Define acceptable ranges for inputs. */ 1424 KASSERT(delta <= (size_t)size); 1425 KASSERT((oldoff == 0) || ((size_t)oldoff >= off)); 1426 KASSERT(oldoff < (off_t)(size + off)); 1427 1428 if ((oldoff == 0) && (delta != 0)) 1429 newoff = off + delta; 1430 else if ((oldoff + delta) < (size + off)) 1431 newoff = oldoff + delta; 1432 else 1433 newoff = (oldoff + delta) - size; 1434 1435 /* Note some interesting axioms */ 1436 KASSERT((delta != 0) || (newoff == oldoff)); 1437 KASSERT((delta == 0) || (newoff != 0)); 1438 KASSERT((delta != (size)) || (newoff == oldoff)); 1439 1440 /* Define acceptable ranges for output. */ 1441 KASSERT((newoff == 0) || ((size_t)newoff >= off)); 1442 KASSERT((size_t)newoff < (size + off)); 1443 return newoff; 1444 } 1445 1446 /* 1447 * wapbl_space_free(avail, head, tail) 1448 * 1449 * Number of bytes free in a circular queue of avail total bytes, 1450 * in which everything from tail to head is used. 1451 */ 1452 static inline size_t 1453 wapbl_space_free(size_t avail, off_t head, off_t tail) 1454 { 1455 1456 return avail - wapbl_space_used(avail, head, tail); 1457 } 1458 1459 /* 1460 * wapbl_advance_head(size, off, delta, headp, tailp) 1461 * 1462 * In a circular queue of size bytes starting at off, given the 1463 * old head and tail offsets *headp and *tailp, store the new head 1464 * and tail offsets in *headp and *tailp resulting from adding 1465 * delta bytes of data to the head. 1466 */ 1467 static inline void 1468 wapbl_advance_head(size_t size, size_t off, size_t delta, off_t *headp, 1469 off_t *tailp) 1470 { 1471 off_t head = *headp; 1472 off_t tail = *tailp; 1473 1474 KASSERT(delta <= wapbl_space_free(size, head, tail)); 1475 head = wapbl_advance(size, off, head, delta); 1476 if ((tail == 0) && (head != 0)) 1477 tail = off; 1478 *headp = head; 1479 *tailp = tail; 1480 } 1481 1482 /* 1483 * wapbl_advance_tail(size, off, delta, headp, tailp) 1484 * 1485 * In a circular queue of size bytes starting at off, given the 1486 * old head and tail offsets *headp and *tailp, store the new head 1487 * and tail offsets in *headp and *tailp resulting from removing 1488 * delta bytes of data from the tail. 1489 */ 1490 static inline void 1491 wapbl_advance_tail(size_t size, size_t off, size_t delta, off_t *headp, 1492 off_t *tailp) 1493 { 1494 off_t head = *headp; 1495 off_t tail = *tailp; 1496 1497 KASSERT(delta <= wapbl_space_used(size, head, tail)); 1498 tail = wapbl_advance(size, off, tail, delta); 1499 if (head == tail) { 1500 head = tail = 0; 1501 } 1502 *headp = head; 1503 *tailp = tail; 1504 } 1505 1506 1507 /****************************************************************/ 1508 1509 /* 1510 * wapbl_truncate(wl, minfree) 1511 * 1512 * Wait until at least minfree bytes are available in the log. 1513 * 1514 * If it was necessary to wait for writes to complete, 1515 * advance the circular queue tail to reflect the new write 1516 * completions and issue a write commit to the log. 1517 * 1518 * => Caller must hold wl->wl_rwlock writer lock. 1519 */ 1520 static int 1521 wapbl_truncate(struct wapbl *wl, size_t minfree) 1522 { 1523 size_t delta; 1524 size_t avail; 1525 off_t head; 1526 off_t tail; 1527 int error = 0; 1528 1529 KASSERT(minfree <= (wl->wl_circ_size - wl->wl_reserved_bytes)); 1530 KASSERT(rw_write_held(&wl->wl_rwlock)); 1531 1532 mutex_enter(&wl->wl_mtx); 1533 1534 /* 1535 * First check to see if we have to do a commit 1536 * at all. 1537 */ 1538 avail = wapbl_space_free(wl->wl_circ_size, wl->wl_head, wl->wl_tail); 1539 if (minfree < avail) { 1540 mutex_exit(&wl->wl_mtx); 1541 return 0; 1542 } 1543 minfree -= avail; 1544 while ((wl->wl_error_count == 0) && 1545 (wl->wl_reclaimable_bytes < minfree)) { 1546 WAPBL_PRINTF(WAPBL_PRINT_TRUNCATE, 1547 ("wapbl_truncate: sleeping on %p wl=%p bytes=%zd " 1548 "minfree=%zd\n", 1549 &wl->wl_reclaimable_bytes, wl, wl->wl_reclaimable_bytes, 1550 minfree)); 1551 1552 cv_wait(&wl->wl_reclaimable_cv, &wl->wl_mtx); 1553 } 1554 if (wl->wl_reclaimable_bytes < minfree) { 1555 KASSERT(wl->wl_error_count); 1556 /* XXX maybe get actual error from buffer instead someday? */ 1557 error = EIO; 1558 } 1559 head = wl->wl_head; 1560 tail = wl->wl_tail; 1561 delta = wl->wl_reclaimable_bytes; 1562 1563 /* If all of the entries are flushed, then be sure to keep 1564 * the reserved bytes reserved. Watch out for discarded transactions, 1565 * which could leave more bytes reserved than are reclaimable. 1566 */ 1567 if (SIMPLEQ_EMPTY(&wl->wl_entries) && 1568 (delta >= wl->wl_reserved_bytes)) { 1569 delta -= wl->wl_reserved_bytes; 1570 } 1571 wapbl_advance_tail(wl->wl_circ_size, wl->wl_circ_off, delta, &head, 1572 &tail); 1573 KDASSERT(wl->wl_reserved_bytes <= 1574 wapbl_space_used(wl->wl_circ_size, head, tail)); 1575 mutex_exit(&wl->wl_mtx); 1576 1577 if (error) 1578 return error; 1579 1580 /* 1581 * This is where head, tail and delta are unprotected 1582 * from races against itself or flush. This is ok since 1583 * we only call this routine from inside flush itself. 1584 * 1585 * XXX: how can it race against itself when accessed only 1586 * from behind the write-locked rwlock? 1587 */ 1588 error = wapbl_write_commit(wl, head, tail); 1589 if (error) 1590 return error; 1591 1592 wl->wl_head = head; 1593 wl->wl_tail = tail; 1594 1595 mutex_enter(&wl->wl_mtx); 1596 KASSERT(wl->wl_reclaimable_bytes >= delta); 1597 wl->wl_reclaimable_bytes -= delta; 1598 mutex_exit(&wl->wl_mtx); 1599 WAPBL_PRINTF(WAPBL_PRINT_TRUNCATE, 1600 ("wapbl_truncate thread %d.%d truncating %zu bytes\n", 1601 curproc->p_pid, curlwp->l_lid, delta)); 1602 1603 return 0; 1604 } 1605 1606 /****************************************************************/ 1607 1608 void 1609 wapbl_biodone(struct buf *bp) 1610 { 1611 struct wapbl_entry *we = bp->b_private; 1612 struct wapbl *wl; 1613 #ifdef WAPBL_DEBUG_BUFBYTES 1614 const int bufsize = bp->b_bufsize; 1615 #endif 1616 1617 mutex_enter(&bufcache_lock); 1618 wl = we->we_wapbl; 1619 mutex_exit(&bufcache_lock); 1620 1621 /* 1622 * Handle possible flushing of buffers after log has been 1623 * decomissioned. 1624 */ 1625 if (!wl) { 1626 KASSERT(we->we_bufcount > 0); 1627 we->we_bufcount--; 1628 #ifdef WAPBL_DEBUG_BUFBYTES 1629 KASSERT(we->we_unsynced_bufbytes >= bufsize); 1630 we->we_unsynced_bufbytes -= bufsize; 1631 #endif 1632 1633 if (we->we_bufcount == 0) { 1634 #ifdef WAPBL_DEBUG_BUFBYTES 1635 KASSERT(we->we_unsynced_bufbytes == 0); 1636 #endif 1637 pool_put(&wapbl_entry_pool, we); 1638 } 1639 1640 brelse(bp, 0); 1641 return; 1642 } 1643 1644 #ifdef ohbother 1645 KDASSERT(bp->b_oflags & BO_DONE); 1646 KDASSERT(!(bp->b_oflags & BO_DELWRI)); 1647 KDASSERT(bp->b_flags & B_ASYNC); 1648 KDASSERT(bp->b_cflags & BC_BUSY); 1649 KDASSERT(!(bp->b_flags & B_LOCKED)); 1650 KDASSERT(!(bp->b_flags & B_READ)); 1651 KDASSERT(!(bp->b_cflags & BC_INVAL)); 1652 KDASSERT(!(bp->b_cflags & BC_NOCACHE)); 1653 #endif 1654 1655 if (bp->b_error) { 1656 /* 1657 * If an error occurs, it would be nice to leave the buffer 1658 * as a delayed write on the LRU queue so that we can retry 1659 * it later. But buffercache(9) can't handle dirty buffer 1660 * reuse, so just mark the log permanently errored out. 1661 */ 1662 mutex_enter(&wl->wl_mtx); 1663 if (wl->wl_error_count == 0) { 1664 wl->wl_error_count++; 1665 cv_broadcast(&wl->wl_reclaimable_cv); 1666 } 1667 mutex_exit(&wl->wl_mtx); 1668 } 1669 1670 /* 1671 * Make sure that the buf doesn't retain the media flags, so that 1672 * e.g. wapbl_allow_fuadpo has immediate effect on any following I/O. 1673 * The flags will be set again if needed by another I/O. 1674 */ 1675 bp->b_flags &= ~B_MEDIA_FLAGS; 1676 1677 /* 1678 * Release the buffer here. wapbl_flush() may wait for the 1679 * log to become empty and we better unbusy the buffer before 1680 * wapbl_flush() returns. 1681 */ 1682 brelse(bp, 0); 1683 1684 mutex_enter(&wl->wl_mtx); 1685 1686 KASSERT(we->we_bufcount > 0); 1687 we->we_bufcount--; 1688 #ifdef WAPBL_DEBUG_BUFBYTES 1689 KASSERT(we->we_unsynced_bufbytes >= bufsize); 1690 we->we_unsynced_bufbytes -= bufsize; 1691 KASSERT(wl->wl_unsynced_bufbytes >= bufsize); 1692 wl->wl_unsynced_bufbytes -= bufsize; 1693 #endif 1694 wl->wl_ev_metawrite.ev_count++; 1695 1696 /* 1697 * If the current transaction can be reclaimed, start 1698 * at the beginning and reclaim any consecutive reclaimable 1699 * transactions. If we successfully reclaim anything, 1700 * then wakeup anyone waiting for the reclaim. 1701 */ 1702 if (we->we_bufcount == 0) { 1703 size_t delta = 0; 1704 int errcnt = 0; 1705 #ifdef WAPBL_DEBUG_BUFBYTES 1706 KDASSERT(we->we_unsynced_bufbytes == 0); 1707 #endif 1708 /* 1709 * clear any posted error, since the buffer it came from 1710 * has successfully flushed by now 1711 */ 1712 while ((we = SIMPLEQ_FIRST(&wl->wl_entries)) && 1713 (we->we_bufcount == 0)) { 1714 delta += we->we_reclaimable_bytes; 1715 if (we->we_error) 1716 errcnt++; 1717 SIMPLEQ_REMOVE_HEAD(&wl->wl_entries, we_entries); 1718 pool_put(&wapbl_entry_pool, we); 1719 } 1720 1721 if (delta) { 1722 wl->wl_reclaimable_bytes += delta; 1723 KASSERT(wl->wl_error_count >= errcnt); 1724 wl->wl_error_count -= errcnt; 1725 cv_broadcast(&wl->wl_reclaimable_cv); 1726 } 1727 } 1728 1729 mutex_exit(&wl->wl_mtx); 1730 } 1731 1732 /* 1733 * wapbl_flush(wl, wait) 1734 * 1735 * Flush pending block writes, deallocations, and inodes from 1736 * the current transaction in memory to the log on disk: 1737 * 1738 * 1. Call the file system's wl_flush callback to flush any 1739 * per-file-system pending updates. 1740 * 2. Wait for enough space in the log for the current transaction. 1741 * 3. Synchronously write the new log records, advancing the 1742 * circular queue head. 1743 * 4. Issue the pending block writes asynchronously, now that they 1744 * are recorded in the log and can be replayed after crash. 1745 * 5. If wait is true, wait for all writes to complete and for the 1746 * log to become empty. 1747 * 1748 * On failure, call the file system's wl_flush_abort callback. 1749 */ 1750 int 1751 wapbl_flush(struct wapbl *wl, int waitfor) 1752 { 1753 struct buf *bp; 1754 struct wapbl_entry *we; 1755 off_t off; 1756 off_t head; 1757 off_t tail; 1758 size_t delta = 0; 1759 size_t flushsize; 1760 size_t reserved; 1761 int error = 0; 1762 1763 /* 1764 * Do a quick check to see if a full flush can be skipped 1765 * This assumes that the flush callback does not need to be called 1766 * unless there are other outstanding bufs. 1767 */ 1768 if (!waitfor) { 1769 size_t nbufs; 1770 mutex_enter(&wl->wl_mtx); /* XXX need mutex here to 1771 protect the KASSERTS */ 1772 nbufs = wl->wl_bufcount; 1773 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0)); 1774 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0)); 1775 mutex_exit(&wl->wl_mtx); 1776 if (nbufs == 0) 1777 return 0; 1778 } 1779 1780 /* 1781 * XXX we may consider using LK_UPGRADE here 1782 * if we want to call flush from inside a transaction 1783 */ 1784 rw_enter(&wl->wl_rwlock, RW_WRITER); 1785 wl->wl_flush(wl->wl_mount, TAILQ_FIRST(&wl->wl_dealloclist)); 1786 1787 /* 1788 * Now that we are exclusively locked and the file system has 1789 * issued any deferred block writes for this transaction, check 1790 * whether there are any blocks to write to the log. If not, 1791 * skip waiting for space or writing any log entries. 1792 * 1793 * XXX Shouldn't this also check wl_dealloccnt and 1794 * wl_inohashcnt? Perhaps wl_dealloccnt doesn't matter if the 1795 * file system didn't produce any blocks as a consequence of 1796 * it, but the same does not seem to be so of wl_inohashcnt. 1797 */ 1798 if (wl->wl_bufcount == 0) { 1799 goto wait_out; 1800 } 1801 1802 #if 0 1803 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1804 ("wapbl_flush thread %d.%d flushing entries with " 1805 "bufcount=%zu bufbytes=%zu\n", 1806 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 1807 wl->wl_bufbytes)); 1808 #endif 1809 1810 /* Calculate amount of space needed to flush */ 1811 flushsize = wapbl_transaction_len(wl); 1812 if (wapbl_verbose_commit) { 1813 struct timespec ts; 1814 getnanotime(&ts); 1815 printf("%s: %lld.%09ld this transaction = %zu bytes\n", 1816 __func__, (long long)ts.tv_sec, 1817 (long)ts.tv_nsec, flushsize); 1818 } 1819 1820 if (flushsize > (wl->wl_circ_size - wl->wl_reserved_bytes)) { 1821 /* 1822 * XXX this could be handled more gracefully, perhaps place 1823 * only a partial transaction in the log and allow the 1824 * remaining to flush without the protection of the journal. 1825 */ 1826 panic("wapbl_flush: current transaction too big to flush"); 1827 } 1828 1829 error = wapbl_truncate(wl, flushsize); 1830 if (error) 1831 goto out; 1832 1833 off = wl->wl_head; 1834 KASSERT((off == 0) || (off >= wl->wl_circ_off)); 1835 KASSERT((off == 0) || (off < wl->wl_circ_off + wl->wl_circ_size)); 1836 error = wapbl_write_blocks(wl, &off); 1837 if (error) 1838 goto out; 1839 error = wapbl_write_revocations(wl, &off); 1840 if (error) 1841 goto out; 1842 error = wapbl_write_inodes(wl, &off); 1843 if (error) 1844 goto out; 1845 1846 reserved = 0; 1847 if (wl->wl_inohashcnt) 1848 reserved = wapbl_transaction_inodes_len(wl); 1849 1850 head = wl->wl_head; 1851 tail = wl->wl_tail; 1852 1853 wapbl_advance_head(wl->wl_circ_size, wl->wl_circ_off, flushsize, 1854 &head, &tail); 1855 1856 KASSERTMSG(head == off, 1857 "lost head! head=%"PRIdMAX" tail=%" PRIdMAX 1858 " off=%"PRIdMAX" flush=%zu", 1859 (intmax_t)head, (intmax_t)tail, (intmax_t)off, 1860 flushsize); 1861 1862 /* Opportunistically move the tail forward if we can */ 1863 mutex_enter(&wl->wl_mtx); 1864 delta = wl->wl_reclaimable_bytes; 1865 mutex_exit(&wl->wl_mtx); 1866 wapbl_advance_tail(wl->wl_circ_size, wl->wl_circ_off, delta, 1867 &head, &tail); 1868 1869 error = wapbl_write_commit(wl, head, tail); 1870 if (error) 1871 goto out; 1872 1873 we = pool_get(&wapbl_entry_pool, PR_WAITOK); 1874 1875 #ifdef WAPBL_DEBUG_BUFBYTES 1876 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1877 ("wapbl_flush: thread %d.%d head+=%zu tail+=%zu used=%zu" 1878 " unsynced=%zu" 1879 "\n\tbufcount=%zu bufbytes=%zu bcount=%zu deallocs=%d " 1880 "inodes=%d\n", 1881 curproc->p_pid, curlwp->l_lid, flushsize, delta, 1882 wapbl_space_used(wl->wl_circ_size, head, tail), 1883 wl->wl_unsynced_bufbytes, wl->wl_bufcount, 1884 wl->wl_bufbytes, wl->wl_bcount, wl->wl_dealloccnt, 1885 wl->wl_inohashcnt)); 1886 #else 1887 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1888 ("wapbl_flush: thread %d.%d head+=%zu tail+=%zu used=%zu" 1889 "\n\tbufcount=%zu bufbytes=%zu bcount=%zu deallocs=%d " 1890 "inodes=%d\n", 1891 curproc->p_pid, curlwp->l_lid, flushsize, delta, 1892 wapbl_space_used(wl->wl_circ_size, head, tail), 1893 wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount, 1894 wl->wl_dealloccnt, wl->wl_inohashcnt)); 1895 #endif 1896 1897 1898 mutex_enter(&bufcache_lock); 1899 mutex_enter(&wl->wl_mtx); 1900 1901 wl->wl_reserved_bytes = reserved; 1902 wl->wl_head = head; 1903 wl->wl_tail = tail; 1904 KASSERT(wl->wl_reclaimable_bytes >= delta); 1905 wl->wl_reclaimable_bytes -= delta; 1906 KDASSERT(wl->wl_dealloccnt == 0); 1907 #ifdef WAPBL_DEBUG_BUFBYTES 1908 wl->wl_unsynced_bufbytes += wl->wl_bufbytes; 1909 #endif 1910 1911 we->we_wapbl = wl; 1912 we->we_bufcount = wl->wl_bufcount; 1913 #ifdef WAPBL_DEBUG_BUFBYTES 1914 we->we_unsynced_bufbytes = wl->wl_bufbytes; 1915 #endif 1916 we->we_reclaimable_bytes = flushsize; 1917 we->we_error = 0; 1918 SIMPLEQ_INSERT_TAIL(&wl->wl_entries, we, we_entries); 1919 1920 /* 1921 * This flushes bufs in order than they were queued, so the LRU 1922 * order is preserved. 1923 */ 1924 while ((bp = TAILQ_FIRST(&wl->wl_bufs)) != NULL) { 1925 if (bbusy(bp, 0, 0, &wl->wl_mtx)) { 1926 continue; 1927 } 1928 bp->b_iodone = wapbl_biodone; 1929 bp->b_private = we; 1930 1931 bremfree(bp); 1932 wapbl_remove_buf_locked(wl, bp); 1933 mutex_exit(&wl->wl_mtx); 1934 mutex_exit(&bufcache_lock); 1935 bawrite(bp); 1936 mutex_enter(&bufcache_lock); 1937 mutex_enter(&wl->wl_mtx); 1938 } 1939 mutex_exit(&wl->wl_mtx); 1940 mutex_exit(&bufcache_lock); 1941 1942 #if 0 1943 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1944 ("wapbl_flush thread %d.%d done flushing entries...\n", 1945 curproc->p_pid, curlwp->l_lid)); 1946 #endif 1947 1948 wait_out: 1949 1950 /* 1951 * If the waitfor flag is set, don't return until everything is 1952 * fully flushed and the on disk log is empty. 1953 */ 1954 if (waitfor) { 1955 error = wapbl_truncate(wl, wl->wl_circ_size - 1956 wl->wl_reserved_bytes); 1957 } 1958 1959 out: 1960 if (error) { 1961 wl->wl_flush_abort(wl->wl_mount, 1962 TAILQ_FIRST(&wl->wl_dealloclist)); 1963 } 1964 1965 #ifdef WAPBL_DEBUG_PRINT 1966 if (error) { 1967 pid_t pid = -1; 1968 lwpid_t lid = -1; 1969 if (curproc) 1970 pid = curproc->p_pid; 1971 if (curlwp) 1972 lid = curlwp->l_lid; 1973 mutex_enter(&wl->wl_mtx); 1974 #ifdef WAPBL_DEBUG_BUFBYTES 1975 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1976 ("wapbl_flush: thread %d.%d aborted flush: " 1977 "error = %d\n" 1978 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 1979 "deallocs=%d inodes=%d\n" 1980 "\terrcnt = %d, reclaimable=%zu reserved=%zu " 1981 "unsynced=%zu\n", 1982 pid, lid, error, wl->wl_bufcount, 1983 wl->wl_bufbytes, wl->wl_bcount, 1984 wl->wl_dealloccnt, wl->wl_inohashcnt, 1985 wl->wl_error_count, wl->wl_reclaimable_bytes, 1986 wl->wl_reserved_bytes, wl->wl_unsynced_bufbytes)); 1987 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 1988 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1989 ("\tentry: bufcount = %zu, reclaimable = %zu, " 1990 "error = %d, unsynced = %zu\n", 1991 we->we_bufcount, we->we_reclaimable_bytes, 1992 we->we_error, we->we_unsynced_bufbytes)); 1993 } 1994 #else 1995 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1996 ("wapbl_flush: thread %d.%d aborted flush: " 1997 "error = %d\n" 1998 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 1999 "deallocs=%d inodes=%d\n" 2000 "\terrcnt = %d, reclaimable=%zu reserved=%zu\n", 2001 pid, lid, error, wl->wl_bufcount, 2002 wl->wl_bufbytes, wl->wl_bcount, 2003 wl->wl_dealloccnt, wl->wl_inohashcnt, 2004 wl->wl_error_count, wl->wl_reclaimable_bytes, 2005 wl->wl_reserved_bytes)); 2006 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 2007 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 2008 ("\tentry: bufcount = %zu, reclaimable = %zu, " 2009 "error = %d\n", we->we_bufcount, 2010 we->we_reclaimable_bytes, we->we_error)); 2011 } 2012 #endif 2013 mutex_exit(&wl->wl_mtx); 2014 } 2015 #endif 2016 2017 rw_exit(&wl->wl_rwlock); 2018 return error; 2019 } 2020 2021 /****************************************************************/ 2022 2023 void 2024 wapbl_jlock_assert(struct wapbl *wl) 2025 { 2026 2027 KASSERT(rw_lock_held(&wl->wl_rwlock)); 2028 } 2029 2030 void 2031 wapbl_junlock_assert(struct wapbl *wl) 2032 { 2033 2034 KASSERT(!rw_write_held(&wl->wl_rwlock)); 2035 } 2036 2037 /****************************************************************/ 2038 2039 /* locks missing */ 2040 void 2041 wapbl_print(struct wapbl *wl, 2042 int full, 2043 void (*pr)(const char *, ...)) 2044 { 2045 struct buf *bp; 2046 struct wapbl_entry *we; 2047 (*pr)("wapbl %p", wl); 2048 (*pr)("\nlogvp = %p, devvp = %p, logpbn = %"PRId64"\n", 2049 wl->wl_logvp, wl->wl_devvp, wl->wl_logpbn); 2050 (*pr)("circ = %zu, header = %zu, head = %"PRIdMAX" tail = %"PRIdMAX"\n", 2051 wl->wl_circ_size, wl->wl_circ_off, 2052 (intmax_t)wl->wl_head, (intmax_t)wl->wl_tail); 2053 (*pr)("fs_dev_bshift = %d, log_dev_bshift = %d\n", 2054 wl->wl_log_dev_bshift, wl->wl_fs_dev_bshift); 2055 #ifdef WAPBL_DEBUG_BUFBYTES 2056 (*pr)("bufcount = %zu, bufbytes = %zu bcount = %zu reclaimable = %zu " 2057 "reserved = %zu errcnt = %d unsynced = %zu\n", 2058 wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount, 2059 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 2060 wl->wl_error_count, wl->wl_unsynced_bufbytes); 2061 #else 2062 (*pr)("bufcount = %zu, bufbytes = %zu bcount = %zu reclaimable = %zu " 2063 "reserved = %zu errcnt = %d\n", wl->wl_bufcount, wl->wl_bufbytes, 2064 wl->wl_bcount, wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 2065 wl->wl_error_count); 2066 #endif 2067 (*pr)("\tdealloccnt = %d, dealloclim = %d\n", 2068 wl->wl_dealloccnt, wl->wl_dealloclim); 2069 (*pr)("\tinohashcnt = %d, inohashmask = 0x%08x\n", 2070 wl->wl_inohashcnt, wl->wl_inohashmask); 2071 (*pr)("entries:\n"); 2072 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 2073 #ifdef WAPBL_DEBUG_BUFBYTES 2074 (*pr)("\tbufcount = %zu, reclaimable = %zu, error = %d, " 2075 "unsynced = %zu\n", 2076 we->we_bufcount, we->we_reclaimable_bytes, 2077 we->we_error, we->we_unsynced_bufbytes); 2078 #else 2079 (*pr)("\tbufcount = %zu, reclaimable = %zu, error = %d\n", 2080 we->we_bufcount, we->we_reclaimable_bytes, we->we_error); 2081 #endif 2082 } 2083 if (full) { 2084 int cnt = 0; 2085 (*pr)("bufs ="); 2086 TAILQ_FOREACH(bp, &wl->wl_bufs, b_wapbllist) { 2087 if (!TAILQ_NEXT(bp, b_wapbllist)) { 2088 (*pr)(" %p", bp); 2089 } else if ((++cnt % 6) == 0) { 2090 (*pr)(" %p,\n\t", bp); 2091 } else { 2092 (*pr)(" %p,", bp); 2093 } 2094 } 2095 (*pr)("\n"); 2096 2097 (*pr)("dealloced blks = "); 2098 { 2099 struct wapbl_dealloc *wd; 2100 cnt = 0; 2101 TAILQ_FOREACH(wd, &wl->wl_dealloclist, wd_entries) { 2102 (*pr)(" %"PRId64":%d,", 2103 wd->wd_blkno, 2104 wd->wd_len); 2105 if ((++cnt % 4) == 0) { 2106 (*pr)("\n\t"); 2107 } 2108 } 2109 } 2110 (*pr)("\n"); 2111 2112 (*pr)("registered inodes = "); 2113 { 2114 int i; 2115 cnt = 0; 2116 for (i = 0; i <= wl->wl_inohashmask; i++) { 2117 struct wapbl_ino_head *wih; 2118 struct wapbl_ino *wi; 2119 2120 wih = &wl->wl_inohash[i]; 2121 LIST_FOREACH(wi, wih, wi_hash) { 2122 if (wi->wi_ino == 0) 2123 continue; 2124 (*pr)(" %"PRIu64"/0%06"PRIo32",", 2125 wi->wi_ino, wi->wi_mode); 2126 if ((++cnt % 4) == 0) { 2127 (*pr)("\n\t"); 2128 } 2129 } 2130 } 2131 (*pr)("\n"); 2132 } 2133 2134 (*pr)("iobufs free ="); 2135 TAILQ_FOREACH(bp, &wl->wl_iobufs, b_wapbllist) { 2136 if (!TAILQ_NEXT(bp, b_wapbllist)) { 2137 (*pr)(" %p", bp); 2138 } else if ((++cnt % 6) == 0) { 2139 (*pr)(" %p,\n\t", bp); 2140 } else { 2141 (*pr)(" %p,", bp); 2142 } 2143 } 2144 (*pr)("\n"); 2145 2146 (*pr)("iobufs busy ="); 2147 TAILQ_FOREACH(bp, &wl->wl_iobufs_busy, b_wapbllist) { 2148 if (!TAILQ_NEXT(bp, b_wapbllist)) { 2149 (*pr)(" %p", bp); 2150 } else if ((++cnt % 6) == 0) { 2151 (*pr)(" %p,\n\t", bp); 2152 } else { 2153 (*pr)(" %p,", bp); 2154 } 2155 } 2156 (*pr)("\n"); 2157 } 2158 } 2159 2160 #if defined(WAPBL_DEBUG) || defined(DDB) 2161 void 2162 wapbl_dump(struct wapbl *wl) 2163 { 2164 #if defined(WAPBL_DEBUG) 2165 if (!wl) 2166 wl = wapbl_debug_wl; 2167 #endif 2168 if (!wl) 2169 return; 2170 wapbl_print(wl, 1, printf); 2171 } 2172 #endif 2173 2174 /****************************************************************/ 2175 2176 int 2177 wapbl_register_deallocation(struct wapbl *wl, daddr_t blk, int len, bool force, 2178 void **cookiep) 2179 { 2180 struct wapbl_dealloc *wd; 2181 int error = 0; 2182 2183 wapbl_jlock_assert(wl); 2184 2185 mutex_enter(&wl->wl_mtx); 2186 2187 if (__predict_false(wl->wl_dealloccnt >= wl->wl_dealloclim)) { 2188 if (!force) { 2189 error = EAGAIN; 2190 goto out; 2191 } 2192 2193 /* 2194 * Forced registration can only be used when: 2195 * 1) the caller can't cope with failure 2196 * 2) the path can be triggered only bounded, small 2197 * times per transaction 2198 * If this is not fullfilled, and the path would be triggered 2199 * many times, this could overflow maximum transaction size 2200 * and panic later. 2201 */ 2202 printf("%s: forced dealloc registration over limit: %d >= %d\n", 2203 wl->wl_mount->mnt_stat.f_mntonname, 2204 wl->wl_dealloccnt, wl->wl_dealloclim); 2205 } 2206 2207 wl->wl_dealloccnt++; 2208 mutex_exit(&wl->wl_mtx); 2209 2210 wd = pool_get(&wapbl_dealloc_pool, PR_WAITOK); 2211 wd->wd_blkno = blk; 2212 wd->wd_len = len; 2213 2214 mutex_enter(&wl->wl_mtx); 2215 TAILQ_INSERT_TAIL(&wl->wl_dealloclist, wd, wd_entries); 2216 2217 if (cookiep) 2218 *cookiep = wd; 2219 2220 out: 2221 mutex_exit(&wl->wl_mtx); 2222 2223 WAPBL_PRINTF(WAPBL_PRINT_ALLOC, 2224 ("wapbl_register_deallocation: blk=%"PRId64" len=%d error=%d\n", 2225 blk, len, error)); 2226 2227 return error; 2228 } 2229 2230 static void 2231 wapbl_deallocation_free(struct wapbl *wl, struct wapbl_dealloc *wd, 2232 bool locked) 2233 { 2234 KASSERT(!locked 2235 || rw_lock_held(&wl->wl_rwlock) || mutex_owned(&wl->wl_mtx)); 2236 2237 if (!locked) 2238 mutex_enter(&wl->wl_mtx); 2239 2240 TAILQ_REMOVE(&wl->wl_dealloclist, wd, wd_entries); 2241 wl->wl_dealloccnt--; 2242 2243 if (!locked) 2244 mutex_exit(&wl->wl_mtx); 2245 2246 pool_put(&wapbl_dealloc_pool, wd); 2247 } 2248 2249 void 2250 wapbl_unregister_deallocation(struct wapbl *wl, void *cookie) 2251 { 2252 KASSERT(cookie != NULL); 2253 wapbl_deallocation_free(wl, cookie, false); 2254 } 2255 2256 /****************************************************************/ 2257 2258 static void 2259 wapbl_inodetrk_init(struct wapbl *wl, u_int size) 2260 { 2261 2262 wl->wl_inohash = hashinit(size, HASH_LIST, true, &wl->wl_inohashmask); 2263 if (atomic_inc_uint_nv(&wapbl_ino_pool_refcount) == 1) { 2264 pool_init(&wapbl_ino_pool, sizeof(struct wapbl_ino), 0, 0, 0, 2265 "wapblinopl", &pool_allocator_nointr, IPL_NONE); 2266 } 2267 } 2268 2269 static void 2270 wapbl_inodetrk_free(struct wapbl *wl) 2271 { 2272 2273 /* XXX this KASSERT needs locking/mutex analysis */ 2274 KASSERT(wl->wl_inohashcnt == 0); 2275 hashdone(wl->wl_inohash, HASH_LIST, wl->wl_inohashmask); 2276 membar_release(); 2277 if (atomic_dec_uint_nv(&wapbl_ino_pool_refcount) == 0) { 2278 membar_acquire(); 2279 pool_destroy(&wapbl_ino_pool); 2280 } 2281 } 2282 2283 static struct wapbl_ino * 2284 wapbl_inodetrk_get(struct wapbl *wl, ino_t ino) 2285 { 2286 struct wapbl_ino_head *wih; 2287 struct wapbl_ino *wi; 2288 2289 KASSERT(mutex_owned(&wl->wl_mtx)); 2290 2291 wih = &wl->wl_inohash[ino & wl->wl_inohashmask]; 2292 LIST_FOREACH(wi, wih, wi_hash) { 2293 if (ino == wi->wi_ino) 2294 return wi; 2295 } 2296 return 0; 2297 } 2298 2299 void 2300 wapbl_register_inode(struct wapbl *wl, ino_t ino, mode_t mode) 2301 { 2302 struct wapbl_ino_head *wih; 2303 struct wapbl_ino *wi; 2304 2305 wi = pool_get(&wapbl_ino_pool, PR_WAITOK); 2306 2307 mutex_enter(&wl->wl_mtx); 2308 if (wapbl_inodetrk_get(wl, ino) == NULL) { 2309 wi->wi_ino = ino; 2310 wi->wi_mode = mode; 2311 wih = &wl->wl_inohash[ino & wl->wl_inohashmask]; 2312 LIST_INSERT_HEAD(wih, wi, wi_hash); 2313 wl->wl_inohashcnt++; 2314 WAPBL_PRINTF(WAPBL_PRINT_INODE, 2315 ("wapbl_register_inode: ino=%"PRId64"\n", ino)); 2316 mutex_exit(&wl->wl_mtx); 2317 } else { 2318 mutex_exit(&wl->wl_mtx); 2319 pool_put(&wapbl_ino_pool, wi); 2320 } 2321 } 2322 2323 void 2324 wapbl_unregister_inode(struct wapbl *wl, ino_t ino, mode_t mode) 2325 { 2326 struct wapbl_ino *wi; 2327 2328 mutex_enter(&wl->wl_mtx); 2329 wi = wapbl_inodetrk_get(wl, ino); 2330 if (wi) { 2331 WAPBL_PRINTF(WAPBL_PRINT_INODE, 2332 ("wapbl_unregister_inode: ino=%"PRId64"\n", ino)); 2333 KASSERT(wl->wl_inohashcnt > 0); 2334 wl->wl_inohashcnt--; 2335 LIST_REMOVE(wi, wi_hash); 2336 mutex_exit(&wl->wl_mtx); 2337 2338 pool_put(&wapbl_ino_pool, wi); 2339 } else { 2340 mutex_exit(&wl->wl_mtx); 2341 } 2342 } 2343 2344 /****************************************************************/ 2345 2346 /* 2347 * wapbl_transaction_inodes_len(wl) 2348 * 2349 * Calculate the number of bytes required for inode registration 2350 * log records in wl. 2351 */ 2352 static inline size_t 2353 wapbl_transaction_inodes_len(struct wapbl *wl) 2354 { 2355 int blocklen = 1<<wl->wl_log_dev_bshift; 2356 int iph; 2357 2358 /* Calculate number of inodes described in a inodelist header */ 2359 iph = (blocklen - offsetof(struct wapbl_wc_inodelist, wc_inodes)) / 2360 sizeof(((struct wapbl_wc_inodelist *)0)->wc_inodes[0]); 2361 2362 KASSERT(iph > 0); 2363 2364 return MAX(1, howmany(wl->wl_inohashcnt, iph)) * blocklen; 2365 } 2366 2367 2368 /* 2369 * wapbl_transaction_len(wl) 2370 * 2371 * Calculate number of bytes required for all log records in wl. 2372 */ 2373 static size_t 2374 wapbl_transaction_len(struct wapbl *wl) 2375 { 2376 int blocklen = 1<<wl->wl_log_dev_bshift; 2377 size_t len; 2378 2379 /* Calculate number of blocks described in a blocklist header */ 2380 len = wl->wl_bcount; 2381 len += howmany(wl->wl_bufcount, wl->wl_brperjblock) * blocklen; 2382 len += howmany(wl->wl_dealloccnt, wl->wl_brperjblock) * blocklen; 2383 len += wapbl_transaction_inodes_len(wl); 2384 2385 return len; 2386 } 2387 2388 /* 2389 * wapbl_cache_sync(wl, msg) 2390 * 2391 * Issue DIOCCACHESYNC to wl->wl_devvp. 2392 * 2393 * If sysctl(vfs.wapbl.verbose_commit) >= 2, print a message 2394 * including msg about the duration of the cache sync. 2395 */ 2396 static int 2397 wapbl_cache_sync(struct wapbl *wl, const char *msg) 2398 { 2399 const bool verbose = wapbl_verbose_commit >= 2; 2400 struct bintime start_time; 2401 int force = 1; 2402 int error; 2403 2404 /* Skip full cache sync if disabled */ 2405 if (!wapbl_flush_disk_cache) { 2406 return 0; 2407 } 2408 if (verbose) { 2409 bintime(&start_time); 2410 } 2411 error = VOP_IOCTL(wl->wl_devvp, DIOCCACHESYNC, &force, 2412 FWRITE, FSCRED); 2413 if (error) { 2414 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 2415 ("wapbl_cache_sync: DIOCCACHESYNC on dev 0x%jx " 2416 "returned %d\n", (uintmax_t)wl->wl_devvp->v_rdev, error)); 2417 } 2418 if (verbose) { 2419 struct bintime d; 2420 struct timespec ts; 2421 2422 bintime(&d); 2423 bintime_sub(&d, &start_time); 2424 bintime2timespec(&d, &ts); 2425 printf("wapbl_cache_sync: %s: dev 0x%jx %ju.%09lu\n", 2426 msg, (uintmax_t)wl->wl_devvp->v_rdev, 2427 (uintmax_t)ts.tv_sec, ts.tv_nsec); 2428 } 2429 2430 wl->wl_ev_cacheflush.ev_count++; 2431 2432 return error; 2433 } 2434 2435 /* 2436 * wapbl_write_commit(wl, head, tail) 2437 * 2438 * Issue a disk cache sync to wait for all pending writes to the 2439 * log to complete, and then synchronously commit the current 2440 * circular queue head and tail to the log, in the next of two 2441 * locations for commit headers on disk. 2442 * 2443 * Increment the generation number. If the generation number 2444 * rolls over to zero, then a subsequent commit would appear to 2445 * have an older generation than this one -- in that case, issue a 2446 * duplicate commit to avoid this. 2447 * 2448 * => Caller must have exclusive access to wl, either by holding 2449 * wl->wl_rwlock for writer or by being wapbl_start before anyone 2450 * else has seen wl. 2451 */ 2452 static int 2453 wapbl_write_commit(struct wapbl *wl, off_t head, off_t tail) 2454 { 2455 struct wapbl_wc_header *wc = wl->wl_wc_header; 2456 struct timespec ts; 2457 int error; 2458 daddr_t pbn; 2459 2460 error = wapbl_buffered_flush(wl, true); 2461 if (error) 2462 return error; 2463 /* 2464 * Flush disk cache to ensure that blocks we've written are actually 2465 * written to the stable storage before the commit header. 2466 * This flushes to disk not only journal blocks, but also all 2467 * metadata blocks, written asynchronously since previous commit. 2468 * 2469 * XXX Calc checksum here, instead we do this for now 2470 */ 2471 wapbl_cache_sync(wl, "1"); 2472 2473 wc->wc_head = head; 2474 wc->wc_tail = tail; 2475 wc->wc_checksum = 0; 2476 wc->wc_version = 1; 2477 getnanotime(&ts); 2478 wc->wc_time = ts.tv_sec; 2479 wc->wc_timensec = ts.tv_nsec; 2480 2481 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2482 ("wapbl_write_commit: head = %"PRIdMAX "tail = %"PRIdMAX"\n", 2483 (intmax_t)head, (intmax_t)tail)); 2484 2485 /* 2486 * write the commit header. 2487 * 2488 * XXX if generation will rollover, then first zero 2489 * over second commit header before trying to write both headers. 2490 */ 2491 2492 pbn = wl->wl_logpbn + (wc->wc_generation % 2); 2493 #ifdef _KERNEL 2494 pbn = btodb(pbn << wc->wc_log_dev_bshift); 2495 #endif 2496 error = wapbl_buffered_write(wc, wc->wc_len, wl, pbn, WAPBL_JFLAGS(wl)); 2497 if (error) 2498 return error; 2499 error = wapbl_buffered_flush(wl, true); 2500 if (error) 2501 return error; 2502 2503 /* 2504 * Flush disk cache to ensure that the commit header is actually 2505 * written before meta data blocks. Commit block is written using 2506 * FUA when enabled, in that case this flush is not needed. 2507 */ 2508 if (!WAPBL_USE_FUA(wl)) 2509 wapbl_cache_sync(wl, "2"); 2510 2511 /* 2512 * If the generation number was zero, write it out a second time. 2513 * This handles initialization and generation number rollover 2514 */ 2515 if (wc->wc_generation++ == 0) { 2516 error = wapbl_write_commit(wl, head, tail); 2517 /* 2518 * This panic should be able to be removed if we do the 2519 * zero'ing mentioned above, and we are certain to roll 2520 * back generation number on failure. 2521 */ 2522 if (error) 2523 panic("wapbl_write_commit: error writing duplicate " 2524 "log header: %d", error); 2525 } 2526 2527 wl->wl_ev_commit.ev_count++; 2528 2529 return 0; 2530 } 2531 2532 /* 2533 * wapbl_write_blocks(wl, offp) 2534 * 2535 * Write all pending physical blocks in the current transaction 2536 * from wapbl_add_buf to the log on disk, adding to the circular 2537 * queue head at byte offset *offp, and returning the new head's 2538 * byte offset in *offp. 2539 */ 2540 static int 2541 wapbl_write_blocks(struct wapbl *wl, off_t *offp) 2542 { 2543 struct wapbl_wc_blocklist *wc = 2544 (struct wapbl_wc_blocklist *)wl->wl_wc_scratch; 2545 int blocklen = 1<<wl->wl_log_dev_bshift; 2546 struct buf *bp; 2547 off_t off = *offp; 2548 int error; 2549 size_t padding; 2550 2551 KASSERT(rw_write_held(&wl->wl_rwlock)); 2552 2553 bp = TAILQ_FIRST(&wl->wl_bufs); 2554 2555 while (bp) { 2556 int cnt; 2557 struct buf *obp = bp; 2558 2559 KASSERT(bp->b_flags & B_LOCKED); 2560 2561 wc->wc_type = WAPBL_WC_BLOCKS; 2562 wc->wc_len = blocklen; 2563 wc->wc_blkcount = 0; 2564 wc->wc_unused = 0; 2565 while (bp && (wc->wc_blkcount < wl->wl_brperjblock)) { 2566 /* 2567 * Make sure all the physical block numbers are up to 2568 * date. If this is not always true on a given 2569 * filesystem, then VOP_BMAP must be called. We 2570 * could call VOP_BMAP here, or else in the filesystem 2571 * specific flush callback, although neither of those 2572 * solutions allow us to take the vnode lock. If a 2573 * filesystem requires that we must take the vnode lock 2574 * to call VOP_BMAP, then we can probably do it in 2575 * bwrite when the vnode lock should already be held 2576 * by the invoking code. 2577 */ 2578 KASSERT((bp->b_vp->v_type == VBLK) || 2579 (bp->b_blkno != bp->b_lblkno)); 2580 KASSERT(bp->b_blkno > 0); 2581 2582 wc->wc_blocks[wc->wc_blkcount].wc_daddr = bp->b_blkno; 2583 wc->wc_blocks[wc->wc_blkcount].wc_dlen = bp->b_bcount; 2584 wc->wc_len += bp->b_bcount; 2585 wc->wc_blkcount++; 2586 bp = TAILQ_NEXT(bp, b_wapbllist); 2587 } 2588 if (wc->wc_len % blocklen != 0) { 2589 padding = blocklen - wc->wc_len % blocklen; 2590 wc->wc_len += padding; 2591 } else { 2592 padding = 0; 2593 } 2594 2595 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2596 ("wapbl_write_blocks: len = %u (padding %zu) off = %"PRIdMAX"\n", 2597 wc->wc_len, padding, (intmax_t)off)); 2598 2599 error = wapbl_circ_write(wl, wc, blocklen, &off); 2600 if (error) 2601 return error; 2602 bp = obp; 2603 cnt = 0; 2604 while (bp && (cnt++ < wl->wl_brperjblock)) { 2605 error = wapbl_circ_write(wl, bp->b_data, 2606 bp->b_bcount, &off); 2607 if (error) 2608 return error; 2609 bp = TAILQ_NEXT(bp, b_wapbllist); 2610 } 2611 if (padding) { 2612 void *zero; 2613 2614 zero = wapbl_alloc(padding); 2615 memset(zero, 0, padding); 2616 error = wapbl_circ_write(wl, zero, padding, &off); 2617 wapbl_free(zero, padding); 2618 if (error) 2619 return error; 2620 } 2621 } 2622 *offp = off; 2623 return 0; 2624 } 2625 2626 /* 2627 * wapbl_write_revocations(wl, offp) 2628 * 2629 * Write all pending deallocations in the current transaction from 2630 * wapbl_register_deallocation to the log on disk, adding to the 2631 * circular queue's head at byte offset *offp, and returning the 2632 * new head's byte offset in *offp. 2633 */ 2634 static int 2635 wapbl_write_revocations(struct wapbl *wl, off_t *offp) 2636 { 2637 struct wapbl_wc_blocklist *wc = 2638 (struct wapbl_wc_blocklist *)wl->wl_wc_scratch; 2639 struct wapbl_dealloc *wd, *lwd; 2640 int blocklen = 1<<wl->wl_log_dev_bshift; 2641 off_t off = *offp; 2642 int error; 2643 2644 KASSERT(rw_write_held(&wl->wl_rwlock)); 2645 2646 if (wl->wl_dealloccnt == 0) 2647 return 0; 2648 2649 while ((wd = TAILQ_FIRST(&wl->wl_dealloclist)) != NULL) { 2650 wc->wc_type = WAPBL_WC_REVOCATIONS; 2651 wc->wc_len = blocklen; 2652 wc->wc_blkcount = 0; 2653 wc->wc_unused = 0; 2654 while (wd && (wc->wc_blkcount < wl->wl_brperjblock)) { 2655 wc->wc_blocks[wc->wc_blkcount].wc_daddr = 2656 wd->wd_blkno; 2657 wc->wc_blocks[wc->wc_blkcount].wc_dlen = 2658 wd->wd_len; 2659 wc->wc_blkcount++; 2660 2661 wd = TAILQ_NEXT(wd, wd_entries); 2662 } 2663 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2664 ("wapbl_write_revocations: len = %u off = %"PRIdMAX"\n", 2665 wc->wc_len, (intmax_t)off)); 2666 error = wapbl_circ_write(wl, wc, blocklen, &off); 2667 if (error) 2668 return error; 2669 2670 /* free all successfully written deallocs */ 2671 lwd = wd; 2672 while ((wd = TAILQ_FIRST(&wl->wl_dealloclist)) != NULL) { 2673 if (wd == lwd) 2674 break; 2675 wapbl_deallocation_free(wl, wd, true); 2676 } 2677 } 2678 *offp = off; 2679 return 0; 2680 } 2681 2682 /* 2683 * wapbl_write_inodes(wl, offp) 2684 * 2685 * Write all pending inode allocations in the current transaction 2686 * from wapbl_register_inode to the log on disk, adding to the 2687 * circular queue's head at byte offset *offp and returning the 2688 * new head's byte offset in *offp. 2689 */ 2690 static int 2691 wapbl_write_inodes(struct wapbl *wl, off_t *offp) 2692 { 2693 struct wapbl_wc_inodelist *wc = 2694 (struct wapbl_wc_inodelist *)wl->wl_wc_scratch; 2695 int i; 2696 int blocklen = 1 << wl->wl_log_dev_bshift; 2697 off_t off = *offp; 2698 int error; 2699 2700 struct wapbl_ino_head *wih; 2701 struct wapbl_ino *wi; 2702 int iph; 2703 2704 iph = (blocklen - offsetof(struct wapbl_wc_inodelist, wc_inodes)) / 2705 sizeof(((struct wapbl_wc_inodelist *)0)->wc_inodes[0]); 2706 2707 i = 0; 2708 wih = &wl->wl_inohash[0]; 2709 wi = 0; 2710 do { 2711 wc->wc_type = WAPBL_WC_INODES; 2712 wc->wc_len = blocklen; 2713 wc->wc_inocnt = 0; 2714 wc->wc_clear = (i == 0); 2715 while ((i < wl->wl_inohashcnt) && (wc->wc_inocnt < iph)) { 2716 while (!wi) { 2717 KASSERT((wih - &wl->wl_inohash[0]) 2718 <= wl->wl_inohashmask); 2719 wi = LIST_FIRST(wih++); 2720 } 2721 wc->wc_inodes[wc->wc_inocnt].wc_inumber = wi->wi_ino; 2722 wc->wc_inodes[wc->wc_inocnt].wc_imode = wi->wi_mode; 2723 wc->wc_inocnt++; 2724 i++; 2725 wi = LIST_NEXT(wi, wi_hash); 2726 } 2727 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2728 ("wapbl_write_inodes: len = %u off = %"PRIdMAX"\n", 2729 wc->wc_len, (intmax_t)off)); 2730 error = wapbl_circ_write(wl, wc, blocklen, &off); 2731 if (error) 2732 return error; 2733 } while (i < wl->wl_inohashcnt); 2734 2735 *offp = off; 2736 return 0; 2737 } 2738 2739 #endif /* _KERNEL */ 2740 2741 /****************************************************************/ 2742 2743 struct wapbl_blk { 2744 LIST_ENTRY(wapbl_blk) wb_hash; 2745 daddr_t wb_blk; 2746 off_t wb_off; /* Offset of this block in the log */ 2747 }; 2748 #define WAPBL_BLKPOOL_MIN 83 2749 2750 static void 2751 wapbl_blkhash_init(struct wapbl_replay *wr, u_int size) 2752 { 2753 if (size < WAPBL_BLKPOOL_MIN) 2754 size = WAPBL_BLKPOOL_MIN; 2755 KASSERT(wr->wr_blkhash == 0); 2756 #ifdef _KERNEL 2757 wr->wr_blkhash = hashinit(size, HASH_LIST, true, &wr->wr_blkhashmask); 2758 #else /* ! _KERNEL */ 2759 /* Manually implement hashinit */ 2760 { 2761 unsigned long i, hashsize; 2762 for (hashsize = 1; hashsize < size; hashsize <<= 1) 2763 continue; 2764 wr->wr_blkhash = wapbl_alloc(hashsize * sizeof(*wr->wr_blkhash)); 2765 for (i = 0; i < hashsize; i++) 2766 LIST_INIT(&wr->wr_blkhash[i]); 2767 wr->wr_blkhashmask = hashsize - 1; 2768 } 2769 #endif /* ! _KERNEL */ 2770 } 2771 2772 static void 2773 wapbl_blkhash_free(struct wapbl_replay *wr) 2774 { 2775 KASSERT(wr->wr_blkhashcnt == 0); 2776 #ifdef _KERNEL 2777 hashdone(wr->wr_blkhash, HASH_LIST, wr->wr_blkhashmask); 2778 #else /* ! _KERNEL */ 2779 wapbl_free(wr->wr_blkhash, 2780 (wr->wr_blkhashmask + 1) * sizeof(*wr->wr_blkhash)); 2781 #endif /* ! _KERNEL */ 2782 } 2783 2784 static struct wapbl_blk * 2785 wapbl_blkhash_get(struct wapbl_replay *wr, daddr_t blk) 2786 { 2787 struct wapbl_blk_head *wbh; 2788 struct wapbl_blk *wb; 2789 wbh = &wr->wr_blkhash[blk & wr->wr_blkhashmask]; 2790 LIST_FOREACH(wb, wbh, wb_hash) { 2791 if (blk == wb->wb_blk) 2792 return wb; 2793 } 2794 return 0; 2795 } 2796 2797 static void 2798 wapbl_blkhash_ins(struct wapbl_replay *wr, daddr_t blk, off_t off) 2799 { 2800 struct wapbl_blk_head *wbh; 2801 struct wapbl_blk *wb; 2802 wb = wapbl_blkhash_get(wr, blk); 2803 if (wb) { 2804 KASSERT(wb->wb_blk == blk); 2805 wb->wb_off = off; 2806 } else { 2807 wb = wapbl_alloc(sizeof(*wb)); 2808 wb->wb_blk = blk; 2809 wb->wb_off = off; 2810 wbh = &wr->wr_blkhash[blk & wr->wr_blkhashmask]; 2811 LIST_INSERT_HEAD(wbh, wb, wb_hash); 2812 wr->wr_blkhashcnt++; 2813 } 2814 } 2815 2816 static void 2817 wapbl_blkhash_rem(struct wapbl_replay *wr, daddr_t blk) 2818 { 2819 struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 2820 if (wb) { 2821 KASSERT(wr->wr_blkhashcnt > 0); 2822 wr->wr_blkhashcnt--; 2823 LIST_REMOVE(wb, wb_hash); 2824 wapbl_free(wb, sizeof(*wb)); 2825 } 2826 } 2827 2828 static void 2829 wapbl_blkhash_clear(struct wapbl_replay *wr) 2830 { 2831 unsigned long i; 2832 for (i = 0; i <= wr->wr_blkhashmask; i++) { 2833 struct wapbl_blk *wb; 2834 2835 while ((wb = LIST_FIRST(&wr->wr_blkhash[i]))) { 2836 KASSERT(wr->wr_blkhashcnt > 0); 2837 wr->wr_blkhashcnt--; 2838 LIST_REMOVE(wb, wb_hash); 2839 wapbl_free(wb, sizeof(*wb)); 2840 } 2841 } 2842 KASSERT(wr->wr_blkhashcnt == 0); 2843 } 2844 2845 /****************************************************************/ 2846 2847 /* 2848 * wapbl_circ_read(wr, data, len, offp) 2849 * 2850 * Read len bytes into data from the circular queue of wr, 2851 * starting at the linear byte offset *offp, and returning the new 2852 * linear byte offset in *offp. 2853 * 2854 * If the starting linear byte offset precedes wr->wr_circ_off, 2855 * the read instead begins at wr->wr_circ_off. XXX WTF? This 2856 * should be a KASSERT, not a conditional. 2857 */ 2858 static int 2859 wapbl_circ_read(struct wapbl_replay *wr, void *data, size_t len, off_t *offp) 2860 { 2861 size_t slen; 2862 off_t off = *offp; 2863 int error; 2864 daddr_t pbn; 2865 2866 KASSERT(((len >> wr->wr_log_dev_bshift) << 2867 wr->wr_log_dev_bshift) == len); 2868 2869 if (off < wr->wr_circ_off) 2870 off = wr->wr_circ_off; 2871 slen = wr->wr_circ_off + wr->wr_circ_size - off; 2872 if (slen < len) { 2873 pbn = wr->wr_logpbn + (off >> wr->wr_log_dev_bshift); 2874 #ifdef _KERNEL 2875 pbn = btodb(pbn << wr->wr_log_dev_bshift); 2876 #endif 2877 error = wapbl_read(data, slen, wr->wr_devvp, pbn); 2878 if (error) 2879 return error; 2880 data = (uint8_t *)data + slen; 2881 len -= slen; 2882 off = wr->wr_circ_off; 2883 } 2884 pbn = wr->wr_logpbn + (off >> wr->wr_log_dev_bshift); 2885 #ifdef _KERNEL 2886 pbn = btodb(pbn << wr->wr_log_dev_bshift); 2887 #endif 2888 error = wapbl_read(data, len, wr->wr_devvp, pbn); 2889 if (error) 2890 return error; 2891 off += len; 2892 if (off >= wr->wr_circ_off + wr->wr_circ_size) 2893 off = wr->wr_circ_off; 2894 *offp = off; 2895 return 0; 2896 } 2897 2898 /* 2899 * wapbl_circ_advance(wr, len, offp) 2900 * 2901 * Compute the linear byte offset of the circular queue of wr that 2902 * is len bytes past *offp, and store it in *offp. 2903 * 2904 * This is as if wapbl_circ_read, but without actually reading 2905 * anything. 2906 * 2907 * If the starting linear byte offset precedes wr->wr_circ_off, it 2908 * is taken to be wr->wr_circ_off instead. XXX WTF? This should 2909 * be a KASSERT, not a conditional. 2910 */ 2911 static void 2912 wapbl_circ_advance(struct wapbl_replay *wr, size_t len, off_t *offp) 2913 { 2914 size_t slen; 2915 off_t off = *offp; 2916 2917 KASSERT(((len >> wr->wr_log_dev_bshift) << 2918 wr->wr_log_dev_bshift) == len); 2919 2920 if (off < wr->wr_circ_off) 2921 off = wr->wr_circ_off; 2922 slen = wr->wr_circ_off + wr->wr_circ_size - off; 2923 if (slen < len) { 2924 len -= slen; 2925 off = wr->wr_circ_off; 2926 } 2927 off += len; 2928 if (off >= wr->wr_circ_off + wr->wr_circ_size) 2929 off = wr->wr_circ_off; 2930 *offp = off; 2931 } 2932 2933 /****************************************************************/ 2934 2935 int 2936 wapbl_replay_start(struct wapbl_replay **wrp, struct vnode *vp, 2937 daddr_t off, size_t count, size_t blksize) 2938 { 2939 struct wapbl_replay *wr; 2940 int error; 2941 struct vnode *devvp; 2942 daddr_t logpbn; 2943 uint8_t *scratch; 2944 struct wapbl_wc_header *wch; 2945 struct wapbl_wc_header *wch2; 2946 /* Use this until we read the actual log header */ 2947 int log_dev_bshift = ilog2(blksize); 2948 size_t used; 2949 daddr_t pbn; 2950 2951 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 2952 ("wapbl_replay_start: vp=%p off=%"PRId64 " count=%zu blksize=%zu\n", 2953 vp, off, count, blksize)); 2954 2955 if (off < 0) 2956 return EINVAL; 2957 2958 if (blksize < DEV_BSIZE) 2959 return EINVAL; 2960 if (blksize % DEV_BSIZE) 2961 return EINVAL; 2962 2963 #ifdef _KERNEL 2964 #if 0 2965 /* XXX vp->v_size isn't reliably set for VBLK devices, 2966 * especially root. However, we might still want to verify 2967 * that the full load is readable */ 2968 if ((off + count) * blksize > vp->v_size) 2969 return EINVAL; 2970 #endif 2971 if ((error = VOP_BMAP(vp, off, &devvp, &logpbn, 0)) != 0) { 2972 return error; 2973 } 2974 #else /* ! _KERNEL */ 2975 devvp = vp; 2976 logpbn = off; 2977 #endif /* ! _KERNEL */ 2978 2979 scratch = wapbl_alloc(MAXBSIZE); 2980 2981 pbn = logpbn; 2982 #ifdef _KERNEL 2983 pbn = btodb(pbn << log_dev_bshift); 2984 #endif 2985 error = wapbl_read(scratch, 2<<log_dev_bshift, devvp, pbn); 2986 if (error) 2987 goto errout; 2988 2989 wch = (struct wapbl_wc_header *)scratch; 2990 wch2 = 2991 (struct wapbl_wc_header *)(scratch + (1<<log_dev_bshift)); 2992 /* XXX verify checksums and magic numbers */ 2993 if (wch->wc_type != WAPBL_WC_HEADER) { 2994 printf("Unrecognized wapbl magic: 0x%08x\n", wch->wc_type); 2995 error = EFTYPE; 2996 goto errout; 2997 } 2998 2999 if (wch2->wc_generation > wch->wc_generation) 3000 wch = wch2; 3001 3002 wr = wapbl_calloc(1, sizeof(*wr)); 3003 3004 wr->wr_logvp = vp; 3005 wr->wr_devvp = devvp; 3006 wr->wr_logpbn = logpbn; 3007 3008 wr->wr_scratch = scratch; 3009 3010 wr->wr_log_dev_bshift = wch->wc_log_dev_bshift; 3011 wr->wr_fs_dev_bshift = wch->wc_fs_dev_bshift; 3012 wr->wr_circ_off = wch->wc_circ_off; 3013 wr->wr_circ_size = wch->wc_circ_size; 3014 wr->wr_generation = wch->wc_generation; 3015 3016 used = wapbl_space_used(wch->wc_circ_size, wch->wc_head, wch->wc_tail); 3017 3018 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 3019 ("wapbl_replay: head=%"PRId64" tail=%"PRId64" off=%"PRId64 3020 " len=%"PRId64" used=%zu\n", 3021 wch->wc_head, wch->wc_tail, wch->wc_circ_off, 3022 wch->wc_circ_size, used)); 3023 3024 wapbl_blkhash_init(wr, (used >> wch->wc_fs_dev_bshift)); 3025 3026 error = wapbl_replay_process(wr, wch->wc_head, wch->wc_tail); 3027 if (error) { 3028 wapbl_replay_stop(wr); 3029 wapbl_replay_free(wr); 3030 return error; 3031 } 3032 3033 *wrp = wr; 3034 return 0; 3035 3036 errout: 3037 wapbl_free(scratch, MAXBSIZE); 3038 return error; 3039 } 3040 3041 void 3042 wapbl_replay_stop(struct wapbl_replay *wr) 3043 { 3044 3045 if (!wapbl_replay_isopen(wr)) 3046 return; 3047 3048 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, ("wapbl_replay_stop called\n")); 3049 3050 wapbl_free(wr->wr_scratch, MAXBSIZE); 3051 wr->wr_scratch = NULL; 3052 3053 wr->wr_logvp = NULL; 3054 3055 wapbl_blkhash_clear(wr); 3056 wapbl_blkhash_free(wr); 3057 } 3058 3059 void 3060 wapbl_replay_free(struct wapbl_replay *wr) 3061 { 3062 3063 KDASSERT(!wapbl_replay_isopen(wr)); 3064 3065 if (wr->wr_inodes) 3066 wapbl_free(wr->wr_inodes, 3067 wr->wr_inodescnt * sizeof(wr->wr_inodes[0])); 3068 wapbl_free(wr, sizeof(*wr)); 3069 } 3070 3071 #ifdef _KERNEL 3072 int 3073 wapbl_replay_isopen1(struct wapbl_replay *wr) 3074 { 3075 3076 return wapbl_replay_isopen(wr); 3077 } 3078 #endif 3079 3080 /* 3081 * calculate the disk address for the i'th block in the wc_blockblist 3082 * offset by j blocks of size blen. 3083 * 3084 * wc_daddr is always a kernel disk address in DEV_BSIZE units that 3085 * was written to the journal. 3086 * 3087 * The kernel needs that address plus the offset in DEV_BSIZE units. 3088 * 3089 * Userland needs that address plus the offset in blen units. 3090 * 3091 */ 3092 static daddr_t 3093 wapbl_block_daddr(struct wapbl_wc_blocklist *wc, int i, int j, int blen) 3094 { 3095 daddr_t pbn; 3096 3097 #ifdef _KERNEL 3098 pbn = wc->wc_blocks[i].wc_daddr + btodb(j * blen); 3099 #else 3100 pbn = dbtob(wc->wc_blocks[i].wc_daddr) / blen + j; 3101 #endif 3102 3103 return pbn; 3104 } 3105 3106 static void 3107 wapbl_replay_process_blocks(struct wapbl_replay *wr, off_t *offp) 3108 { 3109 struct wapbl_wc_blocklist *wc = 3110 (struct wapbl_wc_blocklist *)wr->wr_scratch; 3111 int fsblklen = 1 << wr->wr_fs_dev_bshift; 3112 int i, j, n; 3113 3114 for (i = 0; i < wc->wc_blkcount; i++) { 3115 /* 3116 * Enter each physical block into the hashtable independently. 3117 */ 3118 n = wc->wc_blocks[i].wc_dlen >> wr->wr_fs_dev_bshift; 3119 for (j = 0; j < n; j++) { 3120 wapbl_blkhash_ins(wr, wapbl_block_daddr(wc, i, j, fsblklen), 3121 *offp); 3122 wapbl_circ_advance(wr, fsblklen, offp); 3123 } 3124 } 3125 } 3126 3127 static void 3128 wapbl_replay_process_revocations(struct wapbl_replay *wr) 3129 { 3130 struct wapbl_wc_blocklist *wc = 3131 (struct wapbl_wc_blocklist *)wr->wr_scratch; 3132 int fsblklen = 1 << wr->wr_fs_dev_bshift; 3133 int i, j, n; 3134 3135 for (i = 0; i < wc->wc_blkcount; i++) { 3136 /* 3137 * Remove any blocks found from the hashtable. 3138 */ 3139 n = wc->wc_blocks[i].wc_dlen >> wr->wr_fs_dev_bshift; 3140 for (j = 0; j < n; j++) 3141 wapbl_blkhash_rem(wr, wapbl_block_daddr(wc, i, j, fsblklen)); 3142 } 3143 } 3144 3145 static void 3146 wapbl_replay_process_inodes(struct wapbl_replay *wr, off_t oldoff, off_t newoff) 3147 { 3148 struct wapbl_wc_inodelist *wc = 3149 (struct wapbl_wc_inodelist *)wr->wr_scratch; 3150 void *new_inodes; 3151 const size_t oldsize = wr->wr_inodescnt * sizeof(wr->wr_inodes[0]); 3152 3153 KASSERT(sizeof(wr->wr_inodes[0]) == sizeof(wc->wc_inodes[0])); 3154 3155 /* 3156 * Keep track of where we found this so location won't be 3157 * overwritten. 3158 */ 3159 if (wc->wc_clear) { 3160 wr->wr_inodestail = oldoff; 3161 wr->wr_inodescnt = 0; 3162 if (wr->wr_inodes != NULL) { 3163 wapbl_free(wr->wr_inodes, oldsize); 3164 wr->wr_inodes = NULL; 3165 } 3166 } 3167 wr->wr_inodeshead = newoff; 3168 if (wc->wc_inocnt == 0) 3169 return; 3170 3171 new_inodes = wapbl_alloc((wr->wr_inodescnt + wc->wc_inocnt) * 3172 sizeof(wr->wr_inodes[0])); 3173 if (wr->wr_inodes != NULL) { 3174 memcpy(new_inodes, wr->wr_inodes, oldsize); 3175 wapbl_free(wr->wr_inodes, oldsize); 3176 } 3177 wr->wr_inodes = new_inodes; 3178 memcpy(&wr->wr_inodes[wr->wr_inodescnt], wc->wc_inodes, 3179 wc->wc_inocnt * sizeof(wr->wr_inodes[0])); 3180 wr->wr_inodescnt += wc->wc_inocnt; 3181 } 3182 3183 static int 3184 wapbl_replay_process(struct wapbl_replay *wr, off_t head, off_t tail) 3185 { 3186 off_t off; 3187 int error; 3188 3189 int logblklen = 1 << wr->wr_log_dev_bshift; 3190 3191 wapbl_blkhash_clear(wr); 3192 3193 off = tail; 3194 while (off != head) { 3195 struct wapbl_wc_null *wcn; 3196 off_t saveoff = off; 3197 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 3198 if (error) 3199 goto errout; 3200 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 3201 switch (wcn->wc_type) { 3202 case WAPBL_WC_BLOCKS: 3203 wapbl_replay_process_blocks(wr, &off); 3204 break; 3205 3206 case WAPBL_WC_REVOCATIONS: 3207 wapbl_replay_process_revocations(wr); 3208 break; 3209 3210 case WAPBL_WC_INODES: 3211 wapbl_replay_process_inodes(wr, saveoff, off); 3212 break; 3213 3214 default: 3215 printf("Unrecognized wapbl type: 0x%08x\n", 3216 wcn->wc_type); 3217 error = EFTYPE; 3218 goto errout; 3219 } 3220 wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 3221 if (off != saveoff) { 3222 printf("wapbl_replay: corrupted records\n"); 3223 error = EFTYPE; 3224 goto errout; 3225 } 3226 } 3227 return 0; 3228 3229 errout: 3230 wapbl_blkhash_clear(wr); 3231 return error; 3232 } 3233 3234 #if 0 3235 int 3236 wapbl_replay_verify(struct wapbl_replay *wr, struct vnode *fsdevvp) 3237 { 3238 off_t off; 3239 int mismatchcnt = 0; 3240 int logblklen = 1 << wr->wr_log_dev_bshift; 3241 int fsblklen = 1 << wr->wr_fs_dev_bshift; 3242 void *scratch1 = wapbl_alloc(MAXBSIZE); 3243 void *scratch2 = wapbl_alloc(MAXBSIZE); 3244 int error = 0; 3245 3246 KDASSERT(wapbl_replay_isopen(wr)); 3247 3248 off = wch->wc_tail; 3249 while (off != wch->wc_head) { 3250 struct wapbl_wc_null *wcn; 3251 #ifdef DEBUG 3252 off_t saveoff = off; 3253 #endif 3254 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 3255 if (error) 3256 goto out; 3257 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 3258 switch (wcn->wc_type) { 3259 case WAPBL_WC_BLOCKS: 3260 { 3261 struct wapbl_wc_blocklist *wc = 3262 (struct wapbl_wc_blocklist *)wr->wr_scratch; 3263 int i; 3264 for (i = 0; i < wc->wc_blkcount; i++) { 3265 int foundcnt = 0; 3266 int dirtycnt = 0; 3267 int j, n; 3268 /* 3269 * Check each physical block into the 3270 * hashtable independently 3271 */ 3272 n = wc->wc_blocks[i].wc_dlen >> 3273 wch->wc_fs_dev_bshift; 3274 for (j = 0; j < n; j++) { 3275 struct wapbl_blk *wb = 3276 wapbl_blkhash_get(wr, 3277 wapbl_block_daddr(wc, i, j, fsblklen)); 3278 if (wb && (wb->wb_off == off)) { 3279 foundcnt++; 3280 error = 3281 wapbl_circ_read(wr, 3282 scratch1, fsblklen, 3283 &off); 3284 if (error) 3285 goto out; 3286 error = 3287 wapbl_read(scratch2, 3288 fsblklen, fsdevvp, 3289 wb->wb_blk); 3290 if (error) 3291 goto out; 3292 if (memcmp(scratch1, 3293 scratch2, 3294 fsblklen)) { 3295 printf( 3296 "wapbl_verify: mismatch block %"PRId64" at off %"PRIdMAX"\n", 3297 wb->wb_blk, (intmax_t)off); 3298 dirtycnt++; 3299 mismatchcnt++; 3300 } 3301 } else { 3302 wapbl_circ_advance(wr, 3303 fsblklen, &off); 3304 } 3305 } 3306 #if 0 3307 /* 3308 * If all of the blocks in an entry 3309 * are clean, then remove all of its 3310 * blocks from the hashtable since they 3311 * never will need replay. 3312 */ 3313 if ((foundcnt != 0) && 3314 (dirtycnt == 0)) { 3315 off = saveoff; 3316 wapbl_circ_advance(wr, 3317 logblklen, &off); 3318 for (j = 0; j < n; j++) { 3319 struct wapbl_blk *wb = 3320 wapbl_blkhash_get(wr, 3321 wapbl_block_daddr(wc, i, j, fsblklen)); 3322 if (wb && 3323 (wb->wb_off == off)) { 3324 wapbl_blkhash_rem(wr, wb->wb_blk); 3325 } 3326 wapbl_circ_advance(wr, 3327 fsblklen, &off); 3328 } 3329 } 3330 #endif 3331 } 3332 } 3333 break; 3334 case WAPBL_WC_REVOCATIONS: 3335 case WAPBL_WC_INODES: 3336 break; 3337 default: 3338 KASSERT(0); 3339 } 3340 #ifdef DEBUG 3341 wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 3342 KASSERT(off == saveoff); 3343 #endif 3344 } 3345 out: 3346 wapbl_free(scratch1, MAXBSIZE); 3347 wapbl_free(scratch2, MAXBSIZE); 3348 if (!error && mismatchcnt) 3349 error = EFTYPE; 3350 return error; 3351 } 3352 #endif 3353 3354 int 3355 wapbl_replay_write(struct wapbl_replay *wr, struct vnode *fsdevvp) 3356 { 3357 struct wapbl_blk *wb; 3358 size_t i; 3359 off_t off; 3360 void *scratch; 3361 int error = 0; 3362 int fsblklen = 1 << wr->wr_fs_dev_bshift; 3363 3364 KDASSERT(wapbl_replay_isopen(wr)); 3365 3366 scratch = wapbl_alloc(MAXBSIZE); 3367 3368 for (i = 0; i <= wr->wr_blkhashmask; ++i) { 3369 LIST_FOREACH(wb, &wr->wr_blkhash[i], wb_hash) { 3370 off = wb->wb_off; 3371 error = wapbl_circ_read(wr, scratch, fsblklen, &off); 3372 if (error) 3373 break; 3374 error = wapbl_write(scratch, fsblklen, fsdevvp, 3375 wb->wb_blk); 3376 if (error) 3377 break; 3378 } 3379 } 3380 3381 wapbl_free(scratch, MAXBSIZE); 3382 return error; 3383 } 3384 3385 int 3386 wapbl_replay_can_read(struct wapbl_replay *wr, daddr_t blk, long len) 3387 { 3388 int fsblklen = 1 << wr->wr_fs_dev_bshift; 3389 3390 KDASSERT(wapbl_replay_isopen(wr)); 3391 KASSERT((len % fsblklen) == 0); 3392 3393 while (len != 0) { 3394 struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 3395 if (wb) 3396 return 1; 3397 len -= fsblklen; 3398 } 3399 return 0; 3400 } 3401 3402 int 3403 wapbl_replay_read(struct wapbl_replay *wr, void *data, daddr_t blk, long len) 3404 { 3405 int fsblklen = 1 << wr->wr_fs_dev_bshift; 3406 3407 KDASSERT(wapbl_replay_isopen(wr)); 3408 3409 KASSERT((len % fsblklen) == 0); 3410 3411 while (len != 0) { 3412 struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 3413 if (wb) { 3414 off_t off = wb->wb_off; 3415 int error; 3416 error = wapbl_circ_read(wr, data, fsblklen, &off); 3417 if (error) 3418 return error; 3419 } 3420 data = (uint8_t *)data + fsblklen; 3421 len -= fsblklen; 3422 blk++; 3423 } 3424 return 0; 3425 } 3426 3427 #ifdef _KERNEL 3428 3429 MODULE(MODULE_CLASS_VFS, wapbl, NULL); 3430 3431 static int 3432 wapbl_modcmd(modcmd_t cmd, void *arg) 3433 { 3434 3435 switch (cmd) { 3436 case MODULE_CMD_INIT: 3437 wapbl_init(); 3438 return 0; 3439 case MODULE_CMD_FINI: 3440 return wapbl_fini(); 3441 default: 3442 return ENOTTY; 3443 } 3444 } 3445 #endif /* _KERNEL */ 3446