1 /* $NetBSD: vfs_wapbl.c,v 1.58 2013/09/15 15:59:37 martin Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Wasabi Systems, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * This implements file system independent write ahead filesystem logging. 34 */ 35 36 #define WAPBL_INTERNAL 37 38 #include <sys/cdefs.h> 39 __KERNEL_RCSID(0, "$NetBSD: vfs_wapbl.c,v 1.58 2013/09/15 15:59:37 martin Exp $"); 40 41 #include <sys/param.h> 42 #include <sys/bitops.h> 43 44 #ifdef _KERNEL 45 #include <sys/param.h> 46 #include <sys/namei.h> 47 #include <sys/proc.h> 48 #include <sys/sysctl.h> 49 #include <sys/uio.h> 50 #include <sys/vnode.h> 51 #include <sys/file.h> 52 #include <sys/module.h> 53 #include <sys/resourcevar.h> 54 #include <sys/conf.h> 55 #include <sys/mount.h> 56 #include <sys/kernel.h> 57 #include <sys/kauth.h> 58 #include <sys/mutex.h> 59 #include <sys/atomic.h> 60 #include <sys/wapbl.h> 61 #include <sys/wapbl_replay.h> 62 63 #include <miscfs/specfs/specdev.h> 64 65 #define wapbl_alloc(s) kmem_alloc((s), KM_SLEEP) 66 #define wapbl_free(a, s) kmem_free((a), (s)) 67 #define wapbl_calloc(n, s) kmem_zalloc((n)*(s), KM_SLEEP) 68 69 static struct sysctllog *wapbl_sysctl; 70 static int wapbl_flush_disk_cache = 1; 71 static int wapbl_verbose_commit = 0; 72 73 static inline size_t wapbl_space_free(size_t, off_t, off_t); 74 75 #else /* !_KERNEL */ 76 #include <assert.h> 77 #include <errno.h> 78 #include <stdio.h> 79 #include <stdbool.h> 80 #include <stdlib.h> 81 #include <string.h> 82 83 #include <sys/time.h> 84 #include <sys/wapbl.h> 85 #include <sys/wapbl_replay.h> 86 87 #define KDASSERT(x) assert(x) 88 #define KASSERT(x) assert(x) 89 #define wapbl_alloc(s) malloc(s) 90 #define wapbl_free(a, s) free(a) 91 #define wapbl_calloc(n, s) calloc((n), (s)) 92 93 #endif /* !_KERNEL */ 94 95 /* 96 * INTERNAL DATA STRUCTURES 97 */ 98 99 /* 100 * This structure holds per-mount log information. 101 * 102 * Legend: a = atomic access only 103 * r = read-only after init 104 * l = rwlock held 105 * m = mutex held 106 * lm = rwlock held writing or mutex held 107 * u = unlocked access ok 108 * b = bufcache_lock held 109 */ 110 struct wapbl { 111 struct vnode *wl_logvp; /* r: log here */ 112 struct vnode *wl_devvp; /* r: log on this device */ 113 struct mount *wl_mount; /* r: mountpoint wl is associated with */ 114 daddr_t wl_logpbn; /* r: Physical block number of start of log */ 115 int wl_log_dev_bshift; /* r: logarithm of device block size of log 116 device */ 117 int wl_fs_dev_bshift; /* r: logarithm of device block size of 118 filesystem device */ 119 120 unsigned wl_lock_count; /* m: Count of transactions in progress */ 121 122 size_t wl_circ_size; /* r: Number of bytes in buffer of log */ 123 size_t wl_circ_off; /* r: Number of bytes reserved at start */ 124 125 size_t wl_bufcount_max; /* r: Number of buffers reserved for log */ 126 size_t wl_bufbytes_max; /* r: Number of buf bytes reserved for log */ 127 128 off_t wl_head; /* l: Byte offset of log head */ 129 off_t wl_tail; /* l: Byte offset of log tail */ 130 /* 131 * head == tail == 0 means log is empty 132 * head == tail != 0 means log is full 133 * see assertions in wapbl_advance() for other boundary conditions. 134 * only truncate moves the tail, except when flush sets it to 135 * wl_header_size only flush moves the head, except when truncate 136 * sets it to 0. 137 */ 138 139 struct wapbl_wc_header *wl_wc_header; /* l */ 140 void *wl_wc_scratch; /* l: scratch space (XXX: por que?!?) */ 141 142 kmutex_t wl_mtx; /* u: short-term lock */ 143 krwlock_t wl_rwlock; /* u: File system transaction lock */ 144 145 /* 146 * Must be held while accessing 147 * wl_count or wl_bufs or head or tail 148 */ 149 150 /* 151 * Callback called from within the flush routine to flush any extra 152 * bits. Note that flush may be skipped without calling this if 153 * there are no outstanding buffers in the transaction. 154 */ 155 #if _KERNEL 156 wapbl_flush_fn_t wl_flush; /* r */ 157 wapbl_flush_fn_t wl_flush_abort;/* r */ 158 #endif 159 160 size_t wl_bufbytes; /* m: Byte count of pages in wl_bufs */ 161 size_t wl_bufcount; /* m: Count of buffers in wl_bufs */ 162 size_t wl_bcount; /* m: Total bcount of wl_bufs */ 163 164 LIST_HEAD(, buf) wl_bufs; /* m: Buffers in current transaction */ 165 166 kcondvar_t wl_reclaimable_cv; /* m (obviously) */ 167 size_t wl_reclaimable_bytes; /* m: Amount of space available for 168 reclamation by truncate */ 169 int wl_error_count; /* m: # of wl_entries with errors */ 170 size_t wl_reserved_bytes; /* never truncate log smaller than this */ 171 172 #ifdef WAPBL_DEBUG_BUFBYTES 173 size_t wl_unsynced_bufbytes; /* Byte count of unsynced buffers */ 174 #endif 175 176 daddr_t *wl_deallocblks;/* lm: address of block */ 177 int *wl_dealloclens; /* lm: size of block */ 178 int wl_dealloccnt; /* lm: total count */ 179 int wl_dealloclim; /* l: max count */ 180 181 /* hashtable of inode numbers for allocated but unlinked inodes */ 182 /* synch ??? */ 183 LIST_HEAD(wapbl_ino_head, wapbl_ino) *wl_inohash; 184 u_long wl_inohashmask; 185 int wl_inohashcnt; 186 187 SIMPLEQ_HEAD(, wapbl_entry) wl_entries; /* On disk transaction 188 accounting */ 189 190 u_char *wl_buffer; /* l: buffer for wapbl_buffered_write() */ 191 daddr_t wl_buffer_dblk; /* l: buffer disk block address */ 192 size_t wl_buffer_used; /* l: buffer current use */ 193 }; 194 195 #ifdef WAPBL_DEBUG_PRINT 196 int wapbl_debug_print = WAPBL_DEBUG_PRINT; 197 #endif 198 199 /****************************************************************/ 200 #ifdef _KERNEL 201 202 #ifdef WAPBL_DEBUG 203 struct wapbl *wapbl_debug_wl; 204 #endif 205 206 static int wapbl_write_commit(struct wapbl *wl, off_t head, off_t tail); 207 static int wapbl_write_blocks(struct wapbl *wl, off_t *offp); 208 static int wapbl_write_revocations(struct wapbl *wl, off_t *offp); 209 static int wapbl_write_inodes(struct wapbl *wl, off_t *offp); 210 #endif /* _KERNEL */ 211 212 static int wapbl_replay_process(struct wapbl_replay *wr, off_t, off_t); 213 214 static inline size_t wapbl_space_used(size_t avail, off_t head, 215 off_t tail); 216 217 #ifdef _KERNEL 218 219 static struct pool wapbl_entry_pool; 220 221 #define WAPBL_INODETRK_SIZE 83 222 static int wapbl_ino_pool_refcount; 223 static struct pool wapbl_ino_pool; 224 struct wapbl_ino { 225 LIST_ENTRY(wapbl_ino) wi_hash; 226 ino_t wi_ino; 227 mode_t wi_mode; 228 }; 229 230 static void wapbl_inodetrk_init(struct wapbl *wl, u_int size); 231 static void wapbl_inodetrk_free(struct wapbl *wl); 232 static struct wapbl_ino *wapbl_inodetrk_get(struct wapbl *wl, ino_t ino); 233 234 static size_t wapbl_transaction_len(struct wapbl *wl); 235 static inline size_t wapbl_transaction_inodes_len(struct wapbl *wl); 236 237 #if 0 238 int wapbl_replay_verify(struct wapbl_replay *, struct vnode *); 239 #endif 240 241 static int wapbl_replay_isopen1(struct wapbl_replay *); 242 243 /* 244 * This is useful for debugging. If set, the log will 245 * only be truncated when necessary. 246 */ 247 int wapbl_lazy_truncate = 0; 248 249 struct wapbl_ops wapbl_ops = { 250 .wo_wapbl_discard = wapbl_discard, 251 .wo_wapbl_replay_isopen = wapbl_replay_isopen1, 252 .wo_wapbl_replay_can_read = wapbl_replay_can_read, 253 .wo_wapbl_replay_read = wapbl_replay_read, 254 .wo_wapbl_add_buf = wapbl_add_buf, 255 .wo_wapbl_remove_buf = wapbl_remove_buf, 256 .wo_wapbl_resize_buf = wapbl_resize_buf, 257 .wo_wapbl_begin = wapbl_begin, 258 .wo_wapbl_end = wapbl_end, 259 .wo_wapbl_junlock_assert= wapbl_junlock_assert, 260 261 /* XXX: the following is only used to say "this is a wapbl buf" */ 262 .wo_wapbl_biodone = wapbl_biodone, 263 }; 264 265 static int 266 wapbl_sysctl_init(void) 267 { 268 int rv; 269 const struct sysctlnode *rnode, *cnode; 270 271 wapbl_sysctl = NULL; 272 273 rv = sysctl_createv(&wapbl_sysctl, 0, NULL, &rnode, 274 CTLFLAG_PERMANENT, 275 CTLTYPE_NODE, "vfs", NULL, 276 NULL, 0, NULL, 0, 277 CTL_VFS, CTL_EOL); 278 if (rv) 279 return rv; 280 281 rv = sysctl_createv(&wapbl_sysctl, 0, &rnode, &rnode, 282 CTLFLAG_PERMANENT, 283 CTLTYPE_NODE, "wapbl", 284 SYSCTL_DESCR("WAPBL journaling options"), 285 NULL, 0, NULL, 0, 286 CTL_CREATE, CTL_EOL); 287 if (rv) 288 return rv; 289 290 rv = sysctl_createv(&wapbl_sysctl, 0, &rnode, &cnode, 291 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 292 CTLTYPE_INT, "flush_disk_cache", 293 SYSCTL_DESCR("flush disk cache"), 294 NULL, 0, &wapbl_flush_disk_cache, 0, 295 CTL_CREATE, CTL_EOL); 296 if (rv) 297 return rv; 298 299 rv = sysctl_createv(&wapbl_sysctl, 0, &rnode, &cnode, 300 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 301 CTLTYPE_INT, "verbose_commit", 302 SYSCTL_DESCR("show time and size of wapbl log commits"), 303 NULL, 0, &wapbl_verbose_commit, 0, 304 CTL_CREATE, CTL_EOL); 305 return rv; 306 } 307 308 static void 309 wapbl_init(void) 310 { 311 312 pool_init(&wapbl_entry_pool, sizeof(struct wapbl_entry), 0, 0, 0, 313 "wapblentrypl", &pool_allocator_kmem, IPL_VM); 314 315 wapbl_sysctl_init(); 316 } 317 318 #ifdef notyet 319 static int 320 wapbl_fini(bool interface) 321 { 322 323 if (aio_sysctl != NULL) 324 sysctl_teardown(&aio_sysctl); 325 326 pool_destroy(&wapbl_entry_pool); 327 328 return 0; 329 } 330 #endif 331 332 static int 333 wapbl_start_flush_inodes(struct wapbl *wl, struct wapbl_replay *wr) 334 { 335 int error, i; 336 337 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 338 ("wapbl_start: reusing log with %d inodes\n", wr->wr_inodescnt)); 339 340 /* 341 * Its only valid to reuse the replay log if its 342 * the same as the new log we just opened. 343 */ 344 KDASSERT(!wapbl_replay_isopen(wr)); 345 KASSERT(wl->wl_devvp->v_type == VBLK); 346 KASSERT(wr->wr_devvp->v_type == VBLK); 347 KASSERT(wl->wl_devvp->v_rdev == wr->wr_devvp->v_rdev); 348 KASSERT(wl->wl_logpbn == wr->wr_logpbn); 349 KASSERT(wl->wl_circ_size == wr->wr_circ_size); 350 KASSERT(wl->wl_circ_off == wr->wr_circ_off); 351 KASSERT(wl->wl_log_dev_bshift == wr->wr_log_dev_bshift); 352 KASSERT(wl->wl_fs_dev_bshift == wr->wr_fs_dev_bshift); 353 354 wl->wl_wc_header->wc_generation = wr->wr_generation + 1; 355 356 for (i = 0; i < wr->wr_inodescnt; i++) 357 wapbl_register_inode(wl, wr->wr_inodes[i].wr_inumber, 358 wr->wr_inodes[i].wr_imode); 359 360 /* Make sure new transaction won't overwrite old inodes list */ 361 KDASSERT(wapbl_transaction_len(wl) <= 362 wapbl_space_free(wl->wl_circ_size, wr->wr_inodeshead, 363 wr->wr_inodestail)); 364 365 wl->wl_head = wl->wl_tail = wr->wr_inodeshead; 366 wl->wl_reclaimable_bytes = wl->wl_reserved_bytes = 367 wapbl_transaction_len(wl); 368 369 error = wapbl_write_inodes(wl, &wl->wl_head); 370 if (error) 371 return error; 372 373 KASSERT(wl->wl_head != wl->wl_tail); 374 KASSERT(wl->wl_head != 0); 375 376 return 0; 377 } 378 379 int 380 wapbl_start(struct wapbl ** wlp, struct mount *mp, struct vnode *vp, 381 daddr_t off, size_t count, size_t blksize, struct wapbl_replay *wr, 382 wapbl_flush_fn_t flushfn, wapbl_flush_fn_t flushabortfn) 383 { 384 struct wapbl *wl; 385 struct vnode *devvp; 386 daddr_t logpbn; 387 int error; 388 int log_dev_bshift = ilog2(blksize); 389 int fs_dev_bshift = log_dev_bshift; 390 int run; 391 392 WAPBL_PRINTF(WAPBL_PRINT_OPEN, ("wapbl_start: vp=%p off=%" PRId64 393 " count=%zu blksize=%zu\n", vp, off, count, blksize)); 394 395 if (log_dev_bshift > fs_dev_bshift) { 396 WAPBL_PRINTF(WAPBL_PRINT_OPEN, 397 ("wapbl: log device's block size cannot be larger " 398 "than filesystem's\n")); 399 /* 400 * Not currently implemented, although it could be if 401 * needed someday. 402 */ 403 return ENOSYS; 404 } 405 406 if (off < 0) 407 return EINVAL; 408 409 if (blksize < DEV_BSIZE) 410 return EINVAL; 411 if (blksize % DEV_BSIZE) 412 return EINVAL; 413 414 /* XXXTODO: verify that the full load is writable */ 415 416 /* 417 * XXX check for minimum log size 418 * minimum is governed by minimum amount of space 419 * to complete a transaction. (probably truncate) 420 */ 421 /* XXX for now pick something minimal */ 422 if ((count * blksize) < MAXPHYS) { 423 return ENOSPC; 424 } 425 426 if ((error = VOP_BMAP(vp, off, &devvp, &logpbn, &run)) != 0) { 427 return error; 428 } 429 430 wl = wapbl_calloc(1, sizeof(*wl)); 431 rw_init(&wl->wl_rwlock); 432 mutex_init(&wl->wl_mtx, MUTEX_DEFAULT, IPL_NONE); 433 cv_init(&wl->wl_reclaimable_cv, "wapblrec"); 434 LIST_INIT(&wl->wl_bufs); 435 SIMPLEQ_INIT(&wl->wl_entries); 436 437 wl->wl_logvp = vp; 438 wl->wl_devvp = devvp; 439 wl->wl_mount = mp; 440 wl->wl_logpbn = logpbn; 441 wl->wl_log_dev_bshift = log_dev_bshift; 442 wl->wl_fs_dev_bshift = fs_dev_bshift; 443 444 wl->wl_flush = flushfn; 445 wl->wl_flush_abort = flushabortfn; 446 447 /* Reserve two log device blocks for the commit headers */ 448 wl->wl_circ_off = 2<<wl->wl_log_dev_bshift; 449 wl->wl_circ_size = ((count * blksize) - wl->wl_circ_off); 450 /* truncate the log usage to a multiple of log_dev_bshift */ 451 wl->wl_circ_size >>= wl->wl_log_dev_bshift; 452 wl->wl_circ_size <<= wl->wl_log_dev_bshift; 453 454 /* 455 * wl_bufbytes_max limits the size of the in memory transaction space. 456 * - Since buffers are allocated and accounted for in units of 457 * PAGE_SIZE it is required to be a multiple of PAGE_SIZE 458 * (i.e. 1<<PAGE_SHIFT) 459 * - Since the log device has to be written in units of 460 * 1<<wl_log_dev_bshift it is required to be a mulitple of 461 * 1<<wl_log_dev_bshift. 462 * - Since filesystem will provide data in units of 1<<wl_fs_dev_bshift, 463 * it is convenient to be a multiple of 1<<wl_fs_dev_bshift. 464 * Therefore it must be multiple of the least common multiple of those 465 * three quantities. Fortunately, all of those quantities are 466 * guaranteed to be a power of two, and the least common multiple of 467 * a set of numbers which are all powers of two is simply the maximum 468 * of those numbers. Finally, the maximum logarithm of a power of two 469 * is the same as the log of the maximum power of two. So we can do 470 * the following operations to size wl_bufbytes_max: 471 */ 472 473 /* XXX fix actual number of pages reserved per filesystem. */ 474 wl->wl_bufbytes_max = MIN(wl->wl_circ_size, buf_memcalc() / 2); 475 476 /* Round wl_bufbytes_max to the largest power of two constraint */ 477 wl->wl_bufbytes_max >>= PAGE_SHIFT; 478 wl->wl_bufbytes_max <<= PAGE_SHIFT; 479 wl->wl_bufbytes_max >>= wl->wl_log_dev_bshift; 480 wl->wl_bufbytes_max <<= wl->wl_log_dev_bshift; 481 wl->wl_bufbytes_max >>= wl->wl_fs_dev_bshift; 482 wl->wl_bufbytes_max <<= wl->wl_fs_dev_bshift; 483 484 /* XXX maybe use filesystem fragment size instead of 1024 */ 485 /* XXX fix actual number of buffers reserved per filesystem. */ 486 wl->wl_bufcount_max = (nbuf / 2) * 1024; 487 488 /* XXX tie this into resource estimation */ 489 wl->wl_dealloclim = wl->wl_bufbytes_max / mp->mnt_stat.f_bsize / 2; 490 491 wl->wl_deallocblks = wapbl_alloc(sizeof(*wl->wl_deallocblks) * 492 wl->wl_dealloclim); 493 wl->wl_dealloclens = wapbl_alloc(sizeof(*wl->wl_dealloclens) * 494 wl->wl_dealloclim); 495 496 wl->wl_buffer = wapbl_alloc(MAXPHYS); 497 wl->wl_buffer_used = 0; 498 499 wapbl_inodetrk_init(wl, WAPBL_INODETRK_SIZE); 500 501 /* Initialize the commit header */ 502 { 503 struct wapbl_wc_header *wc; 504 size_t len = 1 << wl->wl_log_dev_bshift; 505 wc = wapbl_calloc(1, len); 506 wc->wc_type = WAPBL_WC_HEADER; 507 wc->wc_len = len; 508 wc->wc_circ_off = wl->wl_circ_off; 509 wc->wc_circ_size = wl->wl_circ_size; 510 /* XXX wc->wc_fsid */ 511 wc->wc_log_dev_bshift = wl->wl_log_dev_bshift; 512 wc->wc_fs_dev_bshift = wl->wl_fs_dev_bshift; 513 wl->wl_wc_header = wc; 514 wl->wl_wc_scratch = wapbl_alloc(len); 515 } 516 517 /* 518 * if there was an existing set of unlinked but 519 * allocated inodes, preserve it in the new 520 * log. 521 */ 522 if (wr && wr->wr_inodescnt) { 523 error = wapbl_start_flush_inodes(wl, wr); 524 if (error) 525 goto errout; 526 } 527 528 error = wapbl_write_commit(wl, wl->wl_head, wl->wl_tail); 529 if (error) { 530 goto errout; 531 } 532 533 *wlp = wl; 534 #if defined(WAPBL_DEBUG) 535 wapbl_debug_wl = wl; 536 #endif 537 538 return 0; 539 errout: 540 wapbl_discard(wl); 541 wapbl_free(wl->wl_wc_scratch, wl->wl_wc_header->wc_len); 542 wapbl_free(wl->wl_wc_header, wl->wl_wc_header->wc_len); 543 wapbl_free(wl->wl_deallocblks, 544 sizeof(*wl->wl_deallocblks) * wl->wl_dealloclim); 545 wapbl_free(wl->wl_dealloclens, 546 sizeof(*wl->wl_dealloclens) * wl->wl_dealloclim); 547 wapbl_free(wl->wl_buffer, MAXPHYS); 548 wapbl_inodetrk_free(wl); 549 wapbl_free(wl, sizeof(*wl)); 550 551 return error; 552 } 553 554 /* 555 * Like wapbl_flush, only discards the transaction 556 * completely 557 */ 558 559 void 560 wapbl_discard(struct wapbl *wl) 561 { 562 struct wapbl_entry *we; 563 struct buf *bp; 564 int i; 565 566 /* 567 * XXX we may consider using upgrade here 568 * if we want to call flush from inside a transaction 569 */ 570 rw_enter(&wl->wl_rwlock, RW_WRITER); 571 wl->wl_flush(wl->wl_mount, wl->wl_deallocblks, wl->wl_dealloclens, 572 wl->wl_dealloccnt); 573 574 #ifdef WAPBL_DEBUG_PRINT 575 { 576 pid_t pid = -1; 577 lwpid_t lid = -1; 578 if (curproc) 579 pid = curproc->p_pid; 580 if (curlwp) 581 lid = curlwp->l_lid; 582 #ifdef WAPBL_DEBUG_BUFBYTES 583 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 584 ("wapbl_discard: thread %d.%d discarding " 585 "transaction\n" 586 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 587 "deallocs=%d inodes=%d\n" 588 "\terrcnt = %u, reclaimable=%zu reserved=%zu " 589 "unsynced=%zu\n", 590 pid, lid, wl->wl_bufcount, wl->wl_bufbytes, 591 wl->wl_bcount, wl->wl_dealloccnt, 592 wl->wl_inohashcnt, wl->wl_error_count, 593 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 594 wl->wl_unsynced_bufbytes)); 595 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 596 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 597 ("\tentry: bufcount = %zu, reclaimable = %zu, " 598 "error = %d, unsynced = %zu\n", 599 we->we_bufcount, we->we_reclaimable_bytes, 600 we->we_error, we->we_unsynced_bufbytes)); 601 } 602 #else /* !WAPBL_DEBUG_BUFBYTES */ 603 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 604 ("wapbl_discard: thread %d.%d discarding transaction\n" 605 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 606 "deallocs=%d inodes=%d\n" 607 "\terrcnt = %u, reclaimable=%zu reserved=%zu\n", 608 pid, lid, wl->wl_bufcount, wl->wl_bufbytes, 609 wl->wl_bcount, wl->wl_dealloccnt, 610 wl->wl_inohashcnt, wl->wl_error_count, 611 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes)); 612 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 613 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 614 ("\tentry: bufcount = %zu, reclaimable = %zu, " 615 "error = %d\n", 616 we->we_bufcount, we->we_reclaimable_bytes, 617 we->we_error)); 618 } 619 #endif /* !WAPBL_DEBUG_BUFBYTES */ 620 } 621 #endif /* WAPBL_DEBUG_PRINT */ 622 623 for (i = 0; i <= wl->wl_inohashmask; i++) { 624 struct wapbl_ino_head *wih; 625 struct wapbl_ino *wi; 626 627 wih = &wl->wl_inohash[i]; 628 while ((wi = LIST_FIRST(wih)) != NULL) { 629 LIST_REMOVE(wi, wi_hash); 630 pool_put(&wapbl_ino_pool, wi); 631 KASSERT(wl->wl_inohashcnt > 0); 632 wl->wl_inohashcnt--; 633 } 634 } 635 636 /* 637 * clean buffer list 638 */ 639 mutex_enter(&bufcache_lock); 640 mutex_enter(&wl->wl_mtx); 641 while ((bp = LIST_FIRST(&wl->wl_bufs)) != NULL) { 642 if (bbusy(bp, 0, 0, &wl->wl_mtx) == 0) { 643 /* 644 * The buffer will be unlocked and 645 * removed from the transaction in brelse 646 */ 647 mutex_exit(&wl->wl_mtx); 648 brelsel(bp, 0); 649 mutex_enter(&wl->wl_mtx); 650 } 651 } 652 mutex_exit(&wl->wl_mtx); 653 mutex_exit(&bufcache_lock); 654 655 /* 656 * Remove references to this wl from wl_entries, free any which 657 * no longer have buffers, others will be freed in wapbl_biodone 658 * when they no longer have any buffers. 659 */ 660 while ((we = SIMPLEQ_FIRST(&wl->wl_entries)) != NULL) { 661 SIMPLEQ_REMOVE_HEAD(&wl->wl_entries, we_entries); 662 /* XXX should we be accumulating wl_error_count 663 * and increasing reclaimable bytes ? */ 664 we->we_wapbl = NULL; 665 if (we->we_bufcount == 0) { 666 #ifdef WAPBL_DEBUG_BUFBYTES 667 KASSERT(we->we_unsynced_bufbytes == 0); 668 #endif 669 pool_put(&wapbl_entry_pool, we); 670 } 671 } 672 673 /* Discard list of deallocs */ 674 wl->wl_dealloccnt = 0; 675 /* XXX should we clear wl_reserved_bytes? */ 676 677 KASSERT(wl->wl_bufbytes == 0); 678 KASSERT(wl->wl_bcount == 0); 679 KASSERT(wl->wl_bufcount == 0); 680 KASSERT(LIST_EMPTY(&wl->wl_bufs)); 681 KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); 682 KASSERT(wl->wl_inohashcnt == 0); 683 684 rw_exit(&wl->wl_rwlock); 685 } 686 687 int 688 wapbl_stop(struct wapbl *wl, int force) 689 { 690 int error; 691 692 WAPBL_PRINTF(WAPBL_PRINT_OPEN, ("wapbl_stop called\n")); 693 error = wapbl_flush(wl, 1); 694 if (error) { 695 if (force) 696 wapbl_discard(wl); 697 else 698 return error; 699 } 700 701 /* Unlinked inodes persist after a flush */ 702 if (wl->wl_inohashcnt) { 703 if (force) { 704 wapbl_discard(wl); 705 } else { 706 return EBUSY; 707 } 708 } 709 710 KASSERT(wl->wl_bufbytes == 0); 711 KASSERT(wl->wl_bcount == 0); 712 KASSERT(wl->wl_bufcount == 0); 713 KASSERT(LIST_EMPTY(&wl->wl_bufs)); 714 KASSERT(wl->wl_dealloccnt == 0); 715 KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); 716 KASSERT(wl->wl_inohashcnt == 0); 717 718 wapbl_free(wl->wl_wc_scratch, wl->wl_wc_header->wc_len); 719 wapbl_free(wl->wl_wc_header, wl->wl_wc_header->wc_len); 720 wapbl_free(wl->wl_deallocblks, 721 sizeof(*wl->wl_deallocblks) * wl->wl_dealloclim); 722 wapbl_free(wl->wl_dealloclens, 723 sizeof(*wl->wl_dealloclens) * wl->wl_dealloclim); 724 wapbl_free(wl->wl_buffer, MAXPHYS); 725 wapbl_inodetrk_free(wl); 726 727 cv_destroy(&wl->wl_reclaimable_cv); 728 mutex_destroy(&wl->wl_mtx); 729 rw_destroy(&wl->wl_rwlock); 730 wapbl_free(wl, sizeof(*wl)); 731 732 return 0; 733 } 734 735 static int 736 wapbl_doio(void *data, size_t len, struct vnode *devvp, daddr_t pbn, int flags) 737 { 738 struct pstats *pstats = curlwp->l_proc->p_stats; 739 struct buf *bp; 740 int error; 741 742 KASSERT((flags & ~(B_WRITE | B_READ)) == 0); 743 KASSERT(devvp->v_type == VBLK); 744 745 if ((flags & (B_WRITE | B_READ)) == B_WRITE) { 746 mutex_enter(devvp->v_interlock); 747 devvp->v_numoutput++; 748 mutex_exit(devvp->v_interlock); 749 pstats->p_ru.ru_oublock++; 750 } else { 751 pstats->p_ru.ru_inblock++; 752 } 753 754 bp = getiobuf(devvp, true); 755 bp->b_flags = flags; 756 bp->b_cflags = BC_BUSY; /* silly & dubious */ 757 bp->b_dev = devvp->v_rdev; 758 bp->b_data = data; 759 bp->b_bufsize = bp->b_resid = bp->b_bcount = len; 760 bp->b_blkno = pbn; 761 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 762 763 WAPBL_PRINTF(WAPBL_PRINT_IO, 764 ("wapbl_doio: %s %d bytes at block %"PRId64" on dev 0x%"PRIx64"\n", 765 BUF_ISWRITE(bp) ? "write" : "read", bp->b_bcount, 766 bp->b_blkno, bp->b_dev)); 767 768 VOP_STRATEGY(devvp, bp); 769 770 error = biowait(bp); 771 putiobuf(bp); 772 773 if (error) { 774 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 775 ("wapbl_doio: %s %zu bytes at block %" PRId64 776 " on dev 0x%"PRIx64" failed with error %d\n", 777 (((flags & (B_WRITE | B_READ)) == B_WRITE) ? 778 "write" : "read"), 779 len, pbn, devvp->v_rdev, error)); 780 } 781 782 return error; 783 } 784 785 int 786 wapbl_write(void *data, size_t len, struct vnode *devvp, daddr_t pbn) 787 { 788 789 return wapbl_doio(data, len, devvp, pbn, B_WRITE); 790 } 791 792 int 793 wapbl_read(void *data, size_t len, struct vnode *devvp, daddr_t pbn) 794 { 795 796 return wapbl_doio(data, len, devvp, pbn, B_READ); 797 } 798 799 /* 800 * Flush buffered data if any. 801 */ 802 static int 803 wapbl_buffered_flush(struct wapbl *wl) 804 { 805 int error; 806 807 if (wl->wl_buffer_used == 0) 808 return 0; 809 810 error = wapbl_doio(wl->wl_buffer, wl->wl_buffer_used, 811 wl->wl_devvp, wl->wl_buffer_dblk, B_WRITE); 812 wl->wl_buffer_used = 0; 813 814 return error; 815 } 816 817 /* 818 * Write data to the log. 819 * Try to coalesce writes and emit MAXPHYS aligned blocks. 820 */ 821 static int 822 wapbl_buffered_write(void *data, size_t len, struct wapbl *wl, daddr_t pbn) 823 { 824 int error; 825 size_t resid; 826 827 /* 828 * If not adjacent to buffered data flush first. Disk block 829 * address is always valid for non-empty buffer. 830 */ 831 if (wl->wl_buffer_used > 0 && 832 pbn != wl->wl_buffer_dblk + btodb(wl->wl_buffer_used)) { 833 error = wapbl_buffered_flush(wl); 834 if (error) 835 return error; 836 } 837 /* 838 * If this write goes to an empty buffer we have to 839 * save the disk block address first. 840 */ 841 if (wl->wl_buffer_used == 0) 842 wl->wl_buffer_dblk = pbn; 843 /* 844 * Remaining space so this buffer ends on a MAXPHYS boundary. 845 * 846 * Cannot become less or equal zero as the buffer would have been 847 * flushed on the last call then. 848 */ 849 resid = MAXPHYS - dbtob(wl->wl_buffer_dblk % btodb(MAXPHYS)) - 850 wl->wl_buffer_used; 851 KASSERT(resid > 0); 852 KASSERT(dbtob(btodb(resid)) == resid); 853 if (len >= resid) { 854 memcpy(wl->wl_buffer + wl->wl_buffer_used, data, resid); 855 wl->wl_buffer_used += resid; 856 error = wapbl_doio(wl->wl_buffer, wl->wl_buffer_used, 857 wl->wl_devvp, wl->wl_buffer_dblk, B_WRITE); 858 data = (uint8_t *)data + resid; 859 len -= resid; 860 wl->wl_buffer_dblk = pbn + btodb(resid); 861 wl->wl_buffer_used = 0; 862 if (error) 863 return error; 864 } 865 KASSERT(len < MAXPHYS); 866 if (len > 0) { 867 memcpy(wl->wl_buffer + wl->wl_buffer_used, data, len); 868 wl->wl_buffer_used += len; 869 } 870 871 return 0; 872 } 873 874 /* 875 * Off is byte offset returns new offset for next write 876 * handles log wraparound 877 */ 878 static int 879 wapbl_circ_write(struct wapbl *wl, void *data, size_t len, off_t *offp) 880 { 881 size_t slen; 882 off_t off = *offp; 883 int error; 884 daddr_t pbn; 885 886 KDASSERT(((len >> wl->wl_log_dev_bshift) << 887 wl->wl_log_dev_bshift) == len); 888 889 if (off < wl->wl_circ_off) 890 off = wl->wl_circ_off; 891 slen = wl->wl_circ_off + wl->wl_circ_size - off; 892 if (slen < len) { 893 pbn = wl->wl_logpbn + (off >> wl->wl_log_dev_bshift); 894 #ifdef _KERNEL 895 pbn = btodb(pbn << wl->wl_log_dev_bshift); 896 #endif 897 error = wapbl_buffered_write(data, slen, wl, pbn); 898 if (error) 899 return error; 900 data = (uint8_t *)data + slen; 901 len -= slen; 902 off = wl->wl_circ_off; 903 } 904 pbn = wl->wl_logpbn + (off >> wl->wl_log_dev_bshift); 905 #ifdef _KERNEL 906 pbn = btodb(pbn << wl->wl_log_dev_bshift); 907 #endif 908 error = wapbl_buffered_write(data, len, wl, pbn); 909 if (error) 910 return error; 911 off += len; 912 if (off >= wl->wl_circ_off + wl->wl_circ_size) 913 off = wl->wl_circ_off; 914 *offp = off; 915 return 0; 916 } 917 918 /****************************************************************/ 919 920 int 921 wapbl_begin(struct wapbl *wl, const char *file, int line) 922 { 923 int doflush; 924 unsigned lockcount; 925 926 KDASSERT(wl); 927 928 /* 929 * XXX this needs to be made much more sophisticated. 930 * perhaps each wapbl_begin could reserve a specified 931 * number of buffers and bytes. 932 */ 933 mutex_enter(&wl->wl_mtx); 934 lockcount = wl->wl_lock_count; 935 doflush = ((wl->wl_bufbytes + (lockcount * MAXPHYS)) > 936 wl->wl_bufbytes_max / 2) || 937 ((wl->wl_bufcount + (lockcount * 10)) > 938 wl->wl_bufcount_max / 2) || 939 (wapbl_transaction_len(wl) > wl->wl_circ_size / 2) || 940 (wl->wl_dealloccnt >= (wl->wl_dealloclim / 2)); 941 mutex_exit(&wl->wl_mtx); 942 943 if (doflush) { 944 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 945 ("force flush lockcnt=%d bufbytes=%zu " 946 "(max=%zu) bufcount=%zu (max=%zu) " 947 "dealloccnt %d (lim=%d)\n", 948 lockcount, wl->wl_bufbytes, 949 wl->wl_bufbytes_max, wl->wl_bufcount, 950 wl->wl_bufcount_max, 951 wl->wl_dealloccnt, wl->wl_dealloclim)); 952 } 953 954 if (doflush) { 955 int error = wapbl_flush(wl, 0); 956 if (error) 957 return error; 958 } 959 960 rw_enter(&wl->wl_rwlock, RW_READER); 961 mutex_enter(&wl->wl_mtx); 962 wl->wl_lock_count++; 963 mutex_exit(&wl->wl_mtx); 964 965 #if defined(WAPBL_DEBUG_PRINT) 966 WAPBL_PRINTF(WAPBL_PRINT_TRANSACTION, 967 ("wapbl_begin thread %d.%d with bufcount=%zu " 968 "bufbytes=%zu bcount=%zu at %s:%d\n", 969 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 970 wl->wl_bufbytes, wl->wl_bcount, file, line)); 971 #endif 972 973 return 0; 974 } 975 976 void 977 wapbl_end(struct wapbl *wl) 978 { 979 980 #if defined(WAPBL_DEBUG_PRINT) 981 WAPBL_PRINTF(WAPBL_PRINT_TRANSACTION, 982 ("wapbl_end thread %d.%d with bufcount=%zu " 983 "bufbytes=%zu bcount=%zu\n", 984 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 985 wl->wl_bufbytes, wl->wl_bcount)); 986 #endif 987 988 #ifdef DIAGNOSTIC 989 size_t flushsize = wapbl_transaction_len(wl); 990 if (flushsize > (wl->wl_circ_size - wl->wl_reserved_bytes)) { 991 /* 992 * XXX this could be handled more gracefully, perhaps place 993 * only a partial transaction in the log and allow the 994 * remaining to flush without the protection of the journal. 995 */ 996 panic("wapbl_end: current transaction too big to flush\n"); 997 } 998 #endif 999 1000 mutex_enter(&wl->wl_mtx); 1001 KASSERT(wl->wl_lock_count > 0); 1002 wl->wl_lock_count--; 1003 mutex_exit(&wl->wl_mtx); 1004 1005 rw_exit(&wl->wl_rwlock); 1006 } 1007 1008 void 1009 wapbl_add_buf(struct wapbl *wl, struct buf * bp) 1010 { 1011 1012 KASSERT(bp->b_cflags & BC_BUSY); 1013 KASSERT(bp->b_vp); 1014 1015 wapbl_jlock_assert(wl); 1016 1017 #if 0 1018 /* 1019 * XXX this might be an issue for swapfiles. 1020 * see uvm_swap.c:1702 1021 * 1022 * XXX2 why require it then? leap of semantics? 1023 */ 1024 KASSERT((bp->b_cflags & BC_NOCACHE) == 0); 1025 #endif 1026 1027 mutex_enter(&wl->wl_mtx); 1028 if (bp->b_flags & B_LOCKED) { 1029 LIST_REMOVE(bp, b_wapbllist); 1030 WAPBL_PRINTF(WAPBL_PRINT_BUFFER2, 1031 ("wapbl_add_buf thread %d.%d re-adding buf %p " 1032 "with %d bytes %d bcount\n", 1033 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, 1034 bp->b_bcount)); 1035 } else { 1036 /* unlocked by dirty buffers shouldn't exist */ 1037 KASSERT(!(bp->b_oflags & BO_DELWRI)); 1038 wl->wl_bufbytes += bp->b_bufsize; 1039 wl->wl_bcount += bp->b_bcount; 1040 wl->wl_bufcount++; 1041 WAPBL_PRINTF(WAPBL_PRINT_BUFFER, 1042 ("wapbl_add_buf thread %d.%d adding buf %p " 1043 "with %d bytes %d bcount\n", 1044 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, 1045 bp->b_bcount)); 1046 } 1047 LIST_INSERT_HEAD(&wl->wl_bufs, bp, b_wapbllist); 1048 mutex_exit(&wl->wl_mtx); 1049 1050 bp->b_flags |= B_LOCKED; 1051 } 1052 1053 static void 1054 wapbl_remove_buf_locked(struct wapbl * wl, struct buf *bp) 1055 { 1056 1057 KASSERT(mutex_owned(&wl->wl_mtx)); 1058 KASSERT(bp->b_cflags & BC_BUSY); 1059 wapbl_jlock_assert(wl); 1060 1061 #if 0 1062 /* 1063 * XXX this might be an issue for swapfiles. 1064 * see uvm_swap.c:1725 1065 * 1066 * XXXdeux: see above 1067 */ 1068 KASSERT((bp->b_flags & BC_NOCACHE) == 0); 1069 #endif 1070 KASSERT(bp->b_flags & B_LOCKED); 1071 1072 WAPBL_PRINTF(WAPBL_PRINT_BUFFER, 1073 ("wapbl_remove_buf thread %d.%d removing buf %p with " 1074 "%d bytes %d bcount\n", 1075 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, bp->b_bcount)); 1076 1077 KASSERT(wl->wl_bufbytes >= bp->b_bufsize); 1078 wl->wl_bufbytes -= bp->b_bufsize; 1079 KASSERT(wl->wl_bcount >= bp->b_bcount); 1080 wl->wl_bcount -= bp->b_bcount; 1081 KASSERT(wl->wl_bufcount > 0); 1082 wl->wl_bufcount--; 1083 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0)); 1084 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0)); 1085 LIST_REMOVE(bp, b_wapbllist); 1086 1087 bp->b_flags &= ~B_LOCKED; 1088 } 1089 1090 /* called from brelsel() in vfs_bio among other places */ 1091 void 1092 wapbl_remove_buf(struct wapbl * wl, struct buf *bp) 1093 { 1094 1095 mutex_enter(&wl->wl_mtx); 1096 wapbl_remove_buf_locked(wl, bp); 1097 mutex_exit(&wl->wl_mtx); 1098 } 1099 1100 void 1101 wapbl_resize_buf(struct wapbl *wl, struct buf *bp, long oldsz, long oldcnt) 1102 { 1103 1104 KASSERT(bp->b_cflags & BC_BUSY); 1105 1106 /* 1107 * XXX: why does this depend on B_LOCKED? otherwise the buf 1108 * is not for a transaction? if so, why is this called in the 1109 * first place? 1110 */ 1111 if (bp->b_flags & B_LOCKED) { 1112 mutex_enter(&wl->wl_mtx); 1113 wl->wl_bufbytes += bp->b_bufsize - oldsz; 1114 wl->wl_bcount += bp->b_bcount - oldcnt; 1115 mutex_exit(&wl->wl_mtx); 1116 } 1117 } 1118 1119 #endif /* _KERNEL */ 1120 1121 /****************************************************************/ 1122 /* Some utility inlines */ 1123 1124 static inline size_t 1125 wapbl_space_used(size_t avail, off_t head, off_t tail) 1126 { 1127 1128 if (tail == 0) { 1129 KASSERT(head == 0); 1130 return 0; 1131 } 1132 return ((head + (avail - 1) - tail) % avail) + 1; 1133 } 1134 1135 #ifdef _KERNEL 1136 /* This is used to advance the pointer at old to new value at old+delta */ 1137 static inline off_t 1138 wapbl_advance(size_t size, size_t off, off_t old, size_t delta) 1139 { 1140 off_t new; 1141 1142 /* Define acceptable ranges for inputs. */ 1143 KASSERT(delta <= (size_t)size); 1144 KASSERT((old == 0) || ((size_t)old >= off)); 1145 KASSERT(old < (off_t)(size + off)); 1146 1147 if ((old == 0) && (delta != 0)) 1148 new = off + delta; 1149 else if ((old + delta) < (size + off)) 1150 new = old + delta; 1151 else 1152 new = (old + delta) - size; 1153 1154 /* Note some interesting axioms */ 1155 KASSERT((delta != 0) || (new == old)); 1156 KASSERT((delta == 0) || (new != 0)); 1157 KASSERT((delta != (size)) || (new == old)); 1158 1159 /* Define acceptable ranges for output. */ 1160 KASSERT((new == 0) || ((size_t)new >= off)); 1161 KASSERT((size_t)new < (size + off)); 1162 return new; 1163 } 1164 1165 static inline size_t 1166 wapbl_space_free(size_t avail, off_t head, off_t tail) 1167 { 1168 1169 return avail - wapbl_space_used(avail, head, tail); 1170 } 1171 1172 static inline void 1173 wapbl_advance_head(size_t size, size_t off, size_t delta, off_t *headp, 1174 off_t *tailp) 1175 { 1176 off_t head = *headp; 1177 off_t tail = *tailp; 1178 1179 KASSERT(delta <= wapbl_space_free(size, head, tail)); 1180 head = wapbl_advance(size, off, head, delta); 1181 if ((tail == 0) && (head != 0)) 1182 tail = off; 1183 *headp = head; 1184 *tailp = tail; 1185 } 1186 1187 static inline void 1188 wapbl_advance_tail(size_t size, size_t off, size_t delta, off_t *headp, 1189 off_t *tailp) 1190 { 1191 off_t head = *headp; 1192 off_t tail = *tailp; 1193 1194 KASSERT(delta <= wapbl_space_used(size, head, tail)); 1195 tail = wapbl_advance(size, off, tail, delta); 1196 if (head == tail) { 1197 head = tail = 0; 1198 } 1199 *headp = head; 1200 *tailp = tail; 1201 } 1202 1203 1204 /****************************************************************/ 1205 1206 /* 1207 * Remove transactions whose buffers are completely flushed to disk. 1208 * Will block until at least minfree space is available. 1209 * only intended to be called from inside wapbl_flush and therefore 1210 * does not protect against commit races with itself or with flush. 1211 */ 1212 static int 1213 wapbl_truncate(struct wapbl *wl, size_t minfree, int waitonly) 1214 { 1215 size_t delta; 1216 size_t avail; 1217 off_t head; 1218 off_t tail; 1219 int error = 0; 1220 1221 KASSERT(minfree <= (wl->wl_circ_size - wl->wl_reserved_bytes)); 1222 KASSERT(rw_write_held(&wl->wl_rwlock)); 1223 1224 mutex_enter(&wl->wl_mtx); 1225 1226 /* 1227 * First check to see if we have to do a commit 1228 * at all. 1229 */ 1230 avail = wapbl_space_free(wl->wl_circ_size, wl->wl_head, wl->wl_tail); 1231 if (minfree < avail) { 1232 mutex_exit(&wl->wl_mtx); 1233 return 0; 1234 } 1235 minfree -= avail; 1236 while ((wl->wl_error_count == 0) && 1237 (wl->wl_reclaimable_bytes < minfree)) { 1238 WAPBL_PRINTF(WAPBL_PRINT_TRUNCATE, 1239 ("wapbl_truncate: sleeping on %p wl=%p bytes=%zd " 1240 "minfree=%zd\n", 1241 &wl->wl_reclaimable_bytes, wl, wl->wl_reclaimable_bytes, 1242 minfree)); 1243 1244 cv_wait(&wl->wl_reclaimable_cv, &wl->wl_mtx); 1245 } 1246 if (wl->wl_reclaimable_bytes < minfree) { 1247 KASSERT(wl->wl_error_count); 1248 /* XXX maybe get actual error from buffer instead someday? */ 1249 error = EIO; 1250 } 1251 head = wl->wl_head; 1252 tail = wl->wl_tail; 1253 delta = wl->wl_reclaimable_bytes; 1254 1255 /* If all of of the entries are flushed, then be sure to keep 1256 * the reserved bytes reserved. Watch out for discarded transactions, 1257 * which could leave more bytes reserved than are reclaimable. 1258 */ 1259 if (SIMPLEQ_EMPTY(&wl->wl_entries) && 1260 (delta >= wl->wl_reserved_bytes)) { 1261 delta -= wl->wl_reserved_bytes; 1262 } 1263 wapbl_advance_tail(wl->wl_circ_size, wl->wl_circ_off, delta, &head, 1264 &tail); 1265 KDASSERT(wl->wl_reserved_bytes <= 1266 wapbl_space_used(wl->wl_circ_size, head, tail)); 1267 mutex_exit(&wl->wl_mtx); 1268 1269 if (error) 1270 return error; 1271 1272 if (waitonly) 1273 return 0; 1274 1275 /* 1276 * This is where head, tail and delta are unprotected 1277 * from races against itself or flush. This is ok since 1278 * we only call this routine from inside flush itself. 1279 * 1280 * XXX: how can it race against itself when accessed only 1281 * from behind the write-locked rwlock? 1282 */ 1283 error = wapbl_write_commit(wl, head, tail); 1284 if (error) 1285 return error; 1286 1287 wl->wl_head = head; 1288 wl->wl_tail = tail; 1289 1290 mutex_enter(&wl->wl_mtx); 1291 KASSERT(wl->wl_reclaimable_bytes >= delta); 1292 wl->wl_reclaimable_bytes -= delta; 1293 mutex_exit(&wl->wl_mtx); 1294 WAPBL_PRINTF(WAPBL_PRINT_TRUNCATE, 1295 ("wapbl_truncate thread %d.%d truncating %zu bytes\n", 1296 curproc->p_pid, curlwp->l_lid, delta)); 1297 1298 return 0; 1299 } 1300 1301 /****************************************************************/ 1302 1303 void 1304 wapbl_biodone(struct buf *bp) 1305 { 1306 struct wapbl_entry *we = bp->b_private; 1307 struct wapbl *wl = we->we_wapbl; 1308 #ifdef WAPBL_DEBUG_BUFBYTES 1309 const int bufsize = bp->b_bufsize; 1310 #endif 1311 1312 /* 1313 * Handle possible flushing of buffers after log has been 1314 * decomissioned. 1315 */ 1316 if (!wl) { 1317 KASSERT(we->we_bufcount > 0); 1318 we->we_bufcount--; 1319 #ifdef WAPBL_DEBUG_BUFBYTES 1320 KASSERT(we->we_unsynced_bufbytes >= bufsize); 1321 we->we_unsynced_bufbytes -= bufsize; 1322 #endif 1323 1324 if (we->we_bufcount == 0) { 1325 #ifdef WAPBL_DEBUG_BUFBYTES 1326 KASSERT(we->we_unsynced_bufbytes == 0); 1327 #endif 1328 pool_put(&wapbl_entry_pool, we); 1329 } 1330 1331 brelse(bp, 0); 1332 return; 1333 } 1334 1335 #ifdef ohbother 1336 KDASSERT(bp->b_oflags & BO_DONE); 1337 KDASSERT(!(bp->b_oflags & BO_DELWRI)); 1338 KDASSERT(bp->b_flags & B_ASYNC); 1339 KDASSERT(bp->b_cflags & BC_BUSY); 1340 KDASSERT(!(bp->b_flags & B_LOCKED)); 1341 KDASSERT(!(bp->b_flags & B_READ)); 1342 KDASSERT(!(bp->b_cflags & BC_INVAL)); 1343 KDASSERT(!(bp->b_cflags & BC_NOCACHE)); 1344 #endif 1345 1346 if (bp->b_error) { 1347 #ifdef notyet /* Can't currently handle possible dirty buffer reuse */ 1348 /* 1349 * XXXpooka: interfaces not fully updated 1350 * Note: this was not enabled in the original patch 1351 * against netbsd4 either. I don't know if comment 1352 * above is true or not. 1353 */ 1354 1355 /* 1356 * If an error occurs, report the error and leave the 1357 * buffer as a delayed write on the LRU queue. 1358 * restarting the write would likely result in 1359 * an error spinloop, so let it be done harmlessly 1360 * by the syncer. 1361 */ 1362 bp->b_flags &= ~(B_DONE); 1363 simple_unlock(&bp->b_interlock); 1364 1365 if (we->we_error == 0) { 1366 mutex_enter(&wl->wl_mtx); 1367 wl->wl_error_count++; 1368 mutex_exit(&wl->wl_mtx); 1369 cv_broadcast(&wl->wl_reclaimable_cv); 1370 } 1371 we->we_error = bp->b_error; 1372 bp->b_error = 0; 1373 brelse(bp); 1374 return; 1375 #else 1376 /* For now, just mark the log permanently errored out */ 1377 1378 mutex_enter(&wl->wl_mtx); 1379 if (wl->wl_error_count == 0) { 1380 wl->wl_error_count++; 1381 cv_broadcast(&wl->wl_reclaimable_cv); 1382 } 1383 mutex_exit(&wl->wl_mtx); 1384 #endif 1385 } 1386 1387 /* 1388 * Release the buffer here. wapbl_flush() may wait for the 1389 * log to become empty and we better unbusy the buffer before 1390 * wapbl_flush() returns. 1391 */ 1392 brelse(bp, 0); 1393 1394 mutex_enter(&wl->wl_mtx); 1395 1396 KASSERT(we->we_bufcount > 0); 1397 we->we_bufcount--; 1398 #ifdef WAPBL_DEBUG_BUFBYTES 1399 KASSERT(we->we_unsynced_bufbytes >= bufsize); 1400 we->we_unsynced_bufbytes -= bufsize; 1401 KASSERT(wl->wl_unsynced_bufbytes >= bufsize); 1402 wl->wl_unsynced_bufbytes -= bufsize; 1403 #endif 1404 1405 /* 1406 * If the current transaction can be reclaimed, start 1407 * at the beginning and reclaim any consecutive reclaimable 1408 * transactions. If we successfully reclaim anything, 1409 * then wakeup anyone waiting for the reclaim. 1410 */ 1411 if (we->we_bufcount == 0) { 1412 size_t delta = 0; 1413 int errcnt = 0; 1414 #ifdef WAPBL_DEBUG_BUFBYTES 1415 KDASSERT(we->we_unsynced_bufbytes == 0); 1416 #endif 1417 /* 1418 * clear any posted error, since the buffer it came from 1419 * has successfully flushed by now 1420 */ 1421 while ((we = SIMPLEQ_FIRST(&wl->wl_entries)) && 1422 (we->we_bufcount == 0)) { 1423 delta += we->we_reclaimable_bytes; 1424 if (we->we_error) 1425 errcnt++; 1426 SIMPLEQ_REMOVE_HEAD(&wl->wl_entries, we_entries); 1427 pool_put(&wapbl_entry_pool, we); 1428 } 1429 1430 if (delta) { 1431 wl->wl_reclaimable_bytes += delta; 1432 KASSERT(wl->wl_error_count >= errcnt); 1433 wl->wl_error_count -= errcnt; 1434 cv_broadcast(&wl->wl_reclaimable_cv); 1435 } 1436 } 1437 1438 mutex_exit(&wl->wl_mtx); 1439 } 1440 1441 /* 1442 * Write transactions to disk + start I/O for contents 1443 */ 1444 int 1445 wapbl_flush(struct wapbl *wl, int waitfor) 1446 { 1447 struct buf *bp; 1448 struct wapbl_entry *we; 1449 off_t off; 1450 off_t head; 1451 off_t tail; 1452 size_t delta = 0; 1453 size_t flushsize; 1454 size_t reserved; 1455 int error = 0; 1456 1457 /* 1458 * Do a quick check to see if a full flush can be skipped 1459 * This assumes that the flush callback does not need to be called 1460 * unless there are other outstanding bufs. 1461 */ 1462 if (!waitfor) { 1463 size_t nbufs; 1464 mutex_enter(&wl->wl_mtx); /* XXX need mutex here to 1465 protect the KASSERTS */ 1466 nbufs = wl->wl_bufcount; 1467 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0)); 1468 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0)); 1469 mutex_exit(&wl->wl_mtx); 1470 if (nbufs == 0) 1471 return 0; 1472 } 1473 1474 /* 1475 * XXX we may consider using LK_UPGRADE here 1476 * if we want to call flush from inside a transaction 1477 */ 1478 rw_enter(&wl->wl_rwlock, RW_WRITER); 1479 wl->wl_flush(wl->wl_mount, wl->wl_deallocblks, wl->wl_dealloclens, 1480 wl->wl_dealloccnt); 1481 1482 /* 1483 * Now that we are fully locked and flushed, 1484 * do another check for nothing to do. 1485 */ 1486 if (wl->wl_bufcount == 0) { 1487 goto out; 1488 } 1489 1490 #if 0 1491 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1492 ("wapbl_flush thread %d.%d flushing entries with " 1493 "bufcount=%zu bufbytes=%zu\n", 1494 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 1495 wl->wl_bufbytes)); 1496 #endif 1497 1498 /* Calculate amount of space needed to flush */ 1499 flushsize = wapbl_transaction_len(wl); 1500 if (wapbl_verbose_commit) { 1501 struct timespec ts; 1502 getnanotime(&ts); 1503 printf("%s: %lld.%09ld this transaction = %zu bytes\n", 1504 __func__, (long long)ts.tv_sec, 1505 (long)ts.tv_nsec, flushsize); 1506 } 1507 1508 if (flushsize > (wl->wl_circ_size - wl->wl_reserved_bytes)) { 1509 /* 1510 * XXX this could be handled more gracefully, perhaps place 1511 * only a partial transaction in the log and allow the 1512 * remaining to flush without the protection of the journal. 1513 */ 1514 panic("wapbl_flush: current transaction too big to flush\n"); 1515 } 1516 1517 error = wapbl_truncate(wl, flushsize, 0); 1518 if (error) 1519 goto out2; 1520 1521 off = wl->wl_head; 1522 KASSERT((off == 0) || ((off >= wl->wl_circ_off) && 1523 (off < wl->wl_circ_off + wl->wl_circ_size))); 1524 error = wapbl_write_blocks(wl, &off); 1525 if (error) 1526 goto out2; 1527 error = wapbl_write_revocations(wl, &off); 1528 if (error) 1529 goto out2; 1530 error = wapbl_write_inodes(wl, &off); 1531 if (error) 1532 goto out2; 1533 1534 reserved = 0; 1535 if (wl->wl_inohashcnt) 1536 reserved = wapbl_transaction_inodes_len(wl); 1537 1538 head = wl->wl_head; 1539 tail = wl->wl_tail; 1540 1541 wapbl_advance_head(wl->wl_circ_size, wl->wl_circ_off, flushsize, 1542 &head, &tail); 1543 #ifdef WAPBL_DEBUG 1544 if (head != off) { 1545 panic("lost head! head=%"PRIdMAX" tail=%" PRIdMAX 1546 " off=%"PRIdMAX" flush=%zu\n", 1547 (intmax_t)head, (intmax_t)tail, (intmax_t)off, 1548 flushsize); 1549 } 1550 #else 1551 KASSERT(head == off); 1552 #endif 1553 1554 /* Opportunistically move the tail forward if we can */ 1555 if (!wapbl_lazy_truncate) { 1556 mutex_enter(&wl->wl_mtx); 1557 delta = wl->wl_reclaimable_bytes; 1558 mutex_exit(&wl->wl_mtx); 1559 wapbl_advance_tail(wl->wl_circ_size, wl->wl_circ_off, delta, 1560 &head, &tail); 1561 } 1562 1563 error = wapbl_write_commit(wl, head, tail); 1564 if (error) 1565 goto out2; 1566 1567 we = pool_get(&wapbl_entry_pool, PR_WAITOK); 1568 1569 #ifdef WAPBL_DEBUG_BUFBYTES 1570 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1571 ("wapbl_flush: thread %d.%d head+=%zu tail+=%zu used=%zu" 1572 " unsynced=%zu" 1573 "\n\tbufcount=%zu bufbytes=%zu bcount=%zu deallocs=%d " 1574 "inodes=%d\n", 1575 curproc->p_pid, curlwp->l_lid, flushsize, delta, 1576 wapbl_space_used(wl->wl_circ_size, head, tail), 1577 wl->wl_unsynced_bufbytes, wl->wl_bufcount, 1578 wl->wl_bufbytes, wl->wl_bcount, wl->wl_dealloccnt, 1579 wl->wl_inohashcnt)); 1580 #else 1581 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1582 ("wapbl_flush: thread %d.%d head+=%zu tail+=%zu used=%zu" 1583 "\n\tbufcount=%zu bufbytes=%zu bcount=%zu deallocs=%d " 1584 "inodes=%d\n", 1585 curproc->p_pid, curlwp->l_lid, flushsize, delta, 1586 wapbl_space_used(wl->wl_circ_size, head, tail), 1587 wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount, 1588 wl->wl_dealloccnt, wl->wl_inohashcnt)); 1589 #endif 1590 1591 1592 mutex_enter(&bufcache_lock); 1593 mutex_enter(&wl->wl_mtx); 1594 1595 wl->wl_reserved_bytes = reserved; 1596 wl->wl_head = head; 1597 wl->wl_tail = tail; 1598 KASSERT(wl->wl_reclaimable_bytes >= delta); 1599 wl->wl_reclaimable_bytes -= delta; 1600 wl->wl_dealloccnt = 0; 1601 #ifdef WAPBL_DEBUG_BUFBYTES 1602 wl->wl_unsynced_bufbytes += wl->wl_bufbytes; 1603 #endif 1604 1605 we->we_wapbl = wl; 1606 we->we_bufcount = wl->wl_bufcount; 1607 #ifdef WAPBL_DEBUG_BUFBYTES 1608 we->we_unsynced_bufbytes = wl->wl_bufbytes; 1609 #endif 1610 we->we_reclaimable_bytes = flushsize; 1611 we->we_error = 0; 1612 SIMPLEQ_INSERT_TAIL(&wl->wl_entries, we, we_entries); 1613 1614 /* 1615 * this flushes bufs in reverse order than they were queued 1616 * it shouldn't matter, but if we care we could use TAILQ instead. 1617 * XXX Note they will get put on the lru queue when they flush 1618 * so we might actually want to change this to preserve order. 1619 */ 1620 while ((bp = LIST_FIRST(&wl->wl_bufs)) != NULL) { 1621 if (bbusy(bp, 0, 0, &wl->wl_mtx)) { 1622 continue; 1623 } 1624 bp->b_iodone = wapbl_biodone; 1625 bp->b_private = we; 1626 bremfree(bp); 1627 wapbl_remove_buf_locked(wl, bp); 1628 mutex_exit(&wl->wl_mtx); 1629 mutex_exit(&bufcache_lock); 1630 bawrite(bp); 1631 mutex_enter(&bufcache_lock); 1632 mutex_enter(&wl->wl_mtx); 1633 } 1634 mutex_exit(&wl->wl_mtx); 1635 mutex_exit(&bufcache_lock); 1636 1637 #if 0 1638 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1639 ("wapbl_flush thread %d.%d done flushing entries...\n", 1640 curproc->p_pid, curlwp->l_lid)); 1641 #endif 1642 1643 out: 1644 1645 /* 1646 * If the waitfor flag is set, don't return until everything is 1647 * fully flushed and the on disk log is empty. 1648 */ 1649 if (waitfor) { 1650 error = wapbl_truncate(wl, wl->wl_circ_size - 1651 wl->wl_reserved_bytes, wapbl_lazy_truncate); 1652 } 1653 1654 out2: 1655 if (error) { 1656 wl->wl_flush_abort(wl->wl_mount, wl->wl_deallocblks, 1657 wl->wl_dealloclens, wl->wl_dealloccnt); 1658 } 1659 1660 #ifdef WAPBL_DEBUG_PRINT 1661 if (error) { 1662 pid_t pid = -1; 1663 lwpid_t lid = -1; 1664 if (curproc) 1665 pid = curproc->p_pid; 1666 if (curlwp) 1667 lid = curlwp->l_lid; 1668 mutex_enter(&wl->wl_mtx); 1669 #ifdef WAPBL_DEBUG_BUFBYTES 1670 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1671 ("wapbl_flush: thread %d.%d aborted flush: " 1672 "error = %d\n" 1673 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 1674 "deallocs=%d inodes=%d\n" 1675 "\terrcnt = %d, reclaimable=%zu reserved=%zu " 1676 "unsynced=%zu\n", 1677 pid, lid, error, wl->wl_bufcount, 1678 wl->wl_bufbytes, wl->wl_bcount, 1679 wl->wl_dealloccnt, wl->wl_inohashcnt, 1680 wl->wl_error_count, wl->wl_reclaimable_bytes, 1681 wl->wl_reserved_bytes, wl->wl_unsynced_bufbytes)); 1682 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 1683 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1684 ("\tentry: bufcount = %zu, reclaimable = %zu, " 1685 "error = %d, unsynced = %zu\n", 1686 we->we_bufcount, we->we_reclaimable_bytes, 1687 we->we_error, we->we_unsynced_bufbytes)); 1688 } 1689 #else 1690 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1691 ("wapbl_flush: thread %d.%d aborted flush: " 1692 "error = %d\n" 1693 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 1694 "deallocs=%d inodes=%d\n" 1695 "\terrcnt = %d, reclaimable=%zu reserved=%zu\n", 1696 pid, lid, error, wl->wl_bufcount, 1697 wl->wl_bufbytes, wl->wl_bcount, 1698 wl->wl_dealloccnt, wl->wl_inohashcnt, 1699 wl->wl_error_count, wl->wl_reclaimable_bytes, 1700 wl->wl_reserved_bytes)); 1701 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 1702 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1703 ("\tentry: bufcount = %zu, reclaimable = %zu, " 1704 "error = %d\n", we->we_bufcount, 1705 we->we_reclaimable_bytes, we->we_error)); 1706 } 1707 #endif 1708 mutex_exit(&wl->wl_mtx); 1709 } 1710 #endif 1711 1712 rw_exit(&wl->wl_rwlock); 1713 return error; 1714 } 1715 1716 /****************************************************************/ 1717 1718 void 1719 wapbl_jlock_assert(struct wapbl *wl) 1720 { 1721 1722 KASSERT(rw_lock_held(&wl->wl_rwlock)); 1723 } 1724 1725 void 1726 wapbl_junlock_assert(struct wapbl *wl) 1727 { 1728 1729 KASSERT(!rw_write_held(&wl->wl_rwlock)); 1730 } 1731 1732 /****************************************************************/ 1733 1734 /* locks missing */ 1735 void 1736 wapbl_print(struct wapbl *wl, 1737 int full, 1738 void (*pr)(const char *, ...)) 1739 { 1740 struct buf *bp; 1741 struct wapbl_entry *we; 1742 (*pr)("wapbl %p", wl); 1743 (*pr)("\nlogvp = %p, devvp = %p, logpbn = %"PRId64"\n", 1744 wl->wl_logvp, wl->wl_devvp, wl->wl_logpbn); 1745 (*pr)("circ = %zu, header = %zu, head = %"PRIdMAX" tail = %"PRIdMAX"\n", 1746 wl->wl_circ_size, wl->wl_circ_off, 1747 (intmax_t)wl->wl_head, (intmax_t)wl->wl_tail); 1748 (*pr)("fs_dev_bshift = %d, log_dev_bshift = %d\n", 1749 wl->wl_log_dev_bshift, wl->wl_fs_dev_bshift); 1750 #ifdef WAPBL_DEBUG_BUFBYTES 1751 (*pr)("bufcount = %zu, bufbytes = %zu bcount = %zu reclaimable = %zu " 1752 "reserved = %zu errcnt = %d unsynced = %zu\n", 1753 wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount, 1754 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 1755 wl->wl_error_count, wl->wl_unsynced_bufbytes); 1756 #else 1757 (*pr)("bufcount = %zu, bufbytes = %zu bcount = %zu reclaimable = %zu " 1758 "reserved = %zu errcnt = %d\n", wl->wl_bufcount, wl->wl_bufbytes, 1759 wl->wl_bcount, wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 1760 wl->wl_error_count); 1761 #endif 1762 (*pr)("\tdealloccnt = %d, dealloclim = %d\n", 1763 wl->wl_dealloccnt, wl->wl_dealloclim); 1764 (*pr)("\tinohashcnt = %d, inohashmask = 0x%08x\n", 1765 wl->wl_inohashcnt, wl->wl_inohashmask); 1766 (*pr)("entries:\n"); 1767 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 1768 #ifdef WAPBL_DEBUG_BUFBYTES 1769 (*pr)("\tbufcount = %zu, reclaimable = %zu, error = %d, " 1770 "unsynced = %zu\n", 1771 we->we_bufcount, we->we_reclaimable_bytes, 1772 we->we_error, we->we_unsynced_bufbytes); 1773 #else 1774 (*pr)("\tbufcount = %zu, reclaimable = %zu, error = %d\n", 1775 we->we_bufcount, we->we_reclaimable_bytes, we->we_error); 1776 #endif 1777 } 1778 if (full) { 1779 int cnt = 0; 1780 (*pr)("bufs ="); 1781 LIST_FOREACH(bp, &wl->wl_bufs, b_wapbllist) { 1782 if (!LIST_NEXT(bp, b_wapbllist)) { 1783 (*pr)(" %p", bp); 1784 } else if ((++cnt % 6) == 0) { 1785 (*pr)(" %p,\n\t", bp); 1786 } else { 1787 (*pr)(" %p,", bp); 1788 } 1789 } 1790 (*pr)("\n"); 1791 1792 (*pr)("dealloced blks = "); 1793 { 1794 int i; 1795 cnt = 0; 1796 for (i = 0; i < wl->wl_dealloccnt; i++) { 1797 (*pr)(" %"PRId64":%d,", 1798 wl->wl_deallocblks[i], 1799 wl->wl_dealloclens[i]); 1800 if ((++cnt % 4) == 0) { 1801 (*pr)("\n\t"); 1802 } 1803 } 1804 } 1805 (*pr)("\n"); 1806 1807 (*pr)("registered inodes = "); 1808 { 1809 int i; 1810 cnt = 0; 1811 for (i = 0; i <= wl->wl_inohashmask; i++) { 1812 struct wapbl_ino_head *wih; 1813 struct wapbl_ino *wi; 1814 1815 wih = &wl->wl_inohash[i]; 1816 LIST_FOREACH(wi, wih, wi_hash) { 1817 if (wi->wi_ino == 0) 1818 continue; 1819 (*pr)(" %"PRIu64"/0%06"PRIo32",", 1820 wi->wi_ino, wi->wi_mode); 1821 if ((++cnt % 4) == 0) { 1822 (*pr)("\n\t"); 1823 } 1824 } 1825 } 1826 (*pr)("\n"); 1827 } 1828 } 1829 } 1830 1831 #if defined(WAPBL_DEBUG) || defined(DDB) 1832 void 1833 wapbl_dump(struct wapbl *wl) 1834 { 1835 #if defined(WAPBL_DEBUG) 1836 if (!wl) 1837 wl = wapbl_debug_wl; 1838 #endif 1839 if (!wl) 1840 return; 1841 wapbl_print(wl, 1, printf); 1842 } 1843 #endif 1844 1845 /****************************************************************/ 1846 1847 void 1848 wapbl_register_deallocation(struct wapbl *wl, daddr_t blk, int len) 1849 { 1850 1851 wapbl_jlock_assert(wl); 1852 1853 mutex_enter(&wl->wl_mtx); 1854 /* XXX should eventually instead tie this into resource estimation */ 1855 /* 1856 * XXX this panic needs locking/mutex analysis and the 1857 * ability to cope with the failure. 1858 */ 1859 /* XXX this XXX doesn't have enough XXX */ 1860 if (__predict_false(wl->wl_dealloccnt >= wl->wl_dealloclim)) 1861 panic("wapbl_register_deallocation: out of resources"); 1862 1863 wl->wl_deallocblks[wl->wl_dealloccnt] = blk; 1864 wl->wl_dealloclens[wl->wl_dealloccnt] = len; 1865 wl->wl_dealloccnt++; 1866 WAPBL_PRINTF(WAPBL_PRINT_ALLOC, 1867 ("wapbl_register_deallocation: blk=%"PRId64" len=%d\n", blk, len)); 1868 mutex_exit(&wl->wl_mtx); 1869 } 1870 1871 /****************************************************************/ 1872 1873 static void 1874 wapbl_inodetrk_init(struct wapbl *wl, u_int size) 1875 { 1876 1877 wl->wl_inohash = hashinit(size, HASH_LIST, true, &wl->wl_inohashmask); 1878 if (atomic_inc_uint_nv(&wapbl_ino_pool_refcount) == 1) { 1879 pool_init(&wapbl_ino_pool, sizeof(struct wapbl_ino), 0, 0, 0, 1880 "wapblinopl", &pool_allocator_nointr, IPL_NONE); 1881 } 1882 } 1883 1884 static void 1885 wapbl_inodetrk_free(struct wapbl *wl) 1886 { 1887 1888 /* XXX this KASSERT needs locking/mutex analysis */ 1889 KASSERT(wl->wl_inohashcnt == 0); 1890 hashdone(wl->wl_inohash, HASH_LIST, wl->wl_inohashmask); 1891 if (atomic_dec_uint_nv(&wapbl_ino_pool_refcount) == 0) { 1892 pool_destroy(&wapbl_ino_pool); 1893 } 1894 } 1895 1896 static struct wapbl_ino * 1897 wapbl_inodetrk_get(struct wapbl *wl, ino_t ino) 1898 { 1899 struct wapbl_ino_head *wih; 1900 struct wapbl_ino *wi; 1901 1902 KASSERT(mutex_owned(&wl->wl_mtx)); 1903 1904 wih = &wl->wl_inohash[ino & wl->wl_inohashmask]; 1905 LIST_FOREACH(wi, wih, wi_hash) { 1906 if (ino == wi->wi_ino) 1907 return wi; 1908 } 1909 return 0; 1910 } 1911 1912 void 1913 wapbl_register_inode(struct wapbl *wl, ino_t ino, mode_t mode) 1914 { 1915 struct wapbl_ino_head *wih; 1916 struct wapbl_ino *wi; 1917 1918 wi = pool_get(&wapbl_ino_pool, PR_WAITOK); 1919 1920 mutex_enter(&wl->wl_mtx); 1921 if (wapbl_inodetrk_get(wl, ino) == NULL) { 1922 wi->wi_ino = ino; 1923 wi->wi_mode = mode; 1924 wih = &wl->wl_inohash[ino & wl->wl_inohashmask]; 1925 LIST_INSERT_HEAD(wih, wi, wi_hash); 1926 wl->wl_inohashcnt++; 1927 WAPBL_PRINTF(WAPBL_PRINT_INODE, 1928 ("wapbl_register_inode: ino=%"PRId64"\n", ino)); 1929 mutex_exit(&wl->wl_mtx); 1930 } else { 1931 mutex_exit(&wl->wl_mtx); 1932 pool_put(&wapbl_ino_pool, wi); 1933 } 1934 } 1935 1936 void 1937 wapbl_unregister_inode(struct wapbl *wl, ino_t ino, mode_t mode) 1938 { 1939 struct wapbl_ino *wi; 1940 1941 mutex_enter(&wl->wl_mtx); 1942 wi = wapbl_inodetrk_get(wl, ino); 1943 if (wi) { 1944 WAPBL_PRINTF(WAPBL_PRINT_INODE, 1945 ("wapbl_unregister_inode: ino=%"PRId64"\n", ino)); 1946 KASSERT(wl->wl_inohashcnt > 0); 1947 wl->wl_inohashcnt--; 1948 LIST_REMOVE(wi, wi_hash); 1949 mutex_exit(&wl->wl_mtx); 1950 1951 pool_put(&wapbl_ino_pool, wi); 1952 } else { 1953 mutex_exit(&wl->wl_mtx); 1954 } 1955 } 1956 1957 /****************************************************************/ 1958 1959 static inline size_t 1960 wapbl_transaction_inodes_len(struct wapbl *wl) 1961 { 1962 int blocklen = 1<<wl->wl_log_dev_bshift; 1963 int iph; 1964 1965 /* Calculate number of inodes described in a inodelist header */ 1966 iph = (blocklen - offsetof(struct wapbl_wc_inodelist, wc_inodes)) / 1967 sizeof(((struct wapbl_wc_inodelist *)0)->wc_inodes[0]); 1968 1969 KASSERT(iph > 0); 1970 1971 return MAX(1, howmany(wl->wl_inohashcnt, iph)) * blocklen; 1972 } 1973 1974 1975 /* Calculate amount of space a transaction will take on disk */ 1976 static size_t 1977 wapbl_transaction_len(struct wapbl *wl) 1978 { 1979 int blocklen = 1<<wl->wl_log_dev_bshift; 1980 size_t len; 1981 int bph; 1982 1983 /* Calculate number of blocks described in a blocklist header */ 1984 bph = (blocklen - offsetof(struct wapbl_wc_blocklist, wc_blocks)) / 1985 sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); 1986 1987 KASSERT(bph > 0); 1988 1989 len = wl->wl_bcount; 1990 len += howmany(wl->wl_bufcount, bph) * blocklen; 1991 len += howmany(wl->wl_dealloccnt, bph) * blocklen; 1992 len += wapbl_transaction_inodes_len(wl); 1993 1994 return len; 1995 } 1996 1997 /* 1998 * wapbl_cache_sync: issue DIOCCACHESYNC 1999 */ 2000 static int 2001 wapbl_cache_sync(struct wapbl *wl, const char *msg) 2002 { 2003 const bool verbose = wapbl_verbose_commit >= 2; 2004 struct bintime start_time; 2005 int force = 1; 2006 int error; 2007 2008 if (!wapbl_flush_disk_cache) { 2009 return 0; 2010 } 2011 if (verbose) { 2012 bintime(&start_time); 2013 } 2014 error = VOP_IOCTL(wl->wl_devvp, DIOCCACHESYNC, &force, 2015 FWRITE, FSCRED); 2016 if (error) { 2017 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 2018 ("wapbl_cache_sync: DIOCCACHESYNC on dev 0x%x " 2019 "returned %d\n", wl->wl_devvp->v_rdev, error)); 2020 } 2021 if (verbose) { 2022 struct bintime d; 2023 struct timespec ts; 2024 2025 bintime(&d); 2026 bintime_sub(&d, &start_time); 2027 bintime2timespec(&d, &ts); 2028 printf("wapbl_cache_sync: %s: dev 0x%jx %ju.%09lu\n", 2029 msg, (uintmax_t)wl->wl_devvp->v_rdev, 2030 (uintmax_t)ts.tv_sec, ts.tv_nsec); 2031 } 2032 return error; 2033 } 2034 2035 /* 2036 * Perform commit operation 2037 * 2038 * Note that generation number incrementation needs to 2039 * be protected against racing with other invocations 2040 * of wapbl_write_commit. This is ok since this routine 2041 * is only invoked from wapbl_flush 2042 */ 2043 static int 2044 wapbl_write_commit(struct wapbl *wl, off_t head, off_t tail) 2045 { 2046 struct wapbl_wc_header *wc = wl->wl_wc_header; 2047 struct timespec ts; 2048 int error; 2049 daddr_t pbn; 2050 2051 error = wapbl_buffered_flush(wl); 2052 if (error) 2053 return error; 2054 /* 2055 * flush disk cache to ensure that blocks we've written are actually 2056 * written to the stable storage before the commit header. 2057 * 2058 * XXX Calc checksum here, instead we do this for now 2059 */ 2060 wapbl_cache_sync(wl, "1"); 2061 2062 wc->wc_head = head; 2063 wc->wc_tail = tail; 2064 wc->wc_checksum = 0; 2065 wc->wc_version = 1; 2066 getnanotime(&ts); 2067 wc->wc_time = ts.tv_sec; 2068 wc->wc_timensec = ts.tv_nsec; 2069 2070 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2071 ("wapbl_write_commit: head = %"PRIdMAX "tail = %"PRIdMAX"\n", 2072 (intmax_t)head, (intmax_t)tail)); 2073 2074 /* 2075 * write the commit header. 2076 * 2077 * XXX if generation will rollover, then first zero 2078 * over second commit header before trying to write both headers. 2079 */ 2080 2081 pbn = wl->wl_logpbn + (wc->wc_generation % 2); 2082 #ifdef _KERNEL 2083 pbn = btodb(pbn << wc->wc_log_dev_bshift); 2084 #endif 2085 error = wapbl_buffered_write(wc, wc->wc_len, wl, pbn); 2086 if (error) 2087 return error; 2088 error = wapbl_buffered_flush(wl); 2089 if (error) 2090 return error; 2091 2092 /* 2093 * flush disk cache to ensure that the commit header is actually 2094 * written before meta data blocks. 2095 */ 2096 wapbl_cache_sync(wl, "2"); 2097 2098 /* 2099 * If the generation number was zero, write it out a second time. 2100 * This handles initialization and generation number rollover 2101 */ 2102 if (wc->wc_generation++ == 0) { 2103 error = wapbl_write_commit(wl, head, tail); 2104 /* 2105 * This panic should be able to be removed if we do the 2106 * zero'ing mentioned above, and we are certain to roll 2107 * back generation number on failure. 2108 */ 2109 if (error) 2110 panic("wapbl_write_commit: error writing duplicate " 2111 "log header: %d\n", error); 2112 } 2113 return 0; 2114 } 2115 2116 /* Returns new offset value */ 2117 static int 2118 wapbl_write_blocks(struct wapbl *wl, off_t *offp) 2119 { 2120 struct wapbl_wc_blocklist *wc = 2121 (struct wapbl_wc_blocklist *)wl->wl_wc_scratch; 2122 int blocklen = 1<<wl->wl_log_dev_bshift; 2123 int bph; 2124 struct buf *bp; 2125 off_t off = *offp; 2126 int error; 2127 size_t padding; 2128 2129 KASSERT(rw_write_held(&wl->wl_rwlock)); 2130 2131 bph = (blocklen - offsetof(struct wapbl_wc_blocklist, wc_blocks)) / 2132 sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); 2133 2134 bp = LIST_FIRST(&wl->wl_bufs); 2135 2136 while (bp) { 2137 int cnt; 2138 struct buf *obp = bp; 2139 2140 KASSERT(bp->b_flags & B_LOCKED); 2141 2142 wc->wc_type = WAPBL_WC_BLOCKS; 2143 wc->wc_len = blocklen; 2144 wc->wc_blkcount = 0; 2145 while (bp && (wc->wc_blkcount < bph)) { 2146 /* 2147 * Make sure all the physical block numbers are up to 2148 * date. If this is not always true on a given 2149 * filesystem, then VOP_BMAP must be called. We 2150 * could call VOP_BMAP here, or else in the filesystem 2151 * specific flush callback, although neither of those 2152 * solutions allow us to take the vnode lock. If a 2153 * filesystem requires that we must take the vnode lock 2154 * to call VOP_BMAP, then we can probably do it in 2155 * bwrite when the vnode lock should already be held 2156 * by the invoking code. 2157 */ 2158 KASSERT((bp->b_vp->v_type == VBLK) || 2159 (bp->b_blkno != bp->b_lblkno)); 2160 KASSERT(bp->b_blkno > 0); 2161 2162 wc->wc_blocks[wc->wc_blkcount].wc_daddr = bp->b_blkno; 2163 wc->wc_blocks[wc->wc_blkcount].wc_dlen = bp->b_bcount; 2164 wc->wc_len += bp->b_bcount; 2165 wc->wc_blkcount++; 2166 bp = LIST_NEXT(bp, b_wapbllist); 2167 } 2168 if (wc->wc_len % blocklen != 0) { 2169 padding = blocklen - wc->wc_len % blocklen; 2170 wc->wc_len += padding; 2171 } else { 2172 padding = 0; 2173 } 2174 2175 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2176 ("wapbl_write_blocks: len = %u (padding %zu) off = %"PRIdMAX"\n", 2177 wc->wc_len, padding, (intmax_t)off)); 2178 2179 error = wapbl_circ_write(wl, wc, blocklen, &off); 2180 if (error) 2181 return error; 2182 bp = obp; 2183 cnt = 0; 2184 while (bp && (cnt++ < bph)) { 2185 error = wapbl_circ_write(wl, bp->b_data, 2186 bp->b_bcount, &off); 2187 if (error) 2188 return error; 2189 bp = LIST_NEXT(bp, b_wapbllist); 2190 } 2191 if (padding) { 2192 void *zero; 2193 2194 zero = wapbl_alloc(padding); 2195 memset(zero, 0, padding); 2196 error = wapbl_circ_write(wl, zero, padding, &off); 2197 wapbl_free(zero, padding); 2198 if (error) 2199 return error; 2200 } 2201 } 2202 *offp = off; 2203 return 0; 2204 } 2205 2206 static int 2207 wapbl_write_revocations(struct wapbl *wl, off_t *offp) 2208 { 2209 struct wapbl_wc_blocklist *wc = 2210 (struct wapbl_wc_blocklist *)wl->wl_wc_scratch; 2211 int i; 2212 int blocklen = 1<<wl->wl_log_dev_bshift; 2213 int bph; 2214 off_t off = *offp; 2215 int error; 2216 2217 if (wl->wl_dealloccnt == 0) 2218 return 0; 2219 2220 bph = (blocklen - offsetof(struct wapbl_wc_blocklist, wc_blocks)) / 2221 sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); 2222 2223 i = 0; 2224 while (i < wl->wl_dealloccnt) { 2225 wc->wc_type = WAPBL_WC_REVOCATIONS; 2226 wc->wc_len = blocklen; 2227 wc->wc_blkcount = 0; 2228 while ((i < wl->wl_dealloccnt) && (wc->wc_blkcount < bph)) { 2229 wc->wc_blocks[wc->wc_blkcount].wc_daddr = 2230 wl->wl_deallocblks[i]; 2231 wc->wc_blocks[wc->wc_blkcount].wc_dlen = 2232 wl->wl_dealloclens[i]; 2233 wc->wc_blkcount++; 2234 i++; 2235 } 2236 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2237 ("wapbl_write_revocations: len = %u off = %"PRIdMAX"\n", 2238 wc->wc_len, (intmax_t)off)); 2239 error = wapbl_circ_write(wl, wc, blocklen, &off); 2240 if (error) 2241 return error; 2242 } 2243 *offp = off; 2244 return 0; 2245 } 2246 2247 static int 2248 wapbl_write_inodes(struct wapbl *wl, off_t *offp) 2249 { 2250 struct wapbl_wc_inodelist *wc = 2251 (struct wapbl_wc_inodelist *)wl->wl_wc_scratch; 2252 int i; 2253 int blocklen = 1 << wl->wl_log_dev_bshift; 2254 off_t off = *offp; 2255 int error; 2256 2257 struct wapbl_ino_head *wih; 2258 struct wapbl_ino *wi; 2259 int iph; 2260 2261 iph = (blocklen - offsetof(struct wapbl_wc_inodelist, wc_inodes)) / 2262 sizeof(((struct wapbl_wc_inodelist *)0)->wc_inodes[0]); 2263 2264 i = 0; 2265 wih = &wl->wl_inohash[0]; 2266 wi = 0; 2267 do { 2268 wc->wc_type = WAPBL_WC_INODES; 2269 wc->wc_len = blocklen; 2270 wc->wc_inocnt = 0; 2271 wc->wc_clear = (i == 0); 2272 while ((i < wl->wl_inohashcnt) && (wc->wc_inocnt < iph)) { 2273 while (!wi) { 2274 KASSERT((wih - &wl->wl_inohash[0]) 2275 <= wl->wl_inohashmask); 2276 wi = LIST_FIRST(wih++); 2277 } 2278 wc->wc_inodes[wc->wc_inocnt].wc_inumber = wi->wi_ino; 2279 wc->wc_inodes[wc->wc_inocnt].wc_imode = wi->wi_mode; 2280 wc->wc_inocnt++; 2281 i++; 2282 wi = LIST_NEXT(wi, wi_hash); 2283 } 2284 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2285 ("wapbl_write_inodes: len = %u off = %"PRIdMAX"\n", 2286 wc->wc_len, (intmax_t)off)); 2287 error = wapbl_circ_write(wl, wc, blocklen, &off); 2288 if (error) 2289 return error; 2290 } while (i < wl->wl_inohashcnt); 2291 2292 *offp = off; 2293 return 0; 2294 } 2295 2296 #endif /* _KERNEL */ 2297 2298 /****************************************************************/ 2299 2300 struct wapbl_blk { 2301 LIST_ENTRY(wapbl_blk) wb_hash; 2302 daddr_t wb_blk; 2303 off_t wb_off; /* Offset of this block in the log */ 2304 }; 2305 #define WAPBL_BLKPOOL_MIN 83 2306 2307 static void 2308 wapbl_blkhash_init(struct wapbl_replay *wr, u_int size) 2309 { 2310 if (size < WAPBL_BLKPOOL_MIN) 2311 size = WAPBL_BLKPOOL_MIN; 2312 KASSERT(wr->wr_blkhash == 0); 2313 #ifdef _KERNEL 2314 wr->wr_blkhash = hashinit(size, HASH_LIST, true, &wr->wr_blkhashmask); 2315 #else /* ! _KERNEL */ 2316 /* Manually implement hashinit */ 2317 { 2318 unsigned long i, hashsize; 2319 for (hashsize = 1; hashsize < size; hashsize <<= 1) 2320 continue; 2321 wr->wr_blkhash = wapbl_alloc(hashsize * sizeof(*wr->wr_blkhash)); 2322 for (i = 0; i < hashsize; i++) 2323 LIST_INIT(&wr->wr_blkhash[i]); 2324 wr->wr_blkhashmask = hashsize - 1; 2325 } 2326 #endif /* ! _KERNEL */ 2327 } 2328 2329 static void 2330 wapbl_blkhash_free(struct wapbl_replay *wr) 2331 { 2332 KASSERT(wr->wr_blkhashcnt == 0); 2333 #ifdef _KERNEL 2334 hashdone(wr->wr_blkhash, HASH_LIST, wr->wr_blkhashmask); 2335 #else /* ! _KERNEL */ 2336 wapbl_free(wr->wr_blkhash, 2337 (wr->wr_blkhashmask + 1) * sizeof(*wr->wr_blkhash)); 2338 #endif /* ! _KERNEL */ 2339 } 2340 2341 static struct wapbl_blk * 2342 wapbl_blkhash_get(struct wapbl_replay *wr, daddr_t blk) 2343 { 2344 struct wapbl_blk_head *wbh; 2345 struct wapbl_blk *wb; 2346 wbh = &wr->wr_blkhash[blk & wr->wr_blkhashmask]; 2347 LIST_FOREACH(wb, wbh, wb_hash) { 2348 if (blk == wb->wb_blk) 2349 return wb; 2350 } 2351 return 0; 2352 } 2353 2354 static void 2355 wapbl_blkhash_ins(struct wapbl_replay *wr, daddr_t blk, off_t off) 2356 { 2357 struct wapbl_blk_head *wbh; 2358 struct wapbl_blk *wb; 2359 wb = wapbl_blkhash_get(wr, blk); 2360 if (wb) { 2361 KASSERT(wb->wb_blk == blk); 2362 wb->wb_off = off; 2363 } else { 2364 wb = wapbl_alloc(sizeof(*wb)); 2365 wb->wb_blk = blk; 2366 wb->wb_off = off; 2367 wbh = &wr->wr_blkhash[blk & wr->wr_blkhashmask]; 2368 LIST_INSERT_HEAD(wbh, wb, wb_hash); 2369 wr->wr_blkhashcnt++; 2370 } 2371 } 2372 2373 static void 2374 wapbl_blkhash_rem(struct wapbl_replay *wr, daddr_t blk) 2375 { 2376 struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 2377 if (wb) { 2378 KASSERT(wr->wr_blkhashcnt > 0); 2379 wr->wr_blkhashcnt--; 2380 LIST_REMOVE(wb, wb_hash); 2381 wapbl_free(wb, sizeof(*wb)); 2382 } 2383 } 2384 2385 static void 2386 wapbl_blkhash_clear(struct wapbl_replay *wr) 2387 { 2388 unsigned long i; 2389 for (i = 0; i <= wr->wr_blkhashmask; i++) { 2390 struct wapbl_blk *wb; 2391 2392 while ((wb = LIST_FIRST(&wr->wr_blkhash[i]))) { 2393 KASSERT(wr->wr_blkhashcnt > 0); 2394 wr->wr_blkhashcnt--; 2395 LIST_REMOVE(wb, wb_hash); 2396 wapbl_free(wb, sizeof(*wb)); 2397 } 2398 } 2399 KASSERT(wr->wr_blkhashcnt == 0); 2400 } 2401 2402 /****************************************************************/ 2403 2404 static int 2405 wapbl_circ_read(struct wapbl_replay *wr, void *data, size_t len, off_t *offp) 2406 { 2407 size_t slen; 2408 off_t off = *offp; 2409 int error; 2410 daddr_t pbn; 2411 2412 KASSERT(((len >> wr->wr_log_dev_bshift) << 2413 wr->wr_log_dev_bshift) == len); 2414 2415 if (off < wr->wr_circ_off) 2416 off = wr->wr_circ_off; 2417 slen = wr->wr_circ_off + wr->wr_circ_size - off; 2418 if (slen < len) { 2419 pbn = wr->wr_logpbn + (off >> wr->wr_log_dev_bshift); 2420 #ifdef _KERNEL 2421 pbn = btodb(pbn << wr->wr_log_dev_bshift); 2422 #endif 2423 error = wapbl_read(data, slen, wr->wr_devvp, pbn); 2424 if (error) 2425 return error; 2426 data = (uint8_t *)data + slen; 2427 len -= slen; 2428 off = wr->wr_circ_off; 2429 } 2430 pbn = wr->wr_logpbn + (off >> wr->wr_log_dev_bshift); 2431 #ifdef _KERNEL 2432 pbn = btodb(pbn << wr->wr_log_dev_bshift); 2433 #endif 2434 error = wapbl_read(data, len, wr->wr_devvp, pbn); 2435 if (error) 2436 return error; 2437 off += len; 2438 if (off >= wr->wr_circ_off + wr->wr_circ_size) 2439 off = wr->wr_circ_off; 2440 *offp = off; 2441 return 0; 2442 } 2443 2444 static void 2445 wapbl_circ_advance(struct wapbl_replay *wr, size_t len, off_t *offp) 2446 { 2447 size_t slen; 2448 off_t off = *offp; 2449 2450 KASSERT(((len >> wr->wr_log_dev_bshift) << 2451 wr->wr_log_dev_bshift) == len); 2452 2453 if (off < wr->wr_circ_off) 2454 off = wr->wr_circ_off; 2455 slen = wr->wr_circ_off + wr->wr_circ_size - off; 2456 if (slen < len) { 2457 len -= slen; 2458 off = wr->wr_circ_off; 2459 } 2460 off += len; 2461 if (off >= wr->wr_circ_off + wr->wr_circ_size) 2462 off = wr->wr_circ_off; 2463 *offp = off; 2464 } 2465 2466 /****************************************************************/ 2467 2468 int 2469 wapbl_replay_start(struct wapbl_replay **wrp, struct vnode *vp, 2470 daddr_t off, size_t count, size_t blksize) 2471 { 2472 struct wapbl_replay *wr; 2473 int error; 2474 struct vnode *devvp; 2475 daddr_t logpbn; 2476 uint8_t *scratch; 2477 struct wapbl_wc_header *wch; 2478 struct wapbl_wc_header *wch2; 2479 /* Use this until we read the actual log header */ 2480 int log_dev_bshift = ilog2(blksize); 2481 size_t used; 2482 daddr_t pbn; 2483 2484 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 2485 ("wapbl_replay_start: vp=%p off=%"PRId64 " count=%zu blksize=%zu\n", 2486 vp, off, count, blksize)); 2487 2488 if (off < 0) 2489 return EINVAL; 2490 2491 if (blksize < DEV_BSIZE) 2492 return EINVAL; 2493 if (blksize % DEV_BSIZE) 2494 return EINVAL; 2495 2496 #ifdef _KERNEL 2497 #if 0 2498 /* XXX vp->v_size isn't reliably set for VBLK devices, 2499 * especially root. However, we might still want to verify 2500 * that the full load is readable */ 2501 if ((off + count) * blksize > vp->v_size) 2502 return EINVAL; 2503 #endif 2504 if ((error = VOP_BMAP(vp, off, &devvp, &logpbn, 0)) != 0) { 2505 return error; 2506 } 2507 #else /* ! _KERNEL */ 2508 devvp = vp; 2509 logpbn = off; 2510 #endif /* ! _KERNEL */ 2511 2512 scratch = wapbl_alloc(MAXBSIZE); 2513 2514 pbn = logpbn; 2515 #ifdef _KERNEL 2516 pbn = btodb(pbn << log_dev_bshift); 2517 #endif 2518 error = wapbl_read(scratch, 2<<log_dev_bshift, devvp, pbn); 2519 if (error) 2520 goto errout; 2521 2522 wch = (struct wapbl_wc_header *)scratch; 2523 wch2 = 2524 (struct wapbl_wc_header *)(scratch + (1<<log_dev_bshift)); 2525 /* XXX verify checksums and magic numbers */ 2526 if (wch->wc_type != WAPBL_WC_HEADER) { 2527 printf("Unrecognized wapbl magic: 0x%08x\n", wch->wc_type); 2528 error = EFTYPE; 2529 goto errout; 2530 } 2531 2532 if (wch2->wc_generation > wch->wc_generation) 2533 wch = wch2; 2534 2535 wr = wapbl_calloc(1, sizeof(*wr)); 2536 2537 wr->wr_logvp = vp; 2538 wr->wr_devvp = devvp; 2539 wr->wr_logpbn = logpbn; 2540 2541 wr->wr_scratch = scratch; 2542 2543 wr->wr_log_dev_bshift = wch->wc_log_dev_bshift; 2544 wr->wr_fs_dev_bshift = wch->wc_fs_dev_bshift; 2545 wr->wr_circ_off = wch->wc_circ_off; 2546 wr->wr_circ_size = wch->wc_circ_size; 2547 wr->wr_generation = wch->wc_generation; 2548 2549 used = wapbl_space_used(wch->wc_circ_size, wch->wc_head, wch->wc_tail); 2550 2551 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 2552 ("wapbl_replay: head=%"PRId64" tail=%"PRId64" off=%"PRId64 2553 " len=%"PRId64" used=%zu\n", 2554 wch->wc_head, wch->wc_tail, wch->wc_circ_off, 2555 wch->wc_circ_size, used)); 2556 2557 wapbl_blkhash_init(wr, (used >> wch->wc_fs_dev_bshift)); 2558 2559 error = wapbl_replay_process(wr, wch->wc_head, wch->wc_tail); 2560 if (error) { 2561 wapbl_replay_stop(wr); 2562 wapbl_replay_free(wr); 2563 return error; 2564 } 2565 2566 *wrp = wr; 2567 return 0; 2568 2569 errout: 2570 wapbl_free(scratch, MAXBSIZE); 2571 return error; 2572 } 2573 2574 void 2575 wapbl_replay_stop(struct wapbl_replay *wr) 2576 { 2577 2578 if (!wapbl_replay_isopen(wr)) 2579 return; 2580 2581 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, ("wapbl_replay_stop called\n")); 2582 2583 wapbl_free(wr->wr_scratch, MAXBSIZE); 2584 wr->wr_scratch = NULL; 2585 2586 wr->wr_logvp = NULL; 2587 2588 wapbl_blkhash_clear(wr); 2589 wapbl_blkhash_free(wr); 2590 } 2591 2592 void 2593 wapbl_replay_free(struct wapbl_replay *wr) 2594 { 2595 2596 KDASSERT(!wapbl_replay_isopen(wr)); 2597 2598 if (wr->wr_inodes) 2599 wapbl_free(wr->wr_inodes, 2600 wr->wr_inodescnt * sizeof(wr->wr_inodes[0])); 2601 wapbl_free(wr, sizeof(*wr)); 2602 } 2603 2604 #ifdef _KERNEL 2605 int 2606 wapbl_replay_isopen1(struct wapbl_replay *wr) 2607 { 2608 2609 return wapbl_replay_isopen(wr); 2610 } 2611 #endif 2612 2613 static void 2614 wapbl_replay_process_blocks(struct wapbl_replay *wr, off_t *offp) 2615 { 2616 struct wapbl_wc_blocklist *wc = 2617 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2618 int fsblklen = 1 << wr->wr_fs_dev_bshift; 2619 int i, j, n; 2620 2621 for (i = 0; i < wc->wc_blkcount; i++) { 2622 /* 2623 * Enter each physical block into the hashtable independently. 2624 */ 2625 n = wc->wc_blocks[i].wc_dlen >> wr->wr_fs_dev_bshift; 2626 for (j = 0; j < n; j++) { 2627 wapbl_blkhash_ins(wr, wc->wc_blocks[i].wc_daddr + btodb(j * fsblklen), 2628 *offp); 2629 wapbl_circ_advance(wr, fsblklen, offp); 2630 } 2631 } 2632 } 2633 2634 static void 2635 wapbl_replay_process_revocations(struct wapbl_replay *wr) 2636 { 2637 struct wapbl_wc_blocklist *wc = 2638 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2639 int fsblklen = 1 << wr->wr_fs_dev_bshift; 2640 int i, j, n; 2641 2642 for (i = 0; i < wc->wc_blkcount; i++) { 2643 /* 2644 * Remove any blocks found from the hashtable. 2645 */ 2646 n = wc->wc_blocks[i].wc_dlen >> wr->wr_fs_dev_bshift; 2647 for (j = 0; j < n; j++) 2648 wapbl_blkhash_rem(wr, wc->wc_blocks[i].wc_daddr + btodb(j * fsblklen)); 2649 } 2650 } 2651 2652 static void 2653 wapbl_replay_process_inodes(struct wapbl_replay *wr, off_t oldoff, off_t newoff) 2654 { 2655 struct wapbl_wc_inodelist *wc = 2656 (struct wapbl_wc_inodelist *)wr->wr_scratch; 2657 void *new_inodes; 2658 const size_t oldsize = wr->wr_inodescnt * sizeof(wr->wr_inodes[0]); 2659 2660 KASSERT(sizeof(wr->wr_inodes[0]) == sizeof(wc->wc_inodes[0])); 2661 2662 /* 2663 * Keep track of where we found this so location won't be 2664 * overwritten. 2665 */ 2666 if (wc->wc_clear) { 2667 wr->wr_inodestail = oldoff; 2668 wr->wr_inodescnt = 0; 2669 if (wr->wr_inodes != NULL) { 2670 wapbl_free(wr->wr_inodes, oldsize); 2671 wr->wr_inodes = NULL; 2672 } 2673 } 2674 wr->wr_inodeshead = newoff; 2675 if (wc->wc_inocnt == 0) 2676 return; 2677 2678 new_inodes = wapbl_alloc((wr->wr_inodescnt + wc->wc_inocnt) * 2679 sizeof(wr->wr_inodes[0])); 2680 if (wr->wr_inodes != NULL) { 2681 memcpy(new_inodes, wr->wr_inodes, oldsize); 2682 wapbl_free(wr->wr_inodes, oldsize); 2683 } 2684 wr->wr_inodes = new_inodes; 2685 memcpy(&wr->wr_inodes[wr->wr_inodescnt], wc->wc_inodes, 2686 wc->wc_inocnt * sizeof(wr->wr_inodes[0])); 2687 wr->wr_inodescnt += wc->wc_inocnt; 2688 } 2689 2690 static int 2691 wapbl_replay_process(struct wapbl_replay *wr, off_t head, off_t tail) 2692 { 2693 off_t off; 2694 int error; 2695 2696 int logblklen = 1 << wr->wr_log_dev_bshift; 2697 2698 wapbl_blkhash_clear(wr); 2699 2700 off = tail; 2701 while (off != head) { 2702 struct wapbl_wc_null *wcn; 2703 off_t saveoff = off; 2704 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 2705 if (error) 2706 goto errout; 2707 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 2708 switch (wcn->wc_type) { 2709 case WAPBL_WC_BLOCKS: 2710 wapbl_replay_process_blocks(wr, &off); 2711 break; 2712 2713 case WAPBL_WC_REVOCATIONS: 2714 wapbl_replay_process_revocations(wr); 2715 break; 2716 2717 case WAPBL_WC_INODES: 2718 wapbl_replay_process_inodes(wr, saveoff, off); 2719 break; 2720 2721 default: 2722 printf("Unrecognized wapbl type: 0x%08x\n", 2723 wcn->wc_type); 2724 error = EFTYPE; 2725 goto errout; 2726 } 2727 wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 2728 if (off != saveoff) { 2729 printf("wapbl_replay: corrupted records\n"); 2730 error = EFTYPE; 2731 goto errout; 2732 } 2733 } 2734 return 0; 2735 2736 errout: 2737 wapbl_blkhash_clear(wr); 2738 return error; 2739 } 2740 2741 #if 0 2742 int 2743 wapbl_replay_verify(struct wapbl_replay *wr, struct vnode *fsdevvp) 2744 { 2745 off_t off; 2746 int mismatchcnt = 0; 2747 int logblklen = 1 << wr->wr_log_dev_bshift; 2748 int fsblklen = 1 << wr->wr_fs_dev_bshift; 2749 void *scratch1 = wapbl_alloc(MAXBSIZE); 2750 void *scratch2 = wapbl_alloc(MAXBSIZE); 2751 int error = 0; 2752 2753 KDASSERT(wapbl_replay_isopen(wr)); 2754 2755 off = wch->wc_tail; 2756 while (off != wch->wc_head) { 2757 struct wapbl_wc_null *wcn; 2758 #ifdef DEBUG 2759 off_t saveoff = off; 2760 #endif 2761 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 2762 if (error) 2763 goto out; 2764 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 2765 switch (wcn->wc_type) { 2766 case WAPBL_WC_BLOCKS: 2767 { 2768 struct wapbl_wc_blocklist *wc = 2769 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2770 int i; 2771 for (i = 0; i < wc->wc_blkcount; i++) { 2772 int foundcnt = 0; 2773 int dirtycnt = 0; 2774 int j, n; 2775 /* 2776 * Check each physical block into the 2777 * hashtable independently 2778 */ 2779 n = wc->wc_blocks[i].wc_dlen >> 2780 wch->wc_fs_dev_bshift; 2781 for (j = 0; j < n; j++) { 2782 struct wapbl_blk *wb = 2783 wapbl_blkhash_get(wr, 2784 wc->wc_blocks[i].wc_daddr + btodb(j * fsblklen)); 2785 if (wb && (wb->wb_off == off)) { 2786 foundcnt++; 2787 error = 2788 wapbl_circ_read(wr, 2789 scratch1, fsblklen, 2790 &off); 2791 if (error) 2792 goto out; 2793 error = 2794 wapbl_read(scratch2, 2795 fsblklen, fsdevvp, 2796 wb->wb_blk); 2797 if (error) 2798 goto out; 2799 if (memcmp(scratch1, 2800 scratch2, 2801 fsblklen)) { 2802 printf( 2803 "wapbl_verify: mismatch block %"PRId64" at off %"PRIdMAX"\n", 2804 wb->wb_blk, (intmax_t)off); 2805 dirtycnt++; 2806 mismatchcnt++; 2807 } 2808 } else { 2809 wapbl_circ_advance(wr, 2810 fsblklen, &off); 2811 } 2812 } 2813 #if 0 2814 /* 2815 * If all of the blocks in an entry 2816 * are clean, then remove all of its 2817 * blocks from the hashtable since they 2818 * never will need replay. 2819 */ 2820 if ((foundcnt != 0) && 2821 (dirtycnt == 0)) { 2822 off = saveoff; 2823 wapbl_circ_advance(wr, 2824 logblklen, &off); 2825 for (j = 0; j < n; j++) { 2826 struct wapbl_blk *wb = 2827 wapbl_blkhash_get(wr, 2828 wc->wc_blocks[i].wc_daddr + btodb(j * fsblklen)); 2829 if (wb && 2830 (wb->wb_off == off)) { 2831 wapbl_blkhash_rem(wr, wb->wb_blk); 2832 } 2833 wapbl_circ_advance(wr, 2834 fsblklen, &off); 2835 } 2836 } 2837 #endif 2838 } 2839 } 2840 break; 2841 case WAPBL_WC_REVOCATIONS: 2842 case WAPBL_WC_INODES: 2843 break; 2844 default: 2845 KASSERT(0); 2846 } 2847 #ifdef DEBUG 2848 wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 2849 KASSERT(off == saveoff); 2850 #endif 2851 } 2852 out: 2853 wapbl_free(scratch1, MAXBSIZE); 2854 wapbl_free(scratch2, MAXBSIZE); 2855 if (!error && mismatchcnt) 2856 error = EFTYPE; 2857 return error; 2858 } 2859 #endif 2860 2861 int 2862 wapbl_replay_write(struct wapbl_replay *wr, struct vnode *fsdevvp) 2863 { 2864 struct wapbl_blk *wb; 2865 size_t i; 2866 off_t off; 2867 void *scratch; 2868 int error = 0; 2869 int fsblklen = 1 << wr->wr_fs_dev_bshift; 2870 2871 KDASSERT(wapbl_replay_isopen(wr)); 2872 2873 scratch = wapbl_alloc(MAXBSIZE); 2874 2875 for (i = 0; i <= wr->wr_blkhashmask; ++i) { 2876 LIST_FOREACH(wb, &wr->wr_blkhash[i], wb_hash) { 2877 off = wb->wb_off; 2878 error = wapbl_circ_read(wr, scratch, fsblklen, &off); 2879 if (error) 2880 break; 2881 error = wapbl_write(scratch, fsblklen, fsdevvp, 2882 wb->wb_blk); 2883 if (error) 2884 break; 2885 } 2886 } 2887 2888 wapbl_free(scratch, MAXBSIZE); 2889 return error; 2890 } 2891 2892 int 2893 wapbl_replay_can_read(struct wapbl_replay *wr, daddr_t blk, long len) 2894 { 2895 int fsblklen = 1 << wr->wr_fs_dev_bshift; 2896 2897 KDASSERT(wapbl_replay_isopen(wr)); 2898 KASSERT((len % fsblklen) == 0); 2899 2900 while (len != 0) { 2901 struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 2902 if (wb) 2903 return 1; 2904 len -= fsblklen; 2905 } 2906 return 0; 2907 } 2908 2909 int 2910 wapbl_replay_read(struct wapbl_replay *wr, void *data, daddr_t blk, long len) 2911 { 2912 int fsblklen = 1 << wr->wr_fs_dev_bshift; 2913 2914 KDASSERT(wapbl_replay_isopen(wr)); 2915 2916 KASSERT((len % fsblklen) == 0); 2917 2918 while (len != 0) { 2919 struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 2920 if (wb) { 2921 off_t off = wb->wb_off; 2922 int error; 2923 error = wapbl_circ_read(wr, data, fsblklen, &off); 2924 if (error) 2925 return error; 2926 } 2927 data = (uint8_t *)data + fsblklen; 2928 len -= fsblklen; 2929 blk++; 2930 } 2931 return 0; 2932 } 2933 2934 #ifdef _KERNEL 2935 /* 2936 * This is not really a module now, but maybe on it's way to 2937 * being one some day. 2938 */ 2939 MODULE(MODULE_CLASS_VFS, wapbl, NULL); 2940 2941 static int 2942 wapbl_modcmd(modcmd_t cmd, void *arg) 2943 { 2944 2945 switch (cmd) { 2946 case MODULE_CMD_INIT: 2947 wapbl_init(); 2948 return 0; 2949 case MODULE_CMD_FINI: 2950 #ifdef notyet 2951 return wapbl_fini(true); 2952 #endif 2953 return EOPNOTSUPP; 2954 default: 2955 return ENOTTY; 2956 } 2957 } 2958 #endif /* _KERNEL */ 2959