1 /* 2 * Copyright (c) 2013-2017 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@dragonflybsd.org> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #include "hammer2.h" 36 37 #define HAMMER2_DOP_READ 1 38 #define HAMMER2_DOP_NEW 2 39 #define HAMMER2_DOP_NEWNZ 3 40 #define HAMMER2_DOP_READQ 4 41 42 /* 43 * Implements an abstraction layer for synchronous and asynchronous 44 * buffered device I/O. Can be used as an OS-abstraction but the main 45 * purpose is to allow larger buffers to be used against hammer2_chain's 46 * using smaller allocations, without causing deadlocks. 47 * 48 * The DIOs also record temporary state with limited persistence. This 49 * feature is used to keep track of dedupable blocks. 50 */ 51 static int hammer2_io_cleanup_callback(hammer2_io_t *dio, void *arg); 52 static void dio_write_stats_update(hammer2_io_t *dio, struct buf *bp); 53 54 static int 55 hammer2_io_cmp(hammer2_io_t *io1, hammer2_io_t *io2) 56 { 57 if (io1->pbase < io2->pbase) 58 return(-1); 59 if (io1->pbase > io2->pbase) 60 return(1); 61 return(0); 62 } 63 64 RB_PROTOTYPE2(hammer2_io_tree, hammer2_io, rbnode, hammer2_io_cmp, off_t); 65 RB_GENERATE2(hammer2_io_tree, hammer2_io, rbnode, hammer2_io_cmp, 66 off_t, pbase); 67 68 struct hammer2_cleanupcb_info { 69 struct hammer2_io_tree tmptree; 70 int count; 71 }; 72 73 #if 0 74 static __inline 75 uint64_t 76 hammer2_io_mask(hammer2_io_t *dio, hammer2_off_t off, u_int bytes) 77 { 78 uint64_t mask; 79 int i; 80 81 if (bytes < 1024) /* smaller chunks not supported */ 82 return 0; 83 84 /* 85 * Calculate crc check mask for larger chunks 86 */ 87 i = (((off & ~HAMMER2_OFF_MASK_RADIX) - dio->pbase) & 88 HAMMER2_PBUFMASK) >> 10; 89 if (i == 0 && bytes == HAMMER2_PBUFSIZE) 90 return((uint64_t)-1); 91 mask = ((uint64_t)1U << (bytes >> 10)) - 1; 92 mask <<= i; 93 94 return mask; 95 } 96 #endif 97 98 /* 99 * Returns the DIO corresponding to the data|radix, creating it if necessary. 100 * 101 * If createit is 0, NULL can be returned indicating that the DIO does not 102 * exist. (btype) is ignored when createit is 0. 103 */ 104 static __inline 105 hammer2_io_t * 106 hammer2_io_alloc(hammer2_dev_t *hmp, hammer2_key_t data_off, uint8_t btype, 107 int createit, int *isgoodp) 108 { 109 hammer2_io_t *dio; 110 hammer2_io_t *xio; 111 hammer2_key_t lbase; 112 hammer2_key_t pbase; 113 hammer2_key_t pmask; 114 uint64_t refs; 115 int lsize; 116 int psize; 117 118 psize = HAMMER2_PBUFSIZE; 119 pmask = ~(hammer2_off_t)(psize - 1); 120 lsize = 1 << (int)(data_off & HAMMER2_OFF_MASK_RADIX); 121 lbase = data_off & ~HAMMER2_OFF_MASK_RADIX; 122 pbase = lbase & pmask; 123 124 if (pbase == 0 || ((lbase + lsize - 1) & pmask) != pbase) { 125 kprintf("Illegal: %016jx %016jx+%08x / %016jx\n", 126 pbase, lbase, lsize, pmask); 127 } 128 KKASSERT(pbase != 0 && ((lbase + lsize - 1) & pmask) == pbase); 129 *isgoodp = 0; 130 131 /* 132 * Access/Allocate the DIO, bump dio->refs to prevent destruction. 133 */ 134 hammer2_spin_sh(&hmp->io_spin); 135 dio = RB_LOOKUP(hammer2_io_tree, &hmp->iotree, pbase); 136 if (dio) { 137 refs = atomic_fetchadd_64(&dio->refs, 1); 138 if ((refs & HAMMER2_DIO_MASK) == 0) { 139 atomic_add_int(&dio->hmp->iofree_count, -1); 140 } 141 if (refs & HAMMER2_DIO_GOOD) 142 *isgoodp = 1; 143 hammer2_spin_unsh(&hmp->io_spin); 144 } else if (createit) { 145 refs = 0; 146 hammer2_spin_unsh(&hmp->io_spin); 147 dio = kmalloc(sizeof(*dio), M_HAMMER2, M_INTWAIT | M_ZERO); 148 dio->hmp = hmp; 149 dio->pbase = pbase; 150 dio->psize = psize; 151 dio->btype = btype; 152 dio->refs = refs + 1; 153 dio->act = 5; 154 hammer2_spin_ex(&hmp->io_spin); 155 xio = RB_INSERT(hammer2_io_tree, &hmp->iotree, dio); 156 if (xio == NULL) { 157 atomic_add_int(&hammer2_dio_count, 1); 158 hammer2_spin_unex(&hmp->io_spin); 159 } else { 160 refs = atomic_fetchadd_64(&xio->refs, 1); 161 if ((refs & HAMMER2_DIO_MASK) == 0) 162 atomic_add_int(&xio->hmp->iofree_count, -1); 163 if (refs & HAMMER2_DIO_GOOD) 164 *isgoodp = 1; 165 hammer2_spin_unex(&hmp->io_spin); 166 kfree(dio, M_HAMMER2); 167 dio = xio; 168 } 169 } else { 170 hammer2_spin_unsh(&hmp->io_spin); 171 return NULL; 172 } 173 dio->ticks = ticks; 174 if (dio->act < 10) 175 ++dio->act; 176 177 return dio; 178 } 179 180 /* 181 * Acquire the requested dio. If DIO_GOOD is not set we must instantiate 182 * a buffer. If set the buffer already exists and is good to go. 183 */ 184 hammer2_io_t * 185 hammer2_io_getblk(hammer2_dev_t *hmp, int btype, off_t lbase, int lsize, int op) 186 { 187 hammer2_io_t *dio; 188 off_t peof; 189 uint64_t orefs; 190 uint64_t nrefs; 191 int isgood; 192 int error; 193 int hce; 194 int notmetaflag = ((btype == HAMMER2_BREF_TYPE_DATA) ? B_NOTMETA : 0); 195 196 KKASSERT((1 << (int)(lbase & HAMMER2_OFF_MASK_RADIX)) == lsize); 197 198 if (op == HAMMER2_DOP_READQ) { 199 dio = hammer2_io_alloc(hmp, lbase, btype, 0, &isgood); 200 if (dio == NULL) 201 return NULL; 202 op = HAMMER2_DOP_READ; 203 } else { 204 dio = hammer2_io_alloc(hmp, lbase, btype, 1, &isgood); 205 } 206 207 for (;;) { 208 orefs = dio->refs; 209 cpu_ccfence(); 210 211 /* 212 * Buffer is already good, handle the op and return. 213 */ 214 if (orefs & HAMMER2_DIO_GOOD) { 215 if (isgood == 0) 216 cpu_mfence(); 217 218 switch(op) { 219 case HAMMER2_DOP_NEW: 220 bzero(hammer2_io_data(dio, lbase), lsize); 221 /* fall through */ 222 case HAMMER2_DOP_NEWNZ: 223 atomic_set_long(&dio->refs, HAMMER2_DIO_DIRTY); 224 break; 225 case HAMMER2_DOP_READ: 226 default: 227 /* nothing to do */ 228 break; 229 } 230 return (dio); 231 } 232 233 /* 234 * Try to own the DIO 235 */ 236 if (orefs & HAMMER2_DIO_INPROG) { 237 nrefs = orefs | HAMMER2_DIO_WAITING; 238 tsleep_interlock(dio, 0); 239 if (atomic_cmpset_64(&dio->refs, orefs, nrefs)) { 240 tsleep(dio, PINTERLOCKED, "h2dio", hz); 241 } 242 /* retry */ 243 } else { 244 nrefs = orefs | HAMMER2_DIO_INPROG; 245 if (atomic_cmpset_64(&dio->refs, orefs, nrefs)) { 246 break; 247 } 248 } 249 } 250 251 /* 252 * We break to here if GOOD is not set and we acquired INPROG for 253 * the I/O. 254 */ 255 KKASSERT(dio->bp == NULL); 256 if (btype == HAMMER2_BREF_TYPE_DATA) 257 hce = hammer2_cluster_data_read; 258 else 259 hce = hammer2_cluster_meta_read; 260 261 error = 0; 262 if (dio->pbase == (lbase & ~HAMMER2_OFF_MASK_RADIX) && 263 dio->psize == lsize) { 264 switch(op) { 265 case HAMMER2_DOP_NEW: 266 case HAMMER2_DOP_NEWNZ: 267 dio->bp = getblk(dio->hmp->devvp, 268 dio->pbase, dio->psize, 269 0, 0); 270 if (op == HAMMER2_DOP_NEW) 271 bzero(dio->bp->b_data, dio->psize); 272 atomic_set_long(&dio->refs, HAMMER2_DIO_DIRTY); 273 break; 274 case HAMMER2_DOP_READ: 275 default: 276 if (hce > 0) { 277 /* 278 * Synchronous cluster I/O for now. 279 */ 280 peof = (dio->pbase + HAMMER2_SEGMASK64) & 281 ~HAMMER2_SEGMASK64; 282 dio->bp = NULL; 283 error = cluster_readx(dio->hmp->devvp, 284 peof, dio->pbase, 285 dio->psize, notmetaflag, 286 dio->psize, 287 HAMMER2_PBUFSIZE*hce, 288 &dio->bp); 289 } else { 290 dio->bp = NULL; 291 error = breadnx(dio->hmp->devvp, dio->pbase, 292 dio->psize, notmetaflag, 293 NULL, NULL, 0, &dio->bp); 294 } 295 } 296 } else { 297 if (hce > 0) { 298 /* 299 * Synchronous cluster I/O for now. 300 */ 301 peof = (dio->pbase + HAMMER2_SEGMASK64) & 302 ~HAMMER2_SEGMASK64; 303 error = cluster_readx(dio->hmp->devvp, 304 peof, dio->pbase, dio->psize, 305 notmetaflag, 306 dio->psize, HAMMER2_PBUFSIZE*hce, 307 &dio->bp); 308 } else { 309 error = breadnx(dio->hmp->devvp, dio->pbase, 310 dio->psize, notmetaflag, 311 NULL, NULL, 0, &dio->bp); 312 } 313 if (dio->bp) { 314 /* 315 * Handle NEW flags 316 */ 317 switch(op) { 318 case HAMMER2_DOP_NEW: 319 bzero(hammer2_io_data(dio, lbase), lsize); 320 /* fall through */ 321 case HAMMER2_DOP_NEWNZ: 322 atomic_set_long(&dio->refs, HAMMER2_DIO_DIRTY); 323 break; 324 case HAMMER2_DOP_READ: 325 default: 326 break; 327 } 328 329 /* 330 * Tell the kernel that the buffer cache is not 331 * meta-data based on the btype. This allows 332 * swapcache to distinguish between data and 333 * meta-data. 334 */ 335 switch(btype) { 336 case HAMMER2_BREF_TYPE_DATA: 337 dio->bp->b_flags |= B_NOTMETA; 338 break; 339 default: 340 break; 341 } 342 } 343 } 344 345 if (dio->bp) { 346 BUF_KERNPROC(dio->bp); 347 dio->bp->b_flags &= ~B_AGE; 348 } 349 dio->error = error; 350 351 /* 352 * Clear INPROG and WAITING, set GOOD wake up anyone waiting. 353 */ 354 for (;;) { 355 orefs = dio->refs; 356 cpu_ccfence(); 357 nrefs = orefs & ~(HAMMER2_DIO_INPROG | HAMMER2_DIO_WAITING); 358 if (error == 0) 359 nrefs |= HAMMER2_DIO_GOOD; 360 if (atomic_cmpset_64(&dio->refs, orefs, nrefs)) { 361 if (orefs & HAMMER2_DIO_WAITING) 362 wakeup(dio); 363 break; 364 } 365 cpu_pause(); 366 } 367 368 /* XXX error handling */ 369 370 return dio; 371 } 372 373 /* 374 * Release our ref on *diop. 375 * 376 * On the 1->0 transition we clear DIO_GOOD, set DIO_INPROG, and dispose 377 * of dio->bp. Then we clean up DIO_INPROG and DIO_WAITING. 378 */ 379 void 380 hammer2_io_putblk(hammer2_io_t **diop) 381 { 382 hammer2_dev_t *hmp; 383 hammer2_io_t *dio; 384 struct buf *bp; 385 off_t peof; 386 off_t pbase; 387 int psize; 388 int limit_dio; 389 uint64_t orefs; 390 uint64_t nrefs; 391 392 dio = *diop; 393 *diop = NULL; 394 hmp = dio->hmp; 395 396 KKASSERT((dio->refs & HAMMER2_DIO_MASK) != 0); 397 398 /* 399 * Drop refs. 400 * 401 * On the 1->0 transition clear GOOD and set INPROG, and break. 402 * On any other transition we can return early. 403 */ 404 for (;;) { 405 orefs = dio->refs; 406 cpu_ccfence(); 407 408 if ((orefs & HAMMER2_DIO_MASK) == 1 && 409 (orefs & HAMMER2_DIO_INPROG) == 0) { 410 /* 411 * Lastdrop case, INPROG can be set. 412 */ 413 nrefs = orefs - 1; 414 nrefs &= ~(HAMMER2_DIO_GOOD | HAMMER2_DIO_DIRTY); 415 nrefs |= HAMMER2_DIO_INPROG; 416 if (atomic_cmpset_64(&dio->refs, orefs, nrefs)) 417 break; 418 } else if ((orefs & HAMMER2_DIO_MASK) == 1) { 419 /* 420 * Lastdrop case, INPROG already set. We must 421 * wait for INPROG to clear. 422 */ 423 nrefs = orefs | HAMMER2_DIO_WAITING; 424 tsleep_interlock(dio, 0); 425 if (atomic_cmpset_64(&dio->refs, orefs, nrefs)) { 426 tsleep(dio, PINTERLOCKED, "h2dio", hz); 427 } 428 /* retry */ 429 } else { 430 /* 431 * Normal drop case. 432 */ 433 nrefs = orefs - 1; 434 if (atomic_cmpset_64(&dio->refs, orefs, nrefs)) 435 return; 436 /* retry */ 437 } 438 cpu_pause(); 439 /* retry */ 440 } 441 442 /* 443 * Lastdrop (1->0 transition). INPROG has been set, GOOD and DIRTY 444 * have been cleared. iofree_count has not yet been incremented, 445 * note that another accessor race will decrement iofree_count so 446 * we have to increment it regardless. 447 * 448 * We can now dispose of the buffer, and should do it before calling 449 * io_complete() in case there's a race against a new reference 450 * which causes io_complete() to chain and instantiate the bp again. 451 */ 452 pbase = dio->pbase; 453 psize = dio->psize; 454 bp = dio->bp; 455 dio->bp = NULL; 456 457 if ((orefs & HAMMER2_DIO_GOOD) && bp) { 458 /* 459 * Non-errored disposal of bp 460 */ 461 if (orefs & HAMMER2_DIO_DIRTY) { 462 int hce; 463 464 dio_write_stats_update(dio, bp); 465 if ((hce = hammer2_cluster_write) > 0) { 466 /* 467 * Allows write-behind to keep the buffer 468 * cache sane. 469 */ 470 peof = (pbase + HAMMER2_SEGMASK64) & 471 ~HAMMER2_SEGMASK64; 472 bp->b_flags |= B_CLUSTEROK; 473 cluster_write(bp, peof, psize, hce); 474 } else { 475 /* 476 * Allows dirty buffers to accumulate and 477 * possibly be canceled (e.g. by a 'rm'), 478 * will burst-write later. 479 */ 480 bp->b_flags |= B_CLUSTEROK; 481 bdwrite(bp); 482 } 483 } else if (bp->b_flags & (B_ERROR | B_INVAL | B_RELBUF)) { 484 brelse(bp); 485 } else { 486 bqrelse(bp); 487 } 488 } else if (bp) { 489 /* 490 * Errored disposal of bp 491 */ 492 brelse(bp); 493 } 494 495 /* 496 * Update iofree_count before disposing of the dio 497 */ 498 hmp = dio->hmp; 499 atomic_add_int(&hmp->iofree_count, 1); 500 501 /* 502 * Clear INPROG, GOOD, and WAITING 503 */ 504 for (;;) { 505 orefs = dio->refs; 506 cpu_ccfence(); 507 nrefs = orefs & ~(HAMMER2_DIO_INPROG | HAMMER2_DIO_GOOD | 508 HAMMER2_DIO_WAITING); 509 if (atomic_cmpset_64(&dio->refs, orefs, nrefs)) { 510 if (orefs & HAMMER2_DIO_WAITING) 511 wakeup(dio); 512 break; 513 } 514 cpu_pause(); 515 } 516 517 /* 518 * We cache free buffers so re-use cases can use a shared lock, but 519 * if too many build up we have to clean them out. 520 */ 521 limit_dio = hammer2_limit_dio; 522 if (limit_dio < 256) 523 limit_dio = 256; 524 if (limit_dio > 1024*1024) 525 limit_dio = 1024*1024; 526 if (hmp->iofree_count > limit_dio) { 527 struct hammer2_cleanupcb_info info; 528 529 RB_INIT(&info.tmptree); 530 hammer2_spin_ex(&hmp->io_spin); 531 if (hmp->iofree_count > limit_dio) { 532 info.count = hmp->iofree_count / 5; 533 RB_SCAN(hammer2_io_tree, &hmp->iotree, NULL, 534 hammer2_io_cleanup_callback, &info); 535 } 536 hammer2_spin_unex(&hmp->io_spin); 537 hammer2_io_cleanup(hmp, &info.tmptree); 538 } 539 } 540 541 /* 542 * Cleanup any dio's with (INPROG | refs) == 0. 543 * 544 * Called to clean up cached DIOs on umount after all activity has been 545 * flushed. 546 */ 547 static 548 int 549 hammer2_io_cleanup_callback(hammer2_io_t *dio, void *arg) 550 { 551 struct hammer2_cleanupcb_info *info = arg; 552 hammer2_io_t *xio; 553 554 if ((dio->refs & (HAMMER2_DIO_MASK | HAMMER2_DIO_INPROG)) == 0) { 555 if (dio->act > 0) { 556 int act; 557 558 act = dio->act - (ticks - dio->ticks) / hz - 1; 559 if (act > 0) { 560 dio->act = act; 561 return 0; 562 } 563 dio->act = 0; 564 } 565 KKASSERT(dio->bp == NULL); 566 if (info->count > 0) { 567 RB_REMOVE(hammer2_io_tree, &dio->hmp->iotree, dio); 568 xio = RB_INSERT(hammer2_io_tree, &info->tmptree, dio); 569 KKASSERT(xio == NULL); 570 --info->count; 571 } 572 } 573 return 0; 574 } 575 576 void 577 hammer2_io_cleanup(hammer2_dev_t *hmp, struct hammer2_io_tree *tree) 578 { 579 hammer2_io_t *dio; 580 581 while ((dio = RB_ROOT(tree)) != NULL) { 582 RB_REMOVE(hammer2_io_tree, tree, dio); 583 KKASSERT(dio->bp == NULL && 584 (dio->refs & (HAMMER2_DIO_MASK | HAMMER2_DIO_INPROG)) == 0); 585 if (dio->refs & HAMMER2_DIO_DIRTY) { 586 kprintf("hammer2_io_cleanup: Dirty buffer " 587 "%016jx/%d (bp=%p)\n", 588 dio->pbase, dio->psize, dio->bp); 589 } 590 kfree(dio, M_HAMMER2); 591 atomic_add_int(&hammer2_dio_count, -1); 592 atomic_add_int(&hmp->iofree_count, -1); 593 } 594 } 595 596 /* 597 * Returns a pointer to the requested data. 598 */ 599 char * 600 hammer2_io_data(hammer2_io_t *dio, off_t lbase) 601 { 602 struct buf *bp; 603 int off; 604 605 bp = dio->bp; 606 KKASSERT(bp != NULL); 607 off = (lbase & ~HAMMER2_OFF_MASK_RADIX) - bp->b_loffset; 608 KKASSERT(off >= 0 && off < bp->b_bufsize); 609 return(bp->b_data + off); 610 } 611 612 int 613 hammer2_io_new(hammer2_dev_t *hmp, int btype, off_t lbase, int lsize, 614 hammer2_io_t **diop) 615 { 616 *diop = hammer2_io_getblk(hmp, btype, lbase, lsize, HAMMER2_DOP_NEW); 617 return ((*diop)->error); 618 } 619 620 int 621 hammer2_io_newnz(hammer2_dev_t *hmp, int btype, off_t lbase, int lsize, 622 hammer2_io_t **diop) 623 { 624 *diop = hammer2_io_getblk(hmp, btype, lbase, lsize, HAMMER2_DOP_NEWNZ); 625 return ((*diop)->error); 626 } 627 628 int 629 hammer2_io_bread(hammer2_dev_t *hmp, int btype, off_t lbase, int lsize, 630 hammer2_io_t **diop) 631 { 632 *diop = hammer2_io_getblk(hmp, btype, lbase, lsize, HAMMER2_DOP_READ); 633 return ((*diop)->error); 634 } 635 636 hammer2_io_t * 637 hammer2_io_getquick(hammer2_dev_t *hmp, off_t lbase, int lsize) 638 { 639 hammer2_io_t *dio; 640 641 dio = hammer2_io_getblk(hmp, 0, lbase, lsize, HAMMER2_DOP_READQ); 642 return dio; 643 } 644 645 void 646 hammer2_io_bawrite(hammer2_io_t **diop) 647 { 648 atomic_set_64(&(*diop)->refs, HAMMER2_DIO_DIRTY); 649 hammer2_io_putblk(diop); 650 } 651 652 void 653 hammer2_io_bdwrite(hammer2_io_t **diop) 654 { 655 atomic_set_64(&(*diop)->refs, HAMMER2_DIO_DIRTY); 656 hammer2_io_putblk(diop); 657 } 658 659 int 660 hammer2_io_bwrite(hammer2_io_t **diop) 661 { 662 atomic_set_64(&(*diop)->refs, HAMMER2_DIO_DIRTY); 663 hammer2_io_putblk(diop); 664 return (0); /* XXX */ 665 } 666 667 void 668 hammer2_io_setdirty(hammer2_io_t *dio) 669 { 670 atomic_set_64(&dio->refs, HAMMER2_DIO_DIRTY); 671 } 672 673 /* 674 * This routine is called when a MODIFIED chain is being DESTROYED, 675 * in an attempt to allow the related buffer cache buffer to be 676 * invalidated and discarded instead of flushing it to disk. 677 * 678 * At the moment this case is only really useful for file meta-data. 679 * File data is already handled via the logical buffer cache associated 680 * with the vnode, and will be discarded if it was never flushed to disk. 681 * File meta-data may include inodes, directory entries, and indirect blocks. 682 * 683 * XXX 684 * However, our DIO buffers are PBUFSIZE'd (64KB), and the area being 685 * invalidated might be smaller. Most of the meta-data structures above 686 * are in the 'smaller' category. For now, don't try to invalidate the 687 * data areas. 688 */ 689 void 690 hammer2_io_inval(hammer2_io_t *dio, hammer2_off_t data_off, u_int bytes) 691 { 692 /* NOP */ 693 } 694 695 void 696 hammer2_io_brelse(hammer2_io_t **diop) 697 { 698 hammer2_io_putblk(diop); 699 } 700 701 void 702 hammer2_io_bqrelse(hammer2_io_t **diop) 703 { 704 hammer2_io_putblk(diop); 705 } 706 707 /* 708 * Set dedup validation bits in a DIO. We do not need the buffer cache 709 * buffer for this. This must be done concurrent with setting bits in 710 * the freemap so as to interlock with bulkfree's clearing of those bits. 711 */ 712 void 713 hammer2_io_dedup_set(hammer2_dev_t *hmp, hammer2_blockref_t *bref) 714 { 715 hammer2_io_t *dio; 716 uint64_t mask; 717 int lsize; 718 int isgood; 719 720 dio = hammer2_io_alloc(hmp, bref->data_off, bref->type, 1, &isgood); 721 lsize = 1 << (int)(bref->data_off & HAMMER2_OFF_MASK_RADIX); 722 mask = hammer2_dedup_mask(dio, bref->data_off, lsize); 723 atomic_clear_64(&dio->dedup_valid, mask); 724 atomic_set_64(&dio->dedup_alloc, mask); 725 hammer2_io_putblk(&dio); 726 } 727 728 /* 729 * Clear dedup validation bits in a DIO. This is typically done when 730 * a modified chain is destroyed or by the bulkfree code. No buffer 731 * is needed for this operation. If the DIO no longer exists it is 732 * equivalent to the bits not being set. 733 */ 734 void 735 hammer2_io_dedup_delete(hammer2_dev_t *hmp, uint8_t btype, 736 hammer2_off_t data_off, u_int bytes) 737 { 738 hammer2_io_t *dio; 739 uint64_t mask; 740 int isgood; 741 742 if ((data_off & ~HAMMER2_OFF_MASK_RADIX) == 0) 743 return; 744 if (btype != HAMMER2_BREF_TYPE_DATA) 745 return; 746 dio = hammer2_io_alloc(hmp, data_off, btype, 0, &isgood); 747 if (dio) { 748 if (data_off < dio->pbase || 749 (data_off & ~HAMMER2_OFF_MASK_RADIX) + bytes > 750 dio->pbase + dio->psize) { 751 panic("hammer2_dedup_delete: DATAOFF BAD " 752 "%016jx/%d %016jx\n", 753 data_off, bytes, dio->pbase); 754 } 755 mask = hammer2_dedup_mask(dio, data_off, bytes); 756 atomic_clear_64(&dio->dedup_alloc, mask); 757 atomic_clear_64(&dio->dedup_valid, mask); 758 hammer2_io_putblk(&dio); 759 } 760 } 761 762 /* 763 * Assert that dedup allocation bits in a DIO are not set. This operation 764 * does not require a buffer. The DIO does not need to exist. 765 */ 766 void 767 hammer2_io_dedup_assert(hammer2_dev_t *hmp, hammer2_off_t data_off, u_int bytes) 768 { 769 hammer2_io_t *dio; 770 int isgood; 771 772 dio = hammer2_io_alloc(hmp, data_off, HAMMER2_BREF_TYPE_DATA, 773 0, &isgood); 774 if (dio) { 775 KASSERT((dio->dedup_alloc & 776 hammer2_dedup_mask(dio, data_off, bytes)) == 0, 777 ("hammer2_dedup_assert: %016jx/%d %016jx/%016jx", 778 data_off, 779 bytes, 780 hammer2_dedup_mask(dio, data_off, bytes), 781 dio->dedup_alloc)); 782 hammer2_io_putblk(&dio); 783 } 784 } 785 786 static 787 void 788 dio_write_stats_update(hammer2_io_t *dio, struct buf *bp) 789 { 790 long *counterp; 791 792 if (bp->b_flags & B_DELWRI) 793 return; 794 795 switch(dio->btype) { 796 case 0: 797 return; 798 case HAMMER2_BREF_TYPE_DATA: 799 counterp = &hammer2_iod_file_write; 800 break; 801 case HAMMER2_BREF_TYPE_DIRENT: 802 case HAMMER2_BREF_TYPE_INODE: 803 counterp = &hammer2_iod_meta_write; 804 break; 805 case HAMMER2_BREF_TYPE_INDIRECT: 806 counterp = &hammer2_iod_indr_write; 807 break; 808 case HAMMER2_BREF_TYPE_FREEMAP_NODE: 809 case HAMMER2_BREF_TYPE_FREEMAP_LEAF: 810 counterp = &hammer2_iod_fmap_write; 811 break; 812 default: 813 counterp = &hammer2_iod_volu_write; 814 break; 815 } 816 *counterp += dio->psize; 817 } 818