1 /* $OpenBSD: subr_hibernate.c,v 1.27 2011/11/18 00:51:27 jasper Exp $ */ 2 3 /* 4 * Copyright (c) 2011 Ariane van der Steldt <ariane@stack.nl> 5 * Copyright (c) 2011 Mike Larkin <mlarkin@openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/hibernate.h> 21 #include <sys/malloc.h> 22 #include <sys/param.h> 23 #include <sys/tree.h> 24 #include <sys/types.h> 25 #include <sys/systm.h> 26 #include <sys/disklabel.h> 27 #include <sys/disk.h> 28 #include <sys/conf.h> 29 #include <sys/buf.h> 30 #include <sys/fcntl.h> 31 #include <sys/stat.h> 32 #include <uvm/uvm.h> 33 #include <machine/hibernate.h> 34 35 struct hibernate_zlib_state *hibernate_state; 36 37 /* Temporary vaddr ranges used during hibernate */ 38 vaddr_t hibernate_temp_page; 39 vaddr_t hibernate_copy_page; 40 41 /* Hibernate info as read from disk during resume */ 42 union hibernate_info disk_hiber_info; 43 paddr_t global_pig_start; 44 vaddr_t global_piglet_va; 45 46 /* 47 * Hib alloc enforced alignment. 48 */ 49 #define HIB_ALIGN 8 /* bytes alignment */ 50 51 /* 52 * sizeof builtin operation, but with alignment constraint. 53 */ 54 #define HIB_SIZEOF(_type) roundup(sizeof(_type), HIB_ALIGN) 55 56 struct hiballoc_entry { 57 size_t hibe_use; 58 size_t hibe_space; 59 RB_ENTRY(hiballoc_entry) hibe_entry; 60 }; 61 62 /* 63 * Compare hiballoc entries based on the address they manage. 64 * 65 * Since the address is fixed, relative to struct hiballoc_entry, 66 * we just compare the hiballoc_entry pointers. 67 */ 68 static __inline int 69 hibe_cmp(struct hiballoc_entry *l, struct hiballoc_entry *r) 70 { 71 return l < r ? -1 : (l > r); 72 } 73 74 RB_PROTOTYPE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp) 75 76 /* 77 * Given a hiballoc entry, return the address it manages. 78 */ 79 static __inline void * 80 hib_entry_to_addr(struct hiballoc_entry *entry) 81 { 82 caddr_t addr; 83 84 addr = (caddr_t)entry; 85 addr += HIB_SIZEOF(struct hiballoc_entry); 86 return addr; 87 } 88 89 /* 90 * Given an address, find the hiballoc that corresponds. 91 */ 92 static __inline struct hiballoc_entry* 93 hib_addr_to_entry(void *addr_param) 94 { 95 caddr_t addr; 96 97 addr = (caddr_t)addr_param; 98 addr -= HIB_SIZEOF(struct hiballoc_entry); 99 return (struct hiballoc_entry*)addr; 100 } 101 102 RB_GENERATE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp) 103 104 /* 105 * Allocate memory from the arena. 106 * 107 * Returns NULL if no memory is available. 108 */ 109 void * 110 hib_alloc(struct hiballoc_arena *arena, size_t alloc_sz) 111 { 112 struct hiballoc_entry *entry, *new_entry; 113 size_t find_sz; 114 115 /* 116 * Enforce alignment of HIB_ALIGN bytes. 117 * 118 * Note that, because the entry is put in front of the allocation, 119 * 0-byte allocations are guaranteed a unique address. 120 */ 121 alloc_sz = roundup(alloc_sz, HIB_ALIGN); 122 123 /* 124 * Find an entry with hibe_space >= find_sz. 125 * 126 * If the root node is not large enough, we switch to tree traversal. 127 * Because all entries are made at the bottom of the free space, 128 * traversal from the end has a slightly better chance of yielding 129 * a sufficiently large space. 130 */ 131 find_sz = alloc_sz + HIB_SIZEOF(struct hiballoc_entry); 132 entry = RB_ROOT(&arena->hib_addrs); 133 if (entry != NULL && entry->hibe_space < find_sz) { 134 RB_FOREACH_REVERSE(entry, hiballoc_addr, &arena->hib_addrs) { 135 if (entry->hibe_space >= find_sz) 136 break; 137 } 138 } 139 140 /* 141 * Insufficient or too fragmented memory. 142 */ 143 if (entry == NULL) 144 return NULL; 145 146 /* 147 * Create new entry in allocated space. 148 */ 149 new_entry = (struct hiballoc_entry*)( 150 (caddr_t)hib_entry_to_addr(entry) + entry->hibe_use); 151 new_entry->hibe_space = entry->hibe_space - find_sz; 152 new_entry->hibe_use = alloc_sz; 153 154 /* 155 * Insert entry. 156 */ 157 if (RB_INSERT(hiballoc_addr, &arena->hib_addrs, new_entry) != NULL) 158 panic("hib_alloc: insert failure"); 159 entry->hibe_space = 0; 160 161 /* Return address managed by entry. */ 162 return hib_entry_to_addr(new_entry); 163 } 164 165 /* 166 * Free a pointer previously allocated from this arena. 167 * 168 * If addr is NULL, this will be silently accepted. 169 */ 170 void 171 hib_free(struct hiballoc_arena *arena, void *addr) 172 { 173 struct hiballoc_entry *entry, *prev; 174 175 if (addr == NULL) 176 return; 177 178 /* 179 * Derive entry from addr and check it is really in this arena. 180 */ 181 entry = hib_addr_to_entry(addr); 182 if (RB_FIND(hiballoc_addr, &arena->hib_addrs, entry) != entry) 183 panic("hib_free: freed item %p not in hib arena", addr); 184 185 /* 186 * Give the space in entry to its predecessor. 187 * 188 * If entry has no predecessor, change its used space into free space 189 * instead. 190 */ 191 prev = RB_PREV(hiballoc_addr, &arena->hib_addrs, entry); 192 if (prev != NULL && 193 (void *)((caddr_t)prev + HIB_SIZEOF(struct hiballoc_entry) + 194 prev->hibe_use + prev->hibe_space) == entry) { 195 /* Merge entry. */ 196 RB_REMOVE(hiballoc_addr, &arena->hib_addrs, entry); 197 prev->hibe_space += HIB_SIZEOF(struct hiballoc_entry) + 198 entry->hibe_use + entry->hibe_space; 199 } else { 200 /* Flip used memory to free space. */ 201 entry->hibe_space += entry->hibe_use; 202 entry->hibe_use = 0; 203 } 204 } 205 206 /* 207 * Initialize hiballoc. 208 * 209 * The allocator will manage memmory at ptr, which is len bytes. 210 */ 211 int 212 hiballoc_init(struct hiballoc_arena *arena, void *p_ptr, size_t p_len) 213 { 214 struct hiballoc_entry *entry; 215 caddr_t ptr; 216 size_t len; 217 218 RB_INIT(&arena->hib_addrs); 219 220 /* 221 * Hib allocator enforces HIB_ALIGN alignment. 222 * Fixup ptr and len. 223 */ 224 ptr = (caddr_t)roundup((vaddr_t)p_ptr, HIB_ALIGN); 225 len = p_len - ((size_t)ptr - (size_t)p_ptr); 226 len &= ~((size_t)HIB_ALIGN - 1); 227 228 /* 229 * Insufficient memory to be able to allocate and also do bookkeeping. 230 */ 231 if (len <= HIB_SIZEOF(struct hiballoc_entry)) 232 return ENOMEM; 233 234 /* 235 * Create entry describing space. 236 */ 237 entry = (struct hiballoc_entry*)ptr; 238 entry->hibe_use = 0; 239 entry->hibe_space = len - HIB_SIZEOF(struct hiballoc_entry); 240 RB_INSERT(hiballoc_addr, &arena->hib_addrs, entry); 241 242 return 0; 243 } 244 245 /* 246 * Zero all free memory. 247 */ 248 void 249 uvm_pmr_zero_everything(void) 250 { 251 struct uvm_pmemrange *pmr; 252 struct vm_page *pg; 253 int i; 254 255 uvm_lock_fpageq(); 256 TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) { 257 /* Zero single pages. */ 258 while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_DIRTY])) 259 != NULL) { 260 uvm_pmr_remove(pmr, pg); 261 uvm_pagezero(pg); 262 atomic_setbits_int(&pg->pg_flags, PG_ZERO); 263 uvmexp.zeropages++; 264 uvm_pmr_insert(pmr, pg, 0); 265 } 266 267 /* Zero multi page ranges. */ 268 while ((pg = RB_ROOT(&pmr->size[UVM_PMR_MEMTYPE_DIRTY])) 269 != NULL) { 270 pg--; /* Size tree always has second page. */ 271 uvm_pmr_remove(pmr, pg); 272 for (i = 0; i < pg->fpgsz; i++) { 273 uvm_pagezero(&pg[i]); 274 atomic_setbits_int(&pg[i].pg_flags, PG_ZERO); 275 uvmexp.zeropages++; 276 } 277 uvm_pmr_insert(pmr, pg, 0); 278 } 279 } 280 uvm_unlock_fpageq(); 281 } 282 283 /* 284 * Mark all memory as dirty. 285 * 286 * Used to inform the system that the clean memory isn't clean for some 287 * reason, for example because we just came back from hibernate. 288 */ 289 void 290 uvm_pmr_dirty_everything(void) 291 { 292 struct uvm_pmemrange *pmr; 293 struct vm_page *pg; 294 int i; 295 296 uvm_lock_fpageq(); 297 TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) { 298 /* Dirty single pages. */ 299 while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_ZERO])) 300 != NULL) { 301 uvm_pmr_remove(pmr, pg); 302 atomic_clearbits_int(&pg->pg_flags, PG_ZERO); 303 uvm_pmr_insert(pmr, pg, 0); 304 } 305 306 /* Dirty multi page ranges. */ 307 while ((pg = RB_ROOT(&pmr->size[UVM_PMR_MEMTYPE_ZERO])) 308 != NULL) { 309 pg--; /* Size tree always has second page. */ 310 uvm_pmr_remove(pmr, pg); 311 for (i = 0; i < pg->fpgsz; i++) 312 atomic_clearbits_int(&pg[i].pg_flags, PG_ZERO); 313 uvm_pmr_insert(pmr, pg, 0); 314 } 315 } 316 317 uvmexp.zeropages = 0; 318 uvm_unlock_fpageq(); 319 } 320 321 /* 322 * Allocate the highest address that can hold sz. 323 * 324 * sz in bytes. 325 */ 326 int 327 uvm_pmr_alloc_pig(paddr_t *addr, psize_t sz) 328 { 329 struct uvm_pmemrange *pmr; 330 struct vm_page *pig_pg, *pg; 331 332 /* 333 * Convert sz to pages, since that is what pmemrange uses internally. 334 */ 335 sz = atop(round_page(sz)); 336 337 uvm_lock_fpageq(); 338 339 TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) { 340 RB_FOREACH_REVERSE(pig_pg, uvm_pmr_addr, &pmr->addr) { 341 if (pig_pg->fpgsz >= sz) { 342 goto found; 343 } 344 } 345 } 346 347 /* 348 * Allocation failure. 349 */ 350 uvm_unlock_pageq(); 351 return ENOMEM; 352 353 found: 354 /* Remove page from freelist. */ 355 uvm_pmr_remove_size(pmr, pig_pg); 356 pig_pg->fpgsz -= sz; 357 pg = pig_pg + pig_pg->fpgsz; 358 if (pig_pg->fpgsz == 0) 359 uvm_pmr_remove_addr(pmr, pig_pg); 360 else 361 uvm_pmr_insert_size(pmr, pig_pg); 362 363 uvmexp.free -= sz; 364 *addr = VM_PAGE_TO_PHYS(pg); 365 366 /* 367 * Update pg flags. 368 * 369 * Note that we trash the sz argument now. 370 */ 371 while (sz > 0) { 372 KASSERT(pg->pg_flags & PQ_FREE); 373 374 atomic_clearbits_int(&pg->pg_flags, 375 PG_PMAP0|PG_PMAP1|PG_PMAP2|PG_PMAP3); 376 377 if (pg->pg_flags & PG_ZERO) 378 uvmexp.zeropages -= sz; 379 atomic_clearbits_int(&pg->pg_flags, 380 PG_ZERO|PQ_FREE); 381 382 pg->uobject = NULL; 383 pg->uanon = NULL; 384 pg->pg_version++; 385 386 /* 387 * Next. 388 */ 389 pg++; 390 sz--; 391 } 392 393 /* Return. */ 394 uvm_unlock_fpageq(); 395 return 0; 396 } 397 398 /* 399 * Allocate a piglet area. 400 * 401 * This is as low as possible. 402 * Piglets are aligned. 403 * 404 * sz and align in bytes. 405 * 406 * The call will sleep for the pagedaemon to attempt to free memory. 407 * The pagedaemon may decide its not possible to free enough memory, causing 408 * the allocation to fail. 409 */ 410 int 411 uvm_pmr_alloc_piglet(vaddr_t *va, paddr_t *pa, vsize_t sz, paddr_t align) 412 { 413 paddr_t pg_addr, piglet_addr; 414 struct uvm_pmemrange *pmr; 415 struct vm_page *pig_pg, *pg; 416 struct pglist pageq; 417 int pdaemon_woken; 418 vaddr_t piglet_va; 419 420 KASSERT((align & (align - 1)) == 0); 421 pdaemon_woken = 0; /* Didn't wake the pagedaemon. */ 422 423 /* 424 * Fixup arguments: align must be at least PAGE_SIZE, 425 * sz will be converted to pagecount, since that is what 426 * pmemrange uses internally. 427 */ 428 if (align < PAGE_SIZE) 429 align = PAGE_SIZE; 430 sz = round_page(sz); 431 432 uvm_lock_fpageq(); 433 434 TAILQ_FOREACH_REVERSE(pmr, &uvm.pmr_control.use, uvm_pmemrange_use, 435 pmr_use) { 436 retry: 437 /* 438 * Search for a range with enough space. 439 * Use the address tree, to ensure the range is as low as 440 * possible. 441 */ 442 RB_FOREACH(pig_pg, uvm_pmr_addr, &pmr->addr) { 443 pg_addr = VM_PAGE_TO_PHYS(pig_pg); 444 piglet_addr = (pg_addr + (align - 1)) & ~(align - 1); 445 446 if (atop(pg_addr) + pig_pg->fpgsz >= 447 atop(piglet_addr) + atop(sz)) 448 goto found; 449 } 450 } 451 452 /* 453 * Try to coerse the pagedaemon into freeing memory 454 * for the piglet. 455 * 456 * pdaemon_woken is set to prevent the code from 457 * falling into an endless loop. 458 */ 459 if (!pdaemon_woken) { 460 pdaemon_woken = 1; 461 if (uvm_wait_pla(ptoa(pmr->low), ptoa(pmr->high) - 1, 462 sz, UVM_PLA_FAILOK) == 0) 463 goto retry; 464 } 465 466 /* Return failure. */ 467 uvm_unlock_fpageq(); 468 return ENOMEM; 469 470 found: 471 /* 472 * Extract piglet from pigpen. 473 */ 474 TAILQ_INIT(&pageq); 475 uvm_pmr_extract_range(pmr, pig_pg, 476 atop(piglet_addr), atop(piglet_addr) + atop(sz), &pageq); 477 478 *pa = piglet_addr; 479 uvmexp.free -= atop(sz); 480 481 /* 482 * Update pg flags. 483 * 484 * Note that we trash the sz argument now. 485 */ 486 TAILQ_FOREACH(pg, &pageq, pageq) { 487 KASSERT(pg->pg_flags & PQ_FREE); 488 489 atomic_clearbits_int(&pg->pg_flags, 490 PG_PMAP0|PG_PMAP1|PG_PMAP2|PG_PMAP3); 491 492 if (pg->pg_flags & PG_ZERO) 493 uvmexp.zeropages--; 494 atomic_clearbits_int(&pg->pg_flags, 495 PG_ZERO|PQ_FREE); 496 497 pg->uobject = NULL; 498 pg->uanon = NULL; 499 pg->pg_version++; 500 } 501 502 uvm_unlock_fpageq(); 503 504 /* 505 * Now allocate a va. 506 * Use direct mappings for the pages. 507 */ 508 509 piglet_va = *va = (vaddr_t)km_alloc(sz, &kv_any, &kp_none, &kd_waitok); 510 if (!piglet_va) { 511 uvm_pglistfree(&pageq); 512 return ENOMEM; 513 } 514 515 /* 516 * Map piglet to va. 517 */ 518 TAILQ_FOREACH(pg, &pageq, pageq) { 519 pmap_kenter_pa(piglet_va, VM_PAGE_TO_PHYS(pg), UVM_PROT_RW); 520 piglet_va += PAGE_SIZE; 521 } 522 pmap_update(pmap_kernel()); 523 524 return 0; 525 } 526 527 /* 528 * Free a piglet area. 529 */ 530 void 531 uvm_pmr_free_piglet(vaddr_t va, vsize_t sz) 532 { 533 paddr_t pa; 534 struct vm_page *pg; 535 536 /* 537 * Fix parameters. 538 */ 539 sz = round_page(sz); 540 541 /* 542 * Find the first page in piglet. 543 * Since piglets are contiguous, the first pg is all we need. 544 */ 545 if (!pmap_extract(pmap_kernel(), va, &pa)) 546 panic("uvm_pmr_free_piglet: piglet 0x%lx has no pages", va); 547 pg = PHYS_TO_VM_PAGE(pa); 548 if (pg == NULL) 549 panic("uvm_pmr_free_piglet: unmanaged page 0x%lx", pa); 550 551 /* 552 * Unmap. 553 */ 554 pmap_kremove(va, sz); 555 pmap_update(pmap_kernel()); 556 557 /* 558 * Free the physical and virtual memory. 559 */ 560 uvm_pmr_freepages(pg, atop(sz)); 561 km_free((void *)va, sz, &kv_any, &kp_none); 562 } 563 564 /* 565 * Physmem RLE compression support. 566 * 567 * Given a physical page address, it will return the number of pages 568 * starting at the address, that are free. 569 * Returns 0 if the page at addr is not free. 570 */ 571 psize_t 572 uvm_page_rle(paddr_t addr) 573 { 574 struct vm_page *pg, *pg_end; 575 struct vm_physseg *vmp; 576 int pseg_idx, off_idx; 577 578 pseg_idx = vm_physseg_find(atop(addr), &off_idx); 579 if (pseg_idx == -1) 580 return 0; 581 582 vmp = &vm_physmem[pseg_idx]; 583 pg = &vmp->pgs[off_idx]; 584 if (!(pg->pg_flags & PQ_FREE)) 585 return 0; 586 587 /* 588 * Search for the first non-free page after pg. 589 * Note that the page may not be the first page in a free pmemrange, 590 * therefore pg->fpgsz cannot be used. 591 */ 592 for (pg_end = pg; pg_end <= vmp->lastpg && 593 (pg_end->pg_flags & PQ_FREE) == PQ_FREE; pg_end++); 594 return pg_end - pg; 595 } 596 597 /* 598 * Fills out the hibernate_info union pointed to by hiber_info 599 * with information about this machine (swap signature block 600 * offsets, number of memory ranges, kernel in use, etc) 601 */ 602 int 603 get_hibernate_info(union hibernate_info *hiber_info, int suspend) 604 { 605 int chunktable_size; 606 struct disklabel dl; 607 char err_string[128], *dl_ret; 608 609 /* Determine I/O function to use */ 610 hiber_info->io_func = get_hibernate_io_function(); 611 if (hiber_info->io_func == NULL) 612 return (1); 613 614 /* Calculate hibernate device */ 615 hiber_info->device = swdevt[0].sw_dev; 616 617 /* Read disklabel (used to calculate signature and image offsets) */ 618 dl_ret = disk_readlabel(&dl, hiber_info->device, err_string, 128); 619 620 if (dl_ret) { 621 printf("Hibernate error reading disklabel: %s\n", dl_ret); 622 return (1); 623 } 624 625 hiber_info->secsize = dl.d_secsize; 626 627 /* Make sure the signature can fit in one block */ 628 KASSERT(sizeof(union hibernate_info)/hiber_info->secsize == 1); 629 630 /* Calculate swap offset from start of disk */ 631 hiber_info->swap_offset = dl.d_partitions[1].p_offset; 632 633 /* Calculate signature block location */ 634 hiber_info->sig_offset = dl.d_partitions[1].p_offset + 635 dl.d_partitions[1].p_size - 636 sizeof(union hibernate_info)/hiber_info->secsize; 637 638 chunktable_size = HIBERNATE_CHUNK_TABLE_SIZE / hiber_info->secsize; 639 640 /* Stash kernel version information */ 641 bzero(&hiber_info->kernel_version, 128); 642 bcopy(version, &hiber_info->kernel_version, 643 min(strlen(version), sizeof(hiber_info->kernel_version)-1)); 644 645 if (suspend) { 646 /* Allocate piglet region */ 647 if (uvm_pmr_alloc_piglet(&hiber_info->piglet_va, 648 &hiber_info->piglet_pa, HIBERNATE_CHUNK_SIZE*3, 649 HIBERNATE_CHUNK_SIZE)) { 650 printf("Hibernate failed to allocate the piglet\n"); 651 return (1); 652 } 653 hiber_info->io_page = (void *)hiber_info->piglet_va; 654 } else { 655 /* 656 * Resuming kernels use a regular I/O page since we won't 657 * have access to the suspended kernel's piglet VA at this 658 * point. No need to free this I/O page as it will vanish 659 * as part of the resume. 660 */ 661 hiber_info->io_page = malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT); 662 if (!hiber_info->io_page) 663 return (1); 664 } 665 666 667 /* 668 * Initialize of the hibernate IO function (for drivers which 669 * need that) 670 */ 671 if (hiber_info->io_func(hiber_info->device, 0, 672 (vaddr_t)NULL, 0, HIB_INIT, hiber_info->io_page)) 673 goto fail; 674 675 if (get_hibernate_info_md(hiber_info)) 676 goto fail; 677 678 /* Calculate memory image location */ 679 hiber_info->image_offset = dl.d_partitions[1].p_offset + 680 dl.d_partitions[1].p_size - 681 (hiber_info->image_size / hiber_info->secsize) - 682 sizeof(union hibernate_info)/hiber_info->secsize - 683 chunktable_size; 684 685 return (0); 686 fail: 687 uvm_pmr_free_piglet(hiber_info->piglet_va, HIBERNATE_CHUNK_SIZE*3); 688 return (1); 689 } 690 691 /* 692 * Allocate nitems*size bytes from the hiballoc area presently in use 693 */ 694 void 695 *hibernate_zlib_alloc(void *unused, int nitems, int size) 696 { 697 return hib_alloc(&hibernate_state->hiballoc_arena, nitems*size); 698 } 699 700 /* 701 * Free the memory pointed to by addr in the hiballoc area presently in 702 * use 703 */ 704 void 705 hibernate_zlib_free(void *unused, void *addr) 706 { 707 hib_free(&hibernate_state->hiballoc_arena, addr); 708 } 709 710 /* 711 * Inflate size bytes from src into dest, skipping any pages in 712 * [src..dest] that are special (see hibernate_inflate_skip) 713 * 714 * This function executes while using the resume-time stack 715 * and pmap, and therefore cannot use ddb/printf/etc. Doing so 716 * will likely hang or reset the machine. 717 */ 718 void 719 hibernate_inflate(union hibernate_info *hiber_info, paddr_t dest, 720 paddr_t src, size_t size) 721 { 722 int i; 723 psize_t rle; 724 725 hibernate_state->hib_stream.avail_in = size; 726 hibernate_state->hib_stream.next_in = (char *)src; 727 728 do { 729 /* Flush cache and TLB */ 730 hibernate_flush(); 731 732 /* 733 * Is this a special page? If yes, redirect the 734 * inflate output to a scratch page (eg, discard it) 735 */ 736 if (hibernate_inflate_skip(hiber_info, dest)) 737 hibernate_enter_resume_mapping( 738 HIBERNATE_INFLATE_PAGE, 739 HIBERNATE_INFLATE_PAGE, 0); 740 else 741 hibernate_enter_resume_mapping( 742 HIBERNATE_INFLATE_PAGE, dest, 0); 743 744 /* Read RLE code */ 745 hibernate_state->hib_stream.avail_out = sizeof(psize_t); 746 hibernate_state->hib_stream.next_out = (char *)&rle; 747 748 i = inflate(&hibernate_state->hib_stream, Z_FULL_FLUSH); 749 if (i != Z_OK && i != Z_STREAM_END) { 750 /* 751 * XXX - this will likely reboot/hang most machines, 752 * but there's not much else we can do here. 753 */ 754 panic("inflate rle error"); 755 } 756 757 /* Skip while RLE code is != 0 */ 758 while (rle != 0) { 759 dest += (rle * PAGE_SIZE); 760 hibernate_state->hib_stream.avail_out = 761 sizeof(psize_t); 762 hibernate_state->hib_stream.next_out = (char *)&rle; 763 764 i = inflate(&hibernate_state->hib_stream, 765 Z_FULL_FLUSH); 766 if (i != Z_OK && i != Z_STREAM_END) { 767 /* 768 * XXX - this will likely reboot/hang most 769 * machines but there's not much else 770 * we can do here. 771 */ 772 panic("inflate rle error 2"); 773 } 774 } 775 776 /* Set up the stream for inflate */ 777 hibernate_state->hib_stream.avail_out = PAGE_SIZE; 778 hibernate_state->hib_stream.next_out = 779 (char *)HIBERNATE_INFLATE_PAGE; 780 781 /* Process next block of data */ 782 i = inflate(&hibernate_state->hib_stream, Z_PARTIAL_FLUSH); 783 if (i != Z_OK && i != Z_STREAM_END) { 784 /* 785 * XXX - this will likely reboot/hang most machines, 786 * but there's not much else we can do here. 787 */ 788 panic("inflate error"); 789 } 790 791 dest += PAGE_SIZE - hibernate_state->hib_stream.avail_out; 792 } while (i != Z_STREAM_END); 793 } 794 795 /* 796 * deflate from src into the I/O page, up to 'remaining' bytes 797 * 798 * Returns number of input bytes consumed, and may reset 799 * the 'remaining' parameter if not all the output space was consumed 800 * (this information is needed to know how much to write to disk 801 */ 802 size_t 803 hibernate_deflate(union hibernate_info *hiber_info, paddr_t src, 804 size_t *remaining) 805 { 806 vaddr_t hibernate_io_page = hiber_info->piglet_va + PAGE_SIZE; 807 808 /* Set up the stream for deflate */ 809 hibernate_state->hib_stream.avail_in = PAGE_SIZE - (src & PAGE_MASK); 810 hibernate_state->hib_stream.avail_out = *remaining; 811 hibernate_state->hib_stream.next_in = (caddr_t)src; 812 hibernate_state->hib_stream.next_out = (caddr_t)hibernate_io_page + 813 (PAGE_SIZE - *remaining); 814 815 /* Process next block of data */ 816 if (deflate(&hibernate_state->hib_stream, Z_PARTIAL_FLUSH) != Z_OK) 817 panic("hibernate zlib deflate error"); 818 819 /* Update pointers and return number of bytes consumed */ 820 *remaining = hibernate_state->hib_stream.avail_out; 821 return (PAGE_SIZE - (src & PAGE_MASK)) - 822 hibernate_state->hib_stream.avail_in; 823 } 824 825 /* 826 * Write the hibernation information specified in hiber_info 827 * to the location in swap previously calculated (last block of 828 * swap), called the "signature block". 829 * 830 * Write the memory chunk table to the area in swap immediately 831 * preceding the signature block. 832 */ 833 int 834 hibernate_write_signature(union hibernate_info *hiber_info) 835 { 836 /* Write hibernate info to disk */ 837 return (hiber_info->io_func(hiber_info->device, hiber_info->sig_offset, 838 (vaddr_t)hiber_info, hiber_info->secsize, HIB_W, 839 hiber_info->io_page)); 840 } 841 842 /* 843 * Write the memory chunk table to the area in swap immediately 844 * preceding the signature block. The chunk table is stored 845 * in the piglet when this function is called. 846 */ 847 int 848 hibernate_write_chunktable(union hibernate_info *hiber_info) 849 { 850 struct hibernate_disk_chunk *chunks; 851 vaddr_t hibernate_chunk_table_start; 852 size_t hibernate_chunk_table_size; 853 daddr_t chunkbase; 854 int i; 855 856 hibernate_chunk_table_size = HIBERNATE_CHUNK_TABLE_SIZE; 857 858 chunkbase = hiber_info->sig_offset - 859 (hibernate_chunk_table_size / hiber_info->secsize); 860 861 hibernate_chunk_table_start = hiber_info->piglet_va + 862 HIBERNATE_CHUNK_SIZE; 863 864 chunks = (struct hibernate_disk_chunk *)(hiber_info->piglet_va + 865 HIBERNATE_CHUNK_SIZE); 866 867 /* Write chunk table */ 868 for (i = 0; i < hibernate_chunk_table_size; i += MAXPHYS) { 869 if (hiber_info->io_func(hiber_info->device, 870 chunkbase + (i/hiber_info->secsize), 871 (vaddr_t)(hibernate_chunk_table_start + i), 872 MAXPHYS, HIB_W, hiber_info->io_page)) 873 return (1); 874 } 875 876 return (0); 877 } 878 879 /* 880 * Write an empty hiber_info to the swap signature block, which is 881 * guaranteed to not match any valid hiber_info. 882 */ 883 int 884 hibernate_clear_signature(void) 885 { 886 union hibernate_info blank_hiber_info; 887 union hibernate_info hiber_info; 888 889 /* Zero out a blank hiber_info */ 890 bzero(&blank_hiber_info, sizeof(hiber_info)); 891 892 if (get_hibernate_info(&hiber_info, 0)) 893 return (1); 894 895 /* Write (zeroed) hibernate info to disk */ 896 /* XXX - use regular kernel write routine for this */ 897 if (hiber_info.io_func(hiber_info.device, hiber_info.sig_offset, 898 (vaddr_t)&blank_hiber_info, hiber_info.secsize, HIB_W, 899 hiber_info.io_page)) 900 panic("error hibernate write 6"); 901 902 return (0); 903 } 904 905 /* 906 * Check chunk range overlap when calculating whether or not to copy a 907 * compressed chunk to the piglet area before decompressing. 908 * 909 * returns zero if the ranges do not overlap, non-zero otherwise. 910 */ 911 int 912 hibernate_check_overlap(paddr_t r1s, paddr_t r1e, paddr_t r2s, paddr_t r2e) 913 { 914 /* case A : end of r1 overlaps start of r2 */ 915 if (r1s < r2s && r1e > r2s) 916 return (1); 917 918 /* case B : r1 entirely inside r2 */ 919 if (r1s >= r2s && r1e <= r2e) 920 return (1); 921 922 /* case C : r2 entirely inside r1 */ 923 if (r2s >= r1s && r2e <= r1e) 924 return (1); 925 926 /* case D : end of r2 overlaps start of r1 */ 927 if (r2s < r1s && r2e > r1s) 928 return (1); 929 930 return (0); 931 } 932 933 /* 934 * Compare two hibernate_infos to determine if they are the same (eg, 935 * we should be performing a hibernate resume on this machine. 936 * Not all fields are checked - just enough to verify that the machine 937 * has the same memory configuration and kernel as the one that 938 * wrote the signature previously. 939 */ 940 int 941 hibernate_compare_signature(union hibernate_info *mine, 942 union hibernate_info *disk) 943 { 944 u_int i; 945 946 if (mine->nranges != disk->nranges) 947 return (1); 948 949 if (strcmp(mine->kernel_version, disk->kernel_version) != 0) 950 return (1); 951 952 for (i = 0; i < mine->nranges; i++) { 953 if ((mine->ranges[i].base != disk->ranges[i].base) || 954 (mine->ranges[i].end != disk->ranges[i].end) ) 955 return (1); 956 } 957 958 return (0); 959 } 960 961 /* 962 * Reads read_size bytes from the hibernate device specified in 963 * hib_info at offset blkctr. Output is placed into the vaddr specified 964 * at dest. 965 * 966 * Separate offsets and pages are used to handle misaligned reads (reads 967 * that span a page boundary). 968 * 969 * blkctr specifies a relative offset (relative to the start of swap), 970 * not an absolute disk offset 971 * 972 */ 973 int 974 hibernate_read_block(union hibernate_info *hib_info, daddr_t blkctr, 975 size_t read_size, vaddr_t dest) 976 { 977 struct buf *bp; 978 struct bdevsw *bdsw; 979 int error; 980 981 bp = geteblk(read_size); 982 bdsw = &bdevsw[major(hib_info->device)]; 983 984 error = (*bdsw->d_open)(hib_info->device, FREAD, S_IFCHR, curproc); 985 if (error) { 986 printf("hibernate_read_block open failed\n"); 987 return (1); 988 } 989 990 bp->b_bcount = read_size; 991 bp->b_blkno = blkctr; 992 CLR(bp->b_flags, B_READ | B_WRITE | B_DONE); 993 SET(bp->b_flags, B_BUSY | B_READ | B_RAW); 994 bp->b_dev = hib_info->device; 995 bp->b_cylinder = 0; 996 (*bdsw->d_strategy)(bp); 997 998 error = biowait(bp); 999 if (error) { 1000 printf("hibernate_read_block biowait failed %d\n", error); 1001 error = (*bdsw->d_close)(hib_info->device, 0, S_IFCHR, 1002 curproc); 1003 if (error) 1004 printf("hibernate_read_block error close failed\n"); 1005 return (1); 1006 } 1007 1008 error = (*bdsw->d_close)(hib_info->device, FREAD, S_IFCHR, curproc); 1009 if (error) { 1010 printf("hibernate_read_block close failed\n"); 1011 return (1); 1012 } 1013 1014 bcopy(bp->b_data, (caddr_t)dest, read_size); 1015 1016 bp->b_flags |= B_INVAL; 1017 brelse(bp); 1018 1019 return (0); 1020 } 1021 1022 /* 1023 * Reads the signature block from swap, checks against the current machine's 1024 * information. If the information matches, perform a resume by reading the 1025 * saved image into the pig area, and unpacking. 1026 */ 1027 void 1028 hibernate_resume(void) 1029 { 1030 union hibernate_info hiber_info; 1031 int s; 1032 1033 /* Get current running machine's hibernate info */ 1034 bzero(&hiber_info, sizeof(hiber_info)); 1035 if (get_hibernate_info(&hiber_info, 0)) 1036 return; 1037 1038 /* Read hibernate info from disk */ 1039 s = splbio(); 1040 1041 /* XXX use regular kernel read routine here */ 1042 if (hiber_info.io_func(hiber_info.device, hiber_info.sig_offset, 1043 (vaddr_t)&disk_hiber_info, hiber_info.secsize, HIB_R, 1044 hiber_info.io_page)) 1045 panic("error in hibernate read"); 1046 1047 /* 1048 * If on-disk and in-memory hibernate signatures match, 1049 * this means we should do a resume from hibernate. 1050 */ 1051 if (hibernate_compare_signature(&hiber_info, &disk_hiber_info)) 1052 return; 1053 1054 /* Read the image from disk into the image (pig) area */ 1055 if (hibernate_read_image(&disk_hiber_info)) 1056 goto fail; 1057 1058 /* Point of no return ... */ 1059 1060 disable_intr(); 1061 cold = 1; 1062 1063 /* Switch stacks */ 1064 hibernate_switch_stack_machdep(); 1065 1066 /* 1067 * Image is now in high memory (pig area), copy to correct location 1068 * in memory. We'll eventually end up copying on top of ourself, but 1069 * we are assured the kernel code here is the same between the 1070 * hibernated and resuming kernel, and we are running on our own 1071 * stack, so the overwrite is ok. 1072 */ 1073 hibernate_unpack_image(&disk_hiber_info); 1074 1075 /* 1076 * Resume the loaded kernel by jumping to the MD resume vector. 1077 * We won't be returning from this call. 1078 */ 1079 hibernate_resume_machdep(); 1080 1081 fail: 1082 printf("Unable to resume hibernated image\n"); 1083 } 1084 1085 /* 1086 * Unpack image from pig area to original location by looping through the 1087 * list of output chunks in the order they should be restored (fchunks). 1088 * This ordering is used to avoid having inflate overwrite a chunk in the 1089 * middle of processing that chunk. This will, of course, happen during the 1090 * final output chunk, where we copy the chunk to the piglet area first, 1091 * before inflating. 1092 */ 1093 void 1094 hibernate_unpack_image(union hibernate_info *hiber_info) 1095 { 1096 struct hibernate_disk_chunk *chunks; 1097 union hibernate_info local_hiber_info; 1098 paddr_t image_cur = global_pig_start; 1099 int *fchunks, i; 1100 char *pva = (char *)hiber_info->piglet_va; 1101 1102 /* Mask off based on arch-specific piglet page size */ 1103 pva = (char *)((paddr_t)pva & (PIGLET_PAGE_MASK)); 1104 fchunks = (int *)(pva + (6 * PAGE_SIZE)); 1105 1106 chunks = (struct hibernate_disk_chunk *)(pva + HIBERNATE_CHUNK_SIZE); 1107 1108 /* Can't use hiber_info that's passed in after here */ 1109 bcopy(hiber_info, &local_hiber_info, sizeof(union hibernate_info)); 1110 1111 hibernate_activate_resume_pt_machdep(); 1112 1113 for (i = 0; i < local_hiber_info.chunk_ctr; i++) { 1114 /* Reset zlib for inflate */ 1115 if (hibernate_zlib_reset(&local_hiber_info, 0) != Z_OK) 1116 panic("hibernate failed to reset zlib for inflate"); 1117 1118 /* 1119 * If there is a conflict, copy the chunk to the piglet area 1120 * before unpacking it to its original location. 1121 */ 1122 if ((chunks[fchunks[i]].flags & HIBERNATE_CHUNK_CONFLICT) == 0) 1123 hibernate_inflate(&local_hiber_info, 1124 chunks[fchunks[i]].base, image_cur, 1125 chunks[fchunks[i]].compressed_size); 1126 else { 1127 bcopy((caddr_t)image_cur, 1128 pva + (HIBERNATE_CHUNK_SIZE * 2), 1129 chunks[fchunks[i]].compressed_size); 1130 hibernate_inflate(&local_hiber_info, 1131 chunks[fchunks[i]].base, 1132 (vaddr_t)(pva + (HIBERNATE_CHUNK_SIZE * 2)), 1133 chunks[fchunks[i]].compressed_size); 1134 } 1135 image_cur += chunks[fchunks[i]].compressed_size; 1136 } 1137 } 1138 1139 /* 1140 * Write a compressed version of this machine's memory to disk, at the 1141 * precalculated swap offset: 1142 * 1143 * end of swap - signature block size - chunk table size - memory size 1144 * 1145 * The function begins by looping through each phys mem range, cutting each 1146 * one into 4MB chunks. These chunks are then compressed individually 1147 * and written out to disk, in phys mem order. Some chunks might compress 1148 * more than others, and for this reason, each chunk's size is recorded 1149 * in the chunk table, which is written to disk after the image has 1150 * properly been compressed and written (in hibernate_write_chunktable). 1151 * 1152 * When this function is called, the machine is nearly suspended - most 1153 * devices are quiesced/suspended, interrupts are off, and cold has 1154 * been set. This means that there can be no side effects once the 1155 * write has started, and the write function itself can also have no 1156 * side effects. 1157 * 1158 * This function uses the piglet area during this process as follows: 1159 * 1160 * offset from piglet base use 1161 * ----------------------- -------------------- 1162 * 0 i/o allocation area 1163 * PAGE_SIZE i/o write area 1164 * 2*PAGE_SIZE temp/scratch page 1165 * 3*PAGE_SIZE temp/scratch page 1166 * 4*PAGE_SIZE hiballoc arena 1167 * 5*PAGE_SIZE to 85*PAGE_SIZE zlib deflate area 1168 * ... 1169 * HIBERNATE_CHUNK_SIZE chunk table temporary area 1170 * 1171 * Some transient piglet content is saved as part of deflate, 1172 * but it is irrelevant during resume as it will be repurposed 1173 * at that time for other things. 1174 */ 1175 int 1176 hibernate_write_chunks(union hibernate_info *hiber_info) 1177 { 1178 paddr_t range_base, range_end, inaddr, temp_inaddr; 1179 size_t nblocks, out_remaining, used, offset = 0; 1180 struct hibernate_disk_chunk *chunks; 1181 vaddr_t hibernate_io_page = hiber_info->piglet_va + PAGE_SIZE; 1182 daddr_t blkctr = hiber_info->image_offset; 1183 int i; 1184 psize_t rle; 1185 1186 hiber_info->chunk_ctr = 0; 1187 1188 /* 1189 * Allocate VA for the temp and copy page. 1190 * These will becomee part of the suspended kernel and will 1191 * be freed in hibernate_free, upon resume. 1192 */ 1193 hibernate_temp_page = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any, 1194 &kp_none, &kd_nowait); 1195 if (!hibernate_temp_page) 1196 return (1); 1197 1198 hibernate_copy_page = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any, 1199 &kp_none, &kd_nowait); 1200 if (!hibernate_copy_page) 1201 return (1); 1202 1203 pmap_kenter_pa(hibernate_copy_page, 1204 (hiber_info->piglet_pa + 3*PAGE_SIZE), VM_PROT_ALL); 1205 1206 /* XXX - not needed on all archs */ 1207 pmap_activate(curproc); 1208 1209 chunks = (struct hibernate_disk_chunk *)(hiber_info->piglet_va + 1210 HIBERNATE_CHUNK_SIZE); 1211 1212 /* Calculate the chunk regions */ 1213 for (i = 0; i < hiber_info->nranges; i++) { 1214 range_base = hiber_info->ranges[i].base; 1215 range_end = hiber_info->ranges[i].end; 1216 1217 inaddr = range_base; 1218 1219 while (inaddr < range_end) { 1220 chunks[hiber_info->chunk_ctr].base = inaddr; 1221 if (inaddr + HIBERNATE_CHUNK_SIZE < range_end) 1222 chunks[hiber_info->chunk_ctr].end = inaddr + 1223 HIBERNATE_CHUNK_SIZE; 1224 else 1225 chunks[hiber_info->chunk_ctr].end = range_end; 1226 1227 inaddr += HIBERNATE_CHUNK_SIZE; 1228 hiber_info->chunk_ctr ++; 1229 } 1230 } 1231 1232 /* Compress and write the chunks in the chunktable */ 1233 for (i = 0; i < hiber_info->chunk_ctr; i++) { 1234 range_base = chunks[i].base; 1235 range_end = chunks[i].end; 1236 1237 chunks[i].offset = blkctr; 1238 1239 /* Reset zlib for deflate */ 1240 if (hibernate_zlib_reset(hiber_info, 1) != Z_OK) 1241 return (1); 1242 1243 inaddr = range_base; 1244 1245 /* 1246 * For each range, loop through its phys mem region 1247 * and write out the chunks (the last chunk might be 1248 * smaller than the chunk size). 1249 */ 1250 while (inaddr < range_end) { 1251 out_remaining = PAGE_SIZE; 1252 while (out_remaining > 0 && inaddr < range_end) { 1253 pmap_kenter_pa(hibernate_temp_page, 1254 inaddr & PMAP_PA_MASK, VM_PROT_ALL); 1255 1256 /* XXX - not needed on all archs */ 1257 pmap_activate(curproc); 1258 1259 bcopy((caddr_t)hibernate_temp_page, 1260 (caddr_t)hibernate_copy_page, PAGE_SIZE); 1261 1262 /* 1263 * Adjust for regions that are not evenly 1264 * divisible by PAGE_SIZE 1265 */ 1266 temp_inaddr = (inaddr & PAGE_MASK) + 1267 hibernate_copy_page; 1268 1269 rle = uvm_page_rle(inaddr); 1270 while (rle > 0 && inaddr < range_end) { 1271 hibernate_state->hib_stream.avail_in = 1272 sizeof(psize_t); 1273 hibernate_state->hib_stream.avail_out = 1274 out_remaining; 1275 hibernate_state->hib_stream.next_in = 1276 (char *)&rle; 1277 hibernate_state->hib_stream.next_out = 1278 (caddr_t)hibernate_io_page + 1279 (PAGE_SIZE - out_remaining); 1280 1281 if (deflate(&hibernate_state->hib_stream, 1282 Z_PARTIAL_FLUSH) != Z_OK) 1283 return (1); 1284 1285 out_remaining = 1286 hibernate_state->hib_stream.avail_out; 1287 inaddr += (rle * PAGE_SIZE); 1288 if (inaddr > range_end) 1289 inaddr = range_end; 1290 else 1291 rle = uvm_page_rle(inaddr); 1292 } 1293 1294 if (out_remaining == 0) { 1295 /* Filled up the page */ 1296 nblocks = PAGE_SIZE / hiber_info->secsize; 1297 1298 if (hiber_info->io_func(hiber_info->device, 1299 blkctr, (vaddr_t)hibernate_io_page, 1300 PAGE_SIZE, HIB_W, hiber_info->io_page)) 1301 return (1); 1302 1303 blkctr += nblocks; 1304 out_remaining = PAGE_SIZE; 1305 } 1306 1307 /* Write '0' RLE code */ 1308 if (inaddr < range_end) { 1309 hibernate_state->hib_stream.avail_in = 1310 sizeof(psize_t); 1311 hibernate_state->hib_stream.avail_out = 1312 out_remaining; 1313 hibernate_state->hib_stream.next_in = 1314 (char *)&rle; 1315 hibernate_state->hib_stream.next_out = 1316 (caddr_t)hibernate_io_page + 1317 (PAGE_SIZE - out_remaining); 1318 1319 if (deflate(&hibernate_state->hib_stream, 1320 Z_PARTIAL_FLUSH) != Z_OK) 1321 return (1); 1322 1323 out_remaining = 1324 hibernate_state->hib_stream.avail_out; 1325 } 1326 1327 if (out_remaining == 0) { 1328 /* Filled up the page */ 1329 nblocks = PAGE_SIZE / hiber_info->secsize; 1330 1331 if (hiber_info->io_func(hiber_info->device, 1332 blkctr, (vaddr_t)hibernate_io_page, 1333 PAGE_SIZE, HIB_W, hiber_info->io_page)) 1334 return (1); 1335 1336 blkctr += nblocks; 1337 out_remaining = PAGE_SIZE; 1338 } 1339 1340 /* Deflate from temp_inaddr to IO page */ 1341 if (inaddr != range_end) 1342 inaddr += hibernate_deflate(hiber_info, 1343 temp_inaddr, &out_remaining); 1344 } 1345 1346 if (out_remaining == 0) { 1347 /* Filled up the page */ 1348 nblocks = PAGE_SIZE / hiber_info->secsize; 1349 1350 if (hiber_info->io_func(hiber_info->device, 1351 blkctr, (vaddr_t)hibernate_io_page, 1352 PAGE_SIZE, HIB_W, hiber_info->io_page)) 1353 return (1); 1354 1355 blkctr += nblocks; 1356 } 1357 } 1358 1359 if (inaddr != range_end) 1360 return (1); 1361 1362 /* 1363 * End of range. Round up to next secsize bytes 1364 * after finishing compress 1365 */ 1366 if (out_remaining == 0) 1367 out_remaining = PAGE_SIZE; 1368 1369 /* Finish compress */ 1370 hibernate_state->hib_stream.avail_in = 0; 1371 hibernate_state->hib_stream.avail_out = out_remaining; 1372 hibernate_state->hib_stream.next_in = (caddr_t)inaddr; 1373 hibernate_state->hib_stream.next_out = 1374 (caddr_t)hibernate_io_page + (PAGE_SIZE - out_remaining); 1375 1376 if (deflate(&hibernate_state->hib_stream, Z_FINISH) != 1377 Z_STREAM_END) 1378 return (1); 1379 1380 out_remaining = hibernate_state->hib_stream.avail_out; 1381 1382 used = PAGE_SIZE - out_remaining; 1383 nblocks = used / hiber_info->secsize; 1384 1385 /* Round up to next block if needed */ 1386 if (used % hiber_info->secsize != 0) 1387 nblocks ++; 1388 1389 /* Write final block(s) for this chunk */ 1390 if (hiber_info->io_func(hiber_info->device, blkctr, 1391 (vaddr_t)hibernate_io_page, nblocks*hiber_info->secsize, 1392 HIB_W, hiber_info->io_page)) 1393 return (1); 1394 1395 blkctr += nblocks; 1396 1397 offset = blkctr; 1398 chunks[i].compressed_size = (offset - chunks[i].offset) * 1399 hiber_info->secsize; 1400 } 1401 1402 return (0); 1403 } 1404 1405 /* 1406 * Reset the zlib stream state and allocate a new hiballoc area for either 1407 * inflate or deflate. This function is called once for each hibernate chunk. 1408 * Calling hiballoc_init multiple times is acceptable since the memory it is 1409 * provided is unmanaged memory (stolen). We use the memory provided to us 1410 * by the piglet allocated via the supplied hiber_info. 1411 */ 1412 int 1413 hibernate_zlib_reset(union hibernate_info *hiber_info, int deflate) 1414 { 1415 vaddr_t hibernate_zlib_start; 1416 size_t hibernate_zlib_size; 1417 char *pva = (char *)hiber_info->piglet_va; 1418 1419 hibernate_state = (struct hibernate_zlib_state *) 1420 (pva + (7 * PAGE_SIZE)); 1421 1422 hibernate_zlib_start = (vaddr_t)(pva + (8 * PAGE_SIZE)); 1423 hibernate_zlib_size = 80 * PAGE_SIZE; 1424 1425 bzero((caddr_t)hibernate_zlib_start, hibernate_zlib_size); 1426 bzero((caddr_t)hibernate_state, PAGE_SIZE); 1427 1428 /* Set up stream structure */ 1429 hibernate_state->hib_stream.zalloc = (alloc_func)hibernate_zlib_alloc; 1430 hibernate_state->hib_stream.zfree = (free_func)hibernate_zlib_free; 1431 1432 /* Initialize the hiballoc arena for zlib allocs/frees */ 1433 hiballoc_init(&hibernate_state->hiballoc_arena, 1434 (caddr_t)hibernate_zlib_start, hibernate_zlib_size); 1435 1436 if (deflate) { 1437 return deflateInit(&hibernate_state->hib_stream, 1438 Z_BEST_SPEED); 1439 } else 1440 return inflateInit(&hibernate_state->hib_stream); 1441 } 1442 1443 /* 1444 * Reads the hibernated memory image from disk, whose location and 1445 * size are recorded in hiber_info. Begin by reading the persisted 1446 * chunk table, which records the original chunk placement location 1447 * and compressed size for each. Next, allocate a pig region of 1448 * sufficient size to hold the compressed image. Next, read the 1449 * chunks into the pig area (calling hibernate_read_chunks to do this), 1450 * and finally, if all of the above succeeds, clear the hibernate signature. 1451 * The function will then return to hibernate_resume, which will proceed 1452 * to unpack the pig image to the correct place in memory. 1453 */ 1454 int 1455 hibernate_read_image(union hibernate_info *hiber_info) 1456 { 1457 size_t compressed_size, disk_size, chunktable_size, pig_sz; 1458 paddr_t image_start, image_end, pig_start, pig_end; 1459 struct hibernate_disk_chunk *chunks; 1460 daddr_t blkctr; 1461 vaddr_t chunktable = (vaddr_t)NULL; 1462 paddr_t piglet_chunktable = hiber_info->piglet_pa + 1463 HIBERNATE_CHUNK_SIZE; 1464 int i; 1465 1466 /* Calculate total chunk table size in disk blocks */ 1467 chunktable_size = HIBERNATE_CHUNK_TABLE_SIZE / hiber_info->secsize; 1468 1469 blkctr = hiber_info->sig_offset - chunktable_size - 1470 hiber_info->swap_offset; 1471 1472 chunktable = (vaddr_t)km_alloc(HIBERNATE_CHUNK_TABLE_SIZE, &kv_any, 1473 &kp_none, &kd_nowait); 1474 1475 if (!chunktable) 1476 return (1); 1477 1478 /* Read the chunktable from disk into the piglet chunktable */ 1479 for (i = 0; i < HIBERNATE_CHUNK_TABLE_SIZE; 1480 i += PAGE_SIZE, blkctr += PAGE_SIZE/hiber_info->secsize) { 1481 pmap_kenter_pa(chunktable + i, piglet_chunktable + i, VM_PROT_ALL); 1482 hibernate_read_block(hiber_info, blkctr, PAGE_SIZE, 1483 chunktable + i); 1484 } 1485 1486 blkctr = hiber_info->image_offset; 1487 compressed_size = 0; 1488 pmap_kenter_pa(chunktable, piglet_chunktable, VM_PROT_ALL); 1489 chunks = (struct hibernate_disk_chunk *)chunktable; 1490 1491 for (i = 0; i < hiber_info->chunk_ctr; i++) 1492 compressed_size += chunks[i].compressed_size; 1493 1494 disk_size = compressed_size; 1495 1496 /* Allocate the pig area */ 1497 pig_sz = compressed_size + HIBERNATE_CHUNK_SIZE; 1498 if (uvm_pmr_alloc_pig(&pig_start, pig_sz) == ENOMEM) 1499 return (1); 1500 1501 pig_end = pig_start + pig_sz; 1502 1503 /* Calculate image extents. Pig image must end on a chunk boundary. */ 1504 image_end = pig_end & ~(HIBERNATE_CHUNK_SIZE - 1); 1505 image_start = pig_start; 1506 1507 image_start = image_end - disk_size; 1508 1509 hibernate_read_chunks(hiber_info, image_start, image_end, disk_size, 1510 chunks); 1511 1512 /* Prepare the resume time pmap/page table */ 1513 hibernate_populate_resume_pt(hiber_info, image_start, image_end); 1514 1515 /* Read complete, clear the signature and return */ 1516 return hibernate_clear_signature(); 1517 } 1518 1519 /* 1520 * Read the hibernated memory chunks from disk (chunk information at this 1521 * point is stored in the piglet) into the pig area specified by 1522 * [pig_start .. pig_end]. Order the chunks so that the final chunk is the 1523 * only chunk with overlap possibilities. 1524 * 1525 * This function uses the piglet area during this process as follows: 1526 * 1527 * offset from piglet base use 1528 * ----------------------- -------------------- 1529 * 0 i/o allocation area 1530 * PAGE_SIZE i/o write area 1531 * 2*PAGE_SIZE temp/scratch page 1532 * 3*PAGE_SIZE temp/scratch page 1533 * 4*PAGE_SIZE to 6*PAGE_SIZE chunk ordering area 1534 * 7*PAGE_SIZE hiballoc arena 1535 * 8*PAGE_SIZE to 88*PAGE_SIZE zlib deflate area 1536 * ... 1537 * HIBERNATE_CHUNK_SIZE chunk table temporary area 1538 */ 1539 int 1540 hibernate_read_chunks(union hibernate_info *hib_info, paddr_t pig_start, 1541 paddr_t pig_end, size_t image_compr_size, 1542 struct hibernate_disk_chunk *chunks) 1543 { 1544 paddr_t img_index, img_cur, r1s, r1e, r2s, r2e; 1545 paddr_t copy_start, copy_end, piglet_cur; 1546 paddr_t piglet_base = hib_info->piglet_pa; 1547 paddr_t piglet_end = piglet_base + HIBERNATE_CHUNK_SIZE; 1548 daddr_t blkctr; 1549 size_t processed, compressed_size, read_size; 1550 int i, j, overlap, found, nchunks; 1551 int nochunks = 0, nfchunks = 0, npchunks = 0; 1552 int *ochunks, *pchunks, *fchunks; 1553 vaddr_t tempva = (vaddr_t)NULL, hibernate_fchunk_area = (vaddr_t)NULL; 1554 1555 global_pig_start = pig_start; 1556 1557 /* XXX - dont need this on all archs */ 1558 pmap_activate(curproc); 1559 1560 /* 1561 * These mappings go into the resuming kernel's page table, and are 1562 * used only during image read. They dissappear from existence 1563 * when the suspended kernel is unpacked on top of us. 1564 */ 1565 tempva = (vaddr_t)km_alloc(2*PAGE_SIZE, &kv_any, &kp_none, &kd_nowait); 1566 if (!tempva) 1567 return (1); 1568 hibernate_fchunk_area = (vaddr_t)km_alloc(3*PAGE_SIZE, &kv_any, 1569 &kp_none, &kd_nowait); 1570 if (!hibernate_fchunk_area) 1571 return (1); 1572 1573 /* Temporary output chunk ordering */ 1574 ochunks = (int *)hibernate_fchunk_area; 1575 1576 /* Piglet chunk ordering */ 1577 pchunks = (int *)(hibernate_fchunk_area + PAGE_SIZE); 1578 1579 /* Final chunk ordering */ 1580 fchunks = (int *)(hibernate_fchunk_area + (2*PAGE_SIZE)); 1581 1582 /* Map the chunk ordering region */ 1583 pmap_kenter_pa(hibernate_fchunk_area, 1584 piglet_base + (4*PAGE_SIZE), VM_PROT_ALL); 1585 pmap_kenter_pa((vaddr_t)pchunks, piglet_base + (5*PAGE_SIZE), 1586 VM_PROT_ALL); 1587 pmap_kenter_pa((vaddr_t)fchunks, piglet_base + (6*PAGE_SIZE), 1588 VM_PROT_ALL); 1589 1590 nchunks = hib_info->chunk_ctr; 1591 1592 /* Initially start all chunks as unplaced */ 1593 for (i = 0; i < nchunks; i++) 1594 chunks[i].flags = 0; 1595 1596 /* 1597 * Search the list for chunks that are outside the pig area. These 1598 * can be placed first in the final output list. 1599 */ 1600 for (i = 0; i < nchunks; i++) { 1601 if (chunks[i].end <= pig_start || chunks[i].base >= pig_end) { 1602 ochunks[nochunks] = (u_int8_t)i; 1603 fchunks[nfchunks] = (u_int8_t)i; 1604 nochunks++; 1605 nfchunks++; 1606 chunks[i].flags |= HIBERNATE_CHUNK_USED; 1607 } 1608 } 1609 1610 /* 1611 * Walk the ordering, place the chunks in ascending memory order. 1612 * Conflicts might arise, these are handled next. 1613 */ 1614 do { 1615 img_index = -1; 1616 found = 0; 1617 j = -1; 1618 for (i = 0; i < nchunks; i++) 1619 if (chunks[i].base < img_index && 1620 chunks[i].flags == 0 ) { 1621 j = i; 1622 img_index = chunks[i].base; 1623 } 1624 1625 if (j != -1) { 1626 found = 1; 1627 ochunks[nochunks] = (short)j; 1628 nochunks++; 1629 chunks[j].flags |= HIBERNATE_CHUNK_PLACED; 1630 } 1631 } while (found); 1632 1633 img_index = pig_start; 1634 1635 /* 1636 * Identify chunk output conflicts (chunks whose pig load area 1637 * corresponds to their original memory placement location) 1638 */ 1639 for (i = 0; i < nochunks ; i++) { 1640 overlap = 0; 1641 r1s = img_index; 1642 r1e = img_index + chunks[ochunks[i]].compressed_size; 1643 r2s = chunks[ochunks[i]].base; 1644 r2e = chunks[ochunks[i]].end; 1645 1646 overlap = hibernate_check_overlap(r1s, r1e, r2s, r2e); 1647 if (overlap) 1648 chunks[ochunks[i]].flags |= HIBERNATE_CHUNK_CONFLICT; 1649 img_index += chunks[ochunks[i]].compressed_size; 1650 } 1651 1652 /* 1653 * Prepare the final output chunk list. Calculate an output 1654 * inflate strategy for overlapping chunks if needed. 1655 */ 1656 img_index = pig_start; 1657 for (i = 0; i < nochunks ; i++) { 1658 /* 1659 * If a conflict is detected, consume enough compressed 1660 * output chunks to fill the piglet 1661 */ 1662 if (chunks[ochunks[i]].flags & HIBERNATE_CHUNK_CONFLICT) { 1663 copy_start = piglet_base; 1664 copy_end = piglet_end; 1665 piglet_cur = piglet_base; 1666 npchunks = 0; 1667 j = i; 1668 while (copy_start < copy_end && j < nochunks) { 1669 piglet_cur += chunks[ochunks[j]].compressed_size; 1670 pchunks[npchunks] = ochunks[j]; 1671 npchunks++; 1672 copy_start += chunks[ochunks[j]].compressed_size; 1673 img_index += chunks[ochunks[j]].compressed_size; 1674 i++; 1675 j++; 1676 } 1677 1678 piglet_cur = piglet_base; 1679 for (j = 0; j < npchunks; j++) { 1680 piglet_cur += chunks[pchunks[j]].compressed_size; 1681 fchunks[nfchunks] = pchunks[j]; 1682 chunks[pchunks[j]].flags |= HIBERNATE_CHUNK_USED; 1683 nfchunks++; 1684 } 1685 } else { 1686 /* 1687 * No conflict, chunk can be added without copying 1688 */ 1689 if ((chunks[ochunks[i]].flags & 1690 HIBERNATE_CHUNK_USED) == 0) { 1691 fchunks[nfchunks] = ochunks[i]; 1692 chunks[ochunks[i]].flags |= HIBERNATE_CHUNK_USED; 1693 nfchunks++; 1694 } 1695 img_index += chunks[ochunks[i]].compressed_size; 1696 } 1697 } 1698 1699 img_index = pig_start; 1700 for (i = 0; i < nfchunks; i++) { 1701 piglet_cur = piglet_base; 1702 img_index += chunks[fchunks[i]].compressed_size; 1703 } 1704 1705 img_cur = pig_start; 1706 1707 for (i = 0; i < nfchunks; i++) { 1708 blkctr = chunks[fchunks[i]].offset - hib_info->swap_offset; 1709 processed = 0; 1710 compressed_size = chunks[fchunks[i]].compressed_size; 1711 1712 while (processed < compressed_size) { 1713 pmap_kenter_pa(tempva, img_cur, VM_PROT_ALL); 1714 pmap_kenter_pa(tempva + PAGE_SIZE, img_cur+PAGE_SIZE, 1715 VM_PROT_ALL); 1716 1717 /* XXX - not needed on all archs */ 1718 pmap_activate(curproc); 1719 if (compressed_size - processed >= PAGE_SIZE) 1720 read_size = PAGE_SIZE; 1721 else 1722 read_size = compressed_size - processed; 1723 1724 hibernate_read_block(hib_info, blkctr, read_size, 1725 tempva + (img_cur & PAGE_MASK)); 1726 1727 blkctr += (read_size / hib_info->secsize); 1728 1729 hibernate_flush(); 1730 pmap_kremove(tempva, PAGE_SIZE); 1731 pmap_kremove(tempva + PAGE_SIZE, PAGE_SIZE); 1732 processed += read_size; 1733 img_cur += read_size; 1734 } 1735 } 1736 1737 return (0); 1738 } 1739 1740 /* 1741 * Hibernating a machine comprises the following operations: 1742 * 1. Calculating this machine's hibernate_info information 1743 * 2. Allocating a piglet and saving the piglet's physaddr 1744 * 3. Calculating the memory chunks 1745 * 4. Writing the compressed chunks to disk 1746 * 5. Writing the chunk table 1747 * 6. Writing the signature block (hibernate_info) 1748 * 1749 * On most architectures, the function calling hibernate_suspend would 1750 * then power off the machine using some MD-specific implementation. 1751 */ 1752 int 1753 hibernate_suspend(void) 1754 { 1755 union hibernate_info hib_info; 1756 1757 /* 1758 * Calculate memory ranges, swap offsets, etc. 1759 * This also allocates a piglet whose physaddr is stored in 1760 * hib_info->piglet_pa and vaddr stored in hib_info->piglet_va 1761 */ 1762 if (get_hibernate_info(&hib_info, 1)) 1763 return (1); 1764 1765 global_piglet_va = hib_info.piglet_va; 1766 1767 /* XXX - Won't need to zero everything with RLE */ 1768 uvm_pmr_zero_everything(); 1769 1770 if (hibernate_write_chunks(&hib_info)) 1771 return (1); 1772 1773 if (hibernate_write_chunktable(&hib_info)) 1774 return (1); 1775 1776 if (hibernate_write_signature(&hib_info)) 1777 return (1); 1778 1779 delay(500000); 1780 return (0); 1781 } 1782 1783 /* 1784 * Free items allocated during hibernate 1785 */ 1786 void 1787 hibernate_free(void) 1788 { 1789 uvm_pmr_free_piglet(global_piglet_va, 3*HIBERNATE_CHUNK_SIZE); 1790 1791 pmap_kremove(hibernate_copy_page, PAGE_SIZE); 1792 pmap_kremove(hibernate_temp_page, PAGE_SIZE); 1793 pmap_update(pmap_kernel()); 1794 1795 km_free((void *)hibernate_copy_page, PAGE_SIZE, &kv_any, &kp_none); 1796 km_free((void *)hibernate_temp_page, PAGE_SIZE, &kv_any, &kp_none); 1797 } 1798