1 /* $OpenBSD: subr_hibernate.c,v 1.121 2017/03/27 20:26:39 deraadt Exp $ */ 2 3 /* 4 * Copyright (c) 2011 Ariane van der Steldt <ariane@stack.nl> 5 * Copyright (c) 2011 Mike Larkin <mlarkin@openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/hibernate.h> 21 #include <sys/malloc.h> 22 #include <sys/param.h> 23 #include <sys/tree.h> 24 #include <sys/systm.h> 25 #include <sys/disklabel.h> 26 #include <sys/disk.h> 27 #include <sys/conf.h> 28 #include <sys/buf.h> 29 #include <sys/fcntl.h> 30 #include <sys/stat.h> 31 #include <sys/atomic.h> 32 33 #include <uvm/uvm.h> 34 #include <uvm/uvm_swap.h> 35 36 #include <machine/hibernate.h> 37 38 /* 39 * Hibernate piglet layout information 40 * 41 * The piglet is a scratch area of memory allocated by the suspending kernel. 42 * Its phys and virt addrs are recorded in the signature block. The piglet is 43 * used to guarantee an unused area of memory that can be used by the resuming 44 * kernel for various things. The piglet is excluded during unpack operations. 45 * The piglet size is presently 4*HIBERNATE_CHUNK_SIZE (typically 4*4MB). 46 * 47 * Offset from piglet_base Purpose 48 * ---------------------------------------------------------------------------- 49 * 0 Private page for suspend I/O write functions 50 * 1*PAGE_SIZE I/O page used during hibernate suspend 51 * 2*PAGE_SIZE I/O page used during hibernate suspend 52 * 3*PAGE_SIZE copy page used during hibernate suspend 53 * 4*PAGE_SIZE final chunk ordering list (24 pages) 54 * 28*PAGE_SIZE RLE utility page 55 * 29*PAGE_SIZE start of hiballoc area 56 * 30*PAGE_SIZE preserved entropy 57 * 110*PAGE_SIZE end of hiballoc area (80 pages) 58 * ... unused 59 * HIBERNATE_CHUNK_SIZE start of hibernate chunk table 60 * 2*HIBERNATE_CHUNK_SIZE bounce area for chunks being unpacked 61 * 4*HIBERNATE_CHUNK_SIZE end of piglet 62 */ 63 64 /* Temporary vaddr ranges used during hibernate */ 65 vaddr_t hibernate_temp_page; 66 vaddr_t hibernate_copy_page; 67 vaddr_t hibernate_rle_page; 68 69 /* Hibernate info as read from disk during resume */ 70 union hibernate_info disk_hib; 71 72 /* 73 * Global copy of the pig start address. This needs to be a global as we 74 * switch stacks after computing it - it can't be stored on the stack. 75 */ 76 paddr_t global_pig_start; 77 78 /* 79 * Global copies of the piglet start addresses (PA/VA). We store these 80 * as globals to avoid having to carry them around as parameters, as the 81 * piglet is allocated early and freed late - its lifecycle extends beyond 82 * that of the hibernate info union which is calculated on suspend/resume. 83 */ 84 vaddr_t global_piglet_va; 85 paddr_t global_piglet_pa; 86 87 /* #define HIB_DEBUG */ 88 #ifdef HIB_DEBUG 89 int hib_debug = 99; 90 #define DPRINTF(x...) do { if (hib_debug) printf(x); } while (0) 91 #define DNPRINTF(n,x...) do { if (hib_debug > (n)) printf(x); } while (0) 92 #else 93 #define DPRINTF(x...) 94 #define DNPRINTF(n,x...) 95 #endif 96 97 #ifndef NO_PROPOLICE 98 extern long __guard_local; 99 #endif /* ! NO_PROPOLICE */ 100 101 void hibernate_copy_chunk_to_piglet(paddr_t, vaddr_t, size_t); 102 int hibernate_calc_rle(paddr_t, paddr_t); 103 int hibernate_write_rle(union hibernate_info *, paddr_t, paddr_t, daddr_t *, 104 size_t *); 105 106 #define MAX_RLE (HIBERNATE_CHUNK_SIZE / PAGE_SIZE) 107 108 /* 109 * Hib alloc enforced alignment. 110 */ 111 #define HIB_ALIGN 8 /* bytes alignment */ 112 113 /* 114 * sizeof builtin operation, but with alignment constraint. 115 */ 116 #define HIB_SIZEOF(_type) roundup(sizeof(_type), HIB_ALIGN) 117 118 struct hiballoc_entry { 119 size_t hibe_use; 120 size_t hibe_space; 121 RBT_ENTRY(hiballoc_entry) hibe_entry; 122 }; 123 124 /* 125 * Sort hibernate memory ranges by ascending PA 126 */ 127 void 128 hibernate_sort_ranges(union hibernate_info *hib_info) 129 { 130 int i, j; 131 struct hibernate_memory_range *ranges; 132 paddr_t base, end; 133 134 ranges = hib_info->ranges; 135 136 for (i = 1; i < hib_info->nranges; i++) { 137 j = i; 138 while (j > 0 && ranges[j - 1].base > ranges[j].base) { 139 base = ranges[j].base; 140 end = ranges[j].end; 141 ranges[j].base = ranges[j - 1].base; 142 ranges[j].end = ranges[j - 1].end; 143 ranges[j - 1].base = base; 144 ranges[j - 1].end = end; 145 j--; 146 } 147 } 148 } 149 150 /* 151 * Compare hiballoc entries based on the address they manage. 152 * 153 * Since the address is fixed, relative to struct hiballoc_entry, 154 * we just compare the hiballoc_entry pointers. 155 */ 156 static __inline int 157 hibe_cmp(const struct hiballoc_entry *l, const struct hiballoc_entry *r) 158 { 159 vaddr_t vl = (vaddr_t)l; 160 vaddr_t vr = (vaddr_t)r; 161 162 return vl < vr ? -1 : (vl > vr); 163 } 164 165 RBT_PROTOTYPE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp) 166 167 /* 168 * Given a hiballoc entry, return the address it manages. 169 */ 170 static __inline void * 171 hib_entry_to_addr(struct hiballoc_entry *entry) 172 { 173 caddr_t addr; 174 175 addr = (caddr_t)entry; 176 addr += HIB_SIZEOF(struct hiballoc_entry); 177 return addr; 178 } 179 180 /* 181 * Given an address, find the hiballoc that corresponds. 182 */ 183 static __inline struct hiballoc_entry* 184 hib_addr_to_entry(void *addr_param) 185 { 186 caddr_t addr; 187 188 addr = (caddr_t)addr_param; 189 addr -= HIB_SIZEOF(struct hiballoc_entry); 190 return (struct hiballoc_entry*)addr; 191 } 192 193 RBT_GENERATE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp); 194 195 /* 196 * Allocate memory from the arena. 197 * 198 * Returns NULL if no memory is available. 199 */ 200 void * 201 hib_alloc(struct hiballoc_arena *arena, size_t alloc_sz) 202 { 203 struct hiballoc_entry *entry, *new_entry; 204 size_t find_sz; 205 206 /* 207 * Enforce alignment of HIB_ALIGN bytes. 208 * 209 * Note that, because the entry is put in front of the allocation, 210 * 0-byte allocations are guaranteed a unique address. 211 */ 212 alloc_sz = roundup(alloc_sz, HIB_ALIGN); 213 214 /* 215 * Find an entry with hibe_space >= find_sz. 216 * 217 * If the root node is not large enough, we switch to tree traversal. 218 * Because all entries are made at the bottom of the free space, 219 * traversal from the end has a slightly better chance of yielding 220 * a sufficiently large space. 221 */ 222 find_sz = alloc_sz + HIB_SIZEOF(struct hiballoc_entry); 223 entry = RBT_ROOT(hiballoc_addr, &arena->hib_addrs); 224 if (entry != NULL && entry->hibe_space < find_sz) { 225 RBT_FOREACH_REVERSE(entry, hiballoc_addr, &arena->hib_addrs) { 226 if (entry->hibe_space >= find_sz) 227 break; 228 } 229 } 230 231 /* 232 * Insufficient or too fragmented memory. 233 */ 234 if (entry == NULL) 235 return NULL; 236 237 /* 238 * Create new entry in allocated space. 239 */ 240 new_entry = (struct hiballoc_entry*)( 241 (caddr_t)hib_entry_to_addr(entry) + entry->hibe_use); 242 new_entry->hibe_space = entry->hibe_space - find_sz; 243 new_entry->hibe_use = alloc_sz; 244 245 /* 246 * Insert entry. 247 */ 248 if (RBT_INSERT(hiballoc_addr, &arena->hib_addrs, new_entry) != NULL) 249 panic("hib_alloc: insert failure"); 250 entry->hibe_space = 0; 251 252 /* Return address managed by entry. */ 253 return hib_entry_to_addr(new_entry); 254 } 255 256 void 257 hib_getentropy(char **bufp, size_t *bufplen) 258 { 259 if (!bufp || !bufplen) 260 return; 261 262 *bufp = (char *)(global_piglet_va + (29 * PAGE_SIZE)); 263 *bufplen = PAGE_SIZE; 264 } 265 266 /* 267 * Free a pointer previously allocated from this arena. 268 * 269 * If addr is NULL, this will be silently accepted. 270 */ 271 void 272 hib_free(struct hiballoc_arena *arena, void *addr) 273 { 274 struct hiballoc_entry *entry, *prev; 275 276 if (addr == NULL) 277 return; 278 279 /* 280 * Derive entry from addr and check it is really in this arena. 281 */ 282 entry = hib_addr_to_entry(addr); 283 if (RBT_FIND(hiballoc_addr, &arena->hib_addrs, entry) != entry) 284 panic("hib_free: freed item %p not in hib arena", addr); 285 286 /* 287 * Give the space in entry to its predecessor. 288 * 289 * If entry has no predecessor, change its used space into free space 290 * instead. 291 */ 292 prev = RBT_PREV(hiballoc_addr, entry); 293 if (prev != NULL && 294 (void *)((caddr_t)prev + HIB_SIZEOF(struct hiballoc_entry) + 295 prev->hibe_use + prev->hibe_space) == entry) { 296 /* Merge entry. */ 297 RBT_REMOVE(hiballoc_addr, &arena->hib_addrs, entry); 298 prev->hibe_space += HIB_SIZEOF(struct hiballoc_entry) + 299 entry->hibe_use + entry->hibe_space; 300 } else { 301 /* Flip used memory to free space. */ 302 entry->hibe_space += entry->hibe_use; 303 entry->hibe_use = 0; 304 } 305 } 306 307 /* 308 * Initialize hiballoc. 309 * 310 * The allocator will manage memmory at ptr, which is len bytes. 311 */ 312 int 313 hiballoc_init(struct hiballoc_arena *arena, void *p_ptr, size_t p_len) 314 { 315 struct hiballoc_entry *entry; 316 caddr_t ptr; 317 size_t len; 318 319 RBT_INIT(hiballoc_addr, &arena->hib_addrs); 320 321 /* 322 * Hib allocator enforces HIB_ALIGN alignment. 323 * Fixup ptr and len. 324 */ 325 ptr = (caddr_t)roundup((vaddr_t)p_ptr, HIB_ALIGN); 326 len = p_len - ((size_t)ptr - (size_t)p_ptr); 327 len &= ~((size_t)HIB_ALIGN - 1); 328 329 /* 330 * Insufficient memory to be able to allocate and also do bookkeeping. 331 */ 332 if (len <= HIB_SIZEOF(struct hiballoc_entry)) 333 return ENOMEM; 334 335 /* 336 * Create entry describing space. 337 */ 338 entry = (struct hiballoc_entry*)ptr; 339 entry->hibe_use = 0; 340 entry->hibe_space = len - HIB_SIZEOF(struct hiballoc_entry); 341 RBT_INSERT(hiballoc_addr, &arena->hib_addrs, entry); 342 343 return 0; 344 } 345 346 /* 347 * Zero all free memory. 348 */ 349 void 350 uvm_pmr_zero_everything(void) 351 { 352 struct uvm_pmemrange *pmr; 353 struct vm_page *pg; 354 int i; 355 356 uvm_lock_fpageq(); 357 TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) { 358 /* Zero single pages. */ 359 while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_DIRTY])) 360 != NULL) { 361 uvm_pmr_remove(pmr, pg); 362 uvm_pagezero(pg); 363 atomic_setbits_int(&pg->pg_flags, PG_ZERO); 364 uvmexp.zeropages++; 365 uvm_pmr_insert(pmr, pg, 0); 366 } 367 368 /* Zero multi page ranges. */ 369 while ((pg = RBT_ROOT(uvm_pmr_size, 370 &pmr->size[UVM_PMR_MEMTYPE_DIRTY])) != NULL) { 371 pg--; /* Size tree always has second page. */ 372 uvm_pmr_remove(pmr, pg); 373 for (i = 0; i < pg->fpgsz; i++) { 374 uvm_pagezero(&pg[i]); 375 atomic_setbits_int(&pg[i].pg_flags, PG_ZERO); 376 uvmexp.zeropages++; 377 } 378 uvm_pmr_insert(pmr, pg, 0); 379 } 380 } 381 uvm_unlock_fpageq(); 382 } 383 384 /* 385 * Mark all memory as dirty. 386 * 387 * Used to inform the system that the clean memory isn't clean for some 388 * reason, for example because we just came back from hibernate. 389 */ 390 void 391 uvm_pmr_dirty_everything(void) 392 { 393 struct uvm_pmemrange *pmr; 394 struct vm_page *pg; 395 int i; 396 397 uvm_lock_fpageq(); 398 TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) { 399 /* Dirty single pages. */ 400 while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_ZERO])) 401 != NULL) { 402 uvm_pmr_remove(pmr, pg); 403 atomic_clearbits_int(&pg->pg_flags, PG_ZERO); 404 uvm_pmr_insert(pmr, pg, 0); 405 } 406 407 /* Dirty multi page ranges. */ 408 while ((pg = RBT_ROOT(uvm_pmr_size, 409 &pmr->size[UVM_PMR_MEMTYPE_ZERO])) != NULL) { 410 pg--; /* Size tree always has second page. */ 411 uvm_pmr_remove(pmr, pg); 412 for (i = 0; i < pg->fpgsz; i++) 413 atomic_clearbits_int(&pg[i].pg_flags, PG_ZERO); 414 uvm_pmr_insert(pmr, pg, 0); 415 } 416 } 417 418 uvmexp.zeropages = 0; 419 uvm_unlock_fpageq(); 420 } 421 422 /* 423 * Allocate an area that can hold sz bytes and doesn't overlap with 424 * the piglet at piglet_pa. 425 */ 426 int 427 uvm_pmr_alloc_pig(paddr_t *pa, psize_t sz, paddr_t piglet_pa) 428 { 429 struct uvm_constraint_range pig_constraint; 430 struct kmem_pa_mode kp_pig = { 431 .kp_constraint = &pig_constraint, 432 .kp_maxseg = 1 433 }; 434 vaddr_t va; 435 436 sz = round_page(sz); 437 438 pig_constraint.ucr_low = piglet_pa + 4 * HIBERNATE_CHUNK_SIZE; 439 pig_constraint.ucr_high = -1; 440 441 va = (vaddr_t)km_alloc(sz, &kv_any, &kp_pig, &kd_nowait); 442 if (va == 0) { 443 pig_constraint.ucr_low = 0; 444 pig_constraint.ucr_high = piglet_pa - 1; 445 446 va = (vaddr_t)km_alloc(sz, &kv_any, &kp_pig, &kd_nowait); 447 if (va == 0) 448 return ENOMEM; 449 } 450 451 pmap_extract(pmap_kernel(), va, pa); 452 return 0; 453 } 454 455 /* 456 * Allocate a piglet area. 457 * 458 * This needs to be in DMA-safe memory. 459 * Piglets are aligned. 460 * 461 * sz and align in bytes. 462 * 463 * The call will sleep for the pagedaemon to attempt to free memory. 464 * The pagedaemon may decide its not possible to free enough memory, causing 465 * the allocation to fail. 466 */ 467 int 468 uvm_pmr_alloc_piglet(vaddr_t *va, paddr_t *pa, vsize_t sz, paddr_t align) 469 { 470 struct kmem_pa_mode kp_piglet = { 471 .kp_constraint = &dma_constraint, 472 .kp_align = align, 473 .kp_maxseg = 1 474 }; 475 476 /* Ensure align is a power of 2 */ 477 KASSERT((align & (align - 1)) == 0); 478 479 /* 480 * Fixup arguments: align must be at least PAGE_SIZE, 481 * sz will be converted to pagecount, since that is what 482 * pmemrange uses internally. 483 */ 484 if (align < PAGE_SIZE) 485 kp_piglet.kp_align = PAGE_SIZE; 486 487 sz = round_page(sz); 488 489 *va = (vaddr_t)km_alloc(sz, &kv_any, &kp_piglet, &kd_nowait); 490 if (*va == 0) 491 return ENOMEM; 492 493 pmap_extract(pmap_kernel(), *va, pa); 494 return 0; 495 } 496 497 /* 498 * Free a piglet area. 499 */ 500 void 501 uvm_pmr_free_piglet(vaddr_t va, vsize_t sz) 502 { 503 /* 504 * Fix parameters. 505 */ 506 sz = round_page(sz); 507 508 /* 509 * Free the physical and virtual memory. 510 */ 511 km_free((void *)va, sz, &kv_any, &kp_dma_contig); 512 } 513 514 /* 515 * Physmem RLE compression support. 516 * 517 * Given a physical page address, return the number of pages starting at the 518 * address that are free. Clamps to the number of pages in 519 * HIBERNATE_CHUNK_SIZE. Returns 0 if the page at addr is not free. 520 */ 521 int 522 uvm_page_rle(paddr_t addr) 523 { 524 struct vm_page *pg, *pg_end; 525 struct vm_physseg *vmp; 526 int pseg_idx, off_idx; 527 528 pseg_idx = vm_physseg_find(atop(addr), &off_idx); 529 if (pseg_idx == -1) 530 return 0; 531 532 vmp = &vm_physmem[pseg_idx]; 533 pg = &vmp->pgs[off_idx]; 534 if (!(pg->pg_flags & PQ_FREE)) 535 return 0; 536 537 /* 538 * Search for the first non-free page after pg. 539 * Note that the page may not be the first page in a free pmemrange, 540 * therefore pg->fpgsz cannot be used. 541 */ 542 for (pg_end = pg; pg_end <= vmp->lastpg && 543 (pg_end->pg_flags & PQ_FREE) == PQ_FREE; pg_end++) 544 ; 545 return min((pg_end - pg), HIBERNATE_CHUNK_SIZE/PAGE_SIZE); 546 } 547 548 /* 549 * Fills out the hibernate_info union pointed to by hib 550 * with information about this machine (swap signature block 551 * offsets, number of memory ranges, kernel in use, etc) 552 */ 553 int 554 get_hibernate_info(union hibernate_info *hib, int suspend) 555 { 556 struct disklabel dl; 557 char err_string[128], *dl_ret; 558 559 #ifndef NO_PROPOLICE 560 /* Save propolice guard */ 561 hib->guard = __guard_local; 562 #endif /* ! NO_PROPOLICE */ 563 564 /* Determine I/O function to use */ 565 hib->io_func = get_hibernate_io_function(swdevt[0].sw_dev); 566 if (hib->io_func == NULL) 567 return (1); 568 569 /* Calculate hibernate device */ 570 hib->dev = swdevt[0].sw_dev; 571 572 /* Read disklabel (used to calculate signature and image offsets) */ 573 dl_ret = disk_readlabel(&dl, hib->dev, err_string, sizeof(err_string)); 574 575 if (dl_ret) { 576 printf("Hibernate error reading disklabel: %s\n", dl_ret); 577 return (1); 578 } 579 580 /* Make sure we have a swap partition. */ 581 if (dl.d_partitions[1].p_fstype != FS_SWAP || 582 DL_GETPSIZE(&dl.d_partitions[1]) == 0) 583 return (1); 584 585 /* Make sure the signature can fit in one block */ 586 if (sizeof(union hibernate_info) > DEV_BSIZE) 587 return (1); 588 589 /* Magic number */ 590 hib->magic = HIBERNATE_MAGIC; 591 592 /* Calculate signature block location */ 593 hib->sig_offset = DL_GETPSIZE(&dl.d_partitions[1]) - 594 sizeof(union hibernate_info)/DEV_BSIZE; 595 596 /* Stash kernel version information */ 597 memset(&hib->kernel_version, 0, 128); 598 bcopy(version, &hib->kernel_version, 599 min(strlen(version), sizeof(hib->kernel_version)-1)); 600 601 if (suspend) { 602 /* Grab the previously-allocated piglet addresses */ 603 hib->piglet_va = global_piglet_va; 604 hib->piglet_pa = global_piglet_pa; 605 hib->io_page = (void *)hib->piglet_va; 606 607 /* 608 * Initialization of the hibernate IO function for drivers 609 * that need to do prep work (such as allocating memory or 610 * setting up data structures that cannot safely be done 611 * during suspend without causing side effects). There is 612 * a matching HIB_DONE call performed after the write is 613 * completed. 614 */ 615 if (hib->io_func(hib->dev, DL_GETPOFFSET(&dl.d_partitions[1]), 616 (vaddr_t)NULL, DL_GETPSIZE(&dl.d_partitions[1]), 617 HIB_INIT, hib->io_page)) 618 goto fail; 619 620 } else { 621 /* 622 * Resuming kernels use a regular private page for the driver 623 * No need to free this I/O page as it will vanish as part of 624 * the resume. 625 */ 626 hib->io_page = malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT); 627 if (!hib->io_page) 628 goto fail; 629 } 630 631 if (get_hibernate_info_md(hib)) 632 goto fail; 633 634 return (0); 635 636 fail: 637 return (1); 638 } 639 640 /* 641 * Allocate nitems*size bytes from the hiballoc area presently in use 642 */ 643 void * 644 hibernate_zlib_alloc(void *unused, int nitems, int size) 645 { 646 struct hibernate_zlib_state *hibernate_state; 647 648 hibernate_state = 649 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 650 651 return hib_alloc(&hibernate_state->hiballoc_arena, nitems*size); 652 } 653 654 /* 655 * Free the memory pointed to by addr in the hiballoc area presently in 656 * use 657 */ 658 void 659 hibernate_zlib_free(void *unused, void *addr) 660 { 661 struct hibernate_zlib_state *hibernate_state; 662 663 hibernate_state = 664 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 665 666 hib_free(&hibernate_state->hiballoc_arena, addr); 667 } 668 669 /* 670 * Inflate next page of data from the image stream. 671 * The rle parameter is modified on exit to contain the number of pages to 672 * skip in the output stream (or 0 if this page was inflated into). 673 * 674 * Returns 0 if the stream contains additional data, or 1 if the stream is 675 * finished. 676 */ 677 int 678 hibernate_inflate_page(int *rle) 679 { 680 struct hibernate_zlib_state *hibernate_state; 681 int i; 682 683 hibernate_state = 684 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 685 686 /* Set up the stream for RLE code inflate */ 687 hibernate_state->hib_stream.next_out = (unsigned char *)rle; 688 hibernate_state->hib_stream.avail_out = sizeof(*rle); 689 690 /* Inflate RLE code */ 691 i = inflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH); 692 if (i != Z_OK && i != Z_STREAM_END) { 693 /* 694 * XXX - this will likely reboot/hang most machines 695 * since the console output buffer will be unmapped, 696 * but there's not much else we can do here. 697 */ 698 panic("rle inflate stream error"); 699 } 700 701 if (hibernate_state->hib_stream.avail_out != 0) { 702 /* 703 * XXX - this will likely reboot/hang most machines 704 * since the console output buffer will be unmapped, 705 * but there's not much else we can do here. 706 */ 707 panic("rle short inflate error"); 708 } 709 710 if (*rle < 0 || *rle > 1024) { 711 /* 712 * XXX - this will likely reboot/hang most machines 713 * since the console output buffer will be unmapped, 714 * but there's not much else we can do here. 715 */ 716 panic("invalid rle count"); 717 } 718 719 if (i == Z_STREAM_END) 720 return (1); 721 722 if (*rle != 0) 723 return (0); 724 725 /* Set up the stream for page inflate */ 726 hibernate_state->hib_stream.next_out = 727 (unsigned char *)HIBERNATE_INFLATE_PAGE; 728 hibernate_state->hib_stream.avail_out = PAGE_SIZE; 729 730 /* Process next block of data */ 731 i = inflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH); 732 if (i != Z_OK && i != Z_STREAM_END) { 733 /* 734 * XXX - this will likely reboot/hang most machines 735 * since the console output buffer will be unmapped, 736 * but there's not much else we can do here. 737 */ 738 panic("inflate error"); 739 } 740 741 /* We should always have extracted a full page ... */ 742 if (hibernate_state->hib_stream.avail_out != 0) { 743 /* 744 * XXX - this will likely reboot/hang most machines 745 * since the console output buffer will be unmapped, 746 * but there's not much else we can do here. 747 */ 748 panic("incomplete page"); 749 } 750 751 return (i == Z_STREAM_END); 752 } 753 754 /* 755 * Inflate size bytes from src into dest, skipping any pages in 756 * [src..dest] that are special (see hibernate_inflate_skip) 757 * 758 * This function executes while using the resume-time stack 759 * and pmap, and therefore cannot use ddb/printf/etc. Doing so 760 * will likely hang or reset the machine since the console output buffer 761 * will be unmapped. 762 */ 763 void 764 hibernate_inflate_region(union hibernate_info *hib, paddr_t dest, 765 paddr_t src, size_t size) 766 { 767 int end_stream = 0, rle; 768 struct hibernate_zlib_state *hibernate_state; 769 770 hibernate_state = 771 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 772 773 hibernate_state->hib_stream.next_in = (unsigned char *)src; 774 hibernate_state->hib_stream.avail_in = size; 775 776 do { 777 /* 778 * Is this a special page? If yes, redirect the 779 * inflate output to a scratch page (eg, discard it) 780 */ 781 if (hibernate_inflate_skip(hib, dest)) { 782 hibernate_enter_resume_mapping( 783 HIBERNATE_INFLATE_PAGE, 784 HIBERNATE_INFLATE_PAGE, 0); 785 } else { 786 hibernate_enter_resume_mapping( 787 HIBERNATE_INFLATE_PAGE, dest, 0); 788 } 789 790 hibernate_flush(); 791 end_stream = hibernate_inflate_page(&rle); 792 793 if (rle == 0) 794 dest += PAGE_SIZE; 795 else 796 dest += (rle * PAGE_SIZE); 797 } while (!end_stream); 798 } 799 800 /* 801 * deflate from src into the I/O page, up to 'remaining' bytes 802 * 803 * Returns number of input bytes consumed, and may reset 804 * the 'remaining' parameter if not all the output space was consumed 805 * (this information is needed to know how much to write to disk 806 */ 807 size_t 808 hibernate_deflate(union hibernate_info *hib, paddr_t src, 809 size_t *remaining) 810 { 811 vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE; 812 struct hibernate_zlib_state *hibernate_state; 813 814 hibernate_state = 815 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 816 817 /* Set up the stream for deflate */ 818 hibernate_state->hib_stream.next_in = (unsigned char *)src; 819 hibernate_state->hib_stream.avail_in = PAGE_SIZE - (src & PAGE_MASK); 820 hibernate_state->hib_stream.next_out = 821 (unsigned char *)hibernate_io_page + (PAGE_SIZE - *remaining); 822 hibernate_state->hib_stream.avail_out = *remaining; 823 824 /* Process next block of data */ 825 if (deflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH) != Z_OK) 826 panic("hibernate zlib deflate error"); 827 828 /* Update pointers and return number of bytes consumed */ 829 *remaining = hibernate_state->hib_stream.avail_out; 830 return (PAGE_SIZE - (src & PAGE_MASK)) - 831 hibernate_state->hib_stream.avail_in; 832 } 833 834 /* 835 * Write the hibernation information specified in hiber_info 836 * to the location in swap previously calculated (last block of 837 * swap), called the "signature block". 838 */ 839 int 840 hibernate_write_signature(union hibernate_info *hib) 841 { 842 /* Write hibernate info to disk */ 843 return (hib->io_func(hib->dev, hib->sig_offset, 844 (vaddr_t)hib, DEV_BSIZE, HIB_W, 845 hib->io_page)); 846 } 847 848 /* 849 * Write the memory chunk table to the area in swap immediately 850 * preceding the signature block. The chunk table is stored 851 * in the piglet when this function is called. Returns errno. 852 */ 853 int 854 hibernate_write_chunktable(union hibernate_info *hib) 855 { 856 vaddr_t hibernate_chunk_table_start; 857 size_t hibernate_chunk_table_size; 858 int i, err; 859 860 hibernate_chunk_table_size = HIBERNATE_CHUNK_TABLE_SIZE; 861 862 hibernate_chunk_table_start = hib->piglet_va + 863 HIBERNATE_CHUNK_SIZE; 864 865 /* Write chunk table */ 866 for (i = 0; i < hibernate_chunk_table_size; i += MAXPHYS) { 867 if ((err = hib->io_func(hib->dev, 868 hib->chunktable_offset + (i/DEV_BSIZE), 869 (vaddr_t)(hibernate_chunk_table_start + i), 870 MAXPHYS, HIB_W, hib->io_page))) { 871 DPRINTF("chunktable write error: %d\n", err); 872 return (err); 873 } 874 } 875 876 return (0); 877 } 878 879 /* 880 * Write an empty hiber_info to the swap signature block, which is 881 * guaranteed to not match any valid hib. 882 */ 883 int 884 hibernate_clear_signature(void) 885 { 886 union hibernate_info blank_hiber_info; 887 union hibernate_info hib; 888 889 /* Zero out a blank hiber_info */ 890 memset(&blank_hiber_info, 0, sizeof(union hibernate_info)); 891 892 /* Get the signature block location */ 893 if (get_hibernate_info(&hib, 0)) 894 return (1); 895 896 /* Write (zeroed) hibernate info to disk */ 897 DPRINTF("clearing hibernate signature block location: %lld\n", 898 hib.sig_offset); 899 if (hibernate_block_io(&hib, 900 hib.sig_offset, 901 DEV_BSIZE, (vaddr_t)&blank_hiber_info, 1)) 902 printf("Warning: could not clear hibernate signature\n"); 903 904 return (0); 905 } 906 907 /* 908 * Compare two hibernate_infos to determine if they are the same (eg, 909 * we should be performing a hibernate resume on this machine. 910 * Not all fields are checked - just enough to verify that the machine 911 * has the same memory configuration and kernel as the one that 912 * wrote the signature previously. 913 */ 914 int 915 hibernate_compare_signature(union hibernate_info *mine, 916 union hibernate_info *disk) 917 { 918 u_int i; 919 920 if (mine->nranges != disk->nranges) { 921 DPRINTF("hibernate memory range count mismatch\n"); 922 return (1); 923 } 924 925 if (strcmp(mine->kernel_version, disk->kernel_version) != 0) { 926 DPRINTF("hibernate kernel version mismatch\n"); 927 return (1); 928 } 929 930 for (i = 0; i < mine->nranges; i++) { 931 if ((mine->ranges[i].base != disk->ranges[i].base) || 932 (mine->ranges[i].end != disk->ranges[i].end) ) { 933 DPRINTF("hib range %d mismatch [%p-%p != %p-%p]\n", 934 i, 935 (void *)mine->ranges[i].base, 936 (void *)mine->ranges[i].end, 937 (void *)disk->ranges[i].base, 938 (void *)disk->ranges[i].end); 939 return (1); 940 } 941 } 942 943 return (0); 944 } 945 946 /* 947 * Transfers xfer_size bytes between the hibernate device specified in 948 * hib_info at offset blkctr and the vaddr specified at dest. 949 * 950 * Separate offsets and pages are used to handle misaligned reads (reads 951 * that span a page boundary). 952 * 953 * blkctr specifies a relative offset (relative to the start of swap), 954 * not an absolute disk offset 955 * 956 */ 957 int 958 hibernate_block_io(union hibernate_info *hib, daddr_t blkctr, 959 size_t xfer_size, vaddr_t dest, int iswrite) 960 { 961 struct buf *bp; 962 struct bdevsw *bdsw; 963 int error; 964 965 bp = geteblk(xfer_size); 966 bdsw = &bdevsw[major(hib->dev)]; 967 968 error = (*bdsw->d_open)(hib->dev, FREAD, S_IFCHR, curproc); 969 if (error) { 970 printf("hibernate_block_io open failed\n"); 971 return (1); 972 } 973 974 if (iswrite) 975 bcopy((caddr_t)dest, bp->b_data, xfer_size); 976 977 bp->b_bcount = xfer_size; 978 bp->b_blkno = blkctr; 979 CLR(bp->b_flags, B_READ | B_WRITE | B_DONE); 980 SET(bp->b_flags, B_BUSY | (iswrite ? B_WRITE : B_READ) | B_RAW); 981 bp->b_dev = hib->dev; 982 (*bdsw->d_strategy)(bp); 983 984 error = biowait(bp); 985 if (error) { 986 printf("hib block_io biowait error %d blk %lld size %zu\n", 987 error, (long long)blkctr, xfer_size); 988 error = (*bdsw->d_close)(hib->dev, 0, S_IFCHR, 989 curproc); 990 if (error) 991 printf("hibernate_block_io error close failed\n"); 992 return (1); 993 } 994 995 error = (*bdsw->d_close)(hib->dev, FREAD, S_IFCHR, curproc); 996 if (error) { 997 printf("hibernate_block_io close failed\n"); 998 return (1); 999 } 1000 1001 if (!iswrite) 1002 bcopy(bp->b_data, (caddr_t)dest, xfer_size); 1003 1004 bp->b_flags |= B_INVAL; 1005 brelse(bp); 1006 1007 return (0); 1008 } 1009 1010 /* 1011 * Preserve one page worth of random data, generated from the resuming 1012 * kernel's arc4random. After resume, this preserved entropy can be used 1013 * to further improve the un-hibernated machine's entropy pool. This 1014 * random data is stored in the piglet, which is preserved across the 1015 * unpack operation, and is restored later in the resume process (see 1016 * hib_getentropy) 1017 */ 1018 void 1019 hibernate_preserve_entropy(union hibernate_info *hib) 1020 { 1021 void *entropy; 1022 1023 entropy = km_alloc(PAGE_SIZE, &kv_any, &kp_none, &kd_nowait); 1024 1025 if (!entropy) 1026 return; 1027 1028 pmap_activate(curproc); 1029 pmap_kenter_pa((vaddr_t)entropy, 1030 (paddr_t)(hib->piglet_pa + (29 * PAGE_SIZE)), 1031 PROT_READ | PROT_WRITE); 1032 1033 arc4random_buf((void *)entropy, PAGE_SIZE); 1034 pmap_kremove((vaddr_t)entropy, PAGE_SIZE); 1035 km_free(entropy, PAGE_SIZE, &kv_any, &kp_none); 1036 } 1037 1038 #ifndef NO_PROPOLICE 1039 vaddr_t 1040 hibernate_unprotect_ssp(void) 1041 { 1042 struct kmem_dyn_mode kd_avoidalias; 1043 vaddr_t va = trunc_page((vaddr_t)&__guard_local); 1044 paddr_t pa; 1045 1046 pmap_extract(pmap_kernel(), va, &pa); 1047 1048 memset(&kd_avoidalias, 0, sizeof kd_avoidalias); 1049 kd_avoidalias.kd_prefer = pa; 1050 kd_avoidalias.kd_waitok = 1; 1051 va = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any, &kp_none, &kd_avoidalias); 1052 if (!va) 1053 panic("hibernate_unprotect_ssp"); 1054 1055 pmap_kenter_pa(va, pa, PROT_READ | PROT_WRITE); 1056 pmap_update(pmap_kernel()); 1057 1058 return va; 1059 } 1060 1061 void 1062 hibernate_reprotect_ssp(vaddr_t va) 1063 { 1064 pmap_kremove(va, PAGE_SIZE); 1065 km_free((void *)va, PAGE_SIZE, &kv_any, &kp_none); 1066 } 1067 #endif /* NO_PROPOLICE */ 1068 1069 /* 1070 * Reads the signature block from swap, checks against the current machine's 1071 * information. If the information matches, perform a resume by reading the 1072 * saved image into the pig area, and unpacking. 1073 * 1074 * Must be called with interrupts enabled. 1075 */ 1076 void 1077 hibernate_resume(void) 1078 { 1079 union hibernate_info hib; 1080 int s; 1081 #ifndef NO_PROPOLICE 1082 vsize_t off = (vaddr_t)&__guard_local - 1083 trunc_page((vaddr_t)&__guard_local); 1084 vaddr_t guard_va; 1085 #endif 1086 1087 /* Get current running machine's hibernate info */ 1088 memset(&hib, 0, sizeof(hib)); 1089 if (get_hibernate_info(&hib, 0)) { 1090 DPRINTF("couldn't retrieve machine's hibernate info\n"); 1091 return; 1092 } 1093 1094 /* Read hibernate info from disk */ 1095 s = splbio(); 1096 1097 DPRINTF("reading hibernate signature block location: %lld\n", 1098 hib.sig_offset); 1099 1100 if (hibernate_block_io(&hib, 1101 hib.sig_offset, 1102 DEV_BSIZE, (vaddr_t)&disk_hib, 0)) { 1103 DPRINTF("error in hibernate read"); 1104 splx(s); 1105 return; 1106 } 1107 1108 /* Check magic number */ 1109 if (disk_hib.magic != HIBERNATE_MAGIC) { 1110 DPRINTF("wrong magic number in hibernate signature: %x\n", 1111 disk_hib.magic); 1112 splx(s); 1113 return; 1114 } 1115 1116 /* 1117 * We (possibly) found a hibernate signature. Clear signature first, 1118 * to prevent accidental resume or endless resume cycles later. 1119 */ 1120 if (hibernate_clear_signature()) { 1121 DPRINTF("error clearing hibernate signature block\n"); 1122 splx(s); 1123 return; 1124 } 1125 1126 /* 1127 * If on-disk and in-memory hibernate signatures match, 1128 * this means we should do a resume from hibernate. 1129 */ 1130 if (hibernate_compare_signature(&hib, &disk_hib)) { 1131 DPRINTF("mismatched hibernate signature block\n"); 1132 splx(s); 1133 return; 1134 } 1135 1136 #ifdef MULTIPROCESSOR 1137 /* XXX - if we fail later, we may need to rehatch APs on some archs */ 1138 DPRINTF("hibernate: quiescing APs\n"); 1139 hibernate_quiesce_cpus(); 1140 #endif /* MULTIPROCESSOR */ 1141 1142 /* Read the image from disk into the image (pig) area */ 1143 if (hibernate_read_image(&disk_hib)) 1144 goto fail; 1145 1146 DPRINTF("hibernate: quiescing devices\n"); 1147 if (config_suspend_all(DVACT_QUIESCE) != 0) 1148 goto fail; 1149 1150 #ifndef NO_PROPOLICE 1151 guard_va = hibernate_unprotect_ssp(); 1152 #endif /* NO_PROPOLICE */ 1153 1154 (void) splhigh(); 1155 hibernate_disable_intr_machdep(); 1156 cold = 1; 1157 1158 DPRINTF("hibernate: suspending devices\n"); 1159 if (config_suspend_all(DVACT_SUSPEND) != 0) { 1160 cold = 0; 1161 hibernate_enable_intr_machdep(); 1162 #ifndef NO_PROPOLICE 1163 hibernate_reprotect_ssp(guard_va); 1164 #endif /* ! NO_PROPOLICE */ 1165 goto fail; 1166 } 1167 1168 hibernate_preserve_entropy(&disk_hib); 1169 1170 printf("Unpacking image...\n"); 1171 1172 /* Switch stacks */ 1173 DPRINTF("hibernate: switching stacks\n"); 1174 hibernate_switch_stack_machdep(); 1175 1176 #ifndef NO_PROPOLICE 1177 /* Start using suspended kernel's propolice guard */ 1178 *(long *)(guard_va + off) = disk_hib.guard; 1179 hibernate_reprotect_ssp(guard_va); 1180 #endif /* ! NO_PROPOLICE */ 1181 1182 /* Unpack and resume */ 1183 hibernate_unpack_image(&disk_hib); 1184 1185 fail: 1186 splx(s); 1187 printf("\nUnable to resume hibernated image\n"); 1188 } 1189 1190 /* 1191 * Unpack image from pig area to original location by looping through the 1192 * list of output chunks in the order they should be restored (fchunks). 1193 * 1194 * Note that due to the stack smash protector and the fact that we have 1195 * switched stacks, it is not permitted to return from this function. 1196 */ 1197 void 1198 hibernate_unpack_image(union hibernate_info *hib) 1199 { 1200 struct hibernate_disk_chunk *chunks; 1201 union hibernate_info local_hib; 1202 paddr_t image_cur = global_pig_start; 1203 short i, *fchunks; 1204 char *pva; 1205 1206 /* Piglet will be identity mapped (VA == PA) */ 1207 pva = (char *)hib->piglet_pa; 1208 1209 fchunks = (short *)(pva + (4 * PAGE_SIZE)); 1210 1211 chunks = (struct hibernate_disk_chunk *)(pva + HIBERNATE_CHUNK_SIZE); 1212 1213 /* Can't use hiber_info that's passed in after this point */ 1214 bcopy(hib, &local_hib, sizeof(union hibernate_info)); 1215 1216 /* VA == PA */ 1217 local_hib.piglet_va = local_hib.piglet_pa; 1218 1219 /* 1220 * Point of no return. Once we pass this point, only kernel code can 1221 * be accessed. No global variables or other kernel data structures 1222 * are guaranteed to be coherent after unpack starts. 1223 * 1224 * The image is now in high memory (pig area), we unpack from the pig 1225 * to the correct location in memory. We'll eventually end up copying 1226 * on top of ourself, but we are assured the kernel code here is the 1227 * same between the hibernated and resuming kernel, and we are running 1228 * on our own stack, so the overwrite is ok. 1229 */ 1230 DPRINTF("hibernate: activating alt. pagetable and starting unpack\n"); 1231 hibernate_activate_resume_pt_machdep(); 1232 1233 for (i = 0; i < local_hib.chunk_ctr; i++) { 1234 /* Reset zlib for inflate */ 1235 if (hibernate_zlib_reset(&local_hib, 0) != Z_OK) 1236 panic("hibernate failed to reset zlib for inflate"); 1237 1238 hibernate_process_chunk(&local_hib, &chunks[fchunks[i]], 1239 image_cur); 1240 1241 image_cur += chunks[fchunks[i]].compressed_size; 1242 1243 } 1244 1245 /* 1246 * Resume the loaded kernel by jumping to the MD resume vector. 1247 * We won't be returning from this call. 1248 */ 1249 hibernate_resume_machdep(); 1250 } 1251 1252 /* 1253 * Bounce a compressed image chunk to the piglet, entering mappings for the 1254 * copied pages as needed 1255 */ 1256 void 1257 hibernate_copy_chunk_to_piglet(paddr_t img_cur, vaddr_t piglet, size_t size) 1258 { 1259 size_t ct, ofs; 1260 paddr_t src = img_cur; 1261 vaddr_t dest = piglet; 1262 1263 /* Copy first partial page */ 1264 ct = (PAGE_SIZE) - (src & PAGE_MASK); 1265 ofs = (src & PAGE_MASK); 1266 1267 if (ct < PAGE_SIZE) { 1268 hibernate_enter_resume_mapping(HIBERNATE_INFLATE_PAGE, 1269 (src - ofs), 0); 1270 hibernate_flush(); 1271 bcopy((caddr_t)(HIBERNATE_INFLATE_PAGE + ofs), (caddr_t)dest, ct); 1272 src += ct; 1273 dest += ct; 1274 } 1275 1276 /* Copy remaining pages */ 1277 while (src < size + img_cur) { 1278 hibernate_enter_resume_mapping(HIBERNATE_INFLATE_PAGE, src, 0); 1279 hibernate_flush(); 1280 ct = PAGE_SIZE; 1281 bcopy((caddr_t)(HIBERNATE_INFLATE_PAGE), (caddr_t)dest, ct); 1282 hibernate_flush(); 1283 src += ct; 1284 dest += ct; 1285 } 1286 } 1287 1288 /* 1289 * Process a chunk by bouncing it to the piglet, followed by unpacking 1290 */ 1291 void 1292 hibernate_process_chunk(union hibernate_info *hib, 1293 struct hibernate_disk_chunk *chunk, paddr_t img_cur) 1294 { 1295 char *pva = (char *)hib->piglet_va; 1296 1297 hibernate_copy_chunk_to_piglet(img_cur, 1298 (vaddr_t)(pva + (HIBERNATE_CHUNK_SIZE * 2)), chunk->compressed_size); 1299 hibernate_inflate_region(hib, chunk->base, 1300 (vaddr_t)(pva + (HIBERNATE_CHUNK_SIZE * 2)), 1301 chunk->compressed_size); 1302 } 1303 1304 /* 1305 * Calculate RLE component for 'inaddr'. Clamps to max RLE pages between 1306 * inaddr and range_end. 1307 */ 1308 int 1309 hibernate_calc_rle(paddr_t inaddr, paddr_t range_end) 1310 { 1311 int rle; 1312 1313 rle = uvm_page_rle(inaddr); 1314 KASSERT(rle >= 0 && rle <= MAX_RLE); 1315 1316 /* Clamp RLE to range end */ 1317 if (rle > 0 && inaddr + (rle * PAGE_SIZE) > range_end) 1318 rle = (range_end - inaddr) / PAGE_SIZE; 1319 1320 return (rle); 1321 } 1322 1323 /* 1324 * Write the RLE byte for page at 'inaddr' to the output stream. 1325 * Returns the number of pages to be skipped at 'inaddr'. 1326 */ 1327 int 1328 hibernate_write_rle(union hibernate_info *hib, paddr_t inaddr, 1329 paddr_t range_end, daddr_t *blkctr, 1330 size_t *out_remaining) 1331 { 1332 int rle, err, *rleloc; 1333 struct hibernate_zlib_state *hibernate_state; 1334 vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE; 1335 1336 hibernate_state = 1337 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 1338 1339 rle = hibernate_calc_rle(inaddr, range_end); 1340 1341 rleloc = (int *)hibernate_rle_page + MAX_RLE - 1; 1342 *rleloc = rle; 1343 1344 /* Deflate the RLE byte into the stream */ 1345 hibernate_deflate(hib, (paddr_t)rleloc, out_remaining); 1346 1347 /* Did we fill the output page? If so, flush to disk */ 1348 if (*out_remaining == 0) { 1349 if ((err = hib->io_func(hib->dev, *blkctr + hib->image_offset, 1350 (vaddr_t)hibernate_io_page, PAGE_SIZE, HIB_W, 1351 hib->io_page))) { 1352 DPRINTF("hib write error %d\n", err); 1353 return (err); 1354 } 1355 1356 *blkctr += PAGE_SIZE / DEV_BSIZE; 1357 *out_remaining = PAGE_SIZE; 1358 1359 /* If we didn't deflate the entire RLE byte, finish it now */ 1360 if (hibernate_state->hib_stream.avail_in != 0) 1361 hibernate_deflate(hib, 1362 (vaddr_t)hibernate_state->hib_stream.next_in, 1363 out_remaining); 1364 } 1365 1366 return (rle); 1367 } 1368 1369 /* 1370 * Write a compressed version of this machine's memory to disk, at the 1371 * precalculated swap offset: 1372 * 1373 * end of swap - signature block size - chunk table size - memory size 1374 * 1375 * The function begins by looping through each phys mem range, cutting each 1376 * one into MD sized chunks. These chunks are then compressed individually 1377 * and written out to disk, in phys mem order. Some chunks might compress 1378 * more than others, and for this reason, each chunk's size is recorded 1379 * in the chunk table, which is written to disk after the image has 1380 * properly been compressed and written (in hibernate_write_chunktable). 1381 * 1382 * When this function is called, the machine is nearly suspended - most 1383 * devices are quiesced/suspended, interrupts are off, and cold has 1384 * been set. This means that there can be no side effects once the 1385 * write has started, and the write function itself can also have no 1386 * side effects. This also means no printfs are permitted (since printf 1387 * has side effects.) 1388 * 1389 * Return values : 1390 * 1391 * 0 - success 1392 * EIO - I/O error occurred writing the chunks 1393 * EINVAL - Failed to write a complete range 1394 * ENOMEM - Memory allocation failure during preparation of the zlib arena 1395 */ 1396 int 1397 hibernate_write_chunks(union hibernate_info *hib) 1398 { 1399 paddr_t range_base, range_end, inaddr, temp_inaddr; 1400 size_t nblocks, out_remaining, used; 1401 struct hibernate_disk_chunk *chunks; 1402 vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE; 1403 daddr_t blkctr = 0; 1404 int i, rle, err; 1405 struct hibernate_zlib_state *hibernate_state; 1406 1407 hibernate_state = 1408 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 1409 1410 hib->chunk_ctr = 0; 1411 1412 /* 1413 * Map the utility VAs to the piglet. See the piglet map at the 1414 * top of this file for piglet layout information. 1415 */ 1416 hibernate_copy_page = hib->piglet_va + 3 * PAGE_SIZE; 1417 hibernate_rle_page = hib->piglet_va + 28 * PAGE_SIZE; 1418 1419 chunks = (struct hibernate_disk_chunk *)(hib->piglet_va + 1420 HIBERNATE_CHUNK_SIZE); 1421 1422 /* Calculate the chunk regions */ 1423 for (i = 0; i < hib->nranges; i++) { 1424 range_base = hib->ranges[i].base; 1425 range_end = hib->ranges[i].end; 1426 1427 inaddr = range_base; 1428 1429 while (inaddr < range_end) { 1430 chunks[hib->chunk_ctr].base = inaddr; 1431 if (inaddr + HIBERNATE_CHUNK_SIZE < range_end) 1432 chunks[hib->chunk_ctr].end = inaddr + 1433 HIBERNATE_CHUNK_SIZE; 1434 else 1435 chunks[hib->chunk_ctr].end = range_end; 1436 1437 inaddr += HIBERNATE_CHUNK_SIZE; 1438 hib->chunk_ctr ++; 1439 } 1440 } 1441 1442 uvm_pmr_dirty_everything(); 1443 uvm_pmr_zero_everything(); 1444 1445 /* Compress and write the chunks in the chunktable */ 1446 for (i = 0; i < hib->chunk_ctr; i++) { 1447 range_base = chunks[i].base; 1448 range_end = chunks[i].end; 1449 1450 chunks[i].offset = blkctr + hib->image_offset; 1451 1452 /* Reset zlib for deflate */ 1453 if (hibernate_zlib_reset(hib, 1) != Z_OK) { 1454 DPRINTF("hibernate_zlib_reset failed for deflate\n"); 1455 return (ENOMEM); 1456 } 1457 1458 inaddr = range_base; 1459 1460 /* 1461 * For each range, loop through its phys mem region 1462 * and write out the chunks (the last chunk might be 1463 * smaller than the chunk size). 1464 */ 1465 while (inaddr < range_end) { 1466 out_remaining = PAGE_SIZE; 1467 while (out_remaining > 0 && inaddr < range_end) { 1468 /* 1469 * Adjust for regions that are not evenly 1470 * divisible by PAGE_SIZE or overflowed 1471 * pages from the previous iteration. 1472 */ 1473 temp_inaddr = (inaddr & PAGE_MASK) + 1474 hibernate_copy_page; 1475 1476 /* Deflate from temp_inaddr to IO page */ 1477 if (inaddr != range_end) { 1478 if (inaddr % PAGE_SIZE == 0) { 1479 rle = hibernate_write_rle(hib, 1480 inaddr, 1481 range_end, 1482 &blkctr, 1483 &out_remaining); 1484 } 1485 1486 if (rle == 0) { 1487 pmap_kenter_pa(hibernate_temp_page, 1488 inaddr & PMAP_PA_MASK, 1489 PROT_READ); 1490 1491 bcopy((caddr_t)hibernate_temp_page, 1492 (caddr_t)hibernate_copy_page, 1493 PAGE_SIZE); 1494 inaddr += hibernate_deflate(hib, 1495 temp_inaddr, 1496 &out_remaining); 1497 } else { 1498 inaddr += rle * PAGE_SIZE; 1499 if (inaddr > range_end) 1500 inaddr = range_end; 1501 } 1502 1503 } 1504 1505 if (out_remaining == 0) { 1506 /* Filled up the page */ 1507 nblocks = PAGE_SIZE / DEV_BSIZE; 1508 1509 if ((err = hib->io_func(hib->dev, 1510 blkctr + hib->image_offset, 1511 (vaddr_t)hibernate_io_page, 1512 PAGE_SIZE, HIB_W, hib->io_page))) { 1513 DPRINTF("hib write error %d\n", 1514 err); 1515 return (err); 1516 } 1517 1518 blkctr += nblocks; 1519 } 1520 } 1521 } 1522 1523 if (inaddr != range_end) { 1524 DPRINTF("deflate range ended prematurely\n"); 1525 return (EINVAL); 1526 } 1527 1528 /* 1529 * End of range. Round up to next secsize bytes 1530 * after finishing compress 1531 */ 1532 if (out_remaining == 0) 1533 out_remaining = PAGE_SIZE; 1534 1535 /* Finish compress */ 1536 hibernate_state->hib_stream.next_in = (unsigned char *)inaddr; 1537 hibernate_state->hib_stream.avail_in = 0; 1538 hibernate_state->hib_stream.next_out = 1539 (unsigned char *)hibernate_io_page + 1540 (PAGE_SIZE - out_remaining); 1541 1542 /* We have an extra output page available for finalize */ 1543 hibernate_state->hib_stream.avail_out = 1544 out_remaining + PAGE_SIZE; 1545 1546 if ((err = deflate(&hibernate_state->hib_stream, Z_FINISH)) != 1547 Z_STREAM_END) { 1548 DPRINTF("deflate error in output stream: %d\n", err); 1549 return (err); 1550 } 1551 1552 out_remaining = hibernate_state->hib_stream.avail_out; 1553 1554 used = 2 * PAGE_SIZE - out_remaining; 1555 nblocks = used / DEV_BSIZE; 1556 1557 /* Round up to next block if needed */ 1558 if (used % DEV_BSIZE != 0) 1559 nblocks ++; 1560 1561 /* Write final block(s) for this chunk */ 1562 if ((err = hib->io_func(hib->dev, blkctr + hib->image_offset, 1563 (vaddr_t)hibernate_io_page, nblocks*DEV_BSIZE, 1564 HIB_W, hib->io_page))) { 1565 DPRINTF("hib final write error %d\n", err); 1566 return (err); 1567 } 1568 1569 blkctr += nblocks; 1570 1571 chunks[i].compressed_size = (blkctr + hib->image_offset - 1572 chunks[i].offset) * DEV_BSIZE; 1573 } 1574 1575 hib->chunktable_offset = hib->image_offset + blkctr; 1576 return (0); 1577 } 1578 1579 /* 1580 * Reset the zlib stream state and allocate a new hiballoc area for either 1581 * inflate or deflate. This function is called once for each hibernate chunk. 1582 * Calling hiballoc_init multiple times is acceptable since the memory it is 1583 * provided is unmanaged memory (stolen). We use the memory provided to us 1584 * by the piglet allocated via the supplied hib. 1585 */ 1586 int 1587 hibernate_zlib_reset(union hibernate_info *hib, int deflate) 1588 { 1589 vaddr_t hibernate_zlib_start; 1590 size_t hibernate_zlib_size; 1591 char *pva = (char *)hib->piglet_va; 1592 struct hibernate_zlib_state *hibernate_state; 1593 1594 hibernate_state = 1595 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 1596 1597 if (!deflate) 1598 pva = (char *)((paddr_t)pva & (PIGLET_PAGE_MASK)); 1599 1600 /* 1601 * See piglet layout information at the start of this file for 1602 * information on the zlib page assignments. 1603 */ 1604 hibernate_zlib_start = (vaddr_t)(pva + (30 * PAGE_SIZE)); 1605 hibernate_zlib_size = 80 * PAGE_SIZE; 1606 1607 memset((void *)hibernate_zlib_start, 0, hibernate_zlib_size); 1608 memset(hibernate_state, 0, PAGE_SIZE); 1609 1610 /* Set up stream structure */ 1611 hibernate_state->hib_stream.zalloc = (alloc_func)hibernate_zlib_alloc; 1612 hibernate_state->hib_stream.zfree = (free_func)hibernate_zlib_free; 1613 1614 /* Initialize the hiballoc arena for zlib allocs/frees */ 1615 hiballoc_init(&hibernate_state->hiballoc_arena, 1616 (caddr_t)hibernate_zlib_start, hibernate_zlib_size); 1617 1618 if (deflate) { 1619 return deflateInit(&hibernate_state->hib_stream, 1620 Z_BEST_SPEED); 1621 } else 1622 return inflateInit(&hibernate_state->hib_stream); 1623 } 1624 1625 /* 1626 * Reads the hibernated memory image from disk, whose location and 1627 * size are recorded in hib. Begin by reading the persisted 1628 * chunk table, which records the original chunk placement location 1629 * and compressed size for each. Next, allocate a pig region of 1630 * sufficient size to hold the compressed image. Next, read the 1631 * chunks into the pig area (calling hibernate_read_chunks to do this), 1632 * and finally, if all of the above succeeds, clear the hibernate signature. 1633 * The function will then return to hibernate_resume, which will proceed 1634 * to unpack the pig image to the correct place in memory. 1635 */ 1636 int 1637 hibernate_read_image(union hibernate_info *hib) 1638 { 1639 size_t compressed_size, disk_size, chunktable_size, pig_sz; 1640 paddr_t image_start, image_end, pig_start, pig_end; 1641 struct hibernate_disk_chunk *chunks; 1642 daddr_t blkctr; 1643 vaddr_t chunktable = (vaddr_t)NULL; 1644 paddr_t piglet_chunktable = hib->piglet_pa + 1645 HIBERNATE_CHUNK_SIZE; 1646 int i, status; 1647 1648 status = 0; 1649 pmap_activate(curproc); 1650 1651 /* Calculate total chunk table size in disk blocks */ 1652 chunktable_size = HIBERNATE_CHUNK_TABLE_SIZE / DEV_BSIZE; 1653 1654 blkctr = hib->chunktable_offset; 1655 1656 chunktable = (vaddr_t)km_alloc(HIBERNATE_CHUNK_TABLE_SIZE, &kv_any, 1657 &kp_none, &kd_nowait); 1658 1659 if (!chunktable) 1660 return (1); 1661 1662 /* Map chunktable pages */ 1663 for (i = 0; i < HIBERNATE_CHUNK_TABLE_SIZE; i += PAGE_SIZE) 1664 pmap_kenter_pa(chunktable + i, piglet_chunktable + i, 1665 PROT_READ | PROT_WRITE); 1666 pmap_update(pmap_kernel()); 1667 1668 /* Read the chunktable from disk into the piglet chunktable */ 1669 for (i = 0; i < HIBERNATE_CHUNK_TABLE_SIZE; 1670 i += MAXPHYS, blkctr += MAXPHYS/DEV_BSIZE) 1671 hibernate_block_io(hib, blkctr, MAXPHYS, 1672 chunktable + i, 0); 1673 1674 blkctr = hib->image_offset; 1675 compressed_size = 0; 1676 1677 chunks = (struct hibernate_disk_chunk *)chunktable; 1678 1679 for (i = 0; i < hib->chunk_ctr; i++) 1680 compressed_size += chunks[i].compressed_size; 1681 1682 disk_size = compressed_size; 1683 1684 printf("unhibernating @ block %lld length %lu bytes\n", 1685 hib->sig_offset - chunktable_size, 1686 compressed_size); 1687 1688 /* Allocate the pig area */ 1689 pig_sz = compressed_size + HIBERNATE_CHUNK_SIZE; 1690 if (uvm_pmr_alloc_pig(&pig_start, pig_sz, hib->piglet_pa) == ENOMEM) { 1691 status = 1; 1692 goto unmap; 1693 } 1694 1695 pig_end = pig_start + pig_sz; 1696 1697 /* Calculate image extents. Pig image must end on a chunk boundary. */ 1698 image_end = pig_end & ~(HIBERNATE_CHUNK_SIZE - 1); 1699 image_start = image_end - disk_size; 1700 1701 hibernate_read_chunks(hib, image_start, image_end, disk_size, 1702 chunks); 1703 1704 /* Prepare the resume time pmap/page table */ 1705 hibernate_populate_resume_pt(hib, image_start, image_end); 1706 1707 unmap: 1708 /* Unmap chunktable pages */ 1709 pmap_kremove(chunktable, HIBERNATE_CHUNK_TABLE_SIZE); 1710 pmap_update(pmap_kernel()); 1711 1712 return (status); 1713 } 1714 1715 /* 1716 * Read the hibernated memory chunks from disk (chunk information at this 1717 * point is stored in the piglet) into the pig area specified by 1718 * [pig_start .. pig_end]. Order the chunks so that the final chunk is the 1719 * only chunk with overlap possibilities. 1720 */ 1721 int 1722 hibernate_read_chunks(union hibernate_info *hib, paddr_t pig_start, 1723 paddr_t pig_end, size_t image_compr_size, 1724 struct hibernate_disk_chunk *chunks) 1725 { 1726 paddr_t img_cur, piglet_base; 1727 daddr_t blkctr; 1728 size_t processed, compressed_size, read_size; 1729 int nchunks, nfchunks, num_io_pages; 1730 vaddr_t tempva, hibernate_fchunk_area; 1731 short *fchunks, i, j; 1732 1733 tempva = (vaddr_t)NULL; 1734 hibernate_fchunk_area = (vaddr_t)NULL; 1735 nfchunks = 0; 1736 piglet_base = hib->piglet_pa; 1737 global_pig_start = pig_start; 1738 1739 /* 1740 * These mappings go into the resuming kernel's page table, and are 1741 * used only during image read. They dissappear from existence 1742 * when the suspended kernel is unpacked on top of us. 1743 */ 1744 tempva = (vaddr_t)km_alloc(MAXPHYS + PAGE_SIZE, &kv_any, &kp_none, 1745 &kd_nowait); 1746 if (!tempva) 1747 return (1); 1748 hibernate_fchunk_area = (vaddr_t)km_alloc(24 * PAGE_SIZE, &kv_any, 1749 &kp_none, &kd_nowait); 1750 if (!hibernate_fchunk_area) 1751 return (1); 1752 1753 /* Final output chunk ordering VA */ 1754 fchunks = (short *)hibernate_fchunk_area; 1755 1756 /* Map the chunk ordering region */ 1757 for(i = 0; i < 24 ; i++) 1758 pmap_kenter_pa(hibernate_fchunk_area + (i * PAGE_SIZE), 1759 piglet_base + ((4 + i) * PAGE_SIZE), 1760 PROT_READ | PROT_WRITE); 1761 pmap_update(pmap_kernel()); 1762 1763 nchunks = hib->chunk_ctr; 1764 1765 /* Initially start all chunks as unplaced */ 1766 for (i = 0; i < nchunks; i++) 1767 chunks[i].flags = 0; 1768 1769 /* 1770 * Search the list for chunks that are outside the pig area. These 1771 * can be placed first in the final output list. 1772 */ 1773 for (i = 0; i < nchunks; i++) { 1774 if (chunks[i].end <= pig_start || chunks[i].base >= pig_end) { 1775 fchunks[nfchunks] = i; 1776 nfchunks++; 1777 chunks[i].flags |= HIBERNATE_CHUNK_PLACED; 1778 } 1779 } 1780 1781 /* 1782 * Walk the ordering, place the chunks in ascending memory order. 1783 */ 1784 for (i = 0; i < nchunks; i++) { 1785 if (chunks[i].flags != HIBERNATE_CHUNK_PLACED) { 1786 fchunks[nfchunks] = i; 1787 nfchunks++; 1788 chunks[i].flags = HIBERNATE_CHUNK_PLACED; 1789 } 1790 } 1791 1792 img_cur = pig_start; 1793 1794 for (i = 0; i < nfchunks; i++) { 1795 blkctr = chunks[fchunks[i]].offset; 1796 processed = 0; 1797 compressed_size = chunks[fchunks[i]].compressed_size; 1798 1799 while (processed < compressed_size) { 1800 if (compressed_size - processed >= MAXPHYS) 1801 read_size = MAXPHYS; 1802 else 1803 read_size = compressed_size - processed; 1804 1805 /* 1806 * We're reading read_size bytes, offset from the 1807 * start of a page by img_cur % PAGE_SIZE, so the 1808 * end will be read_size + (img_cur % PAGE_SIZE) 1809 * from the start of the first page. Round that 1810 * up to the next page size. 1811 */ 1812 num_io_pages = (read_size + (img_cur % PAGE_SIZE) 1813 + PAGE_SIZE - 1) / PAGE_SIZE; 1814 1815 KASSERT(num_io_pages <= MAXPHYS/PAGE_SIZE + 1); 1816 1817 /* Map pages for this read */ 1818 for (j = 0; j < num_io_pages; j ++) 1819 pmap_kenter_pa(tempva + j * PAGE_SIZE, 1820 img_cur + j * PAGE_SIZE, 1821 PROT_READ | PROT_WRITE); 1822 1823 pmap_update(pmap_kernel()); 1824 1825 hibernate_block_io(hib, blkctr, read_size, 1826 tempva + (img_cur & PAGE_MASK), 0); 1827 1828 blkctr += (read_size / DEV_BSIZE); 1829 1830 pmap_kremove(tempva, num_io_pages * PAGE_SIZE); 1831 pmap_update(pmap_kernel()); 1832 1833 processed += read_size; 1834 img_cur += read_size; 1835 } 1836 } 1837 1838 pmap_kremove(hibernate_fchunk_area, 24 * PAGE_SIZE); 1839 pmap_update(pmap_kernel()); 1840 1841 return (0); 1842 } 1843 1844 /* 1845 * Hibernating a machine comprises the following operations: 1846 * 1. Calculating this machine's hibernate_info information 1847 * 2. Allocating a piglet and saving the piglet's physaddr 1848 * 3. Calculating the memory chunks 1849 * 4. Writing the compressed chunks to disk 1850 * 5. Writing the chunk table 1851 * 6. Writing the signature block (hibernate_info) 1852 * 1853 * On most architectures, the function calling hibernate_suspend would 1854 * then power off the machine using some MD-specific implementation. 1855 */ 1856 int 1857 hibernate_suspend(void) 1858 { 1859 union hibernate_info hib; 1860 u_long start, end; 1861 1862 /* 1863 * Calculate memory ranges, swap offsets, etc. 1864 * This also allocates a piglet whose physaddr is stored in 1865 * hib->piglet_pa and vaddr stored in hib->piglet_va 1866 */ 1867 if (get_hibernate_info(&hib, 1)) { 1868 DPRINTF("failed to obtain hibernate info\n"); 1869 return (1); 1870 } 1871 1872 /* Find a page-addressed region in swap [start,end] */ 1873 if (uvm_hibswap(hib.dev, &start, &end)) { 1874 printf("hibernate: cannot find any swap\n"); 1875 return (1); 1876 } 1877 1878 if (end - start < 1000) { 1879 printf("hibernate: insufficient swap (%lu is too small)\n", 1880 end - start); 1881 return (1); 1882 } 1883 1884 /* Calculate block offsets in swap */ 1885 hib.image_offset = ctod(start); 1886 1887 DPRINTF("hibernate @ block %lld max-length %lu blocks\n", 1888 hib.image_offset, ctod(end) - ctod(start)); 1889 1890 pmap_activate(curproc); 1891 DPRINTF("hibernate: writing chunks\n"); 1892 if (hibernate_write_chunks(&hib)) { 1893 DPRINTF("hibernate_write_chunks failed\n"); 1894 return (1); 1895 } 1896 1897 DPRINTF("hibernate: writing chunktable\n"); 1898 if (hibernate_write_chunktable(&hib)) { 1899 DPRINTF("hibernate_write_chunktable failed\n"); 1900 return (1); 1901 } 1902 1903 DPRINTF("hibernate: writing signature\n"); 1904 if (hibernate_write_signature(&hib)) { 1905 DPRINTF("hibernate_write_signature failed\n"); 1906 return (1); 1907 } 1908 1909 /* Allow the disk to settle */ 1910 delay(500000); 1911 1912 /* 1913 * Give the device-specific I/O function a notification that we're 1914 * done, and that it can clean up or shutdown as needed. 1915 */ 1916 hib.io_func(hib.dev, 0, (vaddr_t)NULL, 0, HIB_DONE, hib.io_page); 1917 return (0); 1918 } 1919 1920 int 1921 hibernate_alloc(void) 1922 { 1923 KASSERT(global_piglet_va == 0); 1924 KASSERT(hibernate_temp_page == 0); 1925 1926 pmap_activate(curproc); 1927 pmap_kenter_pa(HIBERNATE_HIBALLOC_PAGE, HIBERNATE_HIBALLOC_PAGE, 1928 PROT_READ | PROT_WRITE); 1929 1930 /* Allocate a piglet, store its addresses in the supplied globals */ 1931 if (uvm_pmr_alloc_piglet(&global_piglet_va, &global_piglet_pa, 1932 HIBERNATE_CHUNK_SIZE * 4, HIBERNATE_CHUNK_SIZE)) 1933 goto unmap; 1934 1935 /* 1936 * Allocate VA for the temp page. 1937 * 1938 * This will become part of the suspended kernel and will 1939 * be freed in hibernate_free, upon resume (or hibernate 1940 * failure) 1941 */ 1942 hibernate_temp_page = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any, 1943 &kp_none, &kd_nowait); 1944 if (!hibernate_temp_page) { 1945 uvm_pmr_free_piglet(global_piglet_va, 1946 4 * HIBERNATE_CHUNK_SIZE); 1947 global_piglet_va = 0; 1948 goto unmap; 1949 } 1950 return (0); 1951 unmap: 1952 pmap_kremove(HIBERNATE_HIBALLOC_PAGE, PAGE_SIZE); 1953 pmap_update(pmap_kernel()); 1954 return (ENOMEM); 1955 } 1956 1957 /* 1958 * Free items allocated by hibernate_alloc() 1959 */ 1960 void 1961 hibernate_free(void) 1962 { 1963 pmap_activate(curproc); 1964 1965 if (global_piglet_va) 1966 uvm_pmr_free_piglet(global_piglet_va, 1967 4 * HIBERNATE_CHUNK_SIZE); 1968 1969 if (hibernate_temp_page) { 1970 pmap_kremove(hibernate_temp_page, PAGE_SIZE); 1971 km_free((void *)hibernate_temp_page, PAGE_SIZE, 1972 &kv_any, &kp_none); 1973 } 1974 1975 global_piglet_va = 0; 1976 hibernate_temp_page = 0; 1977 pmap_kremove(HIBERNATE_HIBALLOC_PAGE, PAGE_SIZE); 1978 pmap_update(pmap_kernel()); 1979 } 1980